config.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281
  1. import datetime as dt
  2. import json
  3. import math
  4. import os
  5. import re
  6. import urllib.parse
  7. def build_url(base_url, params):
  8. if not params:
  9. return base_url
  10. parsed = urllib.parse.urlparse(base_url)
  11. current = urllib.parse.parse_qs(parsed.query)
  12. for k, v in params.items():
  13. current[k] = [str(v)]
  14. query = urllib.parse.urlencode(current, doseq=True)
  15. return urllib.parse.urlunparse(parsed._replace(query=query))
  16. def resolve_path(base_dir, path_value):
  17. path_text = str(path_value or "").strip()
  18. if not path_text:
  19. return ""
  20. if os.path.isabs(path_text):
  21. return os.path.normpath(path_text)
  22. return os.path.normpath(os.path.join(base_dir, path_text))
  23. def get_source_type(cfg):
  24. source_cfg = cfg.get("source", {})
  25. if isinstance(source_cfg, dict):
  26. source_type = str(source_cfg.get("type", "api")).strip().lower()
  27. if source_type:
  28. return source_type
  29. return "api"
  30. def parse_timezone(tz_raw):
  31. if tz_raw is None:
  32. return dt.timezone.utc
  33. s = str(tz_raw).strip().upper()
  34. if s in {"", "UTC", "Z", "+00:00", "+0000"}:
  35. return dt.timezone.utc
  36. m = re.match(r"^([+-])(\d{2}):?(\d{2})$", s)
  37. if not m:
  38. raise ValueError(f"invalid created_time_timezone: {tz_raw}")
  39. sign = 1 if m.group(1) == "+" else -1
  40. hh = int(m.group(2))
  41. mm = int(m.group(3))
  42. if hh > 23 or mm > 59:
  43. raise ValueError(f"invalid created_time_timezone offset: {tz_raw}")
  44. return dt.timezone(sign * dt.timedelta(hours=hh, minutes=mm))
  45. def to_float_or_none(value):
  46. try:
  47. f = float(value)
  48. if not math.isfinite(f):
  49. return None
  50. return f
  51. except Exception:
  52. return None
  53. def build_output_settings(output_cfg, config_path_abs):
  54. # legacy keys (current_domain_file/current_domain_json/substore_vars_file)
  55. # were ambiguous and drifted between server/local configs. Refuse to start
  56. # so operators migrate to the unified key set documented in CLAUDE.md.
  57. legacy_key_map = {
  58. "current_domain_file": "selected_value_file",
  59. "current_domain_json": "selected_value_json",
  60. "substore_vars_file": "export_vars_file",
  61. }
  62. legacy_in_use = [k for k in legacy_key_map if k in output_cfg]
  63. if legacy_in_use:
  64. renames = ", ".join(
  65. "{} -> {}".format(k, legacy_key_map[k]) for k in legacy_in_use
  66. )
  67. raise ValueError(
  68. "deprecated output keys in config: {}. Rename them: {}.".format(
  69. ", ".join(legacy_in_use), renames
  70. )
  71. )
  72. runtime_dir_cfg = output_cfg.get("runtime_dir", "./runtime")
  73. runtime_dir = resolve_path(os.path.dirname(config_path_abs), runtime_dir_cfg)
  74. selected_text_name = output_cfg.get("selected_value_file", "current_domain.txt")
  75. selected_json_name = output_cfg.get("selected_value_json", "current_domain.json")
  76. state_name = output_cfg.get("state_file", "state.json")
  77. vars_name = output_cfg.get("export_vars_file", "substore_vars.json")
  78. return {
  79. "runtime_dir": runtime_dir,
  80. "selected_text_path": os.path.join(runtime_dir, selected_text_name),
  81. "selected_json_path": os.path.join(runtime_dir, selected_json_name),
  82. "state_path": os.path.join(runtime_dir, state_name),
  83. "vars_path": os.path.join(runtime_dir, vars_name),
  84. "selected_json_key": str(output_cfg.get("selected_value_json_key", "domain")).strip() or "domain",
  85. "state_last_good_key": str(output_cfg.get("state_last_good_key", "last_good_domain")).strip() or "last_good_domain",
  86. "vars_value_key": str(output_cfg.get("substore_value_key", "AUTO_DOMAIN")).strip() or "AUTO_DOMAIN",
  87. }
  88. def print_output_settings(config_path_abs, cfg):
  89. output_cfg = cfg.get("output", {})
  90. settings = build_output_settings(output_cfg, config_path_abs)
  91. print(json.dumps(settings, ensure_ascii=True))
  92. def validate_config(cfg):
  93. source_type = get_source_type(cfg)
  94. if source_type not in {"api", "cfst_local"}:
  95. raise ValueError("source.type must be 'api' or 'cfst_local'")
  96. output_cfg = cfg.get("output", {})
  97. if output_cfg and not isinstance(output_cfg, dict):
  98. raise ValueError("output must be an object")
  99. if source_type == "cfst_local":
  100. cfst_cfg = cfg.get("cfst_local")
  101. if not isinstance(cfst_cfg, dict):
  102. raise ValueError("cfst_local is required and must be an object when source.type=cfst_local")
  103. work_dir = str(cfst_cfg.get("work_dir", "")).strip()
  104. if not work_dir:
  105. raise ValueError("cfst_local.work_dir is required")
  106. binary = str(cfst_cfg.get("binary", "")).strip()
  107. if not binary:
  108. raise ValueError("cfst_local.binary is required")
  109. result_file = str(cfst_cfg.get("result_file", "")).strip()
  110. if not result_file:
  111. raise ValueError("cfst_local.result_file is required")
  112. run_args = cfst_cfg.get("run_args", [])
  113. if not isinstance(run_args, list):
  114. raise ValueError("cfst_local.run_args must be an array")
  115. columns_cfg = cfst_cfg.get("columns", {})
  116. if columns_cfg and not isinstance(columns_cfg, dict):
  117. raise ValueError("cfst_local.columns must be an object")
  118. return
  119. # source.type == "api" 校验
  120. api_cfg = cfg.get("api")
  121. if not isinstance(api_cfg, dict):
  122. raise ValueError("api is required and must be an object when source.type=api")
  123. api_url = str(api_cfg.get("url", "")).strip()
  124. if not api_url:
  125. raise ValueError("api.url is required when source.type=api")
  126. parser_cfg = cfg.get("parser")
  127. if parser_cfg is not None and not isinstance(parser_cfg, dict):
  128. raise ValueError("parser must be an object")
  129. selection_cfg = cfg.get("selection")
  130. if selection_cfg is not None and not isinstance(selection_cfg, dict):
  131. raise ValueError("selection must be an object")
  132. domain_filter_cfg = cfg.get("domain_filter")
  133. if domain_filter_cfg is not None and not isinstance(domain_filter_cfg, dict):
  134. raise ValueError("domain_filter must be an object")
  135. record_mapping = cfg.get("record_mapping")
  136. if not isinstance(record_mapping, dict):
  137. raise ValueError("record_mapping is required and must be an object")
  138. records_path = str(record_mapping.get("records_path", "")).strip()
  139. if not records_path:
  140. raise ValueError("record_mapping.records_path is required")
  141. field_map = record_mapping.get("field_map")
  142. if not isinstance(field_map, dict) or not field_map:
  143. raise ValueError("record_mapping.field_map is required and must be a non-empty object")
  144. for key, path in field_map.items():
  145. if not str(key).strip() or not str(path).strip():
  146. raise ValueError("record_mapping.field_map contains empty field name or path")
  147. for required in ["domain", "created_at"]:
  148. if required not in field_map:
  149. raise ValueError(f"record_mapping.field_map.{required} is required")
  150. created_time_formats = record_mapping.get("created_time_formats")
  151. if not isinstance(created_time_formats, list) or not created_time_formats:
  152. raise ValueError("record_mapping.created_time_formats is required and must be a non-empty array")
  153. for fmt in created_time_formats:
  154. if not str(fmt).strip():
  155. raise ValueError("record_mapping.created_time_formats contains empty format")
  156. parse_timezone(record_mapping.get("created_time_timezone", "UTC"))
  157. def ensure_field_registered(field_name, where):
  158. if field_name not in field_map:
  159. raise ValueError(f"{where}: field '{field_name}' is not in record_mapping.field_map")
  160. record_filter = cfg.get("record_filter", {})
  161. if record_filter.get("enabled", False):
  162. rules = record_filter.get("exclude_if_any", [])
  163. if not isinstance(rules, list):
  164. raise ValueError("record_filter.exclude_if_any must be an array")
  165. for i, rule in enumerate(rules):
  166. if not isinstance(rule, dict):
  167. raise ValueError(f"record_filter.exclude_if_any[{i}] must be an object")
  168. field_name = str(rule.get("field", "")).strip()
  169. if not field_name:
  170. raise ValueError(f"record_filter.exclude_if_any[{i}].field is required")
  171. ensure_field_registered(field_name, f"record_filter.exclude_if_any[{i}]")
  172. has_matcher = any(k in rule for k in ["contains", "equals", "regex"])
  173. if not has_matcher:
  174. raise ValueError(f"record_filter.exclude_if_any[{i}] must include one of contains/equals/regex")
  175. scoring = cfg.get("scoring", {})
  176. if scoring.get("enabled", False):
  177. strategy = str(scoring.get("strategy", "")).strip()
  178. if strategy not in {"weighted_average", "lexicographic"}:
  179. raise ValueError("scoring.strategy must be 'weighted_average' or 'lexicographic'")
  180. within_hours = to_float_or_none(scoring.get("within_hours", 0))
  181. if within_hours is None or within_hours < 0:
  182. raise ValueError("scoring.within_hours must be a non-negative number (0 = disabled)")
  183. if strategy == "weighted_average":
  184. weighted_fields = scoring.get("weighted_fields")
  185. if not isinstance(weighted_fields, list) or not weighted_fields:
  186. raise ValueError("scoring.weighted_fields is required for weighted_average strategy")
  187. for i, item in enumerate(weighted_fields):
  188. if not isinstance(item, dict):
  189. raise ValueError(f"scoring.weighted_fields[{i}] must be an object")
  190. field_name = str(item.get("field", "")).strip()
  191. if not field_name:
  192. raise ValueError(f"scoring.weighted_fields[{i}].field is required")
  193. ensure_field_registered(field_name, f"scoring.weighted_fields[{i}]")
  194. weight = to_float_or_none(item.get("weight"))
  195. if weight is None or weight <= 0:
  196. raise ValueError(f"scoring.weighted_fields[{i}].weight must be > 0")
  197. if strategy == "lexicographic":
  198. lex_fields = scoring.get("lexicographic_fields")
  199. if not isinstance(lex_fields, list) or not lex_fields:
  200. raise ValueError("scoring.lexicographic_fields is required for lexicographic strategy")
  201. for i, item in enumerate(lex_fields):
  202. if isinstance(item, str):
  203. field_name = item.strip()
  204. order = ""
  205. elif isinstance(item, dict):
  206. field_name = str(item.get("field", "")).strip()
  207. order = str(item.get("order", "")).strip().lower()
  208. else:
  209. raise ValueError(f"scoring.lexicographic_fields[{i}] must be string or object")
  210. if not field_name:
  211. raise ValueError(f"scoring.lexicographic_fields[{i}] field is required")
  212. ensure_field_registered(field_name, f"scoring.lexicographic_fields[{i}]")
  213. if order and order not in {"asc", "desc"}:
  214. raise ValueError(f"scoring.lexicographic_fields[{i}].order must be asc or desc")
  215. tie_breakers = scoring.get("tie_breakers", [])
  216. if tie_breakers is not None:
  217. if not isinstance(tie_breakers, list):
  218. raise ValueError("scoring.tie_breakers must be an array")
  219. for i, item in enumerate(tie_breakers):
  220. if not isinstance(item, dict):
  221. raise ValueError(f"scoring.tie_breakers[{i}] must be an object")
  222. field_name = str(item.get("field", "")).strip()
  223. order = str(item.get("order", "")).strip().lower()
  224. if not field_name:
  225. raise ValueError(f"scoring.tie_breakers[{i}].field is required")
  226. if order not in {"asc", "desc"}:
  227. raise ValueError(f"scoring.tie_breakers[{i}].order must be asc or desc")
  228. ensure_field_registered(field_name, f"scoring.tie_breakers[{i}]")