config.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283
  1. import datetime as dt
  2. import json
  3. import math
  4. import os
  5. import re
  6. import urllib.parse
  7. def build_url(base_url, params):
  8. if not params:
  9. return base_url
  10. parsed = urllib.parse.urlparse(base_url)
  11. current = urllib.parse.parse_qs(parsed.query)
  12. for k, v in params.items():
  13. current[k] = [str(v)]
  14. query = urllib.parse.urlencode(current, doseq=True)
  15. return urllib.parse.urlunparse(parsed._replace(query=query))
  16. def resolve_path(base_dir, path_value):
  17. path_text = str(path_value or "").strip()
  18. if not path_text:
  19. return ""
  20. if os.path.isabs(path_text):
  21. return os.path.normpath(path_text)
  22. return os.path.normpath(os.path.join(base_dir, path_text))
  23. def get_source_type(cfg):
  24. source_cfg = cfg.get("source", {})
  25. if isinstance(source_cfg, dict):
  26. source_type = str(source_cfg.get("type", "api")).strip().lower()
  27. if source_type:
  28. return source_type
  29. return "api"
  30. def parse_timezone(tz_raw):
  31. if tz_raw is None:
  32. return dt.timezone.utc
  33. s = str(tz_raw).strip().upper()
  34. if s in {"", "UTC", "Z", "+00:00", "+0000"}:
  35. return dt.timezone.utc
  36. m = re.match(r"^([+-])(\d{2}):?(\d{2})$", s)
  37. if not m:
  38. raise ValueError(f"invalid created_time_timezone: {tz_raw}")
  39. sign = 1 if m.group(1) == "+" else -1
  40. hh = int(m.group(2))
  41. mm = int(m.group(3))
  42. if hh > 23 or mm > 59:
  43. raise ValueError(f"invalid created_time_timezone offset: {tz_raw}")
  44. return dt.timezone(sign * dt.timedelta(hours=hh, minutes=mm))
  45. def to_float_or_none(value):
  46. try:
  47. f = float(value)
  48. if not math.isfinite(f):
  49. return None
  50. return f
  51. except Exception:
  52. return None
  53. def build_output_settings(output_cfg, config_path_abs):
  54. # legacy keys (current_domain_file/current_domain_json/substore_vars_file)
  55. # were ambiguous and drifted between server/local configs. Refuse to start
  56. # so operators migrate to the unified key set documented in CLAUDE.md.
  57. legacy_key_map = {
  58. "current_domain_file": "selected_value_file",
  59. "current_domain_json": "selected_value_json",
  60. "substore_vars_file": "export_vars_file",
  61. }
  62. legacy_in_use = [k for k in legacy_key_map if k in output_cfg]
  63. if legacy_in_use:
  64. renames = ", ".join(
  65. "{} -> {}".format(k, legacy_key_map[k]) for k in legacy_in_use
  66. )
  67. raise ValueError(
  68. "deprecated output keys in config: {}. Rename them: {}.".format(
  69. ", ".join(legacy_in_use), renames
  70. )
  71. )
  72. runtime_dir_cfg = output_cfg.get("runtime_dir", "./runtime")
  73. config_dir = os.path.dirname(os.path.abspath(config_path_abs))
  74. base_dir = os.path.dirname(config_dir) if os.path.basename(config_dir) == "configs" else config_dir
  75. runtime_dir = resolve_path(base_dir, runtime_dir_cfg)
  76. selected_text_name = output_cfg.get("selected_value_file", "current_domain.txt")
  77. selected_json_name = output_cfg.get("selected_value_json", "current_domain.json")
  78. state_name = output_cfg.get("state_file", "state.json")
  79. vars_name = output_cfg.get("export_vars_file", "substore_vars.json")
  80. return {
  81. "runtime_dir": runtime_dir,
  82. "selected_text_path": os.path.join(runtime_dir, selected_text_name),
  83. "selected_json_path": os.path.join(runtime_dir, selected_json_name),
  84. "state_path": os.path.join(runtime_dir, state_name),
  85. "vars_path": os.path.join(runtime_dir, vars_name),
  86. "selected_json_key": str(output_cfg.get("selected_value_json_key", "domain")).strip() or "domain",
  87. "state_last_good_key": str(output_cfg.get("state_last_good_key", "last_good_domain")).strip() or "last_good_domain",
  88. "vars_value_key": str(output_cfg.get("substore_value_key", "AUTO_DOMAIN")).strip() or "AUTO_DOMAIN",
  89. }
  90. def print_output_settings(config_path_abs, cfg):
  91. output_cfg = cfg.get("output", {})
  92. settings = build_output_settings(output_cfg, config_path_abs)
  93. print(json.dumps(settings, ensure_ascii=True))
  94. def validate_config(cfg):
  95. source_type = get_source_type(cfg)
  96. if source_type not in {"api", "cfst_local"}:
  97. raise ValueError("source.type must be 'api' or 'cfst_local'")
  98. output_cfg = cfg.get("output", {})
  99. if output_cfg and not isinstance(output_cfg, dict):
  100. raise ValueError("output must be an object")
  101. if source_type == "cfst_local":
  102. cfst_cfg = cfg.get("cfst_local")
  103. if not isinstance(cfst_cfg, dict):
  104. raise ValueError("cfst_local is required and must be an object when source.type=cfst_local")
  105. work_dir = str(cfst_cfg.get("work_dir", "")).strip()
  106. if not work_dir:
  107. raise ValueError("cfst_local.work_dir is required")
  108. binary = str(cfst_cfg.get("binary", "")).strip()
  109. if not binary:
  110. raise ValueError("cfst_local.binary is required")
  111. result_file = str(cfst_cfg.get("result_file", "")).strip()
  112. if not result_file:
  113. raise ValueError("cfst_local.result_file is required")
  114. run_args = cfst_cfg.get("run_args", [])
  115. if not isinstance(run_args, list):
  116. raise ValueError("cfst_local.run_args must be an array")
  117. columns_cfg = cfst_cfg.get("columns", {})
  118. if columns_cfg and not isinstance(columns_cfg, dict):
  119. raise ValueError("cfst_local.columns must be an object")
  120. return
  121. # source.type == "api" 校验
  122. api_cfg = cfg.get("api")
  123. if not isinstance(api_cfg, dict):
  124. raise ValueError("api is required and must be an object when source.type=api")
  125. api_url = str(api_cfg.get("url", "")).strip()
  126. if not api_url:
  127. raise ValueError("api.url is required when source.type=api")
  128. parser_cfg = cfg.get("parser")
  129. if parser_cfg is not None and not isinstance(parser_cfg, dict):
  130. raise ValueError("parser must be an object")
  131. selection_cfg = cfg.get("selection")
  132. if selection_cfg is not None and not isinstance(selection_cfg, dict):
  133. raise ValueError("selection must be an object")
  134. domain_filter_cfg = cfg.get("domain_filter")
  135. if domain_filter_cfg is not None and not isinstance(domain_filter_cfg, dict):
  136. raise ValueError("domain_filter must be an object")
  137. record_mapping = cfg.get("record_mapping")
  138. if not isinstance(record_mapping, dict):
  139. raise ValueError("record_mapping is required and must be an object")
  140. records_path = str(record_mapping.get("records_path", "")).strip()
  141. if not records_path:
  142. raise ValueError("record_mapping.records_path is required")
  143. field_map = record_mapping.get("field_map")
  144. if not isinstance(field_map, dict) or not field_map:
  145. raise ValueError("record_mapping.field_map is required and must be a non-empty object")
  146. for key, path in field_map.items():
  147. if not str(key).strip() or not str(path).strip():
  148. raise ValueError("record_mapping.field_map contains empty field name or path")
  149. for required in ["domain", "created_at"]:
  150. if required not in field_map:
  151. raise ValueError(f"record_mapping.field_map.{required} is required")
  152. created_time_formats = record_mapping.get("created_time_formats")
  153. if not isinstance(created_time_formats, list) or not created_time_formats:
  154. raise ValueError("record_mapping.created_time_formats is required and must be a non-empty array")
  155. for fmt in created_time_formats:
  156. if not str(fmt).strip():
  157. raise ValueError("record_mapping.created_time_formats contains empty format")
  158. parse_timezone(record_mapping.get("created_time_timezone", "UTC"))
  159. def ensure_field_registered(field_name, where):
  160. if field_name not in field_map:
  161. raise ValueError(f"{where}: field '{field_name}' is not in record_mapping.field_map")
  162. record_filter = cfg.get("record_filter", {})
  163. if record_filter.get("enabled", False):
  164. rules = record_filter.get("exclude_if_any", [])
  165. if not isinstance(rules, list):
  166. raise ValueError("record_filter.exclude_if_any must be an array")
  167. for i, rule in enumerate(rules):
  168. if not isinstance(rule, dict):
  169. raise ValueError(f"record_filter.exclude_if_any[{i}] must be an object")
  170. field_name = str(rule.get("field", "")).strip()
  171. if not field_name:
  172. raise ValueError(f"record_filter.exclude_if_any[{i}].field is required")
  173. ensure_field_registered(field_name, f"record_filter.exclude_if_any[{i}]")
  174. has_matcher = any(k in rule for k in ["contains", "equals", "regex"])
  175. if not has_matcher:
  176. raise ValueError(f"record_filter.exclude_if_any[{i}] must include one of contains/equals/regex")
  177. scoring = cfg.get("scoring", {})
  178. if scoring.get("enabled", False):
  179. strategy = str(scoring.get("strategy", "")).strip()
  180. if strategy not in {"weighted_average", "lexicographic"}:
  181. raise ValueError("scoring.strategy must be 'weighted_average' or 'lexicographic'")
  182. within_hours = to_float_or_none(scoring.get("within_hours", 0))
  183. if within_hours is None or within_hours < 0:
  184. raise ValueError("scoring.within_hours must be a non-negative number (0 = disabled)")
  185. if strategy == "weighted_average":
  186. weighted_fields = scoring.get("weighted_fields")
  187. if not isinstance(weighted_fields, list) or not weighted_fields:
  188. raise ValueError("scoring.weighted_fields is required for weighted_average strategy")
  189. for i, item in enumerate(weighted_fields):
  190. if not isinstance(item, dict):
  191. raise ValueError(f"scoring.weighted_fields[{i}] must be an object")
  192. field_name = str(item.get("field", "")).strip()
  193. if not field_name:
  194. raise ValueError(f"scoring.weighted_fields[{i}].field is required")
  195. ensure_field_registered(field_name, f"scoring.weighted_fields[{i}]")
  196. weight = to_float_or_none(item.get("weight"))
  197. if weight is None or weight <= 0:
  198. raise ValueError(f"scoring.weighted_fields[{i}].weight must be > 0")
  199. if strategy == "lexicographic":
  200. lex_fields = scoring.get("lexicographic_fields")
  201. if not isinstance(lex_fields, list) or not lex_fields:
  202. raise ValueError("scoring.lexicographic_fields is required for lexicographic strategy")
  203. for i, item in enumerate(lex_fields):
  204. if isinstance(item, str):
  205. field_name = item.strip()
  206. order = ""
  207. elif isinstance(item, dict):
  208. field_name = str(item.get("field", "")).strip()
  209. order = str(item.get("order", "")).strip().lower()
  210. else:
  211. raise ValueError(f"scoring.lexicographic_fields[{i}] must be string or object")
  212. if not field_name:
  213. raise ValueError(f"scoring.lexicographic_fields[{i}] field is required")
  214. ensure_field_registered(field_name, f"scoring.lexicographic_fields[{i}]")
  215. if order and order not in {"asc", "desc"}:
  216. raise ValueError(f"scoring.lexicographic_fields[{i}].order must be asc or desc")
  217. tie_breakers = scoring.get("tie_breakers", [])
  218. if tie_breakers is not None:
  219. if not isinstance(tie_breakers, list):
  220. raise ValueError("scoring.tie_breakers must be an array")
  221. for i, item in enumerate(tie_breakers):
  222. if not isinstance(item, dict):
  223. raise ValueError(f"scoring.tie_breakers[{i}] must be an object")
  224. field_name = str(item.get("field", "")).strip()
  225. order = str(item.get("order", "")).strip().lower()
  226. if not field_name:
  227. raise ValueError(f"scoring.tie_breakers[{i}].field is required")
  228. if order not in {"asc", "desc"}:
  229. raise ValueError(f"scoring.tie_breakers[{i}].order must be asc or desc")
  230. ensure_field_registered(field_name, f"scoring.tie_breakers[{i}]")