| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281 |
- import datetime as dt
- import json
- import math
- import os
- import re
- import urllib.parse
- def build_url(base_url, params):
- if not params:
- return base_url
- parsed = urllib.parse.urlparse(base_url)
- current = urllib.parse.parse_qs(parsed.query)
- for k, v in params.items():
- current[k] = [str(v)]
- query = urllib.parse.urlencode(current, doseq=True)
- return urllib.parse.urlunparse(parsed._replace(query=query))
- def resolve_path(base_dir, path_value):
- path_text = str(path_value or "").strip()
- if not path_text:
- return ""
- if os.path.isabs(path_text):
- return os.path.normpath(path_text)
- return os.path.normpath(os.path.join(base_dir, path_text))
- def get_source_type(cfg):
- source_cfg = cfg.get("source", {})
- if isinstance(source_cfg, dict):
- source_type = str(source_cfg.get("type", "api")).strip().lower()
- if source_type:
- return source_type
- return "api"
- def parse_timezone(tz_raw):
- if tz_raw is None:
- return dt.timezone.utc
- s = str(tz_raw).strip().upper()
- if s in {"", "UTC", "Z", "+00:00", "+0000"}:
- return dt.timezone.utc
- m = re.match(r"^([+-])(\d{2}):?(\d{2})$", s)
- if not m:
- raise ValueError(f"invalid created_time_timezone: {tz_raw}")
- sign = 1 if m.group(1) == "+" else -1
- hh = int(m.group(2))
- mm = int(m.group(3))
- if hh > 23 or mm > 59:
- raise ValueError(f"invalid created_time_timezone offset: {tz_raw}")
- return dt.timezone(sign * dt.timedelta(hours=hh, minutes=mm))
- def to_float_or_none(value):
- try:
- f = float(value)
- if not math.isfinite(f):
- return None
- return f
- except Exception:
- return None
- def build_output_settings(output_cfg, config_path_abs):
- # legacy keys (current_domain_file/current_domain_json/substore_vars_file)
- # were ambiguous and drifted between server/local configs. Refuse to start
- # so operators migrate to the unified key set documented in CLAUDE.md.
- legacy_key_map = {
- "current_domain_file": "selected_value_file",
- "current_domain_json": "selected_value_json",
- "substore_vars_file": "export_vars_file",
- }
- legacy_in_use = [k for k in legacy_key_map if k in output_cfg]
- if legacy_in_use:
- renames = ", ".join(
- "{} -> {}".format(k, legacy_key_map[k]) for k in legacy_in_use
- )
- raise ValueError(
- "deprecated output keys in config: {}. Rename them: {}.".format(
- ", ".join(legacy_in_use), renames
- )
- )
- runtime_dir_cfg = output_cfg.get("runtime_dir", "./runtime")
- runtime_dir = resolve_path(os.path.dirname(config_path_abs), runtime_dir_cfg)
- selected_text_name = output_cfg.get("selected_value_file", "current_domain.txt")
- selected_json_name = output_cfg.get("selected_value_json", "current_domain.json")
- state_name = output_cfg.get("state_file", "state.json")
- vars_name = output_cfg.get("export_vars_file", "substore_vars.json")
- return {
- "runtime_dir": runtime_dir,
- "selected_text_path": os.path.join(runtime_dir, selected_text_name),
- "selected_json_path": os.path.join(runtime_dir, selected_json_name),
- "state_path": os.path.join(runtime_dir, state_name),
- "vars_path": os.path.join(runtime_dir, vars_name),
- "selected_json_key": str(output_cfg.get("selected_value_json_key", "domain")).strip() or "domain",
- "state_last_good_key": str(output_cfg.get("state_last_good_key", "last_good_domain")).strip() or "last_good_domain",
- "vars_value_key": str(output_cfg.get("substore_value_key", "AUTO_DOMAIN")).strip() or "AUTO_DOMAIN",
- }
- def print_output_settings(config_path_abs, cfg):
- output_cfg = cfg.get("output", {})
- settings = build_output_settings(output_cfg, config_path_abs)
- print(json.dumps(settings, ensure_ascii=True))
- def validate_config(cfg):
- source_type = get_source_type(cfg)
- if source_type not in {"api", "cfst_local"}:
- raise ValueError("source.type must be 'api' or 'cfst_local'")
- output_cfg = cfg.get("output", {})
- if output_cfg and not isinstance(output_cfg, dict):
- raise ValueError("output must be an object")
- if source_type == "cfst_local":
- cfst_cfg = cfg.get("cfst_local")
- if not isinstance(cfst_cfg, dict):
- raise ValueError("cfst_local is required and must be an object when source.type=cfst_local")
- work_dir = str(cfst_cfg.get("work_dir", "")).strip()
- if not work_dir:
- raise ValueError("cfst_local.work_dir is required")
- binary = str(cfst_cfg.get("binary", "")).strip()
- if not binary:
- raise ValueError("cfst_local.binary is required")
- result_file = str(cfst_cfg.get("result_file", "")).strip()
- if not result_file:
- raise ValueError("cfst_local.result_file is required")
- run_args = cfst_cfg.get("run_args", [])
- if not isinstance(run_args, list):
- raise ValueError("cfst_local.run_args must be an array")
- columns_cfg = cfst_cfg.get("columns", {})
- if columns_cfg and not isinstance(columns_cfg, dict):
- raise ValueError("cfst_local.columns must be an object")
- return
- # source.type == "api" 校验
- api_cfg = cfg.get("api")
- if not isinstance(api_cfg, dict):
- raise ValueError("api is required and must be an object when source.type=api")
- api_url = str(api_cfg.get("url", "")).strip()
- if not api_url:
- raise ValueError("api.url is required when source.type=api")
- parser_cfg = cfg.get("parser")
- if parser_cfg is not None and not isinstance(parser_cfg, dict):
- raise ValueError("parser must be an object")
- selection_cfg = cfg.get("selection")
- if selection_cfg is not None and not isinstance(selection_cfg, dict):
- raise ValueError("selection must be an object")
- domain_filter_cfg = cfg.get("domain_filter")
- if domain_filter_cfg is not None and not isinstance(domain_filter_cfg, dict):
- raise ValueError("domain_filter must be an object")
- record_mapping = cfg.get("record_mapping")
- if not isinstance(record_mapping, dict):
- raise ValueError("record_mapping is required and must be an object")
- records_path = str(record_mapping.get("records_path", "")).strip()
- if not records_path:
- raise ValueError("record_mapping.records_path is required")
- field_map = record_mapping.get("field_map")
- if not isinstance(field_map, dict) or not field_map:
- raise ValueError("record_mapping.field_map is required and must be a non-empty object")
- for key, path in field_map.items():
- if not str(key).strip() or not str(path).strip():
- raise ValueError("record_mapping.field_map contains empty field name or path")
- for required in ["domain", "created_at"]:
- if required not in field_map:
- raise ValueError(f"record_mapping.field_map.{required} is required")
- created_time_formats = record_mapping.get("created_time_formats")
- if not isinstance(created_time_formats, list) or not created_time_formats:
- raise ValueError("record_mapping.created_time_formats is required and must be a non-empty array")
- for fmt in created_time_formats:
- if not str(fmt).strip():
- raise ValueError("record_mapping.created_time_formats contains empty format")
- parse_timezone(record_mapping.get("created_time_timezone", "UTC"))
- def ensure_field_registered(field_name, where):
- if field_name not in field_map:
- raise ValueError(f"{where}: field '{field_name}' is not in record_mapping.field_map")
- record_filter = cfg.get("record_filter", {})
- if record_filter.get("enabled", False):
- rules = record_filter.get("exclude_if_any", [])
- if not isinstance(rules, list):
- raise ValueError("record_filter.exclude_if_any must be an array")
- for i, rule in enumerate(rules):
- if not isinstance(rule, dict):
- raise ValueError(f"record_filter.exclude_if_any[{i}] must be an object")
- field_name = str(rule.get("field", "")).strip()
- if not field_name:
- raise ValueError(f"record_filter.exclude_if_any[{i}].field is required")
- ensure_field_registered(field_name, f"record_filter.exclude_if_any[{i}]")
- has_matcher = any(k in rule for k in ["contains", "equals", "regex"])
- if not has_matcher:
- raise ValueError(f"record_filter.exclude_if_any[{i}] must include one of contains/equals/regex")
- scoring = cfg.get("scoring", {})
- if scoring.get("enabled", False):
- strategy = str(scoring.get("strategy", "")).strip()
- if strategy not in {"weighted_average", "lexicographic"}:
- raise ValueError("scoring.strategy must be 'weighted_average' or 'lexicographic'")
- within_hours = to_float_or_none(scoring.get("within_hours", 0))
- if within_hours is None or within_hours < 0:
- raise ValueError("scoring.within_hours must be a non-negative number (0 = disabled)")
- if strategy == "weighted_average":
- weighted_fields = scoring.get("weighted_fields")
- if not isinstance(weighted_fields, list) or not weighted_fields:
- raise ValueError("scoring.weighted_fields is required for weighted_average strategy")
- for i, item in enumerate(weighted_fields):
- if not isinstance(item, dict):
- raise ValueError(f"scoring.weighted_fields[{i}] must be an object")
- field_name = str(item.get("field", "")).strip()
- if not field_name:
- raise ValueError(f"scoring.weighted_fields[{i}].field is required")
- ensure_field_registered(field_name, f"scoring.weighted_fields[{i}]")
- weight = to_float_or_none(item.get("weight"))
- if weight is None or weight <= 0:
- raise ValueError(f"scoring.weighted_fields[{i}].weight must be > 0")
- if strategy == "lexicographic":
- lex_fields = scoring.get("lexicographic_fields")
- if not isinstance(lex_fields, list) or not lex_fields:
- raise ValueError("scoring.lexicographic_fields is required for lexicographic strategy")
- for i, item in enumerate(lex_fields):
- if isinstance(item, str):
- field_name = item.strip()
- order = ""
- elif isinstance(item, dict):
- field_name = str(item.get("field", "")).strip()
- order = str(item.get("order", "")).strip().lower()
- else:
- raise ValueError(f"scoring.lexicographic_fields[{i}] must be string or object")
- if not field_name:
- raise ValueError(f"scoring.lexicographic_fields[{i}] field is required")
- ensure_field_registered(field_name, f"scoring.lexicographic_fields[{i}]")
- if order and order not in {"asc", "desc"}:
- raise ValueError(f"scoring.lexicographic_fields[{i}].order must be asc or desc")
- tie_breakers = scoring.get("tie_breakers", [])
- if tie_breakers is not None:
- if not isinstance(tie_breakers, list):
- raise ValueError("scoring.tie_breakers must be an array")
- for i, item in enumerate(tie_breakers):
- if not isinstance(item, dict):
- raise ValueError(f"scoring.tie_breakers[{i}] must be an object")
- field_name = str(item.get("field", "")).strip()
- order = str(item.get("order", "")).strip().lower()
- if not field_name:
- raise ValueError(f"scoring.tie_breakers[{i}].field is required")
- if order not in {"asc", "desc"}:
- raise ValueError(f"scoring.tie_breakers[{i}].order must be asc or desc")
- ensure_field_registered(field_name, f"scoring.tie_breakers[{i}]")
|