#!/usr/bin/env python3 import argparse import json import os import sqlite3 import sys import time from typing import Any, Dict, List, Tuple PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) if PROJECT_ROOT not in sys.path: sys.path.insert(0, PROJECT_ROOT) from analytics import _last_updated_ts, _normalize_last_updated, _read_dict_rows from config import MONITORING_DB_PATH def _package_name_from_row(row: Dict[str, Any]) -> str: return str(row.get("package_name") or row.get("包名") or row.get("package") or "").strip() def _last_updated_from_row(row: Dict[str, Any]) -> str: return _normalize_last_updated( row.get("last_updated") or row.get("最后更新") or row.get("更新时间") or "" ) def _safe_loads(value: Any) -> Dict[str, Any]: if not value: return {} try: payload = json.loads(value) except (TypeError, ValueError): return {} return payload if isinstance(payload, dict) else {} def _payload_with_last_updated(raw_payload: Any, last_updated: str) -> str: payload = _safe_loads(raw_payload) if not payload: return str(raw_payload or "") payload["last_updated"] = last_updated original_row = payload.get("original_row") if isinstance(original_row, dict): original_row = dict(original_row) original_row["last_updated"] = last_updated original_row["最后更新"] = last_updated original_row["更新时间"] = last_updated payload["original_row"] = original_row return json.dumps(payload, ensure_ascii=False) def _load_csv_updates(csv_paths: List[str]) -> Tuple[Dict[str, str], int, int]: updates: Dict[str, str] = {} listed_rows = 0 rows_with_date = 0 for csv_path in csv_paths: for row in _read_dict_rows(csv_path): listed_rows += 1 package_name = _package_name_from_row(row) last_updated = _last_updated_from_row(row) if not package_name or not last_updated: continue rows_with_date += 1 existing = updates.get(package_name, "") if not existing or _last_updated_ts(last_updated) > _last_updated_ts(existing): updates[package_name] = last_updated return updates, listed_rows, rows_with_date def import_catalog_last_updated(args) -> int: csv_paths = [str(item or "").strip() for item in args.csv_paths if str(item or "").strip()] if not csv_paths: raise ValueError("at least one csv path is required") apply_changes = bool(args.apply) update_payload = not bool(args.skip_payload) incoming_updates, listed_rows, rows_with_date = _load_csv_updates(csv_paths) now_iso = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) changed = 0 missing_packages = [] older_or_same = 0 with sqlite3.connect(str(args.db_path or "").strip()) as connection: connection.row_factory = sqlite3.Row table_row = connection.execute( """ SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'app_catalog' """ ).fetchone() if not table_row: raise SystemExit(f"app_catalog not found in {args.db_path}") for package_name, incoming_last_updated in sorted(incoming_updates.items()): row = connection.execute( """ SELECT package_name, last_updated, task_payload_json FROM app_catalog WHERE package_name = ? """, (package_name,), ).fetchone() if not row: missing_packages.append(package_name) continue existing_last_updated = _normalize_last_updated(row["last_updated"] or "") if existing_last_updated and _last_updated_ts(existing_last_updated) >= _last_updated_ts(incoming_last_updated): older_or_same += 1 continue changed += 1 if apply_changes: if update_payload: connection.execute( """ UPDATE app_catalog SET last_updated = ?, task_payload_json = ?, updated_at = ? WHERE package_name = ? """, ( incoming_last_updated, _payload_with_last_updated(row["task_payload_json"], incoming_last_updated), now_iso, package_name, ), ) else: connection.execute( """ UPDATE app_catalog SET last_updated = ?, updated_at = ? WHERE package_name = ? """, (incoming_last_updated, now_iso, package_name), ) missing_preview = ",".join(missing_packages[:10]) print( "dry_run={dry_run} db={db} csv_files={csv_files} listed_rows={listed_rows} " "rows_with_date={rows_with_date} unique_packages_with_date={unique_packages} " "updated={changed} older_or_same={older_or_same} missing={missing}{missing_suffix}".format( dry_run=0 if apply_changes else 1, db=str(args.db_path or "").strip(), csv_files=len(csv_paths), listed_rows=listed_rows, rows_with_date=rows_with_date, unique_packages=len(incoming_updates), changed=changed, older_or_same=older_or_same, missing=len(missing_packages), missing_suffix=f" missing_preview={missing_preview}" if missing_preview else "", ) ) return 0 def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description="Import last_updated from one or more CSV files without changing catalog collection state." ) parser.add_argument("csv_paths", nargs="+", help="CSV files containing package_name/last_updated columns.") parser.add_argument("--db-path", default=MONITORING_DB_PATH, help="Target monitoring sqlite path.") parser.add_argument("--apply", action="store_true", help="Apply changes. Without this, only prints counts.") parser.add_argument("--skip-payload", action="store_true", help="Do not update task_payload_json.") return parser def main() -> int: return import_catalog_last_updated(build_parser().parse_args()) if __name__ == "__main__": raise SystemExit(main())