autool-dispatcher/scripts/import_catalog_last_updated.py

#!/usr/bin/env python3

import argparse
import json
import os
import sqlite3
import sys
import time
from typing import Any, Dict, List, Tuple


PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if PROJECT_ROOT not in sys.path:
    sys.path.insert(0, PROJECT_ROOT)

from analytics import _last_updated_ts, _normalize_last_updated, _read_dict_rows
from config import MONITORING_DB_PATH


def _package_name_from_row(row: Dict[str, Any]) -> str:
    return str(row.get("package_name") or row.get("包名") or row.get("package") or "").strip()


def _last_updated_from_row(row: Dict[str, Any]) -> str:
    return _normalize_last_updated(
        row.get("last_updated")
        or row.get("最后更新")
        or row.get("更新时间")
        or ""
    )


def _safe_loads(value: Any) -> Dict[str, Any]:
    if not value:
        return {}
    try:
        payload = json.loads(value)
    except (TypeError, ValueError):
        return {}
    return payload if isinstance(payload, dict) else {}


def _payload_with_last_updated(raw_payload: Any, last_updated: str) -> str:
    payload = _safe_loads(raw_payload)
    if not payload:
        return str(raw_payload or "")
    payload["last_updated"] = last_updated
    original_row = payload.get("original_row")
    if isinstance(original_row, dict):
        original_row = dict(original_row)
        original_row["last_updated"] = last_updated
        original_row["最后更新"] = last_updated
        original_row["更新时间"] = last_updated
        payload["original_row"] = original_row
    return json.dumps(payload, ensure_ascii=False)


def _load_csv_updates(csv_paths: List[str]) -> Tuple[Dict[str, str], int, int]:
    updates: Dict[str, str] = {}
    listed_rows = 0
    rows_with_date = 0
    for csv_path in csv_paths:
        for row in _read_dict_rows(csv_path):
            listed_rows += 1
            package_name = _package_name_from_row(row)
            last_updated = _last_updated_from_row(row)
            if not package_name or not last_updated:
                continue
            rows_with_date += 1
            existing = updates.get(package_name, "")
            if not existing or _last_updated_ts(last_updated) > _last_updated_ts(existing):
                updates[package_name] = last_updated
    return updates, listed_rows, rows_with_date


def import_catalog_last_updated(args) -> int:
    csv_paths = [str(item or "").strip() for item in args.csv_paths if str(item or "").strip()]
    if not csv_paths:
        raise ValueError("at least one csv path is required")
    apply_changes = bool(args.apply)
    update_payload = not bool(args.skip_payload)
    incoming_updates, listed_rows, rows_with_date = _load_csv_updates(csv_paths)
    now_iso = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
    changed = 0
    missing_packages = []
    older_or_same = 0
    with sqlite3.connect(str(args.db_path or "").strip()) as connection:
        connection.row_factory = sqlite3.Row
        table_row = connection.execute(
            """
            SELECT name
              FROM sqlite_master
             WHERE type = 'table'
               AND name = 'app_catalog'
            """
        ).fetchone()
        if not table_row:
            raise SystemExit(f"app_catalog not found in {args.db_path}")
        for package_name, incoming_last_updated in sorted(incoming_updates.items()):
            row = connection.execute(
                """
                SELECT package_name, last_updated, task_payload_json
                  FROM app_catalog
                 WHERE package_name = ?
                """,
                (package_name,),
            ).fetchone()
            if not row:
                missing_packages.append(package_name)
                continue
            existing_last_updated = _normalize_last_updated(row["last_updated"] or "")
            if existing_last_updated and _last_updated_ts(existing_last_updated) >= _last_updated_ts(incoming_last_updated):
                older_or_same += 1
                continue
            changed += 1
            if apply_changes:
                if update_payload:
                    connection.execute(
                        """
                        UPDATE app_catalog
                           SET last_updated = ?,
                               task_payload_json = ?,
                               updated_at = ?
                         WHERE package_name = ?
                        """,
                        (
                            incoming_last_updated,
                            _payload_with_last_updated(row["task_payload_json"], incoming_last_updated),
                            now_iso,
                            package_name,
                        ),
                    )
                else:
                    connection.execute(
                        """
                        UPDATE app_catalog
                           SET last_updated = ?,
                               updated_at = ?
                         WHERE package_name = ?
                        """,
                        (incoming_last_updated, now_iso, package_name),
                    )
    missing_preview = ",".join(missing_packages[:10])
    print(
        "dry_run={dry_run} db={db} csv_files={csv_files} listed_rows={listed_rows} "
        "rows_with_date={rows_with_date} unique_packages_with_date={unique_packages} "
        "updated={changed} older_or_same={older_or_same} missing={missing}{missing_suffix}".format(
            dry_run=0 if apply_changes else 1,
            db=str(args.db_path or "").strip(),
            csv_files=len(csv_paths),
            listed_rows=listed_rows,
            rows_with_date=rows_with_date,
            unique_packages=len(incoming_updates),
            changed=changed,
            older_or_same=older_or_same,
            missing=len(missing_packages),
            missing_suffix=f" missing_preview={missing_preview}" if missing_preview else "",
        )
    )
    return 0


def build_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(
        description="Import last_updated from one or more CSV files without changing catalog collection state."
    )
    parser.add_argument("csv_paths", nargs="+", help="CSV files containing package_name/last_updated columns.")
    parser.add_argument("--db-path", default=MONITORING_DB_PATH, help="Target monitoring sqlite path.")
    parser.add_argument("--apply", action="store_true", help="Apply changes. Without this, only prints counts.")
    parser.add_argument("--skip-payload", action="store_true", help="Do not update task_payload_json.")
    return parser


def main() -> int:
    return import_catalog_last_updated(build_parser().parse_args())


if __name__ == "__main__":
    raise SystemExit(main())