#!/usr/bin/env python3 import argparse import os import sys from typing import List REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) if REPO_ROOT not in sys.path: sys.path.insert(0, REPO_ROOT) from analytics import AnalyticsService from config import ANALYTICS_TRAFFIC_ROOT, MONITORING_DB_PATH def _split_packages(raw: str) -> List[str]: return [item.strip() for item in str(raw or "").split(",") if item.strip()] def _has_traffic_domains(traffic_files: List[str], package_name: str) -> bool: """Check whether any traffic file contains at least one valid traffic entry for the package.""" for file_path in traffic_files: try: with open(file_path, "r", encoding="utf-8", errors="ignore") as handle: for raw_line in handle: line = raw_line.strip() if not line: continue fields = [item.strip() for item in line.split(",")] if len(fields) < 7: continue if fields[0].strip() == package_name: return True except OSError: continue return False def _load_zero_traffic_packages(service: AnalyticsService, tag: str = "", limit: int = 0) -> List[str]: normalized_tag = str(tag or "").strip() with service.repo._connect() as connection: summaries = service.repo._list_catalog_summaries(connection) rows = [] for item in summaries: if not item.get("catalog_active"): continue if normalized_tag and normalized_tag not in (item.get("incremental_batch_tags") or []): continue if int(item.get("total_traffic_bytes") or 0) != 0: continue if str(item.get("artifact_status") or "") not in {"missing", "partial"}: continue rows.append(item) rows.sort(key=lambda item: (float(item.get("updated_at") or 0.0), item.get("package_name") or ""), reverse=True) packages = [str(item.get("package_name") or "").strip() for item in rows if str(item.get("package_name") or "").strip()] return packages[:limit] if limit > 0 else packages def main() -> int: parser = argparse.ArgumentParser( description="Rebuild analytics snapshots for zero-traffic packages that already have traffic_count files." ) parser.add_argument("--db-path", default=MONITORING_DB_PATH) parser.add_argument("--traffic-root", default=ANALYTICS_TRAFFIC_ROOT) parser.add_argument("--packages", default="", help="Comma-separated package names. Defaults to zero-traffic packages.") parser.add_argument("--tag", default="", help="Only rebuild packages whose incremental_batch_tag contains this tag.") parser.add_argument("--limit", type=int, default=0) parser.add_argument("--dry-run", action="store_true") args = parser.parse_args() service = AnalyticsService( db_path=args.db_path, traffic_root=args.traffic_root, start_worker=False, ) try: packages = _split_packages(args.packages) or _load_zero_traffic_packages(service, args.tag, args.limit) scanned = 0 matched = 0 rebuilt = 0 skipped = 0 for package_name in packages: scanned += 1 traffic_files = service._find_traffic_files(package_name) if not traffic_files: skipped += 1 continue if not _has_traffic_domains(traffic_files, package_name): skipped += 1 continue matched += 1 if args.dry_run: print(f"DRY-RUN {package_name}: {len(traffic_files)} traffic files") continue summary = service.rebuild_package_now(package_name) rebuilt += 1 print( f"rebuilt {package_name}: files={len(traffic_files)} " f"artifact={summary.get('artifact_status')} bytes={summary.get('total_traffic_bytes')}" ) finally: service.close() print(f"done scanned={scanned} matched={matched} rebuilt={rebuilt} skipped={skipped}") return 0 if __name__ == "__main__": raise SystemExit(main())