110 lines
4.2 KiB
Python
110 lines
4.2 KiB
Python
#!/usr/bin/env python3
|
|
import argparse
|
|
import os
|
|
import sys
|
|
from typing import List
|
|
|
|
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
if REPO_ROOT not in sys.path:
|
|
sys.path.insert(0, REPO_ROOT)
|
|
|
|
from analytics import AnalyticsService
|
|
from config import ANALYTICS_TRAFFIC_ROOT, MONITORING_DB_PATH
|
|
|
|
|
|
def _split_packages(raw: str) -> List[str]:
|
|
return [item.strip() for item in str(raw or "").split(",") if item.strip()]
|
|
|
|
|
|
def _has_traffic_domains(traffic_files: List[str], package_name: str) -> bool:
|
|
"""Check whether any traffic file contains at least one valid traffic entry for the package."""
|
|
for file_path in traffic_files:
|
|
try:
|
|
with open(file_path, "r", encoding="utf-8", errors="ignore") as handle:
|
|
for raw_line in handle:
|
|
line = raw_line.strip()
|
|
if not line:
|
|
continue
|
|
fields = [item.strip() for item in line.split(",")]
|
|
if len(fields) < 7:
|
|
continue
|
|
if fields[0].strip() == package_name:
|
|
return True
|
|
except OSError:
|
|
continue
|
|
return False
|
|
|
|
|
|
def _load_zero_traffic_packages(service: AnalyticsService, tag: str = "", limit: int = 0) -> List[str]:
|
|
normalized_tag = str(tag or "").strip()
|
|
with service.repo._connect() as connection:
|
|
summaries = service.repo._list_catalog_summaries(connection)
|
|
rows = []
|
|
for item in summaries:
|
|
if not item.get("catalog_active"):
|
|
continue
|
|
if normalized_tag and normalized_tag not in (item.get("incremental_batch_tags") or []):
|
|
continue
|
|
if int(item.get("total_traffic_bytes") or 0) != 0:
|
|
continue
|
|
if str(item.get("artifact_status") or "") not in {"missing", "partial"}:
|
|
continue
|
|
rows.append(item)
|
|
rows.sort(key=lambda item: (float(item.get("updated_at") or 0.0), item.get("package_name") or ""), reverse=True)
|
|
packages = [str(item.get("package_name") or "").strip() for item in rows if str(item.get("package_name") or "").strip()]
|
|
return packages[:limit] if limit > 0 else packages
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(
|
|
description="Rebuild analytics snapshots for zero-traffic packages that already have traffic_count files."
|
|
)
|
|
parser.add_argument("--db-path", default=MONITORING_DB_PATH)
|
|
parser.add_argument("--traffic-root", default=ANALYTICS_TRAFFIC_ROOT)
|
|
parser.add_argument("--packages", default="", help="Comma-separated package names. Defaults to zero-traffic packages.")
|
|
parser.add_argument("--tag", default="", help="Only rebuild packages whose incremental_batch_tag contains this tag.")
|
|
parser.add_argument("--limit", type=int, default=0)
|
|
parser.add_argument("--dry-run", action="store_true")
|
|
args = parser.parse_args()
|
|
|
|
service = AnalyticsService(
|
|
db_path=args.db_path,
|
|
traffic_root=args.traffic_root,
|
|
start_worker=False,
|
|
)
|
|
try:
|
|
packages = _split_packages(args.packages) or _load_zero_traffic_packages(service, args.tag, args.limit)
|
|
|
|
scanned = 0
|
|
matched = 0
|
|
rebuilt = 0
|
|
skipped = 0
|
|
for package_name in packages:
|
|
scanned += 1
|
|
traffic_files = service._find_traffic_files(package_name)
|
|
if not traffic_files:
|
|
skipped += 1
|
|
continue
|
|
if not _has_traffic_domains(traffic_files, package_name):
|
|
skipped += 1
|
|
continue
|
|
matched += 1
|
|
if args.dry_run:
|
|
print(f"DRY-RUN {package_name}: {len(traffic_files)} traffic files")
|
|
continue
|
|
summary = service.rebuild_package_now(package_name)
|
|
rebuilt += 1
|
|
print(
|
|
f"rebuilt {package_name}: files={len(traffic_files)} "
|
|
f"artifact={summary.get('artifact_status')} bytes={summary.get('total_traffic_bytes')}"
|
|
)
|
|
finally:
|
|
service.close()
|
|
|
|
print(f"done scanned={scanned} matched={matched} rebuilt={rebuilt} skipped={skipped}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|