#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Analyze a single day's app results and export one CSV report. Rules: - success + light_restricted => qualified - severe_restricted => failed - failure breakdown shares follow the dashboard's monitoring view: overall_share_percent = failure_duration / total_duration error_share_percent = failure_duration / failed_duration """ from __future__ import annotations import argparse import csv import os import sys import tempfile from collections import Counter from datetime import datetime from pathlib import Path from typing import Any, Dict, Iterable, List, Optional, Tuple SCRIPT_DIR = Path(__file__).resolve().parent PROJECT_ROOT = SCRIPT_DIR.parent if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) from analytics import ( # noqa: E402 AnalyticsService, _build_legacy_report_row, _classify_restriction_status, _iter_report_paths, _load_task_csv_packages, _parse_local_datetime, _prefer_legacy_report_candidate, _read_dict_rows, ) from config import ( # noqa: E402 ANALYTICS_TPDPI_APP_LIST, ANALYTICS_TPDPI_URL_LIB, ANALYTICS_TRAFFIC_ROOT, FAILED_TASKS_CSV, REPORT_DIR, RETRY_TASKS_CSV, SUCCESS_TASKS_CSV, TASK_CSV_PATH, ) from monitoring import ( # noqa: E402 SHANGHAI_TZ, _classify_failure, _humanize_failure_domain, _humanize_failure_reason, _humanize_failure_subtype, ) CSV_COLUMNS = [ "record_type", "date", "generated_at", "name", "label", "value", "count", "share_percent", "overall_share_percent", "error_share_percent", "domain_share_percent", "duration_seconds", "avg_duration_seconds", "qualified", "app_name", "package_name", "report_type", "latest_status", "restriction_status", "retryability", "latest_worker_id", "latest_test_time", "latest_failure_type", "latest_task_detail", "failure_domain", "failure_domain_label", "failure_subtype", "failure_subtype_label", "failure_reason", "num_nodes", "unique_domain_count", "unique_second_level_domain_count", "total_traffic_bytes", "self_traffic_bytes", "server_traffic_bytes", "unrecognized_traffic_bytes", "self_ratio", "recognition_ratio", "artifact_status", ] def _date_str_from_ts(ts: float) -> str: return datetime.fromtimestamp(ts, SHANGHAI_TZ).strftime("%Y-%m-%d") def _resolve_allowed_packages(task_csv_path: str) -> Optional[set]: packages = _load_task_csv_packages(task_csv_path) return {item for item in packages if item} or None def _load_daily_latest_rows( *, date_str: str, success_csv_path: str, failed_csv_path: str, retry_csv_path: str, allowed_packages: Optional[set] = None, ) -> Dict[str, Dict[str, Any]]: latest_rows: Dict[str, Dict[str, Any]] = {} report_specs = [ ("success", success_csv_path, True), ("failed", failed_csv_path, False), ("retry", retry_csv_path, False), ] for report_type, report_path, allow_glob in report_specs: sequence = 0 for current_path in _iter_report_paths(report_path, allow_glob=allow_glob): for row in _read_dict_rows(current_path): package_name = str(row.get("包名") or row.get("package_name") or "").strip() if not package_name: continue if allowed_packages is not None and package_name not in allowed_packages: continue row_time = _parse_local_datetime(row.get("时间") or row.get("test_time") or "") if not row_time or _date_str_from_ts(row_time) != date_str: continue sequence += 1 sort_key = (row_time, sequence) current_row = latest_rows.get(package_name) if not _prefer_legacy_report_candidate(current_row, report_type, sort_key): continue latest_rows[package_name] = { "_sort_key": sort_key, "_report_type": report_type, "package_name": package_name, "report_type": report_type, **_build_legacy_report_row(row, report_type, row_time), } for row in latest_rows.values(): row.pop("_sort_key", None) row.pop("_report_type", None) return latest_rows def _build_failure_breakdown(items: List[Dict[str, Any]]) -> Dict[str, Any]: total_duration = sum(float(item.get("duration_seconds") or 0.0) for item in items) failed_items = [item for item in items if not item["qualified"]] failed_duration = sum(float(item.get("duration_seconds") or 0.0) for item in failed_items) domain_buckets: Dict[str, Dict[str, Any]] = {} for item in failed_items: failure_domain = item["failure_domain"] failure_subtype = item["failure_subtype"] duration_seconds = float(item.get("duration_seconds") or 0.0) reason_label = item["failure_reason"] message_label = item["latest_task_detail"] or reason_label sample = { "app_name": item["app_name"], "package_name": item["package_name"], "latest_status": item["latest_status"], "restriction_status": item["restriction_status"], "retryability": item["retryability"], "duration_seconds": round(duration_seconds, 2), "self_ratio": round(float(item.get("self_ratio") or 0.0), 2), "latest_failure_type": item["latest_failure_type"], "latest_task_detail": item["latest_task_detail"], } domain_entry = domain_buckets.setdefault( failure_domain, { "domain": failure_domain, "label": _humanize_failure_domain(failure_domain), "duration_seconds": 0.0, "task_count": 0, "_subtypes": {}, "_reasons": {}, "_messages": {}, "_apps": [], }, ) domain_entry["duration_seconds"] += duration_seconds domain_entry["task_count"] += 1 domain_entry["_apps"].append(sample) domain_reason = domain_entry["_reasons"].setdefault( reason_label, {"label": reason_label, "duration_seconds": 0.0, "task_count": 0}, ) domain_reason["duration_seconds"] += duration_seconds domain_reason["task_count"] += 1 domain_message = domain_entry["_messages"].setdefault( message_label, {"label": message_label, "duration_seconds": 0.0, "task_count": 0}, ) domain_message["duration_seconds"] += duration_seconds domain_message["task_count"] += 1 subtype_entry = domain_entry["_subtypes"].setdefault( failure_subtype, { "domain": failure_domain, "subtype": failure_subtype, "label": _humanize_failure_subtype( failure_domain, failure_subtype, item["latest_failure_type"], item["latest_task_detail"], ), "duration_seconds": 0.0, "task_count": 0, "_reasons": {}, "_messages": {}, "_apps": [], }, ) subtype_entry["duration_seconds"] += duration_seconds subtype_entry["task_count"] += 1 subtype_entry["_apps"].append(sample) subtype_reason = subtype_entry["_reasons"].setdefault( reason_label, {"label": reason_label, "duration_seconds": 0.0, "task_count": 0}, ) subtype_reason["duration_seconds"] += duration_seconds subtype_reason["task_count"] += 1 subtype_message = subtype_entry["_messages"].setdefault( message_label, {"label": message_label, "duration_seconds": 0.0, "task_count": 0}, ) subtype_message["duration_seconds"] += duration_seconds subtype_message["task_count"] += 1 failure_domain_breakdown: List[Dict[str, Any]] = [] failure_subtype_breakdown: List[Dict[str, Any]] = [] for domain_entry in sorted(domain_buckets.values(), key=lambda item: item["duration_seconds"], reverse=True): domain_duration = float(domain_entry["duration_seconds"]) subtypes: List[Dict[str, Any]] = [] for subtype_entry in sorted(domain_entry["_subtypes"].values(), key=lambda item: item["duration_seconds"], reverse=True): subtype_duration = float(subtype_entry["duration_seconds"]) reasons = [ { "label": current["label"], "duration_seconds": round(float(current["duration_seconds"]), 2), "task_count": int(current["task_count"]), "overall_share_percent": round((float(current["duration_seconds"]) / total_duration) * 100, 2) if total_duration > 0 else 0.0, "error_share_percent": round((float(current["duration_seconds"]) / failed_duration) * 100, 2) if failed_duration > 0 else 0.0, "domain_share_percent": round((float(current["duration_seconds"]) / domain_duration) * 100, 2) if domain_duration > 0 else 0.0, "subtype_share_percent": round((float(current["duration_seconds"]) / subtype_duration) * 100, 2) if subtype_duration > 0 else 0.0, } for current in sorted(subtype_entry["_reasons"].values(), key=lambda item: item["duration_seconds"], reverse=True) ] messages = [ { "label": current["label"], "message": current["label"], "duration_seconds": round(float(current["duration_seconds"]), 2), "task_count": int(current["task_count"]), "overall_share_percent": round((float(current["duration_seconds"]) / total_duration) * 100, 2) if total_duration > 0 else 0.0, "error_share_percent": round((float(current["duration_seconds"]) / failed_duration) * 100, 2) if failed_duration > 0 else 0.0, "domain_share_percent": round((float(current["duration_seconds"]) / domain_duration) * 100, 2) if domain_duration > 0 else 0.0, "subtype_share_percent": round((float(current["duration_seconds"]) / subtype_duration) * 100, 2) if subtype_duration > 0 else 0.0, } for current in sorted(subtype_entry["_messages"].values(), key=lambda item: item["duration_seconds"], reverse=True) ] subtype_item = { "domain": subtype_entry["domain"], "subtype": subtype_entry["subtype"], "label": subtype_entry["label"], "duration_seconds": round(subtype_duration, 2), "task_count": int(subtype_entry["task_count"]), "avg_duration_seconds": round(subtype_duration / subtype_entry["task_count"], 2) if subtype_entry["task_count"] else 0.0, "overall_share_percent": round((subtype_duration / total_duration) * 100, 2) if total_duration > 0 else 0.0, "error_share_percent": round((subtype_duration / failed_duration) * 100, 2) if failed_duration > 0 else 0.0, "domain_share_percent": round((subtype_duration / domain_duration) * 100, 2) if domain_duration > 0 else 0.0, "reasons": reasons, "messages": messages, "apps": sorted( subtype_entry["_apps"], key=lambda current: (float(current["duration_seconds"]), current["package_name"]), reverse=True, ), } subtypes.append(subtype_item) failure_subtype_breakdown.append(subtype_item) failure_domain_breakdown.append( { "domain": domain_entry["domain"], "label": domain_entry["label"], "duration_seconds": round(domain_duration, 2), "task_count": int(domain_entry["task_count"]), "overall_share_percent": round((domain_duration / total_duration) * 100, 2) if total_duration > 0 else 0.0, "error_share_percent": round((domain_duration / failed_duration) * 100, 2) if failed_duration > 0 else 0.0, "share_percent": round((domain_duration / failed_duration) * 100, 2) if failed_duration > 0 else 0.0, "subtypes": subtypes, "reasons": [ { "label": current["label"], "duration_seconds": round(float(current["duration_seconds"]), 2), "task_count": int(current["task_count"]), "overall_share_percent": round((float(current["duration_seconds"]) / total_duration) * 100, 2) if total_duration > 0 else 0.0, "error_share_percent": round((float(current["duration_seconds"]) / failed_duration) * 100, 2) if failed_duration > 0 else 0.0, "domain_share_percent": round((float(current["duration_seconds"]) / domain_duration) * 100, 2) if domain_duration > 0 else 0.0, } for current in sorted(domain_entry["_reasons"].values(), key=lambda item: item["duration_seconds"], reverse=True) ], "messages": [ { "label": current["label"], "message": current["label"], "duration_seconds": round(float(current["duration_seconds"]), 2), "task_count": int(current["task_count"]), "overall_share_percent": round((float(current["duration_seconds"]) / total_duration) * 100, 2) if total_duration > 0 else 0.0, "error_share_percent": round((float(current["duration_seconds"]) / failed_duration) * 100, 2) if failed_duration > 0 else 0.0, "domain_share_percent": round((float(current["duration_seconds"]) / domain_duration) * 100, 2) if domain_duration > 0 else 0.0, } for current in sorted(domain_entry["_messages"].values(), key=lambda item: item["duration_seconds"], reverse=True) ], "apps": sorted( domain_entry["_apps"], key=lambda current: (float(current["duration_seconds"]), current["package_name"]), reverse=True, ), } ) failure_reasons = Counter(item["failure_reason"] for item in failed_items if item["failure_reason"]) return { "failure_domain_breakdown": failure_domain_breakdown, "failure_subtype_breakdown": failure_subtype_breakdown, "failure_reasons": [ {"label": label, "count": count} for label, count in failure_reasons.most_common() ], "failed_task_count": len(failed_items), "failed_duration_seconds": round(failed_duration, 2), "total_duration_seconds": round(total_duration, 2), } def _report_overview(items: List[Dict[str, Any]]) -> Dict[str, Any]: total = len(items) success_count = sum(1 for item in items if item["restriction_status"] == "success") light_count = sum(1 for item in items if item["restriction_status"] == "light_restricted") severe_count = sum(1 for item in items if item["restriction_status"] == "severe_restricted") retryable_count = sum( 1 for item in items if item["restriction_status"] == "severe_restricted" and item["retryability"] == "retryable" ) non_retryable_count = sum( 1 for item in items if item["restriction_status"] == "severe_restricted" and item["retryability"] == "non_retryable" ) qualified_count = success_count + light_count failed_count = severe_count total_duration = sum(float(item.get("duration_seconds") or 0.0) for item in items) qualified_duration = sum(float(item.get("duration_seconds") or 0.0) for item in items if item["qualified"]) failed_duration = sum(float(item.get("duration_seconds") or 0.0) for item in items if not item["qualified"]) return { "total_app_count": total, "success_count": success_count, "light_restricted_count": light_count, "failed_count": failed_count, "qualified_count": qualified_count, "qualified_rate_percent": round((qualified_count / total) * 100, 2) if total > 0 else 0.0, "severe_retryable_count": retryable_count, "severe_non_retryable_count": non_retryable_count, "total_duration_seconds": round(total_duration, 2), "qualified_duration_seconds": round(qualified_duration, 2), "failed_duration_seconds": round(failed_duration, 2), } def build_daily_quality_report( *, date_str: str, task_csv_path: str, success_csv_path: str, failed_csv_path: str, retry_csv_path: str, traffic_root: str, app_list_path: str, url_lib_path: str, ) -> Dict[str, Any]: allowed_packages = _resolve_allowed_packages(task_csv_path) daily_rows = _load_daily_latest_rows( date_str=date_str, success_csv_path=success_csv_path, failed_csv_path=failed_csv_path, retry_csv_path=retry_csv_path, allowed_packages=allowed_packages, ) if not daily_rows: return { "date": date_str, "generated_at": datetime.now(SHANGHAI_TZ).strftime("%Y-%m-%d %H:%M:%S"), "source": { "task_csv_path": task_csv_path, "success_csv_path": success_csv_path, "failed_csv_path": failed_csv_path, "retry_csv_path": retry_csv_path, "traffic_root": traffic_root, "app_list_path": app_list_path, "url_lib_path": url_lib_path, }, "overview": _report_overview([]), "failure_analysis": _build_failure_breakdown([]), "apps": [], } with tempfile.TemporaryDirectory(prefix="daily_quality_") as tmp_dir: temp_db_path = os.path.join(tmp_dir, "daily_quality.sqlite3") service = AnalyticsService( db_path=temp_db_path, task_csv_path=task_csv_path, success_tasks_csv_path=success_csv_path, failed_tasks_csv_path=failed_csv_path, retry_tasks_csv_path=retry_csv_path, traffic_root=traffic_root, app_list_path=app_list_path, url_lib_path=url_lib_path, start_worker=False, ) items: List[Dict[str, Any]] = [] for package_name, row in sorted(daily_rows.items(), key=lambda item: item[0]): summary = service.rebuild_package_now(package_name, latest_task_override=row) restriction_status, retryability = _classify_restriction_status( summary.get("latest_status", ""), summary.get("self_ratio", 0.0), summary.get("latest_failure_type", ""), ) qualified = restriction_status in {"success", "light_restricted"} failure_domain = "" failure_subtype = "" if not qualified: failure_domain, failure_subtype = _classify_failure( summary.get("latest_failure_type"), summary.get("latest_task_detail"), ) items.append( { "app_name": summary.get("app_name") or package_name, "package_name": package_name, "report_type": row.get("report_type", ""), "latest_status": summary.get("latest_status", ""), "restriction_status": restriction_status, "retryability": retryability, "qualified": qualified, "latest_worker_id": summary.get("latest_worker_id", ""), "latest_test_time": summary.get("latest_test_time", 0.0), "latest_failure_type": summary.get("latest_failure_type", ""), "latest_task_detail": summary.get("latest_task_detail", ""), "duration_seconds": round(float(summary.get("duration_seconds") or 0.0), 2), "num_nodes": int(summary.get("num_nodes") or 0), "unique_domain_count": int(summary.get("unique_domain_count") or 0), "unique_second_level_domain_count": int(summary.get("unique_second_level_domain_count") or 0), "total_traffic_bytes": int(summary.get("total_traffic_bytes") or 0), "self_traffic_bytes": int(summary.get("self_traffic_bytes") or 0), "server_traffic_bytes": int(summary.get("server_traffic_bytes") or 0), "unrecognized_traffic_bytes": int(summary.get("unrecognized_traffic_bytes") or 0), "self_ratio": round(float(summary.get("self_ratio") or 0.0), 2), "recognition_ratio": round(float(summary.get("recognition_ratio") or 0.0), 2), "artifact_status": summary.get("artifact_status", "missing"), "failure_domain": failure_domain, "failure_domain_label": _humanize_failure_domain(failure_domain) if failure_domain else "", "failure_subtype": failure_subtype, "failure_subtype_label": _humanize_failure_subtype( failure_domain, failure_subtype, summary.get("latest_failure_type"), summary.get("latest_task_detail"), ) if failure_domain else "", "failure_reason": _humanize_failure_reason( summary.get("latest_failure_type"), summary.get("latest_task_detail"), ) if not qualified else "", } ) items.sort( key=lambda item: ( 0 if not item["qualified"] else 1, -float(item["duration_seconds"]), item["package_name"], ) ) return { "date": date_str, "generated_at": datetime.now(SHANGHAI_TZ).strftime("%Y-%m-%d %H:%M:%S"), "source": { "task_csv_path": task_csv_path, "success_csv_path": success_csv_path, "failed_csv_path": failed_csv_path, "retry_csv_path": retry_csv_path, "traffic_root": traffic_root, "app_list_path": app_list_path, "url_lib_path": url_lib_path, "package_filter_enabled": bool(allowed_packages), }, "overview": _report_overview(items), "failure_analysis": _build_failure_breakdown(items), "apps": items, } def build_daily_quality_csv_rows(report: Dict[str, Any]) -> List[Dict[str, Any]]: date_str = str(report.get("date") or "") generated_at = str(report.get("generated_at") or "") overview = report.get("overview") or {} failure_analysis = report.get("failure_analysis") or {} rows: List[Dict[str, Any]] = [] overview_specs = [ ("total_app_count", "应用总数"), ("success_count", "成功"), ("light_restricted_count", "轻度受限"), ("failed_count", "失败"), ("qualified_count", "合格"), ("qualified_rate_percent", "合格率"), ("severe_retryable_count", "严重受限可重试"), ("severe_non_retryable_count", "严重受限不可重试"), ("total_duration_seconds", "总耗时"), ("qualified_duration_seconds", "合格耗时"), ("failed_duration_seconds", "失败耗时"), ] for key, label in overview_specs: rows.append( { "record_type": "overview", "date": date_str, "generated_at": generated_at, "name": key, "label": label, "value": overview.get(key, ""), } ) for item in failure_analysis.get("failure_domain_breakdown") or []: rows.append( { "record_type": "failure_domain", "date": date_str, "generated_at": generated_at, "name": item.get("domain", ""), "label": item.get("label", ""), "count": item.get("task_count", 0), "share_percent": item.get("share_percent", 0.0), "overall_share_percent": item.get("overall_share_percent", 0.0), "error_share_percent": item.get("error_share_percent", 0.0), "duration_seconds": item.get("duration_seconds", 0.0), } ) for item in failure_analysis.get("failure_subtype_breakdown") or []: rows.append( { "record_type": "failure_subtype", "date": date_str, "generated_at": generated_at, "name": item.get("subtype", ""), "label": item.get("label", ""), "count": item.get("task_count", 0), "overall_share_percent": item.get("overall_share_percent", 0.0), "error_share_percent": item.get("error_share_percent", 0.0), "domain_share_percent": item.get("domain_share_percent", 0.0), "duration_seconds": item.get("duration_seconds", 0.0), "avg_duration_seconds": item.get("avg_duration_seconds", 0.0), "failure_domain": item.get("domain", ""), } ) for item in report.get("apps") or []: rows.append( { "record_type": "app", "date": date_str, "generated_at": generated_at, "qualified": "yes" if item.get("qualified") else "no", "app_name": item.get("app_name", ""), "package_name": item.get("package_name", ""), "report_type": item.get("report_type", ""), "latest_status": item.get("latest_status", ""), "restriction_status": item.get("restriction_status", ""), "retryability": item.get("retryability", ""), "latest_worker_id": item.get("latest_worker_id", ""), "latest_test_time": item.get("latest_test_time", ""), "latest_failure_type": item.get("latest_failure_type", ""), "latest_task_detail": item.get("latest_task_detail", ""), "failure_domain": item.get("failure_domain", ""), "failure_domain_label": item.get("failure_domain_label", ""), "failure_subtype": item.get("failure_subtype", ""), "failure_subtype_label": item.get("failure_subtype_label", ""), "failure_reason": item.get("failure_reason", ""), "duration_seconds": item.get("duration_seconds", 0.0), "num_nodes": item.get("num_nodes", 0), "unique_domain_count": item.get("unique_domain_count", 0), "unique_second_level_domain_count": item.get("unique_second_level_domain_count", 0), "total_traffic_bytes": item.get("total_traffic_bytes", 0), "self_traffic_bytes": item.get("self_traffic_bytes", 0), "server_traffic_bytes": item.get("server_traffic_bytes", 0), "unrecognized_traffic_bytes": item.get("unrecognized_traffic_bytes", 0), "self_ratio": item.get("self_ratio", 0.0), "recognition_ratio": item.get("recognition_ratio", 0.0), "artifact_status": item.get("artifact_status", ""), } ) return rows def write_daily_quality_csv(report: Dict[str, Any], output_path: str) -> None: rows = build_daily_quality_csv_rows(report) output_dir = os.path.dirname(output_path) if output_dir: os.makedirs(output_dir, exist_ok=True) with open(output_path, "w", newline="", encoding="utf-8-sig") as handle: writer = csv.DictWriter(handle, fieldnames=CSV_COLUMNS) writer.writeheader() for row in rows: normalized = {column: row.get(column, "") for column in CSV_COLUMNS} writer.writerow(normalized) def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Generate a single-day quality analysis CSV report.") parser.add_argument("--date", required=True, help="Target date in YYYY-MM-DD format.") parser.add_argument( "--output", default="", help="Output CSV path. Default: runtime//reports/daily_quality_.csv", ) parser.add_argument("--task-csv", default=TASK_CSV_PATH, help="Package list CSV path.") parser.add_argument("--success-csv", default=SUCCESS_TASKS_CSV, help="success_tasks.csv path or glob.") parser.add_argument("--failed-csv", default=FAILED_TASKS_CSV, help="failed_tasks.csv path.") parser.add_argument("--retry-csv", default=RETRY_TASKS_CSV, help="retry_tasks.csv path.") parser.add_argument("--traffic-root", default=ANALYTICS_TRAFFIC_ROOT, help="traffic_data root path.") parser.add_argument("--app-list", default=ANALYTICS_TPDPI_APP_LIST, help="TPDPI app list CSV path.") parser.add_argument("--url-lib", default=ANALYTICS_TPDPI_URL_LIB, help="TPDPI url lib CSV path.") return parser.parse_args() def main() -> int: args = parse_args() try: datetime.strptime(args.date, "%Y-%m-%d") except ValueError: print(f"Invalid --date: {args.date}", file=sys.stderr) return 2 output_path = args.output.strip() if not output_path: output_path = os.path.join(REPORT_DIR, f"daily_quality_{args.date.replace('-', '')}.csv") report = build_daily_quality_report( date_str=args.date, task_csv_path=args.task_csv, success_csv_path=args.success_csv, failed_csv_path=args.failed_csv, retry_csv_path=args.retry_csv, traffic_root=args.traffic_root, app_list_path=args.app_list, url_lib_path=args.url_lib, ) write_daily_quality_csv(report, output_path) print(output_path) return 0 if __name__ == "__main__": raise SystemExit(main())