729 lines
31 KiB
Python
729 lines
31 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
|
|
"""
|
|
Analyze a single day's app results and export one CSV report.
|
|
|
|
Rules:
|
|
- success + light_restricted => qualified
|
|
- severe_restricted => failed
|
|
- failure breakdown shares follow the dashboard's monitoring view:
|
|
overall_share_percent = failure_duration / total_duration
|
|
error_share_percent = failure_duration / failed_duration
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import csv
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
from collections import Counter
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from typing import Any, Dict, Iterable, List, Optional, Tuple
|
|
|
|
|
|
SCRIPT_DIR = Path(__file__).resolve().parent
|
|
PROJECT_ROOT = SCRIPT_DIR.parent
|
|
if str(PROJECT_ROOT) not in sys.path:
|
|
sys.path.insert(0, str(PROJECT_ROOT))
|
|
|
|
from analytics import ( # noqa: E402
|
|
AnalyticsService,
|
|
_build_legacy_report_row,
|
|
_classify_restriction_status,
|
|
_iter_report_paths,
|
|
_load_task_csv_packages,
|
|
_parse_local_datetime,
|
|
_prefer_legacy_report_candidate,
|
|
_read_dict_rows,
|
|
)
|
|
from config import ( # noqa: E402
|
|
ANALYTICS_TPDPI_APP_LIST,
|
|
ANALYTICS_TPDPI_URL_LIB,
|
|
ANALYTICS_TRAFFIC_ROOT,
|
|
FAILED_TASKS_CSV,
|
|
REPORT_DIR,
|
|
RETRY_TASKS_CSV,
|
|
SUCCESS_TASKS_CSV,
|
|
TASK_CSV_PATH,
|
|
)
|
|
from monitoring import ( # noqa: E402
|
|
SHANGHAI_TZ,
|
|
_classify_failure,
|
|
_humanize_failure_domain,
|
|
_humanize_failure_reason,
|
|
_humanize_failure_subtype,
|
|
)
|
|
|
|
|
|
CSV_COLUMNS = [
|
|
"record_type",
|
|
"date",
|
|
"generated_at",
|
|
"name",
|
|
"label",
|
|
"value",
|
|
"count",
|
|
"share_percent",
|
|
"overall_share_percent",
|
|
"error_share_percent",
|
|
"domain_share_percent",
|
|
"duration_seconds",
|
|
"avg_duration_seconds",
|
|
"qualified",
|
|
"app_name",
|
|
"package_name",
|
|
"report_type",
|
|
"latest_status",
|
|
"restriction_status",
|
|
"retryability",
|
|
"latest_worker_id",
|
|
"latest_test_time",
|
|
"latest_failure_type",
|
|
"latest_task_detail",
|
|
"failure_domain",
|
|
"failure_domain_label",
|
|
"failure_subtype",
|
|
"failure_subtype_label",
|
|
"failure_reason",
|
|
"num_nodes",
|
|
"unique_domain_count",
|
|
"unique_second_level_domain_count",
|
|
"total_traffic_bytes",
|
|
"self_traffic_bytes",
|
|
"server_traffic_bytes",
|
|
"unrecognized_traffic_bytes",
|
|
"self_ratio",
|
|
"recognition_ratio",
|
|
"artifact_status",
|
|
]
|
|
|
|
|
|
def _date_str_from_ts(ts: float) -> str:
|
|
return datetime.fromtimestamp(ts, SHANGHAI_TZ).strftime("%Y-%m-%d")
|
|
|
|
|
|
def _resolve_allowed_packages(task_csv_path: str) -> Optional[set]:
|
|
packages = _load_task_csv_packages(task_csv_path)
|
|
return {item for item in packages if item} or None
|
|
|
|
|
|
def _load_daily_latest_rows(
|
|
*,
|
|
date_str: str,
|
|
success_csv_path: str,
|
|
failed_csv_path: str,
|
|
retry_csv_path: str,
|
|
allowed_packages: Optional[set] = None,
|
|
) -> Dict[str, Dict[str, Any]]:
|
|
latest_rows: Dict[str, Dict[str, Any]] = {}
|
|
report_specs = [
|
|
("success", success_csv_path, True),
|
|
("failed", failed_csv_path, False),
|
|
("retry", retry_csv_path, False),
|
|
]
|
|
for report_type, report_path, allow_glob in report_specs:
|
|
sequence = 0
|
|
for current_path in _iter_report_paths(report_path, allow_glob=allow_glob):
|
|
for row in _read_dict_rows(current_path):
|
|
package_name = str(row.get("包名") or row.get("package_name") or "").strip()
|
|
if not package_name:
|
|
continue
|
|
if allowed_packages is not None and package_name not in allowed_packages:
|
|
continue
|
|
row_time = _parse_local_datetime(row.get("时间") or row.get("test_time") or "")
|
|
if not row_time or _date_str_from_ts(row_time) != date_str:
|
|
continue
|
|
sequence += 1
|
|
sort_key = (row_time, sequence)
|
|
current_row = latest_rows.get(package_name)
|
|
if not _prefer_legacy_report_candidate(current_row, report_type, sort_key):
|
|
continue
|
|
latest_rows[package_name] = {
|
|
"_sort_key": sort_key,
|
|
"_report_type": report_type,
|
|
"package_name": package_name,
|
|
"report_type": report_type,
|
|
**_build_legacy_report_row(row, report_type, row_time),
|
|
}
|
|
for row in latest_rows.values():
|
|
row.pop("_sort_key", None)
|
|
row.pop("_report_type", None)
|
|
return latest_rows
|
|
|
|
|
|
def _build_failure_breakdown(items: List[Dict[str, Any]]) -> Dict[str, Any]:
|
|
total_duration = sum(float(item.get("duration_seconds") or 0.0) for item in items)
|
|
failed_items = [item for item in items if not item["qualified"]]
|
|
failed_duration = sum(float(item.get("duration_seconds") or 0.0) for item in failed_items)
|
|
|
|
domain_buckets: Dict[str, Dict[str, Any]] = {}
|
|
for item in failed_items:
|
|
failure_domain = item["failure_domain"]
|
|
failure_subtype = item["failure_subtype"]
|
|
duration_seconds = float(item.get("duration_seconds") or 0.0)
|
|
reason_label = item["failure_reason"]
|
|
message_label = item["latest_task_detail"] or reason_label
|
|
sample = {
|
|
"app_name": item["app_name"],
|
|
"package_name": item["package_name"],
|
|
"latest_status": item["latest_status"],
|
|
"restriction_status": item["restriction_status"],
|
|
"retryability": item["retryability"],
|
|
"duration_seconds": round(duration_seconds, 2),
|
|
"self_ratio": round(float(item.get("self_ratio") or 0.0), 2),
|
|
"latest_failure_type": item["latest_failure_type"],
|
|
"latest_task_detail": item["latest_task_detail"],
|
|
}
|
|
|
|
domain_entry = domain_buckets.setdefault(
|
|
failure_domain,
|
|
{
|
|
"domain": failure_domain,
|
|
"label": _humanize_failure_domain(failure_domain),
|
|
"duration_seconds": 0.0,
|
|
"task_count": 0,
|
|
"_subtypes": {},
|
|
"_reasons": {},
|
|
"_messages": {},
|
|
"_apps": [],
|
|
},
|
|
)
|
|
domain_entry["duration_seconds"] += duration_seconds
|
|
domain_entry["task_count"] += 1
|
|
domain_entry["_apps"].append(sample)
|
|
|
|
domain_reason = domain_entry["_reasons"].setdefault(
|
|
reason_label,
|
|
{"label": reason_label, "duration_seconds": 0.0, "task_count": 0},
|
|
)
|
|
domain_reason["duration_seconds"] += duration_seconds
|
|
domain_reason["task_count"] += 1
|
|
|
|
domain_message = domain_entry["_messages"].setdefault(
|
|
message_label,
|
|
{"label": message_label, "duration_seconds": 0.0, "task_count": 0},
|
|
)
|
|
domain_message["duration_seconds"] += duration_seconds
|
|
domain_message["task_count"] += 1
|
|
|
|
subtype_entry = domain_entry["_subtypes"].setdefault(
|
|
failure_subtype,
|
|
{
|
|
"domain": failure_domain,
|
|
"subtype": failure_subtype,
|
|
"label": _humanize_failure_subtype(
|
|
failure_domain,
|
|
failure_subtype,
|
|
item["latest_failure_type"],
|
|
item["latest_task_detail"],
|
|
),
|
|
"duration_seconds": 0.0,
|
|
"task_count": 0,
|
|
"_reasons": {},
|
|
"_messages": {},
|
|
"_apps": [],
|
|
},
|
|
)
|
|
subtype_entry["duration_seconds"] += duration_seconds
|
|
subtype_entry["task_count"] += 1
|
|
subtype_entry["_apps"].append(sample)
|
|
|
|
subtype_reason = subtype_entry["_reasons"].setdefault(
|
|
reason_label,
|
|
{"label": reason_label, "duration_seconds": 0.0, "task_count": 0},
|
|
)
|
|
subtype_reason["duration_seconds"] += duration_seconds
|
|
subtype_reason["task_count"] += 1
|
|
|
|
subtype_message = subtype_entry["_messages"].setdefault(
|
|
message_label,
|
|
{"label": message_label, "duration_seconds": 0.0, "task_count": 0},
|
|
)
|
|
subtype_message["duration_seconds"] += duration_seconds
|
|
subtype_message["task_count"] += 1
|
|
|
|
failure_domain_breakdown: List[Dict[str, Any]] = []
|
|
failure_subtype_breakdown: List[Dict[str, Any]] = []
|
|
for domain_entry in sorted(domain_buckets.values(), key=lambda item: item["duration_seconds"], reverse=True):
|
|
domain_duration = float(domain_entry["duration_seconds"])
|
|
subtypes: List[Dict[str, Any]] = []
|
|
for subtype_entry in sorted(domain_entry["_subtypes"].values(), key=lambda item: item["duration_seconds"], reverse=True):
|
|
subtype_duration = float(subtype_entry["duration_seconds"])
|
|
reasons = [
|
|
{
|
|
"label": current["label"],
|
|
"duration_seconds": round(float(current["duration_seconds"]), 2),
|
|
"task_count": int(current["task_count"]),
|
|
"overall_share_percent": round((float(current["duration_seconds"]) / total_duration) * 100, 2)
|
|
if total_duration > 0
|
|
else 0.0,
|
|
"error_share_percent": round((float(current["duration_seconds"]) / failed_duration) * 100, 2)
|
|
if failed_duration > 0
|
|
else 0.0,
|
|
"domain_share_percent": round((float(current["duration_seconds"]) / domain_duration) * 100, 2)
|
|
if domain_duration > 0
|
|
else 0.0,
|
|
"subtype_share_percent": round((float(current["duration_seconds"]) / subtype_duration) * 100, 2)
|
|
if subtype_duration > 0
|
|
else 0.0,
|
|
}
|
|
for current in sorted(subtype_entry["_reasons"].values(), key=lambda item: item["duration_seconds"], reverse=True)
|
|
]
|
|
messages = [
|
|
{
|
|
"label": current["label"],
|
|
"message": current["label"],
|
|
"duration_seconds": round(float(current["duration_seconds"]), 2),
|
|
"task_count": int(current["task_count"]),
|
|
"overall_share_percent": round((float(current["duration_seconds"]) / total_duration) * 100, 2)
|
|
if total_duration > 0
|
|
else 0.0,
|
|
"error_share_percent": round((float(current["duration_seconds"]) / failed_duration) * 100, 2)
|
|
if failed_duration > 0
|
|
else 0.0,
|
|
"domain_share_percent": round((float(current["duration_seconds"]) / domain_duration) * 100, 2)
|
|
if domain_duration > 0
|
|
else 0.0,
|
|
"subtype_share_percent": round((float(current["duration_seconds"]) / subtype_duration) * 100, 2)
|
|
if subtype_duration > 0
|
|
else 0.0,
|
|
}
|
|
for current in sorted(subtype_entry["_messages"].values(), key=lambda item: item["duration_seconds"], reverse=True)
|
|
]
|
|
subtype_item = {
|
|
"domain": subtype_entry["domain"],
|
|
"subtype": subtype_entry["subtype"],
|
|
"label": subtype_entry["label"],
|
|
"duration_seconds": round(subtype_duration, 2),
|
|
"task_count": int(subtype_entry["task_count"]),
|
|
"avg_duration_seconds": round(subtype_duration / subtype_entry["task_count"], 2)
|
|
if subtype_entry["task_count"]
|
|
else 0.0,
|
|
"overall_share_percent": round((subtype_duration / total_duration) * 100, 2)
|
|
if total_duration > 0
|
|
else 0.0,
|
|
"error_share_percent": round((subtype_duration / failed_duration) * 100, 2)
|
|
if failed_duration > 0
|
|
else 0.0,
|
|
"domain_share_percent": round((subtype_duration / domain_duration) * 100, 2)
|
|
if domain_duration > 0
|
|
else 0.0,
|
|
"reasons": reasons,
|
|
"messages": messages,
|
|
"apps": sorted(
|
|
subtype_entry["_apps"],
|
|
key=lambda current: (float(current["duration_seconds"]), current["package_name"]),
|
|
reverse=True,
|
|
),
|
|
}
|
|
subtypes.append(subtype_item)
|
|
failure_subtype_breakdown.append(subtype_item)
|
|
|
|
failure_domain_breakdown.append(
|
|
{
|
|
"domain": domain_entry["domain"],
|
|
"label": domain_entry["label"],
|
|
"duration_seconds": round(domain_duration, 2),
|
|
"task_count": int(domain_entry["task_count"]),
|
|
"overall_share_percent": round((domain_duration / total_duration) * 100, 2) if total_duration > 0 else 0.0,
|
|
"error_share_percent": round((domain_duration / failed_duration) * 100, 2) if failed_duration > 0 else 0.0,
|
|
"share_percent": round((domain_duration / failed_duration) * 100, 2) if failed_duration > 0 else 0.0,
|
|
"subtypes": subtypes,
|
|
"reasons": [
|
|
{
|
|
"label": current["label"],
|
|
"duration_seconds": round(float(current["duration_seconds"]), 2),
|
|
"task_count": int(current["task_count"]),
|
|
"overall_share_percent": round((float(current["duration_seconds"]) / total_duration) * 100, 2)
|
|
if total_duration > 0
|
|
else 0.0,
|
|
"error_share_percent": round((float(current["duration_seconds"]) / failed_duration) * 100, 2)
|
|
if failed_duration > 0
|
|
else 0.0,
|
|
"domain_share_percent": round((float(current["duration_seconds"]) / domain_duration) * 100, 2)
|
|
if domain_duration > 0
|
|
else 0.0,
|
|
}
|
|
for current in sorted(domain_entry["_reasons"].values(), key=lambda item: item["duration_seconds"], reverse=True)
|
|
],
|
|
"messages": [
|
|
{
|
|
"label": current["label"],
|
|
"message": current["label"],
|
|
"duration_seconds": round(float(current["duration_seconds"]), 2),
|
|
"task_count": int(current["task_count"]),
|
|
"overall_share_percent": round((float(current["duration_seconds"]) / total_duration) * 100, 2)
|
|
if total_duration > 0
|
|
else 0.0,
|
|
"error_share_percent": round((float(current["duration_seconds"]) / failed_duration) * 100, 2)
|
|
if failed_duration > 0
|
|
else 0.0,
|
|
"domain_share_percent": round((float(current["duration_seconds"]) / domain_duration) * 100, 2)
|
|
if domain_duration > 0
|
|
else 0.0,
|
|
}
|
|
for current in sorted(domain_entry["_messages"].values(), key=lambda item: item["duration_seconds"], reverse=True)
|
|
],
|
|
"apps": sorted(
|
|
domain_entry["_apps"],
|
|
key=lambda current: (float(current["duration_seconds"]), current["package_name"]),
|
|
reverse=True,
|
|
),
|
|
}
|
|
)
|
|
|
|
failure_reasons = Counter(item["failure_reason"] for item in failed_items if item["failure_reason"])
|
|
return {
|
|
"failure_domain_breakdown": failure_domain_breakdown,
|
|
"failure_subtype_breakdown": failure_subtype_breakdown,
|
|
"failure_reasons": [
|
|
{"label": label, "count": count}
|
|
for label, count in failure_reasons.most_common()
|
|
],
|
|
"failed_task_count": len(failed_items),
|
|
"failed_duration_seconds": round(failed_duration, 2),
|
|
"total_duration_seconds": round(total_duration, 2),
|
|
}
|
|
|
|
|
|
def _report_overview(items: List[Dict[str, Any]]) -> Dict[str, Any]:
|
|
total = len(items)
|
|
success_count = sum(1 for item in items if item["restriction_status"] == "success")
|
|
light_count = sum(1 for item in items if item["restriction_status"] == "light_restricted")
|
|
severe_count = sum(1 for item in items if item["restriction_status"] == "severe_restricted")
|
|
retryable_count = sum(
|
|
1
|
|
for item in items
|
|
if item["restriction_status"] == "severe_restricted" and item["retryability"] == "retryable"
|
|
)
|
|
non_retryable_count = sum(
|
|
1
|
|
for item in items
|
|
if item["restriction_status"] == "severe_restricted" and item["retryability"] == "non_retryable"
|
|
)
|
|
qualified_count = success_count + light_count
|
|
failed_count = severe_count
|
|
total_duration = sum(float(item.get("duration_seconds") or 0.0) for item in items)
|
|
qualified_duration = sum(float(item.get("duration_seconds") or 0.0) for item in items if item["qualified"])
|
|
failed_duration = sum(float(item.get("duration_seconds") or 0.0) for item in items if not item["qualified"])
|
|
return {
|
|
"total_app_count": total,
|
|
"success_count": success_count,
|
|
"light_restricted_count": light_count,
|
|
"failed_count": failed_count,
|
|
"qualified_count": qualified_count,
|
|
"qualified_rate_percent": round((qualified_count / total) * 100, 2) if total > 0 else 0.0,
|
|
"severe_retryable_count": retryable_count,
|
|
"severe_non_retryable_count": non_retryable_count,
|
|
"total_duration_seconds": round(total_duration, 2),
|
|
"qualified_duration_seconds": round(qualified_duration, 2),
|
|
"failed_duration_seconds": round(failed_duration, 2),
|
|
}
|
|
|
|
|
|
def build_daily_quality_report(
|
|
*,
|
|
date_str: str,
|
|
task_csv_path: str,
|
|
success_csv_path: str,
|
|
failed_csv_path: str,
|
|
retry_csv_path: str,
|
|
traffic_root: str,
|
|
app_list_path: str,
|
|
url_lib_path: str,
|
|
) -> Dict[str, Any]:
|
|
allowed_packages = _resolve_allowed_packages(task_csv_path)
|
|
daily_rows = _load_daily_latest_rows(
|
|
date_str=date_str,
|
|
success_csv_path=success_csv_path,
|
|
failed_csv_path=failed_csv_path,
|
|
retry_csv_path=retry_csv_path,
|
|
allowed_packages=allowed_packages,
|
|
)
|
|
if not daily_rows:
|
|
return {
|
|
"date": date_str,
|
|
"generated_at": datetime.now(SHANGHAI_TZ).strftime("%Y-%m-%d %H:%M:%S"),
|
|
"source": {
|
|
"task_csv_path": task_csv_path,
|
|
"success_csv_path": success_csv_path,
|
|
"failed_csv_path": failed_csv_path,
|
|
"retry_csv_path": retry_csv_path,
|
|
"traffic_root": traffic_root,
|
|
"app_list_path": app_list_path,
|
|
"url_lib_path": url_lib_path,
|
|
},
|
|
"overview": _report_overview([]),
|
|
"failure_analysis": _build_failure_breakdown([]),
|
|
"apps": [],
|
|
}
|
|
|
|
with tempfile.TemporaryDirectory(prefix="daily_quality_") as tmp_dir:
|
|
temp_db_path = os.path.join(tmp_dir, "daily_quality.sqlite3")
|
|
service = AnalyticsService(
|
|
db_path=temp_db_path,
|
|
task_csv_path=task_csv_path,
|
|
success_tasks_csv_path=success_csv_path,
|
|
failed_tasks_csv_path=failed_csv_path,
|
|
retry_tasks_csv_path=retry_csv_path,
|
|
traffic_root=traffic_root,
|
|
app_list_path=app_list_path,
|
|
url_lib_path=url_lib_path,
|
|
start_worker=False,
|
|
)
|
|
items: List[Dict[str, Any]] = []
|
|
for package_name, row in sorted(daily_rows.items(), key=lambda item: item[0]):
|
|
summary = service.rebuild_package_now(package_name, latest_task_override=row)
|
|
restriction_status, retryability = _classify_restriction_status(
|
|
summary.get("latest_status", ""),
|
|
summary.get("self_ratio", 0.0),
|
|
summary.get("latest_failure_type", ""),
|
|
)
|
|
qualified = restriction_status in {"success", "light_restricted"}
|
|
failure_domain = ""
|
|
failure_subtype = ""
|
|
if not qualified:
|
|
failure_domain, failure_subtype = _classify_failure(
|
|
summary.get("latest_failure_type"),
|
|
summary.get("latest_task_detail"),
|
|
)
|
|
items.append(
|
|
{
|
|
"app_name": summary.get("app_name") or package_name,
|
|
"package_name": package_name,
|
|
"report_type": row.get("report_type", ""),
|
|
"latest_status": summary.get("latest_status", ""),
|
|
"restriction_status": restriction_status,
|
|
"retryability": retryability,
|
|
"qualified": qualified,
|
|
"latest_worker_id": summary.get("latest_worker_id", ""),
|
|
"latest_test_time": summary.get("latest_test_time", 0.0),
|
|
"latest_failure_type": summary.get("latest_failure_type", ""),
|
|
"latest_task_detail": summary.get("latest_task_detail", ""),
|
|
"duration_seconds": round(float(summary.get("duration_seconds") or 0.0), 2),
|
|
"num_nodes": int(summary.get("num_nodes") or 0),
|
|
"unique_domain_count": int(summary.get("unique_domain_count") or 0),
|
|
"unique_second_level_domain_count": int(summary.get("unique_second_level_domain_count") or 0),
|
|
"total_traffic_bytes": int(summary.get("total_traffic_bytes") or 0),
|
|
"self_traffic_bytes": int(summary.get("self_traffic_bytes") or 0),
|
|
"server_traffic_bytes": int(summary.get("server_traffic_bytes") or 0),
|
|
"unrecognized_traffic_bytes": int(summary.get("unrecognized_traffic_bytes") or 0),
|
|
"self_ratio": round(float(summary.get("self_ratio") or 0.0), 2),
|
|
"recognition_ratio": round(float(summary.get("recognition_ratio") or 0.0), 2),
|
|
"artifact_status": summary.get("artifact_status", "missing"),
|
|
"failure_domain": failure_domain,
|
|
"failure_domain_label": _humanize_failure_domain(failure_domain) if failure_domain else "",
|
|
"failure_subtype": failure_subtype,
|
|
"failure_subtype_label": _humanize_failure_subtype(
|
|
failure_domain,
|
|
failure_subtype,
|
|
summary.get("latest_failure_type"),
|
|
summary.get("latest_task_detail"),
|
|
)
|
|
if failure_domain
|
|
else "",
|
|
"failure_reason": _humanize_failure_reason(
|
|
summary.get("latest_failure_type"),
|
|
summary.get("latest_task_detail"),
|
|
)
|
|
if not qualified
|
|
else "",
|
|
}
|
|
)
|
|
|
|
items.sort(
|
|
key=lambda item: (
|
|
0 if not item["qualified"] else 1,
|
|
-float(item["duration_seconds"]),
|
|
item["package_name"],
|
|
)
|
|
)
|
|
return {
|
|
"date": date_str,
|
|
"generated_at": datetime.now(SHANGHAI_TZ).strftime("%Y-%m-%d %H:%M:%S"),
|
|
"source": {
|
|
"task_csv_path": task_csv_path,
|
|
"success_csv_path": success_csv_path,
|
|
"failed_csv_path": failed_csv_path,
|
|
"retry_csv_path": retry_csv_path,
|
|
"traffic_root": traffic_root,
|
|
"app_list_path": app_list_path,
|
|
"url_lib_path": url_lib_path,
|
|
"package_filter_enabled": bool(allowed_packages),
|
|
},
|
|
"overview": _report_overview(items),
|
|
"failure_analysis": _build_failure_breakdown(items),
|
|
"apps": items,
|
|
}
|
|
|
|
|
|
def build_daily_quality_csv_rows(report: Dict[str, Any]) -> List[Dict[str, Any]]:
|
|
date_str = str(report.get("date") or "")
|
|
generated_at = str(report.get("generated_at") or "")
|
|
overview = report.get("overview") or {}
|
|
failure_analysis = report.get("failure_analysis") or {}
|
|
rows: List[Dict[str, Any]] = []
|
|
|
|
overview_specs = [
|
|
("total_app_count", "应用总数"),
|
|
("success_count", "成功"),
|
|
("light_restricted_count", "轻度受限"),
|
|
("failed_count", "失败"),
|
|
("qualified_count", "合格"),
|
|
("qualified_rate_percent", "合格率"),
|
|
("severe_retryable_count", "严重受限可重试"),
|
|
("severe_non_retryable_count", "严重受限不可重试"),
|
|
("total_duration_seconds", "总耗时"),
|
|
("qualified_duration_seconds", "合格耗时"),
|
|
("failed_duration_seconds", "失败耗时"),
|
|
]
|
|
for key, label in overview_specs:
|
|
rows.append(
|
|
{
|
|
"record_type": "overview",
|
|
"date": date_str,
|
|
"generated_at": generated_at,
|
|
"name": key,
|
|
"label": label,
|
|
"value": overview.get(key, ""),
|
|
}
|
|
)
|
|
|
|
for item in failure_analysis.get("failure_domain_breakdown") or []:
|
|
rows.append(
|
|
{
|
|
"record_type": "failure_domain",
|
|
"date": date_str,
|
|
"generated_at": generated_at,
|
|
"name": item.get("domain", ""),
|
|
"label": item.get("label", ""),
|
|
"count": item.get("task_count", 0),
|
|
"share_percent": item.get("share_percent", 0.0),
|
|
"overall_share_percent": item.get("overall_share_percent", 0.0),
|
|
"error_share_percent": item.get("error_share_percent", 0.0),
|
|
"duration_seconds": item.get("duration_seconds", 0.0),
|
|
}
|
|
)
|
|
|
|
for item in failure_analysis.get("failure_subtype_breakdown") or []:
|
|
rows.append(
|
|
{
|
|
"record_type": "failure_subtype",
|
|
"date": date_str,
|
|
"generated_at": generated_at,
|
|
"name": item.get("subtype", ""),
|
|
"label": item.get("label", ""),
|
|
"count": item.get("task_count", 0),
|
|
"overall_share_percent": item.get("overall_share_percent", 0.0),
|
|
"error_share_percent": item.get("error_share_percent", 0.0),
|
|
"domain_share_percent": item.get("domain_share_percent", 0.0),
|
|
"duration_seconds": item.get("duration_seconds", 0.0),
|
|
"avg_duration_seconds": item.get("avg_duration_seconds", 0.0),
|
|
"failure_domain": item.get("domain", ""),
|
|
}
|
|
)
|
|
|
|
for item in report.get("apps") or []:
|
|
rows.append(
|
|
{
|
|
"record_type": "app",
|
|
"date": date_str,
|
|
"generated_at": generated_at,
|
|
"qualified": "yes" if item.get("qualified") else "no",
|
|
"app_name": item.get("app_name", ""),
|
|
"package_name": item.get("package_name", ""),
|
|
"report_type": item.get("report_type", ""),
|
|
"latest_status": item.get("latest_status", ""),
|
|
"restriction_status": item.get("restriction_status", ""),
|
|
"retryability": item.get("retryability", ""),
|
|
"latest_worker_id": item.get("latest_worker_id", ""),
|
|
"latest_test_time": item.get("latest_test_time", ""),
|
|
"latest_failure_type": item.get("latest_failure_type", ""),
|
|
"latest_task_detail": item.get("latest_task_detail", ""),
|
|
"failure_domain": item.get("failure_domain", ""),
|
|
"failure_domain_label": item.get("failure_domain_label", ""),
|
|
"failure_subtype": item.get("failure_subtype", ""),
|
|
"failure_subtype_label": item.get("failure_subtype_label", ""),
|
|
"failure_reason": item.get("failure_reason", ""),
|
|
"duration_seconds": item.get("duration_seconds", 0.0),
|
|
"num_nodes": item.get("num_nodes", 0),
|
|
"unique_domain_count": item.get("unique_domain_count", 0),
|
|
"unique_second_level_domain_count": item.get("unique_second_level_domain_count", 0),
|
|
"total_traffic_bytes": item.get("total_traffic_bytes", 0),
|
|
"self_traffic_bytes": item.get("self_traffic_bytes", 0),
|
|
"server_traffic_bytes": item.get("server_traffic_bytes", 0),
|
|
"unrecognized_traffic_bytes": item.get("unrecognized_traffic_bytes", 0),
|
|
"self_ratio": item.get("self_ratio", 0.0),
|
|
"recognition_ratio": item.get("recognition_ratio", 0.0),
|
|
"artifact_status": item.get("artifact_status", ""),
|
|
}
|
|
)
|
|
return rows
|
|
|
|
|
|
def write_daily_quality_csv(report: Dict[str, Any], output_path: str) -> None:
|
|
rows = build_daily_quality_csv_rows(report)
|
|
output_dir = os.path.dirname(output_path)
|
|
if output_dir:
|
|
os.makedirs(output_dir, exist_ok=True)
|
|
with open(output_path, "w", newline="", encoding="utf-8-sig") as handle:
|
|
writer = csv.DictWriter(handle, fieldnames=CSV_COLUMNS)
|
|
writer.writeheader()
|
|
for row in rows:
|
|
normalized = {column: row.get(column, "") for column in CSV_COLUMNS}
|
|
writer.writerow(normalized)
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description="Generate a single-day quality analysis CSV report.")
|
|
parser.add_argument("--date", required=True, help="Target date in YYYY-MM-DD format.")
|
|
parser.add_argument(
|
|
"--output",
|
|
default="",
|
|
help="Output CSV path. Default: runtime/<instance>/reports/daily_quality_<date>.csv",
|
|
)
|
|
parser.add_argument("--task-csv", default=TASK_CSV_PATH, help="Package list CSV path.")
|
|
parser.add_argument("--success-csv", default=SUCCESS_TASKS_CSV, help="success_tasks.csv path or glob.")
|
|
parser.add_argument("--failed-csv", default=FAILED_TASKS_CSV, help="failed_tasks.csv path.")
|
|
parser.add_argument("--retry-csv", default=RETRY_TASKS_CSV, help="retry_tasks.csv path.")
|
|
parser.add_argument("--traffic-root", default=ANALYTICS_TRAFFIC_ROOT, help="traffic_data root path.")
|
|
parser.add_argument("--app-list", default=ANALYTICS_TPDPI_APP_LIST, help="TPDPI app list CSV path.")
|
|
parser.add_argument("--url-lib", default=ANALYTICS_TPDPI_URL_LIB, help="TPDPI url lib CSV path.")
|
|
return parser.parse_args()
|
|
|
|
|
|
def main() -> int:
|
|
args = parse_args()
|
|
try:
|
|
datetime.strptime(args.date, "%Y-%m-%d")
|
|
except ValueError:
|
|
print(f"Invalid --date: {args.date}", file=sys.stderr)
|
|
return 2
|
|
|
|
output_path = args.output.strip()
|
|
if not output_path:
|
|
output_path = os.path.join(REPORT_DIR, f"daily_quality_{args.date.replace('-', '')}.csv")
|
|
|
|
report = build_daily_quality_report(
|
|
date_str=args.date,
|
|
task_csv_path=args.task_csv,
|
|
success_csv_path=args.success_csv,
|
|
failed_csv_path=args.failed_csv,
|
|
retry_csv_path=args.retry_csv,
|
|
traffic_root=args.traffic_root,
|
|
app_list_path=args.app_list,
|
|
url_lib_path=args.url_lib,
|
|
)
|
|
|
|
write_daily_quality_csv(report, output_path)
|
|
print(output_path)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|