1068 lines
43 KiB
Python
1068 lines
43 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
本地 APK 注册表
|
||
|
||
跟踪本地已缓存的 APK 信息，提供检查 APK 是否足够新的方法。
|
||
比较逻辑：download_date >= last_updated 表示此 APK 可用。
|
||
|
||
包含以下子模块：
|
||
- apk_registry: 本地 APK 文件注册 (package_name -> download_date / local_dir / files)
|
||
- apk_pending_download: 等待下载的包队列
|
||
- app_download_record: 下载结果跟踪 (per-package, per-country, per-source)
|
||
|
||
数据存储在 monitoring.sqlite3 中。
|
||
表 schema 由 monitoring.py 的 MonitoringRepository._initialize_schema() 管理。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import os
|
||
import sqlite3
|
||
import sys
|
||
import threading
|
||
import time
|
||
from contextlib import contextmanager
|
||
from datetime import datetime
|
||
from typing import Any, Dict, List, Optional
|
||
|
||
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||
if PROJECT_ROOT not in sys.path:
|
||
sys.path.insert(0, PROJECT_ROOT)
|
||
|
||
from config import MONITORING_DB_PATH, APK_LOCAL_STORAGE_DIR, APK_SMB_DIR
|
||
|
||
_DEFAULT_SMB_DIR = ""
|
||
|
||
|
||
def _parse_date(value: Any) -> Optional[datetime]:
|
||
text = str(value or "").strip()
|
||
if not text:
|
||
return None
|
||
for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d", "%Y/%m/%d", "%Y-%m"):
|
||
try:
|
||
return datetime.strptime(text, fmt)
|
||
except (ValueError, IndexError):
|
||
continue
|
||
return None
|
||
|
||
|
||
def _last_updated_ts(value: Any) -> float:
|
||
dt = _parse_date(value)
|
||
return dt.timestamp() if dt else 0.0
|
||
|
||
|
||
def _is_apk_stale(download_date: Any, last_updated: Any) -> bool:
|
||
return _last_updated_ts(download_date) < _last_updated_ts(last_updated)
|
||
|
||
|
||
def _smb_path(package_name: str, smb_root: str = "") -> str:
|
||
root = str(smb_root or APK_SMB_DIR or _DEFAULT_SMB_DIR).rstrip("\\")
|
||
if not root:
|
||
return ""
|
||
return f"{root}\\{package_name}"
|
||
|
||
|
||
class ApkRegistry:
|
||
def __init__(self, db_path: str = "", storage_dir: str = "",
|
||
smb_dir: str = ""):
|
||
self.db_path = db_path or MONITORING_DB_PATH
|
||
self.storage_dir = storage_dir or APK_LOCAL_STORAGE_DIR
|
||
self.smb_dir = smb_dir or APK_SMB_DIR or _DEFAULT_SMB_DIR
|
||
self._write_lock = threading.Lock()
|
||
self._ensure_tables()
|
||
|
||
@contextmanager
|
||
def _connect(self):
|
||
"""每次操作使用独立连接，避免跨线程共享连接导致游标状态串扰。"""
|
||
connection = sqlite3.connect(self.db_path, timeout=30)
|
||
connection.row_factory = sqlite3.Row
|
||
connection.execute("PRAGMA busy_timeout=30000")
|
||
connection.execute("PRAGMA synchronous=NORMAL")
|
||
connection.execute("PRAGMA journal_mode=WAL")
|
||
try:
|
||
yield connection
|
||
connection.commit()
|
||
except Exception:
|
||
try:
|
||
connection.rollback()
|
||
except Exception:
|
||
pass
|
||
raise
|
||
finally:
|
||
connection.close()
|
||
|
||
def _ensure_tables(self) -> None:
|
||
"""确保 apk_registry、apk_pending_download、app_download_record 表存在。
|
||
|
||
通常由 MonitoringRepository._initialize_schema() 创建，
|
||
此处做兜底确认，避免单独使用 ApkRegistry 时表不存在。
|
||
"""
|
||
os.makedirs(os.path.dirname(self.db_path) or ".", exist_ok=True)
|
||
with self._connect() as conn:
|
||
conn.executescript(
|
||
"""
|
||
CREATE TABLE IF NOT EXISTS apk_registry (
|
||
package_name TEXT PRIMARY KEY,
|
||
download_date TEXT NOT NULL,
|
||
download_time REAL NOT NULL,
|
||
local_dir TEXT,
|
||
smb_dir TEXT,
|
||
version_name TEXT,
|
||
source TEXT,
|
||
apk_files_json TEXT,
|
||
updated_at REAL NOT NULL
|
||
);
|
||
CREATE TABLE IF NOT EXISTS apk_pending_download (
|
||
package_name TEXT PRIMARY KEY,
|
||
created_at REAL NOT NULL
|
||
);
|
||
CREATE TABLE IF NOT EXISTS app_download_record (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
package_name TEXT NOT NULL,
|
||
country_code TEXT NOT NULL DEFAULT '',
|
||
app_name TEXT DEFAULT '',
|
||
last_updated TEXT NOT NULL DEFAULT '',
|
||
downloads INTEGER DEFAULT 0,
|
||
|
||
worker_play_status TEXT NOT NULL DEFAULT 'pending',
|
||
worker_play_error TEXT,
|
||
worker_play_date TEXT,
|
||
|
||
us_play_status TEXT NOT NULL DEFAULT 'pending',
|
||
us_play_error TEXT,
|
||
us_play_date TEXT,
|
||
|
||
us_aurora_status TEXT NOT NULL DEFAULT 'pending',
|
||
us_aurora_error TEXT,
|
||
us_aurora_date TEXT,
|
||
|
||
apkpure_status TEXT NOT NULL DEFAULT 'pending',
|
||
apkpure_error TEXT,
|
||
apkpure_date TEXT,
|
||
|
||
overall_status TEXT NOT NULL DEFAULT 'pending',
|
||
overall_error TEXT,
|
||
apk_version_name TEXT DEFAULT '',
|
||
|
||
local_apk_dir TEXT DEFAULT '',
|
||
local_apk_file_count INTEGER DEFAULT 0,
|
||
local_apk_is_stale INTEGER DEFAULT 1,
|
||
|
||
created_at REAL NOT NULL,
|
||
updated_at REAL NOT NULL,
|
||
|
||
UNIQUE(package_name, country_code)
|
||
);
|
||
CREATE INDEX IF NOT EXISTS idx_download_record_pkg
|
||
ON app_download_record(package_name);
|
||
CREATE INDEX IF NOT EXISTS idx_download_record_country
|
||
ON app_download_record(country_code);
|
||
CREATE INDEX IF NOT EXISTS idx_download_record_overall
|
||
ON app_download_record(overall_status);
|
||
CREATE INDEX IF NOT EXISTS idx_download_record_stale
|
||
ON app_download_record(local_apk_is_stale);
|
||
"""
|
||
)
|
||
# 为已存在的表安全添加 apkpure 列（列已存在时忽略错误）
|
||
for col_def in (
|
||
"apkpure_status TEXT NOT NULL DEFAULT 'pending'",
|
||
"apkpure_error TEXT",
|
||
"apkpure_date TEXT",
|
||
):
|
||
try:
|
||
conn.execute(
|
||
f"ALTER TABLE app_download_record ADD COLUMN {col_def}"
|
||
)
|
||
except sqlite3.OperationalError:
|
||
pass # 列已存在，忽略
|
||
|
||
# ── query ──────────────────────────────────────────────────────
|
||
|
||
def get(self, package_name: str) -> Optional[Dict[str, Any]]:
|
||
with self._connect() as conn:
|
||
row = conn.execute(
|
||
"SELECT * FROM apk_registry WHERE package_name = ?",
|
||
(package_name,),
|
||
).fetchone()
|
||
if not row:
|
||
return None
|
||
return self._row_to_entry(row)
|
||
|
||
def get_local_apk_dir(self, package_name: str) -> Optional[str]:
|
||
entry = self.get(package_name)
|
||
if not entry:
|
||
return None
|
||
local_dir = entry.get("local_dir", "")
|
||
if local_dir and os.path.isdir(local_dir):
|
||
return local_dir
|
||
pkg_dir = os.path.join(self.storage_dir, package_name)
|
||
if os.path.isdir(pkg_dir):
|
||
return pkg_dir
|
||
return None
|
||
|
||
def get_smb_apk_dir(self, package_name: str) -> Optional[str]:
|
||
"""返回 Windows Worker 可访问的 SMB 路径。"""
|
||
entry = self.get(package_name)
|
||
if not entry:
|
||
return None
|
||
smb = entry.get("smb_dir", "")
|
||
if smb:
|
||
return smb
|
||
return _smb_path(package_name, self.smb_dir) or None
|
||
|
||
def is_fresh_enough(self, package_name: str, last_updated: Any) -> bool:
|
||
with self._connect() as conn:
|
||
row = conn.execute(
|
||
"SELECT download_date FROM apk_registry WHERE package_name = ?",
|
||
(package_name,),
|
||
).fetchone()
|
||
if not row:
|
||
return False
|
||
download_date = row["download_date"] or ""
|
||
if not download_date:
|
||
return False
|
||
download_ts = _last_updated_ts(download_date)
|
||
last_ts = _last_updated_ts(last_updated)
|
||
return download_ts >= last_ts
|
||
|
||
def list_all_packages(self) -> List[str]:
|
||
with self._connect() as conn:
|
||
rows = conn.execute(
|
||
"SELECT package_name FROM apk_registry ORDER BY package_name"
|
||
).fetchall()
|
||
return [row["package_name"] for row in rows]
|
||
|
||
def get_download_time(self, package_name: str) -> float:
|
||
with self._connect() as conn:
|
||
row = conn.execute(
|
||
"SELECT download_time FROM apk_registry WHERE package_name = ?",
|
||
(package_name,),
|
||
).fetchone()
|
||
if not row:
|
||
return 0.0
|
||
return float(row["download_time"] or 0.0)
|
||
|
||
# ── pending download tracking ──────────────────────────────────
|
||
|
||
def mark_pending_download(self, package_name: str) -> None:
|
||
with self._write_lock, self._connect() as conn:
|
||
conn.execute(
|
||
"INSERT OR IGNORE INTO apk_pending_download (package_name, created_at) VALUES (?, ?)",
|
||
(package_name, time.time()),
|
||
)
|
||
|
||
def clear_pending_download(self, package_name: str) -> None:
|
||
with self._write_lock, self._connect() as conn:
|
||
conn.execute(
|
||
"DELETE FROM apk_pending_download WHERE package_name = ?",
|
||
(package_name,),
|
||
)
|
||
|
||
def is_pending_download(self, package_name: str) -> bool:
|
||
with self._connect() as conn:
|
||
row = conn.execute(
|
||
"SELECT 1 FROM apk_pending_download WHERE package_name = ?",
|
||
(package_name,),
|
||
).fetchone()
|
||
return bool(row)
|
||
|
||
def get_pending_download_packages(self) -> List[str]:
|
||
with self._connect() as conn:
|
||
rows = conn.execute(
|
||
"SELECT package_name FROM apk_pending_download"
|
||
).fetchall()
|
||
return [row["package_name"] for row in rows]
|
||
|
||
def get_pending_that_are_available(self) -> List[str]:
|
||
"""返回已标记为 pending 且在 apk_registry 中有记录（有 local_dir）的包名。"""
|
||
with self._connect() as conn:
|
||
rows = conn.execute(
|
||
"""
|
||
SELECT p.package_name
|
||
FROM apk_pending_download p
|
||
JOIN apk_registry r ON p.package_name = r.package_name
|
||
WHERE r.local_dir IS NOT NULL AND r.local_dir != ''
|
||
""",
|
||
).fetchall()
|
||
return [row["package_name"] for row in rows]
|
||
|
||
def get_all_pending_downloads(self) -> List[str]:
|
||
with self._connect() as conn:
|
||
rows = conn.execute(
|
||
"SELECT DISTINCT package_name FROM apk_pending_download"
|
||
).fetchall()
|
||
return [row["package_name"] for row in rows]
|
||
|
||
def has_failed_with_same_version(self, package_name: str, last_updated: str) -> bool:
|
||
with self._connect() as conn:
|
||
row = conn.execute(
|
||
"""
|
||
SELECT 1 FROM app_download_record
|
||
WHERE package_name = ?
|
||
AND last_updated = ?
|
||
AND (worker_play_status = 'failed'
|
||
OR us_play_status = 'failed'
|
||
OR us_aurora_status = 'failed'
|
||
OR apkpure_status = 'failed')
|
||
LIMIT 1
|
||
""",
|
||
(package_name, last_updated),
|
||
).fetchone()
|
||
return bool(row)
|
||
|
||
def verify_apk_versions(self, package_name: str) -> bool:
|
||
pkg_dir = os.path.join(self.storage_dir, package_name)
|
||
base = os.path.join(pkg_dir, "base.apk")
|
||
if not os.path.isfile(base):
|
||
return True
|
||
try:
|
||
import subprocess, re
|
||
from log_manager import logger
|
||
out = subprocess.check_output(
|
||
["aapt", "dump", "badging", base],
|
||
stderr=subprocess.DEVNULL, timeout=15,
|
||
).decode()
|
||
m = re.search(r"versionCode='(\d+)'", out)
|
||
base_vc = int(m.group(1)) if m else None
|
||
if base_vc is None:
|
||
return True
|
||
except Exception:
|
||
return True
|
||
|
||
for f in os.listdir(pkg_dir):
|
||
if not f.startswith("split_") or not f.endswith(".apk"):
|
||
continue
|
||
fp = os.path.join(pkg_dir, f)
|
||
try:
|
||
out = subprocess.check_output(
|
||
["aapt", "dump", "badging", fp],
|
||
stderr=subprocess.DEVNULL, timeout=15,
|
||
).decode()
|
||
m = re.search(r"versionCode='(\d+)'", out)
|
||
split_vc = int(m.group(1)) if m else None
|
||
if split_vc is not None and split_vc != base_vc:
|
||
logger.warning("[APK] 版本不匹配 %s: base=%d %s=%d",
|
||
package_name, base_vc, f, split_vc)
|
||
return False
|
||
except Exception:
|
||
pass
|
||
return True
|
||
|
||
# ── update ─────────────────────────────────────────────────────
|
||
|
||
def mark_available(self, package_name: str, download_date: str,
|
||
local_dir: str = "", version_name: str = "",
|
||
apk_files: Optional[List[Dict[str, Any]]] = None,
|
||
source: str = "") -> None:
|
||
apk_files_json = json.dumps(apk_files or [], ensure_ascii=False)
|
||
smb_dir = _smb_path(package_name, self.smb_dir)
|
||
now = time.time()
|
||
with self._write_lock, self._connect() as conn:
|
||
conn.execute(
|
||
"""
|
||
INSERT INTO apk_registry (
|
||
package_name, download_date, download_time, local_dir,
|
||
smb_dir, version_name, source, apk_files_json, updated_at
|
||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||
ON CONFLICT(package_name) DO UPDATE SET
|
||
download_date = excluded.download_date,
|
||
download_time = excluded.download_time,
|
||
local_dir = excluded.local_dir,
|
||
smb_dir = excluded.smb_dir,
|
||
version_name = excluded.version_name,
|
||
source = excluded.source,
|
||
apk_files_json = excluded.apk_files_json,
|
||
updated_at = excluded.updated_at
|
||
""",
|
||
(package_name, download_date, now, local_dir,
|
||
smb_dir, version_name, source or "", apk_files_json, now),
|
||
)
|
||
# 一旦本地有 APK，清除 pending 状态
|
||
conn.execute(
|
||
"DELETE FROM apk_pending_download WHERE package_name = ?",
|
||
(package_name,),
|
||
)
|
||
# 同步刷新 app_download_record 的 stale 标志
|
||
self._sync_stale_for_package(conn, package_name, download_date)
|
||
|
||
def mark_unavailable(self, package_name: str) -> None:
|
||
with self._write_lock, self._connect() as conn:
|
||
conn.execute(
|
||
"DELETE FROM apk_registry WHERE package_name = ?",
|
||
(package_name,),
|
||
)
|
||
conn.execute(
|
||
"UPDATE app_download_record SET local_apk_is_stale = 1, "
|
||
"local_apk_file_count = 0, updated_at = ? "
|
||
"WHERE package_name = ?",
|
||
(time.time(), package_name),
|
||
)
|
||
|
||
def _sync_stale_for_package(self, conn: sqlite3.Connection,
|
||
package_name: str,
|
||
download_date: str) -> None:
|
||
rows = conn.execute(
|
||
"SELECT country_code, last_updated FROM app_download_record "
|
||
"WHERE package_name = ?",
|
||
(package_name,),
|
||
).fetchall()
|
||
now = time.time()
|
||
for row in rows:
|
||
is_stale = 1 if _is_apk_stale(download_date, row["last_updated"]) else 0
|
||
conn.execute(
|
||
"""
|
||
UPDATE app_download_record
|
||
SET local_apk_is_stale = ?,
|
||
local_apk_file_count = (
|
||
SELECT COUNT(*)
|
||
FROM json_each(
|
||
COALESCE(
|
||
(SELECT apk_files_json FROM apk_registry
|
||
WHERE package_name = app_download_record.package_name),
|
||
'[]'
|
||
)
|
||
)
|
||
),
|
||
local_apk_dir = ?,
|
||
apk_version_name = COALESCE(
|
||
(SELECT version_name FROM apk_registry
|
||
WHERE package_name = app_download_record.package_name),
|
||
apk_version_name
|
||
),
|
||
updated_at = ?
|
||
WHERE package_name = ? AND country_code = ?
|
||
""",
|
||
(is_stale, self.storage_dir, now,
|
||
package_name, row["country_code"]),
|
||
)
|
||
|
||
def import_from_download_result(self, package_name: str,
|
||
result: Dict[str, Any],
|
||
local_dir: str = "",
|
||
source: str = "minio") -> None:
|
||
self.mark_available(
|
||
package_name=package_name,
|
||
download_date=result.get("download_date", ""),
|
||
local_dir=local_dir or self.storage_dir,
|
||
version_name=result.get("version_name", ""),
|
||
apk_files=result.get("files", result.get("uploaded_files")),
|
||
source=source,
|
||
)
|
||
|
||
# ── stale detection ────────────────────────────────────────────
|
||
|
||
def get_stale_packages(self, catalog_last_updated: Dict[str, str]) -> List[str]:
|
||
stale = []
|
||
for package_name, last_updated in catalog_last_updated.items():
|
||
if not self.is_fresh_enough(package_name, last_updated):
|
||
stale.append(package_name)
|
||
return stale
|
||
|
||
# ── download record management ─────────────────────────────────
|
||
|
||
def ensure_download_record(self, package_name: str, country_code: str,
|
||
app_name: str = "", last_updated: str = "",
|
||
downloads: int = 0) -> None:
|
||
"""懒初始化：从新目录任务信息同步一条下载记录。
|
||
|
||
仅在 Redis 任务队列中需要处理此包时由中控调用。
|
||
如果记录已存在则更新 catalog 字段（last_updated / downloads）。
|
||
"""
|
||
package_name = str(package_name or "").strip()
|
||
country_code = str(country_code or "").strip()
|
||
if not package_name:
|
||
return
|
||
now = time.time()
|
||
with self._write_lock, self._connect() as conn:
|
||
conn.execute(
|
||
"""
|
||
INSERT INTO app_download_record (
|
||
package_name, country_code, app_name, last_updated,
|
||
downloads, created_at, updated_at
|
||
) VALUES (?, ?, ?, ?, ?, ?, ?)
|
||
ON CONFLICT(package_name, country_code) DO UPDATE SET
|
||
app_name = COALESCE(excluded.app_name, app_download_record.app_name),
|
||
last_updated = excluded.last_updated,
|
||
downloads = excluded.downloads,
|
||
updated_at = excluded.updated_at
|
||
""",
|
||
(package_name, country_code, app_name or "",
|
||
last_updated or "", int(downloads or 0), now, now),
|
||
)
|
||
|
||
def update_source_result(self, package_name: str, country_code: str,
|
||
source: str, status: str, error: str = "",
|
||
download_date: str = "") -> None:
|
||
"""更新单个下载源 (worker_play / us_play / us_aurora / apkpure) 的结果。"""
|
||
package_name = str(package_name or "").strip()
|
||
country_code = str(country_code or "").strip()
|
||
source = str(source or "").strip().lower()
|
||
status = str(status or "pending").strip()
|
||
now = time.time()
|
||
|
||
source_col_map = {
|
||
"worker_play": ("worker_play_status", "worker_play_error", "worker_play_date"),
|
||
"us_play": ("us_play_status", "us_play_error", "us_play_date"),
|
||
"us_aurora": ("us_aurora_status", "us_aurora_error", "us_aurora_date"),
|
||
"apkpure": ("apkpure_status", "apkpure_error", "apkpure_date"),
|
||
}
|
||
if source not in source_col_map:
|
||
return
|
||
|
||
status_col, error_col, date_col = source_col_map[source]
|
||
with self._write_lock, self._connect() as conn:
|
||
conn.execute(
|
||
f"""
|
||
UPDATE app_download_record
|
||
SET {status_col} = ?,
|
||
{error_col} = CASE WHEN ? = 'failed' THEN ? ELSE NULL END,
|
||
{date_col} = CASE WHEN ? = 'success' THEN ? ELSE NULL END,
|
||
updated_at = ?
|
||
WHERE package_name = ? AND country_code = ?
|
||
""",
|
||
(status, status, error or "",
|
||
status, download_date or "", now,
|
||
package_name, country_code),
|
||
)
|
||
self._recompute_overall(conn, package_name, country_code)
|
||
|
||
def _recompute_overall(self, conn: sqlite3.Connection,
|
||
package_name: str, country_code: str) -> None:
|
||
"""根据四个来源的状态计算 overall_status 和 overall_error。"""
|
||
row = conn.execute(
|
||
"""
|
||
SELECT worker_play_status, us_play_status, us_aurora_status,
|
||
apkpure_status,
|
||
worker_play_error, us_play_error, us_aurora_error,
|
||
apkpure_error
|
||
FROM app_download_record
|
||
WHERE package_name = ? AND country_code = ?
|
||
""",
|
||
(package_name, country_code),
|
||
).fetchone()
|
||
if not row:
|
||
return
|
||
|
||
wp_s, up_s, ua_s, ap_s = (
|
||
row["worker_play_status"] or "pending",
|
||
row["us_play_status"] or "pending",
|
||
row["us_aurora_status"] or "pending",
|
||
row["apkpure_status"] or "pending",
|
||
)
|
||
successes = sum(1 for s in (wp_s, up_s, ua_s, ap_s) if s == "success")
|
||
failures = [(col, e) for s, e, col in (
|
||
(wp_s, row["worker_play_error"], "worker_play"),
|
||
(up_s, row["us_play_error"], "us_play"),
|
||
(ua_s, row["us_aurora_error"], "us_aurora"),
|
||
(ap_s, row["apkpure_error"], "apkpure"),
|
||
) if s == "failed"]
|
||
|
||
if successes >= 1:
|
||
overall_status = "success"
|
||
overall_error = ""
|
||
elif failures:
|
||
overall_status = "all_failed"
|
||
error_parts = []
|
||
for src, err in failures:
|
||
label = {
|
||
"worker_play": "Worker Play",
|
||
"us_play": "US Play",
|
||
"us_aurora": "US Aurora",
|
||
"apkpure": "APKPure",
|
||
}.get(src, src)
|
||
error_parts.append(f"{label}: {err or 'unknown'}")
|
||
overall_error = "; ".join(error_parts)
|
||
else:
|
||
overall_status = "pending"
|
||
overall_error = ""
|
||
|
||
conn.execute(
|
||
"""
|
||
UPDATE app_download_record
|
||
SET overall_status = ?,
|
||
overall_error = ?,
|
||
updated_at = ?
|
||
WHERE package_name = ? AND country_code = ?
|
||
""",
|
||
(overall_status, overall_error, time.time(),
|
||
package_name, country_code),
|
||
)
|
||
|
||
def record_download_from_result(self, package_name: str,
|
||
country_code: str,
|
||
result: Dict[str, Any]) -> None:
|
||
"""从 MinIO download result 解析 source_details 写入下载记录。"""
|
||
package_name = str(package_name or "").strip()
|
||
country_code = str(country_code or "").strip()
|
||
if not package_name:
|
||
return
|
||
|
||
existing = self.get_download_record(package_name, country_code)
|
||
if not existing:
|
||
self.ensure_download_record(
|
||
package_name=package_name,
|
||
country_code=country_code,
|
||
app_name=result.get("app_name", ""),
|
||
last_updated=result.get("last_updated", ""),
|
||
downloads=int(result.get("downloads") or 0),
|
||
)
|
||
|
||
source_details = result.get("source_details")
|
||
if isinstance(source_details, dict):
|
||
for source, detail in source_details.items():
|
||
if not isinstance(detail, dict):
|
||
continue
|
||
self.update_source_result(
|
||
package_name=package_name,
|
||
country_code=country_code,
|
||
source=source,
|
||
status=detail.get("status", "not_attempted"),
|
||
error=detail.get("error", ""),
|
||
download_date=detail.get("date", ""),
|
||
)
|
||
|
||
status = result.get("status", "")
|
||
if status == "ok":
|
||
self._update_local_apk_info(package_name, country_code,
|
||
result.get("files", []),
|
||
result.get("version_name", ""),
|
||
result.get("download_date", ""))
|
||
|
||
self._refresh_stale_flag(package_name, country_code)
|
||
|
||
def _update_local_apk_info(self, package_name: str, country_code: str,
|
||
files: List[Dict[str, Any]],
|
||
version_name: str = "",
|
||
download_date: str = "") -> None:
|
||
"""更新本地 APK 文件信息和 stale 标志。"""
|
||
now = time.time()
|
||
with self._write_lock, self._connect() as conn:
|
||
conn.execute(
|
||
"""
|
||
UPDATE app_download_record
|
||
SET apk_version_name = ?,
|
||
local_apk_file_count = ?,
|
||
local_apk_dir = ?,
|
||
local_apk_is_stale = ?,
|
||
updated_at = ?
|
||
WHERE package_name = ? AND country_code = ?
|
||
""",
|
||
(version_name, len(files), self.storage_dir,
|
||
1 if _is_apk_stale(download_date, self._get_last_updated(conn, package_name, country_code)) else 0,
|
||
now, package_name, country_code),
|
||
)
|
||
|
||
def _get_last_updated(self, conn: sqlite3.Connection,
|
||
package_name: str, country_code: str) -> str:
|
||
row = conn.execute(
|
||
"""
|
||
SELECT last_updated FROM app_download_record
|
||
WHERE package_name = ? AND country_code = ?
|
||
""",
|
||
(package_name, country_code),
|
||
).fetchone()
|
||
return row["last_updated"] if row else ""
|
||
|
||
def _refresh_stale_flag(self, package_name: str, country_code: str) -> None:
|
||
"""重新从 apk_registry 检查并刷新 local_apk_is_stale。"""
|
||
entry = self.get(package_name)
|
||
download_date = entry.get("download_date", "") if entry else ""
|
||
last_updated = ""
|
||
with self._connect() as conn:
|
||
last_updated = self._get_last_updated(conn, package_name, country_code)
|
||
|
||
is_stale = 1 if _is_apk_stale(download_date, last_updated) else 0
|
||
now = time.time()
|
||
with self._write_lock, self._connect() as conn:
|
||
conn.execute(
|
||
"""
|
||
UPDATE app_download_record
|
||
SET local_apk_is_stale = ?,
|
||
updated_at = ?
|
||
WHERE package_name = ? AND country_code = ?
|
||
""",
|
||
(is_stale, now, package_name, country_code),
|
||
)
|
||
|
||
def get_stale_records(self) -> List[Dict[str, Any]]:
|
||
"""返回 local_apk_is_stale = 1 的下载记录列表。"""
|
||
with self._connect() as conn:
|
||
rows = conn.execute(
|
||
"""
|
||
SELECT package_name, country_code, app_name, last_updated, downloads,
|
||
worker_play_status, us_play_status, us_aurora_status,
|
||
overall_status, overall_error, apk_version_name,
|
||
local_apk_dir, local_apk_file_count, local_apk_is_stale
|
||
FROM app_download_record
|
||
WHERE local_apk_is_stale = 1
|
||
ORDER BY downloads DESC
|
||
"""
|
||
).fetchall()
|
||
return [dict(row) for row in rows]
|
||
|
||
def get_download_record(self, package_name: str,
|
||
country_code: str = "") -> Optional[Dict[str, Any]]:
|
||
"""按包名和国家码查询下载记录。"""
|
||
package_name = str(package_name or "").strip()
|
||
country_code = str(country_code or "").strip()
|
||
with self._connect() as conn:
|
||
row = conn.execute(
|
||
"""
|
||
SELECT * FROM app_download_record
|
||
WHERE package_name = ? AND country_code = ?
|
||
""",
|
||
(package_name, country_code),
|
||
).fetchone()
|
||
return dict(row) if row else None
|
||
|
||
def list_download_records(self, country_code: str = "",
|
||
overall_status: str = "") -> List[Dict[str, Any]]:
|
||
"""列出下载记录，可按国家码和/或整体状态过滤。"""
|
||
with self._connect() as conn:
|
||
conditions = []
|
||
params: List[Any] = []
|
||
if country_code:
|
||
conditions.append("country_code = ?")
|
||
params.append(country_code)
|
||
if overall_status:
|
||
conditions.append("overall_status = ?")
|
||
params.append(overall_status)
|
||
where = ""
|
||
if conditions:
|
||
where = "WHERE " + " AND ".join(conditions)
|
||
rows = conn.execute(
|
||
f"""
|
||
SELECT * FROM app_download_record
|
||
{where}
|
||
ORDER BY downloads DESC
|
||
""",
|
||
params,
|
||
).fetchall()
|
||
return [dict(row) for row in rows]
|
||
|
||
def cleanup_stale_local(self, package_name: str) -> bool:
|
||
"""删除本地过期的 APK 目录。返回是否成功删除。"""
|
||
pkg_dir = os.path.join(self.storage_dir, package_name)
|
||
if not os.path.isdir(pkg_dir):
|
||
return True
|
||
import shutil
|
||
try:
|
||
shutil.rmtree(pkg_dir, ignore_errors=True)
|
||
self.mark_unavailable(package_name)
|
||
return True
|
||
except OSError:
|
||
return False
|
||
|
||
# ── local disk refresh (enhanced) ───────────────────────────────
|
||
|
||
def refresh_from_local_disk(self) -> int:
|
||
if not os.path.isdir(self.storage_dir):
|
||
return 0
|
||
found = 0
|
||
for entry_name in os.listdir(self.storage_dir):
|
||
pkg_dir = os.path.join(self.storage_dir, entry_name)
|
||
if not os.path.isdir(pkg_dir):
|
||
continue
|
||
manifest_path = os.path.join(pkg_dir, f"{entry_name}.json")
|
||
if os.path.isfile(manifest_path):
|
||
try:
|
||
with open(manifest_path, "r", encoding="utf-8") as fh:
|
||
manifest = json.load(fh)
|
||
except (json.JSONDecodeError, IOError):
|
||
manifest = None
|
||
else:
|
||
manifest = None
|
||
|
||
if manifest:
|
||
package_name = manifest.get("package_name", entry_name)
|
||
download_date = manifest.get("download_date", "")
|
||
version_name = manifest.get("version_name", "")
|
||
apk_files = manifest.get("files")
|
||
source = manifest.get("source", "local_disk")
|
||
else:
|
||
package_name = entry_name
|
||
apk_files_info, earliest_ctime = self._scan_apk_files(pkg_dir)
|
||
if not apk_files_info:
|
||
continue
|
||
download_date = datetime.fromtimestamp(earliest_ctime).strftime("%Y-%m-%d %H:%M:%S")
|
||
version_name = ""
|
||
apk_files = apk_files_info
|
||
source = "local_disk_ctime"
|
||
|
||
self.mark_available(
|
||
package_name=package_name,
|
||
download_date=download_date,
|
||
local_dir=self.storage_dir,
|
||
version_name=version_name,
|
||
apk_files=apk_files,
|
||
source=source,
|
||
)
|
||
found += 1
|
||
return found
|
||
|
||
@staticmethod
|
||
def _scan_apk_files(pkg_dir: str) -> tuple:
|
||
"""扫描目录下的 APK 文件，返回 (文件信息列表, 最早创建时间)。
|
||
|
||
用于没有 manifest JSON 时的本地磁盘恢复。
|
||
"""
|
||
apk_extensions = (".apk", ".xapk", ".apkm")
|
||
file_infos: List[Dict[str, Any]] = []
|
||
earliest_ctime: Optional[float] = None
|
||
for name in sorted(os.listdir(pkg_dir)):
|
||
fp = os.path.join(pkg_dir, name)
|
||
if not os.path.isfile(fp):
|
||
continue
|
||
if not name.lower().endswith(apk_extensions):
|
||
continue
|
||
try:
|
||
st = os.stat(fp)
|
||
except OSError:
|
||
continue
|
||
ctime = st.st_ctime
|
||
if earliest_ctime is None or ctime < earliest_ctime:
|
||
earliest_ctime = ctime
|
||
file_infos.append({
|
||
"filename": name,
|
||
"size": st.st_size,
|
||
})
|
||
return file_infos, earliest_ctime or 0.0
|
||
|
||
# ── internal ───────────────────────────────────────────────────
|
||
|
||
@staticmethod
|
||
def _row_to_entry(row: sqlite3.Row) -> Dict[str, Any]:
|
||
"""将 SQLite Row 转换为与旧 JSON 格式兼容的字典。"""
|
||
apk_files_raw = row["apk_files_json"] or "[]"
|
||
try:
|
||
apk_files = json.loads(apk_files_raw)
|
||
except (json.JSONDecodeError, TypeError):
|
||
apk_files = []
|
||
return {
|
||
"download_date": row["download_date"],
|
||
"download_time": float(row["download_time"] or 0.0),
|
||
"local_dir": row["local_dir"] or "",
|
||
"smb_dir": row["smb_dir"] or "",
|
||
"version_name": row["version_name"] or "",
|
||
"source": row["source"] or "",
|
||
"apk_files": apk_files,
|
||
}
|
||
|
||
|
||
def main() -> int:
|
||
registry = ApkRegistry()
|
||
|
||
# Test basic operations
|
||
registry.mark_available(
|
||
"com.test.app1", download_date="2026-05-19",
|
||
local_dir="/tmp/app1", version_name="1.0", source="test",
|
||
)
|
||
entry = registry.get("com.test.app1")
|
||
assert entry is not None
|
||
assert entry["download_date"] == "2026-05-19"
|
||
assert entry["source"] == "test"
|
||
assert entry["smb_dir"].startswith("\\\\")
|
||
print(" [OK] mark_available + get")
|
||
|
||
# Test freshness check
|
||
assert registry.is_fresh_enough("com.test.app1", "2026-05-18")
|
||
assert registry.is_fresh_enough("com.test.app1", "2026-05-19")
|
||
assert not registry.is_fresh_enough("com.test.app1", "2026-05-20")
|
||
assert not registry.is_fresh_enough("com.nonexistent", "2026-01-01")
|
||
print(" [OK] is_fresh_enough")
|
||
|
||
# Test SMB path
|
||
smb = registry.get_smb_apk_dir("com.test.app1")
|
||
assert smb is not None and "mumu_apk" in smb
|
||
print(f" [OK] get_smb_apk_dir = {smb}")
|
||
|
||
# Test pending download
|
||
registry.mark_pending_download("com.test.need_apk")
|
||
assert registry.is_pending_download("com.test.need_apk")
|
||
assert not registry.is_pending_download("com.test.app1")
|
||
assert "com.test.need_apk" in registry.get_pending_download_packages()
|
||
print(" [OK] pending download tracking")
|
||
|
||
# Test that mark_available clears pending
|
||
registry.mark_available(
|
||
"com.test.need_apk", download_date="2026-05-19",
|
||
local_dir="/tmp/app2",
|
||
)
|
||
assert not registry.is_pending_download("com.test.need_apk")
|
||
print(" [OK] mark_available clears pending")
|
||
|
||
# Test pending_that_are_available
|
||
registry.mark_pending_download("com.test.app1")
|
||
ready = registry.get_pending_that_are_available()
|
||
assert "com.test.app1" in ready # already has local_dir from mark_available
|
||
print(f" [OK] pending_that_are_available: {ready}")
|
||
|
||
# Test stale detection
|
||
catalog = {
|
||
"com.test.app1": "2026-05-19",
|
||
"com.test.app2": "2026-05-25",
|
||
}
|
||
stale = registry.get_stale_packages(catalog)
|
||
assert "com.test.app2" in stale
|
||
assert "com.test.app1" not in stale
|
||
print(" [OK] get_stale_packages")
|
||
|
||
# Test list_all_packages
|
||
all_pkgs = registry.list_all_packages()
|
||
assert "com.test.app1" in all_pkgs
|
||
assert "com.test.need_apk" in all_pkgs
|
||
print(f" [OK] list_all_packages: {all_pkgs}")
|
||
|
||
# Test get_download_time
|
||
dt = registry.get_download_time("com.test.app1")
|
||
assert dt > 0
|
||
print(f" [OK] get_download_time: {dt}")
|
||
|
||
# ── download record tests ──
|
||
print("\n Download record tests ...")
|
||
|
||
# Test ensure_download_record (lazy init)
|
||
registry.ensure_download_record(
|
||
package_name="com.test.dl",
|
||
country_code="US",
|
||
app_name="Test DL App",
|
||
last_updated="2026-05-20",
|
||
downloads=5000000,
|
||
)
|
||
record = registry.get_download_record("com.test.dl", "US")
|
||
assert record is not None
|
||
assert record["app_name"] == "Test DL App"
|
||
assert record["last_updated"] == "2026-05-20"
|
||
assert record["downloads"] == 5000000
|
||
assert record["overall_status"] == "pending"
|
||
print(" [OK] ensure_download_record + get_download_record")
|
||
|
||
# Test ensure_download_record updates existing
|
||
registry.ensure_download_record(
|
||
package_name="com.test.dl",
|
||
country_code="US",
|
||
app_name="Test DL App Updated",
|
||
last_updated="2026-05-25",
|
||
downloads=6000000,
|
||
)
|
||
record = registry.get_download_record("com.test.dl", "US")
|
||
assert record["app_name"] == "Test DL App Updated"
|
||
assert record["last_updated"] == "2026-05-25"
|
||
assert record["downloads"] == 6000000
|
||
print(" [OK] ensure_download_record update")
|
||
|
||
# Test update_source_result
|
||
registry.update_source_result("com.test.dl", "US", "us_play", "success",
|
||
download_date="2026-05-21 10:00:00")
|
||
registry.update_source_result("com.test.dl", "US", "us_aurora", "not_attempted")
|
||
record = registry.get_download_record("com.test.dl", "US")
|
||
assert record["us_play_status"] == "success"
|
||
assert record["us_play_date"] == "2026-05-21 10:00:00"
|
||
assert record["us_aurora_status"] == "not_attempted"
|
||
assert record["overall_status"] == "success"
|
||
print(" [OK] update_source_result (success)")
|
||
|
||
# Test update_source_result (failure)
|
||
registry.update_source_result("com.test.dl", "US", "worker_play", "failed",
|
||
error="network timeout")
|
||
record = registry.get_download_record("com.test.dl", "US")
|
||
assert record["worker_play_status"] == "failed"
|
||
assert record["worker_play_error"] == "network timeout"
|
||
assert record["overall_status"] == "success" # still overall success b/c us_play succeeded
|
||
print(" [OK] update_source_result (failure, but overall still success)")
|
||
|
||
# Test all_failed
|
||
registry.update_source_result("com.test.dl", "US", "us_play", "failed",
|
||
error="region restricted")
|
||
record = registry.get_download_record("com.test.dl", "US")
|
||
assert record["us_play_status"] == "failed"
|
||
assert record["overall_status"] == "all_failed"
|
||
assert "region restricted" in (record["overall_error"] or "")
|
||
print(" [OK] overall_status = all_failed")
|
||
|
||
# Test record_download_from_result (MinIO result)
|
||
result = {
|
||
"status": "ok",
|
||
"download_date": "2026-05-21 11:00:00",
|
||
"version_name": "2.0",
|
||
"files": [{"filename": "base.apk", "size": 5000000}],
|
||
"source_details": {
|
||
"us_play": {"status": "success", "error": None, "date": "2026-05-21 11:00:00"},
|
||
"us_aurora": {"status": "not_attempted", "error": None, "date": None},
|
||
"worker_play": {"status": "not_attempted", "error": None, "date": None},
|
||
},
|
||
}
|
||
registry.record_download_from_result("com.test.dl", "US", result)
|
||
record = registry.get_download_record("com.test.dl", "US")
|
||
assert record["us_play_status"] == "success"
|
||
assert record["overall_status"] == "success"
|
||
assert record["apk_version_name"] == "2.0"
|
||
assert record["local_apk_file_count"] == 1
|
||
print(" [OK] record_download_from_result")
|
||
|
||
# Test record_download_from_result (failed)
|
||
result_failed = {
|
||
"status": "failed",
|
||
"source_details": {
|
||
"us_play": {"status": "failed", "error": "region restricted", "date": None},
|
||
"us_aurora": {"status": "failed", "error": "app not found", "date": None},
|
||
},
|
||
}
|
||
registry.record_download_from_result("com.test.dl", "US", result_failed)
|
||
record = registry.get_download_record("com.test.dl", "US")
|
||
assert record["us_play_status"] == "failed"
|
||
assert record["us_aurora_status"] == "failed"
|
||
assert record["overall_status"] == "all_failed"
|
||
print(" [OK] record_download_from_result (failed)")
|
||
|
||
# Test list_download_records
|
||
all_records = registry.list_download_records()
|
||
assert len(all_records) >= 1
|
||
us_records = registry.list_download_records(country_code="US")
|
||
assert len(us_records) == 1
|
||
failed_records = registry.list_download_records(overall_status="all_failed")
|
||
assert len(failed_records) == 1
|
||
print(f" [OK] list_download_records: total={len(all_records)}, US={len(us_records)}, failed={len(failed_records)}")
|
||
|
||
# Test get_stale_records
|
||
# register an APK older than last_updated to trigger stale
|
||
registry.mark_available(
|
||
"com.test.dl", download_date="2026-05-01",
|
||
local_dir="/tmp/app_dl", version_name="1.0", source="test",
|
||
)
|
||
stale_records = registry.get_stale_records()
|
||
assert any(r["package_name"] == "com.test.dl" for r in stale_records), \
|
||
f"Expected com.test.dl to be stale, got: {[r['package_name'] for r in stale_records]}"
|
||
print(f" [OK] get_stale_records: {len(stale_records)} stale packages")
|
||
|
||
# Test cleanup_stale_local
|
||
# This removes the apk_registry entry (simulating deletion on disk)
|
||
# Since /tmp/app_dl doesn't exist, it should return True
|
||
cleaned = registry.cleanup_stale_local("com.test.dl")
|
||
print(f" [OK] cleanup_stale_local: {cleaned}")
|
||
|
||
# Cleanup download record test data
|
||
with registry._connect() as conn:
|
||
conn.execute("DELETE FROM app_download_record WHERE package_name = ?", ("com.test.dl",))
|
||
print(" [OK] download record cleanup")
|
||
|
||
# Cleanup
|
||
registry.mark_unavailable("com.test.app1")
|
||
registry.clear_pending_download("com.test.app1")
|
||
registry.clear_pending_download("com.test.need_apk")
|
||
registry.mark_unavailable("com.test.need_apk")
|
||
registry.mark_unavailable("com.test.dl")
|
||
assert registry.get("com.test.app1") is None
|
||
|
||
print("\nAll tests passed!")
|
||
print(f"\nDB path: {registry.db_path}")
|
||
print(f"Storage dir: {registry.storage_dir}")
|
||
print(f"SMB dir: {registry.smb_dir}")
|
||
print(f"Packages: {registry.list_all_packages()}")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|