209 lines
8.5 KiB
Python
209 lines
8.5 KiB
Python
#!/usr/bin/env python3
|
||
"""新数据库 schema 轻量访问适配器。"""
|
||
|
||
import json
|
||
import time
|
||
from typing import Dict, List, Any, Optional
|
||
|
||
|
||
class SchemaAdapter:
|
||
"""只访问 app_catalog / collection_task，不做旧表兼容。"""
|
||
|
||
def __init__(self, db_connection, use_new_schema: bool = True):
|
||
self.conn = db_connection
|
||
self.use_new = use_new_schema
|
||
|
||
def _table_exists(self, table_name: str) -> bool:
|
||
"""检查表是否存在"""
|
||
result = self.conn.execute(
|
||
"SELECT name FROM sqlite_master WHERE type = 'table' AND name=?",
|
||
(table_name,)
|
||
).fetchone()
|
||
return result is not None
|
||
|
||
def get_collection_row(self, package_name: str) -> Optional[Dict]:
|
||
"""
|
||
获取应用的完整信息（元数据 + 最新执行状态）
|
||
|
||
直接读取 app_catalog，并按 collection_task 最新记录补充状态。
|
||
"""
|
||
if not (self.use_new and self._table_exists('app_catalog') and self._table_exists('collection_task')):
|
||
return None
|
||
catalog = self.conn.execute(
|
||
"SELECT * FROM app_catalog WHERE package_name = ?",
|
||
(package_name,)
|
||
).fetchone()
|
||
if not catalog:
|
||
return None
|
||
task = self.conn.execute("""
|
||
SELECT *,
|
||
CASE WHEN total_traffic_bytes > 0
|
||
THEN self_traffic_bytes * 100.0 / total_traffic_bytes
|
||
ELSE 0 END AS self_ratio
|
||
FROM collection_task
|
||
WHERE package_name = ?
|
||
ORDER BY
|
||
datetime(COALESCE(NULLIF(completed_at, ''), NULLIF(started_at, ''), created_at)) DESC,
|
||
attempt DESC,
|
||
batch_tag DESC,
|
||
run_kind DESC
|
||
LIMIT 1
|
||
""", (package_name,)).fetchone()
|
||
result = dict(catalog)
|
||
if task:
|
||
task_dict = dict(task)
|
||
result.update({
|
||
"latest_status": task_dict.get("execution_status"),
|
||
"error_category": task_dict.get("error_category"),
|
||
"error_code": task_dict.get("error_code"),
|
||
"num_nodes": task_dict.get("num_nodes"),
|
||
"self_ratio": task_dict.get("self_ratio"),
|
||
"total_traffic_bytes": task_dict.get("total_traffic_bytes"),
|
||
"latest_test_time": task_dict.get("completed_at"),
|
||
})
|
||
return result
|
||
|
||
def upsert_catalog_entry(self, entry: Dict):
|
||
"""
|
||
插入/更新应用目录条目（元数据）
|
||
|
||
UPSERT INTO app_catalog
|
||
"""
|
||
now = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())
|
||
|
||
if self.use_new and self._table_exists('app_catalog'):
|
||
self.conn.execute("""
|
||
INSERT INTO app_catalog (
|
||
package_name, app_name, app_magic_label, downloads, source_order,
|
||
last_updated, country_code, device_type, task_payload_json,
|
||
last_update_interval_days, batch_tags, is_active, updated_at
|
||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||
ON CONFLICT(package_name) DO UPDATE SET
|
||
app_name=excluded.app_name,
|
||
app_magic_label=COALESCE(NULLIF(excluded.app_magic_label, ''), app_catalog.app_magic_label),
|
||
downloads=excluded.downloads,
|
||
source_order=excluded.source_order,
|
||
last_updated=COALESCE(NULLIF(excluded.last_updated, ''), app_catalog.last_updated),
|
||
country_code=excluded.country_code,
|
||
device_type=excluded.device_type,
|
||
task_payload_json=excluded.task_payload_json,
|
||
last_update_interval_days=excluded.last_update_interval_days,
|
||
is_active=excluded.is_active,
|
||
updated_at=excluded.updated_at
|
||
""", (
|
||
entry['package_name'], entry.get('app_name'), entry.get('app_magic_label'),
|
||
entry.get('downloads'), entry.get('source_order', 0),
|
||
entry.get('last_updated', ''), entry.get('country_code', ''),
|
||
entry.get('device_type', ''), json.dumps(entry.get('task_payload', {}), ensure_ascii=False),
|
||
entry.get('last_update_interval_days', 0),
|
||
json.dumps(entry.get('batch_tags', [])),
|
||
entry.get('is_active', 1),
|
||
now
|
||
))
|
||
|
||
def insert_collection_task(self, task: Dict):
|
||
"""
|
||
插入任务执行记录
|
||
|
||
INSERT INTO collection_task
|
||
"""
|
||
now_iso = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())
|
||
|
||
if self.use_new and self._table_exists('collection_task'):
|
||
self.conn.execute("""
|
||
INSERT INTO collection_task (
|
||
package_name, batch_tag, run_kind, attempt,
|
||
execution_status, error_category, error_code, error_reason,
|
||
worker_id, completed_at, duration_seconds,
|
||
total_traffic_bytes, self_traffic_bytes, server_traffic_bytes,
|
||
num_nodes, droidbot_steps, gui_agent_steps,
|
||
created_at
|
||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||
ON CONFLICT(package_name, batch_tag, run_kind, attempt) DO UPDATE SET
|
||
execution_status=excluded.execution_status,
|
||
error_category=excluded.error_category,
|
||
error_code=excluded.error_code,
|
||
error_reason=excluded.error_reason,
|
||
worker_id=excluded.worker_id,
|
||
completed_at=excluded.completed_at,
|
||
duration_seconds=excluded.duration_seconds,
|
||
total_traffic_bytes=excluded.total_traffic_bytes,
|
||
self_traffic_bytes=excluded.self_traffic_bytes,
|
||
server_traffic_bytes=excluded.server_traffic_bytes,
|
||
num_nodes=excluded.num_nodes
|
||
""", (
|
||
task['package_name'], task.get('batch_tag', 'unknown'),
|
||
task.get('run_kind', 'ranking'), task.get('attempt', 1),
|
||
task.get('execution_status'), task.get('error_category'),
|
||
task.get('error_code'), task.get('error_reason'),
|
||
task.get('worker_id'), now_iso, task.get('duration_seconds'),
|
||
task.get('total_traffic_bytes', 0), task.get('self_traffic_bytes', 0),
|
||
task.get('server_traffic_bytes', 0), task.get('num_nodes', 0),
|
||
task.get('droidbot_steps', 0), task.get('gui_agent_steps', 0),
|
||
now_iso
|
||
))
|
||
|
||
def list_pending_tasks(self, limit: int = 1000) -> List[Dict]:
|
||
"""
|
||
列出待处理任务
|
||
|
||
返回 active catalog 中没有成功最新记录的任务。
|
||
"""
|
||
if self.use_new and self._table_exists('app_catalog') and self._table_exists('collection_task'):
|
||
rows = self.conn.execute("""
|
||
SELECT ac.package_name, ac.app_name, ac.downloads, ac.source_order
|
||
FROM app_catalog ac
|
||
WHERE COALESCE(ac.is_active, 1) = 1
|
||
AND NOT EXISTS (
|
||
SELECT 1
|
||
FROM collection_task ct
|
||
WHERE ct.package_name = ac.package_name
|
||
AND ct.execution_status = 'success'
|
||
)
|
||
ORDER BY
|
||
CASE WHEN ac.source_order IS NULL THEN 1 ELSE 0 END ASC,
|
||
ac.source_order ASC,
|
||
ac.package_name ASC
|
||
LIMIT ?
|
||
""", (limit,)).fetchall()
|
||
return [dict(r) for r in rows]
|
||
|
||
return []
|
||
|
||
|
||
# 使用示例
|
||
if __name__ == '__main__':
|
||
import sqlite3
|
||
import os
|
||
|
||
USE_NEW_SCHEMA = os.getenv('USE_NEW_SCHEMA', 'true').lower() == 'true'
|
||
|
||
conn = sqlite3.connect('runtime/main/monitoring.sqlite3')
|
||
conn.row_factory = sqlite3.Row
|
||
|
||
adapter = SchemaAdapter(conn, use_new_schema=USE_NEW_SCHEMA)
|
||
|
||
# 测试查询
|
||
row = adapter.get_collection_row('com.example.test')
|
||
print(f"查询结果: {row}")
|
||
|
||
# 测试写入
|
||
adapter.upsert_catalog_entry({
|
||
'package_name': 'com.test.app',
|
||
'app_name': 'Test App',
|
||
'downloads': 10000,
|
||
'is_active': 1,
|
||
})
|
||
|
||
adapter.insert_collection_task({
|
||
'package_name': 'com.test.app',
|
||
'batch_tag': '2026-6-15',
|
||
'execution_status': 'success',
|
||
'total_traffic_bytes': 5000000,
|
||
})
|
||
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
print("✅ 适配器测试通过")
|