import json import logging import traceback from concurrent.futures import ThreadPoolExecutor, as_completed import threading import time import uuid from datetime import datetime from typing import Any, Dict, List, Optional from config import ( DASHBOARD_HOST, DASHBOARD_PORT, INSTANCE_NAME, MONITORING_TIMEZONE, REDIS_DB, RUN_PIPELINE_DEFAULT_OPTIONS, WORKER_ACTION_MAX_PARALLEL, ) from redis_task_distribute import RedisTaskDispatcher try: from flask import Flask, jsonify, render_template_string, request except ImportError: # pragma: no cover - runtime dependency check Flask = None jsonify = None render_template_string = None request = None def silence_flask_runtime_logs(): try: import flask.cli as flask_cli flask_cli.show_server_banner = lambda *args, **kwargs: None except Exception: pass try: import werkzeug.serving as werkzeug_serving werkzeug_serving._log = lambda *args, **kwargs: None except Exception: pass logging.getLogger("werkzeug").disabled = True class WorkerActionJobManager: def __init__(self, dispatcher): self.dispatcher = dispatcher self._lock = threading.Lock() self._jobs: Dict[str, Dict[str, Any]] = {} def submit(self, worker_ids: List[str], action: str, options: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: job_id = uuid.uuid4().hex now_ts = time.time() job = { "job_id": job_id, "action": action, "worker_ids": list(worker_ids), "options": dict(options or {}), "status": "running", "started_at": now_ts, "finished_at": 0.0, "error": "", "results": [ { "worker_id": worker_id, "ok": None, "message": "等待执行", "steps": [], } for worker_id in worker_ids ], } with self._lock: self._jobs[job_id] = job thread = threading.Thread( target=self._run_job, args=(job_id, list(worker_ids), action, dict(options or {})), daemon=True, ) thread.start() return self.get(job_id) def get(self, job_id: str) -> Optional[Dict[str, Any]]: with self._lock: job = self._jobs.get(job_id) if not job: return None return json.loads(json.dumps(job)) def _run_job(self, job_id: str, worker_ids: List[str], action: str, options: Dict[str, Any]) -> None: results: List[Dict[str, Any]] = [] try: max_workers = min(max(len(worker_ids), 1), WORKER_ACTION_MAX_PARALLEL) with ThreadPoolExecutor(max_workers=max_workers) as executor: futures = { executor.submit( self.dispatcher.run_dashboard_action, [worker_id], action, options=options, progress_callback=lambda current_worker_id, payload: self._record_progress(job_id, current_worker_id, payload), ): worker_id for worker_id in worker_ids } for future in as_completed(futures): worker_id = futures[future] try: worker_results = future.result() except Exception as exc: detail = traceback.format_exc().strip() worker_results = [{ "worker_id": worker_id, "action": action, "ok": False, "message": detail or str(exc), "stderr": detail, }] results.extend(worker_results) except Exception as exc: with self._lock: job = self._jobs.get(job_id) if not job: return job["status"] = "failed" job["finished_at"] = time.time() job["error"] = traceback.format_exc().strip() or str(exc) return with self._lock: job = self._jobs.get(job_id) if not job: return by_worker = {item["worker_id"]: item for item in job["results"]} for result in results: item = by_worker.setdefault( result.get("worker_id", ""), {"worker_id": result.get("worker_id", ""), "ok": None, "message": "", "steps": []}, ) item["ok"] = result.get("ok") item["message"] = self._result_message(result) if isinstance(result.get("steps"), list): item["steps"] = [ { "step": str(step.get("step", "")), "ok": step.get("ok"), "message": str(step.get("message", "")), } for step in result.get("steps", []) if isinstance(step, dict) ] item["stdout"] = result.get("stdout", "") item["stderr"] = result.get("stderr", "") item["command"] = result.get("command", "") job["results"] = list(by_worker.values()) job["finished_at"] = time.time() job["status"] = "failed" if any(item.get("ok") is False for item in job["results"]) else "completed" @staticmethod def _result_message(result: Dict[str, Any]) -> str: base = str(result.get("message", "") or "").strip() prefixes = [] failed_step = str(result.get("failed_step", "") or "").strip() stage = str(result.get("stage", "") or "").strip() if failed_step: prefixes.append(f"step={failed_step}") if stage: prefixes.append(f"stage={stage}") if prefixes: prefix = "[" + " ".join(prefixes) + "]" base = f"{prefix} {base}".strip() if result.get("ok") is not False: return base parts = [base] if base else [] for label, key in (("stdout", "stdout"), ("stderr", "stderr")): value = str(result.get(key, "") or "").strip() if not value or value == base: continue parts.append(f"{label}:\n{value}") return "\n\n".join(parts) def _record_progress(self, job_id: str, worker_id: str, payload: Dict[str, Any]) -> None: with self._lock: job = self._jobs.get(job_id) if not job: return result = None for item in job["results"]: if item["worker_id"] == worker_id: result = item break if result is None: result = {"worker_id": worker_id, "ok": None, "message": "", "steps": []} job["results"].append(result) result["message"] = str(payload.get("message", result.get("message", ""))) step_name = str(payload.get("step", "")).strip() if not step_name: return status = payload.get("status") step_ok = None if status in {"success", "ok"}: step_ok = True elif status in {"failed", "error"}: step_ok = False steps = result.setdefault("steps", []) current_step = None for step in steps: if step.get("step") == step_name: current_step = step break if current_step is None: current_step = {"step": step_name, "ok": step_ok, "message": result["message"]} steps.append(current_step) else: current_step["message"] = result["message"] current_step["ok"] = step_ok TEMPLATE = """ {{ instance_name }} Monitoring Dashboard
Control Center / {{ instance_name }}

Worker Monitoring

实例 {{ instance_name }} / Redis DB {{ redis_db }} 日期 {{ default_date }} 范围 全部 Worker 今日视图自动刷新

运行快照

整体效率

今日采集概况

--

整体运行时间构成

--

合格任务阶段耗时

--

运行状态时间分布

--

各 Worker 采集占比

--
""" def create_app(dispatcher=None): if Flask is None: raise RuntimeError("Flask is not installed. Run `pip install -r requirements.txt` first.") dispatcher = dispatcher or RedisTaskDispatcher() action_jobs = WorkerActionJobManager(dispatcher) app = Flask(__name__) def _get_date_arg() -> str: return request.args.get("date") or datetime.now().strftime("%Y-%m-%d") def _get_int_arg(name, default): value = request.args.get(name, default) try: return int(value) except (TypeError, ValueError): return default def _parse_worker_action_request(): payload = request.get_json(silent=True) or {} worker_ids = payload.get("worker_ids", []) action = str(payload.get("action", "")).strip() allowed_actions = { "run_pipeline", "start", "drain", "recover_and_start", "enable", "disable", "clone", "pull", "pull_pcap_files", "setup", "restart_mumu", "stop_worker", "reboot", "status", "execute_command", "fix_adb_connection", "configure_mumu_network", "recover_mumu_full", "force_reapk", } if not isinstance(worker_ids, list) or not worker_ids: return None, (jsonify({"ok": False, "error": "worker_ids is required"}), 400) if not action: return None, (jsonify({"ok": False, "error": "action is required"}), 400) if action not in allowed_actions: return None, (jsonify({"ok": False, "error": f"unsupported action: {action}"}), 400) return { "worker_ids": worker_ids, "action": action, "options": payload.get("options"), }, None @app.get("/") def index(): return render_template_string( TEMPLATE, instance_name=INSTANCE_NAME, redis_db=REDIS_DB, timezone=MONITORING_TIMEZONE, run_pipeline_defaults=json.dumps(RUN_PIPELINE_DEFAULT_OPTIONS, ensure_ascii=True), default_date=datetime.now().strftime("%Y-%m-%d"), ) @app.get("/api/monitor/overview") def monitor_overview(): return jsonify( dispatcher.get_monitoring_overview( date_str=_get_date_arg(), worker_id=request.args.get("worker_id"), ) ) @app.get("/api/monitor/timeline") def monitor_timeline(): return jsonify( dispatcher.get_monitoring_timeline( date_str=_get_date_arg(), bucket_minutes=_get_int_arg("bucket_minutes", 15), worker_id=request.args.get("worker_id"), ) ) @app.get("/api/monitor/distributions") def monitor_distributions(): return jsonify( dispatcher.get_monitoring_distributions( date_str=_get_date_arg(), worker_id=request.args.get("worker_id"), ) ) @app.get("/api/monitor/workers") def monitor_workers(): return jsonify(dispatcher.get_monitoring_workers(date_str=_get_date_arg())) @app.get("/api/workers/control") def worker_control(): return jsonify( dispatcher.get_worker_control_rows( date_str=_get_date_arg(), worker_id=request.args.get("worker_id"), ) ) @app.post("/api/workers/actions") def worker_actions(): try: parsed, error_response = _parse_worker_action_request() if error_response is not None: return error_response results = dispatcher.run_dashboard_action(parsed["worker_ids"], parsed["action"], options=parsed["options"]) return jsonify({"ok": True, "results": results}) except Exception as e: logger = logging.getLogger("worker-actions") logger.exception("Failed to run worker action") return jsonify({"ok": False, "error": str(e)}), 500 @app.post("/api/workers/actions/submit") def worker_actions_submit(): try: parsed, error_response = _parse_worker_action_request() if error_response is not None: return error_response job = action_jobs.submit(parsed["worker_ids"], parsed["action"], options=parsed["options"]) return jsonify({"ok": True, "job": job}) except Exception as e: logger = logging.getLogger("worker-actions-submit") logger.exception("Failed to submit worker action") return jsonify({"ok": False, "error": str(e)}), 500 @app.get("/api/workers/actions/status") def worker_actions_status(): job_id = str(request.args.get("job_id", "")).strip() if not job_id: return jsonify({"ok": False, "error": "job_id is required"}), 400 job = action_jobs.get(job_id) if not job: return jsonify({"ok": False, "error": "job not found"}), 404 return jsonify({"ok": True, "job": job}) @app.get("/api/monitor/tasks") def monitor_tasks(): return jsonify( dispatcher.get_monitoring_tasks( date_str=_get_date_arg(), worker_id=request.args.get("worker_id"), limit=_get_int_arg("limit", 60), ) ) @app.post("/api/monitor/reset") def monitor_reset(): try: return jsonify(dispatcher.reset_monitoring_dashboard()) except Exception as e: logger = logging.getLogger("monitor-reset") logger.exception("Failed to reset monitoring data") return jsonify({"ok": False, "error": str(e)}), 500 @app.get("/api/analytics/overview") def analytics_overview(): return jsonify( dispatcher.get_analytics_overview( incremental_batch_tag=str(request.args.get("incremental_batch_tag", "")).strip(), top_n=_get_int_arg("top_n", 3000), ) ) @app.get("/api/analytics/apps") def analytics_apps(): return jsonify( dispatcher.get_analytics_apps( q=str(request.args.get("q", "")).strip(), latest_status=str(request.args.get("latest_status", "")).strip(), artifact_status=str(request.args.get("artifact_status", "")).strip(), collection_status=str(request.args.get("collection_status", "")).strip(), restriction_status=str(request.args.get("restriction_status", "")).strip(), retryability=str(request.args.get("retryability", "")).strip(), incremental_batch_tag=str(request.args.get("incremental_batch_tag", "")).strip(), top_n=_get_int_arg("top_n", 3000), sort=str(request.args.get("sort", "updated_at")).strip(), order=str(request.args.get("order", "desc")).strip(), page=_get_int_arg("page", 1), page_size=_get_int_arg("page_size", 50), ) ) @app.get("/api/analytics/apps/") def analytics_app_detail(package_name): payload = dispatcher.get_analytics_app_detail(package_name) if payload is None: return jsonify({"ok": False, "error": "package not found"}), 404 return jsonify(payload) @app.post("/api/analytics/apps//rebuild") def analytics_app_rebuild(package_name): if not str(package_name or "").strip(): return jsonify({"ok": False, "error": "package_name is required"}), 400 job = dispatcher.enqueue_analytics_rebuild(str(package_name).strip()) return jsonify({"ok": True, "job": job}) @app.get("/api/analytics/jobs") def analytics_jobs(): return jsonify( dispatcher.get_analytics_jobs( job_type=str(request.args.get("job_type", "")).strip(), status=str(request.args.get("status", "")).strip(), package_name=str(request.args.get("package_name", "")).strip(), limit=_get_int_arg("limit", 50), ) ) @app.post("/api/analytics/backfill") def analytics_backfill(): payload = request.get_json(silent=True) or {} scope = str(payload.get("scope", "all") or "all").strip() packages = payload.get("packages") or [] if scope not in {"all", "packages"}: return jsonify({"ok": False, "error": "scope must be all or packages"}), 400 if scope == "packages": if not isinstance(packages, list) or not packages: return jsonify({"ok": False, "error": "packages is required when scope=packages"}), 400 packages = [str(item).strip() for item in packages if str(item).strip()] if not packages: return jsonify({"ok": False, "error": "packages is required when scope=packages"}), 400 else: packages = None job = dispatcher.enqueue_analytics_backfill(packages=packages) return jsonify({"ok": True, "job": job}) return app def run_dashboard(dispatcher=None, host=None, port=None): silence_flask_runtime_logs() app = create_app(dispatcher=dispatcher) app.logger.disabled = True app.run( host=host or DASHBOARD_HOST, port=port or DASHBOARD_PORT, debug=False, use_reloader=False, threaded=True, ) def main(): run_dashboard() if __name__ == "__main__": main()