import ahocorasick import pandas as pd import os import sys from pathlib import Path from collections import defaultdict # ================= 配置区域 ================= TARGET_DATE = "20260331" LIB_DATE = "20260224" # 路径配置 SCRIPT_DIR = Path(__file__).resolve().parent PROJECT_ROOT = SCRIPT_DIR.parent APP_LIST_PATH = PROJECT_ROOT / f"Lib/{LIB_DATE}/input/TPDPI_app_list_{LIB_DATE}.csv" URL_LIST_PATH = PROJECT_ROOT / f"Lib/{LIB_DATE}/input/TPDPI_url_lib_{LIB_DATE}.csv" TRAFFIC_DATA_PATH = PROJECT_ROOT / f"TrafficData/traffic_summary/{TARGET_DATE}/traffic_summary.csv" OUTPUT_DIR = PROJECT_ROOT / f"TrafficData/traffic_summary/{TARGET_DATE}/result" # 输出文件定义 OUTPUT_APP_PROFILE = OUTPUT_DIR / f"app_traffic_profile_{TARGET_DATE}.csv" OUTPUT_UNMATCHED_FILE = OUTPUT_DIR / f"unmatched_domains_{TARGET_DATE}.csv" OUTPUT_DETAIL_FILE = OUTPUT_DIR / f"traffic_detail_log_{TARGET_DATE}.csv" # =========================================== def format_bytes(size): """将字节转换为易读格式 (B, KB, MB, GB)，保留2位小数""" power = 2 ** 10 n = 0 power_labels = {0: 'B', 1: 'KB', 2: 'MB', 3: 'GB', 4: 'TB'} while size > power: size /= power n += 1 if n == 0: return f"{int(size)} B" return f"{size:.2f} {power_labels[n]}" class DpiDatabaseTester: def __init__(self): self.automaton = ahocorasick.Automaton() # 基础查找表 self.tp_packages = {} self.tp_pkg_string = {} self.pkg_to_tp_mark = {} # 核心统计容器: Key = Source App ID self.source_app_stats = defaultdict(lambda: { 'app_label': '', 'total_bytes': 0, # 包含 IP 流量 + 域名流量 'total_domain_bytes': 0, # 仅包含域名流量 'self_bytes': 0, 'server_bytes': 0, 'unrec_bytes': 0, 'self_count': 0, 'server_count': 0, 'unrec_count': 0, 'components': defaultdict(lambda: {'bytes': 0, 'is_self': False}) }) # 存储未匹配记录 (用于 unmatched 文件) self.unmatched_records = [] # 存储所有流水的详细记录 (用于 detail_log 文件) self.all_detail_records = [] def check_paths(self): files = [APP_LIST_PATH, URL_LIST_PATH, TRAFFIC_DATA_PATH] missing = [f for f in files if not f.exists()] if missing: print(f"[Error] 缺少文件: {[f.name for f in missing]}") sys.exit(1) if not OUTPUT_DIR.exists(): os.makedirs(OUTPUT_DIR) def load_app_list(self): print(f"[1/5] 加载应用定义 (TPMark): {APP_LIST_PATH.name}") try: df = pd.read_csv(APP_LIST_PATH) for row in df.itertuples(): tp_mark = str(row.app_name).strip() pkg_raw = str(row.package_name) if pd.notna(row.package_name) else "" self.tp_pkg_string[tp_mark] = pkg_raw pkg_list = [p.strip() for p in pkg_raw.split(',') if p.strip()] if tp_mark not in self.tp_packages: self.tp_packages[tp_mark] = set() self.tp_packages[tp_mark].update(pkg_list) for pkg in pkg_list: self.pkg_to_tp_mark[pkg] = tp_mark print(f" - 已加载 {len(self.tp_packages)} 个应用定义。") except Exception as e: print(f"[Error] 加载应用列表失败: {e}") sys.exit(1) def load_url_list(self): print(f"[2/5] 构建 AC 自动机: {URL_LIST_PATH.name}") try: df = pd.read_csv(URL_LIST_PATH) df.columns = [c.strip().lower() for c in df.columns] count = 0 for row in df.itertuples(): suffix = str(row.url).strip().lower() tp_mark = str(row.app).strip() if suffix: self.automaton.add_word(suffix, (suffix, tp_mark)) count += 1 self.automaton.make_automaton() print(f" - 加载了 {count} 条 URL 规则。") except Exception as e: print(f"[Error] 加载 URL 失败: {e}") sys.exit(1) def search_domain(self, domain): """返回 (TPMark, Pattern)""" domain_lower = domain.lower() valid_matches = [] # iter 返回 (end_index, (pattern, tp_mark)) for end_index, (pattern, tp_mark) in self.automaton.iter(domain_lower): if end_index != len(domain_lower) - 1: continue start_index = end_index - len(pattern) + 1 if start_index == 0 or domain_lower[start_index - 1] == '.': valid_matches.append((len(pattern), tp_mark, pattern)) if not valid_matches: return None, None # 取最长匹配 best = max(valid_matches, key=lambda x: x[0]) return best[1], best[2] def process_traffic(self): print(f"[3/5] 分析流量归属: {TRAFFIC_DATA_PATH.name}") try: df = pd.read_csv(TRAFFIC_DATA_PATH) df.columns = [c.strip().replace(' ', '_').replace('(', '_').replace(')', '') for c in df.columns] processed = 0 for row in df.itertuples(): source_app_id = str(row.App_ID).strip() # Filter: 跳过 unknown 和 root if source_app_id.lower() in ['unknown', 'root']: continue domain = str(row.Domain).strip() size = float(row.Traffic_Size_Bytes) source_app_label = str(row.App_Name).strip() stats = self.source_app_stats[source_app_id] stats['app_label'] = source_app_label stats['total_bytes'] += size # === 临时变量，用于构建 Detail Record === detail_tp_result = "" detail_tp_url = "" detail_is_self = False detail_is_ip = False # === 判断是否为无域名IP流量 (model_data) === is_ip_flow = domain.startswith("model_data:") if is_ip_flow: detail_is_ip = True detail_tp_result = "IP Flow" # 或保持空，根据需要 continue stats['unrec_bytes'] += size stats['unrec_count'] += 1 # IP 流量也记入 unmatched_records 供 unmatched文件使用 self.unmatched_records.append({ 'app_id': source_app_id, 'app_label': source_app_label, 'input_url': domain, 'traffic_size': size, 'is_ip_flow': True }) else: # === 普通域名流量处理 === stats['total_domain_bytes'] += size # 匹配域名 matched_tp_mark, matched_pattern = self.search_domain(domain) if matched_tp_mark: # 记录详情 detail_tp_result = matched_tp_mark detail_tp_url = matched_pattern owner_packages = self.tp_packages.get(matched_tp_mark, set()) is_self = source_app_id in owner_packages detail_is_self = is_self if is_self: stats['self_bytes'] += size stats['self_count'] += 1 else: stats['server_bytes'] += size stats['server_count'] += 1 comp = stats['components'][matched_tp_mark] comp['bytes'] += size comp['is_self'] = is_self else: # 域名未匹配 detail_tp_result = "[|Unmatched|]" stats['unrec_bytes'] += size stats['unrec_count'] += 1 self.unmatched_records.append({ 'app_id': source_app_id, 'app_label': source_app_label, 'input_url': domain, 'traffic_size': size, 'is_ip_flow': False }) if detail_tp_result == "[|Unmatched|]": continue # === 保存详细流水记录 === # 注意：此时还没计算 Ratio，因为 total_bytes 还在累加中 # 存下原始数据，在导出时计算 Ratio self.all_detail_records.append({ 'tp_result': detail_tp_result if detail_tp_result else "", 'app_name': source_app_label, 'app_label': source_app_label, 'app_id': source_app_id, 'input_url': domain, 'tp_url': detail_tp_url, 'self': detail_is_self, 'traffic_count_bytes': size, 'is_ip_flow': detail_is_ip }) processed += 1 if processed % 50000 == 0: print(f" - 已处理 {processed} 行...") except Exception as e: print(f"[Error] 处理流量失败: {e}") import traceback traceback.print_exc() sys.exit(1) def export_app_profile(self): print(f"[4/5] 生成画像报告: {OUTPUT_APP_PROFILE.name}") data_rows = [] for app_id, stats in self.source_app_stats.items(): total = stats['total_bytes'] if total == 0: continue tp_mark = self.pkg_to_tp_mark.get(app_id, "") app_label = stats['app_label'] app_name = app_label self_ratio = (stats['self_bytes'] / total) * 100 recog_ratio = ((stats['self_bytes'] + stats['server_bytes']) / total) * 100 analysis_list = [] sorted_comps = sorted(stats['components'].items(), key=lambda x: x[1]['bytes'], reverse=True) for comp_tp_mark, comp_data in sorted_comps: comp_bytes = comp_data['bytes'] comp_percent = (comp_bytes / total) * 100 is_self_flag = '0' if comp_data['is_self'] else '1' comp_pkg_str = self.tp_pkg_string.get(comp_tp_mark, "") analysis_item = [ comp_tp_mark, comp_pkg_str, is_self_flag, f"{comp_percent:.2f}%", int(comp_bytes) ] analysis_list.append(analysis_item) data_rows.append({ 'app_id': app_id, 'app_label': app_label, 'app_name': app_name, 'tp_mark': tp_mark, 'total_traffic(Bytes)': int(total), 'self_traffic_count(Bytes)': format_bytes(stats['self_bytes']), 'server_traffic_count(Bytes)': format_bytes(stats['server_bytes']), 'unrecognized_traffic_count(Bytes)': format_bytes(stats['unrec_bytes']), 'self_label_count': stats['self_count'], 'server_label_count': stats['server_count'], 'unrecognized_label_count': stats['unrec_count'], 'self_ratio': f"{self_ratio:.2f}", 'recognition_ratio': f"{recog_ratio:.2f}", 'traffic_analysis': str(analysis_list) }) df = pd.DataFrame(data_rows) cols = [ 'app_id', 'app_label', 'app_name', 'tp_mark', 'total_traffic(Bytes)', 'self_traffic_count(Bytes)', 'server_traffic_count(Bytes)', 'unrecognized_traffic_count(Bytes)', 'self_label_count', 'server_label_count', 'unrecognized_label_count', 'self_ratio', 'recognition_ratio', 'traffic_analysis' ] if not df.empty: df = df[cols] df.sort_values(by='total_traffic(Bytes)', ascending=False, inplace=True) df.to_csv(OUTPUT_APP_PROFILE, index=False) print(f" - 完成: {OUTPUT_APP_PROFILE.name}") def export_unmatched_list(self): print(f"[5/5] 生成未匹配域名列表: {OUTPUT_UNMATCHED_FILE.name}") if not self.unmatched_records: print(" - 无未匹配记录。") else: export_rows = [] for record in self.unmatched_records: app_id = record['app_id'] size = record['traffic_size'] is_ip = record['is_ip_flow'] stats = self.source_app_stats[app_id] total_bytes = stats['total_bytes'] total_domain_bytes = stats['total_domain_bytes'] traffic_ratio = (size / total_bytes * 100) if total_bytes > 0 else 0 if is_ip: domain_ratio_str = "0.00%" else: domain_ratio = (size / total_domain_bytes * 100) if total_domain_bytes > 0 else 0 domain_ratio_str = f"{domain_ratio:.2f}%" export_rows.append({ 'app_name': record['app_label'], 'app_label': record['app_label'], 'app_id': app_id, 'input_url': record['input_url'], 'traffic_count(Bytes)': int(size), 'traffic_ratio': f"{traffic_ratio:.2f}%", 'domain_traffic_ratio': domain_ratio_str, 'organization': '' }) df = pd.DataFrame(export_rows) if not df.empty: df.sort_values(by='traffic_count(Bytes)', ascending=False, inplace=True) df.to_csv(OUTPUT_UNMATCHED_FILE, index=False) print(f" - 完成: {OUTPUT_UNMATCHED_FILE.name}") def export_detail_log(self): print(f"[Bonus] 生成详细流水日志: {OUTPUT_DETAIL_FILE.name}") if not self.all_detail_records: print(" - 无流水记录。") return export_rows = [] # 批量处理，提升速度 for r in self.all_detail_records: app_id = r['app_id'] size = r['traffic_count_bytes'] is_ip = r['is_ip_flow'] # 获取该 App 的统计数据以计算 Ratio stats = self.source_app_stats[app_id] total_bytes = stats['total_bytes'] total_domain_bytes = stats['total_domain_bytes'] # 1. Total Ratio ratio = (size / total_bytes * 100) if total_bytes > 0 else 0 # 2. Domain Ratio if is_ip: domain_ratio_str = "0.00%" else: d_ratio = (size / total_domain_bytes * 100) if total_domain_bytes > 0 else 0 domain_ratio_str = f"{d_ratio:.2f}%" export_rows.append({ 'tp_result': r['tp_result'], 'app_name': r['app_name'], 'app_label': r['app_label'], 'app_id': app_id, 'input_url': r['input_url'], 'tp_url': r['tp_url'], 'self': str(r['self']), # 转字符串 'traffic_count(Bytes)': int(size), 'traffic_ratio': f"{ratio:.2f}%", 'domain_traffic_ratio': domain_ratio_str, 'organization': '' }) df = pd.DataFrame(export_rows) # 排序建议：先按 App ID 聚类，再按流量大小降序 if not df.empty: df.sort_values(by=['app_id', 'traffic_count(Bytes)'], ascending=[True, False], inplace=True) df.to_csv(OUTPUT_DETAIL_FILE, index=False) print(f" - 完成: {OUTPUT_DETAIL_FILE.name}") if __name__ == "__main__": tester = DpiDatabaseTester() tester.check_paths() tester.load_app_list() tester.load_url_list() tester.process_traffic() tester.export_app_profile() tester.export_unmatched_list() tester.export_detail_log()