413 lines
16 KiB
Python
413 lines
16 KiB
Python
import ahocorasick
|
||
import pandas as pd
|
||
import os
|
||
import sys
|
||
from pathlib import Path
|
||
from collections import defaultdict
|
||
|
||
# ================= 配置区域 =================
|
||
TARGET_DATE = "20260331"
|
||
LIB_DATE = "20260224"
|
||
# 路径配置
|
||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||
PROJECT_ROOT = SCRIPT_DIR.parent
|
||
|
||
APP_LIST_PATH = PROJECT_ROOT / f"Lib/{LIB_DATE}/input/TPDPI_app_list_{LIB_DATE}.csv"
|
||
URL_LIST_PATH = PROJECT_ROOT / f"Lib/{LIB_DATE}/input/TPDPI_url_lib_{LIB_DATE}.csv"
|
||
TRAFFIC_DATA_PATH = PROJECT_ROOT / f"TrafficData/traffic_summary/{TARGET_DATE}/traffic_summary.csv"
|
||
|
||
OUTPUT_DIR = PROJECT_ROOT / f"TrafficData/traffic_summary/{TARGET_DATE}/result"
|
||
|
||
# 输出文件定义
|
||
OUTPUT_APP_PROFILE = OUTPUT_DIR / f"app_traffic_profile_{TARGET_DATE}.csv"
|
||
OUTPUT_UNMATCHED_FILE = OUTPUT_DIR / f"unmatched_domains_{TARGET_DATE}.csv"
|
||
OUTPUT_DETAIL_FILE = OUTPUT_DIR / f"traffic_detail_log_{TARGET_DATE}.csv"
|
||
|
||
|
||
# ===========================================
|
||
|
||
def format_bytes(size):
|
||
"""将字节转换为易读格式 (B, KB, MB, GB)，保留2位小数"""
|
||
power = 2 ** 10
|
||
n = 0
|
||
power_labels = {0: 'B', 1: 'KB', 2: 'MB', 3: 'GB', 4: 'TB'}
|
||
while size > power:
|
||
size /= power
|
||
n += 1
|
||
if n == 0:
|
||
return f"{int(size)} B"
|
||
return f"{size:.2f} {power_labels[n]}"
|
||
|
||
|
||
class DpiDatabaseTester:
|
||
def __init__(self):
|
||
self.automaton = ahocorasick.Automaton()
|
||
|
||
# 基础查找表
|
||
self.tp_packages = {}
|
||
self.tp_pkg_string = {}
|
||
self.pkg_to_tp_mark = {}
|
||
|
||
# 核心统计容器: Key = Source App ID
|
||
self.source_app_stats = defaultdict(lambda: {
|
||
'app_label': '',
|
||
'total_bytes': 0, # 包含 IP 流量 + 域名流量
|
||
'total_domain_bytes': 0, # 仅包含域名流量
|
||
'self_bytes': 0,
|
||
'server_bytes': 0,
|
||
'unrec_bytes': 0,
|
||
'self_count': 0,
|
||
'server_count': 0,
|
||
'unrec_count': 0,
|
||
'components': defaultdict(lambda: {'bytes': 0, 'is_self': False})
|
||
})
|
||
|
||
# 存储未匹配记录 (用于 unmatched 文件)
|
||
self.unmatched_records = []
|
||
|
||
# 存储所有流水的详细记录 (用于 detail_log 文件)
|
||
self.all_detail_records = []
|
||
|
||
def check_paths(self):
|
||
files = [APP_LIST_PATH, URL_LIST_PATH, TRAFFIC_DATA_PATH]
|
||
missing = [f for f in files if not f.exists()]
|
||
if missing:
|
||
print(f"[Error] 缺少文件: {[f.name for f in missing]}")
|
||
sys.exit(1)
|
||
if not OUTPUT_DIR.exists():
|
||
os.makedirs(OUTPUT_DIR)
|
||
|
||
def load_app_list(self):
|
||
print(f"[1/5] 加载应用定义 (TPMark): {APP_LIST_PATH.name}")
|
||
try:
|
||
df = pd.read_csv(APP_LIST_PATH)
|
||
for row in df.itertuples():
|
||
tp_mark = str(row.app_name).strip()
|
||
pkg_raw = str(row.package_name) if pd.notna(row.package_name) else ""
|
||
|
||
self.tp_pkg_string[tp_mark] = pkg_raw
|
||
pkg_list = [p.strip() for p in pkg_raw.split(',') if p.strip()]
|
||
|
||
if tp_mark not in self.tp_packages:
|
||
self.tp_packages[tp_mark] = set()
|
||
self.tp_packages[tp_mark].update(pkg_list)
|
||
|
||
for pkg in pkg_list:
|
||
self.pkg_to_tp_mark[pkg] = tp_mark
|
||
|
||
print(f" - 已加载 {len(self.tp_packages)} 个应用定义。")
|
||
except Exception as e:
|
||
print(f"[Error] 加载应用列表失败: {e}")
|
||
sys.exit(1)
|
||
|
||
def load_url_list(self):
|
||
print(f"[2/5] 构建 AC 自动机: {URL_LIST_PATH.name}")
|
||
try:
|
||
df = pd.read_csv(URL_LIST_PATH)
|
||
df.columns = [c.strip().lower() for c in df.columns]
|
||
count = 0
|
||
for row in df.itertuples():
|
||
suffix = str(row.url).strip().lower()
|
||
tp_mark = str(row.app).strip()
|
||
if suffix:
|
||
self.automaton.add_word(suffix, (suffix, tp_mark))
|
||
count += 1
|
||
self.automaton.make_automaton()
|
||
print(f" - 加载了 {count} 条 URL 规则。")
|
||
except Exception as e:
|
||
print(f"[Error] 加载 URL 失败: {e}")
|
||
sys.exit(1)
|
||
|
||
def search_domain(self, domain):
|
||
"""返回 (TPMark, Pattern)"""
|
||
domain_lower = domain.lower()
|
||
valid_matches = []
|
||
# iter 返回 (end_index, (pattern, tp_mark))
|
||
for end_index, (pattern, tp_mark) in self.automaton.iter(domain_lower):
|
||
if end_index != len(domain_lower) - 1: continue
|
||
start_index = end_index - len(pattern) + 1
|
||
if start_index == 0 or domain_lower[start_index - 1] == '.':
|
||
valid_matches.append((len(pattern), tp_mark, pattern))
|
||
|
||
if not valid_matches: return None, None
|
||
# 取最长匹配
|
||
best = max(valid_matches, key=lambda x: x[0])
|
||
return best[1], best[2]
|
||
|
||
def process_traffic(self):
|
||
print(f"[3/5] 分析流量归属: {TRAFFIC_DATA_PATH.name}")
|
||
try:
|
||
df = pd.read_csv(TRAFFIC_DATA_PATH)
|
||
df.columns = [c.strip().replace(' ', '_').replace('(', '_').replace(')', '') for c in df.columns]
|
||
|
||
processed = 0
|
||
for row in df.itertuples():
|
||
source_app_id = str(row.App_ID).strip()
|
||
|
||
# Filter: 跳过 unknown 和 root
|
||
if source_app_id.lower() in ['unknown', 'root']:
|
||
continue
|
||
|
||
domain = str(row.Domain).strip()
|
||
size = float(row.Traffic_Size_Bytes)
|
||
source_app_label = str(row.App_Name).strip()
|
||
|
||
stats = self.source_app_stats[source_app_id]
|
||
stats['app_label'] = source_app_label
|
||
stats['total_bytes'] += size
|
||
|
||
# === 临时变量，用于构建 Detail Record ===
|
||
detail_tp_result = ""
|
||
detail_tp_url = ""
|
||
detail_is_self = False
|
||
detail_is_ip = False
|
||
|
||
# === 判断是否为无域名IP流量 (model_data) ===
|
||
is_ip_flow = domain.startswith("model_data:")
|
||
|
||
if is_ip_flow:
|
||
detail_is_ip = True
|
||
detail_tp_result = "IP Flow" # 或保持空，根据需要
|
||
continue
|
||
stats['unrec_bytes'] += size
|
||
stats['unrec_count'] += 1
|
||
|
||
# IP 流量也记入 unmatched_records 供 unmatched文件使用
|
||
self.unmatched_records.append({
|
||
'app_id': source_app_id,
|
||
'app_label': source_app_label,
|
||
'input_url': domain,
|
||
'traffic_size': size,
|
||
'is_ip_flow': True
|
||
})
|
||
|
||
else:
|
||
# === 普通域名流量处理 ===
|
||
stats['total_domain_bytes'] += size
|
||
|
||
# 匹配域名
|
||
matched_tp_mark, matched_pattern = self.search_domain(domain)
|
||
|
||
if matched_tp_mark:
|
||
# 记录详情
|
||
detail_tp_result = matched_tp_mark
|
||
detail_tp_url = matched_pattern
|
||
|
||
owner_packages = self.tp_packages.get(matched_tp_mark, set())
|
||
is_self = source_app_id in owner_packages
|
||
detail_is_self = is_self
|
||
|
||
if is_self:
|
||
stats['self_bytes'] += size
|
||
stats['self_count'] += 1
|
||
else:
|
||
stats['server_bytes'] += size
|
||
stats['server_count'] += 1
|
||
|
||
comp = stats['components'][matched_tp_mark]
|
||
comp['bytes'] += size
|
||
comp['is_self'] = is_self
|
||
|
||
else:
|
||
# 域名未匹配
|
||
detail_tp_result = "[|Unmatched|]"
|
||
|
||
stats['unrec_bytes'] += size
|
||
stats['unrec_count'] += 1
|
||
|
||
self.unmatched_records.append({
|
||
'app_id': source_app_id,
|
||
'app_label': source_app_label,
|
||
'input_url': domain,
|
||
'traffic_size': size,
|
||
'is_ip_flow': False
|
||
})
|
||
if detail_tp_result == "[|Unmatched|]":
|
||
continue
|
||
# === 保存详细流水记录 ===
|
||
# 注意：此时还没计算 Ratio，因为 total_bytes 还在累加中
|
||
# 存下原始数据，在导出时计算 Ratio
|
||
self.all_detail_records.append({
|
||
'tp_result': detail_tp_result if detail_tp_result else "",
|
||
'app_name': source_app_label,
|
||
'app_label': source_app_label,
|
||
'app_id': source_app_id,
|
||
'input_url': domain,
|
||
'tp_url': detail_tp_url,
|
||
'self': detail_is_self,
|
||
'traffic_count_bytes': size,
|
||
'is_ip_flow': detail_is_ip
|
||
})
|
||
|
||
processed += 1
|
||
if processed % 50000 == 0:
|
||
print(f" - 已处理 {processed} 行...")
|
||
|
||
except Exception as e:
|
||
print(f"[Error] 处理流量失败: {e}")
|
||
import traceback
|
||
traceback.print_exc()
|
||
sys.exit(1)
|
||
|
||
def export_app_profile(self):
|
||
print(f"[4/5] 生成画像报告: {OUTPUT_APP_PROFILE.name}")
|
||
|
||
data_rows = []
|
||
for app_id, stats in self.source_app_stats.items():
|
||
total = stats['total_bytes']
|
||
if total == 0: continue
|
||
|
||
tp_mark = self.pkg_to_tp_mark.get(app_id, "")
|
||
app_label = stats['app_label']
|
||
app_name = app_label
|
||
|
||
self_ratio = (stats['self_bytes'] / total) * 100
|
||
recog_ratio = ((stats['self_bytes'] + stats['server_bytes']) / total) * 100
|
||
|
||
analysis_list = []
|
||
sorted_comps = sorted(stats['components'].items(), key=lambda x: x[1]['bytes'], reverse=True)
|
||
|
||
for comp_tp_mark, comp_data in sorted_comps:
|
||
comp_bytes = comp_data['bytes']
|
||
comp_percent = (comp_bytes / total) * 100
|
||
is_self_flag = '0' if comp_data['is_self'] else '1'
|
||
comp_pkg_str = self.tp_pkg_string.get(comp_tp_mark, "")
|
||
|
||
analysis_item = [
|
||
comp_tp_mark, comp_pkg_str, is_self_flag, f"{comp_percent:.2f}%", int(comp_bytes)
|
||
]
|
||
analysis_list.append(analysis_item)
|
||
|
||
data_rows.append({
|
||
'app_id': app_id,
|
||
'app_label': app_label,
|
||
'app_name': app_name,
|
||
'tp_mark': tp_mark,
|
||
'total_traffic(Bytes)': int(total),
|
||
'self_traffic_count(Bytes)': format_bytes(stats['self_bytes']),
|
||
'server_traffic_count(Bytes)': format_bytes(stats['server_bytes']),
|
||
'unrecognized_traffic_count(Bytes)': format_bytes(stats['unrec_bytes']),
|
||
'self_label_count': stats['self_count'],
|
||
'server_label_count': stats['server_count'],
|
||
'unrecognized_label_count': stats['unrec_count'],
|
||
'self_ratio': f"{self_ratio:.2f}",
|
||
'recognition_ratio': f"{recog_ratio:.2f}",
|
||
'traffic_analysis': str(analysis_list)
|
||
})
|
||
|
||
df = pd.DataFrame(data_rows)
|
||
cols = [
|
||
'app_id', 'app_label', 'app_name', 'tp_mark',
|
||
'total_traffic(Bytes)', 'self_traffic_count(Bytes)', 'server_traffic_count(Bytes)',
|
||
'unrecognized_traffic_count(Bytes)',
|
||
'self_label_count', 'server_label_count', 'unrecognized_label_count',
|
||
'self_ratio', 'recognition_ratio', 'traffic_analysis'
|
||
]
|
||
if not df.empty:
|
||
df = df[cols]
|
||
df.sort_values(by='total_traffic(Bytes)', ascending=False, inplace=True)
|
||
df.to_csv(OUTPUT_APP_PROFILE, index=False)
|
||
print(f" - 完成: {OUTPUT_APP_PROFILE.name}")
|
||
|
||
def export_unmatched_list(self):
|
||
print(f"[5/5] 生成未匹配域名列表: {OUTPUT_UNMATCHED_FILE.name}")
|
||
if not self.unmatched_records:
|
||
print(" - 无未匹配记录。")
|
||
else:
|
||
export_rows = []
|
||
for record in self.unmatched_records:
|
||
app_id = record['app_id']
|
||
size = record['traffic_size']
|
||
is_ip = record['is_ip_flow']
|
||
|
||
stats = self.source_app_stats[app_id]
|
||
total_bytes = stats['total_bytes']
|
||
total_domain_bytes = stats['total_domain_bytes']
|
||
|
||
traffic_ratio = (size / total_bytes * 100) if total_bytes > 0 else 0
|
||
|
||
if is_ip:
|
||
domain_ratio_str = "0.00%"
|
||
else:
|
||
domain_ratio = (size / total_domain_bytes * 100) if total_domain_bytes > 0 else 0
|
||
domain_ratio_str = f"{domain_ratio:.2f}%"
|
||
|
||
export_rows.append({
|
||
'app_name': record['app_label'],
|
||
'app_label': record['app_label'],
|
||
'app_id': app_id,
|
||
'input_url': record['input_url'],
|
||
'traffic_count(Bytes)': int(size),
|
||
'traffic_ratio': f"{traffic_ratio:.2f}%",
|
||
'domain_traffic_ratio': domain_ratio_str,
|
||
'organization': ''
|
||
})
|
||
|
||
df = pd.DataFrame(export_rows)
|
||
if not df.empty:
|
||
df.sort_values(by='traffic_count(Bytes)', ascending=False, inplace=True)
|
||
df.to_csv(OUTPUT_UNMATCHED_FILE, index=False)
|
||
print(f" - 完成: {OUTPUT_UNMATCHED_FILE.name}")
|
||
|
||
def export_detail_log(self):
|
||
print(f"[Bonus] 生成详细流水日志: {OUTPUT_DETAIL_FILE.name}")
|
||
|
||
if not self.all_detail_records:
|
||
print(" - 无流水记录。")
|
||
return
|
||
|
||
export_rows = []
|
||
# 批量处理，提升速度
|
||
for r in self.all_detail_records:
|
||
app_id = r['app_id']
|
||
size = r['traffic_count_bytes']
|
||
is_ip = r['is_ip_flow']
|
||
|
||
# 获取该 App 的统计数据以计算 Ratio
|
||
stats = self.source_app_stats[app_id]
|
||
total_bytes = stats['total_bytes']
|
||
total_domain_bytes = stats['total_domain_bytes']
|
||
|
||
# 1. Total Ratio
|
||
ratio = (size / total_bytes * 100) if total_bytes > 0 else 0
|
||
|
||
# 2. Domain Ratio
|
||
if is_ip:
|
||
domain_ratio_str = "0.00%"
|
||
else:
|
||
d_ratio = (size / total_domain_bytes * 100) if total_domain_bytes > 0 else 0
|
||
domain_ratio_str = f"{d_ratio:.2f}%"
|
||
|
||
export_rows.append({
|
||
'tp_result': r['tp_result'],
|
||
'app_name': r['app_name'],
|
||
'app_label': r['app_label'],
|
||
'app_id': app_id,
|
||
'input_url': r['input_url'],
|
||
'tp_url': r['tp_url'],
|
||
'self': str(r['self']), # 转字符串
|
||
'traffic_count(Bytes)': int(size),
|
||
'traffic_ratio': f"{ratio:.2f}%",
|
||
'domain_traffic_ratio': domain_ratio_str,
|
||
'organization': ''
|
||
})
|
||
|
||
df = pd.DataFrame(export_rows)
|
||
# 排序建议：先按 App ID 聚类，再按流量大小降序
|
||
if not df.empty:
|
||
df.sort_values(by=['app_id', 'traffic_count(Bytes)'], ascending=[True, False], inplace=True)
|
||
|
||
df.to_csv(OUTPUT_DETAIL_FILE, index=False)
|
||
print(f" - 完成: {OUTPUT_DETAIL_FILE.name}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
tester = DpiDatabaseTester()
|
||
tester.check_paths()
|
||
tester.load_app_list()
|
||
tester.load_url_list()
|
||
tester.process_traffic()
|
||
tester.export_app_profile()
|
||
tester.export_unmatched_list()
|
||
tester.export_detail_log()
|