mcn-hot-trend/test_debug.py at main · yaoshi-labs/mcn-hot-trend · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
from app.crawler.zhihu import fetch_zhihu_trends
from app.crawler.douyin import fetch_douyin_trends
from app.crawler.weibo import fetch_weibo_trends
from app.crawler.baidu import fetch_baidu_trends
from app.processor.dedup import deduplicate_trends
from app.processor.sorter import sort_trends_by_heat_increase
from app.evaluator.rule_based import evaluate_trend
from app.models.database import get_db, HotTrend

# 抓取各平台热点
print("抓取各平台热点...")
douyin_trends = fetch_douyin_trends()
weibo_trends = fetch_weibo_trends()
baidu_trends = fetch_baidu_trends()
zhihu_trends = fetch_zhihu_trends()

print(f"抖音: {len(douyin_trends)} 条")
print(f"微博: {len(weibo_trends)} 条")
print(f"百度: {len(baidu_trends)} 条")
print(f"知乎: {len(zhihu_trends)} 条")

# 合并去重
print("\n合并去重...")
all_trends = douyin_trends + weibo_trends + baidu_trends + zhihu_trends
print(f"总数据量: {len(all_trends)}")

unique_trends = deduplicate_trends(all_trends)
print(f"去重后: {len(unique_trends)}")

# 检查知乎数据是否在去重后保留
zhihu_in_unique = [t for t in unique_trends if t.get('platform') == '知乎']
print(f"知乎数据在去重后: {len(zhihu_in_unique)}")
print("知乎数据示例:", zhihu_in_unique[:2])

# 排序
sorted_trends = sort_trends_by_heat_increase(unique_trends)

# 评估并存储
print("\n评估并存储...")
db = next(get_db())
stored_count = 0
zhihu_stored = 0

for trend in sorted_trends:
    # 评估热点
    evaluated_trend = evaluate_trend(trend)

    # 确保keyword不为None
    keyword = evaluated_trend.get("keyword")
    if keyword:
        # 存储到数据库
        db_trend = HotTrend(
            keyword=keyword,
            sources=evaluated_trend.get("source_url"),
            heat_current=evaluated_trend.get("heat_current"),
            heat_1h_increase=evaluated_trend.get("heat_1h_increase", 0),
            platforms=evaluated_trend.get("platform"),
            ai_level=evaluated_trend.get("ai_level"),
            ai_category=evaluated_trend.get("ai_category"),
            ai_risk_level=evaluated_trend.get("ai_risk_level"),
            ai_recommend_account_type=evaluated_trend.get("ai_recommend_account_type"),
            ai_suggestion=evaluated_trend.get("ai_suggestion"),
            ai_confidence=evaluated_trend.get("ai_confidence")
        )
        db.add(db_trend)
        stored_count += 1
        if evaluated_trend.get("platform") == "知乎":
            zhihu_stored += 1

print(f"存储总数: {stored_count}")
print(f"知乎存储数: {zhihu_stored}")

db.commit()
print("数据已提交到数据库")

# 检查数据库中的知乎数据
db_zhihu_count = db.query(HotTrend).filter(HotTrend.platforms == "知乎").count()
print(f"数据库中知乎数据: {db_zhihu_count} 条")

# 检查数据库中所有平台的数据分布
print("\n数据库中各平台数据分布:")
platforms = db.query(HotTrend.platforms, HotTrend.id).all()
platform_count = {}
for platform, _ in platforms:
    if platform in platform_count:
        platform_count[platform] += 1
    else:
        platform_count[platform] = 1

for platform, count in platform_count.items():
    print(f"{platform}: {count} 条")