08_evaluation.py

"""
Step 8：模型績效評估與視覺化（對應報告第 4 章）
  - 讀取 data/results/ 下所有預測結果 CSV
  - 計算勝率（方向正確率）與累積報酬
  - 交易策略：pred=1 做多（賺實際報酬）；pred=0 做空（賺負實際報酬）
  - 輸出：charts/ 下各公司×門檻的累積報酬走勢圖（對應報告第 4.3 節圖表）
  - 輸出：各模型勝率與平均累積報酬匯總表（對應報告第 4.1、4.2 節表格）
  - TWII 作為 Buy & Hold 基準線顯示於圖表中（不進行新聞預測）
"""

import os
import warnings
import numpy as np
import pandas as pd
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import matplotlib.ticker as mtick
from matplotlib import rcParams

from config import COMPANIES, THRESHOLDS, DATA_DIR, RES_DIR, CHART_DIR, price_fname

warnings.filterwarnings("ignore")

# 讓 matplotlib 顯示中文
rcParams["font.family"] = ["Microsoft JhengHei", "Arial Unicode MS", "sans-serif"]
rcParams["axes.unicode_minus"] = False

MODELS = ["OCSVM", "CNN", "RF"]
PREFIXES = ["baseline", "smote"]

MODEL_COLORS = {
    "baseline_OCSVM": "#E07B54",
    "baseline_CNN":   "#5B8DB8",
    "baseline_RF":    "#6BAE75",
    "smote_OCSVM":    "#C0392B",
    "smote_CNN":      "#2471A3",
    "smote_RF":       "#1E8449",
}
MODEL_LABELS = {
    "baseline_OCSVM": "Baseline OCSVM",
    "baseline_CNN":   "Baseline CNN",
    "baseline_RF":    "Baseline RF",
    "smote_OCSVM":    "SMOTETomek OCSVM",
    "smote_CNN":      "SMOTETomek CNN",
    "smote_RF":       "SMOTETomek RF",
}


# ── 讀取單一結果檔 ──────────────────────────────────────────────

def load_result(prefix: str, company_id: str, threshold: int, model: str) -> pd.DataFrame:
    fname = f"{prefix}_{company_id}_{threshold}pct_{model}.csv"
    path  = os.path.join(RES_DIR, fname)
    if not os.path.exists(path):
        return pd.DataFrame()
    df = pd.read_csv(path, parse_dates=["date"])
    df = df.sort_values("date").reset_index(drop=True)
    return df


# ── 計算績效指標 ────────────────────────────────────────────────

def compute_metrics(df: pd.DataFrame) -> dict:
    """
    df 欄位：date, true_label, pred_label, 漲幅百分比
    回傳：win_rate, cum_return (%), daily_strategy_return series
    """
    if df.empty:
        return {"win_rate": np.nan, "cum_return": np.nan, "daily_returns": pd.Series(dtype=float)}

    y_true = df["true_label"].values
    y_pred = df["pred_label"].values
    pct    = df["漲幅百分比"].values  # 當日實際漲幅（%）

    # 勝率：預測方向正確率
    win_rate = np.mean(y_true == y_pred)

    # 策略報酬：pred=1 做多，pred=0 做空
    # 做空報酬 = −實際漲幅
    direction     = 2 * y_pred - 1          # pred=1 → +1, pred=0 → −1
    daily_ret_pct = direction * pct          # 每日策略報酬（%）
    daily_ret     = daily_ret_pct / 100.0    # 轉成小數

    cum_return = (np.prod(1 + daily_ret) - 1) * 100  # 累積報酬（%）

    series = pd.Series(daily_ret.tolist(), index=df["date"].values)
    return {"win_rate": win_rate, "cum_return": cum_return, "daily_returns": series}


def cumulative_curve(daily_returns: pd.Series) -> pd.Series:
    """將每日報酬序列轉成累積報酬曲線（%）"""
    if daily_returns.empty:
        return pd.Series(dtype=float)
    cum = (1 + daily_returns).cumprod() - 1
    return cum * 100


# ── 讀取大盤／個股買持（Buy & Hold）曲線 ─────────────────────

def load_bah_curve(company_id: str, start_date, end_date) -> pd.Series:
    """
    讀取 {company_id}_{start_year}_{end_year}.csv，截取測試期間，
    計算 Buy & Hold 累積報酬（%）
    """
    path = os.path.join(DATA_DIR, price_fname(company_id))
    if not os.path.exists(path):
        return pd.Series(dtype=float)
    df = pd.read_csv(path, parse_dates=["date"])
    df = df.sort_values("date")
    df = df[(df["date"] >= pd.Timestamp(start_date)) &
            (df["date"] <= pd.Timestamp(end_date))].copy()
    if df.empty:
        return pd.Series(dtype=float)
    ret        = df["漲幅百分比"].values / 100.0
    cum        = (1 + ret).cumprod() - 1
    return pd.Series(cum * 100, index=df["date"].values)


def load_twii_curve(start_date, end_date) -> pd.Series:
    return load_bah_curve("TWII", start_date, end_date)


# ── 繪製單一公司×門檻 累積報酬圖 ──────────────────────────────

def plot_cumulative(company_id: str, threshold: int,
                    curves: dict,        # key=(prefix, model), val=pd.Series
                    bah_series: pd.Series,
                    twii_series: pd.Series):
    """
    curves: {('baseline','RF'): pd.Series, ...}
    """
    fig, ax = plt.subplots(figsize=(12, 6))

    # 大盤 & 個股買持
    if not twii_series.empty:
        ax.plot(twii_series.index, twii_series.values,
                color="black", linewidth=1.2, linestyle="--", label="TWII Buy & Hold")
    if not bah_series.empty:
        ax.plot(bah_series.index, bah_series.values,
                color="gray", linewidth=1.2, linestyle=":", label=f"{company_id} Buy & Hold")

    # 各模型策略曲線
    for (prefix, model), series in curves.items():
        if series is None or series.empty:
            continue
        key   = f"{prefix}_{model}"
        color = MODEL_COLORS.get(key, "#999999")
        label = MODEL_LABELS.get(key, key)
        ax.plot(series.index, series.values, color=color, linewidth=1.5, label=label)

    ax.axhline(0, color="black", linewidth=0.8, linestyle="-")
    ax.yaxis.set_major_formatter(mtick.PercentFormatter())
    ax.set_title(f"{COMPANIES[company_id]['name']}（{company_id}）— 門檻 {threshold}%  累積報酬", fontsize=13)
    ax.set_xlabel("日期")
    ax.set_ylabel("累積報酬（%）")
    ax.legend(loc="best", fontsize=8, ncol=2)
    plt.tight_layout()

    fname = os.path.join(CHART_DIR, f"cumret_{company_id}_{threshold}pct.png")
    plt.savefig(fname, dpi=150)
    plt.close(fig)
    print(f"  圖表已儲存：{fname}")


# ── 匯總表工具 ──────────────────────────────────────────────────

def build_summary_table(records: list) -> pd.DataFrame:
    """
    records: list of dict with keys:
        company, threshold, prefix, model, win_rate, cum_return
    回傳 pivot：index=(company, model), columns=(prefix, threshold)
    """
    df = pd.DataFrame(records)
    if df.empty:
        return df
    df["win_rate_%"]    = (df["win_rate"] * 100).round(2)
    df["cum_return_%"]  = df["cum_return"].round(2)
    return df


def print_win_rate_table(df: pd.DataFrame):
    if df.empty:
        print("  (無資料)")
        return
    pivot = df.pivot_table(
        index=["company", "prefix", "model"],
        columns="threshold",
        values="win_rate_%",
        aggfunc="first"
    )
    print(pivot.to_string())


def print_cum_return_table(df: pd.DataFrame):
    if df.empty:
        print("  (無資料)")
        return
    pivot = df.pivot_table(
        index=["company", "prefix", "model"],
        columns="threshold",
        values="cum_return_%",
        aggfunc="first"
    )
    print(pivot.to_string())


# ── 表 5.1：各模型跨門檻平均累積報酬 ────────────────────────────

def print_avg_cum_return(df: pd.DataFrame):
    """對每個 (company, prefix, model) 取 6 個門檻的平均累積報酬"""
    if df.empty:
        return
    avg = (
        df.groupby(["company", "prefix", "model"])["cum_return_%"]
        .mean()
        .round(2)
        .reset_index()
        .rename(columns={"cum_return_%": "avg_cum_return_%"})
    )
    pivot = avg.pivot_table(
        index=["prefix", "model"],
        columns="company",
        values="avg_cum_return_%",
        aggfunc="first"
    )
    print("\n[Table 5.1] 各模型平均累積報酬（%，跨門檻平均）")
    print(pivot.to_string())


# ── 表 5.2：SMOTETomek vs Baseline 改善計數 ─────────────────────

def print_improvement_table(df: pd.DataFrame):
    """統計 smote 各指標勝過 baseline 的次數"""
    if df.empty:
        return
    rows = []
    for company in df["company"].unique():
        for model in MODELS:
            for t in THRESHOLDS:
                base = df[(df["company"] == company) & (df["prefix"] == "baseline") &
                          (df["model"] == model) & (df["threshold"] == t)]
                smote = df[(df["company"] == company) & (df["prefix"] == "smote") &
                           (df["model"] == model) & (df["threshold"] == t)]
                if base.empty or smote.empty:
                    continue
                wr_base   = base["win_rate_%"].values[0]
                wr_smote  = smote["win_rate_%"].values[0]
                cr_base   = base["cum_return_%"].values[0]
                cr_smote  = smote["cum_return_%"].values[0]
                rows.append({
                    "company"    : company,
                    "model"      : model,
                    "threshold"  : t,
                    "wr_improve" : int(wr_smote > wr_base),
                    "cr_improve" : int(cr_smote > cr_base),
                })

    imp = pd.DataFrame(rows)
    if imp.empty:
        return
    summary = (
        imp.groupby(["company", "model"])[["wr_improve", "cr_improve"]]
        .sum()
        .rename(columns={"wr_improve": "勝率改善次數", "cr_improve": "累積報酬改善次數"})
    )
    print("\n[Table 5.2] SMOTETomek 相對 Baseline 改善次數（共 6 個門檻）")
    print(summary.to_string())


# ── 主程式 ─────────────────────────────────────────────────────

def main():
    print("=== Step 8：績效評估與視覺化（報告第 4 章） ===\n")

    all_records = []

    # 確定測試期間（從任一已有結果檔取日期範圍）
    test_start = "2024-01-01"
    test_end   = "2025-10-31"

    twii_series = load_twii_curve(test_start, test_end)

    for cid, info in COMPANIES.items():
        print(f"\n── {info['name']}（{cid}）──")
        bah_series = load_bah_curve(cid, test_start, test_end)

        # TWII 本身即為大盤基準，圖表中不重複繪製 TWII Buy & Hold 參考線
        ref_twii = pd.Series(dtype=float) if cid == "TWII" else twii_series

        for t in THRESHOLDS:
            curves = {}
            has_any = False

            for prefix in PREFIXES:
                for model in MODELS:
                    df = load_result(prefix, cid, t, model)
                    if df.empty:
                        continue
                    has_any = True

                    # 縮小至測試期間（避免 val 混入）
                    df = df[df["date"] >= pd.Timestamp(test_start)].copy()
                    if df.empty:
                        continue

                    metrics = compute_metrics(df)
                    curves[(prefix, model)] = cumulative_curve(metrics["daily_returns"])

                    all_records.append({
                        "company"    : cid,
                        "threshold"  : t,
                        "prefix"     : prefix,
                        "model"      : model,
                        "win_rate"   : metrics["win_rate"],
                        "cum_return" : metrics["cum_return"],
                    })

                    wr  = metrics["win_rate"]
                    cr  = metrics["cum_return"]
                    tag = f"{prefix}_{model}"
                    print(f"  {tag:<22} 門檻={t}%  勝率={wr*100:.1f}%  累積報酬={cr:.2f}%")

            if has_any:
                plot_cumulative(cid, t, curves, bah_series, ref_twii)

    # ── 匯總表 ──
    summary_df = build_summary_table(all_records)
    if not summary_df.empty:
        print("\n" + "="*70)
        print("[Table 4.x] 勝率（%）一覽")
        print("="*70)
        print_win_rate_table(summary_df)

        print("\n" + "="*70)
        print("[Table 4.x] 累積報酬（%）一覽")
        print("="*70)
        print_cum_return_table(summary_df)

        print_avg_cum_return(summary_df)
        print_improvement_table(summary_df)

        # 儲存 CSV 方便後續查閱
        out_csv = os.path.join(RES_DIR, "evaluation_summary.csv")
        summary_df.to_csv(out_csv, index=False, encoding="utf-8-sig")
        print(f"\n匯總結果已儲存 → {out_csv}")

    print("\n所有評估完成。")


if __name__ == "__main__":
    main()