06_baseline_models.py

"""
Step 6：基礎模型訓練（不加重採樣）（對應報告第 3.2.4 節）
  模型：
    - OCSVM  → 靜態切割，只用正類（label=1，高漲幅日）訓練
    - CNN    → 靜態切割，EarlyStopping
    - RF     → Walk-Forward Validation（每日滾動重訓）
  特徵：TF-IDF（max_features=300）
  輸出：data/results/baseline_{company_id}_{N}pct_{model}.csv
         每列包含：date, true_label, pred_label, 漲幅百分比

  對象：大盤（TWII）及四支個股（2330、3231、2368、3017）皆執行，
        與報告 3.2.4 節以 TWII 為主體說明各模型流程一致。
        OCSVM 訓練於正類（高漲幅日，label=1），
        此邏輯與 Step 7 SMOTETomek 版本一致。
"""

import os
import numpy as np
import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.svm import OneClassSVM
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import classification_report
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Conv1D, GlobalMaxPooling1D, Dropout, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import EarlyStopping

from config import (COMPANIES, THRESHOLDS, DATA_DIR, RES_DIR,
                    TFIDF_MAX_FEATURES, OCSVM_PARAMS, CNN_EPOCHS,
                    CNN_BATCH, CNN_PATIENCE, RF_PARAMS)

tf.get_logger().setLevel("ERROR")


# ── 共用工具 ──────────────────────────────────────────────────

def load_merged(company_id: str, threshold: int) -> pd.DataFrame:
    path = os.path.join(DATA_DIR, f"merged_{company_id}_{threshold}pct.csv")
    if not os.path.exists(path):
        return pd.DataFrame()
    df = pd.read_csv(path, parse_dates=["date"])
    return df


def split_data(df: pd.DataFrame):
    train = df[df["split"] == "train"].copy()
    val   = df[df["split"] == "val"].copy()
    test  = df[df["split"] == "test"].copy()
    return train, val, test


def vectorize(train_df, val_df, test_df):
    vec     = TfidfVectorizer(max_features=TFIDF_MAX_FEATURES)
    X_train = vec.fit_transform(train_df["clean_text"])
    X_val   = vec.transform(val_df["clean_text"])
    X_test  = vec.transform(test_df["clean_text"])
    y_train = train_df["label"].values
    y_val   = val_df["label"].values
    y_test  = test_df["label"].values
    return vec, X_train, X_val, X_test, y_train, y_val, y_test


def save_result(df_split, y_pred, company_id, threshold, model_name, suffix="baseline"):
    out = pd.DataFrame({
        "date"        : df_split["date"].values,
        "true_label"  : df_split["label"].values,
        "pred_label"  : y_pred,
        "漲幅百分比"  : df_split["漲幅百分比"].values,
    })
    fname = f"{suffix}_{company_id}_{threshold}pct_{model_name}.csv"
    out.to_csv(os.path.join(RES_DIR, fname), index=False, encoding="utf-8-sig")
    return out


# ── OCSVM（靜態） ─────────────────────────────────────────────

def run_ocsvm(train_df, val_df, test_df, company_id, threshold):
    _, X_train, X_val, X_test, y_train, y_val, y_test = vectorize(train_df, val_df, test_df)

    # 只用正類訓練
    pos_idx    = np.where(y_train == 1)[0]
    X_train_pos = X_train[pos_idx]
    print(f"    OCSVM 正類樣本數：{X_train_pos.shape[0]}")

    model = OneClassSVM(**OCSVM_PARAMS)
    model.fit(X_train_pos)

    # OCSVM 輸出 1/−1，轉成 1/0
    y_test_pred = np.where(model.predict(X_test) == 1, 1, 0)

    print(f"    測試集結果：\n{classification_report(y_test, y_test_pred, digits=3, zero_division=0)}")
    save_result(test_df, y_test_pred, company_id, threshold, "OCSVM")


# ── CNN（靜態） ───────────────────────────────────────────────

def build_cnn(input_dim: int) -> Sequential:
    model = Sequential([
        Input(shape=(input_dim, 1)),
        Conv1D(32, kernel_size=3, activation="relu"),
        GlobalMaxPooling1D(),
        Dense(64, activation="relu"),
        Dropout(0.5),
        Dense(1, activation="sigmoid"),
    ])
    model.compile(optimizer=Adam(learning_rate=0.001),
                  loss="binary_crossentropy",
                  metrics=["accuracy"])
    return model


def run_cnn(train_df, val_df, test_df, company_id, threshold):
    _, X_train, X_val, X_test, y_train, y_val, y_test = vectorize(train_df, val_df, test_df)

    X_train_d = np.expand_dims(X_train.toarray(), axis=2)
    X_val_d   = np.expand_dims(X_val.toarray(),   axis=2)
    X_test_d  = np.expand_dims(X_test.toarray(),  axis=2)

    model = build_cnn(X_train_d.shape[1])
    es    = EarlyStopping(monitor="val_loss", patience=CNN_PATIENCE, restore_best_weights=True)

    model.fit(X_train_d, y_train,
              validation_data=(X_val_d, y_val),
              epochs=CNN_EPOCHS, batch_size=CNN_BATCH,
              callbacks=[es], verbose=0)

    y_test_pred = (model.predict(X_test_d, verbose=0) > 0.5).astype(int).flatten()
    print(f"    測試集結果：\n{classification_report(y_test, y_test_pred, digits=3, zero_division=0)}")
    save_result(test_df, y_test_pred, company_id, threshold, "CNN")


# ── RF（Walk-Forward Validation） ────────────────────────────

def run_rf(df, company_id, threshold):
    """每個測試日：用截至昨天的所有資料重新訓練 RF，預測當天"""
    test_df    = df[df["split"] == "test"].copy()
    test_dates = sorted(test_df["date"].unique())
    history_df = df[df["split"].isin(["train", "val"])].copy()

    daily_results = []
    for i, cur_date in enumerate(test_dates):
        if history_df.empty:
            break

        vec     = TfidfVectorizer(max_features=TFIDF_MAX_FEATURES)
        X_hist  = vec.fit_transform(history_df["clean_text"])
        y_hist  = history_df["label"].values

        # 若訓練集只有一個類別則跳過
        if len(np.unique(y_hist)) < 2:
            pass  # RF 仍可訓練，但可能預測全同類

        today_df = test_df[test_df["date"] == cur_date]
        if today_df.empty:
            continue

        X_today = vec.transform(today_df["clean_text"])

        rf = RandomForestClassifier(**RF_PARAMS)
        rf.fit(X_hist, y_hist)
        pred = rf.predict(X_today)[0]

        true_label = today_df["label"].values[0]
        ret        = today_df["漲幅百分比"].values[0]
        daily_results.append({
            "date"       : cur_date,
            "true_label" : true_label,
            "pred_label" : pred,
            "漲幅百分比" : ret,
        })

        # 將當天加入歷史
        history_df = pd.concat([history_df, today_df], ignore_index=True)

        if (i + 1) % 20 == 0 or i == len(test_dates) - 1:
            print(f"    RF 進度：{i+1}/{len(test_dates)} 天")

    result_df = pd.DataFrame(daily_results)
    if not result_df.empty:
        y_true = result_df["true_label"].values
        y_pred = result_df["pred_label"].values
        print(f"    測試集結果：\n{classification_report(y_true, y_pred, digits=3, zero_division=0)}")
        fname = f"baseline_{company_id}_{threshold}pct_RF.csv"
        result_df.to_csv(os.path.join(RES_DIR, fname), index=False, encoding="utf-8-sig")


# ── 主程式 ────────────────────────────────────────────────────

def main():
    print("=== Step 6：基礎模型訓練（報告 3.2.4 節） ===\n")

    for cid, info in COMPANIES.items():
        for t in THRESHOLDS:
            print(f"\n── {info['name']} | 門檻 {t}% ──")

            df = load_merged(cid, t)
            if df.empty:
                print("  資料不存在，跳過。")
                continue

            train_df, val_df, test_df = split_data(df)
            if test_df.empty:
                print("  測試集為空，跳過。")
                continue

            # OCSVM
            ocsvm_out = os.path.join(RES_DIR, f"baseline_{cid}_{t}pct_OCSVM.csv")
            if not os.path.exists(ocsvm_out):
                print("  [OCSVM]")
                run_ocsvm(train_df, val_df, test_df, cid, t)
            else:
                print("  [OCSVM] 已完成，跳過。")

            # CNN
            cnn_out = os.path.join(RES_DIR, f"baseline_{cid}_{t}pct_CNN.csv")
            if not os.path.exists(cnn_out):
                print("  [CNN]")
                run_cnn(train_df, val_df, test_df, cid, t)
            else:
                print("  [CNN] 已完成，跳過。")

            # RF（Walk-Forward）
            rf_out = os.path.join(RES_DIR, f"baseline_{cid}_{t}pct_RF.csv")
            if not os.path.exists(rf_out):
                print("  [RF - Walk-Forward]")
                run_rf(df, cid, t)
            else:
                print("  [RF] 已完成，跳過。")

    print("\n基礎模型全部完成。")


if __name__ == "__main__":
    main()