"""
ifo_regime_backtest_v1.py
==========================
Tests whether the IFO Business Climate Index 3-month trend direction
predicts which direction EURUSD z-score trades are more likely to win.

HYPOTHESIS 1 — Directional Bias:
  IFO 3-month trend rising  → LONG signals have higher WR than SHORT
  IFO 3-month trend falling → SHORT signals have higher WR than LONG
  
  Mechanism: the same German economic sentiment that drives IFO also
  drives ECB policy expectations → DE 2Y yield → US-DE spread direction.
  A rising IFO means Germany strengthening → ECB hawkishness rising →
  DE yields rising → spread narrowing → EURUSD structurally bullish.

HYPOTHESIS 2 — Size Scaling:
  Aligned trade   (LONG + IFO rising, or SHORT + IFO falling) → 1.5x risk
  Misaligned trade (LONG + IFO falling, or SHORT + IFO rising) → 0.75x risk
  Neutral (IFO flat within ±0.1)                               → 1.0x risk

WHY THIS IS DIFFERENT from previous research:
  Previous tests (Markov, Hurst, Kalman, OU, EVZ) used derived signals
  from the price/spread data itself — all failed because they were
  redundant with the z-score signal already computed.
  IFO is an independent survey-based leading indicator with no
  mathematical relationship to the yield spread z-score.

Data required:
  data/raw/rates/ifo.csv       ← RAWDATA.csv from lesson zip, renamed
  data/processed/trades_real_costs.csv  ← validated 4,019 trades

Run from project root:
  python src/research/ifo_regime_backtest_v1.py
"""

import pandas as pd
import numpy as np
from pathlib import Path
import sys
import warnings
warnings.filterwarnings('ignore')

BASE_PATH  = Path(__file__).resolve().parents[2]
PROC_PATH  = BASE_PATH / "data" / "processed"
RATES_PATH = BASE_PATH / "data" / "raw" / "rates"
if str(BASE_PATH / "src") not in sys.path:
    sys.path.insert(0, str(BASE_PATH / "src"))

NOTIONAL     = 300_000
TP_BASE      = 0.0020
STOP_PCT     = 0.0025
SPREAD_COST  = 0.0001
ZSCORE_BANDS = [(2.75,3.50,1.0),(3.50,4.50,1.5),(4.50,99.,2.0)]

# IFO trend threshold — change smaller than this = neutral
IFO_NEUTRAL_BAND = 0.10

def get_mult(z):
    for lo, hi, m in ZSCORE_BANDS:
        if lo <= z < hi: return m
    return 2.0


def ifo_regime(trend):
    if trend >  IFO_NEUTRAL_BAND: return "rising"
    if trend < -IFO_NEUTRAL_BAND: return "falling"
    return "neutral"


def alignment(direction, regime):
    """Is the trade direction aligned with the IFO trend?"""
    if direction ==  1 and regime == "rising":  return "aligned"
    if direction == -1 and regime == "falling": return "aligned"
    if direction ==  1 and regime == "falling": return "misaligned"
    if direction == -1 and regime == "rising":  return "misaligned"
    return "neutral"


def score(pnl_arr, label, n_years=18):
    pnl    = np.array(pnl_arr, dtype=float)
    n      = len(pnl)
    if n == 0:
        return {"label":label,"n":0,"wr":0,"sharpe":0,"pf":0,"dd":0,"total":0}
    wr     = (pnl > 0).mean() * 100
    total  = pnl.sum()
    eq     = 100_000 + np.cumsum(pnl)
    peak   = np.maximum.accumulate(eq)
    max_dd = ((eq - peak) / peak * 100).min()
    per_yr = n / n_years
    exc    = pnl / 100_000 - 0.04 / per_yr
    sharpe = exc.mean() / exc.std() * np.sqrt(per_yr) if exc.std() > 0 else 0
    gp = pnl[pnl > 0].sum(); gl = abs(pnl[pnl < 0].sum())
    pf = round(gp / gl, 2) if gl > 0 else 999
    return {"label":label,"n":n,"wr":round(wr,1),"sharpe":round(sharpe,2),
            "pf":pf,"dd":round(max_dd,2),"total":int(total)}


def print_table(rows, title):
    print(f"\n{'='*72}")
    print(title)
    print(f"{'='*72}")
    print(f"  {'Segment':<32}{'N':>5}{'WR%':>6}{'Sharpe':>8}"
          f"{'PF':>6}{'MaxDD%':>8}{'Total$':>11}")
    print(f"  {'─'*68}")
    for r in rows:
        if r['n'] == 0: continue
        print(f"  {r['label']:<32}{r['n']:>5}{r['wr']:>6.1f}"
              f"{r['sharpe']:>8.2f}{r['pf']:>6.2f}"
              f"{r['dd']:>8.2f}{r['total']:>11,.0f}")


def main():
    print("="*72)
    print("IFO REGIME BACKTEST v1")
    print("Testing IFO Business Climate as directional bias for EURUSD trades")
    print("="*72)

    # ── Load IFO data ─────────────────────────────────────────────────────────
    print("\nStep 1: Loading IFO data...")
    ifo_path = next((p for p in [
        RATES_PATH / "ifo.csv",
        RATES_PATH / "RAWDATA.csv",
        RATES_PATH / "ifo_rawdata.csv",
    ] if p.exists()), None)

    if ifo_path is None:
        print("  ERROR: ifo.csv not found in data/raw/rates/")
        print("  Expected: data/raw/rates/ifo.csv")
        print("  Source: RAWDATA.csv from the DAX IFO lesson zip, renamed ifo.csv")
        sys.exit(1)

    ifo = pd.read_csv(ifo_path)
    ifo.columns = ifo.columns.str.strip()

    # Handle both RAWDATA.csv format (ISO BE) and custom naming
    date_col = next(c for c in ifo.columns if 'date' in c.lower() or c == 'Date')
    val_col  = next(c for c in ifo.columns
                    if any(k in c for k in ['ISO BE','ifo','IFO','level'])
                    and 'change' not in c.lower() and 'pct' not in c.lower()
                    and '%' not in c)

    ifo[date_col] = pd.to_datetime(ifo[date_col], dayfirst=True)
    ifo = ifo.rename(columns={val_col: 'ifo_level', date_col: 'date'})
    ifo = ifo[['date','ifo_level']].sort_values('date').reset_index(drop=True)

    # 3-month rolling average trend
    ifo['ifo_3m']     = ifo['ifo_level'].rolling(3).mean()
    ifo['ifo_3m_prev']= ifo['ifo_3m'].shift(1)
    ifo['ifo_trend']  = ifo['ifo_3m'] - ifo['ifo_3m_prev']
    ifo['regime']     = ifo['ifo_trend'].apply(ifo_regime)
    ifo = ifo.dropna().reset_index(drop=True)

    print(f"  Loaded: {len(ifo):,} monthly IFO readings")
    print(f"  Range : {ifo['date'].min().date()} → {ifo['date'].max().date()}")
    print(f"  Level : min={ifo['ifo_level'].min():.1f}  "
          f"mean={ifo['ifo_level'].mean():.1f}  "
          f"max={ifo['ifo_level'].max():.1f}")
    print(f"  Regime distribution:")
    for r, n in ifo['regime'].value_counts().items():
        print(f"    {r:<10}: {n:>3} months ({n/len(ifo)*100:.0f}%)")

    # ── Load validated trades ─────────────────────────────────────────────────
    print("\nStep 2: Loading validated trades...")
    trades_path = next((p for p in [
        PROC_PATH / "trades_real_costs.csv",
        PROC_PATH / "trades" / "trades_real_costs.csv",
    ] if p.exists()), None)

    if trades_path is None:
        print("  ERROR: trades_real_costs.csv not found"); sys.exit(1)

    df = pd.read_csv(trades_path)
    df["entry_time"] = pd.to_datetime(df["entry_time"])
    print(f"  Loaded: {len(df):,} trades")

    # ── Join IFO regime to each trade ─────────────────────────────────────────
    print("\nStep 3: Joining IFO regime to trades...")

    # For each trade, find the most recently published IFO
    # IFO is released around 25th of month — use last available
    ifo_dates  = ifo['date'].values
    ifo_regimes= ifo['regime'].values
    ifo_levels = ifo['ifo_level'].values
    ifo_trends = ifo['ifo_trend'].values

    trade_regimes = []
    trade_levels  = []
    for entry in df["entry_time"]:
        # Most recent IFO at or before entry date
        mask = ifo_dates <= np.datetime64(entry.date())
        if mask.any():
            idx = np.where(mask)[0][-1]
            trade_regimes.append(ifo_regimes[idx])
            trade_levels.append(ifo_levels[idx])
        else:
            trade_regimes.append(None)
            trade_levels.append(None)

    df["ifo_regime"]   = trade_regimes
    df["ifo_level_at"] = trade_levels
    n_before = len(df)
    df = df.dropna(subset=["ifo_regime"])
    n_after  = len(df)
    print(f"  Trades with IFO: {n_after:,} "
          f"({n_before-n_after} dropped — before IFO history starts)")

    # Alignment
    df["alignment"] = [alignment(int(row.signal), row.ifo_regime)
                       for _, row in df.iterrows()]

    years = (df["entry_time"].max() - df["entry_time"].min()).days / 365.25

    # ── SECTION 1: WR by IFO regime and direction ─────────────────────────────
    print(f"\n{'='*72}")
    print("SECTION 1: WIN RATE BY IFO REGIME AND TRADE DIRECTION")
    print("Core question: does IFO trend predict which direction wins more?")
    print(f"{'='*72}")

    print(f"\n  {'Segment':<30}{'N':>6}{'WR%':>8}{'Avg PnL':>10}{'IFO avg':>9}")
    print(f"  {'─'*62}")

    segments = []
    for regime in ['rising','neutral','falling']:
        for direction, dir_label in [(1,'LONG'),(-1,'SHORT')]:
            mask = ((df['ifo_regime'] == regime) & (df['signal'] == direction))
            sub  = df[mask]
            if len(sub) == 0: continue
            wr   = (sub['dollar_pnl_real'] > 0).mean() * 100
            avg  = sub['dollar_pnl_real'].mean()
            ifo  = sub['ifo_level_at'].mean()
            label= f"IFO {regime:<8} | {dir_label}"
            aligned_str = ""
            if (regime=='rising'  and direction==1): aligned_str = " ✓ aligned"
            if (regime=='falling' and direction==-1): aligned_str = " ✓ aligned"
            if (regime=='rising'  and direction==-1): aligned_str = " ✗ misaligned"
            if (regime=='falling' and direction==1): aligned_str = " ✗ misaligned"
            print(f"  {label:<30}{len(sub):>6}{wr:>8.1f}%"
                  f"{avg:>10.0f}  {ifo:>7.1f}{aligned_str}")
            segments.append({'regime':regime,'dir':dir_label,'n':len(sub),
                             'wr':wr,'avg_pnl':avg})

    # ── SECTION 2: Alignment analysis ────────────────────────────────────────
    print(f"\n{'='*72}")
    print("SECTION 2: ALIGNED vs MISALIGNED vs NEUTRAL")
    print(f"{'='*72}")

    align_results = []
    for align in ['aligned','misaligned','neutral']:
        sub = df[df['alignment'] == align]
        if len(sub) == 0: continue
        r = score(sub['dollar_pnl_real'].values, align.capitalize(), years)
        align_results.append(r)

    baseline = score(df['dollar_pnl_real'].values, "All trades (baseline)", years)
    print_table([baseline] + align_results,
                "P&L by alignment vs baseline")

    # Chi-square test: is WR difference statistically significant?
    from scipy import stats as scipy_stats
    aligned    = df[df['alignment']=='aligned']
    misaligned = df[df['alignment']=='misaligned']
    if len(aligned) > 0 and len(misaligned) > 0:
        al_wins   = (aligned['dollar_pnl_real'] > 0).sum()
        al_losses = len(aligned) - al_wins
        mis_wins  = (misaligned['dollar_pnl_real'] > 0).sum()
        mis_losses= len(misaligned) - mis_wins
        chi2, p   = scipy_stats.chi2_contingency(
            [[al_wins, al_losses],[mis_wins, mis_losses]])[:2]
        print(f"\n  Chi-square test (aligned WR vs misaligned WR):")
        print(f"  Aligned    : {al_wins}/{len(aligned)} "
              f"({al_wins/len(aligned)*100:.1f}% WR)")
        print(f"  Misaligned : {mis_wins}/{len(misaligned)} "
              f"({mis_wins/len(misaligned)*100:.1f}% WR)")
        print(f"  Chi²={chi2:.3f}  p={p:.4f}  "
              f"{'Significant (p<0.05)' if p<0.05 else 'Not significant'}")

    # ── SECTION 3: IFO-scaled position sizing ────────────────────────────────
    print(f"\n{'='*72}")
    print("SECTION 3: IFO-SCALED POSITION SIZING")
    print("Aligned → 1.5x risk | Neutral → 1.0x | Misaligned → 0.75x")
    print(f"{'='*72}")

    scale_map  = {"aligned": 1.5, "neutral": 1.0, "misaligned": 0.75}

    for config_label, al_scale, neu_scale, mis_scale in [
        ("Baseline (1.0x flat)",                  1.0,  1.0,  1.0),
        ("IFO scaled (1.5x / 1.0x / 0.75x)",     1.5,  1.0,  0.75),
        ("IFO aggressive (2.0x / 1.0x / 0.5x)",  2.0,  1.0,  0.50),
        ("IFO only aligned (1.5x / 1.0x / 1.0x)",1.5,  1.0,  1.0),
        ("Skip misaligned (1.5x / 1.0x / skip)",  1.5,  1.0,  0.0),
    ]:
        pnl_scaled = []
        for _, row in df.iterrows():
            a = row['alignment']
            if a == 'aligned':    sc = al_scale
            elif a == 'neutral':  sc = neu_scale
            else:                 sc = mis_scale

            if sc == 0.0:
                continue  # skip misaligned

            base_pnl = float(row['dollar_pnl_real'])
            # Scale: base pnl was computed at mult from zscore band
            # We apply alignment scale on top
            pnl_scaled.append(base_pnl * sc)

        r = score(pnl_scaled, config_label, years)
        r['n_used'] = len(pnl_scaled)
        r['pct_used'] = len(pnl_scaled)/n_after*100
        print(f"\n  {config_label}")
        print(f"  N={r['n']:,} ({r.get('pct_used',100):.0f}%)  "
              f"WR={r['wr']:.1f}%  Sharpe={r['sharpe']:.2f}  "
              f"DD={r['dd']:.2f}%  Total=${r['total']:,.0f}")

    # ── SECTION 4: By z-score band within alignment ───────────────────────────
    print(f"\n{'='*72}")
    print("SECTION 4: Z-SCORE BAND + IFO ALIGNMENT COMBINED")
    print("Does alignment improve WR within each z-score band?")
    print(f"{'='*72}")

    bands = [(2.75,3.50,"1x  (2.75-3.50)"),
             (3.50,4.50,"1.5x (3.50-4.50)"),
             (4.50,9.99,"2x  (4.50+)")]

    print(f"\n  {'Band':<18}{'Align':<14}{'N':>5}{'WR%':>7}{'Avg PnL':>10}")
    print(f"  {'─'*55}")
    for lo, hi, blabel in bands:
        band_df = df[(df['zscore_abs']>=lo) & (df['zscore_abs']<hi)]
        for align in ['aligned','misaligned','neutral']:
            sub = band_df[band_df['alignment']==align]
            if len(sub) < 5: continue
            wr  = (sub['dollar_pnl_real']>0).mean()*100
            avg = sub['dollar_pnl_real'].mean()
            print(f"  {blabel:<18}{align:<14}{len(sub):>5}"
                  f"{wr:>7.1f}%{avg:>10.0f}")

    # ── Conclusion ─────────────────────────────────────────────────────────────
    print(f"\n{'='*72}")
    print("CONCLUSION")
    print(f"{'='*72}")

    al_wr  = (df[df['alignment']=='aligned']['dollar_pnl_real']>0).mean()*100
    mis_wr = (df[df['alignment']=='misaligned']['dollar_pnl_real']>0).mean()*100
    base_wr= (df['dollar_pnl_real']>0).mean()*100
    diff   = al_wr - mis_wr

    print(f"\n  Baseline WR    : {base_wr:.1f}%")
    print(f"  Aligned WR     : {al_wr:.1f}%")
    print(f"  Misaligned WR  : {mis_wr:.1f}%")
    print(f"  Difference     : {diff:+.1f} percentage points")

    if diff >= 5 and p < 0.05:
        verdict = ("IMPLEMENT — IFO alignment predicts trade direction.\n"
                   "  Scale up aligned trades, scale down misaligned.")
    elif diff >= 3:
        verdict = ("MARGINAL — Some directional signal but below threshold.\n"
                   "  Monitor live: track aligned vs misaligned WR over 50+ trades.")
    else:
        verdict = ("NO EDGE — IFO regime does not predict EURUSD trade direction.\n"
                   "  The z-score already captures macro direction independently.")

    print(f"\n  VERDICT: {verdict}")
    print(f"""
  KEY INSIGHT:
  ─────────────────────────────────────────────────────────────────
  If aligned WR > misaligned WR by 5%+ with p < 0.05:
    The IFO is adding directional information that the z-score
    alone does not capture. Scale up aligned, scale down misaligned.
    
  If no difference:
    The yield spread z-score already embeds IFO information.
    Rate markets price in business sentiment before IFO is published.
    This would confirm the z-score is the efficient signal.
  ─────────────────────────────────────────────────────────────────
    """)


if __name__ == "__main__":
    main()