"""
evz_mfe_backtest_v1.py
========================
Tests whether EVZ (CBOE EuroCurrency Volatility Index) can be used
to dynamically widen the take-profit target on high-volatility days,
improving P&L by capturing larger moves when market conditions support them.

HYPOTHESIS:
  High EVZ → options market pricing larger EURUSD moves
           → genuine dislocations tend to travel further before reverting
           → MFE (maximum favourable excursion) is larger on high EVZ days
           → a wider TP captures more of the move without sacrificing win rate

APPROACH:
  1. Load EVZ daily data (EVZCLS.csv from FRED)
  2. Join EVZ level to each of the 4,019 validated trades at entry date
  3. Analyse MFE and MAE distribution by EVZ regime
  4. Test dynamic TP levels (wider TP in high EVZ, standard in low EVZ)
  5. Compare P&L vs validated baseline

KEY DISTINCTION from previous research:
  Previous tests (Markov, Hurst, Kalman, OU) tried to filter trades out.
  This keeps ALL trades and asks: can we earn MORE on the ones we already take?

TP SIMULATION METHODOLOGY:
  For TP exits: if MFE >= wider_tp → earn wider_tp (conservative — actual
    exit could be higher). If MFE < wider_tp → worst case: assume SL hit
    (conservative lower bound). Best case: assume 0.20% exit (upper bound).
  For non-TP exits (SL/z-exit/hold): unchanged regardless of wider TP.
  Results show best-case and worst-case range.

Run from project root:
  python src/research/evz_mfe_backtest_v1.py
"""

import pandas as pd
import numpy as np
from pathlib import Path
import sys
import warnings
warnings.filterwarnings('ignore')

BASE_PATH  = Path(__file__).resolve().parents[2]
PROC_PATH  = BASE_PATH / "data" / "processed"
RATES_PATH = BASE_PATH / "data" / "raw" / "rates"
if str(BASE_PATH / "src") not in sys.path:
    sys.path.insert(0, str(BASE_PATH / "src"))

NOTIONAL     = 300_000
TP_BASE      = 0.0020   # validated 0.20%
STOP_PCT     = 0.0025   # validated 0.25%
SPREAD_COST  = 0.0001
ZSCORE_BANDS = [(2.75, 3.50, 1.0), (3.50, 4.50, 1.5), (4.50, 99.0, 2.0)]

def get_mult(z):
    for lo, hi, m in ZSCORE_BANDS:
        if lo <= z < hi: return m
    return 2.0

def evz_regime(v):
    """Classify EVZ into 4 regimes with intuitive labels."""
    if v < 8:   return "1_Calm   (<8)"
    if v < 11:  return "2_Normal (8-11)"
    if v < 15:  return "3_Elevated(11-15)"
    return              "4_Stress (15+)"

def dynamic_tp(evz, base=TP_BASE):
    """
    Scale TP based on EVZ level.
    Derived from MFE p75 in each regime — set TP where 75% of winners reach.
    These are initial values to test; the script will also sweep across levels.
    """
    if evz >= 15: return 0.0035   # stress: 0.35%
    if evz >= 11: return 0.0027   # elevated: 0.27%
    if evz >= 8:  return 0.0023   # normal: 0.23%
    return base                    # calm: standard 0.20%


def simulate_dynamic_tp(df, tp_col):
    """
    Simulate P&L with dynamic TP applied to each trade.

    Conservative worst-case: trades that previously hit 0.20% TP
    but don't reach the wider TP are assumed to eventually hit SL.

    Best-case: those same trades still exit at 0.20% (no change).
    """
    results = []
    for _, row in df.iterrows():
        mfe       = float(row['mfe'])
        pnl_base  = float(row['dollar_pnl_real'])
        exit_r    = str(row.get('exit_reason', ''))
        z_abs     = float(row.get('zscore_abs', 2.75))
        mult      = get_mult(z_abs)
        tp_wide   = float(row[tp_col])

        if exit_r == 'tp':
            # Trade previously hit 0.20% TP
            if mfe >= tp_wide:
                # MFE exceeded wider TP — would have exited at wider level
                pnl_best  = (tp_wide - SPREAD_COST) * mult * NOTIONAL
                pnl_worst = pnl_best  # same — definitely hit
            else:
                # MFE didn't reach wider TP — trade held past 0.20% then reversed
                # Best case: somehow exited at 0.20% anyway (e.g., z-exit near TP)
                pnl_best  = pnl_base
                # Worst case: held until SL
                pnl_worst = -(STOP_PCT + SPREAD_COST) * mult * NOTIONAL
        else:
            # SL / z-exit / hold expiry — not affected by TP widening
            pnl_best  = pnl_base
            pnl_worst = pnl_base

        results.append({
            'pnl_best' : pnl_best,
            'pnl_worst': pnl_worst,
            'pnl_base' : pnl_base,
        })

    return pd.DataFrame(results)


def score(pnl_series, label, n_years=18):
    """Compute key metrics from a P&L series."""
    pnl = pd.to_numeric(pnl_series, errors='coerce').fillna(0).values
    n   = len(pnl)
    if n == 0:
        return {"label": label, "n": 0, "wr": 0, "sharpe": 0,
                "pf": 0, "dd": 0, "total": 0, "per_yr": 0}
    wr  = (pnl > 0).mean() * 100
    tot = pnl.sum()
    eq  = 100_000 + np.cumsum(pnl)
    pk  = np.maximum.accumulate(eq)
    dd  = ((eq - pk) / pk * 100).min()
    per_yr = n / n_years
    exc    = pnl / 100_000 - 0.04 / per_yr
    sharpe = exc.mean() / exc.std() * np.sqrt(per_yr) if exc.std() > 0 else 0
    gp = pnl[pnl > 0].sum()
    gl = abs(pnl[pnl < 0].sum())
    pf = round(gp / gl, 2) if gl > 0 else 999
    return {"label": label, "n": n, "wr": round(wr,1), "sharpe": round(sharpe,2),
            "pf": pf, "dd": round(dd,2), "total": int(round(tot)),
            "per_yr": round(per_yr,1)}


def print_table(rows, title):
    print(f"\n{'='*72}")
    print(title)
    print(f"{'='*72}")
    print(f"  {'Approach':<38}{'N':>5}{'WR%':>6}{'Sharpe':>8}"
          f"{'PF':>6}{'MaxDD%':>8}{'Total$':>11}")
    print(f"  {'─'*70}")
    base_sh = next((r['sharpe'] for r in rows if 'baseline' in r['label'].lower()), 0)
    for r in rows:
        if r['n'] == 0: continue
        gain = r['sharpe'] - base_sh
        mark = f" ◄ +{gain:.2f}" if gain >= 0.1 and 'baseline' not in r['label'].lower() else ""
        print(f"  {r['label']:<38}{r['n']:>5}{r['wr']:>6.1f}"
              f"{r['sharpe']:>8.2f}{r['pf']:>6.2f}{r['dd']:>8.2f}"
              f"{r['total']:>11,.0f}{mark}")


def main():
    print("=" * 72)
    print("EVZ MFE/MAE BACKTEST v1")
    print("Testing EVZ-based dynamic TP on validated 4,019 EURUSD trades")
    print("=" * 72)

    # ── Load EVZ ──────────────────────────────────────────────────────────────
    print("\nStep 1: Loading EVZ data...")
    evz_path = next((p for p in [
        RATES_PATH / "EVZCLS.csv",
        RATES_PATH / "evz.csv",
        RATES_PATH / "EVZ.csv",
    ] if p.exists()), None)

    if evz_path is None:
        print("  ERROR: EVZ CSV not found in data/raw/rates/")
        print("  Expected: EVZCLS.csv (from FRED) or evz.csv")
        sys.exit(1)

    evz_raw = pd.read_csv(evz_path)
    # Handle both FRED format (observation_date,EVZCLS)
    # and Yahoo format (Date,Close,...)
    date_col  = next(c for c in evz_raw.columns
                     if 'date' in c.lower() or c == 'Date')
    val_col   = next(c for c in evz_raw.columns
                     if c not in [date_col] and evz_raw[c].dtype in
                     [float, 'float64', 'object'])

    evz_raw[date_col] = pd.to_datetime(evz_raw[date_col])
    evz_raw[val_col]  = pd.to_numeric(evz_raw[val_col], errors='coerce')
    evz_raw = evz_raw.dropna(subset=[val_col])
    evz_s   = evz_raw.set_index(date_col)[val_col].rename("evz").sort_index()

    # Forward-fill to cover weekends/bank holidays
    full_idx = pd.date_range(evz_s.index.min(), evz_s.index.max(), freq='D')
    evz_ff   = evz_s.reindex(full_idx).ffill()

    print(f"  Loaded: {len(evz_s):,} trading days")
    print(f"  Range : {evz_s.index[0].date()} → {evz_s.index[-1].date()}")
    print(f"  EVZ   : min={evz_s.min():.1f}  mean={evz_s.mean():.1f}  "
          f"max={evz_s.max():.1f}")

    # ── Load validated trades ─────────────────────────────────────────────────
    print("\nStep 2: Loading validated trades...")
    trades_path = next((p for p in [
        PROC_PATH / "trades_real_costs.csv",
        PROC_PATH / "trades" / "trades_real_costs.csv",
    ] if p.exists()), None)

    if trades_path is None:
        print("  ERROR: trades_real_costs.csv not found"); sys.exit(1)

    df = pd.read_csv(trades_path)
    df["entry_time"] = pd.to_datetime(df["entry_time"])
    df["entry_date"] = df["entry_time"].dt.normalize()
    print(f"  Loaded: {len(df):,} trades")
    print(f"  Columns: {list(df.columns)}")

    # ── Join EVZ ──────────────────────────────────────────────────────────────
    print("\nStep 3: Joining EVZ to trades...")
    df["evz"] = df["entry_date"].map(evz_ff)
    n_before  = len(df)
    df        = df.dropna(subset=["evz"])
    n_after   = len(df)
    print(f"  Trades with EVZ  : {n_after:,} "
          f"({n_before - n_after} dropped — before EVZ history starts)")
    print(f"  EVZ at entry     : min={df['evz'].min():.1f}  "
          f"mean={df['evz'].mean():.1f}  max={df['evz'].max():.1f}")

    # Backtest covers n_after trades — estimate years for Sharpe
    years = (df["entry_time"].max() - df["entry_time"].min()).days / 365.25

    # ── EVZ regime assignment ─────────────────────────────────────────────────
    df["evz_regime"] = df["evz"].apply(evz_regime)
    df["dyn_tp"]     = df["evz"].apply(dynamic_tp)

    # ── MFE / MAE Analysis ────────────────────────────────────────────────────
    print(f"\n{'='*72}")
    print("MFE / MAE ANALYSIS BY EVZ REGIME")
    print(f"{'='*72}")
    print(f"\n  {'Regime':<22}{'N':>5}{'EVZ':>6}  "
          f"{'MFE med%':>9}{'MFE p75%':>9}{'MFE p90%':>9}  "
          f"{'MAE med%':>9}{'Ratio':>7}{'WR%':>7}")
    print(f"  {'─'*80}")

    for regime in sorted(df["evz_regime"].unique()):
        sub = df[df["evz_regime"] == regime]
        mfe_med = sub["mfe"].median() * 100
        mfe_p75 = sub["mfe"].quantile(0.75) * 100
        mfe_p90 = sub["mfe"].quantile(0.90) * 100
        mae_med = sub["mae"].abs().median() * 100   # abs — stored as negative
        ratio   = mfe_med / mae_med if mae_med > 0 else 0
        wr      = (sub["dollar_pnl_real"] > 0).mean() * 100
        evz_med = sub["evz"].median()
        print(f"  {regime:<22}{len(sub):>5}{evz_med:>6.1f}  "
              f"{mfe_med:>9.3f}{mfe_p75:>9.3f}{mfe_p90:>9.3f}  "
              f"{mae_med:>9.3f}{ratio:>7.2f}{wr:>7.1f}")

    # ── MAE Analysis — does adverse excursion also scale with EVZ? ──────────
    print(f"\n{'─'*72}")
    print("MAE ANALYSIS — Does adverse excursion also scale with EVZ?")
    print("(If MAE scales equally to MFE, wider TP is offset by more SL hits)")
    print(f"{'─'*72}")
    print(f"\n  {'Regime':<22}{'MAE med%':>10}{'MAE p75%':>10}{'MAE p90%':>10}"
          f"  {'MFE/MAE':>9}{'SL reach%':>11}")
    print(f"  {'─'*72}")
    for regime in sorted(df["evz_regime"].unique()):
        sub     = df[df["evz_regime"] == regime]
        mae_med = sub["mae"].abs().median() * 100
        mae_p75 = sub["mae"].abs().quantile(0.75) * 100
        mae_p90 = sub["mae"].abs().quantile(0.90) * 100
        mfe_med = sub["mfe"].median() * 100
        ratio   = mfe_med / mae_med if mae_med > 0 else 0
        # % of trades where MAE reached the SL level (0.25%)
        sl_reach = (sub["mae"].abs() >= 0.0025).mean() * 100
        print(f"  {regime:<22}{mae_med:>10.3f}{mae_p75:>10.3f}{mae_p90:>10.3f}"
              f"  {ratio:>9.2f}{sl_reach:>10.1f}%")

    print(f"\n  KEY: If MFE/MAE ratio RISES with EVZ → wider TP justified")
    print(f"       If MFE/MAE ratio stays FLAT  → EVZ doesn't help")
    print(f"       If SL reach% rises sharply   → wider TP offset by more losses")

    # ── TP reach analysis ─────────────────────────────────────────────────────
    print(f"\n{'─'*72}")
    print("WHAT % OF TRADES REACH EACH TP LEVEL (by EVZ regime)")
    print(f"{'─'*72}")
    tp_levels = [0.0020, 0.0023, 0.0027, 0.0030, 0.0035, 0.0040]
    header = f"  {'Regime':<22}" + "".join(f"  {tp*100:.2f}%" for tp in tp_levels)
    print(header)
    print(f"  {'─'*70}")
    for regime in sorted(df["evz_regime"].unique()):
        sub  = df[df["evz_regime"] == regime]
        line = f"  {regime:<22}"
        for tp in tp_levels:
            pct = (sub["mfe"] >= tp).mean() * 100
            line += f"  {pct:>5.1f}%"
        print(line)

    # Also show for TP exits only (trades that actually hit TP)
    tp_exits = df[df["exit_reason"] == "tp"]
    print(f"\n  TP EXITS ONLY ({len(tp_exits):,} trades):")
    print(f"  {'Regime':<22}" + "".join(f"  {tp*100:.2f}%" for tp in tp_levels))
    print(f"  {'─'*70}")
    for regime in sorted(df["evz_regime"].unique()):
        sub  = tp_exits[tp_exits["evz_regime"] == regime]
        if len(sub) == 0: continue
        line = f"  {regime:<22}"
        for tp in tp_levels:
            pct = (sub["mfe"] >= tp).mean() * 100
            line += f"  {pct:>5.1f}%"
        print(line)

    # ── Baseline score ────────────────────────────────────────────────────────
    baseline = score(df["dollar_pnl_real"], "Baseline (validated)", years)

    # ── Dynamic TP simulation ─────────────────────────────────────────────────
    print(f"\n{'='*72}")
    print("DYNAMIC TP SIMULATION")
    print(f"{'='*72}")
    print(f"\n  Strategy: widen TP based on EVZ at trade entry")
    print(f"  Calm (<8)    → TP 0.20% (unchanged)")
    print(f"  Normal(8-11) → TP 0.23%")
    print(f"  Elevated(11-15) → TP 0.27%")
    print(f"  Stress(15+)  → TP 0.35%")
    print(f"\n  Simulation methodology:")
    print(f"    Best case : TP exits that miss wider TP still exit at 0.20%")
    print(f"    Worst case: TP exits that miss wider TP eventually hit SL")

    sim = simulate_dynamic_tp(df, "dyn_tp")
    df["pnl_best"]  = sim["pnl_best"]
    df["pnl_worst"] = sim["pnl_worst"]

    r_best  = score(df["pnl_best"],  "Dynamic TP — best case",  years)
    r_worst = score(df["pnl_worst"], "Dynamic TP — worst case", years)

    # How many TP exits would be affected
    tp_ex        = df[df["exit_reason"] == "tp"]
    would_widen  = (tp_ex["dyn_tp"] > TP_BASE).sum()
    would_reach  = ((tp_ex["dyn_tp"] > TP_BASE) & (tp_ex["mfe"] >= tp_ex["dyn_tp"])).sum()
    would_miss   = would_widen - would_reach

    print(f"\n  TP exit analysis:")
    print(f"    Total TP exits         : {len(tp_ex):,}")
    print(f"    TP exits where EVZ>8   : {would_widen:,} ({would_widen/len(tp_ex)*100:.1f}%)")
    print(f"    → MFE reaches wider TP : {would_reach:,} ({would_reach/len(tp_ex)*100:.1f}%) — captures extra profit")
    print(f"    → MFE misses wider TP  : {would_miss:,}  ({would_miss/len(tp_ex)*100:.1f}%) — uncertainty zone")

    all_results = [baseline, r_best, r_worst]

    # ── TP sweep — find optimal ───────────────────────────────────────────────
    print(f"\n{'─'*72}")
    print("TP LEVEL SWEEP — Finding optimal TP per EVZ regime")
    print(f"{'─'*72}")

    sweep_results = [baseline]
    for tp_calm, tp_norm, tp_elev, tp_stress, label in [
        (0.0020, 0.0020, 0.0020, 0.0020, "All 0.20% (baseline)"),
        (0.0020, 0.0023, 0.0027, 0.0035, "Dynamic (0.20/0.23/0.27/0.35)"),
        (0.0020, 0.0025, 0.0030, 0.0040, "Aggressive(0.20/0.25/0.30/0.40)"),
        (0.0020, 0.0020, 0.0025, 0.0030, "Conservative(0.20/0.20/0.25/0.30)"),
        (0.0020, 0.0022, 0.0025, 0.0030, "Gentle (0.20/0.22/0.25/0.30)"),
    ]:
        def tp_fn(evz, c=tp_calm, n=tp_norm, e=tp_elev, s=tp_stress):
            if evz >= 15: return s
            if evz >= 11: return e
            if evz >= 8:  return n
            return c
        df["_tp_test"] = df["evz"].apply(tp_fn)
        sim2 = simulate_dynamic_tp(df, "_tp_test")
        r_b = score(pd.Series(sim2["pnl_best"].values),
                    f"Best  | {label}", years)
        r_w = score(pd.Series(sim2["pnl_worst"].values),
                    f"Worst | {label}", years)
        sweep_results.extend([r_b, r_w])

    print_table(sweep_results, "TP SWEEP — BEST AND WORST CASE")

    # ── By-regime deep dive ───────────────────────────────────────────────────
    print(f"\n{'─'*72}")
    print("EVZ REGIME DETAIL — How often does MFE exceed each level?")
    print(f"{'─'*72}")
    for regime in sorted(df["evz_regime"].unique()):
        sub = df[df["evz_regime"] == regime]
        tp_sub = sub[sub["exit_reason"] == "tp"]
        n_total = len(sub)
        n_tp    = len(tp_sub)
        n_sl    = (sub["exit_reason"] == "sl").sum()
        wr      = (sub["dollar_pnl_real"] > 0).mean() * 100
        avg_mfe = sub["mfe"].mean() * 100
        avg_mae = sub["mae"].abs().mean() * 100
        mfe_mae = avg_mfe / avg_mae if avg_mae > 0 else 0

        print(f"\n  {regime}  (N={n_total}, EVZ avg={sub['evz'].mean():.1f})")
        print(f"    WR={wr:.1f}%  TP exits={n_tp} ({n_tp/n_total*100:.0f}%)  "
              f"SL exits={n_sl} ({n_sl/n_total*100:.0f}%)")
        print(f"    Avg MFE={avg_mfe:.3f}%  Avg MAE={avg_mae:.3f}%  "
              f"MFE/MAE={mfe_mae:.2f}")
        if n_tp > 0:
            print(f"    Of {n_tp} TP exits, MFE reached:")
            for tp in [0.0023, 0.0027, 0.0030, 0.0035, 0.0040]:
                pct = (tp_sub["mfe"] >= tp).mean() * 100
                bar = "█" * int(pct / 5)
                print(f"      {tp*100:.2f}%: {pct:>5.1f}%  {bar}")

    # ── Final conclusion ──────────────────────────────────────────────────────
    print(f"\n{'='*72}")
    print("CONCLUSION")
    print(f"{'='*72}")

    base_sh   = baseline["sharpe"]
    best_sh   = r_best["sharpe"]
    worst_sh  = r_worst["sharpe"]
    best_gain = best_sh - base_sh

    print(f"\n  Baseline          : {baseline['n']:,} trades  "
          f"Sharpe {base_sh:.2f}  Total ${baseline['total']:,.0f}")
    print(f"  Dynamic TP (best) : Sharpe {best_sh:.2f}  "
          f"Total ${r_best['total']:,.0f}  "
          f"({(r_best['total']-baseline['total'])/baseline['total']*100:+.1f}%)")
    print(f"  Dynamic TP (worst): Sharpe {worst_sh:.2f}  "
          f"Total ${r_worst['total']:,.0f}  "
          f"({(r_worst['total']-baseline['total'])/baseline['total']*100:+.1f}%)")

    if best_gain >= 0.3 and worst_sh > base_sh - 0.2:
        verdict = "PROMISING — best case strong, worst case acceptable"
        action  = "Consider implementing with conservative TP levels"
    elif best_gain >= 0.1:
        verdict = "MARGINAL — some upside but below implementation threshold"
        action  = "Check the 0.30% TP level specifically in high EVZ regime"
    else:
        verdict = "NO IMPROVEMENT — fixed 0.20% TP already optimal"
        action  = "MFE does not scale sufficiently with EVZ to justify change"

    print(f"\n  VERDICT: {verdict}")
    print(f"  ACTION : {action}")

    print(f"""
  KEY THINGS TO LOOK FOR IN THE RESULTS ABOVE:
  ─────────────────────────────────────────────────────────────────────
  1. MFE/MAE ratio by regime — if this improves with EVZ, wider TP is
     justified. If it stays flat, EVZ is not predictive of move size.

  2. TP reach % table — the % of TP exits that reach 0.27%, 0.30%
     in Stress regime vs Calm regime is the critical comparison.
     If Stress regime reaches 0.27% 70%+ of the time, that level is safe.

  3. Worst case Sharpe — must not fall below {base_sh - 0.3:.2f} (baseline - 0.3).
     If worst case is destructive, the risk is not worth the upside.
  ─────────────────────────────────────────────────────────────────────
    """)

    out = PROC_PATH / "evz_mfe_results.csv"
    df.to_csv(out, index=False)
    print(f"Trade-level EVZ analysis saved: {out.name}")


if __name__ == "__main__":
    main()