"""
usdjpy_mfe_mae_v1.py
=====================
MFE/MAE analysis on the validated USDJPY best config.

Best config (from v4 sweep):
  z30 t2.0 tp0.60% sl0.65% hold24h fib0.786 london both scaled

MFE = Maximum Favorable Excursion
  How far each trade moved IN OUR FAVOUR before exit.
  If MFE >> TP on winners → we are leaving money on the table.
  If MFE on losers shows cluster near TP → many losers briefly
  went positive before reversing (partial TP could help).

MAE = Maximum Adverse Excursion  
  How far each trade moved AGAINST US before recovering.
  If MAE on winners << SL → SL can be tightened without
  cutting winners (improves Sharpe by reducing loss size).
  If MAE on winners frequently exceeds 0.40% → trades need room.

Run from project root:
  python src/research/usdjpy_mfe_mae_v1.py
"""

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from pathlib import Path
import sys
import warnings
warnings.filterwarnings('ignore')

BASE_PATH  = Path(__file__).resolve().parents[2]
PROC_PATH  = BASE_PATH / "data" / "processed"
RATES_PATH = BASE_PATH / "data" / "raw" / "rates"
OUT_PATH   = BASE_PATH / "data" / "output" / "usdjpy"
OUT_PATH.mkdir(parents=True, exist_ok=True)

if str(BASE_PATH / "src") not in sys.path:
    sys.path.insert(0, str(BASE_PATH / "src"))

# ── Best config ───────────────────────────────────────────────────────────────
Z_WINDOW   = 30
THRESHOLD  = 2.0
TP         = 0.0060
SL         = 0.0065
HOLD       = 24
FIB        = 0.786
NOTIONAL   = 300_000
SPREAD_COST= 0.0001
WF_SPLIT   = pd.Timestamp('2020-01-01')


def load_rates():
    us2y = pd.read_csv(RATES_PATH / "us2y.csv")
    dc = [c for c in us2y.columns if 'date' in c.lower()][0]
    vc = [c for c in us2y.columns if c != dc][0]
    us2y[dc] = pd.to_datetime(us2y[dc])
    us2y = us2y.rename(columns={dc:'date', vc:'us2y'})[['date','us2y']].dropna()
    jp2y = pd.read_csv(RATES_PATH / "jp2y.csv")
    jp2y['date'] = pd.to_datetime(jp2y['date'])
    jp2y = jp2y[['date','jp2y']].dropna()
    rates = pd.merge(us2y, jp2y, on='date', how='outer').sort_values('date')
    rates = rates.set_index('date')
    full_idx = pd.date_range(rates.index.min(), rates.index.max(), freq='D')
    rates = rates.reindex(full_idx).ffill().dropna()
    rates['spread'] = rates['us2y'] - rates['jp2y']
    s = rates['spread']
    z = (s - s.rolling(Z_WINDOW).mean()) / s.rolling(Z_WINDOW).std()
    full_h = pd.date_range(z.index.min(), z.index.max(), freq='h')
    return z.reindex(full_h).ffill()


def load_prices():
    fpath = next(BASE_PATH.rglob("USDJPY_15M*.csv"), None)
    df = pd.read_csv(fpath)
    df.columns = df.columns.str.strip().str.lower()
    dc = next(c for c in df.columns if 'date' in c or 'time' in c)
    df[dc] = pd.to_datetime(df[dc])
    return df.rename(columns={dc:'datetime'}).set_index('datetime').sort_index()


def run_backtest_with_excursions(z_hourly, m15):
    """Same backtest but records MFE and MAE for every trade."""
    trades    = []
    last_exit = None
    hours     = pd.date_range('2003-01-01', '2026-04-01', freq='h')

    for dt in hours:
        if dt not in z_hourly.index: continue
        z = float(z_hourly.at[dt])
        if abs(z) < THRESHOLD: continue
        h_eet = (dt.hour + 2) % 24
        if not (7 <= h_eet <= 17): continue
        if last_exit is not None and dt <= last_exit: continue

        direction = 1 if z >= THRESHOLD else -1
        z_abs = abs(z)
        mult  = 1.0 if z_abs < 2.5 else (1.5 if z_abs < 3.5 else 2.0)

        bar_end   = dt + pd.Timedelta(hours=1) - pd.Timedelta(minutes=1)
        try:
            bar_slice = m15.loc[dt:bar_end]
        except Exception:
            continue
        if bar_slice.empty: continue

        bh = float(bar_slice['high'].max())
        bl = float(bar_slice['low'].min())
        bc = float(bar_slice['close'].iloc[-1])

        if direction == 1:
            pull = bc - bl
            if pull <= 0.05: continue
            target = bc - FIB * pull
        else:
            pull = bh - bc
            if pull <= 0.05: continue
            target = bc + FIB * pull

        entry_price = entry_time = None
        window = m15.loc[dt + pd.Timedelta(minutes=1):
                         dt + pd.Timedelta(hours=6)]
        for edt, ebar in window.iterrows():
            if direction == 1 and float(ebar['low']) <= target:
                entry_price = target; entry_time = edt; break
            elif direction == -1 and float(ebar['high']) >= target:
                entry_price = target; entry_time = edt; break
        if entry_price is None: continue

        tp_px = entry_price * (1 + TP) if direction == 1 else entry_price * (1 - TP)
        sl_px = entry_price * (1 - SL) if direction == 1 else entry_price * (1 + SL)

        hold_bars  = m15.loc[entry_time + pd.Timedelta(minutes=1):
                             entry_time + pd.Timedelta(hours=HOLD)]

        exit_price = None
        exit_reason= "hold_expiry"
        mfe_pct    = 0.0   # max favorable excursion %
        mae_pct    = 0.0   # max adverse excursion %

        for hdt, hbar in hold_bars.iterrows():
            h_high = float(hbar['high'])
            h_low  = float(hbar['low'])

            # Track excursions
            if direction == 1:
                favorable = (h_high - entry_price) / entry_price * 100
                adverse   = (entry_price - h_low)  / entry_price * 100
            else:
                favorable = (entry_price - h_low)  / entry_price * 100
                adverse   = (h_high - entry_price) / entry_price * 100

            mfe_pct = max(mfe_pct, favorable)
            mae_pct = max(mae_pct, adverse)

            if direction == 1:
                if h_high >= tp_px: exit_price=tp_px; exit_reason="tp"; break
                if h_low  <= sl_px: exit_price=sl_px; exit_reason="stop"; break
            else:
                if h_low  <= tp_px: exit_price=tp_px; exit_reason="tp"; break
                if h_high >= sl_px: exit_price=sl_px; exit_reason="stop"; break

        if exit_price is None:
            exit_price = float(hold_bars['close'].iloc[-1]) if not hold_bars.empty else entry_price

        pnl = round(((exit_price - entry_price) / entry_price * direction
                     - SPREAD_COST) * NOTIONAL * mult, 2)
        won = pnl > 0

        trades.append({
            'entry_time':  entry_time,
            'direction':   direction,
            'zscore':      round(z, 4),
            'zscore_abs':  round(z_abs, 4),
            'entry_price': entry_price,
            'exit_price':  exit_price,
            'exit_reason': exit_reason,
            'mfe_pct':     round(mfe_pct, 4),
            'mae_pct':     round(mae_pct, 4),
            'pnl':         pnl,
            'won':         won,
            'mult':        mult,
        })
        last_exit = entry_time + pd.Timedelta(hours=HOLD)

    return pd.DataFrame(trades)


def analyse_and_plot(df):
    winners = df[df['won']]
    losers  = df[~df['won']]

    print(f"\n{'='*72}")
    print("MFE / MAE ANALYSIS")
    print(f"{'='*72}")
    print(f"\n  Total trades: {len(df):,}  "
          f"({len(winners)} winners / {len(losers)} losers)")
    print(f"  TP: {TP*100:.2f}%   SL: {SL*100:.2f}%   Hold: {HOLD}h")

    print(f"\n  {'Metric':<35}{'Winners':>10}{'Losers':>10}")
    print(f"  {'─'*55}")
    for label, col in [('MFE mean %','mfe_pct'),('MAE mean %','mae_pct')]:
        wv = winners[col].mean()
        lv = losers[col].mean()
        print(f"  {label:<35}{wv:>10.3f}{lv:>10.3f}")
    print(f"  {'MFE median %':<35}"
          f"{winners['mfe_pct'].median():>10.3f}"
          f"{losers['mfe_pct'].median():>10.3f}")
    print(f"  {'MAE median %':<35}"
          f"{winners['mae_pct'].median():>10.3f}"
          f"{losers['mae_pct'].median():>10.3f}")
    print(f"  {'MFE/MAE ratio':<35}"
          f"{winners['mfe_pct'].mean()/max(winners['mae_pct'].mean(),0.001):>10.2f}"
          f"{losers['mfe_pct'].mean()/max(losers['mae_pct'].mean(),0.001):>10.2f}")

    # Key diagnostic questions
    print(f"\n  KEY DIAGNOSTICS:")

    # 1. Are winners being cut short?
    mfe_above_tp = (winners['mfe_pct'] > TP*100 * 1.5).mean() * 100
    print(f"\n  1. Winners with MFE > 1.5×TP ({TP*100*1.5:.2f}%): "
          f"{mfe_above_tp:.1f}%")
    if mfe_above_tp > 30:
        print(f"     → TP may be too tight — winners often go further")
    else:
        print(f"     → TP is well calibrated")

    # 2. Can SL be tightened?
    mae_below_half_sl = (winners['mae_pct'] < SL*100 * 0.5).mean() * 100
    print(f"\n  2. Winners with MAE < 0.5×SL ({SL*100*0.5:.3f}%): "
          f"{mae_below_half_sl:.1f}%")
    if mae_below_half_sl > 60:
        print(f"     → SL can likely be tightened without cutting winners")
        suggested_sl = np.percentile(winners['mae_pct'], 90)
        print(f"     → Suggested tighter SL: {suggested_sl:.3f}% "
              f"(90th pct of winner MAE)")
    else:
        print(f"     → Winners need the full SL room — do not tighten")

    # 3. Losers that briefly went positive
    losers_positive_mfe = (losers['mfe_pct'] > TP*100 * 0.5).mean() * 100
    print(f"\n  3. Losers with MFE > 0.5×TP ({TP*100*0.5:.3f}%): "
          f"{losers_positive_mfe:.1f}%")
    if losers_positive_mfe > 25:
        print(f"     → Partial TP at {TP*100*0.5:.2f}% could save some losers")
    else:
        print(f"     → Losers were wrong from the start — partial TP won't help")

    # 4. Optimal TP from MFE distribution
    tp_candidates = [0.004, 0.005, 0.006, 0.007, 0.008, 0.010]
    print(f"\n  4. WR by TP level (from MFE simulation):")
    for tp_c in tp_candidates:
        hits = (winners['mfe_pct'] >= tp_c * 100).mean() * 100
        print(f"     TP {tp_c*100:.2f}%: {hits:.1f}% of winners would have hit it")

    # 5. Optimal SL from MAE distribution
    sl_candidates = [0.003, 0.004, 0.005, 0.006, 0.007, 0.008]
    print(f"\n  5. Winner survival by SL level (from MAE simulation):")
    for sl_c in sl_candidates:
        survives = (winners['mae_pct'] < sl_c * 100).mean() * 100
        print(f"     SL {sl_c*100:.2f}%: {survives:.1f}% of winners survive this SL")

    # ── Charts ────────────────────────────────────────────────────────────────
    fig, axes = plt.subplots(2, 2, figsize=(14, 10))
    fig.suptitle(f'USDJPY MFE/MAE Analysis — z{Z_WINDOW} t{THRESHOLD} '
                 f'TP{TP*100:.2f}% SL{SL*100:.2f}% H{HOLD}h', fontsize=12)

    # Panel 1: MFE distribution by outcome
    ax = axes[0, 0]
    bins = np.linspace(0, 2.0, 50)
    ax.hist(winners['mfe_pct'], bins=bins, alpha=0.6,
            color='#00cc66', label=f'Winners (n={len(winners)})')
    ax.hist(losers['mfe_pct'],  bins=bins, alpha=0.6,
            color='#ff4444', label=f'Losers (n={len(losers)})')
    ax.axvline(TP*100, color='gold', linewidth=2,
               linestyle='--', label=f'TP ({TP*100:.2f}%)')
    ax.set_title('MFE Distribution')
    ax.set_xlabel('MFE %')
    ax.legend(fontsize=8)
    ax.grid(alpha=0.3)

    # Panel 2: MAE distribution by outcome
    ax = axes[0, 1]
    bins2 = np.linspace(0, 2.0, 50)
    ax.hist(winners['mae_pct'], bins=bins2, alpha=0.6,
            color='#00cc66', label=f'Winners (n={len(winners)})')
    ax.hist(losers['mae_pct'],  bins=bins2, alpha=0.6,
            color='#ff4444', label=f'Losers (n={len(losers)})')
    ax.axvline(SL*100, color='gold', linewidth=2,
               linestyle='--', label=f'SL ({SL*100:.2f}%)')
    ax.set_title('MAE Distribution')
    ax.set_xlabel('MAE %')
    ax.legend(fontsize=8)
    ax.grid(alpha=0.3)

    # Panel 3: MFE vs MAE scatter
    ax = axes[1, 0]
    ax.scatter(winners['mae_pct'], winners['mfe_pct'],
               alpha=0.3, color='#00cc66', s=8, label='Winners')
    ax.scatter(losers['mae_pct'],  losers['mfe_pct'],
               alpha=0.3, color='#ff4444', s=8, label='Losers')
    ax.axhline(TP*100, color='gold', linewidth=1,
               linestyle='--', alpha=0.7, label=f'TP')
    ax.axvline(SL*100, color='orange', linewidth=1,
               linestyle='--', alpha=0.7, label=f'SL')
    ax.set_xlabel('MAE %')
    ax.set_ylabel('MFE %')
    ax.set_title('MFE vs MAE Scatter')
    ax.legend(fontsize=8)
    ax.grid(alpha=0.3)
    ax.set_xlim(0, min(SL*100*3, 3.0))
    ax.set_ylim(0, min(TP*100*4, 4.0))

    # Panel 4: Cumulative MFE — what TP would maximise wins
    ax = axes[1, 1]
    tp_range = np.linspace(0.001, 0.020, 100)
    hit_rates = [(winners['mfe_pct'] >= tp*100).mean()*100 for tp in tp_range]
    ax.plot(tp_range*100, hit_rates, color='#00cc66', linewidth=2)
    ax.axvline(TP*100, color='gold', linewidth=2,
               linestyle='--', label=f'Current TP ({TP*100:.2f}%)')
    # Mark optimal TP (where curve starts to flatten)
    gradient   = np.gradient(hit_rates)
    flat_idx   = next((i for i,g in enumerate(gradient) if g > -2), len(gradient)-1)
    optimal_tp = tp_range[flat_idx]*100
    ax.axvline(optimal_tp, color='#4488ff', linewidth=1.5,
               linestyle=':', label=f'Curve flattens ({optimal_tp:.2f}%)')
    ax.set_xlabel('TP %')
    ax.set_ylabel('% of winners hitting TP')
    ax.set_title('Cumulative Winner Hit Rate vs TP Level')
    ax.legend(fontsize=8)
    ax.grid(alpha=0.3)

    fig.tight_layout()
    out_file = OUT_PATH / 'usdjpy_mfe_mae.png'
    fig.savefig(out_file, dpi=150, bbox_inches='tight')
    plt.close(fig)
    print(f"\n  Chart saved: {out_file.name}")

    # ── Recommendation ────────────────────────────────────────────────────────
    print(f"\n{'='*72}")
    print("RECOMMENDATION")
    print(f"{'='*72}")

    suggested_sl = round(np.percentile(winners['mae_pct'], 90) / 100, 4)
    suggested_tp = round(winners['mfe_pct'].median() / 100, 4)

    print(f"\n  Current:   TP={TP*100:.2f}%  SL={SL*100:.2f}%")
    print(f"  MFE-based: TP={suggested_tp*100:.2f}%  "
          f"(median MFE of winners)")
    print(f"  MAE-based: SL={suggested_sl*100:.2f}%  "
          f"(90th pct MAE of winners)")

    if abs(suggested_tp - TP) < 0.001 and abs(suggested_sl - SL) < 0.001:
        print(f"\n  VERDICT: TP and SL are already well calibrated.")
        print(f"  No meaningful improvement available from exit adjustments.")
        print(f"  Lock parameters as-is.")
    else:
        print(f"\n  VERDICT: Run a targeted TP/SL sweep around suggested values")
        print(f"  before locking parameters.")


def main():
    print("="*72)
    print("USDJPY MFE/MAE ANALYSIS v1")
    print(f"Config: z{Z_WINDOW} t{THRESHOLD} TP{TP*100:.2f}% SL{SL*100:.2f}% H{HOLD}h")
    print("="*72)

    print("\nLoading data...")
    z_hourly = load_rates()
    m15      = load_prices()

    print("Running backtest with excursion tracking...")
    df = run_backtest_with_excursions(z_hourly, m15)
    print(f"  {len(df):,} trades with MFE/MAE recorded")

    analyse_and_plot(df)

    out = PROC_PATH / "usdjpy_mfe_mae_trades.csv"
    df.to_csv(out, index=False)
    print(f"\n  Trades saved: {out.name}")


if __name__ == "__main__":
    main()