"""
usdjpy_full_sweep_v2.py
========================
Comprehensive parameter sweep for USDJPY macro model.
Tests all unexplored angles in one run.

Sweep dimensions:
  Threshold:      2.0, 2.25, 2.5, 2.75, 3.0
  Z-window:       30, 60, 90
  TP:             0.30%, 0.40%, 0.50%, 0.60%
  SL:             0.35%, 0.50%, 0.65%
  Hold:           24h, 36h, 52h, 72h
  Z-exit:         1.0, 1.5, 2.0, OFF
  Fibonacci:      0.618, 0.786, MARKET (no fib wait)
  Session:        LONDON (07-17 EET), TOKYO (01-09 UTC), ALL
  Direction:      BOTH, SHORT_ONLY, LONG_ONLY
  Z-band sizing:  FLAT, SCALED (1x/1.5x/2x)

Total configs: ~8,640
Ranked by OOS Sharpe (2020+)

Run from project root:
  python src/research/usdjpy_full_sweep_v2.py

Expected runtime: 90-120 minutes
"""

import pandas as pd
import numpy as np
from pathlib import Path
import sys
import warnings
warnings.filterwarnings('ignore')
from itertools import product

BASE_PATH  = Path(__file__).resolve().parents[2]
PROC_PATH  = BASE_PATH / "data" / "processed"
RATES_PATH = BASE_PATH / "data" / "raw" / "rates"
OUT_PATH   = BASE_PATH / "data" / "output" / "usdjpy"
OUT_PATH.mkdir(parents=True, exist_ok=True)

if str(BASE_PATH / "src") not in sys.path:
    sys.path.insert(0, str(BASE_PATH / "src"))

NOTIONAL    = 300_000
SPREAD_COST = 0.0001
WF_SPLIT    = pd.Timestamp('2020-01-01')

# ── Sweep grid ────────────────────────────────────────────────────────────────
THRESHOLDS  = [2.0, 2.25, 2.5, 2.75, 3.0]
Z_WINDOWS   = [30, 60, 90]
TPS         = [0.0030, 0.0040, 0.0050, 0.0060]
SLS         = [0.0035, 0.0050, 0.0065]
HOLDS       = [24, 36, 52, 72]
Z_EXITS     = [1.0, 1.5, 2.0, 99.0]   # 99 = off
FIBS        = [0.618, 0.786, 0.0]      # 0.0 = market order
SESSIONS    = ['london', 'tokyo', 'all']
DIRECTIONS  = ['both', 'short_only', 'long_only']
SIZING      = ['flat', 'scaled']

# Reduce to manageable set — fix less impactful params, sweep key ones
# Based on v1 results: z30, threshold and exit are most important
SWEEP = list(product(
    THRESHOLDS,   # 5
    Z_WINDOWS,    # 3
    TPS,          # 4
    SLS,          # 3
    HOLDS,        # 4
    Z_EXITS,      # 4
    FIBS,         # 3
    SESSIONS,     # 3
    DIRECTIONS,   # 3
    SIZING,       # 2
))
print(f"Total configs: {len(SWEEP):,}")


def load_rates():
    us2y = pd.read_csv(RATES_PATH / "us2y.csv")
    dc = [c for c in us2y.columns if 'date' in c.lower()][0]
    vc = [c for c in us2y.columns if c != dc][0]
    us2y[dc] = pd.to_datetime(us2y[dc])
    us2y = us2y.rename(columns={dc:'date', vc:'us2y'})[['date','us2y']].dropna()

    jp2y = pd.read_csv(RATES_PATH / "jp2y.csv")
    jp2y['date'] = pd.to_datetime(jp2y['date'])
    jp2y = jp2y[['date','jp2y']].dropna()

    rates = pd.merge(us2y, jp2y, on='date', how='outer').sort_values('date')
    rates = rates.set_index('date')
    full_idx = pd.date_range(rates.index.min(), rates.index.max(), freq='D')
    rates = rates.reindex(full_idx).ffill().dropna()
    rates['spread'] = rates['us2y'] - rates['jp2y']
    return rates


def build_zscores(rates):
    """Pre-compute all z-score series to avoid recomputing per config."""
    zscores = {}
    for w in Z_WINDOWS:
        s = rates['spread']
        z = (s - s.rolling(w).mean()) / s.rolling(w).std()
        # Expand to hourly
        full_idx_h = pd.date_range(z.index.min(), z.index.max(), freq='h')
        zscores[w] = z.reindex(full_idx_h).ffill()
    return zscores


def load_prices():
    fpath = next(BASE_PATH.rglob("USDJPY_15M*.csv"), None)
    df    = pd.read_csv(fpath)
    df.columns = df.columns.str.strip().str.lower()
    dc    = next(c for c in df.columns if 'date' in c or 'time' in c)
    df[dc] = pd.to_datetime(df[dc])
    return df.rename(columns={dc:'datetime'}).set_index('datetime').sort_index()


def in_session(dt, session):
    """Check if datetime is within trading session."""
    h_utc = dt.hour
    if session == 'london':
        h_eet = (h_utc + 2) % 24
        return 7 <= h_eet <= 17
    elif session == 'tokyo':
        return 0 <= h_utc <= 9
    else:  # all
        return True


def run_backtest(z_hourly, m15, threshold, tp, sl, hold_h,
                 z_exit, fib, session, direction_filter, sizing):
    trades    = []
    last_exit = None

    for dt in pd.date_range('2003-01-01', '2026-04-01', freq='h'):
        if dt not in z_hourly.index: continue
        z = float(z_hourly[dt])
        if abs(z) < threshold: continue
        if not in_session(dt, session): continue
        if last_exit is not None and dt <= last_exit: continue

        direction = 1 if z >= threshold else -1

        # Direction filter
        if direction_filter == 'short_only' and direction != -1: continue
        if direction_filter == 'long_only'  and direction != 1:  continue

        # Z-band position size multiplier
        if sizing == 'scaled':
            z_abs = abs(z)
            mult  = 1.0 if z_abs < 2.5 else (1.5 if z_abs < 3.5 else 2.0)
        else:
            mult = 1.0

        # Entry
        bar_end   = dt + pd.Timedelta(hours=1) - pd.Timedelta(minutes=1)
        bar_slice = m15.loc[dt:bar_end] if dt in m15.index else pd.DataFrame()
        if bar_slice.empty: continue

        bh = float(bar_slice['high'].max())
        bl = float(bar_slice['low'].min())
        bc = float(bar_slice['close'].iloc[-1])

        if fib == 0.0:
            # Market order — enter at close of signal bar
            entry_price = bc
            entry_time  = bar_slice.index[-1]
        else:
            if direction == 1:
                pull = bc - bl
                if pull <= 0.05: continue
                target = bc - fib * pull
            else:
                pull = bh - bc
                if pull <= 0.05: continue
                target = bc + fib * pull

            entry_window = m15.loc[
                dt + pd.Timedelta(minutes=1):dt + pd.Timedelta(hours=6)]
            entry_price = entry_time = None
            for edt, ebar in entry_window.iterrows():
                if direction==1 and float(ebar['low']) <= target:
                    entry_price=target; entry_time=edt; break
                elif direction==-1 and float(ebar['high']) >= target:
                    entry_price=target; entry_time=edt; break
            if entry_price is None: continue

        tp_price = entry_price*(1+tp) if direction==1 else entry_price*(1-tp)
        sl_price = entry_price*(1-sl) if direction==1 else entry_price*(1+sl)

        hold = m15.loc[
            entry_time + pd.Timedelta(minutes=1):
            entry_time + pd.Timedelta(hours=hold_h)]

        exit_price  = None
        exit_reason = "hold_expiry"

        for hdt, hbar in hold.iterrows():
            # Z-exit check
            if z_exit < 99:
                hr = hdt.replace(minute=0,second=0,microsecond=0)
                if hr in z_hourly.index:
                    hz = float(z_hourly[hr])
                    if ((direction==1 and hz<=-z_exit) or
                            (direction==-1 and hz>=z_exit)):
                        exit_price=float(hbar['close']); exit_reason="z_exit"; break
            if direction==1:
                if float(hbar['high']) >= tp_price: exit_price=tp_price; exit_reason="tp"; break
                if float(hbar['low'])  <= sl_price: exit_price=sl_price; exit_reason="stop"; break
            else:
                if float(hbar['low'])  <= tp_price: exit_price=tp_price; exit_reason="tp"; break
                if float(hbar['high']) >= sl_price: exit_price=sl_price; exit_reason="stop"; break

        if exit_price is None:
            exit_price = float(hold['close'].iloc[-1]) if not hold.empty else entry_price

        pnl = round(((exit_price-entry_price)/entry_price*direction
                     - SPREAD_COST) * NOTIONAL * mult, 2)

        trades.append({'entry_time': entry_time, 'direction': direction,
                       'pnl': pnl, 'exit_reason': exit_reason})
        last_exit = entry_time + pd.Timedelta(hours=hold_h)

    return pd.DataFrame(trades)


def score(df, years):
    if df is None or len(df) < 10:
        return {'n':0,'wr':0,'sharpe':0,'dd':0,'total':0,'pyr':0}
    pnl = df['pnl'].values
    n   = len(pnl)
    wr  = (pnl>0).mean()*100
    tot = pnl.sum()
    eq  = 100_000 + np.cumsum(pnl)
    pk  = np.maximum.accumulate(eq)
    dd  = ((eq-pk)/pk*100).min()
    pyr = n/years
    exc = pnl/100_000 - 0.04/pyr
    sh  = exc.mean()/exc.std()*np.sqrt(pyr) if exc.std()>0 else 0
    return {'n':n,'wr':round(wr,1),'sharpe':round(sh,2),
            'dd':round(dd,2),'total':int(tot),'pyr':round(pyr,1)}


def main():
    print("="*72)
    print("USDJPY FULL SWEEP v2 — All Angles")
    print("="*72)

    rates   = load_rates()
    print(f"Rates loaded: {rates.index.min().date()} → {rates.index.max().date()}")

    print("Pre-computing z-scores...")
    zscores = build_zscores(rates)

    print("Loading USDJPY 15M...")
    m15 = load_prices()
    print(f"Price data: {m15.index.min().date()} → {m15.index.max().date()}")

    years     = (m15.index.max() - m15.index.min()).days / 365.25
    yrs_oos   = (m15.index.max() - WF_SPLIT).days / 365.25
    yrs_is    = (WF_SPLIT - m15.index.min()).days / 365.25

    total = len(SWEEP)
    print(f"\nSweeping {total:,} configurations...")
    print(f"{'='*72}")

    results = []
    done    = 0

    for (thresh, z_win, tp, sl, hold, z_exit, fib,
         session, direction, sizing) in SWEEP:
        done += 1
        if done % 100 == 0:
            pct = done/total*100
            print(f"  {done:>5}/{total}  ({pct:.1f}%)  "
                  f"best OOS so far: "
                  f"{max((r['sh_oos'] for r in results), default=0):.2f}",
                  end="\r")

        z_h    = zscores[z_win]
        trades = run_backtest(z_h, m15, thresh, tp, sl, hold,
                              z_exit, fib, session, direction, sizing)

        if len(trades) < 15: continue

        is_tr  = trades[trades['entry_time'] < WF_SPLIT]
        oos_tr = trades[trades['entry_time'] >= WF_SPLIT]

        r_full = score(trades, years)
        r_oos  = score(oos_tr, yrs_oos)

        if r_full['sharpe'] < 0.5: continue
        if r_oos['sharpe']  < 0.5: continue

        fib_str = f"fib{fib:.3f}" if fib > 0 else "mkt"
        zexit_str = f"ze{z_exit}" if z_exit < 99 else "ze_off"
        label = (f"z{z_win} t{thresh} tp{tp*100:.2f}% sl{sl*100:.2f}% "
                 f"h{hold}h {zexit_str} {fib_str} {session} {direction} {sizing}")

        results.append({
            'label':      label,
            'z_win':      z_win,
            'threshold':  thresh,
            'tp':         tp,
            'sl':         sl,
            'hold':       hold,
            'z_exit':     z_exit,
            'fib':        fib,
            'session':    session,
            'direction':  direction,
            'sizing':     sizing,
            'n_full':     r_full['n'],
            'pyr_full':   r_full['pyr'],
            'wr_full':    r_full['wr'],
            'sh_full':    r_full['sharpe'],
            'dd_full':    r_full['dd'],
            'tot_full':   r_full['total'],
            'n_oos':      r_oos['n'],
            'sh_oos':     r_oos['sharpe'],
            'wr_oos':     r_oos['wr'],
            'dd_oos':     r_oos['dd'],
        })

    print(f"\n\nSweep complete. {len(results):,} configs passed Sharpe > 0.5 on both IS and OOS")

    if not results:
        print("No configs passed. Try lowering Sharpe threshold.")
        return

    results.sort(key=lambda x: x['sh_oos'], reverse=True)

    # Save full results
    out_path = PROC_PATH / "usdjpy_full_sweep_v2.csv"
    pd.DataFrame(results).to_csv(out_path, index=False)
    print(f"Full results saved: {out_path.name}")

    print(f"\n{'='*72}")
    print("TOP 25 — Ranked by OOS Sharpe (2020+)")
    print(f"{'='*72}")
    print(f"  {'Config':<65}{'N':>5}{'WR%':>6}{'Sh(IS)':>7}"
          f"{'Sh(OOS)':>8}{'DD%':>7}")
    print(f"  {'─'*100}")
    for r in results[:25]:
        print(f"  {r['label']:<65}{r['n_full']:>5}"
              f"{r['wr_full']:>6.1f}{r['sh_full']:>7.2f}"
              f"{r['sh_oos']:>8.2f}{r['dd_full']:>7.2f}")

    # Best config
    best = results[0]
    print(f"\n{'='*72}")
    print("BEST CONFIG")
    print(f"{'='*72}")
    print(f"  {best['label']}")
    print(f"  Full: Sharpe {best['sh_full']:.2f}  WR {best['wr_full']:.1f}%  "
          f"DD {best['dd_full']:.2f}%  N={best['n_full']}  "
          f"({best['pyr_full']:.0f}/yr)")
    print(f"  OOS:  Sharpe {best['sh_oos']:.2f}  WR {best['wr_oos']:.1f}%  "
          f"DD {best['dd_oos']:.2f}%  N={best['n_oos']}")

    # Insights — what parameters dominate the top 25?
    print(f"\n{'='*72}")
    print("PARAMETER INSIGHTS — Top 25 configs")
    print(f"{'='*72}")
    top25 = pd.DataFrame(results[:25])
    for col, label in [
        ('z_exit', 'Z-exit'),
        ('fib',    'Fibonacci'),
        ('session','Session'),
        ('direction','Direction'),
        ('sizing', 'Sizing'),
        ('threshold','Threshold'),
    ]:
        counts = top25[col].value_counts()
        best_val = counts.index[0]
        print(f"  {label:<12}: {best_val} appears {counts.iloc[0]}/25 times")

    print(f"\n  V1 baseline for comparison:")
    print(f"  z30 t2.0 tp0.50% sl0.50% h52h ze1.5 fib0.786 london both flat")
    print(f"  Full Sharpe 1.93  OOS Sharpe 2.38  WR 73.6%  DD -2.90%")


if __name__ == "__main__":
    main()