"""
governor_backtest_v1.py
=======================
Tests a REGIME-BASED NOTIONAL GOVERNOR against the FTMO 2-Step Swing rules.

PROBLEM (from v4): at $300k base (=1x tier, so 2x trades run $600k), daily-DD
breaches happen on busy/volatile days when positions stack. Governor aims to
cut exposure in those regimes -> lift pass rate -> ideally keep $300k's speed.

IMPORTANT TIER NOTE: $300k notional = the 1x tier. The real system sizes
1x/1.5x/2x by z-band, so strong signals already run up to $600k. The 'mult'
(UJ) and zscore-band (EU) encode this. The governor multiplies ON TOP of the
tier, at entry, using ONLY info available at entry (no lookahead).

GOVERNOR MODES (compare):
  none      : flat tiers, $300k base (reproduces v4 ~baseline)
  expo_cap  : cap TOTAL concurrent open notional. If a new trade would push
              concurrent open notional over CAP, scale it down to fit.
  vol_scale : scale notional by trailing realized volatility (per pair).
              High vol -> smaller; low vol -> up to base (NOT above unless
              allow_scaleup=True).
  combined  : expo_cap AND vol_scale.

Also supports allow_scaleup: in quiet/low-vol regimes, permit notional ABOVE
base (your idea). OFF by default — risk-increasing, shown only if you opt in.

This uses the REAL per-trade floating paths already reconstructed conceptually
via mae, but for the governor we recompute daily floating from per-trade entries
scaled by the governor multiplier. We read trades_real_costs (EU) and
usdjpy_trades_real_costs (UJ) and the 15M bars for floating paths.

OUTPUT: prints pass-rate/median comparison per governor mode at base $300k.

NOTE: This is a BACKTEST HARNESS. Parameters at top are meant to be tuned
tomorrow. Defaults are economically sensible, not curve-fit.
"""
import csv, sys, json
import numpy as np
from pathlib import Path
from datetime import datetime, timedelta
from collections import defaultdict

# ---- FTMO rules ----
ACCOUNT=100_000; P1_TARGET=10_000; P2_TARGET=5_000
DAILY_LIMIT=5_000; MAXLOSS_FLOOR=90_000
N_SIMS=5_000; SEED=42
BASE_NOTIONAL=300_000          # = 1x tier
EU_HOLD_H=52; UJ_HOLD_H=24
rng=np.random.default_rng(SEED)

# ---- GOVERNOR PARAMETERS (tune tomorrow) ----
CONCURRENT_CAP   = 600_000     # max total open notional across all positions
                               # (=2x one pair, or 1x+1x both pairs). Tighten to
                               # reduce stacking risk.
VOL_LOOKBACK_DAYS= 20          # trailing window for realized vol
VOL_REF_PCTL     = 50          # vol at/below this percentile -> full size
VOL_HIGH_PCTL    = 90          # vol at/above this -> min multiplier
VOL_MIN_MULT     = 0.5         # smallest the vol-scaler will shrink to
ALLOW_SCALEUP    = False       # if True, quiet regimes can exceed base (risky)
SCALEUP_MAX_MULT = 1.5         # cap on scale-up if allowed

def resolve_base():
    for b in [Path(r"C:\Users\Administrator\OneDrive\fx_macro_intraday"),
              Path(r"C:\Users\paul_\OneDrive\fx_macro_intraday"),
              Path(__file__).resolve().parents[2]]:
        if b.exists(): return b
    sys.exit("base not found")
BASE=resolve_base()

def parse_dt(s):
    s=(s or "").strip()
    for f in ("%Y-%m-%d %H:%M:%S","%Y-%m-%d %H:%M","%Y.%m.%d %H:%M:%S",
              "%Y.%m.%d %H:%M","%d/%m/%Y %H:%M"):
        try: return datetime.strptime(s[:19], f)
        except ValueError: continue
    return None

# ---- Load 15M bars for floating-path + vol ----
def load_bars(path):
    bars=[]
    with open(path, newline="", encoding="utf-8-sig") as f:
        reader=csv.DictReader(f); cols={c.lower():c for c in reader.fieldnames}
        dtc=cols["datetime"]; hc=cols["high"]; lc=cols["low"]; cc=cols["close"]
        for r in reader:
            dt=parse_dt(r.get(dtc))
            if dt is None: continue
            try: hi=float(r[hc]); lo=float(r[lc]); cl=float(r[cc])
            except (ValueError,KeyError): continue
            bars.append((dt,hi,lo,cl))
    bars.sort(key=lambda x:x[0])
    return bars

def bars_by_date(bars):
    idx=defaultdict(list)
    for b in bars: idx[b[0].date()].append(b)
    return idx

def daily_close_series(bars):
    """Last close per date -> for realized vol."""
    by=defaultdict(list)
    for (dt,hi,lo,cl) in bars: by[dt.date()].append(cl)
    dates=sorted(by)
    closes=[by[d][-1] for d in dates]
    return dates, closes

def realized_vol_lookup(dates, closes, lookback):
    """date -> trailing stdev of daily returns (lookback days)."""
    rets=[0.0]+[ (closes[i]/closes[i-1]-1.0) for i in range(1,len(closes)) ]
    vol={}
    for i,d in enumerate(dates):
        lo=max(0,i-lookback)
        window=rets[lo:i+1]
        vol[d]= float(np.std(window)) if len(window)>2 else 0.0
    return vol

# ---- Load trades with tier and direction ----
def load_trades():
    eu_p=BASE/"data"/"processed"/"trades"/"trades_real_costs.csv"
    uj_p=BASE/"data"/"processed"/"usdjpy_trades_real_costs.csv"
    T=[]
    with open(eu_p, newline="", encoding="utf-8-sig") as f:
        for r in csv.DictReader(f):
            e=parse_dt(r.get("entry_time")); x=parse_dt(r.get("exit_time"))
            if e is None: continue
            if x is None: x=e+timedelta(hours=EU_HOLD_H)
            try:
                ep=float(r["entry_price"]); sig=int(float(r["signal"]))
                pnl=float(r["dollar_pnl_real"]); za=float(r.get("zscore_abs",2.0))
            except (ValueError,KeyError): continue
            # EU tier from zscore band (approx live: 1x <3.0, 1.5x 3-3.5, 2x >=3.5)
            tier = 2.0 if za>=3.5 else (1.5 if za>=3.0 else 1.0)
            T.append({"pair":"EU","entry":e,"exit":x,"ep":ep,"dir":sig,
                      "pnl_1x":pnl/tier,"tier":tier})
    with open(uj_p, newline="", encoding="utf-8-sig") as f:
        for r in csv.DictReader(f):
            e=parse_dt(r.get("entry_time"))
            if e is None: continue
            try:
                ep=float(r["entry_price"]); d=int(float(r["direction"]))
                pnl=float(r["pnl_real"]); mult=float(r.get("mult",1.0))
            except (ValueError,KeyError): continue
            T.append({"pair":"UJ","entry":e,"exit":e+timedelta(hours=UJ_HOLD_H),
                      "ep":ep,"dir":d,"pnl_1x":pnl/mult,"tier":mult})
    T.sort(key=lambda t:t["entry"])
    return T

# ---- Governor: decide a multiplier for each trade at ENTRY ----
def governor_multiplier(trade, mode, open_book, vol_eu, vol_uj,
                        vol_ref, vol_hi):
    """Return notional multiplier applied ON TOP of base $300k (not tier).
    open_book: list of (exit_dt, notional) currently open. vol_*: dict date->vol.
    """
    base_mult = 1.0
    # --- volatility scaling ---
    if mode in ("vol_scale","combined"):
        vmap = vol_eu if trade["pair"]=="EU" else vol_uj
        v = vmap.get(trade["entry"].date(), 0.0)
        if v<=vol_ref:
            vm = SCALEUP_MAX_MULT if ALLOW_SCALEUP else 1.0
        elif v>=vol_hi:
            vm = VOL_MIN_MULT
        else:
            # linear interpolate between ref(1.0) and hi(VOL_MIN_MULT)
            frac=(v-vol_ref)/(vol_hi-vol_ref)
            vm = 1.0 - frac*(1.0-VOL_MIN_MULT)
        base_mult *= vm
    # --- concurrent exposure cap ---
    if mode in ("expo_cap","combined"):
        now=trade["entry"]
        open_notional=sum(n for (xt,n) in open_book if xt>now)
        # this trade's intended notional = base * tier * current base_mult
        intended = BASE_NOTIONAL * trade["tier"] * base_mult
        if open_notional + intended > CONCURRENT_CAP:
            room = max(0.0, CONCURRENT_CAP - open_notional)
            if intended>0:
                base_mult *= min(1.0, room/intended)
    return max(0.0, base_mult)

# ---- Build per-day (realized, concurrent_float) under a governor mode ----
def build_days(trades, bars_eu, bars_uj, vol_eu, vol_uj, mode):
    day_real=defaultdict(float); day_float=defaultdict(float); day_n=defaultdict(int)
    open_book=[]  # (exit_dt, notional)
    for t in trades:
        # prune closed
        open_book=[(xt,n) for (xt,n) in open_book if xt>t["entry"]]
        gmult=governor_multiplier(t, mode, open_book, vol_eu, vol_uj,
                                  np.percentile(list(vol_eu.values()),VOL_REF_PCTL) if t["pair"]=="EU"
                                  else np.percentile(list(vol_uj.values()),VOL_REF_PCTL),
                                  np.percentile(list(vol_eu.values()),VOL_HIGH_PCTL) if t["pair"]=="EU"
                                  else np.percentile(list(vol_uj.values()),VOL_HIGH_PCTL))
        eff_notional = BASE_NOTIONAL * t["tier"] * gmult
        scale = eff_notional / (BASE_NOTIONAL * t["tier"]) if t["tier"]>0 else 0
        # realized pnl scales with notional
        day_real[t["exit"].date()] += t["pnl_1x"]*t["tier"]*gmult
        # floating path from bars, scaled
        bidx = bars_eu if t["pair"]=="EU" else bars_uj
        units = eff_notional / t["ep"]
        d=t["entry"].date(); end=t["exit"].date()
        while d<=end:
            worst=0.0
            for (bt,hi,lo,cl) in bidx.get(d,[]):
                if bt<t["entry"] or bt>t["exit"]: continue
                fl=(t["ep"]-lo)*units if t["dir"]>0 else (hi-t["ep"])*units
                if fl>worst: worst=fl
            if worst>0:
                day_float[d]+=worst   # concurrent sum
                day_n[d]+=1
            d=d+timedelta(days=1)
        open_book.append((t["exit"], eff_notional))
    alld=sorted(set(day_real)|set(day_float))
    return [(d, day_real.get(d,0.0), day_float.get(d,0.0), day_n.get(d,0)) for d in alld]

# ---- Challenge sim (same as v4) ----
def run_phase(day_seq, target, start_idx=0):
    equity=ACCOUNT; tdays=0; first=None; i=start_idx; n=len(day_seq)
    while i<n:
        d,dp,df,ntr=day_seq[i]
        if first is None: first=d
        do=equity; floor=do-DAILY_LIMIT
        ilow=do+min(0.0,dp)-df
        if ilow<=MAXLOSS_FLOOR: return ("maxloss",i,(d-first).days)
        if ilow<=floor: return ("daily",i,(d-first).days)
        equity+=dp
        if ntr>0: tdays+=1
        if equity<=MAXLOSS_FLOOR: return ("maxloss",i,(d-first).days)
        if equity>=ACCOUNT+target and tdays>=4: return ("pass",i,(d-first).days)
        i+=1
    return ("incomplete",n-1,(day_seq[-1][0]-first).days if first else 0)

def run_challenge(day_seq):
    r1,i1,e1=run_phase(day_seq,P1_TARGET,0)
    if r1!="pass": return ("fail_p1_"+r1, e1)
    r2,i2,e2=run_phase(day_seq,P2_TARGET,i1+1)
    if r2!="pass": return ("fail_p2_"+r2, e1+e2)
    return ("pass", e1+e2)

def redate(seq):
    out=[]; d=datetime(2000,1,3).date()
    for (_od,pnl,fl,n) in seq:
        out.append((d,pnl,fl,n)); d+=timedelta(days=1)
        while d.weekday()>=5: d+=timedelta(days=1)
    return out

def summ(outs,pd,n):
    p=sum(1 for o in outs if o=="pass")
    f1d=sum(1 for o in outs if o.startswith("fail_p1_daily"))
    f1m=sum(1 for o in outs if o.startswith("fail_p1_maxloss"))
    f2=sum(1 for o in outs if o.startswith("fail_p2"))
    med=float(np.median(np.array(pd)/30.0)) if pd else 0
    return {"pass":100*p/n if n else 0,"daily":100*f1d/n if n else 0,
            "maxloss":100*f1m/n if n else 0,"p2":100*f2/n if n else 0,"med":med}

def m_block(days,n_sims=N_SIMS,blk=21):
    outs=[];pd=[];n=len(days);tl=min(n,24*21)
    for _ in range(n_sims):
        seq=[]
        while len(seq)<tl:
            st=rng.integers(0,max(1,n-blk)); seq.extend(days[st:st+blk])
        seq=redate(seq); o,el=run_challenge(seq); outs.append(o)
        if o=="pass": pd.append(el)
    return summ(outs,pd,n_sims)

def m_seq(days):
    outs=[];pd=[];n=len(days)
    for s in range(n):
        if n-s<8: break
        o,el=run_challenge(days[s:]); outs.append(o)
        if o=="pass": pd.append(el)
    return summ(outs,pd,len(outs))

def main():
    print("="*78)
    print("REGIME GOVERNOR BACKTEST | FTMO 2-Step Swing | base $300k=1x tier")
    print(f"Params: CONCURRENT_CAP=${CONCURRENT_CAP:,} VOL_LOOKBACK={VOL_LOOKBACK_DAYS}d "
          f"VOL_MIN_MULT={VOL_MIN_MULT} ALLOW_SCALEUP={ALLOW_SCALEUP}")
    print("="*78)
    print("Loading bars + trades...")
    bars_eu_raw=load_bars(BASE/"data"/"raw"/"prices"/"EURUSD"/"EURUSD_15M_2003_2026.csv")
    bars_uj_raw=load_bars(BASE/"data"/"raw"/"prices"/"USDJPY"/"USDJPY_15M_2003_2026.csv")
    bars_eu=bars_by_date(bars_eu_raw); bars_uj=bars_by_date(bars_uj_raw)
    de,ce=daily_close_series(bars_eu_raw); du,cu=daily_close_series(bars_uj_raw)
    vol_eu=realized_vol_lookup(de,ce,VOL_LOOKBACK_DAYS)
    vol_uj=realized_vol_lookup(du,cu,VOL_LOOKBACK_DAYS)
    trades=load_trades()
    print(f"Trades {len(trades):,}\n")

    modes=["none","expo_cap","vol_scale","combined"]
    print(f"  {'Mode':>10}{'Method':>9}{'Pass%':>8}{'DailyF%':>9}{'MaxL%':>7}{'P2F%':>7}{'Med':>6}")
    for mode in modes:
        days=build_days(trades,bars_eu,bars_uj,vol_eu,vol_uj,mode)
        rS=m_seq(days); rB=m_block(days)
        print(f"  {mode:>10}{'A-seq':>9}{rS['pass']:>8.1f}{rS['daily']:>9.1f}"
              f"{rS['maxloss']:>7.1f}{rS['p2']:>7.1f}{rS['med']:>6.1f}")
        print(f"  {'':>10}{'B-block':>9}{rB['pass']:>8.1f}{rB['daily']:>9.1f}"
              f"{rB['maxloss']:>7.1f}{rB['p2']:>7.1f}{rB['med']:>6.1f}")
        print()
    print("="*78)
    print("GOAL: a governor mode that lifts Pass% toward 95%+ while keeping Med low.")
    print("If expo_cap/combined keeps median near 'none' but cuts daily fails -> win.")
    print("Tune CONCURRENT_CAP / VOL params at top of file and re-run.")
    print("ALLOW_SCALEUP=True tests quiet-period scale-up (risk-increasing).")
    print("="*78)

if __name__=="__main__":
    main()