"""
usdjpy_sharpe_validation_v1.py
================================
Seven independent statistical tests to validate the USDJPY Sharpe ratio
is genuine and not overfitted — identical methodology to the EURUSD
validation that produced 7/7 passes.

Tests:
  T1: Signal Randomisation     — is the edge in the signal or the prices?
  T2: Block Bootstrap CI       — what is the true Sharpe confidence interval?
  T3: Yearly Consistency       — does the edge hold across individual years?
  T4: Walk-Forward OOS         — does IS Sharpe hold out-of-sample?
  T5: Deflated Sharpe Ratio    — does edge survive multiple-testing correction?
  T6: Min Backtest Length      — do we have enough trades for statistical power?
  T7: Newey-West t-statistic   — is mean return significant after autocorrelation?

Additional metrics (not in EU validation — added for completeness):
  Calmar ratio, Sortino ratio, Recovery factor, Rolling 3-year windows

Data: usdjpy_trades_real_costs.csv (702 trades, 2003-2026)
      pnl_real column — real costs applied, mult scaling applied

Run from project root:
  python src/research/usdjpy_sharpe_validation_v1.py

Output:
  data/output/usdjpy_validation/
    validation_results.txt
    validation_charts.png
"""

import sys
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
from pathlib import Path
from scipy import stats
from datetime import datetime

BASE_PATH  = Path(__file__).resolve().parents[2]
OUTPUT_DIR = BASE_PATH / "data" / "output" / "usdjpy_validation"
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)

# ── Load data ─────────────────────────────────────────────────────────────────
def load_trades():
    path = BASE_PATH / "data" / "processed" / "usdjpy_trades_real_costs.csv"
    if not path.exists():
        raise FileNotFoundError(f"Required: {path}")
    df = pd.read_csv(path)
    df["date"] = pd.to_datetime(df["entry_time"]).dt.normalize()
    df["year"] = df["date"].dt.year
    df = df.sort_values("date").reset_index(drop=True)
    print(f"  Loaded {len(df)} trades | "
          f"{df['date'].iloc[0].date()} → {df['date'].iloc[-1].date()}")
    return df

# ── Core metrics ──────────────────────────────────────────────────────────────
def compute_metrics(pnl_series, label=""):
    """Compute Sharpe, Sortino, Calmar from trade P&L series."""
    n         = len(pnl_series)
    mean_pnl  = np.mean(pnl_series)
    std_pnl   = np.std(pnl_series, ddof=1)
    sharpe    = (mean_pnl / std_pnl * np.sqrt(252)) if std_pnl > 0 else 0
    # Sortino — downside deviation only
    downside  = pnl_series[pnl_series < 0]
    down_std  = np.std(downside, ddof=1) if len(downside) > 1 else std_pnl
    sortino   = (mean_pnl / down_std * np.sqrt(252)) if down_std > 0 else 0
    # Calmar — annualised return / max drawdown
    equity    = np.cumsum(pnl_series)
    peak      = np.maximum.accumulate(equity)
    dd        = (equity - peak)
    max_dd    = abs(dd.min()) if len(dd) > 0 else 1
    years     = n / 252 if n > 252 else 1
    ann_ret   = mean_pnl * 252
    calmar    = ann_ret / max_dd if max_dd > 0 else 0
    # Recovery factor
    total_pnl = pnl_series.sum()
    recovery  = total_pnl / max_dd if max_dd > 0 else 0
    # Win rate
    wins      = (pnl_series > 0).sum()
    wr        = wins / n * 100 if n > 0 else 0
    t_stat    = mean_pnl / (std_pnl / np.sqrt(n)) if std_pnl > 0 else 0
    p_val     = stats.t.sf(abs(t_stat), df=n-1) * 2
    return {
        "n": n, "mean_pnl": mean_pnl, "std_pnl": std_pnl,
        "sharpe": sharpe, "sortino": sortino, "calmar": calmar,
        "recovery": recovery, "win_rate": wr, "max_dd": max_dd,
        "t_stat": t_stat, "p_val": p_val, "label": label,
    }

# ═══════════════════════════════════════════════════════════════════════════════
# T1: Signal Randomisation
# ═══════════════════════════════════════════════════════════════════════════════
def t1_signal_randomisation(df, n_sims=5000, seed=42):
    """
    Randomise signal direction 5,000 times, keeping prices fixed.
    If real Sharpe > 99.9% of random Sharpes, the edge is in the signal.
    """
    print("  T1: Signal Randomisation...")
    rng        = np.random.default_rng(seed)
    real_pnl   = df["pnl_real"].values
    real_sharpe= compute_metrics(real_pnl)["sharpe"]

    random_sharpes = []
    for _ in range(n_sims):
        # Flip signs randomly (preserves magnitude, randomises direction)
        signs  = rng.choice([-1, 1], size=len(real_pnl))
        random = np.abs(real_pnl) * signs
        m      = compute_metrics(random)
        random_sharpes.append(m["sharpe"])

    random_sharpes = np.array(random_sharpes)
    pct_rank = (random_sharpes < real_sharpe).mean()
    z_score  = (real_sharpe - random_sharpes.mean()) / random_sharpes.std()
    # One-tailed p-value
    p_val    = 1 - pct_rank

    result = {
        "test": "T1: Signal Randomisation",
        "pass": pct_rank >= 0.999,
        "real_sharpe": round(real_sharpe, 4),
        "random_mean": round(random_sharpes.mean(), 4),
        "random_std":  round(random_sharpes.std(), 4),
        "percentile":  round(pct_rank * 100, 3),
        "z_score":     round(z_score, 2),
        "p_val":       f"{p_val:.6f}",
        "interpretation": (
            f"Real Sharpe {real_sharpe:.3f} beats {pct_rank*100:.1f}% of random "
            f"signals ({z_score:.1f}σ above random mean). "
            f"{'PASS — edge is in the signal, not the prices.' if pct_rank >= 0.999 else 'FAIL — edge may be random.'}"
        )
    }
    print(f"    Real Sharpe: {real_sharpe:.3f} | Percentile: {pct_rank*100:.2f}% | z={z_score:.1f}σ | "
          f"{'PASS ✅' if result['pass'] else 'FAIL ❌'}")
    return result, random_sharpes

# ═══════════════════════════════════════════════════════════════════════════════
# T2: Block Bootstrap Confidence Interval
# ═══════════════════════════════════════════════════════════════════════════════
def t2_block_bootstrap(df, n_sims=10000, block_size=20, seed=42):
    """
    Block bootstrap — resample in blocks of 20 to preserve autocorrelation.
    Reports 95% CI for the true Sharpe ratio.
    """
    print("  T2: Block Bootstrap CI...")
    rng      = np.random.default_rng(seed)
    pnl      = df["pnl_real"].values
    n        = len(pnl)
    n_blocks = int(np.ceil(n / block_size))
    bs_sharpes = []

    for _ in range(n_sims):
        block_starts = rng.integers(0, n - block_size, size=n_blocks)
        sample = np.concatenate([pnl[s:s+block_size] for s in block_starts])[:n]
        bs_sharpes.append(compute_metrics(sample)["sharpe"])

    bs_sharpes = np.array(bs_sharpes)
    ci_lo = np.percentile(bs_sharpes, 2.5)
    ci_hi = np.percentile(bs_sharpes, 97.5)
    real  = compute_metrics(pnl)["sharpe"]

    result = {
        "test": "T2: Block Bootstrap CI",
        "pass": ci_lo > 1.0,  # CI lower bound must be meaningfully positive
        "real_sharpe": round(real, 4),
        "ci_lo": round(ci_lo, 4),
        "ci_hi": round(ci_hi, 4),
        "bs_mean": round(bs_sharpes.mean(), 4),
        "bs_std":  round(bs_sharpes.std(), 4),
        "interpretation": (
            f"95% CI [{ci_lo:.3f}, {ci_hi:.3f}]. "
            f"{'PASS — lower CI bound > 1.0, Sharpe is robustly positive.' if ci_lo > 1.0 else 'FAIL — CI includes values near zero.'}"
        )
    }
    print(f"    Real: {real:.3f} | 95% CI: [{ci_lo:.3f}, {ci_hi:.3f}] | "
          f"{'PASS ✅' if result['pass'] else 'FAIL ❌'}")
    return result, bs_sharpes

# ═══════════════════════════════════════════════════════════════════════════════
# T3: Yearly Consistency
# ═══════════════════════════════════════════════════════════════════════════════
def t3_yearly_consistency(df, min_trades=5, threshold_pct=75):
    """
    Compute Sharpe per calendar year.
    Pass if positive Sharpe in >= 75% of years with sufficient trades.
    """
    print("  T3: Yearly Consistency...")
    yearly = []
    for yr, grp in df.groupby("year"):
        if len(grp) < min_trades:
            continue
        m = compute_metrics(grp["pnl_real"].values, label=str(yr))
        yearly.append({
            "year": yr, "trades": m["n"], "win_rate": round(m["win_rate"], 1),
            "sharpe": round(m["sharpe"], 3), "pnl": round(grp["pnl_real"].sum(), 0),
            "positive": m["sharpe"] > 0
        })

    df_yr     = pd.DataFrame(yearly)
    n_pos     = df_yr["positive"].sum()
    n_total   = len(df_yr)
    pct_pos   = n_pos / n_total * 100 if n_total > 0 else 0
    pass_test = pct_pos >= threshold_pct

    result = {
        "test": "T3: Yearly Consistency",
        "pass": pass_test,
        "years_positive": int(n_pos),
        "years_total":    int(n_total),
        "pct_positive":   round(pct_pos, 1),
        "threshold":      threshold_pct,
        "yearly_data":    yearly,
        "interpretation": (
            f"{n_pos}/{n_total} years positive ({pct_pos:.1f}%) "
            f"with >= {min_trades} trades. Threshold: {threshold_pct}%. "
            f"{'PASS.' if pass_test else 'FAIL.'}"
        )
    }
    print(f"    {n_pos}/{n_total} years positive ({pct_pos:.1f}%) | "
          f"{'PASS ✅' if pass_test else 'FAIL ❌'}")
    return result, df_yr

# ═══════════════════════════════════════════════════════════════════════════════
# T4: Walk-Forward OOS
# ═══════════════════════════════════════════════════════════════════════════════
def t4_walk_forward(df, cutoff_year=2020):
    """
    Split: IS = 2003-2019, OOS = 2020+
    OOS Sharpe must be positive. Degradation from IS acceptable if <50%.
    """
    print(f"  T4: Walk-Forward OOS (cutoff {cutoff_year})...")
    is_df  = df[df["year"] < cutoff_year]
    oos_df = df[df["year"] >= cutoff_year]

    is_m   = compute_metrics(is_df["pnl_real"].values, "IS")
    oos_m  = compute_metrics(oos_df["pnl_real"].values, "OOS")
    degrad = (is_m["sharpe"] - oos_m["sharpe"]) / is_m["sharpe"] * 100 if is_m["sharpe"] > 0 else 0
    pass_t = oos_m["sharpe"] > 0 and degrad < 60

    result = {
        "test": "T4: Walk-Forward OOS",
        "pass": pass_t,
        "is_sharpe":   round(is_m["sharpe"], 4),
        "oos_sharpe":  round(oos_m["sharpe"], 4),
        "is_trades":   is_m["n"],
        "oos_trades":  oos_m["n"],
        "degradation": round(degrad, 1),
        "is_wr":       round(is_m["win_rate"], 1),
        "oos_wr":      round(oos_m["win_rate"], 1),
        "interpretation": (
            f"IS Sharpe {is_m['sharpe']:.3f} ({is_m['n']} trades) | "
            f"OOS Sharpe {oos_m['sharpe']:.3f} ({oos_m['n']} trades) | "
            f"Degradation {degrad:.1f}%. "
            f"{'PASS — OOS positive.' if pass_t else 'FAIL — OOS negative or excessive degradation.'}"
        )
    }
    print(f"    IS: {is_m['sharpe']:.3f} ({is_m['n']} trades) | "
          f"OOS: {oos_m['sharpe']:.3f} ({oos_m['n']} trades) | "
          f"Degradation: {degrad:.1f}% | {'PASS ✅' if pass_t else 'FAIL ❌'}")
    return result

# ═══════════════════════════════════════════════════════════════════════════════
# T5: Deflated Sharpe Ratio (Bailey & Lopez de Prado 2014)
# ═══════════════════════════════════════════════════════════════════════════════
def t5_deflated_sharpe(df, n_trials=50, seed=42):
    """
    Accounts for selection bias — multiple parameter configurations were tested.
    DSR > 0.95 means the Sharpe survives even aggressive multiple-testing correction.
    """
    print("  T5: Deflated Sharpe Ratio...")
    pnl    = df["pnl_real"].values
    n      = len(pnl)
    sr     = compute_metrics(pnl)["sharpe"] / np.sqrt(252)  # per-trade Sharpe
    skew   = stats.skew(pnl)
    kurt   = stats.kurtosis(pnl)  # excess kurtosis

    # Expected maximum Sharpe under multiple testing
    # Using the formula from Bailey & Lopez de Prado (2014)
    gamma  = 0.5772156649  # Euler-Mascheroni constant
    e_max  = (1 - gamma) * stats.norm.ppf(1 - 1/n_trials) + gamma * stats.norm.ppf(1 - 1/(n_trials * np.e))

    # Variance of Sharpe ratio (non-normal correction)
    sr_var = (1 - skew * sr + (kurt / 4) * sr**2) / (n - 1)
    sr_std = np.sqrt(sr_var) if sr_var > 0 else 0.01

    dsr = stats.norm.cdf((sr - e_max) / sr_std) if sr_std > 0 else 0.0
    pass_t = dsr >= 0.95

    result = {
        "test": "T5: Deflated Sharpe Ratio",
        "pass": pass_t,
        "observed_sr":   round(sr, 6),
        "expected_max":  round(e_max, 6),
        "dsr":           round(dsr, 6),
        "n_trials":      n_trials,
        "skewness":      round(skew, 4),
        "excess_kurt":   round(kurt, 4),
        "interpretation": (
            f"DSR = {dsr:.6f} (threshold 0.95) across {n_trials} assumed trials. "
            f"Skewness {skew:.3f}, excess kurtosis {kurt:.3f}. "
            f"{'PASS — edge survives multiple-testing correction.' if pass_t else 'FAIL — edge may be selection artefact.'}"
        )
    }
    print(f"    DSR: {dsr:.6f} | Trials assumed: {n_trials} | "
          f"Skew: {skew:.3f} | Kurt: {kurt:.3f} | "
          f"{'PASS ✅' if pass_t else 'FAIL ❌'}")
    return result

# ═══════════════════════════════════════════════════════════════════════════════
# T6: Minimum Backtest Length
# ═══════════════════════════════════════════════════════════════════════════════
def t6_min_backtest_length(df, confidence=0.99):
    """
    Bailey & Lopez de Prado (2014) formula for minimum number of trades
    needed to achieve a given confidence level. Need << available trades.
    """
    print("  T6: Minimum Backtest Length...")
    pnl    = df["pnl_real"].values
    n      = len(pnl)
    sr     = compute_metrics(pnl)["sharpe"] / np.sqrt(252)
    skew   = stats.skew(pnl)
    kurt   = stats.kurtosis(pnl)
    z      = stats.norm.ppf(confidence)  # 2.576 for 99%

    # Minimum trades needed
    if sr == 0: sr = 0.001
    n_min = int(np.ceil((z**2 / sr**2) * (1 - skew * sr + (kurt/4) * sr**2))) + 1
    ratio = n / n_min
    pass_t = n >= n_min * 2  # have at least 2× minimum required

    result = {
        "test": "T6: Minimum Backtest Length",
        "pass": pass_t,
        "n_available": n,
        "n_required":  n_min,
        "ratio":       round(ratio, 1),
        "confidence":  confidence,
        "interpretation": (
            f"Have {n} trades, need {n_min} for {confidence*100:.0f}% confidence. "
            f"Ratio {ratio:.1f}× available/required. "
            f"{'PASS — sufficient data.' if pass_t else 'FAIL — insufficient data for confidence level.'}"
        )
    }
    print(f"    Available: {n} | Required: {n_min} | Ratio: {ratio:.1f}× | "
          f"{'PASS ✅' if pass_t else 'FAIL ❌'}")
    return result

# ═══════════════════════════════════════════════════════════════════════════════
# T7: Newey-West t-statistic
# ═══════════════════════════════════════════════════════════════════════════════
def t7_newey_west(df, n_lags=None):
    """
    Autocorrelation-corrected significance test.
    Uses Newey-West HAC standard errors to correct for serial correlation.
    t-statistic must exceed 2.576 (99% confidence, two-tailed).
    """
    print("  T7: Newey-West t-statistic...")
    pnl  = df["pnl_real"].values
    n    = len(pnl)

    if n_lags is None:
        n_lags = int(np.floor(4 * (n/100)**(2/9)))  # Newey-West optimal lag selection

    mean_r = np.mean(pnl)

    # Compute Newey-West HAC variance
    gamma_0 = np.var(pnl, ddof=1)
    nw_var  = gamma_0
    for lag in range(1, n_lags + 1):
        gamma_k = np.cov(pnl[lag:], pnl[:-lag])[0, 1]
        weight  = 1 - lag / (n_lags + 1)
        nw_var += 2 * weight * gamma_k

    nw_se  = np.sqrt(nw_var / n) if nw_var > 0 else 1e-10
    t_stat = mean_r / nw_se
    p_val  = stats.t.sf(abs(t_stat), df=n-1) * 2
    threshold = stats.norm.ppf(0.995)  # 2.576 for 99%
    pass_t = abs(t_stat) > threshold

    result = {
        "test": "T7: Newey-West t-statistic",
        "pass": pass_t,
        "t_stat":    round(t_stat, 4),
        "p_val":     f"{p_val:.2e}",
        "n_lags":    n_lags,
        "nw_se":     round(nw_se, 4),
        "threshold": round(threshold, 3),
        "ratio":     round(abs(t_stat) / threshold, 2),
        "interpretation": (
            f"t = {t_stat:.2f}, p = {p_val:.2e} ({n_lags} HAC lags). "
            f"Threshold: {threshold:.3f} (99% confidence). "
            f"t is {abs(t_stat)/threshold:.1f}× the threshold. "
            f"{'PASS — mean return significant after autocorrelation correction.' if pass_t else 'FAIL.'}"
        )
    }
    print(f"    t = {t_stat:.2f} | p = {p_val:.2e} | "
          f"{abs(t_stat)/threshold:.1f}× threshold | "
          f"{'PASS ✅' if pass_t else 'FAIL ❌'}")
    return result

# ═══════════════════════════════════════════════════════════════════════════════
# Rolling 3-year windows
# ═══════════════════════════════════════════════════════════════════════════════
def rolling_windows(df, window_years=3):
    """
    Test every 3-year rolling window.
    Pass if >= 70% of windows are positive.
    """
    print("  Rolling 3-year windows...")
    results = []
    years   = sorted(df["year"].unique())
    for i in range(len(years) - window_years + 1):
        yr_start = years[i]
        yr_end   = years[i + window_years - 1]
        grp      = df[(df["year"] >= yr_start) & (df["year"] <= yr_end)]
        if len(grp) < 10:
            continue
        m = compute_metrics(grp["pnl_real"].values)
        results.append({
            "window": f"{yr_start}–{yr_end}",
            "trades": m["n"],
            "win_rate": round(m["win_rate"], 1),
            "sharpe": round(m["sharpe"], 3),
            "pnl": round(grp["pnl_real"].sum(), 0),
            "positive": m["sharpe"] > 0,
        })
    df_win  = pd.DataFrame(results)
    n_pos   = df_win["positive"].sum()
    n_total = len(df_win)
    pct_pos = n_pos / n_total * 100 if n_total > 0 else 0
    print(f"    {n_pos}/{n_total} windows positive ({pct_pos:.1f}%)")
    return df_win, n_pos, n_total, pct_pos

# ═══════════════════════════════════════════════════════════════════════════════
# Additional metrics
# ═══════════════════════════════════════════════════════════════════════════════
def additional_metrics(df):
    pnl  = df["pnl_real"].values
    m    = compute_metrics(pnl)
    years = df["year"].nunique()
    return {
        "sharpe":   round(m["sharpe"], 4),
        "sortino":  round(m["sortino"], 4),
        "calmar":   round(m["calmar"], 4),
        "recovery": round(m["recovery"], 2),
        "win_rate": round(m["win_rate"], 2),
        "max_dd":   round(m["max_dd"], 2),
        "t_stat":   round(m["t_stat"], 4),
        "p_val":    f"{m['p_val']:.2e}",
        "n_trades": m["n"],
        "trades_yr": round(m["n"] / years, 1),
        "total_pnl": round(pnl.sum(), 2),
    }

# ═══════════════════════════════════════════════════════════════════════════════
# Charts
# ═══════════════════════════════════════════════════════════════════════════════
def build_charts(df, t1_randoms, t2_bootstraps, df_yr, df_win):
    fig = plt.figure(figsize=(18, 12))
    fig.patch.set_facecolor('#0d1117')
    gs  = gridspec.GridSpec(3, 3, figure=fig, hspace=0.45, wspace=0.35)

    def ax_style(ax, title):
        ax.set_facecolor('#161b22')
        ax.tick_params(colors='#7d8590', labelsize=8)
        ax.set_title(title, color='#e6edf3', fontsize=10, fontweight='bold', pad=8)
        for spine in ax.spines.values(): spine.set_edgecolor('#21262d')

    real_sharpe = compute_metrics(df["pnl_real"].values)["sharpe"]

    # 1 — Equity curve
    ax1 = fig.add_subplot(gs[0, :2])
    equity = df["pnl_real"].cumsum()
    ax1.plot(df["date"], equity, color='#22d3ee', linewidth=1.5)
    ax1.fill_between(df["date"], equity, alpha=0.1, color='#22d3ee')
    ax1.set_ylabel('Cumulative P&L ($)', color='#7d8590', fontsize=8)
    ax_style(ax1, f'USDJPY Equity Curve — {len(df)} Trades | Sharpe {real_sharpe:.3f}')
    ax1.xaxis.label.set_color('#7d8590')
    ax1.tick_params(colors='#7d8590', labelsize=8)

    # 2 — T1: Randomisation distribution
    ax2 = fig.add_subplot(gs[0, 2])
    ax2.hist(t1_randoms, bins=50, color='#484f58', alpha=0.8, edgecolor='none')
    ax2.axvline(real_sharpe, color='#22d3ee', linewidth=2, label=f'Real: {real_sharpe:.3f}')
    ax2.axvline(np.percentile(t1_randoms, 99.9), color='#ffb830', linewidth=1,
                linestyle='--', label='99.9th pct')
    ax2.legend(fontsize=7, labelcolor='#7d8590', facecolor='#161b22', edgecolor='#21262d')
    ax2.set_xlabel('Sharpe', color='#7d8590', fontsize=8)
    ax_style(ax2, 'T1: Signal Randomisation (5,000 sims)')

    # 3 — T2: Bootstrap CI
    ax3 = fig.add_subplot(gs[1, 0])
    ax3.hist(t2_bootstraps, bins=50, color='#3d9fff', alpha=0.7, edgecolor='none')
    ci_lo, ci_hi = np.percentile(t2_bootstraps, [2.5, 97.5])
    ax3.axvline(real_sharpe, color='#00e87a', linewidth=2, label=f'Real: {real_sharpe:.3f}')
    ax3.axvline(ci_lo, color='#ffb830', linewidth=1.5, linestyle='--', label=f'CI [{ci_lo:.2f},{ci_hi:.2f}]')
    ax3.axvline(ci_hi, color='#ffb830', linewidth=1.5, linestyle='--')
    ax3.legend(fontsize=7, labelcolor='#7d8590', facecolor='#161b22', edgecolor='#21262d')
    ax_style(ax3, 'T2: Block Bootstrap (10,000 resamples)')

    # 4 — T3: Yearly Sharpe
    ax4 = fig.add_subplot(gs[1, 1])
    colors = ['#00e87a' if s > 0 else '#ff3d5a' for s in df_yr["sharpe"]]
    ax4.bar(df_yr["year"].astype(str), df_yr["sharpe"], color=colors, alpha=0.8)
    ax4.axhline(0, color='#7d8590', linewidth=1)
    ax4.axhline(real_sharpe, color='#22d3ee', linewidth=1, linestyle='--', alpha=0.5)
    ax4.set_xlabel('Year', color='#7d8590', fontsize=8)
    ax4.tick_params(axis='x', rotation=45)
    ax_style(ax4, f'T3: Yearly Sharpe ({df_yr["positive"].sum()}/{len(df_yr)} positive)')

    # 5 — T4: IS vs OOS
    ax5 = fig.add_subplot(gs[1, 2])
    is_df  = df[df["year"] < 2020]["pnl_real"].values
    oos_df = df[df["year"] >= 2020]["pnl_real"].values
    is_eq  = np.cumsum(is_df)
    oos_eq = np.cumsum(oos_df)
    ax5.plot(range(len(is_eq)),  is_eq,  color='#3d9fff',  linewidth=1.5, label=f'IS 03-19 (S={compute_metrics(is_df)["sharpe"]:.2f})')
    ax5.plot(range(len(is_eq), len(is_eq)+len(oos_eq)), oos_eq + is_eq[-1],
             color='#00e87a', linewidth=1.5, label=f'OOS 20+ (S={compute_metrics(oos_df)["sharpe"]:.2f})')
    ax5.axvline(len(is_eq), color='#ffb830', linewidth=1, linestyle='--')
    ax5.legend(fontsize=7, labelcolor='#7d8590', facecolor='#161b22', edgecolor='#21262d')
    ax_style(ax5, 'T4: Walk-Forward IS vs OOS')

    # 6 — Rolling windows
    ax6 = fig.add_subplot(gs[2, :2])
    win_colors = ['#00e87a' if s > 0 else '#ff3d5a' for s in df_win["sharpe"]]
    ax6.bar(df_win["window"], df_win["sharpe"], color=win_colors, alpha=0.8)
    ax6.axhline(0, color='#7d8590', linewidth=1)
    ax6.tick_params(axis='x', rotation=45, labelsize=7)
    n_pos = df_win["positive"].sum()
    ax_style(ax6, f'Rolling 3-Year Windows ({n_pos}/{len(df_win)} positive)')

    # 7 — P&L distribution
    ax7 = fig.add_subplot(gs[2, 2])
    pnl = df["pnl_real"].values
    wins  = pnl[pnl > 0]
    loses = pnl[pnl < 0]
    ax7.hist(wins,  bins=25, color='#00e87a', alpha=0.7, label=f'Wins ({len(wins)})', edgecolor='none')
    ax7.hist(loses, bins=25, color='#ff3d5a', alpha=0.7, label=f'Losses ({len(loses)})', edgecolor='none')
    ax7.axvline(0, color='#7d8590', linewidth=1)
    ax7.legend(fontsize=7, labelcolor='#7d8590', facecolor='#161b22', edgecolor='#21262d')
    ax_style(ax7, 'P&L Distribution — Trade Population')

    fig.suptitle('USDJPY Sharpe Validation — 7 Statistical Tests',
                 color='#e6edf3', fontsize=13, fontweight='bold', y=0.99)

    path = OUTPUT_DIR / "validation_charts.png"
    plt.savefig(path, dpi=150, bbox_inches='tight', facecolor='#0d1117')
    plt.close()
    print(f"\n  Charts saved: {path}")
    return path

# ═══════════════════════════════════════════════════════════════════════════════
# MAIN
# ═══════════════════════════════════════════════════════════════════════════════
def main():
    print("=" * 65)
    print("USDJPY SHARPE VALIDATION — 7 STATISTICAL TESTS")
    print("Methodology: identical to EURUSD validation (7/7 passed)")
    print("=" * 65)

    print("\nLoading USDJPY trades...")
    df = load_trades()

    print("\nRunning tests:")
    print("-" * 65)

    # Core metrics
    am = additional_metrics(df)
    print(f"\n  Base metrics:")
    print(f"    Sharpe:   {am['sharpe']}")
    print(f"    Sortino:  {am['sortino']}")
    print(f"    Calmar:   {am['calmar']}")
    print(f"    Recovery: {am['recovery']}×")
    print(f"    Win rate: {am['win_rate']}%")
    print()

    # Run all 7 tests
    r1, t1_randoms   = t1_signal_randomisation(df)
    r2, t2_bootstraps= t2_block_bootstrap(df)
    r3, df_yr        = t3_yearly_consistency(df)
    r4               = t4_walk_forward(df)
    r5               = t5_deflated_sharpe(df)
    r6               = t6_min_backtest_length(df)
    r7               = t7_newey_west(df)
    df_win, npos, ntot, pct_win = rolling_windows(df)

    results = [r1, r2, r3, r4, r5, r6, r7]
    n_pass  = sum(1 for r in results if r["pass"])

    print("\n" + "=" * 65)
    print("FINAL SCORECARD")
    print("=" * 65)
    for r in results:
        icon = "✅ PASS" if r["pass"] else "❌ FAIL"
        print(f"  {icon}  {r['test']}")
    print(f"\n  Score: {n_pass}/7 tests passed")
    print(f"  Rolling windows: {npos}/{ntot} positive ({pct_win:.1f}%)")

    # Build report
    lines = [
        "USDJPY SHARPE VALIDATION — RESULTS REPORT",
        f"Generated: {datetime.now().strftime('%Y-%m-%d %H:%M')}",
        f"Data: usdjpy_trades_real_costs.csv ({len(df)} trades, {df['year'].min()}–{df['year'].max()})",
        "=" * 65,
        "",
        "BASE METRICS",
        "-" * 65,
        f"  Sharpe ratio:     {am['sharpe']}",
        f"  Sortino ratio:    {am['sortino']}",
        f"  Calmar ratio:     {am['calmar']}",
        f"  Recovery factor:  {am['recovery']}×",
        f"  Win rate:         {am['win_rate']}%",
        f"  Max drawdown:     ${am['max_dd']:,.0f}",
        f"  t-statistic:      {am['t_stat']} (p={am['p_val']})",
        f"  Trades/year:      {am['trades_yr']}",
        f"  Total P&L:        ${am['total_pnl']:,.2f}",
        "",
        "TEST RESULTS",
        "-" * 65,
    ]
    for r in results:
        lines.append(f"\n  {'✅ PASS' if r['pass'] else '❌ FAIL'}  {r['test']}")
        lines.append(f"  {r['interpretation']}")

    lines += [
        "",
        "-" * 65,
        f"ROLLING 3-YEAR WINDOWS: {npos}/{ntot} positive ({pct_win:.1f}%)",
        "",
    ]
    for _, row in df_win.iterrows():
        lines.append(f"  {row['window']}  {row['trades']:3d} trades  "
                     f"WR {row['win_rate']:.1f}%  "
                     f"Sharpe {row['sharpe']:+.3f}  "
                     f"P&L ${row['pnl']:+,.0f}  "
                     f"{'✅' if row['positive'] else '❌'}")

    lines += [
        "",
        "=" * 65,
        f"VERDICT: {n_pass}/7 tests passed",
        "",
    ]

    if n_pass == 7:
        lines += [
            "CONCLUSION: Sharpe ratio is STATISTICALLY ROBUST.",
            "No evidence of overfitting. The USDJPY edge is genuine.",
            "All 7 tests identical to EURUSD validation methodology.",
            "",
            "The USDJPY model parameters are confirmed optimal.",
            "Paper trading may proceed. Parameters remain LOCKED.",
        ]
    elif n_pass >= 5:
        lines += [
            "CONCLUSION: Sharpe is LIKELY GENUINE but with caveats.",
            f"{7-n_pass} test(s) failed — review individual results above.",
        ]
    else:
        lines += [
            "CONCLUSION: INSUFFICIENT EVIDENCE. Review model.",
        ]

    lines.append("\n" + "=" * 65)

    report = "\n".join(lines)
    print("\n" + report)

    report_path = OUTPUT_DIR / "validation_results.txt"
    report_path.write_text(report, encoding="utf-8")
    print(f"\n  Report saved: {report_path}")

    build_charts(df, t1_randoms, t2_bootstraps, df_yr, df_win)
    print("\n  Done.")

if __name__ == "__main__":
    main()