#!/usr/bin/env python3
"""
build_forecast.py — writes forecast_data.json for the Daily Forecasts page.

ENGINE: replica of vol_range_forecast.py v3 (Parkinson daily variance,
EWMA lambda=0.94, multipliers = median/75th pct of range%/sigma over the
trailing 150 same-weekday sessions, event-conditioned on
FOMC/NFP/USCPI/ECB/BOE days when >=24 samples exist).

Deviations from the standalone tool, both for the rollover build time:
  * SESSION DATING — the forecast session rolls at 17:00 New York
    (= broker midnight). Built Tuesday 5pm NY, it forecasts Wednesday.
  * COMPLETED BARS ONLY — all statistics use completed daily bars,
    exactly matching the OOS validation. The forming bar (nearly empty
    at rollover) contributes only today's OPEN for level anchoring; if
    it hasn't printed yet (Gold/NQ open 18:00 NY), levels anchor to the
    prior close and the server retries until the real open exists.

v3 of this builder also ships M15/H1/H4/D1 candle sets per instrument
so the page can switch timeframes. Statistics remain D1-only.

Save to: src\\execution\\build_forecast.py
Run manually:   python src\\execution\\build_forecast.py
Normally invoked by dashboard_server.py just after the daily rollover.
Requires the MT5 terminal running on this machine.
"""
import json
import math
import os
import statistics
import sys
from datetime import datetime, timedelta, timezone
from pathlib import Path

try:
    import MetaTrader5 as mt5
except ImportError:
    sys.exit("MetaTrader5 package missing — run: python -m pip install MetaTrader5")

BASE_PATH = Path(__file__).resolve().parents[2]
OUT_JSON = BASE_PATH / "forecast_data.json"

# ── Config ────────────────────────────────────────────────────────────────
# (display name, [candidate MT5 symbol names — first that works wins])
# Delete a line to drop an instrument; add one to extend.
INSTRUMENTS = [
    ("GOLD",   ["XAUUSD", "GOLD", "XAUUSD.a", "XAUUSD.r"]),
    ("EURUSD", ["EURUSD", "EURUSD.a", "EURUSD.r"]),
    ("GBPUSD", ["GBPUSD", "GBPUSD.a", "GBPUSD.r"]),
    ("USDJPY", ["USDJPY", "USDJPY.a", "USDJPY.r"]),
    ("NQ",     ["USTEC", "NAS100", "US100", "USTEC.cash", "NAS100.cash",
                "US100.cash"]),
]
CHART_BARS_D1 = 130
INTRADAY_TFS = [            # (name, MT5 timeframe id set in main, bars)
    ("H4", 250),
    ("H1", 260),
    ("M15", 288),
]

# Events file: posrr's live copy first (it gets refreshed), local copy second.
EVENTS_CSV_CANDIDATES = [
    Path(r"C:\Users\Administrator\OneDrive\range_extension_model\src\research\live\calendar_events.csv"),
    BASE_PATH / "calendar_events.csv",
]

# ── vol_range_forecast v3 engine constants (keep in sync with v3) ────────
LAMBDA = 0.94
PULL = 1500
WD_ROLL = 150
EVENT_CLASSES = {
    "FOMC":  ("United States", "Fed Interest Rate Decision"),
    "NFP":   ("United States", "Payroll Jobs Growth"),
    "USCPI": ("United States", "Inflation Rate Year-over-Year"),
    "ECB":   ("Euro Area", "ECB Interest Rate Decision"),
    "BOE":   ("United Kingdom", "BoE Interest Rate Decision"),
}
PAIR_EVENT_CLASSES = {
    "EURUSD": ["FOMC", "ECB", "NFP", "USCPI"],
    "GBPUSD": ["FOMC", "BOE", "NFP", "USCPI"],
}
DEFAULT_EVENT_CLASSES = ["FOMC", "NFP", "USCPI"]


def session_date():
    """Date of the current FX session. Sessions roll at 17:00 New York
    (= broker midnight): after Tue 5pm NY the session is Wednesday, and
    Sunday's rollover opens Monday's session."""
    try:
        from zoneinfo import ZoneInfo
        ny = datetime.now(ZoneInfo("America/New_York"))
    except Exception:
        ny = datetime.now(timezone.utc) - timedelta(hours=4)
    return (ny + timedelta(hours=7)).date()


def load_events():
    """Same parsing as v3; returns ({class: set(dates)}, path_used or None)."""
    import csv
    path = next((p for p in EVENTS_CSV_CANDIDATES if p.exists()), None)
    if path is None:
        return {}, None
    out = {k: set() for k in EVENT_CLASSES}
    try:
        with open(path, encoding="utf-8", errors="replace") as f:
            for row in csv.DictReader(f):
                for cls, (cty, name) in EVENT_CLASSES.items():
                    if row.get("country") == cty and row.get("event") == name:
                        out[cls].add(str(row.get("date", ""))[:10])
        return out, path
    except Exception:
        return {}, path


def q(v, p):
    """Linear-interpolated quantile — identical to v3."""
    s = sorted(v)
    k = (len(s) - 1) * p
    f = int(k)
    return s[f] + (s[min(f + 1, len(s) - 1)] - s[f]) * (k - f)


def forecast_numbers(sym, canonical, target_wd, ev, today):
    """v3 forecast() math on COMPLETED bars, returning structured data.
    `sym` is the broker symbol; `canonical` picks the event-class map;
    `today` (session date, YYYY-MM-DD) selects event conditioning."""
    rates = mt5.copy_rates_from_pos(sym, mt5.TIMEFRAME_D1, 0, PULL)
    if rates is None or len(rates) < 400:
        return {"ok": False, "error": "insufficient D1 history"}

    o_all = [float(r['open']) for r in rates]
    h_all = [float(r['high']) for r in rates]
    l_all = [float(r['low']) for r in rates]
    c_all = [float(r['close']) for r in rates]
    ds_all = [datetime.fromtimestamp(r['time'], timezone.utc)
              .strftime("%Y-%m-%d") for r in rates]

    # Split off the forming bar (today's session) if it has printed.
    if ds_all[-1] == today:
        today_open = o_all[-1]
        anchor, anchor_src = today_open, "open"
        end = len(rates) - 1
    else:
        today_open = None
        anchor, anchor_src = c_all[-1], "prior_close"
        end = len(rates)
    if end < 400:
        return {"ok": False, "error": "insufficient completed D1 history"}

    o = o_all[:end]
    h = h_all[:end]
    l = l_all[:end]
    c = c_all[:end]
    ds = ds_all[:end]
    wd = [datetime.fromtimestamp(r['time'], timezone.utc).weekday()
          for r in rates[:end]]

    classes = [cls for cls in
               PAIR_EVENT_CLASSES.get(canonical, DEFAULT_EVENT_CLASSES)
               if ev and cls in ev]

    def day_class(dstr):
        for cls in classes:
            if dstr in ev[cls]:
                return cls
        return None

    park = [(math.log(h[i] / l[i]) ** 2) / (4 * math.log(2))
            for i in range(len(o))]
    var = statistics.fmean(park[1:31])
    sig = []
    for i in range(1, len(o)):
        sig.append(math.sqrt(var))        # sigma forecast FOR day i (info to i-1)
        var = LAMBDA * var + (1 - LAMBDA) * park[i]
    sig_next = math.sqrt(var)             # forecast for the coming session

    xr_w = {w: [] for w in range(7)}
    xo_w = {w: [] for w in range(7)}
    hist = {cls: [] for cls in classes}
    for i in range(1, len(o)):
        s = sig[i - 1]
        if s <= 0:
            continue
        pair = ((h[i] - l[i]) / o[i] / s, abs(c[i] - o[i]) / o[i] / s)
        cls = day_class(ds[i])
        if cls:
            hist[cls].append(pair)
        else:
            xr_w[wd[i]].append(pair[0])
            xo_w[wd[i]].append(pair[1])

    label = "weekday"
    tcls = day_class(today) if today else None
    if tcls and len(hist[tcls]) >= 24:
        pool = hist[tcls][-150:]
        label = tcls
        xr = [a for a, b in pool]
        xo = [b for a, b in pool]
    else:
        if tcls:
            label = f"{tcls} (thin history — weekday bands)"
        xr = xr_w.get(target_wd, [])[-WD_ROLL:]
        xo = xo_w.get(target_wd, [])[-WD_ROLL:]
        if len(xr) < 40:      # holiday/thin fallback: unconditional
            xr = [v for w in range(5) for v in xr_w[w]][-750:]
            xo = [v for w in range(5) for v in xo_w[w]][-750:]

    ann = sig_next * math.sqrt(252) * 100
    f50r, f75r = sig_next * q(xr, .5), sig_next * q(xr, .75)
    f50o, f75o = sig_next * q(xo, .5), sig_next * q(xo, .75)
    px = c[-1]

    # ── Extras for the page (don't affect the numbers above) ─────────────
    # Walk-forward coverage of the 75th H-L band over the last ~252
    # completed non-event sessions, weekday pools built exactly as above.
    xr_run = {w: [] for w in range(7)}
    inside = 0
    total = 0
    start_eval = max(1, len(o) - 252)
    for i in range(1, len(o)):
        s = sig[i - 1]
        if s <= 0:
            continue
        if day_class(ds[i]):
            continue                     # calibration = weekday bands only
        if i >= start_eval:
            pool = xr_run[wd[i]][-WD_ROLL:]
            if len(pool) >= 40:
                band = s * q(pool, .75)
                total += 1
                if (h[i] - l[i]) / o[i] <= band:
                    inside += 1
        xr_run[wd[i]].append((h[i] - l[i]) / o[i] / s)
    coverage = (
        {"pct": round(100.0 * inside / total, 1), "n": total}
        if total >= 60 else None
    )

    d1_candles = [
        {"t": ds_all[i], "o": o_all[i], "h": h_all[i],
         "l": l_all[i], "c": c_all[i]}
        for i in range(max(0, len(o_all) - CHART_BARS_D1), len(o_all))
    ]

    return {
        "ok": True,
        "label": label,
        "event_conditioned": (label in EVENT_CLASSES),
        "ann_vol": round(ann, 2),
        "hl_med": round(f50r * 100, 4),
        "hl_75": round(f75r * 100, 4),
        "oc_med": round(f50o * 100, 4),
        "oc_75": round(f75o * 100, 4),
        "ref_close": px,
        "today_open": today_open,
        "anchor": anchor,
        "anchor_src": anchor_src,
        "last_completed_bar": ds[-1],
        "pool_n": len(xr),
        "coverage": coverage,
        "candles": {"D1": d1_candles},
    }


def intraday_candles(sym, digits):
    """Pull M15/H1/H4 candle sets for timeframe switching on the page.
    Times are the raw MT5 epoch values (broker clock)."""
    tf_ids = {"H4": mt5.TIMEFRAME_H4, "H1": mt5.TIMEFRAME_H1,
              "M15": mt5.TIMEFRAME_M15}
    out = {}
    for name, bars in INTRADAY_TFS:
        rr = mt5.copy_rates_from_pos(sym, tf_ids[name], 0, bars)
        if rr is None or len(rr) == 0:
            out[name] = []
            continue
        out[name] = [
            {"t": int(r['time']),
             "o": round(float(r['open']), digits),
             "h": round(float(r['high']), digits),
             "l": round(float(r['low']), digits),
             "c": round(float(r['close']), digits)}
            for r in rr
        ]
    return out


def main():
    if not mt5.initialize():
        sys.exit(f"MT5 initialize failed: {mt5.last_error()}")
    try:
        sd = session_date()
        today = sd.isoformat()
        target_wd = sd.weekday()
        ev, ev_path = load_events()
        out = {
            "generated_utc":
                datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S"),
            "session_date": today,
            "session_label": f"{sd:%A, %B %d, %Y}",
            "engine": "vol_range_forecast v3 engine · completed-bar stats · "
                      "session rolls 17:00 New York",
            "events_csv": str(ev_path) if ev_path else None,
            "instruments": [],
        }
        print(f"VOL & RANGE FORECAST build — session {today} ({sd:%A})")
        if ev_path:
            print(f"  events: {ev_path}")
        else:
            print("  events: no calendar_events.csv found — weekday-only bands")
        for display, candidates in INSTRUMENTS:
            resolved = None
            for cand in candidates:
                if mt5.symbol_select(cand, True):
                    test = mt5.copy_rates_from_pos(cand, mt5.TIMEFRAME_D1, 0, 5)
                    if test is not None and len(test) > 0:
                        resolved = cand
                        break
            if resolved is None:
                out["instruments"].append({
                    "display": display, "ok": False,
                    "error": "no symbol found "
                             f"(tried {', '.join(candidates)})",
                })
                print(f"  {display:<8} FAIL  none of {candidates} available")
                continue
            info = mt5.symbol_info(resolved)
            digits = int(info.digits) if info else 5
            d = forecast_numbers(resolved, display, target_wd, ev, today)
            d.update({"display": display, "symbol": resolved,
                      "digits": digits})
            if d.get("ok"):
                d["candles"].update(intraday_candles(resolved, digits))
            out["instruments"].append(d)
            if d.get("ok"):
                tag = "open" if d["anchor_src"] == "open" else "PRIOR-CLOSE anchor"
                print(f"  {display:<8} OK    {resolved:<12} "
                      f"vol {d['ann_vol']:.2f}%  HL {d['hl_med']:.2f}/"
                      f"{d['hl_75']:.2f}  [{d['label']}] ({tag})")
            else:
                print(f"  {display:<8} FAIL  {d.get('error')}")
        out["anchor_pending"] = any(
            i.get("ok") and i.get("anchor_src") == "prior_close"
            for i in out["instruments"]
        )
        tmp = OUT_JSON.with_name(OUT_JSON.name + ".tmp")
        with open(tmp, "w", encoding="utf-8") as f:
            json.dump(out, f)
        os.replace(tmp, OUT_JSON)
        print(f"  wrote {OUT_JSON}"
              + ("  (some anchors pending — server will retry)"
                 if out["anchor_pending"] else ""))
    finally:
        mt5.shutdown()


if __name__ == "__main__":
    main()