"""
fetch_jp2y.py
==============
Downloads Japan JGB yield curve daily data from the Ministry of Finance.
Completely free, no API key needed.

Source: https://www.mof.go.jp/english/policy/jgbs/reference/interest_rate/
Data:   Daily from 1974, includes 2Y, 5Y, 10Y maturities

Run from project root:
  python src/ingestion/fetch_jp2y.py

Output:
  data/raw/rates/jp2y.csv
"""

import pandas as pd
import urllib.request
from pathlib import Path
import ssl

BASE_PATH   = Path(__file__).resolve().parents[2]
OUTPUT_PATH = BASE_PATH / "data" / "raw" / "rates" / "jp2y.csv"

MOF_URL = "https://www.mof.go.jp/english/policy/jgbs/reference/interest_rate/historical/jgbcme_all.csv"


def main():
    print("Downloading Japan JGB yield curve from Ministry of Finance...")
    print(f"Source: {MOF_URL}")
    print(f"Output: {OUTPUT_PATH}")

    # Download — disable SSL verify (MoF cert sometimes causes issues)
    ctx = ssl.create_default_context()
    ctx.check_hostname = False
    ctx.verify_mode    = ssl.CERT_NONE

    try:
        req  = urllib.request.Request(MOF_URL,
               headers={"User-Agent": "Mozilla/5.0"})
        resp = urllib.request.urlopen(req, context=ctx, timeout=30)
        raw  = resp.read().decode("utf-8", errors="replace")
    except Exception as e:
        print(f"Download failed: {e}")
        return

    # Parse — MoF CSV has a header row then date rows
    # Format: Date, 2Y, 3Y, 4Y, 5Y, 6Y, 7Y, 8Y, 9Y, 10Y, 15Y, 20Y, 25Y, 30Y, 40Y
    from io import StringIO
    df = pd.read_csv(StringIO(raw), header=0)
    print(f"\nRaw columns: {list(df.columns[:6])}...")
    print(f"Raw shape: {df.shape}")
    print(df.head(3).to_string())

    # MoF CSV has an extra header row at row 0 with actual column names
    # Row 0: Date, 1Y, 2Y, 3Y, 4Y, 5Y, 6Y, 7Y, 8Y, 9Y, 10Y, 15Y, 20Y, 25Y, 30Y, 40Y
    # Use row 0 as column names then drop it
    df.columns = df.iloc[0].tolist()
    df = df.iloc[1:].reset_index(drop=True)
    print(f"\nActual columns: {list(df.columns[:8])}...")

    date_col = df.columns[0]  # "Date"
    two_yr   = "2Y"
    if two_yr not in df.columns:
        # Fallback — find column containing "2"
        two_yr = next((c for c in df.columns[1:] if str(c).strip() == "2Y"), df.columns[2])
    print(f"Date column: {date_col}")
    print(f"2Y column:   {two_yr}")

    # Build clean output
    out = pd.DataFrame({
        "date": pd.to_datetime(df[date_col], errors="coerce"),
        "jp2y": pd.to_numeric(df[two_yr], errors="coerce"),
    }).dropna().sort_values("date")

    # Filter to model period (2003+)
    out = out[out["date"] >= "2003-01-01"].copy()
    out["date"] = out["date"].dt.strftime("%Y-%m-%d")

    OUTPUT_PATH.parent.mkdir(parents=True, exist_ok=True)
    out.to_csv(OUTPUT_PATH, index=False)

    print(f"\nSaved {len(out):,} rows to {OUTPUT_PATH.name}")
    print(f"Date range: {out['date'].iloc[0]} → {out['date'].iloc[-1]}")
    print(f"Yield range: {out['jp2y'].min():.3f}% → {out['jp2y'].max():.3f}%")
    print(f"\nDone — no API requests used, completely free")


if __name__ == "__main__":
    main()