"""
watchdog_v1.py
==============
Process watchdog for the Dual FX Macro Model live monitor.

Checks every 5 minutes:
  1. live_signal_monitor_v2.py is running
  2. dashboard_server.py is running
  3. State files were updated within the last 2 hours (detects hung/frozen process)
  4. Rate CSVs updated within the last 36 hours (flags data pipeline failure)

If monitor or dashboard are down:
  - Restarts automatically
  - Sends Telegram alert with context (crash vs first start)
  - Logs everything to data/logs/watchdog.log

If state files are stale (process running but frozen):
  - Kills and restarts the monitor
  - Sends Telegram alert

Run from project root as a separate Task Scheduler job:
  Program:   C:\\Users\\Administrator\\AppData\\Local\\Programs\\Python\\Python314\\python.exe
  Arguments: src\\execution\\watchdog_v1.py
  Start in:  C:\\Users\\Administrator\\OneDrive\\fx_macro_intraday
  Trigger:   At system startup
"""

import os
import sys
import time
import json
import logging
import subprocess
import urllib.request
import urllib.parse
from pathlib import Path
from datetime import datetime, timezone, timedelta

# ── Configuration ─────────────────────────────────────────────────────────────
BASE_PATH    = Path(__file__).resolve().parents[2]
LOG_DIR      = BASE_PATH / 'data' / 'logs'
LOG_DIR.mkdir(parents=True, exist_ok=True)

PYTHON_EXE   = sys.executable   # use same Python that's running this script

MONITOR_SCRIPT   = str(BASE_PATH / 'src' / 'execution' / 'live_signal_monitor_v2.py')
DASHBOARD_SCRIPT = str(BASE_PATH / 'src' / 'execution' / 'dashboard_server.py')

STATE_FILES = [
    BASE_PATH / 'data' / 'logs' / 'state_eurusd.json',
    BASE_PATH / 'data' / 'logs' / 'state_usdjpy.json',
]

RATE_FILES = [
    BASE_PATH / 'data' / 'raw' / 'rates' / 'us2y.csv',
    BASE_PATH / 'data' / 'raw' / 'rates' / 'de2y.csv',
    BASE_PATH / 'data' / 'raw' / 'rates' / 'jp2y.csv',
]

# Telegram — reads from same state file approach as monitor
TELEGRAM_TOKEN  = None
TELEGRAM_CHAT   = None

CHECK_INTERVAL_SEC  = 300   # 5 minutes
STATE_STALE_HOURS   = 2     # state file not updated → process likely hung
RATE_STALE_HOURS    = 36    # rate data not updated → pipeline failure

# ── Logging ───────────────────────────────────────────────────────────────────
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s [WATCHDOG] %(message)s',
    handlers=[
        logging.FileHandler(LOG_DIR / 'watchdog.log', encoding='utf-8'),
        logging.StreamHandler(sys.stdout),
    ]
)
log = logging.getLogger('watchdog')

# ── Load Telegram credentials from monitor script ─────────────────────────────
def load_telegram_creds():
    global TELEGRAM_TOKEN, TELEGRAM_CHAT
    try:
        monitor_src = Path(MONITOR_SCRIPT).read_text(encoding='utf-8')
        import re
        tok_m  = re.search(r'TELEGRAM_TOKEN\s*=\s*["\']([^"\']+)["\']', monitor_src)
        chat_m = re.search(r'TELEGRAM_CHAT_ID\s*=\s*["\']([^"\']+)["\']', monitor_src)
        if tok_m:  TELEGRAM_TOKEN = tok_m.group(1)
        if chat_m: TELEGRAM_CHAT  = chat_m.group(1)
        if TELEGRAM_TOKEN and TELEGRAM_CHAT:
            log.info("Telegram credentials loaded from monitor script")
        else:
            log.warning("Could not extract Telegram credentials — alerts disabled")
    except Exception as e:
        log.warning(f"Failed to load Telegram creds: {e}")

def send_telegram(msg):
    if not TELEGRAM_TOKEN or not TELEGRAM_CHAT:
        return
    try:
        url  = f"https://api.telegram.org/bot{TELEGRAM_TOKEN}/sendMessage"
        data = urllib.parse.urlencode({
            'chat_id': TELEGRAM_CHAT,
            'text': msg,
            'parse_mode': 'HTML'
        }).encode()
        req  = urllib.request.Request(url, data=data)
        urllib.request.urlopen(req, timeout=10)
    except Exception as e:
        log.warning(f"Telegram send failed: {e}")

# ── Process management ────────────────────────────────────────────────────────
def is_running(script_name):
    """Check if a Python script is running by script filename."""
    try:
        import psutil
        for proc in psutil.process_iter(['pid','name','cmdline']):
            try:
                cmd = proc.info.get('cmdline') or []
                if any(script_name in str(arg) for arg in cmd):
                    return proc.info['pid']
            except (psutil.NoSuchProcess, psutil.AccessDenied):
                continue
    except ImportError:
        # psutil not available — use tasklist fallback (Windows)
        try:
            result = subprocess.run(
                ['tasklist', '/FO', 'CSV', '/NH'],
                capture_output=True, text=True, timeout=10
            )
            # Can't distinguish script names from tasklist alone
            # Fall back to checking via WMIC
            result2 = subprocess.run(
                ['wmic', 'process', 'get', 'ProcessId,CommandLine', '/FORMAT:CSV'],
                capture_output=True, text=True, timeout=15
            )
            script_base = Path(script_name).name
            for line in result2.stdout.splitlines():
                if script_base in line:
                    parts = line.split(',')
                    try:
                        return int(parts[-1].strip())
                    except:
                        return -1  # running but PID unknown
        except Exception:
            pass
    return None

def kill_process(pid):
    try:
        import psutil
        proc = psutil.Process(pid)
        proc.terminate()
        proc.wait(timeout=10)
        log.info(f"Killed process PID {pid}")
        return True
    except Exception as e:
        log.warning(f"Failed to kill PID {pid}: {e}")
        return False

def start_process(script_path, label):
    try:
        proc = subprocess.Popen(
            [PYTHON_EXE, script_path],
            cwd=str(BASE_PATH),
            creationflags=subprocess.CREATE_NEW_CONSOLE if sys.platform == 'win32' else 0
        )
        log.info(f"Started {label} — PID {proc.pid}")
        return proc.pid
    except Exception as e:
        log.error(f"Failed to start {label}: {e}")
        return None

# ── Health checks ─────────────────────────────────────────────────────────────
def check_state_freshness():
    """Returns list of stale state files (not updated within STATE_STALE_HOURS)."""
    stale = []
    now   = datetime.now(timezone.utc)
    for sf in STATE_FILES:
        if not sf.exists():
            continue  # file not yet created — monitor may not have run yet
        mtime = datetime.fromtimestamp(sf.stat().st_mtime, tz=timezone.utc)
        age_h = (now - mtime).total_seconds() / 3600
        if age_h > STATE_STALE_HOURS:
            stale.append((sf.name, age_h))
    return stale

def check_rate_freshness():
    """Returns list of stale rate files (not updated within RATE_STALE_HOURS)."""
    stale = []
    now   = datetime.now(timezone.utc)
    for rf in RATE_FILES:
        if not rf.exists():
            stale.append((rf.name, 999))
            continue
        mtime = datetime.fromtimestamp(rf.stat().st_mtime, tz=timezone.utc)
        age_h = (now - mtime).total_seconds() / 3600
        if age_h > RATE_STALE_HOURS:
            stale.append((rf.name, age_h))
    return stale

def check_open_positions():
    """Check if either pair has an open position in state files."""
    open_pos = []
    for sf in STATE_FILES:
        if not sf.exists():
            continue
        try:
            state = json.loads(sf.read_text(encoding='utf-8'))
            if state.get('position_open'):
                pair  = sf.stem.replace('state_','').upper()
                entry = state.get('entry_price','?')
                open_pos.append((pair, entry))
        except Exception:
            pass
    return open_pos

# ── Main watchdog loop ────────────────────────────────────────────────────────
def run_watchdog():
    log.info("=" * 60)
    log.info("Watchdog started")
    log.info(f"Monitor script:   {MONITOR_SCRIPT}")
    log.info(f"Dashboard script: {DASHBOARD_SCRIPT}")
    log.info(f"Check interval:   {CHECK_INTERVAL_SEC}s")
    log.info("=" * 60)

    load_telegram_creds()

    send_telegram(
        "🐕 <b>WATCHDOG STARTED</b>\n"
        f"Monitoring: live_signal_monitor_v2 + dashboard_server\n"
        f"Check interval: {CHECK_INTERVAL_SEC//60} minutes\n"
        f"State stale threshold: {STATE_STALE_HOURS}h\n"
        f"Rate stale threshold: {RATE_STALE_HOURS}h"
    )

    restart_counts = {'monitor': 0, 'dashboard': 0}
    last_rate_alert = None

    while True:
        try:
            now_str = datetime.now().strftime('%Y-%m-%d %H:%M:%S')

            # ── 1. Check monitor ──────────────────────────────────────────────
            monitor_pid = is_running('live_signal_monitor_v2')
            if not monitor_pid:
                log.warning("Monitor NOT running — restarting")
                open_pos = check_open_positions()

                new_pid = start_process(MONITOR_SCRIPT, 'live_signal_monitor_v2')
                restart_counts['monitor'] += 1

                alert = (
                    f"⚠️ <b>WATCHDOG: MONITOR RESTARTED</b>\n"
                    f"Time: {now_str}\n"
                    f"Restart #{restart_counts['monitor']}\n"
                )
                if open_pos:
                    pos_str = ' | '.join(f"{p} @ {e}" for p,e in open_pos)
                    alert += f"\n🔴 <b>OPEN POSITIONS AT TIME OF CRASH:</b>\n{pos_str}\n"
                    alert += "\nCheck MT5 immediately — verify positions are managed."
                else:
                    alert += "\nNo open positions at time of crash."
                if new_pid:
                    alert += f"\nRestarted successfully (PID {new_pid})"
                else:
                    alert += "\n❌ RESTART FAILED — manual intervention required"

                send_telegram(alert)
                log.info(f"Monitor restart #{restart_counts['monitor']} complete")

            else:
                log.info(f"Monitor running ✅ PID {monitor_pid}")

                # ── 2. Check for hung/frozen monitor (state file stale) ───────
                stale_states = check_state_freshness()
                if stale_states:
                    stale_str = ', '.join(f"{n} ({h:.1f}h ago)" for n,h in stale_states)
                    log.warning(f"State files stale: {stale_str} — process may be hung")

                    open_pos = check_open_positions()
                    log.warning("Killing and restarting frozen monitor")
                    kill_process(monitor_pid)
                    time.sleep(3)
                    new_pid = start_process(MONITOR_SCRIPT, 'live_signal_monitor_v2')
                    restart_counts['monitor'] += 1

                    alert = (
                        f"⚠️ <b>WATCHDOG: MONITOR FROZEN — RESTARTED</b>\n"
                        f"Stale state files: {stale_str}\n"
                        f"Restart #{restart_counts['monitor']}\n"
                    )
                    if open_pos:
                        pos_str = ' | '.join(f"{p} @ {e}" for p,e in open_pos)
                        alert += f"\n🔴 Open positions: {pos_str}\nCheck MT5 immediately."
                    send_telegram(alert)

            # ── 3. Check dashboard ────────────────────────────────────────────
            dash_pid = is_running('dashboard_server')
            if not dash_pid:
                log.warning("Dashboard NOT running — restarting")
                new_pid = start_process(DASHBOARD_SCRIPT, 'dashboard_server')
                restart_counts['dashboard'] += 1
                send_telegram(
                    f"ℹ️ <b>WATCHDOG: DASHBOARD RESTARTED</b>\n"
                    f"Restart #{restart_counts['dashboard']}\n"
                    f"{'Restarted OK' if new_pid else '❌ Restart failed'}"
                )
            else:
                log.info(f"Dashboard running ✅ PID {dash_pid}")

            # ── 4. Check rate data freshness (once per hour max) ──────────────
            stale_rates = check_rate_freshness()
            if stale_rates:
                now_utc = datetime.now(timezone.utc)
                if last_rate_alert is None or (now_utc - last_rate_alert).total_seconds() > 3600:
                    stale_str = ', '.join(f"{n} ({h:.0f}h ago)" for n,h in stale_rates)
                    log.warning(f"Stale rate files: {stale_str}")
                    send_telegram(
                        f"⚠️ <b>WATCHDOG: RATE DATA STALE</b>\n"
                        f"Files: {stale_str}\n"
                        f"FinanceFlow pipeline may have failed.\n"
                        f"Signals will continue using last known rates — monitor closely."
                    )
                    last_rate_alert = now_utc

            # ── 5. Hourly status summary ──────────────────────────────────────
            current_minute = datetime.now().minute
            if current_minute < (CHECK_INTERVAL_SEC // 60):
                open_pos = check_open_positions()
                pos_str  = (', '.join(f"{p}@{e}" for p,e in open_pos)
                             if open_pos else 'None')
                log.info(
                    f"Health OK | Monitor PID {monitor_pid} | "
                    f"Dashboard PID {dash_pid} | Open: {pos_str}"
                )

        except Exception as e:
            log.error(f"Watchdog loop error: {e}")
            import traceback; traceback.print_exc()

        time.sleep(CHECK_INTERVAL_SEC)

if __name__ == '__main__':
    run_watchdog()