#!/usr/bin/env python3
"""HUYLE STQD v2 experiment runner (one classifier seed, full + one ablation).

All stdout/stderr is captured in v2.log by run_v2.sh. No experiment-specific log files.
The recon decoder and preprocessing are reused from experiments/decoder_lodo.
Run from experiments/huyle/v2, using the parent repo's .venv.
"""
from __future__ import annotations
import argparse
import csv
from datetime import datetime
import json
import os
from pathlib import Path
import subprocess
import sys

HERE=Path(__file__).resolve().parent
EXPERIMENTS=HERE.parents[1]                  # .../experiments
ROOT=HERE.parents[2]                         # project root
BASE=EXPERIMENTS/'decoder_lodo'
EXTERNAL=EXPERIMENTS/'external_data'
PY=sys.executable
DATASETS=('AVGC','DTU','KUL','UHD','USTC')
DEC_BASE='vlaai_ws__eaP_n5_t60_aug_cluster_onset_dom_xFUG_ptSPK'
DEFAULT_VARIANTS=('v2_full','v2_cosine')
VARIANTS=('v2_full','v2_cosine','v2_no_time_bias','v2_cbramod_full','v2_cbramod_cosine')


def dec_names(n):
    return [DEC_BASE+(f'_s{i}' if i else '') for i in range(n)]


def clf_name(v,seed=0,tag=''):
    return f'direct_clf__eaP_n5_t60_{v}{tag}_s{seed}'


def run(cmd,env,label,dry=False):
    cmd=[str(z) for z in cmd]
    print(f'\n===== {datetime.now().isoformat(timespec="seconds")} | {label} =====',flush=True)
    import shlex
    print('$ '+shlex.join(cmd),flush=True)
    if dry:return
    result=subprocess.run(cmd,cwd=HERE,env=env,stdout=None,stderr=None,check=False)
    if result.returncode:
        raise RuntimeError(f'{label} failed (exit={result.returncode}). See v2.log; rerunning will skip completed folds/evidence.')
    print(f'===== COMPLETED {label} =====',flush=True)


def needed_paths(do_pretrain=True, do_external=True, ndec=4):
    req=[ROOT/'reports/eda_full/scripts/common.py',
         ROOT/'baselines/python/SuperHugeAAD',
         BASE/'cache/eaP_n5_t60',
         BASE/'cache/glimpseonorm',
         EXTERNAL/'Fuglsang2020/cache/eaP_n5_t60',
         EXTERNAL/'Fuglsang2020/cache/glimpseonorm']
    if do_pretrain:
        req += [EXTERNAL/'SparrKULee/cache/mat_reac',EXTERNAL/'SparrKULee/cache/glimpseonorm']
    if do_external:
        for ds in ('AASD','Carta2026'):
            req += [EXTERNAL/ds/'cache/eaP_n5_t60', EXTERNAL/ds/'cache/index.csv']
    missing=[str(p) for p in req if not p.exists()]
    for name in dec_names(ndec):
        for ds in DATASETS:
            if not (BASE/'folds'/name/f'{ds}.pkl').exists() and not (HERE/'folds'/name/f'{ds}.pkl').exists():
                missing.append(f'Legacy reconstruction fold: {name}/{ds}.pkl')
    return missing


def external_recon_paths(seed):
    return [f'aasd_c1_best_evidence_s{seed}.pkl',f'ext_c1_Carta2026_best_evidence_s{seed}.pkl']


def recon_external_ready(ndec):
    missing=[]
    for seed in range(ndec):
        for name in external_recon_paths(seed):
            if not any((root/'results/research'/name).exists() for root in (HERE,BASE)):
                missing.append(name)
    return missing


def fusion_done(csvname,experiment,minimum):
    path=HERE/'results/research'/csvname
    if not path.exists(): return False
    try:
        with path.open(newline='') as file:
            return sum(row.get('name')==experiment for row in csv.DictReader(file))>=minimum
    except (OSError,ValueError): return False


def summarize(variants,run_tag):
    rows=[]
    for variant in variants:
        name=clf_name(variant,tag=run_tag)
        path=HERE/'results'/f'{name}.json'
        if not path.exists():continue
        try:
            obj=json.loads(path.read_text())
            mean=obj['causal']['mean']
            row=dict(variant=variant,seed=0,experiment=name,
                     H=mean.get('H'),Acc=mean.get('Acc'),SDT=mean.get('SDT'),L=mean.get('L'))
            row.update({f'H_{ds}':obj['causal']['per_dataset'][ds]['H'] for ds in DATASETS})
            rows.append(row)
        except Exception as e:
            print(f'[WARN] Cannot summarize {path}: {e}',flush=True)
    if rows:
        dest=HERE/'results/v2_classifier_summary.csv'
        with dest.open('w',newline='') as f:
            w=csv.DictWriter(f,fieldnames=list(rows[0]));w.writeheader();w.writerows(rows)
        print(f'\nCLASSIFIER-ONLY LODO SUMMARY ({dest})',flush=True)
        for r in rows: print(f"  {r['variant']:19s} H={r['H']} Acc={r['Acc']} SDT={r['SDT']}",flush=True)
    for filename,output in [('fuse_general.csv','v2_lodo_fusion_summary.csv'),('fuse_external.csv','v2_real_switch_summary.csv')]:
        path=HERE/'results/research'/filename
        if path.exists():
            try:
                with path.open(newline='') as f:
                    data=[r for r in csv.DictReader(f) if r.get('name') in {f'huyle_{v}{run_tag}' for v in variants}]
                if data:
                    with (HERE/'results'/output).open('w',newline='') as f:
                        w=csv.DictWriter(f,fieldnames=list(data[0]));w.writeheader();w.writerows(data)
                    print(f'SUMMARY: wrote results/{output} ({len(data)} rows)',flush=True)
            except Exception as e:print(f'[WARN] Could not summarize {path}: {e}',flush=True)
    return rows


def main():
    ap=argparse.ArgumentParser(description=__doc__)
    ap.add_argument('--variants',default=','.join(DEFAULT_VARIANTS),help='default v2_full,v2_cosine; optional v2_no_time_bias')
    ap.add_argument('--seed',type=int,default=0,help='Only one classifier seed per run; default 0')
    ap.add_argument('--decoder-seeds',type=int,default=4,help='Reuse n existing decoder seeds (unchanged)')
    ap.add_argument('--pretrain-epochs',type=int,default=80)
    ap.add_argument('--fine-epochs',type=int,default=40)
    ap.add_argument('--microbatch',type=int,default=16,help='GPU activation microbatch; effective batch remains 64')
    ap.add_argument('--pretrain-mode',choices=['all','none'],default='all')
    ap.add_argument('--run-tag',default='',help='Optional suffix, e.g. _scratch; must be unique for changed configs')
    ap.add_argument('--skip-lodo',action='store_true')
    ap.add_argument('--skip-fusion',action='store_true')
    ap.add_argument('--skip-external',action='store_true')
    ap.add_argument('--allow-rebuild-decoder-external',action='store_true',help='Not default: permits training unchanged decoder all-five if missing')
    ap.add_argument('--force-fusion',action='store_true',help='Re-score even when CSV rows exist (duplicates appended)')
    ap.add_argument('--dry-run',action='store_true')
    ap.add_argument('--no-preflight',action='store_true',help='For exploratory dry-runs only')
    a=ap.parse_args()
    variants=[v.strip() for v in a.variants.split(',') if v.strip()]
    if not variants or len(set(variants)) != len(variants) or any(v not in VARIANTS for v in variants):
        ap.error(f'--variants must list unique values from {VARIANTS}')
    if a.decoder_seeds < 1:ap.error('--decoder-seeds must be positive')
    if a.microbatch < 1:ap.error('--microbatch must be positive')
    if a.pretrain_epochs < 25 and a.pretrain_mode=='all':ap.error('pretrain-epochs must be >=25')
    if a.fine_epochs < 10:ap.error('fine-epochs must be >=10')
    if a.run_tag and (not a.run_tag.startswith('_') or not a.run_tag[1:].replace('_','').isalnum()):
        ap.error('--run-tag must begin with _ and have alphanumeric/underscore characters')
    if a.pretrain_mode=='none' and not a.run_tag:a.run_tag='_scratch'
    for d in ('results','results/research','folds','models'):(HERE/d).mkdir(parents=True,exist_ok=True)
    print(f'HUYLE STQD v2 | path={HERE}\nConfiguration: {vars(a)}',flush=True)
    print('Fixed input: EEG [B,320,58], speech [B,320,2,3]; VLAAI EEG encoder; one seed; no leakage audit in this exploratory suite.',flush=True)
    print('Decoder reference: '+', '.join(dec_names(a.decoder_seeds)),flush=True)
    print('Log file: v2.log (captured exclusively by run_v2.sh)',flush=True)
    missing=[] if a.no_preflight else needed_paths(a.pretrain_mode=='all',not a.skip_external,a.decoder_seeds)
    if missing:
        print('\nPREFLIGHT MISSING:\n'+'\n'.join(f' - {e}' for e in missing[:80]),flush=True)
        if not a.dry_run:
            raise RuntimeError('Missing required datasets/dependencies. See preflight findings above.')
    if not a.skip_external and not a.allow_rebuild_decoder_external:
        dec_missing=recon_external_ready(a.decoder_seeds)
        if dec_missing:
            print('\nMISSING EXISTING EXTERNAL DECODER EVIDENCE:\n'+'\n'.join(dec_missing),flush=True)
            if not a.dry_run:
                raise RuntimeError('Run unchanged decoder external evidence separately, or use --allow-rebuild-decoder-external')

    env=dict(os.environ,PYTHONUNBUFFERED='1',OMP_NUM_THREADS='4',SCORE_JOBS='2')
    dnames=dec_names(a.decoder_seeds)
    for variant in variants:
        print(f'\n########## VARIANT {variant} seed {a.seed} ##########',flush=True)
        name=clf_name(variant,a.seed,a.run_tag)
        pre=HERE/'models'/f'pretrain_spk_direct_clf_{variant}{a.run_tag}_s{a.seed}.pt'
        if a.pretrain_mode=='all' and not pre.exists():
            cmd=[PY,'direct_classifier.py','--variant',variant,'--window-s','10','--pretrain-spk',
                 '--tag',f'_{variant}{a.run_tag}','--train-seed',str(a.seed),
                 '--min-epochs','25','--microbatch',str(a.microbatch),'--max-epochs',str(a.pretrain_epochs),'--set','model_blocks=3']
            run(cmd,dict(env,GLIMPSE_THR='onorm'),f'SparrKULee pretrain | {variant}',a.dry_run)
        elif a.pretrain_mode=='all':print(f'SKIP pretrained checkpoint {pre}',flush=True)
        if not a.skip_lodo:
            folds=[HERE/'folds'/name/f'{ds}.pkl' for ds in DATASETS]
            result=HERE/'results'/f'{name}.json'
            if not (all(p.exists() for p in folds) and result.exists()):
                cmd=[PY,'direct_classifier.py','--variant',variant,'--prep','eaP_n5_t60','--window-s','10',
                     '--train-seed',str(a.seed),'--min-epochs','10','--microbatch',str(a.microbatch),'--max-epochs',str(a.fine_epochs),
                     '--p-switch','0.5','--aug-set','clf_p_switch2=0.5','--aug-variant','cluster',
                     '--tag',f'_{variant}{a.run_tag}_s{a.seed}']
                if a.pretrain_mode=='all':cmd+=['--init-from',pre]
                run(cmd,dict(env,GLIMPSE_THR='onorm'),f'LODO five folds | {variant}',a.dry_run)
            else:print(f'SKIP completed LODO {name}',flush=True)
        if not a.skip_fusion and (a.force_fusion or not fusion_done('fuse_general.csv',f'huyle_{variant}{a.run_tag}',5*5*3)):
            cmd=[PY,'fuse_general.py','--dec',*dnames,'--clf',name,
                 '--a','0.0','0.3','0.5','0.7','1.0','--pstay','0.99','0.998','0.9997',
                 '--name',f'huyle_{variant}{a.run_tag}']
            run(cmd,env,f'LODO fusion | {variant}',a.dry_run)
        if not a.dry_run:summarize(variants,a.run_tag)

    if not a.skip_external:
        for seed in range(a.decoder_seeds):
            for file in external_recon_paths(seed):
                if not any((root/'results/research'/file).exists() for root in (HERE,BASE)):
                    if not a.allow_rebuild_decoder_external and not a.dry_run:
                        raise RuntimeError('Unexpected missing decoder external evidence: '+file)
                    cmd=[PY,'eval_external.py','--seed',str(seed),'--dataset','AASD,Carta2026',
                         '--prep','eaP_n5_t60','--targets','env,onset,dom',
                         '--extra-train','Fuglsang2020',
                         '--init-from',BASE/'models'/f'pretrain_spk_vlaai_ws_s{seed}.pt',
                         '--evidence-weights','lda','--tag','_best','--ref',dnames[seed]]
                    run(cmd,dict(env,GLIMPSE_THR='norm'),f'EXTERNAL unchanged reconstruction | decoder seed={seed}',a.dry_run)
                    break
        for variant in variants:
            name=clf_name(variant,a.seed,a.run_tag)
            tag=f'_{variant}{a.run_tag}'
            files=[HERE/'results/research'/f'ext_clf_{ds}{tag}_evidence_s{a.seed}.pkl' for ds in ('AASD','Carta2026')]
            if not all(p.exists() for p in files):
                cmd=[PY,'eval_external_clf.py','--seed',str(a.seed),'--variant',variant,
                     '--window-s','10','--prep','eaP_n5_t60','--dataset','AASD,Carta2026',
                     '--tag',tag,'--ref',name,'--min-epochs','10','--microbatch',str(a.microbatch),
                     '--set',f'max_epochs={a.fine_epochs}',
                     '--p-switch','0.5','--aug-set','clf_p_switch2=0.5']
                if a.pretrain_mode=='all':
                    cmd+=['--init-from',HERE/'models'/f'pretrain_spk_direct_clf_{variant}{a.run_tag}_s{a.seed}.pt']
                run(cmd,dict(env,GLIMPSE_THR='onorm'),f'All-five train + REAL switch evidence | {variant}',a.dry_run)
            else:print(f'SKIP cached external evidence {variant}',flush=True)
            if not a.skip_fusion and (a.force_fusion or not fusion_done('fuse_external.csv',f'huyle_{variant}{a.run_tag}',2*5*4)):
                cmd=[PY,'fuse_external.py','--dec',*[f'{d}:{i}' for i,d in enumerate(dnames)],
                     '--clf',f'{name}:{tag}:{a.seed}',
                     '--dataset','AASD,Carta2026','--dec-tag','_best',
                     '--a','0.0','0.3','0.5','0.7','1.0',
                     '--pstay','0.95','0.99','0.998','0.9997',
                     '--name',f'huyle_{variant}{a.run_tag}']
                run(cmd,env,f'AASD/Carta fusion | {variant}',a.dry_run)
    if not a.dry_run:
        summarize(variants,a.run_tag)
        print('\nCOMPLETE; inspect v2.log and results/v2_*_summary.csv',flush=True)
    else:print('\nDRY RUN complete (no training was performed).',flush=True)

if __name__=='__main__':main()
