from pathlib import Path
import json
import subprocess
import sys

import joblib
import numpy as np
import pandas as pd
import torch

from suite7_shared_private_moe import SharedPrivateResidualMoE
from suite7_common import build_historical_bundle, load_fold_caches, FeatureStandardizer


def _support(n,rng):
    hs=rng.integers(0,5,size=n)
    return pd.DataFrame({
        'history_semesters':hs,
        'history_rows':hs+rng.integers(0,4,size=n),
        'new_student':(hs==0).astype(float),
        'course_prior_rows':rng.integers(0,8,size=n),
        'new_course':rng.integers(0,2,size=n).astype(float),
    })


def _make_cache(path,faculty,fold,seed):
    rng=np.random.default_rng(seed);ntr,nte,d=72,24,3
    Xtr=rng.normal(size=(ntr,d));Xte=rng.normal(size=(nte,d))
    ytr=np.zeros(ntr,dtype=int);ytr[rng.choice(ntr,18,replace=False)]=1
    yte=np.zeros(nte,dtype=int);yte[rng.choice(nte,6,replace=False)]=1
    base_tr=np.clip(.12+.55*ytr+rng.normal(0,.12,ntr),.01,.99)
    cf=np.column_stack([np.clip(base_tr+rng.normal(0,.04,ntr),.01,.99) for _ in range(3)])
    base_te=np.clip(.12+.55*yte+rng.normal(0,.12,nte),.01,.99)
    pte=np.column_stack([np.clip(base_te+rng.normal(0,.04,nte),.01,.99) for _ in range(3)])
    cache={
        'feature_names':['f1','f2','f3'],'Xtr':Xtr,'Xte':Xte,'ytr':ytr,'yte':yte,
        'cf_covered_mask':np.ones(ntr,dtype=bool),'cf_probabilities':cf,
        'cf_block_ids':np.repeat([1.,2.,3.],24),
        'support_hist':_support(ntr,rng),'support_target':_support(nte,rng),
        'pte':pte,'qte':pte.mean(axis=1),'test_frame':pd.DataFrame({'dummy':np.arange(nte)}),
    }
    joblib.dump(cache,path/f'{faculty}_fold_{fold}.joblib');return cache


def _score_frame(y,q,shared,adapter,suite8):
    n=len(y);d={'actual':y,'committee':q,'residual_moe_shared':shared,
                'residual_moe_shared_private':adapter,'residual_moe_true_specialist':suite8,
                'correction_logit':np.zeros(n),'weight_null':np.full(n,.2)}
    for k in range(1,5):
        d[f'weight_expert_{k}']=np.full(n,.2);d[f'residual_expert_{k}']=np.zeros(n)
    return pd.DataFrame(d)


def test_suite9_end_to_end_small(tmp_path:Path):
    cache=tmp_path/'cache';shared=tmp_path/'shared';s7=tmp_path/'suite7';s8=tmp_path/'suite8';out=tmp_path/'out'
    for p in [cache,shared,s7,s8]:p.mkdir()
    faculties=['fac_a','fac_b'];common=['f1','f2','f3']
    for fold in [1,2,3]:
        for root in [shared,s7,s8]:
            fd=root/f'fold_{fold}';fd.mkdir()
            (fd/'feature_schema.json').write_text(json.dumps({'fold':fold,'faculties':faculties,'common_raw_features':common,'shared_feature_names':[]}))
        caches={}
        for i,f in enumerate(faculties):
            c=_make_cache(cache,f,fold,1000+fold*10+i);caches[f]=c
            q=c['qte'];y=c['yte'];sh=np.clip(q+(0.01 if f=='fac_a' else -0.01),.001,.999)
            ad=np.clip(sh+(0.005 if f=='fac_a' else -0.005),.001,.999);t8=np.clip(ad+.002,.001,.999)
            sf=_score_frame(y,q,sh,ad,t8)
            sf[['actual','committee','residual_moe_shared','correction_logit','weight_null','weight_expert_1','weight_expert_2','weight_expert_3','weight_expert_4','residual_expert_1','residual_expert_2','residual_expert_3','residual_expert_4']].to_csv(shared/f'fold_{fold}'/f'{f}_scores.csv',index=False)
            sf.drop(columns=['residual_moe_true_specialist']).to_csv(s7/f'fold_{fold}'/f'{f}_scores.csv',index=False)
            sf.to_csv(s8/f'fold_{fold}'/f'{f}_scores.csv',index=False)
        # Compatible fitted Suite7 checkpoint. The smoke test only requires a valid initialization.
        hist=build_historical_bundle(caches,faculties,common)
        torch.manual_seed(42+fold*100003)
        model=SharedPrivateResidualMoE(input_dim=hist.X.shape[1],n_faculties=2,n_experts=4,faculty_dim=3,
            encoder_hidden=16,representation_dim=12,expert_hidden=8,dropout=0.0,adapter_dim=4)
        torch.save({'state_dict':model.state_dict(),'selected_epoch':1,'model_config':{},'faculties':faculties,'common_raw_features':common},s7/f'fold_{fold}'/'shared_private_model.pt')

    script=Path(__file__).resolve().parents[1]/'train_suite9_competitive_responsibility.py'
    cmd=[sys.executable,str(script),'--cache_dir',str(cache),'--shared_results_dir',str(shared),
         '--suite7_results_dir',str(s7),'--suite8_results_dir',str(s8),'--models_dir',str(out),
         '--faculties','all','--dev_folds','1,2','--holdout_folds','3','--device','cpu',
         '--discovery_epochs','2','--router_epochs','2','--joint_refine_epochs','0','--val_blocks','1',
         '--min_train_rows','10','--min_val_rows','5','--batch_size','16','--predict_batch_size','64',
         '--encoder_hidden','16','--representation_dim','12','--expert_hidden','8','--faculty_dim','3','--adapter_dim','4',
         '--dropout','0','--adapter_dropout','0','--print_every','1','--responsibility_top_k','2',
         '--min_precision','0','--max_warning_ratio','10','--min_flags','1','--min_selection_improvement','-10']
    cp=subprocess.run(cmd,cwd=str(script.parent),text=True,capture_output=True,timeout=180)
    assert cp.returncode==0,cp.stdout+'\n'+cp.stderr
    required=['suite9_comparison.csv','suite9_responsibility_diagnostics.csv','suite9_router_agreement.csv',
              'suite9_expert_region_diagnostics.csv','suite9_oracle_gap.csv','suite9_warning_metrics.csv',
              'suite9_transfer_at20.csv','suite9_training_summary.csv']
    for name in required:assert (out/name).exists(),name
    assert (out/'fold_3'/'competitive_responsibility_model.pt').exists()
    assert (out/'fold_3'/'fac_a_scores.csv').exists()
    cmp=pd.read_csv(out/'suite9_comparison.csv')
    assert set(cmp.faculty)==set(faculties)
    assert {'suite9_candidate_ap','suite9_final_ap','suite9_minus_adapter_ap','fallback_rate'} <= set(cmp.columns)
    gap=pd.read_csv(out/'suite9_oracle_gap.csv')
    assert {'oracle_ap','learned_ap','oracle_minus_learned_ap','router_target_top1_accuracy'} <= set(gap.columns)
