from pathlib import Path
import json, subprocess, sys
import joblib, numpy as np, pandas as pd, torch
from suite7_shared_private_moe import SharedPrivateResidualMoE
from suite7_common import build_historical_bundle

RAW_FEATURES=[
 'hist_semester_gpa10_mean','recent3_semester_gpa10_mean','prev_cumulative_gpa10','prev_semester_gpa10',
 'hist_course_grade10_count','hist_course_grade10_mean','recent3_course_grade10_count','recent3_course_grade10_mean',
 'hist_component_grade_count','hist_component_grade_mean','recent3_component_grade_count','recent3_component_grade_mean',
 'hist_conduct_mean','prev_conduct_score','hist_warning_rate','recent3_warning_count','recent3_semester_count','hist_semester_count',
 'hist_same_course_attempts','hist_same_course_component_count','hist_target_course_count','student_gpa_history_missing_flag',
 'course_history_missing_flag','student_prior_course_count_log'
]

def _support(n,rng):
    hs=rng.integers(0,6,size=n); cr=rng.integers(0,150,size=n)
    return pd.DataFrame({'history_semesters':hs,'history_rows':hs*4+rng.integers(0,5,n),'new_student':(hs==0).astype(float),'course_prior_rows':cr,'new_course':(cr==0).astype(float)})

def _raw(n,rng,support):
    X=rng.normal(size=(n,len(RAW_FEATURES))).astype(float);i={n:j for j,n in enumerate(RAW_FEATURES)}
    hist=7+rng.normal(0,.5,n); recent=hist+rng.normal(0,.5,n)
    for h,r in [('hist_semester_gpa10_mean','recent3_semester_gpa10_mean'),('hist_course_grade10_mean','recent3_course_grade10_mean'),('hist_component_grade_mean','recent3_component_grade_mean')]:X[:,i[h]]=hist;X[:,i[r]]=recent
    X[:,i['prev_cumulative_gpa10']]=hist;X[:,i['prev_semester_gpa10']]=recent
    X[:,i['hist_course_grade10_count']]=rng.integers(2,20,n);X[:,i['recent3_course_grade10_count']]=rng.integers(1,10,n)
    X[:,i['hist_component_grade_count']]=rng.integers(2,30,n);X[:,i['recent3_component_grade_count']]=rng.integers(1,15,n)
    X[:,i['hist_conduct_mean']]=85;X[:,i['prev_conduct_score']]=85+rng.normal(0,3,n)
    X[:,i['hist_warning_rate']]=rng.uniform(0,.2,n);X[:,i['recent3_warning_count']]=rng.integers(0,2,n);X[:,i['recent3_semester_count']]=3;X[:,i['hist_semester_count']]=support.history_semesters
    X[:,i['hist_same_course_attempts']]=rng.integers(0,3,n);X[:,i['hist_same_course_component_count']]=rng.integers(0,5,n);X[:,i['hist_target_course_count']]=rng.integers(0,300,n)
    X[:,i['student_gpa_history_missing_flag']]=(support.history_semesters==0).astype(float);X[:,i['course_history_missing_flag']]=(support.course_prior_rows==0).astype(float);X[:,i['student_prior_course_count_log']]=np.log1p(support.history_rows)
    return X

def _make_cache(path,faculty,fold,seed):
    rng=np.random.default_rng(seed);ntr,nte=96,32
    sh=_support(ntr,rng);st=_support(nte,rng);Xtr=_raw(ntr,rng,sh);Xte=_raw(nte,rng,st)
    ytr=(rng.random(ntr)<.20).astype(int);yte=(rng.random(nte)<.22).astype(int)
    bt=np.clip(.15+.55*ytr+rng.normal(0,.12,ntr),.01,.99);cf=np.column_stack([np.clip(bt+rng.normal(0,.04,ntr),.01,.99) for _ in range(3)])
    be=np.clip(.15+.55*yte+rng.normal(0,.12,nte),.01,.99);pte=np.column_stack([np.clip(be+rng.normal(0,.04,nte),.01,.99) for _ in range(3)])
    c={'feature_names':RAW_FEATURES,'Xtr':Xtr,'Xte':Xte,'ytr':ytr,'yte':yte,'cf_covered_mask':np.ones(ntr,bool),'cf_probabilities':cf,'cf_block_ids':np.repeat([1.,2.,3.,4.],24),'support_hist':sh,'support_target':st,'pte':pte,'qte':pte.mean(1),'test_frame':pd.DataFrame({'dummy':range(nte)})}
    joblib.dump(c,path/f'{faculty}_fold_{fold}.joblib');return c

def test_suite10_end_to_end_small(tmp_path:Path):
    cache=tmp_path/'cache';shared=tmp_path/'shared';s7=tmp_path/'s7';out=tmp_path/'out'
    for p in [cache,shared,s7]:p.mkdir()
    fac=['fac_a','fac_b']
    for fold in [1,2,3]:
        for root in [shared,s7]:(root/f'fold_{fold}').mkdir()
        caches={}
        for i,f in enumerate(fac):
            c=_make_cache(cache,f,fold,100+fold*10+i);caches[f]=c;q=c['qte'];y=c['yte'];sh=np.clip(q+.002,.001,.999);ad=np.clip(q+.004,.001,.999)
            pd.DataFrame({'actual':y,'committee':q,'residual_moe_shared':sh}).to_csv(shared/f'fold_{fold}'/f'{f}_scores.csv',index=False)
            pd.DataFrame({'actual':y,'committee':q,'residual_moe_shared_private':ad}).to_csv(s7/f'fold_{fold}'/f'{f}_scores.csv',index=False)
        hist=build_historical_bundle(caches,fac,RAW_FEATURES)
        (shared/f'fold_{fold}'/'feature_schema.json').write_text(json.dumps({'fold':fold,'faculties':fac,'common_raw_features':RAW_FEATURES,'shared_feature_names':hist.feature_names}))
        torch.manual_seed(42+fold*100003);model=SharedPrivateResidualMoE(input_dim=hist.X.shape[1],n_faculties=2,n_experts=4,faculty_dim=3,encoder_hidden=16,representation_dim=12,expert_hidden=8,dropout=0,adapter_dim=4)
        torch.save({'state_dict':model.state_dict(),'selected_epoch':1,'model_config':{'n_experts':4,'faculty_dim':3,'encoder_hidden':16,'representation_dim':12,'expert_hidden':8,'delta_max':2.0,'dropout':0.0,'router_temperature':1.0,'adapter_dim':4,'adapter_dropout':0.0,'adapter_scale':1.0},'faculties':fac,'common_raw_features':RAW_FEATURES},s7/f'fold_{fold}'/'shared_private_model.pt')
    script=Path(__file__).resolve().parents[1]/'train_suite10_semantic_specialists.py'
    cmd=[sys.executable,str(script),'--cache_dir',str(cache),'--shared_results_dir',str(shared),'--suite7_results_dir',str(s7),'--models_dir',str(out),'--faculties','all','--dev_folds','1,2','--holdout_folds','3','--device','cpu','--epochs','2','--val_blocks','2','--min_train_rows','10','--min_val_rows','4','--domain_min_rows','2','--domain_min_failures','1','--batch_size','16','--predict_batch_size','64','--encoder_hidden','16','--representation_dim','12','--expert_hidden','8','--faculty_dim','3','--adapter_dim','4','--dropout','0','--print_every','1','--min_precision','0','--max_warning_ratio','10','--min_flags','1','--min_selection_improvement','-10']
    cp=subprocess.run(cmd,cwd=str(script.parent),text=True,capture_output=True,timeout=180)
    assert cp.returncode==0,cp.stdout+'\n'+cp.stderr
    for name in ['suite10_domain_viability.csv','suite10_domain_overlap.csv','suite10_expert_contracts.csv','suite10_reliability.csv','suite10_authority_diagnostics.csv','suite10_fold_metrics.csv','suite10_warning_metrics.csv','suite10_comparison.csv']:
        assert (out/name).exists(),name
    assert (out/'fold_3'/'semantic_specialist_model.pt').exists()
    df=pd.read_csv(out/'fold_3'/'fac_a_scores.csv')
    assert {'domain_student_sparse','domain_course_sparse','domain_temporal_deterioration','domain_history_relational','weight_null','weight_expert_1','semantic_specialist_candidate'}<=set(df.columns)
