# -*- coding: utf-8 -*-
"""Experiment 2: relational student-course factorization expert.

Tests whether latent student x course interactions provide complementary signal
that the static tabular committee cannot represent directly. Unknown IDs map to
zero/OOV embeddings, making cold-start behavior explicit.
"""
from __future__ import annotations
import argparse, copy
from dataclasses import dataclass
from pathlib import Path
import numpy as np
import pandas as pd
import torch
from torch import nn
from torch.utils.data import DataLoader, TensorDataset
from sklearn.preprocessing import StandardScaler

import train_v3 as core
from breakthrough_common import OOVEncoder, seed_all, load_binary_fold, fit_production_committee, top_raw_selector, safe_ap, safe_auc, score_row, novelty_groups, subgroup_metric_rows, save_csv, parse_folds, parse_float_grid


class RelationalFactorizationModel(nn.Module):
    def __init__(self,n_students,n_courses,n_sections,side_dim,student_dim=16,course_dim=16,section_dim=8,hybrid=False):
        super().__init__();self.hybrid=bool(hybrid);d=min(student_dim,course_dim);self.d=d
        self.student=nn.Embedding(n_students,student_dim,padding_idx=0);self.course=nn.Embedding(n_courses,course_dim,padding_idx=0);self.section=nn.Embedding(max(1,n_sections),section_dim,padding_idx=0)
        self.sb=nn.Embedding(n_students,1,padding_idx=0);self.cb=nn.Embedding(n_courses,1,padding_idx=0);self.xb=nn.Embedding(max(1,n_sections),1,padding_idx=0);self.global_bias=nn.Parameter(torch.zeros(()))
        if self.hybrid:
            self.side_linear=nn.Linear(side_dim,1);self.side_proj=nn.Linear(side_dim,course_dim)
        self.reset_parameters()
    def reset_parameters(self):
        for emb in [self.student,self.course,self.section]: nn.init.normal_(emb.weight,std=.05); emb.weight.data[0].zero_()
        for emb in [self.sb,self.cb,self.xb]: nn.init.zeros_(emb.weight)
        if self.hybrid: nn.init.zeros_(self.side_linear.bias)
    def forward(self,s,c,x,side):
        se=self.student(s);ce=self.course(c);score=self.global_bias+self.sb(s).squeeze(1)+self.cb(c).squeeze(1)+self.xb(x).squeeze(1)+(se[...,:self.d]*ce[...,:self.d]).sum(1)
        if self.hybrid: score=score+self.side_linear(side).squeeze(1)+(self.side_proj(side)*ce).sum(1)/np.sqrt(max(1,ce.shape[1]))
        return score


def _last_temporal_validation(frame,y,min_rows=500,min_failures=20):
    col='target_semester_ordinal' if 'target_semester_ordinal' in frame else ('target_NHHK' if 'target_NHHK' in frame else None)
    if col:
        vals=pd.to_numeric(frame[col],errors='coerce').to_numpy();uniq=sorted(v for v in pd.unique(vals[np.isfinite(vals)]))
        for b in reversed(uniq):
            va=np.where(vals==b)[0];tr=np.where(vals<b)[0]
            if len(va)>=min_rows and y[va].sum()>=min_failures and np.unique(y[tr]).size==2:return tr,va,str(b)
    rng=np.random.default_rng(42);idx=np.arange(len(y));rng.shuffle(idx);cut=max(1,int(.85*len(idx)));return idx[:cut],idx[cut:],'random_fallback'


def _weights(y,beta=.9999): return core.effective_sample_weights(y,beta)[0].astype('float32')

def _predict(model,arrays,device,batch=4096):
    s,c,x,side=arrays;out=[];model.eval()
    ds=TensorDataset(torch.from_numpy(s),torch.from_numpy(c),torch.from_numpy(x),torch.from_numpy(side.astype('float32')))
    with torch.no_grad():
        for bb in DataLoader(ds,batch_size=batch,shuffle=False): out.append(torch.sigmoid(model(*[z.to(device) for z in bb])).cpu().numpy())
    return np.concatenate(out) if out else np.array([])


def _fit_one(arrays,y,train_idx,val_idx,shape,hybrid,seed,epochs,patience,lr,batch_size,device,student_dim=16,course_dim=16,section_dim=8):
    seed_all(seed);model=RelationalFactorizationModel(*shape,student_dim=student_dim,course_dim=course_dim,section_dim=section_dim,hybrid=hybrid).to(device);opt=torch.optim.AdamW(model.parameters(),lr=lr,weight_decay=1e-4);w=_weights(y[train_idx])
    s,c,x,side=arrays;ds=TensorDataset(torch.from_numpy(s[train_idx]),torch.from_numpy(c[train_idx]),torch.from_numpy(x[train_idx]),torch.from_numpy(side[train_idx].astype('float32')),torch.from_numpy(y[train_idx].astype('float32')),torch.from_numpy(w));loader=DataLoader(ds,batch_size=batch_size,shuffle=True)
    best=None;wait=0
    for ep in range(1,epochs+1):
        model.train()
        for bs,bc,bx,bz,by,bw in loader:
            bs,bc,bx,bz,by,bw=[t.to(device) for t in [bs,bc,bx,bz,by,bw]];opt.zero_grad();logit=model(bs,bc,bx,bz);loss=torch.nn.functional.binary_cross_entropy_with_logits(logit,by,weight=bw);loss.backward();opt.step()
        pv=_predict(model,(s[val_idx],c[val_idx],x[val_idx],side[val_idx]),device);ap=safe_ap(y[val_idx],pv);auc=safe_auc(y[val_idx],pv);key=(ap,auc)
        if best is None or key>best[0]:best=(key,ep,copy.deepcopy(model.state_dict()));wait=0
        else:wait+=1
        if wait>=patience:break
    model.load_state_dict(best[2]);return model,best[1],best[0]


def prepare_relational_ids(train,test):
    se=OOVEncoder().fit(train['IDSV'] if 'IDSV' in train else ['<NA>']);ce=OOVEncoder().fit(train['IDMH'] if 'IDMH' in train else ['<NA>']);xe=OOVEncoder().fit(train['IDToHoc'] if 'IDToHoc' in train else ['<NA>'])
    tr_s=se.transform(train['IDSV'] if 'IDSV' in train else ['<NA>']*len(train));te_s=se.transform(test['IDSV'] if 'IDSV' in test else ['<NA>']*len(test));tr_c=ce.transform(train['IDMH'] if 'IDMH' in train else ['<NA>']*len(train));te_c=ce.transform(test['IDMH'] if 'IDMH' in test else ['<NA>']*len(test));tr_x=xe.transform(train['IDToHoc'] if 'IDToHoc' in train else ['<NA>']*len(train));te_x=xe.transform(test['IDToHoc'] if 'IDToHoc' in test else ['<NA>']*len(test))
    return (tr_s,tr_c,tr_x),(te_s,te_c,te_x),se,ce,xe


def make_side_features(Xtr,Xte,ytr,names,fit_idx,topk=48):
    fit_idx=np.asarray(fit_idx,int);selector=top_raw_selector(Xtr[fit_idx],ytr[fit_idx],names,topk);raw_tr=selector.transform(Xtr);raw_te=selector.transform(Xte);sc=StandardScaler().fit(raw_tr[fit_idx]);return sc.transform(raw_tr).astype('float32'),sc.transform(raw_te).astype('float32'),selector

def train_fold(args,fold,outdir):
    d=load_binary_fold(args.splits_dir,args.faculty,fold)
    if d is None:return None
    Xtr,Xte,ytr,yte,tr,te,names=d;_,pe,q=fit_production_committee(Xtr,ytr,Xte,seed=args.seed+fold*11,n_estimators=args.n_estimators,hist_max_iter=args.hist_max_iter)
    tridx,vidx,vname=_last_temporal_validation(tr,ytr,args.val_min_rows,args.val_min_failures);ids_tr,ids_te,se,ce,xe=prepare_relational_ids(tr,te);side_sel_tr,side_sel_te,selector_sel=make_side_features(Xtr,Xte,ytr,names,tridx,args.top_raw_features);atr_sel=(*ids_tr,side_sel_tr);ate_sel=(*ids_te,side_sel_te);shape=(se.size,ce.size,xe.size,side_sel_tr.shape[1]);device=torch.device('cuda' if torch.cuda.is_available() and args.device!='cpu' else 'cpu')
    # Final refit side representation is allowed to use all historical labels, but never target labels.
    full_idx=np.arange(len(ytr));side_full_tr,side_full_te,selector_full=make_side_features(Xtr,Xte,ytr,names,full_idx,args.top_raw_features);atr_full=(*ids_tr,side_full_tr);ate_full=(*ids_te,side_full_te)
    scores={};summ=[]
    for hybrid,name in [(False,'pure_relational'),(True,'hybrid_relational')]:
        model,ep,(vap,vauc)=_fit_one(atr_sel,ytr,tridx,vidx,shape,hybrid,args.seed+fold*100+(1 if hybrid else 0),args.epochs,args.patience,args.lr,args.batch_size,device,args.student_dim,args.course_dim,args.section_dim)
        # Refit fresh on all historical rows for selected epoch count.
        seed_all(args.seed+fold*1000+(1 if hybrid else 0));final=RelationalFactorizationModel(*shape,student_dim=args.student_dim,course_dim=args.course_dim,section_dim=args.section_dim,hybrid=hybrid).to(device);opt=torch.optim.AdamW(final.parameters(),lr=args.lr,weight_decay=1e-4);w=_weights(ytr);ds=TensorDataset(torch.from_numpy(atr_full[0]),torch.from_numpy(atr_full[1]),torch.from_numpy(atr_full[2]),torch.from_numpy(atr_full[3]),torch.from_numpy(ytr.astype('float32')),torch.from_numpy(w));loader=DataLoader(ds,batch_size=args.batch_size,shuffle=True)
        for _ in range(ep):
            final.train()
            for bs,bc,bx,bz,by,bw in loader:
                bs,bc,bx,bz,by,bw=[t.to(device) for t in [bs,bc,bx,bz,by,bw]];opt.zero_grad();z=final(bs,bc,bx,bz);loss=torch.nn.functional.binary_cross_entropy_with_logits(z,by,weight=bw);loss.backward();opt.step()
        p=_predict(final,ate_full,device);scores[name]=p;row={'fold':fold,'model':name,'best_epoch':ep,'validation_block':vname,'validation_ap':vap,'validation_auc':vauc,**score_row(yte,p)};summ.append(row);print(f"  {name}: val AP/AUC={vap:.4f}/{vauc:.4f} ep={ep} | test AP/AUC={row['ap']:.4f}/{row['auc']:.4f}")
    groups=novelty_groups(tr,te);cold={'new_student_rate':float(np.mean(ate_full[0]==0)),'new_course_rate':float(np.mean(ate_full[1]==0)),'new_section_rate':float(np.mean(ate_full[2]==0))}
    tag=f'{args.faculty}_fold_{fold}';pd.DataFrame({'actual':yte,'committee':q,'pure_relational':scores['pure_relational'],'hybrid_relational':scores['hybrid_relational'],'novelty_group':groups}).to_csv(outdir/f'{tag}_scores.csv',index=False);save_csv(summ,outdir/f'{tag}_metrics.csv')
    for name,p in scores.items():save_csv(subgroup_metric_rows(groups,yte,q,p,{'model':name,'fold':fold}),outdir/f'{tag}_{name}_subgroups.csv')
    save_csv([{'fold':fold,**cold}],outdir/f'{tag}_cold_start.csv');print(f"  cold start: new student={cold['new_student_rate']:.1%} new course={cold['new_course_rate']:.1%}")
    return {'fold':fold,'y':yte,'committee':q,**scores,'groups':groups}


def select_blend(records,model_name,grid):
    y=np.concatenate([r['y'] for r in records]);q=np.concatenate([r['committee'] for r in records]);p=np.concatenate([r[model_name] for r in records]);rows=[]
    for a in grid:
        s=(1-a)*q+a*p;rows.append({'model':model_name,'alpha_relational':a,'ap':safe_ap(y,s),'auc':safe_auc(y,s),**{f'best_{k}':v for k,v in score_row(y,s).items() if k in ['best_f2','best_precision','best_recall','best_flag_rate']}})
    return max(rows,key=lambda r:(r['ap'],r['auc'])),rows


def main():
    a=parse_args();seed_all(a.seed);out=Path(a.models_dir);out.mkdir(parents=True,exist_ok=True);print('='*110);print('EXPERIMENT 2 — RELATIONAL FACTORIZATION EXPERT');print('='*110)
    dev=[r for f in parse_folds(a.dev_folds) if (r:=train_fold(a,f,out)) is not None];selected={};allblend=[]
    for name in ['pure_relational','hybrid_relational']:
        b,rows=select_blend(dev,name,a.blend_grid);selected[name]=b['alpha_relational'];allblend+=rows;print(f"DEV blend {name}: alpha={b['alpha_relational']:.2f} AP={b['ap']:.4f} AUC={b['auc']:.4f}")
    save_csv(allblend,out/f'{a.faculty}_development_blend_grid.csv');save_csv([{'model':k,'alpha_relational':v} for k,v in selected.items()],out/f'{a.faculty}_selected_blend.csv')
    hold=[r for f in parse_folds(a.holdout_folds) if (r:=train_fold(a,f,out)) is not None]
    rows=[]
    for split,recs in [('development',dev),('holdout',hold)]:
        if not recs:continue
        y=np.concatenate([r['y'] for r in recs]);q=np.concatenate([r['committee'] for r in recs]);rows.append({'split':split,'model':'committee',**score_row(y,q)})
        for name in ['pure_relational','hybrid_relational']:
            p=np.concatenate([r[name] for r in recs]);a0=selected[name];s=(1-a0)*q+a0*p;rows.append({'split':split,'model':name,**score_row(y,p)});rows.append({'split':split,'model':f'committee_blend_{name}','alpha_relational':a0,**score_row(y,s)});print(f"{split.upper()} {name} blend alpha={a0:.2f} AP/AUC={safe_ap(y,s):.4f}/{safe_auc(y,s):.4f}")
    save_csv(rows,out/f'{a.faculty}_summary.csv')


def parse_args():
    p=argparse.ArgumentParser();p.add_argument('--splits_dir',required=True);p.add_argument('--faculty',default='khoa_cong_nghe_thong_tin');p.add_argument('--dev_folds',default='1,2,3,4,5');p.add_argument('--holdout_folds',default='6,7');p.add_argument('--models_dir',default='results/exp_relational_fm');p.add_argument('--seed',type=int,default=42);p.add_argument('--n_estimators',type=int,default=500);p.add_argument('--hist_max_iter',type=int,default=250);p.add_argument('--top_raw_features',type=int,default=48);p.add_argument('--student_dim',type=int,default=16);p.add_argument('--course_dim',type=int,default=16);p.add_argument('--section_dim',type=int,default=8);p.add_argument('--epochs',type=int,default=60);p.add_argument('--patience',type=int,default=8);p.add_argument('--lr',type=float,default=2e-3);p.add_argument('--batch_size',type=int,default=1024);p.add_argument('--val_min_rows',type=int,default=500);p.add_argument('--val_min_failures',type=int,default=20);p.add_argument('--blend_grid',type=parse_float_grid,default=parse_float_grid('0,0.1,0.2,0.3,0.4,0.5,0.75,1'));p.add_argument('--device',choices=['auto','cpu'],default='auto');return p.parse_args()
if __name__=='__main__':main()
