# -*- coding: utf-8 -*-
from __future__ import annotations
import numpy as np

STUDENT_FEATURES=['DTBHK','DTBTLHK','SOTCDKHK','SoTCDat','SoTCNoTL','SoMonNoHB','SoTCKDat','DTBHKDK','SoTCHKDK','SoTCDatHKDK','DTBCNTLHK','SoTCCNTLHK']

def _slope(v):
    v=np.asarray(v,float)
    if len(v)<2:return 0.
    x=np.arange(len(v),dtype=float);x=x-x.mean();den=np.sum(x*x);return float(np.sum(x*(v-v.mean()))/den) if den>0 else 0.

def build_student_dynamics(student_seq,student_mask,return_names=False):
    seq=np.asarray(student_seq,np.float32);mask=np.asarray(student_mask,bool);n=seq.shape[0];p=min(12,seq.shape[2]); feats=[];names=[]
    per=[]
    for j in range(p):
        arr=np.zeros((n,7),np.float32)
        for i in range(n):
            v=seq[i,mask[i],j].astype(float)
            if len(v)==0:continue
            d1=v[-1]-v[-2] if len(v)>=2 else 0.;acc=v[-1]-2*v[-2]+v[-3] if len(v)>=3 else 0.;sl=_slope(v);recent=v[-min(2,len(v)):].mean();long=v.mean();vol=v.std() if len(v)>1 else 0.;drawdown=v[-1]-v.max();drawup=v[-1]-v.min();arr[i]=[d1,acc,sl,recent-long,vol,drawdown,drawup]
        per.append(arr);prefix=STUDENT_FEATURES[j] if j<len(STUDENT_FEATURES) else f'f{j}';names.extend([f'delta1_{prefix}',f'accel_{prefix}',f'slope_{prefix}',f'recent_minus_long_{prefix}',f'volatility_{prefix}',f'drawdown_{prefix}',f'drawup_{prefix}'])
    x=np.concatenate(per,1) if per else np.zeros((n,0),np.float32)
    support=mask.sum(1,keepdims=True).astype(np.float32);x=np.concatenate([x,support],1);names.append('history_semesters_count')
    return (x,names) if return_names else x

from pathlib import Path
import joblib
import pandas as pd
from suite16_structured_cache import semester_distance
from scipy import sparse
from sklearn.decomposition import TruncatedSVD

COMP_TYPES=['__PAD__','T1','K1','K2','B1','VT','CT','DC','TH','GK','CK','OTHER']

def _coalesce_grade_df(df):
    a=pd.to_numeric(df.get('DiemHP',pd.Series(np.nan,index=df.index)),errors='coerce')
    if 'DiemHPCalc' in df:
        b=pd.to_numeric(df['DiemHPCalc'],errors='coerce');a=a.where(a.notna(),b)
    return a

def _norm_ids(a): return np.asarray([str(x) for x in a],dtype=object)

class LatentFactorStore:
    def __init__(self,hp_frame,cache_dir,dim=12,seed=20260830):
        self.hp=hp_frame.copy();self.cache_dir=Path(cache_dir);self.cache_dir.mkdir(parents=True,exist_ok=True);self.dim=int(dim);self.seed=int(seed)
        self.hp['IDSV']=self.hp['IDSV'].astype(str);self.hp['IDMH']=self.hp['IDMH'].astype(str);self.hp['NHHK']=pd.to_numeric(self.hp['NHHK'],errors='coerce');self.hp['_grade']=_coalesce_grade_df(self.hp)
        self.hp=self.hp[self.hp.NHHK.notna()&self.hp._grade.notna()].copy()
        self._memo={}
    @classmethod
    def from_raw_index(cls,raw_index_dir,cache_dir,dim=12,seed=20260830):
        return cls(joblib.load(Path(raw_index_dir)/'course_history.joblib'),cache_dir,dim,seed)
    @classmethod
    def from_frame(cls,frame,cache_dir,dim=12,seed=20260830):return cls(frame,cache_dir,dim,seed)
    @property
    def feature_names(self):
        return [f'latent_student[{i}]' for i in range(self.dim)]+[f'latent_course[{i}]' for i in range(self.dim)]+['latent_dot','latent_cosine','latent_student_norm','latent_course_norm','latent_student_bias','latent_course_bias','latent_student_seen','latent_course_seen']
    def _path(self,sem):return self.cache_dir/f'latent_sem_{int(float(sem))}_d{self.dim}.joblib'
    def _fit_sem(self,sem):
        key=int(float(sem))
        if key in self._memo:return self._memo[key]
        p=self._path(key)
        if p.exists():
            obj=joblib.load(p);self._memo[key]=obj;return obj
        z=self.hp[self.hp.NHHK<float(sem)].copy()
        if len(z)==0:
            obj={'students':{},'courses':{},'student_emb':np.zeros((0,self.dim),np.float32),'course_emb':np.zeros((0,self.dim),np.float32),'student_bias':{},'course_bias':{},'global_mean':5.}
            joblib.dump(obj,p,compress=3);self._memo[key]=obj;return obj
        agg=z.groupby(['IDSV','IDMH'],as_index=False)['_grade'].mean();students=sorted(agg.IDSV.unique());courses=sorted(agg.IDMH.unique());sm={s:i for i,s in enumerate(students)};cm={c:i for i,c in enumerate(courses)};gm=float(agg._grade.mean())
        vals=(agg._grade.to_numpy(float)-gm)/5.;rows=np.asarray([sm[s] for s in agg.IDSV],int);cols=np.asarray([cm[c] for c in agg.IDMH],int);mat=sparse.coo_matrix((vals,(rows,cols)),shape=(len(students),len(courses))).tocsr()
        k=max(0,min(self.dim,min(mat.shape)-1))
        se=np.zeros((len(students),self.dim),np.float32);ce=np.zeros((len(courses),self.dim),np.float32)
        if k>=1 and mat.nnz:
            svd=TruncatedSVD(n_components=k,n_iter=4,random_state=self.seed+key);se[:,:k]=svd.fit_transform(mat).astype(np.float32);ce[:,:k]=svd.components_.T.astype(np.float32)
        sb=(z.groupby('IDSV')._grade.mean()-gm).to_dict();cb=(z.groupby('IDMH')._grade.mean()-gm).to_dict()
        obj={'students':sm,'courses':cm,'student_emb':se,'course_emb':ce,'student_bias':sb,'course_bias':cb,'global_mean':gm};joblib.dump(obj,p,compress=3);self._memo[key]=obj;return obj
    def features_for_rows(self,student_ids,course_ids,target_semesters):
        sid=_norm_ids(student_ids);cid=_norm_ids(course_ids);sem=np.asarray(target_semesters,float);out=np.zeros((len(sid),2*self.dim+8),np.float32)
        for s in np.unique(sem[np.isfinite(sem)]):
            idx=np.where(sem==s)[0];obj=self._fit_sem(s)
            for i in idx:
                si=obj['students'].get(sid[i]);ci=obj['courses'].get(cid[i]);sv=obj['student_emb'][si] if si is not None else np.zeros(self.dim,np.float32);cv=obj['course_emb'][ci] if ci is not None else np.zeros(self.dim,np.float32);dot=float(np.dot(sv,cv));sn=float(np.linalg.norm(sv));cn=float(np.linalg.norm(cv));cos=dot/(sn*cn+1e-6);off=2*self.dim
                out[i,:self.dim]=sv;out[i,self.dim:2*self.dim]=cv;out[i,off:]=[dot,cos,sn,cn,float(obj['student_bias'].get(sid[i],0.)),float(obj['course_bias'].get(cid[i],0.)),float(si is not None),float(ci is not None)]
        return out

def _is_forward(code):return int(code) in {1,2,3,7,8,9}
def _is_direct(code):return 1<=int(code)<=6

def _group_stats(events,mask):
    n=len(events);out=np.zeros((n,8),np.float32)
    for i in range(n):
        m=np.asarray(mask[i],bool);x=events[i,m]
        if not len(x):continue
        g=x[:,0].astype(float);rec=x[:,3].astype(float);w=np.exp(-.35*np.maximum(rec,0));passed=x[:,2]>0.5;latest=int(np.argmin(rec))
        out[i]=[len(g),g.mean(),g.min(),g.max(),np.mean(~passed),np.sum(w*g)/(w.sum()+1e-6),g[latest],rec.mean()]
    return out

def build_student_curriculum_compatibility(bundle,latent_features=None,return_names=False):
    re=np.asarray(bundle.relation_events,np.float32);rm=np.asarray(bundle.relation_mask,bool);rt=np.asarray(bundle.relation_type,int);n=len(re)
    same=rm&(re[:,:,4]>.5);down=rm&(re[:,:,5]>.5);pre=rm&(re[:,:,6]>.5);related=rm&((rt>0)|down|pre);direct=rm&np.vectorize(_is_direct)(rt)
    groups=[('same_course',same),('prereq',pre),('downstream',down),('related',related)]
    parts=[];names=[]
    for name,m in groups:
        s=_group_stats(re,m);parts.append(s);names += [f'{name}_{x}' for x in ['count','mean_grade','min_grade','max_grade','fail_frac','recency_weighted_grade','latest_grade','mean_recency']]
    # Expected prerequisite neighborhood: target->neighbor reverse relation means neighbor prerequisite of target.
    crt=np.asarray(bundle.course_relation_type,int);cm=np.asarray(bundle.course_neighbor_mask,bool);expected=cm&(crt>0)&(~np.vectorize(_is_forward)(crt));expected_direct=expected&np.vectorize(_is_direct)(crt)
    expc=expected.sum(1).astype(np.float32);expd=expected_direct.sum(1).astype(np.float32);obs=pre.sum(1).astype(np.float32);obsd=(pre&direct).sum(1).astype(np.float32)
    cov=np.divide(np.minimum(obs,expc),expc,out=np.zeros_like(obs),where=expc>0);dcov=np.divide(np.minimum(obsd,expd),expd,out=np.zeros_like(obsd),where=expd>0);readiness=np.column_stack([expc,expd,obs,obsd,np.clip(cov,0,1),np.clip(dcov,0,1),np.maximum(expc-obs,0),np.maximum(expd-obsd,0)]).astype(np.float32);parts.append(readiness);names += ['prereq_expected','direct_prereq_expected','prereq_observed','direct_prereq_observed','prereq_coverage','direct_prereq_coverage','prereq_missing','direct_prereq_missing']
    ct=np.asarray(bundle.course_target,np.float32);course_hist=np.column_stack([ct[:,0],ct[:,12] if ct.shape[1]>12 else 0,ct[:,13] if ct.shape[1]>13 else 0,ct[:,15] if ct.shape[1]>15 else 0]).astype(np.float32);parts.append(course_hist);names += ['target_course_credits','target_course_hist_count','target_course_hist_mean','target_course_hist_fail_rate']
    # Ability gap only, not absolute GPA.
    seq=np.asarray(bundle.student_seq,np.float32);sm=np.asarray(bundle.student_mask,bool);gap=np.zeros((n,2),np.float32)
    for i in range(n):
        if sm[i].any():
            latest=seq[i,np.where(sm[i])[0][-1]];hist_mean=ct[i,13] if ct.shape[1]>13 else 0.;gap[i]=[latest[1]-hist_mean,latest[10]-hist_mean]
    parts.append(gap);names += ['recent_cum_gpa_minus_course_hist_mean','curriculum_gpa_minus_course_hist_mean']
    # Preserve raw relational evidence without a learned bottleneck.
    raw=re.copy();raw[~rm]=0.;parts += [raw.reshape(n,-1),rm.astype(np.float32),np.clip(rt,0,13).astype(np.float32)/13.];names += [f'relation[{i},{j}]' for i in range(re.shape[1]) for j in range(re.shape[2])]+[f'relation_mask[{i}]' for i in range(re.shape[1])]+[f'relation_type_scaled[{i}]' for i in range(re.shape[1])]
    if latent_features is not None:
        lf=np.asarray(latent_features,np.float32);parts.append(lf);names += [f'latent[{i}]' for i in range(lf.shape[1])]
    x=np.concatenate(parts,1).astype(np.float32)
    return (x,names) if return_names else x

class RelativeMasteryStore:
    def __init__(self,comp_frame,cache_dir,min_peer_count=5,max_events=48,relation_lookup=None):
        self.comp=comp_frame.copy();self.cache_dir=Path(cache_dir);self.cache_dir.mkdir(parents=True,exist_ok=True);self.min_peer_count=int(min_peer_count);self.max_events=int(max_events);self.relation_lookup=relation_lookup
        self.comp['IDSV']=self.comp['IDSV'].astype(str);self.comp['IDMH']=self.comp['IDMH'].astype(str);self.comp['NHHK']=pd.to_numeric(self.comp.NHHK,errors='coerce');self.comp['DiemTP']=pd.to_numeric(self.comp.DiemTP,errors='coerce');self.comp['KyHieuTP']=self.comp.get('KyHieuTP','OTHER').fillna('OTHER').astype(str).str.upper();self.comp=self.comp[self.comp.NHHK.notna()&self.comp.DiemTP.notna()].copy();self.by_student={k:g.sort_values('NHHK') for k,g in self.comp.groupby('IDSV')};self._stats={}
    @classmethod
    def from_raw_index(cls,raw_index_dir,cache_dir,min_peer_count=5,max_events=48,relation_lookup=None):return cls(joblib.load(Path(raw_index_dir)/'component_history.joblib'),cache_dir,min_peer_count,max_events,relation_lookup)
    @classmethod
    def from_frame(cls,frame,cache_dir,min_peer_count=5,max_events=48,relation_lookup=None):return cls(frame,cache_dir,min_peer_count,max_events,relation_lookup)
    @property
    def feature_names(self):
        groups=['all','related','direct','same'];stats=['count','mean_z','std_z','min_z','low_tail_frac','recency_weighted_z','weight_weighted_z','recent3_z'];return [f'{g}_{s}' for g in groups for s in stats]+[f'type_{t}_{s}' for t in COMP_TYPES[1:] for s in ('count','mean_z')]+['z_slope','z_recent_minus_long']
    def _stats_sem(self,sem):
        key=int(float(sem))
        if key in self._stats:return self._stats[key]
        cp=self.cache_dir/f'peer_stats_sem_{key}.joblib'
        if cp.exists():
            obj=joblib.load(cp);self._stats[key]=obj;return obj
        z=self.comp[self.comp.NHHK<float(sem)].copy();global_mean=float(z.DiemTP.mean()) if len(z) else 5.;global_std=float(z.DiemTP.std()) if len(z)>1 else 1.;global_std=max(global_std,1e-3)
        ct=z.groupby(['IDMH','KyHieuTP']).DiemTP.agg(['count','mean','std']).reset_index();ct['std']=ct['std'].fillna(0.);tm=z.groupby('KyHieuTP').DiemTP.agg(['count','mean','std']).reset_index();tm['std']=tm['std'].fillna(0.);obj={'course_type':{(str(r.IDMH),str(r.KyHieuTP)):(int(r['count']),float(r['mean']),max(float(r['std']),1e-3)) for _,r in ct.iterrows()},'type':{str(r.KyHieuTP):(int(r['count']),float(r['mean']),max(float(r['std']),1e-3)) for _,r in tm.iterrows()},'global':(global_mean,global_std)};joblib.dump(obj,cp,compress=3);self._stats[key]=obj;return obj
    def _z(self,stats,course,typ,grade):
        a=stats['course_type'].get((str(course),str(typ)))
        if a and a[0]>=self.min_peer_count:return (grade-a[1])/a[2]
        a=stats['type'].get(str(typ))
        if a and a[0]>=self.min_peer_count:return (grade-a[1])/a[2]
        m,s=stats['global'];return (grade-m)/s
    def features_for_rows(self,student_ids,course_ids,target_semesters):
        sid=_norm_ids(student_ids);cid=_norm_ids(course_ids);sem=np.asarray(target_semesters,float);out=np.zeros((len(sid),len(self.feature_names)),np.float32)
        history_cache={}
        for i,(s,c,t) in enumerate(zip(sid,cid,sem)):
            if not np.isfinite(t):continue
            hkey=(str(s),int(float(t)))
            base_rows=history_cache.get(hkey)
            if base_rows is None:
                hist=self.by_student.get(str(s));
                if hist is None:history_cache[hkey]=[];continue
                h=hist[hist.NHHK<float(t)].copy();
                if not len(h):history_cache[hkey]=[];continue
                stats=self._stats_sem(t);base_rows=[]
                for r in h.itertuples(index=False):
                    ec=str(getattr(r,'IDMH'));typ=str(getattr(r,'KyHieuTP'));grade=float(getattr(r,'DiemTP'));rec=semester_distance(t,getattr(r,'NHHK'));rec=0. if not np.isfinite(rec) else max(float(rec),0.);wt=float(pd.to_numeric(getattr(r,'PhTrMH',1.),errors='coerce'));wt=0. if not np.isfinite(wt) else max(wt,0.);zv=float(np.clip(self._z(stats,ec,typ,grade),-5,5));base_rows.append((ec,typ,zv,wt,rec))
                history_cache[hkey]=base_rows
            if not base_rows:continue
            rows=[]
            for ec,typ,zv,wt,rec in base_rows:
                same=float(ec==str(c));code=0;rev=0
                if self.relation_lookup is not None:
                    try:code=int(self.relation_lookup(ec,str(c),t) or 0);rev=int(self.relation_lookup(str(c),ec,t) or 0)
                    except Exception:code=rev=0
                direct=float(_is_direct(code) or _is_direct(rev));related=float(code>0 or rev>0 or same);priority=0 if same else 1 if direct else 2 if related else 3;rows.append((priority,rec,zv,wt,same,direct,related,typ))
            rows=sorted(rows,key=lambda x:(x[0],x[1]))[:self.max_events]
            if not rows:continue
            def agg(sel):
                rr=[r for r in rows if sel(r)]
                if not rr:return [0.]*8
                z=np.array([r[2] for r in rr]);rec=np.array([r[1] for r in rr]);wt=np.array([r[3] for r in rr]);rw=np.exp(-.35*rec);order=np.argsort(rec);return [len(rr),z.mean(),z.std() if len(z)>1 else 0.,z.min(),np.mean(z<-1.),np.sum(rw*z)/(rw.sum()+1e-6),np.sum((wt+1e-3)*z)/np.sum(wt+1e-3),z[order[:min(3,len(order))]].mean()]
            vals=[];vals+=agg(lambda r:True);vals+=agg(lambda r:r[6]>.5);vals+=agg(lambda r:r[5]>.5);vals+=agg(lambda r:r[4]>.5)
            for typ in COMP_TYPES[1:]:
                rr=[r for r in rows if r[7]==typ];vals += [len(rr),float(np.mean([r[2] for r in rr])) if rr else 0.]
            z=np.array([r[2] for r in sorted(rows,key=lambda r:-r[1])]);vals += [_slope(z),float(z[-min(3,len(z)):].mean()-z.mean())]
            out[i]=np.asarray(vals,np.float32)
        return out
