# -*- coding: utf-8 -*-
from __future__ import annotations
import json, zipfile, hashlib
from collections import defaultdict
from pathlib import Path
import numpy as np
import pandas as pd
import joblib

STUDENT_FEATURES=['DTBHK','DTBTLHK','SOTCDKHK','SoTCDat','SoTCNoTL','SoMonNoHB','SoTCKDat','DTBHKDK','SoTCHKDK','SoTCDatHKDK','DTBCNTLHK','SoTCCNTLHK']
COURSE_STATIC=['SoTinChi','SoTietMH','SoTietTuan','SiSoMax','SiSoMin','KhgTinhDTB','IsMHTN','IsMonKT','IsMHKN','IsMHDacThu','IsGDTiengAnh','IsMonNgoaiNgu']
COURSE_HIST=['hist_count','hist_mean','hist_std','hist_fail_rate','hist_pass_rate']
REL_FEATURES=['grade10','credits','passed','recency','same_course','target_prereq_of_event','event_prereq_of_target']
COMP_FEATURES=['grade10','weight','passed','recency','same_course','course_related']
COMP_TYPES=['__PAD__','T1','K1','K2','B1','VT','CT','DC','TH','GK','CK','OTHER']
COMP_MAP={n:i for i,n in enumerate(COMP_TYPES)}
SCHEMA_VERSION=4

def _semester_ordinal(nhhk):
    try:
        x=int(float(nhhk));year=x//10;term=x%10
    except Exception:
        return np.nan
    if term not in (1,2,3):return np.nan
    return float(year*3+(term-1))

def semester_distance(target,event):
    a=_semester_ordinal(target);b=_semester_ordinal(event)
    return float(a-b) if np.isfinite(a) and np.isfinite(b) else np.nan

def date_to_nhhk(value):
    d=pd.to_datetime(value,errors='coerce')
    if pd.isna(d):return None
    y=int(d.year);m=int(d.month)
    if m>=8:return y*10+1
    if m<=5:return (y-1)*10+2
    return (y-1)*10+3

def edge_is_available(available_nhhk,target_nhhk):
    if available_nhhk is None or pd.isna(available_nhhk):return True
    try:return int(available_nhhk)<int(float(target_nhhk))
    except Exception:return False

def coalesce_grade(frame):
    a=pd.to_numeric(frame['DiemHP'],errors='coerce') if 'DiemHP' in frame else pd.Series(np.nan,index=frame.index,dtype=float)
    if 'DiemHPCalc' in frame:
        b=pd.to_numeric(frame['DiemHPCalc'],errors='coerce');a=a.where(a.notna(),b)
    return a

def encode_relation(family,direction,kind):
    family=1 if int(family)==1 else 2;kind=max(1,min(3,int(kind) if not pd.isna(kind) else 2));forward=int(direction)>=0
    if family==1:return kind if forward else 3+kind
    return 6+kind if forward else 9+kind

def relation_is_direct(code):
    try:return 1<=int(code)<=6
    except Exception:return False

def relation_is_forward(code):
    try:return int(code) in {1,2,3,7,8,9}
    except Exception:return False

def evidence_priority(same,rel,rev):
    if bool(same):return 0
    if relation_is_direct(rel) or relation_is_direct(rev):return 1
    if int(rel)>0 or int(rev)>0:return 2
    return 3

def prefer_edge(old,new):
    if old is None:return new
    ot,oa=old;nt,na=new
    op=0 if relation_is_direct(ot) else 1;np_=0 if relation_is_direct(nt) else 1
    if np_<op:return new
    if np_>op:return old
    if int(nt)!=int(ot):return old
    if oa is None or pd.isna(oa):return new if na is not None and not pd.isna(na) else old
    if na is None or pd.isna(na):return old
    return new if int(na)<int(oa) else old

def prior_indices(semesters,target):
    s=np.asarray(semesters,float);return np.where(np.isfinite(s)&(s<float(target)))[0]

def _norm_id(x):
    if pd.isna(x):return None
    try:return int(x)
    except Exception:return str(x)

class RawTableSource:
    """Read raw database tables either from an extracted directory or legacy ZIP archives.

    Directory mode is preferred in production. ZIP mode is retained only for portability/tests.
    """
    def __init__(self,database_dir=None,data1_zip=None,data2_zip=None):
        self.database_dir=Path(database_dir).expanduser().resolve() if database_dir else None
        self.data1_zip=Path(data1_zip).expanduser().resolve() if data1_zip else None
        self.data2_zip=Path(data2_zip).expanduser().resolve() if data2_zip else None
        self._files={};self._zip_members={}
        if self.database_dir is not None:
            if not self.database_dir.is_dir():raise FileNotFoundError(f'Database directory does not exist: {self.database_dir}')
            for fp in self.database_dir.rglob('*.csv'):
                self._files.setdefault(fp.name.lower(),fp)
            if not self._files:raise FileNotFoundError(f'No CSV database tables found under {self.database_dir}')
            self.mode='directory'
        else:
            zips=[z for z in (self.data1_zip,self.data2_zip) if z is not None]
            if not zips:raise FileNotFoundError('Provide --database_dir or both --data1_zip/--data2_zip for Suite12 structured evidence.')
            for zp in zips:
                if not zp.is_file():raise FileNotFoundError(f'Raw database ZIP does not exist: {zp}')
                with zipfile.ZipFile(zp) as z:
                    for member in z.namelist():
                        if member.lower().endswith('.csv'):
                            self._zip_members.setdefault(Path(member).name.lower(),(zp,member))
            self.mode='zip'
    def _resolve(self,name):
        key=Path(name).name.lower()
        if self.mode=='directory':
            fp=self._files.get(key)
            if fp is None:raise FileNotFoundError(f'Raw table {name} not found under {self.database_dir}')
            return ('file',fp)
        item=self._zip_members.get(key)
        if item is None:raise FileNotFoundError(f'Raw table {name} not found in supplied ZIP archives')
        return ('zip',item)
    def read_chunks(self,name,usecols,chunksize=250000):
        wanted=set(usecols) if usecols is not None else None;kind,obj=self._resolve(name)
        kwargs={'usecols':(lambda c:c in wanted) if wanted is not None else None,'chunksize':chunksize,'low_memory':False}
        if kind=='file':
            for ch in pd.read_csv(obj,**kwargs):yield ch
        else:
            zp,member=obj
            with zipfile.ZipFile(zp) as z, z.open(member) as f:
                for ch in pd.read_csv(f,**kwargs):yield ch
    def read_table(self,name,usecols=None):
        wanted=set(usecols) if usecols is not None else None;kind,obj=self._resolve(name)
        kwargs={'usecols':(lambda c:c in wanted) if wanted is not None else None,'low_memory':False}
        if kind=='file':return pd.read_csv(obj,**kwargs)
        zp,member=obj
        with zipfile.ZipFile(zp) as z, z.open(member) as f:return pd.read_csv(f,**kwargs)
    def describe(self):
        if self.mode=='directory':return {'mode':'directory','database_dir':str(self.database_dir)}
        return {'mode':'zip','data1_zip':str(self.data1_zip) if self.data1_zip else None,'data2_zip':str(self.data2_zip) if self.data2_zip else None}


def _filter_table(source,name,usecols,id_col,allowed,chunksize=250000):
    out=[];allowed=set(allowed)
    for ch in source.read_chunks(name,usecols,chunksize):
        if id_col in ch:
            ids=ch[id_col].map(_norm_id);m=ids.isin(allowed);ch=ch.loc[m].copy();ch[id_col]=ids[m].to_numpy()
        if len(ch):out.append(ch)
    return pd.concat(out,ignore_index=True) if out else pd.DataFrame(columns=usecols)


def dedupe_student_semester_latest(frame):
    """Match expanding-fold semester-summary semantics: latest raw record per student/semester."""
    if frame is None or not len(frame):return frame.copy() if frame is not None else pd.DataFrame()
    z=frame.copy();z['IDSV']=z.IDSV.map(_norm_id);z['NHHK']=pd.to_numeric(z.NHHK,errors='coerce')
    z['_suite12_order']=np.arange(len(z),dtype=np.int64)
    if 'NgayThem' in z:z['_suite12_added']=pd.to_datetime(z.NgayThem,errors='coerce')
    else:z['_suite12_added']=pd.NaT
    # NaT first -> any dated record wins; source order breaks ties deterministically.
    z=z.sort_values(['IDSV','NHHK','_suite12_added','_suite12_order'],na_position='first')
    z=z.drop_duplicates(['IDSV','NHHK'],keep='last').drop(columns=['_suite12_order','_suite12_added']).reset_index(drop=True)
    return z


def collect_needed_ids(cache_dir,faculties,folds):
    students=set();courses=set()
    for fac in faculties:
        for fold in folds:
            p=Path(cache_dir)/f'{fac}_fold_{fold}.joblib'
            if not p.exists():continue
            d=joblib.load(p)
            for fr in [d['train_frame'],d['test_frame']]:
                if 'IDSV' in fr:students.update(_norm_id(x) for x in fr.IDSV.dropna())
                if 'IDMH' in fr:courses.update(_norm_id(x) for x in fr.IDMH.dropna())
    students.discard(None);courses.discard(None);return students,courses

ALIGN_COLS=['IDKQDK','IDSV','IDToHoc','target_NHHK','IDMH','target_IDCongDiem','target_grade10','prev_semester_NHHK','prev_semester_gpa10']

def collect_canonical_rows(cache_dir,faculties,folds):
    parts=[]
    for fac in faculties:
        for fold in folds:
            p=Path(cache_dir)/f'{fac}_fold_{int(fold)}.joblib'
            if not p.exists():continue
            d=joblib.load(p)
            for role in ('train_frame','test_frame'):
                fr=d.get(role)
                if fr is None or 'IDKQDK' not in fr:continue
                cols=[c for c in ALIGN_COLS if c in fr]
                z=fr[cols].copy();z['_faculty']=fac;z['_fold']=int(fold);z['_role']=role;parts.append(z)
    if not parts:return pd.DataFrame(columns=ALIGN_COLS)
    allr=pd.concat(parts,ignore_index=True)
    for c in ['IDKQDK','IDSV','IDToHoc','target_NHHK','IDMH','target_IDCongDiem','prev_semester_NHHK']:
        if c in allr:allr[c]=allr[c].map(_norm_id)
    # The expanding protocol repeats historical registrations across train folds. They must retain identical identity/target fields.
    dup=allr[allr.IDKQDK.notna() & allr.IDKQDK.duplicated(False)]
    if len(dup):
        check=[c for c in ['IDSV','IDToHoc','target_NHHK','IDMH','target_IDCongDiem','target_grade10'] if c in dup]
        bad=[]
        for c in check:
            n=dup.groupby('IDKQDK',dropna=False)[c].nunique(dropna=False)
            if (n>1).any():bad.append(c)
        if bad:raise ValueError(f'Canonical cache contains inconsistent repeated registrations for columns {bad}')
    return allr.dropna(subset=['IDKQDK']).drop_duplicates('IDKQDK',keep='last').reset_index(drop=True)


def _agreement(a,b):
    m=a.notna()&b.notna()
    if not m.any():return np.nan,int(m.sum()),0
    aa=a[m].map(_norm_id);bb=b[m].map(_norm_id);eq=(aa.to_numpy()==bb.to_numpy())
    return float(np.mean(eq)),int(m.sum()),int((~eq).sum())


def audit_canonical_alignment(cache_dir,faculties,folds,source,chunksize=250000,hp_frame=None,student_frame=None,strict=True):
    """Verify that the engineered expanding-fold/canonical-cache rows map to the same raw DB records.

    The fold/cache remains authoritative. The database is allowed only to attach prior evidence after this audit.
    """
    can=collect_canonical_rows(cache_dir,faculties,folds)
    if not len(can):raise ValueError('Cannot audit Suite12 raw alignment: no canonical cache rows found.')
    reg_ids=set(can.IDKQDK.dropna().map(_norm_id));to_ids=set(can.IDToHoc.dropna().map(_norm_id));cd_ids=set(can.target_IDCongDiem.dropna().map(_norm_id)) if 'target_IDCongDiem' in can else set()
    mk=_filter_table(source,'MKQDK.csv',['IDKQDK','IDSV','IDToHoc','NHHK'],'IDKQDK',reg_ids,chunksize)
    for c in ['IDKQDK','IDSV','IDToHoc','NHHK']:
        if c in mk:mk[c]=mk[c].map(_norm_id)
    j=can.merge(mk,on='IDKQDK',how='left',suffixes=('_fold','_raw'),indicator=True)
    reg_match=float((j._merge=='both').mean());sid_ag,_,_= _agreement(j.IDSV_fold,j.IDSV_raw);to_ag,_,_=_agreement(j.IDToHoc_fold,j.IDToHoc_raw);sem_ag,_,_=_agreement(j.target_NHHK,j.NHHK)

    kt=_filter_table(source,'KToHoc.csv',['IDToHoc','IDMH','NHHK'],'IDToHoc',to_ids,chunksize)
    for c in ['IDToHoc','IDMH','NHHK']:
        if c in kt:kt[c]=kt[c].map(_norm_id)
    j2=can.merge(kt,on='IDToHoc',how='left',suffixes=('_fold','_raw'),indicator=True)
    course_map=float((j2._merge=='both').mean());mh_ag,_,_=_agreement(j2.IDMH_fold,j2.IDMH_raw);to_sem_ag,_,_=_agreement(j2.target_NHHK,j2.NHHK)

    if hp_frame is None:
        hp=_filter_table(source,'DDTBHP.csv',['IDCongDiem','IDSV','IDMH','NHHK','DiemHP','DiemHPCalc'],'IDCongDiem',cd_ids,chunksize)
    else:
        hp=hp_frame.copy();hp=hp[pd.Series(hp.IDCongDiem.map(_norm_id),index=hp.index).isin(cd_ids)].copy() if 'IDCongDiem' in hp else pd.DataFrame()
    if len(hp):
        for c in ['IDCongDiem','IDSV','IDMH','NHHK']:
            if c in hp:hp[c]=hp[c].map(_norm_id)
        hp['grade_raw']=coalesce_grade(hp)
        hp=hp.drop_duplicates('IDCongDiem',keep='last')
        j3=can.merge(hp[['IDCongDiem','IDSV','IDMH','NHHK','grade_raw']],left_on='target_IDCongDiem',right_on='IDCongDiem',how='left',suffixes=('_fold','_raw'),indicator=True)
        eligible=can.target_IDCongDiem.notna().to_numpy() if 'target_IDCongDiem' in can else np.zeros(len(can),bool)
        target_record_match=float((j3.loc[eligible,'_merge']=='both').mean()) if eligible.any() else np.nan
        hp_sid_ag,_,_=_agreement(j3.IDSV_fold,j3.IDSV_raw);hp_mh_ag,_,_=_agreement(j3.IDMH_fold,j3.IDMH_raw);hp_sem_ag,_,_=_agreement(j3.target_NHHK,j3.NHHK)
        gm=pd.to_numeric(j3.target_grade10,errors='coerce').notna()&pd.to_numeric(j3.grade_raw,errors='coerce').notna()
        grade_match=float(np.isclose(pd.to_numeric(j3.loc[gm,'target_grade10']),pd.to_numeric(j3.loc[gm,'grade_raw']),atol=1e-9,rtol=0).mean()) if gm.any() else np.nan
        grade_n=int(gm.sum())
    else:
        target_record_match=hp_sid_ag=hp_mh_ag=hp_sem_ag=grade_match=np.nan;grade_n=0

    if student_frame is None:
        students=set(can.IDSV.dropna().map(_norm_id));student=_filter_table(source,'DDTBHK.csv',['IDSV','NHHK','DTBHK','NgayThem'],'IDSV',students,chunksize)
        student=dedupe_student_semester_latest(student)
    else:student=dedupe_student_semester_latest(student_frame)
    prev_match=np.nan;prev_n=0
    if len(student) and 'prev_semester_NHHK' in can and 'prev_semester_gpa10' in can:
        student['IDSV']=student.IDSV.map(_norm_id);student['NHHK']=student.NHHK.map(_norm_id);student['DTBHK']=pd.to_numeric(student.DTBHK,errors='coerce')
        jp=can.merge(student[['IDSV','NHHK','DTBHK']],left_on=['IDSV','prev_semester_NHHK'],right_on=['IDSV','NHHK'],how='left')
        pm=pd.to_numeric(jp.prev_semester_gpa10,errors='coerce').notna()&pd.to_numeric(jp.DTBHK,errors='coerce').notna()
        prev_match=float(np.isclose(pd.to_numeric(jp.loc[pm,'prev_semester_gpa10']),pd.to_numeric(jp.loc[pm,'DTBHK']),atol=1e-9,rtol=0).mean()) if pm.any() else np.nan;prev_n=int(pm.sum())

    report={'canonical_rows':int(len(can)),'registration_match_rate':reg_match,'registration_student_agreement':sid_ag,'registration_tohoc_agreement':to_ag,'registration_semester_agreement':sem_ag,
            'course_mapping_match_rate':course_map,'course_id_agreement':mh_ag,'course_semester_agreement':to_sem_ag,
            'target_record_match_rate':target_record_match,'target_student_agreement':hp_sid_ag,'target_course_agreement':hp_mh_ag,'target_semester_agreement':hp_sem_ag,
            'target_grade_match_rate':grade_match,'target_grade_compared':grade_n,'prev_gpa_match_rate':prev_match,'prev_gpa_compared':prev_n,'raw_source':source.describe()}
    core=[reg_match,sid_ag,to_ag,sem_ag,course_map,mh_ag,to_sem_ag,target_record_match,hp_sid_ag,hp_mh_ag,hp_sem_ag,grade_match]
    bad=[x for x in core if np.isfinite(x) and x<1.0-1e-12]
    print('[ALIGNMENT] canonical engineered folds/cache remain authoritative; raw DB is historical evidence only')
    print(f'[ALIGNMENT] rows={len(can):,} MKQDK={reg_match:.6f} IDSV={sid_ag:.6f} IDToHoc={to_ag:.6f} reg_sem={sem_ag:.6f} | KToHoc={course_map:.6f} IDMH={mh_ag:.6f} course_sem={to_sem_ag:.6f}')
    print(f'[ALIGNMENT] DDTBHP={target_record_match:.6f} target_grade={grade_match:.6f} n_grade={grade_n:,} | DDTBHK prev_GPA={prev_match:.6f} n_prev={prev_n:,}')
    if strict and bad:raise ValueError(f'Raw database does not exactly align with canonical expanding-fold identities/targets: {report}')
    return report



def structured_cache_compatible(path):
    p=Path(path)
    if not p.exists():return False
    try:
        obj=joblib.load(p)
        return int(obj.get('config',{}).get('schema_version',-1))==SCHEMA_VERSION
    except Exception:
        return False

def build_raw_index(cache_dir,data1_zip,data2_zip,faculties,folds,out_dir,force=False,chunksize=250000,database_dir=None,strict_alignment=True):
    out=Path(out_dir);out.mkdir(parents=True,exist_ok=True);manifest=out/'raw_index_manifest.json';source=RawTableSource(database_dir=database_dir,data1_zip=data1_zip,data2_zip=data2_zip)
    source_desc=source.describe()
    if manifest.exists() and not force:
        try:
            old=json.loads(manifest.read_text());ok=int(old.get('schema_version',-1))==SCHEMA_VERSION and set(faculties).issubset(set(old.get('faculties',[]))) and set(map(int,folds)).issubset(set(map(int,old.get('folds',[])))) and old.get('raw_source',{}).get('mode')==source_desc.get('mode')
            if source_desc.get('mode')=='directory':ok=ok and old.get('raw_source',{}).get('database_dir')==source_desc.get('database_dir')
        except Exception:ok=False
        if ok:
            print(f'[RAW-INDEX] reuse schema={SCHEMA_VERSION} {out} source={source_desc}');return out
        print(f'[RAW-INDEX] stale/incomplete/source-changed manifest -> rebuild {out}')
    students,target_courses=collect_needed_ids(cache_dir,faculties,folds)
    edge=[]
    mm=source.read_table('MMONTQ.csv',['IDMH','IDMHDK','LoaiTQ','NgayThem']);mm['IDMH']=mm.IDMH.map(_norm_id);mm['IDMHDK']=mm.IDMHDK.map(_norm_id);mm=mm[mm.IDMH.notna()&mm.IDMHDK.notna()];mm=mm[mm.IDMH.isin(target_courses)|mm.IDMHDK.isin(target_courses)];mm['available_nhhk']=mm.get('NgayThem',pd.Series(index=mm.index,dtype=object)).map(date_to_nhhk);edge += [(a,b,1,k,av) for a,b,k,av in mm[['IDMH','IDMHDK','LoaiTQ','available_nhhk']].itertuples(index=False,name=None)]
    mhtq=source.read_table('MMHTQ.csv',['IDMHTQ','IDMH','LoaiTQ','NgayThem']);mhtq['IDMH']=mhtq.IDMH.map(_norm_id);mhtq['added_group']=mhtq.get('NgayThem',pd.Series(index=mhtq.index,dtype=object)).map(date_to_nhhk)
    ct=source.read_table('MMHTQCT.csv',['IDMHTQ','IDMHDK','NgayThem']);ct['IDMHDK']=ct.IDMHDK.map(_norm_id);ct['added_detail']=ct.get('NgayThem',pd.Series(index=ct.index,dtype=object)).map(date_to_nhhk)
    rr=ct.merge(mhtq,on='IDMHTQ',how='inner');rr=rr[rr.IDMH.notna()&rr.IDMHDK.notna()];rr=rr[rr.IDMH.isin(target_courses)|rr.IDMHDK.isin(target_courses)];rr['available_nhhk']=rr[['added_group','added_detail']].max(axis=1,skipna=True);edge += [(a,b,2,k,av) for a,b,k,av in rr[['IDMH','IDMHDK','LoaiTQ','available_nhhk']].itertuples(index=False,name=None)]
    ed=pd.DataFrame(edge,columns=['src','dst','family','kind','available_nhhk']).drop_duplicates();course_pool=set(target_courses)
    if len(ed):course_pool.update(ed.src.tolist());course_pool.update(ed.dst.tolist())
    print(f'[RAW-INDEX] source={source_desc} relevant students={len(students):,} target_courses={len(target_courses):,} target+neighbor_courses={len(course_pool):,}')
    ddtbhk=_filter_table(source,'DDTBHK.csv',['IDSV','NHHK','NgayThem']+STUDENT_FEATURES,'IDSV',students,chunksize);ddtbhk=dedupe_student_semester_latest(ddtbhk);joblib.dump(ddtbhk,out/'student_semester.joblib',compress=3)
    hp_parts=[];cols=['IDCongDiem','IDSV','IDMH','NHHK','DiemHP','DiemHPCalc','SoTinChi','DatHP']
    for ch in source.read_chunks('DDTBHP.csv',cols,chunksize):
        sid=ch.IDSV.map(_norm_id);cid=ch.IDMH.map(_norm_id);m=sid.isin(students)|cid.isin(course_pool)
        if np.any(m):
            z=ch.loc[m].copy();z['IDSV']=sid[m].to_numpy();z['IDMH']=cid[m].to_numpy();hp_parts.append(z)
    hp=pd.concat(hp_parts,ignore_index=True) if hp_parts else pd.DataFrame(columns=cols);joblib.dump(hp,out/'course_history.joblib',compress=3)
    comp=_filter_table(source,'DDiemTPTG.csv',['IDSV','IDMH','NHHK','KyHieuTP','DiemTP','PhTrMH','DatTP'],'IDSV',students,chunksize);comp['IDMH']=comp.IDMH.map(_norm_id);joblib.dump(comp,out/'component_history.joblib',compress=3)
    kd=source.read_table('KDMMH.csv',['IDMH']+COURSE_STATIC);kd['IDMH']=kd.IDMH.map(_norm_id);kd=kd[kd.IDMH.isin(course_pool)].copy();joblib.dump(kd,out/'course_static.joblib',compress=3);joblib.dump(ed,out/'course_edges.joblib',compress=3)
    alignment=audit_canonical_alignment(cache_dir,faculties,folds,source,chunksize,hp_frame=hp,student_frame=ddtbhk,strict=strict_alignment);(out/'alignment_audit.json').write_text(json.dumps(alignment,indent=2,default=str))
    manifest.write_text(json.dumps({'schema_version':SCHEMA_VERSION,'students':len(students),'target_courses':len(target_courses),'course_pool':len(course_pool),'folds':list(map(int,folds)),'faculties':faculties,'raw_source':source_desc,'alignment':alignment},indent=2,default=str));print(f'[RAW-INDEX] wrote {out}')
    return out

class RawEvidenceIndex:
    def __init__(self,root):
        root=Path(root);self.student=joblib.load(root/'student_semester.joblib');self.hp=joblib.load(root/'course_history.joblib');self.comp=joblib.load(root/'component_history.joblib');self.course_static=joblib.load(root/'course_static.joblib');self.edges=joblib.load(root/'course_edges.joblib')
        self.student['NHHK']=pd.to_numeric(self.student.NHHK,errors='coerce');self.hp['NHHK']=pd.to_numeric(self.hp.NHHK,errors='coerce');self.comp['NHHK']=pd.to_numeric(self.comp.NHHK,errors='coerce')
        self.student_groups={k:g.sort_values('NHHK') for k,g in self.student.groupby('IDSV')};self.hp_student={k:g.sort_values('NHHK') for k,g in self.hp.groupby('IDSV')};self.hp_course={k:g.sort_values('NHHK') for k,g in self.hp.groupby('IDMH')};self.comp_student={k:g.sort_values('NHHK') for k,g in self.comp.groupby('IDSV')}
        self.static={r.IDMH:np.array([pd.to_numeric(getattr(r,c,np.nan),errors='coerce') for c in COURSE_STATIC],np.float32) for r in self.course_static.itertuples(index=False)}
        self.neigh=defaultdict(list);self.edge_type={}
        for s,d,fam,kind,av in self.edges[['src','dst','family','kind','available_nhhk']].itertuples(index=False,name=None):
            s=_norm_id(s);d=_norm_id(d);av=None if pd.isna(av) else int(av);fwd=encode_relation(fam,1,kind);rev=encode_relation(fam,-1,kind)
            self.edge_type[(s,d)]=prefer_edge(self.edge_type.get((s,d)),(fwd,av))
            self.edge_type[(d,s)]=prefer_edge(self.edge_type.get((d,s)),(rev,av))
        for (s,d),(t,av) in self.edge_type.items():self.neigh[s].append((d,t,av))
        self._course_vec_cache={};self._neighbors_cache={}
    def course_vec(self,c,target_sem):
        c=_norm_id(c);key=(c,int(float(target_sem)))
        if key in self._course_vec_cache:return self._course_vec_cache[key]
        st=self.static.get(c,np.zeros(len(COURSE_STATIC),np.float32));g=self.hp_course.get(c)
        if g is None:hist=np.zeros(len(COURSE_HIST),np.float32)
        else:
            z=g[g.NHHK<float(target_sem)];grade=coalesce_grade(z).dropna().to_numpy(float)
            if len(grade):hist=np.array([len(grade),grade.mean(),grade.std() if len(grade)>1 else 0.,(grade<5).mean(),(grade>=5).mean()],np.float32)
            else:hist=np.zeros(len(COURSE_HIST),np.float32)
        out=np.nan_to_num(np.concatenate([st,hist])).astype(np.float32);self._course_vec_cache[key]=out;return out
    def relation_type(self,a,b,target_sem=None):
        item=self.edge_type.get((_norm_id(a),_norm_id(b)))
        if item is None:return 0
        t,av=item
        return int(t) if target_sem is None or edge_is_available(av,target_sem) else 0
    def neighbors_for(self,c,target_sem):
        c=_norm_id(c);key=(c,int(float(target_sem)))
        if key in self._neighbors_cache:return self._neighbors_cache[key]
        z=[(d,t,av) for d,t,av in self.neigh.get(c,[]) if edge_is_available(av,target_sem)]
        out=sorted(z,key=lambda x:(0 if relation_is_direct(x[1]) else 1, x[2] if x[2] is not None else -1));self._neighbors_cache[key]=out;return out

def _row_tensor(index:RawEvidenceIndex,idsv,idmh,target,max_sem,max_neigh,max_rel,max_comp,memo=None):
    idsv=_norm_id(idsv);idmh=_norm_id(idmh);target=float(target);memo={} if memo is None else memo;stkey=(idsv,int(target))
    # E1
    if ('sem',stkey) in memo:ss,sm=memo[('sem',stkey)]
    else:
        sf=len(STUDENT_FEATURES)+2;ss=np.zeros((max_sem,sf),np.float32);sm=np.zeros(max_sem,bool);g=index.student_groups.get(idsv)
        z=g[g.NHHK<target].tail(max_sem) if g is not None else None
        if z is not None:
            for j,r in enumerate(z.itertuples(index=False)):
                vals=[pd.to_numeric(getattr(r,c,np.nan),errors='coerce') for c in STUDENT_FEATURES];rec=semester_distance(target,getattr(r,'NHHK'));ss[j]=np.nan_to_num(np.array(vals+[rec,1.],np.float32));sm[j]=1
        memo[('sem',stkey)]=(ss,sm)
    # E2
    ct=index.course_vec(idmh,target);cn=np.zeros((max_neigh,len(ct)),np.float32);cm=np.zeros(max_neigh,bool);crt=np.zeros(max_neigh,np.int64)
    for j,(nb,t,_av) in enumerate(index.neighbors_for(idmh,target)[:max_neigh]):cn[j]=index.course_vec(nb,target);cm[j]=1;crt[j]=int(t)
    # E3
    re=np.zeros((max_rel,len(REL_FEATURES)),np.float32);rm=np.zeros(max_rel,bool);rt=np.zeros(max_rel,np.int64);h=index.hp_student.get(idsv)
    hkey=('hp',stkey)
    if hkey in memo:z=memo[hkey]
    else:z=h[h.NHHK<target] if h is not None else None;memo[hkey]=z
    if z is not None and len(z):
        ranked=[]
        for r in z.itertuples(index=False):
            c=_norm_id(getattr(r,'IDMH'));rel=index.relation_type(c,idmh,target);rev=index.relation_type(idmh,c,target);same=(c==idmh);nh=pd.to_numeric(getattr(r,'NHHK',np.nan),errors='coerce')
            ranked.append((evidence_priority(same,rel,rev),-float(nh) if not pd.isna(nh) else np.inf,r,rel,rev,same))
        ranked.sort(key=lambda x:(x[0],x[1]))
        for j,(_pri,_neg,r,rel,rev,same) in enumerate(ranked[:max_rel]):
            grade=pd.to_numeric(getattr(r,'DiemHP',np.nan),errors='coerce');calc=pd.to_numeric(getattr(r,'DiemHPCalc',np.nan),errors='coerce');grade=calc if pd.isna(grade) else grade;credits=pd.to_numeric(getattr(r,'SoTinChi',np.nan),errors='coerce');passed=pd.to_numeric(getattr(r,'DatHP',np.nan),errors='coerce')
            if pd.isna(passed) and not pd.isna(grade):passed=float(grade>=5)
            rec=semester_distance(target,getattr(r,'NHHK'));re[j]=np.nan_to_num(np.array([grade,credits,passed,rec,float(same),float(relation_is_forward(rel)),float(relation_is_forward(rev))],np.float32));rm[j]=1;rt[j]=13 if same else int(rel or rev)
    # E4
    ce=np.zeros((max_comp,len(COMP_FEATURES)),np.float32);cma=np.zeros(max_comp,bool);ctype=np.zeros(max_comp,np.int64);h=index.comp_student.get(idsv)
    hkey=('comp',stkey)
    if hkey in memo:z=memo[hkey]
    else:z=h[h.NHHK<target] if h is not None else None;memo[hkey]=z
    if z is not None and len(z):
        ranked=[]
        for r in z.itertuples(index=False):
            c=_norm_id(getattr(r,'IDMH'));rel=index.relation_type(c,idmh,target);rev=index.relation_type(idmh,c,target);same=(c==idmh);nh=pd.to_numeric(getattr(r,'NHHK',np.nan),errors='coerce')
            ranked.append((evidence_priority(same,rel,rev),-float(nh) if not pd.isna(nh) else np.inf,r,rel,rev,same))
        ranked.sort(key=lambda x:(x[0],x[1]))
        for j,(_pri,_neg,r,rel,rev,same) in enumerate(ranked[:max_comp]):
            grade=pd.to_numeric(getattr(r,'DiemTP',np.nan),errors='coerce');wt=pd.to_numeric(getattr(r,'PhTrMH',np.nan),errors='coerce');passed=pd.to_numeric(getattr(r,'DatTP',np.nan),errors='coerce')
            if pd.isna(passed) and not pd.isna(grade):passed=float(grade>=5)
            rec=semester_distance(target,getattr(r,'NHHK'));related=float(rel>0 or rev>0);ce[j]=np.nan_to_num(np.array([grade,wt,passed,rec,float(same),related],np.float32));cma[j]=1;typ=str(getattr(r,'KyHieuTP','OTHER')).upper();ctype[j]=COMP_MAP.get(typ,COMP_MAP['OTHER'])
    return ss.copy(),sm.copy(),ct,cn,cm,crt,re,rm,rt,ce,cma,ctype

def materialize_frame(frame,index,max_sem=6,max_neigh=8,max_rel=16,max_comp=24):
    n=len(frame);cd=len(COURSE_STATIC)+len(COURSE_HIST)
    arr={'student_seq':np.zeros((n,max_sem,len(STUDENT_FEATURES)+2),np.float16),'student_mask':np.zeros((n,max_sem),bool),
         'course_target':np.zeros((n,cd),np.float16),'course_neighbors':np.zeros((n,max_neigh,cd),np.float16),'course_neighbor_mask':np.zeros((n,max_neigh),bool),'course_relation_type':np.zeros((n,max_neigh),np.int8),
         'relation_events':np.zeros((n,max_rel,len(REL_FEATURES)),np.float16),'relation_mask':np.zeros((n,max_rel),bool),'relation_type':np.zeros((n,max_rel),np.int8),
         'component_events':np.zeros((n,max_comp,len(COMP_FEATURES)),np.float16),'component_mask':np.zeros((n,max_comp),bool),'component_type':np.zeros((n,max_comp),np.int8)}
    memo={}
    for i,r in enumerate(frame[['IDSV','IDMH','target_NHHK']].itertuples(index=False,name=None)):
        vals=_row_tensor(index,*r,max_sem,max_neigh,max_rel,max_comp,memo)
        for k,v in zip(arr.keys(),vals):arr[k][i]=v
    return arr

def ensure_fold_structured(cache_obj,index,out_path,max_sem=6,max_neigh=8,max_rel=16,max_comp=24,force=False):
    p=Path(out_path)
    if p.exists() and not force:
        try:
            old=joblib.load(p)
            if int(old.get('config',{}).get('schema_version',-1))==SCHEMA_VERSION:return old
        except Exception:pass
        print(f'[STRUCTURED] stale cache -> rebuild {p}')
    mask=np.asarray(cache_obj['cf_covered_mask'],bool)&np.isfinite(np.asarray(cache_obj['cf_probabilities'],float)).all(1);idx=np.where(mask)[0]
    tr=cache_obj['train_frame'].iloc[idx].reset_index(drop=True);te=cache_obj['test_frame'].reset_index(drop=True)
    obj={'train_row_index':idx.astype(np.int32),'train':materialize_frame(tr,index,max_sem,max_neigh,max_rel,max_comp),'target_row_index':np.arange(len(te),dtype=np.int32),'target':materialize_frame(te,index,max_sem,max_neigh,max_rel,max_comp),
         'config':{'schema_version':SCHEMA_VERSION,'max_semesters':max_sem,'max_neighbors':max_neigh,'max_relation_events':max_rel,'max_component_events':max_comp,'course_dim':len(COURSE_STATIC)+len(COURSE_HIST),'student_dim':len(STUDENT_FEATURES)+2,'relation_dim':len(REL_FEATURES),'component_dim':len(COMP_FEATURES),'component_types':COMP_TYPES}}
    p.parent.mkdir(parents=True,exist_ok=True);joblib.dump(obj,p,compress=3);return obj
