"""Suite28 canonical AQ adapter. All outcome-derived course statistics are rebuilt as-of query semester."""
from __future__ import annotations
import dataclasses, collections, pathlib, re, warnings
import numpy as np
import pandas as pd
import joblib

COURSE_STATIC=['SoTinChi','SoTietMH','SoTietTuan','SiSoMax','SiSoMin','KhgTinhDTB','IsMHTN','IsMonKT','IsMHKN','IsMHDacThu','IsGDTiengAnh','IsMonNgoaiNgu']
COURSE_HIST=['hist_count','hist_mean','hist_std','hist_fail_rate','hist_pass_rate']
STUDENT_PREFIX=('prev_semester_','prev_cumulative_','prev_failed_','prev_outstanding_','prev_passed_','hist_semester_','recent3_semester_','hist_warning_','recent3_warning_','hist_cumulative_','hist_registered_','target_semester_registered_','target_semester_difficulty_weighted_workload')
EXCLUDE_SUBSTR=('target_grade','hist_course','recent3_course','target_course_','course_status','prereq','component','IDSV','IDMH','future','label','target_status')

def semester_ordinal(v):
    try:
        x=int(float(v)); q=x%10
        if q not in (1,2,3): raise ValueError(v)
        return x//10*3+q-1
    except Exception as ex: raise ValueError(f'Invalid academic semester {v!r}') from ex

def discover(cache_dir,fold=1):
    suffix=f'_fold_{int(fold)}.joblib';a=sorted(x.name[:-len(suffix)] for x in pathlib.Path(cache_dir).glob('*'+suffix))
    if not a: raise FileNotFoundError(f'No canonical caches in {cache_dir}')
    return a

def valid_context_features(names):
    return [n for n in names if n.startswith(STUDENT_PREFIX) and not any(b in n.lower() for b in EXCLUDE_SUBSTR)]

def _selected(d):
    p=np.asarray(d['cf_probabilities'],float); mask=np.asarray(d['cf_covered_mask'],bool)&np.isfinite(p).all(1)
    return np.flatnonzero(mask)

def _load(path):
    return joblib.load(path)  # Compressed caches are not mmap-compatible.

@dataclasses.dataclass
class Rows:
    raw:np.ndarray; static:np.ndarray; y:np.ndarray; grade:np.ndarray
    student:np.ndarray; course:np.ndarray; faculty:np.ndarray; semester:np.ndarray; source_row:np.ndarray
    committee:np.ndarray; faculty_names:list[str]; raw_names:list[str]
    raw_all:np.ndarray|None=None; full_names:list[str]|None=None
    def __len__(self): return len(self.y)
    def take(self,ids):
        ix=np.asarray(ids); fields={f.name:getattr(self,f.name) for f in dataclasses.fields(self)}
        for k,v in fields.items():
            if isinstance(v,np.ndarray):fields[k]=v[ix]
        return Rows(**fields)

def _normal_ids(values,label):
    raw=pd.Series(values).astype('string').str.strip()
    invalid=raw.isna()|raw.str.lower().isin(['','none','null','nan','<na>']).fillna(True)
    if invalid.any():raise ValueError(f'{label} missing/invalid IDs: {int(invalid.sum())} rows')
    # Normalize spreadsheet numeric ID serialization 123.0 -> 123 but keep leading-zero strings.
    def normalized(x):
        s=str(x)
        if re.fullmatch(r'[0-9]+\.0+',s):return s.split('.')[0]
        return s
    return raw.map(normalized).to_numpy(dtype=str)

def _grade(frame,y):
    if 'target_grade10' not in frame: raise KeyError('target_grade10 grade required on TRAIN frames')
    z=pd.to_numeric(frame['target_grade10'],errors='coerce').to_numpy(float)
    legal=np.isfinite(z)&(z>=0)&(z<=10)
    contradictory=legal&((z<5)!=(np.asarray(y)==1))
    if contradictory.any():raise ValueError(f'Grade versus failure-label mismatch: {contradictory.sum()}')
    return np.where(legal,z,np.nan).astype(np.float32)

def load_fold(cache_dir,structured_dir,faculties,fold,context_names=None):
    root=pathlib.Path(cache_dir);sr=pathlib.Path(structured_dir)
    caches={f:_load(root/f'{f}_fold_{fold}.joblib') for f in faculties}
    shared=set(caches[faculties[0]]['feature_names'])
    for f in faculties[1:]:shared &= set(caches[f]['feature_names'])
    first=list(caches[faculties[0]]['feature_names']); common=[s for s in first if s in shared]
    if context_names is None:context_names=valid_context_features(common)
    if not context_names:raise RuntimeError('No permitted student-context features. Review field ownership.')
    out=[[],[]]
    for fi,f in enumerate(faculties):
        d=caches[f];s=_load(sr/f'{f}_fold_{fold}.joblib');idx=_selected(d)
        if not np.array_equal(np.asarray(s['train_row_index'],int),idx):raise ValueError(f'Canonical/structured alignment mismatch for {f} F{fold}')
        key={n:i for i,n in enumerate(d['feature_names'])}
        if any(n not in key for n in context_names):raise ValueError(f'Context schema differs on {f}')
        for part,(arr,fr,yy,indices,comm) in enumerate([
            (s['train'],d['train_frame'].iloc[idx].reset_index(drop=True),np.asarray(d['ytr'])[idx],idx,np.asarray(d['cf_probabilities'])[idx].mean(1)),
            (s['target'],d['test_frame'].reset_index(drop=True),np.asarray(d['yte']),np.arange(len(d['yte'])),np.asarray(d.get('qte',np.asarray(d['pte']).mean(1))).reshape(-1))]):
            y=np.asarray(yy,np.int8)
            z=np.asarray(arr['course_target'],np.float32)
            if z.shape!=(len(y),17):raise ValueError(f'Course descriptor schema mismatch {f} {part}: {z.shape}')
            raw=np.asarray(d['Xtr'][idx] if part==0 else d['Xte'],np.float32)[:,[key[n] for n in context_names]]
            if 'target_grade10' in fr:
                g=_grade(fr,y)
            elif part==1:g=np.full(len(y),np.nan,np.float32)
            else:raise ValueError('Missing training grade')
            if 'IDSV' not in fr or 'IDMH' not in fr or 'target_NHHK' not in fr:raise ValueError('Missing canonical IDs')
            sem=np.asarray([semester_ordinal(v) for v in fr['target_NHHK']],np.int32)
            raw_all=np.asarray(d['Xtr'][idx] if part==0 else d['Xte'],np.float32)[:,[key[n] for n in common]]
            out[part].append(dict(raw=raw,raw_all=raw_all,static=z[:,:12],y=y,grade=g,student=_normal_ids(fr['IDSV'],'IDSV'),course=_normal_ids(fr['IDMH'],'IDMH'),faculty=np.full(len(y),fi,np.int16),semester=sem,source_row=np.asarray(indices,np.int64),committee=np.asarray(comm,np.float32)))
    def pack(chunks):
        return Rows(**{k:np.concatenate([x[k] for x in chunks]) for k in chunks[0]},faculty_names=list(faculties),raw_names=list(context_names),full_names=list(common))
    hist,tgt=pack(out[0]),pack(out[1]);return hist,tgt

def context_audit(rows:Rows,tol=1e-4,limit=12):
    """Every student-semester must have one shared state. Inconsistent columns are excluded in preflight."""
    rng=np.random.default_rng(3)
    groups=collections.defaultdict(list)
    for i,(f,s,t) in enumerate(zip(rows.faculty,rows.student,rows.semester)):
        groups[(int(f),s,int(t))].append(i)
    n=len(rows.raw_names); bad=np.zeros(n,int);checked=0
    for ids in groups.values():
        if len(ids)<2:continue
        a=rows.raw[np.asarray(ids[:20])];ok=np.isfinite(a)
        for j in range(n):
            v=a[ok[:,j],j]
            if len(v)>1 and float(np.max(v)-np.min(v))>tol:bad[j]+=1
        checked+=1
    return {'groups_checked':checked,'inconsistent':{rows.raw_names[j]:int(bad[j]) for j in range(n) if bad[j]},'allowed':[rows.raw_names[j] for j in range(n) if bad[j]==0]}

def fit_preprocessor(train:Rows):
    mean=np.nanmean(np.where(np.isfinite(train.raw),train.raw,np.nan),0)
    mean=np.nan_to_num(mean,nan=0.,posinf=0.,neginf=0.).astype(np.float32)
    a=np.where(np.isfinite(train.raw),train.raw,mean)
    std=np.maximum(a.std(0),1e-4).astype(np.float32)
    stat=np.asarray(train.static,np.float32);smean=np.nan_to_num(np.nanmean(np.where(np.isfinite(stat),stat,np.nan),0),nan=0.).astype(np.float32)
    sstd=np.maximum(np.nan_to_num(np.nanstd(np.where(np.isfinite(stat),stat,np.nan),0),nan=1.),1e-4).astype(np.float32)
    return {'context_mean':mean,'context_std':std,'static_mean':smean,'static_std':sstd}

def apply_preprocessor(rows:Rows,stats,keep_context=True):
    z=np.where(np.isfinite(rows.raw),rows.raw,stats['context_mean']);context=np.clip((z-stats['context_mean'])/stats['context_std'],-12,12)
    missing=(~np.isfinite(rows.raw)).astype(np.float32)
    context=np.concatenate([context,missing],1).astype(np.float32)
    z=np.where(np.isfinite(rows.static),rows.static,stats['static_mean']);static=np.clip((z-stats['static_mean'])/stats['static_std'],-12,12)
    smissing=(~np.isfinite(rows.static)).astype(np.float32)
    return context,np.concatenate([static,smissing],1).astype(np.float32)

class CourseBank:
    """Frozen TRAIN outcome bank; strict < query semester, faculty-local."""
    def __init__(self,train:Rows,min_id_count=3):
        grouped=collections.defaultdict(list)
        for i in range(len(train)):
            grouped[(int(train.faculty[i]),str(train.course[i]))].append(i)
        self.groups={};self.ids={};self.idcounts={};self.min_id_count=min_id_count
        for key,idx in grouped.items():
            order=np.asarray(sorted(idx,key=lambda j:(int(train.semester[j]),int(train.source_row[j]))),int)
            sem=train.semester[order];grade=train.grade[order];yy=train.y[order]
            valid=np.isfinite(grade)
            cs=np.cumsum(valid);sumg=np.cumsum(np.where(valid,grade,0.));sum2=np.cumsum(np.where(valid,grade**2,0.));fails=np.cumsum(yy);counts=np.arange(1,len(order)+1)
            self.groups[key]=(sem,counts,cs,sumg,sum2,fails)
            if len(order)>=min_id_count:self.ids[key]=len(self.ids)+1
            self.idcounts[key]=len(order)
    def query(self,rows:Rows):
        n=len(rows); features=np.zeros((n,5),np.float32);prior=np.zeros(n,np.float32);cid=np.zeros(n,np.int64)
        for i in range(n):
            key=(int(rows.faculty[i]),str(rows.course[i]));g=self.groups.get(key)
            if g is None:continue
            sem,ct,ng,sm,sm2,fail=g;k=int(np.searchsorted(sem,rows.semester[i],side='left'));prior[i]=k
            if k==0:continue
            a=int(ng[k-1]);s=float(sm[k-1]);sd=max(float(sm2[k-1])/max(a,1)-(s/max(a,1))**2,0.)**.5
            features[i]=[k,s/max(a,1) if a else 0.,sd,float(fail[k-1])/k,1.-float(fail[k-1])/k]
            if key in self.ids:cid[i]=self.ids[key]
        return features,prior,cid
    @property
    def vocab_size(self):return len(self.ids)+1

def build_episode_index(rows:Rows,bank:Rows|None=None,targets:Rows|None=None,history_embargo=0):
    """Separate faculty/student episodes and simultaneous semester groups. No order within term."""
    episodes=collections.defaultdict(lambda:collections.defaultdict(lambda:{'hist':[],'target':[]}))
    if bank is not None:
        for i,(f,s,t) in enumerate(zip(bank.faculty,bank.student,bank.semester)):
            episodes[(int(f),str(s))][int(t)]['hist'].append(i)
    if targets is not None:
        for i,(f,s,t) in enumerate(zip(targets.faculty,targets.student,targets.semester)):
            episodes[(int(f),str(s))][int(t)]['target'].append(i)
    return [sorted([(t,d) for t,d in v.items()],key=lambda q:q[0]) for v in episodes.values()]
