"""AQ canonical cache adapter and strict chronology/data-provenance controls."""
from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
import hashlib, json
import numpy as np
import pandas as pd
import joblib

COURSE_STATIC = ['SoTinChi','SoTietMH','SoTietTuan','SiSoMax','SiSoMin','KhgTinhDTB',
                 'IsMHTN','IsMonKT','IsMHKN','IsMHDacThu','IsGDTiengAnh','IsMonNgoaiNgu']
COURSE_HIST = ['hist_count','hist_mean','hist_std','hist_fail_rate','hist_pass_rate']
COURSE_FEATURES = COURSE_STATIC + COURSE_HIST

def note(tag, **kwargs):
    print(f'[{tag}] '+' '.join(f'{k}={v:.6f}' if isinstance(v, float) else f'{k}={v}' for k,v in kwargs.items()), flush=True)

def stable_seed(*parts):
    return int.from_bytes(hashlib.sha256('|'.join(map(str,parts)).encode()).digest()[:4],'little')

def semester_ordinal(v):
    try:
        z=int(float(v)); term=z%10
        if term not in (1,2,3): raise ValueError(f'Invalid academic term in {v}')
        return (z//10)*3+term-1
    except Exception as e:
        raise ValueError(f'Invalid target_NHHK semester {v!r}') from e

def names_in(cache_dir,fold):
    suffix=f'_fold_{int(fold)}.joblib'
    names=[p.name[:-len(suffix)] for p in sorted(Path(cache_dir).glob(f'*{suffix}'))]
    if not names: raise FileNotFoundError(f'No AQ canonical caches for fold {fold} in {cache_dir}')
    return names

@dataclass
class Part:
    faculty:str
    fold:int
    raw:np.ndarray
    raw_names:list[str]
    course:np.ndarray
    y:np.ndarray
    semester:np.ndarray
    student:np.ndarray
    course_id:np.ndarray
    row_index:np.ndarray
    committee:np.ndarray
    raw_nan:np.ndarray
    def __len__(self): return len(self.y)
    def sub(self,mask):
        ids=np.asarray(mask) if np.asarray(mask).dtype.kind in 'iu' else np.flatnonzero(mask)
        return Part(self.faculty,self.fold,self.raw[ids],self.raw_names,self.course[ids],self.y[ids],self.semester[ids],self.student[ids],self.course_id[ids],self.row_index[ids],self.committee[ids],self.raw_nan[ids])

@dataclass
class FoldData:
    history:dict[str,Part]
    target:dict[str,Part]
    common_names:list[str]
    fold:int
    def faculties(self): return list(self.history)


def _sem(frame,label):
    if 'target_NHHK' not in frame: raise KeyError(f'{label}: missing target_NHHK')
    return np.array([semester_ordinal(s) for s in frame['target_NHHK']],dtype=np.int64)

def _part(d,sc,faculty,fold,train):
    if train:
        if 'cf_probabilities' not in d or 'cf_covered_mask' not in d:
            raise KeyError('Canonical cross-fitted coverage missing; cannot silently use all training rows')
        cf=np.asarray(d['cf_probabilities'],float)
        valid=np.asarray(d['cf_covered_mask'],bool) & np.isfinite(cf).all(axis=1)
        ix=np.flatnonzero(valid)
        src=np.asarray(sc['train_row_index'],int)
        if not np.array_equal(src,ix): raise ValueError(f'{faculty} F{fold}: canonical/structured train row indices differ')
        raw=np.asarray(d['Xtr'])[ix]; y=np.asarray(d['ytr'])[ix]
        frame=d['train_frame'].iloc[ix].reset_index(drop=True)
        course=np.asarray(sc['train']['course_target']);committee=np.asarray(cf[ix]).mean(axis=1)
    else:
        raw=np.asarray(d['Xte']);y=np.asarray(d['yte']);ix=np.arange(len(y))
        frame=d['test_frame'].reset_index(drop=True)
        course=np.asarray(sc['target']['course_target'])
        if 'qte' in d: committee=np.asarray(d['qte'],float)
        elif 'pte' in d: committee=np.asarray(d['pte'],float).mean(axis=1)
        else: committee=np.full(len(y),np.nan)
        if committee.ndim==2:committee=committee.mean(axis=1)
    if raw.ndim!=2 or len(raw)!=len(y) or len(frame)!=len(y) or course.shape!=(len(y),len(COURSE_FEATURES)):
        raise ValueError(f'{faculty} F{fold}: invalid cache/structured input shape')
    if not np.isin(y,[0,1]).all():raise ValueError(f'{faculty} F{fold}: target not binary')
    names=list(d['feature_names'])
    if len(set(names))!=len(names) or len(names)!=raw.shape[1]:raise ValueError(f'{faculty}: duplicate or mismatched raw feature names')
    for col in ('IDSV','IDMH'):
        if col not in frame:raise KeyError(f'{faculty}: missing {col}')
    raw=np.asarray(raw,np.float32)
    nan=~np.isfinite(raw)
    return Part(faculty,fold,raw,names,np.asarray(course,np.float32),np.asarray(y,np.int8),
        _sem(frame,faculty),frame['IDSV'].astype(str).to_numpy(),frame['IDMH'].astype(str).to_numpy(),
        np.asarray(ix,int),np.asarray(committee,np.float32),nan)

def load_fold(cache_dir,structured_dir,faculties,fold,expected_common=None,availability_gap=0):
    hs,qs={},{}
    for faculty in faculties:
        cf=Path(cache_dir)/f'{faculty}_fold_{fold}.joblib';sp=Path(structured_dir)/f'{faculty}_fold_{fold}.joblib'
        if not cf.exists() or not sp.exists():raise FileNotFoundError(f'Missing cache or structured file: {cf} / {sp}')
        d=joblib.load(cf);sc=joblib.load(sp)
        hs[faculty]=_part(d,sc,faculty,fold,True)
        qs[faculty]=_part(d,sc,faculty,fold,False)
        if len(hs[faculty])==0 or len(qs[faculty])==0:raise ValueError(f'{faculty} F{fold}: empty history/target')
        if np.max(hs[faculty].semester)>=np.min(qs[faculty].semester)-availability_gap:
            raise ValueError(f'{faculty} F{fold}: future or unavailable history labels at prediction origin')
        duplicate_count=len(qs[faculty])-len(set(zip(qs[faculty].student,qs[faculty].course_id,qs[faculty].semester)))
        if duplicate_count: note('DATA_WARNING',faculty=faculty,fold=fold,duplicate_student_course_semester=duplicate_count,policy='retain_canonical_row_index')
    if max(np.max(x.semester) for x in hs.values()) >= min(np.min(x.semester) for x in qs.values())-availability_gap:
        raise ValueError('Cross-faculty future-history leak: GLOBAL historical/target origin assertion failed')
    common=set(hs[faculties[0]].raw_names)
    for x in hs.values():common.intersection_update(x.raw_names)
    common=[x for x in hs[faculties[0]].raw_names if x in common]
    if expected_common is not None and common!=expected_common:
        raise ValueError(f'Common feature schema drift across folds: expected={len(expected_common)}, actual={len(common)}')
    return FoldData(hs,qs,common,fold)


def frame_matrix(part,common_names,native_names=None,include_mask=False,mask_type='observed'):
    """Donors carry only verified *common* raw fields; native target extras unavailable in donors.
    Cache NaNs are observed only in cache, and are NOT proof of source missingness.
    """
    fmap={n:i for i,n in enumerate(part.raw_names)}
    if any(n not in fmap for n in common_names):raise ValueError(f'{part.faculty}: common schema mismatch')
    ix=[fmap[n] for n in common_names];common=part.raw[:,ix]
    base=np.concatenate([np.nan_to_num(common,nan=0,posinf=0,neginf=0),np.nan_to_num(part.course,nan=0,posinf=0,neginf=0)],axis=1)
    mask=np.concatenate([part.raw_nan[:,ix],~np.isfinite(part.course)],axis=1).astype(np.float32)
    if native_names:
        extra=np.full((len(part),len(native_names)),np.nan,dtype=np.float32)
        for j,n in enumerate(native_names):
            if n in fmap:
                extra[:,j]=np.where(np.isfinite(part.raw[:,fmap[n]]),part.raw[:,fmap[n]],np.nan)
        base=np.concatenate([base,extra],axis=1)
        mask=np.concatenate([mask,~np.isfinite(extra)],axis=1).astype(np.float32)
    if include_mask:
        if mask_type=='observed': appended=mask
        elif mask_type=='naive_zero':appended=(np.nan_to_num(base,nan=0)==0).astype(np.float32)
        elif mask_type=='placebo':
            # Constant dimension-matched placebo: never use target labels or random target masks.
            appended=np.zeros_like(mask)
        else:raise ValueError(mask_type)
        base=np.concatenate([base,appended],axis=1)
    return np.asarray(base,np.float32)


def feature_schema(data,target,variant='common',include_mask=False,mask_type='observed'):
    """All donor native-only columns are NaN even if a donor has same-named but unverified fields."""
    common=data.common_names
    native=[x for x in data.history[target].raw_names if x not in set(common)] if variant=='native' else []
    names=list(common)+['course::'+n for n in COURSE_FEATURES]+['native::'+x for x in native]
    if include_mask:names+=['indicator::'+x for x in names]
    def matrix(part):
        m=frame_matrix(part,common,native if part.faculty==target else None,include_mask=False)
        if part.faculty!=target and native:m=np.column_stack([m,np.full((len(part),len(native)),np.nan,np.float32)])
        if include_mask:
            if mask_type=='observed':
                # For target: cache-nan; for donor: unavailability of native columns is explicit.
                idx={n:i for i,n in enumerate(part.raw_names)}
                cm=part.raw_nan[:,[idx[n] for n in common]].astype(np.float32)
                native_mask=(part.raw_nan[:,[idx[n] for n in native]].astype(np.float32) if part.faculty==target and native else np.ones((len(part),len(native)),np.float32))
                indicator=np.concatenate([cm,(~np.isfinite(part.course)).astype(np.float32),native_mask],axis=1)
            elif mask_type=='naive_zero':indicator=(np.nan_to_num(m,nan=0)==0).astype(np.float32)
            elif mask_type=='placebo':indicator=np.zeros_like(m)
            else:raise ValueError(mask_type)
            m=np.concatenate([m,indicator],axis=1)
        assert m.shape[1]==len(names)
        return m
    return matrix,names


def status_audit(data,faculty):
    h=data.history[faculty]; q=data.target[faculty]
    missing=float(h.raw_nan.mean())
    zeros=float((h.raw==0).mean())
    return {'fold':data.fold,'faculty':faculty,'raw_columns':len(h.raw_names),
        'common_raw_columns':len(data.common_names),'native_raw_columns':len(h.raw_names)-len(data.common_names),
        'history_rows':len(h),'history_failures':int(h.y.sum()),'target_rows':len(q),
        'target_failures':int(q.y.sum()),'target_prevalence':float(q.y.mean()),
        'raw_cache_nonfinite_rate':missing,'raw_exact_zero_rate':zeros,
        'history_latest_semester':int(h.semester.max()),'target_first_semester':int(q.semester.min()),
        'availability_provenance':'semester_proxy; source release timestamps not verified'}

def feature_ledger(data):
    """Audit observed CACHE states only; never invent database provenance categories."""
    rows=[]
    for f in data.faculties():
        for section,part in [('history',data.history[f]),('target',data.target[f])]:
            for j,name in enumerate(part.raw_names):
                v=np.asarray(part.raw[:,j],float)
                rows.append({'fold':data.fold,'faculty':f,'section':section,'field':name,
                   'field_group':'shared_raw' if name in data.common_names else 'faculty_native_raw',
                   'count':len(v),'finite':int(np.isfinite(v).sum()),
                   'cache_nonfinite':int((~np.isfinite(v)).sum()),
                   'cache_nonfinite_rate':float((~np.isfinite(v)).mean()),
                   'exact_zero':int((v==0).sum()),
                   'exact_zero_rate':float((v==0).mean()),
                   'source_observation_status':'UNKNOWN_NOT_RECOVERABLE_FROM_CACHE_ALONE',
                   'time_availability':'SEMESTER_PROXY_ONLY'})
            for j,name in enumerate(COURSE_FEATURES):
                v=np.asarray(part.course[:,j],float)
                rows.append({'fold':data.fold,'faculty':f,'section':section,'field':'course::'+name,
                   'field_group':'structured_course','count':len(v),'finite':int(np.isfinite(v).sum()),
                   'cache_nonfinite':int((~np.isfinite(v)).sum()),'cache_nonfinite_rate':float((~np.isfinite(v)).mean()),
                   'exact_zero':int((v==0).sum()),'exact_zero_rate':float((v==0).mean()),
                   'source_observation_status':'UNKNOWN_NOT_RECOVERABLE_FROM_CACHE_ALONE',
                   'time_availability':'SEMESTER_PROXY_ONLY'})
    return pd.DataFrame(rows)

def student_overlap_ledger(data):
    """Source overlap is descriptive, not automatic deduplication of legitimate past registrations."""
    rows=[]; sets={f:set(data.history[f].student) for f in data.faculties()}
    for a in data.faculties():
        for b in data.faculties():
            if a==b:continue
            n=len(sets[a]&sets[b]);rows.append({'fold':data.fold,'source':a,'target':b,
                 'unique_source_students':len(sets[a]),'unique_target_students':len(sets[b]),
                 'shared_student_ids':n,'shared_fraction_target':n/max(1,len(sets[b])),
                 'key_semantics':'assumes IDSV is a globally comparable identifier'})
    return pd.DataFrame(rows)
