"""Explicit source contracts, availability checks, and deterministic historical prefixes."""
from __future__ import annotations
import hashlib
import json
from collections.abc import Mapping
from pathlib import Path
import numpy as np
import pandas as pd

IDENTIFIERS = ['row_id','student_id','course_id','semester','faculty']


def read_table(path):
    # IDs are never passed through floating point, including zero-padded numeric IDs.
    p=Path(path)
    if p.suffix.lower()=='.parquet':
        return pd.read_parquet(p)
    return pd.read_csv(p,dtype=str,keep_default_na=False).replace('',np.nan)


def utc(values, name):
    out=pd.to_datetime(values,utc=True,errors='coerce')
    if pd.isna(out).any():
        raise ValueError(f'{name}: missing or invalid UTC timestamp; provide audited publication/availability metadata')
    return out


def sha256(path):
    h=hashlib.sha256()
    with Path(path).open('rb') as f:
        for b in iter(lambda:f.read(1<<20),b''):h.update(b)
    return h.hexdigest()


class EventCorpus:
    def __init__(self,events,calendar,availability_mode='strict',embargo_terms=1):
        self.calendar=calendar.copy()
        req={'semester','order','origin_at','labels_available_at'}
        if not req<=set(calendar):raise ValueError(f'Calendar requires {sorted(req)}')
        self.calendar['semester']=self.calendar.semester.astype(str)
        raw_order=pd.to_numeric(self.calendar['order'],errors='raise')
        if not np.isfinite(raw_order).all() or (raw_order%1!=0).any():raise ValueError('Calendar order must be a finite integer rank')
        self.calendar['order']=raw_order.astype(int)
        self.calendar['origin_at']=utc(self.calendar.origin_at,'calendar.origin_at')
        self.calendar['labels_available_at']=utc(self.calendar.labels_available_at,'calendar.labels_available_at')
        self.calendar=self.calendar.sort_values('order').reset_index(drop=True)
        if self.calendar.semester.duplicated().any() or self.calendar.order.duplicated().any():raise ValueError('Calendar semester/order must be unique')
        if not self.calendar.origin_at.is_monotonic_increasing or self.calendar.origin_at.duplicated().any():raise ValueError('Calendar origins must strictly increase with order')
        if (self.calendar.labels_available_at<=self.calendar.origin_at).any():raise ValueError('Calendar grade availability must follow origin')
        self.orders=self.calendar.set_index('semester')['order'].to_dict()
        self.origins=self.calendar.set_index('semester')['origin_at'].to_dict()
        self.releases=self.calendar.set_index('semester')['labels_available_at'].to_dict()
        e=events.copy(); required={'event_id','student_id','course_id','semester','grade','credits'}
        if not required<=set(e):raise ValueError(f'Events missing {sorted(required-set(e))}; use extract-events with explicit raw mapping')
        for c in ['event_id','student_id','course_id','semester']:
            if e[c].isna().any():raise ValueError(f'Events: missing {c}')
            e[c]=e[c].astype(str)
        if not set(e.semester)<=set(self.orders):raise ValueError('Event semester missing from calendar')
        if availability_mode=='calendar_embargo':
            if embargo_terms<1:raise ValueError('Proxy requires at least one embargo semester')
            terms=self.calendar.semester.tolist();origin=self.calendar.origin_at.tolist()
            avail={t:(origin[i+embargo_terms+1] if i+embargo_terms+1<len(terms) else pd.Timestamp.max.tz_localize('UTC')) for i,t in enumerate(terms)}
            e['available_at']=e.semester.map(avail)
        elif availability_mode not in ('strict','semester_order'):raise ValueError('availability_mode must be strict, semester_order or calendar_embargo')
        elif 'available_at' not in e:raise ValueError('Strict mode requires events.available_at; entry time alone is not certified publication')
        e['available_at']=utc(e.available_at,'events.available_at')
        if (e.available_at<=e.semester.map(self.origins)).any():raise ValueError('Event availability must follow its semester origin')
        original=len(e);e=e.drop_duplicates()
        if e.event_id.duplicated().any():raise ValueError('Event identity conflict: duplicate event_id with differing contents; reconcile revisions/attempts')
        raw=e.grade.copy();e['grade']=pd.to_numeric(e.grade.replace('',np.nan),errors='coerce')
        malformed=raw.notna() & e.grade.isna()
        if malformed.any():raise ValueError('Non-numeric nonmissing event grade; supply explicit numeric grade mapping')
        if ((e.grade.dropna()<0)|(e.grade.dropna()>10)).any():raise ValueError('Event grade outside [0,10]; do not encode missing as -1 or 0')
        e['credits']=pd.to_numeric(e.credits,errors='coerce')
        if not np.isfinite(e.credits).all() or (e.credits<0).any():raise ValueError('Event credits must be finite nonnegative numeric values')
        if 'attempt' not in e:e['attempt']=1.
        e['attempt']=pd.to_numeric(e.attempt,errors='raise')
        if not np.isfinite(e.attempt).all() or (e.attempt<1).any() or (e.attempt%1!=0).any():raise ValueError('Attempt must be a finite integer >=1 when supplied')
        if 'faculty' not in e:e['faculty']='unknown'
        missing=int(e.grade.isna().sum());e=e[e.grade.notna()].copy()
        e['_order']=e.semester.map(self.orders)
        self.events=e.sort_values(['student_id','_order','course_id','event_id']).reset_index(drop=True)
        self.by_student={s:g for s,g in self.events.groupby('student_id',sort=False)}
        self.availability_mode=availability_mode
        self._semester_for_origin={v:k for k,v in self.origins.items()}
        memberships={}
        for i,row in self.events.iterrows():
            names=json.loads(row.faculty_memberships) if 'faculty_memberships' in self.events and pd.notna(row.faculty_memberships) else [str(row.faculty)]
            for name in names:memberships.setdefault(str(name),set()).add(i)
        self._faculty_memberships=memberships
        self.audit={'raw_event_rows':original,'duplicates_removed':original-len(events.drop_duplicates()),'usable_events':len(e),'missing_grades_excluded':missing,'students':int(e.student_id.nunique()),'courses':int(e.course_id.nunique()),'semesters':int(e.semester.nunique()),'availability_mode':availability_mode,'embargo_terms':embargo_terms if availability_mode=='calendar_embargo' else 0}
        if availability_mode=='semester_order':
            self.audit['availability_assumption']='Observed earlier-semester grades assumed available by the next semester; publication timestamps are not observed.'
            self.audit['timestamp_semantics']='Internal ordinal comparison carriers, not actual calendar dates.'

    def before(self,cutoff,faculty=None):
        c=pd.Timestamp(cutoff)
        c=c.tz_localize('UTC') if c.tzinfo is None else c.tz_convert('UTC')
        out=self.events[self.events.available_at<c]
        if faculty is not None:out=out.loc[out.index.isin(self._faculty_memberships.get(str(faculty),set()))]
        return out.copy()

    def describe_origin(self,origin):
        return self._semester_for_origin.get(pd.Timestamp(origin),str(origin)) if self.availability_mode=='semester_order' else str(origin)

    def prefix(self,student,semester,origin):
        e=self.by_student.get(str(student))
        if e is None:return self.events.iloc[:0]
        t=pd.Timestamp(origin);t=t.tz_localize('UTC') if t.tzinfo is None else t.tz_convert('UTC')
        return e[(e['_order']<self.orders[str(semester)]) & (e.available_at<t)]


def grade_band(values):
    v=np.asarray(values,dtype=float)
    # Codes 3..7; exactly zero has a separate band.
    return np.where(v==0,3,np.where(v<4,4,np.where(v<5,5,np.where(v<7,6,7)))).astype(np.int64)


class IndexedHistory(Mapping):
    """Unique prefix tensors and an enrollment-to-prefix index.

    Explicit column access materializes that column for compatibility. Training
    and inference use take()/unique so only a minibatch expands event tensors.
    """
    def __init__(self,unique,row_index):
        self.unique=unique;self.row_index=np.asarray(row_index,dtype=np.int64)
        n=len(unique['course'])
        if self.row_index.ndim!=1 or ((self.row_index<0)|(self.row_index>=n)).any():
            raise ValueError('History row index must reference unique prefixes')
    @property
    def n_rows(self):return len(self.row_index)
    def __len__(self):return len(self.unique)
    def __iter__(self):return iter(self.unique)
    def __getitem__(self,key):return self.unique[key][self.row_index]
    def take(self,rows):
        if hasattr(rows,'detach'):rows=rows.detach().cpu().numpy()
        ix=self.row_index[rows]
        return {k:v[ix] for k,v in self.unique.items()}
    def subset(self,rows):return IndexedHistory(self.unique,self.row_index[rows])


def history_rows(batch):
    return batch.n_rows if isinstance(batch,IndexedHistory) else len(batch['course'])


class SequenceStore:
    def __init__(self,corpus,vocab,max_semesters=16,max_courses=32):
        self.corpus=corpus;self.vocab=vocab;self.max_semesters=max_semesters;self.max_courses=max_courses
        if min(max_semesters,max_courses)<1:raise ValueError('Sequence caps must be positive')

    def _pack(self,histories):
        # Bound padding by observed support. Configured limits still define the
        # retained information; short histories do not allocate 16x32 slots.
        prepared=[]
        for e in histories:
            groups=list(e.groupby('_order',sort=True).indices.items())
            prepared.append((e,groups))
        n=len(prepared)
        s=max(1,min(self.max_semesters,max((len(g) for _,g in prepared),default=0)))
        c=max(1,min(self.max_courses,max((len(ix) for _,g in prepared for _,ix in g[-s:]),default=0)))
        b={'course':np.zeros((n,s,c),np.int64),'grade_band':np.zeros((n,s,c),np.int64),'numeric':np.zeros((n,s,c,3),np.float32),'event_mask':np.zeros((n,s,c),bool),'semester_mask':np.zeros((n,s),bool),'gaps':np.zeros((n,s),np.float32)}
        diag=[]
        for i,(e,groups) in enumerate(prepared):
            retained=groups[-s:];count=0;oov=0
            for j,(order,ix) in enumerate(retained):
                g=e.iloc[ix].sort_values(['course_id','event_id']).iloc[:c];m=len(g);count+=m
                course=np.array([self.vocab.get(str(k),1) for k in g.course_id],dtype=np.int64);oov+=int((course==1).sum())
                b['course'][i,j,:m]=course;b['grade_band'][i,j,:m]=grade_band(g.grade)
                b['numeric'][i,j,:m]=np.stack([g.grade.to_numpy(float)/10,g.credits.to_numpy(float)/10,g.attempt.to_numpy(float)/5],axis=-1)
                b['event_mask'][i,j,:m]=True;b['semester_mask'][i,j]=m>0;b['gaps'][i,j]=order-retained[0][0]
            diag.append({'n_history':len(e),'n_semesters':len(groups),'retained_events':count,'truncated_events':len(e)-count,'truncated_semesters':max(0,len(groups)-self.max_semesters),'oov_rate':oov/max(count,1),'no_history':int(len(e)==0)})
        return b,pd.DataFrame(diag,columns=['n_history','n_semesters','retained_events','truncated_events','truncated_semesters','oov_rate','no_history'])

    def for_rows(self,rows):
        unique=[];lookup={};back=[]
        for r in rows.itertuples():
            key=(str(r.student_id),str(r.semester),pd.Timestamp(r.origin_at))
            if key not in lookup:
                lookup[key]=len(unique);unique.append(self.corpus.prefix(*key))
            back.append(lookup[key])
        batch,diagnostics=self._pack(unique);ix=np.asarray(back,dtype=np.int64)
        return IndexedHistory(batch,ix),diagnostics.iloc[ix].reset_index(drop=True)

    def for_students(self,events,students):
        groups={s:g for s,g in events.groupby('student_id',sort=False)}
        return self._pack([groups.get(str(s),events.iloc[:0]) for s in students])

    def summaries(self,rows):
        names=['hist_events','hist_semesters','hist_grade_mean','hist_grade_std','hist_grade_min','hist_grade_max','hist_failure_rate','hist_credits','hist_earned_credits','hist_recent_mean','hist_recency_mean','hist_grade_trend','target_course_attempts','target_course_mean','history_empty']
        out=[];cache={}
        for r in rows.itertuples():
            key=(str(r.student_id),str(r.semester),pd.Timestamp(r.origin_at))
            if key not in cache:
                e=self.corpus.prefix(*key)
                if len(e)==0:cache[key]=(None,None)
                else:
                    g=e.grade.to_numpy(float);agg=e.groupby('_order').grade.mean()
                    weights=np.exp(-.5*(self.corpus.orders[str(r.semester)]-e['_order'].to_numpy()))
                    trend=float(np.polyfit(np.arange(len(agg)),agg,1)[0]) if len(agg)>1 else 0.
                    common=[len(e),len(agg),g.mean(),g.std(),g.min(),g.max(),(g<5).mean(),e.credits.sum(),e.loc[e.grade>=5,'credits'].sum(),float(agg.iloc[-1]),np.average(g,weights=weights),trend]
                    by_course=e.groupby('course_id').grade.agg(['count','mean']).to_dict('index')
                    cache[key]=(common,by_course)
            common,by_course=cache[key]
            if common is None:out.append([0]*14+[1])
            else:
                target=by_course.get(str(r.course_id),{'count':0,'mean':0.})
                out.append(common+[target['count'],target['mean'],0])
        return np.asarray(out,dtype=np.float32).reshape(len(rows),len(names)) ,names


class NumericPreprocessor:
    def fit(self,x):
        x=np.asarray(x,dtype=float)
        if x.ndim!=2 or not len(x):raise ValueError('Preprocessing needs a nonempty training matrix')
        if np.isinf(x).any():raise ValueError('Infinite engineered feature')
        self.median=np.array([np.median(c[np.isfinite(c)]) if np.isfinite(c).any() else 0. for c in x.T]);z=np.where(np.isnan(x),self.median,x)
        self.mean=z.mean(0);self.scale=z.std(0);self.scale[self.scale<1e-6]=1.
        return self
    def transform(self,x):
        x=np.asarray(x,dtype=float)
        if x.shape[1]!=len(self.median) or np.isinf(x).any():raise ValueError('Invalid engineered feature shape/values')
        m=np.isnan(x);z=(np.where(m,self.median,x)-self.mean)/self.scale
        return np.concatenate([z,m.astype(float)],axis=1).astype(np.float32)
    def fit_transform(self,x):return self.fit(x).transform(x)


def validate_enrollments(frame,corpus,features):
    f=frame.copy()
    required=set(IDENTIFIERS+['label'])
    if not required<=set(f):raise ValueError(f'Enrollments missing {sorted(required-set(f))}')
    if len(set(features))!=len(features) or not features:raise ValueError('Feature names must be nonempty and unique')
    forbidden=set(IDENTIFIERS+['label','y','grade','origin_at','label_available_at','available_at','fold','split'])
    if set(features)&forbidden:raise ValueError(f'Target/identity/timestamp in feature list: {set(features)&forbidden}')
    if not set(features)<=set(f):raise ValueError(f'Enrollment features absent: {set(features)-set(f)}')
    for c in IDENTIFIERS:
        if f[c].isna().any():raise ValueError(f'Missing enrollment {c}')
        f[c]=f[c].astype(str)
    if f.row_id.duplicated().any():raise ValueError('Enrollment row_id not unique within snapshot; distinguish course attempts')
    if not set(f.semester)<=set(corpus.orders):raise ValueError('Enrollment semester absent from calendar')
    f['label']=pd.to_numeric(f.label,errors='raise')
    if not set(f.label.unique())<={0,1}:raise ValueError('Enrollment label must be binary failure grade <5')
    if 'origin_at' not in f:f['origin_at']=f.semester.map(corpus.origins)
    if 'label_available_at' not in f:f['label_available_at']=f.semester.map(corpus.releases)
    f['origin_at']=utc(f.origin_at,'enrollments.origin_at');f['label_available_at']=utc(f.label_available_at,'enrollments.label_available_at')
    # The protocol uses a common start-of-semester prediction origin.
    if (f.origin_at!=f.semester.map(corpus.origins)).any():raise ValueError('Enrollment origin differs from declared semester origin')
    if (f.label_available_at<=f.origin_at).any():raise ValueError('Label availability must follow enrollment origin')
    f['_order']=f.semester.map(corpus.orders)
    for col in features:
        f[col]=pd.to_numeric(f[col],errors='raise')
        if np.isinf(f[col]).any():raise ValueError(f'Infinite feature {col}')
    return f


def load_inputs(cfg,base):
    base=Path(base)
    def path(x):
        p=Path(x);return p if p.is_absolute() else base/p
    if cfg.get('input_mode')=='aq_native':
        from .aq_native import load_native_inputs,semester_ordinal,_origin
        events,calendar,entries,fingerprints,audit=load_native_inputs(cfg,base)
        if cfg.get('aq_history_source','auto')!='canonical':
            from .aq_history import extend_history
            events,raw_audit,raw_fingerprints=extend_history(path(cfg.get('aq_root','/root/disk_3tb/earlywarning/30072026')),events,database_dir=path(cfg['aq_database_dir']) if cfg.get('aq_database_dir') else None,history_index=path(cfg['aq_history_index']) if cfg.get('aq_history_index') else None)
            audit['transcript_source']=raw_audit;fingerprints.update(raw_fingerprints)
            terms=set(calendar.semester.astype(str))|set(events.semester.astype(str))
            calendar=pd.DataFrame([{'semester':t,'order':semester_ordinal(t),'origin_at':_origin(semester_ordinal(t)),'labels_available_at':_origin(semester_ordinal(t))+pd.Timedelta(hours=12)} for t in sorted(terms,key=semester_ordinal)])
        fingerprints['effective_events']=hashlib.sha256(events.to_json(orient='split',date_format='iso',double_precision=15).encode()).hexdigest()
        fingerprints['effective_calendar']=hashlib.sha256(calendar.to_json(orient='split',date_format='iso').encode()).hexdigest()
        # Identity-only join metadata is no longer needed after reconciliation.
        # Pandas propagates attrs into group copies, so retaining this global
        # mapping would duplicate it for every student prefix.
        events.attrs={}
        corpus=EventCorpus(events,calendar,'semester_order')
        corpus.audit.update(audit)
        if not len(corpus.events):raise ValueError('No observed numeric historical grades in canonical cache or DDTBHP; exact grades cannot be reconstructed from binary labels')
        if cfg.get('expected_feature_count') and any(len(e['features'])!=cfg['expected_feature_count'] for e in entries):raise ValueError('Native common feature count differs from explicit expectation')
        # Native preparation already validates all source identities, labels,
        # folds and sidecar alignment while scanning one matrix at a time.
        return corpus,entries,fingerprints
    calendar=read_table(path(cfg['calendar']));events=read_table(path(cfg['events']))
    corpus=EventCorpus(events,calendar,cfg.get('availability_mode','strict'),cfg.get('embargo_terms',1))
    entries=[];fingerprints={'events':sha256(path(cfg['events'])),'calendar':sha256(path(cfg['calendar']))}
    if cfg.get('cohort_manifest'):
        mp=path(cfg['cohort_manifest']);m=json.loads(mp.read_text());fingerprints['cohort_manifest']=sha256(mp)
        for item in m['entries']:
            features=item['features'];trp=mp.parent/item['train_path'];tep=mp.parent/item['test_path']
            tr=validate_enrollments(read_table(trp),corpus,features);te=validate_enrollments(read_table(tep),corpus,features)
            entries.append({'faculty':str(item['faculty']),'fold':int(item['fold']),'train':tr,'test':te,'features':features})
            fingerprints[f'{item["faculty"]}:{item["fold"]}:train']=sha256(trp);fingerprints[f'{item["faculty"]}:{item["fold"]}:test']=sha256(tep)
    else:
        p=path(cfg['enrollments']);features=cfg['feature_columns'];f=validate_enrollments(read_table(p),corpus,features);fingerprints['enrollments']=sha256(p)
        for fold,semester in cfg['folds'].items():
            if str(semester) not in corpus.orders:raise ValueError(f'Unknown target semester {semester}')
            order=corpus.orders[str(semester)]
            for faculty,g in f.groupby('faculty',sort=True):
                te=g[g.semester==str(semester)].copy();tr=g[g['_order']<order].copy()
                if len(te):entries.append({'faculty':str(faculty),'fold':int(fold),'train':tr,'test':te,'features':features})
    for en in entries:
        tr,te=en['train'],en['test']
        if te.semester.nunique()!=1:raise ValueError('Each faculty-fold target snapshot must have one semester')
        if tr.row_id.isin(te.row_id).any():raise ValueError('Train/target row overlap')
        if (tr['_order']>=te['_order'].iloc[0]).any():raise ValueError('Train has current/future semester rows')
        if set(tr.faculty)!={en['faculty']} or set(te.faculty)!={en['faculty']}:raise ValueError('Manifest faculty does not match snapshot')
        if cfg.get('expected_feature_count') and len(en['features'])!=cfg['expected_feature_count']:raise ValueError('Unexpected feature count; verify 136+17 sidecar integration')
    entries.sort(key=lambda e:(e['fold'],e['faculty']))
    if not entries:raise ValueError('No faculty-fold cohorts found')
    validate_matrix(corpus,entries)
    return corpus,entries,fingerprints


def take_batch(batch,indices):return batch.subset(indices) if isinstance(batch,IndexedHistory) else {k:v[indices] for k,v in batch.items()}


def student_partition(students,seed,fraction=.2):
    students=sorted(map(str,students))
    if len(students)<2:raise ValueError('SSL needs >=2 students for student-heldout validation')
    ranked=sorted(students,key=lambda s:hashlib.sha256(f'{seed}|{s}'.encode()).digest())
    n=max(1,min(len(ranked)-1,int(round(len(ranked)*fraction))))
    return ranked[n:],ranked[:n]


def validate_matrix(corpus,entries):
    fold_terms={};seen={}
    for en in entries:
        terms=en['test'].semester.unique()
        if len(terms)!=1:raise ValueError('One target semester required per fold')
        fold=int(en['fold']);term=str(terms[0])
        if fold in fold_terms and fold_terms[fold]!=term:raise ValueError('All faculties must share the same chronological fold semester')
        fold_terms[fold]=term
        for frame in (en['train'],en['test']):
            for r in frame[['row_id','student_id','course_id','semester','faculty','label','label_available_at']].itertuples(index=False,name=None):
                previous=seen.setdefault(r[0],r[1:])
                if previous!=r[1:]:raise ValueError(f'Enrollment identity/label conflict across snapshots for row_id={r[0]}')
    orders=[corpus.orders[fold_terms[k]] for k in sorted(fold_terms)]
    if any(a>=b for a,b in zip(orders,orders[1:])):raise ValueError('Fold numbers must strictly follow global chronological semester order')

    retrospective=[e['test'].origin_at.min() for e in entries if e['fold']>5]
    if retrospective:
        freeze_cutoff=min(retrospective)
        for en in entries:
            if en['fold']<=5 and (en['test'].label_available_at>=freeze_cutoff).any():
                raise ValueError('Development labels unavailable before retrospective origin: protocol freeze cannot use them')
