"""Provenance-checked as-of offering context: no current labels or result-selected rosters."""
from __future__ import annotations
from pathlib import Path
import json, time
import numpy as np
import pandas as pd
from s33_common import log, seed, sha, save_json, save_csv
from s33_data import KEYS,keystr

REQUIRED_ROSTER=['semester','course_id','section_id','student_id','source_faculty',
                 'origin_at','snapshot_at','member_known_at','member_effective_at',
                 'prior_gpa','prior_gpa_released_at']
REQUIRED_MANIFEST={
    'source_kind':'institutional_asof_registration_snapshot',
    'coverage_contract':'all_active_origin_members_irrespective_of_final_grade',
    'origin_cutoffs_certified':True,
    'independent_roster_validation':True,
    'previous_gpa_release_verified':True,
    'historical_grade_release_verified':True,
}

class RosterGateError(RuntimeError):pass

def example_inputs(path):
    base=Path(path);base.mkdir(parents=True,exist_ok=True)
    save_json(base/'roster_manifest.example.json',{
      **REQUIRED_MANIFEST,'roster_csv':'roster_asof.csv','grade_release_csv':'grade_release.csv',
      'roster_version':'REPLACE_FROM_SOURCE_SYSTEM','verification_reference':'REPLACE_WITH_AUDIT_REFERENCE',
      'verified_by':'REPLACE_WITH_RESPONSIBLE_DATA_CUSTODIAN',
      'notes':'Every active student in every historical origin; not final outcome-selected rows. The booleans require real verification.'})
    save_csv(base/'roster_asof.example.csv',[dict(semester='20251',course_id='COURSE',section_id='SECTION',student_id='STUDENT',source_faculty='khoa_x',origin_at='2025-01-01T00:00:00+07:00',snapshot_at='2025-01-01T00:00:00+07:00',member_known_at='2024-12-20T00:00:00+07:00',member_effective_at='2024-12-20T00:00:00+07:00',prior_gpa='3.2',prior_gpa_released_at='2024-12-10T00:00:00+07:00')])
    save_csv(base/'grade_release.example.csv',[{'semester':'20243','latest_grade_released_at':'2024-12-18T00:00:00+07:00'}])

def _times(f,columns):
    for c in columns:
        f[c]=pd.to_datetime(f[c],utc=True,errors='coerce')
    return f

class ContextStore:
    def __init__(self,manifest,strict=True):
        self.path=Path(manifest)
        if not self.path.is_file():raise RosterGateError(f'Missing certified origin-roster manifest: {self.path}. See examples/roster_manifest.example.json')
        meta=json.loads(self.path.read_text());self.meta=meta
        for k,expected in REQUIRED_MANIFEST.items():
            if meta.get(k)!=expected:raise RosterGateError(f'Origin roster provenance failed: manifest {k} must be verified as {expected!r}, got {meta.get(k)!r}')
        for k in ('roster_version','verification_reference','verified_by'):
            if not meta.get(k) or 'REPLACE' in str(meta[k]):raise RosterGateError(f'Missing real source audit field: {k}')
        rp=Path(meta['roster_csv']);gp=Path(meta['grade_release_csv'])
        self.roster_path=rp if rp.is_absolute() else self.path.parent/rp
        self.grades_path=gp if gp.is_absolute() else self.path.parent/gp
        if not self.roster_path.exists() or not self.grades_path.exists():raise RosterGateError(f'Roster or grade-release CSV missing: {self.roster_path} / {self.grades_path}')
        self.hashes=dict(manifest_sha256=sha(self.path),roster_sha256=sha(self.roster_path),grade_release_sha256=sha(self.grades_path))
        self.roster=self._load_roster()
        self.release=self._load_release()
        self.fields={}
        self.eligible_origins=set(self.roster.semester)
        log('ASOF_ROSTER_AUDIT',status='SOURCE_ATTESTED_NOT_INDEPENDENTLY_PROVEN_BY_CODE',rows=len(self.roster),origins=len(self.eligible_origins),source_version=meta['roster_version'],hashes=self.hashes)

    def _load_roster(self):
        t=time.monotonic();r=pd.read_csv(self.roster_path,dtype=str,low_memory=False)
        if not set(REQUIRED_ROSTER).issubset(r):raise RosterGateError('Origin roster missing fields: '+str(sorted(set(REQUIRED_ROSTER)-set(r))))
        r=r[REQUIRED_ROSTER].copy()
        for k in KEYS:r[k]=r[k].map(keystr)
        if r[KEYS].eq('').any().any():raise RosterGateError('Empty origin roster identity keys')
        if r.duplicated(KEYS).any():raise RosterGateError('Duplicate origin roster membership: provider must materialize exact latest as-of state once per member/offering')
        _times(r,['origin_at','snapshot_at','member_known_at','member_effective_at','prior_gpa_released_at'])
        if r[['origin_at','snapshot_at','member_known_at','member_effective_at']].isna().any().any():raise RosterGateError('Missing/unparseable required roster timestamps')
        for k in ('snapshot_at','member_known_at','member_effective_at'):
            bad=r[k]>r.origin_at
            if bad.any():raise RosterGateError(f'Future membership/snapshot time {k} at {int(bad.sum())} origin rows')
        if r.groupby('semester').origin_at.nunique().max()>1:raise RosterGateError('Multiple origin timestamps for same semester; supply one documented institutional prediction origin per semester')
        r['prior_gpa']=pd.to_numeric(r.prior_gpa,errors='coerce');good=r.prior_gpa.notna()
        if (good&r.prior_gpa_released_at.isna()).any():raise RosterGateError('Non-null prior GPA without verifiable release timestamp')
        if (good&(r.prior_gpa_released_at>=r.origin_at)).any():raise RosterGateError('Prior GPA released at/after origin')
        if (good&(~np.isfinite(r.prior_gpa))).any():raise RosterGateError('Nonfinite GPA values')
        if not r.source_faculty.notna().all():raise RosterGateError('Missing faculty identity for matched placebo')
        log('OBSERVATION_PROVENANCE',roster_rows=len(r),gpa_available=int(good.sum()),gpa_missing=int((~good).sum()),origin_count=r.semester.nunique(),seconds=time.monotonic()-t)
        return r

    def _load_release(self):
        a=pd.read_csv(self.grades_path,dtype=str)
        if not {'semester','latest_grade_released_at'}.issubset(a):raise RosterGateError('grade release file requires semester,latest_grade_released_at')
        a=a[['semester','latest_grade_released_at']].copy()
        a.semester=a.semester.map(keystr)
        a['latest_grade_released_at']=pd.to_datetime(a['latest_grade_released_at'],utc=True,errors='coerce')
        if a.semester.duplicated().any() or a.latest_grade_released_at.isna().any():raise RosterGateError('Invalid grade-release semester mapping')
        return dict(zip(a.semester,a.latest_grade_released_at))

    def verify_label_release(self,historical_semesters,target_origin):
        h=set(map(keystr,historical_semesters))
        missing=sorted(h-set(self.release))
        if missing:raise RosterGateError(f'No historical grade release evidence for semesters {missing[:12]} ({len(missing)} missing)')
        for s in h:
            if self.release[s]>=target_origin:raise RosterGateError(f'Historical semester={s} labels were not released before target origin {target_origin}')

    def _feature_rows(self,r,section_name):
        # Input is 1 row per unique student x course x section x semester, with prior GPA released before origin.
        gv=r.prior_gpa
        seen=gv.notna().astype(int);z=gv.fillna(0.)
        g=r.groupby(section_name,sort=False,dropna=False)
        size=g.student_id.transform('size').astype(float)
        count=seen.groupby([r[k] for k in section_name],sort=False).transform('sum').astype(float)
        total=z.groupby([r[k] for k in section_name],sort=False).transform('sum')
        total2=(z*z).groupby([r[k] for k in section_name],sort=False).transform('sum')
        n=count-seen
        numerator=total-z
        mu=(numerator/n.replace(0,np.nan)).astype(float)
        variance=((total2-z*z)/n.replace(0,np.nan)-mu*mu).clip(lower=0.)
        sigma=np.sqrt(variance)
        # Exact empirical CDF of peers below/equal target GPA, leaving out target student.
        rank_min=g.prior_gpa.rank(method='min');rank_max=g.prior_gpa.rank(method='max')
        pct=(rank_min-1+.5*(rank_max-rank_min))/(count-1).replace(0,np.nan)
        return dict(size=size-1,obs=n,coverage=n/(size-1).replace(0,np.nan),mean=mu,std=sigma,relative_pct=pct)

    def _placebo_sections(self,r):
        # Stable within (semester,course,source_faculty) shuffle of section labels;
        # exact section x faculty counts preserved, no target labels used.
        alt=r.section_id.copy();eligible=np.zeros(len(r),bool)
        groups=r.groupby(['semester','course_id','source_faculty'],sort=True).indices
        for (sm,course,faculty),ix in groups.items():
            ix=np.asarray(ix)
            if len(ix)<4 or r.section_id.iloc[ix].nunique()<2:continue
            ss=seed('placebo',sm,course,faculty,2017)
            rs=np.random.default_rng(ss);p=rs.permutation(len(ix))
            if np.array_equal(p,np.arange(len(ix))):p=np.roll(p,1)
            alt.iloc[ix]=r.section_id.iloc[ix].to_numpy()[p]
            eligible[ix]=True
        r['placebo_section']=alt
        r['placebo_matchable']=eligible.astype(float)
        return r

    def compute(self,output_dir):
        r=self.roster.copy().reset_index(drop=True)
        self._placebo_sections(r)
        grp_course=['semester','course_id'];grp_section=['semester','course_id','section_id'];grp_fake=['semester','course_id','placebo_section']
        section=self._feature_rows(r,grp_section);course=self._feature_rows(r,grp_course);fake=self._feature_rows(r,grp_fake)
        fields={}
        # A: sizes and source availability, include same number of students across faculties.
        n_fac=r.groupby(grp_section,sort=False).source_faculty.transform('nunique').astype(float)
        fields.update(s1_sec_peers=section['size'],s1_sec_obs=section['obs'],s1_sec_coverage=section['coverage'],s1_sec_faculties=n_fac,s1_course_peers=course['size'],s1_course_obs=course['obs'])
        fields.update(s2_course_gpa_mean=course['mean'],s2_course_gpa_std=course['std'])
        fields.update(s3_sec_gpa_mean=section['mean'],s3_sec_gpa_std=section['std'],s3_sec_minus_course=section['mean']-course['mean'],s3_sec_std_minus_course=section['std']-course['std'])
        fields.update(s4_gpa_gap=r.prior_gpa-section['mean'],s4_gpa_gap_over_std=(r.prior_gpa-section['mean'])/(section['std']+.1),s4_sec_pct=section['relative_pct'],s4_course_pct=course['relative_pct'])
        # Matched placebo course-wide block remains unchanged; fake section-context only.
        for k,source in [('s1_sec_peers',fake['size']),('s1_sec_obs',fake['obs']),('s1_sec_coverage',fake['coverage'])]:fields['p_'+k]=source
        fields['p_s1_sec_faculties']=r.groupby(grp_fake,sort=False).source_faculty.transform('nunique').astype(float)
        for k,source in [('s3_sec_gpa_mean',fake['mean']),('s3_sec_gpa_std',fake['std']),('s3_sec_minus_course',fake['mean']-course['mean']),('s3_sec_std_minus_course',fake['std']-course['std']),('s4_gpa_gap',r.prior_gpa-fake['mean']),('s4_gpa_gap_over_std',(r.prior_gpa-fake['mean'])/(fake['std']+.1)),('s4_sec_pct',fake['relative_pct'])]:fields['p_'+k]=source
        F=pd.DataFrame(fields,dtype='float32')
        F=F.replace([np.inf,-np.inf],np.nan)
        for c in F:
            if c.startswith('s1_') or c.startswith('p_s1_'):
                F[c]=F[c].fillna(0)
        F['is_placebo_matchable']=r.placebo_matchable.astype('float32')
        full=pd.concat([r[KEYS],F],axis=1)
        if full.duplicated(KEYS).any():raise RosterGateError('Feature rows not unique by registration')
        self.fields=full.set_index(KEYS,verify_integrity=True)
        self.families={
            'S1':['s1_sec_peers','s1_sec_obs','s1_sec_coverage','s1_sec_faculties','s1_course_peers','s1_course_obs'],
            'S2':['s2_course_gpa_mean','s2_course_gpa_std'],
            'S3':['s3_sec_gpa_mean','s3_sec_gpa_std','s3_sec_minus_course','s3_sec_std_minus_course'],
            'S4':['s4_gpa_gap','s4_gpa_gap_over_std','s4_sec_pct','s4_course_pct']
        }
        sec_info=r.groupby(grp_section,sort=False).agg(roster_rows=('student_id','size'),faculty_count=('source_faculty','nunique'),prior_gpa_observed=('prior_gpa','count')).reset_index()
        save_csv(Path(output_dir)/'section_metadata.csv',sec_info)
        save_json(Path(output_dir)/'context_manifest.json',dict(source_hashes=self.hashes,source_version=self.meta['roster_version'],schema={k:len(v) for k,v in self.families.items()},origin_coverage=len(self.eligible_origins)))
        log('SECTION_COHORT_AUDIT',sections=len(sec_info),median_size=float(sec_info.roster_rows.median()),cross_faculty=int((sec_info.faculty_count>1).sum()),placebo_matchable_rate=float(r.placebo_matchable.mean()))
        log('PEER_FEATURE_BLOCK',n=len(full),fields=list(F.columns),total_nonfinite=int(F.isna().sum().sum()),zero_peer=int((section['size']==0).sum()))
        return self

    def fetch(self,keys,min_coverage=.98):
        idx=pd.MultiIndex.from_frame(keys[KEYS].reset_index(drop=True))
        f=self.fields.reindex(idx).reset_index(drop=True)
        matched=np.isfinite(f['s1_sec_peers'].to_numpy())
        coverage=float(matched.mean())
        if coverage<min_coverage:raise RosterGateError(f'Only {coverage:.3%} target/history rows match verified origin membership (< {min_coverage:.1%}). No post-extraction substitution allowed.')
        log('ASOF_ROSTER_AUDIT',registration_rows=len(keys),matched=int(matched.sum()),coverage=coverage,missing=int((~matched).sum()),missing_context_kept=True)
        f=f.replace([np.inf,-np.inf],np.nan)
        return f

    def matrix(self,block,arm):
        families=self.families
        if arm=='S0':return np.zeros((len(block),0),dtype=np.float32)
        cols=families['S1'].copy()
        if arm in ('S2','S3','S4','P1'):cols+=families['S2']
        if arm in ('S3','S4','P1'):cols+=families['S3']
        if arm in ('S4','P1'):cols+=families['S4']
        if arm=='P1':
            cols=[('p_'+c if (c.startswith(('s3_','s4_')) and c!='s4_course_pct') or c in ['s1_sec_peers','s1_sec_obs','s1_sec_coverage','s1_sec_faculties'] else c) for c in cols]
        # All arms receive identical missing-value policy; support counts remain available.
        return np.nan_to_num(block[cols].to_numpy(dtype=np.float32),nan=-999.,posinf=-999.,neginf=-999.)
