"""Exploratory source reconstruction for AQ Suite33.

CRITICAL: Current-state CSVs are NOT immutable historical snapshots. This module
never generates a certified provenance manifest. Every derived feature is marked
as exploratory, and post-origin availability flags are DIAGNOSTICS ONLY.
"""
from __future__ import annotations
from pathlib import Path
import json, time
import numpy as np
import pandas as pd
from s33_common import log,sha,save_csv,save_json
from s33_data import KEYS,keystr,semord
from s33_context import ContextStore,RosterGateError

COLUMN_MAP = {
 'MKQDK':['NHHK','IDKQDK','IDSV','IDToHoc','NgayThem','NgayDoiNhom','IDDVPC'],
 'KToHoc':['NHHK','IDToHoc','IDMH','NgayThem'],
 'HDMNHHK':['NHHK','NgayBDHK'],
 'DDTBHK':['NHHK','IDSV','DTBHK','NgayThem'],
}

def _load(db, name, columns):
    path=Path(db)/(name+'.csv')
    if not path.is_file():raise RosterGateError(f'Raw source missing: {path}')
    hdr=pd.read_csv(path,nrows=0).columns.tolist()
    mandatory=columns[:-1] if name=='MKQDK' else columns
    miss=set(mandatory)-set(hdr)
    if miss:raise RosterGateError(f'{path.name}: missing columns {sorted(miss)}')
    use=[k for k in columns if k in hdr]
    t=time.monotonic();frame=pd.read_csv(path,usecols=use,dtype=str,low_memory=False)
    log('RAW_TABLE',table=name,rows=len(frame),columns=use,seconds=time.monotonic()-t,sha256=sha(path))
    return frame,path

def _parse(x):
    # Source date strings are local-naive; the exploratory source uses Vietnam
    # time exclusively. Normalize into UTC and do not infer other time zones.
    a=pd.to_datetime(x,errors='coerce',format='mixed')
    if hasattr(a.dt,'tz') and a.dt.tz is None:
        return a.dt.tz_localize('Asia/Ho_Chi_Minh',ambiguous='NaT',nonexistent='NaT').dt.tz_convert('UTC')
    return pd.to_datetime(x,errors='coerce',utc=True,format='mixed')

def _ordinal(x):
    a=pd.to_numeric(x,errors='coerce')
    return np.floor(a/10)*3+(a%10)-1

class ReconstructedStore(ContextStore):
    """ContextStore-compatible source with explicitly UNVERIFIED membership.

    Source current registrations are not timestamp-filtered in the *main*
    exploratory context, because a large majority is date-stamped after the
    semester start. A separate restrictive source cohort is computed for
    descriptive origin-eligibility sensitivity without changing the task.
    """
    def __init__(self, database_dir, output_dir, *, cutoff='semester_start'):
        self.db=Path(database_dir)
        self.output=Path(output_dir)
        self.output.mkdir(parents=True,exist_ok=True)
        if cutoff!='semester_start':raise RosterGateError('Only semester_start origin proxy is implemented; changing cutoff post hoc is prohibited')
        src={};srcpath={}
        for tab,cols in COLUMN_MAP.items():src[tab],srcpath[tab]=_load(self.db,tab,cols)
        self.hashes={tab+'_sha256':sha(p) for tab,p in srcpath.items()}
        self.meta={'roster_version':'AQ_RAW_CURRENT_STATE_EXPLORATORY_V2','source_kind':'CURRENT_STATE_EXPORT_NOT_HISTORICAL_SNAPSHOT','origin_cutoffs_certified':False,'independent_roster_validation':False,'previous_gpa_release_verified':False,'historical_grade_release_verified':False}
        self.release={};self.fields={}
        calendar=src['HDMNHHK'].copy()
        calendar['semester']=calendar.NHHK.map(keystr)
        calendar['origin_at']=_parse(calendar.NgayBDHK)
        if calendar.semester.duplicated().any() or calendar.origin_at.isna().any():raise RosterGateError('Semester calendar has duplicate/invalid cutoff values')
        origin=calendar.set_index('semester').origin_at
        reg=src['MKQDK'].copy()
        reg['semester']=reg.NHHK.map(keystr)
        reg['student_id']=reg.IDSV.map(keystr)
        reg['section_id']=reg.IDToHoc.map(keystr)
        reg['registered_created_at']=_parse(reg.NgayThem)
        reg['group_changed_at']=_parse(reg.NgayDoiNhom)
        sections=src['KToHoc'].copy()
        sections['semester']=sections.NHHK.map(keystr)
        sections['section_id']=sections.IDToHoc.map(keystr)
        sections['course_id']=sections.IDMH.map(keystr)
        sections['section_created_at']=_parse(sections.NgayThem)
        k=['semester','section_id']
        if sections.duplicated(k).any():raise RosterGateError('Raw course-section mapping is ambiguous')
        reg=reg.merge(sections[k+['course_id','section_created_at']],on=k,how='left',validate='many_to_one',sort=False,indicator=True)
        reg['origin_at']=reg.semester.map(origin)
        bad_key=reg[KEYS].isna().any(axis=1)|reg[KEYS].eq('').any(axis=1)
        audit=dict(registration_rows=len(reg),unmatched_section=int((reg['_merge']!='both').sum()),unmapped_origin=int(reg.origin_at.isna().sum()),blank_registration_keys=int(bad_key.sum()),duplicate_idkqdk=int(reg.IDKQDK.duplicated().sum()))
        reg=reg[(reg._merge=='both')&reg.origin_at.notna()&~bad_key].copy()
        reg['registration_late']=reg.registered_created_at.isna()|(reg.registered_created_at>reg.origin_at)
        reg['section_late']=reg.section_created_at.isna()|(reg.section_created_at>reg.origin_at)
        reg['section_changed_late']=reg.group_changed_at.notna()&(reg.group_changed_at>reg.origin_at)
        reg['origin_member_candidate']=~(reg.registration_late|reg.section_late|reg.section_changed_late)
        reg['source_faculty']=('unit_'+reg.IDDVPC.fillna('UNKNOWN').map(keystr)) if 'IDDVPC' in reg else 'unknown_unit'
        # Enforce exactly one current section per student/course/semester.
        # Current-state duplicates cannot safely be interpreted as parallel classes.
        gk=['semester','course_id','student_id']
        dup=reg.duplicated(gk,keep=False)
        duplicates=reg.loc[dup,gk].drop_duplicates()
        audit['ambiguous_student_course_keys']=len(duplicates)
        audit['ambiguous_registration_rows']=int(dup.sum())
        reg=reg[~dup].copy()
        if reg.duplicated(KEYS).any():raise RosterGateError('Reconstructed membership remains duplicate')
        audit['retained_current_members']=len(reg)
        audit['late_registration_fraction']=float(reg.registration_late.mean())
        audit['late_section_fraction']=float(reg.section_late.mean())
        audit['late_change_fraction']=float(reg.section_changed_late.mean())
        audit['preorigin_candidate_fraction']=float(reg.origin_member_candidate.mean())
        # Latest prior GPA for EACH target semester is constructed from older
        # SEMESTERS, never current grade values. Entry time is a conservative
        # proxy, not an independently verified release time.
        gpa=src['DDTBHK'].copy()
        gpa['student_id']=gpa.IDSV.map(keystr);gpa['semester']=gpa.NHHK.map(keystr)
        gpa['ordinal']=_ordinal(gpa.semester)
        gpa['entry_at']=_parse(gpa.NgayThem)
        gpa['gpa_value']=pd.to_numeric(gpa.DTBHK,errors='coerce')
        gpa=gpa[gpa.student_id.ne('')&gpa.gpa_value.between(0,10)&gpa.semester.str.fullmatch(r'\d{4,5}').fillna(False)&pd.to_numeric(gpa.semester,errors='coerce').mod(10).isin([1,2,3])&gpa.entry_at.notna()].copy()
        # Ambiguous multiple revisions in one student/semester: choose latest
        # entry <= origin; cannot infer overwritten historic values.
        eligible=[]
        reg['prior_gpa']=np.nan;reg['prior_gpa_released_at']=pd.Series(pd.NaT,index=reg.index,dtype='datetime64[ns, UTC]')
        for sm,part in reg.groupby('semester',sort=False):
            cut=part.origin_at.iloc[0];o=semord(sm)
            hist=gpa[(gpa.ordinal<o)&(gpa.entry_at<cut)]
            if len(hist):
                newest=hist.sort_values(['ordinal','entry_at'],kind='stable').drop_duplicates('student_id',keep='last')
                look=newest.set_index('student_id')
                ids=part.student_id.to_numpy()
                reg.loc[part.index,'prior_gpa']=pd.Series(ids).map(look.gpa_value).to_numpy()
                reg.loc[part.index,'prior_gpa_released_at']=pd.Series(ids).map(look.entry_at).to_numpy()
            eligible.append(dict(semester=sm,registrations=len(part),candidate_preorigin_members=int(part.origin_member_candidate.sum()),late_registrations=int(part.registration_late.sum()),late_sections=int(part.section_late.sum()),late_changes=int(part.section_changed_late.sum()),peer_gpa_with_date_proxy=int(reg.loc[part.index,'prior_gpa'].notna().sum()),origin_proxy=str(cut)))
        # Source-affecting flags remain in a separate audit and NEVER enter the
        # model feature matrix: they may reveal future administrative activity.
        q=reg[KEYS+['origin_member_candidate','registration_late','section_late','section_changed_late']].copy()
        for col in ['origin_member_candidate','registration_late','section_late','section_changed_late']:q[col]=q[col].astype('int8')
        save_csv(self.output/'reconstruction_quality_per_membership.csv',q)
        save_csv(self.output/'reconstruction_quality_by_semester.csv',eligible)
        save_json(self.output/'raw_reconstruction_audit.json',dict(**audit,source_hashes=self.hashes,cutoff_policy='HDMNHHK.NgayBDHK_local_time',caveats=['CURRENT_STATE_NOT_ASOF','LATER_SECTION_CHANGES_NOT_REVERSIBLE','WITHDRAWALS_MAY_BE_MISSING','DDTBHK.NgayThem_NOT_GPA_RELEASE_PROOF','CACHED_OWN_HISTORY_SEMESTER_PROXY']))
        # This is a computational adapter, NOT a provider certification.
        # The inherited compute() uses exactly these fields.
        reg['snapshot_at']=reg['origin_at'];reg['member_known_at']=reg['origin_at'];reg['member_effective_at']=reg['origin_at']
        self.roster=reg[KEYS+['source_faculty','origin_at','snapshot_at','member_known_at','member_effective_at','prior_gpa','prior_gpa_released_at']].copy()
        self.roster=self.roster.reset_index(drop=True)
        self.eligible_origins=set(self.roster.semester)
        self.quality=q.set_index(KEYS,verify_integrity=True)
        log('RECONSTRUCTION_SOURCE_AUDIT',**audit)
        log('RECONSTRUCTION_MODE',status='EXPLORATORY_NOT_ORIGIN_VERIFIED',information='CURRENT_STATE_ROSTER',cutoff='SEMESTER_START_PROXY',gpa='PRIOR_SEMESTER_AND_ENTRY_DATE_PROXY',source_faculty_proxy='MKQDK.IDDVPC_ADMIN_UNIT_NOT_VERIFIED_FACULTY',affected_semesters=len(eligible))
        for row in eligible:log('RECONSTRUCTION_SEMESTER',**row)

    def verify_label_release(self,historical_semesters,target_origin):
        # No false release certification. Cache chronology still enforced in
        # train_suite33.py by semester ordinal.
        log('LABEL_AVAILABILITY',status='UNVERIFIED_SEMESTER_PROXY_ONLY',historical_semesters=len(set(historical_semesters)),target_origin=str(target_origin))

    def fetch(self,keys,min_coverage=0.0):
        f=super().fetch(keys,min_coverage=min_coverage)
        idx=pd.MultiIndex.from_frame(keys[KEYS].reset_index(drop=True))
        qa=self.quality.reindex(idx).reset_index(drop=True)
        # Additional fields for descriptive subgroup evaluation ONLY.
        f['origin_member_candidate_DIAGNOSTIC_ONLY']=qa.origin_member_candidate.fillna(0).astype('float32')
        f['registration_late_DIAGNOSTIC_ONLY']=qa.registration_late.fillna(1).astype('float32')
        f['section_late_DIAGNOSTIC_ONLY']=qa.section_late.fillna(1).astype('float32')
        f['section_changed_late_DIAGNOSTIC_ONLY']=qa.section_changed_late.fillna(0).astype('float32')
        return f

    def compute(self,output_dir):
        super().compute(output_dir)
        # Do not mislabel source hashes as certified data.
        save_json(Path(output_dir)/'SOURCE_WARNING.json',{'status':'EXPLORATORY_CURRENT_STATE_ONLY','certified_origin_rosters':False,'data_availability_verified':False,'raw_database':str(self.db),'do_not_deploy':True})
        return self
