"""Course-conditioned chronological evidence, with bounded raw-path feedback.

No auxiliary risk score enters a tree. Cached retrieval arrays belong to an
immutable reader. Values use historical outcomes; course keys never use them.
"""
import hashlib
import math
import time
import numpy as np
import pandas as pd
import torch
from torch import nn
from torch.nn import functional as F
from data import HistoryIndex,build_history,shuffle_history,_validate_history
from protocol import prior_evidence

MEMORY_NAMES=('failure_supported','performance_supported','performance_sd',
              'failure_relevance_contrast','effective_support_log','retained_history_log',
              'full_history_log','recency_log','same_course_mass','observed_grade_mass',
              'attention_entropy','history_present','prior_failure','prior_performance')


def reader_fingerprint(reader,include_heads=True):
    if reader is None: return None
    h=hashlib.sha256()
    for name,value in sorted(reader.state_dict().items()):
        if not include_heads and 'head' in name: continue
        h.update(name.encode()); h.update(value.detach().cpu().contiguous().numpy().tobytes())
    return h.hexdigest()


def summarize_torch(batch,logits,no_outcomes=False,kappa=4.):
    mask=batch['mask']; present=mask.any(1).float()
    a=torch.softmax(logits.masked_fill(~mask,-1e9),dim=1)*mask
    a=a/a.sum(1,keepdim=True).clamp_min(1e-12)
    ess=present/a.square().sum(1).clamp_min(1e-12); support=ess/(ess+kappa)
    prior=batch['prior']; y=batch['h_y']; performance=batch['h_performance']; grades=batch['h_grade_present']
    if no_outcomes:
        y=prior[:,0,None].expand_as(y); performance=prior[:,1,None].expand_as(performance); grades=torch.zeros_like(grades)
    fail=(a*y).sum(1); perf=(a*performance).sum(1)
    count=mask.sum(1).float()
    uniform=(y*mask).sum(1)/count.clamp_min(1)
    variance=(a*(performance-perf[:,None]).square()).sum(1).clamp_min(0)
    # Explicit safe branch avoids sqrt'(0) on supported older Torch runtimes.
    sd=torch.where(variance>0,variance.clamp_min(1e-12).sqrt(),torch.zeros_like(variance))
    ent=-(a*a.clamp_min(1e-12).log()).sum(1)/count.clamp_min(2).log()
    values=[support*fail+(1-support)*prior[:,0],support*perf+(1-support)*prior[:,1],sd,
            (fail-uniform)*present,torch.log1p(ess),torch.log1p(count),torch.log1p(batch['full_count']),
            (a*torch.log1p(batch['age'])).sum(1),(a*batch['same']).sum(1),(a*grades).sum(1),
            ent,present,prior[:,0],prior[:,1]]
    return torch.stack(values,dim=1),a


def summarize_numpy(batch,logits,no_outcomes=False,kappa=4.):
    mask=batch['mask']; present=mask.any(1).astype(np.float32)
    z=np.where(mask,logits,-1e9); z=z-z.max(1,keepdims=True)
    a=np.exp(z)*mask; a=a/np.maximum(a.sum(1,keepdims=True),1e-12)
    ess=present/np.maximum((a*a).sum(1),1e-12); support=ess/(ess+kappa)
    prior=batch['prior']; y=batch['h_y']; perfvals=batch['h_performance']; grades=batch['h_grade_present']
    if no_outcomes:
        y=np.broadcast_to(prior[:,0,None],y.shape); perfvals=np.broadcast_to(prior[:,1,None],perfvals.shape); grades=np.zeros_like(grades)
    fail=(a*y).sum(1); perf=(a*perfvals).sum(1); count=mask.sum(1).astype(np.float32)
    uniform=(y*mask).sum(1)/np.maximum(count,1)
    sd=np.sqrt(np.maximum((a*(perfvals-perf[:,None])**2).sum(1),0))
    ent=-(a*np.log(np.maximum(a,1e-12))).sum(1)/np.log(np.maximum(count,2))
    values=[support*fail+(1-support)*prior[:,0],support*perf+(1-support)*prior[:,1],sd,
            (fail-uniform)*present,np.log1p(ess),np.log1p(count),np.log1p(batch['full_count']),
            (a*np.log1p(batch['age'])).sum(1),(a*batch['same']).sum(1),(a*grades).sum(1),
            ent,present,prior[:,0],prior[:,1]]
    return np.stack(values,axis=1).astype(np.float32),a


class Reader(nn.Module):
    def __init__(self,n_courses,meta_dim=34,hidden=24):
        super().__init__(); self.config={'n_courses':n_courses,'meta_dim':meta_dim,'hidden':hidden}
        self.embedding=nn.Embedding(n_courses,hidden,padding_idx=0)
        self.encoder=nn.Sequential(nn.Linear(hidden+meta_dim,hidden),nn.Tanh(),nn.Linear(hidden,hidden))
        self.logit_scale=nn.Parameter(torch.tensor(0.)); self.age_decay=nn.Parameter(torch.tensor(-1.5))
        self.same_bonus=nn.Parameter(torch.tensor(.5))
        self.failure_head=nn.Linear(len(MEMORY_NAMES),1); self.grade_head=nn.Linear(len(MEMORY_NAMES),1)

    def encode(self,ids,meta):
        return F.normalize(self.encoder(torch.cat((self.embedding(ids),meta),dim=-1)),dim=-1,eps=1e-8)

    def root_logits(self,batch):
        q=self.encode(batch['q_course_id'],batch['q_meta'])
        k=self.encode(batch['h_course_id'],batch['h_meta'])
        return 4*torch.sigmoid(self.logit_scale)*(q[:,None,:]*k).sum(-1)-F.softplus(self.age_decay).clamp(max=2)*torch.log1p(batch['age'])+2*torch.tanh(self.same_bonus)*batch['same']

    def evidence(self,batch,path_bias=None,uniform=False,no_outcomes=False):
        logits=torch.zeros_like(batch['age']) if uniform else self.root_logits(batch)
        if path_bias is not None and not uniform: logits=logits+path_bias
        return summarize_torch(batch,logits,no_outcomes)


class EvidenceProvider:
    def __init__(self,bank,query,history,reader=None,path_strength=0.,mode='learned',
                 device='cpu',batch_size=2048,embargo=0,cache=True,intervention='baseline'):
        if mode not in ('current','deterministic','learned'): raise ValueError('Unknown memory mode')
        if mode=='learned' and reader is None: raise ValueError('Learned memory requires Reader')
        if path_strength<0 or not np.isfinite(path_strength): raise ValueError('Invalid path strength')
        _validate_history(bank.table,query.table,history)
        self.bank=bank; self.query=query; self.history=history; self.reader=reader
        self.raw=query.x; self.raw_names=list(query.table.feature_names)+['present:'+n for n in query.table.feature_names]+['faculty:'+f for f in query.table.faculty_names]
        self.comparable_features=2*len(query.table.feature_names)
        self.mode=mode; self.path_strength=float(path_strength); self.device=torch.device(device)
        self.batch_size=int(batch_size); self.embargo=int(embargo)
        if self.batch_size<1: raise ValueError('Positive provider batch size required')
        if self.embargo<0 or self.embargo!=embargo: raise ValueError('Nonnegative integer embargo required')
        for lo in range(0,len(query),self.batch_size):
            h=history.indices[lo:lo+self.batch_size]; valid=h>=0
            qrow,column=np.nonzero(valid); bank_rows=h[valid]
            if not len(bank_rows): continue
            distance=query.table.semester[lo+qrow]-bank.table.semester[bank_rows]
            if (distance<=self.embargo).any(): raise ValueError('History violates configured embargo')
            if intervention!='shuffled_history' and (query.table.student[lo+qrow]!=bank.table.student[bank_rows]).any():
                raise ValueError('History contains a different student outside the explicit shuffle intervention')
            expected_same=(query.table.course[lo+qrow]==bank.table.course[bank_rows]).astype(np.float32)
            if not np.array_equal(history.same_course[lo:lo+len(h)][valid],expected_same): raise ValueError('History same-course metadata mismatch')
        self.memory_names=[] if mode=='current' else list(MEMORY_NAMES)
        self.reader_hash=reader_fingerprint(reader,False)
        self.prior=prior_evidence(bank.table,query.table,embargo)
        self.no_outcomes=False; self.uniform=False; self.intervention=intervention
        self._root_logits=None; self._root_features=None
        if reader is not None: reader.to(self.device)
        if cache and mode!='current': self._cache_root()

    def assert_immutable(self):
        if reader_fingerprint(self.reader,False)!=self.reader_hash:
            raise ValueError('Reader changed beneath cached evidence/forest: reconstruct provider and regrow all trees')

    def _numpy_batch(self,indices):
        idx=np.asarray(indices,dtype=np.int64); h=self.history.indices[idx]
        valid=h>=0; safe=np.maximum(h,0)
        grade=self.bank.table.grade[safe]; y=self.bank.table.y[safe].astype(np.float32)
        return {'mask':valid,'age':self.history.recency[idx].astype(np.float32),
                'same':self.history.same_course[idx].astype(np.float32),
                'h_y':y,'h_performance':np.where(np.isfinite(grade),grade/10.,1-y).astype(np.float32),
                'h_grade_present':np.isfinite(grade).astype(np.float32),'full_count':self.history.full_count[idx].astype(np.float32),
                'prior':self.prior[idx]}

    def batch(self,indices):
        idx=np.asarray(indices,dtype=np.int64); safe=np.maximum(self.history.indices[idx],0)
        values=self._numpy_batch(idx)
        values.update(q_course_id=self.query.course_id[idx],q_meta=self.query.course_meta[idx],
                      h_course_id=self.bank.course_id[safe],h_meta=self.bank.course_meta[safe],
                      target_y=self.query.table.y[idx].astype(np.float32),
                      target_grade=np.nan_to_num(self.query.table.grade[idx]/10.,nan=0.).astype(np.float32),
                      grade_known=np.isfinite(self.query.table.grade[idx]))
        return {k:torch.as_tensor(v,device=self.device) for k,v in values.items()}

    def _cache_root(self):
        n=len(self.query); width=self.history.max_history
        self._root_logits=np.empty((n,width),dtype=np.float32)
        if self.mode=='deterministic':
            self._root_logits[:]=-.5*np.log1p(self.history.recency)
        else:
            # Encode each bank event once. Gather only compact course encodings,
            # never the N x history x raw-feature tensor.
            self.reader.eval()
            with torch.no_grad():
                base=np.empty((len(self.bank),self.reader.config['hidden']),np.float32)
                for lo in range(0,len(self.bank),self.batch_size):
                    hi=min(lo+self.batch_size,len(self.bank))
                    base[lo:hi]=self.reader.encode(torch.as_tensor(self.bank.course_id[lo:hi],device=self.device),torch.as_tensor(self.bank.course_meta[lo:hi],device=self.device)).cpu().numpy()
                scale=float((4*torch.sigmoid(self.reader.logit_scale)).cpu()); decay=float(F.softplus(self.reader.age_decay).clamp(max=2).cpu()); same=float((2*torch.tanh(self.reader.same_bonus)).cpu())
                for lo in range(0,n,self.batch_size):
                    hi=min(lo+self.batch_size,n)
                    q=self.reader.encode(torch.as_tensor(self.query.course_id[lo:hi],device=self.device),torch.as_tensor(self.query.course_meta[lo:hi],device=self.device)).cpu().numpy()
                    keys=base[np.maximum(self.history.indices[lo:hi],0)]
                    self._root_logits[lo:hi]=scale*np.einsum('nd,nhd->nh',q,keys)-decay*np.log1p(self.history.recency[lo:hi])+same*self.history.same_course[lo:hi]

    def path_bias(self,indices,path):
        idx=np.asarray(indices,dtype=np.int64); history=self.history.indices[idx]
        valid=history>=0; safe=np.maximum(history,0)
        out=np.zeros(history.shape,np.float32); count=0
        if self.path_strength==0: return out
        for feature,threshold,left in path:
            feature=int(feature)
            if feature<0 or feature>=self.comparable_features: continue
            agrees=(self.bank.x[safe,feature]<=float(threshold))==bool(left)
            out+=np.where(agrees,1.,-1.); count+=1
        if count: out*=self.path_strength/count
        return out*valid

    def features(self,indices,path=()):
        idx=np.asarray(indices,dtype=np.int64)
        if self.mode=='current': return np.empty((len(idx),0),np.float32)
        if self._root_logits is None: self._cache_root()
        out=np.empty((len(idx),len(MEMORY_NAMES)),np.float32)
        for lo in range(0,len(idx),self.batch_size):
            rows=idx[lo:lo+self.batch_size]
            logits=np.zeros_like(self._root_logits[rows]) if self.uniform else self._root_logits[rows]+self.path_bias(rows,path)
            out[lo:lo+len(rows)]=summarize_numpy(self._numpy_batch(rows),logits,self.no_outcomes)[0]
        return out

    def root_features(self):
        if self._root_features is None: self._root_features=self.features(np.arange(len(self.query)))
        return self._root_features

    def clone(self,intervention,seed=42):
        h=self.history; strength=self.path_strength
        if intervention=='no_memory':
            h=HistoryIndex(np.full_like(h.indices,-1),np.zeros_like(h.recency),np.zeros_like(h.same_course),np.zeros_like(h.full_count),np.zeros_like(h.retained_count))
        elif intervention=='history_8': h=build_history(self.bank.table,self.query.table,min(8,h.max_history),embargo=self.embargo)
        elif intervention=='shuffled_history': h,_=shuffle_history(self.bank.table,self.query.table,h,seed)
        elif intervention not in ('no_outcomes','uniform_attention','no_path'): raise ValueError(f'Unknown intervention {intervention}')
        if intervention in ('no_path','uniform_attention'): strength=0.
        obj=EvidenceProvider(self.bank,self.query,h,self.reader,strength,self.mode,self.device,self.batch_size,self.embargo,intervention=intervention)
        obj.no_outcomes=intervention=='no_outcomes'; obj.uniform=intervention=='uniform_attention'; obj.intervention=intervention
        obj._root_features=None
        return obj

    def diagnostics(self):
        m=self.root_features(); t=self.query.table
        frame=pd.DataFrame({'sample_id':t.sample_id,'faculty':[t.faculty_names[i] for i in t.faculty],
                            'history_full':self.history.full_count,'history_retained':self.history.retained_count,
                            'known_course':self.query.course_id>0,'grade_known_query':np.isfinite(t.grade)})
        for j,name in enumerate(self.memory_names): frame[name]=m[:,j]
        return frame


def auxiliary_loss(reader,batch,weights,evidence=None,attention=None):
    if evidence is None: evidence,attention=reader.evidence(batch)
    failure=reader.failure_head(evidence).squeeze(1)
    bce=(F.binary_cross_entropy_with_logits(failure,batch['target_y'],reduction='none')*weights).mean()
    known=batch['grade_known'].float(); grade=torch.sigmoid(reader.grade_head(evidence).squeeze(1))
    mse=((grade-batch['target_grade']).square()*known*weights).sum()/(known*weights).sum().clamp_min(1.)
    # Penalize concentration (zero on a uniform nonempty history).
    concentration=(batch['mask'].sum(1)*attention.square().sum(1)-1).clamp_min(0)
    penalty=(concentration*weights).mean()
    return bce+.15*mse+.002*penalty,{'aux_bce':float(bce.detach()),'aux_grade_mse':float(mse.detach()),'attention_concentration':float(penalty.detach())}


def retrieval_parameters(reader): return [p for n,p in reader.named_parameters() if 'head' not in n]


def grad_norm(parameters):
    return math.sqrt(sum(float(p.grad.detach().square().sum()) for p in parameters if p.grad is not None))


def pretrain_reader(reader,provider,weights,epochs=3,max_rows=120000,batch_size=512,lr=.001,seed=42,context=None):
    if epochs<0 or max_rows<1 or batch_size<1: raise ValueError('Invalid reader training budget')
    reader.to(provider.device); reader.train(); rng=np.random.default_rng(seed)
    weights=np.asarray(weights,np.float32)
    optimizer=torch.optim.AdamW(reader.parameters(),lr=lr,weight_decay=1e-4)
    records=[]; before=reader_fingerprint(reader); before_r=reader_fingerprint(reader,False)
    original=[p.detach().clone() for p in retrieval_parameters(reader)]
    for epoch in range(epochs):
        rows=rng.permutation(len(provider.query))[:max_rows]; loss_sum=0.; grad_sum=0.; head_sum=0.; n=0; start=time.perf_counter()
        for lo in range(0,len(rows),batch_size):
            idx=rows[lo:lo+batch_size]; b=provider.batch(idx)
            loss,parts=auxiliary_loss(reader,b,torch.as_tensor(weights[idx],device=provider.device))
            optimizer.zero_grad(); loss.backward()
            grad_sum+=grad_norm(retrieval_parameters(reader)); head_sum+=grad_norm([p for name,p in reader.named_parameters() if 'head' in name])
            torch.nn.utils.clip_grad_norm_(reader.parameters(),2.); optimizer.step()
            loss_sum+=float(loss.detach()); n+=1
        record={**(context or {}),'epoch':epoch+1,'rows':len(rows),'aux_loss':loss_sum/max(n,1),
                'retrieval_grad_norm':grad_sum/max(n,1),'auxiliary_head_grad_norm':head_sum/max(n,1),
                'seconds':time.perf_counter()-start,'initial_reader_hash':before,'reader_hash':reader_fingerprint(reader),
                'initial_retrieval_hash':before_r,'retrieval_hash':reader_fingerprint(reader,False),
                'retrieval_parameter_delta':math.sqrt(sum(float((p.detach()-old).square().sum()) for p,old in zip(retrieval_parameters(reader),original)))}
        records.append(record); print('Reader pretrain:',record,flush=True)
    reader.eval(); return records
