"""Native AQ contract tests use tiny real joblib/CSV files, not AQ exports."""
import hashlib
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch

import joblib
import numpy as np
import pandas as pd

from pace.aq_native import COURSE_FEATURES, COURSE_STATIC, load_native_inputs, semester_ordinal


class NativeAQTests(unittest.TestCase):
    def setUp(self):
        self.tmp = tempfile.TemporaryDirectory()
        self.root = Path(self.tmp.name)
        self.cache = self.root / 'results/cross_faculty_residual_moe/cache'
        self.structured = self.root / 'results/suite12_structured_cache'
        self.cache.mkdir(parents=True)
        self.structured.mkdir(parents=True)

    def tearDown(self):
        self.tmp.cleanup()

    def frame(self, ids, semesters, grades, courses=None, students=None):
        n = len(ids)
        return pd.DataFrame({'IDKQDK': ids, 'IDSV': students or ['s'] * n,
                             'IDMH': courses or ['c'] * n, 'target_NHHK': semesters,
                             'target_grade10': grades, 'SoTinChi': [3.] * n,
                             'IDToHoc': ['section'] * n})

    def write(self, faculty='A', fold=1, train=None, test=None, sidecar=True,
              mask=None, probabilities=None, names=None):
        if train is None:
            train = self.frame(['r0', 'r1', 'r2'], [20231, 20232, 20232], [0., 8., 6.])
        if test is None:
            test = self.frame(['t0', 't1'], [20233, 20233], [9., 2.], courses=['c', 'unknown'])
        names = names or ['raw', 'SoTinChi']
        ytr = (pd.to_numeric(train.target_grade10) < 5).to_numpy(np.int8)
        yte = (pd.to_numeric(test.target_grade10) < 5).to_numpy(np.int8)
        n = len(train)
        mask = np.array([False, True, True]) if mask is None and n == 3 else np.ones(n, bool) if mask is None else np.asarray(mask)
        probabilities = np.full((n, 2), .5) if probabilities is None else np.asarray(probabilities)
        rawtr = np.column_stack([np.arange(n) + j for j in range(len(names))]).astype(float)
        rawte = np.column_stack([np.arange(len(test)) + 10 + j for j in range(len(names))]).astype(float)
        for j, name in enumerate(names):
            if name == 'SoTinChi':
                rawtr[:, j] = 3.; rawte[:, j] = 3.
        cache = dict(feature_names=names, Xtr=rawtr, Xte=rawte, ytr=ytr, yte=yte,
                     train_frame=train, test_frame=test, cf_covered_mask=mask,
                     cf_probabilities=probabilities)
        path = self.cache / f'{faculty}_fold_{fold}.joblib'
        joblib.dump(cache, path)
        sc = None
        if sidecar:
            covered = mask.astype(bool) & np.isfinite(probabilities).all(axis=1)
            ix = np.flatnonzero(covered)[::-1]
            sc = {'train_row_index': ix,
                  'train': {'course_target': (100 + np.arange(len(ix) * 17)).reshape(len(ix), 17).astype(float)},
                  'target_row_index': np.arange(len(test))[::-1],
                  'target': {'course_target': (1000 + np.arange(len(test) * 17)).reshape(len(test), 17).astype(float)}}
            joblib.dump(sc, self.structured / path.name)
        return cache, sc, path

    def load(self, **extra):
        return load_native_inputs(dict(input_mode='aq_native', aq_root=str(self.root), aq_folds=[1], **extra), self.root)

    def test_reordered_sidecar_exact_values_and_course_ids(self):
        cache, sc, _ = self.write()
        events, calendar, entries, hashes, audit = self.load()
        en = next(iter(entries)); tr = en['train']; te = en['test']
        cols = ['course::' + x for x in COURSE_FEATURES]
        np.testing.assert_array_equal(tr.loc[sc['train_row_index'], cols], sc['train']['course_target'])
        np.testing.assert_array_equal(te.loc[sc['target_row_index'], cols], sc['target']['course_target'])
        self.assertEqual(te.course_id.tolist(), ['c', 'unknown'])
        self.assertEqual(en['features'], ['raw', 'SoTinChi'] + cols)
        self.assertEqual(calendar.set_index('semester').loc['20231', 'order'], semester_ordinal(20231))

    def test_finite_cf_coverage_not_just_mask(self):
        self.write(mask=[True, True, True], probabilities=[[np.nan, .3], [.1, .2], [.5, .6]])
        _, _, entries, _, _ = self.load(aq_history_support='covered')
        self.assertEqual(next(iter(entries))['train'].row_id.tolist(), ['IDKQDK|r1', 'IDKQDK|r2'])
        # A sidecar that includes the nonfinite CF row must fail, even though the mask says covered.
        p = self.structured / 'A_fold_1.joblib'; sc = joblib.load(p)
        sc['train_row_index'] = np.array([0, 1, 2]); sc['train']['course_target'] = np.ones((3, 17)); joblib.dump(sc, p)
        with self.assertRaisesRegex(ValueError, 'finite CF'):
            self.load()

    def test_full_and_covered_support_preserve_no_history_rows(self):
        self.write()
        events, _, entries, _, audit = self.load()
        self.assertEqual(len(next(iter(entries))['train']), 3)
        self.assertEqual(len(events), 3)
        _, _, covered, _, _ = self.load(aq_history_support='covered')
        self.assertEqual(len(next(iter(covered))['train']), 2)

    def test_missing_sidecar_prior_semester_stats_are_deduplicated(self):
        self.write(sidecar=False)
        # Duplicate historical events in a later fold are not new observations.
        tr = self.frame(['r0', 'r1', 'r2', 't0'], [20231, 20232, 20232, 20233], [0., 8., 6., 9.])
        te = self.frame(['future'], [20241], [8.])
        self.write(fold=2, train=tr, test=te, sidecar=False)
        events, _, entries, _, audit = self.load()
        en = next(iter(entries)); tr = en['train']; te = en['test']
        self.assertEqual(len(events), 4)  # Later fold's training history may add a legitimate event.
        self.assertEqual(tr['course::hist_count'].tolist(), [0., 1., 1.])
        np.testing.assert_allclose(tr['course::hist_mean'].iloc[1:], [0., 0.])
        self.assertEqual(te['course::hist_count'].tolist(), [3., 0.])
        self.assertAlmostEqual(te['course::hist_mean'].iloc[0], 14 / 3)
        self.assertAlmostEqual(te['course::hist_std'].iloc[0], np.std([0, 8, 6], ddof=0))
        self.assertTrue(np.isnan(te['course::hist_mean'].iloc[1]))
        self.assertEqual(tr['course::SoTinChi'].tolist(), [3., 3., 3.])

    def test_target_only_grades_never_enter_events(self):
        self.write(sidecar=False)
        events, _, _, _, audit = self.load()
        self.assertEqual(set(events.event_id), {'IDKQDK|r0', 'IDKQDK|r1', 'IDKQDK|r2'})
        self.assertIn(0., events.grade.tolist())
        self.assertTrue(audit['target_grades_excluded_from_event_corpus'])

    def test_missing_grade_is_not_inferred_from_binary_label(self):
        tr = self.frame(['r0', 'r1', 'r2'], [20231, 20232, 20232], [0., np.nan, 6.])
        cache, _, p = self.write(train=tr, sidecar=False)
        cache['ytr'][1] = 1; joblib.dump(cache, p)
        events, _, entries, _, audit = self.load()
        self.assertEqual(events.grade.isna().sum(), 1)
        self.assertEqual(audit['history_grade_missing_events'], 1)
        self.assertEqual(len(next(iter(entries))['train']), 3)

    def test_grade_consistency_and_identity_conflicts_rejected(self):
        cache, _, p = self.write(sidecar=False)
        cache['ytr'][0] = 0; joblib.dump(cache, p)
        with self.assertRaisesRegex(ValueError, 'grade/failure'):
            self.load()
        cache['ytr'][0] = 1; joblib.dump(cache, p)
        tr = self.frame(['r0'], [20231], [1.])
        self.write('B', train=tr, sidecar=False)
        with self.assertRaisesRegex(ValueError, 'grade conflict'):
            self.load()

    def test_crossfaculty_membership_and_numeric_id_normalization(self):
        tr = self.frame(['r0', 'r1'], [20231, 20232], [0., 8.], courses=[17., '001'], students=['17.0', '001'])
        te = self.frame(['t0'], [20233], [7.])
        self.write(train=tr, test=te, sidecar=False)
        self.write('B', train=tr, test=te, sidecar=False)
        events, _, entries, _, audit = self.load()
        self.assertEqual(len(events), 2)
        self.assertEqual(events.student_id.tolist(), ['17', '001'])
        self.assertEqual(events.course_id.tolist(), ['17', '001'])
        self.assertEqual(events.faculty_memberships.tolist(), ['["A", "B"]'] * 2)

    def test_source_hash_is_upstream_bytes_and_changes(self):
        cache, _, p = self.write(sidecar=False)
        _, _, _, hashes, _ = self.load()
        self.assertIn(hashlib.sha256(p.read_bytes()).hexdigest(), hashes.values())
        cache['Xtr'][0, 0] += 1; joblib.dump(cache, p)
        _, _, _, changed, _ = self.load()
        self.assertNotEqual(hashes, changed)

    def test_entries_are_reiterable_and_metadata_does_not_load_matrices(self):
        self.write()
        _, _, entries, _, _ = self.load()
        self.assertEqual(len(entries), 1)
        with patch('pace.aq_native.joblib.load', wraps=joblib.load) as loads:
            for en in entries:
                self.assertEqual(en['faculty'], 'A'); self.assertEqual(en['fold'], 1); self.assertTrue(en['features'])
            self.assertEqual(loads.call_count, 0)
            en = next(iter(entries)); _ = en['train']; _ = en['test']
            self.assertEqual(loads.call_count, 2)  # One canonical and one sidecar.
            fresh = next(iter(entries)); self.assertNotIn('train', dict(fresh))
            _ = fresh['test']; self.assertEqual(loads.call_count, 4)

    def test_common_raw_intersection_is_across_requested_folds(self):
        self.write(names=['raw', 'SoTinChi', 'only_first'], sidecar=False)
        tr = self.frame(['r0', 'r1', 'r2', 't0'], [20231, 20232, 20232, 20233], [0., 8., 6., 9.])
        self.write(fold=2, train=tr, test=self.frame(['future'], [20241], [8.]), names=['SoTinChi', 'raw', 'other'], sidecar=False)
        _, _, entries, _, _ = load_native_inputs(dict(aq_root=str(self.root), aq_folds=[1, 2]), self.root)
        self.assertEqual(next(iter(entries))['features'][:2], ['raw', 'SoTinChi'])

    def test_sidecar_index_duplicates_and_oob_rejected(self):
        for ix in ([1, 1], [1, 3]):
            with self.subTest(ix=ix):
                self.write(); p = self.structured / 'A_fold_1.joblib'; sc = joblib.load(p)
                sc['train_row_index'] = np.array(ix); joblib.dump(sc, p)
                with self.assertRaisesRegex(ValueError, 'train_row_index'):
                    self.load()

    def test_auto_csv_fallback_and_only_missing_target_rows_dropped(self):
        splits = self.root.parent / (self.root.name + '_split')
        # Explicit path keeps this temporary sibling self contained.
        fold = splits / 'fold_1'; fold.mkdir(parents=True)
        tr = self.frame(['r0', 'r1', 'r2'], [20231, 20232, 20232], [0., 8., np.nan], students=['001', 's', 'missing'])
        tr['raw'] = [1., np.nan, 3.]
        te = self.frame(['t0'], [20233], [9.]); te['raw'] = 5.
        tr.to_csv(fold / 'A_train.csv', index=False); te.to_csv(fold / 'A_test.csv', index=False)
        try:
            events, _, entries, hashes, audit = self.load(aq_splits_dir=str(splits))
            en = next(iter(entries))
            self.assertEqual(len(en['train']), 2)
            self.assertEqual(en['train'].student_id.iloc[0], '001')
            self.assertEqual(set(en['features'][:2]), {'SoTinChi', 'raw'})
            self.assertTrue(np.isnan(en['train']['raw'].iloc[1]))
            self.assertEqual(len(events), 2)
            self.assertEqual(audit['csv_rows_missing_target_dropped'], 1)
        finally:
            import shutil
            shutil.rmtree(splits)

    def test_csv_categorical_encoding_fits_training_values_only(self):
        fold = self.root / 'data/splits/expanding/fold_1'
        fold.mkdir(parents=True)
        tr = self.frame(['r0','r1','r2'], [20231,20232,20232], [0.,8.,6.])
        te = self.frame(['t0'], [20233], [9.])
        tr['category'] = ['red','blue',None]; te['category'] = ['unseen']
        tr['raw'] = [1.,2.,3.]; te['raw'] = [4.]
        tr.to_csv(fold/'A_train.csv',index=False); te.to_csv(fold/'A_test.csv',index=False)
        _, _, entries, _, _ = self.load()
        en = next(iter(entries)); names = [x for x in en['features'] if x.startswith('category_')]
        self.assertEqual(set(names), {'category_red','category_blue','category___MISSING__'})
        self.assertEqual(en['train'][names].to_numpy().sum(), 3.)
        self.assertEqual(en['test'][names].to_numpy().sum(), 0.)

    def test_calendar_uses_internal_ordinal_carriers(self):
        self.write()
        events, calendar, _, _, audit = self.load()
        row = calendar.set_index('semester').loc['20231']
        self.assertEqual(row.origin_at, pd.Timestamp('1900-01-01', tz='UTC') + pd.Timedelta(days=semester_ordinal(20231)))
        self.assertEqual(row.labels_available_at - row.origin_at, pd.Timedelta(hours=12))
        self.assertIn('not actual dates', audit['timestamp_semantics'])

    def test_grade_record_identity_is_retained_and_deduplicates_history(self):
        tr = self.frame(['registration1', 'registration2', 'r3'], [20231, 20231, 20232], [8., 8., 0.])
        tr['target_IDCongDiem'] = ['17.0', 17, None]
        self.write(train=tr, sidecar=False)
        events, _, entries, _, audit = self.load()
        self.assertEqual(len(next(iter(entries))['train']), 3)
        self.assertEqual(len(events), 2)
        self.assertEqual(events.loc[events.grade == 8., 'grade_record_id'].tolist(), ['17'])
        self.assertEqual(audit['history_grade_identity_duplicates_removed'], 1)

    def test_same_registration_cannot_change_grade_record_identity(self):
        tr = self.frame(['r0', 'r1', 'r2'], [20231, 20232, 20232], [0., 8., 6.])
        tr['target_IDCongDiem'] = ['g0', 'g1', 'g2']
        self.write(train=tr, sidecar=False)
        tr2 = pd.concat([tr, self.frame(['t0'], [20233], [9.])], ignore_index=True)
        tr2.loc[0, 'target_IDCongDiem'] = 'different'
        self.write(fold=2, train=tr2, test=self.frame(['future'], [20241], [8.]), sidecar=False)
        with self.assertRaisesRegex(ValueError, 'grade record identity'):
            load_native_inputs(dict(aq_root=str(self.root), aq_folds=[1, 2]), self.root)

    def test_target_only_students_supply_join_metadata_without_target_grades(self):
        self.write(test=self.frame(['only_target'], [20233], [1.], students=['new_student']), sidecar=False)
        events, _, _, _, _ = self.load()
        self.assertNotIn('new_student', set(events.student_id))
        self.assertEqual(events.attrs.get('student_faculty_memberships', {}).get('new_student'), ['A'])

    def test_reconstruction_matches_preserved_population_std_and_singleton_zero(self):
        tr = self.frame(['r0', 'r1', 'r2'], [20231, 20231, 20232], [0., 8., 6.])
        self.write(train=tr, sidecar=False)
        _, _, entries, _, _ = self.load()
        en = next(iter(entries))
        self.assertEqual(en['train'].loc[2, 'course::hist_std'], 4.)
        tr = self.frame(['r0', 'r1', 'r2'], [20231, 20232, 20232], [0., 8., 6.])
        tr['SoTinChi'] = [None, 3., 3.]
        self.write(train=tr, sidecar=False)
        events, _, entries, _, _ = self.load()
        self.assertEqual(next(iter(entries))['train'].loc[1, 'course::hist_std'], 0.)
        row = events[events.grade == 0.].iloc[0]
        self.assertTrue(row.credits_missing)
        self.assertEqual(row.credits, 0.)


if __name__ == '__main__':
    unittest.main()
