"""Transcript reconciliation uses real files and hand-checked event counts."""
import hashlib
import importlib
import json
from pathlib import Path
import tempfile
import unittest

import joblib
import numpy as np
import pandas as pd

from pace.aq_native import _origin, semester_ordinal
from pace.data import EventCorpus


class AQHistoryTests(unittest.TestCase):
    def setUp(self):
        self.tmp = tempfile.TemporaryDirectory()
        self.addCleanup(self.tmp.cleanup)
        self.root = Path(self.tmp.name) / 'project'
        self.root.mkdir()
        self.database = self.root.parent / 'database'

    def extend(self, events=None, **kwargs):
        module = importlib.import_module('pace.aq_history')
        return module.extend_history(self.root, self.canonical() if events is None else events, **kwargs)

    def canonical(self, rows=None):
        rows = rows or [dict(event_id='registration', student_id='s', course_id='c', semester='20232', grade=8.)]
        frame = pd.DataFrame(rows)
        for name, value in [('credits', 3.), ('attempt', 1), ('faculty', 'A'), ('faculty_memberships', '["A"]')]:
            if name not in frame:
                frame[name] = value
        frame['available_at'] = [_origin(semester_ordinal(t)) + pd.Timedelta(hours=12) for t in frame.semester]
        return frame

    def raw(self, **extra):
        return dict(IDCongDiem='g', IDSV='s', IDMH='c', NHHK='20232', DiemHP='8', DiemHPCalc='', SoTinChi='3', DatHP='1', **extra)

    def write_csv(self, rows):
        self.database.mkdir(exist_ok=True)
        path = self.database / 'DDTBHP.csv'
        pd.DataFrame(rows).to_csv(path, index=False)
        return path

    def test_absent_automatic_source_leaves_canonical_unchanged(self):
        canonical = self.canonical()
        events, audit, hashes = self.extend(canonical)
        pd.testing.assert_frame_equal(events, canonical)
        self.assertEqual(audit['status'], 'canonical_only_no_source')
        self.assertIn('not found', audit['reason'])
        self.assertEqual(hashes, {})

    def test_raw_invalid_semesters_are_excluded_audited_without_changing_sources(self):
        invalid=[0,'0.0','',None,'unknown',20234,20232.5,-20231,'inf']
        for legacy in (False,True):
            with self.subTest(legacy=legacy):
                rows=[self.raw()]
                for i,value in enumerate(invalid):
                    row=self.raw();row.update(IDCongDiem=f'invalid{i}',NHHK=value,IDMH='c' if i==0 else None)
                    rows.append(row)
                prior=self.raw();prior.update(IDCongDiem='prior',NHHK=20231,DiemHP=0.)
                rows.append(prior)
                if legacy:
                    source=self.root/'course_history.joblib';joblib.dump(pd.DataFrame(rows),source)
                    kwargs={'history_index':source}
                else:
                    source=self.write_csv(rows);kwargs={}
                before=source.read_bytes()
                events,audit,_=self.extend(**kwargs)
                self.assertEqual(len(events),2)
                self.assertEqual(set(events.semester),{'20231','20232'})
                self.assertEqual(events.loc[events.semester=='20231','grade'].tolist(),[0.])
                self.assertEqual(audit['raw_invalid_semester_rows_dropped'],len(invalid))
                self.assertIn('0',audit['raw_invalid_semester_examples'])
                self.assertIn('not inferred',audit['raw_semester_policy'])
                self.assertEqual(source.read_bytes(),before)

    def test_invalid_canonical_semester_still_fails_instead_of_dropping_enrollments(self):
        self.write_csv([self.raw()])
        canonical=self.canonical();canonical['semester']='0'
        with self.assertRaisesRegex(ValueError,'semester'):
            self.extend(canonical)

    def test_explicit_missing_or_invalid_sources_fail(self):
        with self.assertRaises(FileNotFoundError):
            self.extend(database_dir=self.database)
        with self.assertRaises(FileNotFoundError):
            self.extend(history_index=self.root / 'missing.joblib')
        self.write_csv([dict(unknown='x')])
        with self.assertRaisesRegex(ValueError, 'schema'):
            self.extend(database_dir=self.database)
        path = self.root / 'bad.joblib'; joblib.dump({'unknown': []}, path)
        with self.assertRaisesRegex(ValueError, 'DataFrame'):
            self.extend(history_index=path)

    def test_automatic_csv_supplements_selected_students_and_hashes_bytes(self):
        older = self.raw(); older.update(IDCongDiem='a/|', NHHK='20231', DiemHP='0')
        other = self.raw(); other.update(IDSV='unselected', IDCongDiem='other')
        path = self.write_csv([self.raw(), older, other])
        before = path.read_bytes()
        events, audit, hashes = self.extend()
        self.assertEqual(set(events.event_id), {'grade|g', 'grade|a%2F%7C'})
        self.assertEqual(events.sort_values('semester').grade.tolist(), [0., 8.])
        self.assertEqual(audit['raw_rows_unselected_students'], 1)
        self.assertIn(hashlib.sha256(before).hexdigest(), hashes.values())
        self.assertEqual(path.read_bytes(), before)

    def test_legacy_index_without_grade_ids_reconciles_and_is_stable(self):
        directory = self.root / 'results/suite12_structured_cache/raw_index'
        directory.mkdir(parents=True)
        first = self.raw(); first.update(NHHK='20231', DiemHP='4')
        raw = pd.DataFrame([first, self.raw()]).drop(columns='IDCongDiem')
        path = directory / 'course_history.joblib'; joblib.dump(raw, path)
        events, audit, hashes = self.extend()
        self.assertEqual(len(events), 2)
        self.assertEqual(events.sort_values('semester').attempt.tolist(), [1, 2])
        self.assertTrue(all(x.startswith('raw|') for x in events.event_id))
        self.assertIn(hashlib.sha256(path.read_bytes()).hexdigest(), hashes.values())
        again, _, _ = self.extend()
        pd.testing.assert_frame_equal(events, again)

    def test_grade_coalescence_zero_and_invalid_counts(self):
        rows = []
        for i, (primary, fallback) in enumerate([('', '0'), ('', ''), ('bad', '8'), ('11', ''), ('inf', ''), ('4', '9')]):
            row = self.raw(); row.update(IDCongDiem=str(i), IDMH=str(i), DiemHP=primary, DiemHPCalc=fallback, SoTinChi='')
            rows.append(row)
        self.write_csv(rows)
        events, audit, _ = self.extend()
        raw = events[events.event_id.str.startswith('grade|')]
        self.assertEqual(raw.grade.tolist(), [0., 8., 4.])
        self.assertEqual(raw.credits.tolist(), [0., 0., 0.])
        self.assertEqual(audit['raw_grades_coalesced'], 2)
        self.assertEqual(audit['raw_invalid_primary_grades_recovered'], 1)
        self.assertEqual(audit['raw_missing_grade_rows_dropped'], 1)
        self.assertEqual(audit['raw_invalid_grade_rows_dropped'], 1)
        self.assertEqual(audit['raw_out_of_range_grade_rows_dropped'], 1)
        self.assertEqual(audit['raw_missing_credits_default_zero'], 3)

    def test_nonnumeric_primary_uses_numeric_calc_but_out_of_range_does_not(self):
        status = self.raw(); status.update(IDCongDiem='status', IDMH='status', DiemHP='not graded', DiemHPCalc='6')
        invalid = self.raw(); invalid.update(IDCongDiem='invalid', IDMH='invalid', DiemHP='bad', DiemHPCalc='bad')
        outside = self.raw(); outside.update(IDCongDiem='outside', IDMH='outside', DiemHP='11', DiemHPCalc='8')
        self.write_csv([status, invalid, outside])
        events, audit, _ = self.extend()
        selected = events[events.event_id.str.startswith('grade|')]
        self.assertEqual(selected.event_id.tolist(), ['grade|status'])
        self.assertEqual(selected.grade.tolist(), [6.])
        self.assertEqual(audit['raw_grades_coalesced'], 1)
        self.assertEqual(audit['raw_invalid_primary_grades_recovered'], 1)
        self.assertEqual(audit['raw_invalid_grade_rows_dropped'], 1)
        self.assertEqual(audit['raw_out_of_range_grade_rows_dropped'], 1)

    def test_exact_id_recovers_missing_grade_and_retains_canonical_registration(self):
        row = dict(event_id='r', student_id='s', course_id='c', semester='20232', grade=np.nan, grade_record_id='17.0', failure_label=0)
        unmatched = dict(row, event_id='registration-only', course_id='other', grade_record_id=None)
        source = self.raw(); source['IDCongDiem'] = '17'
        self.write_csv([source])
        events, audit, _ = self.extend(self.canonical([row, unmatched]))
        self.assertEqual(len(events), 2)
        self.assertEqual(events.set_index('event_id').loc['grade|17', 'grade'], 8.)
        self.assertTrue(pd.isna(events.set_index('event_id').loc['registration-only', 'grade']))
        self.assertEqual(audit['canonical_missing_grades_recovered'], 1)

    def test_exact_id_rejects_identity_grade_or_label_conflict(self):
        self.write_csv([self.raw()])
        base = dict(event_id='r', student_id='s', course_id='c', semester='20232', grade=8., grade_record_id='g')
        for change in [dict(course_id='wrong'), dict(semester='20231'), dict(grade=7.), dict(grade=np.nan, failure_label=1)]:
            with self.subTest(change=change):
                row = dict(base, **change)
                with self.assertRaisesRegex(ValueError, 'conflict'):
                    self.extend(self.canonical([row]))

    def test_identical_duplicate_grade_ids_removed_but_conflicts_rejected(self):
        self.write_csv([self.raw(), self.raw()])
        events, audit, _ = self.extend()
        self.assertEqual(len(events), 1)
        self.assertEqual(audit['raw_duplicate_grade_ids_removed'], 1)
        for column, value in [('IDSV', 'other'), ('IDMH', 'other'), ('DiemHP', '7')]:
            other = self.raw(); other[column] = value
            self.write_csv([self.raw(), other])
            with self.subTest(column=column), self.assertRaisesRegex(ValueError, 'duplicate grade.*conflict'):
                self.extend()

    def test_matching_consumes_one_raw_attempt_per_canonical_record(self):
        second = self.raw(); second['IDCongDiem'] = 'g2'
        third = self.raw(); third.update(IDCongDiem='g3', DiemHP='4')
        self.write_csv([self.raw(), second, third])
        events, audit, _ = self.extend()
        self.assertEqual(len(events), 3)
        self.assertEqual(set(events.event_id), {'grade|g', 'grade|g2', 'grade|g3'})
        self.assertEqual(audit['canonical_tuple_matches'], 1)
        canonical = self.canonical([dict(event_id=x, student_id='s', course_id='c', semester='20232', grade=8.) for x in ['r1', 'r2']])
        events, audit, _ = self.extend(canonical)
        self.assertEqual(len(events), 3)
        self.assertEqual(audit['canonical_tuple_matches'], 2)

    def test_unique_tuple_recovers_missing_grade_but_ambiguity_fails(self):
        canonical = self.canonical([dict(event_id='r', student_id='s', course_id='c', semester='20232', grade=np.nan)])
        self.write_csv([self.raw()])
        events, _, _ = self.extend(canonical)
        self.assertEqual(events.grade.tolist(), [8.])
        second = self.raw(); second.update(IDCongDiem='g2', DiemHP='4')
        self.write_csv([self.raw(), second])
        with self.assertRaisesRegex(ValueError, 'ambiguous'):
            self.extend(canonical)

    def test_observed_tuple_with_distinct_candidate_credits_is_ambiguous(self):
        second = self.raw(); second.update(IDCongDiem='g2', SoTinChi='4')
        self.write_csv([self.raw(), second])
        with self.assertRaisesRegex(ValueError, 'ambiguous'):
            self.extend()

    def test_raw_credits_recover_only_canonical_values_marked_missing(self):
        for exact, marked, expected in [(True, True, 3.), (False, True, 3.), (True, False, 0.), (False, False, 0.)]:
            with self.subTest(exact=exact, credits_missing=marked):
                row = dict(event_id='r', student_id='s', course_id='c', semester='20232', grade=8., credits=0., credits_missing=marked)
                if exact:
                    row['grade_record_id'] = 'g'
                prior = self.raw(); prior.update(IDCongDiem='prior', NHHK='20231', SoTinChi='4')
                self.write_csv([self.raw(), prior])
                events, audit, _ = self.extend(self.canonical([row]))
                indexed = events.set_index('event_id')
                self.assertEqual(indexed.loc['grade|g', 'credits'], expected)
                self.assertFalse(indexed.loc['grade|g', 'credits_missing'])
                self.assertFalse(indexed.loc['grade|prior', 'credits_missing'])
                self.assertEqual(audit['canonical_missing_credits_recovered'], int(marked))
                self.assertEqual(audit['canonical_raw_observed_credit_differences'], int(not marked))

    def test_two_unidentified_registrations_cannot_duplicate_one_raw_grade(self):
        canonical = self.canonical([dict(event_id=x, student_id='s', course_id='c', semester='20232', grade=8.) for x in ['r1', 'r2']])
        self.write_csv([self.raw()])
        with self.assertRaisesRegex(ValueError, 'ambiguous.*registrations.*raw'):
            self.extend(canonical)
        second = self.raw(); second['IDCongDiem'] = 'g2'
        self.write_csv([self.raw(), second])
        events, audit, _ = self.extend(canonical)
        self.assertEqual(set(events.event_id), {'grade|g', 'grade|g2'})
        self.assertEqual(len(events), 2)
        self.assertEqual(audit['canonical_tuple_matches'], 2)

    def test_unknown_grade_identity_cannot_turn_tuple_grade_conflict_into_two_attempts(self):
        for grade_id in ['g', None]:
            with self.subTest(raw_grade_record_id=grade_id):
                raw = self.raw(); raw.update(IDCongDiem=grade_id, DiemHP='4')
                self.write_csv([raw])
                with self.assertRaisesRegex(ValueError, 'tuple.*grade mismatch') as error:
                    self.extend()
                self.assertIn("('s', 'c', '20232')", str(error.exception))
                self.assertNotIn('supply', str(error.exception))

    def test_known_canonical_grade_id_cannot_hide_legacy_tuple_grade_conflict(self):
        canonical = self.canonical([dict(event_id='r', student_id='s', course_id='c', semester='20232', grade=8., grade_record_id='canonical')])
        raw = self.raw(); raw.update(IDCongDiem=None, DiemHP='4')
        self.write_csv([raw])
        with self.assertRaisesRegex(ValueError, 'tuple.*grade mismatch'):
            self.extend(canonical)

    def test_unidentified_registration_cannot_reuse_consumed_matching_grade(self):
        canonical = self.canonical([dict(event_id='a-known', student_id='s', course_id='c', semester='20232', grade=8., grade_record_id='g'), dict(event_id='b-unknown', student_id='s', course_id='c', semester='20232', grade=8., grade_record_id=None)])
        for raw_id in ['g', None]:
            with self.subTest(raw_grade_record_id=raw_id):
                raw = self.raw(); raw['IDCongDiem'] = raw_id
                self.write_csv([raw])
                with self.assertRaisesRegex(ValueError, 'ambiguous.*consumed'):
                    self.extend(canonical)

    def test_unidentified_conflicting_or_missing_grade_cannot_survive_consumed_tuple(self):
        for grade in [4., np.nan]:
            for raw_id in ['g', None]:
                with self.subTest(unknown_grade=grade, raw_grade_record_id=raw_id):
                    canonical = self.canonical([dict(event_id='a-known', student_id='s', course_id='c', semester='20232', grade=8., grade_record_id='g'), dict(event_id='b-unknown', student_id='s', course_id='c', semester='20232', grade=grade, grade_record_id=None)])
                    raw = self.raw(); raw['IDCongDiem'] = raw_id
                    self.write_csv([raw])
                    with self.assertRaisesRegex(ValueError, 'ambiguous.*consumed'):
                        self.extend(canonical)

    def test_distinct_known_grade_ids_remain_distinct_attempts(self):
        canonical = self.canonical([dict(event_id='r', student_id='s', course_id='c', semester='20232', grade=8., grade_record_id='canonical')])
        self.write_csv([self.raw()])
        events, _, _ = self.extend(canonical)
        self.assertEqual(len(events), 2)
        self.assertEqual(set(events.grade_record_id), {'g', 'canonical'})
        raw = self.raw(); raw['DiemHP'] = '4'
        self.write_csv([raw])
        events, _, _ = self.extend(canonical)
        self.assertEqual(sorted(events.grade.tolist()), [4., 8.])
        self.assertEqual(set(events.grade_record_id), {'g', 'canonical'})
        canonical = self.canonical([dict(event_id=x, student_id='s', course_id='c', semester='20232', grade=8., grade_record_id=x) for x in ['g', 'h']])
        legacy = self.raw(); legacy['IDCongDiem'] = None
        self.write_csv([legacy])
        events, _, _ = self.extend(canonical)
        self.assertEqual(len(events), 2)
        self.assertEqual(set(events.grade_record_id), {'g', 'h'})

    def test_missing_grade_canonical_duplicates_cannot_arbitrarily_claim_one_raw(self):
        canonical = self.canonical([dict(event_id=x, student_id='s', course_id='c', semester='20232', grade=np.nan) for x in ['r1', 'r2']])
        self.write_csv([self.raw()])
        with self.assertRaisesRegex(ValueError, 'ambiguous'):
            self.extend(canonical)

    def test_canonical_known_grade_id_survives_legacy_tuple_match(self):
        directory = self.root / 'legacy'; directory.mkdir()
        path = directory / 'course_history.joblib'
        joblib.dump(pd.DataFrame([self.raw()]).drop(columns='IDCongDiem'), path)
        canonical = self.canonical([dict(event_id='registration', student_id='s', course_id='c', semester='20232', grade=8., grade_record_id='canonical/id')])
        events, _, _ = self.extend(canonical, history_index=path)
        self.assertEqual(events.event_id.tolist(), ['grade|canonical%2Fid'])
        self.assertEqual(events.grade_record_id.tolist(), ['canonical/id'])

    def test_numeric_ids_normalize_without_losing_zero_padding(self):
        canonical = self.canonical([dict(event_id='r', student_id=17., course_id='17.0', semester='20232', grade=8.), dict(event_id='z', student_id='001', course_id='001', semester='20232', grade=8.)])
        a = self.raw(); a.update(IDSV='17.0', IDMH='17', IDCongDiem='17.0')
        b = self.raw(); b.update(IDSV='001', IDMH='001', IDCongDiem='001')
        self.write_csv([a, b])
        events, _, _ = self.extend(canonical)
        self.assertEqual(set(zip(events.student_id, events.course_id)), {('17', '17'), ('001', '001')})
        self.assertEqual(set(events.grade_record_id), {'17', '001'})

    def test_metadata_memberships_select_target_only_students_without_target_grades(self):
        canonical = self.canonical()
        canonical.attrs['student_faculty_memberships'] = {'new.0': ['B', 'C']}
        past = self.raw(); past.update(IDSV='new.0', IDCongDiem='new-past', NHHK='20231', DiemHP='4')
        future = self.raw(); future.update(IDSV='new.0', IDCongDiem='new-future', NHHK='20233', DiemHP='9')
        self.write_csv([self.raw(), past, future])
        events, _, _ = self.extend(canonical)
        selected = events[events.student_id == 'new.0']
        self.assertEqual(selected.grade.tolist(), [4., 9.])
        self.assertEqual(selected.faculty_memberships.tolist(), ['["B", "C"]'] * 2)
        calendar = pd.DataFrame([dict(semester=t, order=semester_ordinal(t), origin_at=_origin(semester_ordinal(t)), labels_available_at=_origin(semester_ordinal(t)) + pd.Timedelta(hours=12)) for t in ['20231', '20232', '20233']])
        corpus = EventCorpus(events, calendar, 'semester_order')
        self.assertEqual(corpus.prefix('new.0', '20233', _origin(semester_ordinal('20233'))).grade.tolist(), [4.])

    def test_membership_union_and_strict_cutoffs_exclude_future_records(self):
        canonical = self.canonical([dict(event_id='a', student_id='s', course_id='c', semester='20232', grade=8., faculty='A', faculty_memberships='["A", "B"]'), dict(event_id='b', student_id='s', course_id='other', semester='20231', grade=7., faculty='C', faculty_memberships='["C"]')])
        rows = [self.raw()]
        for sem, gid in [('20231', 'past'), ('20233', 'future')]:
            row = self.raw(); row.update(NHHK=sem, IDCongDiem=gid)
            rows.append(row)
        self.write_csv(rows)
        events, audit, _ = self.extend(canonical)
        self.assertTrue(all(json.loads(x) == ['A', 'B', 'C'] for x in events.faculty_memberships))
        terms = ['20231', '20232', '20233']
        calendar = pd.DataFrame([dict(semester=t, order=semester_ordinal(t), origin_at=_origin(semester_ordinal(t)), labels_available_at=_origin(semester_ordinal(t)) + pd.Timedelta(hours=12)) for t in terms])
        corpus = EventCorpus(events, calendar, 'semester_order')
        origin = _origin(semester_ordinal('20232'))
        self.assertEqual(len(corpus.prefix('s', '20232', origin)), 2)
        self.assertEqual(len(corpus.before(origin, faculty='B')), 2)
        exact_release = origin + pd.Timedelta(hours=12)
        self.assertEqual(len(corpus.before(exact_release)), 2)
        self.assertEqual(events[events.course_id == 'c'].sort_values('semester').attempt.tolist(), [1, 2, 3])
        self.assertIn('not actual dates', audit['timestamp_semantics'])
        self.assertIn('assum', audit['availability_assumption'])


if __name__ == '__main__':
    unittest.main()
