"""Source adapters are exercised only on synthetic CSVs and trusted joblibs."""
import importlib.util
import json
import tempfile
import unittest
from pathlib import Path

import joblib
import numpy as np
import pandas as pd

# A missing module is an assertion failure during the first red run.
ADAPTER_SPEC = importlib.util.find_spec('pace.adapters')
if ADAPTER_SPEC:
    from pace.adapters import AdapterError, export_aq, extract_events, inspect_sources, resolve_key_path


class AdapterTests(unittest.TestCase):
    def setUp(self):
        self.assertIsNotNone(ADAPTER_SPEC, 'pace.adapters must implement source adapters')
        self.temp = tempfile.TemporaryDirectory()
        self.addCleanup(self.temp.cleanup)
        self.root = Path(self.temp.name)

    def write_json(self, name, value):
        path = self.root / name
        path.write_text(json.dumps(value), encoding='utf-8')
        return path

    def raw(self):
        (self.root / 'raw.csv').write_text(
            'reg,student,course,term,score,cr,published,faculty_key\n'
            '0001,0007,0042,20201,4.5,3,2020-05-15,F1\n'
            '0002,0008,0043,20201,,2,2020-05-15,F1\n', encoding='utf-8')
        (self.root / 'faculties.csv').write_text('code,name\nF1,Science\n', encoding='utf-8')
        return {
            'source': {'path': 'raw.csv', 'read_csv': {}},
            'joins': [{'path': 'faculties.csv', 'left_on': ['faculty_key'],
                       'right_on': ['code'], 'validate': 'many_to_one'}],
            'columns': {'event_id': 'reg', 'student_id': 'student', 'course_id': 'course',
                        'semester': 'term', 'grade': 'score', 'credits': 'cr',
                        'available_at': 'published', 'faculty': 'name'},
            'missing_grade': 'retain',
            'timestamp_timezone': 'UTC',
        }

    def aq(self, coverage='covered', sidecar=True):
        train = pd.DataFrame({'IDSV': ['001', '002', '003'], 'IDMH': ['c1', 'c2', 'c3'],
                              'target_NHHK': ['t1', 't1', 't1'], 'registration': ['r1', 'r2', 'r3']})
        test = pd.DataFrame({'IDSV': ['001'], 'IDMH': ['c4'],
                             'target_NHHK': ['t2'], 'registration': ['r4']})
        xfull = np.array([[1., 3.], [2., 4.], [5., 6.]])
        payload = {'model': {'Xtr': xfull if coverage == 'full' else xfull[[0, 2]],
                             'Xte': np.array([[8., 9.]]),
                             'ytr': np.array([0, 1, 1]) if coverage == 'full' else np.array([0, 1]),
                             'yte': np.array([1]), 'train_frame': train, 'test_frame': test,
                             'feature_names': ['f_a', 'f_b'],
                             'cf_covered_mask': np.array([True, False, True])}}
        cache = self.root / 'cache' / 'SCI_fold_1.joblib'
        cache.parent.mkdir()
        joblib.dump(payload, cache)
        cfg = {'trusted_joblib': True, 'faculties': ['SCI'], 'folds': [1],
               'cache_pattern': 'cache/{faculty}_fold_{fold}.joblib',
               'key_paths': {key: 'model.' + key for key in payload['model']},
               'xtr_coverage': coverage, 'columns': {'row_id': 'registration'},
               'calendar': {'path': 'calendar.csv', 'timestamp_timezone': 'UTC'}}
        if sidecar:
            joblib.dump({'target': {'train': np.array(['c1', 'c3']), 'test': np.array(['c4']),
                                   'indices': np.array([0, 2])}}, self.root / 'sidecar_SCI_1.joblib')
            cfg['sidecar_pattern'] = 'sidecar_{faculty}_{fold}.joblib'
            cfg['sidecar_key_paths'] = {'train_course_target': 'target.train',
                                         'test_course_target': 'target.test',
                                         'train_row_index': 'target.indices'}
        (self.root / 'calendar.csv').write_text(
            'semester,order,origin_at,labels_available_at\n'
            't1,1,2020-01-01,2020-06-01\nt2,2,2021-01-01,2021-06-01\n', encoding='utf-8')
        return cfg, cache

    def test_raw_preserves_string_ids_missing_grades_and_join(self):
        audit = extract_events(self.write_json('map.json', self.raw()), self.root / 'events.csv')
        got = pd.read_csv(self.root / 'events.csv', dtype=str)
        self.assertEqual(got.event_id.tolist(), ['0001', '0002'])
        self.assertEqual(got.student_id.tolist(), ['0007', '0008'])
        self.assertEqual(got.course_id.tolist(), ['0042', '0043'])
        self.assertEqual(got.faculty.tolist(), ['Science', 'Science'])
        self.assertTrue(pd.isna(got.grade.iloc[1]))
        self.assertEqual(audit['missing_grade_rows'], 1)
        self.assertEqual(audit['output_rows'], 2)

    def test_raw_explicit_missing_grade_exclusion_is_audited(self):
        cfg = self.raw()
        cfg['missing_grade'] = 'exclude'
        audit = extract_events(self.write_json('map.json', cfg), self.root / 'events.csv')
        self.assertEqual(audit['excluded_missing_grade_rows'], 1)
        self.assertEqual(audit['output_rows'], 1)

    def test_raw_requires_numeric_grades_and_explicit_availability(self):
        cfg = self.raw()
        raw = self.root / 'raw.csv'
        raw.write_text(raw.read_text().replace('4.5,3', 'fail,3'))
        with self.assertRaisesRegex(AdapterError, 'grade'):
            extract_events(self.write_json('map.json', cfg), self.root / 'events.csv')
        del cfg['columns']['available_at']
        with self.assertRaisesRegex(AdapterError, 'available_at'):
            extract_events(self.write_json('map2.json', cfg), self.root / 'events.csv')

    def test_raw_strict_default_requires_available_at_mapping(self):
        cfg = self.raw()
        del cfg['columns']['available_at']
        with self.assertRaisesRegex(AdapterError, 'available_at'):
            extract_events(self.write_json('map.json', cfg), self.root / 'events.csv')

    def test_raw_explicit_calendar_embargo_exports_no_inferred_timestamp(self):
        cfg = self.raw()
        cfg['availability_mode'] = 'calendar_embargo'
        del cfg['columns']['available_at']
        del cfg['timestamp_timezone']
        audit = extract_events(self.write_json('map.json', cfg), self.root / 'events.csv')
        got = pd.read_csv(self.root / 'events.csv', dtype=str)
        self.assertNotIn('available_at', got)
        self.assertEqual(got.event_id.tolist(), ['0001', '0002'])
        self.assertEqual(audit['availability_mode'], 'calendar_embargo')
        self.assertIn('proxy', audit['availability_semantics'])
        self.assertIn('runner', audit['availability_semantics'])

    def test_raw_calendar_embargo_does_not_repurpose_mapped_entry_timestamp(self):
        cfg = self.raw()
        cfg['availability_mode'] = 'calendar_embargo'
        raw = self.root / 'raw.csv'
        raw.write_text(raw.read_text().replace('2020-05-15', 'unverified-grade-entry-time'))
        audit = extract_events(self.write_json('map.json', cfg), self.root / 'events.csv')
        got = pd.read_csv(self.root / 'events.csv')
        self.assertNotIn('available_at', got)
        self.assertTrue(audit['ignored_available_at_mapping'])

    def test_raw_rejects_unrecognized_availability_mode(self):
        cfg = self.raw()
        cfg['availability_mode'] = 'guess_from_entry_time'
        with self.assertRaisesRegex(AdapterError, 'availability_mode'):
            extract_events(self.write_json('map.json', cfg), self.root / 'events.csv')

    def test_raw_many_to_one_join_rejects_duplicate_right_keys(self):
        cfg = self.raw()
        (self.root / 'faculties.csv').write_text('code,name\nF1,Science\nF1,Other\n')
        with self.assertRaisesRegex(AdapterError, 'many.to.one|not unique'):
            extract_events(self.write_json('map.json', cfg), self.root / 'events.csv')

    def test_raw_requires_timezone_for_naive_availability(self):
        cfg = self.raw()
        del cfg['timestamp_timezone']
        with self.assertRaisesRegex(AdapterError, 'timestamp_timezone'):
            extract_events(self.write_json('map.json', cfg), self.root / 'events.csv')

    def test_raw_preserves_explicit_timezone_semantics(self):
        cfg = self.raw()
        del cfg['timestamp_timezone']
        raw = self.root / 'raw.csv'
        raw.write_text(raw.read_text().replace('2020-05-15', '2020-05-15T07:00:00+07:00'))
        extract_events(self.write_json('map.json', cfg), self.root / 'events.csv')
        got = pd.read_csv(self.root / 'events.csv')
        self.assertEqual(got.available_at.iloc[0], '2020-05-15T00:00:00+00:00')

    def test_raw_ambiguous_date_requires_explicit_format(self):
        cfg = self.raw()
        raw = self.root / 'raw.csv'
        raw.write_text(raw.read_text().replace('2020-05-15', '05/07/2020'))
        with self.assertRaisesRegex(AdapterError, 'timestamp_format'):
            extract_events(self.write_json('map.json', cfg), self.root / 'events.csv')
        cfg['timestamp_format'] = '%d/%m/%Y'
        extract_events(self.write_json('map2.json', cfg), self.root / 'events.csv')
        got = pd.read_csv(self.root / 'events.csv')
        self.assertEqual(got.available_at.iloc[0], '2020-07-05T00:00:00+00:00')

    def test_raw_rejects_missing_attempt_when_attempt_column_is_declared(self):
        cfg = self.raw()
        cfg['columns']['attempt'] = 'attempt'
        raw = pd.read_csv(self.root / 'raw.csv', dtype=str, keep_default_na=False)
        raw['attempt'] = ['1', '']
        raw.to_csv(self.root / 'raw.csv', index=False)
        with self.assertRaisesRegex(AdapterError, 'attempt'):
            extract_events(self.write_json('map.json', cfg), self.root / 'events.csv')

    def test_aq_covered_export_uses_exact_feature_rows_and_calendar(self):
        cfg, _ = self.aq()
        audit = export_aq(self.write_json('export.json', cfg), self.root / 'normalized')
        manifest = json.loads((self.root / 'normalized/manifest.json').read_text())
        entry = manifest['entries'][0]
        self.assertEqual(entry['features'], ['f_a', 'f_b'])
        train = pd.read_csv(self.root / 'normalized' / entry['train_path'], dtype={'student_id': str})
        self.assertEqual(train.student_id.tolist(), ['001', '003'])
        self.assertEqual(train.row_id.tolist(), ['r1', 'r3'])
        self.assertEqual(train.label.tolist(), [0, 1])
        self.assertEqual(train[['f_a', 'f_b']].to_numpy().tolist(), [[1., 3.], [5., 6.]])
        self.assertEqual(train.label_available_at.iloc[0], '2020-06-01T00:00:00+00:00')
        self.assertEqual(audit['entries'][0]['uncovered_train_rows'], 1)
        self.assertTrue((self.root / 'normalized' / manifest['calendar']).is_file())

    def test_aq_full_export_preserves_uncovered_rows_when_course_is_known(self):
        cfg, _ = self.aq('full')
        export_aq(self.write_json('export.json', cfg), self.root / 'normalized')
        train = pd.read_csv(self.root / 'normalized' / 'SCI_fold_1_train.csv')
        self.assertEqual(train.row_id.tolist(), ['r1', 'r2', 'r3'])
        self.assertEqual(train.f_a.tolist(), [1., 2., 5.])

    def test_aq_requires_explicit_coverage_convention(self):
        cfg, _ = self.aq()
        del cfg['xtr_coverage']
        with self.assertRaisesRegex(AdapterError, 'xtr_coverage'):
            export_aq(self.write_json('export.json', cfg), self.root / 'normalized')

    def test_aq_rejects_sidecar_row_order_mismatch(self):
        cfg, _ = self.aq()
        joblib.dump({'target': {'train': ['c3', 'c1'], 'test': ['c4'], 'indices': [2, 0]}},
                    self.root / 'sidecar_SCI_1.joblib')
        with self.assertRaisesRegex(AdapterError, 'train_row_index.*flatnonzero'):
            export_aq(self.write_json('export.json', cfg), self.root / 'normalized')

    def test_aq_uses_sidecar_targets_when_frames_have_no_course(self):
        cfg, cache = self.aq()
        payload = joblib.load(cache)
        payload['model']['train_frame'] = payload['model']['train_frame'].drop(columns='IDMH')
        payload['model']['test_frame'] = payload['model']['test_frame'].drop(columns='IDMH')
        joblib.dump(payload, cache)
        export_aq(self.write_json('export.json', cfg), self.root / 'normalized')
        train = pd.read_csv(self.root / 'normalized/SCI_fold_1_train.csv')
        self.assertEqual(train.course_id.tolist(), ['c1', 'c3'])

    def test_aq_full_features_never_guess_uncovered_course_targets(self):
        cfg, cache = self.aq('full')
        payload = joblib.load(cache)
        payload['model']['train_frame'] = payload['model']['train_frame'].drop(columns='IDMH')
        joblib.dump(payload, cache)
        with self.assertRaisesRegex(AdapterError, 'uncovered'):
            export_aq(self.write_json('export.json', cfg), self.root / 'normalized')

    def test_aq_rejects_sidecar_course_disagreement(self):
        cfg, _ = self.aq()
        joblib.dump({'target': {'train': ['wrong', 'c3'], 'test': ['c4'], 'indices': [0, 2]}},
                    self.root / 'sidecar_SCI_1.joblib')
        with self.assertRaisesRegex(AdapterError, 'disagrees'):
            export_aq(self.write_json('export.json', cfg), self.root / 'normalized')

    def structured(self, cfg):
        path = self.root / 'sidecar_SCI_1.joblib'
        payload = joblib.load(path)
        payload['extra'] = {'train': np.array([[11.], [33.]]), 'test': np.array([[44.]]),
                            'names': ['structured_a']}
        joblib.dump(payload, path)
        cfg['structured_features'] = {'enabled': True, 'train_key_path': 'extra.train',
                                      'test_key_path': 'extra.test',
                                      'feature_names_key_path': 'extra.names', 'expected_count': 1}
        cfg['expected_total_features'] = 3
        return path

    def test_aq_appends_explicit_structured_features_to_covered_rows(self):
        cfg, _ = self.aq()
        self.structured(cfg)
        export_aq(self.write_json('export.json', cfg), self.root / 'normalized')
        train = pd.read_csv(self.root / 'normalized/SCI_fold_1_train.csv')
        test = pd.read_csv(self.root / 'normalized/SCI_fold_1_test.csv')
        manifest = json.loads((self.root / 'normalized/manifest.json').read_text())
        self.assertEqual(manifest['entries'][0]['features'], ['f_a', 'f_b', 'structured_a'])
        self.assertEqual(train.structured_a.tolist(), [11., 33.])
        self.assertEqual(test.structured_a.tolist(), [44.])

    def test_aq_full_rows_receive_explicit_missing_structured_values(self):
        cfg, _ = self.aq('full')
        self.structured(cfg)
        audit = export_aq(self.write_json('export.json', cfg), self.root / 'normalized')
        train = pd.read_csv(self.root / 'normalized/SCI_fold_1_train.csv')
        self.assertEqual(train.row_id.tolist(), ['r1', 'r2', 'r3'])
        self.assertIn('structured_a', train)
        self.assertEqual(train.structured_a.iloc[[0, 2]].tolist(), [11., 33.])
        self.assertTrue(pd.isna(train.structured_a.iloc[1]))
        self.assertEqual(audit['entries'][0]['structured_missing_train_rows'], 1)

    def test_aq_structured_alignment_never_ignores_row_index_mismatch(self):
        cfg, _ = self.aq()
        sidecar = self.structured(cfg)
        payload = joblib.load(sidecar)
        payload['target']['indices'] = [2, 0]
        joblib.dump(payload, sidecar)
        with self.assertRaisesRegex(AdapterError, 'train_row_index.*flatnonzero'):
            export_aq(self.write_json('export.json', cfg), self.root / 'normalized')

    def test_aq_rejects_duplicate_structured_feature_names(self):
        cfg, _ = self.aq()
        sidecar = self.structured(cfg)
        payload = joblib.load(sidecar)
        payload['extra']['names'] = ['f_a']
        joblib.dump(payload, sidecar)
        with self.assertRaisesRegex(AdapterError, 'duplicate|unique|overlap'):
            export_aq(self.write_json('export.json', cfg), self.root / 'normalized')

    def test_aq_checks_opt_in_expected_total_feature_count(self):
        cfg, _ = self.aq()
        cfg['expected_total_features'] = 153
        with self.assertRaisesRegex(AdapterError, 'expected_total_features.*153'):
            export_aq(self.write_json('export.json', cfg), self.root / 'normalized')

    def test_aq_disabled_structured_section_keeps_canonical_features(self):
        cfg, _ = self.aq()
        self.structured(cfg)
        cfg['structured_features']['enabled'] = False
        cfg['expected_total_features'] = 2
        export_aq(self.write_json('export.json', cfg), self.root / 'normalized')
        manifest = json.loads((self.root / 'normalized/manifest.json').read_text())
        self.assertEqual(manifest['entries'][0]['features'], ['f_a', 'f_b'])

    def test_aq_structured_features_require_explicit_source_keys(self):
        cfg, _ = self.aq()
        self.structured(cfg)
        del cfg['structured_features']['train_key_path']
        with self.assertRaisesRegex(AdapterError, 'train_key_path'):
            export_aq(self.write_json('export.json', cfg), self.root / 'normalized')

    def test_aq_structured_count_mismatch_is_actionable(self):
        cfg, _ = self.aq()
        self.structured(cfg)
        cfg['structured_features']['expected_count'] = 17
        with self.assertRaisesRegex(AdapterError, 'structured.*expected_count.*17'):
            export_aq(self.write_json('export.json', cfg), self.root / 'normalized')

    def test_aq_rejects_mask_length_mismatch(self):
        cfg, cache = self.aq()
        payload = joblib.load(cache)
        payload['model']['cf_covered_mask'] = np.array([True, False])
        joblib.dump(payload, cache)
        with self.assertRaisesRegex(AdapterError, 'cf_covered_mask.*aligned'):
            export_aq(self.write_json('export.json', cfg), self.root / 'normalized')

    def test_aq_checks_full_ytr_coverage_before_selecting_covered_rows(self):
        cfg, cache = self.aq()
        cfg['ytr_coverage'] = 'full'
        payload = joblib.load(cache)
        payload['model']['ytr'] = np.array([0, 0, 1])
        joblib.dump(payload, cache)
        export_aq(self.write_json('export.json', cfg), self.root / 'normalized')
        train = pd.read_csv(self.root / 'normalized/SCI_fold_1_train.csv')
        self.assertEqual(train.label.tolist(), [0, 1])

    def test_aq_rejects_nonbinary_labels_and_no_partial_output(self):
        cfg, cache = self.aq()
        payload = joblib.load(cache)
        payload['model']['yte'] = np.array([4.5])
        joblib.dump(payload, cache)
        with self.assertRaisesRegex(AdapterError, 'binary'):
            export_aq(self.write_json('export.json', cfg), self.root / 'normalized')
        self.assertFalse((self.root / 'normalized').exists())

    def test_aq_rejects_naive_calendar_without_timezone(self):
        cfg, _ = self.aq()
        del cfg['calendar']['timestamp_timezone']
        with self.assertRaisesRegex(AdapterError, 'timestamp_timezone'):
            export_aq(self.write_json('export.json', cfg), self.root / 'normalized')

    def test_aq_rejects_calendar_labels_available_at_origin(self):
        cfg, _ = self.aq()
        calendar = self.root / 'calendar.csv'
        calendar.write_text(calendar.read_text().replace('2020-06-01', '2020-01-01'))
        with self.assertRaisesRegex(AdapterError, 'labels_available_at.*follow.*origin_at'):
            export_aq(self.write_json('export.json', cfg), self.root / 'normalized')

    def test_aq_rejects_feature_order_mismatch(self):
        cfg, cache = self.aq()
        payload = joblib.load(cache)
        payload['model']['Xte'] = pd.DataFrame([[8., 9.]], columns=['f_b', 'f_a'])
        joblib.dump(payload, cache)
        with self.assertRaisesRegex(AdapterError, 'feature.*order'):
            export_aq(self.write_json('export.json', cfg), self.root / 'normalized')

    def test_aq_preserves_represented_float32_feature_values(self):
        cfg, cache = self.aq()
        payload = joblib.load(cache)
        values = np.array([[0.1, 1.234567], [0.3, 2.345678]], dtype=np.float32)
        payload['model']['Xtr'] = values
        joblib.dump(payload, cache)
        export_aq(self.write_json('export.json', cfg), self.root / 'normalized')
        got = pd.read_csv(self.root / 'normalized/SCI_fold_1_train.csv', float_precision='round_trip')
        np.testing.assert_array_equal(got[['f_a', 'f_b']].to_numpy(), values.astype(np.float64))

    def test_aq_rejects_categorical_features_actionably(self):
        cfg, cache = self.aq()
        payload = joblib.load(cache)
        payload['model']['Xte'] = np.array([['a', 'b']])
        joblib.dump(payload, cache)
        with self.assertRaisesRegex(AdapterError, 'numeric.*encod|encod.*numeric'):
            export_aq(self.write_json('export.json', cfg), self.root / 'normalized')

    def test_aq_key_mismatch_reports_available_keys(self):
        cfg, _ = self.aq()
        cfg['key_paths']['Xtr'] = 'model.missing'
        with self.assertRaisesRegex(AdapterError, 'missing.*available.*Xtr'):
            export_aq(self.write_json('export.json', cfg), self.root / 'normalized')

    def test_aq_requires_explicit_identity_fields_without_row_key(self):
        cfg, cache = self.aq(sidecar=False)
        cfg['columns'] = {}
        with self.assertRaisesRegex(AdapterError, 'identity_fields'):
            export_aq(self.write_json('export.json', cfg), self.root / 'normalized')
        cfg['identity_fields'] = ['student_id', 'course_id', 'semester', 'faculty']
        export_aq(self.write_json('export.json', cfg), self.root / 'normalized')
        first = pd.read_csv(self.root / 'normalized/SCI_fold_1_train.csv').row_id.tolist()
        export_aq(self.root / 'export.json', self.root / 'normalized_again')
        second = pd.read_csv(self.root / 'normalized_again/SCI_fold_1_train.csv').row_id.tolist()
        self.assertEqual(first, second)

    def test_aq_keeps_each_fold_feature_snapshot(self):
        cfg, cache = self.aq(sidecar=False)
        payload = joblib.load(cache)
        payload['model']['Xtr'] += 100
        joblib.dump(payload, self.root / 'cache/SCI_fold_2.joblib')
        cfg['folds'] = [1, 2]
        export_aq(self.write_json('export.json', cfg), self.root / 'normalized')
        manifest = json.loads((self.root / 'normalized/manifest.json').read_text())
        self.assertEqual(len(manifest['entries']), 2)
        a = pd.read_csv(self.root / 'normalized' / manifest['entries'][0]['train_path'])
        b = pd.read_csv(self.root / 'normalized' / manifest['entries'][1]['train_path'])
        self.assertEqual(a.row_id.tolist(), b.row_id.tolist())
        self.assertEqual((b.f_a - a.f_a).tolist(), [100., 100.])

    def test_inspection_exposes_schema_and_never_student_values(self):
        _, cache = self.aq()
        report = inspect_sources([cache])
        serialized = json.dumps(report)
        self.assertIn('IDSV', serialized)
        self.assertIn('shape', serialized)
        self.assertNotIn('001', serialized)
        self.assertNotIn('r1', serialized)

    def test_key_resolver_supports_literal_dotted_mapping_keys(self):
        self.assertEqual(resolve_key_path({'nested': {'a.b': 7}}, 'nested.a.b'), 7)


if __name__ == '__main__':
    unittest.main()
