#!/usr/bin/env python3
"""
Read-only structural audit for the EEG-sAAD Grand Challenge datasets.

The script inventories files and MATLAB variables, inspects MATLAB structs
without changing source data, and writes reports under --out.
It does NOT filter, resample, rereference, epoch, or overwrite EEG data.
"""
from __future__ import annotations

import argparse
import csv
import json
import os
import re
import sys
import traceback
from collections import Counter
from pathlib import Path
from typing import Any

DATASET_NAMES = [
    "AVGCAAD2024", "DTU2018", "KUL2016",
    "KUL_AAD_UltraHD2021", "USTC2026"
]
MAT_NAME_RE = re.compile(
    r"^(dataStim|pre_dataStim|dataSub|pre_dataSub).*\.mat$", re.I
)
MATLAB_CODE_HINTS = re.compile(
    r"\b(load|save|pop_biosig|pop_loadset|pop_eegfiltnew|"
    r"pop_reref|pop_resample|eeg_checkset|srate|chanlocs|"
    r"event|trigger|boundary|filter|resample|reref|"
    r"removeBadChannels|preprocess)\b", re.I
)

def safe_rel(path: Path, root: Path) -> str:
    try:
        return str(path.relative_to(root))
    except Exception:
        return str(path)

def jsonable(x: Any):
    try:
        import numpy as np
        if isinstance(x, np.ndarray):
            if x.size <= 30:
                return x.tolist()
            return {"shape": list(x.shape), "dtype": str(x.dtype)}
        if isinstance(x, (np.integer, np.floating)):
            return x.item()
    except Exception:
        pass
    if isinstance(x, Path):
        return str(x)
    return str(x)

def summarize_value(value: Any, depth=0, max_depth=3):
    """Compact summary; avoid dumping large arrays."""
    try:
        import numpy as np
        if isinstance(value, dict):
            if depth >= max_depth:
                return {"type": "dict", "keys": list(value.keys())[:100]}
            return {
                str(k): summarize_value(v, depth + 1, max_depth)
                for k, v in value.items()
            }
        if isinstance(value, np.ndarray):
            info = {"type": "ndarray", "shape": list(value.shape),
                    "dtype": str(value.dtype), "size": int(value.size)}
            if value.size <= 20:
                info["values"] = value.tolist()
            elif np.issubdtype(value.dtype, np.number):
                flat = value.ravel()
                sample = flat[: min(10000, flat.size)]
                finite = sample[np.isfinite(sample)]
                if finite.size:
                    info["sample_stats"] = {
                        "sample_n": int(finite.size),
                        "min": float(finite.min()),
                        "max": float(finite.max()),
                        "mean": float(finite.mean()),
                        "std": float(finite.std()),
                    }
            return info
        if hasattr(value, "_fieldnames"):  # scipy MATLAB mat_struct
            fields = {}
            for f in value._fieldnames:
                fields[f] = summarize_value(getattr(value, f), depth + 1, max_depth)
            return {"type": "mat_struct", "fields": fields}
        if isinstance(value, (list, tuple)):
            return {"type": type(value).__name__, "length": len(value),
                    "items": [summarize_value(v, depth+1, max_depth)
                              for v in value[:10]]}
        if isinstance(value, (str, int, float, bool)) or value is None:
            return value
        return {"type": type(value).__name__, "repr": repr(value)[:200]}
    except Exception as e:
        return {"summary_error": str(e), "type": type(value).__name__}

def inspect_mat(path: Path, max_load_mb: float):
    record = {"path": str(path), "size_bytes": path.stat().st_size,
              "format": "unknown", "variables": [], "details": {},
              "warnings": []}
    size_mb = path.stat().st_size / (1024 * 1024)
    try:
        from scipy.io import whosmat, loadmat
        try:
            variables = whosmat(str(path))
            record["format"] = "MATLAB pre-v7.3 / scipy-readable"
            record["variables"] = [
                {"name": n, "shape": list(shape), "matlab_class": cls}
                for n, shape, cls in variables
            ]
            # Load small files only. This is metadata inspection, not preprocessing.
            if size_mb <= max_load_mb:
                loaded = loadmat(str(path), simplify_cells=True)
                for name, value in loaded.items():
                    if name.startswith("__"):
                        continue
                    record["details"][name] = summarize_value(value)
            else:
                record["warnings"].append(
                    f"Skipped value-level load ({size_mb:.1f} MB > "
                    f"{max_load_mb:.1f} MB limit); header metadata only."
                )
            return record
        except Exception as scipy_error:
            record["warnings"].append(f"SciPy reader: {scipy_error}")
    except ImportError:
        record["warnings"].append("scipy is not installed; cannot inspect MAT header.")

    # MATLAB v7.3 files are HDF5 containers.
    try:
        import h5py
        with h5py.File(path, "r") as h5:
            record["format"] = "MATLAB v7.3 / HDF5"
            def walk(group, prefix=""):
                for key, obj in group.items():
                    name = f"{prefix}/{key}".strip("/")
                    if isinstance(obj, h5py.Dataset):
                        record["variables"].append({
                            "name": name, "shape": list(obj.shape),
                            "dtype": str(obj.dtype), "matlab_class":
                            str(obj.attrs.get("MATLAB_class", "")),
                        })
                    elif isinstance(obj, h5py.Group):
                        walk(obj, name)
            walk(h5)
    except Exception as e:
        record["warnings"].append(f"HDF5 fallback failed: {e}")
    return record

def inspect_m_file(path: Path):
    result = {"path": str(path), "bytes": path.stat().st_size,
              "functions": [], "interesting_lines": []}
    try:
        text = path.read_text(errors="replace")
        for line_no, line in enumerate(text.splitlines(), 1):
            stripped = line.strip()
            if re.match(r"^\s*function\b", line):
                result["functions"].append({"line": line_no, "text": stripped[:300]})
            if MATLAB_CODE_HINTS.search(line) and not stripped.startswith("%"):
                result["interesting_lines"].append(
                    {"line": line_no, "text": stripped[:350]}
                )
    except Exception as e:
        result["error"] = str(e)
    return result

def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--root", required=True, help="Dataset root containing the five dataset folders")
    ap.add_argument("--out", required=True, help="Output directory for audit reports")
    ap.add_argument("--datasets", nargs="*", default=DATASET_NAMES,
                    help="Dataset folder names; defaults to the five shown in the screenshots")
    ap.add_argument("--max-load-mb", type=float, default=128,
                    help="Only load small MAT files for nested value summaries (default 128 MB)")
    args = ap.parse_args()

    root, out = Path(args.root).expanduser(), Path(args.out).expanduser()
    if not root.is_dir():
        raise SystemExit(f"ERROR: dataset root does not exist: {root}")
    out.mkdir(parents=True, exist_ok=True)

    all_inventory, mat_records, m_records, dataset_summary = [], [], [], []
    for ds_name in args.datasets:
        ds = root / ds_name
        if not ds.is_dir():
            dataset_summary.append({"dataset": ds_name, "status": "MISSING"})
            continue

        files = [p for p in ds.rglob("*") if p.is_file()]
        ext_counts = Counter((p.suffix.lower() or "[no extension]") for p in files)
        folder_counts = Counter(safe_rel(p.parent, ds) for p in files)
        total_bytes = sum(p.stat().st_size for p in files)

        dataset_summary.append({
            "dataset": ds_name, "status": "OK",
            "file_count": len(files), "total_bytes": total_bytes,
            "extensions": dict(sorted(ext_counts.items())),
            "folder_file_counts": dict(sorted(folder_counts.items())),
            "has_dataCND": any(p.is_dir() and p.name.lower() == "datacnd"
                               for p in ds.rglob("*")),
        })

        for p in files:
            row = {"dataset": ds_name, "relative_path": safe_rel(p, ds),
                   "extension": p.suffix.lower(), "size_bytes": p.stat().st_size}
            all_inventory.append(row)
            if p.suffix.lower() == ".mat":
                rec = inspect_mat(p, args.max_load_mb)
                rec["dataset"] = ds_name
                rec["relative_path"] = safe_rel(p, ds)
                rec["likely_role"] = (
                    "stimulus metadata/envelope" if "stim" in p.name.lower()
                    else "subject EEG or subject metadata" if "sub" in p.name.lower()
                    else "other MATLAB data"
                )
                mat_records.append(rec)
            elif p.suffix.lower() == ".m":
                rec = inspect_m_file(p)
                rec["dataset"] = ds_name
                rec["relative_path"] = safe_rel(p, ds)
                m_records.append(rec)

    report = {
        "root": str(root), "output_dir": str(out),
        "dataset_summary": dataset_summary,
        "mat_files": mat_records, "matlab_scripts": m_records,
        "notes": [
            "This is a read-only audit; no source files were modified.",
            "A .mat file named dataSub/pre_dataSub is not assumed to be raw EEG until its variables and dimensions confirm that.",
            "MATLAB v7.3 files are inspected as HDF5; reference/cell decoding may require MATLAB or h5py-specific follow-up.",
            "Sampling rates, units, channel order, event timing, and preprocessing history must be verified from metadata and scripts, not inferred from filenames."
        ]
    }

    (out / "audit_report.json").write_text(
        json.dumps(report, indent=2, ensure_ascii=False, default=jsonable),
        encoding="utf-8"
    )

    with (out / "file_inventory.csv").open("w", newline="", encoding="utf-8") as f:
        w = csv.DictWriter(f, fieldnames=["dataset", "relative_path", "extension", "size_bytes"])
        w.writeheader(); w.writerows(all_inventory)

    with (out / "matlab_scripts_index.json").open("w", encoding="utf-8") as f:
        json.dump(m_records, f, indent=2, ensure_ascii=False)

    with (out / "mat_files_index.json").open("w", encoding="utf-8") as f:
        json.dump(mat_records, f, indent=2, ensure_ascii=False, default=jsonable)

    # Human-readable summary
    lines = ["# EEG dataset structural audit", "", f"- Root: `{root}`",
             f"- Datasets requested: {', '.join(args.datasets)}", "",
             "## Dataset overview", "",
             "| Dataset | Status | Files | Size (GB) | Extensions |",
             "|---|---:|---:|---:|---|"]
    for d in dataset_summary:
        lines.append(
            f"| {d['dataset']} | {d['status']} | {d.get('file_count','-')} | "
            f"{d.get('total_bytes',0)/1e9:.3f} | "
            f"{', '.join(f'{k}: {v}' for k,v in d.get('extensions',{}).items())} |"
        )
    lines += ["", "## MATLAB files", ""]
    for m in mat_records:
        lines += [f"### {m['dataset']} / `{m['relative_path']}`",
                  f"- Format: {m['format']}; size: {m['size_bytes']/1e6:.2f} MB",
                  f"- Variables: {len(m['variables'])}"]
        for v in m["variables"]:
            lines.append(f"  - `{v['name']}`: shape={v['shape']}, class={v.get('matlab_class', v.get('dtype','?'))}")
        for warning in m["warnings"]:
            lines.append(f"- Warning: {warning}")
        lines.append("")
    lines += ["## MATLAB scripts", ""]
    for m in m_records:
        lines.append(f"### {m['dataset']} / `{m['relative_path']}`")
        for fn in m["functions"]:
            lines.append(f"- Function: line {fn['line']}: `{fn['text']}`")
        for ln in m["interesting_lines"][:80]:
            lines.append(f"- L{ln['line']}: `{ln['text']}`")
        lines.append("")
    lines += ["## Interpretation cautions", ""]
    lines += [f"- {n}" for n in report["notes"]]
    (out / "README_AUDIT.md").write_text("\n".join(lines), encoding="utf-8")

    print(f"Audit complete. Output: {out}")
    print(f"Datasets found: {sum(d['status']=='OK' for d in dataset_summary)}/{len(args.datasets)}")
    print(f"Files indexed: {len(all_inventory)} | MAT: {len(mat_records)} | M scripts: {len(m_records)}")
    print("Read README_AUDIT.md first, then audit_report.json for detailed nested summaries.")

if __name__ == "__main__":
    main()
