"""Build EEG-sAAD_onboarding_guide.pdf: one document for people joining the challenge team.

It combines (1) the corpus summary in reports/eda_raw, (2) the example-trial EDA 'EEG-sAAD Data Story.html'
(one trial per dataset; source of hypotheses H1-H12) and (3) the full-corpus EDA in reports/eda_full (verdicts).
Tables are read from reports/eda_full/tables at build time; run make_figures.py first.
"""
import os
import re
from xml.sax.saxutils import escape

import numpy as np
import pandas as pd
from PIL import Image as PImage
from reportlab.lib import colors
from reportlab.lib.enums import TA_JUSTIFY
from reportlab.lib.pagesizes import A4
from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet
from reportlab.lib.units import cm
from reportlab.pdfbase import pdfmetrics
from reportlab.pdfbase.ttfonts import TTFont
from reportlab.platypus import (Image, KeepTogether, ListFlowable, ListItem, PageBreak, Paragraph, SimpleDocTemplate,
                                Spacer, Table, TableStyle)

HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
FULL = os.path.join(ROOT, "reports", "eda_full")
TAB = os.path.join(FULL, "tables")
LAB = {"AVGC": "KUL-AV-GC", "DTU": "DTU", "KUL": "KUL2016", "UHD": "KUL-UHD", "USTC": "USTC"}
DSK = list(LAB)

FD = "/usr/share/fonts/truetype/dejavu/"
pdfmetrics.registerFont(TTFont("DV", FD + "DejaVuSans.ttf"))
pdfmetrics.registerFont(TTFont("DVB", FD + "DejaVuSans-Bold.ttf"))
pdfmetrics.registerFont(TTFont("DVI", FD + "DejaVuSans-Oblique.ttf"))
pdfmetrics.registerFontFamily("DV", normal="DV", bold="DVB", italic="DVI", boldItalic="DVB")
ss = getSampleStyleSheet()
NAVY = colors.HexColor("#1f3a5f")
S = dict(
    body=ParagraphStyle("b", parent=ss["Normal"], fontName="DV", fontSize=9.2, leading=12.6, alignment=TA_JUSTIFY, spaceAfter=5),
    title=ParagraphStyle("t", parent=ss["Title"], fontName="DVB", fontSize=22, leading=27, textColor=NAVY, spaceAfter=10),
    sub=ParagraphStyle("s", parent=ss["Normal"], fontName="DV", fontSize=11.5, leading=15, textColor=colors.HexColor("#333333"), spaceAfter=14),
    part=ParagraphStyle("p", parent=ss["Heading1"], fontName="DVB", fontSize=16, leading=20, textColor=NAVY, spaceBefore=4, spaceAfter=8),
    h2=ParagraphStyle("h2", parent=ss["Heading2"], fontName="DVB", fontSize=12, leading=15.5, textColor=NAVY, spaceBefore=9, spaceAfter=4,
                      keepWithNext=1),
    cap=ParagraphStyle("c", parent=ss["Normal"], fontName="DVI", fontSize=7.8, leading=10, textColor=colors.HexColor("#444444"), spaceAfter=8),
    cell=ParagraphStyle("cell", parent=ss["Normal"], fontName="DV", fontSize=7.0, leading=8.6),
    li=ParagraphStyle("li", parent=ss["Normal"], fontName="DV", fontSize=9.2, leading=12.4, alignment=TA_JUSTIFY, spaceAfter=2.5),
    box=ParagraphStyle("box", parent=ss["Normal"], fontName="DV", fontSize=8.4, leading=11, spaceAfter=0),
)
S["head"] = ParagraphStyle("head", parent=S["cell"], fontName="DVB")
PW = A4[0] - 3.4 * cm
story = []
PENDING = []          # a heading waiting to be glued to the next block (avoids headings orphaned at page bottoms)


def add(*flowables):
    """Append flowables; a pending heading is kept together with the first of them."""
    global PENDING
    fl = list(flowables)
    if PENDING and fl:
        first = fl.pop(0)
        inner = list(first._content) if isinstance(first, KeepTogether) else [first]
        story.append(KeepTogether(PENDING + inner))
        PENDING = []
    story.extend(fl)


def rich(t):
    """Escape, then **bold** -> <b>, *italic* -> <i>."""
    t = escape(t)
    t = re.sub(r"\*\*(.+?)\*\*", r"<b>\1</b>", t)
    t = re.sub(r"(?<!\*)\*(?!\s)(.+?)(?<!\s)\*(?!\*)", r"<i>\1</i>", t)
    return t


def P(t, st="body"):
    add(Paragraph(rich(t), S[st]))


def H(t):
    global PENDING
    PENDING = [Paragraph(escape(t), S["h2"])]


def PART(t):
    story.append(PageBreak())
    story.append(Paragraph(escape(t), S["part"]))


def UL(items):
    add(ListFlowable([ListItem(Paragraph(rich(x), S["li"]), leftIndent=10, value="•") for x in items],
                              bulletType="bullet", start="•", leftIndent=12, bulletFontName="DV", bulletFontSize=8))
    story.append(Spacer(1, 4))


def BOX(title, text, bg="#f4f7fb", edge="#1f3a5f"):
    b = Table([[Paragraph(f"<b>{escape(title)}</b>", S["box"])], [Paragraph(rich(text), S["box"])]], colWidths=[PW])
    b.setStyle(TableStyle([("BACKGROUND", (0, 0), (-1, -1), colors.HexColor(bg)),
                           ("LINEBEFORE", (0, 0), (0, -1), 2.5, colors.HexColor(edge)),
                           ("LEFTPADDING", (0, 0), (-1, -1), 8), ("RIGHTPADDING", (0, 0), (-1, -1), 6),
                           ("TOPPADDING", (0, 0), (-1, -1), 2), ("BOTTOMPADDING", (0, 0), (-1, -1), 3)]))
    add(KeepTogether([b, Spacer(1, 7)]))


def HOW(text):
    BOX("How to read this", text, bg="#fff8e6", edge="#e0c27a")


def TABLE(rows, header, widths, caption=None, keep=True):
    data = [[Paragraph(rich(str(h)), S["head"]) for h in header]] + [[Paragraph(rich(str(c)), S["cell"]) for c in r] for r in rows]
    tb = Table(data, colWidths=[w * PW for w in widths], repeatRows=1)
    tb.setStyle(TableStyle([("BACKGROUND", (0, 0), (-1, 0), colors.HexColor("#e8eef5")),
                            ("GRID", (0, 0), (-1, -1), 0.3, colors.HexColor("#b0b8c4")),
                            ("VALIGN", (0, 0), (-1, -1), "TOP"),
                            ("ROWBACKGROUNDS", (0, 1), (-1, -1), [colors.white, colors.HexColor("#f7f9fb")]),
                            ("LEFTPADDING", (0, 0), (-1, -1), 3), ("RIGHTPADDING", (0, 0), (-1, -1), 3),
                            ("TOPPADDING", (0, 0), (-1, -1), 2), ("BOTTOMPADDING", (0, 0), (-1, -1), 2)]))
    parts = ([Paragraph(rich(caption), S["cap"])] if caption else []) + [tb, Spacer(1, 7)]
    if keep:
        add(KeepTogether(parts))
    else:
        add(*parts)


def FIG(path, caption, max_h=10.5):
    w, h = PImage.open(path).size
    W = PW
    Hh = W * h / w
    if Hh > max_h * cm:
        Hh = max_h * cm
        W = Hh * w / h
    add(KeepTogether([Image(path, width=W, height=Hh), Paragraph(rich(caption), S["cap"])]))


def T(name):
    return pd.read_csv(os.path.join(TAB, name))


def pc(x, nd=1):
    return f"{100 * x:.{nd}f}%"


def ci(m, lo, hi, nd=1, scale=100):
    return f"{m * scale:.{nd}f} [{lo * scale:.{nd}f}, {hi * scale:.{nd}f}]"


FULLF = lambda n: os.path.join(FULL, "figures", n)      # noqa: E731
OWN = lambda n: os.path.join(HERE, "figures", n)        # noqa: E731

# =====================================================================================================================
# Title page
# =====================================================================================================================
story.append(Spacer(1, 2.2 * cm))
story.append(Paragraph("EEG-sAAD Grand Challenge<br/>Onboarding guide to the training data", S["title"]))
story.append(Paragraph("What is in the five training datasets, what our two exploratory analyses found, which traps to "
                       "avoid, and how to start building a decoder.", S["sub"]))
BOX("Who this is for",
    "New team members joining our ICASSP 2027 EEG-sAAD (selective auditory attention decoding) challenge work. "
    "You need basic machine-learning and signal-processing background; EEG-specific terms are explained in the glossary "
    "(Appendix A).")
BOX("What this document combines",
    "**1. Corpus summary** (reports/eda_raw): inventory and integrity check of all 1,979 trials.  "
    "**2. Example-trial EDA** ('EEG-sAAD Data Story'): a deep look at ONE trial per dataset. It produced twelve hypotheses "
    "(H1-H12) and a method roadmap.  "
    "**3. Full-corpus EDA** (reports/eda_full): every trial of every subject, used to confirm or refute H1-H12 with "
    "subject-level statistics.  "
    "Where the two EDAs disagree, **the full-corpus result wins**; example-trial numbers appear only as 'first estimate'.")
BOX("How to use it",
    "Read **Part 1 (Start here)** first: it takes ten minutes and lists what everyone on the team must know. Parts 2-6 are "
    "reference material. Part 7 tells you where the files are and how to run things. Every number comes from code that was "
    "run on the data; the source tables are named so you can check them.")
story.append(Spacer(1, 0.8 * cm))
story.append(Paragraph(escape("Built 2026-09-25 by reports/onboarding/build_guide.py from reports/eda_full/tables, "
                              "reports/eda_raw and 'EEG-sAAD Data Story.html'."), S["cap"]))

# =====================================================================================================================
# Contents
# =====================================================================================================================
story.append(PageBreak())
story.append(Paragraph("Contents", S["part"]))
TABLE([["1", "Start here: twelve things to know"], ["2", "The challenge in two pages: task, rules, metric, baseline"],
       ["3", "The data: datasets, files, labels and conventions"],
       ["4", "How we built our understanding: from example-trial hypotheses to full-corpus verdicts"],
       ["5", "What the data show: dialects, labels, the attention signal, shortcuts, switch detection"],
       ["6", "What to do about it: recommended pipeline, validation checklist, roadmap, open questions"],
       ["7", "Getting started: environment, directory map, commands"],
       ["A", "Glossary"], ["B", "References (literature cited by the example-trial EDA)"]],
      ["Part", "Topic"], [0.08, 0.92])

# =====================================================================================================================
# Part 1
# =====================================================================================================================
PART("1. Start here: twelve things to know")
UL([
    "**The task.** Two people talk at once; the listener attends to one. From the listener's EEG (plus the two speech "
    "envelopes) we must output, for every 128 Hz sample, which talker (or which side) is attended. Attention may switch "
    "within a trial. The test subjects and recording labs are unseen.",
    "**The hard rule.** A label at time t may use data only up to t + 1 s, counting every filter, lag, window and smoother. "
    "Zero-phase filtering (filtfilt) breaks this rule. Subject, trial and dataset IDs must never be model inputs.",
    "**The score.** H = harmonic mean of sample accuracy (Acc) and a switch-latency score L = median of exp(-SDT/60). "
    "The published baseline scores Acc 68.4%, SDT 18.53 s, H 0.7002 on the KUL-AV-GC hold-out.",
    "**The data.** Five datasets: 95 subjects, 1,979 trials, 77 h of EEG at 128 Hz (Part 3). They come from different labs, "
    "amplifiers, languages and montages; only **57 channel names** are common to all five.",
    "**'Raw' means five different things.** The amplitude scale differs 16.5x between datasets; DTU, KUL-UHD and USTC "
    "carry DC offsets of thousands of units; every KUL-UHD and USTC trial starts and ends with a large jump. Normalise "
    "causally before pooling (Part 5.1).",
    "**There is no speaker switch in the training data.** The only genuine switches are the 99 KUL-AV-GC side switches at "
    "exactly t = 300 s, where the SAME talker moves to the other side. Speaker-switch detection can only be trained and "
    "tested with synthetic switches (Part 5.2).",
    "**Never use the trial-level label for direction.** It is the side at trial onset, so it is wrong for 50% of KUL-AV-GC "
    "samples. Use the per-sample CND labels (attendedStream, attendedAzimuth).",
    "**The attention signal is a whisper.** Across datasets, 1-s decisions are right about 53-56% of the time. Accuracy "
    "grows predictably with window length: accuracy is about Phi(0.9 x SNR1 x sqrt(w)), where SNR1 is roughly 0.07-0.13 "
    "across datasets and 0.14-0.20 within a subject (Part 5.3).",
    "**Shortcuts are everywhere.** Labels follow trial parity (KUL-UHD, USTC); the attended envelope is louder in KUL2016 "
    "(a loudness-only classifier gets 82.5%); covariances identify the dataset at 99.7%; spatial decoders memorise trials. "
    "Validate so that none of these can help (Part 5.4).",
    "**Spatial (direction-only) decoders do not transfer.** Across datasets they are at chance (47-54%); on the 99 genuine "
    "KUL-AV-GC switches the best one reaches 51.7% sample accuracy.",
    "**The scorer has quirks.** A PERFECT decoder is charged a missed switch when the next switch reverts within 50 s, and "
    "a decoder with zero information still earns L of about 0.45-0.73. Always report L next to its null (Part 5.5).",
    "**Causal preprocessing costs nothing.** Causal filtering is never worse than zero-phase filtering or the organisers' "
    "preprocessed 32 Hz data. Start from the pipeline in Part 6.",
])

# =====================================================================================================================
# Part 2
# =====================================================================================================================
PART("2. The challenge in two pages")
H("2.1 Task and test sets")
P("Selective auditory attention decoding (sAAD): a listener hears two simultaneous talkers, one on each side, and attends "
  "to one of them. The EEG follows the slow loudness contour (envelope) of the attended talker slightly more closely than "
  "that of the ignored one. A decoder exploits this difference, or other spatial cues, to say who (or which side) is "
  "attended, and must keep doing so when attention switches.")
P("The blind test sets are **KUL-DLAAD** (BioSemi, Dutch) and **NJU-SUSTech-sAAD-GC** (NeuroScan, Mandarin). Both have "
  "unseen subjects, attention switches at random times within trials, and eye-gaze control, so the eyes cannot give the "
  "attended side away. The example-trial EDA quotes the challenge page: KUL-DLAAD has 19 subjects with 8 trials and about "
  "50 min of EEG each; NJU-SUSTech has 12 subjects with about 60 min each. The switch rate and cueing are not published.")
H("2.2 Rules that shape every design decision")
TABLE([["Lookahead", "A label at time t may use data only up to t + 1 s. This budget covers every filter, decoder lag, "
        "window and smoother (Part 6 shows how our pipeline spends it)."],
       ["Output", "One label per 128 Hz sample: attended stream (0 = first, 1 = second) or attended side (0 = left, 1 = right)."],
       ["Identity", "Dataset, subject and trial IDs must never be predictive inputs; no statistics pooled across a subject's "
        "test trials."],
       ["Adaptation", "Test-time updates are allowed only within a trial and only causally."]],
      ["Rule", "What it means"], [0.15, 0.85], "Table 2.1. Challenge rules (from the challenge prompt and the organisers' "
      "baseline README).")
H("2.3 The metric")
P("For each ground-truth switch at t0 the scorer searches [t0, t0 + 100 s] for the split point tau* that best explains the "
  "predictions as 'old label, then new label'. The detection counts only if the 0.5 s after tau* all carry the new label; "
  "otherwise the search moves on, and a switch never confirmed scores 0. SDT = tau* - t0 and the per-switch score is "
  "exp(-SDT/60). L is the median over a subject's switches, averaged over subjects; Acc is the fraction of correct "
  "samples, averaged over subjects; **H = 2 Acc L / (Acc + L)**.")
TABLE([["5 s", "0.92"], ["10 s", "0.85"], ["18.5 s (baseline)", "0.73"], ["60 s", "0.37"], ["missed", "0"]],
      ["SDT", "exp(-SDT/60)"], [0.3, 0.2], "Table 2.2. The latency score is lenient: even 18.5 s keeps 73% of the credit.")
H("2.4 The official baseline")
P("baselines/matlab_linear trains one backward (stimulus-reconstruction) decoder on the pooled training datasets, decodes "
  "1-s windows on a held-out dataset and smooths the decisions with a causal hidden Markov model (HMM). On the KUL-AV-GC "
  "hold-out it reaches 53.94% at 1 s and 73.20% at 50 s without smoothing, and **Acc 68.4%, SDT 18.53 s, H 0.7002** with "
  "the HMM. The baseline also contains generateSyntheticSwitches.m, which imposes switches on fixed-attention trials by "
  "swapping the two streams' roles. The official scorer is baselines/matlab_scoring. MATLAB is not installed here, so "
  "we use a verified Python port (reports/eda_full/scripts/scorer.py; 34/34 hand-made tests plus 3,000 random cases).")

# =====================================================================================================================
# Part 3
# =====================================================================================================================
PART("3. The data")
H("3.1 The five training datasets")
inv = T("corpus_inventory.csv").set_index("ds")
dev = {"AVGC": ("BioSemi ActiveTwo", "Dutch", "±90°", "audiovisual, gaze-controlled; side switch at 300 s"),
       "DTU": ("BioSemi", "Danish", "±60°", "reverberation conditions"),
       "KUL": ("BioSemi", "Dutch", "±90°", "dry vs HRTF rendering"),
       "UHD": ("SynAmps RT (per CND file)*", "Dutch", "±90°", "ultra-high-density montage"),
       "USTC": ("NeuroScan SynAmps", "Mandarin", "±90°", "")}
rows = []
for d in DSK:
    r = inv.loc[d]
    rows.append([LAB[d], int(r.subjects), int(r.trials), f"{r.hours:.1f}", f"{r.dur_med:.0f}", int(r.channels), dev[d][0],
                 dev[d][1], dev[d][2], dev[d][3]])
TABLE(rows, ["dataset", "subj.", "trials", "hours", "trial (s)", "ch.", "amplifier", "lang.", "azimuth", "notes"],
      [0.1, 0.06, 0.065, 0.065, 0.07, 0.05, 0.16, 0.085, 0.09, 0.255],
      "Table 3.1. Training corpus (counts from reports/eda_full/tables/corpus_inventory.csv). *The challenge page lists "
      "KUL-UHD as BioSemi/255 channels; the CND files say SynAmps RT; 253 channels are stored after dropping two extra "
      "electrodes.")
FIG(os.path.join(ROOT, "reports", "eda_raw", "06_example_raw_traces.png"),
    "Figure 3.1 (from the corpus summary, reports/eda_raw). The first 10 s of subject 1, trial 1 of each dataset: three "
    "EEG channels and the two envelopes, each standardised for display. Note the eye blinks (KUL-AV-GC), the slow drift "
    "(DTU) and the huge jump at t = 0 in KUL-UHD and USTC.", max_h=13)
H("3.2 Where the data live and what the files contain")
TABLE([["archives/datasets/", "original ZIP archives + SHA256.txt (never modify)"],
       ["data/cnd/<DATASET>/dataCND/dataSub<N>.mat", "EEG ('neural'): per trial EEG [time x channels], chanlocs, fs, and "
        "PER-SAMPLE ground truth: attendedStream (1-based stream index) and attendedAzimuth (degrees). Some datasets add "
        "extChan (EOG), origTrialPosition, deviceName."],
       ["data/cnd/<DATASET>/dataCND/dataStim<N>.mat", "'stim': envelope_stream1/2 per trial, streamWav (story file names), "
        "condIdxs/condNames. KUL-UHD uses one shared dataStim.mat."],
       ["data/cnd/.../pre_*.mat", "the organisers' preprocessed variant: zero-phase 1-9 Hz band-pass, 32 Hz, plus bad-channel "
        "or bad-segment repair"],
       ["data/superhuge/raw/", "the same data converted for the Python DNN baselines: eeg/dataset-DDD-subject-SSS-trial-TTT.npy "
        "(time x channels, 128 Hz), stimuli/env/..._env.npy (time x 1 x 2 streams), meta/metadata.pkl"],
       ["data/superhuge/preprocessed/", "same layout for the pre_* (32 Hz) files"],
       ["baselines/", "matlab_linear (decoder + HMM), matlab_scoring (official scorer), python/SuperHugeAAD (DNNs)"],
       ["tools/convert_cnd_to_superhuge.py", "the CND -> SuperHuge converter; its conventions are listed below"]],
      ["path", "contents"], [0.34, 0.66], "Table 3.2. Directory map of the workspace data.")
H("3.3 Conventions that trip people up")
UL([
    "**SuperHuge column 0 = the stream attended for most of the trial.** This is a training convention; it will not hold at "
    "test time, and it silently leaks the answer if a model can see column order. Columns were checked against the CND "
    "files for all 1,979 trials (EEG identical, envelope r = 1.000).",
    "**The metadata 'label' is trial level:** the attended side at trial onset ('left'/'right' or 'L'/'R'). It is not a "
    "per-sample ground truth. Per-sample labels exist only in the CND files.",
    "**Dataset IDs differ between systems:** the converter numbers datasets 1 (KUL-AV-GC), 3 (DTU), 4 (KUL2016), 5 (KUL-UHD) "
    "and 7 (USTC). Subjects are renumbered 1..N in file order, except USTC; KUL-UHD files start at dataSub0.",
    "**Channels must be matched by upper-cased name, never by index.** The BioSemi sets spell 'Afz'; USTC uses upper case "
    "(FP1, FPZ) and adds M1, M2, CB1, CB2, PO5, PO6; KUL-UHD uses 10-5 names. The common montages are 57 channels (all "
    "five datasets), 58 (BioSemi-64 and USTC, identical to the baseline's list) and 62 (BioSemi-64 and KUL-UHD).",
    "**Story names are unreliable.** In KUL-AV-GC, 29 of 30 distinct audios carry more than one streamWav name. All KUL-UHD "
    "audio also occurs in KUL2016. Group folds by envelope identity (tables/stim_identity_clusters.csv), not by file name.",
])

# =====================================================================================================================
# Part 4
# =====================================================================================================================
PART("4. How we built our understanding")
P("Three analyses were run, each on a larger evidence base than the last. The example-trial EDA was fast and "
  "suggestive; the full-corpus EDA was designed to test its claims at scale, with statistics over subjects.")
TABLE([["Corpus summary", "reports/eda_raw", "all 1,979 trials, summary statistics", "inventory, integrity (0 missing files, "
        "0 non-finite values), durations, label counts, channel statistics"],
       ["Example-trial EDA", "EEG-sAAD Data Story.html", "one trial per dataset (subject 1, trial 1) + corpus statistics",
        "12 hypotheses (H1-H12), a literature-backed method roadmap, a re-implemented scorer, synthetic-switch simulations"],
       ["Full-corpus EDA", "reports/eda_full", "every trial and subject; subject-level medians with 95% bootstrap CIs, "
        "Wilcoxon tests, FDR", "verdicts on H1-H12, LOTO/LOSO/LODO decoders, SNR law, shortcut audits, official-scorer port, "
        "integrator comparison, lookahead unit test"]],
      ["analysis", "where", "evidence base", "main output"], [0.15, 0.2, 0.28, 0.37],
      "Table 4.1. The three analyses behind this guide.")
H("4.1 The twelve hypotheses and their verdicts")
vd = T("hypotheses_verdicts.csv").set_index("id")
short = {
    "H1": ("Amplitude scales differ ~16x; DTU, KUL-UHD, USTC carry large DC offsets",
           "Yes: 16.5x; offsets of thousands of units in DTU, KUL-UHD, USTC."),
    "H2": ("Different pre-filtering; strong 50 Hz in KUL-UHD and USTC",
           "KUL-AV-GC is band-limited 1-41 Hz and notched; the 50 Hz line in KUL-UHD/USTC is real but 3-4x weaker than "
           "the example suggested."),
    "H3": ("Edge transients in USTC and KUL-UHD; bad channels exist",
           "Yes: 100% of USTC and KUL-UHD trials vs 3-5% elsewhere; 5.1% of KUL-UHD channels bad."),
    "H4": ("Accuracy = Phi(SNR1 sqrt(w)); SNR1 ~0.16 within subject, ~0.10 for the baseline",
           "Yes: k = 0.74-0.98; KUL-AV-GC SNR1 0.185 within subject, 0.107 across datasets (baseline implies 0.099)."),
    "H5": ("Causal filtering is no worse than zero-phase",
           "Yes: never worse; zero-phase is slightly worse where significant."),
    "H6": ("The KUL-AV-GC 5-min switch changes side, not talker",
           "Yes: 0 speaker switches, 99 side switches (all at 300 s); cross-half test 84% vs 20% at 60 s."),
    "H7": ("The attended envelope is louder in KUL2016 and KUL-AV-GC",
           "KUL2016 yes (65% of trials; loudness-only classifier 82.5%); KUL-AV-GC only weakly."),
    "H8": ("Label follows trial parity in KUL-UHD and USTC", "Yes: 100% of trials, every subject."),
    "H9": ("USTC stream columns may be swapped", "No: the held-out decoder prefers column 0 in 75% of USTC trials."),
    "H10": ("Covariance fingerprints the dataset; alignment removes only part",
            "Yes: 99.7% dataset ID; still 84% after EA and 87% after causal running EA (chance 31%)."),
    "H11": ("Spatial decoders exploit trial fingerprints", "Yes: trial ID 63-89%; -22 pp when trials are held out; chance across datasets."),
    "H12": ("No alpha or EOG marker at the KUL-AV-GC switch",
            "Yes (all p >= 0.08), but KUL-AV-GC is high-passed ~1 Hz, so weak evidence of absence."),
}
rows = [[h, short[h][0], vd.loc[h, "previous"], short[h][1], vd.loc[h, "verdict"]] for h in short]
TABLE(rows, ["ID", "claim from the example trial", "first estimate (n = 1 trial)", "full corpus", "verdict"],
      [0.05, 0.25, 0.22, 0.34, 0.14], "Table 4.2. Hypotheses H1-H12: example-trial estimate vs full-corpus result "
      "(details and CIs: reports/eda_full/tables/hypotheses_verdicts.csv).", keep=False)
H("4.2 What changed when we went from one trial to all trials")
UL([
    "**Confirmed and strengthened:** the direction-only KUL-AV-GC switch (now in all 99 trials, from the CND labels "
    "themselves), the parity lock, the dataset fingerprint, the SNR law and the harmlessness of causal filtering.",
    "**Refuted:** the suspected USTC column swap. USTC is simply the hardest dataset (weakest SNR1, edge transients, a "
    "different envelope extraction), not a mislabelled one.",
    "**Weaker than first thought:** the 50 Hz mains line (3-4x smaller), the 'louder attended column' in KUL-AV-GC (not "
    "usable by a classifier), and the gain from integrator tuning. On synthetic switches with within-subject evidence, "
    "the example trial gave H of about 0.80 for a 5-10 s moving average. With honest cross-dataset evidence the best "
    "integrator reaches H 0.63-0.75, only 0.02-0.07 above plain argmax.",
    "**Worse than first thought:** Euclidean alignment removes less of the fingerprint (84-87% remain vs 76% in the example), "
    "and the chance floor of L depends on the dataset (0.45-0.73 rather than 0.49-0.57), because it depends on trial "
    "length and switch spacing.",
    "**New:** the scorer's 50-s property (a perfect decoder can miss a switch), unreliable story names, the fact that all "
    "KUL-UHD audio is reused from KUL2016, and that pooling strategy (pooled covariance vs averaged decoders) makes no "
    "difference across datasets.",
])
FIG(OWN("ds_fig4.png"), "Figure 4.1 (example-trial EDA, n = 1 KUL-AV-GC trial). The SNR law as first seen: window "
    "accuracy tracks Phi(0.16 sqrt(w)). Compare with Figure 5.5, the same law confirmed on all 95 subjects. "
    "(Colours of example-trial figures follow that document, not the dataset colours used elsewhere.)", max_h=7.5)

# =====================================================================================================================
# Part 5
# =====================================================================================================================
PART("5. What the data show")
P("Dataset colours used in all full-corpus figures: **KUL-AV-GC red, DTU blue, KUL2016 green, KUL-UHD purple, USTC "
  "orange**; grey marks a baseline or null. Accuracies are medians over subjects with 95% bootstrap CIs.")
H("5.1 Five labs, five dialects of 'raw'")
qc = T("qc_dataset_summary.csv").set_index("ds")
inf = T("prefilter_inference.csv").set_index("ds")
rows = []
for d in DSK:
    r = qc.loc[d]
    dc = f"{r.median_abs_dc:,.0f}" if r.median_abs_dc >= 10 else f"{r.median_abs_dc:.3f}"
    rows.append([LAB[d], f"{r.median_sd:.3g}", f"{r.median_rms_sd:.3g}", dc,
                 inf.loc[d, "inferred_highpass"], inf.loc[d, "inferred_lowpass"].replace(" (only the resampling anti-alias at the 64 Hz Nyquist)", ""),
                 f"{r.line_ratio_50:.2g}", pc(r.share_edge_transient_trials, 0), pc(r.share_bad_channels, 1)])
TABLE(rows, ["dataset", "median ch. SD", "RMS / SD", "|DC|", "high-pass", "low-pass", "50 Hz ratio",
             "edge-jump trials", "bad ch."], [0.1, 0.08, 0.065, 0.07, 0.2, 0.19, 0.075, 0.12, 0.1],
      "Table 5.1. Recording 'dialects' (subject-level medians over 1,979 trials; stored units). RMS/SD = 1 means no DC "
      "offset; 50 Hz ratio > 1 means mains noise, < 1 a notch.")
FIG(FULLF("a2_psd_median_iqr.png"), "Figure 5.1. Median-channel power spectra (median over trials, IQR band). KUL-AV-GC "
    "(red) was band-limited to about 1-41 Hz before release; KUL-UHD and USTC show a 50 Hz line; DTU, KUL-UHD and USTC keep "
    "large power below 1 Hz because they were never high-passed.", max_h=8.5)
HOW("x = frequency, y = power, both logarithmic. Curves that differ in shape mean the datasets were processed differently "
    "before release, which a model can use to tell datasets apart.")
P("**Consequence:** the front end must be causal and per-trial: match channels by name, high-pass with the filter state "
  "initialised to the first sample (otherwise the DC offsets ring for seconds), re-reference, band-pass, and scale with "
  "a running robust z-score. Trim or warm up the first second, because every KUL-UHD and USTC trial starts with a jump. "
  "None of the datasets is stored average-referenced; common-average referencing measurably helps (Part 5.3).")

H("5.2 Labels: there is no speaker switch")
lab = T("label_audit_summary.csv").set_index("ds")
rows = [[LAB[d], int(lab.loc[d, "trials"]), int(lab.loc[d, "speaker_switches"]), int(lab.loc[d, "direction_switches"]),
         pc(lab.loc[d, "frac_direction_samples_label_wrong"], 0)] for d in DSK]
TABLE(rows, ["dataset", "trials", "within-trial speaker switches", "within-trial side switches",
             "side samples mislabelled by the trial label"], [0.16, 0.1, 0.25, 0.22, 0.27],
      "Table 5.2. Ground truth from the per-sample CND labels (reports/eda_full/tables/label_audit_summary.csv).")
FIG(OWN("g1_avgc_label_timeline.png"), "Figure 5.2. What happens in a KUL-AV-GC trial: the attended talker never changes; "
    "at t = 300 s the two talkers swap sides, so the attended SIDE flips. The trial-level label is wrong for the second half.",
    max_h=8)
xh = T("crosshalf_h6.csv")
rows = []
for w in (1, 10, 60):
    a = xh[(xh.w == w) & xh.hypothesis.str.startswith("hypA")].iloc[0]
    b = xh[(xh.w == w) & xh.hypothesis.str.startswith("hypB")].iloc[0]
    rows.append([f"{w} s", ci(a.median_acc, a.ci_lo, a.ci_hi), ci(b.median_acc, b.ci_lo, b.ci_hi)])
TABLE(rows, ["window", "if the talker switched at 5 min", "if the same talker is attended all trial"], [0.2, 0.4, 0.4],
      "Table 5.3. Cross-half decoding test on all 99 KUL-AV-GC trials (13 subjects): a decoder trained on one half and "
      "tested on the other. Accuracy (%) expected under each explanation.")
FIG(OWN("ds_fig5.png"), "Figure 5.3 (example-trial EDA, n = 1 trial). The same test on one trial first raised the "
    "hypothesis; Table 5.3 confirms it on all trials (the full-corpus 60-s values are 84% vs 20%, less extreme than 89% vs 0%).",
    max_h=6)
BOX("What this means for you",
    "For a stimulus-reconstruction (speaker) decoder there is no genuine switch anywhere in the training data, so speaker-"
    "switch latency can only be measured with SYNTHETIC switches. For a direction decoder, take labels from the per-sample "
    "azimuth. The KUL-AV-GC side switches become 'free' for a speaker decoder only if the test data tell us where each "
    "stream is; ask the organisers (Part 6.4).")

H("5.3 The attention signal: small, consistent, predictable")
FIG(FULLF("c1_tracking_curves.png"), "Figure 5.4. Envelope-EEG cross-correlation (RMS over 57 channels, median over "
    "subjects). Solid = attended, dashed = ignored talker. The attended talker is tracked more strongly in every dataset "
    "(peak 94-188 ms; Wilcoxon p <= 0.01), but correlations are only about 0.01-0.02.", max_h=5)
acc = T("decoder_accuracy_vs_window.csv")
acc = acc[acc.lag == "0-500ms"]
rows = []
for sc, name in (("loto", "within subject (LOTO)"), ("loso", "new subject (LOSO)"), ("lodo_pool", "new dataset (LODO)")):
    for w in (1, 10, 60):
        r = [name if w == 1 else "", f"{w} s"]
        for d in DSK:
            g = acc[(acc.scheme == sc) & (acc.w == w) & (acc.ds == d)]
            r.append(ci(g.median_acc.iloc[0], g.ci_lo.iloc[0], g.ci_hi.iloc[0]) if len(g) else "n/a")
        rows.append(r)
TABLE(rows, ["validation", "window"] + [LAB[d] for d in DSK], [0.17, 0.08] + [0.15] * 5,
      "Table 5.4a. Backward-decoder accuracy (%) by validation scheme and window (causal pipeline, 57 channels, lags "
      "0-500 ms). DTU trials are too short for 60-s windows.")
snr = T("decoder_snr_law.csv")
snr = snr[snr.lag == "0-500ms"]
rows = []
for sc, name in (("loto", "within subject"), ("loso", "new subject"), ("lodo_pool", "new dataset")):
    r = [name]
    for d in DSK:
        g = snr[(snr.scheme == sc) & (snr.ds == d)].iloc[0]
        r.append(f"{g.snr1_median:.3f} [{g.snr1_ci_lo:.3f}, {g.snr1_ci_hi:.3f}]")
    rows.append(r)
kr = snr.k
TABLE(rows, ["SNR1", *[LAB[d] for d in DSK]], [0.16] + [0.168] * 5,
      f"Table 5.4b. 1-s evidence SNR1 = mean(d)/SD(d), d = r(attended) - r(ignored). The official baseline's 53.94% at 1 s "
      f"implies 0.099. Fitted k in Phi(k SNR1 sqrt(w)): {kr.min():.2f}-{kr.max():.2f}.")
FIG(FULLF("c4_snr_law.png"), "Figure 5.5. The SNR law on all 95 subjects: accuracy predicted from each subject's 1-s SNR "
    "(x) vs measured accuracy (y), one dot per subject and window length.", max_h=5.5)
HOW("Dots on the diagonal mean one number (SNR1) predicts accuracy at every window length. k < 1 means consecutive "
    "seconds are correlated, so longer windows help slightly less than sqrt(w). The practical reading: a better decoder "
    "(higher SNR1) improves every window length at once.")
P("**Pipeline choices (LODO, 95 subjects, paired tests):** zero-phase filtering is never better (up to 1.3 pp worse); the "
  "organisers' preprocessed 32 Hz data are not better; dropping the common average reference costs up to 1.4 pp; 57, 58 or "
  "62 channels and 0-250 vs 0-500 ms lags make no reliable difference; a causal 0.5-30 Hz variant was +1.3 pp at 10 s "
  "(p = 0.04, uncorrected), which is worth a follow-up. How the training data are pooled across datasets (dataset-equal "
  "pooled covariance, sample-weighted pooling, or averaging per-subject decoders) changes accuracy by 0.3 pp at most.")

H("5.4 Shortcuts and traps")
fp = T("fingerprint_results.csv")
g = fp[(fp.task == "dataset") & fp.features.str.startswith("cov")].set_index("alignment")
TABLE([["Label = trial parity", "KUL-UHD, USTC", "100% of trials: odd trials one side, even the other", "Anything slow (drift, "
        "fatigue, electrode settling) or order-related predicts the label", "Fold by subject; never feed trial order or slow "
        "drift features"],
       ["Louder attended envelope", "KUL2016", "65% of trials; a loudness-only classifier picks the attended stream at 82.5% "
        "for unseen subjects", "A model can 'decode' attention without looking at the EEG", "z-score each stream per window; "
        "swap stream columns at random in training"],
       ["Column 0 = attended", "all (training files)", "converter convention", "Any access to column order leaks the answer",
        "shuffle column order; never rely on it"],
       ["Reused stimuli, bad names", "KUL2016/KUL-UHD, KUL-AV-GC", "all KUL-UHD audio occurs in KUL2016; KUL2016 has 8 audios; "
        "KUL-AV-GC names do not match audio", "Story leakage between folds", "group folds by envelope identity"],
       ["Dataset fingerprint", "all", f"covariance identifies the dataset at {pc(g.loc['none', 'acc'])} ({pc(g.loc['ea', 'acc'])} "
        f"after EA, {pc(g.loc['running_ea', 'acc'])} after causal running EA; chance 31%); raw amplitude alone 98.9%",
        "Models learn the lab instead of attention", "causal normalisation; LODO validation"],
       ["Trial fingerprint", "spatial decoders", "trial ID predicted at 63-89% from 5-s covariances", "Random-window splits "
        "inflate direction accuracy by a median 22 pp (up to 49 pp)", "never score windows from training trials"],
       ["Gaze / EOG", "KUL2016, USTC", "USTC's recorded HEO separates left/right trials (d = 0.43); test sets are gaze-controlled",
        "Eye cues will be absent or reversed at test time", "exclude EOG and frontal-proxy features from direction models"]],
      ["trap", "where", "evidence", "why it hurts", "defence"], [0.13, 0.13, 0.29, 0.23, 0.22],
      "Table 5.5. Shortcuts found in the training data (full-corpus numbers).", keep=False)
FIG(FULLF("d1_embedding.png"), "Figure 5.6. Each dot is a 5-s EEG segment placed by its spatial covariance; colour = "
    "dataset. Separate islands = a dataset fingerprint. Alignment mixes the colours only partly.", max_h=4.8)
sp = T("spatial_summary.csv")
sa = sp[(sp.ds == "AVGC") & (sp.window_s.astype(str) == "5") & (sp.model == "fbcsp")].set_index("scheme")
P(f"**Spatial decoders in numbers (KUL-AV-GC, FBCSP, 5-s windows):** within-trial split {pc(sa.loc['within_trial', 'median_acc'])}, "
  f"held-out trial {pc(sa.loc['LOTO', 'median_acc'])}, new subject {pc(sa.loc['LOSO', 'median_acc'])}, new dataset "
  f"{pc(sa.loc['LODO', 'median_acc'])}. In KUL-UHD the same decoder drops from 92.7% to 44.3% when whole trials are held out.")

H("5.5 Switch detection and the metric")
FIG(OWN("g2_scorer_quirk.png"), "Figure 5.7. The official scorer (Python port) on a decoder that is correct on every "
    "sample. When the next switch reverts within 50 s (top), 'old label until the end of the 100-s window' wins the split "
    "search, fails the 0.5-s sustain check, and the first switch is scored as missed. With 60 s spacing (bottom) it is "
    "detected instantly.", max_h=7.5)
P("Under the default synthetic schedule (gaps drawn from 30-90 s) this happens to 414 of 3,113 switches for a perfect "
  "decoder. It is invisible in the perfect decoder's L (still 1.00, because L is a median) but it costs real decoders. "
  "A second property: the 0.5-s sustain check can read the next trial's labels for a switch at the very end of a trial.")
sw = T("switch_results_c19-common57-car_lodo_pool_0-500ms.csv")
rows = []
for d in DSK:
    b = sw[(sw.test_ds == d) & (sw.integrator == "ma_10s") & (sw.switches == "syn")]
    real, null = b[~b.null].iloc[0], b[b.null].iloc[0]
    o = sw[(sw.test_ds == d) & (sw.integrator == "ORACLE")].iloc[0]
    ag = sw[(sw.test_ds == d) & (sw.integrator == "argmax") & (sw.switches == "syn") & (~sw.null)].iloc[0]
    rows.append([LAB[d], int(real.nSwitches), f"{real.Acc:.1f}", f"{real.SDT:.1f}", f"{real.L:.3f}", f"{real.H:.3f}",
                 f"{ag.H:.3f}", f"{null.L:.3f}", f"{int(o.nUndetected)}/{int(o.nSwitches)}"])
TABLE(rows, ["held-out dataset", "SYNTHETIC switches", "Acc %", "SDT s", "L", "H", "H argmax", "null L", "oracle misses"],
      [0.14, 0.12, 0.08, 0.08, 0.08, 0.08, 0.11, 0.1, 0.12],
      "Table 5.6. Switch detection on SYNTHETIC switches with cross-dataset (LODO) evidence; best integrator on the median "
      "held-out dataset = 10-s causal moving average. 'null L' = the same pipeline on sign-randomised evidence.")
FIG(FULLF("e1_tradeoff_c19-common57-car_lodo_pool_0-500ms.png"), "Figure 5.8. Accuracy vs latency score for all "
    "integrator settings (colour) and their nulls (grey), with iso-H lines. Everything sits in a narrow band well above "
    "the null and far below the perfect decoder.", max_h=4.5)
ss2 = T("spatial_switch_results.csv")
best = ss2[~ss2.null].sort_values("H", ascending=False).iloc[0]
P(f"**Genuine switches.** On the 99 real KUL-AV-GC side switches, direction evidence from spatial decoders trained on the "
  f"other four datasets gives Acc {best.Acc:.1f}%, SDT {best.SDT:.1f} s, L {best.L:.3f}, H {best.H:.3f} (best setting; "
  f"null L {ss2[ss2.null].L.median():.3f}). Direction is essentially not decodable across datasets with spatial features. "
  "Mapping a speaker decoder through known stream positions gives a high-looking H of about 0.91, but that number is "
  "invalid for our purposes: column 0 is the attended talker for the whole trial by construction, and it assumes stream "
  "positions are known at test time.")
P("**Lookahead unit test.** Perturbing every input after t + 1 s leaves all labels up to t unchanged (21/21 checks). "
  "Zero-phase filtering and a 2-s budget are both caught as violations; leaks shorter than about 0.5 s may escape the test.")

# =====================================================================================================================
# Part 6
# =====================================================================================================================
PART("6. What to do about it")
H("6.1 Recommended starting pipeline")
FIG(OWN("g3_front_end.png"), "Figure 6.1. The causal pipeline used for every full-corpus decoding result, and how it "
    "spends the 1-s lookahead budget exactly.", max_h=7.5)
H("6.2 Validation checklist")
UL([
    "Outer loop: **leave-one-dataset-out**, with KUL-AV-GC as the main proxy fold; hyperparameters (including the "
    "integrator) in an inner loop over the training datasets only. Choose settings on the median held-out dataset, not "
    "the mean.",
    "Aggregate per subject first, then across subjects; report medians with bootstrap CIs.",
    "Never score windows drawn from a trial that contributed training data. For classification with balanced labels, "
    "plain leave-one-trial-out is biased below 50%; use balanced leave-pair-out.",
    "Group folds by **envelope identity**, and keep KUL2016 and KUL-UHD on the same side of any split.",
    "Report Acc, the SDT distribution, the miss rate, L and H separately, with the **null L** (sign-randomised evidence) "
    "next to every L. Label SYNTHETIC results as such.",
    "Check the synthetic switch spacing against the scorer's 50-s property; closely spaced switches penalise even a "
    "perfect decoder.",
    "Run the lookahead unit test (reports/eda_full/scripts/e3_lookahead_test.py) on the full submission pipeline.",
    "Confirm that no step reads trial or subject IDs, trial order, column order, or statistics pooled across test trials.",
])
H("6.3 Roadmap: the example-trial proposals, updated with full-corpus evidence")
TABLE([["P0", "Remove the lab dialects causally (name-matched montage, state-initialised HP, CAR, running robust z)",
        "Confirmed necessary: amplitude alone identifies the dataset at 98.9%; causal filtering costs nothing; CAR helps",
        "keep"],
       ["P0", "Make envelopes scale-symmetric (per-window z-score, random column swap)",
        "Confirmed: loudness-only classifier 82.5% in KUL2016", "keep"],
       ["P0", "Retune the integrator against the official scorer",
        "Gain is real but small with cross-dataset evidence (+0.02-0.07 H); a 10-s moving average or variance-normalised HMM is "
        "near the frontier", "keep, but do not expect big wins"],
       ["P1", "Raise SNR1: pooled-covariance ('avgcorr') decoders, CCA",
        "Pooling strategy made no difference under LODO (<= 0.3 pp); CCA not yet tested", "test CCA next"],
       ["P1", "Causal running Euclidean alignment", "The fingerprint survives (87%); effect on decoding untested", "test, low priority"],
       ["P2", "Within-trial unsupervised adaptation of the decoder", "Untested here; allowed by the rules if causal",
        "promising for long trials"],
       ["P3", "Causal DNN envelope decoders with shortcut defences", "Untested here; the lookahead rule caps the receptive field",
        "only with LODO validation"],
       ["avoid", "Spatial / direction-only decoders", "Confirmed: chance across datasets; memorise trials", "auxiliary at most"]],
      ["priority", "proposal (example-trial EDA)", "full-corpus evidence", "status"], [0.08, 0.32, 0.42, 0.18],
      "Table 6.1. Method roadmap. The literature behind each proposal is listed in Appendix B.", keep=False)
H("6.4 Open questions to raise with the organisers")
UL([
    "Are the stream positions (azimuths) available at test time? If so, a speaker decoder also yields direction labels.",
    "Which label is scored on each blind set (attended stream or attended side), and how are the streams ordered?",
    "What is the switch spacing in the blind sets, and is the scorer's behaviour for switches closer than 50 s intended?",
    "How were the envelopes computed? USTC envelopes differ in shape (syllable-rate bump, platykurtic), and KUL-AV-GC "
    "envelopes are high-passed at about 1 Hz.",
    "Which amplifier recorded KUL-UHD (the challenge page and the CND files disagree), and which two channels were dropped?",
])
H("6.5 Five experiments this analysis justifies")
UL([
    "**Integrator tuning** on LODO evidence against switch schedules that match the test protocol.",
    "**Raise SNR1** before touching the integrator: CCA with an envelope filter bank, causal within-trial adaptation, or a "
    "small DNN trained LODO, all compared on the SNR1 yardstick.",
    "**Causal front end** as in Figure 6.1, with explicit edge handling for KUL-UHD and USTC.",
    "**Keep spatial decoders out** unless they survive LOTO and LODO; clarify stream-position availability first.",
    "**Story-disjoint, order-aware folds** in every model-selection loop.",
])

# =====================================================================================================================
# Part 7
# =====================================================================================================================
PART("7. Getting started")
H("7.1 Environment")
P("cd /root/disk_3tb/EEG-sAAD-Grand-Challenge-Training-Dataset && source .venv/bin/activate. The environment has Python "
  "3.12, PyTorch with CUDA (a Tesla V100 is available), the SuperHugeAAD package, numpy/scipy/scikit-learn/pyriemann and "
  "reportlab. MATLAB, Octave and MNE are not installed. The machine is shared (8 CPUs, 23 GB RAM): use at most about 4 "
  "worker processes.")
H("7.2 Where to find things")
TABLE([["reports/onboarding/", "this guide (build_guide.py, make_figures.py, figures/)"],
       ["reports/eda_raw/", "corpus summary: README.md, trial_inventory.csv, channel statistics"],
       ["EEG-sAAD Data Story.html", "example-trial EDA (1.6 MB with embedded images; open it in a browser)"],
       ["reports/eda_full/README.md, EEG-sAAD_full_corpus_EDA_report.pdf", "full-corpus EDA report (all tables, figures and "
        "'how to read' notes)"],
       ["reports/eda_full/tables/", "68 result tables; start with hypotheses_verdicts.csv, per_subject_metrics.csv, "
        "label_audit.csv (per trial: labels, switches, column check, story names)"],
       ["reports/eda_full/scripts/", "resumable analysis scripts; run_all.sh drives them; scorer.py is the verified scorer port"],
       ["reports/eda_full/cache/cnd/<key>.npz", "per-trial ground truth extracted from CND: att_stream, att_az, att_col, env_cnd, "
        "EOG, story names, condition"],
       ["reports/eda_full/config.yaml", "every analysis parameter (seed 20270101)"],
       ["reports/eda_full/NEXT_STEPS.md", "the five experiments above, with their evidence"]],
      ["path", "what it is"], [0.38, 0.62], "Table 7.1. Where the analysis outputs live.")
H("7.3 Useful commands")
UL(["Re-run the full-corpus EDA (resumable): cd reports/eda_full && bash scripts/run_all.sh",
    "Score a prediction directory with the Python scorer: use score_subject() / aggregate() in reports/eda_full/scripts/scorer.py "
    "on the same CSV format as baselines/matlab_scoring (time_index, label, trial; 0-based)",
    "Check the scorer port: .venv/bin/python reports/eda_full/scripts/e0_scorer_tests.py (must print PASSED 34/34)",
    "Load one trial in the causal pipeline: load_pp(row, 'c19', 'common57') in reports/eda_full/scripts/common.py",
    "Rebuild this guide: cd reports/onboarding && ../../.venv/bin/python make_figures.py && ../../.venv/bin/python build_guide.py"])
BOX("Good first tasks for a new team member",
    "1. Read reports/eda_full/tables/label_audit.csv for one dataset and find the KUL-AV-GC switch in the CND labels.  "
    "2. Reproduce one LODO accuracy number from Table 5.4a with the scripts.  "
    "3. Score the sign-randomised null for your own integrator and compare it with Table 5.6.")

# =====================================================================================================================
# Appendix A: glossary
# =====================================================================================================================
PART("Appendix A. Glossary")
TABLE([["sAAD / AAD", "(selective) auditory attention decoding: inferring which of several talkers a listener attends to from EEG"],
       ["Trial", "one continuous recording (50 s to 10 min) with two simultaneous talkers"],
       ["Envelope", "the slow loudness contour of a talker's speech (roughly 1-9 Hz); the thing the brain tracks"],
       ["Backward decoder", "a linear filter mapping EEG (here 0-500 ms after each moment) onto the speech envelope; the talker "
        "whose envelope correlates better with the reconstruction is predicted as attended"],
       ["Forward model / TRF", "the reverse direction: how EEG responds over time to the envelope"],
       ["r(att), r(ign), d", "correlation of the reconstruction with the attended / ignored envelope in a window; d = r(att) - r(ign)"],
       ["SNR1", "mean(d)/SD(d) over 1-s windows; accuracy is about Phi(k SNR1 sqrt(w)) (the SNR law)"],
       ["LOTO / LOSO / LODO", "leave-one-trial-out within a subject / leave-one-subject-out within a dataset / "
        "leave-one-dataset-out (closest proxy for the blind test sets)"],
       ["Within-trial split", "train and test windows from the same trial; leaky, only used to measure memorisation"],
       ["Acc, SDT, L, H", "official metrics: sample accuracy; switch detection time; median exp(-SDT/60); harmonic mean of Acc and L"],
       ["SYNTHETIC switch", "a switch imposed on a fixed-attention trial by swapping which stream counts as attended "
        "(generateSyntheticSwitches.m); measures reaction speed where no real switch happened"],
       ["Integrator", "causal post-processor turning noisy 1-s evidence into stable labels (argmax, moving average, HMM, ladder)"],
       ["Null (sign-randomised)", "the same pipeline fed evidence with random sign: what zero information scores"],
       ["CAR", "common average reference: subtract the mean of all channels at each sample"],
       ["EA / running EA", "Euclidean alignment: whiten each recording by its mean covariance; the running version uses only the past"],
       ["Causal / zero-phase", "causal filters use only past samples (required); zero-phase filters also use future samples"],
       ["CND", "the 'continuous-event neural data' MATLAB format the organisers distribute (neural + stim structs)"],
       ["SuperHuge", "the npy/pickle layout used by the Python DNN baselines, produced by tools/convert_cnd_to_superhuge.py"],
       ["Montage 57 / 58 / 62", "channel sets matched by name: common to all five datasets / BioSemi-64 and USTC / "
        "BioSemi-64 and KUL-UHD"],
       ["ALI, hEOG", "alpha lateralisation index (right minus left parieto-occipital 8-13 Hz power, normalised); horizontal "
        "eye-movement proxy from frontal electrodes"]],
      ["term", "meaning"], [0.2, 0.8], keep=False)

# =====================================================================================================================
# Appendix B: references
# =====================================================================================================================
PART("Appendix B. References")
P("Sources cited by the example-trial EDA ('EEG-sAAD Data Story'). They were **not re-checked** for this guide; items that "
  "document marked as unverified (mostly 2026 preprints) remain unverified. Challenge and dataset pages come first.")
refs = [
    ("Challenge page (rules, metric, baseline numbers)", "https://www.codabench.org/competitions/17856/"),
    ("KUL2016 dataset (Zenodo 4004271)", "https://zenodo.org/records/4004271"),
    ("KUL-AV-GC dataset (Zenodo 11058711)", "https://zenodo.org/records/11058711"),
    ("KUL-UHD dataset (Zenodo 4518754)", "https://zenodo.org/records/4518754"),
    ("USTC dataset (Zenodo 18366436)", "https://zenodo.org/records/18366436"),
    ("Heintz et al. 2025, HMM post-processing of AAD decisions (arXiv 2506.24024)", "https://arxiv.org/abs/2506.24024"),
    ("Geirnaert et al. 2021, Electroencephalography-based AAD: an overview (SPM; arXiv 2008.04569)", "https://arxiv.org/abs/2008.04569"),
    ("Geirnaert et al. 2020, MESD metric for AAD (TNSRE)", "https://www.biorxiv.org/content/10.1101/745695v2.full"),
    ("Geirnaert et al. 2021, Unsupervised self-adaptive AAD (JBHI)", "https://simongeirnaert.com/publication/unsupervised2021geirnaert/unsupervised2021geirnaert.pdf"),
    ("Time-adaptive unsupervised AAD (bioRxiv 2022.01.07.475386)", "https://www.biorxiv.org/content/10.1101/2022.01.07.475386v3.full"),
    ("Initialisation-bias fixes for unsupervised AAD (arXiv 2509.14764)", "https://arxiv.org/abs/2509.14764"),
    ("Wong et al. 2018, comparison of regularisation methods for AAD (Front. Neurosci.)", "https://www.frontiersin.org/journals/neuroscience/articles/10.3389/fnins.2018.00531/full"),
    ("Miran et al. 2018, real-time tracking of selective attention (Front. Neurosci.)", "https://www.frontiersin.org/journals/neuroscience/articles/10.3389/fnins.2018.00262/full"),
    ("de Cheveigné et al. 2018, decoding the auditory brain with CCA (NeuroImage)", "https://www.sciencedirect.com/science/article/pii/S1053811918300338"),
    ("de Cheveigné et al., stepwise match-mismatch study (bioRxiv 2020.11.05.370072)", "https://www.biorxiv.org/content/10.1101/2020.11.05.370072v1.full"),
    ("Accou et al. 2023, VLAAI decoder (Sci. Rep.)", "https://www.nature.com/articles/s41598-022-27332-2"),
    ("Puffay et al. 2023, relating EEG to continuous speech: review of pitfalls (arXiv 2302.01736)", "https://arxiv.org/abs/2302.01736"),
    ("Rotaru et al. 2024, gaze-controlled AV AAD dataset and spatial decoders (bioRxiv 2023.07.13.548824)", "https://www.biorxiv.org/content/10.1101/2023.07.13.548824v2"),
    ("Rotaru et al. 2024, spatial AAD under gaze control (arXiv 2412.01401)", "https://arxiv.org/abs/2412.01401"),
    ("Yan et al. 2025, block-design shortcuts in DNN AAD (Interspeech)", "https://www.isca-archive.org/interspeech_2025/yan25_interspeech.html"),
    ("ListenNet (IJCAI 2025)", "https://www.ijcai.org/proceedings/2025/0461.pdf"),
    ("He & Wu, Euclidean alignment (arXiv 1808.05464)", "https://arxiv.org/abs/1808.05464"),
    ("Revisiting Euclidean alignment (arXiv 2502.09203)", "https://arxiv.org/abs/2502.09203"),
    ("Wilroth et al. 2023, Riemannian transfer for locus-of-attention decoding", "https://pubmed.ncbi.nlm.nih.gov/37988748/"),
    ("Adams & MacKay 2007, Bayesian online change-point detection (arXiv 0710.3742)", "https://arxiv.org/abs/0710.3742"),
    ("Closed-loop CCA AAD system (bioRxiv 2024.03.01.582668)", "https://www.biorxiv.org/content/10.1101/2024.03.01.582668v1"),
    ("Contrastive transformer ensembles for match-mismatch (Sci. Rep. 2025)", "https://www.nature.com/articles/s41598-025-13646-4"),
    ("HappyQuokka system, ICASSP 2023 challenge (arXiv 2305.06806)", "https://arxiv.org/abs/2305.06806"),
    ("Wang et al., speech features for match-mismatch (arXiv 2401.04964)", "https://arxiv.org/abs/2401.04964"),
    ("Deng et al. 2020, alpha lateralisation in spatial attention (NeuroImage)", "https://www.cmu.edu/dietrich/psychology/shinn/publications/pdfs/2019/2019neuroimage_deng.pdf"),
    ("Alpha + tracking fusion (JSLHR 2021)", "https://pubs.asha.org/doi/10.1044/2021_JSLHR-20-00608"),
    ("Speech onsets and EEG (J. Neural Eng. 2024)", "https://iopscience.iop.org/article/10.1088/1741-2552/ad1784"),
    ("Nested cross-validation for EEG (arXiv 2510.19174)", "https://arxiv.org/abs/2510.19174"),
    ("Ivucic et al. 2024, causal time-series splits (EMBC)", "https://doi.org/10.1109/embc53108.2024.10782636"),
    ("EEG-AAD 2026 locus-of-attention challenge", "https://fchest.github.io/icassp-aad/"),
    ("Unverified 2026 preprints cited: channel adaptation (arXiv 2604.23091), unbalanced datasets (arXiv 2605.25605), "
     "learned switch gate (arXiv 2608.01618), Markov-switching model (arXiv 2602.13447); EEG foundation-model study "
     "(OpenReview 5Xwm8e6vbh)", "https://openreview.net/forum?id=5Xwm8e6vbh"),
]
TABLE([[str(i + 1), t, u] for i, (t, u) in enumerate(refs)], ["#", "source", "link"], [0.04, 0.5, 0.46], keep=False)


def footer(canvas, doc):
    canvas.saveState()
    canvas.setFont("DV", 7.5)
    canvas.setFillColor(colors.HexColor("#666666"))
    canvas.drawString(1.7 * cm, 1.0 * cm, "EEG-sAAD Grand Challenge: onboarding guide to the training data")
    canvas.drawRightString(A4[0] - 1.7 * cm, 1.0 * cm, f"page {doc.page}")
    canvas.restoreState()


out = os.path.join(HERE, "EEG-sAAD_onboarding_guide.pdf")
doc = SimpleDocTemplate(out, pagesize=A4, leftMargin=1.7 * cm, rightMargin=1.7 * cm, topMargin=1.5 * cm, bottomMargin=1.6 * cm,
                        title="EEG-sAAD Grand Challenge: onboarding guide to the training data",
                        author="EEG-sAAD challenge team")
doc.build(story, onFirstPage=footer, onLaterPages=footer)
print("wrote", out)
