"""B6 follow-up: stimulus identity from the envelopes themselves (names ignored), vs the CND streamWav names.

Every (trial, column) envelope is reduced to 8 Hz (anti-aliased) and compared with every other envelope of the
same dataset by zero-lag correlation over their common prefix (up to 40 s, trials start at stimulus onset).
r > 0.99 = same audio. Connected components = envelope-identity clusters. A cluster holding several names, or a
name spread over several clusters, is a naming inconsistency. Cross-dataset: same test between KUL2016 and KUL-UHD.
Writes tables/stim_identity_clusters.csv (per envelope), tables/stim_identity_summary.csv
"""
import os
import sys

import numpy as np
import pandas as pd
from scipy import signal
from scipy.sparse.csgraph import connected_components

sys.path.insert(0, os.path.dirname(__file__))
from common import DS_KEYS, TAB, get_logger, load_env  # noqa: E402

log = get_logger("b4_envelope_identity")
N8 = 320                      # 40 s at 8 Hz

if __name__ == "__main__":
    la = pd.read_csv(os.path.join(TAB, "label_audit.csv"))
    items, Z = [], []
    for _, r in la.iterrows():
        e = load_env(r["key"]).astype(float)
        for c in (0, 1):
            x = signal.resample_poly(e[:, c] - e[:, c].mean(), 1, 16)[:N8]
            x = (x - x.mean()) / (x.std() + 1e-12)
            items.append(dict(key=r["key"], ds=r["ds"], subject=r["subject"], trial=r["trial"], col=c,
                              wav=r[f"wav_col{c}"], attended=(c == 0)))
            Z.append(x)
    it = pd.DataFrame(items)
    Z = np.array(Z) / np.sqrt(N8)
    it["cluster"] = -1
    summ = []
    for ds in DS_KEYS:
        m = np.flatnonzero(it.ds.values == ds)
        R = Z[m] @ Z[m].T
        n, lab = connected_components(R > 0.99, directed=False)
        it.loc[m, "cluster"] = [f"{ds}-{k}" for k in lab]
        g = it.iloc[m]
        names_per_cluster = g.groupby("cluster").wav.nunique()
        clusters_per_name = g.groupby("wav").cluster.nunique()
        summ.append(dict(ds=ds, n_envelopes=len(m), n_names=g.wav.nunique(), n_identity_clusters=n,
                         clusters_with_several_names=int((names_per_cluster > 1).sum()),
                         names_in_several_clusters=int((clusters_per_name > 1).sum()),
                         envelopes_in_mixed_clusters=int(g.cluster.isin(names_per_cluster[names_per_cluster > 1].index).sum()),
                         max_uses_of_one_audio=int(g.cluster.value_counts().max()),
                         max_subjects_one_audio=int(g.groupby("cluster").subject.nunique().max())))
    # cross-dataset KUL2016 vs KUL-UHD
    a, b = np.flatnonzero(it.ds.values == "KUL"), np.flatnonzero(it.ds.values == "UHD")
    X = (Z[a] @ Z[b].T) > 0.99
    shared = pd.DataFrame(dict(kul=it.cluster.values[a][np.nonzero(X)[0]], uhd=it.cluster.values[b][np.nonzero(X)[1]])).drop_duplicates()
    s = pd.DataFrame(summ)
    s["kul_uhd_shared_audio_clusters"] = np.nan
    s.loc[s.ds.isin(["KUL", "UHD"]), "kul_uhd_shared_audio_clusters"] = shared.uhd.nunique()
    it.to_csv(os.path.join(TAB, "stim_identity_clusters.csv"), index=False)
    s.to_csv(os.path.join(TAB, "stim_identity_summary.csv"), index=False)
    log.info("\n" + s.to_string())
    log.info(f"KUL2016<->KUL-UHD identical audio: {shared.uhd.nunique()} UHD clusters match {shared.kul.nunique()} KUL clusters; "
             f"UHD envelopes whose audio also occurs in KUL2016: {int(it.iloc[b].cluster.isin(shared.uhd).sum())}/{len(b)}")
    mixed = it[it.cluster.isin(it.groupby("cluster").wav.nunique().loc[lambda v: v > 1].index)]
    log.info("examples of clusters holding several names:\n"
             + mixed.groupby("cluster").wav.apply(lambda v: sorted(set(v))[:3]).head(8).to_string())
