_` prefix."""
+ warc_path = tmp_path / "empty.warc.gz"
+ _write_synthetic_warc(warc_path, "http://example.com/", b"")
+ fake_a = _FakeModel(("__label__shared",))
+ fake_b = _FakeModel(("__label__shared",))
+ _patch_models(
+ monkeypatch,
+ {
+ ("rA/m", "a.bin"): fake_a,
+ ("rB/m", "b.bin"): fake_b,
+ },
+ )
+
+ output_path = tmp_path / "out.csv"
+ args = _default_args(
+ warc_paths=[str(warc_path)],
+ output=str(output_path),
+ model_repo=["rA/m", "rB/m"],
+ model_file=["a.bin", "b.bin"],
+ )
+ rc = ClassifyWarcCommand().run(args)
+ assert rc == 0
+
+ with output_path.open(newline="", encoding="utf-8") as fh:
+ rows = list(csv.reader(fh))
+ assert rows[0] == [
+ "URL",
+ "score_m0___label__shared",
+ "score_m1___label__shared",
+ "warc_filename",
+ "warc_record_index",
+ ]
+
+
+def test_run_rejects_mismatched_model_lengths(caplog):
+ """--model-repo and --model-file of different lengths: rc=2, no work done."""
+ caplog.set_level(logging.ERROR, logger="ccoa.commands.classify_warc")
+ args = _default_args(
+ model_repo=["r1/m", "r2/m"],
+ model_file=["only_one.bin"],
+ output="-",
+ )
+ rc = ClassifyWarcCommand().run(args)
+ assert rc == 2
+
+
+def test_run_scores_all_resolved_labels(monkeypatch, tmp_path):
+ """Each record is scored against every requested label; outputs land in column order."""
+ warc_path = tmp_path / "doc.warc.gz"
+ html = (
+ b"T "
+ b"Long enough body to survive trafilatura's minimum-content gates. "
+ b"We just need a single response record to flow through the pipeline.
"
+ b""
+ )
+ _write_synthetic_warc(warc_path, "http://example.com/doc", html)
+ fake = _FakeModel(("__label__alpha", "__label__beta"))
+ _patch_models(monkeypatch, {("r/x", "x.bin"): fake})
+
+ output_path = tmp_path / "out.csv"
+ args = _default_args(
+ warc_paths=[str(warc_path)],
+ output=str(output_path),
+ model_repo=["r/x"],
+ model_file=["x.bin"],
+ )
+ rc = ClassifyWarcCommand().run(args)
+ assert rc == 0
+
+ with output_path.open(newline="", encoding="utf-8") as fh:
+ rows = list(csv.reader(fh))
+ assert rows[0] == [
+ "URL",
+ "score___label__alpha",
+ "score___label__beta",
+ "warc_filename",
+ "warc_record_index",
+ ]
+ assert len(rows) == 2
+ url, alpha, beta, warc_filename, record_index = rows[1]
+ assert url == "http://example.com/doc"
+ # `_FakeModel.predict` is rigged: 1.0 for the first label, 0.0 for the rest.
+ assert float(alpha) == 1.0
+ assert float(beta) == 0.0
+ assert warc_filename == str(warc_path)
+ assert record_index == "0"
From 9d3cf9c9dd2db428963cad0bf1e2b739b633edcf Mon Sep 17 00:00:00 2001
From: malteos
Date: Mon, 1 Jun 2026 16:21:31 +0200
Subject: [PATCH 2/3] added notebook for multi label eval
---
notebooks/compare_multi_label_scores.ipynb | 980 +++++++++++++++++++++
1 file changed, 980 insertions(+)
create mode 100644 notebooks/compare_multi_label_scores.ipynb
diff --git a/notebooks/compare_multi_label_scores.ipynb b/notebooks/compare_multi_label_scores.ipynb
new file mode 100644
index 0000000..333e033
--- /dev/null
+++ b/notebooks/compare_multi_label_scores.ipynb
@@ -0,0 +1,980 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "mlc-00-title",
+ "metadata": {},
+ "source": [
+ "# Multi-classifier comparison: Science + Quality, Baseline vs Focus\n",
+ "\n",
+ "This notebook compares `ccoa classify-warc` **multi-model** output for two Common Crawl runs:\n",
+ "\n",
+ "- **CC-MAIN-2024-18** — baseline\n",
+ "- **CC-SUPPLEMENTAL-2026-22** — focus (Open Athena project)\n",
+ "\n",
+ "Each row has four score columns produced in a single pass:\n",
+ "\n",
+ "| Column | Source | Meaning |\n",
+ "|------------------------------|---------------------------------------------|----------------------------------|\n",
+ "| `score_m0___label__science` | `ibm-granite/GneissWeb.Sci_classifier` | P(page is science) |\n",
+ "| `score_m0___label__cc` | `ibm-granite/GneissWeb.Sci_classifier` | P(page is generic Common Crawl) |\n",
+ "| `score_m1___label__hq` | `ibm-granite/GneissWeb.Quality_annotator` | P(page is high quality) |\n",
+ "| `score_m1___label__cc` | `ibm-granite/GneissWeb.Quality_annotator` | P(page is generic Common Crawl) |\n",
+ "\n",
+ "The two scores from each model sum to 1.0 per row by construction (softmax).\n",
+ "\n",
+ "We analyse the data along two axes:\n",
+ "\n",
+ "1. **Continuous scores** — how each crawl's distribution sits in `[0, 1]` for each label.\n",
+ "2. **Binary labels** — pick the larger score of each pair (`is_science = score_science > score_cc`, same for `is_hq`); then look at the **joint** label `is_science AND is_hq` — pages both classifiers agree are *high-quality science*. The headline question is whether the focus crawl produces more such pages, both in absolute count and as a share of the crawl.\n",
+ "\n",
+ "## Setup\n",
+ "\n",
+ "First-time install of notebook extras:\n",
+ "\n",
+ "```bash\n",
+ "uv pip install -e .[notebooks]\n",
+ "```\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "id": "mlc-01-imports",
+ "metadata": {
+ "execution": {
+ "iopub.execute_input": "2026-06-01T14:17:13.835417Z",
+ "iopub.status.busy": "2026-06-01T14:17:13.835268Z",
+ "iopub.status.idle": "2026-06-01T14:17:15.214316Z",
+ "shell.execute_reply": "2026-06-01T14:17:15.213944Z"
+ }
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Baseline CSV: ../data/gneissweb-science-top-10k/CC-MAIN-2024-18_1m_scores_sci-hq.csv (exists=True)\n",
+ "Focus CSV: ../data/gneissweb-science-top-10k/CC-SUPPLEMENTAL-2026-22_1m_scores_sci-hq.csv (exists=True)\n"
+ ]
+ }
+ ],
+ "source": [
+ "from __future__ import annotations\n",
+ "\n",
+ "from pathlib import Path\n",
+ "\n",
+ "import matplotlib.pyplot as plt\n",
+ "import numpy as np\n",
+ "import pandas as pd\n",
+ "from scipy import stats\n",
+ "\n",
+ "# Run from repo root or from inside notebooks/.\n",
+ "DATA_DIR = Path(\"data\") if Path(\"data\").exists() else Path(\"..\") / \"data\"\n",
+ "SCORES_DIR = DATA_DIR / \"gneissweb-science-top-10k\"\n",
+ "\n",
+ "BASELINE_NAME = \"CC-MAIN-2024-18\"\n",
+ "FOCUS_NAME = \"CC-SUPPLEMENTAL-2026-22\"\n",
+ "\n",
+ "# Download from S3:\n",
+ "# aws s3 sync s3://commoncrawl-dev/cc-focus-tools/experiments/open-athena/gneissweb-science-top-10k/ ./data/gneissweb-science-top-10k/\n",
+ "BASELINE_CSV = SCORES_DIR / f\"{BASELINE_NAME}_1m_scores_sci-hq.csv\"\n",
+ "FOCUS_CSV = SCORES_DIR / f\"{FOCUS_NAME}_1m_scores_sci-hq.csv\"\n",
+ "\n",
+ "# The four score columns produced by the dual-model classify-warc run.\n",
+ "SCI_SCIENCE = \"score_m0___label__science\"\n",
+ "SCI_CC = \"score_m0___label__cc\"\n",
+ "HQ_HQ = \"score_m1___label__hq\"\n",
+ "HQ_CC = \"score_m1___label__cc\"\n",
+ "SCORE_COLS = [SCI_SCIENCE, SCI_CC, HQ_HQ, HQ_CC]\n",
+ "\n",
+ "print(f\"Baseline CSV: {BASELINE_CSV} (exists={BASELINE_CSV.exists()})\")\n",
+ "print(f\"Focus CSV: {FOCUS_CSV} (exists={FOCUS_CSV.exists()})\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "mlc-02-load-md",
+ "metadata": {},
+ "source": [
+ "## 1. Load the score data\n",
+ "\n",
+ "Read just the four score columns from each CSV (URL/warc-filename/warc-record-index aren't needed for distribution analysis)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "id": "mlc-03-load",
+ "metadata": {
+ "execution": {
+ "iopub.execute_input": "2026-06-01T14:17:15.215422Z",
+ "iopub.status.busy": "2026-06-01T14:17:15.215336Z",
+ "iopub.status.idle": "2026-06-01T14:17:16.112684Z",
+ "shell.execute_reply": "2026-06-01T14:17:16.112294Z"
+ }
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "CC-MAIN-2024-18: 1,000,000 rows\n",
+ "CC-SUPPLEMENTAL-2026-22: 999,774 rows\n",
+ "m0 row sum (baseline, top values): {1.0: np.int64(1000000)}\n",
+ "m1 row sum (baseline, top values): {1.0: np.int64(1000000)}\n"
+ ]
+ }
+ ],
+ "source": [
+ "def load_scores(csv_path: Path) -> pd.DataFrame:\n",
+ " \"\"\"Load the four score columns from a multi-model classify-warc CSV.\"\"\"\n",
+ " return pd.read_csv(csv_path, usecols=SCORE_COLS, dtype=float)\n",
+ "\n",
+ "\n",
+ "baseline = load_scores(BASELINE_CSV)\n",
+ "focus = load_scores(FOCUS_CSV)\n",
+ "\n",
+ "# Quick softmax sanity check — m0 and m1 each sum to 1.0 per row.\n",
+ "m0_sum = (baseline[SCI_SCIENCE] + baseline[SCI_CC]).round(4).value_counts().head(3)\n",
+ "m1_sum = (baseline[HQ_HQ] + baseline[HQ_CC]).round(4).value_counts().head(3)\n",
+ "print(f\"{BASELINE_NAME}: {len(baseline):,} rows\")\n",
+ "print(f\"{FOCUS_NAME}: {len(focus):,} rows\")\n",
+ "print(f\"m0 row sum (baseline, top values): {dict(m0_sum)}\")\n",
+ "print(f\"m1 row sum (baseline, top values): {dict(m1_sum)}\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "mlc-04-stats-md",
+ "metadata": {},
+ "source": [
+ "## 2. Continuous-score summary\n",
+ "\n",
+ "Where does each score sit per crawl? `delta` is `focus - baseline`."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "id": "mlc-05-stats",
+ "metadata": {
+ "execution": {
+ "iopub.execute_input": "2026-06-01T14:17:16.113688Z",
+ "iopub.status.busy": "2026-06-01T14:17:16.113638Z",
+ "iopub.status.idle": "2026-06-01T14:17:16.362014Z",
+ "shell.execute_reply": "2026-06-01T14:17:16.361615Z"
+ }
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " CC-MAIN-2024-18 \n",
+ " CC-SUPPLEMENTAL-2026-22 \n",
+ " delta \n",
+ " \n",
+ " \n",
+ " column \n",
+ " stat \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " score_m0___label__science \n",
+ " mean \n",
+ " 0.142090 \n",
+ " 0.517199 \n",
+ " 0.375109 \n",
+ " \n",
+ " \n",
+ " median \n",
+ " 0.006990 \n",
+ " 0.576132 \n",
+ " 0.569142 \n",
+ " \n",
+ " \n",
+ " p25 \n",
+ " 0.000052 \n",
+ " 0.019895 \n",
+ " 0.019843 \n",
+ " \n",
+ " \n",
+ " p75 \n",
+ " 0.179258 \n",
+ " 0.989210 \n",
+ " 0.809953 \n",
+ " \n",
+ " \n",
+ " p95 \n",
+ " 0.731946 \n",
+ " 1.000009 \n",
+ " 0.268063 \n",
+ " \n",
+ " \n",
+ " p99 \n",
+ " 0.988988 \n",
+ " 1.000010 \n",
+ " 0.011022 \n",
+ " \n",
+ " \n",
+ " score_m0___label__cc \n",
+ " mean \n",
+ " 0.857930 \n",
+ " 0.482821 \n",
+ " -0.375109 \n",
+ " \n",
+ " \n",
+ " median \n",
+ " 0.993030 \n",
+ " 0.423888 \n",
+ " -0.569142 \n",
+ " \n",
+ " \n",
+ " p25 \n",
+ " 0.820763 \n",
+ " 0.010810 \n",
+ " -0.809953 \n",
+ " \n",
+ " \n",
+ " p75 \n",
+ " 0.999968 \n",
+ " 0.980125 \n",
+ " -0.019843 \n",
+ " \n",
+ " \n",
+ " p95 \n",
+ " 1.000010 \n",
+ " 0.999974 \n",
+ " -0.000036 \n",
+ " \n",
+ " \n",
+ " p99 \n",
+ " 1.000010 \n",
+ " 1.000010 \n",
+ " 0.000000 \n",
+ " \n",
+ " \n",
+ " score_m1___label__hq \n",
+ " mean \n",
+ " 0.106053 \n",
+ " 0.174356 \n",
+ " 0.068303 \n",
+ " \n",
+ " \n",
+ " median \n",
+ " 0.000295 \n",
+ " 0.019288 \n",
+ " 0.018993 \n",
+ " \n",
+ " \n",
+ " p25 \n",
+ " 0.000010 \n",
+ " 0.000082 \n",
+ " 0.000072 \n",
+ " \n",
+ " \n",
+ " p75 \n",
+ " 0.048318 \n",
+ " 0.230062 \n",
+ " 0.181744 \n",
+ " \n",
+ " \n",
+ " p95 \n",
+ " 0.834057 \n",
+ " 0.882294 \n",
+ " 0.048237 \n",
+ " \n",
+ " \n",
+ " p99 \n",
+ " 0.999977 \n",
+ " 0.997152 \n",
+ " -0.002825 \n",
+ " \n",
+ " \n",
+ " score_m1___label__cc \n",
+ " mean \n",
+ " 0.893967 \n",
+ " 0.825664 \n",
+ " -0.068303 \n",
+ " \n",
+ " \n",
+ " median \n",
+ " 0.999725 \n",
+ " 0.980732 \n",
+ " -0.018993 \n",
+ " \n",
+ " \n",
+ " p25 \n",
+ " 0.951702 \n",
+ " 0.769958 \n",
+ " -0.181744 \n",
+ " \n",
+ " \n",
+ " p75 \n",
+ " 1.000010 \n",
+ " 0.999938 \n",
+ " -0.000072 \n",
+ " \n",
+ " \n",
+ " p95 \n",
+ " 1.000010 \n",
+ " 1.000010 \n",
+ " 0.000000 \n",
+ " \n",
+ " \n",
+ " p99 \n",
+ " 1.000010 \n",
+ " 1.000010 \n",
+ " 0.000000 \n",
+ " \n",
+ " \n",
+ "
\n",
+ "
"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "fig, axes = plt.subplots(2, 4, figsize=(20, 8), sharey=\"row\")\n",
+ "bins = np.linspace(0.0, 1.0, 81)\n",
+ "\n",
+ "for col_idx, col in enumerate(SCORE_COLS):\n",
+ " ax = axes[0, col_idx]\n",
+ " ax.hist(baseline[col], bins=bins, alpha=0.5, label=BASELINE_NAME, color=\"C0\", density=True)\n",
+ " ax.hist(focus[col], bins=bins, alpha=0.5, label=FOCUS_NAME, color=\"C1\", density=True)\n",
+ " ax.set_yscale(\"log\")\n",
+ " ax.set_title(col, fontsize=9)\n",
+ " ax.set_xlabel(\"score\")\n",
+ " if col_idx == 0:\n",
+ " ax.set_ylabel(\"density (log)\")\n",
+ " ax.legend(fontsize=8)\n",
+ " ax.grid(True, alpha=0.3)\n",
+ "\n",
+ " ax = axes[1, col_idx]\n",
+ " for series, label, color in [\n",
+ " (baseline[col], BASELINE_NAME, \"C0\"),\n",
+ " (focus[col], FOCUS_NAME, \"C1\"),\n",
+ " ]:\n",
+ " x = np.sort(series.to_numpy())\n",
+ " y = np.arange(1, len(x) + 1) / len(x)\n",
+ " ax.plot(x, y, label=label, color=color)\n",
+ " ax.set_title(f\"ECDF — {col}\", fontsize=9)\n",
+ " ax.set_xlabel(\"score\")\n",
+ " if col_idx == 0:\n",
+ " ax.set_ylabel(\"P(score \\u2264 x)\")\n",
+ " ax.legend(fontsize=8)\n",
+ " ax.grid(True, alpha=0.3)\n",
+ "\n",
+ "plt.tight_layout()\n",
+ "plt.show()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "mlc-08-binary-md",
+ "metadata": {},
+ "source": [
+ "## 4. Binary labels from the score pairs\n",
+ "\n",
+ "We collapse each classifier's two scores into a single boolean by picking the larger one:\n",
+ "\n",
+ "- `is_science = score_m0___label__science > score_m0___label__cc`\n",
+ "- `is_hq = score_m1___label__hq > score_m1___label__cc`\n",
+ "\n",
+ "And then the joint label that interests us most:\n",
+ "\n",
+ "- `is_sci_and_hq = is_science AND is_hq`"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "id": "mlc-09-binary",
+ "metadata": {
+ "execution": {
+ "iopub.execute_input": "2026-06-01T14:17:19.226495Z",
+ "iopub.status.busy": "2026-06-01T14:17:19.226421Z",
+ "iopub.status.idle": "2026-06-01T14:17:19.240514Z",
+ "shell.execute_reply": "2026-06-01T14:17:19.240141Z"
+ }
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " CC-MAIN-2024-18 \n",
+ " CC-SUPPLEMENTAL-2026-22 \n",
+ " lift (focus / baseline) \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " n \n",
+ " 1000000.000000 \n",
+ " 999774.000000 \n",
+ " 0.999774 \n",
+ " \n",
+ " \n",
+ " is_science_count \n",
+ " 113543.000000 \n",
+ " 521658.000000 \n",
+ " 4.594365 \n",
+ " \n",
+ " \n",
+ " is_science_share \n",
+ " 0.113543 \n",
+ " 0.521776 \n",
+ " 4.595404 \n",
+ " \n",
+ " \n",
+ " is_hq_count \n",
+ " 85984.000000 \n",
+ " 149159.000000 \n",
+ " 1.734730 \n",
+ " \n",
+ " \n",
+ " is_hq_share \n",
+ " 0.085984 \n",
+ " 0.149193 \n",
+ " 1.735122 \n",
+ " \n",
+ " \n",
+ " is_sci_and_hq_count \n",
+ " 21508.000000 \n",
+ " 118576.000000 \n",
+ " 5.513111 \n",
+ " \n",
+ " \n",
+ " is_sci_and_hq_share \n",
+ " 0.021508 \n",
+ " 0.118603 \n",
+ " 5.514358 \n",
+ " \n",
+ " \n",
+ "
\n",
+ "
"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "labels = [\"is_science\", \"is_hq\", \"is_science AND is_hq\"]\n",
+ "baseline_counts = [\n",
+ " int(baseline_lbl[\"is_science\"].sum()),\n",
+ " int(baseline_lbl[\"is_hq\"].sum()),\n",
+ " int(baseline_lbl[\"is_sci_and_hq\"].sum()),\n",
+ "]\n",
+ "focus_counts = [\n",
+ " int(focus_lbl[\"is_science\"].sum()),\n",
+ " int(focus_lbl[\"is_hq\"].sum()),\n",
+ " int(focus_lbl[\"is_sci_and_hq\"].sum()),\n",
+ "]\n",
+ "baseline_shares = [c / len(baseline_lbl) for c in baseline_counts]\n",
+ "focus_shares = [c / len(focus_lbl) for c in focus_counts]\n",
+ "\n",
+ "fig, axes = plt.subplots(1, 2, figsize=(13, 5))\n",
+ "x = np.arange(len(labels))\n",
+ "w = 0.4\n",
+ "\n",
+ "axes[0].bar(x - w / 2, baseline_counts, w, label=BASELINE_NAME, color=\"C0\")\n",
+ "axes[0].bar(x + w / 2, focus_counts, w, label=FOCUS_NAME, color=\"C1\")\n",
+ "for i, (b, f) in enumerate(zip(baseline_counts, focus_counts, strict=True)):\n",
+ " axes[0].text(i - w / 2, b, f\"{b:,}\", ha=\"center\", va=\"bottom\", fontsize=8)\n",
+ " axes[0].text(i + w / 2, f, f\"{f:,}\", ha=\"center\", va=\"bottom\", fontsize=8)\n",
+ "axes[0].set_xticks(x)\n",
+ "axes[0].set_xticklabels(labels)\n",
+ "axes[0].set_ylabel(\"page count\")\n",
+ "axes[0].set_title(\"Absolute counts\")\n",
+ "axes[0].legend()\n",
+ "axes[0].grid(axis=\"y\", alpha=0.3)\n",
+ "\n",
+ "axes[1].bar(x - w / 2, baseline_shares, w, label=BASELINE_NAME, color=\"C0\")\n",
+ "axes[1].bar(x + w / 2, focus_shares, w, label=FOCUS_NAME, color=\"C1\")\n",
+ "for i, (b, f) in enumerate(zip(baseline_shares, focus_shares, strict=True)):\n",
+ " axes[1].text(i - w / 2, b, f\"{b:.3f}\", ha=\"center\", va=\"bottom\", fontsize=8)\n",
+ " axes[1].text(i + w / 2, f, f\"{f:.3f}\", ha=\"center\", va=\"bottom\", fontsize=8)\n",
+ "axes[1].set_xticks(x)\n",
+ "axes[1].set_xticklabels(labels)\n",
+ "axes[1].set_ylabel(\"share of crawl\")\n",
+ "axes[1].set_title(\"Share of crawl\")\n",
+ "axes[1].legend()\n",
+ "axes[1].grid(axis=\"y\", alpha=0.3)\n",
+ "\n",
+ "plt.tight_layout()\n",
+ "plt.show()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "mlc-14-tests-md",
+ "metadata": {},
+ "source": [
+ "## 7. Significance and effect size on the binary labels\n",
+ "\n",
+ "For each binary label we run a 2×2 chi-square test on the (focus vs baseline) × (label vs not-label) contingency table. With ~1M rows per crawl, p-values will be effectively 0 for any non-trivial difference, so the **rate ratio** (focus rate / baseline rate) is the substantive answer.\n",
+ "\n",
+ "We complement the chi-square with a **two-sample Kolmogorov–Smirnov test** on the underlying continuous scores (`score___label__science` and `score___label__hq`) so the comparison is grounded in distributional differences, not just thresholded labels."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "id": "mlc-15-tests",
+ "metadata": {
+ "execution": {
+ "iopub.execute_input": "2026-06-01T14:17:19.385657Z",
+ "iopub.status.busy": "2026-06-01T14:17:19.385608Z",
+ "iopub.status.idle": "2026-06-01T14:17:19.560335Z",
+ "shell.execute_reply": "2026-06-01T14:17:19.559969Z"
+ }
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Chi-square on binary labels:\n",
+ " CC-MAIN-2024-18 rate CC-SUPPLEMENTAL-2026-22 rate rate ratio (focus / baseline) chi2 p\n",
+ "label \n",
+ "is_science 0.113543 0.521776 4.595404 384404.736904 0.0\n",
+ "is_hq 0.085984 0.149193 1.735122 19250.278261 0.0\n",
+ "is_sci_and_hq 0.021508 0.118603 5.514358 72349.694228 0.0\n",
+ "\n",
+ "Kolmogorov–Smirnov on continuous scores:\n",
+ " name KS statistic KS p\n",
+ "column \n",
+ "score_m0___label__science science prob. 0.409693 0.0\n",
+ "score_m1___label__hq HQ prob. 0.213082 0.0\n"
+ ]
+ }
+ ],
+ "source": [
+ "rows = []\n",
+ "for label in (\"is_science\", \"is_hq\", \"is_sci_and_hq\"):\n",
+ " f_pos = int(focus_lbl[label].sum())\n",
+ " f_neg = len(focus_lbl) - f_pos\n",
+ " b_pos = int(baseline_lbl[label].sum())\n",
+ " b_neg = len(baseline_lbl) - b_pos\n",
+ " table = np.array([[f_pos, f_neg], [b_pos, b_neg]])\n",
+ " chi2, p, _dof, _ = stats.chi2_contingency(table)\n",
+ " p_focus = f_pos / len(focus_lbl)\n",
+ " p_base = b_pos / len(baseline_lbl) if len(baseline_lbl) else 0.0\n",
+ " ratio = p_focus / p_base if p_base else float(\"inf\")\n",
+ " rows.append(\n",
+ " {\n",
+ " \"label\": label,\n",
+ " f\"{BASELINE_NAME} rate\": p_base,\n",
+ " f\"{FOCUS_NAME} rate\": p_focus,\n",
+ " \"rate ratio (focus / baseline)\": ratio,\n",
+ " \"chi2\": chi2,\n",
+ " \"p\": p,\n",
+ " }\n",
+ " )\n",
+ "chi_df = pd.DataFrame(rows).set_index(\"label\")\n",
+ "\n",
+ "ks_rows = []\n",
+ "for col, friendly in [(SCI_SCIENCE, \"science prob.\"), (HQ_HQ, \"HQ prob.\")]:\n",
+ " ks = stats.ks_2samp(baseline[col], focus[col], method=\"auto\")\n",
+ " ks_rows.append(\n",
+ " {\n",
+ " \"column\": col,\n",
+ " \"name\": friendly,\n",
+ " \"KS statistic\": ks.statistic,\n",
+ " \"KS p\": ks.pvalue,\n",
+ " }\n",
+ " )\n",
+ "ks_df = pd.DataFrame(ks_rows).set_index(\"column\")\n",
+ "\n",
+ "print(\"Chi-square on binary labels:\")\n",
+ "print(chi_df.round(6).to_string())\n",
+ "print(\"\\nKolmogorov\\u2013Smirnov on continuous scores:\")\n",
+ "print(ks_df.round(6).to_string())"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "mlc-16-verdict-md",
+ "metadata": {},
+ "source": [
+ "## 8. Verdict\n",
+ "\n",
+ "One-paragraph summary computed from the cells above."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "id": "mlc-17-verdict",
+ "metadata": {
+ "execution": {
+ "iopub.execute_input": "2026-06-01T14:17:19.561253Z",
+ "iopub.status.busy": "2026-06-01T14:17:19.561203Z",
+ "iopub.status.idle": "2026-06-01T14:17:19.565547Z",
+ "shell.execute_reply": "2026-06-01T14:17:19.565300Z"
+ }
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Comparison: CC-SUPPLEMENTAL-2026-22 vs CC-MAIN-2024-18\n",
+ " is_science focus=0.5218 baseline=0.1135 lift=4.60x\n",
+ " is_hq focus=0.1492 baseline=0.0860 lift=1.74x\n",
+ " is_sci_and_hq focus=0.1186 baseline=0.0215 lift=5.51x\n",
+ "\n",
+ "Joint pages (science AND high quality):\n",
+ " CC-MAIN-2024-18: 21,508 of 1,000,000\n",
+ " CC-SUPPLEMENTAL-2026-22: 118,576 of 999,774\n",
+ " absolute delta: +97,068 (focus has 5.51x the rate of the baseline)\n"
+ ]
+ }
+ ],
+ "source": [
+ "sci_lift = focus_lbl[\"is_science\"].mean() / max(baseline_lbl[\"is_science\"].mean(), 1e-12)\n",
+ "hq_lift = focus_lbl[\"is_hq\"].mean() / max(baseline_lbl[\"is_hq\"].mean(), 1e-12)\n",
+ "joint_lift = focus_lbl[\"is_sci_and_hq\"].mean() / max(baseline_lbl[\"is_sci_and_hq\"].mean(), 1e-12)\n",
+ "\n",
+ "b_joint = int(baseline_lbl[\"is_sci_and_hq\"].sum())\n",
+ "f_joint = int(focus_lbl[\"is_sci_and_hq\"].sum())\n",
+ "\n",
+ "print(f\"Comparison: {FOCUS_NAME} vs {BASELINE_NAME}\")\n",
+ "print(f\" is_science focus={focus_lbl['is_science'].mean():.4f} baseline={baseline_lbl['is_science'].mean():.4f} lift={sci_lift:.2f}x\")\n",
+ "print(f\" is_hq focus={focus_lbl['is_hq'].mean():.4f} baseline={baseline_lbl['is_hq'].mean():.4f} lift={hq_lift:.2f}x\")\n",
+ "print(f\" is_sci_and_hq focus={focus_lbl['is_sci_and_hq'].mean():.4f} baseline={baseline_lbl['is_sci_and_hq'].mean():.4f} lift={joint_lift:.2f}x\")\n",
+ "print()\n",
+ "print(\"Joint pages (science AND high quality):\")\n",
+ "print(f\" {BASELINE_NAME}: {b_joint:,} of {len(baseline_lbl):,}\")\n",
+ "print(f\" {FOCUS_NAME}: {f_joint:,} of {len(focus_lbl):,}\")\n",
+ "print(f\" absolute delta: {f_joint - b_joint:+,d} (focus has {joint_lift:.2f}x the rate of the baseline)\")"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.12.13"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
From 6a67460d20c5363a42614bd0a3ac3277b782bf1b Mon Sep 17 00:00:00 2001
From: malteos
Date: Mon, 1 Jun 2026 16:30:47 +0200
Subject: [PATCH 3/3] fixed notebook
---
notebooks/compare_multi_label_scores.ipynb | 470 ++++-----------------
1 file changed, 85 insertions(+), 385 deletions(-)
diff --git a/notebooks/compare_multi_label_scores.ipynb b/notebooks/compare_multi_label_scores.ipynb
index 333e033..bbbea50 100644
--- a/notebooks/compare_multi_label_scores.ipynb
+++ b/notebooks/compare_multi_label_scores.ipynb
@@ -12,14 +12,12 @@
"- **CC-MAIN-2024-18** — baseline\n",
"- **CC-SUPPLEMENTAL-2026-22** — focus (Open Athena project)\n",
"\n",
- "Each row has four score columns produced in a single pass:\n",
+ "Each row of the input CSVs has four score columns produced in a single pass:\n",
"\n",
- "| Column | Source | Meaning |\n",
- "|------------------------------|---------------------------------------------|----------------------------------|\n",
- "| `score_m0___label__science` | `ibm-granite/GneissWeb.Sci_classifier` | P(page is science) |\n",
- "| `score_m0___label__cc` | `ibm-granite/GneissWeb.Sci_classifier` | P(page is generic Common Crawl) |\n",
- "| `score_m1___label__hq` | `ibm-granite/GneissWeb.Quality_annotator` | P(page is high quality) |\n",
- "| `score_m1___label__cc` | `ibm-granite/GneissWeb.Quality_annotator` | P(page is generic Common Crawl) |\n",
+ "- `score_m0___label__science` — `ibm-granite/GneissWeb.Sci_classifier`, P(page is science)\n",
+ "- `score_m0___label__cc` — same model, P(page is generic Common Crawl)\n",
+ "- `score_m1___label__hq` — `ibm-granite/GneissWeb.Quality_annotator`, P(page is high quality)\n",
+ "- `score_m1___label__cc` — same model, P(page is generic Common Crawl)\n",
"\n",
"The two scores from each model sum to 1.0 per row by construction (softmax).\n",
"\n",
@@ -43,10 +41,10 @@
"id": "mlc-01-imports",
"metadata": {
"execution": {
- "iopub.execute_input": "2026-06-01T14:17:13.835417Z",
- "iopub.status.busy": "2026-06-01T14:17:13.835268Z",
- "iopub.status.idle": "2026-06-01T14:17:15.214316Z",
- "shell.execute_reply": "2026-06-01T14:17:15.213944Z"
+ "iopub.execute_input": "2026-06-01T14:26:01.911346Z",
+ "iopub.status.busy": "2026-06-01T14:26:01.911221Z",
+ "iopub.status.idle": "2026-06-01T14:26:02.772015Z",
+ "shell.execute_reply": "2026-06-01T14:26:02.771662Z"
}
},
"outputs": [
@@ -108,10 +106,10 @@
"id": "mlc-03-load",
"metadata": {
"execution": {
- "iopub.execute_input": "2026-06-01T14:17:15.215422Z",
- "iopub.status.busy": "2026-06-01T14:17:15.215336Z",
- "iopub.status.idle": "2026-06-01T14:17:16.112684Z",
- "shell.execute_reply": "2026-06-01T14:17:16.112294Z"
+ "iopub.execute_input": "2026-06-01T14:26:02.773165Z",
+ "iopub.status.busy": "2026-06-01T14:26:02.773067Z",
+ "iopub.status.idle": "2026-06-01T14:26:03.721906Z",
+ "shell.execute_reply": "2026-06-01T14:26:03.721445Z"
}
},
"outputs": [
@@ -160,259 +158,44 @@
"id": "mlc-05-stats",
"metadata": {
"execution": {
- "iopub.execute_input": "2026-06-01T14:17:16.113688Z",
- "iopub.status.busy": "2026-06-01T14:17:16.113638Z",
- "iopub.status.idle": "2026-06-01T14:17:16.362014Z",
- "shell.execute_reply": "2026-06-01T14:17:16.361615Z"
+ "iopub.execute_input": "2026-06-01T14:26:03.722891Z",
+ "iopub.status.busy": "2026-06-01T14:26:03.722832Z",
+ "iopub.status.idle": "2026-06-01T14:26:03.963331Z",
+ "shell.execute_reply": "2026-06-01T14:26:03.962976Z"
}
},
"outputs": [
{
- "data": {
- "text/html": [
- "\n",
- "\n",
- "
\n",
- " \n",
- " \n",
- " CC-MAIN-2024-18 \n",
- " CC-SUPPLEMENTAL-2026-22 \n",
- " delta \n",
- " \n",
- " \n",
- " column \n",
- " stat \n",
- " \n",
- " \n",
- " \n",
- " \n",
- " score_m0___label__science \n",
- " mean \n",
- " 0.142090 \n",
- " 0.517199 \n",
- " 0.375109 \n",
- " \n",
- " \n",
- " median \n",
- " 0.006990 \n",
- " 0.576132 \n",
- " 0.569142 \n",
- " \n",
- " \n",
- " p25 \n",
- " 0.000052 \n",
- " 0.019895 \n",
- " 0.019843 \n",
- " \n",
- " \n",
- " p75 \n",
- " 0.179258 \n",
- " 0.989210 \n",
- " 0.809953 \n",
- " \n",
- " \n",
- " p95 \n",
- " 0.731946 \n",
- " 1.000009 \n",
- " 0.268063 \n",
- " \n",
- " \n",
- " p99 \n",
- " 0.988988 \n",
- " 1.000010 \n",
- " 0.011022 \n",
- " \n",
- " \n",
- " score_m0___label__cc \n",
- " mean \n",
- " 0.857930 \n",
- " 0.482821 \n",
- " -0.375109 \n",
- " \n",
- " \n",
- " median \n",
- " 0.993030 \n",
- " 0.423888 \n",
- " -0.569142 \n",
- " \n",
- " \n",
- " p25 \n",
- " 0.820763 \n",
- " 0.010810 \n",
- " -0.809953 \n",
- " \n",
- " \n",
- " p75 \n",
- " 0.999968 \n",
- " 0.980125 \n",
- " -0.019843 \n",
- " \n",
- " \n",
- " p95 \n",
- " 1.000010 \n",
- " 0.999974 \n",
- " -0.000036 \n",
- " \n",
- " \n",
- " p99 \n",
- " 1.000010 \n",
- " 1.000010 \n",
- " 0.000000 \n",
- " \n",
- " \n",
- " score_m1___label__hq \n",
- " mean \n",
- " 0.106053 \n",
- " 0.174356 \n",
- " 0.068303 \n",
- " \n",
- " \n",
- " median \n",
- " 0.000295 \n",
- " 0.019288 \n",
- " 0.018993 \n",
- " \n",
- " \n",
- " p25 \n",
- " 0.000010 \n",
- " 0.000082 \n",
- " 0.000072 \n",
- " \n",
- " \n",
- " p75 \n",
- " 0.048318 \n",
- " 0.230062 \n",
- " 0.181744 \n",
- " \n",
- " \n",
- " p95 \n",
- " 0.834057 \n",
- " 0.882294 \n",
- " 0.048237 \n",
- " \n",
- " \n",
- " p99 \n",
- " 0.999977 \n",
- " 0.997152 \n",
- " -0.002825 \n",
- " \n",
- " \n",
- " score_m1___label__cc \n",
- " mean \n",
- " 0.893967 \n",
- " 0.825664 \n",
- " -0.068303 \n",
- " \n",
- " \n",
- " median \n",
- " 0.999725 \n",
- " 0.980732 \n",
- " -0.018993 \n",
- " \n",
- " \n",
- " p25 \n",
- " 0.951702 \n",
- " 0.769958 \n",
- " -0.181744 \n",
- " \n",
- " \n",
- " p75 \n",
- " 1.000010 \n",
- " 0.999938 \n",
- " -0.000072 \n",
- " \n",
- " \n",
- " p95 \n",
- " 1.000010 \n",
- " 1.000010 \n",
- " 0.000000 \n",
- " \n",
- " \n",
- " p99 \n",
- " 1.000010 \n",
- " 1.000010 \n",
- " 0.000000 \n",
- " \n",
- " \n",
- "
\n",
- "
\n",
- "\n",
- "
\n",
- " \n",
- " \n",
- " CC-MAIN-2024-18 \n",
- " CC-SUPPLEMENTAL-2026-22 \n",
- " lift (focus / baseline) \n",
- " \n",
- " \n",
- " \n",
- " \n",
- " n \n",
- " 1000000.000000 \n",
- " 999774.000000 \n",
- " 0.999774 \n",
- " \n",
- " \n",
- " is_science_count \n",
- " 113543.000000 \n",
- " 521658.000000 \n",
- " 4.594365 \n",
- " \n",
- " \n",
- " is_science_share \n",
- " 0.113543 \n",
- " 0.521776 \n",
- " 4.595404 \n",
- " \n",
- " \n",
- " is_hq_count \n",
- " 85984.000000 \n",
- " 149159.000000 \n",
- " 1.734730 \n",
- " \n",
- " \n",
- " is_hq_share \n",
- " 0.085984 \n",
- " 0.149193 \n",
- " 1.735122 \n",
- " \n",
- " \n",
- " is_sci_and_hq_count \n",
- " 21508.000000 \n",
- " 118576.000000 \n",
- " 5.513111 \n",
- " \n",
- " \n",
- " is_sci_and_hq_share \n",
- " 0.021508 \n",
- " 0.118603 \n",
- " 5.514358 \n",
- " \n",
- " \n",
- "
\n",
- "