Patching the WARC filename parser to support additional filename schema

src/main/java/org/commoncrawl/spark/CCIndex2Table.java

@@ -44,7 +44,7 @@ protected static class CdxLine extends IndexTable.CdxLine {
 		String digest;
 		String mime, mimeDetected;
 		String filename;
-		int offset, length;
+		long offset, length;
 		short status;
 		String crawl, segment, subset;
 		String charset, languages;
@@ -61,8 +61,8 @@ public CdxLine(String line) throws IOException {
 			mimeDetected = getString("mime-detected");
 
 			filename = getString("filename");
-			offset = getInt("offset");
-			length = getInt("length");
+			offset = getLong("offset");
+			length = getLong("length");
 			status = getHttpStatus("status");
 
 			try {

src/main/java/org/commoncrawl/spark/util/CCWarcFilenameParser.java

@@ -30,7 +30,14 @@ public class CCWarcFilenameParser {
 	 */
 	protected static final Pattern filenameAnalyzer = Pattern.compile(
 			"^(?:common-crawl/)?crawl-data/([^/]+)/segments/([^/]+)/(crawldiagnostics|robotstxt|warc|wat|wet)/");
-
+	/**
+	 * Supplemental crawl filename pattern:
+	 * <code>projects/PROJECT/CC-SUPPLEMENTAL-YYYY-WW[-...]/SEGMENT/SUBSET/*.warc.gz</code>
+	 * e.g.
+	 * <code>projects/cc-open-athena-test/CC-SUPPLEMENTAL-2026-22-bis/20260522101936/warc/CC-SUPPLEMENTAL-2026-22-bis-20260522102012-20260522152012-00000.warc.gz</code>
+	 */
+	protected static final Pattern supplementalFilenameAnalyzer = Pattern.compile(
+			"^[^/]+/[^/]+/(CC-SUPPLEMENTAL-[^/]+)/segments/(\\d+)/(crawldiagnostics|robotstxt|warc)/");
 	/**
 	 * News crawl filename pattern:
 	 * <code>s3://commoncrawl/crawl-data/CC-NEWS/YYYY/MM/*.warc.gz</code> e.g.
@@ -67,11 +74,15 @@ public static FilenameParts getParts(String filename) throws FilenameParseError
 		if (m.find()) {
 			return new FilenameParts(m.group(1), m.group(2), m.group(3));
 		}
+		m = supplementalFilenameAnalyzer.matcher(filename);
+		if (m.find()) {
+			return new FilenameParts(m.group(1), m.group(2), m.group(3));
+		}
 		m = newsFilenameAnalyzer.matcher(filename);
 		if (m.find()) {
 			String crawl = String.format("CC-NEWS-%s-%s", m.group(1), m.group(2));
 			return new FilenameParts(crawl, m.group(3), "news-warc");
 		}
-		throw new FilenameParseError("Filename not parseable (tried main and news): " + filename);
+		throw new FilenameParseError("Filename not parseable (tried main, supplemental and news): " + filename);
 	}
 }

src/main/resources/schema/cc-index-schema-flat.json

@@ -265,7 +265,7 @@
     },
     {
       "name": "warc_record_offset",
-      "type": "integer",
+      "type": "long",
       "nullable": false,
       "metadata": {
         "description": "Offset of the WARC record",
@@ -275,7 +275,7 @@
     },
     {
       "name": "warc_record_length",
-      "type": "integer",
+      "type": "long",
       "nullable": false,
       "metadata": {
         "description": "Length of the WARC record",

src/main/resources/schema/cc-index-schema-nested.json

@@ -310,7 +310,7 @@
           },
           {
             "name": "record_offset",
-            "type": "integer",
+            "type": "long",
             "nullable": false,
             "metadata": {
               "description": "Offset of the WARC record",
@@ -320,7 +320,7 @@
           },
           {
             "name": "record_length",
-            "type": "integer",
+            "type": "long",
             "nullable": false,
             "metadata": {
               "description": "Length of the WARC record",

src/main/resources/schema/index-schema-simple-nested.json

@@ -287,7 +287,7 @@
           },
           {
             "name": "record_offset",
-            "type": "integer",
+            "type": "long",
             "nullable": false,
             "metadata": {
               "description": "Offset of the WARC record",
@@ -297,7 +297,7 @@
           },
           {
             "name": "record_length",
-            "type": "integer",
+            "type": "long",
             "nullable": false,
             "metadata": {
               "description": "Length of the WARC record",

src/main/resources/schema/index-schema-simple.json

@@ -239,7 +239,7 @@
     },
     {
       "name": "warc_record_offset",
-      "type": "integer",
+      "type": "long",
       "nullable": false,
       "metadata": {
         "description": "Offset of the WARC record",
@@ -249,7 +249,7 @@
     },
     {
       "name": "warc_record_length",
-      "type": "integer",
+      "type": "long",
       "nullable": false,
       "metadata": {
         "description": "Length of the WARC record",

src/test/java/org/commoncrawl/spark/TestCCWarcFilenameParser.java

@@ -33,6 +33,24 @@ public void testMainWarcFilename() throws FilenameParseError {
 		assertEquals("warc", parts.subset);
 	}
 
+	@Test
+	public void testSupplementalWarcFilenameCrawlDiagnostics() throws FilenameParseError {
+		String filename = "projects/cc-open-athena-test/CC-SUPPLEMENTAL-2026-22/segments/20260522204839/crawldiagnostics/CC-SUPPLEMENTAL-2026-22-20260522204937-20260523014937-00119.warc.gz";
+		FilenameParts parts = CCWarcFilenameParser.getParts(filename);
+		assertEquals("CC-SUPPLEMENTAL-2026-22", parts.crawl);
+		assertEquals("20260522204839", parts.segment);
+		assertEquals("crawldiagnostics", parts.subset);
+	}
+
+		@Test
+	public void testSupplementalWarcFilenameRobotsTxt() throws FilenameParseError {
+		String filename = "projects/cc-open-athena-test/CC-SUPPLEMENTAL-2026-22/segments/20260522204839/robotstxt/CC-SUPPLEMENTAL-2026-22-20260522204937-20260523014937-00119.warc.gz";
+		FilenameParts parts = CCWarcFilenameParser.getParts(filename);
+		assertEquals("CC-SUPPLEMENTAL-2026-22", parts.crawl);
+		assertEquals("20260522204839", parts.segment);
+		assertEquals("robotstxt", parts.subset);
+	}
+
 	@Test
 	public void testMainWat() throws FilenameParseError {
 		String filename = "crawl-data/CC-MAIN-2018-47/segments/1542039741324.15/wat/CC-MAIN-20181113153141-20181113174452-00011.warc.wat.gz";