Update to latest crawl and disable throttling, seems not necessary currently

centic9 · centic9 · commit 9756d08fa054 · 2025-11-02T09:19:48.000+01:00
diff --git a/src/main/java/org/dstadler/commoncrawl/index/DownloadURLIndex.java b/src/main/java/org/dstadler/commoncrawl/index/DownloadURLIndex.java
@@ -30,7 +30,7 @@ public class DownloadURLIndex {
     private static final Logger log = LoggerFactory.make();
 
     // https://commoncrawl.org/blog/
-    public static final String CURRENT_CRAWL = "CC-MAIN-2024-42";
+    public static final String CURRENT_CRAWL = "CC-MAIN-2025-43";
 	public static final File COMMON_CRAWL_FILE = new File("commoncrawl-" + CURRENT_CRAWL + ".txt");
 
 	private static final int START_INDEX = 0;
@@ -144,7 +144,7 @@ protected static void handleInputStream(String url, InputStream stream, int inde
 							StringUtils.abbreviate(FOUND_MIME_TYPES.sortedMap().toString(), 95));
 					lastLog = System.currentTimeMillis();
 
-					Utils.throttleDownloads();
+					//Utils.throttleDownloads();
 				}
 			}
 		}

Original file line number	Diff line number	Diff line change
`@@ -30,7 +30,7 @@ public class DownloadURLIndex {`
`30`	`30`	`private static final Logger log = LoggerFactory.make();`
`31`	`31`
`32`	`32`	`// https://commoncrawl.org/blog/`
`33`		`- public static final String CURRENT_CRAWL = "CC-MAIN-2024-42";`
	`33`	`+ public static final String CURRENT_CRAWL = "CC-MAIN-2025-43";`
`34`	`34`	`public static final File COMMON_CRAWL_FILE = new File("commoncrawl-" + CURRENT_CRAWL + ".txt");`
`35`	`35`
`36`	`36`	`private static final int START_INDEX = 0;`
`@@ -144,7 +144,7 @@ protected static void handleInputStream(String url, InputStream stream, int inde`
`144`	`144`	`StringUtils.abbreviate(FOUND_MIME_TYPES.sortedMap().toString(), 95));`
`145`	`145`	`lastLog = System.currentTimeMillis();`
`146`	`146`
`147`		`- Utils.throttleDownloads();`
	`147`	`+ //Utils.throttleDownloads();`
`148`	`148`	`}`
`149`	`149`	`}`
`150`	`150`	`}`