Skip to content

NUTCH-3164 Catch specific exceptions in CrawlDbFilter so plugin errors no longer silently drop URLs #918

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
prakharchaube wants to merge 1 commit into
base: master
Choose a base branch
from
+29 −6
Open

NUTCH-3164 Catch specific exceptions in CrawlDbFilter so plugin errors no longer silently drop URLs #918

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 29 additions & 6 deletions src/java/org/apache/nutch/crawl/CrawlDbFilter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,17 @@

import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.net.MalformedURLException;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Counter;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.metrics.ErrorTracker;
import org.apache.nutch.metrics.NutchMetrics;
import org.apache.nutch.net.URLFilterException;
import org.apache.nutch.net.URLFilters;
import org.apache.nutch.net.URLNormalizers;

Expand Down Expand Up @@ -56,6 +59,8 @@ public class CrawlDbFilter extends
private Counter orphanRecordsRemovedCounter;
private Counter urlsFilteredCounter;

private ErrorTracker errorTracker;

private static final Logger LOG = LoggerFactory
.getLogger(MethodHandles.lookup().lookupClass());

Expand All @@ -77,6 +82,9 @@ public void setup(Mapper<Text, CrawlDatum, Text, CrawlDatum>.Context context) {

// Initialize cached counter references
initCounters(context);

// Initialize error tracker with cached counters (NUTCH-3164)
errorTracker = new ErrorTracker(NutchMetrics.GROUP_CRAWLDB_FILTER, context);
}

/**
Expand Down Expand Up @@ -114,17 +122,32 @@ public void map(Text key, CrawlDatum value,
if (url != null && urlNormalizers) {
try {
url = normalizers.normalize(url, scope); // normalize the url
} catch (Exception e) {
LOG.warn("Skipping {}: ", url, e);
url = null;
} catch (MalformedURLException e) {
// NUTCH-3164: malformed URL is a legitimate reason to drop; tracked via
// ErrorTracker, not urlsFilteredCounter (which conflates filtering with
// malformed input).
LOG.error("Skipping malformed URL {}: {}", url, e.getMessage());
errorTracker.incrementCounters(e);
return;
} catch (RuntimeException e) {
// NUTCH-3164: a normalizer plugin bug must not silently delete URLs.
LOG.error("Unexpected exception normalizing {}, keeping URL: ", url, e);
errorTracker.incrementCounters(e);
}
}
if (url != null && urlFiltering) {
try {
url = filters.filter(url); // filter the url
} catch (Exception e) {
LOG.warn("Skipping {}: ", url, e);
url = null;
} catch (URLFilterException e) {
// NUTCH-3164: URLFilterException signals an internal filter failure,
// not URL rejection (rejection is communicated by returning null).
// Track via ErrorTracker; do not drop the URL.
LOG.error("Filter error for {}, keeping URL: {}", url, e.getMessage());
errorTracker.incrementCounters(e);
} catch (RuntimeException e) {
// NUTCH-3164: a filter plugin bug must not silently delete URLs.
LOG.error("Unexpected exception filtering {}, keeping URL: ", url, e);
errorTracker.incrementCounters(e);
}
}
if (url == null) {
Expand Down