diff --git a/core/src/main/java/org/apache/stormcrawler/filtering/host/HostURLFilter.java b/core/src/main/java/org/apache/stormcrawler/filtering/host/HostURLFilter.java index 645b7b6d8..d14541fc0 100644 --- a/core/src/main/java/org/apache/stormcrawler/filtering/host/HostURLFilter.java +++ b/core/src/main/java/org/apache/stormcrawler/filtering/host/HostURLFilter.java @@ -62,7 +62,7 @@ public void configure(@NotNull Map stormConf, @NotNull JsonNode // always the same if (!ignoreOutsideHost) { JsonNode filterByDomainNode = filterParams.get("ignoreOutsideDomain"); - if (filterByHostNode == null) { + if (filterByDomainNode == null) { ignoreOutsideDomain = false; } else { ignoreOutsideDomain = filterByDomainNode.asBoolean(false); diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/HostURLFilterTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/HostURLFilterTest.java index f70ca90b4..c44579037 100644 --- a/core/src/test/java/org/apache/stormcrawler/filtering/HostURLFilterTest.java +++ b/core/src/test/java/org/apache/stormcrawler/filtering/HostURLFilterTest.java @@ -36,10 +36,14 @@ class HostURLFilterTest { private HostURLFilter createFilter(boolean ignoreOutsideHost, boolean ignoreOutsideDomain) { - HostURLFilter filter = new HostURLFilter(); ObjectNode filterParams = new ObjectNode(JsonNodeFactory.instance); filterParams.put("ignoreOutsideHost", Boolean.valueOf(ignoreOutsideHost)); filterParams.put("ignoreOutsideDomain", Boolean.valueOf(ignoreOutsideDomain)); + return createFilter(filterParams); + } + + private HostURLFilter createFilter(ObjectNode filterParams) { + HostURLFilter filter = new HostURLFilter(); Map conf = new HashMap<>(); filter.configure(conf, filterParams); return filter; @@ -108,4 +112,35 @@ void testWithinDomain() throws MalformedURLException { allAllowed.filter(sourceURL, metadata, "http://sub.sourcedomain.com/index.html"); Assertions.assertEquals("http://sub.sourcedomain.com/index.html", filterResult); } + + /** The two modes are independent, so ignoreOutsideDomain must be honoured on its own. */ + @Test + void testWithinDomainWithoutHostParameter() throws MalformedURLException { + ObjectNode filterParams = new ObjectNode(JsonNodeFactory.instance); + filterParams.put("ignoreOutsideDomain", Boolean.TRUE); + HostURLFilter withinDomain = createFilter(filterParams); + URL sourceURL = URLUtil.toURL("http://www.sourcedomain.com/index.html"); + Metadata metadata = new Metadata(); + String filterResult = + withinDomain.filter(sourceURL, metadata, "http://sub.sourcedomain.com/index.html"); + Assertions.assertEquals("http://sub.sourcedomain.com/index.html", filterResult); + filterResult = + withinDomain.filter(sourceURL, metadata, "http://www.anotherDomain.com/index.html"); + Assertions.assertNull(filterResult); + } + + /** + * A configuration which sets ignoreOutsideHost only must not fail on the missing domain key. + */ + @Test + void testHostParameterWithoutDomainParameter() throws MalformedURLException { + ObjectNode filterParams = new ObjectNode(JsonNodeFactory.instance); + filterParams.put("ignoreOutsideHost", Boolean.FALSE); + HostURLFilter allAllowed = createFilter(filterParams); + URL sourceURL = URLUtil.toURL("http://www.sourcedomain.com/index.html"); + Metadata metadata = new Metadata(); + String filterResult = + allAllowed.filter(sourceURL, metadata, "http://www.anotherDomain.com/index.html"); + Assertions.assertEquals("http://www.anotherDomain.com/index.html", filterResult); + } }