diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java b/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java index 2f846f8b2..a24afe62b 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java @@ -21,6 +21,7 @@ import com.google.common.primitives.Ints; import crawlercommons.robots.BaseRobotRules; import java.net.URL; +import java.util.ArrayList; import java.util.HashSet; import java.util.LinkedList; import java.util.List; @@ -44,6 +45,10 @@ public class HttpRobotRulesParser extends RobotRulesParser { protected boolean allow5xx = false; + protected boolean allowCrossOriginRedirects = false; + + protected boolean allowRefusedRedirect = false; + protected Metadata fetchRobotsMd; private static final int MAX_NUM_REDIRECTS = 5; @@ -63,6 +68,36 @@ public void setConf(Config conf) { int robotsTxtContentLimit = ConfUtils.getInt(conf, "http.robots.content.limit", -1); fetchRobotsMd.addValue("http.content.limit", Integer.toString(robotsTxtContentLimit)); allow5xx = ConfUtils.getBoolean(conf, "http.robots.5xx.allow", false); + allowCrossOriginRedirects = + ConfUtils.getBoolean(conf, "http.robots.redirect.crossorigin.allow", false); + allowRefusedRedirect = + ConfUtils.getBoolean(conf, "http.robots.redirect.refused.allow", false); + if (allowCrossOriginRedirects) { + logForwardedRequestHeaders(conf); + } + } + + /** + * Logs which of the configured request headers are sent to the target of a robots.txt redirect + * to another host. The headers are configured per protocol instance and the redirect is + * followed by re-fetching the target through the same instance, so every hop of the chain is + * requested with them. + */ + private static void logForwardedRequestHeaders(Config conf) { + List sources = new ArrayList<>(); + if (StringUtils.isNotBlank(ConfUtils.getString(conf, "http.basicauth.user", null))) { + sources.add("http.basicauth.user"); + } + if (!ConfUtils.loadListFromConf("http.custom.headers", conf).isEmpty()) { + sources.add("http.custom.headers"); + } + if (!sources.isEmpty()) { + LOG.warn( + "http.robots.redirect.crossorigin.allow is set: a robots.txt redirect is fetched " + + "from the host named in the Location header, and the request headers " + + "configured by {} are sent to it as to any other host", + String.join(" and ", sources)); + } } /** Compose unique key to store and access robot rules in cache for given URL. */ @@ -81,6 +116,71 @@ protected static String getCacheKey(URL url) { return protocol + ":" + host + ":" + port; } + /** + * Checks whether a redirect while fetching a robots.txt is followed. The target must use the + * http or https scheme and, unless {@code http.robots.redirect.crossorigin.allow} is set, + * either share scheme, host and port with the URL it was reached from, or be the same host and + * port reached over https instead of http. + * + * @param from URL which returned the redirect + * @param target resolved value of the Location header + * @return true if the target may be fetched + */ + protected boolean followRedirect(URL from, URL target) { + String scheme = target.getProtocol().toLowerCase(Locale.ROOT); + if (!"http".equals(scheme) && !"https".equals(scheme)) { + return false; + } + return allowCrossOriginRedirects + || getCacheKey(from).equals(getCacheKey(target)) + || isSchemeUpgrade(from, target); + } + + /** + * Checks whether a redirect only replaces http with https while staying on the same host and + * port, e.g. {@code http://example.com/robots.txt} to {@code https://example.com/robots.txt}. + * {@link #getCacheKey(URL)} derives the default port from the scheme, so such a target has a + * different key although no other host is involved. It is therefore fetched even when redirects + * to a different origin are not followed. + * + *

Only this direction is exempt. The reverse, https to http, would send the request headers + * configured for the fetch, e.g. the Authorization header built from {@code + * http.basicauth.user}, unencrypted, and would let the rules read over a plain text connection + * replace the ones the redirect was reached from. A site relying on that redirect needs {@code + * http.robots.redirect.crossorigin.allow}. + */ + private static boolean isSchemeUpgrade(URL from, URL target) { + if (!"http".equals(from.getProtocol().toLowerCase(Locale.ROOT)) + || !"https".equals(target.getProtocol().toLowerCase(Locale.ROOT))) { + return false; + } + if (!from.getHost().equalsIgnoreCase(target.getHost())) { + return false; + } + // either the port is unchanged or both sides use the default port of their scheme + return from.getPort() == target.getPort() || (isDefaultPort(from) && isDefaultPort(target)); + } + + /** Checks whether a URL uses the default port of its scheme, explicitly or implicitly. */ + private static boolean isDefaultPort(URL url) { + return url.getPort() == -1 || url.getPort() == url.getDefaultPort(); + } + + /** Describes for logging why the target of a redirect is not fetched. */ + private static String redirectRefusalReason(URL from, URL target) { + String targetScheme = target.getProtocol().toLowerCase(Locale.ROOT); + if (!"http".equals(targetScheme) && !"https".equals(targetScheme)) { + return "on neither http nor https"; + } + if (!from.getHost().equalsIgnoreCase(target.getHost())) { + return "on a different host than " + from; + } + if (!from.getProtocol().toLowerCase(Locale.ROOT).equals(targetScheme)) { + return "on a different scheme than " + from; + } + return "on a different port than " + from; + } + /** * Returns the robots rules from the cache or empty rules if not found. * @@ -123,6 +223,7 @@ public BaseRobotRules getRobotRulesSet(Protocol http, URL url) { } boolean cacheRule = true; + boolean redirectRefused = false; Set redirectCacheKeys = new HashSet<>(); URL robotsUrl = null; @@ -149,7 +250,25 @@ public BaseRobotRules getRobotRulesSet(Protocol http, URL url) { String redirection = response.getMetadata().getFirstValue(HttpHeaders.LOCATION); LOG.debug("Redirected from {} to {}", redir, redirection); if (StringUtils.isNotBlank(redirection)) { - redir = URLUtil.resolveUrl(redir, redirection); + URL target = URLUtil.resolveUrl(redir, redirection); + if (!followRedirect(redir, target)) { + redirectRefused = true; + LOG.warn( + "Robots for {} redirected to {} which is not fetched, the target " + + "is {}. As a result {}. Set " + + "http.robots.redirect.crossorigin.allow to true to follow " + + "such redirects, or " + + "http.robots.redirect.refused.allow to change what " + + "happens when one of them is not followed.", + url, + target, + redirectRefusalReason(redir, target), + allowRefusedRedirect + ? "the host is crawled without any rules" + : "nothing is crawled on that host"); + break; + } + redir = target; if (redir.getPath().equals("/robots.txt") && redir.getQuery() == null) { // only if the path (including the query part) of the redirect target is // `/robots.txt` we can get/put the rules from/to the cache under the host @@ -212,6 +331,10 @@ public BaseRobotRules getRobotRulesSet(Protocol http, URL url) { if (allow5xx) { robotRules = EMPTY_RULES; // allow all } + } else if (redirectRefused) { + // the redirect was not followed, so no rules were obtained for this host + cacheRule = false; + robotRules = allowRefusedRedirect ? EMPTY_RULES : FORBID_ALL_RULES; } else { robotRules = EMPTY_RULES; // allow all } @@ -235,11 +358,19 @@ public BaseRobotRules getRobotRulesSet(Protocol http, URL url) { // cache robot rules for redirections // get here only if the target has not been found in the cache - for (String keyredir : redirectCacheKeys) { - // keyredir isn't null only if the robots.txt file of the target is - // at the root - LOG.debug("Caching robots for {} under key {} in cache {}", redir, keyredir, cacheName); - cacheToUse.put(keyredir, cached); + // a chain ending in a redirect which was not followed produced no rules for the hosts on + // it, so nothing is stored under their keys + if (!redirectRefused) { + for (String keyredir : redirectCacheKeys) { + // keyredir isn't null only if the robots.txt file of the target is + // at the root + LOG.debug( + "Caching robots for {} under key {} in cache {}", + redir, + keyredir, + cacheName); + cacheToUse.put(keyredir, cached); + } } RobotRules live = new RobotRules(robotRules); diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/IPFilterRules.java b/core/src/main/java/org/apache/stormcrawler/protocol/IPFilterRules.java index d9888d25d..2fbb5aecd 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/IPFilterRules.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/IPFilterRules.java @@ -116,6 +116,9 @@ private static List> parseIPRules( case "sitelocal": rules.add(InetAddress::isSiteLocalAddress); break; + case "linklocal": + rules.add(InetAddress::isLinkLocalAddress); + break; default: try { CIDR cidr = new CIDR(ipRule); diff --git a/core/src/main/resources/crawler-default.yaml b/core/src/main/resources/crawler-default.yaml index 0343f644f..692d527d0 100644 --- a/core/src/main/resources/crawler-default.yaml +++ b/core/src/main/resources/crawler-default.yaml @@ -156,10 +156,11 @@ config: # - a CIDR block, e.g. "192.168.0.0/16" or "fd00::/8" # - "localhost" / "loopback" (matches InetAddress.isLoopbackAddress()) # - "sitelocal" (matches InetAddress.isSiteLocalAddress()) + # - "linklocal" (matches InetAddress.isLinkLocalAddress()) # Only addresses matching an include rule are fetched (empty means all are # allowed), addresses matching an exclude rule are always blocked. # http.filter.ipaddress.include: - # http.filter.ipaddress.exclude: "localhost,sitelocal" + # http.filter.ipaddress.exclude: "localhost,sitelocal,linklocal" # Allow all if robots.txt cannot be parsed due to code 403 (Forbidden): http.robots.403.allow: true @@ -167,6 +168,22 @@ config: # Allow all if robots.txt cannot be parsed due to a server error (5xx): http.robots.5xx.allow: false + # Follow a robots.txt redirect whose target is on a different scheme, host or + # port than the URL it was reached from? Redirects to schemes other than http + # and https are never followed. A redirect which only replaces http with https + # while staying on the same host and port is always followed, whatever this is + # set to. When this is enabled, the robots.txt is re-fetched from the host + # named in the Location header, for up to 5 hops, and each of these requests + # carries the headers configured through http.basicauth.* and + # http.custom.headers as any other request does; http.filter.ipaddress.exclude + # can be used to restrict the addresses those fetches may reach. + http.robots.redirect.crossorigin.allow: false + + # Allow all if a robots.txt redirect was not followed because of the setting + # above? If false, nothing is crawled on that host until the redirect is + # followed or this is set to true. + http.robots.redirect.refused.allow: false + # ignore directives from robots.txt files? http.robots.file.skip: false diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserRedirectTargetTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserRedirectTargetTest.java new file mode 100644 index 000000000..ba1ac9092 --- /dev/null +++ b/core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserRedirectTargetTest.java @@ -0,0 +1,295 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.stormcrawler.protocol; + +import crawlercommons.robots.BaseRobotRules; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.storm.Config; +import org.apache.stormcrawler.Metadata; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** Checks which URLs are fetched when a robots.txt responds with a redirect. */ +class HttpRobotRulesParserRedirectTargetTest { + + private static final String RULES = "User-agent: this_is_only_a_test\nDisallow: /restricted/"; + + /** Protocol stub recording the URLs it is asked to fetch. */ + private static class RecordingProtocol implements Protocol { + + final List requested = new ArrayList<>(); + + final Map responses = new HashMap<>(); + + void redirect(String from, String to) { + Metadata md = new Metadata(); + md.setValue("location", to); + responses.put(from, new ProtocolResponse(new byte[0], 301, md)); + } + + void rules(String url) { + Metadata md = new Metadata(); + md.setValue("content-type", "text/plain"); + responses.put( + url, new ProtocolResponse(RULES.getBytes(StandardCharsets.UTF_8), 200, md)); + } + + @Override + public void configure(Config conf) {} + + @Override + public ProtocolResponse getProtocolOutput(String url, Metadata metadata) { + requested.add(url); + ProtocolResponse response = responses.get(url); + if (response == null) { + return new ProtocolResponse(new byte[0], 404, new Metadata()); + } + return response; + } + + @Override + public BaseRobotRules getRobotRules(String url) { + return null; + } + + @Override + public void cleanup() {} + } + + private static Config conf() { + Config conf = new Config(); + conf.put("http.agent.name", "this_is_only_a_test"); + return conf; + } + + private static HttpRobotRulesParser parser(Config conf) { + HttpRobotRulesParser parser = new HttpRobotRulesParser(); + parser.setConf(conf); + return parser; + } + + @Test + void redirectToSameHostIsFollowed() { + RecordingProtocol protocol = new RecordingProtocol(); + protocol.redirect("http://same.example.org/robots.txt", "/robots/rules.txt"); + protocol.rules("http://same.example.org/robots/rules.txt"); + BaseRobotRules rules = + parser(conf()).getRobotRulesSet(protocol, "http://same.example.org/"); + Assertions.assertTrue( + protocol.requested.contains("http://same.example.org/robots/rules.txt"), + "expected the redirect target to be fetched, requested: " + protocol.requested); + Assertions.assertTrue(rules.isAllowed("http://same.example.org/index.html")); + Assertions.assertFalse(rules.isAllowed("http://same.example.org/restricted/index.html")); + } + + @Test + void redirectToOtherHostIsNotFollowed() { + RecordingProtocol protocol = new RecordingProtocol(); + protocol.redirect( + "http://host.example.org/robots.txt", "http://elsewhere.example.org/robots.txt"); + protocol.rules("http://elsewhere.example.org/robots.txt"); + BaseRobotRules rules = + parser(conf()).getRobotRulesSet(protocol, "http://host.example.org/"); + Assertions.assertFalse( + protocol.requested.contains("http://elsewhere.example.org/robots.txt"), + "robots.txt of another host should not be fetched, requested: " + + protocol.requested); + // no rules obtained, so nothing is crawled on that host + Assertions.assertTrue(rules.isAllowNone()); + } + + @Test + void redirectToOtherPortIsNotFollowed() { + RecordingProtocol protocol = new RecordingProtocol(); + protocol.redirect( + "http://port.example.org/robots.txt", "http://port.example.org:8080/robots.txt"); + protocol.rules("http://port.example.org:8080/robots.txt"); + BaseRobotRules rules = + parser(conf()).getRobotRulesSet(protocol, "http://port.example.org/"); + Assertions.assertFalse( + protocol.requested.contains("http://port.example.org:8080/robots.txt"), + "robots.txt on another port should not be fetched, requested: " + + protocol.requested); + Assertions.assertTrue(rules.isAllowNone()); + } + + @Test + void redirectToOtherSchemeIsNotFollowed() { + RecordingProtocol protocol = new RecordingProtocol(); + protocol.redirect("http://scheme.example.org/robots.txt", "file:/tmp/robots.txt"); + parser(conf()).getRobotRulesSet(protocol, "http://scheme.example.org/"); + Assertions.assertFalse( + protocol.requested.contains("file:/tmp/robots.txt"), + "robots.txt fetch should stay on http(s), requested: " + protocol.requested); + } + + @Test + void redirectToOtherHostIsFollowedIfConfigured() { + RecordingProtocol protocol = new RecordingProtocol(); + protocol.redirect("http://cdn.example.org/robots.txt", "http://cdn.example.com/robots.txt"); + protocol.rules("http://cdn.example.com/robots.txt"); + Config conf = conf(); + conf.put("http.robots.redirect.crossorigin.allow", true); + BaseRobotRules rules = parser(conf).getRobotRulesSet(protocol, "http://cdn.example.org/"); + Assertions.assertTrue( + protocol.requested.contains("http://cdn.example.com/robots.txt"), + "expected the redirect target to be fetched, requested: " + protocol.requested); + Assertions.assertFalse(rules.isAllowed("http://cdn.example.org/restricted/index.html")); + } + + @Test + void schemeUpgradeOnSameHostIsFollowed() { + RecordingProtocol protocol = new RecordingProtocol(); + protocol.redirect( + "http://upgrade.example.org/robots.txt", "https://upgrade.example.org/robots.txt"); + protocol.rules("https://upgrade.example.org/robots.txt"); + BaseRobotRules rules = + parser(conf()).getRobotRulesSet(protocol, "http://upgrade.example.org/"); + Assertions.assertTrue( + protocol.requested.contains("https://upgrade.example.org/robots.txt"), + "expected the redirect target to be fetched, requested: " + protocol.requested); + Assertions.assertFalse(rules.isAllowed("http://upgrade.example.org/restricted/index.html")); + } + + @Test + void schemeUpgradeWithExplicitDefaultPortIsFollowed() { + RecordingProtocol protocol = new RecordingProtocol(); + protocol.redirect( + "http://upgrade.example.org/robots.txt", + "https://upgrade.example.org:443/robots.txt"); + protocol.rules("https://upgrade.example.org:443/robots.txt"); + BaseRobotRules rules = + parser(conf()).getRobotRulesSet(protocol, "http://upgrade.example.org/"); + Assertions.assertTrue( + protocol.requested.contains("https://upgrade.example.org:443/robots.txt"), + "expected the redirect target to be fetched, requested: " + protocol.requested); + Assertions.assertFalse(rules.isAllowed("http://upgrade.example.org/restricted/index.html")); + } + + @Test + void schemeUpgradeKeepingExplicitPortIsFollowed() { + RecordingProtocol protocol = new RecordingProtocol(); + protocol.redirect( + "http://upgrade.example.org:8080/robots.txt", + "https://upgrade.example.org:8080/robots.txt"); + protocol.rules("https://upgrade.example.org:8080/robots.txt"); + BaseRobotRules rules = + parser(conf()).getRobotRulesSet(protocol, "http://upgrade.example.org:8080/"); + Assertions.assertTrue( + protocol.requested.contains("https://upgrade.example.org:8080/robots.txt"), + "expected the redirect target to be fetched, requested: " + protocol.requested); + Assertions.assertFalse( + rules.isAllowed("http://upgrade.example.org:8080/restricted/index.html")); + } + + @Test + void schemeDowngradeOnSameHostIsNotFollowed() { + RecordingProtocol protocol = new RecordingProtocol(); + protocol.redirect( + "https://downgrade.example.org/robots.txt", + "http://downgrade.example.org/robots.txt"); + protocol.rules("http://downgrade.example.org/robots.txt"); + BaseRobotRules rules = + parser(conf()).getRobotRulesSet(protocol, "https://downgrade.example.org/"); + Assertions.assertFalse( + protocol.requested.contains("http://downgrade.example.org/robots.txt"), + "robots.txt should not be fetched over http after https, requested: " + + protocol.requested); + Assertions.assertTrue(rules.isAllowNone()); + } + + @Test + void schemeUpgradeToOtherHostIsNotFollowed() { + RecordingProtocol protocol = new RecordingProtocol(); + protocol.redirect( + "http://one.example.org/robots.txt", "https://other.example.org/robots.txt"); + protocol.rules("https://other.example.org/robots.txt"); + parser(conf()).getRobotRulesSet(protocol, "http://one.example.org/"); + Assertions.assertFalse( + protocol.requested.contains("https://other.example.org/robots.txt"), + "robots.txt of another host should not be fetched, requested: " + + protocol.requested); + } + + @Test + void schemeUpgradeToOtherPortIsNotFollowed() { + RecordingProtocol protocol = new RecordingProtocol(); + protocol.redirect( + "http://upgrade.example.org:8080/robots.txt", + "https://upgrade.example.org:9443/robots.txt"); + protocol.rules("https://upgrade.example.org:9443/robots.txt"); + parser(conf()).getRobotRulesSet(protocol, "http://upgrade.example.org:8080/"); + Assertions.assertFalse( + protocol.requested.contains("https://upgrade.example.org:9443/robots.txt"), + "robots.txt on another port should not be fetched, requested: " + + protocol.requested); + } + + @Test + void redirectWhichIsNotFollowedIsNotCachedAsSuccess() throws MalformedURLException { + RecordingProtocol protocol = new RecordingProtocol(); + protocol.redirect( + "http://cached.example.org/robots.txt", "http://elsewhere.example.org/robots.txt"); + HttpRobotRulesParser parser = parser(conf()); + BaseRobotRules rules = parser.getRobotRulesSet(protocol, "http://cached.example.org/"); + Assertions.assertTrue(rules.isAllowNone()); + // the cache of successfully fetched rules, which the robots URL filter reads, is untouched + Assertions.assertTrue( + parser.getRobotRulesSetFromCache(new URL("http://cached.example.org/")) + .isAllowAll()); + } + + @Test + void redirectWhichIsNotFollowedAllowsCrawlingIfConfigured() { + RecordingProtocol protocol = new RecordingProtocol(); + protocol.redirect( + "http://lenient.example.org/robots.txt", "http://elsewhere.example.org/robots.txt"); + Config conf = conf(); + conf.put("http.robots.redirect.refused.allow", true); + BaseRobotRules rules = + parser(conf).getRobotRulesSet(protocol, "http://lenient.example.org/"); + Assertions.assertFalse( + protocol.requested.contains("http://elsewhere.example.org/robots.txt"), + "robots.txt of another host should not be fetched, requested: " + + protocol.requested); + Assertions.assertTrue(rules.isAllowAll()); + } + + @Test + void redirectWhichIsNotFollowedDoesNotApplyToTheHostsOnTheChain() { + RecordingProtocol protocol = new RecordingProtocol(); + protocol.redirect( + "http://first.example.org/robots.txt", "http://second.example.org/robots.txt"); + protocol.redirect("http://second.example.org/robots.txt", "file:/tmp/robots.txt"); + Config conf = conf(); + conf.put("http.robots.redirect.crossorigin.allow", true); + HttpRobotRulesParser parser = parser(conf); + Assertions.assertTrue( + parser.getRobotRulesSet(protocol, "http://first.example.org/").isAllowNone()); + // the second host is asked again instead of inheriting the outcome of the chain above + protocol.rules("http://second.example.org/robots.txt"); + BaseRobotRules rules = parser.getRobotRulesSet(protocol, "http://second.example.org/"); + Assertions.assertFalse(rules.isAllowed("http://second.example.org/restricted/index.html")); + } +} diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserRedirectTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserRedirectTest.java index d3e1eccf0..034489d65 100644 --- a/core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserRedirectTest.java +++ b/core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserRedirectTest.java @@ -20,6 +20,8 @@ import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; import static com.github.tomakehurst.wiremock.client.WireMock.configureFor; import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.matching; import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; @@ -29,6 +31,7 @@ import com.github.tomakehurst.wiremock.core.WireMockConfiguration; import crawlercommons.robots.BaseRobotRules; import org.apache.storm.Config; +import org.apache.stormcrawler.protocol.okhttp.HttpProtocol; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -44,6 +47,7 @@ class HttpRobotRulesParserRedirectTest { private String body; + private String url0 = "http://localhost:" + ports[0]; private String url1 = "http://localhost:" + ports[1]; private String url7 = "http://localhost:" + ports[7]; private String url8 = "http://localhost:" + ports[8]; @@ -106,6 +110,10 @@ void tearDown() { @Test void testRedirects() { + // the redirect chains tested here point at other ports of the same host + Config crossOriginConf = new Config(); + crossOriginConf.putAll(conf); + crossOriginConf.put("http.robots.redirect.crossorigin.allow", true); // Test for 5 consecutive redirects configureFor(mockServer1.getClient()); stubFor( @@ -162,7 +170,7 @@ void testRedirects() { get(urlPathEqualTo("/robots.txt")) .willReturn(aResponse().withBody(body).withStatus(200))); HttpRobotRulesParser httpRobotRulesParser = new HttpRobotRulesParser(); - httpRobotRulesParser.setConf(conf); + httpRobotRulesParser.setConf(crossOriginConf); BaseRobotRules robotRules = httpRobotRulesParser.getRobotRulesSet(protocol, url1); Assertions.assertFalse(robotRules.isAllowAll()); Assertions.assertFalse(robotRules.isAllowNone()); @@ -178,7 +186,7 @@ void testRedirects() { .withBody(body) .withStatus(301))); httpRobotRulesParser = new HttpRobotRulesParser(); - httpRobotRulesParser.setConf(conf); + httpRobotRulesParser.setConf(crossOriginConf); robotRules = httpRobotRulesParser.getRobotRulesSet(protocol, url1); Assertions.assertTrue(robotRules.isAllowAll()); // Test relative redirects @@ -212,7 +220,7 @@ void testRedirects() { .withStatus(302))); // from here the redirect leads to the same robots.txt as in the first test block httpRobotRulesParser = new HttpRobotRulesParser(); - httpRobotRulesParser.setConf(conf); + httpRobotRulesParser.setConf(crossOriginConf); robotRules = httpRobotRulesParser.getRobotRulesSet(protocol, url7); Assertions.assertFalse(robotRules.isAllowAll()); Assertions.assertFalse(robotRules.isAllowNone()); @@ -233,6 +241,82 @@ void testRedirects() { Assertions.assertFalse(robotRules.isAllowed(url1 + "/restricted/index.html")); } + @Test + void testRedirectOnTheSameAuthorityIsFollowedByDefault() { + configureFor(mockServer0.getClient()); + stubFor( + get(urlPathEqualTo("/robots.txt")) + .willReturn( + aResponse() + .withHeader("location", "/robots/rules.txt") + .withStatus(301))); + stubFor( + get(urlPathEqualTo("/robots/rules.txt")) + .willReturn(aResponse().withBody(body).withStatus(200))); + HttpRobotRulesParser httpRobotRulesParser = new HttpRobotRulesParser(); + httpRobotRulesParser.setConf(conf); + BaseRobotRules robotRules = httpRobotRulesParser.getRobotRulesSet(protocol, url0); + Assertions.assertFalse(robotRules.isAllowAll()); + Assertions.assertFalse(robotRules.isAllowNone()); + Assertions.assertTrue(robotRules.isAllowed(url0 + "/index.html")); + Assertions.assertFalse(robotRules.isAllowed(url0 + "/restricted/index.html")); + } + + @Test + void testRedirectToAnotherAuthorityIsNotFollowedByDefault() { + configureFor(mockServer0.getClient()); + stubFor( + get(urlPathEqualTo("/robots.txt")) + .willReturn( + aResponse() + .withHeader("location", url1 + "/robots.txt") + .withStatus(301))); + configureFor(mockServer1.getClient()); + stubFor( + get(urlPathEqualTo("/robots.txt")) + .willReturn(aResponse().withBody(body).withStatus(200))); + HttpRobotRulesParser httpRobotRulesParser = new HttpRobotRulesParser(); + httpRobotRulesParser.setConf(conf); + BaseRobotRules robotRules = httpRobotRulesParser.getRobotRulesSet(protocol, url0); + mockServer1.verify(0, getRequestedFor(urlPathEqualTo("/robots.txt"))); + Assertions.assertTrue(robotRules.isAllowNone()); + } + + @Test + void testHeadersNotSentToAnUnfollowedRedirectTarget() { + Config authConf = new Config(); + authConf.putAll(conf); + authConf.put("http.basicauth.user", "this_is_only_a_test"); + authConf.put("http.basicauth.password", "this_is_only_a_test"); + HttpProtocol authProtocol = new HttpProtocol(); + authProtocol.configure(authConf); + try { + configureFor(mockServer0.getClient()); + stubFor( + get(urlPathEqualTo("/robots.txt")) + .willReturn( + aResponse() + .withHeader("location", url1 + "/robots.txt") + .withStatus(301))); + configureFor(mockServer1.getClient()); + stubFor( + get(urlPathEqualTo("/robots.txt")) + .willReturn(aResponse().withBody(body).withStatus(200))); + HttpRobotRulesParser httpRobotRulesParser = new HttpRobotRulesParser(); + httpRobotRulesParser.setConf(authConf); + httpRobotRulesParser.getRobotRulesSet(authProtocol, url0); + // the configured Authorization header is sent to the host the robots.txt belongs to + mockServer0.verify( + 1, + getRequestedFor(urlPathEqualTo("/robots.txt")) + .withHeader("Authorization", matching("Basic .+"))); + // the target of the redirect is on another port and is not contacted at all + mockServer1.verify(0, getRequestedFor(urlPathEqualTo("/robots.txt"))); + } finally { + authProtocol.cleanup(); + } + } + private static class MockServer extends WireMockServer { public MockServer(Options options) { super(options); diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/IPFilterRulesTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/IPFilterRulesTest.java index 109529ce0..fa42d190f 100644 --- a/core/src/test/java/org/apache/stormcrawler/protocol/IPFilterRulesTest.java +++ b/core/src/test/java/org/apache/stormcrawler/protocol/IPFilterRulesTest.java @@ -63,6 +63,23 @@ void excludeLoopbackAndSitelocalAsCommaSeparatedString() { assertTrue(r.accept(ip("8.8.8.8"))); } + @Test + void excludeLinklocal() { + IPFilterRules r = rules(null, "linklocal"); + assertFalse(r.isEmpty()); + assertFalse(r.accept(ip("169.254.169.254"))); + assertFalse(r.accept(ip("fe80::1"))); + assertTrue(r.accept(ip("8.8.8.8"))); + } + + @Test + void linklocalIsCoveredByNeitherLoopbackNorSitelocal() { + IPFilterRules r = rules(null, "localhost,sitelocal"); + assertTrue( + r.accept(ip("169.254.169.254")), + "a link-local address needs the linklocal rule of its own"); + } + @Test void excludeAsYamlList() { IPFilterRules r = rules(null, Arrays.asList("loopback", "192.168.0.0/16")); diff --git a/docs/src/main/asciidoc/configuration.adoc b/docs/src/main/asciidoc/configuration.adoc index 7aaa0ccf7..b2c241959 100644 --- a/docs/src/main/asciidoc/configuration.adoc +++ b/docs/src/main/asciidoc/configuration.adoc @@ -220,6 +220,8 @@ implementation. | http.robots.file.skip | false | Ignore robots.txt rules entirely. | http.robots.headers.skip | false | Ignore robots directives from HTTP headers. | http.robots.meta.skip | false | Ignore robots directives from HTML meta tags. +| http.robots.redirect.crossorigin.allow | false | Follow a robots.txt redirect pointing at a different scheme, host or port than the URL it was reached from. Redirects to schemes other than http and https are never followed. A redirect which only replaces http with https while staying on the same host and port is always followed, whatever this is set to. When enabled, the robots.txt is re-fetched from the host named in the `Location` header, for up to 5 hops, and each of these requests carries the headers configured through `http.basicauth.*` and `http.custom.headers` as any other request does; see `http.filter.ipaddress.exclude` to restrict the addresses those fetches may reach. +| http.robots.redirect.refused.allow | false | Allow crawling when a robots.txt redirect was not followed because of `http.robots.redirect.crossorigin.allow`. If false, nothing is crawled on that host until the redirect is followed or this is set to true. | http.skip.robots | false | Deprecated (replaced by http.robots.file.skip). | robots.noFollow.strict | true | If true, remove all outlinks from pages marked as noFollow. | http.store.headers | false | Whether to store response headers. @@ -283,14 +285,20 @@ Each property accepts a comma-separated string or a YAML list of rules. A rule c * a CIDR block, e.g. `192.168.0.0/16` or `fd00::/8` * `localhost` or `loopback` — matches any loopback address (`InetAddress.isLoopbackAddress()`) * `sitelocal` — matches any site-local address (`InetAddress.isSiteLocalAddress()`) +* `linklocal` — matches any link-local address (`InetAddress.isLinkLocalAddress()`), i.e. `169.254.0.0/16` +and `fe80::/10` For example, to block crawling of localhost and private address spaces: [source,yaml] ---- -http.filter.ipaddress.exclude: "localhost,sitelocal" +http.filter.ipaddress.exclude: "localhost,sitelocal,linklocal" ---- +`sitelocal` follows the JDK and covers `10.0.0.0/8`, `172.16.0.0/12` and `192.168.0.0/16` only. Carrier +grade NAT (`100.64.0.0/10`) and IPv6 unique local addresses (`fd00::/8`) have no keyword and are added +as CIDR blocks when they are also to be blocked. + When a connection to a blocked address is attempted, the fetch fails with an `IOException` and a warning is logged. If neither property is set, no IP filtering is performed.