From dbf0bd1525bce5b0c636e47769fb2570831a6921 Mon Sep 17 00:00:00 2001 From: Richard Zowalla <13417392+rzo1@users.noreply.github.com> Date: Thu, 27 Aug 2026 14:20:10 +0200 Subject: [PATCH] Only follow robots.txt redirects within the same origin HttpRobotRulesParser followed the Location header of a robots.txt response for up to 5 hops, honouring an absolute target verbatim and re-fetching it with no restriction, so the crawled site chose which host was requested. A redirect is now only followed when the target shares scheme, host and port with the URL it was reached from, unless http.robots.redirect.crossorigin.allow is set. A redirect which only replaces http with https while staying on the same host and port is always followed. getCacheKey derives the default port from the scheme, so such a target has a different key although no other host is involved, and refusing it left the very common apex redirect without any rules. Only that direction is exempt: https to http would send the headers configured for the fetch unencrypted and would let rules read over a plain text connection replace the ones the redirect was reached from. A redirect which is not followed no longer ends as allow-all in the success cache. It yields no rules for the host, cached in the error cache like a 429 or a 5xx, so nothing is crawled there until the redirect is resolved. http.robots.redirect.refused.allow: true restores the previous behaviour and crawls the host without any rules. The refusal is logged at warn level, naming the target, why it was not followed and which setting changes it. Following a redirect to another origin re-fetches the robots.txt with the headers configured through http.basicauth.* and http.custom.headers, as any other request does. That is now stated where the setting is defined and logged once at configuration time when either is set, and http.filter.ipaddress.exclude gained a linklocal rule so that the addresses those fetches may reach can be restricted: link local addresses are matched by neither loopback nor sitelocal, so the documented example did not cover them. --- .../protocol/HttpRobotRulesParser.java | 143 ++++++++- .../stormcrawler/protocol/IPFilterRules.java | 3 + core/src/main/resources/crawler-default.yaml | 19 +- ...ttpRobotRulesParserRedirectTargetTest.java | 295 ++++++++++++++++++ .../HttpRobotRulesParserRedirectTest.java | 90 +++++- .../protocol/IPFilterRulesTest.java | 17 + docs/src/main/asciidoc/configuration.adoc | 10 +- 7 files changed, 566 insertions(+), 11 deletions(-) create mode 100644 core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserRedirectTargetTest.java diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java b/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java index 2f846f8b2..a24afe62b 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java @@ -21,6 +21,7 @@ import com.google.common.primitives.Ints; import crawlercommons.robots.BaseRobotRules; import java.net.URL; +import java.util.ArrayList; import java.util.HashSet; import java.util.LinkedList; import java.util.List; @@ -44,6 +45,10 @@ public class HttpRobotRulesParser extends RobotRulesParser { protected boolean allow5xx = false; + protected boolean allowCrossOriginRedirects = false; + + protected boolean allowRefusedRedirect = false; + protected Metadata fetchRobotsMd; private static final int MAX_NUM_REDIRECTS = 5; @@ -63,6 +68,36 @@ public void setConf(Config conf) { int robotsTxtContentLimit = ConfUtils.getInt(conf, "http.robots.content.limit", -1); fetchRobotsMd.addValue("http.content.limit", Integer.toString(robotsTxtContentLimit)); allow5xx = ConfUtils.getBoolean(conf, "http.robots.5xx.allow", false); + allowCrossOriginRedirects = + ConfUtils.getBoolean(conf, "http.robots.redirect.crossorigin.allow", false); + allowRefusedRedirect = + ConfUtils.getBoolean(conf, "http.robots.redirect.refused.allow", false); + if (allowCrossOriginRedirects) { + logForwardedRequestHeaders(conf); + } + } + + /** + * Logs which of the configured request headers are sent to the target of a robots.txt redirect + * to another host. The headers are configured per protocol instance and the redirect is + * followed by re-fetching the target through the same instance, so every hop of the chain is + * requested with them. + */ + private static void logForwardedRequestHeaders(Config conf) { + List sources = new ArrayList<>(); + if (StringUtils.isNotBlank(ConfUtils.getString(conf, "http.basicauth.user", null))) { + sources.add("http.basicauth.user"); + } + if (!ConfUtils.loadListFromConf("http.custom.headers", conf).isEmpty()) { + sources.add("http.custom.headers"); + } + if (!sources.isEmpty()) { + LOG.warn( + "http.robots.redirect.crossorigin.allow is set: a robots.txt redirect is fetched " + + "from the host named in the Location header, and the request headers " + + "configured by {} are sent to it as to any other host", + String.join(" and ", sources)); + } } /** Compose unique key to store and access robot rules in cache for given URL. */ @@ -81,6 +116,71 @@ protected static String getCacheKey(URL url) { return protocol + ":" + host + ":" + port; } + /** + * Checks whether a redirect while fetching a robots.txt is followed. The target must use the + * http or https scheme and, unless {@code http.robots.redirect.crossorigin.allow} is set, + * either share scheme, host and port with the URL it was reached from, or be the same host and + * port reached over https instead of http. + * + * @param from URL which returned the redirect + * @param target resolved value of the Location header + * @return true if the target may be fetched + */ + protected boolean followRedirect(URL from, URL target) { + String scheme = target.getProtocol().toLowerCase(Locale.ROOT); + if (!"http".equals(scheme) && !"https".equals(scheme)) { + return false; + } + return allowCrossOriginRedirects + || getCacheKey(from).equals(getCacheKey(target)) + || isSchemeUpgrade(from, target); + } + + /** + * Checks whether a redirect only replaces http with https while staying on the same host and + * port, e.g. {@code http://example.com/robots.txt} to {@code https://example.com/robots.txt}. + * {@link #getCacheKey(URL)} derives the default port from the scheme, so such a target has a + * different key although no other host is involved. It is therefore fetched even when redirects + * to a different origin are not followed. + * + *

Only this direction is exempt. The reverse, https to http, would send the request headers + * configured for the fetch, e.g. the Authorization header built from {@code + * http.basicauth.user}, unencrypted, and would let the rules read over a plain text connection + * replace the ones the redirect was reached from. A site relying on that redirect needs {@code + * http.robots.redirect.crossorigin.allow}. + */ + private static boolean isSchemeUpgrade(URL from, URL target) { + if (!"http".equals(from.getProtocol().toLowerCase(Locale.ROOT)) + || !"https".equals(target.getProtocol().toLowerCase(Locale.ROOT))) { + return false; + } + if (!from.getHost().equalsIgnoreCase(target.getHost())) { + return false; + } + // either the port is unchanged or both sides use the default port of their scheme + return from.getPort() == target.getPort() || (isDefaultPort(from) && isDefaultPort(target)); + } + + /** Checks whether a URL uses the default port of its scheme, explicitly or implicitly. */ + private static boolean isDefaultPort(URL url) { + return url.getPort() == -1 || url.getPort() == url.getDefaultPort(); + } + + /** Describes for logging why the target of a redirect is not fetched. */ + private static String redirectRefusalReason(URL from, URL target) { + String targetScheme = target.getProtocol().toLowerCase(Locale.ROOT); + if (!"http".equals(targetScheme) && !"https".equals(targetScheme)) { + return "on neither http nor https"; + } + if (!from.getHost().equalsIgnoreCase(target.getHost())) { + return "on a different host than " + from; + } + if (!from.getProtocol().toLowerCase(Locale.ROOT).equals(targetScheme)) { + return "on a different scheme than " + from; + } + return "on a different port than " + from; + } + /** * Returns the robots rules from the cache or empty rules if not found. * @@ -123,6 +223,7 @@ public BaseRobotRules getRobotRulesSet(Protocol http, URL url) { } boolean cacheRule = true; + boolean redirectRefused = false; Set redirectCacheKeys = new HashSet<>(); URL robotsUrl = null; @@ -149,7 +250,25 @@ public BaseRobotRules getRobotRulesSet(Protocol http, URL url) { String redirection = response.getMetadata().getFirstValue(HttpHeaders.LOCATION); LOG.debug("Redirected from {} to {}", redir, redirection); if (StringUtils.isNotBlank(redirection)) { - redir = URLUtil.resolveUrl(redir, redirection); + URL target = URLUtil.resolveUrl(redir, redirection); + if (!followRedirect(redir, target)) { + redirectRefused = true; + LOG.warn( + "Robots for {} redirected to {} which is not fetched, the target " + + "is {}. As a result {}. Set " + + "http.robots.redirect.crossorigin.allow to true to follow " + + "such redirects, or " + + "http.robots.redirect.refused.allow to change what " + + "happens when one of them is not followed.", + url, + target, + redirectRefusalReason(redir, target), + allowRefusedRedirect + ? "the host is crawled without any rules" + : "nothing is crawled on that host"); + break; + } + redir = target; if (redir.getPath().equals("/robots.txt") && redir.getQuery() == null) { // only if the path (including the query part) of the redirect target is // `/robots.txt` we can get/put the rules from/to the cache under the host @@ -212,6 +331,10 @@ public BaseRobotRules getRobotRulesSet(Protocol http, URL url) { if (allow5xx) { robotRules = EMPTY_RULES; // allow all } + } else if (redirectRefused) { + // the redirect was not followed, so no rules were obtained for this host + cacheRule = false; + robotRules = allowRefusedRedirect ? EMPTY_RULES : FORBID_ALL_RULES; } else { robotRules = EMPTY_RULES; // allow all } @@ -235,11 +358,19 @@ public BaseRobotRules getRobotRulesSet(Protocol http, URL url) { // cache robot rules for redirections // get here only if the target has not been found in the cache - for (String keyredir : redirectCacheKeys) { - // keyredir isn't null only if the robots.txt file of the target is - // at the root - LOG.debug("Caching robots for {} under key {} in cache {}", redir, keyredir, cacheName); - cacheToUse.put(keyredir, cached); + // a chain ending in a redirect which was not followed produced no rules for the hosts on + // it, so nothing is stored under their keys + if (!redirectRefused) { + for (String keyredir : redirectCacheKeys) { + // keyredir isn't null only if the robots.txt file of the target is + // at the root + LOG.debug( + "Caching robots for {} under key {} in cache {}", + redir, + keyredir, + cacheName); + cacheToUse.put(keyredir, cached); + } } RobotRules live = new RobotRules(robotRules); diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/IPFilterRules.java b/core/src/main/java/org/apache/stormcrawler/protocol/IPFilterRules.java index d9888d25d..2fbb5aecd 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/IPFilterRules.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/IPFilterRules.java @@ -116,6 +116,9 @@ private static List> parseIPRules( case "sitelocal": rules.add(InetAddress::isSiteLocalAddress); break; + case "linklocal": + rules.add(InetAddress::isLinkLocalAddress); + break; default: try { CIDR cidr = new CIDR(ipRule); diff --git a/core/src/main/resources/crawler-default.yaml b/core/src/main/resources/crawler-default.yaml index 0343f644f..692d527d0 100644 --- a/core/src/main/resources/crawler-default.yaml +++ b/core/src/main/resources/crawler-default.yaml @@ -156,10 +156,11 @@ config: # - a CIDR block, e.g. "192.168.0.0/16" or "fd00::/8" # - "localhost" / "loopback" (matches InetAddress.isLoopbackAddress()) # - "sitelocal" (matches InetAddress.isSiteLocalAddress()) + # - "linklocal" (matches InetAddress.isLinkLocalAddress()) # Only addresses matching an include rule are fetched (empty means all are # allowed), addresses matching an exclude rule are always blocked. # http.filter.ipaddress.include: - # http.filter.ipaddress.exclude: "localhost,sitelocal" + # http.filter.ipaddress.exclude: "localhost,sitelocal,linklocal" # Allow all if robots.txt cannot be parsed due to code 403 (Forbidden): http.robots.403.allow: true @@ -167,6 +168,22 @@ config: # Allow all if robots.txt cannot be parsed due to a server error (5xx): http.robots.5xx.allow: false + # Follow a robots.txt redirect whose target is on a different scheme, host or + # port than the URL it was reached from? Redirects to schemes other than http + # and https are never followed. A redirect which only replaces http with https + # while staying on the same host and port is always followed, whatever this is + # set to. When this is enabled, the robots.txt is re-fetched from the host + # named in the Location header, for up to 5 hops, and each of these requests + # carries the headers configured through http.basicauth.* and + # http.custom.headers as any other request does; http.filter.ipaddress.exclude + # can be used to restrict the addresses those fetches may reach. + http.robots.redirect.crossorigin.allow: false + + # Allow all if a robots.txt redirect was not followed because of the setting + # above? If false, nothing is crawled on that host until the redirect is + # followed or this is set to true. + http.robots.redirect.refused.allow: false + # ignore directives from robots.txt files? http.robots.file.skip: false diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserRedirectTargetTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserRedirectTargetTest.java new file mode 100644 index 000000000..ba1ac9092 --- /dev/null +++ b/core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserRedirectTargetTest.java @@ -0,0 +1,295 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.stormcrawler.protocol; + +import crawlercommons.robots.BaseRobotRules; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.storm.Config; +import org.apache.stormcrawler.Metadata; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** Checks which URLs are fetched when a robots.txt responds with a redirect. */ +class HttpRobotRulesParserRedirectTargetTest { + + private static final String RULES = "User-agent: this_is_only_a_test\nDisallow: /restricted/"; + + /** Protocol stub recording the URLs it is asked to fetch. */ + private static class RecordingProtocol implements Protocol { + + final List requested = new ArrayList<>(); + + final Map responses = new HashMap<>(); + + void redirect(String from, String to) { + Metadata md = new Metadata(); + md.setValue("location", to); + responses.put(from, new ProtocolResponse(new byte[0], 301, md)); + } + + void rules(String url) { + Metadata md = new Metadata(); + md.setValue("content-type", "text/plain"); + responses.put( + url, new ProtocolResponse(RULES.getBytes(StandardCharsets.UTF_8), 200, md)); + } + + @Override + public void configure(Config conf) {} + + @Override + public ProtocolResponse getProtocolOutput(String url, Metadata metadata) { + requested.add(url); + ProtocolResponse response = responses.get(url); + if (response == null) { + return new ProtocolResponse(new byte[0], 404, new Metadata()); + } + return response; + } + + @Override + public BaseRobotRules getRobotRules(String url) { + return null; + } + + @Override + public void cleanup() {} + } + + private static Config conf() { + Config conf = new Config(); + conf.put("http.agent.name", "this_is_only_a_test"); + return conf; + } + + private static HttpRobotRulesParser parser(Config conf) { + HttpRobotRulesParser parser = new HttpRobotRulesParser(); + parser.setConf(conf); + return parser; + } + + @Test + void redirectToSameHostIsFollowed() { + RecordingProtocol protocol = new RecordingProtocol(); + protocol.redirect("http://same.example.org/robots.txt", "/robots/rules.txt"); + protocol.rules("http://same.example.org/robots/rules.txt"); + BaseRobotRules rules = + parser(conf()).getRobotRulesSet(protocol, "http://same.example.org/"); + Assertions.assertTrue( + protocol.requested.contains("http://same.example.org/robots/rules.txt"), + "expected the redirect target to be fetched, requested: " + protocol.requested); + Assertions.assertTrue(rules.isAllowed("http://same.example.org/index.html")); + Assertions.assertFalse(rules.isAllowed("http://same.example.org/restricted/index.html")); + } + + @Test + void redirectToOtherHostIsNotFollowed() { + RecordingProtocol protocol = new RecordingProtocol(); + protocol.redirect( + "http://host.example.org/robots.txt", "http://elsewhere.example.org/robots.txt"); + protocol.rules("http://elsewhere.example.org/robots.txt"); + BaseRobotRules rules = + parser(conf()).getRobotRulesSet(protocol, "http://host.example.org/"); + Assertions.assertFalse( + protocol.requested.contains("http://elsewhere.example.org/robots.txt"), + "robots.txt of another host should not be fetched, requested: " + + protocol.requested); + // no rules obtained, so nothing is crawled on that host + Assertions.assertTrue(rules.isAllowNone()); + } + + @Test + void redirectToOtherPortIsNotFollowed() { + RecordingProtocol protocol = new RecordingProtocol(); + protocol.redirect( + "http://port.example.org/robots.txt", "http://port.example.org:8080/robots.txt"); + protocol.rules("http://port.example.org:8080/robots.txt"); + BaseRobotRules rules = + parser(conf()).getRobotRulesSet(protocol, "http://port.example.org/"); + Assertions.assertFalse( + protocol.requested.contains("http://port.example.org:8080/robots.txt"), + "robots.txt on another port should not be fetched, requested: " + + protocol.requested); + Assertions.assertTrue(rules.isAllowNone()); + } + + @Test + void redirectToOtherSchemeIsNotFollowed() { + RecordingProtocol protocol = new RecordingProtocol(); + protocol.redirect("http://scheme.example.org/robots.txt", "file:/tmp/robots.txt"); + parser(conf()).getRobotRulesSet(protocol, "http://scheme.example.org/"); + Assertions.assertFalse( + protocol.requested.contains("file:/tmp/robots.txt"), + "robots.txt fetch should stay on http(s), requested: " + protocol.requested); + } + + @Test + void redirectToOtherHostIsFollowedIfConfigured() { + RecordingProtocol protocol = new RecordingProtocol(); + protocol.redirect("http://cdn.example.org/robots.txt", "http://cdn.example.com/robots.txt"); + protocol.rules("http://cdn.example.com/robots.txt"); + Config conf = conf(); + conf.put("http.robots.redirect.crossorigin.allow", true); + BaseRobotRules rules = parser(conf).getRobotRulesSet(protocol, "http://cdn.example.org/"); + Assertions.assertTrue( + protocol.requested.contains("http://cdn.example.com/robots.txt"), + "expected the redirect target to be fetched, requested: " + protocol.requested); + Assertions.assertFalse(rules.isAllowed("http://cdn.example.org/restricted/index.html")); + } + + @Test + void schemeUpgradeOnSameHostIsFollowed() { + RecordingProtocol protocol = new RecordingProtocol(); + protocol.redirect( + "http://upgrade.example.org/robots.txt", "https://upgrade.example.org/robots.txt"); + protocol.rules("https://upgrade.example.org/robots.txt"); + BaseRobotRules rules = + parser(conf()).getRobotRulesSet(protocol, "http://upgrade.example.org/"); + Assertions.assertTrue( + protocol.requested.contains("https://upgrade.example.org/robots.txt"), + "expected the redirect target to be fetched, requested: " + protocol.requested); + Assertions.assertFalse(rules.isAllowed("http://upgrade.example.org/restricted/index.html")); + } + + @Test + void schemeUpgradeWithExplicitDefaultPortIsFollowed() { + RecordingProtocol protocol = new RecordingProtocol(); + protocol.redirect( + "http://upgrade.example.org/robots.txt", + "https://upgrade.example.org:443/robots.txt"); + protocol.rules("https://upgrade.example.org:443/robots.txt"); + BaseRobotRules rules = + parser(conf()).getRobotRulesSet(protocol, "http://upgrade.example.org/"); + Assertions.assertTrue( + protocol.requested.contains("https://upgrade.example.org:443/robots.txt"), + "expected the redirect target to be fetched, requested: " + protocol.requested); + Assertions.assertFalse(rules.isAllowed("http://upgrade.example.org/restricted/index.html")); + } + + @Test + void schemeUpgradeKeepingExplicitPortIsFollowed() { + RecordingProtocol protocol = new RecordingProtocol(); + protocol.redirect( + "http://upgrade.example.org:8080/robots.txt", + "https://upgrade.example.org:8080/robots.txt"); + protocol.rules("https://upgrade.example.org:8080/robots.txt"); + BaseRobotRules rules = + parser(conf()).getRobotRulesSet(protocol, "http://upgrade.example.org:8080/"); + Assertions.assertTrue( + protocol.requested.contains("https://upgrade.example.org:8080/robots.txt"), + "expected the redirect target to be fetched, requested: " + protocol.requested); + Assertions.assertFalse( + rules.isAllowed("http://upgrade.example.org:8080/restricted/index.html")); + } + + @Test + void schemeDowngradeOnSameHostIsNotFollowed() { + RecordingProtocol protocol = new RecordingProtocol(); + protocol.redirect( + "https://downgrade.example.org/robots.txt", + "http://downgrade.example.org/robots.txt"); + protocol.rules("http://downgrade.example.org/robots.txt"); + BaseRobotRules rules = + parser(conf()).getRobotRulesSet(protocol, "https://downgrade.example.org/"); + Assertions.assertFalse( + protocol.requested.contains("http://downgrade.example.org/robots.txt"), + "robots.txt should not be fetched over http after https, requested: " + + protocol.requested); + Assertions.assertTrue(rules.isAllowNone()); + } + + @Test + void schemeUpgradeToOtherHostIsNotFollowed() { + RecordingProtocol protocol = new RecordingProtocol(); + protocol.redirect( + "http://one.example.org/robots.txt", "https://other.example.org/robots.txt"); + protocol.rules("https://other.example.org/robots.txt"); + parser(conf()).getRobotRulesSet(protocol, "http://one.example.org/"); + Assertions.assertFalse( + protocol.requested.contains("https://other.example.org/robots.txt"), + "robots.txt of another host should not be fetched, requested: " + + protocol.requested); + } + + @Test + void schemeUpgradeToOtherPortIsNotFollowed() { + RecordingProtocol protocol = new RecordingProtocol(); + protocol.redirect( + "http://upgrade.example.org:8080/robots.txt", + "https://upgrade.example.org:9443/robots.txt"); + protocol.rules("https://upgrade.example.org:9443/robots.txt"); + parser(conf()).getRobotRulesSet(protocol, "http://upgrade.example.org:8080/"); + Assertions.assertFalse( + protocol.requested.contains("https://upgrade.example.org:9443/robots.txt"), + "robots.txt on another port should not be fetched, requested: " + + protocol.requested); + } + + @Test + void redirectWhichIsNotFollowedIsNotCachedAsSuccess() throws MalformedURLException { + RecordingProtocol protocol = new RecordingProtocol(); + protocol.redirect( + "http://cached.example.org/robots.txt", "http://elsewhere.example.org/robots.txt"); + HttpRobotRulesParser parser = parser(conf()); + BaseRobotRules rules = parser.getRobotRulesSet(protocol, "http://cached.example.org/"); + Assertions.assertTrue(rules.isAllowNone()); + // the cache of successfully fetched rules, which the robots URL filter reads, is untouched + Assertions.assertTrue( + parser.getRobotRulesSetFromCache(new URL("http://cached.example.org/")) + .isAllowAll()); + } + + @Test + void redirectWhichIsNotFollowedAllowsCrawlingIfConfigured() { + RecordingProtocol protocol = new RecordingProtocol(); + protocol.redirect( + "http://lenient.example.org/robots.txt", "http://elsewhere.example.org/robots.txt"); + Config conf = conf(); + conf.put("http.robots.redirect.refused.allow", true); + BaseRobotRules rules = + parser(conf).getRobotRulesSet(protocol, "http://lenient.example.org/"); + Assertions.assertFalse( + protocol.requested.contains("http://elsewhere.example.org/robots.txt"), + "robots.txt of another host should not be fetched, requested: " + + protocol.requested); + Assertions.assertTrue(rules.isAllowAll()); + } + + @Test + void redirectWhichIsNotFollowedDoesNotApplyToTheHostsOnTheChain() { + RecordingProtocol protocol = new RecordingProtocol(); + protocol.redirect( + "http://first.example.org/robots.txt", "http://second.example.org/robots.txt"); + protocol.redirect("http://second.example.org/robots.txt", "file:/tmp/robots.txt"); + Config conf = conf(); + conf.put("http.robots.redirect.crossorigin.allow", true); + HttpRobotRulesParser parser = parser(conf); + Assertions.assertTrue( + parser.getRobotRulesSet(protocol, "http://first.example.org/").isAllowNone()); + // the second host is asked again instead of inheriting the outcome of the chain above + protocol.rules("http://second.example.org/robots.txt"); + BaseRobotRules rules = parser.getRobotRulesSet(protocol, "http://second.example.org/"); + Assertions.assertFalse(rules.isAllowed("http://second.example.org/restricted/index.html")); + } +} diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserRedirectTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserRedirectTest.java index d3e1eccf0..034489d65 100644 --- a/core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserRedirectTest.java +++ b/core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserRedirectTest.java @@ -20,6 +20,8 @@ import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; import static com.github.tomakehurst.wiremock.client.WireMock.configureFor; import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.matching; import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; @@ -29,6 +31,7 @@ import com.github.tomakehurst.wiremock.core.WireMockConfiguration; import crawlercommons.robots.BaseRobotRules; import org.apache.storm.Config; +import org.apache.stormcrawler.protocol.okhttp.HttpProtocol; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -44,6 +47,7 @@ class HttpRobotRulesParserRedirectTest { private String body; + private String url0 = "http://localhost:" + ports[0]; private String url1 = "http://localhost:" + ports[1]; private String url7 = "http://localhost:" + ports[7]; private String url8 = "http://localhost:" + ports[8]; @@ -106,6 +110,10 @@ void tearDown() { @Test void testRedirects() { + // the redirect chains tested here point at other ports of the same host + Config crossOriginConf = new Config(); + crossOriginConf.putAll(conf); + crossOriginConf.put("http.robots.redirect.crossorigin.allow", true); // Test for 5 consecutive redirects configureFor(mockServer1.getClient()); stubFor( @@ -162,7 +170,7 @@ void testRedirects() { get(urlPathEqualTo("/robots.txt")) .willReturn(aResponse().withBody(body).withStatus(200))); HttpRobotRulesParser httpRobotRulesParser = new HttpRobotRulesParser(); - httpRobotRulesParser.setConf(conf); + httpRobotRulesParser.setConf(crossOriginConf); BaseRobotRules robotRules = httpRobotRulesParser.getRobotRulesSet(protocol, url1); Assertions.assertFalse(robotRules.isAllowAll()); Assertions.assertFalse(robotRules.isAllowNone()); @@ -178,7 +186,7 @@ void testRedirects() { .withBody(body) .withStatus(301))); httpRobotRulesParser = new HttpRobotRulesParser(); - httpRobotRulesParser.setConf(conf); + httpRobotRulesParser.setConf(crossOriginConf); robotRules = httpRobotRulesParser.getRobotRulesSet(protocol, url1); Assertions.assertTrue(robotRules.isAllowAll()); // Test relative redirects @@ -212,7 +220,7 @@ void testRedirects() { .withStatus(302))); // from here the redirect leads to the same robots.txt as in the first test block httpRobotRulesParser = new HttpRobotRulesParser(); - httpRobotRulesParser.setConf(conf); + httpRobotRulesParser.setConf(crossOriginConf); robotRules = httpRobotRulesParser.getRobotRulesSet(protocol, url7); Assertions.assertFalse(robotRules.isAllowAll()); Assertions.assertFalse(robotRules.isAllowNone()); @@ -233,6 +241,82 @@ void testRedirects() { Assertions.assertFalse(robotRules.isAllowed(url1 + "/restricted/index.html")); } + @Test + void testRedirectOnTheSameAuthorityIsFollowedByDefault() { + configureFor(mockServer0.getClient()); + stubFor( + get(urlPathEqualTo("/robots.txt")) + .willReturn( + aResponse() + .withHeader("location", "/robots/rules.txt") + .withStatus(301))); + stubFor( + get(urlPathEqualTo("/robots/rules.txt")) + .willReturn(aResponse().withBody(body).withStatus(200))); + HttpRobotRulesParser httpRobotRulesParser = new HttpRobotRulesParser(); + httpRobotRulesParser.setConf(conf); + BaseRobotRules robotRules = httpRobotRulesParser.getRobotRulesSet(protocol, url0); + Assertions.assertFalse(robotRules.isAllowAll()); + Assertions.assertFalse(robotRules.isAllowNone()); + Assertions.assertTrue(robotRules.isAllowed(url0 + "/index.html")); + Assertions.assertFalse(robotRules.isAllowed(url0 + "/restricted/index.html")); + } + + @Test + void testRedirectToAnotherAuthorityIsNotFollowedByDefault() { + configureFor(mockServer0.getClient()); + stubFor( + get(urlPathEqualTo("/robots.txt")) + .willReturn( + aResponse() + .withHeader("location", url1 + "/robots.txt") + .withStatus(301))); + configureFor(mockServer1.getClient()); + stubFor( + get(urlPathEqualTo("/robots.txt")) + .willReturn(aResponse().withBody(body).withStatus(200))); + HttpRobotRulesParser httpRobotRulesParser = new HttpRobotRulesParser(); + httpRobotRulesParser.setConf(conf); + BaseRobotRules robotRules = httpRobotRulesParser.getRobotRulesSet(protocol, url0); + mockServer1.verify(0, getRequestedFor(urlPathEqualTo("/robots.txt"))); + Assertions.assertTrue(robotRules.isAllowNone()); + } + + @Test + void testHeadersNotSentToAnUnfollowedRedirectTarget() { + Config authConf = new Config(); + authConf.putAll(conf); + authConf.put("http.basicauth.user", "this_is_only_a_test"); + authConf.put("http.basicauth.password", "this_is_only_a_test"); + HttpProtocol authProtocol = new HttpProtocol(); + authProtocol.configure(authConf); + try { + configureFor(mockServer0.getClient()); + stubFor( + get(urlPathEqualTo("/robots.txt")) + .willReturn( + aResponse() + .withHeader("location", url1 + "/robots.txt") + .withStatus(301))); + configureFor(mockServer1.getClient()); + stubFor( + get(urlPathEqualTo("/robots.txt")) + .willReturn(aResponse().withBody(body).withStatus(200))); + HttpRobotRulesParser httpRobotRulesParser = new HttpRobotRulesParser(); + httpRobotRulesParser.setConf(authConf); + httpRobotRulesParser.getRobotRulesSet(authProtocol, url0); + // the configured Authorization header is sent to the host the robots.txt belongs to + mockServer0.verify( + 1, + getRequestedFor(urlPathEqualTo("/robots.txt")) + .withHeader("Authorization", matching("Basic .+"))); + // the target of the redirect is on another port and is not contacted at all + mockServer1.verify(0, getRequestedFor(urlPathEqualTo("/robots.txt"))); + } finally { + authProtocol.cleanup(); + } + } + private static class MockServer extends WireMockServer { public MockServer(Options options) { super(options); diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/IPFilterRulesTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/IPFilterRulesTest.java index 109529ce0..fa42d190f 100644 --- a/core/src/test/java/org/apache/stormcrawler/protocol/IPFilterRulesTest.java +++ b/core/src/test/java/org/apache/stormcrawler/protocol/IPFilterRulesTest.java @@ -63,6 +63,23 @@ void excludeLoopbackAndSitelocalAsCommaSeparatedString() { assertTrue(r.accept(ip("8.8.8.8"))); } + @Test + void excludeLinklocal() { + IPFilterRules r = rules(null, "linklocal"); + assertFalse(r.isEmpty()); + assertFalse(r.accept(ip("169.254.169.254"))); + assertFalse(r.accept(ip("fe80::1"))); + assertTrue(r.accept(ip("8.8.8.8"))); + } + + @Test + void linklocalIsCoveredByNeitherLoopbackNorSitelocal() { + IPFilterRules r = rules(null, "localhost,sitelocal"); + assertTrue( + r.accept(ip("169.254.169.254")), + "a link-local address needs the linklocal rule of its own"); + } + @Test void excludeAsYamlList() { IPFilterRules r = rules(null, Arrays.asList("loopback", "192.168.0.0/16")); diff --git a/docs/src/main/asciidoc/configuration.adoc b/docs/src/main/asciidoc/configuration.adoc index 7aaa0ccf7..b2c241959 100644 --- a/docs/src/main/asciidoc/configuration.adoc +++ b/docs/src/main/asciidoc/configuration.adoc @@ -220,6 +220,8 @@ implementation. | http.robots.file.skip | false | Ignore robots.txt rules entirely. | http.robots.headers.skip | false | Ignore robots directives from HTTP headers. | http.robots.meta.skip | false | Ignore robots directives from HTML meta tags. +| http.robots.redirect.crossorigin.allow | false | Follow a robots.txt redirect pointing at a different scheme, host or port than the URL it was reached from. Redirects to schemes other than http and https are never followed. A redirect which only replaces http with https while staying on the same host and port is always followed, whatever this is set to. When enabled, the robots.txt is re-fetched from the host named in the `Location` header, for up to 5 hops, and each of these requests carries the headers configured through `http.basicauth.*` and `http.custom.headers` as any other request does; see `http.filter.ipaddress.exclude` to restrict the addresses those fetches may reach. +| http.robots.redirect.refused.allow | false | Allow crawling when a robots.txt redirect was not followed because of `http.robots.redirect.crossorigin.allow`. If false, nothing is crawled on that host until the redirect is followed or this is set to true. | http.skip.robots | false | Deprecated (replaced by http.robots.file.skip). | robots.noFollow.strict | true | If true, remove all outlinks from pages marked as noFollow. | http.store.headers | false | Whether to store response headers. @@ -283,14 +285,20 @@ Each property accepts a comma-separated string or a YAML list of rules. A rule c * a CIDR block, e.g. `192.168.0.0/16` or `fd00::/8` * `localhost` or `loopback` — matches any loopback address (`InetAddress.isLoopbackAddress()`) * `sitelocal` — matches any site-local address (`InetAddress.isSiteLocalAddress()`) +* `linklocal` — matches any link-local address (`InetAddress.isLinkLocalAddress()`), i.e. `169.254.0.0/16` +and `fe80::/10` For example, to block crawling of localhost and private address spaces: [source,yaml] ---- -http.filter.ipaddress.exclude: "localhost,sitelocal" +http.filter.ipaddress.exclude: "localhost,sitelocal,linklocal" ---- +`sitelocal` follows the JDK and covers `10.0.0.0/8`, `172.16.0.0/12` and `192.168.0.0/16` only. Carrier +grade NAT (`100.64.0.0/10`) and IPv6 unique local addresses (`fd00::/8`) have no keyword and are added +as CIDR blocks when they are also to be blocked. + When a connection to a blocked address is attempted, the fetch fails with an `IOException` and a warning is logged. If neither property is set, no IP filtering is performed.