Only follow robots.txt redirects on the same host

core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java

@@ -21,6 +21,7 @@
 import com.google.common.primitives.Ints;
 import crawlercommons.robots.BaseRobotRules;
 import java.net.URL;
+import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
@@ -44,6 +45,10 @@ public class HttpRobotRulesParser extends RobotRulesParser {
 
     protected boolean allow5xx = false;
 
+    protected boolean allowCrossOriginRedirects = false;
+
+    protected boolean allowRefusedRedirect = false;
+
     protected Metadata fetchRobotsMd;
 
     private static final int MAX_NUM_REDIRECTS = 5;
@@ -63,6 +68,36 @@ public void setConf(Config conf) {
         int robotsTxtContentLimit = ConfUtils.getInt(conf, "http.robots.content.limit", -1);
         fetchRobotsMd.addValue("http.content.limit", Integer.toString(robotsTxtContentLimit));
         allow5xx = ConfUtils.getBoolean(conf, "http.robots.5xx.allow", false);
+        allowCrossOriginRedirects =
+                ConfUtils.getBoolean(conf, "http.robots.redirect.crossorigin.allow", false);
+        allowRefusedRedirect =
+                ConfUtils.getBoolean(conf, "http.robots.redirect.refused.allow", false);
+        if (allowCrossOriginRedirects) {
+            logForwardedRequestHeaders(conf);
+        }
+    }
+
+    /**
+     * Logs which of the configured request headers are sent to the target of a robots.txt redirect
+     * to another host. The headers are configured per protocol instance and the redirect is
+     * followed by re-fetching the target through the same instance, so every hop of the chain is
+     * requested with them.
+     */
+    private static void logForwardedRequestHeaders(Config conf) {
+        List<String> sources = new ArrayList<>();
+        if (StringUtils.isNotBlank(ConfUtils.getString(conf, "http.basicauth.user", null))) {
+            sources.add("http.basicauth.user");
+        }
+        if (!ConfUtils.loadListFromConf("http.custom.headers", conf).isEmpty()) {
+            sources.add("http.custom.headers");
+        }
+        if (!sources.isEmpty()) {
+            LOG.warn(
+                    "http.robots.redirect.crossorigin.allow is set: a robots.txt redirect is fetched "
+                            + "from the host named in the Location header, and the request headers "
+                            + "configured by {} are sent to it as to any other host",
+                    String.join(" and ", sources));
+        }
     }
 
     /** Compose unique key to store and access robot rules in cache for given URL. */
@@ -81,6 +116,71 @@ protected static String getCacheKey(URL url) {
         return protocol + ":" + host + ":" + port;
     }
 
+    /**
+     * Checks whether a redirect while fetching a robots.txt is followed. The target must use the
+     * http or https scheme and, unless {@code http.robots.redirect.crossorigin.allow} is set,
+     * either share scheme, host and port with the URL it was reached from, or be the same host and
+     * port reached over https instead of http.
+     *
+     * @param from URL which returned the redirect
+     * @param target resolved value of the Location header
+     * @return true if the target may be fetched
+     */
+    protected boolean followRedirect(URL from, URL target) {
+        String scheme = target.getProtocol().toLowerCase(Locale.ROOT);
+        if (!"http".equals(scheme) && !"https".equals(scheme)) {
+            return false;
+        }
+        return allowCrossOriginRedirects
+                || getCacheKey(from).equals(getCacheKey(target))
+                || isSchemeUpgrade(from, target);
+    }
+
+    /**
+     * Checks whether a redirect only replaces http with https while staying on the same host and
+     * port, e.g. {@code http://example.com/robots.txt} to {@code https://example.com/robots.txt}.
+     * {@link #getCacheKey(URL)} derives the default port from the scheme, so such a target has a
+     * different key although no other host is involved. It is therefore fetched even when redirects
+     * to a different origin are not followed.
+     *
+     * <p>Only this direction is exempt. The reverse, https to http, would send the request headers
+     * configured for the fetch, e.g. the Authorization header built from {@code
+     * http.basicauth.user}, unencrypted, and would let the rules read over a plain text connection
+     * replace the ones the redirect was reached from. A site relying on that redirect needs {@code
+     * http.robots.redirect.crossorigin.allow}.
+     */
+    private static boolean isSchemeUpgrade(URL from, URL target) {
+        if (!"http".equals(from.getProtocol().toLowerCase(Locale.ROOT))
+                || !"https".equals(target.getProtocol().toLowerCase(Locale.ROOT))) {
+            return false;
+        }
+        if (!from.getHost().equalsIgnoreCase(target.getHost())) {
+            return false;
+        }
+        // either the port is unchanged or both sides use the default port of their scheme
+        return from.getPort() == target.getPort() || (isDefaultPort(from) && isDefaultPort(target));
+    }
+
+    /** Checks whether a URL uses the default port of its scheme, explicitly or implicitly. */
+    private static boolean isDefaultPort(URL url) {
+        return url.getPort() == -1 || url.getPort() == url.getDefaultPort();
+    }
+
+    /** Describes for logging why the target of a redirect is not fetched. */
+    private static String redirectRefusalReason(URL from, URL target) {
+        String targetScheme = target.getProtocol().toLowerCase(Locale.ROOT);
+        if (!"http".equals(targetScheme) && !"https".equals(targetScheme)) {
+            return "on neither http nor https";
+        }
+        if (!from.getHost().equalsIgnoreCase(target.getHost())) {
+            return "on a different host than " + from;
+        }
+        if (!from.getProtocol().toLowerCase(Locale.ROOT).equals(targetScheme)) {
+            return "on a different scheme than " + from;
+        }
+        return "on a different port than " + from;
+    }
+
     /**
      * Returns the robots rules from the cache or empty rules if not found.
      *
@@ -123,6 +223,7 @@ public BaseRobotRules getRobotRulesSet(Protocol http, URL url) {
         }
 
         boolean cacheRule = true;
+        boolean redirectRefused = false;
         Set<String> redirectCacheKeys = new HashSet<>();
 
         URL robotsUrl = null;
@@ -149,7 +250,25 @@ public BaseRobotRules getRobotRulesSet(Protocol http, URL url) {
                 String redirection = response.getMetadata().getFirstValue(HttpHeaders.LOCATION);
                 LOG.debug("Redirected from {} to {}", redir, redirection);
                 if (StringUtils.isNotBlank(redirection)) {
-                    redir = URLUtil.resolveUrl(redir, redirection);
+                    URL target = URLUtil.resolveUrl(redir, redirection);
+                    if (!followRedirect(redir, target)) {
+                        redirectRefused = true;
+                        LOG.warn(
+                                "Robots for {} redirected to {} which is not fetched, the target "
+                                        + "is {}. As a result {}. Set "
+                                        + "http.robots.redirect.crossorigin.allow to true to follow "
+                                        + "such redirects, or "
+                                        + "http.robots.redirect.refused.allow to change what "
+                                        + "happens when one of them is not followed.",
+                                url,
+                                target,
+                                redirectRefusalReason(redir, target),
+                                allowRefusedRedirect
+                                        ? "the host is crawled without any rules"
+                                        : "nothing is crawled on that host");
+                        break;
+                    }
+                    redir = target;
                     if (redir.getPath().equals("/robots.txt") && redir.getQuery() == null) {
                         // only if the path (including the query part) of the redirect target is
                         // `/robots.txt` we can get/put the rules from/to the cache under the host
@@ -212,6 +331,10 @@ public BaseRobotRules getRobotRulesSet(Protocol http, URL url) {
                 if (allow5xx) {
                     robotRules = EMPTY_RULES; // allow all
                 }
+            } else if (redirectRefused) {
+                // the redirect was not followed, so no rules were obtained for this host
+                cacheRule = false;
+                robotRules = allowRefusedRedirect ? EMPTY_RULES : FORBID_ALL_RULES;
             } else {
                 robotRules = EMPTY_RULES; // allow all
             }
@@ -235,11 +358,19 @@ public BaseRobotRules getRobotRulesSet(Protocol http, URL url) {
 
         // cache robot rules for redirections
         // get here only if the target has not been found in the cache
-        for (String keyredir : redirectCacheKeys) {
-            // keyredir isn't null only if the robots.txt file of the target is
-            // at the root
-            LOG.debug("Caching robots for {} under key {} in cache {}", redir, keyredir, cacheName);
-            cacheToUse.put(keyredir, cached);
+        // a chain ending in a redirect which was not followed produced no rules for the hosts on
+        // it, so nothing is stored under their keys
+        if (!redirectRefused) {
+            for (String keyredir : redirectCacheKeys) {
+                // keyredir isn't null only if the robots.txt file of the target is
+                // at the root
+                LOG.debug(
+                        "Caching robots for {} under key {} in cache {}",
+                        redir,
+                        keyredir,
+                        cacheName);
+                cacheToUse.put(keyredir, cached);
+            }
         }
 
         RobotRules live = new RobotRules(robotRules);

core/src/main/java/org/apache/stormcrawler/protocol/IPFilterRules.java

@@ -116,6 +116,9 @@ private static List<Predicate<InetAddress>> parseIPRules(
                     case "sitelocal":
                         rules.add(InetAddress::isSiteLocalAddress);
                         break;
+                    case "linklocal":
+                        rules.add(InetAddress::isLinkLocalAddress);
+                        break;
                     default:
                         try {
                             CIDR cidr = new CIDR(ipRule);

core/src/main/resources/crawler-default.yaml

@@ -156,17 +156,34 @@ config:
   #   - a CIDR block, e.g. "192.168.0.0/16" or "fd00::/8"
   #   - "localhost" / "loopback" (matches InetAddress.isLoopbackAddress())
   #   - "sitelocal" (matches InetAddress.isSiteLocalAddress())
+  #   - "linklocal" (matches InetAddress.isLinkLocalAddress())
   # Only addresses matching an include rule are fetched (empty means all are
   # allowed), addresses matching an exclude rule are always blocked.
   # http.filter.ipaddress.include:
-  # http.filter.ipaddress.exclude: "localhost,sitelocal"
+  # http.filter.ipaddress.exclude: "localhost,sitelocal,linklocal"
 
   # Allow all if robots.txt cannot be parsed due to code 403 (Forbidden):
   http.robots.403.allow: true
 
   # Allow all if robots.txt cannot be parsed due to a server error (5xx):
   http.robots.5xx.allow: false
 
+  # Follow a robots.txt redirect whose target is on a different scheme, host or
+  # port than the URL it was reached from? Redirects to schemes other than http
+  # and https are never followed. A redirect which only replaces http with https
+  # while staying on the same host and port is always followed, whatever this is
+  # set to. When this is enabled, the robots.txt is re-fetched from the host
+  # named in the Location header, for up to 5 hops, and each of these requests
+  # carries the headers configured through http.basicauth.* and
+  # http.custom.headers as any other request does; http.filter.ipaddress.exclude
+  # can be used to restrict the addresses those fetches may reach.
+  http.robots.redirect.crossorigin.allow: false
+
+  # Allow all if a robots.txt redirect was not followed because of the setting
+  # above? If false, nothing is crawled on that host until the redirect is
+  # followed or this is set to true.
+  http.robots.redirect.refused.allow: false
+
   # ignore directives from robots.txt files?
   http.robots.file.skip: false