Scope cookies to the host that set them

core/src/main/java/org/apache/stormcrawler/protocol/AbstractHttpProtocol.java

@@ -70,6 +70,13 @@ public abstract class AbstractHttpProtocol implements Protocol {
 
     protected static final String RESPONSE_COOKIES_HEADER = "set-cookie";
 
+    /**
+     * Metadata key holding the url whose response carried the cookies stored under {@link
+     * #RESPONSE_COOKIES_HEADER}. It is written by the protocol and not taken from a response
+     * header, and is used to scope cookies without a domain attribute to the host which set them.
+     */
+    protected static final String RESPONSE_COOKIES_ORIGIN = "set-cookie-origin";
+
     protected static final String SET_HEADER_BY_REQUEST = "set-header";
 
     protected String protocolMetadataPrefix = "";

core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java

@@ -23,6 +23,7 @@
 import java.net.InetSocketAddress;
 import java.net.MalformedURLException;
 import java.net.Proxy;
+import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.security.cert.CertificateException;
 import java.util.ArrayList;
@@ -35,6 +36,7 @@
 import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 import javax.net.ssl.HostnameVerifier;
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.SSLSession;
@@ -103,6 +105,9 @@ public class HttpProtocol extends AbstractHttpProtocol {
     // track the time spent for each URL in DNS resolution
     private final Map<String, Long> DNStimes = new ConcurrentHashMap<>();
 
+    // makes sure that a missing cookie origin is reported once and not for every url
+    private final AtomicBoolean missingCookieOriginLogged = new AtomicBoolean();
+
     private OkHttpClient.Builder builder;
 
     private static final TrustManager[] trustAllCerts =
@@ -287,14 +292,51 @@ private void addCookiesToRequest(Builder rb, String url, Metadata md) {
         }
         try {
             final List<Cookie> cookies =
-                    CookieConverter.getCookies(cookieStrings, URLUtil.toURL(url));
+                    CookieConverter.getCookies(
+                            cookieStrings, getCookieOrigin(md, url), URLUtil.toURL(url));
             for (Cookie c : cookies) {
                 rb.addHeader("Cookie", c.getName() + "=" + c.getValue());
             }
         } catch (MalformedURLException e) { // Bad url , nothing to do
         }
     }
 
+    /**
+     * Returns the url whose response set the cookies, or null when it was not recorded, in which
+     * case cookies without a domain attribute are not sent.
+     */
+    private URL getCookieOrigin(Metadata md, String url) {
+        final String origin = md.getFirstValue(RESPONSE_COOKIES_ORIGIN, protocolMetadataPrefix);
+        if (StringUtils.isBlank(origin)) {
+            if (missingCookieOriginLogged.compareAndSet(false, true)) {
+                LOG.warn(
+                        "No {}{} for {}, cookies without a domain attribute are not sent. Add {}{}"
+                                + " to metadata.transfer and metadata.persist next to {}{}.",
+                        protocolMetadataPrefix,
+                        RESPONSE_COOKIES_ORIGIN,
+                        url,
+                        protocolMetadataPrefix,
+                        RESPONSE_COOKIES_ORIGIN,
+                        protocolMetadataPrefix,
+                        RESPONSE_COOKIES_HEADER);
+            } else {
+                LOG.debug("No {}{} for {}", protocolMetadataPrefix, RESPONSE_COOKIES_ORIGIN, url);
+            }
+            return null;
+        }
+        try {
+            return URLUtil.toURL(origin);
+        } catch (MalformedURLException e) {
+            LOG.warn(
+                    "Invalid {}{} {} for {}",
+                    protocolMetadataPrefix,
+                    RESPONSE_COOKIES_ORIGIN,
+                    origin,
+                    url);
+            return null;
+        }
+    }
+
     protected void addHeadersToRequest(Builder rb, Metadata md) {
         final String[] headerStrings = md.getValues(SET_HEADER_BY_REQUEST, protocolMetadataPrefix);
 
@@ -445,6 +487,16 @@ public ProtocolResponse getProtocolOutput(String url, final Metadata metadata)
                 responsemetadata.addValue(key.toLowerCase(Locale.ROOT), value);
             }
 
+            // the Set-Cookie header does not say which host sent it: record the url of this
+            // response so that the cookies can be scoped to it when they are sent back. The
+            // key is dropped first so that a server sending a header of that name can not
+            // forge the origin of the cookies inherited from another page.
+            responsemetadata.remove(RESPONSE_COOKIES_ORIGIN);
+            if (responsemetadata.getFirstValue(RESPONSE_COOKIES_HEADER) != null) {
+                responsemetadata.setValue(
+                        RESPONSE_COOKIES_ORIGIN, response.request().url().toString());
+            }
+
             final MutableObject<TrimmedContentReason> trimmed =
                     new MutableObject<>(TrimmedContentReason.NOT_TRIMMED);
             final byte[] bytes = toByteArray(response.body(), pageMaxContent, trimmed);