diff --git a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/job/BulkRequestContainerJob.java b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/job/BulkRequestContainerJob.java
index c7d0269ff24..63a10da53e9 100644
--- a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/job/BulkRequestContainerJob.java
+++ b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/job/BulkRequestContainerJob.java
@@ -93,6 +93,7 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
+import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.Semaphore;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
@@ -113,6 +114,7 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
import org.dcache.services.bulk.util.BulkRequestTarget.State;
import org.dcache.services.bulk.util.BulkRequestTargetBuilder;
import org.dcache.services.bulk.util.BulkServiceStatistics;
+import org.dcache.services.bulk.util.BulkTaskThrottle;
import org.dcache.util.CacheExceptionFactory;
import org.dcache.util.SignalAware;
import org.dcache.util.list.DirectoryEntry;
@@ -174,12 +176,16 @@ public int compare(DirTarget o1, DirTarget o2) {
}
/**
- * Encapsulates manipulation of the semaphore.
+ * Encapsulates manipulation of the semaphore or throttle.
+ *
+ * A task uses either a semaphore (blocking acquisition) or a throttle (non-blocking; the
+ * permit is granted by the throttle when the task is launched).
*/
class PermitHolder {
private final AtomicBoolean holdingPermit = new AtomicBoolean(false);
private Semaphore taskSemaphore;
+ private BulkTaskThrottle taskThrottle;
void acquireIfNotHoldingPermit() throws InterruptedException {
if (taskSemaphore == null) {
@@ -191,24 +197,35 @@ void acquireIfNotHoldingPermit() throws InterruptedException {
}
}
+ /**
+ * The throttle has acquired a permit on behalf of this task.
+ */
+ void permitGranted() {
+ holdingPermit.set(true);
+ }
+
void throttledRelease() {
activity.throttle();
releaseIfHoldingPermit();
}
void releaseIfHoldingPermit() {
- if (taskSemaphore == null) {
- return;
- }
-
if (holdingPermit.compareAndSet(true, false)) {
- taskSemaphore.release();
+ if (taskThrottle != null) {
+ taskThrottle.release();
+ } else if (taskSemaphore != null) {
+ taskSemaphore.release();
+ }
}
}
void setTaskSemaphore(Semaphore taskSemaphore) {
this.taskSemaphore = taskSemaphore;
}
+
+ void setTaskThrottle(BulkTaskThrottle taskThrottle) {
+ this.taskThrottle = taskThrottle;
+ }
}
/**
@@ -258,11 +275,10 @@ void expandDepthFirst(Long id, PID pid, FsPath path, FileAttributes dirAttribute
throws BulkServiceException, CacheException {
LOGGER.debug("{} - expandDepthFirst, {}, {}, {}, {}", ruid, id, pid, path,
dirAttributes);
- try {
- new DirListTask(id, pid, path, dirAttributes).submitAsync();
- } catch (InterruptedException e) {
- LOGGER.trace("{} - expandDepthFirst {} interrupted.", ruid, id);
- }
+ /*
+ * Does not block: the listing is queued until a permit is available.
+ */
+ new DirListTask(id, pid, path, dirAttributes).submitAsync();
}
void submitAsync() throws InterruptedException {
@@ -316,7 +332,67 @@ class DirListTask extends ContainerTask {
this.path = path;
this.dirAttributes = dirAttributes;
taskExecutor = listExecutor;
- permitHolder.setTaskSemaphore(dirListSemaphore);
+ permitHolder.setTaskThrottle(dirListThrottle);
+ }
+
+ /**
+ * Registers the task as running and queues it for listing without blocking.
+ *
+ * Blocking for a permit here would deadlock: this is called from listing tasks which
+ * hold permits themselves, so a sufficiently wide directory tree would end up with all
+ * permits held by tasks waiting for permits (GitHub issue #8062).
+ *
+ * The task is added to running immediately so that the transition to directory
+ * processing waits for queued listings as well.
+ */
+ @Override
+ void submitAsync() {
+ synchronized (running) {
+ if (jobTarget.isTerminated()) {
+ return;
+ }
+
+ running.put(seqNo, this);
+ LOGGER.debug("{} - submitAsync (queued) {}, task count is now {}.", ruid, seqNo,
+ running.size());
+ }
+
+ dirListThrottle.submit(this::launch);
+ }
+
+ /**
+ * Called by the throttle holding a permit for this task.
+ */
+ private void launch() {
+ permitHolder.permitGranted();
+
+ RejectedExecutionException rejected = null;
+ synchronized (running) {
+ if (!jobTarget.isTerminated() && running.containsKey(seqNo)) {
+ try {
+ taskFuture = taskExecutor.submit(this);
+ return;
+ } catch (RejectedExecutionException e) {
+ rejected = e;
+ }
+ }
+ }
+
+ if (rejected != null) {
+ /*
+ * Fail fast rather than silently skipping this subtree.
+ */
+ LOGGER.error("{} - could not launch listing of {}: {}.", ruid, path,
+ rejected.toString());
+ containerState = ContainerState.STOP;
+ jobTarget.setErrorObject(rejected);
+ update();
+ }
+
+ /*
+ * Cancelled while queued (or rejected): return the permit.
+ */
+ remove();
}
void doInner() {
@@ -703,7 +779,7 @@ private void storeOrUpdate(Throwable error) {
private ExecutorService executor;
private ExecutorService listExecutor;
private ExecutorService callbackExecutor;
- private Semaphore dirListSemaphore;
+ private BulkTaskThrottle dirListThrottle;
private Semaphore inFlightSemaphore;
private volatile ContainerState containerState;
@@ -882,8 +958,8 @@ public void run() {
}
}
- public void setDirListSemaphore(Semaphore dirListSemaphore) {
- this.dirListSemaphore = dirListSemaphore;
+ public void setDirListThrottle(BulkTaskThrottle dirListThrottle) {
+ this.dirListThrottle = dirListThrottle;
}
public void setInFlightSemaphore(Semaphore inFlightSemaphore) {
diff --git a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/job/RequestContainerJobFactory.java b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/job/RequestContainerJobFactory.java
index 5e145773772..e7a6d0d5e05 100644
--- a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/job/RequestContainerJobFactory.java
+++ b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/job/RequestContainerJobFactory.java
@@ -80,6 +80,7 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
import org.dcache.services.bulk.util.BulkRequestTarget.PID;
import org.dcache.services.bulk.util.BulkRequestTargetBuilder;
import org.dcache.services.bulk.util.BulkServiceStatistics;
+import org.dcache.services.bulk.util.BulkTaskThrottle;
import org.dcache.util.BoundedCachedExecutor;
import org.dcache.util.list.ListDirectoryHandler;
import org.dcache.vehicles.FileAttributes;
@@ -101,7 +102,7 @@ public final class RequestContainerJobFactory {
private ListDirectoryHandler listHandler;
private BulkTargetStore targetStore;
private BulkServiceStatistics statistics;
- private Semaphore dirListSemaphore;
+ private BulkTaskThrottle dirListThrottle;
private Semaphore inFlightSemaphore;
private BoundedCachedExecutor taskExecutor;
private BoundedCachedExecutor callbackExecutor;
@@ -134,7 +135,7 @@ public BulkRequestContainerJob createRequestJob(BulkRequest request)
containerJob.setNamespaceHandler(pnfsHandler);
containerJob.setTargetStore(targetStore);
containerJob.setListHandler(listHandler);
- containerJob.setDirListSemaphore(dirListSemaphore);
+ containerJob.setDirListThrottle(dirListThrottle);
containerJob.setInFlightSemaphore(inFlightSemaphore);
containerJob.setExecutor(taskExecutor);
containerJob.setListExecutor(listExecutor);
@@ -144,7 +145,11 @@ public BulkRequestContainerJob createRequestJob(BulkRequest request)
}
public int getDirListSemaphoreAvailable() {
- return dirListSemaphore.availablePermits();
+ return dirListThrottle.availablePermits();
+ }
+
+ public int getDirListPending() {
+ return dirListThrottle.pendingCount();
}
public int getInFlightSemaphoreAvailable() {
@@ -173,7 +178,7 @@ public void setListExecutor(BoundedCachedExecutor listExecutor) {
@Required
public void setDirListSemaphore(int permits) {
- dirListSemaphore = new Semaphore(permits);
+ dirListThrottle = new BulkTaskThrottle(permits);
}
@Required
diff --git a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/util/BulkServiceStatistics.java b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/util/BulkServiceStatistics.java
index 461c1173777..3729c0925a8 100644
--- a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/util/BulkServiceStatistics.java
+++ b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/util/BulkServiceStatistics.java
@@ -180,6 +180,8 @@ public void getInfo(PrintWriter pw) {
pw.println(String.format(CONCURRENCY_FORMAT, "Available permits for directory listing",
factory.getDirListSemaphoreAvailable()));
+ pw.println(String.format(CONCURRENCY_FORMAT, "Directory listings waiting for permit",
+ factory.getDirListPending()));
pw.println(String.format(CONCURRENCY_FORMAT, "Available permits for in-flight targets",
factory.getInFlightSemaphoreAvailable()));
pw.println();
diff --git a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/util/BulkTaskThrottle.java b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/util/BulkTaskThrottle.java
new file mode 100644
index 00000000000..f97840e99f2
--- /dev/null
+++ b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/util/BulkTaskThrottle.java
@@ -0,0 +1,160 @@
+/*
+COPYRIGHT STATUS:
+Dec 1st 2001, Fermi National Accelerator Laboratory (FNAL) documents and
+software are sponsored by the U.S. Department of Energy under Contract No.
+DE-AC02-76CH03000. Therefore, the U.S. Government retains a world-wide
+non-exclusive, royalty-free license to publish or reproduce these documents
+and software for U.S. Government purposes. All documents and software
+available from this server are private under the U.S. and Foreign
+Copyright Laws, and FNAL reserves all rights.
+
+Distribution of the software available from this server is free of
+charge subject to the user following the terms of the Fermitools
+Software Legal Information.
+
+Redistribution and/or modification of the software shall be accompanied
+by the Fermitools Software Legal Information (including the copyright
+notice).
+
+The user is asked to feed back problems, benefits, and/or suggestions
+about the software to the Fermilab Software Providers.
+
+Neither the name of Fermilab, the URA, nor the names of the contributors
+may be used to endorse or promote products derived from this software
+without specific prior written permission.
+
+DISCLAIMER OF LIABILITY (BSD):
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL FERMILAB,
+OR THE URA, OR THE U.S. DEPARTMENT of ENERGY, OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Liabilities of the Government:
+
+This software is provided by URA, independent from its Prime Contract
+with the U.S. Department of Energy. URA is acting independently from
+the Government and in its own private capacity and is not acting on
+behalf of the U.S. Government, nor as its contractor nor its agent.
+Correspondingly, it is understood and agreed that the U.S. Government
+has no connection to this software and in no manner whatsoever shall
+be liable for nor assume any responsibility or obligation for any claim,
+cost, or damages arising out of or resulting from the use of the software
+available from this server.
+
+Export Control:
+
+All documents and software available from this server are subject to U.S.
+export control laws. Anyone downloading information from this server is
+obligated to secure any necessary Government licenses before exporting
+documents or software obtained from this server.
+ */
+package org.dcache.services.bulk.util;
+
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.Semaphore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Non-blocking admission control for bulk tasks.
+ *
+ * Tasks are queued and launched only when a permit is available. Submission never blocks, so a
+ * task holding a permit may safely submit further tasks (e.g., a directory listing discovering
+ * subdirectories) without risk of all permits being held by tasks waiting for permits.
+ *
+ * A launcher is invoked holding a permit, which it owns from that point on; it must eventually
+ * call {@link #release()} exactly once. If the launcher throws instead, the permit is returned
+ * by the throttle and the launcher must not release it. The throttle may be shared among
+ * several requests.
+ */
+public final class BulkTaskThrottle {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(BulkTaskThrottle.class);
+
+ private final Semaphore permits;
+ private final Queue pending = new ConcurrentLinkedQueue<>();
+
+ /*
+ * Per-thread re-entrancy guard; it does NOT exclude other threads from draining
+ * concurrently (concurrent drains coordinate solely through the semaphore and the
+ * concurrent queue).
+ *
+ * A launcher runs inside drain() and may call release() or submit() in the same
+ * thread (e.g., its request was cancelled while queued). Rather than recurring once
+ * per pending task, the nested call returns, and the loop of the enclosing drain() in
+ * this same thread picks up the change on its next iteration.
+ */
+ private final ThreadLocal drainingInThisThread
+ = ThreadLocal.withInitial(() -> false);
+
+ public BulkTaskThrottle(int permits) {
+ this.permits = new Semaphore(permits);
+ }
+
+ public int availablePermits() {
+ return permits.availablePermits();
+ }
+
+ public int pendingCount() {
+ return pending.size();
+ }
+
+ /**
+ * Queue the launcher; it is run (possibly in the calling thread) as soon as a permit is
+ * available. Never blocks waiting for a permit.
+ */
+ public void submit(Runnable launcher) {
+ pending.add(launcher);
+ drain();
+ }
+
+ /**
+ * Return a permit acquired on behalf of a launcher, and launch pending tasks if any.
+ */
+ public void release() {
+ permits.release();
+ drain();
+ }
+
+ /*
+ * Both submit and release publish their change (enqueue / release) before checking
+ * for the other condition, so a pending launcher cannot be stranded while a permit
+ * is available.
+ */
+ private void drain() {
+ if (drainingInThisThread.get()) {
+ return;
+ }
+
+ drainingInThisThread.set(true);
+ try {
+ while (!pending.isEmpty() && permits.tryAcquire()) {
+ Runnable launcher = pending.poll();
+ if (launcher == null) {
+ permits.release();
+ continue;
+ }
+
+ try {
+ launcher.run();
+ } catch (RuntimeException e) {
+ LOGGER.error("Bulk task launcher failed; please report to support@dcache.org",
+ e);
+ permits.release();
+ }
+ }
+ } finally {
+ drainingInThisThread.set(false);
+ }
+ }
+}
diff --git a/modules/dcache-bulk/src/test/java/org/dcache/services/bulk/job/BulkRequestContainerJobTest.java b/modules/dcache-bulk/src/test/java/org/dcache/services/bulk/job/BulkRequestContainerJobTest.java
new file mode 100644
index 00000000000..22356da4658
--- /dev/null
+++ b/modules/dcache-bulk/src/test/java/org/dcache/services/bulk/job/BulkRequestContainerJobTest.java
@@ -0,0 +1,302 @@
+/*
+COPYRIGHT STATUS:
+Dec 1st 2001, Fermi National Accelerator Laboratory (FNAL) documents and
+software are sponsored by the U.S. Department of Energy under Contract No.
+DE-AC02-76CH03000. Therefore, the U.S. Government retains a world-wide
+non-exclusive, royalty-free license to publish or reproduce these documents
+and software for U.S. Government purposes. All documents and software
+available from this server are private under the U.S. and Foreign
+Copyright Laws, and FNAL reserves all rights.
+
+Distribution of the software available from this server is free of
+charge subject to the user following the terms of the Fermitools
+Software Legal Information.
+
+Redistribution and/or modification of the software shall be accompanied
+by the Fermitools Software Legal Information (including the copyright
+notice).
+
+The user is asked to feed back problems, benefits, and/or suggestions
+about the software to the Fermilab Software Providers.
+
+Neither the name of Fermilab, the URA, nor the names of the contributors
+may be used to endorse or promote products derived from this software
+without specific prior written permission.
+
+DISCLAIMER OF LIABILITY (BSD):
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL FERMILAB,
+OR THE URA, OR THE U.S. DEPARTMENT of ENERGY, OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Liabilities of the Government:
+
+This software is provided by URA, independent from its Prime Contract
+with the U.S. Department of Energy. URA is acting independently from
+the Government and in its own private capacity and is not acting on
+behalf of the U.S. Government, nor as its contractor nor its agent.
+Correspondingly, it is understood and agreed that the U.S. Government
+has no connection to this software and in no manner whatsoever shall
+be liable for nor assume any responsibility or obligation for any claim,
+cost, or damages arising out of or resulting from the use of the software
+available from this server.
+
+Export Control:
+
+All documents and software available from this server are subject to U.S.
+export control laws. Anyone downloading information from this server is
+obligated to secure any necessary Government licenses before exporting
+documents or software obtained from this server.
+ */
+package org.dcache.services.bulk.job;
+
+import static org.dcache.services.bulk.util.BulkRequestTarget.PLACEHOLDER_PNFSID;
+import static org.dcache.services.bulk.util.BulkRequestTarget.ROOT_REQUEST_PATH;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import diskCacheV111.util.FsPath;
+import diskCacheV111.util.PnfsHandler;
+import diskCacheV111.util.PnfsId;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.dcache.namespace.FileType;
+import org.dcache.services.bulk.BulkRequest;
+import org.dcache.services.bulk.BulkRequest.Depth;
+import org.dcache.services.bulk.activity.BulkActivity;
+import org.dcache.services.bulk.store.BulkTargetStore;
+import org.dcache.services.bulk.util.BulkRequestTarget;
+import org.dcache.services.bulk.util.BulkRequestTarget.PID;
+import org.dcache.services.bulk.util.BulkRequestTarget.State;
+import org.dcache.services.bulk.util.BulkRequestTargetBuilder;
+import org.dcache.services.bulk.util.BulkTaskThrottle;
+import org.dcache.util.list.DirectoryEntry;
+import org.dcache.util.list.DirectoryStream;
+import org.dcache.util.list.ListDirectoryHandler;
+import org.dcache.vehicles.FileAttributes;
+import org.dcache.vehicles.PnfsGetFileAttributes;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Recursive expansion of a directory tree wider than the number of directory listing permits
+ * (GitHub issue #8062).
+ */
+public class BulkRequestContainerJobTest {
+
+ private static final FsPath ROOT = FsPath.create("/test");
+ private static final int FANOUT = 5;
+ private static final int DEPTH = 4;
+ private static final int LIST_PERMITS = 2;
+
+ private final AtomicLong pnfsIds = new AtomicLong();
+ private final Map performed = new ConcurrentHashMap<>();
+ private final List performedDirs = Collections.synchronizedList(new ArrayList<>());
+
+ private volatile long listDelayMs;
+
+ private ExecutorService executor;
+ private ExecutorService listExecutor;
+ private ExecutorService callbackExecutor;
+ private BulkTaskThrottle throttle;
+ private BulkRequestTarget jobTarget;
+ private BulkRequestContainerJob job;
+
+ @Before
+ public void setUp() throws Exception {
+ executor = Executors.newCachedThreadPool();
+ /*
+ * As in production, the list executor has as many threads as there are permits.
+ */
+ listExecutor = Executors.newFixedThreadPool(LIST_PERMITS);
+ callbackExecutor = Executors.newCachedThreadPool();
+ throttle = new BulkTaskThrottle(LIST_PERMITS);
+
+ BulkRequest request = new BulkRequest();
+ request.setId(1L);
+ request.setUid("test-request");
+ request.setActivity("TEST");
+ request.setExpandDirectories(Depth.ALL);
+ request.setTargetPrefix("/");
+
+ BulkActivity