From e78d2bbbecd54a0fb4071549a835c33a6c1acdc2 Mon Sep 17 00:00:00 2001 From: Dmitry Litvintsev Date: Fri, 20 Nov 2026 15:52:39 -0600 Subject: [PATCH] bulk: fix deadlock in recursive directory expansion Motivation: ----------- Issue https://github.com/dCache/dcache/issues/8062 reports that recursive bulk requests (e.g. DELETE with expandDirectories=ALL) on large directory trees stay in RUNNING state forever, with "Available permits for directory listing" dropping to 0. Increasing bulk.limits.dir-list-semaphore only delays the hang and, at very high values, exhausts the JVM memory. A directory listing task holds a dir-list permit while iterating over the directory and, for each subdirectory found, blocks acquiring a permit for the child listing. Once the number of directories being listed concurrently reaches the number of permits, all permits are held by tasks waiting for permits and nothing progresses. Since the semaphore is shared by all requests, a single large request also blocks directory listing for every other request. A similar cycle exists for an initial directory target holding an in-flight permit while waiting for a dir-list permit. Modification: ------------- Replaced the dir-list semaphore with a new non-blocking BulkTaskThrottle, shared by all requests. Directory listing tasks are registered as running immediately (so that the transition to directory processing still waits for them) and queued in the throttle; a queued listing is launched as soon as a permit is available, and every released permit launches the next queued listing, regardless of which request it belongs to. Submission never blocks, so a listing task can queue its subdirectories while holding its own permit. A listing cancelled while queued returns its permit without running. A listing rejected by the executor fails the request instead of silently skipping the subtree. Added "Directory listings waiting for permit" to the service info output. Added unit tests for BulkTaskThrottle and for BulkRequestContainerJob expanding a directory tree wider than the number of listing permits (this test hangs without the fix), including cancellation during expansion. Result: ------- Recursive bulk requests on large directory trees complete with the default bulk.limits.dir-list-semaphore, which now only limits the number of concurrent directory listings sent to PnfsManager. Ticket: https://github.com/dCache/dcache/issues/8062 Acked-by: Tigran Target: trunk Request: 11.2, 12.0, 12.1 Require-book: no Require-notes: yes --- .../bulk/job/BulkRequestContainerJob.java | 106 +++++- .../bulk/job/RequestContainerJobFactory.java | 13 +- .../bulk/util/BulkServiceStatistics.java | 2 + .../services/bulk/util/BulkTaskThrottle.java | 160 ++++++++++ .../bulk/job/BulkRequestContainerJobTest.java | 302 ++++++++++++++++++ .../bulk/util/BulkTaskThrottleTest.java | 185 +++++++++++ 6 files changed, 749 insertions(+), 19 deletions(-) create mode 100644 modules/dcache-bulk/src/main/java/org/dcache/services/bulk/util/BulkTaskThrottle.java create mode 100644 modules/dcache-bulk/src/test/java/org/dcache/services/bulk/job/BulkRequestContainerJobTest.java create mode 100644 modules/dcache-bulk/src/test/java/org/dcache/services/bulk/util/BulkTaskThrottleTest.java diff --git a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/job/BulkRequestContainerJob.java b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/job/BulkRequestContainerJob.java index c7d0269ff24..63a10da53e9 100644 --- a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/job/BulkRequestContainerJob.java +++ b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/job/BulkRequestContainerJob.java @@ -93,6 +93,7 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.Semaphore; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -113,6 +114,7 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING import org.dcache.services.bulk.util.BulkRequestTarget.State; import org.dcache.services.bulk.util.BulkRequestTargetBuilder; import org.dcache.services.bulk.util.BulkServiceStatistics; +import org.dcache.services.bulk.util.BulkTaskThrottle; import org.dcache.util.CacheExceptionFactory; import org.dcache.util.SignalAware; import org.dcache.util.list.DirectoryEntry; @@ -174,12 +176,16 @@ public int compare(DirTarget o1, DirTarget o2) { } /** - * Encapsulates manipulation of the semaphore. + * Encapsulates manipulation of the semaphore or throttle. + *

+ * A task uses either a semaphore (blocking acquisition) or a throttle (non-blocking; the + * permit is granted by the throttle when the task is launched). */ class PermitHolder { private final AtomicBoolean holdingPermit = new AtomicBoolean(false); private Semaphore taskSemaphore; + private BulkTaskThrottle taskThrottle; void acquireIfNotHoldingPermit() throws InterruptedException { if (taskSemaphore == null) { @@ -191,24 +197,35 @@ void acquireIfNotHoldingPermit() throws InterruptedException { } } + /** + * The throttle has acquired a permit on behalf of this task. + */ + void permitGranted() { + holdingPermit.set(true); + } + void throttledRelease() { activity.throttle(); releaseIfHoldingPermit(); } void releaseIfHoldingPermit() { - if (taskSemaphore == null) { - return; - } - if (holdingPermit.compareAndSet(true, false)) { - taskSemaphore.release(); + if (taskThrottle != null) { + taskThrottle.release(); + } else if (taskSemaphore != null) { + taskSemaphore.release(); + } } } void setTaskSemaphore(Semaphore taskSemaphore) { this.taskSemaphore = taskSemaphore; } + + void setTaskThrottle(BulkTaskThrottle taskThrottle) { + this.taskThrottle = taskThrottle; + } } /** @@ -258,11 +275,10 @@ void expandDepthFirst(Long id, PID pid, FsPath path, FileAttributes dirAttribute throws BulkServiceException, CacheException { LOGGER.debug("{} - expandDepthFirst, {}, {}, {}, {}", ruid, id, pid, path, dirAttributes); - try { - new DirListTask(id, pid, path, dirAttributes).submitAsync(); - } catch (InterruptedException e) { - LOGGER.trace("{} - expandDepthFirst {} interrupted.", ruid, id); - } + /* + * Does not block: the listing is queued until a permit is available. + */ + new DirListTask(id, pid, path, dirAttributes).submitAsync(); } void submitAsync() throws InterruptedException { @@ -316,7 +332,67 @@ class DirListTask extends ContainerTask { this.path = path; this.dirAttributes = dirAttributes; taskExecutor = listExecutor; - permitHolder.setTaskSemaphore(dirListSemaphore); + permitHolder.setTaskThrottle(dirListThrottle); + } + + /** + * Registers the task as running and queues it for listing without blocking. + *

+ * Blocking for a permit here would deadlock: this is called from listing tasks which + * hold permits themselves, so a sufficiently wide directory tree would end up with all + * permits held by tasks waiting for permits (GitHub issue #8062). + *

+ * The task is added to running immediately so that the transition to directory + * processing waits for queued listings as well. + */ + @Override + void submitAsync() { + synchronized (running) { + if (jobTarget.isTerminated()) { + return; + } + + running.put(seqNo, this); + LOGGER.debug("{} - submitAsync (queued) {}, task count is now {}.", ruid, seqNo, + running.size()); + } + + dirListThrottle.submit(this::launch); + } + + /** + * Called by the throttle holding a permit for this task. + */ + private void launch() { + permitHolder.permitGranted(); + + RejectedExecutionException rejected = null; + synchronized (running) { + if (!jobTarget.isTerminated() && running.containsKey(seqNo)) { + try { + taskFuture = taskExecutor.submit(this); + return; + } catch (RejectedExecutionException e) { + rejected = e; + } + } + } + + if (rejected != null) { + /* + * Fail fast rather than silently skipping this subtree. + */ + LOGGER.error("{} - could not launch listing of {}: {}.", ruid, path, + rejected.toString()); + containerState = ContainerState.STOP; + jobTarget.setErrorObject(rejected); + update(); + } + + /* + * Cancelled while queued (or rejected): return the permit. + */ + remove(); } void doInner() { @@ -703,7 +779,7 @@ private void storeOrUpdate(Throwable error) { private ExecutorService executor; private ExecutorService listExecutor; private ExecutorService callbackExecutor; - private Semaphore dirListSemaphore; + private BulkTaskThrottle dirListThrottle; private Semaphore inFlightSemaphore; private volatile ContainerState containerState; @@ -882,8 +958,8 @@ public void run() { } } - public void setDirListSemaphore(Semaphore dirListSemaphore) { - this.dirListSemaphore = dirListSemaphore; + public void setDirListThrottle(BulkTaskThrottle dirListThrottle) { + this.dirListThrottle = dirListThrottle; } public void setInFlightSemaphore(Semaphore inFlightSemaphore) { diff --git a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/job/RequestContainerJobFactory.java b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/job/RequestContainerJobFactory.java index 5e145773772..e7a6d0d5e05 100644 --- a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/job/RequestContainerJobFactory.java +++ b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/job/RequestContainerJobFactory.java @@ -80,6 +80,7 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING import org.dcache.services.bulk.util.BulkRequestTarget.PID; import org.dcache.services.bulk.util.BulkRequestTargetBuilder; import org.dcache.services.bulk.util.BulkServiceStatistics; +import org.dcache.services.bulk.util.BulkTaskThrottle; import org.dcache.util.BoundedCachedExecutor; import org.dcache.util.list.ListDirectoryHandler; import org.dcache.vehicles.FileAttributes; @@ -101,7 +102,7 @@ public final class RequestContainerJobFactory { private ListDirectoryHandler listHandler; private BulkTargetStore targetStore; private BulkServiceStatistics statistics; - private Semaphore dirListSemaphore; + private BulkTaskThrottle dirListThrottle; private Semaphore inFlightSemaphore; private BoundedCachedExecutor taskExecutor; private BoundedCachedExecutor callbackExecutor; @@ -134,7 +135,7 @@ public BulkRequestContainerJob createRequestJob(BulkRequest request) containerJob.setNamespaceHandler(pnfsHandler); containerJob.setTargetStore(targetStore); containerJob.setListHandler(listHandler); - containerJob.setDirListSemaphore(dirListSemaphore); + containerJob.setDirListThrottle(dirListThrottle); containerJob.setInFlightSemaphore(inFlightSemaphore); containerJob.setExecutor(taskExecutor); containerJob.setListExecutor(listExecutor); @@ -144,7 +145,11 @@ public BulkRequestContainerJob createRequestJob(BulkRequest request) } public int getDirListSemaphoreAvailable() { - return dirListSemaphore.availablePermits(); + return dirListThrottle.availablePermits(); + } + + public int getDirListPending() { + return dirListThrottle.pendingCount(); } public int getInFlightSemaphoreAvailable() { @@ -173,7 +178,7 @@ public void setListExecutor(BoundedCachedExecutor listExecutor) { @Required public void setDirListSemaphore(int permits) { - dirListSemaphore = new Semaphore(permits); + dirListThrottle = new BulkTaskThrottle(permits); } @Required diff --git a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/util/BulkServiceStatistics.java b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/util/BulkServiceStatistics.java index 461c1173777..3729c0925a8 100644 --- a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/util/BulkServiceStatistics.java +++ b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/util/BulkServiceStatistics.java @@ -180,6 +180,8 @@ public void getInfo(PrintWriter pw) { pw.println(String.format(CONCURRENCY_FORMAT, "Available permits for directory listing", factory.getDirListSemaphoreAvailable())); + pw.println(String.format(CONCURRENCY_FORMAT, "Directory listings waiting for permit", + factory.getDirListPending())); pw.println(String.format(CONCURRENCY_FORMAT, "Available permits for in-flight targets", factory.getInFlightSemaphoreAvailable())); pw.println(); diff --git a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/util/BulkTaskThrottle.java b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/util/BulkTaskThrottle.java new file mode 100644 index 00000000000..f97840e99f2 --- /dev/null +++ b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/util/BulkTaskThrottle.java @@ -0,0 +1,160 @@ +/* +COPYRIGHT STATUS: +Dec 1st 2001, Fermi National Accelerator Laboratory (FNAL) documents and +software are sponsored by the U.S. Department of Energy under Contract No. +DE-AC02-76CH03000. Therefore, the U.S. Government retains a world-wide +non-exclusive, royalty-free license to publish or reproduce these documents +and software for U.S. Government purposes. All documents and software +available from this server are private under the U.S. and Foreign +Copyright Laws, and FNAL reserves all rights. + +Distribution of the software available from this server is free of +charge subject to the user following the terms of the Fermitools +Software Legal Information. + +Redistribution and/or modification of the software shall be accompanied +by the Fermitools Software Legal Information (including the copyright +notice). + +The user is asked to feed back problems, benefits, and/or suggestions +about the software to the Fermilab Software Providers. + +Neither the name of Fermilab, the URA, nor the names of the contributors +may be used to endorse or promote products derived from this software +without specific prior written permission. + +DISCLAIMER OF LIABILITY (BSD): + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL FERMILAB, +OR THE URA, OR THE U.S. DEPARTMENT of ENERGY, OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT +OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Liabilities of the Government: + +This software is provided by URA, independent from its Prime Contract +with the U.S. Department of Energy. URA is acting independently from +the Government and in its own private capacity and is not acting on +behalf of the U.S. Government, nor as its contractor nor its agent. +Correspondingly, it is understood and agreed that the U.S. Government +has no connection to this software and in no manner whatsoever shall +be liable for nor assume any responsibility or obligation for any claim, +cost, or damages arising out of or resulting from the use of the software +available from this server. + +Export Control: + +All documents and software available from this server are subject to U.S. +export control laws. Anyone downloading information from this server is +obligated to secure any necessary Government licenses before exporting +documents or software obtained from this server. + */ +package org.dcache.services.bulk.util; + +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.Semaphore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Non-blocking admission control for bulk tasks. + *

+ * Tasks are queued and launched only when a permit is available. Submission never blocks, so a + * task holding a permit may safely submit further tasks (e.g., a directory listing discovering + * subdirectories) without risk of all permits being held by tasks waiting for permits. + *

+ * A launcher is invoked holding a permit, which it owns from that point on; it must eventually + * call {@link #release()} exactly once. If the launcher throws instead, the permit is returned + * by the throttle and the launcher must not release it. The throttle may be shared among + * several requests. + */ +public final class BulkTaskThrottle { + + private static final Logger LOGGER = LoggerFactory.getLogger(BulkTaskThrottle.class); + + private final Semaphore permits; + private final Queue pending = new ConcurrentLinkedQueue<>(); + + /* + * Per-thread re-entrancy guard; it does NOT exclude other threads from draining + * concurrently (concurrent drains coordinate solely through the semaphore and the + * concurrent queue). + * + * A launcher runs inside drain() and may call release() or submit() in the same + * thread (e.g., its request was cancelled while queued). Rather than recurring once + * per pending task, the nested call returns, and the loop of the enclosing drain() in + * this same thread picks up the change on its next iteration. + */ + private final ThreadLocal drainingInThisThread + = ThreadLocal.withInitial(() -> false); + + public BulkTaskThrottle(int permits) { + this.permits = new Semaphore(permits); + } + + public int availablePermits() { + return permits.availablePermits(); + } + + public int pendingCount() { + return pending.size(); + } + + /** + * Queue the launcher; it is run (possibly in the calling thread) as soon as a permit is + * available. Never blocks waiting for a permit. + */ + public void submit(Runnable launcher) { + pending.add(launcher); + drain(); + } + + /** + * Return a permit acquired on behalf of a launcher, and launch pending tasks if any. + */ + public void release() { + permits.release(); + drain(); + } + + /* + * Both submit and release publish their change (enqueue / release) before checking + * for the other condition, so a pending launcher cannot be stranded while a permit + * is available. + */ + private void drain() { + if (drainingInThisThread.get()) { + return; + } + + drainingInThisThread.set(true); + try { + while (!pending.isEmpty() && permits.tryAcquire()) { + Runnable launcher = pending.poll(); + if (launcher == null) { + permits.release(); + continue; + } + + try { + launcher.run(); + } catch (RuntimeException e) { + LOGGER.error("Bulk task launcher failed; please report to support@dcache.org", + e); + permits.release(); + } + } + } finally { + drainingInThisThread.set(false); + } + } +} diff --git a/modules/dcache-bulk/src/test/java/org/dcache/services/bulk/job/BulkRequestContainerJobTest.java b/modules/dcache-bulk/src/test/java/org/dcache/services/bulk/job/BulkRequestContainerJobTest.java new file mode 100644 index 00000000000..22356da4658 --- /dev/null +++ b/modules/dcache-bulk/src/test/java/org/dcache/services/bulk/job/BulkRequestContainerJobTest.java @@ -0,0 +1,302 @@ +/* +COPYRIGHT STATUS: +Dec 1st 2001, Fermi National Accelerator Laboratory (FNAL) documents and +software are sponsored by the U.S. Department of Energy under Contract No. +DE-AC02-76CH03000. Therefore, the U.S. Government retains a world-wide +non-exclusive, royalty-free license to publish or reproduce these documents +and software for U.S. Government purposes. All documents and software +available from this server are private under the U.S. and Foreign +Copyright Laws, and FNAL reserves all rights. + +Distribution of the software available from this server is free of +charge subject to the user following the terms of the Fermitools +Software Legal Information. + +Redistribution and/or modification of the software shall be accompanied +by the Fermitools Software Legal Information (including the copyright +notice). + +The user is asked to feed back problems, benefits, and/or suggestions +about the software to the Fermilab Software Providers. + +Neither the name of Fermilab, the URA, nor the names of the contributors +may be used to endorse or promote products derived from this software +without specific prior written permission. + +DISCLAIMER OF LIABILITY (BSD): + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL FERMILAB, +OR THE URA, OR THE U.S. DEPARTMENT of ENERGY, OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT +OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Liabilities of the Government: + +This software is provided by URA, independent from its Prime Contract +with the U.S. Department of Energy. URA is acting independently from +the Government and in its own private capacity and is not acting on +behalf of the U.S. Government, nor as its contractor nor its agent. +Correspondingly, it is understood and agreed that the U.S. Government +has no connection to this software and in no manner whatsoever shall +be liable for nor assume any responsibility or obligation for any claim, +cost, or damages arising out of or resulting from the use of the software +available from this server. + +Export Control: + +All documents and software available from this server are subject to U.S. +export control laws. Anyone downloading information from this server is +obligated to secure any necessary Government licenses before exporting +documents or software obtained from this server. + */ +package org.dcache.services.bulk.job; + +import static org.dcache.services.bulk.util.BulkRequestTarget.PLACEHOLDER_PNFSID; +import static org.dcache.services.bulk.util.BulkRequestTarget.ROOT_REQUEST_PATH; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import diskCacheV111.util.FsPath; +import diskCacheV111.util.PnfsHandler; +import diskCacheV111.util.PnfsId; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import org.dcache.namespace.FileType; +import org.dcache.services.bulk.BulkRequest; +import org.dcache.services.bulk.BulkRequest.Depth; +import org.dcache.services.bulk.activity.BulkActivity; +import org.dcache.services.bulk.store.BulkTargetStore; +import org.dcache.services.bulk.util.BulkRequestTarget; +import org.dcache.services.bulk.util.BulkRequestTarget.PID; +import org.dcache.services.bulk.util.BulkRequestTarget.State; +import org.dcache.services.bulk.util.BulkRequestTargetBuilder; +import org.dcache.services.bulk.util.BulkTaskThrottle; +import org.dcache.util.list.DirectoryEntry; +import org.dcache.util.list.DirectoryStream; +import org.dcache.util.list.ListDirectoryHandler; +import org.dcache.vehicles.FileAttributes; +import org.dcache.vehicles.PnfsGetFileAttributes; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** + * Recursive expansion of a directory tree wider than the number of directory listing permits + * (GitHub issue #8062). + */ +public class BulkRequestContainerJobTest { + + private static final FsPath ROOT = FsPath.create("/test"); + private static final int FANOUT = 5; + private static final int DEPTH = 4; + private static final int LIST_PERMITS = 2; + + private final AtomicLong pnfsIds = new AtomicLong(); + private final Map performed = new ConcurrentHashMap<>(); + private final List performedDirs = Collections.synchronizedList(new ArrayList<>()); + + private volatile long listDelayMs; + + private ExecutorService executor; + private ExecutorService listExecutor; + private ExecutorService callbackExecutor; + private BulkTaskThrottle throttle; + private BulkRequestTarget jobTarget; + private BulkRequestContainerJob job; + + @Before + public void setUp() throws Exception { + executor = Executors.newCachedThreadPool(); + /* + * As in production, the list executor has as many threads as there are permits. + */ + listExecutor = Executors.newFixedThreadPool(LIST_PERMITS); + callbackExecutor = Executors.newCachedThreadPool(); + throttle = new BulkTaskThrottle(LIST_PERMITS); + + BulkRequest request = new BulkRequest(); + request.setId(1L); + request.setUid("test-request"); + request.setActivity("TEST"); + request.setExpandDirectories(Depth.ALL); + request.setTargetPrefix("/"); + + BulkActivity activity = new BulkActivity<>("TEST", + BulkActivity.TargetType.BOTH) { + @Override + public ListenableFuture perform(String rid, long tid, String prefix, + FsPath path, FileAttributes attributes) { + performed.merge(path, 1, Integer::sum); + if (attributes.getFileType() == FileType.DIR) { + performedDirs.add(path); + } + return Futures.immediateFuture(path); + } + + @Override + protected void configure(Map arguments) { + } + + @Override + public void handleCompletion(BulkRequestTarget target, Future future) { + target.setState(State.COMPLETED); + } + }; + + FileAttributes placeholder = new FileAttributes(); + placeholder.setFileType(FileType.SPECIAL); + placeholder.setPnfsId(PLACEHOLDER_PNFSID); + jobTarget = BulkRequestTargetBuilder.builder(null).activity("TEST").rid(1L) + .ruid("test-request").pid(PID.ROOT).attributes(placeholder) + .path(ROOT_REQUEST_PATH).build(); + + BulkRequestTarget initial = BulkRequestTargetBuilder.builder(null).activity("TEST") + .id(2L).rid(1L).ruid("test-request").pid(PID.INITIAL).path(ROOT).build(); + + BulkTargetStore targetStore = mock(BulkTargetStore.class); + when(targetStore.getInitialTargets(1L, true)).thenReturn(List.of(initial)); + + PnfsHandler pnfsHandler = mock(PnfsHandler.class); + when(pnfsHandler.requestAsync(any())).thenAnswer(invocation -> { + PnfsGetFileAttributes message = invocation.getArgument(0); + message.setFileAttributes(attributes(FileType.DIR)); + return Futures.immediateFuture(message); + }); + + ListDirectoryHandler listHandler = mock(ListDirectoryHandler.class); + when(listHandler.list(any(), any(), any(), any(), any(), any())).thenAnswer( + invocation -> listing(invocation.getArgument(2))); + + job = new BulkRequestContainerJob(activity, jobTarget, request, null); + job.setNamespaceHandler(pnfsHandler); + job.setTargetStore(targetStore); + job.setListHandler(listHandler); + job.setDirListThrottle(throttle); + job.setInFlightSemaphore(new Semaphore(100)); + job.setExecutor(executor); + job.setListExecutor(listExecutor); + job.setCallbackExecutor(callbackExecutor); + job.initialize(); + } + + @After + public void tearDown() { + executor.shutdownNow(); + listExecutor.shutdownNow(); + callbackExecutor.shutdownNow(); + } + + @Test(timeout = 60_000) + public void shouldCompleteRecursiveExpansionWiderThanListingPermits() throws Exception { + executor.submit(job); + + while (jobTarget.getState() != State.COMPLETED) { + assertTrue("job terminated in state " + jobTarget.getState(), + !jobTarget.isTerminated() || jobTarget.getState() == State.COMPLETED); + TimeUnit.MILLISECONDS.sleep(20); + } + + int dirs = 0; + int files = 0; + for (int level = 0, width = 1; level <= DEPTH; ++level, width *= FANOUT) { + dirs += width; /* directories at this level */ + files += width; /* each directory holds one file */ + } + + assertEquals(dirs + files, performed.size()); + assertTrue("some target performed more than once", + performed.values().stream().allMatch(n -> n == 1)); + assertEquals(dirs, performedDirs.size()); + + /* + * Directories must be processed deepest first (children before parents). + */ + for (int i = 1; i < performedDirs.size(); ++i) { + assertTrue(depth(performedDirs.get(i - 1)) >= depth(performedDirs.get(i))); + } + + assertEquals(LIST_PERMITS, throttle.availablePermits()); + assertEquals(0, throttle.pendingCount()); + } + + @Test(timeout = 60_000) + public void shouldReturnAllListingPermitsWhenCancelledDuringExpansion() throws Exception { + listDelayMs = 5; + executor.submit(job); + + while (performed.size() < 20) { + TimeUnit.MILLISECONDS.sleep(5); + } + + assertTrue("listings should be waiting for permits", throttle.pendingCount() > 0); + + job.cancel(); + + assertEquals(State.CANCELLED, jobTarget.getState()); + while (throttle.availablePermits() != LIST_PERMITS || throttle.pendingCount() != 0) { + TimeUnit.MILLISECONDS.sleep(10); + } + + assertTrue("no directory should be processed after cancellation", + performedDirs.isEmpty()); + } + + private FileAttributes attributes(FileType type) { + FileAttributes attributes = new FileAttributes(); + attributes.setFileType(type); + attributes.setPnfsId(new PnfsId(String.format("%036X", pnfsIds.incrementAndGet()))); + return attributes; + } + + private static int depth(FsPath path) { + return path.toString().split("/").length; + } + + private DirectoryStream listing(FsPath path) throws InterruptedException { + if (listDelayMs > 0) { + TimeUnit.MILLISECONDS.sleep(listDelayMs); + } + + List entries = new ArrayList<>(); + entries.add(new DirectoryEntry("file", attributes(FileType.REGULAR))); + if (depth(path) - depth(ROOT) < DEPTH) { + for (int i = 0; i < FANOUT; ++i) { + entries.add(new DirectoryEntry("dir" + i, attributes(FileType.DIR))); + } + } + + return new DirectoryStream() { + @Override + public void close() { + } + + @Override + public Iterator iterator() { + return entries.iterator(); + } + }; + } +} diff --git a/modules/dcache-bulk/src/test/java/org/dcache/services/bulk/util/BulkTaskThrottleTest.java b/modules/dcache-bulk/src/test/java/org/dcache/services/bulk/util/BulkTaskThrottleTest.java new file mode 100644 index 00000000000..269004016f8 --- /dev/null +++ b/modules/dcache-bulk/src/test/java/org/dcache/services/bulk/util/BulkTaskThrottleTest.java @@ -0,0 +1,185 @@ +/* +COPYRIGHT STATUS: +Dec 1st 2001, Fermi National Accelerator Laboratory (FNAL) documents and +software are sponsored by the U.S. Department of Energy under Contract No. +DE-AC02-76CH03000. Therefore, the U.S. Government retains a world-wide +non-exclusive, royalty-free license to publish or reproduce these documents +and software for U.S. Government purposes. All documents and software +available from this server are private under the U.S. and Foreign +Copyright Laws, and FNAL reserves all rights. + +Distribution of the software available from this server is free of +charge subject to the user following the terms of the Fermitools +Software Legal Information. + +Redistribution and/or modification of the software shall be accompanied +by the Fermitools Software Legal Information (including the copyright +notice). + +The user is asked to feed back problems, benefits, and/or suggestions +about the software to the Fermilab Software Providers. + +Neither the name of Fermilab, the URA, nor the names of the contributors +may be used to endorse or promote products derived from this software +without specific prior written permission. + +DISCLAIMER OF LIABILITY (BSD): + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL FERMILAB, +OR THE URA, OR THE U.S. DEPARTMENT of ENERGY, OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT +OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Liabilities of the Government: + +This software is provided by URA, independent from its Prime Contract +with the U.S. Department of Energy. URA is acting independently from +the Government and in its own private capacity and is not acting on +behalf of the U.S. Government, nor as its contractor nor its agent. +Correspondingly, it is understood and agreed that the U.S. Government +has no connection to this software and in no manner whatsoever shall +be liable for nor assume any responsibility or obligation for any claim, +cost, or damages arising out of or resulting from the use of the software +available from this server. + +Export Control: + +All documents and software available from this server are subject to U.S. +export control laws. Anyone downloading information from this server is +obligated to secure any necessary Government licenses before exporting +documents or software obtained from this server. + */ +package org.dcache.services.bulk.util; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class BulkTaskThrottleTest { + + private static final int PERMITS = 3; + + private BulkTaskThrottle throttle; + private ExecutorService executor; + private final AtomicInteger active = new AtomicInteger(); + private final AtomicInteger maxActive = new AtomicInteger(); + + @Before + public void setUp() { + throttle = new BulkTaskThrottle(PERMITS); + executor = Executors.newCachedThreadPool(); + } + + @After + public void tearDown() { + executor.shutdownNow(); + } + + @Test + public void shouldRunLaunchersImmediatelyWhilePermitsAreAvailable() { + AtomicInteger launched = new AtomicInteger(); + for (int i = 0; i < PERMITS; ++i) { + throttle.submit(launched::incrementAndGet); + } + + assertEquals(PERMITS, launched.get()); + assertEquals(0, throttle.availablePermits()); + assertEquals(0, throttle.pendingCount()); + } + + @Test + public void shouldQueueWithoutBlockingWhenNoPermitIsAvailable() { + AtomicInteger launched = new AtomicInteger(); + for (int i = 0; i < PERMITS + 2; ++i) { + throttle.submit(launched::incrementAndGet); + } + + assertEquals(PERMITS, launched.get()); + assertEquals(2, throttle.pendingCount()); + + throttle.release(); + assertEquals(PERMITS + 1, launched.get()); + assertEquals(1, throttle.pendingCount()); + } + + @Test + public void shouldReturnPermitIfLauncherFails() { + throttle.submit(() -> { + throw new IllegalStateException("test"); + }); + + assertEquals(PERMITS, throttle.availablePermits()); + } + + @Test + public void shouldNotRecurWhenLaunchersReleaseImmediately() { + throttle = new BulkTaskThrottle(1); + throttle.submit(() -> { /* hold the only permit */ }); + for (int i = 0; i < 100_000; ++i) { + throttle.submit(throttle::release); + } + + throttle.release(); + assertEquals(1, throttle.availablePermits()); + assertEquals(0, throttle.pendingCount()); + } + + /* + * Each task submits children while holding its permit, as directory listing does. + * A blocking acquisition would deadlock here. + */ + @Test(timeout = 30_000) + public void shouldCompleteTreeOfNestedSubmissions() throws Exception { + int fanout = 4; + int depth = 6; + int total = 0; + for (int level = 0, width = 1; level <= depth; ++level, width *= fanout) { + total += width; + } + + CountDownLatch done = new CountDownLatch(total); + submitNode(0, depth, fanout, done); + assertTrue(done.await(30, TimeUnit.SECONDS)); + + while (throttle.availablePermits() != PERMITS) { + TimeUnit.MILLISECONDS.sleep(1); + } + + assertEquals(0, throttle.pendingCount()); + assertTrue("max concurrency " + maxActive.get(), maxActive.get() <= PERMITS); + } + + private void submitNode(int level, int depth, int fanout, CountDownLatch done) { + throttle.submit(() -> executor.execute(() -> { + int now = active.incrementAndGet(); + maxActive.accumulateAndGet(now, Math::max); + try { + if (level < depth) { + for (int i = 0; i < fanout; ++i) { + submitNode(level + 1, depth, fanout, done); + } + } + } finally { + active.decrementAndGet(); + done.countDown(); + throttle.release(); + } + })); + } +}