diff --git a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/job/BulkRequestContainerJob.java b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/job/BulkRequestContainerJob.java index c7d0269ff24..63a10da53e9 100644 --- a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/job/BulkRequestContainerJob.java +++ b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/job/BulkRequestContainerJob.java @@ -93,6 +93,7 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.Semaphore; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -113,6 +114,7 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING import org.dcache.services.bulk.util.BulkRequestTarget.State; import org.dcache.services.bulk.util.BulkRequestTargetBuilder; import org.dcache.services.bulk.util.BulkServiceStatistics; +import org.dcache.services.bulk.util.BulkTaskThrottle; import org.dcache.util.CacheExceptionFactory; import org.dcache.util.SignalAware; import org.dcache.util.list.DirectoryEntry; @@ -174,12 +176,16 @@ public int compare(DirTarget o1, DirTarget o2) { } /** - * Encapsulates manipulation of the semaphore. + * Encapsulates manipulation of the semaphore or throttle. + *

+ * A task uses either a semaphore (blocking acquisition) or a throttle (non-blocking; the + * permit is granted by the throttle when the task is launched). */ class PermitHolder { private final AtomicBoolean holdingPermit = new AtomicBoolean(false); private Semaphore taskSemaphore; + private BulkTaskThrottle taskThrottle; void acquireIfNotHoldingPermit() throws InterruptedException { if (taskSemaphore == null) { @@ -191,24 +197,35 @@ void acquireIfNotHoldingPermit() throws InterruptedException { } } + /** + * The throttle has acquired a permit on behalf of this task. + */ + void permitGranted() { + holdingPermit.set(true); + } + void throttledRelease() { activity.throttle(); releaseIfHoldingPermit(); } void releaseIfHoldingPermit() { - if (taskSemaphore == null) { - return; - } - if (holdingPermit.compareAndSet(true, false)) { - taskSemaphore.release(); + if (taskThrottle != null) { + taskThrottle.release(); + } else if (taskSemaphore != null) { + taskSemaphore.release(); + } } } void setTaskSemaphore(Semaphore taskSemaphore) { this.taskSemaphore = taskSemaphore; } + + void setTaskThrottle(BulkTaskThrottle taskThrottle) { + this.taskThrottle = taskThrottle; + } } /** @@ -258,11 +275,10 @@ void expandDepthFirst(Long id, PID pid, FsPath path, FileAttributes dirAttribute throws BulkServiceException, CacheException { LOGGER.debug("{} - expandDepthFirst, {}, {}, {}, {}", ruid, id, pid, path, dirAttributes); - try { - new DirListTask(id, pid, path, dirAttributes).submitAsync(); - } catch (InterruptedException e) { - LOGGER.trace("{} - expandDepthFirst {} interrupted.", ruid, id); - } + /* + * Does not block: the listing is queued until a permit is available. + */ + new DirListTask(id, pid, path, dirAttributes).submitAsync(); } void submitAsync() throws InterruptedException { @@ -316,7 +332,67 @@ class DirListTask extends ContainerTask { this.path = path; this.dirAttributes = dirAttributes; taskExecutor = listExecutor; - permitHolder.setTaskSemaphore(dirListSemaphore); + permitHolder.setTaskThrottle(dirListThrottle); + } + + /** + * Registers the task as running and queues it for listing without blocking. + *

+ * Blocking for a permit here would deadlock: this is called from listing tasks which + * hold permits themselves, so a sufficiently wide directory tree would end up with all + * permits held by tasks waiting for permits (GitHub issue #8062). + *

+ * The task is added to running immediately so that the transition to directory + * processing waits for queued listings as well. + */ + @Override + void submitAsync() { + synchronized (running) { + if (jobTarget.isTerminated()) { + return; + } + + running.put(seqNo, this); + LOGGER.debug("{} - submitAsync (queued) {}, task count is now {}.", ruid, seqNo, + running.size()); + } + + dirListThrottle.submit(this::launch); + } + + /** + * Called by the throttle holding a permit for this task. + */ + private void launch() { + permitHolder.permitGranted(); + + RejectedExecutionException rejected = null; + synchronized (running) { + if (!jobTarget.isTerminated() && running.containsKey(seqNo)) { + try { + taskFuture = taskExecutor.submit(this); + return; + } catch (RejectedExecutionException e) { + rejected = e; + } + } + } + + if (rejected != null) { + /* + * Fail fast rather than silently skipping this subtree. + */ + LOGGER.error("{} - could not launch listing of {}: {}.", ruid, path, + rejected.toString()); + containerState = ContainerState.STOP; + jobTarget.setErrorObject(rejected); + update(); + } + + /* + * Cancelled while queued (or rejected): return the permit. + */ + remove(); } void doInner() { @@ -703,7 +779,7 @@ private void storeOrUpdate(Throwable error) { private ExecutorService executor; private ExecutorService listExecutor; private ExecutorService callbackExecutor; - private Semaphore dirListSemaphore; + private BulkTaskThrottle dirListThrottle; private Semaphore inFlightSemaphore; private volatile ContainerState containerState; @@ -882,8 +958,8 @@ public void run() { } } - public void setDirListSemaphore(Semaphore dirListSemaphore) { - this.dirListSemaphore = dirListSemaphore; + public void setDirListThrottle(BulkTaskThrottle dirListThrottle) { + this.dirListThrottle = dirListThrottle; } public void setInFlightSemaphore(Semaphore inFlightSemaphore) { diff --git a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/job/RequestContainerJobFactory.java b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/job/RequestContainerJobFactory.java index 5e145773772..e7a6d0d5e05 100644 --- a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/job/RequestContainerJobFactory.java +++ b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/job/RequestContainerJobFactory.java @@ -80,6 +80,7 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING import org.dcache.services.bulk.util.BulkRequestTarget.PID; import org.dcache.services.bulk.util.BulkRequestTargetBuilder; import org.dcache.services.bulk.util.BulkServiceStatistics; +import org.dcache.services.bulk.util.BulkTaskThrottle; import org.dcache.util.BoundedCachedExecutor; import org.dcache.util.list.ListDirectoryHandler; import org.dcache.vehicles.FileAttributes; @@ -101,7 +102,7 @@ public final class RequestContainerJobFactory { private ListDirectoryHandler listHandler; private BulkTargetStore targetStore; private BulkServiceStatistics statistics; - private Semaphore dirListSemaphore; + private BulkTaskThrottle dirListThrottle; private Semaphore inFlightSemaphore; private BoundedCachedExecutor taskExecutor; private BoundedCachedExecutor callbackExecutor; @@ -134,7 +135,7 @@ public BulkRequestContainerJob createRequestJob(BulkRequest request) containerJob.setNamespaceHandler(pnfsHandler); containerJob.setTargetStore(targetStore); containerJob.setListHandler(listHandler); - containerJob.setDirListSemaphore(dirListSemaphore); + containerJob.setDirListThrottle(dirListThrottle); containerJob.setInFlightSemaphore(inFlightSemaphore); containerJob.setExecutor(taskExecutor); containerJob.setListExecutor(listExecutor); @@ -144,7 +145,11 @@ public BulkRequestContainerJob createRequestJob(BulkRequest request) } public int getDirListSemaphoreAvailable() { - return dirListSemaphore.availablePermits(); + return dirListThrottle.availablePermits(); + } + + public int getDirListPending() { + return dirListThrottle.pendingCount(); } public int getInFlightSemaphoreAvailable() { @@ -173,7 +178,7 @@ public void setListExecutor(BoundedCachedExecutor listExecutor) { @Required public void setDirListSemaphore(int permits) { - dirListSemaphore = new Semaphore(permits); + dirListThrottle = new BulkTaskThrottle(permits); } @Required diff --git a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/util/BulkServiceStatistics.java b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/util/BulkServiceStatistics.java index 461c1173777..3729c0925a8 100644 --- a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/util/BulkServiceStatistics.java +++ b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/util/BulkServiceStatistics.java @@ -180,6 +180,8 @@ public void getInfo(PrintWriter pw) { pw.println(String.format(CONCURRENCY_FORMAT, "Available permits for directory listing", factory.getDirListSemaphoreAvailable())); + pw.println(String.format(CONCURRENCY_FORMAT, "Directory listings waiting for permit", + factory.getDirListPending())); pw.println(String.format(CONCURRENCY_FORMAT, "Available permits for in-flight targets", factory.getInFlightSemaphoreAvailable())); pw.println(); diff --git a/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/util/BulkTaskThrottle.java b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/util/BulkTaskThrottle.java new file mode 100644 index 00000000000..f97840e99f2 --- /dev/null +++ b/modules/dcache-bulk/src/main/java/org/dcache/services/bulk/util/BulkTaskThrottle.java @@ -0,0 +1,160 @@ +/* +COPYRIGHT STATUS: +Dec 1st 2001, Fermi National Accelerator Laboratory (FNAL) documents and +software are sponsored by the U.S. Department of Energy under Contract No. +DE-AC02-76CH03000. Therefore, the U.S. Government retains a world-wide +non-exclusive, royalty-free license to publish or reproduce these documents +and software for U.S. Government purposes. All documents and software +available from this server are private under the U.S. and Foreign +Copyright Laws, and FNAL reserves all rights. + +Distribution of the software available from this server is free of +charge subject to the user following the terms of the Fermitools +Software Legal Information. + +Redistribution and/or modification of the software shall be accompanied +by the Fermitools Software Legal Information (including the copyright +notice). + +The user is asked to feed back problems, benefits, and/or suggestions +about the software to the Fermilab Software Providers. + +Neither the name of Fermilab, the URA, nor the names of the contributors +may be used to endorse or promote products derived from this software +without specific prior written permission. + +DISCLAIMER OF LIABILITY (BSD): + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL FERMILAB, +OR THE URA, OR THE U.S. DEPARTMENT of ENERGY, OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT +OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Liabilities of the Government: + +This software is provided by URA, independent from its Prime Contract +with the U.S. Department of Energy. URA is acting independently from +the Government and in its own private capacity and is not acting on +behalf of the U.S. Government, nor as its contractor nor its agent. +Correspondingly, it is understood and agreed that the U.S. Government +has no connection to this software and in no manner whatsoever shall +be liable for nor assume any responsibility or obligation for any claim, +cost, or damages arising out of or resulting from the use of the software +available from this server. + +Export Control: + +All documents and software available from this server are subject to U.S. +export control laws. Anyone downloading information from this server is +obligated to secure any necessary Government licenses before exporting +documents or software obtained from this server. + */ +package org.dcache.services.bulk.util; + +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.Semaphore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Non-blocking admission control for bulk tasks. + *

+ * Tasks are queued and launched only when a permit is available. Submission never blocks, so a + * task holding a permit may safely submit further tasks (e.g., a directory listing discovering + * subdirectories) without risk of all permits being held by tasks waiting for permits. + *

+ * A launcher is invoked holding a permit, which it owns from that point on; it must eventually + * call {@link #release()} exactly once. If the launcher throws instead, the permit is returned + * by the throttle and the launcher must not release it. The throttle may be shared among + * several requests. + */ +public final class BulkTaskThrottle { + + private static final Logger LOGGER = LoggerFactory.getLogger(BulkTaskThrottle.class); + + private final Semaphore permits; + private final Queue pending = new ConcurrentLinkedQueue<>(); + + /* + * Per-thread re-entrancy guard; it does NOT exclude other threads from draining + * concurrently (concurrent drains coordinate solely through the semaphore and the + * concurrent queue). + * + * A launcher runs inside drain() and may call release() or submit() in the same + * thread (e.g., its request was cancelled while queued). Rather than recurring once + * per pending task, the nested call returns, and the loop of the enclosing drain() in + * this same thread picks up the change on its next iteration. + */ + private final ThreadLocal drainingInThisThread + = ThreadLocal.withInitial(() -> false); + + public BulkTaskThrottle(int permits) { + this.permits = new Semaphore(permits); + } + + public int availablePermits() { + return permits.availablePermits(); + } + + public int pendingCount() { + return pending.size(); + } + + /** + * Queue the launcher; it is run (possibly in the calling thread) as soon as a permit is + * available. Never blocks waiting for a permit. + */ + public void submit(Runnable launcher) { + pending.add(launcher); + drain(); + } + + /** + * Return a permit acquired on behalf of a launcher, and launch pending tasks if any. + */ + public void release() { + permits.release(); + drain(); + } + + /* + * Both submit and release publish their change (enqueue / release) before checking + * for the other condition, so a pending launcher cannot be stranded while a permit + * is available. + */ + private void drain() { + if (drainingInThisThread.get()) { + return; + } + + drainingInThisThread.set(true); + try { + while (!pending.isEmpty() && permits.tryAcquire()) { + Runnable launcher = pending.poll(); + if (launcher == null) { + permits.release(); + continue; + } + + try { + launcher.run(); + } catch (RuntimeException e) { + LOGGER.error("Bulk task launcher failed; please report to support@dcache.org", + e); + permits.release(); + } + } + } finally { + drainingInThisThread.set(false); + } + } +} diff --git a/modules/dcache-bulk/src/test/java/org/dcache/services/bulk/job/BulkRequestContainerJobTest.java b/modules/dcache-bulk/src/test/java/org/dcache/services/bulk/job/BulkRequestContainerJobTest.java new file mode 100644 index 00000000000..22356da4658 --- /dev/null +++ b/modules/dcache-bulk/src/test/java/org/dcache/services/bulk/job/BulkRequestContainerJobTest.java @@ -0,0 +1,302 @@ +/* +COPYRIGHT STATUS: +Dec 1st 2001, Fermi National Accelerator Laboratory (FNAL) documents and +software are sponsored by the U.S. Department of Energy under Contract No. +DE-AC02-76CH03000. Therefore, the U.S. Government retains a world-wide +non-exclusive, royalty-free license to publish or reproduce these documents +and software for U.S. Government purposes. All documents and software +available from this server are private under the U.S. and Foreign +Copyright Laws, and FNAL reserves all rights. + +Distribution of the software available from this server is free of +charge subject to the user following the terms of the Fermitools +Software Legal Information. + +Redistribution and/or modification of the software shall be accompanied +by the Fermitools Software Legal Information (including the copyright +notice). + +The user is asked to feed back problems, benefits, and/or suggestions +about the software to the Fermilab Software Providers. + +Neither the name of Fermilab, the URA, nor the names of the contributors +may be used to endorse or promote products derived from this software +without specific prior written permission. + +DISCLAIMER OF LIABILITY (BSD): + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL FERMILAB, +OR THE URA, OR THE U.S. DEPARTMENT of ENERGY, OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT +OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Liabilities of the Government: + +This software is provided by URA, independent from its Prime Contract +with the U.S. Department of Energy. URA is acting independently from +the Government and in its own private capacity and is not acting on +behalf of the U.S. Government, nor as its contractor nor its agent. +Correspondingly, it is understood and agreed that the U.S. Government +has no connection to this software and in no manner whatsoever shall +be liable for nor assume any responsibility or obligation for any claim, +cost, or damages arising out of or resulting from the use of the software +available from this server. + +Export Control: + +All documents and software available from this server are subject to U.S. +export control laws. Anyone downloading information from this server is +obligated to secure any necessary Government licenses before exporting +documents or software obtained from this server. + */ +package org.dcache.services.bulk.job; + +import static org.dcache.services.bulk.util.BulkRequestTarget.PLACEHOLDER_PNFSID; +import static org.dcache.services.bulk.util.BulkRequestTarget.ROOT_REQUEST_PATH; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import diskCacheV111.util.FsPath; +import diskCacheV111.util.PnfsHandler; +import diskCacheV111.util.PnfsId; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import org.dcache.namespace.FileType; +import org.dcache.services.bulk.BulkRequest; +import org.dcache.services.bulk.BulkRequest.Depth; +import org.dcache.services.bulk.activity.BulkActivity; +import org.dcache.services.bulk.store.BulkTargetStore; +import org.dcache.services.bulk.util.BulkRequestTarget; +import org.dcache.services.bulk.util.BulkRequestTarget.PID; +import org.dcache.services.bulk.util.BulkRequestTarget.State; +import org.dcache.services.bulk.util.BulkRequestTargetBuilder; +import org.dcache.services.bulk.util.BulkTaskThrottle; +import org.dcache.util.list.DirectoryEntry; +import org.dcache.util.list.DirectoryStream; +import org.dcache.util.list.ListDirectoryHandler; +import org.dcache.vehicles.FileAttributes; +import org.dcache.vehicles.PnfsGetFileAttributes; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** + * Recursive expansion of a directory tree wider than the number of directory listing permits + * (GitHub issue #8062). + */ +public class BulkRequestContainerJobTest { + + private static final FsPath ROOT = FsPath.create("/test"); + private static final int FANOUT = 5; + private static final int DEPTH = 4; + private static final int LIST_PERMITS = 2; + + private final AtomicLong pnfsIds = new AtomicLong(); + private final Map performed = new ConcurrentHashMap<>(); + private final List performedDirs = Collections.synchronizedList(new ArrayList<>()); + + private volatile long listDelayMs; + + private ExecutorService executor; + private ExecutorService listExecutor; + private ExecutorService callbackExecutor; + private BulkTaskThrottle throttle; + private BulkRequestTarget jobTarget; + private BulkRequestContainerJob job; + + @Before + public void setUp() throws Exception { + executor = Executors.newCachedThreadPool(); + /* + * As in production, the list executor has as many threads as there are permits. + */ + listExecutor = Executors.newFixedThreadPool(LIST_PERMITS); + callbackExecutor = Executors.newCachedThreadPool(); + throttle = new BulkTaskThrottle(LIST_PERMITS); + + BulkRequest request = new BulkRequest(); + request.setId(1L); + request.setUid("test-request"); + request.setActivity("TEST"); + request.setExpandDirectories(Depth.ALL); + request.setTargetPrefix("/"); + + BulkActivity activity = new BulkActivity<>("TEST", + BulkActivity.TargetType.BOTH) { + @Override + public ListenableFuture perform(String rid, long tid, String prefix, + FsPath path, FileAttributes attributes) { + performed.merge(path, 1, Integer::sum); + if (attributes.getFileType() == FileType.DIR) { + performedDirs.add(path); + } + return Futures.immediateFuture(path); + } + + @Override + protected void configure(Map arguments) { + } + + @Override + public void handleCompletion(BulkRequestTarget target, Future future) { + target.setState(State.COMPLETED); + } + }; + + FileAttributes placeholder = new FileAttributes(); + placeholder.setFileType(FileType.SPECIAL); + placeholder.setPnfsId(PLACEHOLDER_PNFSID); + jobTarget = BulkRequestTargetBuilder.builder(null).activity("TEST").rid(1L) + .ruid("test-request").pid(PID.ROOT).attributes(placeholder) + .path(ROOT_REQUEST_PATH).build(); + + BulkRequestTarget initial = BulkRequestTargetBuilder.builder(null).activity("TEST") + .id(2L).rid(1L).ruid("test-request").pid(PID.INITIAL).path(ROOT).build(); + + BulkTargetStore targetStore = mock(BulkTargetStore.class); + when(targetStore.getInitialTargets(1L, true)).thenReturn(List.of(initial)); + + PnfsHandler pnfsHandler = mock(PnfsHandler.class); + when(pnfsHandler.requestAsync(any())).thenAnswer(invocation -> { + PnfsGetFileAttributes message = invocation.getArgument(0); + message.setFileAttributes(attributes(FileType.DIR)); + return Futures.immediateFuture(message); + }); + + ListDirectoryHandler listHandler = mock(ListDirectoryHandler.class); + when(listHandler.list(any(), any(), any(), any(), any(), any())).thenAnswer( + invocation -> listing(invocation.getArgument(2))); + + job = new BulkRequestContainerJob(activity, jobTarget, request, null); + job.setNamespaceHandler(pnfsHandler); + job.setTargetStore(targetStore); + job.setListHandler(listHandler); + job.setDirListThrottle(throttle); + job.setInFlightSemaphore(new Semaphore(100)); + job.setExecutor(executor); + job.setListExecutor(listExecutor); + job.setCallbackExecutor(callbackExecutor); + job.initialize(); + } + + @After + public void tearDown() { + executor.shutdownNow(); + listExecutor.shutdownNow(); + callbackExecutor.shutdownNow(); + } + + @Test(timeout = 60_000) + public void shouldCompleteRecursiveExpansionWiderThanListingPermits() throws Exception { + executor.submit(job); + + while (jobTarget.getState() != State.COMPLETED) { + assertTrue("job terminated in state " + jobTarget.getState(), + !jobTarget.isTerminated() || jobTarget.getState() == State.COMPLETED); + TimeUnit.MILLISECONDS.sleep(20); + } + + int dirs = 0; + int files = 0; + for (int level = 0, width = 1; level <= DEPTH; ++level, width *= FANOUT) { + dirs += width; /* directories at this level */ + files += width; /* each directory holds one file */ + } + + assertEquals(dirs + files, performed.size()); + assertTrue("some target performed more than once", + performed.values().stream().allMatch(n -> n == 1)); + assertEquals(dirs, performedDirs.size()); + + /* + * Directories must be processed deepest first (children before parents). + */ + for (int i = 1; i < performedDirs.size(); ++i) { + assertTrue(depth(performedDirs.get(i - 1)) >= depth(performedDirs.get(i))); + } + + assertEquals(LIST_PERMITS, throttle.availablePermits()); + assertEquals(0, throttle.pendingCount()); + } + + @Test(timeout = 60_000) + public void shouldReturnAllListingPermitsWhenCancelledDuringExpansion() throws Exception { + listDelayMs = 5; + executor.submit(job); + + while (performed.size() < 20) { + TimeUnit.MILLISECONDS.sleep(5); + } + + assertTrue("listings should be waiting for permits", throttle.pendingCount() > 0); + + job.cancel(); + + assertEquals(State.CANCELLED, jobTarget.getState()); + while (throttle.availablePermits() != LIST_PERMITS || throttle.pendingCount() != 0) { + TimeUnit.MILLISECONDS.sleep(10); + } + + assertTrue("no directory should be processed after cancellation", + performedDirs.isEmpty()); + } + + private FileAttributes attributes(FileType type) { + FileAttributes attributes = new FileAttributes(); + attributes.setFileType(type); + attributes.setPnfsId(new PnfsId(String.format("%036X", pnfsIds.incrementAndGet()))); + return attributes; + } + + private static int depth(FsPath path) { + return path.toString().split("/").length; + } + + private DirectoryStream listing(FsPath path) throws InterruptedException { + if (listDelayMs > 0) { + TimeUnit.MILLISECONDS.sleep(listDelayMs); + } + + List entries = new ArrayList<>(); + entries.add(new DirectoryEntry("file", attributes(FileType.REGULAR))); + if (depth(path) - depth(ROOT) < DEPTH) { + for (int i = 0; i < FANOUT; ++i) { + entries.add(new DirectoryEntry("dir" + i, attributes(FileType.DIR))); + } + } + + return new DirectoryStream() { + @Override + public void close() { + } + + @Override + public Iterator iterator() { + return entries.iterator(); + } + }; + } +} diff --git a/modules/dcache-bulk/src/test/java/org/dcache/services/bulk/util/BulkTaskThrottleTest.java b/modules/dcache-bulk/src/test/java/org/dcache/services/bulk/util/BulkTaskThrottleTest.java new file mode 100644 index 00000000000..269004016f8 --- /dev/null +++ b/modules/dcache-bulk/src/test/java/org/dcache/services/bulk/util/BulkTaskThrottleTest.java @@ -0,0 +1,185 @@ +/* +COPYRIGHT STATUS: +Dec 1st 2001, Fermi National Accelerator Laboratory (FNAL) documents and +software are sponsored by the U.S. Department of Energy under Contract No. +DE-AC02-76CH03000. Therefore, the U.S. Government retains a world-wide +non-exclusive, royalty-free license to publish or reproduce these documents +and software for U.S. Government purposes. All documents and software +available from this server are private under the U.S. and Foreign +Copyright Laws, and FNAL reserves all rights. + +Distribution of the software available from this server is free of +charge subject to the user following the terms of the Fermitools +Software Legal Information. + +Redistribution and/or modification of the software shall be accompanied +by the Fermitools Software Legal Information (including the copyright +notice). + +The user is asked to feed back problems, benefits, and/or suggestions +about the software to the Fermilab Software Providers. + +Neither the name of Fermilab, the URA, nor the names of the contributors +may be used to endorse or promote products derived from this software +without specific prior written permission. + +DISCLAIMER OF LIABILITY (BSD): + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL FERMILAB, +OR THE URA, OR THE U.S. DEPARTMENT of ENERGY, OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT +OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Liabilities of the Government: + +This software is provided by URA, independent from its Prime Contract +with the U.S. Department of Energy. URA is acting independently from +the Government and in its own private capacity and is not acting on +behalf of the U.S. Government, nor as its contractor nor its agent. +Correspondingly, it is understood and agreed that the U.S. Government +has no connection to this software and in no manner whatsoever shall +be liable for nor assume any responsibility or obligation for any claim, +cost, or damages arising out of or resulting from the use of the software +available from this server. + +Export Control: + +All documents and software available from this server are subject to U.S. +export control laws. Anyone downloading information from this server is +obligated to secure any necessary Government licenses before exporting +documents or software obtained from this server. + */ +package org.dcache.services.bulk.util; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class BulkTaskThrottleTest { + + private static final int PERMITS = 3; + + private BulkTaskThrottle throttle; + private ExecutorService executor; + private final AtomicInteger active = new AtomicInteger(); + private final AtomicInteger maxActive = new AtomicInteger(); + + @Before + public void setUp() { + throttle = new BulkTaskThrottle(PERMITS); + executor = Executors.newCachedThreadPool(); + } + + @After + public void tearDown() { + executor.shutdownNow(); + } + + @Test + public void shouldRunLaunchersImmediatelyWhilePermitsAreAvailable() { + AtomicInteger launched = new AtomicInteger(); + for (int i = 0; i < PERMITS; ++i) { + throttle.submit(launched::incrementAndGet); + } + + assertEquals(PERMITS, launched.get()); + assertEquals(0, throttle.availablePermits()); + assertEquals(0, throttle.pendingCount()); + } + + @Test + public void shouldQueueWithoutBlockingWhenNoPermitIsAvailable() { + AtomicInteger launched = new AtomicInteger(); + for (int i = 0; i < PERMITS + 2; ++i) { + throttle.submit(launched::incrementAndGet); + } + + assertEquals(PERMITS, launched.get()); + assertEquals(2, throttle.pendingCount()); + + throttle.release(); + assertEquals(PERMITS + 1, launched.get()); + assertEquals(1, throttle.pendingCount()); + } + + @Test + public void shouldReturnPermitIfLauncherFails() { + throttle.submit(() -> { + throw new IllegalStateException("test"); + }); + + assertEquals(PERMITS, throttle.availablePermits()); + } + + @Test + public void shouldNotRecurWhenLaunchersReleaseImmediately() { + throttle = new BulkTaskThrottle(1); + throttle.submit(() -> { /* hold the only permit */ }); + for (int i = 0; i < 100_000; ++i) { + throttle.submit(throttle::release); + } + + throttle.release(); + assertEquals(1, throttle.availablePermits()); + assertEquals(0, throttle.pendingCount()); + } + + /* + * Each task submits children while holding its permit, as directory listing does. + * A blocking acquisition would deadlock here. + */ + @Test(timeout = 30_000) + public void shouldCompleteTreeOfNestedSubmissions() throws Exception { + int fanout = 4; + int depth = 6; + int total = 0; + for (int level = 0, width = 1; level <= depth; ++level, width *= fanout) { + total += width; + } + + CountDownLatch done = new CountDownLatch(total); + submitNode(0, depth, fanout, done); + assertTrue(done.await(30, TimeUnit.SECONDS)); + + while (throttle.availablePermits() != PERMITS) { + TimeUnit.MILLISECONDS.sleep(1); + } + + assertEquals(0, throttle.pendingCount()); + assertTrue("max concurrency " + maxActive.get(), maxActive.get() <= PERMITS); + } + + private void submitNode(int level, int depth, int fanout, CountDownLatch done) { + throttle.submit(() -> executor.execute(() -> { + int now = active.incrementAndGet(); + maxActive.accumulateAndGet(now, Math::max); + try { + if (level < depth) { + for (int i = 0; i < fanout; ++i) { + submitNode(level + 1, depth, fanout, done); + } + } + } finally { + active.decrementAndGet(); + done.countDown(); + throttle.release(); + } + })); + } +}