Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.Semaphore;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
Expand All @@ -113,6 +114,7 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
import org.dcache.services.bulk.util.BulkRequestTarget.State;
import org.dcache.services.bulk.util.BulkRequestTargetBuilder;
import org.dcache.services.bulk.util.BulkServiceStatistics;
import org.dcache.services.bulk.util.BulkTaskThrottle;
import org.dcache.util.CacheExceptionFactory;
import org.dcache.util.SignalAware;
import org.dcache.util.list.DirectoryEntry;
Expand Down Expand Up @@ -174,12 +176,16 @@ public int compare(DirTarget o1, DirTarget o2) {
}

/**
* Encapsulates manipulation of the semaphore.
* Encapsulates manipulation of the semaphore or throttle.
* <p>
* A task uses either a semaphore (blocking acquisition) or a throttle (non-blocking; the
* permit is granted by the throttle when the task is launched).
*/
class PermitHolder {

private final AtomicBoolean holdingPermit = new AtomicBoolean(false);
private Semaphore taskSemaphore;
private BulkTaskThrottle taskThrottle;

void acquireIfNotHoldingPermit() throws InterruptedException {
if (taskSemaphore == null) {
Expand All @@ -191,24 +197,35 @@ void acquireIfNotHoldingPermit() throws InterruptedException {
}
}

/**
* The throttle has acquired a permit on behalf of this task.
*/
void permitGranted() {
holdingPermit.set(true);
}

void throttledRelease() {
activity.throttle();
releaseIfHoldingPermit();
}

void releaseIfHoldingPermit() {
if (taskSemaphore == null) {
return;
}

if (holdingPermit.compareAndSet(true, false)) {
taskSemaphore.release();
if (taskThrottle != null) {
taskThrottle.release();
} else if (taskSemaphore != null) {
taskSemaphore.release();
}
}
}

void setTaskSemaphore(Semaphore taskSemaphore) {
this.taskSemaphore = taskSemaphore;
}

void setTaskThrottle(BulkTaskThrottle taskThrottle) {
this.taskThrottle = taskThrottle;
}
}

/**
Expand Down Expand Up @@ -258,11 +275,10 @@ void expandDepthFirst(Long id, PID pid, FsPath path, FileAttributes dirAttribute
throws BulkServiceException, CacheException {
LOGGER.debug("{} - expandDepthFirst, {}, {}, {}, {}", ruid, id, pid, path,
dirAttributes);
try {
new DirListTask(id, pid, path, dirAttributes).submitAsync();
} catch (InterruptedException e) {
LOGGER.trace("{} - expandDepthFirst {} interrupted.", ruid, id);
}
/*
* Does not block: the listing is queued until a permit is available.
*/
new DirListTask(id, pid, path, dirAttributes).submitAsync();
}

void submitAsync() throws InterruptedException {
Expand Down Expand Up @@ -316,7 +332,67 @@ class DirListTask extends ContainerTask {
this.path = path;
this.dirAttributes = dirAttributes;
taskExecutor = listExecutor;
permitHolder.setTaskSemaphore(dirListSemaphore);
permitHolder.setTaskThrottle(dirListThrottle);
}

/**
* Registers the task as running and queues it for listing without blocking.
* <p>
* Blocking for a permit here would deadlock: this is called from listing tasks which
* hold permits themselves, so a sufficiently wide directory tree would end up with all
* permits held by tasks waiting for permits (GitHub issue #8062).
* <p>
* The task is added to running immediately so that the transition to directory
* processing waits for queued listings as well.
*/
@Override
void submitAsync() {
synchronized (running) {
if (jobTarget.isTerminated()) {
return;
}

running.put(seqNo, this);
LOGGER.debug("{} - submitAsync (queued) {}, task count is now {}.", ruid, seqNo,
running.size());
}

dirListThrottle.submit(this::launch);
}

/**
* Called by the throttle holding a permit for this task.
*/
private void launch() {
permitHolder.permitGranted();

RejectedExecutionException rejected = null;
synchronized (running) {
if (!jobTarget.isTerminated() && running.containsKey(seqNo)) {
try {
taskFuture = taskExecutor.submit(this);
return;
} catch (RejectedExecutionException e) {
rejected = e;
}
}
}

if (rejected != null) {
/*
* Fail fast rather than silently skipping this subtree.
*/
LOGGER.error("{} - could not launch listing of {}: {}.", ruid, path,
rejected.toString());
containerState = ContainerState.STOP;
jobTarget.setErrorObject(rejected);
update();
}

/*
* Cancelled while queued (or rejected): return the permit.
*/
remove();
}

void doInner() {
Expand Down Expand Up @@ -703,7 +779,7 @@ private void storeOrUpdate(Throwable error) {
private ExecutorService executor;
private ExecutorService listExecutor;
private ExecutorService callbackExecutor;
private Semaphore dirListSemaphore;
private BulkTaskThrottle dirListThrottle;
private Semaphore inFlightSemaphore;

private volatile ContainerState containerState;
Expand Down Expand Up @@ -882,8 +958,8 @@ public void run() {
}
}

public void setDirListSemaphore(Semaphore dirListSemaphore) {
this.dirListSemaphore = dirListSemaphore;
public void setDirListThrottle(BulkTaskThrottle dirListThrottle) {
this.dirListThrottle = dirListThrottle;
}

public void setInFlightSemaphore(Semaphore inFlightSemaphore) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
import org.dcache.services.bulk.util.BulkRequestTarget.PID;
import org.dcache.services.bulk.util.BulkRequestTargetBuilder;
import org.dcache.services.bulk.util.BulkServiceStatistics;
import org.dcache.services.bulk.util.BulkTaskThrottle;
import org.dcache.util.BoundedCachedExecutor;
import org.dcache.util.list.ListDirectoryHandler;
import org.dcache.vehicles.FileAttributes;
Expand All @@ -101,7 +102,7 @@ public final class RequestContainerJobFactory {
private ListDirectoryHandler listHandler;
private BulkTargetStore targetStore;
private BulkServiceStatistics statistics;
private Semaphore dirListSemaphore;
private BulkTaskThrottle dirListThrottle;
private Semaphore inFlightSemaphore;
private BoundedCachedExecutor taskExecutor;
private BoundedCachedExecutor callbackExecutor;
Expand Down Expand Up @@ -134,7 +135,7 @@ public BulkRequestContainerJob createRequestJob(BulkRequest request)
containerJob.setNamespaceHandler(pnfsHandler);
containerJob.setTargetStore(targetStore);
containerJob.setListHandler(listHandler);
containerJob.setDirListSemaphore(dirListSemaphore);
containerJob.setDirListThrottle(dirListThrottle);
containerJob.setInFlightSemaphore(inFlightSemaphore);
containerJob.setExecutor(taskExecutor);
containerJob.setListExecutor(listExecutor);
Expand All @@ -144,7 +145,11 @@ public BulkRequestContainerJob createRequestJob(BulkRequest request)
}

public int getDirListSemaphoreAvailable() {
return dirListSemaphore.availablePermits();
return dirListThrottle.availablePermits();
}

public int getDirListPending() {
return dirListThrottle.pendingCount();
}

public int getInFlightSemaphoreAvailable() {
Expand Down Expand Up @@ -173,7 +178,7 @@ public void setListExecutor(BoundedCachedExecutor listExecutor) {

@Required
public void setDirListSemaphore(int permits) {
dirListSemaphore = new Semaphore(permits);
dirListThrottle = new BulkTaskThrottle(permits);
}

@Required
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,8 @@ public void getInfo(PrintWriter pw) {

pw.println(String.format(CONCURRENCY_FORMAT, "Available permits for directory listing",
factory.getDirListSemaphoreAvailable()));
pw.println(String.format(CONCURRENCY_FORMAT, "Directory listings waiting for permit",
factory.getDirListPending()));
pw.println(String.format(CONCURRENCY_FORMAT, "Available permits for in-flight targets",
factory.getInFlightSemaphoreAvailable()));
pw.println();
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,160 @@
/*
COPYRIGHT STATUS:
Dec 1st 2001, Fermi National Accelerator Laboratory (FNAL) documents and
software are sponsored by the U.S. Department of Energy under Contract No.
DE-AC02-76CH03000. Therefore, the U.S. Government retains a world-wide
non-exclusive, royalty-free license to publish or reproduce these documents
and software for U.S. Government purposes. All documents and software
available from this server are private under the U.S. and Foreign
Copyright Laws, and FNAL reserves all rights.

Distribution of the software available from this server is free of
charge subject to the user following the terms of the Fermitools
Software Legal Information.

Redistribution and/or modification of the software shall be accompanied
by the Fermitools Software Legal Information (including the copyright
notice).

The user is asked to feed back problems, benefits, and/or suggestions
about the software to the Fermilab Software Providers.

Neither the name of Fermilab, the URA, nor the names of the contributors
may be used to endorse or promote products derived from this software
without specific prior written permission.

DISCLAIMER OF LIABILITY (BSD):

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL FERMILAB,
OR THE URA, OR THE U.S. DEPARTMENT of ENERGY, OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

Liabilities of the Government:

This software is provided by URA, independent from its Prime Contract
with the U.S. Department of Energy. URA is acting independently from
the Government and in its own private capacity and is not acting on
behalf of the U.S. Government, nor as its contractor nor its agent.
Correspondingly, it is understood and agreed that the U.S. Government
has no connection to this software and in no manner whatsoever shall
be liable for nor assume any responsibility or obligation for any claim,
cost, or damages arising out of or resulting from the use of the software
available from this server.

Export Control:

All documents and software available from this server are subject to U.S.
export control laws. Anyone downloading information from this server is
obligated to secure any necessary Government licenses before exporting
documents or software obtained from this server.
*/
package org.dcache.services.bulk.util;

import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.Semaphore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
* Non-blocking admission control for bulk tasks.
* <p>
* Tasks are queued and launched only when a permit is available. Submission never blocks, so a
* task holding a permit may safely submit further tasks (e.g., a directory listing discovering
* subdirectories) without risk of all permits being held by tasks waiting for permits.
* <p>
* A launcher is invoked holding a permit, which it owns from that point on; it must eventually
* call {@link #release()} exactly once. If the launcher throws instead, the permit is returned
* by the throttle and the launcher must not release it. The throttle may be shared among
* several requests.
*/
public final class BulkTaskThrottle {

private static final Logger LOGGER = LoggerFactory.getLogger(BulkTaskThrottle.class);

private final Semaphore permits;
private final Queue<Runnable> pending = new ConcurrentLinkedQueue<>();

/*
* Per-thread re-entrancy guard; it does NOT exclude other threads from draining
* concurrently (concurrent drains coordinate solely through the semaphore and the
* concurrent queue).
*
* A launcher runs inside drain() and may call release() or submit() in the same
* thread (e.g., its request was cancelled while queued). Rather than recurring once
* per pending task, the nested call returns, and the loop of the enclosing drain() in
* this same thread picks up the change on its next iteration.
*/
private final ThreadLocal<Boolean> drainingInThisThread
= ThreadLocal.withInitial(() -> false);

public BulkTaskThrottle(int permits) {
this.permits = new Semaphore(permits);
}

public int availablePermits() {
return permits.availablePermits();
}

public int pendingCount() {
return pending.size();
}

/**
* Queue the launcher; it is run (possibly in the calling thread) as soon as a permit is
* available. Never blocks waiting for a permit.
*/
public void submit(Runnable launcher) {
pending.add(launcher);
drain();
}

/**
* Return a permit acquired on behalf of a launcher, and launch pending tasks if any.
*/
public void release() {
permits.release();
drain();
}

/*
* Both submit and release publish their change (enqueue / release) before checking
* for the other condition, so a pending launcher cannot be stranded while a permit
* is available.
*/
private void drain() {
if (drainingInThisThread.get()) {
return;
}

drainingInThisThread.set(true);
try {
while (!pending.isEmpty() && permits.tryAcquire()) {
Runnable launcher = pending.poll();
if (launcher == null) {
permits.release();
continue;
}

try {
launcher.run();
} catch (RuntimeException e) {
LOGGER.error("Bulk task launcher failed; please report to support@dcache.org",
e);
permits.release();
}
}
} finally {
drainingInThisThread.set(false);
}
}
}
Loading
Loading