diff --git a/src/Common/Apidrvr.h b/src/Common/Apidrvr.h index bf9f56a707..3cd32b9d52 100644 --- a/src/Common/Apidrvr.h +++ b/src/Common/Apidrvr.h @@ -427,6 +427,8 @@ typedef struct #define VC_ENCRYPTION_ITEM_COUNT DRIVER_STR("VeraCryptEncryptionItemCount") #define VC_ENCRYPTION_FRAGMENT_SIZE DRIVER_STR("VeraCryptEncryptionFragmentSize") #define VC_ENCRYPTION_MAX_WORK_ITEMS DRIVER_STR("VeraCryptEncryptionMaxWorkItems") +#define VC_MOUNTED_VOLUME_FAST_READ_IO DRIVER_STR("VeraCryptMountedVolumeFastReadIo") +#define VC_MOUNTED_VOLUME_FAST_WRITE_IO DRIVER_STR("VeraCryptMountedVolumeFastWriteIo") #define VC_ERASE_KEYS_SHUTDOWN DRIVER_STR("VeraCryptEraseKeysShutdown") diff --git a/src/Driver/EncryptedIoQueue.c b/src/Driver/EncryptedIoQueue.c index ec7f5a9a8b..125c528b18 100644 --- a/src/Driver/EncryptedIoQueue.c +++ b/src/Driver/EncryptedIoQueue.c @@ -19,6 +19,16 @@ #include "Volumes.h" #include +#define VC_DIRECT_HOST_READ_MAX_IN_FLIGHT 8 +#define VC_DIRECT_HOST_WRITE_MAX_IN_FLIGHT 8 + +typedef struct +{ + LIST_ENTRY ListEntry; + ULONGLONG Start; + ULONGLONG EndExclusive; +} VC_DIRECT_HOST_WRITE_RANGE; + // Returns STATUS_SUCCESS on success and sets *outVa. // On failure, returns STATUS_INVALID_USER_BUFFER or STATUS_INSUFFICIENT_RESOURCES // and leaves *outVa as NULL. If *outTempMdl not NULL, the caller must unlock/free it at completion. @@ -238,6 +248,256 @@ static void DecrementOutstandingIoCount (EncryptedIoQueue *queue) } +static BOOL IsWriteBarrierIrp (PIRP irp) +{ + UCHAR majorFunction = IoGetCurrentIrpStackLocation (irp)->MajorFunction; + return majorFunction == IRP_MJ_WRITE || majorFunction == IRP_MJ_FLUSH_BUFFERS; +} + + +static void ReleaseWriteBarrier (EncryptedIoQueue *queue) +{ + LONG remaining = InterlockedDecrement (&queue->PendingWriteBarrierCount); + ASSERT (remaining >= 0); +} + + +static void WaitForDirectHostReads (EncryptedIoQueue *queue) +{ + for (;;) + { + KIRQL oldIrql; + + KeAcquireSpinLock (&queue->DirectHostReadLock, &oldIrql); + if (queue->DirectHostReadsInFlight == 0) + { + KeReleaseSpinLock (&queue->DirectHostReadLock, oldIrql); + return; + } + KeReleaseSpinLock (&queue->DirectHostReadLock, oldIrql); + + KeWaitForSingleObject (&queue->NoDirectHostReadsEvent, Executive, KernelMode, FALSE, NULL); + } +} + + +static void BeginLegacyIoRequest (EncryptedIoQueue *queue) +{ + if (InterlockedIncrement (&queue->LegacyIoRequestsInFlight) == 1) + KeClearEvent (&queue->NoLegacyIoRequestsEvent); +} + + +static void CompleteLegacyIoRequest (EncryptedIoQueue *queue) +{ + LONG remaining = InterlockedDecrement (&queue->LegacyIoRequestsInFlight); + ASSERT (remaining >= 0); + if (remaining == 0) + KeSetEvent (&queue->NoLegacyIoRequestsEvent, IO_DISK_INCREMENT, FALSE); +} + + +static void WaitForLegacyIoRequests (EncryptedIoQueue *queue) +{ + while (InterlockedCompareExchange (&queue->LegacyIoRequestsInFlight, 0, 0) != 0) + KeWaitForSingleObject (&queue->NoLegacyIoRequestsEvent, Executive, KernelMode, FALSE, NULL); +} + + +static BOOL DirectHostWriteRangeOverlaps ( + const VC_DIRECT_HOST_WRITE_RANGE *left, + const VC_DIRECT_HOST_WRITE_RANGE *right) +{ + return left->Start < right->EndExclusive && right->Start < left->EndExclusive; +} + + +static void BeginDirectHostWrite ( + EncryptedIoQueue *queue, + VC_DIRECT_HOST_WRITE_RANGE *range) +{ + KIRQL oldIrql; + NTSTATUS status = KeWaitForSingleObject ( + &queue->DirectHostWriteSlots, + Executive, + KernelMode, + FALSE, + NULL); + + if (!NT_SUCCESS (status)) + TC_BUG_CHECK (status); + + for (;;) + { + PLIST_ENTRY entry; + BOOL overlaps = FALSE; + + KeAcquireSpinLock (&queue->DirectHostWriteLock, &oldIrql); + for (entry = queue->DirectHostWriteRanges.Flink; + entry != &queue->DirectHostWriteRanges; + entry = entry->Flink) + { + VC_DIRECT_HOST_WRITE_RANGE *activeRange = + CONTAINING_RECORD (entry, VC_DIRECT_HOST_WRITE_RANGE, ListEntry); + + if (DirectHostWriteRangeOverlaps (range, activeRange)) + { + overlaps = TRUE; + break; + } + } + + if (!overlaps) + { + InsertTailList (&queue->DirectHostWriteRanges, &range->ListEntry); + if (queue->DirectHostWritesInFlight++ == 0) + KeClearEvent (&queue->NoDirectHostWritesEvent); + KeReleaseSpinLock (&queue->DirectHostWriteLock, oldIrql); + return; + } + + KeClearEvent (&queue->DirectHostWriteRangeChangedEvent); + KeReleaseSpinLock (&queue->DirectHostWriteLock, oldIrql); + + status = KeWaitForSingleObject ( + &queue->DirectHostWriteRangeChangedEvent, + Executive, + KernelMode, + FALSE, + NULL); + if (!NT_SUCCESS (status)) + TC_BUG_CHECK (status); + } +} + + +static void CompleteDirectHostWrite ( + EncryptedIoQueue *queue, + VC_DIRECT_HOST_WRITE_RANGE *range) +{ + KIRQL oldIrql; + + KeAcquireSpinLock (&queue->DirectHostWriteLock, &oldIrql); + RemoveEntryList (&range->ListEntry); + ASSERT (queue->DirectHostWritesInFlight > 0); + if (--queue->DirectHostWritesInFlight == 0) + KeSetEvent (&queue->NoDirectHostWritesEvent, IO_DISK_INCREMENT, FALSE); + KeSetEvent (&queue->DirectHostWriteRangeChangedEvent, IO_DISK_INCREMENT, FALSE); + KeReleaseSpinLock (&queue->DirectHostWriteLock, oldIrql); + + KeReleaseSemaphore (&queue->DirectHostWriteSlots, IO_DISK_INCREMENT, 1, FALSE); +} + + +static void WaitForDirectHostWrites (EncryptedIoQueue *queue) +{ + for (;;) + { + KIRQL oldIrql; + + KeAcquireSpinLock (&queue->DirectHostWriteLock, &oldIrql); + if (queue->DirectHostWritesInFlight == 0) + { + KeReleaseSpinLock (&queue->DirectHostWriteLock, oldIrql); + return; + } + KeReleaseSpinLock (&queue->DirectHostWriteLock, oldIrql); + + KeWaitForSingleObject (&queue->NoDirectHostWritesEvent, Executive, KernelMode, FALSE, NULL); + } +} + + +_Function_class_(KSTART_ROUTINE) +static VOID DirectHostIoThreadProc (PVOID threadArg) +{ + EncryptedIoQueue *queue = (EncryptedIoQueue *) threadArg; + + KeSetPriorityThread (KeGetCurrentThread (), LOW_REALTIME_PRIORITY); + + for (;;) + { + VC_DIRECT_HOST_IO_TASK *task = NULL; + KIRQL oldIrql; + NTSTATUS status = KeWaitForSingleObject ( + &queue->DirectHostIoQueueSemaphore, + Executive, + KernelMode, + FALSE, + NULL); + + if (!NT_SUCCESS (status)) + continue; + + KeAcquireSpinLock (&queue->DirectHostIoQueueLock, &oldIrql); + if (!IsListEmpty (&queue->DirectHostIoQueue)) + { + PLIST_ENTRY entry = RemoveHeadList (&queue->DirectHostIoQueue); + task = CONTAINING_RECORD (entry, VC_DIRECT_HOST_IO_TASK, ListEntry); + } + else if (InterlockedCompareExchange (&queue->DirectHostIoExitRequested, 0, 0) != 0) + { + KeReleaseSpinLock (&queue->DirectHostIoQueueLock, oldIrql); + break; + } + KeReleaseSpinLock (&queue->DirectHostIoQueueLock, oldIrql); + + if (task && task->Routine) + task->Routine (task); + } + + PsTerminateSystemThread (STATUS_SUCCESS); +} + + +BOOL EncryptedIoQueueSubmitDirectIoTask (EncryptedIoQueue *queue, VC_DIRECT_HOST_IO_TASK *task) +{ + KIRQL oldIrql; + + if (!task || !task->Routine) + return FALSE; + + KeAcquireSpinLock (&queue->DirectHostIoQueueLock, &oldIrql); + if (queue->DirectHostIoThreadCount == 0 + || queue->StopPending + || InterlockedCompareExchange (&queue->DirectHostIoExitRequested, 0, 0) != 0) + { + KeReleaseSpinLock (&queue->DirectHostIoQueueLock, oldIrql); + return FALSE; + } + + if (task->IrpToMarkPending) + IoMarkIrpPending (task->IrpToMarkPending); + InsertTailList (&queue->DirectHostIoQueue, &task->ListEntry); + KeReleaseSpinLock (&queue->DirectHostIoQueueLock, oldIrql); + KeReleaseSemaphore (&queue->DirectHostIoQueueSemaphore, IO_DISK_INCREMENT, 1, FALSE); + return TRUE; +} + + +static void StopDirectHostIoThreads (EncryptedIoQueue *queue) +{ + ULONG i; + + if (queue->DirectHostIoThreadCount == 0) + return; + + InterlockedExchange (&queue->DirectHostIoExitRequested, TRUE); + KeReleaseSemaphore ( + &queue->DirectHostIoQueueSemaphore, + IO_DISK_INCREMENT, + (LONG) queue->DirectHostIoThreadCount, + FALSE); + + for (i = 0; i < queue->DirectHostIoThreadCount; ++i) + { + TCStopThread (queue->DirectHostIoThreads[i], NULL); + queue->DirectHostIoThreads[i] = NULL; + } + queue->DirectHostIoThreadCount = 0; +} + + static void OnItemCompleted (EncryptedIoQueueItem *item, BOOL freeItem) { if (item->TempUserMdl) { @@ -246,15 +506,18 @@ static void OnItemCompleted (EncryptedIoQueueItem *item, BOOL freeItem) item->TempUserMdl = NULL; } + if (item->WriteBarrierTracked) + ReleaseWriteBarrier (item->Queue); + DecrementOutstandingIoCount (item->Queue); IoReleaseRemoveLock (&item->Queue->RemoveLock, item->OriginalIrp); if (NT_SUCCESS (item->Status) && !item->Flush) { if (item->Write) - item->Queue->TotalBytesWritten += item->OriginalLength; + InterlockedAdd64 (&item->Queue->TotalBytesWritten, item->OriginalLength); else - item->Queue->TotalBytesRead += item->OriginalLength; + InterlockedAdd64 (&item->Queue->TotalBytesRead, item->OriginalLength); } if (freeItem) @@ -564,6 +827,276 @@ static NTSTATUS TCCachedRead (EncryptedIoQueue *queue, IO_STATUS_BLOCK *ioStatus } +static BOOL IsAlignedToSize (ULONGLONG value, ULONG size) +{ + return size == 0 || (value % size) == 0; +} + + +static BOOL CanUseDirectHostWrite (EncryptedIoQueue *queue, PIRP irp, EncryptedIoQueueItem *item) +{ + ULONGLONG hostEndExclusive; + ULONG hostSectorSize; + + if (InterlockedCompareExchange (&queue->DirectHostWriteEnabled, 0, 0) == 0 + || !item->Write + || item->Flush) + return FALSE; + + if (queue->IsFilterDevice || !queue->HostDeviceObject || !queue->HostFileObject) + return FALSE; + + if (item->OriginalLength == 0 || item->OriginalOffset.QuadPart < 0) + return FALSE; + + if (queue->HostDeviceIoMaxSize == 0 + || queue->DirectHostIoMaxRequestSize == 0 + || item->OriginalLength > queue->DirectHostIoMaxRequestSize) + return FALSE; + + if (!queue->CryptoInfo || queue->ThreadBlockReadWrite || queue->StopPending || queue->SuspendPending || queue->Suspended) + return FALSE; + + if (queue->bSupportPartialEncryption || queue->EncryptedAreaEndUpdatePending || queue->RemapEncryptedArea) + return FALSE; + + if ((queue->SecRegionData != NULL && queue->SecRegionSize > 512) + || queue->CryptoInfo->hiddenVolume + || queue->CryptoInfo->bProtectHiddenVolume + || queue->CryptoInfo->bHiddenVolProtectionAction + || queue->CryptoInfo->bPartitionInInactiveSysEncScope) + { + return FALSE; + } + + if ((irp->Flags & (IRP_PAGING_IO | IRP_SYNCHRONOUS_PAGING_IO)) != 0) + return FALSE; + + if (!IsAlignedToSize ((ULONGLONG) item->OriginalLength, ENCRYPTION_DATA_UNIT_SIZE) + || !IsAlignedToSize ((ULONGLONG) item->OriginalOffset.QuadPart, ENCRYPTION_DATA_UNIT_SIZE)) + { + return FALSE; + } + + hostSectorSize = queue->HostBytesPerSector != 0 ? queue->HostBytesPerSector : ENCRYPTION_DATA_UNIT_SIZE; + if (!IsAlignedToSize ((ULONGLONG) item->OriginalLength, hostSectorSize) + || !IsAlignedToSize ((ULONGLONG) item->OriginalOffset.QuadPart, hostSectorSize)) + { + return FALSE; + } + + if (queue->MaxReadAheadOffset.QuadPart > 0 + && (S_OK != ULongLongAdd ((ULONGLONG) item->OriginalOffset.QuadPart, (ULONGLONG) item->OriginalLength, &hostEndExclusive) + || hostEndExclusive > (ULONGLONG) queue->MaxReadAheadOffset.QuadPart)) + { + return FALSE; + } + + return TRUE; +} + + +static BOOL IsDirectHostIoCompatibilityFailure (NTSTATUS status) +{ + switch (status) + { + case STATUS_ACCESS_DENIED: + case STATUS_DATATYPE_MISALIGNMENT: + case STATUS_INVALID_DEVICE_REQUEST: + case STATUS_INVALID_PARAMETER: + case STATUS_NOT_SUPPORTED: + return TRUE; + + default: + return FALSE; + } +} + + +typedef struct +{ + VC_DIRECT_HOST_IO_TASK Task; + EncryptedIoQueue *Queue; + EncryptedIoQueueItem *Item; + PUCHAR BufferAllocation; + PUCHAR DataBuffer; + ULONG Length; + LARGE_INTEGER HostOffset; + VC_DIRECT_HOST_WRITE_RANGE Range; +} VC_DIRECT_HOST_WRITE_CONTEXT; + + +static void DirectHostWriteTask (VC_DIRECT_HOST_IO_TASK *task) +{ + VC_DIRECT_HOST_WRITE_CONTEXT *writeContext = + CONTAINING_RECORD (task, VC_DIRECT_HOST_WRITE_CONTEXT, Task); + EncryptedIoQueue *queue = writeContext->Queue; + EncryptedIoQueueItem *item = writeContext->Item; + PIRP originalIrp = item->OriginalIrp; + PUCHAR bufferAllocation = writeContext->BufferAllocation; + PUCHAR dataBuffer = writeContext->DataBuffer; + ULONG length = writeContext->Length; + LARGE_INTEGER hostOffset = writeContext->HostOffset; + IO_STATUS_BLOCK ioStatus = { STATUS_PENDING, 0 }; + ULONG completedLength = 0; + BOOL useLegacyIo = FALSE; + NTSTATUS status; + + if (originalIrp->Cancel) + { + status = STATUS_CANCELLED; + } + else if (queue->ThreadBlockReadWrite) + { + status = STATUS_DEVICE_BUSY; + } + else + { + status = STATUS_SUCCESS; + while (completedLength < length) + { + ULONG segmentLength = min (length - completedLength, queue->HostDeviceIoMaxSize); + ULONG_PTR segmentInformation = 0; + LARGE_INTEGER segmentOffset; + PUCHAR segmentBuffer = dataBuffer + completedLength; + + if (originalIrp->Cancel) + { + status = STATUS_CANCELLED; + break; + } + + if (segmentLength == 0) + { + status = STATUS_INVALID_PARAMETER; + break; + } + + segmentOffset.QuadPart = hostOffset.QuadPart + completedLength; + if (!useLegacyIo) + { + status = TCWriteDeviceUsingFileObjectWithInformation ( + queue->HostDeviceObject, + queue->HostFileObject, + segmentBuffer, + segmentOffset, + segmentLength, + &segmentInformation); + + if (!NT_SUCCESS (status) || segmentInformation != segmentLength) + { + if (segmentInformation == 0 + && IsDirectHostIoCompatibilityFailure (status)) + { + InterlockedExchange (&queue->DirectHostWriteEnabled, FALSE); + useLegacyIo = TRUE; + } + else + { + if (NT_SUCCESS (status)) + status = STATUS_END_OF_FILE; + break; + } + } + } + + if (useLegacyIo) + { + ioStatus.Status = STATUS_PENDING; + ioStatus.Information = 0; + status = ZwWriteFile ( + queue->HostFileHandle, + NULL, + NULL, + NULL, + &ioStatus, + segmentBuffer, + segmentLength, + &segmentOffset, + NULL); + segmentInformation = ioStatus.Information; + if (NT_SUCCESS (status) && segmentInformation != segmentLength) + status = STATUS_END_OF_FILE; + } + + if (!NT_SUCCESS (status)) + break; + + completedLength += segmentLength; + } + + } + + TCfree (bufferAllocation); + CompleteDirectHostWrite (queue, &writeContext->Range); + TCfree (writeContext); + CompleteOriginalIrp ( + item, + status, + NT_SUCCESS (status) && completedLength == length ? length : 0); +} + + +static BOOL QueueDirectHostWrite ( + EncryptedIoQueue *queue, + EncryptedIoQueueItem *item, + const uint8 *sourceBuffer) +{ + VC_DIRECT_HOST_WRITE_CONTEXT *context; + PUCHAR bufferAllocation; + PUCHAR dataBuffer; + ULONG allocationSize; + ULONG alignmentMask = queue->HostAlignmentMask; + UINT64_STRUCT dataUnit; + + if (S_OK != ULongAdd (item->OriginalLength, alignmentMask, &allocationSize)) + return FALSE; + + context = (VC_DIRECT_HOST_WRITE_CONTEXT *) TCalloc (sizeof (*context)); + if (!context) + return FALSE; + + bufferAllocation = (PUCHAR) TCalloc (allocationSize); + if (!bufferAllocation) + { + TCfree (context); + return FALSE; + } + + dataBuffer = (PUCHAR) ((((ULONG_PTR) bufferAllocation) + alignmentMask) + & ~((ULONG_PTR) alignmentMask)); + memcpy (dataBuffer, sourceBuffer, item->OriginalLength); + + dataUnit.Value = item->OriginalOffset.QuadPart / ENCRYPTION_DATA_UNIT_SIZE; + EncryptDataUnits ( + dataBuffer, + &dataUnit, + item->OriginalLength / ENCRYPTION_DATA_UNIT_SIZE, + queue->CryptoInfo); + + context->Queue = queue; + context->Task.Routine = DirectHostWriteTask; + context->Task.IrpToMarkPending = NULL; + context->Item = item; + context->BufferAllocation = bufferAllocation; + context->DataBuffer = dataBuffer; + context->Length = item->OriginalLength; + context->HostOffset = item->OriginalOffset; + context->Range.Start = (ULONGLONG) item->OriginalOffset.QuadPart; + context->Range.EndExclusive = context->Range.Start + item->OriginalLength; + + BeginDirectHostWrite (queue, &context->Range); + if (!EncryptedIoQueueSubmitDirectIoTask (queue, &context->Task)) + { + CompleteDirectHostWrite (queue, &context->Range); + TCfree (bufferAllocation); + TCfree (context); + return FALSE; + } + return TRUE; +} + + static VOID IoThreadProc (PVOID threadArg) { EncryptedIoQueue *queue = (EncryptedIoQueue *) threadArg; @@ -614,6 +1147,7 @@ static VOID IoThreadProc (PVOID threadArg) HandleCompleteOriginalIrp (queue, request); ReleasePoolBuffer (queue, request); + CompleteLegacyIoRequest (queue); continue; } @@ -695,9 +1229,22 @@ static VOID IoThreadProc (PVOID threadArg) IO_STATUS_BLOCK ioStatus; if (request->Item->Write) - request->Item->Status = ZwWriteFile (queue->HostFileHandle, NULL, NULL, NULL, &ioStatus, request->Data, request->Length, &request->Offset, NULL); + { + request->Item->Status = ZwWriteFile ( + queue->HostFileHandle, + NULL, + NULL, + NULL, + &ioStatus, + request->Data, + request->Length, + &request->Offset, + NULL); + } else + { request->Item->Status = TCCachedRead (queue, &ioStatus, request->Data, request->Offset, request->Length); + } if (NT_SUCCESS (request->Item->Status) && ioStatus.Information != request->Length) request->Item->Status = STATUS_END_OF_FILE; @@ -765,6 +1312,8 @@ static VOID IoThreadProc (PVOID threadArg) DecrementOutstandingIoCount (queue); } } + + CompleteLegacyIoRequest (queue); } } @@ -787,6 +1336,7 @@ static VOID MainThreadProc (PVOID threadArg) uint32 intersectLength; ULONGLONG addResult; HRESULT hResult; + BOOL useDirectHostWrite; if (IsEncryptionThreadPoolRunning()) KeSetPriorityThread (KeGetCurrentThread(), LOW_REALTIME_PRIORITY); @@ -808,6 +1358,8 @@ static VOID MainThreadProc (PVOID threadArg) if (!item) { TCCompleteDiskIrp (irp, STATUS_INSUFFICIENT_RESOURCES, 0); + if (IsWriteBarrierIrp (irp)) + ReleaseWriteBarrier (queue); DecrementOutstandingIoCount (queue); IoReleaseRemoveLock (&queue->RemoveLock, irp); @@ -819,13 +1371,7 @@ static VOID MainThreadProc (PVOID threadArg) item->TempUserMdl = NULL; item->Status = STATUS_SUCCESS; item->Flush = FALSE; - - IoSetCancelRoutine (irp, NULL); - if (irp->Cancel) - { - CompleteOriginalIrp (item, STATUS_CANCELLED, 0); - continue; - } + item->WriteBarrierTracked = IsWriteBarrierIrp (irp); switch (irpSp->MajorFunction) { @@ -853,18 +1399,31 @@ static VOID MainThreadProc (PVOID threadArg) continue; } + IoSetCancelRoutine (irp, NULL); + if (irp->Cancel) + { + CompleteOriginalIrp (item, STATUS_CANCELLED, 0); + continue; + } + + if (item->WriteBarrierTracked) + WaitForDirectHostReads (queue); + #ifdef TC_TRACE_IO_QUEUE item->OriginalIrpOffset = item->OriginalOffset; #endif if (item->Flush) { + WaitForDirectHostWrites (queue); InterlockedIncrement (&queue->IoThreadPendingRequestCount); + BeginLegacyIoRequest (queue); request = GetPoolBuffer (queue, sizeof (EncryptedIoRequest)); if (!request) { InterlockedDecrement (&queue->IoThreadPendingRequestCount); + CompleteLegacyIoRequest (queue); CompleteOriginalIrp (item, STATUS_INSUFFICIENT_RESOURCES, 0); continue; } @@ -1042,20 +1601,35 @@ static VOID MainThreadProc (PVOID threadArg) } dataBuffer = NULL; - NTSTATUS mapStatus = MapIrpDataBuffer( - irp, - item->Write, - item->OriginalLength, - &dataBuffer, - &item->TempUserMdl); - if (!NT_SUCCESS(mapStatus)) { - CompleteOriginalIrp (item, mapStatus, 0); - continue; + NTSTATUS mapStatus = MapIrpDataBuffer( + irp, + item->Write, + item->OriginalLength, + &dataBuffer, + &item->TempUserMdl); + if (!NT_SUCCESS (mapStatus)) + { + CompleteOriginalIrp (item, mapStatus, 0); + continue; + } } - // Divide data block to fragments to enable efficient overlapping of encryption and IO operations + useDirectHostWrite = CanUseDirectHostWrite (queue, irp, item); + if (useDirectHostWrite) + { + // Preserve the legacy/direct ordering boundary while allowing + // adjacent eligible writes to overlap below this queue. + WaitForLegacyIoRequests (queue); + queue->ReadAheadBufferValid = FALSE; + if (QueueDirectHostWrite (queue, item, dataBuffer)) + continue; + } + // Reads and legacy requests must not overtake a direct batch. + WaitForDirectHostWrites (queue); + + // Divide data block to fragments to enable efficient overlapping of encryption and IO operations dataRemaining = item->OriginalLength; fragmentOffset = item->OriginalOffset; @@ -1068,12 +1642,14 @@ static VOID MainThreadProc (PVOID threadArg) activeFragmentBuffer = (activeFragmentBuffer == queue->FragmentBufferA ? queue->FragmentBufferB : queue->FragmentBufferA); InterlockedIncrement (&queue->IoThreadPendingRequestCount); + BeginLegacyIoRequest (queue); // Create IO request request = GetPoolBuffer (queue, sizeof (EncryptedIoRequest)); if (!request) { InterlockedDecrement(&queue->IoThreadPendingRequestCount); + CompleteLegacyIoRequest (queue); CompleteOriginalIrp (item, STATUS_INSUFFICIENT_RESOURCES, 0); break; } @@ -1146,9 +1722,116 @@ static VOID MainThreadProc (PVOID threadArg) } +NTSTATUS EncryptedIoQueueBeginDirectRead (EncryptedIoQueue *queue, PIRP irp) +{ + NTSTATUS status; + + status = IoAcquireRemoveLock (&queue->RemoveLock, irp); + if (!NT_SUCCESS (status)) + return status; + + InterlockedIncrement (&queue->OutstandingIoCount); + if (queue->StartPending + || queue->StopPending + || queue->ThreadExitRequested + || queue->SuspendPending + || queue->Suspended + || queue->ThreadBlockReadWrite) + { + status = STATUS_DEVICE_NOT_READY; + goto err; + } + + return STATUS_SUCCESS; + +err: + DecrementOutstandingIoCount (queue); + IoReleaseRemoveLock (&queue->RemoveLock, irp); + return status; +} + + +NTSTATUS EncryptedIoQueueAdmitDirectRead (EncryptedIoQueue *queue) +{ + KIRQL oldIrql; + NTSTATUS status = STATUS_SUCCESS; + + if (queue->StartPending + || queue->StopPending + || queue->ThreadExitRequested + || queue->SuspendPending + || queue->Suspended + || queue->ThreadBlockReadWrite) + { + return STATUS_DEVICE_NOT_READY; + } + + // A pending write blocks new direct reads. If this read wins the race, + // MainThread observes it under the same lock and waits before writing. + KeAcquireSpinLock (&queue->DirectHostReadLock, &oldIrql); + if (InterlockedCompareExchange (&queue->PendingWriteBarrierCount, 0, 0) != 0 + || queue->DirectHostReadsInFlight >= VC_DIRECT_HOST_READ_MAX_IN_FLIGHT) + { + status = STATUS_DEVICE_BUSY; + } + else if (queue->DirectHostReadsInFlight++ == 0) + KeClearEvent (&queue->NoDirectHostReadsEvent); + KeReleaseSpinLock (&queue->DirectHostReadLock, oldIrql); + return status; +} + + +static void ReleaseDirectReadAdmission (EncryptedIoQueue *queue) +{ + KIRQL oldIrql; + + KeAcquireSpinLock (&queue->DirectHostReadLock, &oldIrql); + ASSERT (queue->DirectHostReadsInFlight > 0); + if (--queue->DirectHostReadsInFlight == 0) + KeSetEvent (&queue->NoDirectHostReadsEvent, IO_DISK_INCREMENT, FALSE); + KeReleaseSpinLock (&queue->DirectHostReadLock, oldIrql); +} + + +static void ReleaseDirectReadLifetime (EncryptedIoQueue *queue, PIRP irp) +{ + DecrementOutstandingIoCount (queue); + IoReleaseRemoveLock (&queue->RemoveLock, irp); +} + + +void EncryptedIoQueueCancelDirectRead (EncryptedIoQueue *queue, PIRP irp) +{ + ReleaseDirectReadLifetime (queue, irp); +} + + +void EncryptedIoQueueAbortDirectRead (EncryptedIoQueue *queue, PIRP irp) +{ + ReleaseDirectReadAdmission (queue); + ReleaseDirectReadLifetime (queue, irp); +} + + +void EncryptedIoQueueCompleteDirectRead (EncryptedIoQueue *queue, PIRP irp, ULONG length, NTSTATUS status, ULONG_PTR information) +{ + if (NT_SUCCESS (status)) + InterlockedAdd64 (&queue->TotalBytesRead, length); + + ReleaseDirectReadAdmission (queue); + ReleaseDirectReadLifetime (queue, irp); + TCCompleteDiskIrp (irp, status, information); +} + + NTSTATUS EncryptedIoQueueAddIrp (EncryptedIoQueue *queue, PIRP irp) { NTSTATUS status; + BOOL writeBarrierTracked = IsWriteBarrierIrp (irp); + + // Publish the barrier before queuing so a later direct read cannot overtake it. + if (writeBarrierTracked) + InterlockedIncrement (&queue->PendingWriteBarrierCount); InterlockedIncrement (&queue->OutstandingIoCount); if (queue->StopPending) @@ -1181,6 +1864,8 @@ NTSTATUS EncryptedIoQueueAddIrp (EncryptedIoQueue *queue, PIRP irp) return STATUS_PENDING; err: + if (writeBarrierTracked) + ReleaseWriteBarrier (queue); DecrementOutstandingIoCount (queue); return status; } @@ -1279,11 +1964,29 @@ NTSTATUS EncryptedIoQueueStart (EncryptedIoQueue *queue) queue->OutstandingIoCount = 0; queue->IoThreadPendingRequestCount = 0; + queue->DirectHostReadsInFlight = 0; + queue->DirectHostWritesInFlight = 0; + queue->LegacyIoRequestsInFlight = 0; + queue->PendingWriteBarrierCount = 0; + queue->DirectHostIoThreadCount = 0; + queue->DirectHostIoExitRequested = FALSE; + RtlZeroMemory (queue->DirectHostIoThreads, sizeof (queue->DirectHostIoThreads)); queue->FirstPoolBuffer = NULL; KeInitializeMutex (&queue->BufferPoolMutex, 0); KeInitializeEvent (&queue->NoOutstandingIoEvent, SynchronizationEvent, FALSE); + KeInitializeEvent (&queue->NoDirectHostReadsEvent, NotificationEvent, TRUE); + KeInitializeSpinLock (&queue->DirectHostReadLock); + KeInitializeEvent (&queue->NoDirectHostWritesEvent, NotificationEvent, TRUE); + KeInitializeEvent (&queue->DirectHostWriteRangeChangedEvent, NotificationEvent, FALSE); + KeInitializeSpinLock (&queue->DirectHostWriteLock); + KeInitializeSemaphore (&queue->DirectHostWriteSlots, VC_DIRECT_HOST_WRITE_MAX_IN_FLIGHT, VC_DIRECT_HOST_WRITE_MAX_IN_FLIGHT); + InitializeListHead (&queue->DirectHostWriteRanges); + InitializeListHead (&queue->DirectHostIoQueue); + KeInitializeSpinLock (&queue->DirectHostIoQueueLock); + KeInitializeSemaphore (&queue->DirectHostIoQueueSemaphore, 0, MAXLONG); + KeInitializeEvent (&queue->NoLegacyIoRequestsEvent, NotificationEvent, TRUE); KeInitializeEvent (&queue->PoolBufferFreeEvent, SynchronizationEvent, FALSE); KeInitializeEvent (&queue->QueueResumedEvent, SynchronizationEvent, FALSE); @@ -1445,6 +2148,27 @@ NTSTATUS EncryptedIoQueueStart (EncryptedIoQueue *queue) goto err; } + if (InterlockedCompareExchange (&queue->DirectHostReadEnabled, 0, 0) != 0 + || queue->DirectHostWriteConfigured) + { + for (i = 0; i < VC_DIRECT_HOST_IO_THREAD_COUNT; ++i) + { + status = TCStartThread ( + DirectHostIoThreadProc, + queue, + &queue->DirectHostIoThreads[queue->DirectHostIoThreadCount]); + if (!NT_SUCCESS (status)) + { + StopDirectHostIoThreads (queue); + InterlockedExchange (&queue->DirectHostReadEnabled, FALSE); + InterlockedExchange (&queue->DirectHostWriteEnabled, FALSE); + queue->DirectHostWriteConfigured = FALSE; + break; + } + ++queue->DirectHostIoThreadCount; + } + } + #ifdef TC_TRACE_IO_QUEUE GetElapsedTimeInit (&queue->LastPerformanceCounter); #endif @@ -1499,6 +2223,7 @@ NTSTATUS EncryptedIoQueueStop (EncryptedIoQueue *queue) Dump ("Queue stopping out=%d\n", queue->OutstandingIoCount); + StopDirectHostIoThreads (queue); queue->ThreadExitRequested = TRUE; TCStopThread (queue->MainThread, &queue->MainThreadQueueNotEmptyEvent); diff --git a/src/Driver/EncryptedIoQueue.h b/src/Driver/EncryptedIoQueue.h index 93f946d491..f28e4d387f 100644 --- a/src/Driver/EncryptedIoQueue.h +++ b/src/Driver/EncryptedIoQueue.h @@ -27,6 +27,9 @@ #define TC_ENC_IO_QUEUE_PREALLOCATED_IO_REQUEST_MAX_COUNT 8192 #define VC_MAX_WORK_ITEMS 1024 +#define VC_MOUNTED_VOLUME_FAST_IO_MAX_REQUEST_SIZE (1024U * 1024U) +#define VC_MOUNTED_VOLUME_FAST_IO_MAX_SEGMENT_SIZE VC_MOUNTED_VOLUME_FAST_IO_MAX_REQUEST_SIZE +#define VC_DIRECT_HOST_IO_THREAD_COUNT 8 typedef struct EncryptedIoQueueBufferStruct { @@ -48,6 +51,16 @@ typedef struct _COMPLETE_IRP_WORK_ITEM LIST_ENTRY ListEntry; // For managing free work items } COMPLETE_IRP_WORK_ITEM, * PCOMPLETE_IRP_WORK_ITEM; +typedef struct _VC_DIRECT_HOST_IO_TASK VC_DIRECT_HOST_IO_TASK; +typedef void (*VC_DIRECT_HOST_IO_TASK_ROUTINE) (VC_DIRECT_HOST_IO_TASK *task); + +struct _VC_DIRECT_HOST_IO_TASK +{ + LIST_ENTRY ListEntry; + VC_DIRECT_HOST_IO_TASK_ROUTINE Routine; + PIRP IrpToMarkPending; +}; + typedef struct { PDEVICE_OBJECT DeviceObject; @@ -59,6 +72,31 @@ typedef struct // File-handle-based IO HANDLE HostFileHandle; + PDEVICE_OBJECT HostDeviceObject; + PFILE_OBJECT HostFileObject; + ULONG HostDeviceIoMaxSize; + ULONG DirectHostIoMaxRequestSize; + ULONG HostAlignmentMask; + uint32 HostBytesPerSector; + BOOL DirectHostWriteConfigured; + volatile LONG DirectHostReadEnabled; + volatile LONG DirectHostWriteEnabled; + KSPIN_LOCK DirectHostReadLock; + KEVENT NoDirectHostReadsEvent; + volatile LONG DirectHostReadsInFlight; + KSPIN_LOCK DirectHostWriteLock; + KEVENT NoDirectHostWritesEvent; + KEVENT DirectHostWriteRangeChangedEvent; + KSEMAPHORE DirectHostWriteSlots; + LIST_ENTRY DirectHostWriteRanges; + volatile LONG DirectHostWritesInFlight; + volatile LONG PendingWriteBarrierCount; + PKTHREAD DirectHostIoThreads[VC_DIRECT_HOST_IO_THREAD_COUNT]; + ULONG DirectHostIoThreadCount; + LIST_ENTRY DirectHostIoQueue; + KSPIN_LOCK DirectHostIoQueueLock; + KSEMAPHORE DirectHostIoQueueSemaphore; + volatile LONG DirectHostIoExitRequested; BOOL bSupportPartialEncryption; int64 VirtualDeviceLength; SECURITY_CLIENT_CONTEXT *SecurityClientContext; @@ -110,6 +148,8 @@ typedef struct volatile LONG OutstandingIoCount; KEVENT NoOutstandingIoEvent; volatile LONG IoThreadPendingRequestCount; + volatile LONG LegacyIoRequestsInFlight; + KEVENT NoLegacyIoRequestsEvent; KEVENT PoolBufferFreeEvent; @@ -158,6 +198,7 @@ typedef struct LARGE_INTEGER OriginalOffset; NTSTATUS Status; PMDL TempUserMdl; // NULL if none. Used in MapIrpDataBuffer logic + BOOL WriteBarrierTracked; #ifdef TC_TRACE_IO_QUEUE LARGE_INTEGER OriginalIrpOffset; @@ -190,6 +231,12 @@ NTSTATUS EncryptedIoQueueResumeFromHold (EncryptedIoQueue *queue); NTSTATUS EncryptedIoQueueStart (EncryptedIoQueue *queue); NTSTATUS EncryptedIoQueueStop (EncryptedIoQueue *queue); NTSTATUS EncryptedIoQueueHoldWhenIdle (EncryptedIoQueue *queue, int64 timeout); +NTSTATUS EncryptedIoQueueBeginDirectRead (EncryptedIoQueue *queue, PIRP irp); +NTSTATUS EncryptedIoQueueAdmitDirectRead (EncryptedIoQueue *queue); +void EncryptedIoQueueCancelDirectRead (EncryptedIoQueue *queue, PIRP irp); +void EncryptedIoQueueAbortDirectRead (EncryptedIoQueue *queue, PIRP irp); +void EncryptedIoQueueCompleteDirectRead (EncryptedIoQueue *queue, PIRP irp, ULONG length, NTSTATUS status, ULONG_PTR information); +BOOL EncryptedIoQueueSubmitDirectIoTask (EncryptedIoQueue *queue, VC_DIRECT_HOST_IO_TASK *task); #endif // TC_HEADER_DRIVER_ENCRYPTED_IO_QUEUE diff --git a/src/Driver/Ntdriver.c b/src/Driver/Ntdriver.c index 5b32992ec1..e4a28c6fc8 100644 --- a/src/Driver/Ntdriver.c +++ b/src/Driver/Ntdriver.c @@ -145,6 +145,8 @@ static BOOL EnableExtendedIoctlSupport = FALSE; static BOOL AllowTrimCommand = FALSE; static BOOL OrderedFlushBarriersEnabled = FALSE; static BOOL RamEncryptionActivated = FALSE; +static BOOL MountedVolumeFastReadIoEnabled = FALSE; +static BOOL MountedVolumeFastWriteIoEnabled = FALSE; int EncryptionIoRequestCount = 0; int EncryptionItemCount = 0; int EncryptionFragmentSize = 0; @@ -155,6 +157,16 @@ BOOL IsOrderedFlushBarriersEnabled () return OrderedFlushBarriersEnabled; } +BOOL IsMountedVolumeFastReadIoEnabled () +{ + return MountedVolumeFastReadIoEnabled; +} + +BOOL IsMountedVolumeFastWriteIoEnabled () +{ + return MountedVolumeFastWriteIoEnabled; +} + PDEVICE_OBJECT VirtualVolumeDeviceObjects[MAX_MOUNTED_VOLUME_DRIVE_NUMBER + 1]; BOOL AlignValue (ULONG ulValue, ULONG ulAlignment, ULONG *pulResult) @@ -602,6 +614,304 @@ PDEVICE_OBJECT GetVirtualVolumeDeviceObject (int driveNumber) } +typedef struct +{ + VC_DIRECT_HOST_IO_TASK Task; + EncryptedIoQueue *Queue; + PIRP OriginalIrp; + PUCHAR DataBuffer; + ULONG Length; + LARGE_INTEGER HostOffset; +} VC_DIRECT_HOST_READ_CONTEXT; + + +static BOOL IsAlignedDirectHostIoValue (ULONGLONG value, ULONG size) +{ + return size == 0 || (value % size) == 0; +} + + +static BOOL IsDirectHostReadCompatibilityFailure (NTSTATUS status) +{ + switch (status) + { + case STATUS_ACCESS_DENIED: + case STATUS_DATATYPE_MISALIGNMENT: + case STATUS_INVALID_DEVICE_REQUEST: + case STATUS_INVALID_PARAMETER: + case STATUS_NOT_SUPPORTED: + return TRUE; + + default: + return FALSE; + } +} + + +static BOOL VcCanUseDirectHostRead (PEXTENSION extension, PIRP irp, PIO_STACK_LOCATION irpSp, LARGE_INTEGER *hostOffset) +{ + EncryptedIoQueue *queue = &extension->Queue; + ULONGLONG virtualEndExclusive; + ULONGLONG hostOffsetValue; + ULONGLONG hostEndExclusive; + LONGLONG virtualOffset = irpSp->Parameters.Read.ByteOffset.QuadPart; + ULONG length = irpSp->Parameters.Read.Length; + ULONG hostSectorSize; + + hostOffset->QuadPart = 0; + + if (irpSp->MajorFunction != IRP_MJ_READ + || KeGetCurrentIrql () > APC_LEVEL + || !irp->MdlAddress + || MmGetMdlByteCount (irp->MdlAddress) < length) + { + return FALSE; + } + + if (!extension->IsVolumeDevice + || extension->bRootDevice + || extension->IsDriveFilterDevice + || extension->IsVolumeFilterDevice + || !extension->bRawDevice + || queue->IsFilterDevice + || !queue->HostDeviceObject + || !queue->HostFileObject) + { + return FALSE; + } + + if (!queue->CryptoInfo + || !EncryptedIoQueueIsRunning (queue) + || EncryptedIoQueueIsSuspended (queue) + || queue->SuspendPending + || queue->StopPending + || queue->ThreadBlockReadWrite) + { + return FALSE; + } + + if (queue->bSupportPartialEncryption + || queue->EncryptedAreaEndUpdatePending + || queue->RemapEncryptedArea + || (queue->SecRegionData != NULL && queue->SecRegionSize > 512) + || queue->CryptoInfo->hiddenVolume + || queue->CryptoInfo->bProtectHiddenVolume + || queue->CryptoInfo->bHiddenVolProtectionAction + || queue->CryptoInfo->bPartitionInInactiveSysEncScope + || (irp->Flags & (IRP_PAGING_IO | IRP_SYNCHRONOUS_PAGING_IO)) != 0) + { + return FALSE; + } + + if (length == 0 + || virtualOffset < 0 + || queue->HostDeviceIoMaxSize == 0 + || queue->DirectHostIoMaxRequestSize == 0 + || length > queue->DirectHostIoMaxRequestSize + || !IsAlignedDirectHostIoValue ((ULONGLONG) length, ENCRYPTION_DATA_UNIT_SIZE) + || !IsAlignedDirectHostIoValue ((ULONGLONG) virtualOffset, ENCRYPTION_DATA_UNIT_SIZE) + || S_OK != ULongLongAdd ((ULONGLONG) virtualOffset, (ULONGLONG) length, &virtualEndExclusive) + || virtualEndExclusive > (ULONGLONG) queue->VirtualDeviceLength + || S_OK != ULongLongAdd ((ULONGLONG) virtualOffset, queue->CryptoInfo->volDataAreaOffset, &hostOffsetValue)) + { + return FALSE; + } + + hostSectorSize = queue->HostBytesPerSector != 0 ? queue->HostBytesPerSector : ENCRYPTION_DATA_UNIT_SIZE; + if (!IsAlignedDirectHostIoValue ((ULONGLONG) length, hostSectorSize) + || !IsAlignedDirectHostIoValue (hostOffsetValue, hostSectorSize) + || (queue->MaxReadAheadOffset.QuadPart > 0 + && (S_OK != ULongLongAdd (hostOffsetValue, (ULONGLONG) length, &hostEndExclusive) + || hostEndExclusive > (ULONGLONG) queue->MaxReadAheadOffset.QuadPart))) + { + return FALSE; + } + + hostOffset->QuadPart = (LONGLONG) hostOffsetValue; + return TRUE; +} + + +static void VcDirectHostReadTask (VC_DIRECT_HOST_IO_TASK *task) +{ + VC_DIRECT_HOST_READ_CONTEXT *readContext = + CONTAINING_RECORD (task, VC_DIRECT_HOST_READ_CONTEXT, Task); + EncryptedIoQueue *queue = readContext->Queue; + PIRP originalIrp = readContext->OriginalIrp; + PUCHAR dataBuffer = readContext->DataBuffer; + ULONG length = readContext->Length; + LARGE_INTEGER hostOffset = readContext->HostOffset; + IO_STATUS_BLOCK ioStatus = { STATUS_PENDING, 0 }; + ULONG_PTR information = 0; + ULONG completedLength = 0; + BOOL useLegacyIo = FALSE; + NTSTATUS status; + + if (originalIrp->Cancel) + { + status = STATUS_CANCELLED; + } + else if (!queue->CryptoInfo + || !EncryptedIoQueueIsRunning (queue) + || EncryptedIoQueueIsSuspended (queue) + || queue->SuspendPending + || queue->StopPending + || queue->ThreadBlockReadWrite) + { + status = STATUS_DEVICE_NOT_READY; + } + else + { + status = STATUS_SUCCESS; + while (completedLength < length) + { + ULONG segmentLength = min (length - completedLength, queue->HostDeviceIoMaxSize); + ULONG_PTR segmentInformation = 0; + LARGE_INTEGER segmentOffset; + PUCHAR segmentBuffer = dataBuffer + completedLength; + UINT64_STRUCT dataUnit; + + if (originalIrp->Cancel) + { + status = STATUS_CANCELLED; + break; + } + + if (segmentLength == 0) + { + status = STATUS_INVALID_PARAMETER; + break; + } + + segmentOffset.QuadPart = hostOffset.QuadPart + completedLength; + if (!useLegacyIo) + { + status = TCReadDeviceUsingFileObjectWithInformation ( + queue->HostDeviceObject, + queue->HostFileObject, + segmentBuffer, + segmentOffset, + segmentLength, + &segmentInformation); + + if (!NT_SUCCESS (status) || segmentInformation != segmentLength) + { + if (segmentInformation == 0 + && IsDirectHostReadCompatibilityFailure (status)) + { + InterlockedExchange (&queue->DirectHostReadEnabled, FALSE); + useLegacyIo = TRUE; + } + else + { + if (NT_SUCCESS (status)) + status = STATUS_END_OF_FILE; + break; + } + } + } + + if (useLegacyIo) + { + ioStatus.Status = STATUS_PENDING; + ioStatus.Information = 0; + status = ZwReadFile ( + queue->HostFileHandle, + NULL, + NULL, + NULL, + &ioStatus, + segmentBuffer, + segmentLength, + &segmentOffset, + NULL); + segmentInformation = ioStatus.Information; + if (NT_SUCCESS (status) && segmentInformation != segmentLength) + status = STATUS_END_OF_FILE; + } + + if (!NT_SUCCESS (status)) + break; + + dataUnit.Value = segmentOffset.QuadPart / ENCRYPTION_DATA_UNIT_SIZE; + DecryptDataUnits ( + segmentBuffer, + &dataUnit, + segmentLength / ENCRYPTION_DATA_UNIT_SIZE, + queue->CryptoInfo); + completedLength += segmentLength; + } + + if (NT_SUCCESS (status) && completedLength == length) + { + information = length; + } + else if (NT_SUCCESS (status)) + status = STATUS_END_OF_FILE; + } + + TCfree (readContext); + EncryptedIoQueueCompleteDirectRead (queue, originalIrp, length, status, information); +} + + +static BOOL VcTryDirectHostRead (PEXTENSION extension, PIRP irp, PIO_STACK_LOCATION irpSp) +{ + EncryptedIoQueue *queue = &extension->Queue; + VC_DIRECT_HOST_READ_CONTEXT *context; + LARGE_INTEGER hostOffset; + PUCHAR dataBuffer; + NTSTATUS status; + + status = EncryptedIoQueueBeginDirectRead (queue, irp); + if (!NT_SUCCESS (status)) + return FALSE; + + if (InterlockedCompareExchange (&queue->DirectHostReadEnabled, FALSE, FALSE) == FALSE) + goto fallback; + + if (!VcCanUseDirectHostRead (extension, irp, irpSp, &hostOffset)) + goto fallback; + + dataBuffer = (PUCHAR) MmGetSystemAddressForMdlSafe (irp->MdlAddress, HighPagePriority | MdlMappingNoExecute); + if (!dataBuffer + || (queue->HostAlignmentMask != 0 + && (((ULONG_PTR) dataBuffer) & queue->HostAlignmentMask) != 0)) + goto fallback; + + context = (VC_DIRECT_HOST_READ_CONTEXT *) TCalloc (sizeof (*context)); + if (!context) + goto fallback; + + context->Task.Routine = VcDirectHostReadTask; + context->Task.IrpToMarkPending = irp; + context->Queue = queue; + context->OriginalIrp = irp; + context->DataBuffer = dataBuffer; + context->Length = irpSp->Parameters.Read.Length; + context->HostOffset = hostOffset; + + status = EncryptedIoQueueAdmitDirectRead (queue); + if (!NT_SUCCESS (status)) + { + TCfree (context); + goto fallback; + } + + if (!EncryptedIoQueueSubmitDirectIoTask (queue, &context->Task)) + { + TCfree (context); + EncryptedIoQueueAbortDirectRead (queue, irp); + return FALSE; + } + return TRUE; + +fallback: + EncryptedIoQueueCancelDirectRead (queue, irp); + return FALSE; +} + + /* TCDispatchQueueIRP queues any IRP's so that they can be processed later by the thread -- or in some cases handles them immediately! */ NTSTATUS TCDispatchQueueIRP (PDEVICE_OBJECT DeviceObject, PIRP Irp) @@ -718,6 +1028,17 @@ NTSTATUS TCDispatchQueueIRP (PDEVICE_OBJECT DeviceObject, PIRP Irp) switch (irpSp->MajorFunction) { case IRP_MJ_READ: + if (MountedVolumeFastReadIoEnabled + && VcTryDirectHostRead (Extension, Irp, irpSp)) + return STATUS_PENDING; + + ntStatus = EncryptedIoQueueAddIrp (&Extension->Queue, Irp); + + if (ntStatus != STATUS_PENDING) + TCCompleteDiskIrp (Irp, ntStatus, 0); + + return ntStatus; + case IRP_MJ_WRITE: ntStatus = EncryptedIoQueueAddIrp (&Extension->Queue, Irp); @@ -739,7 +1060,9 @@ NTSTATUS TCDispatchQueueIRP (PDEVICE_OBJECT DeviceObject, PIRP Irp) return STATUS_PENDING; case IRP_MJ_FLUSH_BUFFERS: - if (!OrderedFlushBarriersEnabled || Extension->hDeviceFile == NULL || Extension->bReadOnly) + if ((!OrderedFlushBarriersEnabled && !Extension->Queue.DirectHostWriteConfigured) + || Extension->hDeviceFile == NULL + || Extension->bReadOnly) return TCCompleteDiskIrp (Irp, STATUS_SUCCESS, 0); if (!EncryptedIoQueueIsRunning (&Extension->Queue)) @@ -2677,7 +3000,6 @@ NTSTATUS ProcessMainDeviceControlIrp (PDEVICE_OBJECT DeviceObject, PEXTENSION Ex prop->totalBytesRead = ListExtension->Queue.TotalBytesRead; prop->totalBytesWritten = ListExtension->Queue.TotalBytesWritten; - prop->volFormatVersion = ListExtension->cryptoInfo->LegacyVolume ? TC_VOLUME_FORMAT_VERSION_PRE_6_0 : TC_VOLUME_FORMAT_VERSION; Irp->IoStatus.Status = STATUS_SUCCESS; @@ -3425,6 +3747,48 @@ VOID VolumeThreadProc (PVOID Context) Extension->Queue.DeviceObject = DeviceObject; Extension->Queue.CryptoInfo = Extension->cryptoInfo; Extension->Queue.HostFileHandle = Extension->hDeviceFile; + Extension->Queue.HostDeviceObject = Extension->bRawDevice ? Extension->pFastIoFsdDevice : NULL; + Extension->Queue.HostFileObject = Extension->bRawDevice ? Extension->pfoFastIoDeviceFile : NULL; + Extension->Queue.HostDeviceIoMaxSize = VC_MOUNTED_VOLUME_FAST_IO_MAX_SEGMENT_SIZE; + if (Extension->HostMaximumTransferLength != 0 + && Extension->HostMaximumTransferLength < Extension->Queue.HostDeviceIoMaxSize) + { + Extension->Queue.HostDeviceIoMaxSize = Extension->HostMaximumTransferLength; + } + if (Extension->HostMaximumPhysicalPages != 0) + { + if (Extension->HostMaximumPhysicalPages == 1) + { + Extension->Queue.HostDeviceIoMaxSize = 0; + } + else + { + // Reserve one page for an arbitrary offset within the first buffer page. + ULONGLONG physicalPageTransferLimit = + (ULONGLONG) (Extension->HostMaximumPhysicalPages - 1) * PAGE_SIZE; + if (physicalPageTransferLimit < Extension->Queue.HostDeviceIoMaxSize) + Extension->Queue.HostDeviceIoMaxSize = (ULONG) physicalPageTransferLimit; + } + } + Extension->Queue.HostAlignmentMask = Extension->HostAlignmentMask; + Extension->Queue.HostBytesPerSector = Extension->HostBytesPerSector; + if (Extension->Queue.HostBytesPerSector != 0) + Extension->Queue.HostDeviceIoMaxSize -= Extension->Queue.HostDeviceIoMaxSize % Extension->Queue.HostBytesPerSector; + Extension->Queue.DirectHostIoMaxRequestSize = VC_MOUNTED_VOLUME_FAST_IO_MAX_REQUEST_SIZE; + Extension->Queue.DirectHostReadEnabled = Extension->bRawDevice + && Extension->Queue.HostDeviceObject + && Extension->Queue.HostFileObject + && Extension->Queue.HostDeviceIoMaxSize >= ENCRYPTION_DATA_UNIT_SIZE + && Extension->Queue.DirectHostIoMaxRequestSize >= ENCRYPTION_DATA_UNIT_SIZE + && MountedVolumeFastReadIoEnabled; + Extension->Queue.DirectHostWriteConfigured = Extension->bRawDevice + && Extension->Queue.HostDeviceObject + && Extension->Queue.HostFileObject + && Extension->Queue.HostDeviceIoMaxSize >= ENCRYPTION_DATA_UNIT_SIZE + && Extension->Queue.DirectHostIoMaxRequestSize >= ENCRYPTION_DATA_UNIT_SIZE + && !Extension->bReadOnly + && MountedVolumeFastWriteIoEnabled; + Extension->Queue.DirectHostWriteEnabled = Extension->Queue.DirectHostWriteConfigured; Extension->Queue.VirtualDeviceLength = Extension->DiskLength; Extension->Queue.MaxReadAheadOffset.QuadPart = Extension->HostLength; if (bDevice && pThreadBlock->mount->bPartitionInInactiveSysEncScope @@ -3579,9 +3943,8 @@ LPWSTR TCTranslateCode (ULONG ulCode) TC_CASE_RET_NAME (TC_IOCTL_WRITE_BOOT_DRIVE_SECTOR); TC_CASE_RET_NAME (VC_IOCTL_GET_DRIVE_GEOMETRY_EX); TC_CASE_RET_NAME (VC_IOCTL_EMERGENCY_CLEAR_ALL_KEYS); - TC_CASE_RET_NAME (VC_IOCTL_IS_RAM_ENCRYPTION_ENABLED); - TC_CASE_RET_NAME (VC_IOCTL_ENCRYPTION_QUEUE_PARAMS); - + TC_CASE_RET_NAME (VC_IOCTL_IS_RAM_ENCRYPTION_ENABLED); + TC_CASE_RET_NAME (VC_IOCTL_ENCRYPTION_QUEUE_PARAMS); TC_CASE_RET_NAME (IOCTL_VOLUME_GET_VOLUME_DISK_EXTENTS); TC_CASE_RET_NAME(IOCTL_DISK_GET_DRIVE_GEOMETRY); @@ -3992,20 +4355,32 @@ void TCCloseFsVolume (HANDLE volumeHandle, PFILE_OBJECT fileObject) } -static NTSTATUS TCReadWriteDevice (BOOL write, PDEVICE_OBJECT deviceObject, PVOID buffer, LARGE_INTEGER offset, ULONG length) +static NTSTATUS TCReadWriteDevice (BOOL write, PDEVICE_OBJECT deviceObject, PFILE_OBJECT fileObject, PVOID buffer, LARGE_INTEGER offset, ULONG length, ULONG_PTR *information) { NTSTATUS status; - IO_STATUS_BLOCK ioStatusBlock; + IO_STATUS_BLOCK ioStatusBlock = { STATUS_PENDING, 0 }; PIRP irp; KEVENT completionEvent; ASSERT (KeGetCurrentIrql() <= APC_LEVEL); + if (information) + *information = 0; + KeInitializeEvent (&completionEvent, NotificationEvent, FALSE); irp = IoBuildSynchronousFsdRequest (write ? IRP_MJ_WRITE : IRP_MJ_READ, deviceObject, buffer, length, &offset, &completionEvent, &ioStatusBlock); if (!irp) return STATUS_INSUFFICIENT_RESOURCES; + if (fileObject) + { + PIO_STACK_LOCATION irpSp = IoGetNextIrpStackLocation (irp); + irpSp->FileObject = fileObject; + irp->Flags |= IRP_NOCACHE; + if (write) + irpSp->Flags |= SL_WRITE_THROUGH; + } + ObReferenceObject (deviceObject); status = IoCallDriver (deviceObject, irp); @@ -4016,6 +4391,9 @@ static NTSTATUS TCReadWriteDevice (BOOL write, PDEVICE_OBJECT deviceObject, PVOI status = ioStatusBlock.Status; } + if (information) + *information = ioStatusBlock.Information; + ObDereferenceObject (deviceObject); return status; } @@ -4023,13 +4401,25 @@ static NTSTATUS TCReadWriteDevice (BOOL write, PDEVICE_OBJECT deviceObject, PVOI NTSTATUS TCReadDevice (PDEVICE_OBJECT deviceObject, PVOID buffer, LARGE_INTEGER offset, ULONG length) { - return TCReadWriteDevice (FALSE, deviceObject, buffer, offset, length); + return TCReadWriteDevice (FALSE, deviceObject, NULL, buffer, offset, length, NULL); } NTSTATUS TCWriteDevice (PDEVICE_OBJECT deviceObject, PVOID buffer, LARGE_INTEGER offset, ULONG length) { - return TCReadWriteDevice (TRUE, deviceObject, buffer, offset, length); + return TCReadWriteDevice (TRUE, deviceObject, NULL, buffer, offset, length, NULL); +} + + +NTSTATUS TCReadDeviceUsingFileObjectWithInformation (PDEVICE_OBJECT deviceObject, PFILE_OBJECT fileObject, PVOID buffer, LARGE_INTEGER offset, ULONG length, ULONG_PTR *information) +{ + return TCReadWriteDevice (FALSE, deviceObject, fileObject, buffer, offset, length, information); +} + + +NTSTATUS TCWriteDeviceUsingFileObjectWithInformation (PDEVICE_OBJECT deviceObject, PFILE_OBJECT fileObject, PVOID buffer, LARGE_INTEGER offset, ULONG length, ULONG_PTR *information) +{ + return TCReadWriteDevice (TRUE, deviceObject, fileObject, buffer, offset, length, information); } @@ -4937,54 +5327,57 @@ NTSTATUS ReadRegistryConfigFlags (BOOL driverEntry) NTSTATUS status; uint32 flags = 0; - OrderedFlushBarriersEnabled = FALSE; - RtlInitUnicodeString (&name, L"\\REGISTRY\\MACHINE\\SYSTEM\\CurrentControlSet\\Services\\veracrypt"); status = TCReadRegistryKey (&name, TC_DRIVER_CONFIG_REG_VALUE_NAME, &data); if (NT_SUCCESS (status)) { - if (data->Type == REG_DWORD) + if (data->Type == REG_DWORD && data->DataLength == sizeof (flags)) { flags = *(uint32 *) data->Data; Dump ("Configuration flags = 0x%x\n", flags); + } + else + status = STATUS_INVALID_PARAMETER; - if (driverEntry) - { - if (flags & (TC_DRIVER_CONFIG_CACHE_BOOT_PASSWORD | TC_DRIVER_CONFIG_CACHE_BOOT_PASSWORD_FOR_SYS_FAVORITES)) - CacheBootPassword = TRUE; + TCfree (data); + } + else if (status == STATUS_OBJECT_NAME_NOT_FOUND) + status = STATUS_SUCCESS; - if (flags & TC_DRIVER_CONFIG_DISABLE_NONADMIN_SYS_FAVORITES_ACCESS) - NonAdminSystemFavoritesAccessDisabled = TRUE; + if (NT_SUCCESS (status)) + { + if (driverEntry) + { + if (flags & (TC_DRIVER_CONFIG_CACHE_BOOT_PASSWORD | TC_DRIVER_CONFIG_CACHE_BOOT_PASSWORD_FOR_SYS_FAVORITES)) + CacheBootPassword = TRUE; - if (flags & TC_DRIVER_CONFIG_CACHE_BOOT_PIM) - CacheBootPim = TRUE; + if (flags & TC_DRIVER_CONFIG_DISABLE_NONADMIN_SYS_FAVORITES_ACCESS) + NonAdminSystemFavoritesAccessDisabled = TRUE; - if (flags & VC_DRIVER_CONFIG_BLOCK_SYS_TRIM) - BlockSystemTrimCommand = TRUE; + if (flags & TC_DRIVER_CONFIG_CACHE_BOOT_PIM) + CacheBootPim = TRUE; - /* clear VC_DRIVER_CONFIG_CLEAR_KEYS_ON_NEW_DEVICE_INSERTION if it is set */ - if (flags & VC_DRIVER_CONFIG_CLEAR_KEYS_ON_NEW_DEVICE_INSERTION) - { - flags ^= VC_DRIVER_CONFIG_CLEAR_KEYS_ON_NEW_DEVICE_INSERTION; - WriteRegistryConfigFlags (flags); - } + if (flags & VC_DRIVER_CONFIG_BLOCK_SYS_TRIM) + BlockSystemTrimCommand = TRUE; - RamEncryptionActivated = (flags & VC_DRIVER_CONFIG_ENABLE_RAM_ENCRYPTION) ? TRUE : FALSE; + /* clear VC_DRIVER_CONFIG_CLEAR_KEYS_ON_NEW_DEVICE_INSERTION if it is set */ + if (flags & VC_DRIVER_CONFIG_CLEAR_KEYS_ON_NEW_DEVICE_INSERTION) + { + flags ^= VC_DRIVER_CONFIG_CLEAR_KEYS_ON_NEW_DEVICE_INSERTION; + WriteRegistryConfigFlags (flags); } - EnableHwEncryption ((flags & TC_DRIVER_CONFIG_DISABLE_HARDWARE_ENCRYPTION) ? FALSE : TRUE); - EnableCpuRng ((flags & VC_DRIVER_CONFIG_ENABLE_CPU_RNG) ? TRUE : FALSE); - - EnableExtendedIoctlSupport = (flags & TC_DRIVER_CONFIG_ENABLE_EXTENDED_IOCTL)? TRUE : FALSE; - AllowTrimCommand = (flags & VC_DRIVER_CONFIG_ALLOW_NONSYS_TRIM)? TRUE : FALSE; - AllowWindowsDefrag = (flags & VC_DRIVER_CONFIG_ALLOW_WINDOWS_DEFRAG)? TRUE : FALSE; - OrderedFlushBarriersEnabled = (flags & VC_DRIVER_CONFIG_ENABLE_ORDERED_FLUSH_BARRIERS)? TRUE : FALSE; + RamEncryptionActivated = (flags & VC_DRIVER_CONFIG_ENABLE_RAM_ENCRYPTION) ? TRUE : FALSE; } - else - status = STATUS_INVALID_PARAMETER; - TCfree (data); + EnableHwEncryption ((flags & TC_DRIVER_CONFIG_DISABLE_HARDWARE_ENCRYPTION) ? FALSE : TRUE); + EnableCpuRng ((flags & VC_DRIVER_CONFIG_ENABLE_CPU_RNG) ? TRUE : FALSE); + + EnableExtendedIoctlSupport = (flags & TC_DRIVER_CONFIG_ENABLE_EXTENDED_IOCTL)? TRUE : FALSE; + AllowTrimCommand = (flags & VC_DRIVER_CONFIG_ALLOW_NONSYS_TRIM)? TRUE : FALSE; + AllowWindowsDefrag = (flags & VC_DRIVER_CONFIG_ALLOW_WINDOWS_DEFRAG)? TRUE : FALSE; + OrderedFlushBarriersEnabled = (flags & VC_DRIVER_CONFIG_ENABLE_ORDERED_FLUSH_BARRIERS)? TRUE : FALSE; } if (driverEntry && NT_SUCCESS (TCReadRegistryKey (&name, TC_ENCRYPTION_FREE_CPU_COUNT_REG_VALUE_NAME, &data))) @@ -5027,6 +5420,24 @@ NTSTATUS ReadRegistryConfigFlags (BOOL driverEntry) TCfree(data); } + MountedVolumeFastReadIoEnabled = FALSE; + if (NT_SUCCESS (TCReadRegistryKey (&name, VC_MOUNTED_VOLUME_FAST_READ_IO, &data))) + { + if (data->Type == REG_DWORD) + MountedVolumeFastReadIoEnabled = (*(uint32 *) data->Data) ? TRUE : FALSE; + + TCfree (data); + } + + MountedVolumeFastWriteIoEnabled = FALSE; + if (NT_SUCCESS (TCReadRegistryKey (&name, VC_MOUNTED_VOLUME_FAST_WRITE_IO, &data))) + { + if (data->Type == REG_DWORD) + MountedVolumeFastWriteIoEnabled = (*(uint32 *) data->Data) ? TRUE : FALSE; + + TCfree (data); + } + if (driverEntry) { if (EncryptionIoRequestCount < TC_ENC_IO_QUEUE_PREALLOCATED_IO_REQUEST_COUNT) diff --git a/src/Driver/Ntdriver.h b/src/Driver/Ntdriver.h index b367c87ae9..b7e99a93ac 100644 --- a/src/Driver/Ntdriver.h +++ b/src/Driver/Ntdriver.h @@ -67,6 +67,11 @@ typedef struct EXTENSION HANDLE hDeviceFile; /* Device handle for this device */ PFILE_OBJECT pfoDeviceFile; /* Device fileobject for this device */ PDEVICE_OBJECT pFsdDevice; /* lower level device handle */ + PFILE_OBJECT pfoDirectDeviceFile; /* File object referenced from hDeviceFile */ + PDEVICE_OBJECT pDirectFsdDevice; /* Related device for pfoDirectDeviceFile */ + HANDLE hFastIoDeviceFile; /* Asynchronous handle used by direct host I/O */ + PFILE_OBJECT pfoFastIoDeviceFile; /* File object referenced from hFastIoDeviceFile */ + PDEVICE_OBJECT pFastIoFsdDevice; /* Related device for pfoFastIoDeviceFile */ CRYPTO_INFO *cryptoInfo; /* Cryptographic and other information for this device */ @@ -162,6 +167,8 @@ NTSTATUS TCCreateRootDeviceObject (PDRIVER_OBJECT DriverObject); NTSTATUS TCCreateDeviceObject (PDRIVER_OBJECT DriverObject, PDEVICE_OBJECT * ppDeviceObject, MOUNT_STRUCT * mount); NTSTATUS TCReadDevice (PDEVICE_OBJECT deviceObject, PVOID buffer, LARGE_INTEGER offset, ULONG length); NTSTATUS TCWriteDevice (PDEVICE_OBJECT deviceObject, PVOID buffer, LARGE_INTEGER offset, ULONG length); +NTSTATUS TCReadDeviceUsingFileObjectWithInformation (PDEVICE_OBJECT deviceObject, PFILE_OBJECT fileObject, PVOID buffer, LARGE_INTEGER offset, ULONG length, ULONG_PTR *information); +NTSTATUS TCWriteDeviceUsingFileObjectWithInformation (PDEVICE_OBJECT deviceObject, PFILE_OBJECT fileObject, PVOID buffer, LARGE_INTEGER offset, ULONG length, ULONG_PTR *information); NTSTATUS TCStartThread (PKSTART_ROUTINE threadProc, PVOID threadArg, PKTHREAD *kThread); NTSTATUS TCStartThreadInProcess (PKSTART_ROUTINE threadProc, PVOID threadArg, PKTHREAD *kThread, PEPROCESS process); NTSTATUS TCStartVolumeThread (PDEVICE_OBJECT DeviceObject, PEXTENSION Extension, MOUNT_STRUCT * mount); @@ -196,6 +203,8 @@ NTSTATUS TCCompleteDiskIrp (PIRP irp, NTSTATUS status, ULONG_PTR information); NTSTATUS ProbeRealDriveSize (PDEVICE_OBJECT driveDeviceObject, LARGE_INTEGER *driveSize); BOOL UserCanAccessDriveDevice (); BOOL IsOrderedFlushBarriersEnabled (); +BOOL IsMountedVolumeFastReadIoEnabled (); +BOOL IsMountedVolumeFastWriteIoEnabled (); size_t GetCpuCount (WORD* pGroupCount); USHORT GetCpuGroup (size_t index); void SetThreadCpuGroupAffinity (USHORT index); diff --git a/src/Driver/Ntvol.c b/src/Driver/Ntvol.c index 17eef47c8f..ace7659bab 100644 --- a/src/Driver/Ntvol.c +++ b/src/Driver/Ntvol.c @@ -75,7 +75,12 @@ NTSTATUS TCOpenVolume (PDEVICE_OBJECT DeviceObject, BOOL bAutoCachePassword = mount->bProtectHiddenVolume? FALSE : mount->bCache; Extension->pfoDeviceFile = NULL; + Extension->pfoDirectDeviceFile = NULL; + Extension->pDirectFsdDevice = NULL; + Extension->pfoFastIoDeviceFile = NULL; + Extension->pFastIoFsdDevice = NULL; Extension->hDeviceFile = NULL; + Extension->hFastIoDeviceFile = NULL; Extension->bTimeStampValid = FALSE; /* default value for storage alignment */ @@ -447,17 +452,68 @@ NTSTATUS TCOpenVolume (PDEVICE_OBJECT DeviceObject, } else { - // Try to gain "raw" access to the partition in case there is a live filesystem on it (otherwise, - // the NTFS driver guards hidden sectors and prevents mounting using a backup header e.g. after the user - // accidentally quick-formats a dismounted partition-hosted TrueCrypt volume as NTFS). - - PFILE_OBJECT pfoTmpDeviceFile = NULL; + // Preserve raw access for legacy reads, including backup headers that a + // live filesystem would otherwise hide. + if (NT_SUCCESS (ObReferenceObjectByHandle ( + Extension->hDeviceFile, + 0, + *IoFileObjectType, + KernelMode, + &Extension->pfoDirectDeviceFile, + NULL)) + && Extension->pfoDirectDeviceFile != NULL) + { + Extension->pDirectFsdDevice = IoGetRelatedDeviceObject (Extension->pfoDirectDeviceFile); + TCFsctlCall (Extension->pfoDirectDeviceFile, FSCTL_ALLOW_EXTENDED_DASD_IO, NULL, 0, NULL, 0); + } - if (NT_SUCCESS (ObReferenceObjectByHandle (Extension->hDeviceFile, FILE_ALL_ACCESS, *IoFileObjectType, KernelMode, &pfoTmpDeviceFile, NULL)) - && pfoTmpDeviceFile != NULL) + if (IsMountedVolumeFastReadIoEnabled () + || (IsMountedVolumeFastWriteIoEnabled () && !Extension->bReadOnly)) { - TCFsctlCall (pfoTmpDeviceFile, FSCTL_ALLOW_EXTENDED_DASD_IO, NULL, 0, NULL, 0); - ObDereferenceObject (pfoTmpDeviceFile); + ACCESS_MASK directAccess = GENERIC_READ; + NTSTATUS directStatus; + + if (IsMountedVolumeFastWriteIoEnabled () && !Extension->bReadOnly) + directAccess |= GENERIC_WRITE; + + // Direct requests are issued concurrently by queue-owned threads. + // Keep their asynchronous file object separate from the synchronous + // legacy handle and its file-object serialization. + directStatus = ZwCreateFile ( + &Extension->hFastIoDeviceFile, + directAccess, + &oaFileAttributes, + &IoStatusBlock, + NULL, + FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_SYSTEM, + FILE_SHARE_READ | FILE_SHARE_WRITE, + FILE_OPEN, + FILE_RANDOM_ACCESS | FILE_WRITE_THROUGH | FILE_NO_INTERMEDIATE_BUFFERING, + NULL, + 0); + + if (NT_SUCCESS (directStatus)) + { + directStatus = ObReferenceObjectByHandle ( + Extension->hFastIoDeviceFile, + 0, + *IoFileObjectType, + KernelMode, + &Extension->pfoFastIoDeviceFile, + NULL); + + if (NT_SUCCESS (directStatus) && Extension->pfoFastIoDeviceFile != NULL) + { + Extension->pFastIoFsdDevice = IoGetRelatedDeviceObject (Extension->pfoFastIoDeviceFile); + TCFsctlCall (Extension->pfoFastIoDeviceFile, FSCTL_ALLOW_EXTENDED_DASD_IO, NULL, 0, NULL, 0); + } + else + { + ZwClose (Extension->hFastIoDeviceFile); + Extension->hFastIoDeviceFile = NULL; + Extension->pfoFastIoDeviceFile = NULL; + } + } } } @@ -910,6 +966,9 @@ NTSTATUS TCOpenVolume (PDEVICE_OBJECT DeviceObject, if (Extension->hDeviceFile != NULL) ZwClose (Extension->hDeviceFile); + if (Extension->hFastIoDeviceFile != NULL) + ZwClose (Extension->hFastIoDeviceFile); + /* The cryptoInfo pointer is deallocated if the readheader routines fail so there is no need to deallocate here */ @@ -917,6 +976,12 @@ NTSTATUS TCOpenVolume (PDEVICE_OBJECT DeviceObject, if (Extension->pfoDeviceFile != NULL) ObDereferenceObject (Extension->pfoDeviceFile); + if (Extension->pfoDirectDeviceFile != NULL) + ObDereferenceObject (Extension->pfoDirectDeviceFile); + + if (Extension->pfoFastIoDeviceFile != NULL) + ObDereferenceObject (Extension->pfoFastIoDeviceFile); + /* Free the tmp IO buffers */ if (readBuffer != NULL) TCfree (readBuffer); @@ -935,7 +1000,8 @@ void TCCloseVolume (PDEVICE_OBJECT DeviceObject, PEXTENSION Extension) { RestoreTimeStamp (Extension); } - if (!Extension->bReadOnly && IsOrderedFlushBarriersEnabled ()) + if (!Extension->bReadOnly + && (IsOrderedFlushBarriersEnabled () || Extension->Queue.DirectHostWriteConfigured)) { IO_STATUS_BLOCK ioStatus; NTSTATUS flushStatus = ZwFlushBuffersFile (Extension->hDeviceFile, &ioStatus); @@ -945,11 +1011,28 @@ void TCCloseVolume (PDEVICE_OBJECT DeviceObject, PEXTENSION Extension) ZwClose (Extension->hDeviceFile); Extension->hDeviceFile = NULL; } + if (Extension->hFastIoDeviceFile != NULL) + { + ZwClose (Extension->hFastIoDeviceFile); + Extension->hFastIoDeviceFile = NULL; + } if (Extension->pfoDeviceFile != NULL) { ObDereferenceObject (Extension->pfoDeviceFile); Extension->pfoDeviceFile = NULL; } + if (Extension->pfoDirectDeviceFile != NULL) + { + ObDereferenceObject (Extension->pfoDirectDeviceFile); + Extension->pfoDirectDeviceFile = NULL; + Extension->pDirectFsdDevice = NULL; + } + if (Extension->pfoFastIoDeviceFile != NULL) + { + ObDereferenceObject (Extension->pfoFastIoDeviceFile); + Extension->pfoFastIoDeviceFile = NULL; + Extension->pFastIoFsdDevice = NULL; + } if (Extension->cryptoInfo) { crypto_close (Extension->cryptoInfo);