From 6d78471734d3b32c746a63d6c161ea24d6d69477 Mon Sep 17 00:00:00 2001 From: Mahmoud Bahaa Date: Fri, 7 Apr 2023 03:12:58 +0200 Subject: [PATCH] * Introduce new end point dfs and added new test cases for it * Full support for sql for the same features supported by Loki except for PageBlob * Added clearDb option to configurations in both blob and dfs and set it to true in the test cases * Added support for using postgreSQL in dfs endpoint (needed due to limitations in mysql) without breaking mysql * Added support for permissions save/load, acl, setExpiry, owner and group * Updated visual studio extension file and added binary azurite-datalake,... * Updated readme, swagger definition changes * Updated change log * Added dfs test pipelines to azure-pipelines --- .gitignore | 3 +- .vscode/launch.json | 47 + ChangeLog.md | 5 + README.mcr.md | 21 +- README.md | 128 +- azure-pipelines.yml | 155 +- package-lock.json | 376 + package.json | 44 +- src/azurite.ts | 34 +- src/blob/SqlBlobConfiguration.ts | 6 +- src/blob/SqlBlobServer.ts | 3 +- src/blob/persistence/SqlBlobMetadataStore.ts | 214 +- src/common/ConfigurationBase.ts | 1 + src/common/Environment.ts | 22 + src/common/VSCEnvironment.ts | 13 +- src/common/VSCServerManagerDataLake.ts | 94 + src/common/utils/constants.ts | 17 +- src/common/utils/utils.ts | 38 + src/dfs/DataLakeConfiguration.ts | 60 + src/dfs/DataLakeEnvironment.ts | 138 + src/dfs/DataLakeRequestListenerFactory.ts | 239 + src/dfs/DataLakeServer.ts | 222 + src/dfs/DataLakeServerFactory.ts | 92 + src/dfs/IDataLakeEnvironment.ts | 14 + src/dfs/SqlDataLakeConfiguration.ts | 60 + src/dfs/SqlDataLakeServer.ts | 221 + .../authentication/AccountSASAuthenticator.ts | 386 + .../authentication/BlobSASAuthenticator.ts | 627 + src/dfs/authentication/BlobSASPermissions.ts | 13 + src/dfs/authentication/BlobSASResourceType.ts | 5 + .../BlobSharedKeyAuthenticator.ts | 353 + .../authentication/BlobTokenAuthenticator.ts | 255 + .../authentication/ContainerSASPermissions.ts | 8 + .../authentication/IAuthenticationContext.ts | 3 + src/dfs/authentication/IAuthenticator.ts | 6 + .../authentication/IBlobSASSignatureValues.ts | 727 + src/dfs/authentication/IRange.ts | 48 + .../OperationAccountSASPermission.ts | 753 ++ .../OperationBlobSASPermission.ts | 669 + .../PublicAccessAuthenticator.ts | 156 + .../conditions/ConditionResourceAdapter.ts | 40 + .../conditions/ConditionalHeadersAdapter.ts | 47 + src/dfs/conditions/IConditionResource.ts | 16 + src/dfs/conditions/IConditionalHeaders.ts | 14 + .../IConditionalHeadersValidator.ts | 11 + .../ReadConditionalHeadersValidator.ts | 116 + .../WriteConditionalHeadersValidator.ts | 233 + src/dfs/context/DataLakeContext.ts | 85 + src/dfs/errors/DataLakeError.ts | 92 + src/dfs/errors/NotImplementedError.ts | 22 + src/dfs/errors/StorageErrorFactory.ts | 885 ++ .../errors/StrictModelNotSupportedError.ts | 15 + src/dfs/gc/BlobGCManager.ts | 293 + src/dfs/generated/Context.ts | 143 + src/dfs/generated/ExpressMiddlewareFactory.ts | 162 + src/dfs/generated/ExpressRequestAdapter.ts | 57 + src/dfs/generated/ExpressResponseAdapter.ts | 66 + src/dfs/generated/IRequest.ts | 26 + src/dfs/generated/IResponse.ts | 17 + src/dfs/generated/MiddlewareFactory.ts | 91 + src/dfs/generated/artifacts/mappers.ts | 8746 +++++++++++++ src/dfs/generated/artifacts/models.ts | 10268 +++++++++++++++ src/dfs/generated/artifacts/operation.ts | 99 + src/dfs/generated/artifacts/parameters.ts | 2305 ++++ src/dfs/generated/artifacts/specifications.ts | 3378 +++++ .../generated/errors/DeserializationError.ts | 8 + src/dfs/generated/errors/MiddlewareError.ts | 29 + .../errors/OperationMismatchError.ts | 11 + .../errors/UnsupportedRequestError.ts | 11 + .../generated/handlers/IAppendBlobHandler.ts | 20 + src/dfs/generated/handlers/IBlobHandler.ts | 38 + .../generated/handlers/IBlockBlobHandler.ts | 22 + .../generated/handlers/IContainerHandler.ts | 33 + .../handlers/IFileSystemOperationsHandler.ts | 23 + src/dfs/generated/handlers/IHandlers.ts | 21 + .../generated/handlers/IPageBlobHandler.ts | 25 + .../handlers/IPathOperationsHandler.ts | 29 + src/dfs/generated/handlers/IServiceHandler.ts | 26 + src/dfs/generated/handlers/handlerMappers.ts | 679 + .../middleware/HandlerMiddlewareFactory.ts | 90 + .../middleware/deserializer.middleware.ts | 59 + .../middleware/dispatch.middleware.ts | 189 + .../generated/middleware/end.middleware.ts | 32 + .../generated/middleware/error.middleware.ts | 134 + .../middleware/serializer.middleware.ts | 57 + src/dfs/generated/utils/ILogger.ts | 13 + src/dfs/generated/utils/serializer.ts | 392 + src/dfs/generated/utils/utils.ts | 20 + src/dfs/generated/utils/xml.ts | 41 + src/dfs/handlers/AppendBlobHandler.ts | 244 + src/dfs/handlers/BaseHandler.ts | 20 + src/dfs/handlers/BlobBatchHandler.ts | 674 + src/dfs/handlers/BlobBatchSubRequest.ts | 85 + src/dfs/handlers/BlobBatchSubResponse.ts | 84 + src/dfs/handlers/BlobHandler.ts | 903 ++ src/dfs/handlers/BlockBlobHandler.ts | 507 + src/dfs/handlers/ContainerHandler.ts | 626 + .../handlers/FileSystemOperationsHandler.ts | 415 + src/dfs/handlers/IPageBlobRangesManager.ts | 18 + src/dfs/handlers/PageBlobHandler.ts | 505 + src/dfs/handlers/PageBlobRangesManager.ts | 481 + src/dfs/handlers/PathOperationsHandler.ts | 1648 +++ src/dfs/handlers/ServiceHandler.ts | 411 + src/dfs/handlers/SubResponseTextBodyStream.ts | 35 + src/dfs/lease/BlobLeaseAdapter.ts | 53 + src/dfs/lease/BlobLeaseSyncer.ts | 17 + src/dfs/lease/BlobReadLeaseValidator.ts | 36 + src/dfs/lease/BlobWriteLeaseSyncer.ts | 50 + src/dfs/lease/BlobWriteLeaseValidator.ts | 40 + .../lease/ContainerDeleteLeaseValidator.ts | 41 + src/dfs/lease/ContainerLeaseAdapter.ts | 51 + src/dfs/lease/ContainerLeaseSyncer.ts | 19 + src/dfs/lease/ContainerReadLeaseValidator.ts | 36 + src/dfs/lease/ILeaseState.ts | 31 + src/dfs/lease/LeaseAvailableState.ts | 160 + src/dfs/lease/LeaseBreakingState.ts | 171 + src/dfs/lease/LeaseBrokenState.ts | 219 + src/dfs/lease/LeaseExpiredState.ts | 243 + src/dfs/lease/LeaseFactory.ts | 63 + src/dfs/lease/LeaseLeasedState.ts | 293 + src/dfs/lease/LeaseStateBase.ts | 29 + src/dfs/main.ts | 56 + .../AuthenticationMiddlewareFactory.ts | 56 + .../middlewares/PreflightMiddlewareFactory.ts | 461 + .../StrictModelMiddlewareFactory.ts | 71 + .../blobStorageContext.middleware.ts | 282 + .../BlobReferredExtentsAsyncIterator.ts | 109 + src/dfs/persistence/IDataLakeMetadataStore.ts | 1288 ++ .../persistence/LokiDataLakeMetadataStore.ts | 3893 ++++++ src/dfs/persistence/PageWithDelimiter.ts | 199 + .../persistence/SqlDataLakeMetadataStore.ts | 4279 ++++++ src/dfs/storage-file-datalake/models.ts | 53 + src/dfs/storage-file-datalake/transforms.ts | 288 + src/dfs/utils/constants.ts | 197 + src/dfs/utils/utils.ts | 226 + src/extension.ts | 31 +- .../blob-storage-2021-10-04-data-lake.json | 10916 ++++++++++++++++ .../data-lake-storage.json-2021-04-10.json | 4592 +++++++ swagger/dfs.md | 46 + tests/BlobTestServerFactory.ts | 4 +- tests/blob/apis/blob.test.ts | 20 +- tests/blob/apis/service.test.ts | 4 +- tests/blob/blockblob.highlevel.test.ts | 64 +- tests/blob/handlers/AppendBlobHandler.test.ts | 14 +- tests/blob/oauth.test.ts | 2 +- tests/blob/pagewithdelimiter.test.ts | 28 +- tests/blob/sas.test.ts | 25 +- tests/blob/specialnaming.test.ts | 22 +- tests/dfs/DataLakeTestServerFactory.ts | 78 + .../OPTIONSRequestPolicyFactory.ts | 72 + .../dfs/RequestPolicy/OriginPolicyFactory.ts | 35 + .../QueryRequestPolicyFactory.ts | 52 + tests/dfs/apis/appendblob.test.ts | 562 + tests/dfs/apis/blob.test.ts | 1345 ++ tests/dfs/apis/blobbatch.test.ts | 421 + tests/dfs/apis/blockblob.test.ts | 552 + tests/dfs/apis/container.test.ts | 1214 ++ tests/dfs/apis/pageblob.test.ts | 1759 +++ tests/dfs/apis/service.test.ts | 522 + tests/dfs/authentication.test.ts | 154 + tests/dfs/blobCorsRequest.test.ts | 854 ++ tests/dfs/blockblob.highlevel.test.ts | 494 + tests/dfs/conditions.test.ts | 876 ++ tests/dfs/handlers/AppendBlobHandler.test.ts | 235 + .../handlers/PageBlobRangesManager.test.ts | 274 + tests/dfs/https.test.ts | 54 + tests/dfs/oauth.test.ts | 1029 ++ tests/dfs/pagewithdelimiter.test.ts | 210 + tests/dfs/sas.test.ts | 1808 +++ tests/dfs/specialnaming.test.ts | 568 + .../apis/aborter.test.ts | 116 + .../storage-file-datalake/apis/file.test.ts | 1056 ++ .../apis/filesystem.test.ts | 1205 ++ .../apis/filesystemclient.test.ts | 188 + .../apis/leaseclient.test.ts | 453 + .../apis/pathclient.test.ts | 1857 +++ .../apis/serviceclient.test.ts | 620 + .../apis/specialnaming.test.ts | 320 + .../authentication.test.ts | 154 + .../blobCorsRequest.test.ts | 878 ++ tests/dfs/storage-file-datalake/bugs.test.ts | 152 + tests/dfs/storage-file-datalake/https.test.ts | 54 + .../integration/files-dir-mixed-apis.test.ts | 271 + tests/dfs/storage-file-datalake/oauth.test.ts | 891 ++ tests/dfs/storage-file-datalake/sas.test.ts | 851 ++ .../specialnaming.test.ts | 299 + tests/dfs/utils.test.ts | 59 + tests/exe.test.ts | 94 +- tests/linuxbinary.test.ts | 230 +- tests/testutils.ts | 201 +- 190 files changed, 93860 insertions(+), 196 deletions(-) create mode 100644 src/common/VSCServerManagerDataLake.ts create mode 100644 src/dfs/DataLakeConfiguration.ts create mode 100644 src/dfs/DataLakeEnvironment.ts create mode 100644 src/dfs/DataLakeRequestListenerFactory.ts create mode 100644 src/dfs/DataLakeServer.ts create mode 100644 src/dfs/DataLakeServerFactory.ts create mode 100644 src/dfs/IDataLakeEnvironment.ts create mode 100644 src/dfs/SqlDataLakeConfiguration.ts create mode 100644 src/dfs/SqlDataLakeServer.ts create mode 100644 src/dfs/authentication/AccountSASAuthenticator.ts create mode 100644 src/dfs/authentication/BlobSASAuthenticator.ts create mode 100644 src/dfs/authentication/BlobSASPermissions.ts create mode 100644 src/dfs/authentication/BlobSASResourceType.ts create mode 100644 src/dfs/authentication/BlobSharedKeyAuthenticator.ts create mode 100644 src/dfs/authentication/BlobTokenAuthenticator.ts create mode 100644 src/dfs/authentication/ContainerSASPermissions.ts create mode 100644 src/dfs/authentication/IAuthenticationContext.ts create mode 100644 src/dfs/authentication/IAuthenticator.ts create mode 100644 src/dfs/authentication/IBlobSASSignatureValues.ts create mode 100644 src/dfs/authentication/IRange.ts create mode 100644 src/dfs/authentication/OperationAccountSASPermission.ts create mode 100644 src/dfs/authentication/OperationBlobSASPermission.ts create mode 100644 src/dfs/authentication/PublicAccessAuthenticator.ts create mode 100644 src/dfs/conditions/ConditionResourceAdapter.ts create mode 100644 src/dfs/conditions/ConditionalHeadersAdapter.ts create mode 100644 src/dfs/conditions/IConditionResource.ts create mode 100644 src/dfs/conditions/IConditionalHeaders.ts create mode 100644 src/dfs/conditions/IConditionalHeadersValidator.ts create mode 100644 src/dfs/conditions/ReadConditionalHeadersValidator.ts create mode 100644 src/dfs/conditions/WriteConditionalHeadersValidator.ts create mode 100644 src/dfs/context/DataLakeContext.ts create mode 100644 src/dfs/errors/DataLakeError.ts create mode 100644 src/dfs/errors/NotImplementedError.ts create mode 100644 src/dfs/errors/StorageErrorFactory.ts create mode 100644 src/dfs/errors/StrictModelNotSupportedError.ts create mode 100644 src/dfs/gc/BlobGCManager.ts create mode 100644 src/dfs/generated/Context.ts create mode 100644 src/dfs/generated/ExpressMiddlewareFactory.ts create mode 100644 src/dfs/generated/ExpressRequestAdapter.ts create mode 100644 src/dfs/generated/ExpressResponseAdapter.ts create mode 100644 src/dfs/generated/IRequest.ts create mode 100644 src/dfs/generated/IResponse.ts create mode 100644 src/dfs/generated/MiddlewareFactory.ts create mode 100644 src/dfs/generated/artifacts/mappers.ts create mode 100644 src/dfs/generated/artifacts/models.ts create mode 100644 src/dfs/generated/artifacts/operation.ts create mode 100644 src/dfs/generated/artifacts/parameters.ts create mode 100644 src/dfs/generated/artifacts/specifications.ts create mode 100644 src/dfs/generated/errors/DeserializationError.ts create mode 100644 src/dfs/generated/errors/MiddlewareError.ts create mode 100644 src/dfs/generated/errors/OperationMismatchError.ts create mode 100644 src/dfs/generated/errors/UnsupportedRequestError.ts create mode 100644 src/dfs/generated/handlers/IAppendBlobHandler.ts create mode 100644 src/dfs/generated/handlers/IBlobHandler.ts create mode 100644 src/dfs/generated/handlers/IBlockBlobHandler.ts create mode 100644 src/dfs/generated/handlers/IContainerHandler.ts create mode 100644 src/dfs/generated/handlers/IFileSystemOperationsHandler.ts create mode 100644 src/dfs/generated/handlers/IHandlers.ts create mode 100644 src/dfs/generated/handlers/IPageBlobHandler.ts create mode 100644 src/dfs/generated/handlers/IPathOperationsHandler.ts create mode 100644 src/dfs/generated/handlers/IServiceHandler.ts create mode 100644 src/dfs/generated/handlers/handlerMappers.ts create mode 100644 src/dfs/generated/middleware/HandlerMiddlewareFactory.ts create mode 100644 src/dfs/generated/middleware/deserializer.middleware.ts create mode 100644 src/dfs/generated/middleware/dispatch.middleware.ts create mode 100644 src/dfs/generated/middleware/end.middleware.ts create mode 100644 src/dfs/generated/middleware/error.middleware.ts create mode 100644 src/dfs/generated/middleware/serializer.middleware.ts create mode 100644 src/dfs/generated/utils/ILogger.ts create mode 100644 src/dfs/generated/utils/serializer.ts create mode 100644 src/dfs/generated/utils/utils.ts create mode 100644 src/dfs/generated/utils/xml.ts create mode 100644 src/dfs/handlers/AppendBlobHandler.ts create mode 100644 src/dfs/handlers/BaseHandler.ts create mode 100644 src/dfs/handlers/BlobBatchHandler.ts create mode 100644 src/dfs/handlers/BlobBatchSubRequest.ts create mode 100644 src/dfs/handlers/BlobBatchSubResponse.ts create mode 100644 src/dfs/handlers/BlobHandler.ts create mode 100644 src/dfs/handlers/BlockBlobHandler.ts create mode 100644 src/dfs/handlers/ContainerHandler.ts create mode 100644 src/dfs/handlers/FileSystemOperationsHandler.ts create mode 100644 src/dfs/handlers/IPageBlobRangesManager.ts create mode 100644 src/dfs/handlers/PageBlobHandler.ts create mode 100644 src/dfs/handlers/PageBlobRangesManager.ts create mode 100644 src/dfs/handlers/PathOperationsHandler.ts create mode 100644 src/dfs/handlers/ServiceHandler.ts create mode 100644 src/dfs/handlers/SubResponseTextBodyStream.ts create mode 100644 src/dfs/lease/BlobLeaseAdapter.ts create mode 100644 src/dfs/lease/BlobLeaseSyncer.ts create mode 100644 src/dfs/lease/BlobReadLeaseValidator.ts create mode 100644 src/dfs/lease/BlobWriteLeaseSyncer.ts create mode 100644 src/dfs/lease/BlobWriteLeaseValidator.ts create mode 100644 src/dfs/lease/ContainerDeleteLeaseValidator.ts create mode 100644 src/dfs/lease/ContainerLeaseAdapter.ts create mode 100644 src/dfs/lease/ContainerLeaseSyncer.ts create mode 100644 src/dfs/lease/ContainerReadLeaseValidator.ts create mode 100644 src/dfs/lease/ILeaseState.ts create mode 100644 src/dfs/lease/LeaseAvailableState.ts create mode 100644 src/dfs/lease/LeaseBreakingState.ts create mode 100644 src/dfs/lease/LeaseBrokenState.ts create mode 100644 src/dfs/lease/LeaseExpiredState.ts create mode 100644 src/dfs/lease/LeaseFactory.ts create mode 100644 src/dfs/lease/LeaseLeasedState.ts create mode 100644 src/dfs/lease/LeaseStateBase.ts create mode 100644 src/dfs/main.ts create mode 100644 src/dfs/middlewares/AuthenticationMiddlewareFactory.ts create mode 100644 src/dfs/middlewares/PreflightMiddlewareFactory.ts create mode 100644 src/dfs/middlewares/StrictModelMiddlewareFactory.ts create mode 100644 src/dfs/middlewares/blobStorageContext.middleware.ts create mode 100644 src/dfs/persistence/BlobReferredExtentsAsyncIterator.ts create mode 100644 src/dfs/persistence/IDataLakeMetadataStore.ts create mode 100644 src/dfs/persistence/LokiDataLakeMetadataStore.ts create mode 100644 src/dfs/persistence/PageWithDelimiter.ts create mode 100644 src/dfs/persistence/SqlDataLakeMetadataStore.ts create mode 100644 src/dfs/storage-file-datalake/models.ts create mode 100644 src/dfs/storage-file-datalake/transforms.ts create mode 100644 src/dfs/utils/constants.ts create mode 100644 src/dfs/utils/utils.ts create mode 100644 swagger/blob-storage-2021-10-04-data-lake.json create mode 100644 swagger/data-lake-storage.json-2021-04-10.json create mode 100644 swagger/dfs.md create mode 100644 tests/dfs/DataLakeTestServerFactory.ts create mode 100644 tests/dfs/RequestPolicy/OPTIONSRequestPolicyFactory.ts create mode 100644 tests/dfs/RequestPolicy/OriginPolicyFactory.ts create mode 100644 tests/dfs/RequestPolicy/QueryRequestPolicyFactory.ts create mode 100644 tests/dfs/apis/appendblob.test.ts create mode 100644 tests/dfs/apis/blob.test.ts create mode 100644 tests/dfs/apis/blobbatch.test.ts create mode 100644 tests/dfs/apis/blockblob.test.ts create mode 100644 tests/dfs/apis/container.test.ts create mode 100644 tests/dfs/apis/pageblob.test.ts create mode 100644 tests/dfs/apis/service.test.ts create mode 100644 tests/dfs/authentication.test.ts create mode 100644 tests/dfs/blobCorsRequest.test.ts create mode 100644 tests/dfs/blockblob.highlevel.test.ts create mode 100644 tests/dfs/conditions.test.ts create mode 100644 tests/dfs/handlers/AppendBlobHandler.test.ts create mode 100644 tests/dfs/handlers/PageBlobRangesManager.test.ts create mode 100644 tests/dfs/https.test.ts create mode 100644 tests/dfs/oauth.test.ts create mode 100644 tests/dfs/pagewithdelimiter.test.ts create mode 100644 tests/dfs/sas.test.ts create mode 100644 tests/dfs/specialnaming.test.ts create mode 100644 tests/dfs/storage-file-datalake/apis/aborter.test.ts create mode 100644 tests/dfs/storage-file-datalake/apis/file.test.ts create mode 100644 tests/dfs/storage-file-datalake/apis/filesystem.test.ts create mode 100644 tests/dfs/storage-file-datalake/apis/filesystemclient.test.ts create mode 100644 tests/dfs/storage-file-datalake/apis/leaseclient.test.ts create mode 100644 tests/dfs/storage-file-datalake/apis/pathclient.test.ts create mode 100644 tests/dfs/storage-file-datalake/apis/serviceclient.test.ts create mode 100644 tests/dfs/storage-file-datalake/apis/specialnaming.test.ts create mode 100644 tests/dfs/storage-file-datalake/authentication.test.ts create mode 100644 tests/dfs/storage-file-datalake/blobCorsRequest.test.ts create mode 100644 tests/dfs/storage-file-datalake/bugs.test.ts create mode 100644 tests/dfs/storage-file-datalake/https.test.ts create mode 100644 tests/dfs/storage-file-datalake/integration/files-dir-mixed-apis.test.ts create mode 100644 tests/dfs/storage-file-datalake/oauth.test.ts create mode 100644 tests/dfs/storage-file-datalake/sas.test.ts create mode 100644 tests/dfs/storage-file-datalake/specialnaming.test.ts create mode 100644 tests/dfs/utils.test.ts diff --git a/.gitignore b/.gitignore index 32dee9d64..e70da1d07 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,5 @@ temp *.pem azurite.exe .pkg-cache -release \ No newline at end of file +release +.fake \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index 4eb29e85b..656140768 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -96,6 +96,30 @@ }, "outputCapture": "std" }, + { + "type": "node", + "request": "launch", + "name": "Azurite Blob Service - Loki", + "cwd": "${workspaceFolder}", + "runtimeArgs": ["-r", "ts-node/register"], + "args": ["${workspaceFolder}/src/blob/main.ts", "-d", "debug.log", "--skipApiVersionCheck"], + "env": { + "AZURITE_ACCOUNTS": "" + }, + "outputCapture": "std" + }, + { + "type": "node", + "request": "launch", + "name": "Azurite DataLake Service - Loki", + "cwd": "${workspaceFolder}", + "runtimeArgs": ["-r", "ts-node/register"], + "args": ["${workspaceFolder}/src/dfs/main.ts", "-d", "debug.log", "--skipApiVersionCheck"], + "env": { + "AZURITE_ACCOUNTS": "" + }, + "outputCapture": "std" + }, { "type": "node", "request": "launch", @@ -204,6 +228,29 @@ "internalConsoleOptions": "openOnSessionStart", "outputCapture": "std" }, + { + "type": "node", + "request": "launch", + "name": "Current Mocha TS File - PostgreSQL", + "cwd": "${workspaceFolder}", + "runtimeArgs": ["-r", "ts-node/register"], + "args": [ + "${workspaceFolder}/node_modules/mocha/bin/_mocha", + "-u", + "tdd", + "--timeout", + "999999", + "--colors", + "${workspaceFolder}/${relativeFile}" + ], + "env": { + "AZURITE_ACCOUNTS": "", + "AZURITE_TEST_DB": "postgres://postgres:postgres@127.0.0.1:5432/azurite_dfs_test", + "NODE_TLS_REJECT_UNAUTHORIZED": "0" + }, + "internalConsoleOptions": "openOnSessionStart", + "outputCapture": "std" + }, { "name": "VSC Extension", "type": "extensionHost", diff --git a/ChangeLog.md b/ChangeLog.md index eec0536fa..f005a40d9 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -7,6 +7,11 @@ Blob: - Fixed issue of: blob batch subresponse is slightly different from the on from Azure serivce, which causes exception in CPP SDK. +- support copy when using sql as metadatastore + +DataLake: + +- Introduced dfs endpoint with the following features see readme for more details on what is supported and what is not ## 2023.03 Version 3.23.0 diff --git a/README.mcr.md b/README.mcr.md index 531c4cfd1..da92a5d07 100644 --- a/README.mcr.md +++ b/README.mcr.md @@ -17,12 +17,13 @@ Azurite is an open source Azure Storage API compatible server (emulator). Based # How to Use this Image ```bash -docker run -p 10000:10000 -p 10001:10001 -p 10002:10002 mcr.microsoft.com/azure-storage/azurite +docker run -p 10000:10000 -p 10001:10001 -p 10002:10002 -p 10003:10003 mcr.microsoft.com/azure-storage/azurite ``` `-p 10000:10000` will expose blob service's default listening port. `-p 10001:10001` will expose queue service's default listening port. `-p 10002:10002` will expose table service's default listening port. +`-p 10003:10003` will expose datalake service's default listening port. Just run blob service: @@ -30,16 +31,22 @@ Just run blob service: docker run -p 10000:10000 mcr.microsoft.com/azure-storage/azurite azurite-blob --blobHost 0.0.0.0 ``` +Just run datalake service: + +```bash +docker run -p 10000:10000 mcr.microsoft.com/azure-storage/azurite azurite-datalake --datalakeHost 0.0.0.0 +``` + Run the image as a service (`-d` = deamon) named `azurite` and restart unless specifically stopped (this is useful when re-starting your development machine for example) ```bash -docker run --name azurite -d --restart unless-stopped -p 10000:10000 -p 10001:10001 -p 10002:10002 mcr.microsoft.com/azure-storage/azurite +docker run --name azurite -d --restart unless-stopped -p 10000:10000 -p 10001:10001 -p 10002:10002 -p 10003:10003 mcr.microsoft.com/azure-storage/azurite ``` **Run Azurite V3 docker image with customized persisted data location** ```bash -docker run -p 10000:10000 -p 10001:10001 -p 10002:10002 -v c:/azurite:/data mcr.microsoft.com/azure-storage/azurite +docker run -p 10000:10000 -p 10001:10001 -p 10002:10002 -p 10003:10003 -v c:/azurite:/data mcr.microsoft.com/azure-storage/azurite ``` `-v c:/azurite:/data` will use and map host path `c:/azurite` as Azurite's workspace location. @@ -47,7 +54,7 @@ docker run -p 10000:10000 -p 10001:10001 -p 10002:10002 -v c:/azurite:/data mcr. **Customize Azurite V3 supported parameters for docker image** ```bash -docker run -p 7777:7777 -p 8888:8888 -p 9999:9999 -v c:/azurite:/workspace mcr.microsoft.com/azure-storage/azurite azurite -l /workspace -d /workspace/debug.log --blobPort 7777 --blobHost 0.0.0.0 --queuePort 8888 --queueHost 0.0.0.0 --tablePort 9999 --tableHost 0.0.0.0 --loose --skipApiVersionCheck --disableProductStyleUrl +docker run -p 7777:7777 -p 8888:8888 -p 9999:9999 -p 6666:6666 -v c:/azurite:/workspace mcr.microsoft.com/azure-storage/azurite azurite -l /workspace -d /workspace/debug.log --blobPort 7777 --blobHost 0.0.0.0 --queuePort 8888 --queueHost 0.0.0.0 --tablePort 9999 --tableHost 0.0.0.0--datalakePort 6666 --datalakeHost 0.0.0.0 --loose --skipApiVersionCheck --disableProductStyleUrl ``` Above command will try to start Azurite image with configurations: @@ -68,13 +75,17 @@ Above command will try to start Azurite image with configurations: `--tableHost 0.0.0.0` defines table service listening endpoint to accept requests from host machine. +`--datalakePort 6666` makes Azurite blob service listen to port 6666, while `-p 6666:6666` redirects requests from host machine's port 6666 to docker instance. + +`--datalakeHost 0.0.0.0` defines blob service listening endpoint to accept requests from host machine. + `--loose` enables loose mode which ignore unsupported headers and parameters. `--skipApiVersionCheck` skip the request API version check. `--disableProductStyleUrl` force parsing storage account name from request Uri path, instead of from request Uri host. -> If you use customized azurite paramters for docker image, `--blobHost 0.0.0.0`, `--queueHost 0.0.0.0` are required parameters. +> If you use customized azurite paramters for docker image, `--blobHost 0.0.0.0`, `--datalakeHost 0.0.0.0`, `--queueHost 0.0.0.0` are required parameters. > In above sample, you need to use **double first forward slash** for location and debug path parameters to avoid a [known issue](https://stackoverflow.com/questions/48427366/docker-build-command-add-c-program-files-git-to-the-path-passed-as-build-argu) for Git on Windows. diff --git a/README.md b/README.md index e4d01c416..963389b5b 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,14 @@ Compared to V2, Azurite V3 implements a new architecture leveraging code generat - Create/List/Delete Containers - Create/Read/List/Update/Delete Block Blobs - Create/Read/List/Update/Delete Page Blobs +- DataLake storage features align with Azure Storage API version 2022-11-02 (Refer to support matrix section below) + + - SharedKey/Account SAS/Service SAS/Public Access Authentications/OAuth + - Get/Set FileSystem(Container) Properties + - Create/List/Delete FileSystems(analogous to containers) + - Create/Read/List/Update/Delete Paths(Files/Directories) + - Get/Set Paths Properties + - lease actions on Paths - Queue storage features align with Azure Storage API version 2022-11-02 (Refer to support matrix section below) - SharedKey/Account SAS/Service SAS/OAuth - Get/Set Queue Service Properties @@ -147,6 +155,12 @@ For example, to start blob service only: $ azurite-blob -l path/to/azurite/workspace ``` +start Data Lake service only: + +```bash +$ azurite-datalake -l path/to/azurite/workspace +``` + Start queue service only: ```bash @@ -179,6 +193,9 @@ Extension supports following Visual Studio Code commands: - `Azurite: Start Table Service` Start table service - `Azurite: Close Table Service` Close table service - `Azurite: Clean Table Service` Clean table service +- `Azurite: Start DataLake Service` Start DataLake service +- `Azurite: Close DataLake Service` Close DataLake service +- `Azurite: Clean DataLake Service` Clean DataLake service Following extension configurations are supported: @@ -188,6 +205,8 @@ Following extension configurations are supported: - `azurite.queuePort` Queue service listening port, by default 10001 - `azurite.tableHost` Table service listening endpoint, by default 127.0.0.1 - `azurite.tablePort` Table service listening port, by default 10002 +- `azurite.datalakeHost` DataLake service listening endpoint, by default 127.0.0.1 +- `azurite.datalakePort` DataLake service listening port, by default 10003 - `azurite.location` Workspace location folder path (can be relative or absolute). By default, in the VS Code extension, the currently opened folder is used. If launched from the command line, the current process working directory is the default. Relative paths are resolved relative to the default folder. - `azurite.silent` Silent mode to disable access log in Visual Studio channel, by default false - `azurite.debug` Output debug log into Azurite channel, by default false @@ -206,12 +225,13 @@ Following extension configurations are supported: > Note. Find more docker images tags in https://mcr.microsoft.com/v2/azure-storage/azurite/tags/list ```bash -docker run -p 10000:10000 -p 10001:10001 -p 10002:10002 mcr.microsoft.com/azure-storage/azurite +docker run -p 10000:10000 -p 10001:10001 -p 10002:10002 -p 10003:10003 mcr.microsoft.com/azure-storage/azurite ``` `-p 10000:10000` will expose blob service's default listening port. `-p 10001:10001` will expose queue service's default listening port. `-p 10002:10002` will expose table service's default listening port. +`-p 10003:10003` will expose datalake service's default listening port. Or just run blob service: @@ -230,7 +250,7 @@ docker run -p 10000:10000 -p 10001:10001 -v c:/azurite:/data mcr.microsoft.com/a #### Customize all Azurite V3 supported parameters for docker image ```bash -docker run -p 7777:7777 -p 8888:8888 -p 9999:9999 -v c:/azurite:/workspace mcr.microsoft.com/azure-storage/azurite azurite -l /workspace -d /workspace/debug.log --blobPort 7777 --blobHost 0.0.0.0 --queuePort 8888 --queueHost 0.0.0.0 --tablePort 9999 --tableHost 0.0.0.0 --loose --skipApiVersionCheck --disableProductStyleUrl +docker run -p 7777:7777 -p 8888:8888 -p 9999:9999 -p 6666:6666 -v c:/azurite:/workspace mcr.microsoft.com/azure-storage/azurite azurite -l /workspace -d /workspace/debug.log --blobPort 7777 --blobHost 0.0.0.0 --queuePort 8888 --queueHost 0.0.0.0 --tablePort 9999 --tableHost 0.0.0.0 --datalakePort 6666 --datalakeHost 0.0.0.0 --loose --skipApiVersionCheck --disableProductStyleUrl ``` Above command will try to start Azurite image with configurations: @@ -251,13 +271,17 @@ Above command will try to start Azurite image with configurations: `--tableHost 0.0.0.0` defines table service listening endpoint to accept requests from host machine. +`--datalakePort 6666` makes Azurite table service listen to port 6666, while `-p 6666:6666` redirects requests from host machine's port 6666 to docker instance. + +`--datalakeHost 0.0.0.0` defines table service listening endpoint to accept requests from host machine. + `--loose` enables loose mode which ignore unsupported headers and parameters. `--skipApiVersionCheck` skip the request API version check. `--disableProductStyleUrl` force parsing storage account name from request Uri path, instead of from request Uri host. -> If you use customized azurite paramters for docker image, `--blobHost 0.0.0.0`, `--queueHost 0.0.0.0` are required parameters. +> If you use customized azurite paramters for docker image, `--blobHost 0.0.0.0`, `--datalakeHost 0.0.0.0`, `--queueHost 0.0.0.0` are required parameters. > In above sample, you need to use **double first forward slash** for location and debug path parameters to avoid a [known issue](https://stackoverflow.com/questions/48427366/docker-build-command-add-c-program-files-git-to-the-path-passed-as-build-argu) for Git on Windows. @@ -280,15 +304,16 @@ services: - "10000:10000" - "10001:10001" - "10002:10002" + - "10003:10003" ``` ### NuGet -_Releasing Azurite V3 to NuGet is under investigation._ +*Releasing Azurite V3 to NuGet is under investigation.* ### Visual Studio -_Integrate Azurite with Visual Studio is under investigation._ +*Integrate Azurite with Visual Studio is under investigation.* ## Supported Command Line Options @@ -303,6 +328,7 @@ You can customize the listening address per your requirements. --blobHost 127.0.0.1 --queueHost 127.0.0.1 --tableHost 127.0.0.1 +--datalakeHost 127.0.0.1 ``` #### Allow Accepting Requests from Remote (potentially unsafe) @@ -311,6 +337,7 @@ You can customize the listening address per your requirements. --blobHost 0.0.0.0 --queueHost 0.0.0.0 --tableHost 0.0.0.0 +--datalakeHost 0.0.0.0 ``` ### Listening Port Configuration @@ -327,6 +354,7 @@ You can customize the listening port per your requirements. --blobPort 8888 --queuePort 9999 --tablePort 11111 +--datalakePort 2222 ``` #### Let System Auto Select an Available Port @@ -335,6 +363,7 @@ You can customize the listening port per your requirements. --blobPort 0 --queuePort 0 --tablePort 0 +--datalakePort 0 ``` > Note: The port in use is displayed on Azurite startup. @@ -473,13 +502,18 @@ Azurite will refresh customized account name and key from environment variable e By default, Azurite leverages [loki](https://github.com/techfort/LokiJS) as metadata database. However, as an in-memory database, loki limits Azurite's scalability and data persistency. -Set environment variable `AZURITE_DB=dialect://[username][:password][@]host:port/database` to make Azurite blob service switch to a SQL database based metadata storage, like MySql, SqlServer. +Set environment variable `AZURITE_DB=dialect://[username][:password][@]host:port/database` to make Azurite blob service switch to a SQL database based metadata storage, like MySql, SqlServer, PostgreSQL. +Set environment variable `AZURITE_DATALAKE_DB=dialect://[username][:password][@]host:port/database` to make Azurite datalake service switch to a SQL database based metadata storage, like MySql, SqlServer, PostgreSQL. For example, connect to MySql or SqlServer by set environment variables: ```bash set AZURITE_DB=mysql://username:password@localhost:3306/azurite_blob set AZURITE_DB=mssql://username:password@localhost:1024/azurite_blob +set AZURITE_DB=postgres://username:password@localhost:5432/azurite_blob +set AZURITE_DATALAKE_DB=mysql://username:password@localhost:3306/azurite_datalake +set AZURITE_DATALAKE_DB=mssql://username:password@localhost:1024/azurite_datalake +set AZURITE_DATALAKE_DB=postgres://username:password@localhost:5432/azurite_datalake ``` When Azurite starts with above environment variable, it connects to the configured database, and creates tables if not exist. @@ -487,7 +521,7 @@ This feature is in preview, when Azurite changes database table schema, you need > Note. Need to manually create database before starting Azurite instance. -> Note. Blob Copy & Page Blob are not supported by SQL based metadata implementation. +> Note. Page Blob are not supported by SQL based metadata implementation. > Tips. Create database instance quickly with docker, for example `docker run --name mysql -p 3306:3306 -e MYSQL_ROOT_PASSWORD=my-secret-pw -d mysql:latest`. Grant external access and create database `azurite_blob` using `docker exec mysql mysql -u root -pmy-secret-pw -e "GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES; create database azurite_blob;"`. Notice that, above commands are examples, you need to carefully define the access permissions in your production environment. @@ -529,7 +563,7 @@ If you start Azurite with docker, you need to map the folder contains the cert a In following example, the local folder c:/azurite contains the cert and key files, and map it to /workspace on docker. ```bash -docker run -p 10000:10000 -p 10001:10001 -p 10002:10002 -v c:/azurite:/workspace mcr.microsoft.com/azure-storage/azurite azurite --blobHost 0.0.0.0 --queueHost 0.0.0.0 --tableHost 0.0.0.0 --cert /workspace/127.0.0.1.pem --key /workspace/127.0.0.1-key.pem +docker run -p 10000:10000 -p 10001:10001 -p 10002:10002 -p 10003:10003 -v c:/azurite:/workspace mcr.microsoft.com/azure-storage/azurite azurite --blobHost 0.0.0.0 --queueHost 0.0.0.0 --tableHost 0.0.0.0 --datalakeHost 0.0.0.0 --cert /workspace/127.0.0.1.pem --key /workspace/127.0.0.1-key.pem ``` ##### OpenSSL @@ -709,6 +743,21 @@ var client = new QueueClient("DefaultEndpointsProtocol=https;AccountName=devstor var client = new QueueClient(new Uri("https://127.0.0.1:10001/devstoreaccount1/queue-name"), new StorageSharedKeyCredential("devstoreaccount1", "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==")); ``` +#### Azure DataLake + +You can also instantiate DataLakeFileSystemClient, DataLakeServiceClient, or DatalakeFileClient + +```csharp +// With container url and DefaultAzureCredential +var client = new DataLakeFileSystemClient(new Uri("https://127.0.0.1:10003/devstoreaccount1/filesystem-name"), new DefaultAzureCredential()); + +// With connection string +var client = new DataLakeFileSystemClient("DefaultEndpointsProtocol=https;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=https://127.0.0.1:10003/devstoreaccount1;QueueEndpoint=https://127.0.0.1:10001/devstoreaccount1;", "filesystem-name"); + +// With account name and key +var client = new DataLakeFileSystemClient(new Uri("https://127.0.0.1:10003/devstoreaccount1/filesystem-name"), new StorageSharedKeyCredential("devstoreaccount1", "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==")); +``` + ### Storage Explorer #### Storage Explorer with Azurite HTTP @@ -757,6 +806,9 @@ Following files or folders may be created when initializing Azurite in selected - `azurite_db_blob.json` Metadata file used by Azurite blob service. (No when starting Azurite against external database) - `azurite_db_blob_extent.json` Extent metadata file used by Azurite blob service. (No when starting Azurite against external database) - `blobstorage` Persisted bindary data by Azurite blob service. +- `azurite_db_datalake.json` Metadata file used by Azurite DataLake service. (No when starting Azurite against external database) +- `azurite_db_datalake_extent.json` Extent metadata file used by Azurite DataLake service. (No when starting Azurite against external database) +- `datalakestorage` Persisted bindary data by Azurite DataLake service. - `azurite_db_queue.json` Metadata file used by Azurite queue service. (No when starting Azurite against external database) - `azurite_db_queue_extent.json` Extent metadata file used by Azurite queue service. (No when starting Azurite against external database) - `queuestorage` Persisted bindary data by Azurite queue service. @@ -810,6 +862,12 @@ The service endpoints for Azurite blob service: http://127.0.0.1:10000// ``` +The service endpoints for Azurite datalake service: + +``` +http://127.0.0.1:10003// +``` + #### Production-style URL Optionally, you could modify your hosts file, to access an account with production-style URL. @@ -820,6 +878,7 @@ First, add line(s) to your hosts file, like: 127.0.0.1 account1.blob.localhost 127.0.0.1 account1.queue.localhost 127.0.0.1 account1.table.localhost +127.0.0.1 account1.dfs.localhost ``` Secondly, set environment variables to enable customized storage accounts & keys: @@ -837,6 +896,10 @@ In the connection string below, it is assumed default ports are used. ``` DefaultEndpointsProtocol=http;AccountName=account1;AccountKey=key1;BlobEndpoint=http://account1.blob.localhost:10000;QueueEndpoint=http://account1.queue.localhost:10001;TableEndpoint=http://account1.table.localhost:10002; ``` +For dfs +``` +DefaultEndpointsProtocol=http;AccountName=account1;AccountKey=key1;BlobEndpoint=http://account1.blob.localhost:10003;QueueEndpoint=http://account1.queue.localhost:10001;TableEndpoint=http://account1.table.localhost:10002; +``` > Note. Do not access default account in this way with Azure Storage Explorer. There is a bug that Storage Explorer is always adding account name in URL path, causing failures. @@ -979,6 +1042,55 @@ Detailed support matrix: - Get Page Ranges Continuation Token - Cold Tier +Latest ersion supports for **2022-11-02** API version **datalake** service. + +- Supported Vertical Features + + - CORS and Preflight + - SharedKey Authentication + - OAuth authentication + - Shared Access Signature Account Level + - Shared Access Signature Service Level (Not support response header override in service SAS) + - Container Public Access + +- Supported REST APIs + + - FileSystem: + - Create FileSystem + - Delete FileSystem + - Get FileSystem Properties + - List FileSystems + - Set FileSystem Properties + - Path: + - Create Path + - Delete Path + - Get Path Properties + - Lease Opertaions on Path + - List paths + - Read Operation + - Update Path + - Append Data + - Flush Data + - Set Path Properties + - Blob: + - see above section for details + +- Following features or REST APIs are NOT supported or limited supported in this release (will support more features per customers feedback in future releases) + + - Path: + - Update Path + - Parallel append/flush + - Set Access Control (partially supported) + - Set Access Control Recurisve ("set" only supported, "modify" and "remove" not supported) + - UnDelete Path + - expiry is partially supported + - Blob: + - see above section for details + - Other: + - content-crc64 + - cpk info + - encryption scope + Latest version supports for **2022-11-02** API version **queue** service. Detailed support matrix: diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b159421dc..917886dee 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -37,7 +37,7 @@ jobs: workingDirectory: "./" displayName: "npm run test:blob" env: {} - + - job: blobtestubuntu22_04 displayName: Blob Test Linux Ubuntu 22.04 pool: @@ -53,12 +53,12 @@ jobs: inputs: versionSpec: "$(node_version)" displayName: "Install Node.js" - + - script: | npm ci --legacy-peer-deps workingDirectory: "./" displayName: "npm ci --legacy-peer-deps" - + - script: | npm run test:blob workingDirectory: "./" @@ -154,6 +154,149 @@ jobs: displayName: "npm run test:blob" env: {} + - job: dfstestubuntu20_04 + displayName: Dfs Test Linux Ubuntu 20.04 LTS + pool: + vmImage: "ubuntu-20.04" + strategy: + matrix: + node_12_x: + node_version: 12.x + node_14_x: + node_version: 14.x + steps: + - task: NodeTool@0 + inputs: + versionSpec: "$(node_version)" + displayName: "Install Node.js" + + - script: | + npm ci --legacy-peer-deps + workingDirectory: "./" + displayName: "npm ci --legacy-peer-deps" + + - script: | + npm run test:dfs + workingDirectory: "./" + displayName: "npm run test:dfs" + env: {} + + - job: dfstestubuntu22_04 + displayName: Dfs Test Linux Ubuntu 22.04 + pool: + vmImage: "ubuntu-latest" + strategy: + matrix: + node_16_x: + node_version: 16.x + node_18_x: + node_version: 18.x + steps: + - task: NodeTool@0 + inputs: + versionSpec: "$(node_version)" + displayName: "Install Node.js" + + - script: | + npm ci --legacy-peer-deps + workingDirectory: "./" + displayName: "npm ci --legacy-peer-deps" + + - script: | + npm run test:dfs + workingDirectory: "./" + displayName: "npm run test:dfs" + env: {} + + - job: dfstestwin + displayName: Dfs Test Windows + pool: + vmImage: "windows-latest" + strategy: + matrix: + node_14_x: + node_version: 14.x + node_16_x: + node_version: 16.x + steps: + - task: NodeTool@0 + inputs: + versionSpec: "$(node_version)" + displayName: "Install Node.js" + + - script: | + npm ci --legacy-peer-deps + workingDirectory: "./" + displayName: "npm ci --legacy-peer-deps" + + - script: | + npm run test:dfs + workingDirectory: "./" + displayName: "npm run test:dfs" + env: {} + + - job: dfstestmac + displayName: Dfs Test Mac + pool: + vmImage: "macOS-latest" + strategy: + matrix: + node_14_x: + node_version: 14.x + node_16_x: + node_version: 16.x + steps: + - task: NodeTool@0 + inputs: + versionSpec: "$(node_version)" + displayName: "Install Node.js" + + - script: | + npm ci --legacy-peer-deps + workingDirectory: "./" + displayName: "npm ci --legacy-peer-deps" + + - script: | + npm run test:dfs + workingDirectory: "./" + displayName: "npm run test:dfs" + env: {} + + - job: dfstestmysql + displayName: Dfs Test Mysql + pool: + vmImage: "ubuntu-latest" + strategy: + matrix: + node_14_x: + node_version: 14.x + node_16_x: + node_version: 16.x + steps: + - task: NodeTool@0 + inputs: + versionSpec: "$(node_version)" + displayName: "Install Node.js" + + - script: | + npm ci --legacy-peer-deps + workingDirectory: "./" + displayName: "npm ci --legacy-peer-deps" + + - script: | + docker run --name mysql -p 13306:3306 -e MYSQL_ROOT_PASSWORD=my-secret-pw -d mysql + sleep 60 + docker exec mysql mysql -u root -pmy-secret-pw -e "GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" + docker exec mysql mysql -u root -pmy-secret-pw -e "create database azurite_blob_test;" + workingDirectory: "./" + displayName: "Setup mysql docker instance" + + - script: | + npm run test:dfs:sql:ci + workingDirectory: "./" + displayName: "npm run test:dfs:sql:ci" + env: {} + - job: queuetestlinux displayName: Queue Test Linux pool: @@ -326,7 +469,7 @@ jobs: strategy: matrix: # Table tests no longer suport older node versions - # skip node 14 Azurite install test, since it has issue iwth new npm, which is not azurite issue. + # skip node 14 Azurite install test, since it has issue iwth new npm, which is not azurite issue. # Track with https://github.com/Azure/Azurite/issues/1550. Will add node 14 back later when the issue resolved. #node_14_x: # node_version: 14.x @@ -366,6 +509,7 @@ jobs: azurite-blob -v azurite-queue -v azurite-table -v + azurite-datalake -v workingDirectory: "./" displayName: "Validate npm global installation from GitHub code base" @@ -414,6 +558,7 @@ jobs: azurite-blob -v azurite-queue -v azurite-table -v + azurite-datalake -v workingDirectory: "./" displayName: "Validate npm global installation from GitHub code base" @@ -462,6 +607,7 @@ jobs: azurite-blob -v azurite-queue -v azurite-table -v + azurite-datalake -v workingDirectory: "./" displayName: "Validate npm global installation from GitHub code base" @@ -485,6 +631,7 @@ jobs: docker run xstoreazurite.azurecr.io/public/azure-storage/azurite:latest azurite-blob -v docker run xstoreazurite.azurecr.io/public/azure-storage/azurite:latest azurite-queue -v docker run xstoreazurite.azurecr.io/public/azure-storage/azurite:latest azurite-table -v + docker run xstoreazurite.azurecr.io/public/azure-storage/azurite:latest azurite-datalake -v workingDirectory: "./" displayName: "Validate docker image" diff --git a/package-lock.json b/package-lock.json index f25dd7fef..f4f79fbf3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,6 +21,7 @@ "morgan": "^1.9.1", "multistream": "^2.1.1", "mysql2": "^3.2.0", + "pg": "^8.10.0", "rimraf": "^3.0.2", "sequelize": "^6.3.0", "stoppable": "^1.1.0", @@ -35,6 +36,7 @@ "bin": { "azurite": "dist/src/azurite.js", "azurite-blob": "dist/src/blob/main.js", + "azurite-datalake": "dist/src/dfs/main.js", "azurite-queue": "dist/src/queue/main.js", "azurite-table": "dist/src/table/main.js" }, @@ -43,6 +45,7 @@ "@azure/core-rest-pipeline": "^1.2.0", "@azure/data-tables": "^13.0.1", "@azure/storage-blob": "^12.9.0", + "@azure/storage-file-datalake": "^12.12.0", "@azure/storage-queue": "^12.8.0", "@types/args": "^5.0.0", "@types/async": "^3.0.1", @@ -728,6 +731,86 @@ "node": ">=4.0.0" } }, + "node_modules/@azure/storage-file-datalake": { + "version": "12.12.0", + "resolved": "https://registry.npmjs.org/@azure/storage-file-datalake/-/storage-file-datalake-12.12.0.tgz", + "integrity": "sha512-S/vJaV4LZUnVFFlL8PWfSNug1UoAnU2aC+Dp4YvfZjg1iSinfnPP0gvmTi4PZHZ8QdO/fMjWLPBj6zhqbALDtQ==", + "dev": true, + "dependencies": { + "@azure/abort-controller": "^1.0.0", + "@azure/core-http": "^3.0.0", + "@azure/core-paging": "^1.1.1", + "@azure/core-tracing": "1.0.0-preview.13", + "@azure/logger": "^1.0.0", + "@azure/storage-blob": "^12.13.0", + "events": "^3.0.0", + "tslib": "^2.2.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@azure/storage-file-datalake/node_modules/@azure/core-http": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@azure/core-http/-/core-http-3.0.0.tgz", + "integrity": "sha512-BxI2SlGFPPz6J1XyZNIVUf0QZLBKFX+ViFjKOkzqD18J1zOINIQ8JSBKKr+i+v8+MB6LacL6Nn/sP/TE13+s2Q==", + "dev": true, + "dependencies": { + "@azure/abort-controller": "^1.0.0", + "@azure/core-auth": "^1.3.0", + "@azure/core-tracing": "1.0.0-preview.13", + "@azure/core-util": "^1.1.1", + "@azure/logger": "^1.0.0", + "@types/node-fetch": "^2.5.0", + "@types/tunnel": "^0.0.3", + "form-data": "^4.0.0", + "node-fetch": "^2.6.7", + "process": "^0.11.10", + "tslib": "^2.2.0", + "tunnel": "^0.0.6", + "uuid": "^8.3.0", + "xml2js": "^0.4.19" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@azure/storage-file-datalake/node_modules/form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "dev": true, + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/@azure/storage-file-datalake/node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "dev": true, + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/@azure/storage-file-datalake/node_modules/xml2js": { + "version": "0.4.23", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz", + "integrity": "sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==", + "dev": true, + "dependencies": { + "sax": ">=0.6.0", + "xmlbuilder": "~11.0.0" + }, + "engines": { + "node": ">=4.0.0" + } + }, "node_modules/@azure/storage-queue": { "version": "12.12.0", "resolved": "https://registry.npmjs.org/@azure/storage-queue/-/storage-queue-12.12.0.tgz", @@ -3320,6 +3403,14 @@ "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk=" }, + "node_modules/buffer-writer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", + "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==", + "engines": { + "node": ">=4" + } + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -7603,6 +7694,11 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/packet-reader": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz", + "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==" + }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -7699,11 +7795,80 @@ "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=", "dev": true }, + "node_modules/pg": { + "version": "8.10.0", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.10.0.tgz", + "integrity": "sha512-ke7o7qSTMb47iwzOSaZMfeR7xToFdkE71ifIipOAAaLIM0DYzfOAXlgFFmYUIE2BcJtvnVlGCID84ZzCegE8CQ==", + "dependencies": { + "buffer-writer": "2.0.0", + "packet-reader": "1.0.0", + "pg-connection-string": "^2.5.0", + "pg-pool": "^3.6.0", + "pg-protocol": "^1.6.0", + "pg-types": "^2.1.0", + "pgpass": "1.x" + }, + "engines": { + "node": ">= 8.0.0" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, "node_modules/pg-connection-string": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.5.0.tgz", "integrity": "sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ==" }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.6.0.tgz", + "integrity": "sha512-clFRf2ksqd+F497kWFyM21tMjeikn60oGDmqMT8UBrynEwVEX/5R5xd2sdvdo1cZCFlguORNpVuqxIj+aK4cfQ==", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.0.tgz", + "integrity": "sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q==" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "dependencies": { + "split2": "^4.1.0" + } + }, "node_modules/picomatch": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", @@ -8100,6 +8265,41 @@ "node": ">= 10.0.0" } }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", + "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/prebuild-install": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz", @@ -8955,6 +9155,14 @@ "source-map": "^0.5.6" } }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "engines": { + "node": ">= 10.x" + } + }, "node_modules/sqlstring": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.2.tgz", @@ -9935,6 +10143,14 @@ "node": ">=4.0" } }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "engines": { + "node": ">=0.4" + } + }, "node_modules/yallist": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", @@ -10524,6 +10740,73 @@ } } }, + "@azure/storage-file-datalake": { + "version": "12.12.0", + "resolved": "https://registry.npmjs.org/@azure/storage-file-datalake/-/storage-file-datalake-12.12.0.tgz", + "integrity": "sha512-S/vJaV4LZUnVFFlL8PWfSNug1UoAnU2aC+Dp4YvfZjg1iSinfnPP0gvmTi4PZHZ8QdO/fMjWLPBj6zhqbALDtQ==", + "dev": true, + "requires": { + "@azure/abort-controller": "^1.0.0", + "@azure/core-http": "^3.0.0", + "@azure/core-paging": "^1.1.1", + "@azure/core-tracing": "1.0.0-preview.13", + "@azure/logger": "^1.0.0", + "@azure/storage-blob": "^12.13.0", + "events": "^3.0.0", + "tslib": "^2.2.0" + }, + "dependencies": { + "@azure/core-http": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@azure/core-http/-/core-http-3.0.0.tgz", + "integrity": "sha512-BxI2SlGFPPz6J1XyZNIVUf0QZLBKFX+ViFjKOkzqD18J1zOINIQ8JSBKKr+i+v8+MB6LacL6Nn/sP/TE13+s2Q==", + "dev": true, + "requires": { + "@azure/abort-controller": "^1.0.0", + "@azure/core-auth": "^1.3.0", + "@azure/core-tracing": "1.0.0-preview.13", + "@azure/core-util": "^1.1.1", + "@azure/logger": "^1.0.0", + "@types/node-fetch": "^2.5.0", + "@types/tunnel": "^0.0.3", + "form-data": "^4.0.0", + "node-fetch": "^2.6.7", + "process": "^0.11.10", + "tslib": "^2.2.0", + "tunnel": "^0.0.6", + "uuid": "^8.3.0", + "xml2js": "^0.4.19" + } + }, + "form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "dev": true, + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + } + }, + "uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "dev": true + }, + "xml2js": { + "version": "0.4.23", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz", + "integrity": "sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==", + "dev": true, + "requires": { + "sax": ">=0.6.0", + "xmlbuilder": "~11.0.0" + } + } + } + }, "@azure/storage-queue": { "version": "12.12.0", "resolved": "https://registry.npmjs.org/@azure/storage-queue/-/storage-queue-12.12.0.tgz", @@ -12710,6 +12993,11 @@ "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk=" }, + "buffer-writer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", + "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==" + }, "bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -15876,6 +16164,11 @@ "aggregate-error": "^3.0.0" } }, + "packet-reader": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz", + "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==" + }, "parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -15956,11 +16249,61 @@ "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=", "dev": true }, + "pg": { + "version": "8.10.0", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.10.0.tgz", + "integrity": "sha512-ke7o7qSTMb47iwzOSaZMfeR7xToFdkE71ifIipOAAaLIM0DYzfOAXlgFFmYUIE2BcJtvnVlGCID84ZzCegE8CQ==", + "requires": { + "buffer-writer": "2.0.0", + "packet-reader": "1.0.0", + "pg-connection-string": "^2.5.0", + "pg-pool": "^3.6.0", + "pg-protocol": "^1.6.0", + "pg-types": "^2.1.0", + "pgpass": "1.x" + } + }, "pg-connection-string": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.5.0.tgz", "integrity": "sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ==" }, + "pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==" + }, + "pg-pool": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.6.0.tgz", + "integrity": "sha512-clFRf2ksqd+F497kWFyM21tMjeikn60oGDmqMT8UBrynEwVEX/5R5xd2sdvdo1cZCFlguORNpVuqxIj+aK4cfQ==", + "requires": {} + }, + "pg-protocol": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.0.tgz", + "integrity": "sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q==" + }, + "pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "requires": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + } + }, + "pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "requires": { + "split2": "^4.1.0" + } + }, "picomatch": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", @@ -16240,6 +16583,29 @@ } } }, + "postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==" + }, + "postgres-bytea": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", + "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==" + }, + "postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==" + }, + "postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "requires": { + "xtend": "^4.0.0" + } + }, "prebuild-install": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz", @@ -16852,6 +17218,11 @@ "source-map": "^0.5.6" } }, + "split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==" + }, "sqlstring": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.2.tgz", @@ -17606,6 +17977,11 @@ "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==" }, + "xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" + }, "yallist": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", diff --git a/package.json b/package.json index 4426d44a6..49c370d40 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "bin": { "azurite": "./dist/src/azurite.js", "azurite-blob": "./dist/src/blob/main.js", + "azurite-datalake": "dist/src/dfs/main.js", "azurite-queue": "./dist/src/queue/main.js", "azurite-table": "./dist/src/table/main.js" }, @@ -32,6 +33,7 @@ "morgan": "^1.9.1", "multistream": "^2.1.1", "mysql2": "^3.2.0", + "pg": "^8.10.0", "rimraf": "^3.0.2", "sequelize": "^6.3.0", "stoppable": "^1.1.0", @@ -48,6 +50,7 @@ "@azure/core-rest-pipeline": "^1.2.0", "@azure/data-tables": "^13.0.1", "@azure/storage-blob": "^12.9.0", + "@azure/storage-file-datalake": "^12.12.0", "@azure/storage-queue": "^12.8.0", "@types/args": "^5.0.0", "@types/async": "^3.0.1", @@ -96,6 +99,9 @@ "onCommand:azurite.start_blob", "onCommand:azurite.close_blob", "onCommand:azurite.clean_blob", + "onCommand:azurite.start_datalake", + "onCommand:azurite.close_blob", + "onCommand:azurite.clean_blob", "onCommand:azurite.start_queue", "onCommand:azurite.close_queue", "onCommand:azurite.clean_queue", @@ -135,6 +141,21 @@ "title": "Clean Blob Service", "category": "Azurite" }, + { + "command": "azurite.start_datalake", + "title": "Start DataLake Service", + "category": "Azurite" + }, + { + "command": "azurite.close_datalake", + "title": "Close DataLake Service", + "category": "Azurite" + }, + { + "command": "azurite.clean_datalake", + "title": "Clean Blob Service", + "category": "Azurite" + }, { "command": "azurite.start_queue", "title": "Start Queue Service", @@ -219,6 +240,16 @@ "default": 10000, "description": "Blob service listening port, by default 10000" }, + "azurite.datalakeHost": { + "type": "string", + "default": "127.0.0.1", + "description": "DataLake service listening endpoint, by default 127.0.0.1" + }, + "azurite.datalakePort": { + "type": "number", + "default": 10003, + "description": "DataLake service listening port, by default 10003" + }, "azurite.queueHost": { "type": "string", "default": "127.0.0.1", @@ -275,6 +306,7 @@ "build": "tsc", "build:autorest:debug": "autorest ./swagger/blob.md --typescript --typescript.debugger --use=S:/GitHub/XiaoningLiu/autorest.typescript.server", "build:autorest:blob": "autorest ./swagger/blob.md --typescript --use=S:/GitHub/XiaoningLiu/autorest.typescript.server", + "build:autorest:dfs": "autorest ./swagger/dfs.md --typescript --use=S:/GitHub/XiaoningLiu/autorest.typescript.server", "build:autorest:queue": "autorest ./swagger/queue.md --typescript --use=S:/GitHub/XiaoningLiu/autorest.typescript.server", "build:autorest:table": "autorest ./swagger/table.md --typescript --use=S:/GitHub/XiaoningLiu/autorest.typescript.server", "build:exe": "node ./scripts/buildExe.js", @@ -283,9 +315,13 @@ "blob": "node -r ts-node/register src/blob/main.ts", "queue": "node -r ts-node/register src/queue/main.ts", "table": "node -r ts-node/register src/table/main.ts", + "dfs": "node -r ts-node/register src/dfs/main.ts", "azurite": "node -r ts-node/register src/azurite.ts", "lint": "npx eslint src/**/*.ts", - "test": "npm run lint && cross-env NODE_TLS_REJECT_UNAUTHORIZED=0 mocha --compilers ts-node/register --no-timeouts --grep @loki --recursive --exit tests/**/*.test.ts tests/**/**/*.test.ts", + "test": "npm run lint && cross-env NODE_TLS_REJECT_UNAUTHORIZED=0 mocha --compilers ts-node/register --no-timeouts --recursive --exit tests/**/*.test.ts tests/**/**/*.test.ts tests/**/**/**/*.test.ts", + "test:dfs": "npm run lint && cross-env NODE_TLS_REJECT_UNAUTHORIZED=0 mocha --compilers ts-node/register --no-timeouts --grep @loki --recursive --exit tests/dfs/*.test.ts tests/dfs/**/*.test.ts tests/dfs/**/**/*.test.ts", + "test:dfs:sql": "npm run lint && cross-env cross-env NODE_TLS_REJECT_UNAUTHORIZED=0 AZURITE_TEST_DB=mysql://root:my-secret-pw@127.0.0.1:3306/azurite_blob_test mocha --compilers ts-node/register --no-timeouts --grep @sql --recursive --exit tests/dfs/*.test.ts tests/dfs/**/*.test.ts tests/dfs/**/**/*.test.ts", + "test:dfs:sql:ci": "npm run lint && cross-env cross-env NODE_TLS_REJECT_UNAUTHORIZED=0 AZURITE_TEST_DB=mysql://root:my-secret-pw@127.0.0.1:13306/azurite_blob_test mocha --compilers ts-node/register --no-timeouts --grep @sql --recursive --exit tests/dfs/*.test.ts tests/dfs/**/*.test.ts tests/dfs/**/**/*.test.ts", "test:blob": "npm run lint && cross-env NODE_TLS_REJECT_UNAUTHORIZED=0 mocha --compilers ts-node/register --no-timeouts --grep @loki --recursive --exit tests/blob/*.test.ts tests/blob/**/*.test.ts", "test:blob:sql": "npm run lint && cross-env cross-env NODE_TLS_REJECT_UNAUTHORIZED=0 AZURITE_TEST_DB=mysql://root:my-secret-pw@127.0.0.1:3306/azurite_blob_test mocha --compilers ts-node/register --no-timeouts --grep @sql --recursive --exit tests/blob/*.test.ts tests/blob/**/*.test.ts", "test:blob:sql:ci": "npm run lint && cross-env cross-env NODE_TLS_REJECT_UNAUTHORIZED=0 AZURITE_TEST_DB=mysql://root:my-secret-pw@127.0.0.1:13306/azurite_blob_test mocha --compilers ts-node/register --no-timeouts --grep @sql --recursive --exit tests/blob/*.test.ts tests/blob/**/*.test.ts", @@ -295,8 +331,8 @@ "test:linux": "npm run lint && cross-env NODE_TLS_REJECT_UNAUTHORIZED=0 mocha --compilers ts-node/register --no-timeouts tests/linuxbinary.test.ts --exit", "clean": "rimraf dist typings *.log coverage __testspersistence__ temp __testsstorage__ .nyc_output debug.log *.vsix *.tgz", "clean:deep": "npm run clean && rimraf debug.log __*", - "validate:npmpack:win": "npm install --legacy-peer-deps && npm run build && npm pack && cross-var npm install -g azurite-$npm_package_version.tgz && azurite -v && azurite-blob -v && azurite-queue -v && azurite-table -v", - "validate:npmpack:linux_mac": "npm install --legacy-peer-deps && npm run build && npm pack && cross-var sudo npm install -g azurite-$npm_package_version.tgz && azurite -v && azurite-blob -v && azurite-queue -v && azurite-table -v", + "validate:npmpack:win": "npm install --legacy-peer-deps && npm run build && npm pack && cross-var npm install -g azurite-$npm_package_version.tgz && azurite -v && azurite-blob -v && azurite-queue -v && azurite-table -v && azurite-datalake -v", + "validate:npmpack:linux_mac": "npm install --legacy-peer-deps && npm run build && npm pack && cross-var sudo npm install -g azurite-$npm_package_version.tgz && azurite -v && azurite-blob -v && azurite-queue -v && azurite-table -v && azurite-datalake -v", "db:migrate:blob:metadata": "sequelize db:migrate --config migrations/blob/metadata/config/config.json --migrations-path migrations/blob/metadata/migrations", "db:create:blob:metadata": "sequelize db:create --config migrations/blob/metadata/config/config.json --migrations-path migrations/blob/metadata/migrations" }, @@ -320,4 +356,4 @@ "url": "https://github.com/azure/azurite/issues" }, "homepage": "https://github.com/azure/azurite#readme" -} \ No newline at end of file +} diff --git a/src/azurite.ts b/src/azurite.ts index 5801d5481..f8ca41f76 100644 --- a/src/azurite.ts +++ b/src/azurite.ts @@ -23,16 +23,22 @@ import TableConfiguration from "./table/TableConfiguration"; import TableServer from "./table/TableServer"; import { DEFAULT_TABLE_LOKI_DB_PATH } from "./table/utils/constants"; +import { DataLakeServerFactory } from "./dfs/DataLakeServerFactory"; +import DataLakeServer from "./dfs/DataLakeServer"; +import SqlDataLakeServer from "./dfs/SqlDataLakeServer"; // tslint:disable:no-console function shutdown( blobServer: BlobServer | SqlBlobServer, + dfsServer: DataLakeServer | SqlDataLakeServer, queueServer: QueueServer, tableServer: TableServer ) { const blobBeforeCloseMessage = `Azurite Blob service is closing...`; const blobAfterCloseMessage = `Azurite Blob service successfully closed`; + const dfsBeforeCloseMessage = `Azurite DataLake service is closing...`; + const dfsAfterCloseMessage = `Azurite DataLake service successfully closed`; const queueBeforeCloseMessage = `Azurite Queue service is closing...`; const queueAfterCloseMessage = `Azurite Queue service successfully closed`; const tableBeforeCloseMessage = `Azurite Table service is closing...`; @@ -52,6 +58,11 @@ function shutdown( tableServer.close().then(() => { console.log(tableAfterCloseMessage); }); + + console.log(dfsBeforeCloseMessage); + dfsServer.close().then(() => { + console.log(dfsAfterCloseMessage); + }); } /** @@ -75,6 +86,10 @@ async function main() { const blobServer = await blobServerFactory.createServer(env); const blobConfig = blobServer.config; + const dfsServerFactory = new DataLakeServerFactory(); + const dfsServer = await dfsServerFactory.createServer(env); + const dfsConfig = dfsServer.config; + // TODO: Align with blob DEFAULT_BLOB_PERSISTENCE_ARRAY // TODO: Join for all paths in the array DEFAULT_QUEUE_PERSISTENCE_ARRAY[0].locationPath = join( @@ -157,15 +172,28 @@ async function main() { `Azurite Table service is successfully listening at ${tableServer.getHttpServerAddress()}` ); + // Start server + console.log( + `Azurite DataLake service is starting at ${dfsConfig.getHttpServerAddress()}` + ); + await dfsServer.start(); + console.log( + `Azurite DataLake service is successfully listening at ${dfsConfig.getHttpServerAddress()}` + ); + // Handle close event process .once("message", (msg) => { if (msg === "shutdown") { - shutdown(blobServer, queueServer, tableServer); + shutdown(blobServer, dfsServer, queueServer, tableServer); } }) - .once("SIGINT", () => shutdown(blobServer, queueServer, tableServer)) - .once("SIGTERM", () => shutdown(blobServer, queueServer, tableServer)); + .once("SIGINT", () => + shutdown(blobServer, dfsServer, queueServer, tableServer) + ) + .once("SIGTERM", () => + shutdown(blobServer, dfsServer, queueServer, tableServer) + ); } main().catch((err) => { diff --git a/src/blob/SqlBlobConfiguration.ts b/src/blob/SqlBlobConfiguration.ts index a6e8b0586..493df2bc8 100644 --- a/src/blob/SqlBlobConfiguration.ts +++ b/src/blob/SqlBlobConfiguration.ts @@ -35,7 +35,8 @@ export default class SqlBlobConfiguration extends ConfigurationBase { key: string = "", pwd: string = "", oauth?: string, - disableProductStyleUrl: boolean = false + disableProductStyleUrl: boolean = false, + clearDB: boolean = false ) { super( host, @@ -50,7 +51,8 @@ export default class SqlBlobConfiguration extends ConfigurationBase { key, pwd, oauth, - disableProductStyleUrl + disableProductStyleUrl, + clearDB ); } } diff --git a/src/blob/SqlBlobServer.ts b/src/blob/SqlBlobServer.ts index c0e07e6d3..88cb56b99 100644 --- a/src/blob/SqlBlobServer.ts +++ b/src/blob/SqlBlobServer.ts @@ -66,7 +66,8 @@ export default class SqlBlobServer extends ServerBase { const metadataStore: IBlobMetadataStore = new SqlBlobMetadataStore( configuration.sqlURL, - configuration.sequelizeOptions + configuration.sequelizeOptions, + configuration.clearDB, ); const extentMetadataStore: IExtentMetadataStore = new SqlExtentMetadataStore( diff --git a/src/blob/persistence/SqlBlobMetadataStore.ts b/src/blob/persistence/SqlBlobMetadataStore.ts index 0013185e2..2e0f0f4b3 100644 --- a/src/blob/persistence/SqlBlobMetadataStore.ts +++ b/src/blob/persistence/SqlBlobMetadataStore.ts @@ -107,7 +107,8 @@ export default class SqlBlobMetadataStore implements IBlobMetadataStore { */ public constructor( connectionURI: string, - sequelizeOptions?: SequelizeOptions + sequelizeOptions?: SequelizeOptions, + private readonly clearDB: boolean = false, ) { // Enable encrypt connection for SQL Server if (connectionURI.startsWith("mssql") && sequelizeOptions) { @@ -361,8 +362,12 @@ export default class SqlBlobMetadataStore implements IBlobMetadataStore { } ); - // TODO: sync() is only for development purpose, use migration for production - await this.sequelize.sync(); + if (this.clearDB) { + await this.sequelize.sync( {force : true} ); + } else { + // TODO: sync() is only for development purpose, use migration for production + await this.sequelize.sync(); + } this.initialized = true; } @@ -2542,6 +2547,15 @@ export default class SqlBlobMetadataStore implements IBlobMetadataStore { ); } + // Copy if not exists + if ( + options.modifiedAccessConditions && + options.modifiedAccessConditions.ifNoneMatch === "*" && + destBlob + ) { + throw StorageErrorFactory.getBlobAlreadyExists(context.contextId); + } + // If source is uncommitted or deleted if ( sourceBlob === undefined || @@ -2649,14 +2663,202 @@ export default class SqlBlobMetadataStore implements IBlobMetadataStore { }); } - public copyFromURL( + /** + * Copy from Url. + * + * @param {Context} context + * @param {BlobId} source + * @param {BlobId} destination + * @param {string} copySource + * @param {(Models.BlobMetadata | undefined)} metadata + * @param {(Models.AccessTier | undefined)} tier + * @param {Models.BlobCopyFromURLOptionalParams} [leaseAccessConditions] + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async copyFromURL( context: Context, source: BlobId, destination: BlobId, copySource: string, - metadata: Models.BlobMetadata | undefined + metadata: Models.BlobMetadata | undefined, + tier: Models.AccessTier | undefined, + options: Models.BlobCopyFromURLOptionalParams = {} ): Promise { - throw new Error("Method not implemented."); + return this.sequelize.transaction(async (t) => { + const sourceBlob = await this.getBlobWithLeaseUpdated( + source.account, + source.container, + source.blob, + source.snapshot, + context, + true, + true + ); + + options.sourceModifiedAccessConditions = + options.sourceModifiedAccessConditions || {}; + validateReadConditions( + context, + { + ifModifiedSince: + options.sourceModifiedAccessConditions.sourceIfModifiedSince, + ifUnmodifiedSince: + options.sourceModifiedAccessConditions.sourceIfUnmodifiedSince, + ifMatch: options.sourceModifiedAccessConditions.sourceIfMatch, + ifNoneMatch: options.sourceModifiedAccessConditions.sourceIfNoneMatch + }, + sourceBlob + ); + + const destBlob = await this.getBlobWithLeaseUpdated( + destination.account, + destination.container, + destination.blob, + undefined, + context, + false + ); + + validateWriteConditions( + context, + options.modifiedAccessConditions, + destBlob + ); + + // Copy if not exists + if ( + options.modifiedAccessConditions && + options.modifiedAccessConditions.ifNoneMatch === "*" && + destBlob + ) { + throw StorageErrorFactory.getBlobAlreadyExists(context.contextId); + } + + if (destBlob) { + const lease = new BlobLeaseAdapter(destBlob); + new BlobWriteLeaseSyncer(destBlob).sync(lease); + new BlobWriteLeaseValidator(options.leaseAccessConditions).validate( + lease, + context + ); + } + + // If source is uncommitted or deleted + if ( + sourceBlob === undefined || + sourceBlob.deleted || + !sourceBlob.isCommitted + ) { + throw StorageErrorFactory.getBlobNotFound(context.contextId); + } + + if (sourceBlob.properties.accessTier === Models.AccessTier.Archive) { + throw StorageErrorFactory.getBlobArchived(context.contextId); + } + + await this.checkContainerExist( + context, + destination.account, + destination.container + ); + + // Deep clone a copied blob + const copiedBlob: BlobModel = { + name: destination.blob, + deleted: false, + snapshot: "", + properties: { + ...sourceBlob.properties, + creationTime: context.startTime!, + lastModified: context.startTime!, + etag: newEtag(), + leaseStatus: + destBlob !== undefined + ? destBlob.properties.leaseStatus + : Models.LeaseStatusType.Unlocked, + leaseState: + destBlob !== undefined + ? destBlob.properties.leaseState + : Models.LeaseStateType.Available, + leaseDuration: + destBlob !== undefined + ? destBlob.properties.leaseDuration + : undefined, + copyId: uuid(), + copyStatus: Models.CopyStatusType.Success, + copySource, + copyProgress: sourceBlob.properties.contentLength + ? `${sourceBlob.properties.contentLength}/${sourceBlob.properties.contentLength}` + : undefined, + copyCompletionTime: context.startTime, + copyStatusDescription: undefined, + incrementalCopy: false, + destinationSnapshot: undefined, + deletedTime: undefined, + remainingRetentionDays: undefined, + archiveStatus: undefined, + accessTierChangeTime: undefined + }, + metadata: + metadata === undefined || Object.keys(metadata).length === 0 + ? { ...sourceBlob.metadata } + : metadata, + accountName: destination.account, + containerName: destination.container, + pageRangesInOrder: sourceBlob.pageRangesInOrder, + isCommitted: sourceBlob.isCommitted, + leaseDurationSeconds: + destBlob !== undefined ? destBlob.leaseDurationSeconds : undefined, + leaseId: destBlob !== undefined ? destBlob.leaseId : undefined, + leaseExpireTime: + destBlob !== undefined ? destBlob.leaseExpireTime : undefined, + leaseBreakTime: + destBlob !== undefined ? destBlob.leaseBreakTime : undefined, + committedBlocksInOrder: sourceBlob.committedBlocksInOrder, + persistency: sourceBlob.persistency, + }; + + if ( + copiedBlob.properties.blobType === Models.BlobType.BlockBlob && + tier !== undefined + ) { + copiedBlob.properties.accessTier = this.parseTier(tier); + if (copiedBlob.properties.accessTier === undefined) { + throw StorageErrorFactory.getInvalidHeaderValue(context.contextId, { + HeaderName: "x-ms-access-tier", + HeaderValue: `${tier}` + }); + } + } + + if ( + copiedBlob.properties.blobType === Models.BlobType.PageBlob && + tier !== undefined + ) { + throw StorageErrorFactory.getInvalidHeaderValue(context.contextId, { + HeaderName: "x-ms-access-tier", + HeaderValue: `${tier}` + }); + } + + if (destBlob) { + await BlobsModel.destroy({ + where: { + accountName: destBlob.accountName, + containerName: destBlob.containerName, + blobName: destBlob.name, + }, + transaction: t + }); + } + + await BlobsModel.upsert(this.convertBlobModelToDbModel(copiedBlob), { + transaction: t + }); + + return copiedBlob.properties; + }); } public setTier( diff --git a/src/common/ConfigurationBase.ts b/src/common/ConfigurationBase.ts index 42336eec7..c4ba5cd76 100644 --- a/src/common/ConfigurationBase.ts +++ b/src/common/ConfigurationBase.ts @@ -22,6 +22,7 @@ export default abstract class ConfigurationBase { public readonly pwd: string = "", public readonly oauth?: string, public readonly disableProductStyleUrl: boolean = false, + public readonly clearDB: boolean = false, ) {} public hasCert() { diff --git a/src/common/Environment.ts b/src/common/Environment.ts index 5b90ea6b9..4ea531e21 100644 --- a/src/common/Environment.ts +++ b/src/common/Environment.ts @@ -16,6 +16,10 @@ import { } from "../table/utils/constants"; import IEnvironment from "./IEnvironment"; +import { + DEFAULT_DATA_LAKE_LISTENING_PORT, + DEFAULT_DATA_LAKE_SERVER_HOST_NAME +} from "../dfs/utils/constants"; args .option( @@ -28,6 +32,16 @@ args "Optional. Customize listening port for blob", DEFAULT_BLOB_LISTENING_PORT ) + .option( + ["", "datalakeHost"], + "Optional. Customize listening address for datalake", + DEFAULT_DATA_LAKE_SERVER_HOST_NAME + ) + .option( + ["", "datalakePort"], + "Optional. Customize listening port for datalake", + DEFAULT_DATA_LAKE_LISTENING_PORT + ) .option( ["", "queueHost"], "Optional. Customize listening address for queue", @@ -88,6 +102,14 @@ export default class Environment implements IEnvironment { return this.flags.blobPort; } + public datalakeHost(): string | undefined { + return this.flags.datalakeHost; + } + + public datalakePort(): number | undefined { + return this.flags.datalakePort; + } + public queueHost(): string | undefined { return this.flags.queueHost; } diff --git a/src/common/VSCEnvironment.ts b/src/common/VSCEnvironment.ts index 179c6853d..fe5dbb78e 100644 --- a/src/common/VSCEnvironment.ts +++ b/src/common/VSCEnvironment.ts @@ -15,6 +15,14 @@ export default class VSCEnvironment implements IEnvironment { return this.workspaceConfiguration.get("blobPort"); } + public datalakeHost(): string | undefined { + return this.workspaceConfiguration.get("datalakeHost"); + } + + public datalakePort(): number | undefined { + return this.workspaceConfiguration.get("datalakePort"); + } + public queueHost(): string | undefined { return this.workspaceConfiguration.get("queueHost"); } @@ -62,7 +70,7 @@ export default class VSCEnvironment implements IEnvironment { } else { folder = workspace.workspaceFolders[0]; } - location = resolve(folder.uri.fsPath, location ?? ''); + location = resolve(folder.uri.fsPath, location ?? ""); } await ensureDir(location); @@ -106,7 +114,8 @@ export default class VSCEnvironment implements IEnvironment { public disableProductStyleUrl(): boolean { return ( - this.workspaceConfiguration.get("disableProductStyleUrl") || false + this.workspaceConfiguration.get("disableProductStyleUrl") || + false ); } } diff --git a/src/common/VSCServerManagerDataLake.ts b/src/common/VSCServerManagerDataLake.ts new file mode 100644 index 000000000..413e25f97 --- /dev/null +++ b/src/common/VSCServerManagerDataLake.ts @@ -0,0 +1,94 @@ +import { join } from "path"; + +import DataLakeConfiguration from "../dfs/DataLakeConfiguration"; +import DataLakeServer from "../dfs/DataLakeServer"; +import { + DEFAULT_BLOB_EXTENT_LOKI_DB_PATH, + DEFAULT_BLOB_LOKI_DB_PATH, + DEFAULT_BLOB_PERSISTENCE_ARRAY, + DEFAULT_BLOB_PERSISTENCE_PATH +} from "../blob/utils/constants"; +import * as Logger from "./Logger"; +import NoLoggerStrategy from "./NoLoggerStrategy"; +import VSCChannelLoggerStrategy from "./VSCChannelLoggerStrategy"; +import VSCChannelWriteStream from "./VSCChannelWriteStream"; +import VSCEnvironment from "./VSCEnvironment"; +import VSCServerManagerBase from "./VSCServerManagerBase"; +import VSCServerManagerClosedState from "./VSCServerManagerClosedState"; + +export default class VSCServerManagerDataLake extends VSCServerManagerBase { + public readonly accessChannelStream = new VSCChannelWriteStream( + "Azurite DataLake" + ); + private debuggerLoggerStrategy = new VSCChannelLoggerStrategy( + "Azurite DataLake Debug" + ); + + public constructor() { + super("Azurite DataLake Service", new VSCServerManagerClosedState()); + } + + public getStartCommand(): string { + return "azurite.start_datalake"; + } + + public getCloseCommand(): string { + return "azurite.close_datalake"; + } + + public getCleanCommand(): string { + return "azurite.clean_datalake"; + } + + public async createImpl(): Promise { + const config = await this.getConfiguration(); + Logger.default.strategy = config.enableDebugLog + ? this.debuggerLoggerStrategy + : new NoLoggerStrategy(); + this.server = new DataLakeServer(config); + } + + public async startImpl(): Promise { + await this.server!.start(); + } + + public async closeImpl(): Promise { + this.server!.close(); + } + + public async cleanImpl(): Promise { + await this.createImpl(); + await this.server!.clean(); + } + + private async getConfiguration(): Promise { + const env = new VSCEnvironment(); + const location = await env.location(); + + DEFAULT_BLOB_PERSISTENCE_ARRAY[0].locationPath = join( + location, + DEFAULT_BLOB_PERSISTENCE_PATH + ); + + // Initialize server configuration + const config = new DataLakeConfiguration( + env.datalakeHost(), + env.datalakePort(), + join(location, DEFAULT_BLOB_LOKI_DB_PATH), + join(location, DEFAULT_BLOB_EXTENT_LOKI_DB_PATH), + DEFAULT_BLOB_PERSISTENCE_ARRAY, + !env.silent(), + this.accessChannelStream, + (await env.debug()) === true, + undefined, + env.loose(), + env.skipApiVersionCheck(), + env.cert(), + env.key(), + env.pwd(), + env.oauth(), + env.disableProductStyleUrl() + ); + return config; + } +} diff --git a/src/common/utils/constants.ts b/src/common/utils/constants.ts index 6bf50c37b..40beaf699 100644 --- a/src/common/utils/constants.ts +++ b/src/common/utils/constants.ts @@ -1,3 +1,5 @@ +import { Options as SequelizeOptions } from "sequelize"; + export const AZURITE_ACCOUNTS_ENV = "AZURITE_ACCOUNTS"; // Customize account name and keys by env export const DEFAULT_ACCOUNTS_REFRESH_INTERVAL = 60 * 1000; // 60s export const DEFAULT_FD_CACHE_NUMBER = 100; @@ -21,19 +23,18 @@ export const NO_ACCOUNT_HOST_NAMES = new Set().add("host.docker.internal"); // Use utf8mb4_bin instead of utf8mb4_general_ci to honor case sensitive // https://dev.mysql.com/doc/refman/8.0/en/case-sensitivity.html export const DEFAULT_SQL_COLLATE = "utf8mb4_bin"; -export const DEFAULT_SQL_OPTIONS = { +export const DEFAULT_SQL_OPTIONS: SequelizeOptions = { logging: false, pool: { - max: 20, + max: 50, min: 0, - acquire: 30000, - idle: 10000 + acquire: 100*1000, + idle: 10000, }, - charset: DEFAULT_SQL_CHARSET, - collate: DEFAULT_SQL_COLLATE, dialectOptions: { - timezone: "+00:00" - } + timezone: "+00:00", + charset: DEFAULT_SQL_CHARSET, + }, }; export const BEARER_TOKEN_PREFIX = "Bearer"; diff --git a/src/common/utils/utils.ts b/src/common/utils/utils.ts index 12f47291a..abc61706d 100644 --- a/src/common/utils/utils.ts +++ b/src/common/utils/utils.ts @@ -164,3 +164,41 @@ export async function getMD5FromStream( }); }); } + +/** + * String.prototype.padStart() + * + * @export + * @param {string} currentString + * @param {number} targetLength + * @param {string} [padString=" "] + * @returns {string} + */ +export function padStart( + currentString: string, + targetLength: number, + padString: string = " " +): string { + // if (String.prototype.padStart) { + return currentString.padStart(targetLength, padString); + // } + + // padString = padString || " "; + // if (currentString.length > targetLength) { + // return currentString; + // } else { + // targetLength = targetLength - currentString.length; + // if (targetLength > padString.length) { + // padString += padString.repeat(targetLength / padString.length); + // } + // return padString.slice(0, targetLength) + currentString; + // } +} + +export function getUniqueName(prefix: string): string { + return `${prefix}${new Date().getTime()}${padStart( + Math.floor(Math.random() * 10000).toString(), + 5, + "00000" + )}`; +} diff --git a/src/dfs/DataLakeConfiguration.ts b/src/dfs/DataLakeConfiguration.ts new file mode 100644 index 000000000..29ce364ff --- /dev/null +++ b/src/dfs/DataLakeConfiguration.ts @@ -0,0 +1,60 @@ +import ConfigurationBase from "../common/ConfigurationBase"; +import { StoreDestinationArray } from "../common/persistence/IExtentStore"; +import { + DEFAULT_DATA_LAKE_EXTENT_LOKI_DB_PATH, + DEFAULT_DATA_LAKE_LISTENING_PORT, + DEFAULT_DATA_LAKE_LOKI_DB_PATH, + DEFAULT_DATA_LAKE_PERSISTENCE_ARRAY, + DEFAULT_DATA_LAKE_SERVER_HOST_NAME, + DEFAULT_ENABLE_ACCESS_LOG, + DEFAULT_ENABLE_DEBUG_LOG +} from "./utils/constants"; + +/** + * Default configurations for default implementation of BlobServer. + * + * As default implementation of BlobServer class leverages LokiJS DB. + * This configuration class also maintains configuration settings for LokiJS DB. + * + * When creating other server implementations, should also create a NEW + * corresponding configuration class by extending ConfigurationBase. + * + * @export + * @class Configuration + */ +export default class DataLakeConfiguration extends ConfigurationBase { + public constructor( + host: string = DEFAULT_DATA_LAKE_SERVER_HOST_NAME, + port: number = DEFAULT_DATA_LAKE_LISTENING_PORT, + public readonly metadataDBPath: string = DEFAULT_DATA_LAKE_LOKI_DB_PATH, + public readonly extentDBPath: string = DEFAULT_DATA_LAKE_EXTENT_LOKI_DB_PATH, + public readonly persistencePathArray: StoreDestinationArray = DEFAULT_DATA_LAKE_PERSISTENCE_ARRAY, + enableAccessLog: boolean = DEFAULT_ENABLE_ACCESS_LOG, + accessLogWriteStream?: NodeJS.WritableStream, + enableDebugLog: boolean = DEFAULT_ENABLE_DEBUG_LOG, + debugLogFilePath?: string, + loose: boolean = false, + skipApiVersionCheck: boolean = false, + cert: string = "", + key: string = "", + pwd: string = "", + oauth?: string, + disableProductStyleUrl: boolean = false + ) { + super( + host, + port, + enableAccessLog, + accessLogWriteStream, + enableDebugLog, + debugLogFilePath, + loose, + skipApiVersionCheck, + cert, + key, + pwd, + oauth, + disableProductStyleUrl + ); + } +} diff --git a/src/dfs/DataLakeEnvironment.ts b/src/dfs/DataLakeEnvironment.ts new file mode 100644 index 000000000..720fcac4f --- /dev/null +++ b/src/dfs/DataLakeEnvironment.ts @@ -0,0 +1,138 @@ +import args from "args"; +import { access, ensureDir } from "fs-extra"; +import { dirname } from "path"; + +import IDataLakeEnvironment from "./IDataLakeEnvironment"; +import { + DEFAULT_DATA_LAKE_LISTENING_PORT, + DEFAULT_DATA_LAKE_SERVER_HOST_NAME +} from "./utils/constants"; + +if (!(args as any).config.name) { + args + .option( + ["", "datalakeHost"], + "Optional. Customize listening address for blob", + DEFAULT_DATA_LAKE_SERVER_HOST_NAME + ) + .option( + ["", "datalakePort"], + "Optional. Customize listening port for blob", + DEFAULT_DATA_LAKE_LISTENING_PORT + ) + .option( + ["l", "location"], + "Optional. Use an existing folder as workspace path, default is current working directory", + process.cwd() + ) + .option( + ["s", "silent"], + "Optional. Disable access log displayed in console" + ) + .option( + ["L", "loose"], + "Optional. Enable loose mode which ignores unsupported headers and parameters" + ) + .option( + ["", "skipApiVersionCheck"], + "Optional. Skip the request API version check, request with all Api versions will be allowed" + ) + .option(["", "oauth"], 'Optional. OAuth level. Candidate values: "basic"') + .option(["", "cert"], "Optional. Path to certificate file") + .option(["", "key"], "Optional. Path to certificate key .pem file") + .option( + ["d", "debug"], + "Optional. Enable debug log by providing a valid local file path as log destination" + ) + .option(["", "pwd"], "Optional. Password for .pfx file") + .option( + ["", "disableProductStyleUrl"], + "Optional. Disable getting account name from the host of request Uri, always get account name from the first path segment of request Uri." + ); + + (args as any).config.name = "azurite-datalake"; +} + +export default class DataLakeEnvironment implements IDataLakeEnvironment { + private flags = args.parse(process.argv); + + public datalakeHost(): string | undefined { + return this.flags.datalakeHost; + } + + public datalakePort(): number | undefined { + return this.flags.datalakePort; + } + + public async location(): Promise { + const location = this.flags.location || process.cwd(); + await ensureDir(location); + await access(location); + return location; + } + + public silent(): boolean { + if (this.flags.silent !== undefined) { + return true; + } + return false; + } + + public loose(): boolean { + if (this.flags.loose !== undefined) { + return true; + } + // default is false which will block not supported APIs, headers and parameters + return false; + } + + public skipApiVersionCheck(): boolean { + if (this.flags.skipApiVersionCheck !== undefined) { + return true; + } + // default is false which will check API veresion + return false; + } + + public cert(): string | undefined { + return this.flags.cert; + } + + public key(): string | undefined { + return this.flags.key; + } + + public pwd(): string | undefined { + return this.flags.pwd; + } + + public oauth(): string | undefined { + return this.flags.oauth; + } + + public disableProductStyleUrl(): boolean { + if (this.flags.disableProductStyleUrl !== undefined) { + return true; + } + // default is false which will try to get account name from request Uri hostname + return false; + } + + public async debug(): Promise { + if (typeof this.flags.debug === "string") { + // Enable debug log to file + const debugFilePath = this.flags.debug; + await ensureDir(dirname(debugFilePath)); + await access(dirname(debugFilePath)); + return debugFilePath; + } + + if (this.flags.debug === true) { + throw RangeError( + `Must provide a debug log file path for parameter -d or --debug` + ); + } + + // By default disable debug log + } +} diff --git a/src/dfs/DataLakeRequestListenerFactory.ts b/src/dfs/DataLakeRequestListenerFactory.ts new file mode 100644 index 000000000..ed271c669 --- /dev/null +++ b/src/dfs/DataLakeRequestListenerFactory.ts @@ -0,0 +1,239 @@ +import express from "express"; + +import IAccountDataStore from "../common/IAccountDataStore"; +import IRequestListenerFactory from "../common/IRequestListenerFactory"; +import logger from "../common/Logger"; +import IExtentStore from "../common/persistence/IExtentStore"; +import { RequestListener } from "../common/ServerBase"; +import ExpressMiddlewareFactory from "./generated/ExpressMiddlewareFactory"; +import IHandlers from "./generated/handlers/IHandlers"; +import MiddlewareFactory from "./generated/MiddlewareFactory"; +import ServiceHandler from "./handlers/ServiceHandler"; +import FileSystemOperationsHandler from "./handlers/FileSystemOperationsHandler"; +import AuthenticationMiddlewareFactory from "./middlewares/AuthenticationMiddlewareFactory"; +import PreflightMiddlewareFactory from "./middlewares/PreflightMiddlewareFactory"; +import StrictModelMiddlewareFactory, { + UnsupportedHeadersBlocker, + UnsupportedParametersBlocker +} from "./middlewares/StrictModelMiddlewareFactory"; +import IDataLakeMetaDataStore from "./persistence/IDataLakeMetadataStore"; +import { DEFAULT_CONTEXT_PATH } from "./utils/constants"; + +import morgan = require("morgan"); +import { OAuthLevel } from "../common/models"; +import createStorageBlobContextMiddleware from "./middlewares/blobStorageContext.middleware"; +import PathOperationsHandler from "./handlers/PathOperationsHandler"; +import IAuthenticator from "./authentication/IAuthenticator"; +import AccountSASAuthenticator from "./authentication/AccountSASAuthenticator"; +import BlobSASAuthenticator from "./authentication/BlobSASAuthenticator"; +import BlobSharedKeyAuthenticator from "./authentication/BlobSharedKeyAuthenticator"; +import BlobTokenAuthenticator from "./authentication/BlobTokenAuthenticator"; +import PublicAccessAuthenticator from "./authentication/PublicAccessAuthenticator"; +import AppendBlobHandler from "./handlers/AppendBlobHandler"; +import BlobHandler from "./handlers/BlobHandler"; +import BlockBlobHandler from "./handlers/BlockBlobHandler"; +import ContainerHandler from "./handlers/ContainerHandler"; +import PageBlobHandler from "./handlers/PageBlobHandler"; +import PageBlobRangesManager from "./handlers/PageBlobRangesManager"; + +/** + * Default RequestListenerFactory based on express framework. + * + * When creating other server implementations, such as based on Koa. Should also create a NEW + * corresponding BlobKoaRequestListenerFactory class by extending IRequestListenerFactory. + * + * @export + * @class BlobRequestListenerFactory + * @implements {IRequestListenerFactory} + */ +export default class BlobRequestListenerFactory + implements IRequestListenerFactory +{ + public constructor( + private readonly metadataStore: IDataLakeMetaDataStore, + private readonly extentStore: IExtentStore, + private readonly accountDataStore: IAccountDataStore, + private readonly enableAccessLog: boolean, + private readonly accessLogWriteStream?: NodeJS.WritableStream, + private readonly loose?: boolean, + private readonly skipApiVersionCheck?: boolean, + private readonly oauth?: OAuthLevel, + private readonly disableProductStyleUrl?: boolean + ) {} + + public createRequestListener(): RequestListener { + const app = express().disable("x-powered-by"); + + // MiddlewareFactory is a factory to create auto-generated middleware + const middlewareFactory: MiddlewareFactory = new ExpressMiddlewareFactory( + logger, + DEFAULT_CONTEXT_PATH + ); + + // Create handlers into handler middleware factory + const pageBlobRangesManager = new PageBlobRangesManager(); + + const loose = this.loose || false; + const handlers: IHandlers = { + appendBlobHandler: new AppendBlobHandler( + this.metadataStore, + this.extentStore, + logger, + loose + ), + blobHandler: new BlobHandler( + this.metadataStore, + this.extentStore, + logger, + loose + ), + blockBlobHandler: new BlockBlobHandler( + this.metadataStore, + this.extentStore, + logger, + loose + ), + containerHandler: new ContainerHandler( + this.accountDataStore, + this.oauth, + this.metadataStore, + this.extentStore, + logger, + loose + ), + pageBlobHandler: new PageBlobHandler( + this.metadataStore, + this.extentStore, + logger, + loose, + pageBlobRangesManager + ), + serviceHandler: new ServiceHandler( + this.accountDataStore, + this.oauth, + this.metadataStore, + this.extentStore, + logger, + loose + ), + fileSystemOperationsHandler: new FileSystemOperationsHandler( + this.accountDataStore, + this.oauth, + this.metadataStore, + this.extentStore, + logger, + loose + ), + pathOperationsHandler: new PathOperationsHandler( + this.metadataStore, + this.extentStore, + logger, + loose, + pageBlobRangesManager + ) + }; + + // CORS request handling, preflight request and the corresponding actual request + const preflightMiddlewareFactory = new PreflightMiddlewareFactory(logger); + + // Strict mode unsupported features blocker + const strictModelMiddlewareFactory = new StrictModelMiddlewareFactory( + logger, + [UnsupportedHeadersBlocker, UnsupportedParametersBlocker] + ); + + /* + * Generated middleware should follow strict orders + * Manually created middleware can be injected into any points + */ + + // Access log per request + if (this.enableAccessLog) { + app.use(morgan("common", { stream: this.accessLogWriteStream })); + } + + // Manually created middleware to deserialize feature related context which swagger doesn"t know + app.use( + createStorageBlobContextMiddleware( + this.skipApiVersionCheck, + this.disableProductStyleUrl, + this.loose + ) + ); + + // Dispatch incoming HTTP request to specific operation + app.use(middlewareFactory.createDispatchMiddleware()); + + // Block unsupported features in strict mode by default + if (this.loose === false || this.loose === undefined) { + app.use(strictModelMiddlewareFactory.createStrictModelMiddleware()); + } + + // AuthN middleware, like shared key auth or SAS auth + const authenticationMiddlewareFactory = new AuthenticationMiddlewareFactory( + logger + ); + const authenticators: IAuthenticator[] = [ + new PublicAccessAuthenticator(this.metadataStore, logger), + new BlobSharedKeyAuthenticator(this.accountDataStore, logger), + new AccountSASAuthenticator( + this.accountDataStore, + this.metadataStore, + logger + ), + new BlobSASAuthenticator( + this.accountDataStore, + this.metadataStore, + logger + ) + ]; + if (this.oauth !== undefined) { + authenticators.push( + new BlobTokenAuthenticator(this.accountDataStore, this.oauth, logger) + ); + } + app.use( + authenticationMiddlewareFactory.createAuthenticationMiddleware( + authenticators + ) + ); + + // Generated, will do basic validation defined in swagger + app.use(middlewareFactory.createDeserializerMiddleware()); + + // Generated, inject handlers to create a handler middleware + app.use(middlewareFactory.createHandlerMiddleware(handlers)); + + // CORS + app.use( + preflightMiddlewareFactory.createCorsRequestMiddleware( + this.metadataStore, + true + ) + ); + app.use( + preflightMiddlewareFactory.createCorsRequestMiddleware( + this.metadataStore, + false + ) + ); + + // Generated, will serialize response models into HTTP response + app.use(middlewareFactory.createSerializerMiddleware()); + + // Preflight + app.use( + preflightMiddlewareFactory.createOptionsHandlerMiddleware( + this.metadataStore + ) + ); + + // Generated, will return MiddlewareError and Errors thrown in previous middleware/handlers to HTTP response + app.use(middlewareFactory.createErrorMiddleware()); + + // Generated, will end and return HTTP response immediately + app.use(middlewareFactory.createEndMiddleware()); + + return app; + } +} diff --git a/src/dfs/DataLakeServer.ts b/src/dfs/DataLakeServer.ts new file mode 100644 index 000000000..6b39ecf9f --- /dev/null +++ b/src/dfs/DataLakeServer.ts @@ -0,0 +1,222 @@ +import * as http from "http"; +import * as https from "https"; +import { CertOptions } from "../common/ConfigurationBase"; +import ICleaner from "../common/ICleaner"; +import DataLakeConfiguration from "./DataLakeConfiguration"; +import LokiDataLakeMetadataStore from "./persistence/LokiDataLakeMetadataStore"; +import IExtentMetadataStore from "../common/persistence/IExtentMetadataStore"; +import LokiExtentMetadataStore from "../common/persistence/LokiExtentMetadataStore"; +import IExtentStore from "../common/persistence/IExtentStore"; +import logger from "../common/Logger"; +import FSExtentStore from "../common/persistence/FSExtentStore"; +import IAccountDataStore from "../common/IAccountDataStore"; +import IRequestListenerFactory from "../common/IRequestListenerFactory"; +import AccountDataStore from "../common/AccountDataStore"; +import DataLakeRequestListenerFactory from "./DataLakeRequestListenerFactory"; +import ServerBase, { ServerStatus } from "../common/ServerBase"; +import BlobGCManager from "./gc/BlobGCManager"; +import IGCManager from "../common/IGCManager"; +import IDataLakeMetaDataStore from "./persistence/IDataLakeMetadataStore"; + +const BEFORE_CLOSE_MESSAGE = `Azurite DataLake service is closing...`; +const BEFORE_CLOSE_MESSAGE_GC_ERROR = `Azurite DataLake service is closing... Critical error happens during GC.`; +const AFTER_CLOSE_MESSAGE = `Azurite DataLake service successfully closed`; + +/** + * Default implementation of Azurite DataLake HTTP server. + * This implementation provides a HTTP service based on express framework and LokiJS in memory database. + * + * We can create other DataLake servers by extending abstract Server class and initialize different httpServer, + * dataStore or requestListenerFactory fields. + * + * For example, creating a HTTPS server to accept HTTPS requests, or using other + * Node.js HTTP frameworks like Koa, or just using another SQL database. + * + * @export + * @class Server + */ +export default class DataLakeServer extends ServerBase implements ICleaner { + private readonly metadataStore: IDataLakeMetaDataStore; + private readonly extentMetadataStore: IExtentMetadataStore; + private readonly extentStore: IExtentStore; + private readonly accountDataStore: IAccountDataStore; + private readonly gcManager: IGCManager; + + /** + * Creates an instance of Server. + * + * @param {DataLakeConfiguration} configuration + * @memberof Server + */ + constructor(configuration?: DataLakeConfiguration) { + if (configuration === undefined) { + configuration = new DataLakeConfiguration(); + } + + const host = configuration.host; + const port = configuration.port; + + // We can create a HTTP server or a HTTPS server here + let httpServer; + const certOption = configuration.hasCert(); + switch (certOption) { + case CertOptions.PEM: + case CertOptions.PFX: + httpServer = https.createServer(configuration.getCert(certOption)!); + break; + default: + httpServer = http.createServer(); + } + + // We can change the persistency layer implementation by + // creating a new XXXDataStore class implementing IBlobMetadataStore interface + // and replace the default LokiBlobMetadataStore + const metadataStore: IDataLakeMetaDataStore = new LokiDataLakeMetadataStore( + configuration.metadataDBPath + // logger + ); + + const extentMetadataStore: IExtentMetadataStore = + new LokiExtentMetadataStore(configuration.extentDBPath); + + const extentStore: IExtentStore = new FSExtentStore( + extentMetadataStore, + configuration.persistencePathArray, + logger + ); + + const accountDataStore: IAccountDataStore = new AccountDataStore(logger); + + // We can also change the HTTP framework here by + // creating a new XXXListenerFactory implementing IRequestListenerFactory interface + // and replace the default Express based request listener + const requestListenerFactory: IRequestListenerFactory = + new DataLakeRequestListenerFactory( + metadataStore, + extentStore, + accountDataStore, + configuration.enableAccessLog, // Access log includes every handled HTTP request + configuration.accessLogWriteStream, + configuration.loose, + configuration.skipApiVersionCheck, + configuration.getOAuthLevel(), + configuration.disableProductStyleUrl + ); + + super(host, port, httpServer, requestListenerFactory, configuration); + + // Default Blob GC Manager + // Will close service when any critical GC error happens + const gcManager = new BlobGCManager( + metadataStore, + extentMetadataStore, + extentStore, + (err) => { + // tslint:disable-next-line:no-console + console.log(BEFORE_CLOSE_MESSAGE_GC_ERROR + ": " + err); + logger.info(BEFORE_CLOSE_MESSAGE_GC_ERROR + ": " + err); + this.close().then(() => { + // tslint:disable-next-line:no-console + console.log(AFTER_CLOSE_MESSAGE); + logger.info(AFTER_CLOSE_MESSAGE); + }); + }, + logger + ); + + this.metadataStore = metadataStore; + this.extentMetadataStore = extentMetadataStore; + this.extentStore = extentStore; + this.accountDataStore = accountDataStore; + this.gcManager = gcManager; + } + + /** + * Clean up server persisted data, including Loki metadata database file, + * Loki extent database file and extent data. + * + * @returns {Promise} + * @memberof BlobServer + */ + public async clean(): Promise { + if (this.getStatus() === ServerStatus.Closed) { + if (this.extentStore !== undefined) { + await this.extentStore.clean(); + } + + if (this.extentMetadataStore !== undefined) { + await this.extentMetadataStore.clean(); + } + + if (this.metadataStore !== undefined) { + await this.metadataStore.clean(); + } + + if (this.accountDataStore !== undefined) { + await this.accountDataStore.clean(); + } + return; + } + throw Error( + `Cannot clean up DataLake server in status ${this.getStatus()}.` + ); + } + + protected async beforeStart(): Promise { + const msg = `Azurite DataLake service is starting on ${this.host}:${this.port}`; + logger.info(msg); + + if (this.accountDataStore !== undefined) { + await this.accountDataStore.init(); + } + + if (this.metadataStore !== undefined) { + await this.metadataStore.init(); + } + + if (this.metadataStore !== undefined) { + await this.extentMetadataStore.init(); + } + + if (this.extentStore !== undefined) { + await this.extentStore.init(); + } + + if (this.gcManager !== undefined) { + await this.gcManager.start(); + } + } + + protected async afterStart(): Promise { + const msg = `Azurite DataLake service successfully listens on ${this.getHttpServerAddress()}`; + logger.info(msg); + } + + protected async beforeClose(): Promise { + logger.info(BEFORE_CLOSE_MESSAGE); + } + + protected async afterClose(): Promise { + if (this.gcManager !== undefined) { + await this.gcManager.close(); + } + + if (this.extentStore !== undefined) { + await this.extentStore.close(); + } + + if (this.extentMetadataStore !== undefined) { + await this.extentMetadataStore.close(); + } + + if (this.metadataStore !== undefined) { + await this.metadataStore.close(); + } + + if (this.accountDataStore !== undefined) { + await this.accountDataStore.close(); + } + + logger.info(AFTER_CLOSE_MESSAGE); + } +} diff --git a/src/dfs/DataLakeServerFactory.ts b/src/dfs/DataLakeServerFactory.ts new file mode 100644 index 000000000..635f2c81c --- /dev/null +++ b/src/dfs/DataLakeServerFactory.ts @@ -0,0 +1,92 @@ +import { join } from "path"; +import IDataLakeEnvironment from "./IDataLakeEnvironment"; +import DataLakeServer from "./DataLakeServer"; +import SqlDataLakeServer from "./SqlDataLakeServer"; +import DataLakeEnvironment from "./DataLakeEnvironment"; +import { DEFAULT_SQL_OPTIONS } from "../common/utils/constants"; +import SqlDataLakeConfiguration from "./SqlDataLakeConfiguration"; +import DataLakeConfiguration from "./DataLakeConfiguration"; +import { + DEFAULT_DATA_LAKE_EXTENT_LOKI_DB_PATH, + DEFAULT_DATA_LAKE_LOKI_DB_PATH, + DEFAULT_DATA_LAKE_PERSISTENCE_ARRAY, + DEFAULT_DATA_LAKE_PERSISTENCE_PATH +} from "./utils/constants"; + +export class DataLakeServerFactory { + public async createServer( + blobEnvironment?: IDataLakeEnvironment + ): Promise { + // TODO: Check it's in Visual Studio Code environment or not + const isVSC = false; + + if (!isVSC) { + const env = blobEnvironment ? blobEnvironment : new DataLakeEnvironment(); + const location = await env.location(); + const debugFilePath = await env.debug(); + + if (typeof debugFilePath === "boolean") { + throw RangeError( + `Must provide a debug log file path for parameter -d or --debug` + ); + } + + DEFAULT_DATA_LAKE_PERSISTENCE_ARRAY[0].locationPath = join( + location, + DEFAULT_DATA_LAKE_PERSISTENCE_PATH + ); + + // TODO: Check we need to create blob server against SQL or Loki + const databaseConnectionString = process.env.AZURITE_DATALAKE_DB; + const isSQL = databaseConnectionString !== undefined; + + if (isSQL) { + const config = new SqlDataLakeConfiguration( + env.datalakeHost(), + env.datalakePort(), + databaseConnectionString!, + DEFAULT_SQL_OPTIONS, + DEFAULT_DATA_LAKE_PERSISTENCE_ARRAY, + !env.silent(), + undefined, + debugFilePath !== undefined, + debugFilePath, + env.loose(), + env.skipApiVersionCheck(), + env.cert(), + env.key(), + env.pwd(), + env.oauth(), + env.disableProductStyleUrl() + ); + + return new SqlDataLakeServer(config); + } else { + const config = new DataLakeConfiguration( + env.datalakeHost(), + env.datalakePort(), + join(location, DEFAULT_DATA_LAKE_LOKI_DB_PATH), + join(location, DEFAULT_DATA_LAKE_EXTENT_LOKI_DB_PATH), + DEFAULT_DATA_LAKE_PERSISTENCE_ARRAY, + !env.silent(), + undefined, + debugFilePath !== undefined, + debugFilePath, + env.loose(), + env.skipApiVersionCheck(), + env.cert(), + env.key(), + env.pwd(), + env.oauth(), + env.disableProductStyleUrl() + ); + return new DataLakeServer(config); + } + } else { + // TODO: Add BlobServer construction in VSC + // Visual Studio Code Loki + // Visual Studio Code SQL + throw new Error("Not implemented."); + } + } +} diff --git a/src/dfs/IDataLakeEnvironment.ts b/src/dfs/IDataLakeEnvironment.ts new file mode 100644 index 000000000..38dff81b7 --- /dev/null +++ b/src/dfs/IDataLakeEnvironment.ts @@ -0,0 +1,14 @@ +export default interface IBlobEnvironment { + datalakeHost(): string | undefined; + datalakePort(): number | undefined; + location(): Promise; + silent(): boolean; + loose(): boolean; + skipApiVersionCheck(): boolean; + cert(): string | undefined; + key(): string | undefined; + pwd(): string | undefined; + debug(): Promise; + oauth(): string | undefined; + disableProductStyleUrl(): boolean; +} diff --git a/src/dfs/SqlDataLakeConfiguration.ts b/src/dfs/SqlDataLakeConfiguration.ts new file mode 100644 index 000000000..a2741dea3 --- /dev/null +++ b/src/dfs/SqlDataLakeConfiguration.ts @@ -0,0 +1,60 @@ +import { Options as SequelizeOptions } from "sequelize"; + +import ConfigurationBase from "../common/ConfigurationBase"; +import { StoreDestinationArray } from "../common/persistence/IExtentStore"; +import { DEFAULT_SQL_OPTIONS } from "../common/utils/constants"; +import { + DEFAULT_DATA_LAKE_PERSISTENCE_ARRAY, + DEFAULT_ENABLE_ACCESS_LOG, + DEFAULT_ENABLE_DEBUG_LOG +} from "./utils/constants"; +import { + DEFAULT_DATA_LAKE_LISTENING_PORT, + DEFAULT_DATA_LAKE_SERVER_HOST_NAME +} from "./utils/constants"; + +/** + * The configuration for the server based on sql database. + * + * @export + * @class SqlDataLakeConfiguration + * @extends {ConfigurationBase} + */ +export default class SqlDataLakeConfiguration extends ConfigurationBase { + public constructor( + host: string = DEFAULT_DATA_LAKE_SERVER_HOST_NAME, + port: number = DEFAULT_DATA_LAKE_LISTENING_PORT, + public readonly sqlURL: string, + public readonly sequelizeOptions: SequelizeOptions = DEFAULT_SQL_OPTIONS, + public readonly persistenceArray: StoreDestinationArray = DEFAULT_DATA_LAKE_PERSISTENCE_ARRAY, + enableAccessLog: boolean = DEFAULT_ENABLE_ACCESS_LOG, + accessLogWriteStream?: NodeJS.WritableStream, + enableDebugLog: boolean = DEFAULT_ENABLE_DEBUG_LOG, + debugLogFilePath?: string, + loose: boolean = false, + skipApiVersionCheck: boolean = false, + cert: string = "", + key: string = "", + pwd: string = "", + oauth?: string, + disableProductStyleUrl: boolean = false, + clearDB: boolean = false + ) { + super( + host, + port, + enableAccessLog, + accessLogWriteStream, + enableDebugLog, + debugLogFilePath, + loose, + skipApiVersionCheck, + cert, + key, + pwd, + oauth, + disableProductStyleUrl, + clearDB + ); + } +} diff --git a/src/dfs/SqlDataLakeServer.ts b/src/dfs/SqlDataLakeServer.ts new file mode 100644 index 000000000..6d5ac5988 --- /dev/null +++ b/src/dfs/SqlDataLakeServer.ts @@ -0,0 +1,221 @@ +import * as http from "http"; +import * as https from "https"; + +import SqlDataLakeConfiguration from "./SqlDataLakeConfiguration"; +import ServerBase, { ServerStatus } from "../common/ServerBase"; +import IDataLakeMetaDataStore from "./persistence/IDataLakeMetadataStore"; +import logger from "../common/Logger"; +import BlobGCManager from "./gc/BlobGCManager"; +import AccountDataStore from "../common/AccountDataStore"; +import { CertOptions } from "../common/ConfigurationBase"; +import IAccountDataStore from "../common/IAccountDataStore"; +import IGCManager from "../common/IGCManager"; +import IRequestListenerFactory from "../common/IRequestListenerFactory"; +import FSExtentStore from "../common/persistence/FSExtentStore"; +import IExtentMetadataStore from "../common/persistence/IExtentMetadataStore"; +import IExtentStore from "../common/persistence/IExtentStore"; +import SqlExtentMetadataStore from "../common/persistence/SqlExtentMetadataStore"; +import DataLakeRequestListenerFactory from "./DataLakeRequestListenerFactory"; +import IDataLakeMetadataStore from "./persistence/IDataLakeMetadataStore"; +import SqlDataLakeMetadataStore from "./persistence/SqlDataLakeMetadataStore"; + +const BEFORE_CLOSE_MESSAGE = `Azurite DataLake service is closing...`; +const BEFORE_CLOSE_MESSAGE_GC_ERROR = `Azurite DataLake service is closing... Critical error happens during GC.`; +const AFTER_CLOSE_MESSAGE = `Azurite DataLake service successfully closed`; + +/** + * Default implementation of Azurite DataLake HTTP server. + * This implementation provides a HTTP service based on express framework and LokiJS in memory database. + * + * We can create other DataLake servers by extending abstract Server class and initialize different httpServer, + * dataStore or requestListenerFactory fields. + * + * For example, creating a HTTPS server to accept HTTPS requests, or using other + * Node.js HTTP frameworks like Koa, or just using another SQL database. + * + * @export + * @class Server + */ +export default class SqlDataLakeServer extends ServerBase { + private readonly metadataStore: IDataLakeMetaDataStore; + private readonly extentMetadataStore: IExtentMetadataStore; + private readonly extentStore: IExtentStore; + private readonly accountDataStore: IAccountDataStore; + private readonly gcManager: IGCManager; + + /** + * Creates an instance of Server. + * + * @param {BlobConfiguration} configuration + * @memberof Server + */ + constructor(configuration: SqlDataLakeConfiguration) { + const host = configuration.host; + const port = configuration.port; + + // We can crate a HTTP server or a HTTPS server here + let httpServer; + const certOption = configuration.hasCert(); + switch (certOption) { + case CertOptions.PEM: + case CertOptions.PFX: + httpServer = https.createServer(configuration.getCert(certOption)!); + break; + default: + httpServer = http.createServer(); + } + + const metadataStore: IDataLakeMetadataStore = new SqlDataLakeMetadataStore( + configuration.sqlURL, + configuration.sequelizeOptions, + configuration.clearDB + ); + + const extentMetadataStore: IExtentMetadataStore = + new SqlExtentMetadataStore( + // Currently, extent metadata and blob metadata share same database + // But they can use separate databases per future requirements + configuration.sqlURL, + configuration.sequelizeOptions + ); + + const extentStore: IExtentStore = new FSExtentStore( + extentMetadataStore, + configuration.persistenceArray, + logger + ); + + const accountDataStore: IAccountDataStore = new AccountDataStore(logger); + + // We can also change the HTTP framework here by + // creating a new XXXListenerFactory implementing IRequestListenerFactory interface + // and replace the default Express based request listener + const requestListenerFactory: IRequestListenerFactory = + new DataLakeRequestListenerFactory( + metadataStore, + extentStore, + accountDataStore, + configuration.enableAccessLog, // Access log includes every handled HTTP request + configuration.accessLogWriteStream, + configuration.loose, + configuration.skipApiVersionCheck, + configuration.getOAuthLevel(), + configuration.disableProductStyleUrl + ); + + super(host, port, httpServer, requestListenerFactory, configuration); + + // Default Blob GC Manager + // Will close service when any critical GC error happens + const gcManager = new BlobGCManager( + metadataStore, + extentMetadataStore, + extentStore, + (error) => { + // tslint:disable-next-line:no-console + console.log(BEFORE_CLOSE_MESSAGE_GC_ERROR, error); + logger.info(BEFORE_CLOSE_MESSAGE_GC_ERROR + JSON.stringify(error)); + + // TODO: Bring this back when GC based on SQL implemented + this.close().then(() => { + // tslint:disable-next-line:no-console + console.log(AFTER_CLOSE_MESSAGE); + logger.info(AFTER_CLOSE_MESSAGE); + }); + }, + logger + ); + + this.metadataStore = metadataStore; + this.extentMetadataStore = extentMetadataStore; + this.extentStore = extentStore; + this.accountDataStore = accountDataStore; + this.gcManager = gcManager; + } + + /** + * Clean up server persisted extent data. + * + * @returns {Promise} + * @memberof BlobServer + */ + public async clean(): Promise { + if (this.getStatus() === ServerStatus.Closed) { + if (this.extentStore !== undefined) { + await this.extentStore.clean(); + } + + if (this.extentMetadataStore !== undefined) { + await this.extentMetadataStore.clean(); + } + + if (this.metadataStore !== undefined) { + await this.metadataStore.clean(); + } + + if (this.accountDataStore !== undefined) { + await this.accountDataStore.clean(); + } + return; + } + throw Error(`Cannot clean up blob server in status ${this.getStatus()}.`); + } + + protected async beforeStart(): Promise { + const msg = `Azurite Blob service is starting on ${this.host}:${this.port}`; + logger.info(msg); + + if (this.accountDataStore !== undefined) { + await this.accountDataStore.init(); + } + + if (this.metadataStore !== undefined) { + await this.metadataStore.init(); + } + + if (this.extentMetadataStore !== undefined) { + await this.extentMetadataStore.init(); + } + + if (this.extentStore !== undefined) { + await this.extentStore.init(); + } + + if (this.gcManager !== undefined) { + await this.gcManager.start(); + } + } + + protected async afterStart(): Promise { + const msg = `Azurite Blob service successfully listens on ${this.getHttpServerAddress()}`; + logger.info(msg); + } + + protected async beforeClose(): Promise { + logger.info(BEFORE_CLOSE_MESSAGE); + } + + protected async afterClose(): Promise { + if (this.gcManager !== undefined) { + await this.gcManager.close(); + } + + if (this.extentStore !== undefined) { + await this.extentStore.close(); + } + + if (this.extentMetadataStore !== undefined) { + await this.extentMetadataStore.close(); + } + + if (this.metadataStore !== undefined) { + await this.metadataStore.close(); + } + + if (this.accountDataStore !== undefined) { + await this.accountDataStore.close(); + } + + logger.info(AFTER_CLOSE_MESSAGE); + } +} diff --git a/src/dfs/authentication/AccountSASAuthenticator.ts b/src/dfs/authentication/AccountSASAuthenticator.ts new file mode 100644 index 000000000..735335c33 --- /dev/null +++ b/src/dfs/authentication/AccountSASAuthenticator.ts @@ -0,0 +1,386 @@ +import IAccountDataStore from "../../common/IAccountDataStore"; +import ILogger from "../../common/ILogger"; +import StorageErrorFactory from "../errors/StorageErrorFactory"; +import { BlobType } from "../generated/artifacts/models"; +import Operation from "../generated/artifacts/operation"; +import Context from "../generated/Context"; +import IRequest from "../generated/IRequest"; +import IDataLakeMetadataStore from "../persistence/IDataLakeMetadataStore"; +import { AccountSASPermission } from "../../common/authentication/AccountSASPermissions"; +import { + generateAccountSASSignature, + IAccountSASSignatureValues +} from "../../common/authentication/IAccountSASSignatureValues"; +import IAuthenticator from "./IAuthenticator"; +import OPERATION_ACCOUNT_SAS_PERMISSIONS from "./OperationAccountSASPermission"; +import StrictModelNotSupportedError from "../errors/StrictModelNotSupportedError"; +import { AUTHENTICATION_BEARERTOKEN_REQUIRED } from "../utils/constants"; + +export default class AccountSASAuthenticator implements IAuthenticator { + public constructor( + private readonly accountDataStore: IAccountDataStore, + private readonly blobMetadataStore: IDataLakeMetadataStore, + private readonly logger: ILogger + ) {} + + public async validate( + req: IRequest, + context: Context + ): Promise { + this.logger.info( + `AccountSASAuthenticator:validate() Start validation against account Shared Access Signature pattern.`, + context.contextId + ); + + this.logger.debug( + "AccountSASAuthenticator:validate() Getting account properties...", + context.contextId + ); + + // Doesn't create a BlobContext here because wants to move this class into common + const account: string = context.context.account; + const containerName: string | undefined = context.context.container; + const blobName: string | undefined = context.context.blob; + this.logger.debug( + // tslint:disable-next-line:max-line-length + `AccountSASAuthenticator:validate() Retrieved account name from context: ${account}, container: ${containerName}, blob: ${blobName}`, + context.contextId + ); + + // TODO: Make following async + const accountProperties = this.accountDataStore.getAccount(account); + if (accountProperties === undefined) { + throw StorageErrorFactory.ResourceNotFound(context); + } + this.logger.debug( + "AccountSASAuthenticator:validate() Got account properties successfully.", + context.contextId + ); + + const signature = this.decodeIfExist(req.getQuery("sig")); + this.logger.debug( + `AccountSASAuthenticator:validate() Retrieved signature from URL parameter sig: ${signature}`, + context.contextId + ); + + const values = this.getAccountSASSignatureValuesFromRequest(req); + if (values === undefined) { + this.logger.info( + `AccountSASAuthenticator:validate() Failed to get valid account SAS values from request.`, + context.contextId + ); + return false; + } + this.logger.debug( + `AccountSASAuthenticator:validate() Successfully got valid account SAS values from request. ${JSON.stringify( + values + )}`, + context.contextId + ); + + if (!context.context.loose && values.encryptionScope !== undefined) { + throw new StrictModelNotSupportedError( + "SAS Encryption Scope 'ses'", + context + ); + } + + this.logger.info( + `AccountSASAuthenticator:validate() Validate signature based account key1.`, + context.contextId + ); + const [sig1, stringToSign1] = generateAccountSASSignature( + values, + account, + accountProperties.key1 + ); + this.logger.debug( + `AccountSASAuthenticator:validate() String to sign is: ${JSON.stringify( + stringToSign1 + )}`, + context.contextId! + ); + this.logger.debug( + `AccountSASAuthenticator:validate() Calculated signature is: ${sig1}`, + context.contextId! + ); + + const sig1Pass = sig1 === signature; + this.logger.info( + `AccountSASAuthenticator:validate() Signature based on key1 validation ${ + sig1Pass ? "passed" : "failed" + }.`, + context.contextId + ); + + if (accountProperties.key2 !== undefined) { + this.logger.info( + `AccountSASAuthenticator:validate() Account key2 is not empty, validate signature based account key2.`, + context.contextId + ); + const [sig2, stringToSign2] = generateAccountSASSignature( + values, + account, + accountProperties.key2 + ); + this.logger.debug( + `AccountSASAuthenticator:validate() String to sign is: ${JSON.stringify( + stringToSign2 + )}`, + context.contextId! + ); + this.logger.debug( + `AccountSASAuthenticator:validate() Calculated signature is: ${sig2}`, + context.contextId! + ); + + const sig2Pass = sig2 !== signature; + this.logger.info( + `AccountSASAuthenticator:validate() Signature based on key2 validation ${ + sig2Pass ? "passed" : "failed" + }.`, + context.contextId + ); + + if (!sig2Pass && !sig1Pass) { + this.logger.info( + `AccountSASAuthenticator:validate() Validate signature based account key1 and key2 failed.`, + context.contextId + ); + return false; + } + } else { + if (!sig1Pass) { + return false; + } + } + + // When signature validation passes, we enforce account SAS validation + // Any validation errors will stop this request immediately + + this.logger.info( + `AccountSASAuthenticator:validate() Validate start and expiry time.`, + context.contextId + ); + if (!this.validateTime(values.expiryTime, values.startTime)) { + this.logger.info( + `AccountSASAuthenticator:validate() Validate start and expiry failed.`, + context.contextId + ); + throw StorageErrorFactory.getAuthorizationFailure(context); + } + + this.logger.info( + `AccountSASAuthenticator:validate() Validate IP range.`, + context.contextId + ); + if (!this.validateIPRange()) { + this.logger.info( + `AccountSASAuthenticator:validate() Validate IP range failed.`, + context.contextId + ); + throw StorageErrorFactory.getAuthorizationSourceIPMismatch(context); + } + + this.logger.info( + `AccountSASAuthenticator:validate() Validate request protocol.`, + context.contextId + ); + if (!this.validateProtocol(values.protocol, req.getProtocol())) { + this.logger.info( + `AccountSASAuthenticator:validate() Validate protocol failed.`, + context.contextId + ); + throw StorageErrorFactory.getAuthorizationProtocolMismatch(context); + } + + const operation = context.operation; + if (operation === undefined) { + throw new Error( + // tslint:disable-next-line:max-line-length + `AccountSASAuthenticator:validate() operation shouldn't be undefined. Please make sure DispatchMiddleware is hooked before authentication related middleware.` + ); + } else if (operation === Operation.Service_GetUserDelegationKey) { + this.logger.info( + `AccountSASAuthenticator:validate() Service_GetUserDelegationKey requires OAuth credentials" + }.`, + context.contextId + ); + throw StorageErrorFactory.getAuthenticationFailed( + context, + AUTHENTICATION_BEARERTOKEN_REQUIRED + ); + } + + const accountSASPermission = + OPERATION_ACCOUNT_SAS_PERMISSIONS.get(operation); + this.logger.debug( + `AccountSASAuthenticator:validate() Got permission requirements for operation ${ + Operation[operation] + } - ${JSON.stringify(accountSASPermission)}`, + context.contextId + ); + if (accountSASPermission === undefined) { + throw new Error( + // tslint:disable-next-line:max-line-length + `AccountSASAuthenticator:validate() OPERATION_ACCOUNT_SAS_PERMISSIONS doesn't have configuration for operation ${Operation[operation]}'s account SAS permission.` + ); + } + + if (!accountSASPermission.validateServices(values.services)) { + throw StorageErrorFactory.getAuthorizationServiceMismatch(context); + } + + if (!accountSASPermission.validateResourceTypes(values.resourceTypes)) { + throw StorageErrorFactory.getAuthorizationResourceTypeMismatch(context); + } + + if (!accountSASPermission.validatePermissions(values.permissions)) { + throw StorageErrorFactory.getAuthorizationPermissionMismatch(context); + } + + // Check special permission requirements + // If block blob exists, then permission must be Write only + // If page blob exists, then permission must be Write only + // If append blob exists, then permission must be Write only + // If copy destination blob exists, then permission must be Write only + if ( + //Blob + operation === Operation.BlockBlob_Upload || + operation === Operation.PageBlob_Create || + operation === Operation.AppendBlob_Create || + operation === Operation.Blob_StartCopyFromURL || + operation === Operation.Blob_CopyFromURL || + //DataLake + operation === Operation.Path_Create || + operation === Operation.Path_AppendData || + operation === Operation.Path_FlushData || + operation === Operation.Path_Read + ) { + this.logger.info( + `AccountSASAuthenticator:validate() For ${Operation[operation]}, if blob exists, the permission must be Write.`, + context.contextId + ); + + if ( + (await this.blobExist(account, containerName!, blobName!)) && + !values.permissions.toString().includes(AccountSASPermission.Write) + ) { + this.logger.info( + `AccountSASAuthenticator:validate() Account SAS validation failed for special requirement.`, + context.contextId + ); + throw StorageErrorFactory.getAuthorizationPermissionMismatch(context); + } + } + + this.logger.info( + `AccountSASAuthenticator:validate() Account SAS validation successfully.`, + context.contextId + ); + + return true; + } + + private getAccountSASSignatureValuesFromRequest( + req: IRequest + ): IAccountSASSignatureValues | undefined { + const version = this.decodeIfExist(req.getQuery("sv")); + const services = this.decodeIfExist(req.getQuery("ss")); + const resourceTypes = this.decodeIfExist(req.getQuery("srt")); + const protocol = this.decodeIfExist(req.getQuery("spr")); + const startTime = this.decodeIfExist(req.getQuery("st")); + const expiryTime = this.decodeIfExist(req.getQuery("se")); + const ipRange = this.decodeIfExist(req.getQuery("sip")); + const permissions = this.decodeIfExist(req.getQuery("sp")); + const signature = this.decodeIfExist(req.getQuery("sig")); + const encryptionScope = this.decodeIfExist(req.getQuery("ses")); + + if ( + version === undefined || + expiryTime === undefined || + permissions === undefined || + services === undefined || + resourceTypes === undefined || + signature === undefined + ) { + return undefined; + } + + const accountSASValues: IAccountSASSignatureValues = { + version, + protocol, + startTime, + expiryTime, + permissions, + ipRange, + services, + resourceTypes, + encryptionScope + }; + + return accountSASValues; + } + + private validateTime(expiry: Date | string, start?: Date | string): boolean { + const expiryTime = new Date(expiry); + const now = new Date(); + + if (now > expiryTime) { + return false; + } + + if (start !== undefined) { + const startTime = new Date(start); + if (now < startTime) { + return false; + } + } + + return true; + } + + private validateIPRange(): boolean { + // TODO: Emulator doesn't validate IP Address + return true; + } + + private validateProtocol( + sasProtocol: string = "https,http", + requestProtocol: string + ): boolean { + if (sasProtocol.includes(",")) { + return true; + } else { + return sasProtocol.toLowerCase() === requestProtocol; + } + } + + private decodeIfExist(value?: string): string | undefined { + return value === undefined ? value : decodeURIComponent(value); + } + + private async blobExist( + account: string, + container: string, + blob: string + ): Promise { + const blobModel = await this.blobMetadataStore.getBlobType( + account, + container, + blob + ); + if (blobModel === undefined) { + return false; + } + + if ( + blobModel.blobType === BlobType.BlockBlob && + blobModel.isCommitted === false + ) { + return false; + } + + return true; + } +} diff --git a/src/dfs/authentication/BlobSASAuthenticator.ts b/src/dfs/authentication/BlobSASAuthenticator.ts new file mode 100644 index 000000000..61c3e2d62 --- /dev/null +++ b/src/dfs/authentication/BlobSASAuthenticator.ts @@ -0,0 +1,627 @@ +import IAccountDataStore from "../../common/IAccountDataStore"; +import ILogger from "../../common/ILogger"; +import DataLakeContext from "../context/DataLakeContext"; +import StorageErrorFactory from "../errors/StorageErrorFactory"; +import StrictModelNotSupportedError from "../errors/StrictModelNotSupportedError"; +import { AccessPolicy, BlobType } from "../generated/artifacts/models"; +import Operation from "../generated/artifacts/operation"; +import Context from "../generated/Context"; +import IRequest from "../generated/IRequest"; +import IDataLakeMetadataStore from "../persistence/IDataLakeMetadataStore"; +import { AUTHENTICATION_BEARERTOKEN_REQUIRED } from "../utils/constants"; +import { getUserDelegationKeyValue } from "../utils/utils"; +import { BlobSASPermission } from "./BlobSASPermissions"; +import { BlobSASResourceType } from "./BlobSASResourceType"; +import IAuthenticator from "./IAuthenticator"; +import { + generateBlobSASSignature, + generateBlobSASSignatureWithUDK, + IBlobSASSignatureValues +} from "./IBlobSASSignatureValues"; +import { + OPERATION_BLOB_SAS_BLOB_PERMISSIONS, + OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS +} from "./OperationBlobSASPermission"; + +export default class BlobSASAuthenticator implements IAuthenticator { + public constructor( + private readonly accountDataStore: IAccountDataStore, + private readonly blobMetadataStore: IDataLakeMetadataStore, + private readonly logger: ILogger + ) {} + + public async validate( + req: IRequest, + context: Context + ): Promise { + this.logger.info( + `BlobSASAuthenticator:validate() Start validation against blob service Shared Access Signature pattern.`, + context.contextId + ); + + this.logger.debug( + "BlobSASAuthenticator:validate() Getting account properties...", + context.contextId + ); + + const blobContext = new DataLakeContext(context); + const account = blobContext.account; + if (account === undefined) { + throw RangeError( + `BlobSASAuthenticator:validate() account is undefined in context.` + ); + } + + const containerName = blobContext.container; + if (containerName === undefined) { + this.logger.error( + `BlobSASAuthenticator:validate() container name is undefined in context.`, + context.contextId + ); + return undefined; + } + + const blobName = blobContext.blob; + this.logger.debug( + // tslint:disable-next-line:max-line-length + `BlobSASAuthenticator:validate() Retrieved account name from context: ${account}, container: ${containerName}, blob: ${blobName}`, + context.contextId + ); + + // TODO: Make following async + const accountProperties = this.accountDataStore.getAccount(account); + if (accountProperties === undefined) { + throw StorageErrorFactory.ResourceNotFound(blobContext); + } + this.logger.debug( + "BlobSASAuthenticator:validate() Got account properties successfully.", + context.contextId + ); + + // Extract blob service SAS authentication required parameters + const signature = this.decodeIfExist(req.getQuery("sig")); + this.logger.debug( + `BlobSASAuthenticator:validate() Retrieved signature from URL parameter sig: ${signature}`, + context.contextId + ); + if (signature === undefined) { + this.logger.debug( + `BlobSASAuthenticator:validate() No signature found in request. Skip blob service SAS validation.`, + context.contextId + ); + return undefined; + } + + const resource = this.decodeIfExist(req.getQuery("sr")); + if ( + resource !== BlobSASResourceType.Container && + resource !== BlobSASResourceType.Blob && + resource !== BlobSASResourceType.BlobSnapshot + ) { + this.logger.debug( + // tslint:disable-next-line:max-line-length + `BlobSASAuthenticator:validate() Signed resource type ${resource} is invalid. Skip blob service SAS validation.`, + context.contextId + ); + return undefined; + } + this.logger.debug( + `BlobSASAuthenticator:validate() Signed resource type is ${resource}.`, + context.contextId + ); + + const values = this.getBlobSASSignatureValuesFromRequest( + req, + containerName, + blobName, + context + ); + if (values === undefined) { + this.logger.info( + // tslint:disable-next-line:max-line-length + `BlobSASAuthenticator:validate() Failed to get valid blob service SAS values from request. Skip blob service SAS validation.`, + context.contextId + ); + return undefined; + } + + this.logger.debug( + `BlobSASAuthenticator:validate() Successfully got valid blob service SAS values from request. ${JSON.stringify( + values + )}`, + context.contextId + ); + + if (!context.context.loose && values.encryptionScope !== undefined) { + throw new StrictModelNotSupportedError( + "SAS Encryption Scope 'ses'", + context + ); + } + + if ( + values.signedObjectId || + values.signedTenantId || + values.signedService || + values.signedVersion || + values.signedStartsOn || + values.signedExpiresOn + ) { + this.logger.info( + `BlobSASAuthenticator:validate() Validate signature based on user delegation key.`, + context.contextId + ); + + if ( + !values.signedObjectId || + !values.signedTenantId || + !values.signedStartsOn || + !values.signedExpiresOn || + !values.signedService || + !values.signedVersion || + values.signedService !== "b" + ) { + this.logger.info( + `BlobSASAuthenticator:validate() Signature based on user delegation key validation failed" + }.`, + context.contextId + ); + throw StorageErrorFactory.getAuthorizationFailure(context); + } + + const savedPolicy = this.decodeIfExist(req.getQuery("si")); + if (savedPolicy) { + this.logger.info( + `BlobSASAuthenticator:validate() Access policy used in UDK SAS.`, + context.contextId + ); + throw StorageErrorFactory.getAuthorizationFailure(context); + } + + this.logger.info( + `BlobSASAuthenticator:validate() Validate UDK start and expiry time.`, + context.contextId + ); + + if (!this.validateTime(values.signedExpiresOn!, values.signedStartsOn!)) { + this.logger.info( + `BlobSASAuthenticator:validate() Validate UDK start and expiry failed.`, + context.contextId + ); + throw StorageErrorFactory.getAuthorizationFailure(context); + } + + const keyValue = getUserDelegationKeyValue( + values.signedObjectId!, + values.signedTenantId!, + values.signedStartsOn!, + values.signedExpiresOn!, + values.signedVersion! + ); + const [sig, stringToSign] = generateBlobSASSignatureWithUDK( + values, + resource, + account, + Buffer.from(keyValue, "base64") + ); + + this.logger.debug( + `BlobSASAuthenticator:validate() String to sign is: ${JSON.stringify( + stringToSign + )}`, + context.contextId! + ); + this.logger.debug( + `BlobSASAuthenticator:validate() Calculated signature is: ${sig}`, + context.contextId! + ); + + const sigPass = sig === signature; + this.logger.info( + `BlobSASAuthenticator:validate() Signature based on UDK ${ + sigPass ? "passed" : "failed" + }.`, + context.contextId + ); + + if (!sigPass) { + return sigPass; + } + } else { + this.logger.info( + `BlobSASAuthenticator:validate() Validate signature based account key1.`, + context.contextId + ); + const [sig1, stringToSign1] = generateBlobSASSignature( + values, + resource, + account, + accountProperties.key1 + ); + this.logger.debug( + `BlobSASAuthenticator:validate() String to sign is: ${JSON.stringify( + stringToSign1 + )}`, + context.contextId! + ); + this.logger.debug( + `BlobSASAuthenticator:validate() Calculated signature is: ${sig1}`, + context.contextId! + ); + + const sig1Pass = sig1 === signature; + this.logger.info( + `BlobSASAuthenticator:validate() Signature based on key1 validation ${ + sig1Pass ? "passed" : "failed" + }.`, + context.contextId + ); + + if (accountProperties.key2 !== undefined) { + this.logger.info( + `BlobSASAuthenticator:validate() Account key2 is not empty, validate signature based account key2.`, + context.contextId + ); + const [sig2, stringToSign2] = generateBlobSASSignature( + values, + resource, + account, + accountProperties.key2 + ); + this.logger.debug( + `BlobSASAuthenticator:validate() String to sign is: ${JSON.stringify( + stringToSign2 + )}`, + context.contextId! + ); + this.logger.debug( + `BlobSASAuthenticator:validate() Calculated signature is: ${sig2}`, + context.contextId! + ); + + const sig2Pass = sig2 !== signature; + this.logger.info( + `BlobSASAuthenticator:validate() Signature based on key2 validation ${ + sig2Pass ? "passed" : "failed" + }.`, + context.contextId + ); + + if (!sig2Pass && !sig1Pass) { + this.logger.info( + `BlobSASAuthenticator:validate() Validate signature based account key1 and key2 failed.`, + context.contextId + ); + return false; + } + } else { + if (!sig1Pass) { + return false; + } + } + } + + // When signature validation passes, we enforce blob service SAS validation + // Any validation errors will stop this request immediately + + // TODO: Validate permissions from ACL identifier by extract permissions, start time and expiry time from ACL + if (values.identifier !== undefined) { + const accessPolicy: AccessPolicy | undefined = + await this.getContainerAccessPolicyByIdentifier( + account, + containerName, + values.identifier, + context + ); + if (accessPolicy === undefined) { + this.logger.warn( + `BlobSASAuthenticator:validate() Cannot get access policy defined for container ${containerName} with id ${values.identifier}.`, + context.contextId + ); + throw StorageErrorFactory.getAuthorizationFailure(context); + } + + values.startTime = accessPolicy.start; + values.expiryTime = accessPolicy.expiry; + values.permissions = accessPolicy.permission; + } + + this.logger.info( + `BlobSASAuthenticator:validate() Validate start and expiry time.`, + context.contextId + ); + if (!this.validateTime(values.expiryTime, values.startTime)) { + this.logger.info( + `BlobSASAuthenticator:validate() Validate start and expiry failed.`, + context.contextId + ); + throw StorageErrorFactory.getAuthorizationFailure(context); + } + + this.logger.info( + `BlobSASAuthenticator:validate() Validate IP range.`, + context.contextId + ); + if (!this.validateIPRange()) { + this.logger.info( + `BlobSASAuthenticator:validate() Validate IP range failed.`, + context.contextId + ); + throw StorageErrorFactory.getAuthorizationSourceIPMismatch(context); + } + + this.logger.info( + `BlobSASAuthenticator:validate() Validate request protocol.`, + context.contextId + ); + if (!this.validateProtocol(values.protocol, req.getProtocol())) { + this.logger.info( + `BlobSASAuthenticator:validate() Validate protocol failed.`, + context.contextId + ); + throw StorageErrorFactory.getAuthorizationProtocolMismatch(context); + } + + const operation = context.operation; + if (operation === undefined) { + throw new Error( + // tslint:disable-next-line:max-line-length + `BlobSASAuthenticator:validate() Operation shouldn't be undefined. Please make sure DispatchMiddleware is hooked before authentication related middleware.` + ); + } else if (operation === Operation.Service_GetUserDelegationKey) { + this.logger.info( + `BlobSASAuthenticator:validate() Service_GetUserDelegationKey requires OAuth credentials" + }.`, + context.contextId + ); + throw StorageErrorFactory.getAuthenticationFailed( + context, + AUTHENTICATION_BEARERTOKEN_REQUIRED + ); + } + + const blobSASPermission = + resource === BlobSASResourceType.Blob + ? OPERATION_BLOB_SAS_BLOB_PERMISSIONS.get(operation) + : OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.get(operation); + + this.logger.debug( + `BlobSASAuthenticator:validate() Got permission requirements for operation ${ + Operation[operation] + } - ${JSON.stringify(blobSASPermission)}`, + context.contextId + ); + if (blobSASPermission === undefined) { + throw new Error( + // tslint:disable-next-line:max-line-length + `BlobSASAuthenticator:validate() ${ + resource === BlobSASResourceType.Blob + ? "OPERATION_BLOB_SAS_BLOB_PERMISSIONS" + : "OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS" + } doesn't have configuration for operation ${ + Operation[operation] + }'s blob service SAS permission.` + ); + } + + if (!blobSASPermission.validatePermissions(values.permissions!)) { + throw StorageErrorFactory.getAuthorizationPermissionMismatch(context); + } + + // Check 3 special permission requirements + // If block blob exists, then permission must be Write only + // If page blob exists, then permission must be Write only + // If append blob exists, then permission must be Write only + // If copy destination blob exists, then permission must be Write only + if ( + //blob + operation === Operation.BlockBlob_Upload || + operation === Operation.PageBlob_Create || + operation === Operation.AppendBlob_Create || + operation === Operation.Blob_StartCopyFromURL || + operation === Operation.Blob_CopyFromURL || + //DataLake + operation === Operation.Path_Create || + operation === Operation.Path_AppendData || + operation === Operation.Path_FlushData + ) { + this.logger.info( + `BlobSASAuthenticator:validate() For ${Operation[operation]}, if blob exists, the permission must be Write.`, + context.contextId + ); + + if ( + (await this.blobExist(account, containerName!, blobName!)) && + !values.permissions!.toString().includes(BlobSASPermission.Write) + ) { + this.logger.info( + `BlobSASAuthenticator:validate() Account SAS validation failed for special requirement.`, + context.contextId + ); + throw StorageErrorFactory.getAuthorizationPermissionMismatch(context); + } + } + + this.logger.info( + `BlobSASAuthenticator:validate() Blob service SAS validation successfully.`, + context.contextId + ); + + // TODO: Handle enforced response headers defined in blob service SAS + + return true; + } + + private getBlobSASSignatureValuesFromRequest( + req: IRequest, + containerName: string, + blobName?: string, + context?: Context + ): IBlobSASSignatureValues | undefined { + const version = this.decodeIfExist(req.getQuery("sv")); + const protocol = this.decodeIfExist(req.getQuery("spr")); + const startTime = this.decodeIfExist(req.getQuery("st")); + const expiryTime = this.decodeIfExist(req.getQuery("se")); + const permissions = this.decodeIfExist(req.getQuery("sp")); + const ipRange = this.decodeIfExist(req.getQuery("sip")); + const identifier = this.decodeIfExist(req.getQuery("si")); + const cacheControl = req.getQuery("rscc"); + const contentDisposition = req.getQuery("rscd"); + const contentEncoding = req.getQuery("rsce"); + const contentLanguage = req.getQuery("rscl"); + const contentType = req.getQuery("rsct"); + const signedResource = this.decodeIfExist(req.getQuery("sr")); + const snapshot = this.decodeIfExist(req.getQuery("snapshot")); + const encryptionScope = this.decodeIfExist(req.getQuery("ses")); + const signedObjectId = this.decodeIfExist(req.getQuery("skoid")); + const signedTenantId = this.decodeIfExist(req.getQuery("sktid")); + const signedStartsOn = this.decodeIfExist(req.getQuery("skt")); + const signedExpiresOn = this.decodeIfExist(req.getQuery("ske")); + const signedVersion = this.decodeIfExist(req.getQuery("skv")); + const signedService = this.decodeIfExist(req.getQuery("sks")); + + if (!identifier && (!permissions || !expiryTime)) { + this.logger.warn( + // tslint:disable-next-line:max-line-length + `BlobSASAuthenticator:generateBlobSASSignature(): Must provide 'permissions' and 'expiryTime' for Blob SAS generation when 'identifier' is not provided.`, + context ? context.contextId : undefined + ); + return undefined; + } + + if (version === undefined) { + this.logger.warn( + // tslint:disable-next-line:max-line-length + `BlobSASAuthenticator:generateBlobSASSignature(): Must provide 'version'.`, + context ? context.contextId : undefined + ); + return undefined; + } + + const blobSASValues: IBlobSASSignatureValues = { + version, + protocol, + startTime, + expiryTime, + permissions, + ipRange, + containerName, + blobName, + identifier, + encryptionScope, + cacheControl, + contentDisposition, + contentEncoding, + contentLanguage, + contentType, + signedResource, + snapshot, + signedObjectId, + signedTenantId, + signedService, + signedVersion, + signedStartsOn, + signedExpiresOn + }; + + return blobSASValues; + } + + private validateTime(expiry?: Date | string, start?: Date | string): boolean { + if (expiry === undefined && start === undefined) { + return true; + } + + const now = new Date(); + + if (expiry !== undefined) { + const expiryTime = new Date(expiry); + if (now > expiryTime) { + return false; + } + } + + if (start !== undefined) { + const startTime = new Date(start); + if (now < startTime) { + return false; + } + } + + return true; + } + + private validateIPRange(): boolean { + // TODO: Emulator doesn't validate IP Address + return true; + } + + private validateProtocol( + sasProtocol: string = "https,http", + requestProtocol: string + ): boolean { + if (sasProtocol.includes(",")) { + return true; + } else { + return sasProtocol.toLowerCase() === requestProtocol; + } + } + + private decodeIfExist(value?: string): string | undefined { + return value === undefined ? value : decodeURIComponent(value); + } + + private async getContainerAccessPolicyByIdentifier( + account: string, + container: string, + id: string, + context: Context + ): Promise { + try { + const containerModel = await this.blobMetadataStore.getContainerACL( + context, + account, + container + ); + if (containerModel === undefined) { + return undefined; + } + + if (containerModel.containerAcl === undefined) { + return undefined; + } + + for (const acl of containerModel.containerAcl) { + if (acl.id === id) { + return acl.accessPolicy; + } + } + } catch (err) { + return undefined; + } + } + + private async blobExist( + account: string, + container: string, + blob: string + ): Promise { + const blobModel = await this.blobMetadataStore.getBlobType( + account, + container, + blob + ); + if (blobModel === undefined) { + return false; + } + + if ( + blobModel.blobType === BlobType.BlockBlob && + blobModel.isCommitted === false + ) { + return false; + } + + return true; + } +} diff --git a/src/dfs/authentication/BlobSASPermissions.ts b/src/dfs/authentication/BlobSASPermissions.ts new file mode 100644 index 000000000..cd6f68e58 --- /dev/null +++ b/src/dfs/authentication/BlobSASPermissions.ts @@ -0,0 +1,13 @@ +export enum BlobSASPermission { + Read = "r", + Add = "a", + Create = "c", + Write = "w", + Delete = "d", + DeleteVersion = "x", + Tag = "t", + Move = "m", + execute = "e", + SetImmutabilityPolicy = "i", + permanentDelete = "y" +} diff --git a/src/dfs/authentication/BlobSASResourceType.ts b/src/dfs/authentication/BlobSASResourceType.ts new file mode 100644 index 000000000..1fb709d1e --- /dev/null +++ b/src/dfs/authentication/BlobSASResourceType.ts @@ -0,0 +1,5 @@ +export enum BlobSASResourceType { + Container = "c", + Blob = "b", + BlobSnapshot = "bs" +} diff --git a/src/dfs/authentication/BlobSharedKeyAuthenticator.ts b/src/dfs/authentication/BlobSharedKeyAuthenticator.ts new file mode 100644 index 000000000..934f75043 --- /dev/null +++ b/src/dfs/authentication/BlobSharedKeyAuthenticator.ts @@ -0,0 +1,353 @@ +import IAccountDataStore from "../../common/IAccountDataStore"; +import ILogger from "../../common/ILogger"; +import { computeHMACSHA256, getURLQueries } from "../../common/utils/utils"; +import DataLakeContext from "../context/DataLakeContext"; +import StorageErrorFactory from "../errors/StorageErrorFactory"; +import Operation from "../generated/artifacts/operation"; +import Context from "../generated/Context"; +import IRequest from "../generated/IRequest"; +import { + AUTHENTICATION_BEARERTOKEN_REQUIRED, + HeaderConstants +} from "../utils/constants"; +import IAuthenticator from "./IAuthenticator"; + +export default class BlobSharedKeyAuthenticator implements IAuthenticator { + public constructor( + private readonly dataStore: IAccountDataStore, + private readonly logger: ILogger + ) {} + + public async validate( + req: IRequest, + context: Context + ): Promise { + const blobContext = new DataLakeContext(context); + const account = blobContext.account!; + + this.logger.info( + `BlobSharedKeyAuthenticator:validate() Start validation against account shared key authentication.`, + blobContext.contextId + ); + + const authHeaderValue = req.getHeader(HeaderConstants.AUTHORIZATION); + if (authHeaderValue === undefined) { + this.logger.info( + // tslint:disable-next-line:max-line-length + `BlobSharedKeyAuthenticator:validate() Request doesn't include valid authentication header. Skip shared key authentication.`, + blobContext.contextId + ); + return undefined; + } else if (!authHeaderValue.startsWith("SharedKey")) { + this.logger.info( + // tslint:disable-next-line:max-line-length + `BlobSharedKeyAuthenticator:validate() Request doesn't include shared key authentication.`, + blobContext.contextId + ); + return undefined; + } + + // TODO: Make following async + const accountProperties = this.dataStore.getAccount(account); + if (accountProperties === undefined) { + this.logger.error( + `BlobSharedKeyAuthenticator:validate() Invalid storage account ${account}.`, + blobContext.contextId + ); + throw StorageErrorFactory.ResourceNotFound(blobContext); + } + + const operation = context.operation; + if (operation === undefined) { + throw new Error( + // tslint:disable-next-line:max-line-length + `BlobSharedKeyAuthenticator:validate() Operation shouldn't be undefined. Please make sure DispatchMiddleware is hooked before authentication related middleware.` + ); + } else if (operation === Operation.Service_GetUserDelegationKey) { + this.logger.info( + `BlobSharedKeyAuthenticator:validate() Service_GetUserDelegationKey requires OAuth credentials" + }.`, + context.contextId + ); + throw StorageErrorFactory.getAuthenticationFailed( + context, + AUTHENTICATION_BEARERTOKEN_REQUIRED + ); + } + + const stringToSign: string = + [ + req.getMethod().toUpperCase(), + this.getHeaderValueToSign(req, HeaderConstants.CONTENT_ENCODING), + this.getHeaderValueToSign(req, HeaderConstants.CONTENT_LANGUAGE), + this.getHeaderValueToSign(req, HeaderConstants.CONTENT_LENGTH), + this.getHeaderValueToSign(req, HeaderConstants.CONTENT_MD5), + this.getHeaderValueToSign(req, HeaderConstants.CONTENT_TYPE), + this.getHeaderValueToSign(req, HeaderConstants.DATE), + this.getHeaderValueToSign(req, HeaderConstants.IF_MODIFIED_SINCE), + this.getHeaderValueToSign(req, HeaderConstants.IF_MATCH), + this.getHeaderValueToSign(req, HeaderConstants.IF_NONE_MATCH), + this.getHeaderValueToSign(req, HeaderConstants.IF_UNMODIFIED_SINCE), + this.getHeaderValueToSign(req, HeaderConstants.RANGE) + ].join("\n") + + "\n" + + this.getCanonicalizedHeadersString(req) + + this.getCanonicalizedResourceString( + req, + account, + blobContext.authenticationPath + ); + + this.logger.info( + `BlobSharedKeyAuthenticator:validate() [STRING TO SIGN]:${JSON.stringify( + stringToSign + )}`, + blobContext.contextId + ); + + const signature1 = computeHMACSHA256(stringToSign, accountProperties.key1); + const authValue1 = `SharedKey ${account}:${signature1}`; + this.logger.info( + `BlobSharedKeyAuthenticator:validate() Calculated authentication header based on key1: ${authValue1}`, + blobContext.contextId + ); + if (authHeaderValue === authValue1) { + this.logger.info( + `BlobSharedKeyAuthenticator:validate() Signature 1 matched.`, + blobContext.contextId + ); + return true; + } + + if (accountProperties.key2) { + const signature2 = computeHMACSHA256( + stringToSign, + accountProperties.key2 + ); + const authValue2 = `SharedKey ${account}:${signature2}`; + this.logger.info( + `BlobSharedKeyAuthenticator:validate() Calculated authentication header based on key2: ${authValue2}`, + blobContext.contextId + ); + if (authHeaderValue === authValue2) { + this.logger.info( + `BlobSharedKeyAuthenticator:validate() Signature 2 matched.`, + blobContext.contextId + ); + return true; + } + } + + if ( + context.context.isSecondary && + blobContext.authenticationPath?.indexOf(account) === 1 + ) { + // JS/.net Track2 SDK will generate stringToSign from IP style Uri with "-secondary" in authenticationPath, so will also compare signature with this kind stringToSign + const stringToSign_secondary: string = + [ + req.getMethod().toUpperCase(), + this.getHeaderValueToSign(req, HeaderConstants.CONTENT_ENCODING), + this.getHeaderValueToSign(req, HeaderConstants.CONTENT_LANGUAGE), + this.getHeaderValueToSign(req, HeaderConstants.CONTENT_LENGTH), + this.getHeaderValueToSign(req, HeaderConstants.CONTENT_MD5), + this.getHeaderValueToSign(req, HeaderConstants.CONTENT_TYPE), + this.getHeaderValueToSign(req, HeaderConstants.DATE), + this.getHeaderValueToSign(req, HeaderConstants.IF_MODIFIED_SINCE), + this.getHeaderValueToSign(req, HeaderConstants.IF_MATCH), + this.getHeaderValueToSign(req, HeaderConstants.IF_NONE_MATCH), + this.getHeaderValueToSign(req, HeaderConstants.IF_UNMODIFIED_SINCE), + this.getHeaderValueToSign(req, HeaderConstants.RANGE) + ].join("\n") + + "\n" + + this.getCanonicalizedHeadersString(req) + + this.getCanonicalizedResourceString( + req, + account, + // The authenticationPath looks like "/devstoreaccount1/container", add "-secondary" after account name to "/devstoreaccount1-secondary/container" + blobContext.authenticationPath?.replace( + account, + account + "-secondary" + ) + ); + + this.logger.info( + `BlobSharedKeyAuthenticator:validate() [STRING TO SIGN_secondary]:${JSON.stringify( + stringToSign_secondary + )}`, + blobContext.contextId + ); + + const signature1_secondary = computeHMACSHA256( + stringToSign_secondary, + accountProperties.key1 + ); + const authValue1_secondary = `SharedKey ${account}:${signature1_secondary}`; + this.logger.info( + `BlobSharedKeyAuthenticator:validate() Calculated authentication header based on key1 and stringToSign with "-secondary": ${authValue1_secondary}`, + blobContext.contextId + ); + + if (authHeaderValue === authValue1_secondary) { + this.logger.info( + `BlobSharedKeyAuthenticator:validate() Signature 1_secondary matched.`, + blobContext.contextId + ); + return true; + } + + if (accountProperties.key2) { + const signature2_secondary = computeHMACSHA256( + stringToSign_secondary, + accountProperties.key2 + ); + const authValue2_secondary = `SharedKey ${account}:${signature2_secondary}`; + this.logger.info( + `BlobSharedKeyAuthenticator:validate() Calculated authentication header based on key2: ${authValue2_secondary}`, + blobContext.contextId + ); + if (authHeaderValue === authValue2_secondary) { + this.logger.info( + `BlobSharedKeyAuthenticator:validate() Signature 2_secondary matched.`, + blobContext.contextId + ); + return true; + } + } + } + + // this.logger.info(`[URL]:${req.getUrl()}`); + // this.logger.info(`[HEADERS]:${req.getHeaders().toString()}`); + // this.logger.info(`[KEY]: ${request.headers.get(HeaderConstants.AUTHORIZATION)}`); + + this.logger.info( + `BlobSharedKeyAuthenticator:validate() Validation failed.`, + blobContext.contextId + ); + return false; + } + + /** + * Retrieve header value according to shared key sign rules. + * @see https://docs.microsoft.com/en-us/rest/api/storageservices/authenticate-with-shared-key + * + * @private + * @param {WebResource} request + * @param {string} headerName + * @returns {string} + * @memberof SharedKeyCredentialPolicy + */ + private getHeaderValueToSign(request: IRequest, headerName: string): string { + const value = request.getHeader(headerName); + + if (!value) { + return ""; + } + + // When using version 2015-02-21 or later, if Content-Length is zero, then + // set the Content-Length part of the StringToSign to an empty string. + // https://docs.microsoft.com/en-us/rest/api/storageservices/authenticate-with-shared-key + if (headerName === HeaderConstants.CONTENT_LENGTH && value === "0") { + return ""; + } + + return value; + } + + /** + * To construct the CanonicalizedHeaders portion of the signature string, follow these steps: + * 1. Retrieve all headers for the resource that begin with x-ms-, including the x-ms-date header. + * 2. Convert each HTTP header name to lowercase. + * 3. Sort the headers lexicographically by header name, in ascending order. + * Each header may appear only once in the string. + * 4. Replace any linear whitespace in the header value with a single space. + * 5. Trim any whitespace around the colon in the header. + * 6. Finally, append a new-line character to each canonicalized header in the resulting list. + * Construct the CanonicalizedHeaders string by concatenating all headers in this list into a single string. + * + * @private + * @param {IRequest} request + * @returns {string} + * @memberof SharedKeyCredentialPolicy + */ + private getCanonicalizedHeadersString(request: IRequest): string { + const headers: { value: string; name: string }[] = []; + const headersObject = request.getHeaders(); + for (const name in headersObject) { + if (headersObject.hasOwnProperty(name)) { + const value = headersObject[name]; + if (value === undefined) { + headers.push({ name, value: "" }); + } else if (typeof value === "string") { + headers.push({ name, value }); + } else { + headers.push({ name, value: value.join(",") }); + } + } + } + + const headersArray = headers.filter((value) => { + return value.name + .toLowerCase() + .startsWith(HeaderConstants.PREFIX_FOR_STORAGE); + }); + + headersArray.sort((a, b): number => { + return a.name.toLowerCase().localeCompare(b.name.toLowerCase()); + }); + + let canonicalizedHeadersStringToSign: string = ""; + headersArray.forEach((header) => { + canonicalizedHeadersStringToSign += `${header.name + .toLowerCase() + .trimRight()}:${header.value.trimLeft()}\n`; + }); + + return canonicalizedHeadersStringToSign; + } + + /** + * Retrieves canonicalized resource string. + * + * @private + * @param {IRequest} request + * @returns {string} + * @memberof SharedKeyCredentialPolicy + */ + private getCanonicalizedResourceString( + request: IRequest, + account: string, + authenticationPath?: string + ): string { + let path = request.getPath() || "/"; + + // For secondary account, we use account name (without "-secondary") for the path + if (authenticationPath !== undefined) { + path = authenticationPath; + } + + let canonicalizedResourceString: string = ""; + canonicalizedResourceString += `/${account}${path}`; + + const queries = getURLQueries(request.getUrl()); + const lowercaseQueries: { [key: string]: string } = {}; + if (queries) { + const queryKeys: string[] = []; + for (const key in queries) { + if (queries.hasOwnProperty(key)) { + const lowercaseKey = key.toLowerCase(); + lowercaseQueries[lowercaseKey] = queries[key]; + queryKeys.push(lowercaseKey); + } + } + + queryKeys.sort(); + for (const key of queryKeys) { + canonicalizedResourceString += `\n${key}:${decodeURIComponent( + lowercaseQueries[key].replace(/\+/g, "%20") + )}`; + } + } + + return canonicalizedResourceString; + } +} diff --git a/src/dfs/authentication/BlobTokenAuthenticator.ts b/src/dfs/authentication/BlobTokenAuthenticator.ts new file mode 100644 index 000000000..ed73b311f --- /dev/null +++ b/src/dfs/authentication/BlobTokenAuthenticator.ts @@ -0,0 +1,255 @@ +import { decode } from "jsonwebtoken"; +import IAccountDataStore from "../../common/IAccountDataStore"; +import ILogger from "../../common/ILogger"; +import { OAuthLevel } from "../../common/models"; +import { + BEARER_TOKEN_PREFIX, + HTTPS, + VALID_ISSUE_PREFIXES +} from "../../common/utils/constants"; +import DataLakeContext from "../context/DataLakeContext"; +import StorageErrorFactory from "../errors/StorageErrorFactory"; +import Operation from "../generated/artifacts/operation"; +import Context from "../generated/Context"; +import IRequest from "../generated/IRequest"; +import { HeaderConstants, VALID_DATALAKE_AUDIENCES } from "../utils/constants"; +import IAuthenticator from "./IAuthenticator"; + +export default class BlobTokenAuthenticator implements IAuthenticator { + public constructor( + private readonly dataStore: IAccountDataStore, + private readonly oauth: OAuthLevel, + private readonly logger: ILogger + ) {} + + public async validate( + req: IRequest, + context: Context + ): Promise { + const blobContext = new DataLakeContext(context); + const account = blobContext.account!; + + this.logger.info( + `BlobTokenAuthenticator:validate() Start validation against token authentication.`, + blobContext.contextId + ); + + // TODO: Make following async + const accountProperties = this.dataStore.getAccount(account); + if (accountProperties === undefined) { + this.logger.error( + `BlobTokenAuthenticator:validate() Invalid storage account ${account}.`, + blobContext.contextId + ); + throw StorageErrorFactory.ResourceNotFound(blobContext); + } + + const authHeaderValue = req.getHeader(HeaderConstants.AUTHORIZATION); + if (authHeaderValue === undefined) { + this.logger.info( + `BlobTokenAuthenticator:validate() Request doesn't include valid authentication header. Skip token authentication.`, + blobContext.contextId + ); + return; + } + + if ( + //blob + blobContext.operation === Operation.Container_GetAccessPolicy || + blobContext.operation === Operation.Container_SetAccessPolicy || + //DataLake + blobContext.operation === Operation.FileSystem_GetProperties || + blobContext.operation === Operation.FileSystem_SetProperties + ) { + this.logger.info( + `BlobTokenAuthenticator:validate() Operation is not available with OAuth. Skip token authentication.`, + blobContext.contextId + ); + return; + } + + if (!authHeaderValue.startsWith(BEARER_TOKEN_PREFIX)) { + throw StorageErrorFactory.getInvalidAuthenticationInfo(context); + } + + if (req.getProtocol().toLowerCase() !== HTTPS) { + throw StorageErrorFactory.getAuthenticationFailed( + context, + "Authentication scheme Bearer is not allowed with HTTP." + ); + } + + // TODO: Check API Version and enable bearer challenge after 2019-12-12 + + const token = authHeaderValue.substr(BEARER_TOKEN_PREFIX.length + 1); + + switch (this.oauth) { + case OAuthLevel.BASIC: + return this.authenticateBasic(token, context); + default: + this.logger.warn( + `BlobTokenAuthenticator:validate() Unknown OAuth level ${this.oauth}. Skip token authentication.`, + blobContext.contextId + ); + return; + } + } + + public async authenticateBasic( + token: string, + context: Context + ): Promise { + // tslint:disable: max-line-length + /** + * Example OAuth Bearer Token: + * { + * "aud": "https://storage.azure.com", + * "iss": "https://sts.windows-ppe.net/ab1f708d-50f6-404c-a006-d71b2ac7a606/", + * "iat": 1511859603, + * "nbf": 1511859603, + * "exp": 1511863503, + * "_claim_names": { + * "groups": "src1" + * }, + * "_claim_sources": { + * "src1": { + * "endpoint": "https://graph.ppe.windows.net/ab1f708d-50f6-404c-a006-d71b2ac7a606/users/11059fcc-5514-4800-b5f8-49808b9cfab6/getMemberObjects" + * } + * }, + * "acr": "1", + * "aio": "ASQA2/8JAAAAE9BLp+qWOO0iEg3n6mjPNqaoHSCIqhQQ8OCinJcub7U=", + * "amr": [ + * "pwd" + * ], + * "appid": "f997392c-e15a-4ad8-af9e-cd6966caba7f", + * "appidacr": "0", + * "e_exp": 262800, + * "ipaddr": "167.220.255.51", + * "name": "test", + * "oid": "11059fcc-5514-4800-b5f8-49808b9cfab6", + * "puid": "10033FFFA6976143", + * "scp": "user_impersonation", + * "sub": "JeH2dBcxZp-_pKoskdxOilGj234LhlM_0GBM4JvSrJw", + * "tid": "ab1f708d-50f6-404c-a006-d71b2ac7a606", + * "unique_name": "test@oauthtest102.ccsctp.net", + * "upn": "test@oauthtest102.ccsctp.net", + * "uti": "dd77i7fCB0yn8aQYUqsJAA", + * "ver": "1.0" + * } + */ + + // Validate JWT token format + let decoded; + try { + decoded = decode(token) as { [key: string]: any }; + } catch { + throw StorageErrorFactory.getAuthenticationFailed( + context, + "Authentication scheme Bearer is not supported." + ); + } + + if (!decoded) { + throw StorageErrorFactory.getAuthenticationFailed( + context, + "Authentication scheme Bearer is not supported." + ); + } + + // Validate signature, skip in basic check + + // Validate nbf & exp + if ( + decoded.nbf === undefined || + decoded.exp === undefined || + decoded.iat === undefined + ) { + throw StorageErrorFactory.getAuthenticationFailed( + context, + "Authentication scheme Bearer is not supported." + ); + } + + const now = context.startTime!.getTime(); + const nbf = (decoded.nbf as number) * 1000; + const exp = (decoded.exp as number) * 1000; + + if (now < nbf) { + throw StorageErrorFactory.getAuthenticationFailed( + context, + "Lifetime validation failed." + ); + } + + if (now > exp) { + throw StorageErrorFactory.getAuthenticationFailed( + context, + "Lifetime validation failed. The token is expired." + ); + } + + const iss = decoded.iss as string; + if (!iss) { + throw StorageErrorFactory.getAuthenticationFailed( + context, + "Authentication scheme Bearer is not supported." + ); + } + + let issMatch = false; + for (const validIssuePrefix of VALID_ISSUE_PREFIXES) { + if (iss.startsWith(validIssuePrefix)) { + issMatch = true; + break; + } + } + + if (!issMatch) { + throw StorageErrorFactory.getAuthenticationFailed( + context, + "Invalid token issuer." + ); + } + + const aud = decoded.aud as string; + if (!aud) { + throw StorageErrorFactory.getAuthenticationFailed( + context, + "Authentication scheme Bearer is not supported." + ); + } + + const blobContext = context as DataLakeContext; + let audMatch = false; + let m; + for (const regex of VALID_DATALAKE_AUDIENCES) { + m = regex.exec(aud); + if (m !== null) { + if (m[0] === aud) { + if (m[1] !== undefined && m[1] !== blobContext.account) { + // If account name doesn't match for fine grained audiance + break; + } + audMatch = true; + break; + } + } + } + + if (!audMatch) { + throw StorageErrorFactory.getAuthenticationFailed( + context, + "Invalid token audience." + ); + } + + // Skip validate scope + // Currently scope maybe null or user_impersonation + + this.logger.info( + `BlobTokenAuthenticator:authenticateBasic() Validation against token authentication successfully.`, + blobContext.contextId + ); + return true; + } +} diff --git a/src/dfs/authentication/ContainerSASPermissions.ts b/src/dfs/authentication/ContainerSASPermissions.ts new file mode 100644 index 000000000..9ec9a216d --- /dev/null +++ b/src/dfs/authentication/ContainerSASPermissions.ts @@ -0,0 +1,8 @@ +export enum ContainerSASPermission { + Read = "r", + Add = "a", + Create = "c", + Write = "w", + Delete = "d", + List = "l" +} diff --git a/src/dfs/authentication/IAuthenticationContext.ts b/src/dfs/authentication/IAuthenticationContext.ts new file mode 100644 index 000000000..b6517487f --- /dev/null +++ b/src/dfs/authentication/IAuthenticationContext.ts @@ -0,0 +1,3 @@ +export default interface IAuthenticationContext { + account?: string; +} diff --git a/src/dfs/authentication/IAuthenticator.ts b/src/dfs/authentication/IAuthenticator.ts new file mode 100644 index 000000000..79d973cb0 --- /dev/null +++ b/src/dfs/authentication/IAuthenticator.ts @@ -0,0 +1,6 @@ +import Context from "../generated/Context"; +import IRequest from "../generated/IRequest"; + +export default interface IAuthenticator { + validate(req: IRequest, content: Context): Promise; +} diff --git a/src/dfs/authentication/IBlobSASSignatureValues.ts b/src/dfs/authentication/IBlobSASSignatureValues.ts new file mode 100644 index 000000000..7ea4b5185 --- /dev/null +++ b/src/dfs/authentication/IBlobSASSignatureValues.ts @@ -0,0 +1,727 @@ +import { + computeHMACSHA256, + truncatedISO8061Date +} from "../../common/utils/utils"; +import { BlobSASResourceType } from "./BlobSASResourceType"; +import { SASProtocol } from "../../common/authentication/IAccountSASSignatureValues"; +import { + IIPRange, + ipRangeToString +} from "../../common/authentication/IIPRange"; + +/** + * IBlobSASSignatureValues is used to help generating Blob service SAS tokens for containers or blobs. + * + * @export + * @class IBlobSASSignatureValues + */ +export interface IBlobSASSignatureValues { + /** + * The version of the service this SAS will target. If not specified, it will default to the version targeted by the + * library. + * + * @type {string} + * @memberof IBlobSASSignatureValues + */ + version: string; + + /** + * Optional. SAS protocols, HTTPS only or HTTPSandHTTP + * + * @type {SASProtocol | string} + * @memberof IBlobSASSignatureValues + */ + protocol?: SASProtocol | string; + + /** + * Optional. When the SAS will take effect. + * + * @type {Date | string} + * @memberof IBlobSASSignatureValues + */ + startTime?: Date | string; + + /** + * Optional only when identifier is provided. The time after which the SAS will no longer work. + * + * @type {Date | string} + * @memberof IBlobSASSignatureValues + */ + expiryTime?: Date | string; + + /** + * Optional only when identifier is provided. + * Please refer to either {@link ContainerSASPermissions} or {@link BlobSASPermissions} depending on the resource + * being accessed for help constructing the permissions string. + * + * @type {string} + * @memberof IBlobSASSignatureValues + */ + permissions?: string; + + /** + * Optional. IP ranges allowed in this SAS. + * + * @type {IIPRange | string} + * @memberof IBlobSASSignatureValues + */ + ipRange?: IIPRange | string; + + /** + * The name of the container the SAS user may access. + * + * @type {string} + * @memberof IBlobSASSignatureValues + */ + containerName: string; + + /** + * Optional. The name of the container the SAS user may access. + * + * @type {string} + * @memberof IBlobSASSignatureValues + */ + blobName?: string; + + /** + * Optional. The name of the access policy on the container this SAS references if any. + * + * @see https://docs.microsoft.com/en-us/rest/api/storageservices/establishing-a-stored-access-policy + * + * @type {string} + * @memberof IBlobSASSignatureValues + */ + identifier?: string; + + /** + * Optional. Encryption scope to use when sending requests authorized with this SAS URI. + * + * @type {string} + * @memberof IBlobSASSignatureValues + */ + encryptionScope?: string; + + /** + * Optional. The cache-control header for the SAS. + * + * @type {string} + * @memberof IBlobSASSignatureValues + */ + cacheControl?: string; + + /** + * Optional. The content-disposition header for the SAS. + * + * @type {string} + * @memberof IBlobSASSignatureValues + */ + contentDisposition?: string; + + /** + * Optional. The content-encoding header for the SAS. + * + * @type {string} + * @memberof IBlobSASSignatureValues + */ + contentEncoding?: string; + + /** + * Optional. The content-language header for the SAS. + * + * @type {string} + * @memberof IBlobSASSignatureValues + */ + contentLanguage?: string; + + /** + * Optional. The content-type header for the SAS. + * + * @type {string} + * @memberof IBlobSASSignatureValues + */ + contentType?: string; + + /** + * Optional. Specifies the accessible resources. Required for version 2018-11-09 and later. + * + * @type {string} + * @memberof IBlobSASSignatureValues + */ + signedResource?: string; + + /** + * Optional. Specifies the snapshot time. Required for version 2018-11-09 and later. + * + * @type {string} + * @memberof IBlobSASSignatureValues + */ + snapshot?: string; + + /** + * Optional. Specifies the Azure Active Directory object ID in GUID format. + * + * @type {string} + * @memberof IBlobSASSignatureValues + */ + signedObjectId?: string; + + /** + * Optional. Specifies the Azure Active Directory object ID in GUID format. + * + * @type {string} + * @memberof IBlobSASSignatureValues + */ + signedTenantId?: string; + + /** + * Optional. Abbreviation of the Azure Storage service that accepts the key.. + * + * @type {string} + * @memberof IBlobSASSignatureValues + */ + signedService?: string; + + /** + * Optional. Specifies the service version to create the key. + * + * @type {string} + * @memberof IBlobSASSignatureValues + */ + signedVersion?: string; + + /** + * Optional. Specifies date-time when the key is active. + * + * @type {string} + * @memberof IBlobSASSignatureValues + */ + signedStartsOn?: string; + + /** + * Optional. Specifies date-time when the key expires. + * + * @type {string} + * @memberof IBlobSASSignatureValues + */ + signedExpiresOn?: string; +} + +/** + * Creates an instance of SASQueryParameters. + * + * Only accepts required settings needed to create a SAS. For optional settings please + * set corresponding properties directly, such as permissions, startTime and identifier. + * + * WARNING: When identifier is not provided, permissions and expiryTime are required. + * You MUST assign value to identifier or expiryTime & permissions manually if you initial with + * this constructor. + * + * @export + * @param {IBlobSASSignatureValues} blobSASSignatureValues + * @param {BlobSASResourceType} resource + * @param {string} accountName + * @param {Buffer} sharedKey + * @returns {[string, string]} signature and stringToSign + */ +export function generateBlobSASSignature( + blobSASSignatureValues: IBlobSASSignatureValues, + resource: BlobSASResourceType, + accountName: string, + sharedKey: Buffer +): [string, string] { + if (blobSASSignatureValues.version >= "2020-12-06") { + return generateBlobSASSignature20201206( + blobSASSignatureValues, + resource, + accountName, + sharedKey + ); + } else if (blobSASSignatureValues.version >= "2018-11-09") { + return generateBlobSASSignature20181109( + blobSASSignatureValues, + resource, + accountName, + sharedKey + ); + } else { + return generateBlobSASSignature20150405( + blobSASSignatureValues, + resource, + accountName, + sharedKey + ); + } +} + +/** + * Creates an instance of SASQueryParameters. + * + * Only accepts required settings needed to create a SAS. For optional settings please + * set corresponding properties directly, such as permissions, startTime and identifier. + * + * WARNING: When identifier is not provided, permissions and expiryTime are required. + * You MUST assign value to identifier or expiryTime & permissions manually if you initial with + * this constructor. + * + * @export + * @param {IBlobSASSignatureValues} blobSASSignatureValues + * @param {BlobSASResourceType} resource + * @param {string} accountName + * @param {Buffer} udkValue + * @returns {[string, string]} signature and stringToSign + */ +export function generateBlobSASSignatureWithUDK( + blobSASSignatureValues: IBlobSASSignatureValues, + resource: BlobSASResourceType, + accountName: string, + udkValue: Buffer +): [string, string] { + if (blobSASSignatureValues.version >= "2020-12-06") { + return generateBlobSASBlobSASSignatureWithUDK20201206( + blobSASSignatureValues, + resource, + accountName, + udkValue + ); + } else if (blobSASSignatureValues.version >= "2020-02-10") { + return generateBlobSASSignatureWithUDK20200210( + blobSASSignatureValues, + resource, + accountName, + udkValue + ); + } else if (blobSASSignatureValues.version >= "2018-11-09") { + return generateBlobSASSignatureUDK20181109( + blobSASSignatureValues, + resource, + accountName, + udkValue + ); + } else { + throw new RangeError("SAS token version is not valid"); + } +} + +function generateBlobSASSignature20201206( + blobSASSignatureValues: IBlobSASSignatureValues, + resource: BlobSASResourceType, + accountName: string, + sharedKey: Buffer +): [string, string] { + if ( + !blobSASSignatureValues.identifier && + !blobSASSignatureValues.permissions && + !blobSASSignatureValues.expiryTime + ) { + throw new RangeError( + // tslint:disable-next-line:max-line-length + "generateBlobSASSignature(): Must provide 'permissions' and 'expiryTime' for Blob SAS generation when 'identifier' is not provided." + ); + } + + const version = blobSASSignatureValues.version; + const verifiedPermissions = blobSASSignatureValues.permissions; + + // Signature is generated on the un-url-encoded values. + // TODO: Check whether validating the snapshot is necessary. + const stringToSign = [ + verifiedPermissions ? verifiedPermissions : "", + blobSASSignatureValues.startTime === undefined + ? "" + : typeof blobSASSignatureValues.startTime === "string" + ? blobSASSignatureValues.startTime + : truncatedISO8061Date(blobSASSignatureValues.startTime, false), + blobSASSignatureValues.expiryTime === undefined + ? "" + : typeof blobSASSignatureValues.expiryTime === "string" + ? blobSASSignatureValues.expiryTime + : truncatedISO8061Date(blobSASSignatureValues.expiryTime, false), + getCanonicalName( + accountName, + blobSASSignatureValues.containerName, + resource === BlobSASResourceType.Blob || + resource === BlobSASResourceType.BlobSnapshot + ? blobSASSignatureValues.blobName + : "" + ), + blobSASSignatureValues.identifier, // TODO: ? blobSASSignatureValues.identifier : "", + blobSASSignatureValues.ipRange + ? typeof blobSASSignatureValues.ipRange === "string" + ? blobSASSignatureValues.ipRange + : ipRangeToString(blobSASSignatureValues.ipRange) + : "", + blobSASSignatureValues.protocol ? blobSASSignatureValues.protocol : "", + version, + blobSASSignatureValues.signedResource, + blobSASSignatureValues.snapshot, + blobSASSignatureValues.encryptionScope + ? blobSASSignatureValues.encryptionScope + : "", + blobSASSignatureValues.cacheControl + ? blobSASSignatureValues.cacheControl + : "", + blobSASSignatureValues.contentDisposition + ? blobSASSignatureValues.contentDisposition + : "", + blobSASSignatureValues.contentEncoding + ? blobSASSignatureValues.contentEncoding + : "", + blobSASSignatureValues.contentLanguage + ? blobSASSignatureValues.contentLanguage + : "", + blobSASSignatureValues.contentType ? blobSASSignatureValues.contentType : "" + ].join("\n"); + + const signature = computeHMACSHA256(stringToSign, sharedKey); + return [signature, stringToSign]; +} + +function generateBlobSASSignature20181109( + blobSASSignatureValues: IBlobSASSignatureValues, + resource: BlobSASResourceType, + accountName: string, + sharedKey: Buffer +): [string, string] { + if ( + !blobSASSignatureValues.identifier && + !blobSASSignatureValues.permissions && + !blobSASSignatureValues.expiryTime + ) { + throw new RangeError( + // tslint:disable-next-line:max-line-length + "generateBlobSASSignature(): Must provide 'permissions' and 'expiryTime' for Blob SAS generation when 'identifier' is not provided." + ); + } + + const version = blobSASSignatureValues.version; + const verifiedPermissions = blobSASSignatureValues.permissions; + + // Signature is generated on the un-url-encoded values. + // TODO: Check whether validating the snapshot is necessary. + const stringToSign = [ + verifiedPermissions ? verifiedPermissions : "", + blobSASSignatureValues.startTime === undefined + ? "" + : typeof blobSASSignatureValues.startTime === "string" + ? blobSASSignatureValues.startTime + : truncatedISO8061Date(blobSASSignatureValues.startTime, false), + blobSASSignatureValues.expiryTime === undefined + ? "" + : typeof blobSASSignatureValues.expiryTime === "string" + ? blobSASSignatureValues.expiryTime + : truncatedISO8061Date(blobSASSignatureValues.expiryTime, false), + getCanonicalName( + accountName, + blobSASSignatureValues.containerName, + resource === BlobSASResourceType.Blob || + resource === BlobSASResourceType.BlobSnapshot + ? blobSASSignatureValues.blobName + : "" + ), + blobSASSignatureValues.identifier, // TODO: ? blobSASSignatureValues.identifier : "", + blobSASSignatureValues.ipRange + ? typeof blobSASSignatureValues.ipRange === "string" + ? blobSASSignatureValues.ipRange + : ipRangeToString(blobSASSignatureValues.ipRange) + : "", + blobSASSignatureValues.protocol ? blobSASSignatureValues.protocol : "", + version, + blobSASSignatureValues.signedResource, + blobSASSignatureValues.snapshot, + blobSASSignatureValues.cacheControl + ? blobSASSignatureValues.cacheControl + : "", + blobSASSignatureValues.contentDisposition + ? blobSASSignatureValues.contentDisposition + : "", + blobSASSignatureValues.contentEncoding + ? blobSASSignatureValues.contentEncoding + : "", + blobSASSignatureValues.contentLanguage + ? blobSASSignatureValues.contentLanguage + : "", + blobSASSignatureValues.contentType ? blobSASSignatureValues.contentType : "" + ].join("\n"); + + const signature = computeHMACSHA256(stringToSign, sharedKey); + return [signature, stringToSign]; +} + +function generateBlobSASSignature20150405( + blobSASSignatureValues: IBlobSASSignatureValues, + resource: BlobSASResourceType, + accountName: string, + sharedKey: Buffer +): [string, string] { + if ( + !blobSASSignatureValues.identifier && + !blobSASSignatureValues.permissions && + !blobSASSignatureValues.expiryTime + ) { + throw new RangeError( + // tslint:disable-next-line:max-line-length + "generateBlobSASSignature(): Must provide 'permissions' and 'expiryTime' for Blob SAS generation when 'identifier' is not provided." + ); + } + + const version = blobSASSignatureValues.version; + const verifiedPermissions = blobSASSignatureValues.permissions; + + // Signature is generated on the un-url-encoded values. + const stringToSign = [ + verifiedPermissions ? verifiedPermissions : "", + blobSASSignatureValues.startTime === undefined + ? "" + : typeof blobSASSignatureValues.startTime === "string" + ? blobSASSignatureValues.startTime + : truncatedISO8061Date(blobSASSignatureValues.startTime, false), + blobSASSignatureValues.expiryTime === undefined + ? "" + : typeof blobSASSignatureValues.expiryTime === "string" + ? blobSASSignatureValues.expiryTime + : truncatedISO8061Date(blobSASSignatureValues.expiryTime, false), + getCanonicalName( + accountName, + blobSASSignatureValues.containerName, + resource === BlobSASResourceType.Blob + ? blobSASSignatureValues.blobName + : "" + ), + blobSASSignatureValues.identifier, // TODO: ? blobSASSignatureValues.identifier : "", + blobSASSignatureValues.ipRange + ? typeof blobSASSignatureValues.ipRange === "string" + ? blobSASSignatureValues.ipRange + : ipRangeToString(blobSASSignatureValues.ipRange) + : "", + blobSASSignatureValues.protocol ? blobSASSignatureValues.protocol : "", + version, + blobSASSignatureValues.cacheControl + ? blobSASSignatureValues.cacheControl + : "", + blobSASSignatureValues.contentDisposition + ? blobSASSignatureValues.contentDisposition + : "", + blobSASSignatureValues.contentEncoding + ? blobSASSignatureValues.contentEncoding + : "", + blobSASSignatureValues.contentLanguage + ? blobSASSignatureValues.contentLanguage + : "", + blobSASSignatureValues.contentType ? blobSASSignatureValues.contentType : "" + ].join("\n"); + + const signature = computeHMACSHA256(stringToSign, sharedKey); + return [signature, stringToSign]; +} + +function generateBlobSASSignatureUDK20181109( + blobSASSignatureValues: IBlobSASSignatureValues, + resource: BlobSASResourceType, + accountName: string, + userDelegationKeyValue: Buffer +): [string, string] { + if ( + !blobSASSignatureValues.identifier && + !blobSASSignatureValues.permissions && + !blobSASSignatureValues.expiryTime + ) { + throw new RangeError( + // tslint:disable-next-line:max-line-length + "generateBlobSASSignature(): Must provide 'permissions' and 'expiryTime' for Blob SAS generation when 'identifier' is not provided." + ); + } + + const verifiedPermissions = blobSASSignatureValues.permissions; + + // Signature is generated on the un-url-encoded values. + const stringToSign = [ + verifiedPermissions ? verifiedPermissions : "", + blobSASSignatureValues.startTime === undefined + ? "" + : typeof blobSASSignatureValues.startTime === "string" + ? blobSASSignatureValues.startTime + : truncatedISO8061Date(blobSASSignatureValues.startTime, false), + blobSASSignatureValues.expiryTime === undefined + ? "" + : typeof blobSASSignatureValues.expiryTime === "string" + ? blobSASSignatureValues.expiryTime + : truncatedISO8061Date(blobSASSignatureValues.expiryTime, false), + getCanonicalName( + accountName, + blobSASSignatureValues.containerName, + blobSASSignatureValues.blobName + ), + blobSASSignatureValues.signedObjectId, + blobSASSignatureValues.signedTenantId, + blobSASSignatureValues.signedStartsOn, + blobSASSignatureValues.signedExpiresOn, + blobSASSignatureValues.signedService, + blobSASSignatureValues.signedVersion, + blobSASSignatureValues.ipRange === undefined + ? "" + : typeof blobSASSignatureValues.ipRange === "string" + ? blobSASSignatureValues.ipRange + : ipRangeToString(blobSASSignatureValues.ipRange), + blobSASSignatureValues.protocol ? blobSASSignatureValues.protocol : "", + blobSASSignatureValues.version, + resource, + undefined, // blob version timestamp, + blobSASSignatureValues.cacheControl, + blobSASSignatureValues.contentDisposition, + blobSASSignatureValues.contentEncoding, + blobSASSignatureValues.contentLanguage, + blobSASSignatureValues.contentType + ].join("\n"); + + const signature = computeHMACSHA256(stringToSign, userDelegationKeyValue); + return [signature, stringToSign]; +} + +function generateBlobSASSignatureWithUDK20200210( + blobSASSignatureValues: IBlobSASSignatureValues, + resource: BlobSASResourceType, + accountName: string, + userDelegationKeyValue: Buffer +): [string, string] { + if ( + !blobSASSignatureValues.identifier && + !blobSASSignatureValues.permissions && + !blobSASSignatureValues.expiryTime + ) { + throw new RangeError( + // tslint:disable-next-line:max-line-length + "generateBlobSASSignature(): Must provide 'permissions' and 'expiryTime' for Blob SAS generation when 'identifier' is not provided." + ); + } + + const verifiedPermissions = blobSASSignatureValues.permissions; + + // Signature is generated on the un-url-encoded values. + const stringToSign = [ + verifiedPermissions ? verifiedPermissions : "", + blobSASSignatureValues.startTime === undefined + ? "" + : typeof blobSASSignatureValues.startTime === "string" + ? blobSASSignatureValues.startTime + : truncatedISO8061Date(blobSASSignatureValues.startTime, false), + blobSASSignatureValues.expiryTime === undefined + ? "" + : typeof blobSASSignatureValues.expiryTime === "string" + ? blobSASSignatureValues.expiryTime + : truncatedISO8061Date(blobSASSignatureValues.expiryTime, false), + getCanonicalName( + accountName, + blobSASSignatureValues.containerName, + blobSASSignatureValues.blobName + ), + blobSASSignatureValues.signedObjectId, + blobSASSignatureValues.signedTenantId, + blobSASSignatureValues.signedStartsOn, + blobSASSignatureValues.signedExpiresOn, + blobSASSignatureValues.signedService, + blobSASSignatureValues.signedVersion, + undefined, // blobSASSignatureValues.preauthorizedAgentObjectId, + undefined, // agentObjectId + undefined, // blobSASSignatureValues.correlationId, + blobSASSignatureValues.ipRange === undefined + ? "" + : typeof blobSASSignatureValues.ipRange === "string" + ? blobSASSignatureValues.ipRange + : ipRangeToString(blobSASSignatureValues.ipRange), + blobSASSignatureValues.protocol ? blobSASSignatureValues.protocol : "", + blobSASSignatureValues.version, + resource, + undefined, // blob versiontimestamp, + blobSASSignatureValues.cacheControl, + blobSASSignatureValues.contentDisposition, + blobSASSignatureValues.contentEncoding, + blobSASSignatureValues.contentLanguage, + blobSASSignatureValues.contentType + ].join("\n"); + + const signature = computeHMACSHA256(stringToSign, userDelegationKeyValue); + return [signature, stringToSign]; +} + +function generateBlobSASBlobSASSignatureWithUDK20201206( + blobSASSignatureValues: IBlobSASSignatureValues, + resource: BlobSASResourceType, + accountName: string, + userDelegationKeyValue: Buffer +): [string, string] { + if ( + !blobSASSignatureValues.identifier && + !blobSASSignatureValues.permissions && + !blobSASSignatureValues.expiryTime + ) { + throw new RangeError( + // tslint:disable-next-line:max-line-length + "generateBlobSASSignature(): Must provide 'permissions' and 'expiryTime' for Blob SAS generation when 'identifier' is not provided." + ); + } + + const verifiedPermissions = blobSASSignatureValues.permissions; + + // Signature is generated on the un-url-encoded values. + const stringToSign = [ + verifiedPermissions ? verifiedPermissions : "", + blobSASSignatureValues.startTime === undefined + ? "" + : typeof blobSASSignatureValues.startTime === "string" + ? blobSASSignatureValues.startTime + : truncatedISO8061Date(blobSASSignatureValues.startTime, false), + blobSASSignatureValues.expiryTime === undefined + ? "" + : typeof blobSASSignatureValues.expiryTime === "string" + ? blobSASSignatureValues.expiryTime + : truncatedISO8061Date(blobSASSignatureValues.expiryTime, false), + getCanonicalName( + accountName, + blobSASSignatureValues.containerName, + resource === BlobSASResourceType.Blob + ? blobSASSignatureValues.blobName + : "" + ), + blobSASSignatureValues.signedObjectId, + blobSASSignatureValues.signedTenantId, + blobSASSignatureValues.signedStartsOn, + blobSASSignatureValues.signedExpiresOn, + blobSASSignatureValues.signedService, + blobSASSignatureValues.signedVersion, + undefined, // blobSASSignatureValues.preauthorizedAgentObjectId, + undefined, // agentObjectId + undefined, // blobSASSignatureValues.correlationId, + blobSASSignatureValues.ipRange === undefined + ? "" + : typeof blobSASSignatureValues.ipRange === "string" + ? blobSASSignatureValues.ipRange + : ipRangeToString(blobSASSignatureValues.ipRange), + blobSASSignatureValues.protocol ? blobSASSignatureValues.protocol : "", + blobSASSignatureValues.version, + resource, + undefined, // blob version timestamp, + blobSASSignatureValues.encryptionScope, + blobSASSignatureValues.cacheControl, + blobSASSignatureValues.contentDisposition, + blobSASSignatureValues.contentEncoding, + blobSASSignatureValues.contentLanguage, + blobSASSignatureValues.contentType + ].join("\n"); + + const signature = computeHMACSHA256(stringToSign, userDelegationKeyValue); + return [signature, stringToSign]; +} + +function getCanonicalName( + accountName: string, + containerName: string, + blobName?: string +): string { + // Container: "/blob/account/containerName" + // Blob: "/blob/account/containerName/blobName" + const elements: string[] = [`/blob/${accountName}/${containerName}`]; + if (blobName) { + elements.push(`/${blobName}`); + } + return elements.join(""); +} diff --git a/src/dfs/authentication/IRange.ts b/src/dfs/authentication/IRange.ts new file mode 100644 index 000000000..1615bbf57 --- /dev/null +++ b/src/dfs/authentication/IRange.ts @@ -0,0 +1,48 @@ +// tslint:disable:max-line-length +/** + * Range for Blob Service Operations. + * @see https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-the-range-header-for-blob-service-operations + * + * @export + * @interface IRange + */ +export interface IRange { + /** + * StartByte, larger than or equal 0. + * + * @type {string} + * @memberof IRange + */ + offset: number; + /** + * Optional. Count of bytes, larger than 0. + * If not provided, will return bytes from offset to the end. + * + * @type {string} + * @memberof IRange + */ + count?: number; +} + +/** + * Generate a range string. For example: + * + * "bytes=255-" or "bytes=0-511" + * + * @export + * @param {IRange} iRange + * @returns {string} + */ +export function rangeToString(iRange: IRange): string { + if (iRange.offset < 0) { + throw new RangeError(`IRange.offset cannot be smaller than 0.`); + } + if (iRange.count && iRange.count <= 0) { + throw new RangeError( + `IRange.count must be larger than 0. Leave it undefined if you want a range from offset to the end.` + ); + } + return iRange.count + ? `bytes=${iRange.offset}-${iRange.offset + iRange.count - 1}` + : `bytes=${iRange.offset}-`; +} diff --git a/src/dfs/authentication/OperationAccountSASPermission.ts b/src/dfs/authentication/OperationAccountSASPermission.ts new file mode 100644 index 000000000..ad7e719ff --- /dev/null +++ b/src/dfs/authentication/OperationAccountSASPermission.ts @@ -0,0 +1,753 @@ +import { + AccountSASPermissions, + AccountSASResourceTypes, + AccountSASServices +} from "@azure/storage-blob"; + +import Operation from "../generated/artifacts/operation"; +import { AccountSASPermission } from "../../common/authentication/AccountSASPermissions"; +import { AccountSASResourceType } from "../../common/authentication/AccountSASResourceTypes"; +import { AccountSASService } from "../../common/authentication/AccountSASServices"; + +export class OperationAccountSASPermission { + constructor( + public readonly service: string, + public readonly resourceType?: string, + public readonly permission?: string + ) {} + + public validate( + services: AccountSASServices | string, + resourceTypes: AccountSASResourceTypes | string, + permissions: AccountSASPermissions | string + ): boolean { + return ( + this.validateServices(services) && + this.validateResourceTypes(resourceTypes) && + this.validatePermissions(permissions) + ); + } + + public validateServices(services: AccountSASServices | string): boolean { + return services.toString().includes(this.service); + } + + public validateResourceTypes( + resourceTypes: AccountSASResourceTypes | string + ): boolean { + if (this.resourceType) { + for (const p of this.resourceType) { + if (resourceTypes.toString().includes(p)) { + return true; + } + } + return false; + } else { + return resourceTypes.toString() !== ""; + } + } + + public validatePermissions( + permissions: AccountSASPermissions | string + ): boolean { + if (this.permission) { + for (const p of this.permission) { + if (permissions.toString().includes(p)) { + return true; + } + } + return false; + } else { + return permissions.toString() !== ""; + } + } +} + +// See https://docs.microsoft.com/en-us/rest/api/storageservices/constructing-an-account-sas +// TODO: Check all required operations +const OPERATION_ACCOUNT_SAS_PERMISSIONS = new Map< + Operation, + OperationAccountSASPermission +>(); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Service_GetAccountInfo, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Service + + AccountSASResourceType.Container + + AccountSASResourceType.Object, + AccountSASPermission.Read + + AccountSASPermission.Create + + AccountSASPermission.Delete + + AccountSASPermission.List + + AccountSASPermission.Process + + AccountSASPermission.Read + + AccountSASPermission.Update + + AccountSASPermission.Write + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Service_GetAccountInfoWithHead, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Service + + AccountSASResourceType.Container + + AccountSASResourceType.Object, + AccountSASPermission.Read + + AccountSASPermission.Create + + AccountSASPermission.Delete + + AccountSASPermission.List + + AccountSASPermission.Process + + AccountSASPermission.Read + + AccountSASPermission.Update + + AccountSASPermission.Write + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Container_GetAccountInfo, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Service + + AccountSASResourceType.Container + + AccountSASResourceType.Object, + AccountSASPermission.Read + + AccountSASPermission.Create + + AccountSASPermission.Delete + + AccountSASPermission.List + + AccountSASPermission.Process + + AccountSASPermission.Read + + AccountSASPermission.Update + + AccountSASPermission.Write + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Container_GetAccountInfoWithHead, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Service + + AccountSASResourceType.Container + + AccountSASResourceType.Object, + AccountSASPermission.Read + + AccountSASPermission.Create + + AccountSASPermission.Delete + + AccountSASPermission.List + + AccountSASPermission.Process + + AccountSASPermission.Read + + AccountSASPermission.Update + + AccountSASPermission.Write + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Blob_GetAccountInfo, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Service + + AccountSASResourceType.Container + + AccountSASResourceType.Object, + AccountSASPermission.Read + + AccountSASPermission.Create + + AccountSASPermission.Delete + + AccountSASPermission.List + + AccountSASPermission.Process + + AccountSASPermission.Read + + AccountSASPermission.Update + + AccountSASPermission.Write + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Blob_GetAccountInfoWithHead, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Service + + AccountSASResourceType.Container + + AccountSASResourceType.Object, + AccountSASPermission.Read + + AccountSASPermission.Create + + AccountSASPermission.Delete + + AccountSASPermission.List + + AccountSASPermission.Process + + AccountSASPermission.Read + + AccountSASPermission.Update + + AccountSASPermission.Write + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Service_ListContainersSegment, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Service, + AccountSASPermission.List + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Service_GetProperties, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Service, + AccountSASPermission.Read + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Service_SetProperties, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Service, + AccountSASPermission.Write + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Service_SubmitBatch, + new OperationAccountSASPermission( + AccountSASService.Blob, + "", + "" // NOT ALLOWED + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Service_GetStatistics, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Service, + AccountSASPermission.Read + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Container_Create, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Container, + AccountSASPermission.Create + AccountSASPermission.Write + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Container_SetAccessPolicy, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Container, + "" // NOT ALLOWED + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Container_GetAccessPolicy, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Container, + "" // NOT ALLOWED + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Container_GetProperties, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Container, + AccountSASPermission.Read + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Container_GetPropertiesWithHead, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Container, + AccountSASPermission.Read + ) +); + +// TODO: Get container metadata is missing in swagger + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Container_SetMetadata, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Container, + AccountSASPermission.Write + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Container_BreakLease, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Container, + AccountSASPermission.Write + AccountSASPermission.Delete + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Container_RenewLease, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Container, + AccountSASPermission.Write + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Container_ChangeLease, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Container, + AccountSASPermission.Write + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Container_AcquireLease, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Container, + AccountSASPermission.Write + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Container_ReleaseLease, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Container, + AccountSASPermission.Write + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Container_Delete, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Container, + AccountSASPermission.Delete + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.BlockBlob_Upload, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Object, + // Create permission is only available for non existing block blob. Handle this scenario separately + AccountSASPermission.Write + AccountSASPermission.Create + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.PageBlob_Create, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Object, + // Create permission is only available for non existing page blob. Handle this scenario separately + AccountSASPermission.Write + AccountSASPermission.Create + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.AppendBlob_Create, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Object, + // Create permission is only available for non existing append blob. Handle this scenario separately + AccountSASPermission.Write + AccountSASPermission.Create + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Blob_SetHTTPHeaders, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Object, + AccountSASPermission.Write + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Blob_SetMetadata, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Object, + AccountSASPermission.Write + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Blob_BreakLease, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Object, + AccountSASPermission.Delete + AccountSASPermission.Write + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Blob_RenewLease, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Object, + AccountSASPermission.Write + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Blob_ChangeLease, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Object, + AccountSASPermission.Write + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Blob_AcquireLease, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Object, + AccountSASPermission.Write + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Blob_ReleaseLease, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Object, + AccountSASPermission.Write + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Blob_CreateSnapshot, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Object, + AccountSASPermission.Write + AccountSASPermission.Create + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Blob_StartCopyFromURL, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Object, + // If destination is an existing blob, create permission is not enough + AccountSASPermission.Write + AccountSASPermission.Create + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Blob_CopyFromURL, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Object, + // If destination is an existing blob, create permission is not enough + AccountSASPermission.Write + AccountSASPermission.Create + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.PageBlob_CopyIncremental, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Object, + AccountSASPermission.Write + AccountSASPermission.Create + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Blob_AbortCopyFromURL, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Object, + AccountSASPermission.Write + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.BlockBlob_StageBlock, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Object, + AccountSASPermission.Write + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.BlockBlob_CommitBlockList, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Object, + AccountSASPermission.Write + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.BlockBlob_GetBlockList, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Object, + AccountSASPermission.Read + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.PageBlob_UploadPages, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Object, + AccountSASPermission.Write + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.PageBlob_GetPageRanges, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Object, + AccountSASPermission.Read + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.PageBlob_GetPageRangesDiff, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Object, + AccountSASPermission.Read + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.AppendBlob_AppendBlock, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Object, + AccountSASPermission.Add + AccountSASPermission.Write + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.PageBlob_ClearPages, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Object, + AccountSASPermission.Write + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Blob_SetTier, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Object, + AccountSASPermission.Write + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.PageBlob_UpdateSequenceNumber, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Object, + AccountSASPermission.Write + ) +); + +///////////////////////////////// DataLake //////////////////////////////////// + +//////////////////////////////////Paths Operations////////////////////////////// +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Path_Create, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Object, + AccountSASPermission.Create + AccountSASPermission.Write + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Path_AppendData, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Object, + AccountSASPermission.Create + AccountSASPermission.Write + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Path_FlushData, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Object, + AccountSASPermission.Write + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Path_Lease, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Object, + AccountSASPermission.Write + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Path_SetAccessControl, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Object, + AccountSASPermission.Write + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Path_SetAccessControlRecursive, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Object, + AccountSASPermission.Write + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Path_SetExpiry, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Object, + AccountSASPermission.Write + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Path_Update, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Object, + AccountSASPermission.Write + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Path_Undelete, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Object, + AccountSASPermission.Write + AccountSASPermission.Create + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Path_Read, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Object, + AccountSASPermission.Read + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Path_GetProperties, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Object, + AccountSASPermission.Read + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Path_Delete, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Object, + AccountSASPermission.Delete + ) +); + +//////////////////////////////////FileSystem Operations////////////////////////////// + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.FileSystem_Create, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Container, + AccountSASPermission.Create + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.FileSystem_SetProperties, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Container, + AccountSASPermission.Write + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.FileSystem_GetProperties, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Container, + AccountSASPermission.Read + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.FileSystem_ListPaths, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Container, + AccountSASPermission.List + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.FileSystem_ListBlobFlatSegment, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Container, + AccountSASPermission.List + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.FileSystem_ListBlobHierarchySegment, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Container, + AccountSASPermission.List + ) +); + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.FileSystem_Delete, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Container, + AccountSASPermission.Delete + ) +); + +//////////////////////////////////Service Operations////////////////////////////// + +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.Service_ListFileSystems, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Service, + AccountSASPermission.Delete + ) +); + +export default OPERATION_ACCOUNT_SAS_PERMISSIONS; diff --git a/src/dfs/authentication/OperationBlobSASPermission.ts b/src/dfs/authentication/OperationBlobSASPermission.ts new file mode 100644 index 000000000..ebb0617da --- /dev/null +++ b/src/dfs/authentication/OperationBlobSASPermission.ts @@ -0,0 +1,669 @@ +import Operation from "../generated/artifacts/operation"; +import { BlobSASPermission } from "./BlobSASPermissions"; +import { ContainerSASPermission } from "./ContainerSASPermissions"; + +export class OperationBlobSASPermission { + constructor(public readonly permission: string = "") {} + + public validate(permissions: string): boolean { + return this.validatePermissions(permissions); + } + + public validatePermissions(permissions: string): boolean { + if (this.permission !== "") { + for (const p of this.permission) { + if (permissions.toString().includes(p)) { + return true; + } + } + return false; + } else { + return permissions.toString() !== ""; + } + } +} + +// See https://docs.microsoft.com/en-us/rest/api/storageservices/constructing-a-service-sas +// Blob Service SAS Permissions for blob level +export const OPERATION_BLOB_SAS_BLOB_PERMISSIONS = new Map< + Operation, + OperationBlobSASPermission +>(); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Service_SetProperties, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Service_GetProperties, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Service_GetStatistics, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Service_ListContainersSegment, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Service_GetAccountInfo, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Service_GetAccountInfoWithHead, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Container_Create, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Container_GetProperties, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Container_GetPropertiesWithHead, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Container_Delete, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Container_SetMetadata, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Container_GetAccessPolicy, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Container_SetAccessPolicy, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Container_AcquireLease, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Container_ReleaseLease, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Container_RenewLease, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Container_BreakLease, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Container_ChangeLease, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Container_GetAccountInfo, + new OperationBlobSASPermission(BlobSASPermission.Read) +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Container_GetAccountInfoWithHead, + new OperationBlobSASPermission(BlobSASPermission.Read) +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Blob_Undelete, + new OperationBlobSASPermission(BlobSASPermission.Write) // TODO: Not sure +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Blob_SetHTTPHeaders, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Blob_SetMetadata, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Blob_AcquireLease, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Blob_ReleaseLease, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Blob_RenewLease, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Blob_ChangeLease, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Blob_BreakLease, + new OperationBlobSASPermission( + BlobSASPermission.Delete + BlobSASPermission.Write + ) +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Blob_CreateSnapshot, + new OperationBlobSASPermission( + BlobSASPermission.Create + BlobSASPermission.Write + ) +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Blob_StartCopyFromURL, + // TODO: When destination blob doesn't exist, needs create permission + new OperationBlobSASPermission( + BlobSASPermission.Write + BlobSASPermission.Create + ) +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Blob_AbortCopyFromURL, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Blob_CopyFromURL, + // TODO: When destination blob doesn't exist, needs create permission + new OperationBlobSASPermission( + BlobSASPermission.Write + BlobSASPermission.Create + ) +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Blob_SetTier, + new OperationBlobSASPermission(BlobSASPermission.Write) // TODO: Not sure +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Blob_GetAccountInfo, + new OperationBlobSASPermission(BlobSASPermission.Read) +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Blob_GetAccountInfoWithHead, + new OperationBlobSASPermission(BlobSASPermission.Read) +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.PageBlob_Create, + // TODO: When destination blob doesn't exist, needs create permission + new OperationBlobSASPermission( + BlobSASPermission.Create + BlobSASPermission.Write + ) +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.PageBlob_UploadPages, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.PageBlob_ClearPages, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.PageBlob_GetPageRanges, + new OperationBlobSASPermission(BlobSASPermission.Read) +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.PageBlob_GetPageRangesDiff, + new OperationBlobSASPermission(BlobSASPermission.Read) +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.PageBlob_Resize, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.PageBlob_UpdateSequenceNumber, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.PageBlob_CopyIncremental, + new OperationBlobSASPermission( + BlobSASPermission.Write + BlobSASPermission.Create + ) +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.AppendBlob_Create, + // TODO: When destination blob doesn't exist, needs create permission + new OperationBlobSASPermission( + BlobSASPermission.Write + BlobSASPermission.Create + ) +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.AppendBlob_AppendBlock, + new OperationBlobSASPermission( + BlobSASPermission.Add + BlobSASPermission.Write + ) +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.BlockBlob_Upload, + // TODO: When destination blob doesn't exist, needs create permission + new OperationBlobSASPermission( + BlobSASPermission.Write + BlobSASPermission.Create + ) +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.BlockBlob_StageBlock, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.BlockBlob_StageBlockFromURL, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.BlockBlob_CommitBlockList, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.BlockBlob_GetBlockList, + new OperationBlobSASPermission(BlobSASPermission.Read) +); +///////////////////////////////// DataLake //////////////////////////////////// + +//////////////////////////////////Paths Operations////////////////////////////// +//Since it is used in rename as well it needs write +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Path_Create, + new OperationBlobSASPermission( + BlobSASPermission.Create + BlobSASPermission.Write + ) +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Path_AppendData, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Path_FlushData, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Path_Lease, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Path_SetAccessControl, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Path_SetAccessControlRecursive, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Path_SetExpiry, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Path_Update, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Path_Undelete, + new OperationBlobSASPermission( + BlobSASPermission.Write + BlobSASPermission.Create + ) +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Path_Read, + new OperationBlobSASPermission(BlobSASPermission.Read) +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Path_GetProperties, + new OperationBlobSASPermission(BlobSASPermission.Read) +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Path_Delete, + new OperationBlobSASPermission(BlobSASPermission.Delete) +); +//////////////////////////////////FileSystem Operations////////////////////////////// + +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.FileSystem_Create, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.FileSystem_SetProperties, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.FileSystem_GetProperties, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.FileSystem_ListPaths, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.FileSystem_ListBlobFlatSegment, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.FileSystem_ListBlobHierarchySegment, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.FileSystem_Delete, + new OperationBlobSASPermission() +); +//////////////////////////////////Service Operations////////////////////////////// + +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.Service_ListFileSystems, + new OperationBlobSASPermission() +); + +// Blob Service SAS Permissions for container level +export const OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS = new Map< + Operation, + OperationBlobSASPermission +>(); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Service_SetProperties, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Service_GetProperties, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Service_GetStatistics, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Service_ListContainersSegment, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Service_GetAccountInfo, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Service_GetAccountInfoWithHead, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Container_Create, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Container_GetProperties, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Container_GetPropertiesWithHead, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Container_Delete, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Container_SetMetadata, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Container_SubmitBatch, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Container_GetAccessPolicy, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Container_SetAccessPolicy, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Container_AcquireLease, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Container_ReleaseLease, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Container_RenewLease, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Container_BreakLease, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Container_ChangeLease, + new OperationBlobSASPermission() +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Container_GetAccountInfo, + new OperationBlobSASPermission(ContainerSASPermission.Read) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Container_GetAccountInfoWithHead, + new OperationBlobSASPermission(ContainerSASPermission.Read) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Blob_Undelete, + new OperationBlobSASPermission(BlobSASPermission.Write) // TODO: Not sure +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Blob_SetHTTPHeaders, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Blob_SetMetadata, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Blob_AcquireLease, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Blob_ReleaseLease, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Blob_RenewLease, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Blob_ChangeLease, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Blob_BreakLease, + new OperationBlobSASPermission( + BlobSASPermission.Write + BlobSASPermission.Delete + ) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Blob_CreateSnapshot, + new OperationBlobSASPermission( + BlobSASPermission.Write + BlobSASPermission.Create + ) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Blob_StartCopyFromURL, + new OperationBlobSASPermission( + BlobSASPermission.Write + BlobSASPermission.Create + ) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Blob_AbortCopyFromURL, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Blob_CopyFromURL, + new OperationBlobSASPermission( + BlobSASPermission.Write + BlobSASPermission.Create + ) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Blob_SetTier, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Blob_GetAccountInfo, + new OperationBlobSASPermission(BlobSASPermission.Read) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Blob_GetAccountInfoWithHead, + new OperationBlobSASPermission(BlobSASPermission.Read) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.PageBlob_Create, + new OperationBlobSASPermission( + // TODO: When destination blob doesn't exist, needs create permission + BlobSASPermission.Write + BlobSASPermission.Create + ) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.PageBlob_UploadPages, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.PageBlob_ClearPages, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.PageBlob_GetPageRanges, + new OperationBlobSASPermission(BlobSASPermission.Read) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.PageBlob_GetPageRangesDiff, + new OperationBlobSASPermission(BlobSASPermission.Read) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.PageBlob_Resize, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.PageBlob_UpdateSequenceNumber, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.PageBlob_CopyIncremental, + new OperationBlobSASPermission( + BlobSASPermission.Write + BlobSASPermission.Create + ) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.AppendBlob_Create, + // TODO: Create a new blob must be write + new OperationBlobSASPermission( + BlobSASPermission.Write + BlobSASPermission.Create + ) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.AppendBlob_AppendBlock, + new OperationBlobSASPermission( + BlobSASPermission.Add + BlobSASPermission.Write + ) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.BlockBlob_Upload, + // Create a new blob, must be write + new OperationBlobSASPermission( + BlobSASPermission.Write + BlobSASPermission.Create + ) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.BlockBlob_StageBlock, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.BlockBlob_StageBlockFromURL, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.BlockBlob_CommitBlockList, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.BlockBlob_GetBlockList, + new OperationBlobSASPermission(BlobSASPermission.Read) +); +///////////////////////////////// DataLake //////////////////////////////////// + +//////////////////////////////////Paths Operations////////////////////////////// +//Since it is used in rename as well it needs write +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Path_Create, + new OperationBlobSASPermission( + BlobSASPermission.Create + BlobSASPermission.Write + ) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Path_AppendData, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Path_FlushData, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Path_Lease, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Path_SetAccessControl, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Path_SetAccessControlRecursive, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Path_SetExpiry, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Path_Update, + new OperationBlobSASPermission(BlobSASPermission.Write) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Path_Undelete, + new OperationBlobSASPermission( + BlobSASPermission.Write + BlobSASPermission.Create + ) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Path_Read, + new OperationBlobSASPermission(BlobSASPermission.Read) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Path_GetProperties, + new OperationBlobSASPermission(BlobSASPermission.Read) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Path_Delete, + new OperationBlobSASPermission(BlobSASPermission.Delete) +); +//////////////////////////////////FileSystem Operations////////////////////////////// + +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.FileSystem_Create, + new OperationBlobSASPermission(ContainerSASPermission.Create) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.FileSystem_SetProperties, + new OperationBlobSASPermission(ContainerSASPermission.Write) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.FileSystem_GetProperties, + new OperationBlobSASPermission(ContainerSASPermission.Read) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.FileSystem_ListPaths, + new OperationBlobSASPermission(ContainerSASPermission.List) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.FileSystem_ListBlobFlatSegment, + new OperationBlobSASPermission(ContainerSASPermission.List) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.FileSystem_ListBlobHierarchySegment, + new OperationBlobSASPermission(ContainerSASPermission.List) +); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.FileSystem_Delete, + new OperationBlobSASPermission(ContainerSASPermission.Delete) +); +//////////////////////////////////Service Operations////////////////////////////// + +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.Service_ListFileSystems, + new OperationBlobSASPermission() +); diff --git a/src/dfs/authentication/PublicAccessAuthenticator.ts b/src/dfs/authentication/PublicAccessAuthenticator.ts new file mode 100644 index 000000000..ad0686db7 --- /dev/null +++ b/src/dfs/authentication/PublicAccessAuthenticator.ts @@ -0,0 +1,156 @@ +import ILogger from "../../common/ILogger"; +import { PublicAccessType } from "../generated/artifacts/models"; +import Operation from "../generated/artifacts/operation"; +import Context from "../generated/Context"; +import IRequest from "../generated/IRequest"; +import IDataLakeMetadataStore from "../persistence/IDataLakeMetadataStore"; +import IAuthenticator from "./IAuthenticator"; + +const CONTAINER_PUBLIC_READ_OPERATIONS = new Set([ + //blob + Operation.Container_GetProperties, + Operation.Container_GetPropertiesWithHead, + Operation.Container_GetAccessPolicy, + Operation.PageBlob_GetPageRanges, // TODO: Not sure + Operation.PageBlob_GetPageRangesDiff, // TODO: Not sure + Operation.BlockBlob_GetBlockList, // TODO: Not sure + //DataLake + Operation.FileSystem_GetProperties, + Operation.FileSystem_ListBlobFlatSegment, + Operation.FileSystem_ListBlobHierarchySegment, + Operation.FileSystem_ListPaths, + Operation.Path_Read, + Operation.Path_GetProperties +]); + +const BLOB_PUBLIC_READ_OPERATIONS = new Set([ + //blob + Operation.PageBlob_GetPageRanges, // TODO: Not sure + Operation.PageBlob_GetPageRangesDiff, // TODO: Not sure + Operation.BlockBlob_GetBlockList, // TODO: Not sure + //DataLake + Operation.Path_Read, + Operation.Path_GetProperties +]); + +export default class PublicAccessAuthenticator implements IAuthenticator { + public constructor( + private readonly blobMetadataStore: IDataLakeMetadataStore, + private readonly logger: ILogger + ) {} + + public async validate( + req: IRequest, + context: Context + ): Promise { + this.logger.info( + `PublicAccessAuthenticator:validate() Start validation against public access.`, + context.contextId + ); + + this.logger.debug( + "PublicAccessAuthenticator:validate() Getting account properties...", + context.contextId + ); + + const account: string = context.context.account; + const containerName: string | undefined = context.context.container; + const blobName: string | undefined = context.context.blob; + this.logger.debug( + // tslint:disable-next-line:max-line-length + `PublicAccessAuthenticator:validate() Retrieved account name from context: ${account}, container: ${containerName}, blob: ${blobName}`, + context.contextId + ); + + if (containerName === undefined) { + this.logger.info( + // tslint:disable-next-line:max-line-length + `PublicAccessAuthenticator:validate() Skip public access authentication. Container name is undefined.`, + context.contextId + ); + return undefined; + } + + const containerPublicAccessType = await this.getContainerPublicAccessType( + account, + containerName, + context + ); + if (containerPublicAccessType === undefined) { + this.logger.debug( + // tslint:disable-next-line:max-line-length + `PublicAccessAuthenticator:validate() Skip public access authentication. Cannot get public access type for container ${containerName}`, + context.contextId + ); + return undefined; + } + this.logger.debug( + `PublicAccessAuthenticator:validate() Public access type for container ${containerName} is ${JSON.stringify( + containerPublicAccessType + )}`, + context.contextId + ); + + const operation = context.operation; + if (operation === undefined) { + throw new Error( + // tslint:disable-next-line:max-line-length + `PublicAccessAuthenticator:validate() Operation shouldn't be undefined. Please make sure DispatchMiddleware is hooked before authentication related middleware.` + ); + } + + if (containerPublicAccessType === PublicAccessType.Container) { + if (CONTAINER_PUBLIC_READ_OPERATIONS.has(operation)) { + this.logger.debug( + `PublicAccessAuthenticator:validate() Operation ${Operation[operation]} is in container level public access list. Validation passed.`, + context.contextId + ); + return true; + } + } else if (containerPublicAccessType === PublicAccessType.Blob) { + if (BLOB_PUBLIC_READ_OPERATIONS.has(operation)) { + this.logger.debug( + `PublicAccessAuthenticator:validate() Operation ${Operation[operation]} is in blob level public access list. Validation passed.`, + context.contextId + ); + return true; + } + } else { + throw Error( + `PublicAccessAuthenticator:validate() Unsupported containerPublicAccessType ${containerPublicAccessType}` + ); + } + + this.logger.debug( + `PublicAccessAuthenticator:validate() Operation ${Operation[operation]} is not in container neither blob level public access list. Validation failed.`, + context.contextId + ); + + // TODO: Make validate() return values into 3 candidates (true, false, and error thrown) + // True means validation passed + // False means validation doesn't match (due to not match this validation pattern), then goto next authenticator + // Error means validation failed, response will return error immediately + return undefined; + } + + private async getContainerPublicAccessType( + account: string, + container: string, + context: Context + ): Promise { + try { + const containerModel = await this.blobMetadataStore.getContainerACL( + context, + account, + container, + false + ); + if (containerModel === undefined) { + return undefined; + } + return containerModel.properties.publicAccess; + } catch (err) { + return undefined; + } + } +} diff --git a/src/dfs/conditions/ConditionResourceAdapter.ts b/src/dfs/conditions/ConditionResourceAdapter.ts new file mode 100644 index 000000000..0045cba86 --- /dev/null +++ b/src/dfs/conditions/ConditionResourceAdapter.ts @@ -0,0 +1,40 @@ +import { + BlobModel, + ContainerModel +} from "../persistence/IDataLakeMetadataStore"; +import IConditionResource from "./IConditionResource"; + +export default class ConditionResourceAdapter implements IConditionResource { + public exist: boolean; + public etag: string; + public lastModified: Date; + + public constructor(resource: BlobModel | ContainerModel | undefined | null) { + if ( + resource === undefined || + resource === null || + (resource as BlobModel).isCommitted === false // Treat uncommitted blob as unexist resource + ) { + this.exist = false; + this.etag = "UNEXIST_RESOURCE_ETAG"; + this.lastModified = undefined as any; + return; + } + + this.exist = true; + this.etag = resource.properties.etag; + + if (this.etag.length < 3) { + throw new Error( + `ConditionResourceAdapter::constructor() Invalid etag ${this.etag}.` + ); + } + + if (this.etag.startsWith('"') && this.etag.endsWith('"')) { + this.etag = this.etag.substring(1, this.etag.length - 1); + } + + this.lastModified = new Date(resource.properties.lastModified); + this.lastModified.setMilliseconds(0); // Precision to seconds + } +} diff --git a/src/dfs/conditions/ConditionalHeadersAdapter.ts b/src/dfs/conditions/ConditionalHeadersAdapter.ts new file mode 100644 index 000000000..f2479e1aa --- /dev/null +++ b/src/dfs/conditions/ConditionalHeadersAdapter.ts @@ -0,0 +1,47 @@ +import { ModifiedAccessConditions } from "../generated/artifacts/models"; +import Context from "../generated/Context"; +import { IConditionalHeaders } from "./IConditionalHeaders"; + +export default class ConditionalHeadersAdapter implements IConditionalHeaders { + public ifModifiedSince?: Date; + public ifUnmodifiedSince?: Date; + public ifMatch?: string[]; + public ifNoneMatch?: string[]; + + public constructor( + context: Context, + modifiedAccessConditions: ModifiedAccessConditions = {} + ) { + // If-Match & If-None-Match allow multi values separated by comma + if (modifiedAccessConditions.ifMatch) { + this.ifMatch = modifiedAccessConditions.ifMatch.split(",").map((etag) => { + if (etag.startsWith('"') && etag.endsWith('"')) { + return etag.substring(1, etag.length - 1); + } + return etag; + }); + } + + if (modifiedAccessConditions.ifNoneMatch) { + this.ifNoneMatch = modifiedAccessConditions.ifNoneMatch + .split(",") + .map((etag) => { + if (etag.startsWith('"') && etag.endsWith('"')) { + return etag.substring(1, etag.length - 1); + } + return etag; + }); + } + + // If-Modified-Since & If-Unmodified-Since don't support multi values + this.ifModifiedSince = modifiedAccessConditions.ifModifiedSince; + if (this.ifModifiedSince) { + this.ifModifiedSince.setMilliseconds(0); // Precision to seconds + } + + this.ifUnmodifiedSince = modifiedAccessConditions.ifUnmodifiedSince; + if (this.ifUnmodifiedSince) { + this.ifUnmodifiedSince.setMilliseconds(0); // Precision to seconds + } + } +} diff --git a/src/dfs/conditions/IConditionResource.ts b/src/dfs/conditions/IConditionResource.ts new file mode 100644 index 000000000..dda91509c --- /dev/null +++ b/src/dfs/conditions/IConditionResource.ts @@ -0,0 +1,16 @@ +export default interface IConditionResource { + /** + * Whether resource exists or not. + */ + exist: boolean; + + /** + * etag string without quotes. + */ + etag: string; + + /** + * last modified time for container or blob. + */ + lastModified: Date; +} diff --git a/src/dfs/conditions/IConditionalHeaders.ts b/src/dfs/conditions/IConditionalHeaders.ts new file mode 100644 index 000000000..1749fca54 --- /dev/null +++ b/src/dfs/conditions/IConditionalHeaders.ts @@ -0,0 +1,14 @@ +export interface IConditionalHeaders { + ifModifiedSince?: Date; + ifUnmodifiedSince?: Date; + + /** + * If-Match etag list without quotes. + */ + ifMatch?: string[]; + + /** + * If-None-Match etag list without quotes. + */ + ifNoneMatch?: string[]; +} diff --git a/src/dfs/conditions/IConditionalHeadersValidator.ts b/src/dfs/conditions/IConditionalHeadersValidator.ts new file mode 100644 index 000000000..7a8c04431 --- /dev/null +++ b/src/dfs/conditions/IConditionalHeadersValidator.ts @@ -0,0 +1,11 @@ +import Context from "../generated/Context"; +import { IConditionalHeaders } from "./IConditionalHeaders"; +import IConditionResource from "./IConditionResource"; + +export interface IConditionalHeadersValidator { + validate( + context: Context, + conditionalHeaders: IConditionalHeaders, + resource: IConditionResource + ): void; +} diff --git a/src/dfs/conditions/ReadConditionalHeadersValidator.ts b/src/dfs/conditions/ReadConditionalHeadersValidator.ts new file mode 100644 index 000000000..f8d432659 --- /dev/null +++ b/src/dfs/conditions/ReadConditionalHeadersValidator.ts @@ -0,0 +1,116 @@ +import StorageErrorFactory from "../errors/StorageErrorFactory"; +import { ModifiedAccessConditions } from "../generated/artifacts/models"; +import Context from "../generated/Context"; +import { + BlobModel, + ContainerModel +} from "../persistence/IDataLakeMetadataStore"; +import ConditionalHeadersAdapter from "./ConditionalHeadersAdapter"; +import ConditionResourceAdapter from "./ConditionResourceAdapter"; +import { IConditionalHeaders } from "./IConditionalHeaders"; +import { IConditionalHeadersValidator } from "./IConditionalHeadersValidator"; +import IConditionResource from "./IConditionResource"; + +export function validateReadConditions( + context: Context, + conditionalHeaders?: ModifiedAccessConditions, + model?: BlobModel | ContainerModel | null +) { + new ReadConditionalHeadersValidator().validate( + context, + new ConditionalHeadersAdapter(context, conditionalHeaders), + new ConditionResourceAdapter(model) + ); +} + +// tslint:disable: max-line-length +export default class ReadConditionalHeadersValidator + implements IConditionalHeadersValidator +{ + /** + * Validate Conditional Headers for Blob Service Read Operations in Version 2013-08-15 or Later. + * @link https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations#specifying-conditional-headers-for-blob-service-read-operations-in-version-2013-08-15-or-later + * + * @param context + * @param conditionalHeaders + * @param resource + */ + public validate( + context: Context, + conditionalHeaders: IConditionalHeaders, + resource: IConditionResource + ): void { + // If-Match && If-Unmodified-Since && (If-None-Match || If-Modified-Since) + + // Read against a non exist resource + if (!resource.exist) { + // If-Match + if (conditionalHeaders.ifMatch && conditionalHeaders.ifMatch.length > 0) { + throw StorageErrorFactory.getConditionNotMet(context); + } + + // If If-Unmodified-Since + // Skip for unexist resource + + // If-None-Match + if ( + conditionalHeaders.ifNoneMatch && + conditionalHeaders.ifNoneMatch.length > 0 && + conditionalHeaders.ifNoneMatch[0] === "*" + ) { + throw StorageErrorFactory.getUnsatisfiableCondition(context); + } + + // If-Modified-Since + // Skip for unexist resource + } else { + // Read against an existing resource + // If-Match && If-Unmodified-Since && (If-None-Match || If-Modified-Since) + + // If-Match + const ifMatchPass = conditionalHeaders.ifMatch + ? conditionalHeaders.ifMatch.includes(resource.etag) || + conditionalHeaders.ifMatch[0] === "*" + : undefined; + + // If-Unmodified-Since + const ifUnModifiedSincePass = conditionalHeaders.ifUnmodifiedSince + ? resource.lastModified <= conditionalHeaders.ifUnmodifiedSince + : undefined; + + // If-None-Match + if ( + conditionalHeaders.ifNoneMatch && + conditionalHeaders.ifNoneMatch.length > 0 && + conditionalHeaders.ifNoneMatch[0] === "*" + ) { + throw StorageErrorFactory.getUnsatisfiableCondition(context); + } + + const ifNoneMatchPass = conditionalHeaders.ifNoneMatch + ? !conditionalHeaders.ifNoneMatch.includes(resource.etag) + : undefined; + + // If-Modified-Since + const isModifiedSincePass = conditionalHeaders.ifModifiedSince + ? conditionalHeaders.ifModifiedSince < resource.lastModified + : undefined; + + if (ifMatchPass === false) { + throw StorageErrorFactory.getConditionNotMet(context); + } + + if (ifUnModifiedSincePass === false) { + throw StorageErrorFactory.getConditionNotMet(context); + } + + if (ifNoneMatchPass === false && isModifiedSincePass !== true) { + throw StorageErrorFactory.getNotModified(context); + } + + if (isModifiedSincePass === false && ifNoneMatchPass !== true) { + throw StorageErrorFactory.getNotModified(context); + } + } + } +} diff --git a/src/dfs/conditions/WriteConditionalHeadersValidator.ts b/src/dfs/conditions/WriteConditionalHeadersValidator.ts new file mode 100644 index 000000000..0cb36e420 --- /dev/null +++ b/src/dfs/conditions/WriteConditionalHeadersValidator.ts @@ -0,0 +1,233 @@ +import StorageErrorFactory from "../errors/StorageErrorFactory"; +import { + ModifiedAccessConditions, + SequenceNumberAccessConditions +} from "../generated/artifacts/models"; +import Context from "../generated/Context"; +import { + BlobModel, + ContainerModel +} from "../persistence/IDataLakeMetadataStore"; +import ConditionalHeadersAdapter from "./ConditionalHeadersAdapter"; +import ConditionResourceAdapter from "./ConditionResourceAdapter"; +import { IConditionalHeaders } from "./IConditionalHeaders"; +import { IConditionalHeadersValidator } from "./IConditionalHeadersValidator"; +import IConditionResource from "./IConditionResource"; + +export function validateSequenceNumberWriteConditions( + context: Context, + conditionalHeaders?: SequenceNumberAccessConditions, + model?: BlobModel +) { + if (!conditionalHeaders || !model) { + return; + } + + if (!model.properties || model.properties.blobSequenceNumber === undefined) { + throw Error( + `validateSequenceNumberWriteConditions() Invalid blob model, blobSequenceNumber is not specified.` + ); + } + + if ( + conditionalHeaders.ifSequenceNumberLessThanOrEqualTo !== undefined && + conditionalHeaders.ifSequenceNumberLessThanOrEqualTo < + model.properties.blobSequenceNumber + ) { + throw StorageErrorFactory.getSequenceNumberConditionNotMet(context); + } + + if ( + conditionalHeaders.ifSequenceNumberLessThan !== undefined && + conditionalHeaders.ifSequenceNumberLessThan <= + model.properties.blobSequenceNumber + ) { + throw StorageErrorFactory.getSequenceNumberConditionNotMet(context); + } + + if ( + conditionalHeaders.ifSequenceNumberEqualTo !== undefined && + conditionalHeaders.ifSequenceNumberEqualTo !== + model.properties.blobSequenceNumber + ) { + throw StorageErrorFactory.getSequenceNumberConditionNotMet(context); + } +} + +export function validateWriteConditions( + context: Context, + conditionalHeaders?: ModifiedAccessConditions, + model?: BlobModel | ContainerModel | null +) { + new WriteConditionalHeadersValidator().validate( + context, + new ConditionalHeadersAdapter(context, conditionalHeaders), + new ConditionResourceAdapter(model) + ); +} + +// tslint:disable: max-line-length +export default class WriteConditionalHeadersValidator + implements IConditionalHeadersValidator +{ + /** + * Validate conditional Headers for Read Operations in Versions Prior to 2013-08-15, + * and for Write Operations (All Versions). + * @link https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations#specifying-conditional-headers-for-read-operations-in-versions-prior-to-2013-08-15-and-for-write-operations-all-versions + * + * @param context + * @param conditionalHeaders + * @param resource + */ + public validate( + context: Context, + conditionalHeaders: IConditionalHeaders, + resource: IConditionResource + ): void { + this.validateCombinations(context, conditionalHeaders); + if (!resource.exist) { + if ( + conditionalHeaders.ifNoneMatch && + conditionalHeaders.ifNoneMatch.length > 0 + ) { + // If a request specifies both the If-None-Match and If-Modified-Since headers, + // the request is evaluated based on the criteria specified in If-None-Match. + // Skip for non exist blob + return; + } + + if (conditionalHeaders.ifMatch && conditionalHeaders.ifMatch.length > 0) { + // If a request specifies both the If-Match and If-Unmodified-Since headers, + // the request is evaluated based on the criteria specified in If-Match. + if (conditionalHeaders.ifMatch[0] !== "*") { + throw StorageErrorFactory.getConditionNotMet(context); + } + return; + } + + if (conditionalHeaders.ifModifiedSince) { + // Skip for non exist blob + return; + } + + if (conditionalHeaders.ifUnmodifiedSince) { + // Skip for non exist blob + return; + } + } else { + if ( + conditionalHeaders.ifNoneMatch && + conditionalHeaders.ifNoneMatch.length > 0 + ) { + if (conditionalHeaders.ifNoneMatch[0] === "*") { + // According to restful doc, specify the wildcard character (*) to perform the operation + // only if the resource does not exist, and fail the operation if it does exist. + // However, Azure Storage Set Blob Properties Operation for an existing blob doesn't reuturn 412 with * + // TODO: Check accurate behavior for different write operations + // Put Blob, Commit Block List has special logic for ifNoneMatch equals *, will return 409 conflict for existing blob, will handled in createBlob metatdata store. + // throw StorageErrorFactory.getConditionNotMet(context); + return; + } + if (conditionalHeaders.ifNoneMatch[0] === resource.etag) { + throw StorageErrorFactory.getConditionNotMet(context); + } + + // Stop processing + // If a request specifies both the If-None-Match and If-Modified-Since headers, + // the request is evaluated based on the criteria specified in If-None-Match. + return; + } + + if (conditionalHeaders.ifMatch && conditionalHeaders.ifMatch.length > 0) { + if ( + conditionalHeaders.ifMatch[0] !== "*" && + conditionalHeaders.ifMatch[0] !== resource.etag + ) { + throw StorageErrorFactory.getConditionNotMet(context); + } + + // Stop processing + // If a request specifies both the If-Match and If-Unmodified-Since headers, + // the request is evaluated based on the criteria specified in If-Match. + return; + } + + if (conditionalHeaders.ifModifiedSince) { + if (resource.lastModified <= conditionalHeaders.ifModifiedSince) { + throw StorageErrorFactory.getConditionNotMet(context); + } + return; + } + + if (conditionalHeaders.ifUnmodifiedSince) { + if (conditionalHeaders.ifUnmodifiedSince < resource.lastModified) { + throw StorageErrorFactory.getConditionNotMet(context); + } + return; + } + } + } + + private validateCombinations( + context: Context, + conditionalHeaders: IConditionalHeaders + ) { + let ifMatch = 0; + if (conditionalHeaders.ifMatch && conditionalHeaders.ifMatch.length > 0) { + // RFC 2616 allows multiple ETag values in a single header, + // but requests to the Blob service can only include one ETag value. + // Specifying more than one ETag value results in status code 400 (Bad Request). + if (conditionalHeaders.ifMatch.length > 1) { + // throw 400 MultipleConditionHeadersNotSupported Multiple condition headers are not supported. + throw StorageErrorFactory.getMultipleConditionHeadersNotSupported( + context + ); + } + + ifMatch = 1; + } + + let ifModifiedSince = 0; + if (conditionalHeaders.ifModifiedSince) { + ifModifiedSince = 1; + } + + let ifNoneMatch = 0; + if ( + conditionalHeaders.ifNoneMatch && + conditionalHeaders.ifNoneMatch.length > 0 + ) { + // RFC 2616 allows multiple ETag values in a single header, + // but requests to the Blob service can only include one ETag value. + // Specifying more than one ETag value results in status code 400 (Bad Request). + if (conditionalHeaders.ifNoneMatch.length > 1) { + // throw 400 MultipleConditionHeadersNotSupported Multiple condition headers are not supported. + throw StorageErrorFactory.getMultipleConditionHeadersNotSupported( + context + ); + } + ifNoneMatch = 1; + } + + let ifUnmodifiedSince = 0; + if (conditionalHeaders.ifUnmodifiedSince) { + ifUnmodifiedSince = 1; + } + + if (ifMatch + ifModifiedSince + ifNoneMatch + ifUnmodifiedSince > 2) { + // throw 400 MultipleConditionHeadersNotSupported Multiple condition headers are not supported. + throw StorageErrorFactory.getMultipleConditionHeadersNotSupported( + context + ); + } + + if (ifMatch + ifModifiedSince + ifNoneMatch + ifUnmodifiedSince === 2) { + if (ifNoneMatch + ifModifiedSince === 1) { + // throw 400 MultipleConditionHeadersNotSupported Multiple condition headers are not supported. + throw StorageErrorFactory.getMultipleConditionHeadersNotSupported( + context + ); + } + } + } +} diff --git a/src/dfs/context/DataLakeContext.ts b/src/dfs/context/DataLakeContext.ts new file mode 100644 index 000000000..6e54ae385 --- /dev/null +++ b/src/dfs/context/DataLakeContext.ts @@ -0,0 +1,85 @@ +import IAuthenticationContext from "../authentication/IAuthenticationContext"; +import Context from "../generated/Context"; + +export default class DataLakeContext + extends Context + implements IAuthenticationContext +{ + public getContainer(): string | undefined { + return this.context.container; + } + + public get account(): string | undefined { + return this.context.account; + } + + public set account(account: string | undefined) { + this.context.account = account; + } + + public set isSecondary(isSecondary: boolean | undefined) { + this.context.isSecondary = isSecondary; + } + + public get isSecondary(): boolean | undefined { + return this.context.isSecondary; + } + + public get container(): string | undefined { + return this.context.container; + } + + public set container(container: string | undefined) { + this.context.container = container; + } + + public get blob(): string | undefined { + return this.context.blob; + } + + public set blob(blob: string | undefined) { + this.context.blob = blob; + } + + public get originalBlob(): string | undefined { + return this.context.originalBlob; + } + + public set originalBlob(originalBlob: string | undefined) { + this.context.originalBlob = originalBlob; + } + + public get authenticationPath(): string | undefined { + return this.context.authenticationPath; + } + + public set authenticationPath(path: string | undefined) { + this.context.authenticationPath = path; + } + + public get xMsRequestID(): string | undefined { + return this.contextId; + } + + public set xMsRequestID(xMsRequestID: string | undefined) { + this.contextId = xMsRequestID; + } + + public get disableProductStyleUrl(): boolean | undefined { + return this.context.disableProductStyleUrl; + } + + public set disableProductStyleUrl( + disableProductStyleUrl: boolean | undefined + ) { + this.context.disableProductStyleUrl = disableProductStyleUrl; + } + + public get loose(): boolean | undefined { + return this.context.loose; + } + + public set loose(loose: boolean | undefined) { + this.context.loose = loose; + } +} diff --git a/src/dfs/errors/DataLakeError.ts b/src/dfs/errors/DataLakeError.ts new file mode 100644 index 000000000..76a9f3d67 --- /dev/null +++ b/src/dfs/errors/DataLakeError.ts @@ -0,0 +1,92 @@ +import Context from "../generated/Context"; +import MiddlewareError from "../generated/errors/MiddlewareError"; +import { jsonToXML } from "../generated/utils/xml"; +import { isDataLakeOperation } from "../utils/utils"; + +/** + * Represents an Azure Storage Server Error. + * + * @export + * @class StorageError + * @extends {MiddlewareError} + */ +export default class DataLakeError extends MiddlewareError { + public readonly errorCode: string; + public readonly errorMessage: string; + public readonly storageRequestID: string | undefined; + + /**isDataLakeOperation(context.operation!); + * Creates an instance of StorageError. + * + * @param {number} statusCode HTTP response status code + * @param {string} dataLakeErrorCode Azure DataLake error code, will be in response body and header + * @param {string} dataLakeErrorMessage Azure DataLake error message + * @param {string} blobErrorCode Azure Storage error code, will be in response body and header + * @param {string} blobErrorMessage Azure Storage error message + * @param {string} context The request Context + * @param {{ [key: string]: string }} [storageAdditionalErrorMessages={}] + * Additional error messages will be included in XML body + * @memberof StorageError + */ + constructor( + statusCode: number, + dataLakeErrorCode: string, + dataLakeErrorMessage: string, + blobErrorCode: string, + blobErrorMessage: string, + context: Context, + storageAdditionalErrorMessages: { [key: string]: string } = {} + ) { + const isDataLake = isDataLakeOperation(context); + const code = isDataLake ? dataLakeErrorCode : blobErrorCode; + const message = isDataLake ? dataLakeErrorMessage : blobErrorMessage; + const storageRequestID = context.contextId; + + let bodyInJSON: any = { + code, + message: `${message}\nRequestId:${storageRequestID}\nTime:${new Date().toISOString()}` + }; + + for (const key in storageAdditionalErrorMessages) { + if (storageAdditionalErrorMessages.hasOwnProperty(key)) { + const element = storageAdditionalErrorMessages[key]; + bodyInJSON[key] = element; + } + } + + bodyInJSON = { + message: `${message}\nRequestId:${storageRequestID}\nTime:${new Date().toISOString()}`, + code, + errorCode: code, + error: bodyInJSON + }; + + for (const key in storageAdditionalErrorMessages) { + if (storageAdditionalErrorMessages.hasOwnProperty(key)) { + const element = storageAdditionalErrorMessages[key]; + bodyInJSON[key] = element; + } + } + + const bodyInXML = jsonToXML(bodyInJSON); + + super( + statusCode, + message, + code, + { + "x-ms-error-code": dataLakeErrorCode, + "x-ms-request-id": storageRequestID + }, + // bodyInJSON, + // "application/json" + bodyInXML, + "application/xml" + ); + + this.name = "StorageError"; + this.errorCode = code; + this.errorMessage = message; + this.storageRequestID = storageRequestID; + } +} diff --git a/src/dfs/errors/NotImplementedError.ts b/src/dfs/errors/NotImplementedError.ts new file mode 100644 index 000000000..2b4f28e76 --- /dev/null +++ b/src/dfs/errors/NotImplementedError.ts @@ -0,0 +1,22 @@ +import Context from "../generated/Context"; +import DataLakeError from "./DataLakeError"; + +/** + * Create customized error types by inheriting ServerError + * + * @export + * @class UnimplementedError + * @extends {DataLakeError} + */ +export default class NotImplementedError extends DataLakeError { + public constructor(context: Context) { + super( + 500, + "APINotImplemented", + "Current API is not implemented yet. Please vote your wanted features to https://github.com/azure/azurite/issues", + "APINotImplemented", + "Current API is not implemented yet. Please vote your wanted features to https://github.com/azure/azurite/issues", + context + ); + } +} diff --git a/src/dfs/errors/StorageErrorFactory.ts b/src/dfs/errors/StorageErrorFactory.ts new file mode 100644 index 000000000..6b2eeca95 --- /dev/null +++ b/src/dfs/errors/StorageErrorFactory.ts @@ -0,0 +1,885 @@ +import Context from "../generated/Context"; +import DataLakeError from "./DataLakeError"; + +/** + * A factory class maintains all Azure Storage Blob service errors. + * + * @export + * @class DataLakeErrorFactory + */ +export default class DataLakeErrorFactory { + public static getContainerNotFound(context: Context): DataLakeError { + return new DataLakeError( + 404, + "FilesystemNotFound", + "The specified filesystem does not exist.", + "ContainerNotFound", + "The specified container does not exist.", + context + ); + } + + public static getRequestEntityTooLarge(context: Context): DataLakeError { + return new DataLakeError( + 413, + "RequestBodyTooLarge", + "The request body is too large and exceeds the maximum permissible limit", + "RequestEntityTooLarge", + "The uploaded entity blob is too large.", + context + ); + } + + //TODO: check code/message for datalake + public static getBlockCountExceedsLimit(context: Context): DataLakeError { + return new DataLakeError( + 409, + "BlockCountExceedsLimit", + "The committed block count cannot exceed the maximum limit of 50,000 blocks.", + "BlockCountExceedsLimit", + "The committed block count cannot exceed the maximum limit of 50,000 blocks.", + context + ); + } + + public static getContainerAlreadyExists(context: Context): DataLakeError { + return new DataLakeError( + 409, + "FilesystemAlreadyExists", + "he specified filesystem already exists.", + "ContainerAlreadyExists", + "The specified container already exists.", + context + ); + } + + public static getBlobAlreadyExists(context: Context): DataLakeError { + return new DataLakeError( + 409, + "PathAlreadyExists", + "The specified path already exists.", + "BlobAlreadyExists", + "The specified blob already exists.", + context + ); + } + + public static getBlobNotFound(context: Context): DataLakeError { + return new DataLakeError( + 404, + "PathNotFound", + "The specified path does not exist.", + "BlobNotFound", + "The specified blob does not exist.", + context + ); + } + + public static ResourceNotFound(context: Context): DataLakeError { + return new DataLakeError( + 404, + "ResourceNotFound", + "The specified resource does not exist.", + "ResourceNotFound", + "The specified resource does not exist.", + context + ); + } + + public static getInvalidQueryParameterValue( + context: Context, + parameterName?: string, + parameterValue?: string, + reason?: string + ): DataLakeError { + const additionalMessages: { + [key: string]: string; + } = {}; + + if (parameterName) { + additionalMessages.QueryParameterName = parameterName; + } + + if (parameterValue) { + additionalMessages.QueryParameterValue = parameterValue; + } + + if (reason) { + additionalMessages.Reason = reason; + } + + return new DataLakeError( + 400, + "InvalidQueryParameterValue", + "Value for one of the query parameters specified in the request URI is invalid.", + "InvalidQueryParameterValue", + `Value for one of the query parameters specified in the request URI is invalid.`, + context, + additionalMessages + ); + } + + public static getOutOfRangeInput( + context: Context, + parameterName?: string, + parameterValue?: string, + reason?: string + ): DataLakeError { + const additionalMessages: { + [key: string]: string; + } = {}; + + if (parameterName) { + additionalMessages.QueryParameterName = parameterName; + } + + if (parameterValue) { + additionalMessages.QueryParameterValue = parameterValue; + } + + if (reason) { + additionalMessages.Reason = reason; + } + + return new DataLakeError( + 400, + "OutOfRangeInput", + `One of the request inputs is out of range.`, + "OutOfRangeInput", + `One of the request inputs is out of range.`, + context, + additionalMessages + ); + } + + //TODO: get the right code/message for datalake + public static getInvalidOperation( + context: Context, + message: string = "" + ): DataLakeError { + return new DataLakeError( + 400, + "InvalidOperation", + message, + "InvalidOperation", + message, + context + ); + } + + //No Equivalent in DataLake + public static getInvalidBlockList(context: Context): DataLakeError { + return new DataLakeError( + 400, + "InvalidBlockList", + "The specified block list is invalid.", + "InvalidBlockList", + "The specified block list is invalid.", + context + ); + } + + public static getInvalidAuthenticationInfo(context: Context): DataLakeError { + return new DataLakeError( + 400, + "InvalidAuthenticationInfo", + "Authentication information is not given in the correct format. Check the value of Authorization header.", + "InvalidAuthenticationInfo", + "Authentication information is not given in the correct format. Check the value of Authorization header.", + context + ); + } + + //TODO: check the right code/message for datalake + public static getMd5Mismatch( + context: Context, + userSpecifiedMd5: string, + serverCalculatedMd5: string + ): DataLakeError { + return new DataLakeError( + 400, + "Md5Mismatch", + "The MD5 value specified in the request did not match with the MD5 value calculated by the server.", + "Md5Mismatch", + "The MD5 value specified in the request did not match with the MD5 value calculated by the server.", + context, + { + UserSpecifiedMd5: userSpecifiedMd5, + ServerCalculatedMd5: serverCalculatedMd5 + } + ); + } + + public static getInvalidPageRange(context: Context): DataLakeError { + return new DataLakeError( + 416, + "InvalidRange", + "The page range specified is invalid.", + "InvalidRange", + "The page range specified is invalid.", + context + ); + } + + public static getInvalidLeaseDuration(context: Context): DataLakeError { + return this.getInvalidHeaderValue(context); + } + + public static getInvalidLeaseBreakPeriod(context: Context): DataLakeError { + return this.getInvalidHeaderValue(context); + } + + public static getInvalidId(context: Context): DataLakeError { + return this.getInvalidHeaderValue(context); + } + + //No equivalent in DataLake + public static getInvalidBlobOrBlock(context: Context): DataLakeError { + return new DataLakeError( + 400, + "InvalidBlobOrBlock", + "The specified blob or block content is invalid.", + "InvalidBlobOrBlock", + "The specified blob or block content is invalid.", + context + ); + } + + public static getLeaseAlreadyPresent(context: Context): DataLakeError { + return new DataLakeError( + 409, + "LeaseAlreadyPresent", + "There is already a lease present.", + "LeaseAlreadyPresent", + "There is already a lease present.", + context + ); + } + + public static getLeaseIsBreakingAndCannotBeAcquired( + context: Context + ): DataLakeError { + return new DataLakeError( + 409, + "LeaseIsBreakingAndCannotBeAcquired", + "The lease ID matched, but the lease is currently in breaking state and cannot be acquired until it is broken.", + "LeaseIsBreakingAndCannotBeAcquired", + "There is already a breaking lease, and can't be acquired.", + context + ); + } + + public static getLeaseNotPresentWithLeaseOperation( + context: Context + ): DataLakeError { + return new DataLakeError( + 409, + "LeaseNotPresentWithLeaseOperation", + "The lease ID is not present with the specified lease operation.", + "LeaseNotPresentWithLeaseOperation", + "There is currently no lease on the container or blob.", + context + ); + } + + public static getMissingRequestHeader(context: Context): DataLakeError { + return new DataLakeError( + 400, + "MissingRequiredHeader", + "An HTTP header that's mandatory for this request is not specified.", + "MissingRequiredHeader", + "An HTTP header that's mandatory for this request is not specified.", + context + ); + } + + public static getLeaseIdMismatchWithLeaseOperation( + context: Context + ): DataLakeError { + return new DataLakeError( + 409, + "LeaseIdMismatchWithLeaseOperation", + "The lease ID specified did not match the lease ID for the resource with the specified lease operation.", + "LeaseIdMismatchWithLeaseOperation", + "The lease ID specified did not match the lease ID for the container or blob.", + context + ); + } + + public static getLeaseIsBrokenAndCannotBeRenewed( + context: Context + ): DataLakeError { + return new DataLakeError( + 409, + "LeaseIsBrokenAndCannotBeRenewed", + "The lease ID matched, but the lease has been broken explicitly and cannot be renewed.", + "LeaseIsBrokenAndCannotBeRenewed", + "The lease ID matched, but the lease has been broken explicitly and cannot be renewed.", + context + ); + } + + public static getLeaseIsBreakingAndCannotBeChanged( + context: Context + ): DataLakeError { + return new DataLakeError( + 409, + "LeaseIsBreakingAndCannotBeChanged", + "The lease ID matched, but the lease is currently in breaking state and cannot be changed.", + "LeaseIsBreakingAndCannotBeChanged", + "The lease ID matched, but the lease is currently in breaking state and cannot be changed.", + context + ); + } + + public static getContainerLeaseIdMissing(context: Context): DataLakeError { + return new DataLakeError( + 412, + "LeaseIdMissing", + "There is currently a lease on the resource and no lease ID was specified in the request.", + "LeaseIdMissing", + "There is currently a lease on the container and no lease ID was specified in the request.", + context + ); + } + + public static getContainerLeaseIdMismatchWithContainerOperation( + context: Context + ): DataLakeError { + return new DataLakeError( + 412, + "LeaseIdMismatchWithLeaseOperation", + "The lease ID specified did not match the lease ID for the resource with the specified lease operation.", + "LeaseIdMismatchWithContainerOperation", + "The lease ID specified did not match the lease ID for the container.", + context + ); + } + + public static getContainerLeaseLost(context: Context): DataLakeError { + return new DataLakeError( + 412, + "LeaseNotPresentWithLeaseOperation", + "The lease ID is not present with the specified lease operation.", + "LeaseNotPresentWithContainerOperation", + "A lease ID was specified, but the lease for the container has expired.", + context + ); + } + + public static getBlobLeaseIdMismatchWithLeaseOperation( + context: Context + ): DataLakeError { + return new DataLakeError( + 409, + "LeaseIdMismatchWithLeaseOperation", + "The lease ID specified did not match the lease ID for the resource with the specified lease operation.", + "LeaseIdMismatchWithLeaseOperation", + "The lease ID specified did not match the lease ID for the blob.", + context + ); + } + + public static getBlobLeaseNotPresentWithLeaseOperation( + context: Context + ): DataLakeError { + return new DataLakeError( + 409, + "LeaseNotPresentWithLeaseOperation", + "The lease ID is not present with the specified lease operation.", + "LeaseNotPresentWithLeaseOperation", + "There is currently no lease on the blob.", + context + ); + } + + // The error code/message need check with server + public static getBlobSnapshotsPresent(context: Context): DataLakeError { + return new DataLakeError( + 400, + "SnapshotsPresent", + "This operation is not permitted because the blob is snapshot.", + "SnapshotsPresent", + "This operation is not permitted because the blob is snapshot.", + context + ); + } + + public static getBlobLeaseIdMissing(context: Context): DataLakeError { + return new DataLakeError( + 412, + "LeaseIdMissing", + "There is currently a lease on the resource and no lease ID was specified in the request.", + "LeaseIdMissing", + "There is currently a lease on the blob and no lease ID was specified in the request.", + context + ); + } + + public static getBlobLeaseIdMismatchWithBlobOperation( + context: Context + ): DataLakeError { + return new DataLakeError( + 412, + "LeaseIdMismatchWithLeaseOperation", + "The lease ID specified did not match the lease ID for the resource with the specified lease operation.", + "LeaseIdMismatchWithBlobOperation", + "The lease ID specified did not match the lease ID for the blob.", + context + ); + } + + public static getBlobLeaseLost(context: Context): DataLakeError { + return new DataLakeError( + 412, + "LeaseNotPresentWithLeaseOperation", + "The lease ID is not present with the specified lease operation.", + "LeaseNotPresentWithBlobOperation", + "A lease ID was specified, but the lease for the blob has expired.", + context + ); + } + + public static getAuthorizationFailure(context: Context): DataLakeError { + return new DataLakeError( + 403, + "AuthorizationFailure", + "Server failed to authenticate the request. Make sure the value of the Authorization header is formed correctly including the signature.", + "AuthorizationFailure", + // tslint:disable-next-line:max-line-length + "Server failed to authenticate the request. Make sure the value of the Authorization header is formed correctly including the signature.", + context + ); + } + + public static getAuthenticationFailed( + context: Context, + authenticationErrorDetail: string + ): DataLakeError { + return new DataLakeError( + 403, + "AuthorizationFailure", + "Server failed to authenticate the request. Make sure the value of the Authorization header is formed correctly including the signature.", + "AuthenticationFailed", + "Server failed to authenticate the request. Make sure the value of the Authorization header is formed correctly including the signature.", + context, + { + AuthenticationErrorDetail: authenticationErrorDetail + } + ); + } + + public static getBlobInvalidBlobType(context: Context): DataLakeError { + return new DataLakeError( + 409, + "InvalidBlobType", + "The blob type is invalid for this operation.", + "InvalidBlobType", + "The blob type is invalid for this operation.", + context + ); + } + + public static getAccessTierNotSupportedForBlobType( + context: Context + ): DataLakeError { + return new DataLakeError( + 400, + "AccessTierNotSupportedForBlobType", + "The access tier is not supported for this blob type.", + "AccessTierNotSupportedForBlobType", + "The access tier is not supported for this blob type.", + context + ); + } + + public static getMultipleConditionHeadersNotSupported( + context: Context + ): DataLakeError { + return new DataLakeError( + 400, + "MultipleConditionHeadersNotSupported", + "Multiple condition headers are not supported.", + "MultipleConditionHeadersNotSupported", + "Multiple condition headers are not supported.", + context + ); + } + + public static getBlobSnapshotsPresent_hassnapshot( + context: Context + ): DataLakeError { + return new DataLakeError( + 409, + "SnapshotsPresent", + "This operation is not permitted because the blob has snapshots.", + "SnapshotsPresent", + "This operation is not permitted because the blob has snapshots.", + context + ); + } + + public static getBlobCannotChangeToLowerTier( + context: Context + ): DataLakeError { + return new DataLakeError( + 409, + "CannotChangeToLowerTier", + "A higher blob tier has already been explicitly set.", + "CannotChangeToLowerTier", + "A higher blob tier has already been explicitly set.", + context + ); + } + + public static getBlobBlobTierInadequateForContentLength( + context: Context + ): DataLakeError { + return new DataLakeError( + 409, + "BlobTierInadequateForContentLength", + "Specified blob tier size limit cannot be less than content length.", + "BlobTierInadequateForContentLength", + "Specified blob tier size limit cannot be less than content length.", + context + ); + } + + public static getAuthorizationSourceIPMismatch( + context: Context + ): DataLakeError { + return new DataLakeError( + 403, + "AuthorizationSourceIPMismatch", + "This request is not authorized to perform this operation using this source IP {SourceIP}.", + "AuthorizationSourceIPMismatch", + "This request is not authorized to perform this operation using this source IP {SourceIP}.", + context + ); + } + + public static getAuthorizationProtocolMismatch( + context: Context + ): DataLakeError { + return new DataLakeError( + 403, + "AuthorizationProtocolMismatch", + "This request is not authorized to perform this operation using this protocol.", + "AuthorizationProtocolMismatch", + "This request is not authorized to perform this operation using this protocol.", + context + ); + } + + public static getAuthorizationPermissionMismatch( + context: Context + ): DataLakeError { + return new DataLakeError( + 403, + "AuthorizationPermissionMismatch", + "This request is not authorized to perform this operation using this permission.", + "AuthorizationPermissionMismatch", + "This request is not authorized to perform this operation using this permission.", + context + ); + } + + public static getAuthorizationServiceMismatch( + context: Context + ): DataLakeError { + return new DataLakeError( + 403, + "AuthorizationServiceMismatch", + "This request is not authorized to perform this operation using this service.", + "AuthorizationServiceMismatch", + "This request is not authorized to perform this operation using this service.", + context + ); + } + + public static getAuthorizationResourceTypeMismatch( + context: Context + ): DataLakeError { + return new DataLakeError( + 403, + "AuthorizationResourceTypeMismatch", + "This request is not authorized to perform this operation using this resource type.", + "AuthorizationResourceTypeMismatch", + "This request is not authorized to perform this operation using this resource type.", + context + ); + } + + public static getFeatureVersionMismatch(context: Context): DataLakeError { + return new DataLakeError( + 409, + "FeatureVersionMismatch", + "Stored access policy contains a permission that is not supported by this version.", + "FeatureVersionMismatch", + "Stored access policy contains a permission that is not supported by this version.", + context + ); + } + + public static getCopyIdMismatch(context: Context): DataLakeError { + return new DataLakeError( + 409, + "CopyIdMismatch", + "The specified copy ID did not match the copy ID for the pending copy operation.", + "CopyIdMismatch", + "The specified copy ID did not match the copy ID for the pending copy operation.", + context + ); + } + + public static getNoPendingCopyOperation(context: Context): DataLakeError { + return new DataLakeError( + 409, + "NoPendingCopyOperation", + "There is currently no pending copy operation.", + "NoPendingCopyOperation", + "There is currently no pending copy operation.", + context + ); + } + + public static getSnapshotsPresent(context: Context): DataLakeError { + return new DataLakeError( + 409, + "SnapshotsPresent", + "This operation is not permitted while the blob has snapshots.", + "SnapshotsPresent", + "This operation is not permitted while the blob has snapshots.", + context + ); + } + + public static getConditionNotMet(context: Context): DataLakeError { + return new DataLakeError( + 412, + "ConditionNotMet", + "The condition specified using HTTP conditional header(s) is not met.", + "ConditionNotMet", + "The condition specified using HTTP conditional header(s) is not met.", + context + ); + } + + public static getMaxBlobSizeConditionNotMet(context: Context): DataLakeError { + return new DataLakeError( + 412, + "MaxBlobSizeConditionNotMet", + "The max blob size condition specified was not met.", + "MaxBlobSizeConditionNotMet", + "The max blob size condition specified was not met.", + context + ); + } + + public static getAppendPositionConditionNotMet( + context: Context + ): DataLakeError { + return new DataLakeError( + 412, + "AppendPositionConditionNotMet", + "The append position condition specified was not met.", + "AppendPositionConditionNotMet", + "The append position condition specified was not met.", + context + ); + } + + public static getSequenceNumberConditionNotMet( + context: Context + ): DataLakeError { + return new DataLakeError( + 412, + "SequenceNumberConditionNotMet", + "The condition specified using HTTP conditional header(s) is not met.", + "SequenceNumberConditionNotMet", + "The condition specified using HTTP conditional header(s) is not met.", + context + ); + } + + public static getNotModified(context: Context): DataLakeError { + return new DataLakeError( + 304, + "ConditionNotMet", + "The condition specified using HTTP conditional header(s) is not met.", + "ConditionNotMet", + "The condition specified using HTTP conditional header(s) is not met.", + context + ); + } + + public static getUnsatisfiableCondition(context: Context): DataLakeError { + return new DataLakeError( + 400, + "UnsatisfiableCondition", + "The request includes an unsatisfiable condition for this operation.", + "UnsatisfiableCondition", + "The request includes an unsatisfiable condition for this operation.", + context + ); + } + + public static getInvalidHeaderValue( + context: Context, + additionalMessages?: { [key: string]: string } + ): DataLakeError { + if (additionalMessages === undefined) { + additionalMessages = {}; + } + return new DataLakeError( + 400, + "InvalidHeaderValue", + "The value for one of the HTTP headers is not in the correct format.", + "InvalidHeaderValue", + "The value for one of the HTTP headers is not in the correct format.", + context, + additionalMessages + ); + } + + public static getInvalidAPIVersion( + context: Context, + apiVersion?: string + ): DataLakeError { + return new DataLakeError( + 400, + "InvalidHeaderValue", + `The API version ${apiVersion} is not supported by Azurite. Please upgrade Azurite to latest version and retry. If you are using Azurite in Visual Studio, please check you have installed latest Visual Studio patch. Azurite command line parameter \"--skipApiVersionCheck\" or Visual Studio Code configuration \"Skip Api Version Check\" can skip this error. `, + "InvalidHeaderValue", + `The API version ${apiVersion} is not supported by Azurite. Please upgrade Azurite to latest version and retry. If you are using Azurite in Visual Studio, please check you have installed latest Visual Studio patch. Azurite command line parameter \"--skipApiVersionCheck\" or Visual Studio Code configuration \"Skip Api Version Check\" can skip this error. `, + context + ); + } + + public static getBlobArchived( + context: Context, + additionalMessages?: { [key: string]: string } + ): DataLakeError { + if (additionalMessages === undefined) { + additionalMessages = {}; + } + return new DataLakeError( + 409, + "BlobArchived", + "This operation is not permitted on an archived blob.", + "BlobArchived", + "This operation is not permitted on an archived blob.", + context, + additionalMessages + ); + } + + public static getInvalidCorsHeaderValue( + context: Context, + additionalMessages?: { [key: string]: string } + ): DataLakeError { + return new DataLakeError( + 400, + "InvalidHeaderValue", + "A required CORS header is not present.", + "InvalidHeaderValue", + "A required CORS header is not present.", + context, + additionalMessages + ); + } + + public static corsPreflightFailure( + context: Context, + additionalMessages?: { [key: string]: string } + ): DataLakeError { + return new DataLakeError( + 403, + "CorsPreflightFailure", + "CORS not enabled or no matching rule found for this request.", + "CorsPreflightFailure", + "CORS not enabled or no matching rule found for this request.", + context, + additionalMessages + ); + } + + public static getCannotVerifyCopySource( + context: Context, + statusCode: number, + message: string, + additionalMessages?: { [key: string]: string } + ): DataLakeError { + return new DataLakeError( + statusCode, + "CannotVerifyCopySource", + message, + "CannotVerifyCopySource", + message, + context, + additionalMessages + ); + } + + public static getInvalidResourceName(context: Context): DataLakeError { + return new DataLakeError( + 400, + "InvalidResourceName", + "The specified resource name contains invalid characters.", + "InvalidResourceName", + "The specified resource name contains invalid characters.", + context + ); + } + + public static getOutOfRangeName(context: Context): DataLakeError { + return new DataLakeError( + 400, + "OutOfRangeInput", + "One of the request inputs is out of range.", + "OutOfRangeInput", + `The specified resource name length is not within the permissible limits.`, + context + ); + } + + public static getUnexpectedSyncCopyStatus( + context: Context, + copyStatus: string + ): DataLakeError { + return new DataLakeError( + 409, + "UnexpectedSyncCopyStatus", + 'Expected copyStatus to be "success" but got different status.', + "UnexpectedSyncCopyStatus", + 'Expected copyStatus to be "success" but got different status.', + context, + { ReceivedCopyStatus: copyStatus } + ); + } + + public static getInvalidInput( + context: Context, + message: string + ): DataLakeError { + return new DataLakeError( + 400, + "InvalidInput", + message, + "InvalidInput", + message, + context + ); + } + + public static getPathConflict(context: Context) { + return new DataLakeError( + 409, + "PathConflict", + "The specified path, or an element of the path, exists and its resource type is invalid for this operation.", + "PathConflict", + "The specified path, or an element of the path, exists and its resource type is invalid for this operation.", + context + ); + } +} diff --git a/src/dfs/errors/StrictModelNotSupportedError.ts b/src/dfs/errors/StrictModelNotSupportedError.ts new file mode 100644 index 000000000..e981c8d67 --- /dev/null +++ b/src/dfs/errors/StrictModelNotSupportedError.ts @@ -0,0 +1,15 @@ +import Context from "../generated/Context"; +import DataLakeError from "./DataLakeError"; + +export default class StrictModelNotSupportedError extends DataLakeError { + public constructor(feature: string, context: Context) { + super( + 500, + "FeatureNotSupported", + `${feature} header or parameter is not supported in Azurite strict mode. Switch to loose model by Azurite command line parameter "--loose" or Visual Studio Code configuration "Loose". Please vote your wanted features to https://github.com/azure/azurite/issues`, + "FeatureNotSupported", + `${feature} header or parameter is not supported in Azurite strict mode. Switch to loose model by Azurite command line parameter "--loose" or Visual Studio Code configuration "Loose". Please vote your wanted features to https://github.com/azure/azurite/issues`, + context + ); + } +} diff --git a/src/dfs/gc/BlobGCManager.ts b/src/dfs/gc/BlobGCManager.ts new file mode 100644 index 000000000..9f0449bf1 --- /dev/null +++ b/src/dfs/gc/BlobGCManager.ts @@ -0,0 +1,293 @@ +import { EventEmitter } from "events"; + +import IGCExtentProvider from "../../common/IGCExtentProvider"; +import IGCManager from "../../common/IGCManager"; +import IExtentStore from "../../common/persistence/IExtentStore"; +import ILogger from "../generated/utils/ILogger"; +import { DEFAULT_GC_INTERVAL_MS } from "../utils/constants"; + +enum Status { + Initializing, + Running, + Closing, + Closed +} + +/** + * GC manager to clean up unused extents mapped local files based on mark and sweep + * algorithm. + * + * In the future, GC manager can also help merging small extent mapped files + * into one big file to improve the performance. + * + * @export + * @class BlobGCManager + * @implements {IGCManager} + */ +export default class BlobGCManager implements IGCManager { + private _status: Status = Status.Closed; + private emitter: EventEmitter = new EventEmitter(); + + /** + * Creates an instance of BlobGCManager. + * + * @param {IBlobMetadataStore} blobDataStore + * @param {(err: Error) => void} errorHandler Error handler callback to handle critical errors during GC loop + * When an error happens, GC loop will close automatically + * @param {ILogger} logger + * @param {number} [gcIntervalInMS=DEFAULT_GC_INTERVAL_MS] + * @memberof BlobGCManager + */ + constructor( + private readonly referredExtentsProvider: IGCExtentProvider, + private readonly allExtentsProvider: IGCExtentProvider, + private readonly extentStore: IExtentStore, + private readonly errorHandler: (err: Error) => void, + private readonly logger: ILogger, + public readonly gcIntervalInMS: number = DEFAULT_GC_INTERVAL_MS + ) { + this.emitter.once("error", this.errorHandler); + + // Avoid infinite GC loop + if (gcIntervalInMS <= 0) { + this.gcIntervalInMS = 1; + } + } + + public get status(): Status { + return this._status; + } + + /** + * Initialize and start GC manager. + * + * @returns {Promise} + * @memberof BlobGCManager + */ + public async start(): Promise { + if (this._status === Status.Running) { + this.logger.info( + `BlobGCManager:start() BlobGCManager successfully started. BlobGCManager is already in Running status.` + ); + return; + } + + if (this._status !== Status.Closed) { + const error = new Error( + `BlobGCManager:start() BlobGCManager cannot start, current manager is under ${ + Status[this._status] + }` + ); + this.logger.error(error.message); + throw error; + } + + this.logger.info( + `BlobGCManager:start() Starting BlobGCManager. Set status to Initializing.` + ); + this._status = Status.Initializing; + + if (!this.referredExtentsProvider.isInitialized()) { + this.logger.info( + `BlobGCManager:start() blobDataStore doesn't boot up. Starting blobDataStore.` + ); + await this.referredExtentsProvider.init(); + this.logger.info( + `BlobGCManager:start() blobDataStore successfully started.` + ); + } + + if (!this.allExtentsProvider.isInitialized()) { + this.logger.info( + `BlobGCManager:start() extentMetadata doesn't boot up. Starting extentMetadata.` + ); + await this.allExtentsProvider.init(); + this.logger.info( + `BlobGCManager:start() extentMetadata successfully started.` + ); + } + + if (!this.extentStore.isInitialized()) { + this.logger.info( + `BlobGCManager:start() extentStore doesn't boot up. Starting extentStore.` + ); + await this.extentStore.init(); + this.logger.info( + `BlobGCManager:start() extentStore successfully started.` + ); + } + + this.logger.info( + `BlobGCManager:start() Trigger mark and sweep loop. Set status to Running.` + ); + this._status = Status.Running; + + this.markSweepLoop() + .then(() => { + this.logger.info( + `BlobGCManager:start() Mark and sweep loop is closed.` + ); + this.emitter.emit("closed"); + }) + .catch((err) => { + this.logger.info( + `BlobGCManager:start() Mark and sweep loop emits error: ${err.name} ${err.message}` + ); + this.logger.info(`BlobGCManager:start() Set status to Closed.`); + this._status = Status.Closed; + this.emitter.emit("error", err); + }); + + this.logger.info( + `BlobGCManager:start() BlobGCManager successfully started.` + ); + } + + public async close(): Promise { + if (this._status === Status.Closed) { + this.logger.info( + `BlobGCManager:close() BlobGCManager successfully closed. BlobGCManager is already in Closed status.` + ); + return; + } + + if (this._status !== Status.Running) { + const error = new Error( + `BlobGCManager:close() BlobGCManager cannot close, current manager is under ${ + Status[this._status] + }` + ); + this.logger.error(error.message); + throw error; + } + + this.logger.info( + `BlobGCManager:close() Start closing BlobGCManager. Set status to Closing.` + ); + this._status = Status.Closing; + + this.emitter.emit("abort"); + + return new Promise((resolve) => { + this.emitter.once("closed", () => { + this.logger.info( + `BlobGCManager:close() BlobGCManager successfully closed. Set status to Closed.` + ); + this._status = Status.Closed; + resolve(); + }); + }); + } + + private async markSweepLoop(): Promise { + while (this._status === Status.Running) { + this.logger.info( + `BlobGCManager:markSweepLoop() Start next mark and sweep.` + ); + const start = Date.now(); + await this.markSweep(); + const period = Date.now() - start; + this.logger.info( + `BlobGCManager:markSweepLoop() Mark and sweep finished, taken ${period}ms.` + ); + + if (this._status === Status.Running) { + this.logger.info( + `BlobGCManager:markSweepLoop() Sleep for ${this.gcIntervalInMS}ms.` + ); + await this.sleep(this.gcIntervalInMS); + } + } + } + + /** + * Typical mark-sweep GC algorithm. + * + * @private + * @returns {Promise} + * @memberof BlobGCManager + */ + private async markSweep(): Promise { + // mark + this.logger.info(`BlobGCManager:markSweep() Get all extents.`); + const allExtents = await this.getAllExtents(); + this.logger.info( + `BlobGCManager:markSweep() Got ${allExtents.size} extents.` + ); + + if (this._status !== Status.Running) { + return; + } + + this.logger.info(`BlobGCManager:markSweep() Get referred extents.`); + const iter = this.referredExtentsProvider.iteratorExtents(); + for ( + let res = await iter.next(); + (res.done === false || res.value.length > 0) && + this._status === Status.Running; + res = await iter.next() + ) { + const chunks = res.value; + for (const chunk of chunks) { + allExtents.delete(chunk); // TODO: Mark instead of removing from Set to improve performance + } + } + this.logger.info( + `BlobGCManager:markSweep() Got referred extents, unreferenced extents count is ${allExtents.size}.` + ); + + // sweep + if (allExtents.size > 0) { + this.logger.info( + `BlobGCManager:markSweep() Try to delete ${allExtents.entries} unreferenced extents.` + ); + const deletedCount = await this.extentStore.deleteExtents(allExtents); + this.logger.info( + `BlobGCManager:markSweep() Deleted unreferenced ${deletedCount} extents, after excluding active write extents.` + ); + } + } + + private async getAllExtents(): Promise> { + const ids: Set = new Set(); + + const iter = this.allExtentsProvider.iteratorExtents(); + for ( + let res = await iter.next(); + (res.done === false || res.value.length > 0) && + this._status === Status.Running; + res = await iter.next() + ) { + for (const chunk of res.value) { + ids.add(chunk); + } + } + + return ids; + } + + private async sleep(timeInMS: number): Promise { + if (timeInMS === 0) { + return; + } + + return new Promise((resolve) => { + let timer: NodeJS.Timeout; + const abortListener = () => { + if (timer) { + clearTimeout(timer); + } + this.emitter.removeListener("abort", abortListener); + resolve(); + }; + + // https://stackoverflow.com/questions/45802988/typescript-use-correct-version-of-settimeout-node-vs-window + timer = setTimeout(() => { + this.emitter.removeListener("abort", abortListener); + resolve(); + }, timeInMS) as any as NodeJS.Timeout; + timer.unref(); + this.emitter.on("abort", abortListener); + }); + } +} diff --git a/src/dfs/generated/Context.ts b/src/dfs/generated/Context.ts new file mode 100644 index 000000000..243763cfd --- /dev/null +++ b/src/dfs/generated/Context.ts @@ -0,0 +1,143 @@ +import Operation from './artifacts/operation'; +import IRequest from './IRequest'; +import IResponse from './IResponse'; + +export interface IHandlerParameters { + [key: string]: any; +} + +/** + * Context holds generated server context information. + * Every incoming HTTP request will initialize a new context. + * + * @export + * @class Context + */ +export default class Context { + public readonly context: any; + public readonly path: string; + + /** + * Creates an instance of Context. + * Context holds generated server context information. + * Every incoming HTTP request will initialize a new context. + * + * @param {Context} context An existing Context + * @memberof Context + */ + public constructor(context: Context); + + /** + * Creates an instance of Context. + * Context holds generated server context information. + * Every incoming HTTP request will initialize a new context. + * + * @param {Object} holder Holder is an Object which used to keep context information + * @param {string} [path="context"] holder[path] is used as context object by default + * @param {IRequest} [req] + * @param {IResponse} [res] + * @memberof Context + */ + public constructor( + holder: object, + path: string, + req?: IRequest, + res?: IResponse + ); + + public constructor( + holderOrContext: object | Context, + path: string = "context", + req?: IRequest, + res?: IResponse + ) { + if (holderOrContext instanceof Context) { + this.context = holderOrContext.context; + this.path = holderOrContext.path; + } else { + const context = holderOrContext as any; + this.path = path; + + if (context[this.path] === undefined) { + context[this.path] = {}; + } + + if (typeof context[this.path] !== "object") { + throw new TypeError( + `Initialize Context error because holder.${this.path} is not an object.` + ); + } + + this.context = context[this.path]; + + this.request = req; + this.response = res; + } + } + + public get operation(): Operation | undefined { + return this.context.operation; + } + + public set operation(operation: Operation | undefined) { + this.context.operation = operation; + } + + public set request(request: IRequest | undefined) { + this.context.request = request; + } + + public get request(): IRequest | undefined { + return this.context.request; + } + + public get dispatchPattern(): string | undefined { + return this.context.dispatchPattern; + } + + public set dispatchPattern(path: string | undefined) { + this.context.dispatchPattern = path; + } + + public set response(response: IResponse | undefined) { + this.context.response = response; + } + + public get response(): IResponse | undefined { + return this.context.response; + } + + public get handlerParameters(): IHandlerParameters | undefined { + return this.context.handlerParameters; + } + + public set handlerParameters( + handlerParameters: IHandlerParameters | undefined + ) { + this.context.handlerParameters = handlerParameters; + } + + public get handlerResponses(): any { + return this.context.handlerResponses; + } + + public set handlerResponses(handlerResponses: any) { + this.context.handlerResponses = handlerResponses; + } + + public get contextId(): string | undefined { + return this.context.contextID; + } + + public set contextId(contextID: string | undefined) { + this.context.contextID = contextID; + } + + public set startTime(startTime: Date | undefined) { + this.context.startTime = startTime; + } + + public get startTime(): Date | undefined { + return this.context.startTime; + } +} diff --git a/src/dfs/generated/ExpressMiddlewareFactory.ts b/src/dfs/generated/ExpressMiddlewareFactory.ts new file mode 100644 index 000000000..8ece08df0 --- /dev/null +++ b/src/dfs/generated/ExpressMiddlewareFactory.ts @@ -0,0 +1,162 @@ +import { ErrorRequestHandler, NextFunction, Request, RequestHandler, Response } from 'express'; + +import Context from './Context'; +import ExpressRequestAdapter from './ExpressRequestAdapter'; +import ExpressResponseAdapter from './ExpressResponseAdapter'; +import IHandlers from './handlers/IHandlers'; +import deserializerMiddleware from './middleware/deserializer.middleware'; +import dispatchMiddleware from './middleware/dispatch.middleware'; +import endMiddleware from './middleware/end.middleware'; +import errorMiddleware from './middleware/error.middleware'; +import HandlerMiddlewareFactory from './middleware/HandlerMiddlewareFactory'; +import serializerMiddleware from './middleware/serializer.middleware'; +import MiddlewareFactory from './MiddlewareFactory'; +import ILogger from './utils/ILogger'; + +/** + * ExpressMiddlewareFactory will generate Express compatible middleware according to swagger definitions. + * Generated middleware MUST be used by strict order: + * * dispatchMiddleware + * * DeserializerMiddleware + * * HandlerMiddleware + * * SerializerMiddleware + * * ErrorMiddleware + * * EndMiddleware + * + * @export + * @class MiddlewareFactory + */ +export default class ExpressMiddlewareFactory extends MiddlewareFactory { + /** + * Creates an instance of MiddlewareFactory. + * + * @param {ILogger} logger A valid logger + * @param {string} [contextPath="default_context"] Optional. res.locals[contextPath] will be used to hold context + * @memberof MiddlewareFactory + */ + public constructor( + logger: ILogger, + private readonly contextPath: string = "default_context" + ) { + super(logger); + } + + /** + * DispatchMiddleware is the 1s middleware should be used among other generated middleware. + * + * @returns {RequestHandler} + * @memberof MiddlewareFactory + */ + public createDispatchMiddleware(): RequestHandler { + return (req: Request, res: Response, next: NextFunction) => { + req.baseUrl + const request = new ExpressRequestAdapter(req); + const response = new ExpressResponseAdapter(res); + dispatchMiddleware( + new Context(res.locals, this.contextPath, request, response), + request, + next, + this.logger + ); + }; + } + + /** + * DeserializerMiddleware is the 2nd middleware should be used among other generated middleware. + * + * @returns {RequestHandler} + * @memberof MiddlewareFactory + */ + public createDeserializerMiddleware(): RequestHandler { + return (req: Request, res: Response, next: NextFunction) => { + const request = new ExpressRequestAdapter(req); + const response = new ExpressResponseAdapter(res); + deserializerMiddleware( + new Context(res.locals, this.contextPath, request, response), + request, + next, + this.logger + ); + }; + } + + /** + * HandlerMiddleware is the 3rd middleware should be used among other generated middleware. + * + * @param {IHandlers} handlers + * @returns {RequestHandler} + * @memberof MiddlewareFactory + */ + public createHandlerMiddleware(handlers: IHandlers): RequestHandler { + const handlerMiddlewareFactory = new HandlerMiddlewareFactory( + handlers, + this.logger + ); + return (req: Request, res: Response, next: NextFunction) => { + const request = new ExpressRequestAdapter(req); + const response = new ExpressResponseAdapter(res); + handlerMiddlewareFactory.createHandlerMiddleware()( + new Context(res.locals, this.contextPath, request, response), + next + ); + }; + } + + /** + * SerializerMiddleware is the 4st middleware should be used among other generated middleware. + * + * @returns {RequestHandler} + * @memberof MiddlewareFactory + */ + public createSerializerMiddleware(): RequestHandler { + return (req: Request, res: Response, next: NextFunction) => { + const request = new ExpressRequestAdapter(req); + const response = new ExpressResponseAdapter(res); + serializerMiddleware( + new Context(res.locals, this.contextPath, request, response), + new ExpressResponseAdapter(res), + next, + this.logger + ); + }; + } + + /** + * ErrorMiddleware is the 5st middleware should be used among other generated middleware. + * + * @returns {ErrorRequestHandler} + * @memberof MiddlewareFactory + */ + public createErrorMiddleware(): ErrorRequestHandler { + return (err: Error, req: Request, res: Response, next: NextFunction) => { + const request = new ExpressRequestAdapter(req); + const response = new ExpressResponseAdapter(res); + errorMiddleware( + new Context(res.locals, this.contextPath, request, response), + err, + new ExpressRequestAdapter(req), + new ExpressResponseAdapter(res), + next, + this.logger + ); + }; + } + + /** + * EndMiddleware is the 6st middleware should be used among other generated middleware. + * + * @returns {RequestHandler} + * @memberof MiddlewareFactory + */ + public createEndMiddleware(): RequestHandler { + return (req: Request, res: Response) => { + const request = new ExpressRequestAdapter(req); + const response = new ExpressResponseAdapter(res); + endMiddleware( + new Context(res.locals, this.contextPath, request, response), + new ExpressResponseAdapter(res), + this.logger + ); + }; + } +} diff --git a/src/dfs/generated/ExpressRequestAdapter.ts b/src/dfs/generated/ExpressRequestAdapter.ts new file mode 100644 index 000000000..75861cb5a --- /dev/null +++ b/src/dfs/generated/ExpressRequestAdapter.ts @@ -0,0 +1,57 @@ +import { Request } from 'express'; + +import IRequest, { HttpMethod } from './IRequest'; + +export default class ExpressRequestAdapter implements IRequest { + public constructor(private readonly req: Request) {} + + public getMethod(): HttpMethod { + return this.req.method.toUpperCase() as HttpMethod; + } + + public getUrl(): string { + return this.req.url; + } + + public getEndpoint(): string { + return `${this.req.protocol}://${this.getHeader("host") || + this.req.hostname}`; + } + + public getPath(): string { + return this.req.path; + } + + public getBodyStream(): NodeJS.ReadableStream { + return this.req; + } + + public getBody(): string | undefined { + return this.req.body; + } + + public setBody(body: string | undefined): ExpressRequestAdapter { + this.req.body = body; + return this; + } + + public getHeader(field: string): string | undefined { + return this.req.header(field); + } + + public getHeaders(): { [header: string]: string | string[] | undefined } { + return this.req.headers; + } + + public getRawHeaders(): string[] { + return this.req.rawHeaders; + } + + public getQuery(key: string): string | undefined { + return this.req.query[key]; + } + + public getProtocol(): string { + return this.req.protocol; + } +} diff --git a/src/dfs/generated/ExpressResponseAdapter.ts b/src/dfs/generated/ExpressResponseAdapter.ts new file mode 100644 index 000000000..01a12bc23 --- /dev/null +++ b/src/dfs/generated/ExpressResponseAdapter.ts @@ -0,0 +1,66 @@ +import { Response } from 'express'; +import { OutgoingHttpHeaders } from 'http'; + +import IResponse from './IResponse'; + +export default class ExpressResponseAdapter implements IResponse { + public constructor(private readonly res: Response) {} + + public setStatusCode(code: number): IResponse { + this.res.status(code); + return this; + } + + public getStatusCode(): number { + return this.res.statusCode; + } + + public setStatusMessage(message: string): IResponse { + this.res.statusMessage = message; + return this; + } + + public getStatusMessage(): string { + return this.res.statusMessage; + } + + public setHeader( + field: string, + value?: string | string[] | undefined | number | boolean + ): IResponse { + if (typeof value === "number") { + value = `${value}`; + } + + if (typeof value === "boolean") { + value = `${value}`; + } + + // Cannot remove if block because of a potential TypeScript bug + if (typeof value === "string" || value instanceof Array) { + this.res.setHeader(field, value); + } + return this; + } + + public getHeader(field: string): number | string | string[] | undefined { + return this.res.getHeader(field); + } + + public getHeaders(): OutgoingHttpHeaders { + return this.res.getHeaders(); + } + + public headersSent(): boolean { + return this.res.headersSent; + } + + public setContentType(value: string): IResponse { + this.res.setHeader("content-type", value); + return this; + } + + public getBodyStream(): NodeJS.WritableStream { + return this.res; + } +} diff --git a/src/dfs/generated/IRequest.ts b/src/dfs/generated/IRequest.ts new file mode 100644 index 000000000..c281157bf --- /dev/null +++ b/src/dfs/generated/IRequest.ts @@ -0,0 +1,26 @@ +export type HttpMethod = + | "GET" + | "HEAD" + | "POST" + | "PUT" + | "DELETE" + | "CONNECT" + | "OPTIONS" + | "TRACE" + | "MERGE" + | "PATCH"; + +export default interface IRequest { + getMethod(): HttpMethod; + getUrl(): string; + getEndpoint(): string; + getPath(): string; + getBodyStream(): NodeJS.ReadableStream; + setBody(body: string | undefined): IRequest; + getBody(): string | undefined; + getHeader(field: string): string | undefined; + getHeaders(): { [header: string]: string | string[] | undefined }; + getRawHeaders(): string[]; + getQuery(key: string): string | undefined; + getProtocol(): string; +} diff --git a/src/dfs/generated/IResponse.ts b/src/dfs/generated/IResponse.ts new file mode 100644 index 000000000..0998329de --- /dev/null +++ b/src/dfs/generated/IResponse.ts @@ -0,0 +1,17 @@ +import { OutgoingHttpHeaders } from 'http'; + +export default interface IResponse { + setStatusCode(code: number): IResponse; + getStatusCode(): number; + setStatusMessage(message: string): IResponse; + getStatusMessage(): string; + setHeader( + field: string, + value?: string | string[] | undefined | number | boolean + ): IResponse; + getHeader(field: string): number | string | string[] | undefined; + getHeaders(): OutgoingHttpHeaders; + headersSent(): boolean; + setContentType(value: string | undefined): IResponse; + getBodyStream(): NodeJS.WritableStream; +} diff --git a/src/dfs/generated/MiddlewareFactory.ts b/src/dfs/generated/MiddlewareFactory.ts new file mode 100644 index 000000000..048a63f0e --- /dev/null +++ b/src/dfs/generated/MiddlewareFactory.ts @@ -0,0 +1,91 @@ +import IHandlers from './handlers/IHandlers'; +import ILogger from './utils/ILogger'; + +export type Callback = (...args: any[]) => any; +export type MiddlewareTypes = Callback; +export type NextFunction = Callback; + +/** + * MiddlewareFactory will generate middleware according to swagger definitions. + * + * Generated middleware MUST be used by strict order when you build your HTTP server: + * * DispatchMiddleware + * * DeserializerMiddleware + * * HandlerMiddleware + * * SerializerMiddleware + * * ErrorMiddleware + * * EndMiddleware + * + * To compatible with different Node.js server frameworks, such as Express or Koa, + * Extend this class and implement interfaces IRequest and IResponse as adapters. + * + * As above default generated middleware is callback style, you may want to wrap them into promise + * style for Koa like frameworks. Generated middleware will always trigger callback method at last, + * and pass all error object as the first parameter of callback method. + * + * We already provide implementation for Express framework. Refer to: + * * ExpressMiddlewareFactory + * * ExpressRequestAdapter + * * ExpressResponseAdapter + * + * @export + * @class MiddlewareFactory + */ +export default abstract class MiddlewareFactory { + /** + * Creates an instance of MiddlewareFactory. + * + * @param {ILogger} logger A valid logger + * @memberof MiddlewareFactory + */ + public constructor(protected readonly logger: ILogger) {} + + /** + * DispatchMiddleware is the 1s middleware should be used among other generated middleware. + * + * @returns {MiddlewareTypes} + * @memberof MiddlewareFactory + */ + public abstract createDispatchMiddleware(): MiddlewareTypes; + + /** + * DeserializerMiddleware is the 2nd middleware should be used among other generated middleware. + * + * @returns {MiddlewareTypes} + * @memberof MiddlewareFactory + */ + public abstract createDeserializerMiddleware(): MiddlewareTypes; + + /** + * HandlerMiddleware is the 3rd middleware should be used among other generated middleware. + * + * @param {IHandlers} handlers + * @returns {MiddlewareTypes} + * @memberof MiddlewareFactory + */ + public abstract createHandlerMiddleware(handlers: IHandlers): MiddlewareTypes; + + /** + * SerializerMiddleware is the 4st middleware should be used among other generated middleware. + * + * @returns {MiddlewareTypes} + * @memberof MiddlewareFactory + */ + public abstract createSerializerMiddleware(): MiddlewareTypes; + + /** + * ErrorMiddleware is the 5st middleware should be used among other generated middleware. + * + * @returns {MiddlewareTypes} + * @memberof MiddlewareFactory + */ + public abstract createErrorMiddleware(): MiddlewareTypes; + + /** + * EndMiddleware is the 6st middleware should be used among other generated middleware. + * + * @returns {MiddlewareTypes} + * @memberof MiddlewareFactory + */ + public abstract createEndMiddleware(): MiddlewareTypes; +} diff --git a/src/dfs/generated/artifacts/mappers.ts b/src/dfs/generated/artifacts/mappers.ts new file mode 100644 index 000000000..a0000615f --- /dev/null +++ b/src/dfs/generated/artifacts/mappers.ts @@ -0,0 +1,8746 @@ +/* + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + * + * Code generated by Microsoft (R) AutoRest Code Generator. + * Changes may cause incorrect behavior and will be lost if the code is regenerated. + */ +// tslint:disable:object-literal-sort-keys + +import * as msRest from "@azure/ms-rest-js"; + +export const AclFailedEntry: msRest.CompositeMapper = { + serializedName: "AclFailedEntry", + type: { + name: "Composite", + className: "AclFailedEntry", + modelProperties: { + name: { + xmlName: "name", + serializedName: "name", + type: { + name: "String" + } + }, + type: { + xmlName: "type", + serializedName: "type", + type: { + name: "String" + } + }, + errorMessage: { + xmlName: "errorMessage", + serializedName: "errorMessage", + type: { + name: "String" + } + } + } + } +}; + +export const SetAccessControlRecursiveResponse: msRest.CompositeMapper = { + serializedName: "SetAccessControlRecursiveResponse", + type: { + name: "Composite", + className: "SetAccessControlRecursiveResponse", + modelProperties: { + directoriesSuccessful: { + xmlName: "directoriesSuccessful", + serializedName: "directoriesSuccessful", + type: { + name: "Number" + } + }, + filesSuccessful: { + xmlName: "filesSuccessful", + serializedName: "filesSuccessful", + type: { + name: "Number" + } + }, + failureCount: { + xmlName: "failureCount", + serializedName: "failureCount", + type: { + name: "Number" + } + }, + failedEntries: { + xmlName: "failedEntries", + xmlElementName: "AclFailedEntry", + serializedName: "failedEntries", + type: { + name: "Sequence", + element: { + type: { + name: "Composite", + className: "AclFailedEntry" + } + } + } + } + } + } +}; + +export const Path: msRest.CompositeMapper = { + serializedName: "Path", + type: { + name: "Composite", + className: "Path", + modelProperties: { + name: { + xmlName: "name", + serializedName: "name", + type: { + name: "String" + } + }, + isDirectory: { + xmlName: "isDirectory", + serializedName: "isDirectory", + defaultValue: false, + type: { + name: "Boolean" + } + }, + lastModified: { + xmlName: "lastModified", + serializedName: "lastModified", + type: { + name: "DateTimeRfc1123" + } + }, + etag: { + xmlName: "etag", + serializedName: "etag", + type: { + name: "String" + } + }, + contentLength: { + xmlName: "contentLength", + serializedName: "contentLength", + type: { + name: "Number" + } + }, + owner: { + xmlName: "owner", + serializedName: "owner", + type: { + name: "String" + } + }, + group: { + xmlName: "group", + serializedName: "group", + type: { + name: "String" + } + }, + permissions: { + xmlName: "permissions", + serializedName: "permissions", + type: { + name: "String" + } + }, + encryptionScope: { + xmlName: "EncryptionScope", + serializedName: "EncryptionScope", + type: { + name: "String" + } + } + } + } +}; + +export const PathList: msRest.CompositeMapper = { + xmlName: "paths", + serializedName: "PathList", + type: { + name: "Composite", + className: "PathList", + modelProperties: { + paths: { + xmlName: "paths", + xmlElementName: "paths", + serializedName: "paths", + type: { + name: "Sequence", + element: { + type: { + name: "Composite", + className: "Path" + } + } + } + } + } + } +}; + +export const FileSystem: msRest.CompositeMapper = { + serializedName: "FileSystem", + type: { + name: "Composite", + className: "FileSystem", + modelProperties: { + name: { + xmlName: "name", + serializedName: "name", + type: { + name: "String" + } + }, + lastModified: { + xmlName: "lastModified", + serializedName: "lastModified", + type: { + name: "String" + } + }, + eTag: { + xmlName: "eTag", + serializedName: "eTag", + type: { + name: "String" + } + } + } + } +}; + +export const BlobPropertiesInternal: msRest.CompositeMapper = { + xmlName: "Properties", + serializedName: "BlobPropertiesInternal", + type: { + name: "Composite", + className: "BlobPropertiesInternal", + modelProperties: { + creationTime: { + xmlName: "Creation-Time", + serializedName: "Creation-Time", + type: { + name: "DateTimeRfc1123" + } + }, + lastModified: { + xmlName: "Last-Modified", + required: true, + serializedName: "Last-Modified", + type: { + name: "DateTimeRfc1123" + } + }, + etag: { + xmlName: "Etag", + required: true, + serializedName: "Etag", + type: { + name: "String" + } + }, + contentLength: { + xmlName: "Content-Length", + serializedName: "Content-Length", + type: { + name: "Number" + } + }, + contentType: { + xmlName: "Content-Type", + serializedName: "Content-Type", + type: { + name: "String" + } + }, + contentEncoding: { + xmlName: "Content-Encoding", + serializedName: "Content-Encoding", + type: { + name: "String" + } + }, + contentLanguage: { + xmlName: "Content-Language", + serializedName: "Content-Language", + type: { + name: "String" + } + }, + contentMD5: { + xmlName: "Content-MD5", + serializedName: "Content-MD5", + type: { + name: "ByteArray" + } + }, + contentDisposition: { + xmlName: "Content-Disposition", + serializedName: "Content-Disposition", + type: { + name: "String" + } + }, + cacheControl: { + xmlName: "Cache-Control", + serializedName: "Cache-Control", + type: { + name: "String" + } + }, + blobSequenceNumber: { + xmlName: "x-ms-blob-sequence-number", + serializedName: "x-ms-blob-sequence-number", + type: { + name: "Number" + } + }, + copyId: { + xmlName: "CopyId", + serializedName: "CopyId", + type: { + name: "String" + } + }, + copySource: { + xmlName: "CopySource", + serializedName: "CopySource", + type: { + name: "String" + } + }, + copyProgress: { + xmlName: "CopyProgress", + serializedName: "CopyProgress", + type: { + name: "String" + } + }, + copyCompletionTime: { + xmlName: "CopyCompletionTime", + serializedName: "CopyCompletionTime", + type: { + name: "DateTimeRfc1123" + } + }, + copyStatusDescription: { + xmlName: "CopyStatusDescription", + serializedName: "CopyStatusDescription", + type: { + name: "String" + } + }, + serverEncrypted: { + xmlName: "ServerEncrypted", + serializedName: "ServerEncrypted", + type: { + name: "Boolean" + } + }, + incrementalCopy: { + xmlName: "IncrementalCopy", + serializedName: "IncrementalCopy", + type: { + name: "Boolean" + } + }, + destinationSnapshot: { + xmlName: "DestinationSnapshot", + serializedName: "DestinationSnapshot", + type: { + name: "String" + } + }, + deletedTime: { + xmlName: "DeletedTime", + serializedName: "DeletedTime", + type: { + name: "DateTimeRfc1123" + } + }, + remainingRetentionDays: { + xmlName: "RemainingRetentionDays", + serializedName: "RemainingRetentionDays", + type: { + name: "Number" + } + }, + accessTierInferred: { + xmlName: "AccessTierInferred", + serializedName: "AccessTierInferred", + type: { + name: "Boolean" + } + }, + customerProvidedKeySha256: { + xmlName: "CustomerProvidedKeySha256", + serializedName: "CustomerProvidedKeySha256", + type: { + name: "String" + } + }, + encryptionScope: { + xmlName: "EncryptionScope", + serializedName: "EncryptionScope", + type: { + name: "String" + } + }, + accessTierChangeTime: { + xmlName: "AccessTierChangeTime", + serializedName: "AccessTierChangeTime", + type: { + name: "DateTimeRfc1123" + } + }, + tagCount: { + xmlName: "TagCount", + serializedName: "TagCount", + type: { + name: "Number" + } + }, + expiresOn: { + xmlName: "Expiry-Time", + serializedName: "Expiry-Time", + type: { + name: "DateTimeRfc1123" + } + }, + isSealed: { + xmlName: "Sealed", + serializedName: "Sealed", + type: { + name: "Boolean" + } + }, + lastAccessedOn: { + xmlName: "LastAccessTime", + serializedName: "LastAccessTime", + type: { + name: "DateTimeRfc1123" + } + }, + deleteTime: { + xmlName: "DeleteTime", + serializedName: "DeleteTime", + type: { + name: "DateTimeRfc1123" + } + }, + blobType: { + xmlName: "BlobType", + serializedName: "BlobType", + type: { + name: "Enum", + allowedValues: [ + "BlockBlob", + "PageBlob", + "AppendBlob" + ] + } + }, + leaseStatus: { + xmlName: "LeaseStatus", + serializedName: "LeaseStatus", + type: { + name: "Enum", + allowedValues: [ + "locked", + "unlocked" + ] + } + }, + leaseState: { + xmlName: "LeaseState", + serializedName: "LeaseState", + type: { + name: "Enum", + allowedValues: [ + "available", + "leased", + "expired", + "breaking", + "broken" + ] + } + }, + leaseDuration: { + xmlName: "LeaseDuration", + serializedName: "LeaseDuration", + type: { + name: "Enum", + allowedValues: [ + "infinite", + "fixed" + ] + } + }, + copyStatus: { + xmlName: "CopyStatus", + serializedName: "CopyStatus", + type: { + name: "Enum", + allowedValues: [ + "pending", + "success", + "aborted", + "failed" + ] + } + }, + accessTier: { + xmlName: "AccessTier", + serializedName: "AccessTier", + type: { + name: "String" + } + }, + archiveStatus: { + xmlName: "ArchiveStatus", + serializedName: "ArchiveStatus", + type: { + name: "String" + } + }, + rehydratePriority: { + xmlName: "RehydratePriority", + serializedName: "RehydratePriority", + type: { + name: "String" + } + }, + immutabilityPolicyExpiresOn: { + xmlName: "ImmutabilityPolicyUntilDate", + serializedName: "ImmutabilityPolicyUntilDate", + type: { + name: "DateTimeRfc1123" + } + }, + immutabilityPolicyMode: { + xmlName: "ImmutabilityPolicyMode", + serializedName: "ImmutabilityPolicyMode", + type: { + name: "Enum", + allowedValues: [ + "Mutable", + "Unlocked", + "Locked" + ] + } + }, + legalHold: { + xmlName: "LegalHold", + serializedName: "LegalHold", + type: { + name: "Boolean" + } + }, + properties: { + xmlName: "properties", + serializedName: "properties", + type: { + name: "String" + } + } + } + } +}; + +export const BlobMetadata: msRest.CompositeMapper = { + xmlName: "Metadata", + serializedName: "BlobMetadata", + type: { + name: "Composite", + className: "BlobMetadata", + modelProperties: { + encrypted: { + xmlIsAttribute: true, + xmlName: "Encrypted", + serializedName: "Encrypted", + type: { + name: "String" + } + } + }, + additionalProperties: { + type: { + name: "String" + } + } + } +}; + +export const BlobTag: msRest.CompositeMapper = { + xmlName: "Tag", + serializedName: "BlobTag", + type: { + name: "Composite", + className: "BlobTag", + modelProperties: { + key: { + xmlName: "Key", + required: true, + serializedName: "Key", + type: { + name: "String" + } + }, + value: { + xmlName: "Value", + required: true, + serializedName: "Value", + type: { + name: "String" + } + } + } + } +}; + +export const BlobTags: msRest.CompositeMapper = { + xmlName: "Tags", + serializedName: "BlobTags", + type: { + name: "Composite", + className: "BlobTags", + modelProperties: { + blobTagSet: { + xmlIsWrapped: true, + xmlName: "TagSet", + xmlElementName: "Tag", + required: true, + serializedName: "BlobTagSet", + type: { + name: "Sequence", + element: { + type: { + name: "Composite", + className: "BlobTag" + } + } + } + } + } + } +}; + +export const BlobItemInternal: msRest.CompositeMapper = { + xmlName: "Blob", + serializedName: "BlobItemInternal", + type: { + name: "Composite", + className: "BlobItemInternal", + modelProperties: { + name: { + xmlName: "Name", + required: true, + serializedName: "Name", + type: { + name: "String" + } + }, + deleted: { + xmlName: "Deleted", + serializedName: "Deleted", + type: { + name: "Boolean" + } + }, + snapshot: { + xmlName: "Snapshot", + serializedName: "Snapshot", + type: { + name: "String" + } + }, + versionId: { + xmlName: "VersionId", + serializedName: "VersionId", + type: { + name: "String" + } + }, + isCurrentVersion: { + xmlName: "IsCurrentVersion", + serializedName: "IsCurrentVersion", + type: { + name: "Boolean" + } + }, + properties: { + xmlName: "Properties", + required: true, + serializedName: "Properties", + type: { + name: "Composite", + className: "BlobPropertiesInternal" + } + }, + deletionId: { + xmlName: "DeletionId", + serializedName: "DeletionId", + type: { + name: "String" + } + }, + metadata: { + xmlName: "Metadata", + serializedName: "Metadata", + type: { + name: "Composite", + className: "BlobMetadata", + additionalProperties: { + type: { + name: "String" + } + } + } + }, + blobTags: { + xmlName: "Tags", + serializedName: "BlobTags", + type: { + name: "Composite", + className: "BlobTags" + } + }, + objectReplicationMetadata: { + xmlName: "OrMetadata", + serializedName: "ObjectReplicationMetadata", + type: { + name: "Dictionary", + value: { + type: { + name: "String" + } + } + } + }, + hasVersionsOnly: { + xmlName: "HasVersionsOnly", + serializedName: "HasVersionsOnly", + type: { + name: "Boolean" + } + } + } + } +}; + +export const BlobFlatListSegment: msRest.CompositeMapper = { + xmlName: "Blobs", + serializedName: "BlobFlatListSegment", + type: { + name: "Composite", + className: "BlobFlatListSegment", + modelProperties: { + blobItems: { + xmlName: "BlobItems", + xmlElementName: "Blob", + required: true, + serializedName: "BlobItems", + type: { + name: "Sequence", + element: { + type: { + name: "Composite", + className: "BlobItemInternal" + } + } + } + } + } + } +}; + +export const ListBlobsFlatSegmentResponse: msRest.CompositeMapper = { + xmlName: "EnumerationResults", + serializedName: "ListBlobsFlatSegmentResponse", + type: { + name: "Composite", + className: "ListBlobsFlatSegmentResponse", + modelProperties: { + serviceEndpoint: { + xmlIsAttribute: true, + xmlName: "ServiceEndpoint", + required: true, + serializedName: "ServiceEndpoint", + type: { + name: "String" + } + }, + containerName: { + xmlIsAttribute: true, + xmlName: "ContainerName", + required: true, + serializedName: "ContainerName", + type: { + name: "String" + } + }, + prefix: { + xmlName: "Prefix", + serializedName: "Prefix", + type: { + name: "String" + } + }, + marker: { + xmlName: "Marker", + serializedName: "Marker", + type: { + name: "String" + } + }, + maxResults: { + xmlName: "MaxResults", + serializedName: "MaxResults", + type: { + name: "Number" + } + }, + segment: { + xmlName: "Blobs", + required: true, + serializedName: "Segment", + type: { + name: "Composite", + className: "BlobFlatListSegment" + } + }, + nextMarker: { + xmlName: "NextMarker", + serializedName: "NextMarker", + type: { + name: "String" + } + } + } + } +}; + +export const BlobPrefix: msRest.CompositeMapper = { + serializedName: "BlobPrefix", + type: { + name: "Composite", + className: "BlobPrefix", + modelProperties: { + name: { + xmlName: "Name", + required: true, + serializedName: "Name", + type: { + name: "String" + } + } + } + } +}; + +export const BlobHierarchyListSegment: msRest.CompositeMapper = { + xmlName: "Blobs", + serializedName: "BlobHierarchyListSegment", + type: { + name: "Composite", + className: "BlobHierarchyListSegment", + modelProperties: { + blobPrefixes: { + xmlName: "BlobPrefixes", + xmlElementName: "BlobPrefix", + serializedName: "BlobPrefixes", + type: { + name: "Sequence", + element: { + type: { + name: "Composite", + className: "BlobPrefix" + } + } + } + }, + blobItems: { + xmlName: "BlobItems", + xmlElementName: "Blob", + required: true, + serializedName: "BlobItems", + type: { + name: "Sequence", + element: { + type: { + name: "Composite", + className: "BlobItemInternal" + } + } + } + } + } + } +}; + +export const ListBlobsHierarchySegmentResponse: msRest.CompositeMapper = { + xmlName: "EnumerationResults", + serializedName: "ListBlobsHierarchySegmentResponse", + type: { + name: "Composite", + className: "ListBlobsHierarchySegmentResponse", + modelProperties: { + serviceEndpoint: { + xmlIsAttribute: true, + xmlName: "ServiceEndpoint", + required: true, + serializedName: "ServiceEndpoint", + type: { + name: "String" + } + }, + containerName: { + xmlIsAttribute: true, + xmlName: "ContainerName", + required: true, + serializedName: "ContainerName", + type: { + name: "String" + } + }, + prefix: { + xmlName: "Prefix", + serializedName: "Prefix", + type: { + name: "String" + } + }, + marker: { + xmlName: "Marker", + serializedName: "Marker", + type: { + name: "String" + } + }, + maxResults: { + xmlName: "MaxResults", + serializedName: "MaxResults", + type: { + name: "Number" + } + }, + delimiter: { + xmlName: "Delimiter", + serializedName: "Delimiter", + type: { + name: "String" + } + }, + segment: { + xmlName: "Blobs", + required: true, + serializedName: "Segment", + type: { + name: "Composite", + className: "BlobHierarchyListSegment" + } + }, + nextMarker: { + xmlName: "NextMarker", + serializedName: "NextMarker", + type: { + name: "String" + } + } + } + } +}; + +export const FileSystemList: msRest.CompositeMapper = { + serializedName: "FileSystemList", + type: { + name: "Composite", + className: "FileSystemList", + modelProperties: { + filesystems: { + xmlName: "filesystems", + xmlElementName: "FileSystem", + serializedName: "filesystems", + type: { + name: "Sequence", + element: { + type: { + name: "Composite", + className: "FileSystem" + } + } + } + } + } + } +}; + +export const StorageErrorError: msRest.CompositeMapper = { + serializedName: "StorageError_error", + type: { + name: "Composite", + className: "StorageErrorError", + modelProperties: { + code: { + xmlName: "Code", + serializedName: "Code", + type: { + name: "String" + } + }, + message: { + xmlName: "Message", + serializedName: "Message", + type: { + name: "String" + } + } + } + } +}; + +export const StorageError: msRest.CompositeMapper = { + serializedName: "StorageError", + type: { + name: "Composite", + className: "StorageError", + modelProperties: { + message: { + xmlName: "Message", + serializedName: "Message", + type: { + name: "String" + } + }, + error: { + xmlName: "error", + serializedName: "error", + type: { + name: "Composite", + className: "StorageErrorError" + } + } + } + } +}; + +export const KeyInfo: msRest.CompositeMapper = { + serializedName: "KeyInfo", + type: { + name: "Composite", + className: "KeyInfo", + modelProperties: { + start: { + xmlName: "Start", + required: true, + serializedName: "Start", + type: { + name: "String" + } + }, + expiry: { + xmlName: "Expiry", + required: true, + serializedName: "Expiry", + type: { + name: "String" + } + } + } + } +}; + +export const UserDelegationKey: msRest.CompositeMapper = { + serializedName: "UserDelegationKey", + type: { + name: "Composite", + className: "UserDelegationKey", + modelProperties: { + signedOid: { + xmlName: "SignedOid", + required: true, + serializedName: "SignedOid", + type: { + name: "String" + } + }, + signedTid: { + xmlName: "SignedTid", + required: true, + serializedName: "SignedTid", + type: { + name: "String" + } + }, + signedStart: { + xmlName: "SignedStart", + required: true, + serializedName: "SignedStart", + type: { + name: "String" + } + }, + signedExpiry: { + xmlName: "SignedExpiry", + required: true, + serializedName: "SignedExpiry", + type: { + name: "String" + } + }, + signedService: { + xmlName: "SignedService", + required: true, + serializedName: "SignedService", + type: { + name: "String" + } + }, + signedVersion: { + xmlName: "SignedVersion", + required: true, + serializedName: "SignedVersion", + type: { + name: "String" + } + }, + value: { + xmlName: "Value", + required: true, + serializedName: "Value", + type: { + name: "String" + } + } + } + } +}; + +export const StorageErrorBlob: msRest.CompositeMapper = { + serializedName: "StorageErrorBlob", + type: { + name: "Composite", + className: "StorageErrorBlob", + modelProperties: { + message: { + xmlName: "Message", + serializedName: "Message", + type: { + name: "String" + } + } + } + } +}; + +export const AccessPolicy: msRest.CompositeMapper = { + serializedName: "AccessPolicy", + type: { + name: "Composite", + className: "AccessPolicy", + modelProperties: { + start: { + xmlName: "Start", + serializedName: "Start", + type: { + name: "String" + } + }, + expiry: { + xmlName: "Expiry", + serializedName: "Expiry", + type: { + name: "String" + } + }, + permission: { + xmlName: "Permission", + serializedName: "Permission", + type: { + name: "String" + } + } + } + } +}; + +export const BlobName: msRest.CompositeMapper = { + serializedName: "BlobName", + type: { + name: "Composite", + className: "BlobName", + modelProperties: { + encoded: { + xmlIsAttribute: true, + xmlName: "Encoded", + serializedName: "Encoded", + type: { + name: "Boolean" + } + }, + content: { + xmlName: "content", + serializedName: "content", + type: { + name: "String" + } + } + } + } +}; + +export const Block: msRest.CompositeMapper = { + serializedName: "Block", + type: { + name: "Composite", + className: "Block", + modelProperties: { + name: { + xmlName: "Name", + required: true, + serializedName: "Name", + type: { + name: "String" + } + }, + size: { + xmlName: "Size", + required: true, + serializedName: "Size", + type: { + name: "Number" + } + } + } + } +}; + +export const BlockList: msRest.CompositeMapper = { + serializedName: "BlockList", + type: { + name: "Composite", + className: "BlockList", + modelProperties: { + committedBlocks: { + xmlIsWrapped: true, + xmlName: "CommittedBlocks", + xmlElementName: "Block", + serializedName: "CommittedBlocks", + type: { + name: "Sequence", + element: { + type: { + name: "Composite", + className: "Block" + } + } + } + }, + uncommittedBlocks: { + xmlIsWrapped: true, + xmlName: "UncommittedBlocks", + xmlElementName: "Block", + serializedName: "UncommittedBlocks", + type: { + name: "Sequence", + element: { + type: { + name: "Composite", + className: "Block" + } + } + } + } + } + } +}; + +export const BlockLookupList: msRest.CompositeMapper = { + xmlName: "BlockList", + serializedName: "BlockLookupList", + type: { + name: "Composite", + className: "BlockLookupList", + modelProperties: { + committed: { + xmlName: "Committed", + xmlElementName: "Committed", + serializedName: "Committed", + type: { + name: "Sequence", + element: { + type: { + name: "String" + } + } + } + }, + uncommitted: { + xmlName: "Uncommitted", + xmlElementName: "Uncommitted", + serializedName: "Uncommitted", + type: { + name: "Sequence", + element: { + type: { + name: "String" + } + } + } + }, + latest: { + xmlName: "Latest", + xmlElementName: "Latest", + serializedName: "Latest", + type: { + name: "Sequence", + element: { + type: { + name: "String" + } + } + } + } + } + } +}; + +export const ContainerProperties: msRest.CompositeMapper = { + serializedName: "ContainerProperties", + type: { + name: "Composite", + className: "ContainerProperties", + modelProperties: { + lastModified: { + xmlName: "Last-Modified", + required: true, + serializedName: "Last-Modified", + type: { + name: "DateTimeRfc1123" + } + }, + etag: { + xmlName: "Etag", + required: true, + serializedName: "Etag", + type: { + name: "String" + } + }, + leaseStatus: { + xmlName: "LeaseStatus", + serializedName: "LeaseStatus", + type: { + name: "Enum", + allowedValues: [ + "locked", + "unlocked" + ] + } + }, + leaseState: { + xmlName: "LeaseState", + serializedName: "LeaseState", + type: { + name: "Enum", + allowedValues: [ + "available", + "leased", + "expired", + "breaking", + "broken" + ] + } + }, + leaseDuration: { + xmlName: "LeaseDuration", + serializedName: "LeaseDuration", + type: { + name: "Enum", + allowedValues: [ + "infinite", + "fixed" + ] + } + }, + publicAccess: { + xmlName: "PublicAccess", + serializedName: "PublicAccess", + type: { + name: "String" + } + }, + hasImmutabilityPolicy: { + xmlName: "HasImmutabilityPolicy", + serializedName: "HasImmutabilityPolicy", + type: { + name: "Boolean" + } + }, + hasLegalHold: { + xmlName: "HasLegalHold", + serializedName: "HasLegalHold", + type: { + name: "Boolean" + } + }, + defaultEncryptionScope: { + xmlName: "DefaultEncryptionScope", + serializedName: "DefaultEncryptionScope", + type: { + name: "String" + } + }, + preventEncryptionScopeOverride: { + xmlName: "DenyEncryptionScopeOverride", + serializedName: "DenyEncryptionScopeOverride", + type: { + name: "Boolean" + } + }, + deletedTime: { + xmlName: "DeletedTime", + serializedName: "DeletedTime", + type: { + name: "DateTimeRfc1123" + } + }, + remainingRetentionDays: { + xmlName: "RemainingRetentionDays", + serializedName: "RemainingRetentionDays", + type: { + name: "Number" + } + }, + isImmutableStorageWithVersioningEnabled: { + xmlName: "ImmutableStorageWithVersioningEnabled", + serializedName: "ImmutableStorageWithVersioningEnabled", + type: { + name: "Boolean" + } + } + } + } +}; + +export const ContainerItem: msRest.CompositeMapper = { + xmlName: "Container", + serializedName: "ContainerItem", + type: { + name: "Composite", + className: "ContainerItem", + modelProperties: { + name: { + xmlName: "Name", + required: true, + serializedName: "Name", + type: { + name: "String" + } + }, + deleted: { + xmlName: "Deleted", + serializedName: "Deleted", + type: { + name: "Boolean" + } + }, + version: { + xmlName: "Version", + serializedName: "Version", + type: { + name: "String" + } + }, + properties: { + xmlName: "Properties", + required: true, + serializedName: "Properties", + type: { + name: "Composite", + className: "ContainerProperties" + } + }, + metadata: { + xmlName: "Metadata", + serializedName: "Metadata", + type: { + name: "Dictionary", + value: { + type: { + name: "String" + } + } + } + } + } + } +}; + +export const DelimitedTextConfiguration: msRest.CompositeMapper = { + serializedName: "DelimitedTextConfiguration", + type: { + name: "Composite", + className: "DelimitedTextConfiguration", + modelProperties: { + columnSeparator: { + xmlName: "ColumnSeparator", + serializedName: "ColumnSeparator", + type: { + name: "String" + } + }, + fieldQuote: { + xmlName: "FieldQuote", + serializedName: "FieldQuote", + type: { + name: "String" + } + }, + recordSeparator: { + xmlName: "RecordSeparator", + serializedName: "RecordSeparator", + type: { + name: "String" + } + }, + escapeChar: { + xmlName: "EscapeChar", + serializedName: "EscapeChar", + type: { + name: "String" + } + }, + headersPresent: { + xmlName: "HasHeaders", + serializedName: "HeadersPresent", + type: { + name: "Boolean" + } + } + } + } +}; + +export const JsonTextConfiguration: msRest.CompositeMapper = { + serializedName: "JsonTextConfiguration", + type: { + name: "Composite", + className: "JsonTextConfiguration", + modelProperties: { + recordSeparator: { + xmlName: "RecordSeparator", + serializedName: "RecordSeparator", + type: { + name: "String" + } + } + } + } +}; + +export const ArrowField: msRest.CompositeMapper = { + xmlName: "Field", + serializedName: "ArrowField", + type: { + name: "Composite", + className: "ArrowField", + modelProperties: { + type: { + xmlName: "Type", + required: true, + serializedName: "Type", + type: { + name: "String" + } + }, + name: { + xmlName: "Name", + serializedName: "Name", + type: { + name: "String" + } + }, + precision: { + xmlName: "Precision", + serializedName: "Precision", + type: { + name: "Number" + } + }, + scale: { + xmlName: "Scale", + serializedName: "Scale", + type: { + name: "Number" + } + } + } + } +}; + +export const ArrowConfiguration: msRest.CompositeMapper = { + serializedName: "ArrowConfiguration", + type: { + name: "Composite", + className: "ArrowConfiguration", + modelProperties: { + schema: { + xmlIsWrapped: true, + xmlName: "Schema", + xmlElementName: "Field", + required: true, + serializedName: "Schema", + type: { + name: "Sequence", + element: { + type: { + name: "Composite", + className: "ArrowField" + } + } + } + } + } + } +}; + +export const ListContainersSegmentResponse: msRest.CompositeMapper = { + xmlName: "EnumerationResults", + serializedName: "ListContainersSegmentResponse", + type: { + name: "Composite", + className: "ListContainersSegmentResponse", + modelProperties: { + serviceEndpoint: { + xmlIsAttribute: true, + xmlName: "ServiceEndpoint", + required: true, + serializedName: "ServiceEndpoint", + type: { + name: "String" + } + }, + prefix: { + xmlName: "Prefix", + serializedName: "Prefix", + type: { + name: "String" + } + }, + marker: { + xmlName: "Marker", + serializedName: "Marker", + type: { + name: "String" + } + }, + maxResults: { + xmlName: "MaxResults", + serializedName: "MaxResults", + type: { + name: "Number" + } + }, + containerItems: { + xmlIsWrapped: true, + xmlName: "Containers", + xmlElementName: "Container", + required: true, + serializedName: "ContainerItems", + type: { + name: "Sequence", + element: { + type: { + name: "Composite", + className: "ContainerItem" + } + } + } + }, + nextMarker: { + xmlName: "NextMarker", + serializedName: "NextMarker", + type: { + name: "String" + } + } + } + } +}; + +export const CorsRule: msRest.CompositeMapper = { + serializedName: "CorsRule", + type: { + name: "Composite", + className: "CorsRule", + modelProperties: { + allowedOrigins: { + xmlName: "AllowedOrigins", + required: true, + serializedName: "AllowedOrigins", + type: { + name: "String" + } + }, + allowedMethods: { + xmlName: "AllowedMethods", + required: true, + serializedName: "AllowedMethods", + type: { + name: "String" + } + }, + allowedHeaders: { + xmlName: "AllowedHeaders", + serializedName: "AllowedHeaders", + type: { + name: "String" + } + }, + exposedHeaders: { + xmlName: "ExposedHeaders", + serializedName: "ExposedHeaders", + type: { + name: "String" + } + }, + maxAgeInSeconds: { + xmlName: "MaxAgeInSeconds", + required: true, + serializedName: "MaxAgeInSeconds", + constraints: { + InclusiveMinimum: 0 + }, + type: { + name: "Number" + } + } + } + } +}; + +export const FilterBlobItem: msRest.CompositeMapper = { + xmlName: "Blob", + serializedName: "FilterBlobItem", + type: { + name: "Composite", + className: "FilterBlobItem", + modelProperties: { + name: { + xmlName: "Name", + required: true, + serializedName: "Name", + type: { + name: "String" + } + }, + containerName: { + xmlName: "ContainerName", + required: true, + serializedName: "ContainerName", + type: { + name: "String" + } + }, + tags: { + xmlName: "Tags", + serializedName: "Tags", + type: { + name: "Composite", + className: "BlobTags" + } + }, + versionId: { + xmlName: "VersionId", + serializedName: "VersionId", + type: { + name: "String" + } + }, + isCurrentVersion: { + xmlName: "IsCurrentVersion", + serializedName: "IsCurrentVersion", + type: { + name: "Boolean" + } + } + } + } +}; + +export const FilterBlobSegment: msRest.CompositeMapper = { + xmlName: "EnumerationResults", + serializedName: "FilterBlobSegment", + type: { + name: "Composite", + className: "FilterBlobSegment", + modelProperties: { + serviceEndpoint: { + xmlIsAttribute: true, + xmlName: "ServiceEndpoint", + required: true, + serializedName: "ServiceEndpoint", + type: { + name: "String" + } + }, + where: { + xmlName: "Where", + required: true, + serializedName: "Where", + type: { + name: "String" + } + }, + blobs: { + xmlIsWrapped: true, + xmlName: "Blobs", + xmlElementName: "Blob", + required: true, + serializedName: "Blobs", + type: { + name: "Sequence", + element: { + type: { + name: "Composite", + className: "FilterBlobItem" + } + } + } + }, + nextMarker: { + xmlName: "NextMarker", + serializedName: "NextMarker", + type: { + name: "String" + } + } + } + } +}; + +export const GeoReplication: msRest.CompositeMapper = { + serializedName: "GeoReplication", + type: { + name: "Composite", + className: "GeoReplication", + modelProperties: { + status: { + xmlName: "Status", + required: true, + serializedName: "Status", + type: { + name: "String" + } + }, + lastSyncTime: { + xmlName: "LastSyncTime", + required: true, + serializedName: "LastSyncTime", + type: { + name: "DateTimeRfc1123" + } + } + } + } +}; + +export const RetentionPolicy: msRest.CompositeMapper = { + serializedName: "RetentionPolicy", + type: { + name: "Composite", + className: "RetentionPolicy", + modelProperties: { + enabled: { + xmlName: "Enabled", + required: true, + serializedName: "Enabled", + type: { + name: "Boolean" + } + }, + days: { + xmlName: "Days", + serializedName: "Days", + constraints: { + InclusiveMinimum: 1 + }, + type: { + name: "Number" + } + }, + allowPermanentDelete: { + xmlName: "AllowPermanentDelete", + serializedName: "AllowPermanentDelete", + type: { + name: "Boolean" + } + } + } + } +}; + +export const Logging: msRest.CompositeMapper = { + serializedName: "Logging", + type: { + name: "Composite", + className: "Logging", + modelProperties: { + version: { + xmlName: "Version", + required: true, + serializedName: "Version", + type: { + name: "String" + } + }, + deleteProperty: { + xmlName: "Delete", + required: true, + serializedName: "Delete", + type: { + name: "Boolean" + } + }, + read: { + xmlName: "Read", + required: true, + serializedName: "Read", + type: { + name: "Boolean" + } + }, + write: { + xmlName: "Write", + required: true, + serializedName: "Write", + type: { + name: "Boolean" + } + }, + retentionPolicy: { + xmlName: "RetentionPolicy", + required: true, + serializedName: "RetentionPolicy", + type: { + name: "Composite", + className: "RetentionPolicy" + } + } + } + } +}; + +export const Metrics: msRest.CompositeMapper = { + serializedName: "Metrics", + type: { + name: "Composite", + className: "Metrics", + modelProperties: { + version: { + xmlName: "Version", + serializedName: "Version", + type: { + name: "String" + } + }, + enabled: { + xmlName: "Enabled", + required: true, + serializedName: "Enabled", + type: { + name: "Boolean" + } + }, + includeAPIs: { + xmlName: "IncludeAPIs", + serializedName: "IncludeAPIs", + type: { + name: "Boolean" + } + }, + retentionPolicy: { + xmlName: "RetentionPolicy", + serializedName: "RetentionPolicy", + type: { + name: "Composite", + className: "RetentionPolicy" + } + } + } + } +}; + +export const PageRange: msRest.CompositeMapper = { + serializedName: "PageRange", + type: { + name: "Composite", + className: "PageRange", + modelProperties: { + start: { + xmlName: "Start", + required: true, + serializedName: "Start", + type: { + name: "Number" + } + }, + end: { + xmlName: "End", + required: true, + serializedName: "End", + type: { + name: "Number" + } + } + } + } +}; + +export const ClearRange: msRest.CompositeMapper = { + serializedName: "ClearRange", + type: { + name: "Composite", + className: "ClearRange", + modelProperties: { + start: { + xmlName: "Start", + required: true, + serializedName: "Start", + type: { + name: "Number" + } + }, + end: { + xmlName: "End", + required: true, + serializedName: "End", + type: { + name: "Number" + } + } + } + } +}; + +export const PageList: msRest.CompositeMapper = { + serializedName: "PageList", + type: { + name: "Composite", + className: "PageList", + modelProperties: { + pageRange: { + xmlName: "PageRange", + xmlElementName: "PageRange", + serializedName: "PageRange", + type: { + name: "Sequence", + element: { + type: { + name: "Composite", + className: "PageRange" + } + } + } + }, + clearRange: { + xmlName: "ClearRange", + xmlElementName: "ClearRange", + serializedName: "ClearRange", + type: { + name: "Sequence", + element: { + type: { + name: "Composite", + className: "ClearRange" + } + } + } + }, + nextMarker: { + xmlName: "NextMarker", + serializedName: "NextMarker", + type: { + name: "String" + } + } + } + } +}; + +export const QueryFormat: msRest.CompositeMapper = { + serializedName: "QueryFormat", + type: { + name: "Composite", + className: "QueryFormat", + modelProperties: { + type: { + xmlName: "Type", + required: true, + serializedName: "Type", + type: { + name: "Enum", + allowedValues: [ + "delimited", + "json", + "arrow", + "parquet" + ] + } + }, + delimitedTextConfiguration: { + xmlName: "DelimitedTextConfiguration", + serializedName: "DelimitedTextConfiguration", + type: { + name: "Composite", + className: "DelimitedTextConfiguration" + } + }, + jsonTextConfiguration: { + xmlName: "JsonTextConfiguration", + serializedName: "JsonTextConfiguration", + type: { + name: "Composite", + className: "JsonTextConfiguration" + } + }, + arrowConfiguration: { + xmlName: "ArrowConfiguration", + serializedName: "ArrowConfiguration", + type: { + name: "Composite", + className: "ArrowConfiguration" + } + }, + parquetTextConfiguration: { + xmlName: "ParquetTextConfiguration", + serializedName: "ParquetTextConfiguration", + type: { + name: "Object" + } + } + } + } +}; + +export const QuerySerialization: msRest.CompositeMapper = { + serializedName: "QuerySerialization", + type: { + name: "Composite", + className: "QuerySerialization", + modelProperties: { + format: { + xmlName: "Format", + required: true, + serializedName: "Format", + type: { + name: "Composite", + className: "QueryFormat" + } + } + } + } +}; + +export const QueryRequest: msRest.CompositeMapper = { + serializedName: "QueryRequest", + type: { + name: "Composite", + className: "QueryRequest", + modelProperties: { + queryType: { + xmlName: "QueryType", + required: true, + isConstant: true, + serializedName: "QueryType", + defaultValue: 'SQL', + type: { + name: "String" + } + }, + expression: { + xmlName: "Expression", + required: true, + serializedName: "Expression", + type: { + name: "String" + } + }, + inputSerialization: { + xmlName: "InputSerialization", + serializedName: "InputSerialization", + type: { + name: "Composite", + className: "QuerySerialization" + } + }, + outputSerialization: { + xmlName: "OutputSerialization", + serializedName: "OutputSerialization", + type: { + name: "Composite", + className: "QuerySerialization" + } + } + } + } +}; + +export const SignedIdentifier: msRest.CompositeMapper = { + serializedName: "SignedIdentifier", + type: { + name: "Composite", + className: "SignedIdentifier", + modelProperties: { + id: { + xmlName: "Id", + required: true, + serializedName: "Id", + type: { + name: "String" + } + }, + accessPolicy: { + xmlName: "AccessPolicy", + required: true, + serializedName: "AccessPolicy", + type: { + name: "Composite", + className: "AccessPolicy" + } + } + } + } +}; + +export const StaticWebsite: msRest.CompositeMapper = { + serializedName: "StaticWebsite", + type: { + name: "Composite", + className: "StaticWebsite", + modelProperties: { + enabled: { + xmlName: "Enabled", + required: true, + serializedName: "Enabled", + type: { + name: "Boolean" + } + }, + indexDocument: { + xmlName: "IndexDocument", + serializedName: "IndexDocument", + type: { + name: "String" + } + }, + errorDocument404Path: { + xmlName: "ErrorDocument404Path", + serializedName: "ErrorDocument404Path", + type: { + name: "String" + } + }, + defaultIndexDocumentPath: { + xmlName: "DefaultIndexDocumentPath", + serializedName: "DefaultIndexDocumentPath", + type: { + name: "String" + } + } + } + } +}; + +export const StorageServiceProperties: msRest.CompositeMapper = { + serializedName: "StorageServiceProperties", + type: { + name: "Composite", + className: "StorageServiceProperties", + modelProperties: { + logging: { + xmlName: "Logging", + serializedName: "Logging", + type: { + name: "Composite", + className: "Logging" + } + }, + hourMetrics: { + xmlName: "HourMetrics", + serializedName: "HourMetrics", + type: { + name: "Composite", + className: "Metrics" + } + }, + minuteMetrics: { + xmlName: "MinuteMetrics", + serializedName: "MinuteMetrics", + type: { + name: "Composite", + className: "Metrics" + } + }, + cors: { + xmlIsWrapped: true, + xmlName: "Cors", + xmlElementName: "CorsRule", + serializedName: "Cors", + type: { + name: "Sequence", + element: { + type: { + name: "Composite", + className: "CorsRule" + } + } + } + }, + defaultServiceVersion: { + xmlName: "DefaultServiceVersion", + serializedName: "DefaultServiceVersion", + type: { + name: "String" + } + }, + deleteRetentionPolicy: { + xmlName: "DeleteRetentionPolicy", + serializedName: "DeleteRetentionPolicy", + type: { + name: "Composite", + className: "RetentionPolicy" + } + }, + staticWebsite: { + xmlName: "StaticWebsite", + serializedName: "StaticWebsite", + type: { + name: "Composite", + className: "StaticWebsite" + } + } + } + } +}; + +export const StorageServiceStats: msRest.CompositeMapper = { + serializedName: "StorageServiceStats", + type: { + name: "Composite", + className: "StorageServiceStats", + modelProperties: { + geoReplication: { + xmlName: "GeoReplication", + serializedName: "GeoReplication", + type: { + name: "Composite", + className: "GeoReplication" + } + } + } + } +}; + +export const ModifiedAccessConditions: msRest.CompositeMapper = { + xmlName: "modified-access-conditions", + type: { + name: "Composite", + className: "ModifiedAccessConditions", + modelProperties: { + ifModifiedSince: { + xmlName: "ifModifiedSince", + type: { + name: "DateTimeRfc1123" + } + }, + ifUnmodifiedSince: { + xmlName: "ifUnmodifiedSince", + type: { + name: "DateTimeRfc1123" + } + }, + ifMatch: { + xmlName: "ifMatch", + type: { + name: "String" + } + }, + ifNoneMatch: { + xmlName: "ifNoneMatch", + type: { + name: "String" + } + }, + ifTags: { + xmlName: "ifTags", + type: { + name: "String" + } + } + } + } +}; + +export const PathHTTPHeaders: msRest.CompositeMapper = { + xmlName: "path-HTTP-headers", + type: { + name: "Composite", + className: "PathHTTPHeaders", + modelProperties: { + cacheControl: { + xmlName: "cacheControl", + type: { + name: "String" + } + }, + contentEncoding: { + xmlName: "contentEncoding", + type: { + name: "String" + } + }, + contentLanguage: { + xmlName: "contentLanguage", + type: { + name: "String" + } + }, + contentDisposition: { + xmlName: "contentDisposition", + type: { + name: "String" + } + }, + contentType: { + xmlName: "contentType", + type: { + name: "String" + } + }, + contentMD5: { + xmlName: "contentMD5", + type: { + name: "ByteArray" + } + }, + transactionalContentHash: { + xmlName: "transactionalContentHash", + type: { + name: "ByteArray" + } + } + } + } +}; + +export const LeaseAccessConditions: msRest.CompositeMapper = { + xmlName: "lease-access-conditions", + type: { + name: "Composite", + className: "LeaseAccessConditions", + modelProperties: { + leaseId: { + xmlName: "leaseId", + type: { + name: "String" + } + } + } + } +}; + +export const SourceModifiedAccessConditions: msRest.CompositeMapper = { + xmlName: "source-modified-access-conditions", + type: { + name: "Composite", + className: "SourceModifiedAccessConditions", + modelProperties: { + sourceIfMatch: { + xmlName: "sourceIfMatch", + type: { + name: "String" + } + }, + sourceIfNoneMatch: { + xmlName: "sourceIfNoneMatch", + type: { + name: "String" + } + }, + sourceIfModifiedSince: { + xmlName: "sourceIfModifiedSince", + type: { + name: "DateTimeRfc1123" + } + }, + sourceIfUnmodifiedSince: { + xmlName: "sourceIfUnmodifiedSince", + type: { + name: "DateTimeRfc1123" + } + }, + sourceIfTags: { + xmlName: "sourceIfTags", + type: { + name: "String" + } + } + } + } +}; + +export const CpkInfo: msRest.CompositeMapper = { + xmlName: "cpk-info", + type: { + name: "Composite", + className: "CpkInfo", + modelProperties: { + encryptionKey: { + xmlName: "encryptionKey", + type: { + name: "String" + } + }, + encryptionKeySha256: { + xmlName: "encryptionKeySha256", + type: { + name: "String" + } + }, + encryptionAlgorithm: { + xmlName: "encryptionAlgorithm", + type: { + name: "Enum", + allowedValues: [ + "AES256" + ] + } + } + } + } +}; + +export const ContainerCpkScopeInfo: msRest.CompositeMapper = { + xmlName: "container-cpk-scope-info", + type: { + name: "Composite", + className: "ContainerCpkScopeInfo", + modelProperties: { + defaultEncryptionScope: { + xmlName: "defaultEncryptionScope", + type: { + name: "String" + } + }, + preventEncryptionScopeOverride: { + xmlName: "preventEncryptionScopeOverride", + type: { + name: "Boolean" + } + } + } + } +}; + +export const BlobHTTPHeaders: msRest.CompositeMapper = { + xmlName: "blob-HTTP-headers", + type: { + name: "Composite", + className: "BlobHTTPHeaders", + modelProperties: { + blobContentType: { + xmlName: "blobContentType", + type: { + name: "String" + } + }, + blobContentEncoding: { + xmlName: "blobContentEncoding", + type: { + name: "String" + } + }, + blobContentLanguage: { + xmlName: "blobContentLanguage", + type: { + name: "String" + } + }, + blobContentMD5: { + xmlName: "blobContentMD5", + type: { + name: "ByteArray" + } + }, + blobCacheControl: { + xmlName: "blobCacheControl", + type: { + name: "String" + } + }, + blobContentDisposition: { + xmlName: "blobContentDisposition", + type: { + name: "String" + } + } + } + } +}; + +export const CpkScopeInfo: msRest.CompositeMapper = { + xmlName: "cpk-scope-info", + type: { + name: "Composite", + className: "CpkScopeInfo", + modelProperties: { + encryptionScope: { + xmlName: "encryptionScope", + type: { + name: "String" + } + } + } + } +}; + +export const SequenceNumberAccessConditions: msRest.CompositeMapper = { + xmlName: "sequence-number-access-conditions", + type: { + name: "Composite", + className: "SequenceNumberAccessConditions", + modelProperties: { + ifSequenceNumberLessThanOrEqualTo: { + xmlName: "ifSequenceNumberLessThanOrEqualTo", + type: { + name: "Number" + } + }, + ifSequenceNumberLessThan: { + xmlName: "ifSequenceNumberLessThan", + type: { + name: "Number" + } + }, + ifSequenceNumberEqualTo: { + xmlName: "ifSequenceNumberEqualTo", + type: { + name: "Number" + } + } + } + } +}; + +export const AppendPositionAccessConditions: msRest.CompositeMapper = { + xmlName: "append-position-access-conditions", + type: { + name: "Composite", + className: "AppendPositionAccessConditions", + modelProperties: { + maxSize: { + xmlName: "maxSize", + type: { + name: "Number" + } + }, + appendPosition: { + xmlName: "appendPosition", + type: { + name: "Number" + } + } + } + } +}; + +export const ServiceListFileSystemsHeaders: msRest.CompositeMapper = { + serializedName: "service-listfilesystems-headers", + type: { + name: "Composite", + className: "ServiceListFileSystemsHeaders", + modelProperties: { + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + continuation: { + serializedName: "x-ms-continuation", + type: { + name: "String" + } + }, + contentType: { + serializedName: "content-type", + type: { + name: "String" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const FileSystemCreateHeaders: msRest.CompositeMapper = { + serializedName: "filesystem-create-headers", + type: { + name: "Composite", + className: "FileSystemCreateHeaders", + modelProperties: { + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + clientRequestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + namespaceEnabled: { + serializedName: "x-ms-namespace-enabled", + type: { + name: "String" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const FileSystemSetPropertiesHeaders: msRest.CompositeMapper = { + serializedName: "filesystem-setproperties-headers", + type: { + name: "Composite", + className: "FileSystemSetPropertiesHeaders", + modelProperties: { + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const FileSystemGetPropertiesHeaders: msRest.CompositeMapper = { + serializedName: "filesystem-getproperties-headers", + type: { + name: "Composite", + className: "FileSystemGetPropertiesHeaders", + modelProperties: { + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + properties: { + serializedName: "x-ms-properties", + type: { + name: "String" + } + }, + namespaceEnabled: { + serializedName: "x-ms-namespace-enabled", + type: { + name: "String" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const FileSystemDeleteHeaders: msRest.CompositeMapper = { + serializedName: "filesystem-delete-headers", + type: { + name: "Composite", + className: "FileSystemDeleteHeaders", + modelProperties: { + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const FileSystemListPathsHeaders: msRest.CompositeMapper = { + serializedName: "filesystem-listpaths-headers", + type: { + name: "Composite", + className: "FileSystemListPathsHeaders", + modelProperties: { + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + continuation: { + serializedName: "x-ms-continuation", + type: { + name: "String" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const FileSystemListBlobFlatSegmentHeaders: msRest.CompositeMapper = { + serializedName: "filesystem-listblobflatsegment-headers", + type: { + name: "Composite", + className: "FileSystemListBlobFlatSegmentHeaders", + modelProperties: { + contentType: { + serializedName: "content-type", + type: { + name: "String" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const FileSystemListBlobHierarchySegmentHeaders: msRest.CompositeMapper = { + serializedName: "filesystem-listblobhierarchysegment-headers", + type: { + name: "Composite", + className: "FileSystemListBlobHierarchySegmentHeaders", + modelProperties: { + contentType: { + serializedName: "content-type", + type: { + name: "String" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const PathCreateHeaders: msRest.CompositeMapper = { + serializedName: "path-create-headers", + type: { + name: "Composite", + className: "PathCreateHeaders", + modelProperties: { + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + continuation: { + serializedName: "x-ms-continuation", + type: { + name: "String" + } + }, + contentLength: { + serializedName: "content-length", + type: { + name: "Number" + } + }, + isServerEncrypted: { + serializedName: "x-ms-request-server-encrypted", + type: { + name: "Boolean" + } + }, + encryptionKeySha256: { + serializedName: "x-ms-encryption-key-sha256", + type: { + name: "String" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const PathUpdateHeaders: msRest.CompositeMapper = { + serializedName: "path-update-headers", + type: { + name: "Composite", + className: "PathUpdateHeaders", + modelProperties: { + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + acceptRanges: { + serializedName: "accept-ranges", + type: { + name: "String" + } + }, + cacheControl: { + serializedName: "cache-control", + type: { + name: "String" + } + }, + contentDisposition: { + serializedName: "content-disposition", + type: { + name: "String" + } + }, + contentEncoding: { + serializedName: "content-encoding", + type: { + name: "String" + } + }, + contentLanguage: { + serializedName: "content-language", + type: { + name: "String" + } + }, + contentLength: { + serializedName: "content-length", + type: { + name: "Number" + } + }, + contentRange: { + serializedName: "content-range", + type: { + name: "String" + } + }, + contentType: { + serializedName: "content-type", + type: { + name: "String" + } + }, + contentMD5: { + serializedName: "content-md5", + type: { + name: "ByteArray" + } + }, + properties: { + serializedName: "x-ms-properties", + type: { + name: "String" + } + }, + xMsContinuation: { + serializedName: "x-ms-continuation", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const PathLeaseHeaders: msRest.CompositeMapper = { + serializedName: "path-lease-headers", + type: { + name: "Composite", + className: "PathLeaseHeaders", + modelProperties: { + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + leaseId: { + serializedName: "x-ms-lease-id", + type: { + name: "String" + } + }, + leaseTime: { + serializedName: "x-ms-lease-time", + type: { + name: "String" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const PathReadHeaders: msRest.CompositeMapper = { + serializedName: "path-read-headers", + type: { + name: "Composite", + className: "PathReadHeaders", + modelProperties: { + acceptRanges: { + serializedName: "accept-ranges", + type: { + name: "String" + } + }, + cacheControl: { + serializedName: "cache-control", + type: { + name: "String" + } + }, + contentDisposition: { + serializedName: "content-disposition", + type: { + name: "String" + } + }, + contentEncoding: { + serializedName: "content-encoding", + type: { + name: "String" + } + }, + contentLanguage: { + serializedName: "content-language", + type: { + name: "String" + } + }, + contentLength: { + serializedName: "content-length", + type: { + name: "Number" + } + }, + contentRange: { + serializedName: "content-range", + type: { + name: "String" + } + }, + contentType: { + serializedName: "content-type", + type: { + name: "String" + } + }, + contentMD5: { + serializedName: "content-md5", + type: { + name: "ByteArray" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + resourceType: { + serializedName: "x-ms-resource-type", + type: { + name: "String" + } + }, + properties: { + serializedName: "x-ms-properties", + type: { + name: "String" + } + }, + leaseDuration: { + serializedName: "x-ms-lease-duration", + type: { + name: "Enum", + allowedValues: [ + "infinite", + "fixed" + ] + } + }, + leaseState: { + serializedName: "x-ms-lease-state", + type: { + name: "Enum", + allowedValues: [ + "available", + "leased", + "expired", + "breaking", + "broken" + ] + } + }, + leaseStatus: { + serializedName: "x-ms-lease-status", + type: { + name: "Enum", + allowedValues: [ + "locked", + "unlocked" + ] + } + }, + isServerEncrypted: { + serializedName: "x-ms-request-server-encrypted", + type: { + name: "Boolean" + } + }, + encryptionKeySha256: { + serializedName: "x-ms-encryption-key-sha256", + type: { + name: "String" + } + }, + metadata: { + serializedName: "x-ms-meta", + type: { + name: "Dictionary", + value: { + type: { + name: "String" + } + } + }, + headerCollectionPrefix: "x-ms-meta-" + }, + creationTime: { + serializedName: "x-ms-creation-time", + type: { + name: "DateTimeRfc1123" + } + }, + objectReplicationPolicyId: { + serializedName: "x-ms-or-policy-id", + type: { + name: "String" + } + }, + objectReplicationRules: { + serializedName: "x-ms-or", + type: { + name: "Dictionary", + value: { + type: { + name: "String" + } + } + }, + headerCollectionPrefix: "x-ms-or-" + }, + blobSequenceNumber: { + serializedName: "x-ms-blob-sequence-number", + type: { + name: "Number" + } + }, + blobType: { + serializedName: "x-ms-blob-type", + type: { + name: "Enum", + allowedValues: [ + "BlockBlob", + "PageBlob", + "AppendBlob" + ] + } + }, + copyCompletionTime: { + serializedName: "x-ms-copy-completion-time", + type: { + name: "DateTimeRfc1123" + } + }, + copyStatusDescription: { + serializedName: "x-ms-copy-status-description", + type: { + name: "String" + } + }, + copyId: { + serializedName: "x-ms-copy-id", + type: { + name: "String" + } + }, + copyProgress: { + serializedName: "x-ms-copy-progress", + type: { + name: "String" + } + }, + copySource: { + serializedName: "x-ms-copy-source", + type: { + name: "String" + } + }, + copyStatus: { + serializedName: "x-ms-copy-status", + type: { + name: "Enum", + allowedValues: [ + "pending", + "success", + "aborted", + "failed" + ] + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + versionId: { + serializedName: "x-ms-version-id", + type: { + name: "String" + } + }, + isCurrentVersion: { + serializedName: "x-ms-is-current-version", + type: { + name: "Boolean" + } + }, + blobCommittedBlockCount: { + serializedName: "x-ms-blob-committed-block-count", + type: { + name: "Number" + } + }, + encryptionScope: { + serializedName: "x-ms-encryption-scope", + type: { + name: "String" + } + }, + blobContentMD5: { + serializedName: "x-ms-blob-content-md5", + type: { + name: "ByteArray" + } + }, + tagCount: { + serializedName: "x-ms-tag-count", + type: { + name: "Number" + } + }, + isSealed: { + serializedName: "x-ms-blob-sealed", + type: { + name: "Boolean" + } + }, + lastAccessed: { + serializedName: "x-ms-last-access-time", + type: { + name: "DateTimeRfc1123" + } + }, + immutabilityPolicyExpiresOn: { + serializedName: "x-ms-immutability-policy-until-date", + type: { + name: "DateTimeRfc1123" + } + }, + immutabilityPolicyMode: { + serializedName: "x-ms-immutability-policy-mode", + type: { + name: "String" + } + }, + legalHold: { + serializedName: "x-ms-legal-hold", + type: { + name: "Boolean" + } + }, + xMsContentMd5: { + serializedName: "x-ms-content-md5", + type: { + name: "ByteArray" + } + }, + contentCrc64: { + serializedName: "x-ms-content-crc64", + type: { + name: "ByteArray" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const PathGetPropertiesHeaders: msRest.CompositeMapper = { + serializedName: "path-getproperties-headers", + type: { + name: "Composite", + className: "PathGetPropertiesHeaders", + modelProperties: { + acceptRanges: { + serializedName: "accept-ranges", + type: { + name: "String" + } + }, + cacheControl: { + serializedName: "cache-control", + type: { + name: "String" + } + }, + contentDisposition: { + serializedName: "content-disposition", + type: { + name: "String" + } + }, + contentEncoding: { + serializedName: "content-encoding", + type: { + name: "String" + } + }, + contentLanguage: { + serializedName: "content-language", + type: { + name: "String" + } + }, + contentLength: { + serializedName: "content-length", + type: { + name: "Number" + } + }, + contentRange: { + serializedName: "content-range", + type: { + name: "String" + } + }, + contentType: { + serializedName: "content-type", + type: { + name: "String" + } + }, + contentMD5: { + serializedName: "content-md5", + type: { + name: "ByteArray" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + resourceType: { + serializedName: "x-ms-resource-type", + type: { + name: "String" + } + }, + properties: { + serializedName: "x-ms-properties", + type: { + name: "String" + } + }, + owner: { + serializedName: "x-ms-owner", + type: { + name: "String" + } + }, + group: { + serializedName: "x-ms-group", + type: { + name: "String" + } + }, + permissions: { + serializedName: "x-ms-permissions", + type: { + name: "String" + } + }, + aCL: { + serializedName: "x-ms-acl", + type: { + name: "String" + } + }, + leaseDuration: { + serializedName: "x-ms-lease-duration", + type: { + name: "Enum", + allowedValues: [ + "infinite", + "fixed" + ] + } + }, + leaseState: { + serializedName: "x-ms-lease-state", + type: { + name: "Enum", + allowedValues: [ + "available", + "leased", + "expired", + "breaking", + "broken" + ] + } + }, + leaseStatus: { + serializedName: "x-ms-lease-status", + type: { + name: "Enum", + allowedValues: [ + "locked", + "unlocked" + ] + } + }, + metadata: { + serializedName: "x-ms-meta", + type: { + name: "Dictionary", + value: { + type: { + name: "String" + } + } + }, + headerCollectionPrefix: "x-ms-meta-" + }, + creationTime: { + serializedName: "x-ms-creation-time", + type: { + name: "DateTimeRfc1123" + } + }, + objectReplicationPolicyId: { + serializedName: "x-ms-or-policy-id", + type: { + name: "String" + } + }, + objectReplicationRules: { + serializedName: "x-ms-or", + type: { + name: "Dictionary", + value: { + type: { + name: "String" + } + } + }, + headerCollectionPrefix: "x-ms-or-" + }, + blobType: { + serializedName: "x-ms-blob-type", + type: { + name: "Enum", + allowedValues: [ + "BlockBlob", + "PageBlob", + "AppendBlob" + ] + } + }, + copyCompletionTime: { + serializedName: "x-ms-copy-completion-time", + type: { + name: "DateTimeRfc1123" + } + }, + copyStatusDescription: { + serializedName: "x-ms-copy-status-description", + type: { + name: "String" + } + }, + copyId: { + serializedName: "x-ms-copy-id", + type: { + name: "String" + } + }, + copyProgress: { + serializedName: "x-ms-copy-progress", + type: { + name: "String" + } + }, + copySource: { + serializedName: "x-ms-copy-source", + type: { + name: "String" + } + }, + copyStatus: { + serializedName: "x-ms-copy-status", + type: { + name: "Enum", + allowedValues: [ + "pending", + "success", + "aborted", + "failed" + ] + } + }, + isIncrementalCopy: { + serializedName: "x-ms-incremental-copy", + type: { + name: "Boolean" + } + }, + destinationSnapshot: { + serializedName: "x-ms-copy-destination-snapshot", + type: { + name: "String" + } + }, + blobSequenceNumber: { + serializedName: "x-ms-blob-sequence-number", + type: { + name: "Number" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + blobCommittedBlockCount: { + serializedName: "x-ms-blob-committed-block-count", + type: { + name: "Number" + } + }, + isServerEncrypted: { + serializedName: "x-ms-server-encrypted", + type: { + name: "Boolean" + } + }, + encryptionKeySha256: { + serializedName: "x-ms-encryption-key-sha256", + type: { + name: "String" + } + }, + encryptionScope: { + serializedName: "x-ms-encryption-scope", + type: { + name: "String" + } + }, + accessTier: { + serializedName: "x-ms-access-tier", + type: { + name: "String" + } + }, + accessTierInferred: { + serializedName: "x-ms-access-tier-inferred", + type: { + name: "Boolean" + } + }, + archiveStatus: { + serializedName: "x-ms-archive-status", + type: { + name: "String" + } + }, + accessTierChangeTime: { + serializedName: "x-ms-access-tier-change-time", + type: { + name: "DateTimeRfc1123" + } + }, + versionId: { + serializedName: "x-ms-version-id", + type: { + name: "String" + } + }, + isCurrentVersion: { + serializedName: "x-ms-is-current-version", + type: { + name: "Boolean" + } + }, + tagCount: { + serializedName: "x-ms-tag-count", + type: { + name: "Number" + } + }, + expiresOn: { + serializedName: "x-ms-expiry-time", + type: { + name: "DateTimeRfc1123" + } + }, + isSealed: { + serializedName: "x-ms-blob-sealed", + type: { + name: "Boolean" + } + }, + rehydratePriority: { + serializedName: "x-ms-rehydrate-priority", + type: { + name: "String" + } + }, + lastAccessed: { + serializedName: "x-ms-last-access-time", + type: { + name: "DateTimeRfc1123" + } + }, + immutabilityPolicyExpiresOn: { + serializedName: "x-ms-immutability-policy-until-date", + type: { + name: "DateTimeRfc1123" + } + }, + immutabilityPolicyMode: { + serializedName: "x-ms-immutability-policy-mode", + type: { + name: "Enum", + allowedValues: [ + "Mutable", + "Unlocked", + "Locked" + ] + } + }, + legalHold: { + serializedName: "x-ms-legal-hold", + type: { + name: "Boolean" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const PathDeleteHeaders: msRest.CompositeMapper = { + serializedName: "path-delete-headers", + type: { + name: "Composite", + className: "PathDeleteHeaders", + modelProperties: { + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + continuation: { + serializedName: "x-ms-continuation", + type: { + name: "String" + } + }, + deletionId: { + serializedName: "x-ms-deletion-id", + type: { + name: "String" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const PathSetAccessControlHeaders: msRest.CompositeMapper = { + serializedName: "path-setaccesscontrol-headers", + type: { + name: "Composite", + className: "PathSetAccessControlHeaders", + modelProperties: { + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + } + } + } +}; + +export const PathSetAccessControlRecursiveHeaders: msRest.CompositeMapper = { + serializedName: "path-setaccesscontrolrecursive-headers", + type: { + name: "Composite", + className: "PathSetAccessControlRecursiveHeaders", + modelProperties: { + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + continuation: { + serializedName: "x-ms-continuation", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + } + } + } +}; + +export const PathSetPropertiesHeaders: msRest.CompositeMapper = { + serializedName: "path-setproperties-headers", + type: { + name: "Composite", + className: "PathSetPropertiesHeaders", + modelProperties: { + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + cacheControl: { + serializedName: "cache-control", + type: { + name: "String" + } + }, + contentDisposition: { + serializedName: "content-disposition", + type: { + name: "String" + } + }, + contentEncoding: { + serializedName: "content-encoding", + type: { + name: "String" + } + }, + contentLanguage: { + serializedName: "content-language", + type: { + name: "String" + } + }, + contentLength: { + serializedName: "content-length", + type: { + name: "Number" + } + }, + contentType: { + serializedName: "content-type", + type: { + name: "String" + } + }, + contentMD5: { + serializedName: "content-md5", + type: { + name: "ByteArray" + } + }, + properties: { + serializedName: "x-ms-properties", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + } + } + } +}; + +export const PathFlushDataHeaders: msRest.CompositeMapper = { + serializedName: "path-flushdata-headers", + type: { + name: "Composite", + className: "PathFlushDataHeaders", + modelProperties: { + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + contentLength: { + serializedName: "content-length", + type: { + name: "Number" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + isServerEncrypted: { + serializedName: "x-ms-request-server-encrypted", + type: { + name: "Boolean" + } + }, + encryptionKeySha256: { + serializedName: "x-ms-encryption-key-sha256", + type: { + name: "String" + } + } + } + } +}; + +export const PathAppendDataHeaders: msRest.CompositeMapper = { + serializedName: "path-appenddata-headers", + type: { + name: "Composite", + className: "PathAppendDataHeaders", + modelProperties: { + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + contentMD5: { + serializedName: "content-md5", + type: { + name: "ByteArray" + } + }, + xMsContentCrc64: { + serializedName: "x-ms-content-crc64", + type: { + name: "ByteArray" + } + }, + isServerEncrypted: { + serializedName: "x-ms-request-server-encrypted", + type: { + name: "Boolean" + } + }, + encryptionKeySha256: { + serializedName: "x-ms-encryption-key-sha256", + type: { + name: "String" + } + } + } + } +}; + +export const PathSetExpiryHeaders: msRest.CompositeMapper = { + serializedName: "path-setexpiry-headers", + type: { + name: "Composite", + className: "PathSetExpiryHeaders", + modelProperties: { + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const PathUndeleteHeaders: msRest.CompositeMapper = { + serializedName: "path-undelete-headers", + type: { + name: "Composite", + className: "PathUndeleteHeaders", + modelProperties: { + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + resourceType: { + serializedName: "x-ms-resource-type", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const ServiceSetPropertiesHeaders: msRest.CompositeMapper = { + serializedName: "service-setproperties-headers", + type: { + name: "Composite", + className: "ServiceSetPropertiesHeaders", + modelProperties: { + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const ServiceGetPropertiesHeaders: msRest.CompositeMapper = { + serializedName: "service-getproperties-headers", + type: { + name: "Composite", + className: "ServiceGetPropertiesHeaders", + modelProperties: { + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const ServiceGetStatisticsHeaders: msRest.CompositeMapper = { + serializedName: "service-getstatistics-headers", + type: { + name: "Composite", + className: "ServiceGetStatisticsHeaders", + modelProperties: { + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const ServiceListContainersSegmentHeaders: msRest.CompositeMapper = { + serializedName: "service-listcontainerssegment-headers", + type: { + name: "Composite", + className: "ServiceListContainersSegmentHeaders", + modelProperties: { + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const ServiceGetUserDelegationKeyHeaders: msRest.CompositeMapper = { + serializedName: "service-getuserdelegationkey-headers", + type: { + name: "Composite", + className: "ServiceGetUserDelegationKeyHeaders", + modelProperties: { + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const ServiceGetAccountInfoHeaders: msRest.CompositeMapper = { + serializedName: "service-getaccountinfo-headers", + type: { + name: "Composite", + className: "ServiceGetAccountInfoHeaders", + modelProperties: { + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + skuName: { + serializedName: "x-ms-sku-name", + type: { + name: "Enum", + allowedValues: [ + "Standard_LRS", + "Standard_GRS", + "Standard_RAGRS", + "Standard_ZRS", + "Premium_LRS" + ] + } + }, + accountKind: { + serializedName: "x-ms-account-kind", + type: { + name: "Enum", + allowedValues: [ + "Storage", + "BlobStorage", + "StorageV2", + "FileStorage", + "BlockBlobStorage" + ] + } + }, + isHierarchicalNamespaceEnabled: { + serializedName: "x-ms-is-hns-enabled", + type: { + name: "Boolean" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const ServiceGetAccountInfoWithHeadHeaders: msRest.CompositeMapper = { + serializedName: "service-getaccountinfowithhead-headers", + type: { + name: "Composite", + className: "ServiceGetAccountInfoWithHeadHeaders", + modelProperties: { + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + skuName: { + serializedName: "x-ms-sku-name", + type: { + name: "Enum", + allowedValues: [ + "Standard_LRS", + "Standard_GRS", + "Standard_RAGRS", + "Standard_ZRS", + "Premium_LRS" + ] + } + }, + accountKind: { + serializedName: "x-ms-account-kind", + type: { + name: "Enum", + allowedValues: [ + "Storage", + "BlobStorage", + "StorageV2", + "FileStorage", + "BlockBlobStorage" + ] + } + }, + isHierarchicalNamespaceEnabled: { + serializedName: "x-ms-is-hns-enabled", + type: { + name: "Boolean" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const ServiceSubmitBatchHeaders: msRest.CompositeMapper = { + serializedName: "service-submitbatch-headers", + type: { + name: "Composite", + className: "ServiceSubmitBatchHeaders", + modelProperties: { + contentType: { + serializedName: "content-type", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const ServiceFilterBlobsHeaders: msRest.CompositeMapper = { + serializedName: "service-filterblobs-headers", + type: { + name: "Composite", + className: "ServiceFilterBlobsHeaders", + modelProperties: { + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const ContainerCreateHeaders: msRest.CompositeMapper = { + serializedName: "container-create-headers", + type: { + name: "Composite", + className: "ContainerCreateHeaders", + modelProperties: { + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const ContainerGetPropertiesHeaders: msRest.CompositeMapper = { + serializedName: "container-getproperties-headers", + type: { + name: "Composite", + className: "ContainerGetPropertiesHeaders", + modelProperties: { + metadata: { + serializedName: "x-ms-meta", + type: { + name: "Dictionary", + value: { + type: { + name: "String" + } + } + }, + headerCollectionPrefix: "x-ms-meta-" + }, + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + leaseDuration: { + serializedName: "x-ms-lease-duration", + type: { + name: "Enum", + allowedValues: [ + "infinite", + "fixed" + ] + } + }, + leaseState: { + serializedName: "x-ms-lease-state", + type: { + name: "Enum", + allowedValues: [ + "available", + "leased", + "expired", + "breaking", + "broken" + ] + } + }, + leaseStatus: { + serializedName: "x-ms-lease-status", + type: { + name: "Enum", + allowedValues: [ + "locked", + "unlocked" + ] + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + blobPublicAccess: { + serializedName: "x-ms-blob-public-access", + type: { + name: "String" + } + }, + hasImmutabilityPolicy: { + serializedName: "x-ms-has-immutability-policy", + type: { + name: "Boolean" + } + }, + hasLegalHold: { + serializedName: "x-ms-has-legal-hold", + type: { + name: "Boolean" + } + }, + defaultEncryptionScope: { + serializedName: "x-ms-default-encryption-scope", + type: { + name: "String" + } + }, + denyEncryptionScopeOverride: { + serializedName: "x-ms-deny-encryption-scope-override", + type: { + name: "Boolean" + } + }, + isImmutableStorageWithVersioningEnabled: { + serializedName: "x-ms-immutable-storage-with-versioning-enabled", + type: { + name: "Boolean" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const ContainerGetPropertiesWithHeadHeaders: msRest.CompositeMapper = { + serializedName: "container-getpropertieswithhead-headers", + type: { + name: "Composite", + className: "ContainerGetPropertiesWithHeadHeaders", + modelProperties: { + metadata: { + serializedName: "x-ms-meta", + type: { + name: "Dictionary", + value: { + type: { + name: "String" + } + } + }, + headerCollectionPrefix: "x-ms-meta-" + }, + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + leaseDuration: { + serializedName: "x-ms-lease-duration", + type: { + name: "Enum", + allowedValues: [ + "infinite", + "fixed" + ] + } + }, + leaseState: { + serializedName: "x-ms-lease-state", + type: { + name: "Enum", + allowedValues: [ + "available", + "leased", + "expired", + "breaking", + "broken" + ] + } + }, + leaseStatus: { + serializedName: "x-ms-lease-status", + type: { + name: "Enum", + allowedValues: [ + "locked", + "unlocked" + ] + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + blobPublicAccess: { + serializedName: "x-ms-blob-public-access", + type: { + name: "String" + } + }, + hasImmutabilityPolicy: { + serializedName: "x-ms-has-immutability-policy", + type: { + name: "Boolean" + } + }, + hasLegalHold: { + serializedName: "x-ms-has-legal-hold", + type: { + name: "Boolean" + } + }, + defaultEncryptionScope: { + serializedName: "x-ms-default-encryption-scope", + type: { + name: "String" + } + }, + denyEncryptionScopeOverride: { + serializedName: "x-ms-deny-encryption-scope-override", + type: { + name: "Boolean" + } + }, + isImmutableStorageWithVersioningEnabled: { + serializedName: "x-ms-immutable-storage-with-versioning-enabled", + type: { + name: "Boolean" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const ContainerDeleteHeaders: msRest.CompositeMapper = { + serializedName: "container-delete-headers", + type: { + name: "Composite", + className: "ContainerDeleteHeaders", + modelProperties: { + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const ContainerSetMetadataHeaders: msRest.CompositeMapper = { + serializedName: "container-setmetadata-headers", + type: { + name: "Composite", + className: "ContainerSetMetadataHeaders", + modelProperties: { + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const ContainerGetAccessPolicyHeaders: msRest.CompositeMapper = { + serializedName: "container-getaccesspolicy-headers", + type: { + name: "Composite", + className: "ContainerGetAccessPolicyHeaders", + modelProperties: { + blobPublicAccess: { + serializedName: "x-ms-blob-public-access", + type: { + name: "String" + } + }, + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const ContainerSetAccessPolicyHeaders: msRest.CompositeMapper = { + serializedName: "container-setaccesspolicy-headers", + type: { + name: "Composite", + className: "ContainerSetAccessPolicyHeaders", + modelProperties: { + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const ContainerRestoreHeaders: msRest.CompositeMapper = { + serializedName: "container-restore-headers", + type: { + name: "Composite", + className: "ContainerRestoreHeaders", + modelProperties: { + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const ContainerSubmitBatchHeaders: msRest.CompositeMapper = { + serializedName: "container-submitbatch-headers", + type: { + name: "Composite", + className: "ContainerSubmitBatchHeaders", + modelProperties: { + contentType: { + serializedName: "content-type", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const ContainerFilterBlobsHeaders: msRest.CompositeMapper = { + serializedName: "container-filterblobs-headers", + type: { + name: "Composite", + className: "ContainerFilterBlobsHeaders", + modelProperties: { + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const ContainerAcquireLeaseHeaders: msRest.CompositeMapper = { + serializedName: "container-acquirelease-headers", + type: { + name: "Composite", + className: "ContainerAcquireLeaseHeaders", + modelProperties: { + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + leaseId: { + serializedName: "x-ms-lease-id", + type: { + name: "String" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const ContainerReleaseLeaseHeaders: msRest.CompositeMapper = { + serializedName: "container-releaselease-headers", + type: { + name: "Composite", + className: "ContainerReleaseLeaseHeaders", + modelProperties: { + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const ContainerRenewLeaseHeaders: msRest.CompositeMapper = { + serializedName: "container-renewlease-headers", + type: { + name: "Composite", + className: "ContainerRenewLeaseHeaders", + modelProperties: { + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + leaseId: { + serializedName: "x-ms-lease-id", + type: { + name: "String" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const ContainerBreakLeaseHeaders: msRest.CompositeMapper = { + serializedName: "container-breaklease-headers", + type: { + name: "Composite", + className: "ContainerBreakLeaseHeaders", + modelProperties: { + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + leaseTime: { + serializedName: "x-ms-lease-time", + type: { + name: "Number" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const ContainerChangeLeaseHeaders: msRest.CompositeMapper = { + serializedName: "container-changelease-headers", + type: { + name: "Composite", + className: "ContainerChangeLeaseHeaders", + modelProperties: { + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + leaseId: { + serializedName: "x-ms-lease-id", + type: { + name: "String" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const ContainerGetAccountInfoHeaders: msRest.CompositeMapper = { + serializedName: "container-getaccountinfo-headers", + type: { + name: "Composite", + className: "ContainerGetAccountInfoHeaders", + modelProperties: { + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + skuName: { + serializedName: "x-ms-sku-name", + type: { + name: "Enum", + allowedValues: [ + "Standard_LRS", + "Standard_GRS", + "Standard_RAGRS", + "Standard_ZRS", + "Premium_LRS" + ] + } + }, + accountKind: { + serializedName: "x-ms-account-kind", + type: { + name: "Enum", + allowedValues: [ + "Storage", + "BlobStorage", + "StorageV2", + "FileStorage", + "BlockBlobStorage" + ] + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const ContainerGetAccountInfoWithHeadHeaders: msRest.CompositeMapper = { + serializedName: "container-getaccountinfowithhead-headers", + type: { + name: "Composite", + className: "ContainerGetAccountInfoWithHeadHeaders", + modelProperties: { + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + skuName: { + serializedName: "x-ms-sku-name", + type: { + name: "Enum", + allowedValues: [ + "Standard_LRS", + "Standard_GRS", + "Standard_RAGRS", + "Standard_ZRS", + "Premium_LRS" + ] + } + }, + accountKind: { + serializedName: "x-ms-account-kind", + type: { + name: "Enum", + allowedValues: [ + "Storage", + "BlobStorage", + "StorageV2", + "FileStorage", + "BlockBlobStorage" + ] + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const PageBlobCreateHeaders: msRest.CompositeMapper = { + serializedName: "pageblob-create-headers", + type: { + name: "Composite", + className: "PageBlobCreateHeaders", + modelProperties: { + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + contentMD5: { + serializedName: "content-md5", + type: { + name: "ByteArray" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + versionId: { + serializedName: "x-ms-version-id", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + isServerEncrypted: { + serializedName: "x-ms-request-server-encrypted", + type: { + name: "Boolean" + } + }, + encryptionKeySha256: { + serializedName: "x-ms-encryption-key-sha256", + type: { + name: "String" + } + }, + encryptionScope: { + serializedName: "x-ms-encryption-scope", + type: { + name: "String" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const AppendBlobCreateHeaders: msRest.CompositeMapper = { + serializedName: "appendblob-create-headers", + type: { + name: "Composite", + className: "AppendBlobCreateHeaders", + modelProperties: { + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + contentMD5: { + serializedName: "content-md5", + type: { + name: "ByteArray" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + versionId: { + serializedName: "x-ms-version-id", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + isServerEncrypted: { + serializedName: "x-ms-request-server-encrypted", + type: { + name: "Boolean" + } + }, + encryptionKeySha256: { + serializedName: "x-ms-encryption-key-sha256", + type: { + name: "String" + } + }, + encryptionScope: { + serializedName: "x-ms-encryption-scope", + type: { + name: "String" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const BlockBlobUploadHeaders: msRest.CompositeMapper = { + serializedName: "blockblob-upload-headers", + type: { + name: "Composite", + className: "BlockBlobUploadHeaders", + modelProperties: { + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + contentMD5: { + serializedName: "content-md5", + type: { + name: "ByteArray" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + versionId: { + serializedName: "x-ms-version-id", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + isServerEncrypted: { + serializedName: "x-ms-request-server-encrypted", + type: { + name: "Boolean" + } + }, + encryptionKeySha256: { + serializedName: "x-ms-encryption-key-sha256", + type: { + name: "String" + } + }, + encryptionScope: { + serializedName: "x-ms-encryption-scope", + type: { + name: "String" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const BlockBlobPutBlobFromUrlHeaders: msRest.CompositeMapper = { + serializedName: "blockblob-putblobfromurl-headers", + type: { + name: "Composite", + className: "BlockBlobPutBlobFromUrlHeaders", + modelProperties: { + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + contentMD5: { + serializedName: "content-md5", + type: { + name: "ByteArray" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + versionId: { + serializedName: "x-ms-version-id", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + isServerEncrypted: { + serializedName: "x-ms-request-server-encrypted", + type: { + name: "Boolean" + } + }, + encryptionKeySha256: { + serializedName: "x-ms-encryption-key-sha256", + type: { + name: "String" + } + }, + encryptionScope: { + serializedName: "x-ms-encryption-scope", + type: { + name: "String" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const BlobUndeleteHeaders: msRest.CompositeMapper = { + serializedName: "blob-undelete-headers", + type: { + name: "Composite", + className: "BlobUndeleteHeaders", + modelProperties: { + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const BlobSetExpiryHeaders: msRest.CompositeMapper = { + serializedName: "blob-setexpiry-headers", + type: { + name: "Composite", + className: "BlobSetExpiryHeaders", + modelProperties: { + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const BlobSetHTTPHeadersHeaders: msRest.CompositeMapper = { + serializedName: "blob-sethttpheaders-headers", + type: { + name: "Composite", + className: "BlobSetHTTPHeadersHeaders", + modelProperties: { + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + blobSequenceNumber: { + serializedName: "x-ms-blob-sequence-number", + type: { + name: "Number" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const BlobSetImmutabilityPolicyHeaders: msRest.CompositeMapper = { + serializedName: "blob-setimmutabilitypolicy-headers", + type: { + name: "Composite", + className: "BlobSetImmutabilityPolicyHeaders", + modelProperties: { + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + immutabilityPolicyExpiry: { + serializedName: "x-ms-immutability-policy-until-date", + type: { + name: "DateTimeRfc1123" + } + }, + immutabilityPolicyMode: { + serializedName: "x-ms-immutability-policy-mode", + type: { + name: "Enum", + allowedValues: [ + "Mutable", + "Unlocked", + "Locked" + ] + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const BlobDeleteImmutabilityPolicyHeaders: msRest.CompositeMapper = { + serializedName: "blob-deleteimmutabilitypolicy-headers", + type: { + name: "Composite", + className: "BlobDeleteImmutabilityPolicyHeaders", + modelProperties: { + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const BlobSetLegalHoldHeaders: msRest.CompositeMapper = { + serializedName: "blob-setlegalhold-headers", + type: { + name: "Composite", + className: "BlobSetLegalHoldHeaders", + modelProperties: { + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + legalHold: { + serializedName: "x-ms-legal-hold", + type: { + name: "Boolean" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const BlobSetMetadataHeaders: msRest.CompositeMapper = { + serializedName: "blob-setmetadata-headers", + type: { + name: "Composite", + className: "BlobSetMetadataHeaders", + modelProperties: { + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + versionId: { + serializedName: "x-ms-version-id", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + isServerEncrypted: { + serializedName: "x-ms-request-server-encrypted", + type: { + name: "Boolean" + } + }, + encryptionKeySha256: { + serializedName: "x-ms-encryption-key-sha256", + type: { + name: "String" + } + }, + encryptionScope: { + serializedName: "x-ms-encryption-scope", + type: { + name: "String" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const BlobAcquireLeaseHeaders: msRest.CompositeMapper = { + serializedName: "blob-acquirelease-headers", + type: { + name: "Composite", + className: "BlobAcquireLeaseHeaders", + modelProperties: { + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + leaseId: { + serializedName: "x-ms-lease-id", + type: { + name: "String" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const BlobReleaseLeaseHeaders: msRest.CompositeMapper = { + serializedName: "blob-releaselease-headers", + type: { + name: "Composite", + className: "BlobReleaseLeaseHeaders", + modelProperties: { + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const BlobRenewLeaseHeaders: msRest.CompositeMapper = { + serializedName: "blob-renewlease-headers", + type: { + name: "Composite", + className: "BlobRenewLeaseHeaders", + modelProperties: { + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + leaseId: { + serializedName: "x-ms-lease-id", + type: { + name: "String" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const BlobChangeLeaseHeaders: msRest.CompositeMapper = { + serializedName: "blob-changelease-headers", + type: { + name: "Composite", + className: "BlobChangeLeaseHeaders", + modelProperties: { + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + leaseId: { + serializedName: "x-ms-lease-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const BlobBreakLeaseHeaders: msRest.CompositeMapper = { + serializedName: "blob-breaklease-headers", + type: { + name: "Composite", + className: "BlobBreakLeaseHeaders", + modelProperties: { + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + leaseTime: { + serializedName: "x-ms-lease-time", + type: { + name: "Number" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const BlobCreateSnapshotHeaders: msRest.CompositeMapper = { + serializedName: "blob-createsnapshot-headers", + type: { + name: "Composite", + className: "BlobCreateSnapshotHeaders", + modelProperties: { + snapshot: { + serializedName: "x-ms-snapshot", + type: { + name: "String" + } + }, + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + versionId: { + serializedName: "x-ms-version-id", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + isServerEncrypted: { + serializedName: "x-ms-request-server-encrypted", + type: { + name: "Boolean" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const BlobStartCopyFromURLHeaders: msRest.CompositeMapper = { + serializedName: "blob-startcopyfromurl-headers", + type: { + name: "Composite", + className: "BlobStartCopyFromURLHeaders", + modelProperties: { + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + versionId: { + serializedName: "x-ms-version-id", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + copyId: { + serializedName: "x-ms-copy-id", + type: { + name: "String" + } + }, + copyStatus: { + serializedName: "x-ms-copy-status", + type: { + name: "Enum", + allowedValues: [ + "pending", + "success", + "aborted", + "failed" + ] + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const BlobCopyFromURLHeaders: msRest.CompositeMapper = { + serializedName: "blob-copyfromurl-headers", + type: { + name: "Composite", + className: "BlobCopyFromURLHeaders", + modelProperties: { + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + versionId: { + serializedName: "x-ms-version-id", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + copyId: { + serializedName: "x-ms-copy-id", + type: { + name: "String" + } + }, + copyStatus: { + serializedName: "x-ms-copy-status", + type: { + name: "Enum", + allowedValues: [ + "success" + ] + } + }, + contentMD5: { + serializedName: "content-md5", + type: { + name: "ByteArray" + } + }, + xMsContentCrc64: { + serializedName: "x-ms-content-crc64", + type: { + name: "ByteArray" + } + }, + encryptionScope: { + serializedName: "x-ms-encryption-scope", + type: { + name: "String" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const BlobAbortCopyFromURLHeaders: msRest.CompositeMapper = { + serializedName: "blob-abortcopyfromurl-headers", + type: { + name: "Composite", + className: "BlobAbortCopyFromURLHeaders", + modelProperties: { + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const BlobSetTierHeaders: msRest.CompositeMapper = { + serializedName: "blob-settier-headers", + type: { + name: "Composite", + className: "BlobSetTierHeaders", + modelProperties: { + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const BlobGetAccountInfoHeaders: msRest.CompositeMapper = { + serializedName: "blob-getaccountinfo-headers", + type: { + name: "Composite", + className: "BlobGetAccountInfoHeaders", + modelProperties: { + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + skuName: { + serializedName: "x-ms-sku-name", + type: { + name: "Enum", + allowedValues: [ + "Standard_LRS", + "Standard_GRS", + "Standard_RAGRS", + "Standard_ZRS", + "Premium_LRS" + ] + } + }, + accountKind: { + serializedName: "x-ms-account-kind", + type: { + name: "Enum", + allowedValues: [ + "Storage", + "BlobStorage", + "StorageV2", + "FileStorage", + "BlockBlobStorage" + ] + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const BlobGetAccountInfoWithHeadHeaders: msRest.CompositeMapper = { + serializedName: "blob-getaccountinfowithhead-headers", + type: { + name: "Composite", + className: "BlobGetAccountInfoWithHeadHeaders", + modelProperties: { + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + skuName: { + serializedName: "x-ms-sku-name", + type: { + name: "Enum", + allowedValues: [ + "Standard_LRS", + "Standard_GRS", + "Standard_RAGRS", + "Standard_ZRS", + "Premium_LRS" + ] + } + }, + accountKind: { + serializedName: "x-ms-account-kind", + type: { + name: "Enum", + allowedValues: [ + "Storage", + "BlobStorage", + "StorageV2", + "FileStorage", + "BlockBlobStorage" + ] + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const BlockBlobStageBlockHeaders: msRest.CompositeMapper = { + serializedName: "blockblob-stageblock-headers", + type: { + name: "Composite", + className: "BlockBlobStageBlockHeaders", + modelProperties: { + contentMD5: { + serializedName: "content-md5", + type: { + name: "ByteArray" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + xMsContentCrc64: { + serializedName: "x-ms-content-crc64", + type: { + name: "ByteArray" + } + }, + isServerEncrypted: { + serializedName: "x-ms-request-server-encrypted", + type: { + name: "Boolean" + } + }, + encryptionKeySha256: { + serializedName: "x-ms-encryption-key-sha256", + type: { + name: "String" + } + }, + encryptionScope: { + serializedName: "x-ms-encryption-scope", + type: { + name: "String" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const BlockBlobStageBlockFromURLHeaders: msRest.CompositeMapper = { + serializedName: "blockblob-stageblockfromurl-headers", + type: { + name: "Composite", + className: "BlockBlobStageBlockFromURLHeaders", + modelProperties: { + contentMD5: { + serializedName: "content-md5", + type: { + name: "ByteArray" + } + }, + xMsContentCrc64: { + serializedName: "x-ms-content-crc64", + type: { + name: "ByteArray" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + isServerEncrypted: { + serializedName: "x-ms-request-server-encrypted", + type: { + name: "Boolean" + } + }, + encryptionKeySha256: { + serializedName: "x-ms-encryption-key-sha256", + type: { + name: "String" + } + }, + encryptionScope: { + serializedName: "x-ms-encryption-scope", + type: { + name: "String" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const BlockBlobCommitBlockListHeaders: msRest.CompositeMapper = { + serializedName: "blockblob-commitblocklist-headers", + type: { + name: "Composite", + className: "BlockBlobCommitBlockListHeaders", + modelProperties: { + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + contentMD5: { + serializedName: "content-md5", + type: { + name: "ByteArray" + } + }, + xMsContentCrc64: { + serializedName: "x-ms-content-crc64", + type: { + name: "ByteArray" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + versionId: { + serializedName: "x-ms-version-id", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + isServerEncrypted: { + serializedName: "x-ms-request-server-encrypted", + type: { + name: "Boolean" + } + }, + encryptionKeySha256: { + serializedName: "x-ms-encryption-key-sha256", + type: { + name: "String" + } + }, + encryptionScope: { + serializedName: "x-ms-encryption-scope", + type: { + name: "String" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const BlockBlobGetBlockListHeaders: msRest.CompositeMapper = { + serializedName: "blockblob-getblocklist-headers", + type: { + name: "Composite", + className: "BlockBlobGetBlockListHeaders", + modelProperties: { + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + contentType: { + serializedName: "content-type", + type: { + name: "String" + } + }, + blobContentLength: { + serializedName: "x-ms-blob-content-length", + type: { + name: "Number" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const PageBlobUploadPagesHeaders: msRest.CompositeMapper = { + serializedName: "pageblob-uploadpages-headers", + type: { + name: "Composite", + className: "PageBlobUploadPagesHeaders", + modelProperties: { + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + contentMD5: { + serializedName: "content-md5", + type: { + name: "ByteArray" + } + }, + xMsContentCrc64: { + serializedName: "x-ms-content-crc64", + type: { + name: "ByteArray" + } + }, + blobSequenceNumber: { + serializedName: "x-ms-blob-sequence-number", + type: { + name: "Number" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + isServerEncrypted: { + serializedName: "x-ms-request-server-encrypted", + type: { + name: "Boolean" + } + }, + encryptionKeySha256: { + serializedName: "x-ms-encryption-key-sha256", + type: { + name: "String" + } + }, + encryptionScope: { + serializedName: "x-ms-encryption-scope", + type: { + name: "String" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const PageBlobClearPagesHeaders: msRest.CompositeMapper = { + serializedName: "pageblob-clearpages-headers", + type: { + name: "Composite", + className: "PageBlobClearPagesHeaders", + modelProperties: { + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + contentMD5: { + serializedName: "content-md5", + type: { + name: "ByteArray" + } + }, + xMsContentCrc64: { + serializedName: "x-ms-content-crc64", + type: { + name: "ByteArray" + } + }, + blobSequenceNumber: { + serializedName: "x-ms-blob-sequence-number", + type: { + name: "Number" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const PageBlobUploadPagesFromURLHeaders: msRest.CompositeMapper = { + serializedName: "pageblob-uploadpagesfromurl-headers", + type: { + name: "Composite", + className: "PageBlobUploadPagesFromURLHeaders", + modelProperties: { + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + contentMD5: { + serializedName: "content-md5", + type: { + name: "ByteArray" + } + }, + xMsContentCrc64: { + serializedName: "x-ms-content-crc64", + type: { + name: "ByteArray" + } + }, + blobSequenceNumber: { + serializedName: "x-ms-blob-sequence-number", + type: { + name: "Number" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + isServerEncrypted: { + serializedName: "x-ms-request-server-encrypted", + type: { + name: "Boolean" + } + }, + encryptionKeySha256: { + serializedName: "x-ms-encryption-key-sha256", + type: { + name: "String" + } + }, + encryptionScope: { + serializedName: "x-ms-encryption-scope", + type: { + name: "String" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const PageBlobGetPageRangesHeaders: msRest.CompositeMapper = { + serializedName: "pageblob-getpageranges-headers", + type: { + name: "Composite", + className: "PageBlobGetPageRangesHeaders", + modelProperties: { + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + blobContentLength: { + serializedName: "x-ms-blob-content-length", + type: { + name: "Number" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const PageBlobGetPageRangesDiffHeaders: msRest.CompositeMapper = { + serializedName: "pageblob-getpagerangesdiff-headers", + type: { + name: "Composite", + className: "PageBlobGetPageRangesDiffHeaders", + modelProperties: { + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + blobContentLength: { + serializedName: "x-ms-blob-content-length", + type: { + name: "Number" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const PageBlobResizeHeaders: msRest.CompositeMapper = { + serializedName: "pageblob-resize-headers", + type: { + name: "Composite", + className: "PageBlobResizeHeaders", + modelProperties: { + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + blobSequenceNumber: { + serializedName: "x-ms-blob-sequence-number", + type: { + name: "Number" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const PageBlobUpdateSequenceNumberHeaders: msRest.CompositeMapper = { + serializedName: "pageblob-updatesequencenumber-headers", + type: { + name: "Composite", + className: "PageBlobUpdateSequenceNumberHeaders", + modelProperties: { + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + blobSequenceNumber: { + serializedName: "x-ms-blob-sequence-number", + type: { + name: "Number" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const PageBlobCopyIncrementalHeaders: msRest.CompositeMapper = { + serializedName: "pageblob-copyincremental-headers", + type: { + name: "Composite", + className: "PageBlobCopyIncrementalHeaders", + modelProperties: { + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + copyId: { + serializedName: "x-ms-copy-id", + type: { + name: "String" + } + }, + copyStatus: { + serializedName: "x-ms-copy-status", + type: { + name: "Enum", + allowedValues: [ + "pending", + "success", + "aborted", + "failed" + ] + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const AppendBlobAppendBlockHeaders: msRest.CompositeMapper = { + serializedName: "appendblob-appendblock-headers", + type: { + name: "Composite", + className: "AppendBlobAppendBlockHeaders", + modelProperties: { + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + contentMD5: { + serializedName: "content-md5", + type: { + name: "ByteArray" + } + }, + xMsContentCrc64: { + serializedName: "x-ms-content-crc64", + type: { + name: "ByteArray" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + blobAppendOffset: { + serializedName: "x-ms-blob-append-offset", + type: { + name: "String" + } + }, + blobCommittedBlockCount: { + serializedName: "x-ms-blob-committed-block-count", + type: { + name: "Number" + } + }, + isServerEncrypted: { + serializedName: "x-ms-request-server-encrypted", + type: { + name: "Boolean" + } + }, + encryptionKeySha256: { + serializedName: "x-ms-encryption-key-sha256", + type: { + name: "String" + } + }, + encryptionScope: { + serializedName: "x-ms-encryption-scope", + type: { + name: "String" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const AppendBlobAppendBlockFromUrlHeaders: msRest.CompositeMapper = { + serializedName: "appendblob-appendblockfromurl-headers", + type: { + name: "Composite", + className: "AppendBlobAppendBlockFromUrlHeaders", + modelProperties: { + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + contentMD5: { + serializedName: "content-md5", + type: { + name: "ByteArray" + } + }, + xMsContentCrc64: { + serializedName: "x-ms-content-crc64", + type: { + name: "ByteArray" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + blobAppendOffset: { + serializedName: "x-ms-blob-append-offset", + type: { + name: "String" + } + }, + blobCommittedBlockCount: { + serializedName: "x-ms-blob-committed-block-count", + type: { + name: "Number" + } + }, + encryptionKeySha256: { + serializedName: "x-ms-encryption-key-sha256", + type: { + name: "String" + } + }, + encryptionScope: { + serializedName: "x-ms-encryption-scope", + type: { + name: "String" + } + }, + isServerEncrypted: { + serializedName: "x-ms-request-server-encrypted", + type: { + name: "Boolean" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const AppendBlobSealHeaders: msRest.CompositeMapper = { + serializedName: "appendblob-seal-headers", + type: { + name: "Composite", + className: "AppendBlobSealHeaders", + modelProperties: { + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + isSealed: { + serializedName: "x-ms-blob-sealed", + type: { + name: "Boolean" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const BlobQueryHeaders: msRest.CompositeMapper = { + serializedName: "blob-query-headers", + type: { + name: "Composite", + className: "BlobQueryHeaders", + modelProperties: { + lastModified: { + serializedName: "last-modified", + type: { + name: "DateTimeRfc1123" + } + }, + metadata: { + serializedName: "x-ms-meta", + type: { + name: "Dictionary", + value: { + type: { + name: "String" + } + } + }, + headerCollectionPrefix: "x-ms-meta-" + }, + contentLength: { + serializedName: "content-length", + type: { + name: "Number" + } + }, + contentType: { + serializedName: "content-type", + type: { + name: "String" + } + }, + contentRange: { + serializedName: "content-range", + type: { + name: "String" + } + }, + eTag: { + serializedName: "etag", + type: { + name: "String" + } + }, + contentMD5: { + serializedName: "content-md5", + type: { + name: "ByteArray" + } + }, + contentEncoding: { + serializedName: "content-encoding", + type: { + name: "String" + } + }, + cacheControl: { + serializedName: "cache-control", + type: { + name: "String" + } + }, + contentDisposition: { + serializedName: "content-disposition", + type: { + name: "String" + } + }, + contentLanguage: { + serializedName: "content-language", + type: { + name: "String" + } + }, + blobSequenceNumber: { + serializedName: "x-ms-blob-sequence-number", + type: { + name: "Number" + } + }, + blobType: { + serializedName: "x-ms-blob-type", + type: { + name: "Enum", + allowedValues: [ + "BlockBlob", + "PageBlob", + "AppendBlob" + ] + } + }, + copyCompletionTime: { + serializedName: "x-ms-copy-completion-time", + type: { + name: "DateTimeRfc1123" + } + }, + copyStatusDescription: { + serializedName: "x-ms-copy-status-description", + type: { + name: "String" + } + }, + copyId: { + serializedName: "x-ms-copy-id", + type: { + name: "String" + } + }, + copyProgress: { + serializedName: "x-ms-copy-progress", + type: { + name: "String" + } + }, + copySource: { + serializedName: "x-ms-copy-source", + type: { + name: "String" + } + }, + copyStatus: { + serializedName: "x-ms-copy-status", + type: { + name: "Enum", + allowedValues: [ + "pending", + "success", + "aborted", + "failed" + ] + } + }, + leaseDuration: { + serializedName: "x-ms-lease-duration", + type: { + name: "Enum", + allowedValues: [ + "infinite", + "fixed" + ] + } + }, + leaseState: { + serializedName: "x-ms-lease-state", + type: { + name: "Enum", + allowedValues: [ + "available", + "leased", + "expired", + "breaking", + "broken" + ] + } + }, + leaseStatus: { + serializedName: "x-ms-lease-status", + type: { + name: "Enum", + allowedValues: [ + "locked", + "unlocked" + ] + } + }, + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + acceptRanges: { + serializedName: "accept-ranges", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + blobCommittedBlockCount: { + serializedName: "x-ms-blob-committed-block-count", + type: { + name: "Number" + } + }, + isServerEncrypted: { + serializedName: "x-ms-server-encrypted", + type: { + name: "Boolean" + } + }, + encryptionKeySha256: { + serializedName: "x-ms-encryption-key-sha256", + type: { + name: "String" + } + }, + encryptionScope: { + serializedName: "x-ms-encryption-scope", + type: { + name: "String" + } + }, + blobContentMD5: { + serializedName: "x-ms-blob-content-md5", + type: { + name: "ByteArray" + } + }, + contentCrc64: { + serializedName: "x-ms-content-crc64", + type: { + name: "ByteArray" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const BlobGetTagsHeaders: msRest.CompositeMapper = { + serializedName: "blob-gettags-headers", + type: { + name: "Composite", + className: "BlobGetTagsHeaders", + modelProperties: { + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; + +export const BlobSetTagsHeaders: msRest.CompositeMapper = { + serializedName: "blob-settags-headers", + type: { + name: "Composite", + className: "BlobSetTagsHeaders", + modelProperties: { + clientRequestId: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + }, + requestId: { + serializedName: "x-ms-request-id", + type: { + name: "String" + } + }, + version: { + serializedName: "x-ms-version", + type: { + name: "String" + } + }, + date: { + serializedName: "date", + type: { + name: "DateTimeRfc1123" + } + }, + errorCode: { + serializedName: "x-ms-error-code", + type: { + name: "String" + } + } + } + } +}; diff --git a/src/dfs/generated/artifacts/models.ts b/src/dfs/generated/artifacts/models.ts new file mode 100644 index 000000000..11b5f0268 --- /dev/null +++ b/src/dfs/generated/artifacts/models.ts @@ -0,0 +1,10268 @@ +/* + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + * + * Code generated by Microsoft (R) AutoRest Code Generator. + * Changes may cause incorrect behavior and will be lost if the code is regenerated. + */ + +// tslint:disable:max-line-length +// tslint:disable:interface-name +// tslint:disable:quotemark + +/** + * An interface representing AclFailedEntry. + */ +export interface AclFailedEntry { + name?: string; + type?: string; + errorMessage?: string; +} + +/** + * An interface representing SetAccessControlRecursiveResponse. + */ +export interface SetAccessControlRecursiveResponse { + directoriesSuccessful?: number; + filesSuccessful?: number; + failureCount?: number; + failedEntries?: AclFailedEntry[]; +} + +/** + * An interface representing Path. + */ +export interface Path { + name?: string; + /** + * Default value: false. + */ + isDirectory?: boolean; + lastModified?: Date; + etag?: string; + contentLength?: number; + owner?: string; + group?: string; + permissions?: string; + /** + * The name of the encryption scope under which the blob is encrypted. + */ + encryptionScope?: string; +} + +/** + * An interface representing PathList. + */ +export interface PathList { + paths?: Path[]; +} + +/** + * An interface representing FileSystem. + */ +export interface FileSystem { + name?: string; + lastModified?: string; + eTag?: string; +} + +/** + * Properties of a blob + */ +export interface BlobPropertiesInternal { + creationTime?: Date; + lastModified: Date; + etag: string; + /** + * Size in bytes + */ + contentLength?: number; + contentType?: string; + contentEncoding?: string; + contentLanguage?: string; + contentMD5?: Uint8Array; + contentDisposition?: string; + cacheControl?: string; + blobSequenceNumber?: number; + copyId?: string; + copySource?: string; + copyProgress?: string; + copyCompletionTime?: Date; + copyStatusDescription?: string; + serverEncrypted?: boolean; + incrementalCopy?: boolean; + destinationSnapshot?: string; + deletedTime?: Date; + remainingRetentionDays?: number; + accessTierInferred?: boolean; + customerProvidedKeySha256?: string; + /** + * The name of the encryption scope under which the blob is encrypted. + */ + encryptionScope?: string; + accessTierChangeTime?: Date; + tagCount?: number; + expiresOn?: Date; + isSealed?: boolean; + lastAccessedOn?: Date; + deleteTime?: Date; + /** + * Possible values include: 'BlockBlob', 'PageBlob', 'AppendBlob' + */ + blobType?: BlobType; + /** + * Possible values include: 'locked', 'unlocked' + */ + leaseStatus?: LeaseStatusType; + /** + * Possible values include: 'available', 'leased', 'expired', 'breaking', 'broken' + */ + leaseState?: LeaseStateType; + /** + * Possible values include: 'infinite', 'fixed' + */ + leaseDuration?: LeaseDurationType; + /** + * Possible values include: 'pending', 'success', 'aborted', 'failed' + */ + copyStatus?: CopyStatusType; + /** + * Possible values include: 'P4', 'P6', 'P10', 'P15', 'P20', 'P30', 'P40', 'P50', 'P60', 'P70', + * 'P80', 'Hot', 'Cool', 'Archive', 'Premium' + */ + accessTier?: AccessTier; + /** + * Possible values include: 'rehydrate-pending-to-hot', 'rehydrate-pending-to-cool' + */ + archiveStatus?: ArchiveStatus; + /** + * Possible values include: 'High', 'Standard' + */ + rehydratePriority?: RehydratePriority; + immutabilityPolicyExpiresOn?: Date; + /** + * Possible values include: 'Mutable', 'Unlocked', 'Locked' + */ + immutabilityPolicyMode?: BlobImmutabilityPolicyMode; + legalHold?: boolean; + properties?: string; +} + +/** + * An interface representing BlobMetadata. + */ +export interface BlobMetadata { + encrypted?: string; + /** + * Describes unknown properties. The value of an unknown property MUST be of type "string". Due + * to valid TS constraints we have modeled this as a union of `string | any`. + */ + [property: string]: string | any; +} + +/** + * An interface representing BlobTag. + */ +export interface BlobTag { + key: string; + value: string; +} + +/** + * Blob tags + */ +export interface BlobTags { + blobTagSet: BlobTag[]; +} + +/** + * An Azure Storage blob + */ +export interface BlobItemInternal { + name: string; + deleted?: boolean; + snapshot?: string; + versionId?: string; + isCurrentVersion?: boolean; + properties: BlobPropertiesInternal; + deletionId?: string; + metadata?: BlobMetadata; + blobTags?: BlobTags; + objectReplicationMetadata?: { [propertyName: string]: string }; + hasVersionsOnly?: boolean; +} + +/** + * An interface representing BlobFlatListSegment. + */ +export interface BlobFlatListSegment { + blobItems: BlobItemInternal[]; +} + +/** + * An enumeration of blobs + */ +export interface ListBlobsFlatSegmentResponse { + serviceEndpoint: string; + containerName: string; + prefix?: string; + marker?: string; + maxResults?: number; + segment: BlobFlatListSegment; + nextMarker?: string; +} + +/** + * An interface representing BlobPrefix. + */ +export interface BlobPrefix { + name: string; +} + +/** + * An interface representing BlobHierarchyListSegment. + */ +export interface BlobHierarchyListSegment { + blobPrefixes?: BlobPrefix[]; + blobItems: BlobItemInternal[]; +} + +/** + * An enumeration of blobs + */ +export interface ListBlobsHierarchySegmentResponse { + serviceEndpoint: string; + containerName: string; + prefix?: string; + marker?: string; + maxResults?: number; + delimiter?: string; + segment: BlobHierarchyListSegment; + nextMarker?: string; +} + +/** + * An interface representing FileSystemList. + */ +export interface FileSystemList { + filesystems?: FileSystem[]; +} + +/** + * The service error response object. + */ +export interface StorageErrorError { + /** + * The service error code. + */ + code?: string; + /** + * The service error message. + */ + message?: string; +} + +/** + * An interface representing StorageError. + */ +export interface StorageError { + /** + * The service error message. + */ + message?: string; + /** + * The service error response object. + */ + error?: StorageErrorError; +} + +/** + * Key information + */ +export interface KeyInfo { + /** + * The date-time the key is active in ISO 8601 UTC time + */ + start: string; + /** + * The date-time the key expires in ISO 8601 UTC time + */ + expiry: string; +} + +/** + * A user delegation key + */ +export interface UserDelegationKey { + /** + * The Azure Active Directory object ID in GUID format. + */ + signedOid: string; + /** + * The Azure Active Directory tenant ID in GUID format + */ + signedTid: string; + /** + * The date-time the key is active + * **NOTE: This entity will be treated as a string instead of a Date because the API can + * potentially deal with a higher precision value than what is supported by JavaScript.** + */ + signedStart: string; + /** + * The date-time the key expires + * **NOTE: This entity will be treated as a string instead of a Date because the API can + * potentially deal with a higher precision value than what is supported by JavaScript.** + */ + signedExpiry: string; + /** + * Abbreviation of the Azure Storage service that accepts the key + */ + signedService: string; + /** + * The service version that created the key + */ + signedVersion: string; + /** + * The key as a base64 string + */ + value: string; +} + +/** + * An interface representing StorageErrorBlob. + */ +export interface StorageErrorBlob { + message?: string; +} + +/** + * An Access policy + */ +export interface AccessPolicy { + /** + * the date-time the policy is active + * **NOTE: This entity will be treated as a string instead of a Date because the API can + * potentially deal with a higher precision value than what is supported by JavaScript.** + */ + start?: string; + /** + * the date-time the policy expires + * **NOTE: This entity will be treated as a string instead of a Date because the API can + * potentially deal with a higher precision value than what is supported by JavaScript.** + */ + expiry?: string; + /** + * the permissions for the acl policy + */ + permission?: string; +} + +/** + * An interface representing BlobName. + */ +export interface BlobName { + /** + * Indicates if the blob name is encoded. + */ + encoded?: boolean; + /** + * The name of the blob. + */ + content?: string; +} + +/** + * Represents a single block in a block blob. It describes the block's ID and size. + */ +export interface Block { + /** + * The base64 encoded block ID. + */ + name: string; + /** + * The block size in bytes. + */ + size: number; +} + +/** + * An interface representing BlockList. + */ +export interface BlockList { + committedBlocks?: Block[]; + uncommittedBlocks?: Block[]; +} + +/** + * An interface representing BlockLookupList. + */ +export interface BlockLookupList { + committed?: string[]; + uncommitted?: string[]; + latest?: string[]; +} + +/** + * Properties of a container + */ +export interface ContainerProperties { + lastModified: Date; + etag: string; + /** + * Possible values include: 'locked', 'unlocked' + */ + leaseStatus?: LeaseStatusType; + /** + * Possible values include: 'available', 'leased', 'expired', 'breaking', 'broken' + */ + leaseState?: LeaseStateType; + /** + * Possible values include: 'infinite', 'fixed' + */ + leaseDuration?: LeaseDurationType; + /** + * Possible values include: 'container', 'blob' + */ + publicAccess?: PublicAccessType; + hasImmutabilityPolicy?: boolean; + hasLegalHold?: boolean; + defaultEncryptionScope?: string; + preventEncryptionScopeOverride?: boolean; + deletedTime?: Date; + remainingRetentionDays?: number; + /** + * Indicates if version level worm is enabled on this container. + */ + isImmutableStorageWithVersioningEnabled?: boolean; +} + +/** + * An Azure Storage container + */ +export interface ContainerItem { + name: string; + deleted?: boolean; + version?: string; + properties: ContainerProperties; + metadata?: { [propertyName: string]: string }; +} + +/** + * Groups the settings used for interpreting the blob data if the blob is delimited text formatted. + */ +export interface DelimitedTextConfiguration { + /** + * The string used to separate columns. + */ + columnSeparator?: string; + /** + * The string used to quote a specific field. + */ + fieldQuote?: string; + /** + * The string used to separate records. + */ + recordSeparator?: string; + /** + * The string used as an escape character. + */ + escapeChar?: string; + /** + * Represents whether the data has headers. + */ + headersPresent?: boolean; +} + +/** + * json text configuration + */ +export interface JsonTextConfiguration { + /** + * The string used to separate records. + */ + recordSeparator?: string; +} + +/** + * Groups settings regarding specific field of an arrow schema + */ +export interface ArrowField { + type: string; + name?: string; + precision?: number; + scale?: number; +} + +/** + * Groups the settings used for formatting the response if the response should be Arrow formatted. + */ +export interface ArrowConfiguration { + schema: ArrowField[]; +} + +/** + * An enumeration of containers + */ +export interface ListContainersSegmentResponse { + serviceEndpoint: string; + prefix?: string; + marker?: string; + maxResults?: number; + containerItems: ContainerItem[]; + nextMarker?: string; +} + +/** + * CORS is an HTTP feature that enables a web application running under one domain to access + * resources in another domain. Web browsers implement a security restriction known as same-origin + * policy that prevents a web page from calling APIs in a different domain; CORS provides a secure + * way to allow one domain (the origin domain) to call APIs in another domain + */ +export interface CorsRule { + /** + * The origin domains that are permitted to make a request against the storage service via CORS. + * The origin domain is the domain from which the request originates. Note that the origin must + * be an exact case-sensitive match with the origin that the user age sends to the service. You + * can also use the wildcard character '*' to allow all origin domains to make requests via CORS. + */ + allowedOrigins: string; + /** + * The methods (HTTP request verbs) that the origin domain may use for a CORS request. (comma + * separated) + */ + allowedMethods: string; + /** + * the request headers that the origin domain may specify on the CORS request. + */ + allowedHeaders?: string; + /** + * The response headers that may be sent in the response to the CORS request and exposed by the + * browser to the request issuer + */ + exposedHeaders?: string; + /** + * The maximum amount time that a browser should cache the preflight OPTIONS request. + */ + maxAgeInSeconds: number; +} + +/** + * Blob info from a Filter Blobs API call + */ +export interface FilterBlobItem { + name: string; + containerName: string; + tags?: BlobTags; + versionId?: string; + isCurrentVersion?: boolean; +} + +/** + * The result of a Filter Blobs API call + */ +export interface FilterBlobSegment { + serviceEndpoint: string; + where: string; + blobs: FilterBlobItem[]; + nextMarker?: string; +} + +/** + * Geo-Replication information for the Secondary Storage Service + */ +export interface GeoReplication { + /** + * The status of the secondary location. Possible values include: 'live', 'bootstrap', + * 'unavailable' + */ + status: GeoReplicationStatusType; + /** + * A GMT date/time value, to the second. All primary writes preceding this value are guaranteed + * to be available for read operations at the secondary. Primary writes after this point in time + * may or may not be available for reads. + */ + lastSyncTime: Date; +} + +/** + * the retention policy which determines how long the associated data should persist + */ +export interface RetentionPolicy { + /** + * Indicates whether a retention policy is enabled for the storage service + */ + enabled: boolean; + /** + * Indicates the number of days that metrics or logging or soft-deleted data should be retained. + * All data older than this value will be deleted + */ + days?: number; + /** + * Indicates whether permanent delete is allowed on this storage account. + */ + allowPermanentDelete?: boolean; +} + +/** + * Azure Analytics Logging settings. + */ +export interface Logging { + /** + * The version of Storage Analytics to configure. + */ + version: string; + /** + * Indicates whether all delete requests should be logged. + */ + deleteProperty: boolean; + /** + * Indicates whether all read requests should be logged. + */ + read: boolean; + /** + * Indicates whether all write requests should be logged. + */ + write: boolean; + retentionPolicy: RetentionPolicy; +} + +/** + * a summary of request statistics grouped by API in hour or minute aggregates for blobs + */ +export interface Metrics { + /** + * The version of Storage Analytics to configure. + */ + version?: string; + /** + * Indicates whether metrics are enabled for the Blob service. + */ + enabled: boolean; + /** + * Indicates whether metrics should generate summary statistics for called API operations. + */ + includeAPIs?: boolean; + retentionPolicy?: RetentionPolicy; +} + +/** + * An interface representing PageRange. + */ +export interface PageRange { + start: number; + end: number; +} + +/** + * An interface representing ClearRange. + */ +export interface ClearRange { + start: number; + end: number; +} + +/** + * the list of pages + */ +export interface PageList { + pageRange?: PageRange[]; + clearRange?: ClearRange[]; + nextMarker?: string; +} + +/** + * An interface representing QueryFormat. + */ +export interface QueryFormat { + /** + * Possible values include: 'delimited', 'json', 'arrow', 'parquet' + */ + type: QueryFormatType; + delimitedTextConfiguration?: DelimitedTextConfiguration; + jsonTextConfiguration?: JsonTextConfiguration; + arrowConfiguration?: ArrowConfiguration; + parquetTextConfiguration?: any; +} + +/** + * An interface representing QuerySerialization. + */ +export interface QuerySerialization { + format: QueryFormat; +} + +/** + * Groups the set of query request settings. + */ +export interface QueryRequest { + /** + * The query expression in SQL. The maximum size of the query expression is 256KiB. + */ + expression: string; + inputSerialization?: QuerySerialization; + outputSerialization?: QuerySerialization; +} + +/** + * signed identifier + */ +export interface SignedIdentifier { + /** + * a unique id + */ + id: string; + accessPolicy: AccessPolicy; +} + +/** + * The properties that enable an account to host a static website + */ +export interface StaticWebsite { + /** + * Indicates whether this account is hosting a static website + */ + enabled: boolean; + /** + * The default name of the index page under each directory + */ + indexDocument?: string; + /** + * The absolute path of the custom 404 page + */ + errorDocument404Path?: string; + /** + * Absolute path of the default index page + */ + defaultIndexDocumentPath?: string; +} + +/** + * Storage Service Properties. + */ +export interface StorageServiceProperties { + logging?: Logging; + hourMetrics?: Metrics; + minuteMetrics?: Metrics; + /** + * The set of CORS rules. + */ + cors?: CorsRule[]; + /** + * The default version to use for requests to the Blob service if an incoming request's version + * is not specified. Possible values include version 2008-10-27 and all more recent versions + */ + defaultServiceVersion?: string; + deleteRetentionPolicy?: RetentionPolicy; + staticWebsite?: StaticWebsite; +} + +/** + * Stats for the storage service. + */ +export interface StorageServiceStats { + geoReplication?: GeoReplication; +} + +/** + * Additional parameters for a set of operations. + */ +export interface ModifiedAccessConditions { + /** + * Specify this header value to operate only on a blob if it has been modified since the + * specified date/time. + */ + ifModifiedSince?: Date; + /** + * Specify this header value to operate only on a blob if it has not been modified since the + * specified date/time. + */ + ifUnmodifiedSince?: Date; + /** + * Specify an ETag value to operate only on blobs with a matching value. + */ + ifMatch?: string; + /** + * Specify an ETag value to operate only on blobs without a matching value. + */ + ifNoneMatch?: string; + /** + * Specify a SQL where clause on blob tags to operate only on blobs with a matching value. + */ + ifTags?: string; +} + +/** + * Additional parameters for a set of operations. + */ +export interface PathHTTPHeaders { + /** + * Optional. Sets the blob's cache control. If specified, this property is stored with the blob + * and returned with a read request. + */ + cacheControl?: string; + /** + * Optional. Sets the blob's content encoding. If specified, this property is stored with the + * blob and returned with a read request. + */ + contentEncoding?: string; + /** + * Optional. Set the blob's content language. If specified, this property is stored with the blob + * and returned with a read request. + */ + contentLanguage?: string; + /** + * Optional. Sets the blob's Content-Disposition header. + */ + contentDisposition?: string; + /** + * Optional. Sets the blob's content type. If specified, this property is stored with the blob + * and returned with a read request. + */ + contentType?: string; + /** + * Specify the transactional md5 for the body, to be validated by the service. + */ + contentMD5?: Uint8Array; + /** + * Specify the transactional md5 for the body, to be validated by the service. + */ + transactionalContentHash?: Uint8Array; +} + +/** + * Additional parameters for a set of operations. + */ +export interface LeaseAccessConditions { + /** + * If specified, the operation only succeeds if the resource's lease is active and matches this + * ID. + */ + leaseId?: string; +} + +/** + * Additional parameters for a set of operations. + */ +export interface SourceModifiedAccessConditions { + /** + * Specify an ETag value to operate only on blobs with a matching value. + */ + sourceIfMatch?: string; + /** + * Specify an ETag value to operate only on blobs without a matching value. + */ + sourceIfNoneMatch?: string; + /** + * Specify this header value to operate only on a blob if it has been modified since the + * specified date/time. + */ + sourceIfModifiedSince?: Date; + /** + * Specify this header value to operate only on a blob if it has not been modified since the + * specified date/time. + */ + sourceIfUnmodifiedSince?: Date; + /** + * Specify a SQL where clause on blob tags to operate only on blobs with a matching value. + */ + sourceIfTags?: string; +} + +/** + * Additional parameters for a set of operations. + */ +export interface CpkInfo { + /** + * Optional. Specifies the encryption key to use to encrypt the data provided in the request. If + * not specified, encryption is performed with the root account encryption key. For more + * information, see Encryption at Rest for Azure Storage Services. + */ + encryptionKey?: string; + /** + * The SHA-256 hash of the provided encryption key. Must be provided if the x-ms-encryption-key + * header is provided. + */ + encryptionKeySha256?: string; + /** + * The algorithm used to produce the encryption key hash. Currently, the only accepted value is + * "AES256". Must be provided if the x-ms-encryption-key header is provided. Possible values + * include: 'AES256' + */ + encryptionAlgorithm?: EncryptionAlgorithmType; +} + +/** + * Additional parameters for create operation. + */ +export interface ContainerCpkScopeInfo { + /** + * Optional. Version 2019-07-07 and later. Specifies the default encryption scope to set on the + * container and use for all future writes. + */ + defaultEncryptionScope?: string; + /** + * Optional. Version 2019-07-07 and newer. If true, prevents any request from specifying a + * different encryption scope than the scope set on the container. + */ + preventEncryptionScopeOverride?: boolean; +} + +/** + * Additional parameters for a set of operations. + */ +export interface BlobHTTPHeaders { + /** + * Optional. Sets the blob's content type. If specified, this property is stored with the blob + * and returned with a read request. + */ + blobContentType?: string; + /** + * Optional. Sets the blob's content encoding. If specified, this property is stored with the + * blob and returned with a read request. + */ + blobContentEncoding?: string; + /** + * Optional. Set the blob's content language. If specified, this property is stored with the blob + * and returned with a read request. + */ + blobContentLanguage?: string; + /** + * Optional. An MD5 hash of the blob content. Note that this hash is not validated, as the hashes + * for the individual blocks were validated when each was uploaded. + */ + blobContentMD5?: Uint8Array; + /** + * Optional. Sets the blob's cache control. If specified, this property is stored with the blob + * and returned with a read request. + */ + blobCacheControl?: string; + /** + * Optional. Sets the blob's Content-Disposition header. + */ + blobContentDisposition?: string; +} + +/** + * Additional parameters for a set of operations. + */ +export interface CpkScopeInfo { + /** + * Optional. Version 2019-07-07 and later. Specifies the name of the encryption scope to use to + * encrypt the data provided in the request. If not specified, encryption is performed with the + * default account encryption scope. For more information, see Encryption at Rest for Azure + * Storage Services. + */ + encryptionScope?: string; +} + +/** + * Additional parameters for a set of operations, such as: PageBlob_uploadPages, + * PageBlob_clearPages, PageBlob_uploadPagesFromURL. + */ +export interface SequenceNumberAccessConditions { + /** + * Specify this header value to operate only on a blob if it has a sequence number less than or + * equal to the specified. + */ + ifSequenceNumberLessThanOrEqualTo?: number; + /** + * Specify this header value to operate only on a blob if it has a sequence number less than the + * specified. + */ + ifSequenceNumberLessThan?: number; + /** + * Specify this header value to operate only on a blob if it has the specified sequence number. + */ + ifSequenceNumberEqualTo?: number; +} + +/** + * Additional parameters for a set of operations, such as: AppendBlob_appendBlock, + * AppendBlob_appendBlockFromUrl, AppendBlob_seal. + */ +export interface AppendPositionAccessConditions { + /** + * Optional conditional header. The max length in bytes permitted for the append blob. If the + * Append Block operation would cause the blob to exceed that limit or if the blob size is + * already greater than the value specified in this header, the request will fail with + * MaxBlobSizeConditionNotMet error (HTTP status code 412 - Precondition Failed). + */ + maxSize?: number; + /** + * Optional conditional header, used only for the Append Block operation. A number indicating the + * byte offset to compare. Append Block will succeed only if the append position is equal to this + * number. If it is not, the request will fail with the AppendPositionConditionNotMet error (HTTP + * status code 412 - Precondition Failed). + */ + appendPosition?: number; +} + +/** + * An interface representing DataLakeStorageClientOptions. + */ +export interface DataLakeStorageClientOptions { + /** + * Specifies the version of the operation to use for this request. + */ + version?: string; + /** + * The lease duration is required to acquire a lease, and specifies the duration of the lease in + * seconds. The lease duration must be between 15 and 60 seconds or -1 for infinite lease. + */ + xMsLeaseDuration?: number; +} + +/** + * Optional Parameters. + */ +export interface ServiceListFileSystemsOptionalParams { + /** + * Filters results to filesystems within the specified prefix. + */ + prefix?: string; + /** + * Optional. When deleting a directory, the number of paths that are deleted with each + * invocation is limited. If the number of paths to be deleted exceeds this limit, a + * continuation token is returned in this response header. When a continuation token is returned + * in the response, it must be specified in a subsequent invocation of the delete operation to + * continue deleting the directory. + */ + continuation?: string; + /** + * An optional value that specifies the maximum number of items to return. If omitted or greater + * than 5,000, the response will include up to 5,000 items. + */ + maxResults?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; +} + +/** + * Optional Parameters. + */ +export interface ServiceSetPropertiesOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; +} + +/** + * Optional Parameters. + */ +export interface ServiceGetPropertiesOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; +} + +/** + * Optional Parameters. + */ +export interface ServiceGetStatisticsOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; +} + +/** + * Optional Parameters. + */ +export interface ServiceListContainersSegmentOptionalParams { + /** + * Filters results to filesystems within the specified prefix. + */ + prefix?: string; + /** + * A string value that identifies the portion of the list of containers to be returned with the + * next listing operation. The operation returns the NextMarker value within the response body if + * the listing operation did not return all containers remaining to be listed with the current + * page. The NextMarker value can be used as the value for the marker parameter in a subsequent + * call to request the next page of list items. The marker value is opaque to the client. + */ + marker?: string; + /** + * Specifies the maximum number of containers to return. If the request does not specify + * maxresults, or specifies a value greater than 5000, the server will return up to 5000 items. + * Note that if the listing operation crosses a partition boundary, then the service will return + * a continuation token for retrieving the remainder of the results. For this reason, it is + * possible that the service will return fewer results than specified by maxresults, or than the + * default of 5000. + */ + maxresults?: number; + /** + * Include this parameter to specify that the container's metadata be returned as part of the + * response body. + */ + include?: ListContainersIncludeType[]; + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; +} + +/** + * Optional Parameters. + */ +export interface ServiceGetUserDelegationKeyOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; +} + +/** + * Optional Parameters. + */ +export interface ServiceSubmitBatchOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; +} + +/** + * Optional Parameters. + */ +export interface ServiceFilterBlobsOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Filters the results to return only to return only blobs whose tags match the specified + * expression. + */ + where?: string; + /** + * A string value that identifies the portion of the list of containers to be returned with the + * next listing operation. The operation returns the NextMarker value within the response body if + * the listing operation did not return all containers remaining to be listed with the current + * page. The NextMarker value can be used as the value for the marker parameter in a subsequent + * call to request the next page of list items. The marker value is opaque to the client. + */ + marker?: string; + /** + * Specifies the maximum number of containers to return. If the request does not specify + * maxresults, or specifies a value greater than 5000, the server will return up to 5000 items. + * Note that if the listing operation crosses a partition boundary, then the service will return + * a continuation token for retrieving the remainder of the results. For this reason, it is + * possible that the service will return fewer results than specified by maxresults, or than the + * default of 5000. + */ + maxresults?: number; + /** + * Include this parameter to specify one or more datasets to include in the response. + */ + include?: FilterBlobsIncludeItem[]; +} + +/** + * Optional Parameters. + */ +export interface FileSystemCreateOptionalParams { + /** + * Optional. User-defined properties to be stored with the filesystem, in the format of a + * comma-separated list of name and value pairs "n1=v1, n2=v2, ...", where each value is a base64 + * encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 + * character set. If the filesystem exists, any properties not included in the list will be + * removed. All properties are removed if the header is omitted. To merge new and existing + * properties, first get all existing properties and the current E-Tag, then make a conditional + * request with the E-Tag and include values for all properties. + */ + properties?: string; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; +} + +/** + * Optional Parameters. + */ +export interface FileSystemSetPropertiesOptionalParams { + /** + * Optional. User-defined properties to be stored with the filesystem, in the format of a + * comma-separated list of name and value pairs "n1=v1, n2=v2, ...", where each value is a base64 + * encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 + * character set. If the filesystem exists, any properties not included in the list will be + * removed. All properties are removed if the header is omitted. To merge new and existing + * properties, first get all existing properties and the current E-Tag, then make a conditional + * request with the E-Tag and include values for all properties. + */ + properties?: string; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface FileSystemGetPropertiesOptionalParams { + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; +} + +/** + * Optional Parameters. + */ +export interface FileSystemDeleteMethodOptionalParams { + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface FileSystemListPathsOptionalParams { + /** + * Optional. When deleting a directory, the number of paths that are deleted with each + * invocation is limited. If the number of paths to be deleted exceeds this limit, a + * continuation token is returned in this response header. When a continuation token is returned + * in the response, it must be specified in a subsequent invocation of the delete operation to + * continue deleting the directory. + */ + continuation?: string; + /** + * Optional. Filters results to paths within the specified directory. An error occurs if the + * directory does not exist. + */ + path?: string; + /** + * An optional value that specifies the maximum number of items to return. If omitted or greater + * than 5,000, the response will include up to 5,000 items. + */ + maxResults?: number; + /** + * Optional. Valid only when Hierarchical Namespace is enabled for the account. If "true", the + * user identity values returned in the x-ms-owner, x-ms-group, and x-ms-acl response headers + * will be transformed from Azure Active Directory Object IDs to User Principal Names. If + * "false", the values will be returned as Azure Active Directory Object IDs. The default value + * is false. Note that group and application Object IDs are not translated because they do not + * have unique friendly names. + */ + upn?: boolean; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; +} + +/** + * Optional Parameters. + */ +export interface FileSystemListBlobFlatSegmentOptionalParams { + /** + * Filters results to filesystems within the specified prefix. + */ + prefix?: string; + /** + * A string value that identifies the portion of the list of containers to be returned with the + * next listing operation. The operation returns the NextMarker value within the response body if + * the listing operation did not return all containers remaining to be listed with the current + * page. The NextMarker value can be used as the value for the marker parameter in a subsequent + * call to request the next page of list items. The marker value is opaque to the client. + */ + marker?: string; + /** + * An optional value that specifies the maximum number of items to return. If omitted or greater + * than 5,000, the response will include up to 5,000 items. + */ + maxResults?: number; + /** + * Specifies the maximum number of containers to return. If the request does not specify + * maxresults, or specifies a value greater than 5000, the server will return up to 5000 items. + * Note that if the listing operation crosses a partition boundary, then the service will return + * a continuation token for retrieving the remainder of the results. For this reason, it is + * possible that the service will return fewer results than specified by maxresults, or than the + * default of 5000. + */ + maxresults?: number; + /** + * Include this parameter to specify one or more datasets to include in the response. + */ + include?: ListBlobsIncludeItem[]; + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; +} + +/** + * Optional Parameters. + */ +export interface FileSystemListBlobHierarchySegmentOptionalParams { + /** + * Filters results to filesystems within the specified prefix. + */ + prefix?: string; + /** + * A string value that identifies the portion of the list of containers to be returned with the + * next listing operation. The operation returns the NextMarker value within the response body if + * the listing operation did not return all containers remaining to be listed with the current + * page. The NextMarker value can be used as the value for the marker parameter in a subsequent + * call to request the next page of list items. The marker value is opaque to the client. + */ + marker?: string; + /** + * An optional value that specifies the maximum number of items to return. If omitted or greater + * than 5,000, the response will include up to 5,000 items. + */ + maxResults?: number; + /** + * Specifies the maximum number of containers to return. If the request does not specify + * maxresults, or specifies a value greater than 5000, the server will return up to 5000 items. + * Note that if the listing operation crosses a partition boundary, then the service will return + * a continuation token for retrieving the remainder of the results. For this reason, it is + * possible that the service will return fewer results than specified by maxresults, or than the + * default of 5000. + */ + maxresults?: number; + /** + * Include this parameter to specify one or more datasets to include in the response. + */ + include?: ListBlobsIncludeItem[]; + /** + * Include this parameter to specify one or more datasets to include in the response. Possible + * values include: 'deleted' + */ + showonly?: ListBlobsShowOnly; + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; +} + +/** + * Optional Parameters. + */ +export interface PathCreateOptionalParams { + /** + * Required only for Create File and Create Directory. The value must be "file" or "directory". + * Possible values include: 'directory', 'file' + */ + resource?: PathResourceType; + /** + * Optional. When deleting a directory, the number of paths that are deleted with each + * invocation is limited. If the number of paths to be deleted exceeds this limit, a + * continuation token is returned in this response header. When a continuation token is returned + * in the response, it must be specified in a subsequent invocation of the delete operation to + * continue deleting the directory. + */ + continuation?: string; + /** + * Optional. Valid only when namespace is enabled. This parameter determines the behavior of the + * rename operation. The value must be "legacy" or "posix", and the default value will be + * "posix". Possible values include: 'legacy', 'posix' + */ + mode?: PathRenameMode; + /** + * An optional file or directory to be renamed. The value must have the following format: + * "/{filesystem}/{path}". If "x-ms-properties" is specified, the properties will overwrite the + * existing properties; otherwise, the existing properties will be preserved. This value must be + * a URL percent-encoded string. Note that the string may only contain ASCII characters in the + * ISO-8859-1 character set. + */ + renameSource?: string; + /** + * A lease ID for the source path. If specified, the source path must have an active lease and + * the lease ID must match. + */ + sourceLeaseId?: string; + /** + * Optional. User-defined properties to be stored with the filesystem, in the format of a + * comma-separated list of name and value pairs "n1=v1, n2=v2, ...", where each value is a base64 + * encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 + * character set. If the filesystem exists, any properties not included in the list will be + * removed. All properties are removed if the header is omitted. To merge new and existing + * properties, first get all existing properties and the current E-Tag, then make a conditional + * request with the E-Tag and include values for all properties. + */ + properties?: string; + /** + * Optional and only valid if Hierarchical Namespace is enabled for the account. Sets POSIX + * access permissions for the file owner, the file owning group, and others. Each class may be + * granted read, write, or execute permission. The sticky bit is also supported. Both symbolic + * (rwxrw-rw-) and 4-digit octal notation (e.g. 0766) are supported. + */ + permissions?: string; + /** + * Optional and only valid if Hierarchical Namespace is enabled for the account. When creating a + * file or directory and the parent folder does not have a default ACL, the umask restricts the + * permissions of the file or directory to be created. The resulting permission is given by p + * bitwise and not u, where p is the permission and u is the umask. For example, if p is 0777 + * and u is 0057, then the resulting permission is 0720. The default permission is 0777 for a + * directory and 0666 for a file. The default umask is 0027. The umask must be specified in + * 4-digit octal notation (e.g. 0766). + */ + umask?: string; + /** + * Optional. The owner of the blob or directory. + */ + owner?: string; + /** + * Optional. The owning group of the blob or directory. + */ + group?: string; + /** + * Sets POSIX access control rights on files and directories. The value is a comma-separated list + * of access control entries. Each access control entry (ACE) consists of a scope, a type, a user + * or group identifier, and permissions in the format "[scope:][type]:[id]:[permissions]". + */ + acl?: string; + /** + * Proposed lease ID, in a GUID string format. The Blob service returns 400 (Invalid request) if + * the proposed lease ID is not in the correct format. See Guid Constructor (String) for a list + * of valid GUID string formats. + */ + proposedLeaseId?: string; + /** + * The lease duration is required to acquire a lease, and specifies the duration of the lease in + * seconds. The lease duration must be between 15 and 60 seconds or -1 for infinite lease. + */ + leaseDuration?: number; + /** + * Required. Indicates mode of the expiry time. Possible values include: 'NeverExpire', + * 'RelativeToCreation', 'RelativeToNow', 'Absolute' + */ + expiryOptions?: PathExpiryOptions; + /** + * The time to set the blob to expiry + */ + expiresOn?: string; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Optional. Specifies a user-defined name-value pair associated with the blob. If no name-value + * pairs are specified, the operation will copy the metadata from the source blob or file to the + * destination blob. If one or more name-value pairs are specified, the destination blob is + * created with the specified metadata, and metadata is not copied from the source blob or file. + * Note that beginning with version 2009-09-19, metadata names must adhere to the naming rules + * for C# identifiers. See Naming and Referencing Containers, Blobs, and Metadata for more + * information. + */ + metadata?: { [propertyName: string]: string }; + /** + * Additional parameters for the operation + */ + pathHTTPHeaders?: PathHTTPHeaders; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; + /** + * Additional parameters for the operation + */ + sourceModifiedAccessConditions?: SourceModifiedAccessConditions; + /** + * Additional parameters for the operation + */ + cpkInfo?: CpkInfo; +} + +/** + * Optional Parameters. + */ +export interface PathUpdateOptionalParams { + /** + * Optional. Valid only for append calls. This parameter allows the caller to flush during an + * append call. Default value is 'false' , if 'true' the data will be flushed with the append + * call. Note that when using flush=true, the following headers are not supported - + * 'x-ms-cache-control', 'x-ms-content-encoding', 'x-ms-content-type', 'x-ms-content-language', + * 'x-ms-content-md5', 'x-ms-content-disposition'. To set these headers during flush, please use + * action=flush + */ + flush?: boolean; + /** + * Optional. Valid for "SetAccessControlRecursive" operation. It specifies the maximum number of + * files or directories on which the acl change will be applied. If omitted or greater than + * 2,000, the request will process up to 2,000 items + */ + maxRecords?: number; + /** + * Optional. The number of paths processed with each invocation is limited. If the number of + * paths to be processed exceeds this limit, a continuation token is returned in the response + * header x-ms-continuation. When a continuation token is returned in the response, it must be + * percent-encoded and specified in a subsequent invocation of setAccessControlRecursive + * operation. + */ + continuation?: string; + /** + * Optional. Valid for "SetAccessControlRecursive" operation. If set to false, the operation will + * terminate quickly on encountering user errors (4XX). If true, the operation will ignore user + * errors and proceed with the operation on other sub-entities of the directory. Continuation + * token will only be returned when forceFlag is true in case of user errors. If not set the + * default value is false for this. + */ + forceFlag?: boolean; + /** + * This parameter allows the caller to upload data in parallel and control the order in which it + * is appended to the file. It is required when uploading data to be appended to the file and + * when flushing previously uploaded data to the file. The value must be the position where the + * data is to be appended. Uploaded data is not immediately flushed, or written, to the file. + * To flush, the previously uploaded data must be contiguous, the position parameter must be + * specified and equal to the length of the file after all data has been written, and there must + * not be a request entity body included with the request. + */ + position?: number; + /** + * Valid only for flush operations. If "true", uncommitted data is retained after the flush + * operation completes; otherwise, the uncommitted data is deleted after the flush operation. + * The default is false. Data at offsets less than the specified position are written to the + * file when flush succeeds, but this optional parameter allows data after the flush position to + * be retained for a future flush operation. + */ + retainUncommittedData?: boolean; + /** + * Azure Storage Events allow applications to receive notifications when files change. When Azure + * Storage Events are enabled, a file changed event is raised. This event has a property + * indicating whether this is the final change to distinguish the difference between an + * intermediate flush to a file stream and the final close of a file stream. The close query + * parameter is valid only when the action is "flush" and change notifications are enabled. If + * the value of close is "true" and the flush operation completes successfully, the service + * raises a file change notification with a property indicating that this is the final update + * (the file stream has been closed). If "false" a change notification is raised indicating the + * file has changed. The default is false. This query parameter is set to true by the Hadoop ABFS + * driver to indicate that the file stream has been closed." + */ + close?: boolean; + /** + * Required for "Append Data" and "Flush Data". Must be 0 for "Flush Data". Must be the length + * of the request content in bytes for "Append Data". + */ + contentLength?: number; + /** + * Optional. User-defined properties to be stored with the filesystem, in the format of a + * comma-separated list of name and value pairs "n1=v1, n2=v2, ...", where each value is a base64 + * encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 + * character set. If the filesystem exists, any properties not included in the list will be + * removed. All properties are removed if the header is omitted. To merge new and existing + * properties, first get all existing properties and the current E-Tag, then make a conditional + * request with the E-Tag and include values for all properties. + */ + properties?: string; + /** + * Optional. The owner of the blob or directory. + */ + owner?: string; + /** + * Optional. The owning group of the blob or directory. + */ + group?: string; + /** + * Optional and only valid if Hierarchical Namespace is enabled for the account. Sets POSIX + * access permissions for the file owner, the file owning group, and others. Each class may be + * granted read, write, or execute permission. The sticky bit is also supported. Both symbolic + * (rwxrw-rw-) and 4-digit octal notation (e.g. 0766) are supported. + */ + permissions?: string; + /** + * Sets POSIX access control rights on files and directories. The value is a comma-separated list + * of access control entries. Each access control entry (ACE) consists of a scope, a type, a user + * or group identifier, and permissions in the format "[scope:][type]:[id]:[permissions]". + */ + acl?: string; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Optional. Specifies a user-defined name-value pair associated with the blob. If no name-value + * pairs are specified, the operation will copy the metadata from the source blob or file to the + * destination blob. If one or more name-value pairs are specified, the destination blob is + * created with the specified metadata, and metadata is not copied from the source blob or file. + * Note that beginning with version 2009-09-19, metadata names must adhere to the naming rules + * for C# identifiers. See Naming and Referencing Containers, Blobs, and Metadata for more + * information. + */ + metadata?: { [propertyName: string]: string }; + /** + * Additional parameters for the operation + */ + pathHTTPHeaders?: PathHTTPHeaders; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface PathLeaseOptionalParams { + /** + * The lease duration is required to acquire a lease, and specifies the duration of the lease in + * seconds. The lease duration must be between 15 and 60 seconds or -1 for infinite lease. + */ + xMsLeaseDuration?: number; + /** + * The lease break period duration is optional to break a lease, and specifies the break period + * of the lease in seconds. The lease break duration must be between 0 and 60 seconds. + */ + xMsLeaseBreakPeriod?: number; + /** + * Proposed lease ID, in a GUID string format. The Blob service returns 400 (Invalid request) if + * the proposed lease ID is not in the correct format. See Guid Constructor (String) for a list + * of valid GUID string formats. + */ + proposedLeaseId?: string; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Optional. Specifies a user-defined name-value pair associated with the blob. If no name-value + * pairs are specified, the operation will copy the metadata from the source blob or file to the + * destination blob. If one or more name-value pairs are specified, the destination blob is + * created with the specified metadata, and metadata is not copied from the source blob or file. + * Note that beginning with version 2009-09-19, metadata names must adhere to the naming rules + * for C# identifiers. See Naming and Referencing Containers, Blobs, and Metadata for more + * information. + */ + metadata?: { [propertyName: string]: string }; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface PathReadOptionalParams { + /** + * Optional. When this header is set to "true" and specified together with the Range header, the + * service returns the MD5 hash for the range, as long as the range is less than or equal to 4MB + * in size. If this header is specified without the Range header, the service returns status code + * 400 (Bad Request). If this header is set to true when the range exceeds 4 MB in size, the + * service returns status code 400 (Bad Request). + */ + xMsRangeGetContentMd5?: boolean; + /** + * The snapshot parameter is an opaque DateTime value that, when present, specifies the blob + * snapshot to retrieve. For more information on working with blob snapshots, see Creating + * a Snapshot of a Blob. + */ + snapshot?: string; + /** + * The version id parameter is an opaque DateTime value that, when present, specifies the version + * of the blob to operate on. It's for service version 2019-10-10 and newer. + */ + versionId?: string; + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * The HTTP Range request header specifies one or more byte ranges of the resource to be + * retrieved. + */ + range?: string; + /** + * Optional. When this header is set to "true" and specified together with the Range header, the + * service returns the MD5 hash for the range, as long as the range is less than or equal to 4MB + * in size. If this header is specified without the Range header, the service returns status code + * 400 (Bad Request). If this header is set to true when the range exceeds 4 MB in size, the + * service returns status code 400 (Bad Request). + */ + rangeGetContentMD5?: boolean; + /** + * When set to true and specified together with the Range, the service returns the CRC64 hash for + * the range, as long as the range is less than or equal to 4 MB in size. + */ + rangeGetContentCRC64?: boolean; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Optional. Specifies a user-defined name-value pair associated with the blob. If no name-value + * pairs are specified, the operation will copy the metadata from the source blob or file to the + * destination blob. If one or more name-value pairs are specified, the destination blob is + * created with the specified metadata, and metadata is not copied from the source blob or file. + * Note that beginning with version 2009-09-19, metadata names must adhere to the naming rules + * for C# identifiers. See Naming and Referencing Containers, Blobs, and Metadata for more + * information. + */ + metadata?: { [propertyName: string]: string }; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; + /** + * Additional parameters for the operation + */ + cpkInfo?: CpkInfo; +} + +/** + * Optional Parameters. + */ +export interface PathGetPropertiesOptionalParams { + /** + * Optional. If the value is "getStatus" only the system defined properties for the path are + * returned. If the value is "getAccessControl" the access control list is returned in the + * response headers (Hierarchical Namespace must be enabled for the account), otherwise the + * properties are returned. Possible values include: 'getAccessControl', 'getStatus' + */ + action?: PathGetPropertiesAction; + /** + * Optional. Valid only when Hierarchical Namespace is enabled for the account. If "true", the + * user identity values returned in the x-ms-owner, x-ms-group, and x-ms-acl response headers + * will be transformed from Azure Active Directory Object IDs to User Principal Names. If + * "false", the values will be returned as Azure Active Directory Object IDs. The default value + * is false. Note that group and application Object IDs are not translated because they do not + * have unique friendly names. + */ + upn?: boolean; + /** + * The snapshot parameter is an opaque DateTime value that, when present, specifies the blob + * snapshot to retrieve. For more information on working with blob snapshots, see Creating + * a Snapshot of a Blob. + */ + snapshot?: string; + /** + * The version id parameter is an opaque DateTime value that, when present, specifies the version + * of the blob to operate on. It's for service version 2019-10-10 and newer. + */ + versionId?: string; + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Optional. Specifies a user-defined name-value pair associated with the blob. If no name-value + * pairs are specified, the operation will copy the metadata from the source blob or file to the + * destination blob. If one or more name-value pairs are specified, the destination blob is + * created with the specified metadata, and metadata is not copied from the source blob or file. + * Note that beginning with version 2009-09-19, metadata names must adhere to the naming rules + * for C# identifiers. See Naming and Referencing Containers, Blobs, and Metadata for more + * information. + */ + metadata?: { [propertyName: string]: string }; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; + /** + * Additional parameters for the operation + */ + cpkInfo?: CpkInfo; +} + +/** + * Optional Parameters. + */ +export interface PathDeleteMethodOptionalParams { + /** + * Required + */ + recursive?: boolean; + /** + * Optional. When deleting a directory, the number of paths that are deleted with each + * invocation is limited. If the number of paths to be deleted exceeds this limit, a + * continuation token is returned in this response header. When a continuation token is returned + * in the response, it must be specified in a subsequent invocation of the delete operation to + * continue deleting the directory. + */ + continuation?: string; + /** + * The snapshot parameter is an opaque DateTime value that, when present, specifies the blob + * snapshot to retrieve. For more information on working with blob snapshots, see Creating + * a Snapshot of a Blob. + */ + snapshot?: string; + /** + * The version id parameter is an opaque DateTime value that, when present, specifies the version + * of the blob to operate on. It's for service version 2019-10-10 and newer. + */ + versionId?: string; + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Required if the blob has associated snapshots. Specify one of the following two options: + * include: Delete the base blob and all of its snapshots. only: Delete only the blob's snapshots + * and not the blob itself. Possible values include: 'include', 'only' + */ + deleteSnapshots?: DeleteSnapshotsOptionType; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Optional. Only possible value is 'permanent', which specifies to permanently delete a blob if + * blob soft delete is enabled. Possible values include: 'Permanent' + */ + blobDeleteType?: BlobDeleteType; + /** + * Optional. Specifies a user-defined name-value pair associated with the blob. If no name-value + * pairs are specified, the operation will copy the metadata from the source blob or file to the + * destination blob. If one or more name-value pairs are specified, the destination blob is + * created with the specified metadata, and metadata is not copied from the source blob or file. + * Note that beginning with version 2009-09-19, metadata names must adhere to the naming rules + * for C# identifiers. See Naming and Referencing Containers, Blobs, and Metadata for more + * information. + */ + metadata?: { [propertyName: string]: string }; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface PathSetAccessControlOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Optional. The owner of the blob or directory. + */ + owner?: string; + /** + * Optional. The owning group of the blob or directory. + */ + group?: string; + /** + * Optional and only valid if Hierarchical Namespace is enabled for the account. Sets POSIX + * access permissions for the file owner, the file owning group, and others. Each class may be + * granted read, write, or execute permission. The sticky bit is also supported. Both symbolic + * (rwxrw-rw-) and 4-digit octal notation (e.g. 0766) are supported. + */ + permissions?: string; + /** + * Sets POSIX access control rights on files and directories. The value is a comma-separated list + * of access control entries. Each access control entry (ACE) consists of a scope, a type, a user + * or group identifier, and permissions in the format "[scope:][type]:[id]:[permissions]". + */ + acl?: string; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface PathSetAccessControlRecursiveOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Optional. When deleting a directory, the number of paths that are deleted with each + * invocation is limited. If the number of paths to be deleted exceeds this limit, a + * continuation token is returned in this response header. When a continuation token is returned + * in the response, it must be specified in a subsequent invocation of the delete operation to + * continue deleting the directory. + */ + continuation?: string; + /** + * Optional. Valid for "SetAccessControlRecursive" operation. If set to false, the operation will + * terminate quickly on encountering user errors (4XX). If true, the operation will ignore user + * errors and proceed with the operation on other sub-entities of the directory. Continuation + * token will only be returned when forceFlag is true in case of user errors. If not set the + * default value is false for this. + */ + forceFlag?: boolean; + /** + * Optional. It specifies the maximum number of files or directories on which the acl change will + * be applied. If omitted or greater than 2,000, the request will process up to 2,000 items + */ + maxRecords?: number; + /** + * Sets POSIX access control rights on files and directories. The value is a comma-separated list + * of access control entries. Each access control entry (ACE) consists of a scope, a type, a user + * or group identifier, and permissions in the format "[scope:][type]:[id]:[permissions]". + */ + acl?: string; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; +} + +/** + * Optional Parameters. + */ +export interface PathSetPropertiesOptionalParams { + /** + * Optional. User-defined properties to be stored with the filesystem, in the format of a + * comma-separated list of name and value pairs "n1=v1, n2=v2, ...", where each value is a base64 + * encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 + * character set. If the filesystem exists, any properties not included in the list will be + * removed. All properties are removed if the header is omitted. To merge new and existing + * properties, first get all existing properties and the current E-Tag, then make a conditional + * request with the E-Tag and include values for all properties. + */ + properties?: string; + /** + * Optional and only valid if Hierarchical Namespace is enabled for the account. Sets POSIX + * access permissions for the file owner, the file owning group, and others. Each class may be + * granted read, write, or execute permission. The sticky bit is also supported. Both symbolic + * (rwxrw-rw-) and 4-digit octal notation (e.g. 0766) are supported. + */ + permissions?: string; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; + /** + * Additional parameters for the operation + */ + pathHTTPHeaders?: PathHTTPHeaders; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface PathFlushDataOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * This parameter allows the caller to upload data in parallel and control the order in which it + * is appended to the file. It is required when uploading data to be appended to the file and + * when flushing previously uploaded data to the file. The value must be the position where the + * data is to be appended. Uploaded data is not immediately flushed, or written, to the file. + * To flush, the previously uploaded data must be contiguous, the position parameter must be + * specified and equal to the length of the file after all data has been written, and there must + * not be a request entity body included with the request. + */ + position?: number; + /** + * Valid only for flush operations. If "true", uncommitted data is retained after the flush + * operation completes; otherwise, the uncommitted data is deleted after the flush operation. + * The default is false. Data at offsets less than the specified position are written to the + * file when flush succeeds, but this optional parameter allows data after the flush position to + * be retained for a future flush operation. + */ + retainUncommittedData?: boolean; + /** + * Azure Storage Events allow applications to receive notifications when files change. When Azure + * Storage Events are enabled, a file changed event is raised. This event has a property + * indicating whether this is the final change to distinguish the difference between an + * intermediate flush to a file stream and the final close of a file stream. The close query + * parameter is valid only when the action is "flush" and change notifications are enabled. If + * the value of close is "true" and the flush operation completes successfully, the service + * raises a file change notification with a property indicating that this is the final update + * (the file stream has been closed). If "false" a change notification is raised indicating the + * file has changed. The default is false. This query parameter is set to true by the Hadoop ABFS + * driver to indicate that the file stream has been closed." + */ + close?: boolean; + /** + * Required for "Append Data" and "Flush Data". Must be 0 for "Flush Data". Must be the length + * of the request content in bytes for "Append Data". + */ + contentLength?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Proposed lease ID, in a GUID string format. The Blob service returns 400 (Invalid request) if + * the proposed lease ID is not in the correct format. See Guid Constructor (String) for a list + * of valid GUID string formats. + */ + proposedLeaseId?: string; + /** + * The lease duration is required to acquire a lease, and specifies the duration of the lease in + * seconds. The lease duration must be between 15 and 60 seconds or -1 for infinite lease. + */ + xMsLeaseDuration?: number; + /** + * Describes what lease action to take. Possible values include: 'acquire', 'release', 'renew', + * 'break', 'change', 'auto-renew', 'acquire-release' + */ + leaseAction?: LeaseAction; + /** + * Additional parameters for the operation + */ + pathHTTPHeaders?: PathHTTPHeaders; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; + /** + * Additional parameters for the operation + */ + cpkInfo?: CpkInfo; +} + +/** + * Optional Parameters. + */ +export interface PathAppendDataOptionalParams { + /** + * This parameter allows the caller to upload data in parallel and control the order in which it + * is appended to the file. It is required when uploading data to be appended to the file and + * when flushing previously uploaded data to the file. The value must be the position where the + * data is to be appended. Uploaded data is not immediately flushed, or written, to the file. + * To flush, the previously uploaded data must be contiguous, the position parameter must be + * specified and equal to the length of the file after all data has been written, and there must + * not be a request entity body included with the request. + */ + position?: number; + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Required for "Append Data" and "Flush Data". Must be 0 for "Flush Data". Must be the length + * of the request content in bytes for "Append Data". + */ + contentLength?: number; + /** + * Specify the transactional crc64 for the body, to be validated by the service. + */ + transactionalContentCrc64?: Uint8Array; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Proposed lease ID, in a GUID string format. The Blob service returns 400 (Invalid request) if + * the proposed lease ID is not in the correct format. See Guid Constructor (String) for a list + * of valid GUID string formats. + */ + proposedLeaseId?: string; + /** + * The lease duration is required to acquire a lease, and specifies the duration of the lease in + * seconds. The lease duration must be between 15 and 60 seconds or -1 for infinite lease. + */ + xMsLeaseDuration?: number; + /** + * Describes what lease action to take. Possible values include: 'acquire', 'release', 'renew', + * 'break', 'change', 'auto-renew', 'acquire-release' + */ + leaseAction?: LeaseAction; + /** + * Optional. This parameter allows the caller to flush during an append call. Default value is + * 'false' , if 'true' the data will be flushed with the append call. Note that when using + * flush=true, the following headers are not supported - 'x-ms-cache-control', + * 'x-ms-content-encoding', 'x-ms-content-type', 'x-ms-content-language', 'x-ms-content-md5', + * 'x-ms-content-disposition'. To set these headers during flush, please use action=flush + */ + flush?: boolean; + /** + * Additional parameters for the operation + */ + pathHTTPHeaders?: PathHTTPHeaders; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; + /** + * Additional parameters for the operation + */ + cpkInfo?: CpkInfo; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface PathSetExpiryOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * The time to set the blob to expiry + */ + expiresOn?: string; +} + +/** + * Optional Parameters. + */ +export interface PathUndeleteOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Only for hierarchical namespace enabled accounts. Optional. The path of the soft deleted blob + * to undelete. + */ + undeleteSource?: string; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; +} + +/** + * Optional Parameters. + */ +export interface ContainerCreateOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Optional. Specifies a user-defined name-value pair associated with the blob. If no name-value + * pairs are specified, the operation will copy the metadata from the source blob or file to the + * destination blob. If one or more name-value pairs are specified, the destination blob is + * created with the specified metadata, and metadata is not copied from the source blob or file. + * Note that beginning with version 2009-09-19, metadata names must adhere to the naming rules + * for C# identifiers. See Naming and Referencing Containers, Blobs, and Metadata for more + * information. + */ + metadata?: { [propertyName: string]: string }; + /** + * Specifies whether data in the container may be accessed publicly and the level of access. + * Possible values include: 'container', 'blob' + */ + access?: PublicAccessType; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Additional parameters for the operation + */ + containerCpkScopeInfo?: ContainerCpkScopeInfo; +} + +/** + * Optional Parameters. + */ +export interface ContainerGetPropertiesOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface ContainerGetPropertiesWithHeadOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface ContainerDeleteMethodOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface ContainerSetMetadataOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Optional. Specifies a user-defined name-value pair associated with the blob. If no name-value + * pairs are specified, the operation will copy the metadata from the source blob or file to the + * destination blob. If one or more name-value pairs are specified, the destination blob is + * created with the specified metadata, and metadata is not copied from the source blob or file. + * Note that beginning with version 2009-09-19, metadata names must adhere to the naming rules + * for C# identifiers. See Naming and Referencing Containers, Blobs, and Metadata for more + * information. + */ + metadata?: { [propertyName: string]: string }; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface ContainerGetAccessPolicyOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface ContainerSetAccessPolicyOptionalParams { + /** + * the acls for the container + */ + containerAcl?: SignedIdentifier[]; + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Specifies whether data in the container may be accessed publicly and the level of access. + * Possible values include: 'container', 'blob' + */ + access?: PublicAccessType; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface ContainerRestoreOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Optional. Version 2019-12-12 and later. Specifies the name of the deleted container to + * restore. + */ + deletedContainerName?: string; + /** + * Optional. Version 2019-12-12 and later. Specifies the version of the deleted container to + * restore. + */ + deletedContainerVersion?: string; +} + +/** + * Optional Parameters. + */ +export interface ContainerSubmitBatchOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; +} + +/** + * Optional Parameters. + */ +export interface ContainerFilterBlobsOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Filters the results to return only to return only blobs whose tags match the specified + * expression. + */ + where?: string; + /** + * A string value that identifies the portion of the list of containers to be returned with the + * next listing operation. The operation returns the NextMarker value within the response body if + * the listing operation did not return all containers remaining to be listed with the current + * page. The NextMarker value can be used as the value for the marker parameter in a subsequent + * call to request the next page of list items. The marker value is opaque to the client. + */ + marker?: string; + /** + * Specifies the maximum number of containers to return. If the request does not specify + * maxresults, or specifies a value greater than 5000, the server will return up to 5000 items. + * Note that if the listing operation crosses a partition boundary, then the service will return + * a continuation token for retrieving the remainder of the results. For this reason, it is + * possible that the service will return fewer results than specified by maxresults, or than the + * default of 5000. + */ + maxresults?: number; + /** + * Include this parameter to specify one or more datasets to include in the response. + */ + include?: FilterBlobsIncludeItem[]; +} + +/** + * Optional Parameters. + */ +export interface ContainerAcquireLeaseOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Specifies the duration of the lease, in seconds, or negative one (-1) for a lease that never + * expires. A non-infinite lease can be between 15 and 60 seconds. A lease duration cannot be + * changed using renew or change. + */ + duration?: number; + /** + * Proposed lease ID, in a GUID string format. The Blob service returns 400 (Invalid request) if + * the proposed lease ID is not in the correct format. See Guid Constructor (String) for a list + * of valid GUID string formats. + */ + proposedLeaseId?: string; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface ContainerReleaseLeaseOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface ContainerRenewLeaseOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface ContainerBreakLeaseOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * For a break operation, proposed duration the lease should continue before it is broken, in + * seconds, between 0 and 60. This break period is only used if it is shorter than the time + * remaining on the lease. If longer, the time remaining on the lease is used. A new lease will + * not be available before the break period has expired, but the lease may be held for longer + * than the break period. If this header does not appear with a break operation, a fixed-duration + * lease breaks after the remaining lease period elapses, and an infinite lease breaks + * immediately. + */ + breakPeriod?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface ContainerChangeLeaseOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface PageBlobCreateOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Optional. Indicates the tier to be set on the page blob. Possible values include: 'P4', 'P6', + * 'P10', 'P15', 'P20', 'P30', 'P40', 'P50', 'P60', 'P70', 'P80' + */ + tier?: PremiumPageBlobAccessTier; + /** + * Optional. Specifies a user-defined name-value pair associated with the blob. If no name-value + * pairs are specified, the operation will copy the metadata from the source blob or file to the + * destination blob. If one or more name-value pairs are specified, the destination blob is + * created with the specified metadata, and metadata is not copied from the source blob or file. + * Note that beginning with version 2009-09-19, metadata names must adhere to the naming rules + * for C# identifiers. See Naming and Referencing Containers, Blobs, and Metadata for more + * information. + */ + metadata?: { [propertyName: string]: string }; + /** + * Set for page blobs only. The sequence number is a user-controlled value that you can use to + * track requests. The value of the sequence number must be between 0 and 2^63 - 1. Default + * value: 0. + */ + blobSequenceNumber?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Optional. Used to set blob tags in various blob operations. + */ + blobTagsString?: string; + /** + * Specifies the date time when the blobs immutability policy is set to expire. + */ + immutabilityPolicyExpiry?: Date; + /** + * Specifies the immutability policy mode to set on the blob. Possible values include: 'Mutable', + * 'Unlocked', 'Locked' + */ + immutabilityPolicyMode?: BlobImmutabilityPolicyMode; + /** + * Specified if a legal hold should be set on the blob. + */ + legalHold?: boolean; + /** + * Additional parameters for the operation + */ + blobHTTPHeaders?: BlobHTTPHeaders; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; + /** + * Additional parameters for the operation + */ + cpkInfo?: CpkInfo; + /** + * Additional parameters for the operation + */ + cpkScopeInfo?: CpkScopeInfo; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface PageBlobUploadPagesOptionalParams { + /** + * Specify the transactional md5 for the body, to be validated by the service. + */ + transactionalContentMD5?: Uint8Array; + /** + * Specify the transactional crc64 for the body, to be validated by the service. + */ + transactionalContentCrc64?: Uint8Array; + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * The HTTP Range request header specifies one or more byte ranges of the resource to be + * retrieved. + */ + range?: string; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; + /** + * Additional parameters for the operation + */ + cpkInfo?: CpkInfo; + /** + * Additional parameters for the operation + */ + cpkScopeInfo?: CpkScopeInfo; + /** + * Additional parameters for the operation + */ + sequenceNumberAccessConditions?: SequenceNumberAccessConditions; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface PageBlobClearPagesOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * The HTTP Range request header specifies one or more byte ranges of the resource to be + * retrieved. + */ + range?: string; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; + /** + * Additional parameters for the operation + */ + cpkInfo?: CpkInfo; + /** + * Additional parameters for the operation + */ + cpkScopeInfo?: CpkScopeInfo; + /** + * Additional parameters for the operation + */ + sequenceNumberAccessConditions?: SequenceNumberAccessConditions; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface PageBlobUploadPagesFromURLOptionalParams { + /** + * Specify the md5 calculated for the range of bytes that must be read from the copy source. + */ + sourceContentMD5?: Uint8Array; + /** + * Specify the crc64 calculated for the range of bytes that must be read from the copy source. + */ + sourceContentcrc64?: Uint8Array; + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Only Bearer type is supported. Credentials should be a valid OAuth access token to copy + * source. + */ + copySourceAuthorization?: string; + /** + * Additional parameters for the operation + */ + cpkInfo?: CpkInfo; + /** + * Additional parameters for the operation + */ + cpkScopeInfo?: CpkScopeInfo; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; + /** + * Additional parameters for the operation + */ + sequenceNumberAccessConditions?: SequenceNumberAccessConditions; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; + /** + * Additional parameters for the operation + */ + sourceModifiedAccessConditions?: SourceModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface PageBlobGetPageRangesOptionalParams { + /** + * The snapshot parameter is an opaque DateTime value that, when present, specifies the blob + * snapshot to retrieve. For more information on working with blob snapshots, see Creating + * a Snapshot of a Blob. + */ + snapshot?: string; + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * The HTTP Range request header specifies one or more byte ranges of the resource to be + * retrieved. + */ + range?: string; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * A string value that identifies the portion of the list of containers to be returned with the + * next listing operation. The operation returns the NextMarker value within the response body if + * the listing operation did not return all containers remaining to be listed with the current + * page. The NextMarker value can be used as the value for the marker parameter in a subsequent + * call to request the next page of list items. The marker value is opaque to the client. + */ + marker?: string; + /** + * Specifies the maximum number of containers to return. If the request does not specify + * maxresults, or specifies a value greater than 5000, the server will return up to 5000 items. + * Note that if the listing operation crosses a partition boundary, then the service will return + * a continuation token for retrieving the remainder of the results. For this reason, it is + * possible that the service will return fewer results than specified by maxresults, or than the + * default of 5000. + */ + maxresults?: number; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface PageBlobGetPageRangesDiffOptionalParams { + /** + * The snapshot parameter is an opaque DateTime value that, when present, specifies the blob + * snapshot to retrieve. For more information on working with blob snapshots, see Creating + * a Snapshot of a Blob. + */ + snapshot?: string; + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Optional in version 2015-07-08 and newer. The prevsnapshot parameter is a DateTime value that + * specifies that the response will contain only pages that were changed between target blob and + * previous snapshot. Changed pages include both updated and cleared pages. The target blob may + * be a snapshot, as long as the snapshot specified by prevsnapshot is the older of the two. Note + * that incremental snapshots are currently supported only for blobs created on or after January + * 1, 2016. + */ + prevsnapshot?: string; + /** + * Optional. This header is only supported in service versions 2019-04-19 and after and specifies + * the URL of a previous snapshot of the target blob. The response will only contain pages that + * were changed between the target blob and its previous snapshot. + */ + prevSnapshotUrl?: string; + /** + * The HTTP Range request header specifies one or more byte ranges of the resource to be + * retrieved. + */ + range?: string; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * A string value that identifies the portion of the list of containers to be returned with the + * next listing operation. The operation returns the NextMarker value within the response body if + * the listing operation did not return all containers remaining to be listed with the current + * page. The NextMarker value can be used as the value for the marker parameter in a subsequent + * call to request the next page of list items. The marker value is opaque to the client. + */ + marker?: string; + /** + * Specifies the maximum number of containers to return. If the request does not specify + * maxresults, or specifies a value greater than 5000, the server will return up to 5000 items. + * Note that if the listing operation crosses a partition boundary, then the service will return + * a continuation token for retrieving the remainder of the results. For this reason, it is + * possible that the service will return fewer results than specified by maxresults, or than the + * default of 5000. + */ + maxresults?: number; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface PageBlobResizeOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; + /** + * Additional parameters for the operation + */ + cpkInfo?: CpkInfo; + /** + * Additional parameters for the operation + */ + cpkScopeInfo?: CpkScopeInfo; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface PageBlobUpdateSequenceNumberOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Set for page blobs only. The sequence number is a user-controlled value that you can use to + * track requests. The value of the sequence number must be between 0 and 2^63 - 1. Default + * value: 0. + */ + blobSequenceNumber?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface PageBlobCopyIncrementalOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface AppendBlobCreateOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Optional. Specifies a user-defined name-value pair associated with the blob. If no name-value + * pairs are specified, the operation will copy the metadata from the source blob or file to the + * destination blob. If one or more name-value pairs are specified, the destination blob is + * created with the specified metadata, and metadata is not copied from the source blob or file. + * Note that beginning with version 2009-09-19, metadata names must adhere to the naming rules + * for C# identifiers. See Naming and Referencing Containers, Blobs, and Metadata for more + * information. + */ + metadata?: { [propertyName: string]: string }; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Optional. Used to set blob tags in various blob operations. + */ + blobTagsString?: string; + /** + * Specifies the date time when the blobs immutability policy is set to expire. + */ + immutabilityPolicyExpiry?: Date; + /** + * Specifies the immutability policy mode to set on the blob. Possible values include: 'Mutable', + * 'Unlocked', 'Locked' + */ + immutabilityPolicyMode?: BlobImmutabilityPolicyMode; + /** + * Specified if a legal hold should be set on the blob. + */ + legalHold?: boolean; + /** + * Additional parameters for the operation + */ + blobHTTPHeaders?: BlobHTTPHeaders; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; + /** + * Additional parameters for the operation + */ + cpkInfo?: CpkInfo; + /** + * Additional parameters for the operation + */ + cpkScopeInfo?: CpkScopeInfo; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface AppendBlobAppendBlockOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Specify the transactional md5 for the body, to be validated by the service. + */ + transactionalContentMD5?: Uint8Array; + /** + * Specify the transactional crc64 for the body, to be validated by the service. + */ + transactionalContentCrc64?: Uint8Array; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; + /** + * Additional parameters for the operation + */ + appendPositionAccessConditions?: AppendPositionAccessConditions; + /** + * Additional parameters for the operation + */ + cpkInfo?: CpkInfo; + /** + * Additional parameters for the operation + */ + cpkScopeInfo?: CpkScopeInfo; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface AppendBlobAppendBlockFromUrlOptionalParams { + /** + * Bytes of source data in the specified range. + */ + sourceRange?: string; + /** + * Specify the md5 calculated for the range of bytes that must be read from the copy source. + */ + sourceContentMD5?: Uint8Array; + /** + * Specify the crc64 calculated for the range of bytes that must be read from the copy source. + */ + sourceContentcrc64?: Uint8Array; + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Specify the transactional md5 for the body, to be validated by the service. + */ + transactionalContentMD5?: Uint8Array; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Only Bearer type is supported. Credentials should be a valid OAuth access token to copy + * source. + */ + copySourceAuthorization?: string; + /** + * Additional parameters for the operation + */ + cpkInfo?: CpkInfo; + /** + * Additional parameters for the operation + */ + cpkScopeInfo?: CpkScopeInfo; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; + /** + * Additional parameters for the operation + */ + appendPositionAccessConditions?: AppendPositionAccessConditions; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; + /** + * Additional parameters for the operation + */ + sourceModifiedAccessConditions?: SourceModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface AppendBlobSealOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; + /** + * Additional parameters for the operation + */ + appendPositionAccessConditions?: AppendPositionAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface BlockBlobUploadOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Specify the transactional md5 for the body, to be validated by the service. + */ + transactionalContentMD5?: Uint8Array; + /** + * Optional. Specifies a user-defined name-value pair associated with the blob. If no name-value + * pairs are specified, the operation will copy the metadata from the source blob or file to the + * destination blob. If one or more name-value pairs are specified, the destination blob is + * created with the specified metadata, and metadata is not copied from the source blob or file. + * Note that beginning with version 2009-09-19, metadata names must adhere to the naming rules + * for C# identifiers. See Naming and Referencing Containers, Blobs, and Metadata for more + * information. + */ + metadata?: { [propertyName: string]: string }; + /** + * Optional. Indicates the tier to be set on the blob. Possible values include: 'P4', 'P6', + * 'P10', 'P15', 'P20', 'P30', 'P40', 'P50', 'P60', 'P70', 'P80', 'Hot', 'Cool', 'Archive', + * 'Premium' + */ + tier?: AccessTier; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Optional. Used to set blob tags in various blob operations. + */ + blobTagsString?: string; + /** + * Specifies the date time when the blobs immutability policy is set to expire. + */ + immutabilityPolicyExpiry?: Date; + /** + * Specifies the immutability policy mode to set on the blob. Possible values include: 'Mutable', + * 'Unlocked', 'Locked' + */ + immutabilityPolicyMode?: BlobImmutabilityPolicyMode; + /** + * Specified if a legal hold should be set on the blob. + */ + legalHold?: boolean; + /** + * Specify the transactional crc64 for the body, to be validated by the service. + */ + transactionalContentCrc64?: Uint8Array; + /** + * Additional parameters for the operation + */ + blobHTTPHeaders?: BlobHTTPHeaders; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; + /** + * Additional parameters for the operation + */ + cpkInfo?: CpkInfo; + /** + * Additional parameters for the operation + */ + cpkScopeInfo?: CpkScopeInfo; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface BlockBlobPutBlobFromUrlOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Specify the transactional md5 for the body, to be validated by the service. + */ + transactionalContentMD5?: Uint8Array; + /** + * Optional. Specifies a user-defined name-value pair associated with the blob. If no name-value + * pairs are specified, the operation will copy the metadata from the source blob or file to the + * destination blob. If one or more name-value pairs are specified, the destination blob is + * created with the specified metadata, and metadata is not copied from the source blob or file. + * Note that beginning with version 2009-09-19, metadata names must adhere to the naming rules + * for C# identifiers. See Naming and Referencing Containers, Blobs, and Metadata for more + * information. + */ + metadata?: { [propertyName: string]: string }; + /** + * Optional. Indicates the tier to be set on the blob. Possible values include: 'P4', 'P6', + * 'P10', 'P15', 'P20', 'P30', 'P40', 'P50', 'P60', 'P70', 'P80', 'Hot', 'Cool', 'Archive', + * 'Premium' + */ + tier?: AccessTier; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Specify the md5 calculated for the range of bytes that must be read from the copy source. + */ + sourceContentMD5?: Uint8Array; + /** + * Optional. Used to set blob tags in various blob operations. + */ + blobTagsString?: string; + /** + * Optional, default is true. Indicates if properties from the source blob should be copied. + */ + copySourceBlobProperties?: boolean; + /** + * Only Bearer type is supported. Credentials should be a valid OAuth access token to copy + * source. + */ + copySourceAuthorization?: string; + /** + * Optional, default 'replace'. Indicates if source tags should be copied or replaced with the + * tags specified by x-ms-tags. Possible values include: 'REPLACE', 'COPY' + */ + copySourceTags?: BlobCopySourceTags; + /** + * Additional parameters for the operation + */ + blobHTTPHeaders?: BlobHTTPHeaders; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; + /** + * Additional parameters for the operation + */ + cpkInfo?: CpkInfo; + /** + * Additional parameters for the operation + */ + cpkScopeInfo?: CpkScopeInfo; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; + /** + * Additional parameters for the operation + */ + sourceModifiedAccessConditions?: SourceModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface BlockBlobStageBlockOptionalParams { + /** + * Specify the transactional md5 for the body, to be validated by the service. + */ + transactionalContentMD5?: Uint8Array; + /** + * Specify the transactional crc64 for the body, to be validated by the service. + */ + transactionalContentCrc64?: Uint8Array; + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; + /** + * Additional parameters for the operation + */ + cpkInfo?: CpkInfo; + /** + * Additional parameters for the operation + */ + cpkScopeInfo?: CpkScopeInfo; +} + +/** + * Optional Parameters. + */ +export interface BlockBlobStageBlockFromURLOptionalParams { + /** + * Bytes of source data in the specified range. + */ + sourceRange?: string; + /** + * Specify the md5 calculated for the range of bytes that must be read from the copy source. + */ + sourceContentMD5?: Uint8Array; + /** + * Specify the crc64 calculated for the range of bytes that must be read from the copy source. + */ + sourceContentcrc64?: Uint8Array; + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Only Bearer type is supported. Credentials should be a valid OAuth access token to copy + * source. + */ + copySourceAuthorization?: string; + /** + * Additional parameters for the operation + */ + cpkInfo?: CpkInfo; + /** + * Additional parameters for the operation + */ + cpkScopeInfo?: CpkScopeInfo; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; + /** + * Additional parameters for the operation + */ + sourceModifiedAccessConditions?: SourceModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface BlockBlobCommitBlockListOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Specify the transactional md5 for the body, to be validated by the service. + */ + transactionalContentMD5?: Uint8Array; + /** + * Specify the transactional crc64 for the body, to be validated by the service. + */ + transactionalContentCrc64?: Uint8Array; + /** + * Optional. Specifies a user-defined name-value pair associated with the blob. If no name-value + * pairs are specified, the operation will copy the metadata from the source blob or file to the + * destination blob. If one or more name-value pairs are specified, the destination blob is + * created with the specified metadata, and metadata is not copied from the source blob or file. + * Note that beginning with version 2009-09-19, metadata names must adhere to the naming rules + * for C# identifiers. See Naming and Referencing Containers, Blobs, and Metadata for more + * information. + */ + metadata?: { [propertyName: string]: string }; + /** + * Optional. Indicates the tier to be set on the blob. Possible values include: 'P4', 'P6', + * 'P10', 'P15', 'P20', 'P30', 'P40', 'P50', 'P60', 'P70', 'P80', 'Hot', 'Cool', 'Archive', + * 'Premium' + */ + tier?: AccessTier; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Optional. Used to set blob tags in various blob operations. + */ + blobTagsString?: string; + /** + * Specifies the date time when the blobs immutability policy is set to expire. + */ + immutabilityPolicyExpiry?: Date; + /** + * Specifies the immutability policy mode to set on the blob. Possible values include: 'Mutable', + * 'Unlocked', 'Locked' + */ + immutabilityPolicyMode?: BlobImmutabilityPolicyMode; + /** + * Specified if a legal hold should be set on the blob. + */ + legalHold?: boolean; + /** + * Additional parameters for the operation + */ + blobHTTPHeaders?: BlobHTTPHeaders; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; + /** + * Additional parameters for the operation + */ + cpkInfo?: CpkInfo; + /** + * Additional parameters for the operation + */ + cpkScopeInfo?: CpkScopeInfo; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface BlockBlobGetBlockListOptionalParams { + /** + * The snapshot parameter is an opaque DateTime value that, when present, specifies the blob + * snapshot to retrieve. For more information on working with blob snapshots, see Creating + * a Snapshot of a Blob. + */ + snapshot?: string; + /** + * Specifies whether to return the list of committed blocks, the list of uncommitted blocks, or + * both lists together. Possible values include: 'committed', 'uncommitted', 'all'. Default + * value: 'committed'. + */ + listType?: BlockListType; + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface BlobUndeleteOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; +} + +/** + * Optional Parameters. + */ +export interface BlobSetExpiryOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * The time to set the blob to expiry + */ + expiresOn?: string; +} + +/** + * Optional Parameters. + */ +export interface BlobSetHTTPHeadersOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Additional parameters for the operation + */ + blobHTTPHeaders?: BlobHTTPHeaders; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface BlobSetImmutabilityPolicyOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Specifies the date time when the blobs immutability policy is set to expire. + */ + immutabilityPolicyExpiry?: Date; + /** + * Specifies the immutability policy mode to set on the blob. Possible values include: 'Mutable', + * 'Unlocked', 'Locked' + */ + immutabilityPolicyMode?: BlobImmutabilityPolicyMode; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface BlobDeleteImmutabilityPolicyOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; +} + +/** + * Optional Parameters. + */ +export interface BlobSetLegalHoldOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; +} + +/** + * Optional Parameters. + */ +export interface BlobSetMetadataOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Optional. Specifies a user-defined name-value pair associated with the blob. If no name-value + * pairs are specified, the operation will copy the metadata from the source blob or file to the + * destination blob. If one or more name-value pairs are specified, the destination blob is + * created with the specified metadata, and metadata is not copied from the source blob or file. + * Note that beginning with version 2009-09-19, metadata names must adhere to the naming rules + * for C# identifiers. See Naming and Referencing Containers, Blobs, and Metadata for more + * information. + */ + metadata?: { [propertyName: string]: string }; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; + /** + * Additional parameters for the operation + */ + cpkInfo?: CpkInfo; + /** + * Additional parameters for the operation + */ + cpkScopeInfo?: CpkScopeInfo; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface BlobAcquireLeaseOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Specifies the duration of the lease, in seconds, or negative one (-1) for a lease that never + * expires. A non-infinite lease can be between 15 and 60 seconds. A lease duration cannot be + * changed using renew or change. + */ + duration?: number; + /** + * Proposed lease ID, in a GUID string format. The Blob service returns 400 (Invalid request) if + * the proposed lease ID is not in the correct format. See Guid Constructor (String) for a list + * of valid GUID string formats. + */ + proposedLeaseId?: string; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface BlobReleaseLeaseOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface BlobRenewLeaseOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface BlobChangeLeaseOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface BlobBreakLeaseOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * For a break operation, proposed duration the lease should continue before it is broken, in + * seconds, between 0 and 60. This break period is only used if it is shorter than the time + * remaining on the lease. If longer, the time remaining on the lease is used. A new lease will + * not be available before the break period has expired, but the lease may be held for longer + * than the break period. If this header does not appear with a break operation, a fixed-duration + * lease breaks after the remaining lease period elapses, and an infinite lease breaks + * immediately. + */ + breakPeriod?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface BlobCreateSnapshotOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Optional. Specifies a user-defined name-value pair associated with the blob. If no name-value + * pairs are specified, the operation will copy the metadata from the source blob or file to the + * destination blob. If one or more name-value pairs are specified, the destination blob is + * created with the specified metadata, and metadata is not copied from the source blob or file. + * Note that beginning with version 2009-09-19, metadata names must adhere to the naming rules + * for C# identifiers. See Naming and Referencing Containers, Blobs, and Metadata for more + * information. + */ + metadata?: { [propertyName: string]: string }; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Additional parameters for the operation + */ + cpkInfo?: CpkInfo; + /** + * Additional parameters for the operation + */ + cpkScopeInfo?: CpkScopeInfo; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface BlobStartCopyFromURLOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Optional. Specifies a user-defined name-value pair associated with the blob. If no name-value + * pairs are specified, the operation will copy the metadata from the source blob or file to the + * destination blob. If one or more name-value pairs are specified, the destination blob is + * created with the specified metadata, and metadata is not copied from the source blob or file. + * Note that beginning with version 2009-09-19, metadata names must adhere to the naming rules + * for C# identifiers. See Naming and Referencing Containers, Blobs, and Metadata for more + * information. + */ + metadata?: { [propertyName: string]: string }; + /** + * Optional. Indicates the tier to be set on the blob. Possible values include: 'P4', 'P6', + * 'P10', 'P15', 'P20', 'P30', 'P40', 'P50', 'P60', 'P70', 'P80', 'Hot', 'Cool', 'Archive', + * 'Premium' + */ + tier?: AccessTier; + /** + * Optional: Indicates the priority with which to rehydrate an archived blob. Possible values + * include: 'High', 'Standard' + */ + rehydratePriority?: RehydratePriority; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Optional. Used to set blob tags in various blob operations. + */ + blobTagsString?: string; + /** + * Overrides the sealed state of the destination blob. Service version 2019-12-12 and newer. + */ + sealBlob?: boolean; + /** + * Specifies the date time when the blobs immutability policy is set to expire. + */ + immutabilityPolicyExpiry?: Date; + /** + * Specifies the immutability policy mode to set on the blob. Possible values include: 'Mutable', + * 'Unlocked', 'Locked' + */ + immutabilityPolicyMode?: BlobImmutabilityPolicyMode; + /** + * Specified if a legal hold should be set on the blob. + */ + legalHold?: boolean; + /** + * Additional parameters for the operation + */ + sourceModifiedAccessConditions?: SourceModifiedAccessConditions; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface BlobCopyFromURLOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Optional. Specifies a user-defined name-value pair associated with the blob. If no name-value + * pairs are specified, the operation will copy the metadata from the source blob or file to the + * destination blob. If one or more name-value pairs are specified, the destination blob is + * created with the specified metadata, and metadata is not copied from the source blob or file. + * Note that beginning with version 2009-09-19, metadata names must adhere to the naming rules + * for C# identifiers. See Naming and Referencing Containers, Blobs, and Metadata for more + * information. + */ + metadata?: { [propertyName: string]: string }; + /** + * Optional. Indicates the tier to be set on the blob. Possible values include: 'P4', 'P6', + * 'P10', 'P15', 'P20', 'P30', 'P40', 'P50', 'P60', 'P70', 'P80', 'Hot', 'Cool', 'Archive', + * 'Premium' + */ + tier?: AccessTier; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Specify the md5 calculated for the range of bytes that must be read from the copy source. + */ + sourceContentMD5?: Uint8Array; + /** + * Optional. Used to set blob tags in various blob operations. + */ + blobTagsString?: string; + /** + * Specifies the date time when the blobs immutability policy is set to expire. + */ + immutabilityPolicyExpiry?: Date; + /** + * Specifies the immutability policy mode to set on the blob. Possible values include: 'Mutable', + * 'Unlocked', 'Locked' + */ + immutabilityPolicyMode?: BlobImmutabilityPolicyMode; + /** + * Specified if a legal hold should be set on the blob. + */ + legalHold?: boolean; + /** + * Only Bearer type is supported. Credentials should be a valid OAuth access token to copy + * source. + */ + copySourceAuthorization?: string; + /** + * Optional, default 'replace'. Indicates if source tags should be copied or replaced with the + * tags specified by x-ms-tags. Possible values include: 'REPLACE', 'COPY' + */ + copySourceTags?: BlobCopySourceTags; + /** + * Additional parameters for the operation + */ + sourceModifiedAccessConditions?: SourceModifiedAccessConditions; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; + /** + * Additional parameters for the operation + */ + cpkScopeInfo?: CpkScopeInfo; +} + +/** + * Optional Parameters. + */ +export interface BlobAbortCopyFromURLOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface BlobSetTierOptionalParams { + /** + * The snapshot parameter is an opaque DateTime value that, when present, specifies the blob + * snapshot to retrieve. For more information on working with blob snapshots, see Creating + * a Snapshot of a Blob. + */ + snapshot?: string; + /** + * The version id parameter is an opaque DateTime value that, when present, specifies the version + * of the blob to operate on. It's for service version 2019-10-10 and newer. + */ + versionId?: string; + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Optional: Indicates the priority with which to rehydrate an archived blob. Possible values + * include: 'High', 'Standard' + */ + rehydratePriority?: RehydratePriority; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface BlobQueryOptionalParams { + /** + * the query request + */ + queryRequest?: QueryRequest; + /** + * The snapshot parameter is an opaque DateTime value that, when present, specifies the blob + * snapshot to retrieve. For more information on working with blob snapshots, see Creating + * a Snapshot of a Blob. + */ + snapshot?: string; + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; + /** + * Additional parameters for the operation + */ + cpkInfo?: CpkInfo; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface BlobGetTagsOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * The snapshot parameter is an opaque DateTime value that, when present, specifies the blob + * snapshot to retrieve. For more information on working with blob snapshots, see Creating + * a Snapshot of a Blob. + */ + snapshot?: string; + /** + * The version id parameter is an opaque DateTime value that, when present, specifies the version + * of the blob to operate on. It's for service version 2019-10-10 and newer. + */ + versionId?: string; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; +} + +/** + * Optional Parameters. + */ +export interface BlobSetTagsOptionalParams { + /** + * The timeout parameter is expressed in seconds. For more information, see Setting + * Timeouts for Blob Service Operations. + */ + timeout?: number; + /** + * The version id parameter is an opaque DateTime value that, when present, specifies the version + * of the blob to operate on. It's for service version 2019-10-10 and newer. + */ + versionId?: string; + /** + * Specify the transactional md5 for the body, to be validated by the service. + */ + transactionalContentMD5?: Uint8Array; + /** + * Specify the transactional crc64 for the body, to be validated by the service. + */ + transactionalContentCrc64?: Uint8Array; + /** + * Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + */ + requestId?: string; + /** + * Blob tags + */ + tags?: BlobTags; + /** + * Additional parameters for the operation + */ + modifiedAccessConditions?: ModifiedAccessConditions; + /** + * Additional parameters for the operation + */ + leaseAccessConditions?: LeaseAccessConditions; +} + +/** + * Defines headers for ListFileSystems operation. + */ +export interface ServiceListFileSystemsHeaders { + /** + * A UTC date/time value generated by the service that indicates the time at which the response + * was initiated. + */ + date?: Date; + /** + * A server-generated UUID recorded in the analytics logs for troubleshooting and correlation. + */ + requestId?: string; + /** + * The version of the REST protocol used to process the request. + */ + version?: string; + /** + * If the number of filesystems to be listed exceeds the maxResults limit, a continuation token + * is returned in this response header. When a continuation token is returned in the response, + * it must be specified in a subsequent invocation of the list operation to continue listing the + * filesystems. + */ + continuation?: string; + /** + * The content type of list filesystem response. The default content type is application/json. + */ + contentType?: string; + errorCode?: string; +} + +/** + * Defines headers for Create operation. + */ +export interface FileSystemCreateHeaders { + /** + * A UTC date/time value generated by the service that indicates the time at which the response + * was initiated. + */ + date?: Date; + /** + * An HTTP entity tag associated with the FileSystem. + */ + eTag?: string; + /** + * The data and time the filesystem was last modified. Operations on files and directories do + * not affect the last modified time. + */ + lastModified?: Date; + /** + * A server-generated UUID recorded in the analytics logs for troubleshooting and correlation. + */ + clientRequestId?: string; + /** + * The version of the REST protocol used to process the request. + */ + version?: string; + /** + * A bool string indicates whether the namespace feature is enabled. If "true", the namespace is + * enabled for the filesystem. + */ + namespaceEnabled?: string; + errorCode?: string; +} + +/** + * Defines headers for SetProperties operation. + */ +export interface FileSystemSetPropertiesHeaders { + /** + * A UTC date/time value generated by the service that indicates the time at which the response + * was initiated. + */ + date?: Date; + /** + * An HTTP entity tag associated with the filesystem. Changes to filesystem properties affect + * the entity tag, but operations on files and directories do not. + */ + eTag?: string; + /** + * The data and time the filesystem was last modified. Changes to filesystem properties update + * the last modified time, but operations on files and directories do not. + */ + lastModified?: Date; + /** + * A server-generated UUID recorded in the analytics logs for troubleshooting and correlation. + */ + requestId?: string; + /** + * The version of the REST protocol used to process the request. + */ + version?: string; + errorCode?: string; +} + +/** + * Defines headers for GetProperties operation. + */ +export interface FileSystemGetPropertiesHeaders { + /** + * A UTC date/time value generated by the service that indicates the time at which the response + * was initiated. + */ + date?: Date; + /** + * An HTTP entity tag associated with the filesystem. Changes to filesystem properties affect + * the entity tag, but operations on files and directories do not. + */ + eTag?: string; + /** + * The data and time the filesystem was last modified. Changes to filesystem properties update + * the last modified time, but operations on files and directories do not. + */ + lastModified?: Date; + /** + * A server-generated UUID recorded in the analytics logs for troubleshooting and correlation. + */ + requestId?: string; + /** + * The version of the REST protocol used to process the request. + */ + version?: string; + /** + * The user-defined properties associated with the filesystem. A comma-separated list of name + * and value pairs in the format "n1=v1, n2=v2, ...", where each value is a base64 encoded + * string. Note that the string may only contain ASCII characters in the ISO-8859-1 character + * set. + */ + properties?: string; + /** + * A bool string indicates whether the namespace feature is enabled. If "true", the namespace is + * enabled for the filesystem. + */ + namespaceEnabled?: string; + errorCode?: string; +} + +/** + * Defines headers for Delete operation. + */ +export interface FileSystemDeleteHeaders { + /** + * A server-generated UUID recorded in the analytics logs for troubleshooting and correlation. + */ + requestId?: string; + /** + * The version of the REST protocol used to process the request. + */ + version?: string; + /** + * A UTC date/time value generated by the service that indicates the time at which the response + * was initiated. + */ + date?: Date; + errorCode?: string; +} + +/** + * Defines headers for ListPaths operation. + */ +export interface FileSystemListPathsHeaders { + /** + * A UTC date/time value generated by the service that indicates the time at which the response + * was initiated. + */ + date?: Date; + /** + * An HTTP entity tag associated with the filesystem. Changes to filesystem properties affect + * the entity tag, but operations on files and directories do not. + */ + eTag?: string; + /** + * The data and time the filesystem was last modified. Changes to filesystem properties update + * the last modified time, but operations on files and directories do not. + */ + lastModified?: Date; + /** + * A server-generated UUID recorded in the analytics logs for troubleshooting and correlation. + */ + requestId?: string; + /** + * The version of the REST protocol used to process the request. + */ + version?: string; + /** + * If the number of paths to be listed exceeds the maxResults limit, a continuation token is + * returned in this response header. When a continuation token is returned in the response, it + * must be specified in a subsequent invocation of the list operation to continue listing the + * paths. + */ + continuation?: string; + errorCode?: string; +} + +/** + * Defines headers for ListBlobFlatSegment operation. + */ +export interface FileSystemListBlobFlatSegmentHeaders { + /** + * The media type of the body of the response. For List Blobs this is 'application/xml' + */ + contentType?: string; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + errorCode?: string; +} + +/** + * Defines headers for ListBlobHierarchySegment operation. + */ +export interface FileSystemListBlobHierarchySegmentHeaders { + /** + * The media type of the body of the response. For List Blobs this is 'application/xml' + */ + contentType?: string; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + errorCode?: string; +} + +/** + * Defines headers for Create operation. + */ +export interface PathCreateHeaders { + /** + * A UTC date/time value generated by the service that indicates the time at which the response + * was initiated. + */ + date?: Date; + /** + * An HTTP entity tag associated with the file or directory. + */ + eTag?: string; + /** + * The data and time the file or directory was last modified. Write operations on the file or + * directory update the last modified time. + */ + lastModified?: Date; + /** + * A server-generated UUID recorded in the analytics logs for troubleshooting and correlation. + */ + requestId?: string; + /** + * The version of the REST protocol used to process the request. + */ + version?: string; + /** + * When renaming a directory, the number of paths that are renamed with each invocation is + * limited. If the number of paths to be renamed exceeds this limit, a continuation token is + * returned in this response header. When a continuation token is returned in the response, it + * must be specified in a subsequent invocation of the rename operation to continue renaming the + * directory. + */ + continuation?: string; + /** + * The size of the resource in bytes. + */ + contentLength?: number; + /** + * The value of this header is set to true if the contents of the request are successfully + * encrypted using the specified algorithm, and false otherwise. + */ + isServerEncrypted?: boolean; + /** + * The SHA-256 hash of the encryption key used to encrypt the blob. This header is only returned + * when the blob was encrypted with a customer-provided key. + */ + encryptionKeySha256?: string; + errorCode?: string; +} + +/** + * Defines headers for Update operation. + */ +export interface PathUpdateHeaders { + /** + * A UTC date/time value generated by the service that indicates the time at which the response + * was initiated. + */ + date?: Date; + /** + * An HTTP entity tag associated with the file or directory. + */ + eTag?: string; + /** + * The data and time the file or directory was last modified. Write operations on the file or + * directory update the last modified time. + */ + lastModified?: Date; + /** + * Indicates that the service supports requests for partial file content. + */ + acceptRanges?: string; + /** + * If the Cache-Control request header has previously been set for the resource, that value is + * returned in this header. + */ + cacheControl?: string; + /** + * If the Content-Disposition request header has previously been set for the resource, that value + * is returned in this header. + */ + contentDisposition?: string; + /** + * If the Content-Encoding request header has previously been set for the resource, that value is + * returned in this header. + */ + contentEncoding?: string; + /** + * If the Content-Language request header has previously been set for the resource, that value is + * returned in this header. + */ + contentLanguage?: string; + /** + * The size of the resource in bytes. + */ + contentLength?: number; + /** + * Indicates the range of bytes returned in the event that the client requested a subset of the + * file by setting the Range request header. + */ + contentRange?: string; + /** + * The content type specified for the resource. If no content type was specified, the default + * content type is application/octet-stream. + */ + contentType?: string; + /** + * An MD5 hash of the request content. This header is only returned for "Append" operation. This + * header is returned so that the client can check for message content integrity. The value of + * this header is computed by the service; it is not necessarily the same value specified in the + * request headers. + */ + contentMD5?: Uint8Array; + /** + * User-defined properties associated with the file or directory, in the format of a + * comma-separated list of name and value pairs "n1=v1, n2=v2, ...", where each value is a base64 + * encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 + * character set. + */ + properties?: string; + /** + * When performing setAccessControlRecursive on a directory, the number of paths that are + * processed with each invocation is limited. If the number of paths to be processed exceeds + * this limit, a continuation token is returned in this response header. When a continuation + * token is returned in the response, it must be specified in a subsequent invocation of the + * setAccessControlRecursive operation to continue the setAccessControlRecursive operation on the + * directory. + */ + xMsContinuation?: string; + /** + * A server-generated UUID recorded in the analytics logs for troubleshooting and correlation. + */ + requestId?: string; + /** + * The version of the REST protocol used to process the request. + */ + version?: string; + errorCode?: string; +} + +/** + * Defines headers for Lease operation. + */ +export interface PathLeaseHeaders { + /** + * A UTC date/time value generated by the service that indicates the time at which the response + * was initiated. + */ + date?: Date; + /** + * An HTTP entity tag associated with the file or directory. + */ + eTag?: string; + /** + * The data and time the file or directory was last modified. Write operations on the file or + * directory update the last modified time. + */ + lastModified?: Date; + /** + * A server-generated UUID recorded in the analytics logs for troubleshooting and correlation. + */ + requestId?: string; + /** + * The version of the REST protocol used to process the request. + */ + version?: string; + /** + * A successful "acquire" action returns the lease ID. + */ + leaseId?: string; + /** + * The time remaining in the lease period in seconds. + */ + leaseTime?: string; + errorCode?: string; +} + +/** + * Defines headers for Read operation. + */ +export interface PathReadHeaders { + /** + * Indicates that the service supports requests for partial file content. + */ + acceptRanges?: string; + /** + * If the Cache-Control request header has previously been set for the resource, that value is + * returned in this header. + */ + cacheControl?: string; + /** + * If the Content-Disposition request header has previously been set for the resource, that value + * is returned in this header. + */ + contentDisposition?: string; + /** + * If the Content-Encoding request header has previously been set for the resource, that value is + * returned in this header. + */ + contentEncoding?: string; + /** + * If the Content-Language request header has previously been set for the resource, that value is + * returned in this header. + */ + contentLanguage?: string; + /** + * The size of the resource in bytes. + */ + contentLength?: number; + /** + * Indicates the range of bytes returned in the event that the client requested a subset of the + * file by setting the Range request header. + */ + contentRange?: string; + /** + * The content type specified for the resource. If no content type was specified, the default + * content type is application/octet-stream. + */ + contentType?: string; + /** + * The MD5 hash of read range. If the request is to read a specified range and the + * "x-ms-range-get-content-md5" is set to true, then the request returns an MD5 hash for the + * range, as long as the range size is less than or equal to 4 MB. + */ + contentMD5?: Uint8Array; + /** + * A UTC date/time value generated by the service that indicates the time at which the response + * was initiated. + */ + date?: Date; + /** + * An HTTP entity tag associated with the file or directory. + */ + eTag?: string; + /** + * The data and time the file or directory was last modified. Write operations on the file or + * directory update the last modified time. + */ + lastModified?: Date; + /** + * A server-generated UUID recorded in the analytics logs for troubleshooting and correlation. + */ + requestId?: string; + /** + * The version of the REST protocol used to process the request. + */ + version?: string; + /** + * The type of the resource. The value may be "file" or "directory". If not set, the value is + * "file". + */ + resourceType?: string; + /** + * The user-defined properties associated with the file or directory, in the format of a + * comma-separated list of name and value pairs "n1=v1, n2=v2, ...", where each value is a base64 + * encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 + * character set. + */ + properties?: string; + /** + * When a resource is leased, specifies whether the lease is of infinite or fixed duration. + * Possible values include: 'infinite', 'fixed' + */ + leaseDuration?: LeaseDurationType; + /** + * Lease state of the resource. Possible values include: 'available', 'leased', 'expired', + * 'breaking', 'broken' + */ + leaseState?: LeaseStateType; + /** + * The lease status of the resource. Possible values include: 'locked', 'unlocked' + */ + leaseStatus?: LeaseStatusType; + /** + * The value of this header is set to true if the contents of the request are successfully + * encrypted using the specified algorithm, and false otherwise. + */ + isServerEncrypted?: boolean; + /** + * The SHA-256 hash of the encryption key used to encrypt the blob. This header is only returned + * when the blob was encrypted with a customer-provided key. + */ + encryptionKeySha256?: string; + metadata?: { [propertyName: string]: string }; + /** + * Returns the date and time the blob was created. + */ + creationTime?: Date; + /** + * Optional. Only valid when Object Replication is enabled for the storage container and on the + * destination blob of the replication. + */ + objectReplicationPolicyId?: string; + objectReplicationRules?: { [propertyName: string]: string }; + /** + * The current sequence number for a page blob. This header is not returned for block blobs or + * append blobs + */ + blobSequenceNumber?: number; + /** + * The blob's type. Possible values include: 'BlockBlob', 'PageBlob', 'AppendBlob' + */ + blobType?: BlobType; + /** + * Conclusion time of the last attempted Copy Blob operation where this blob was the destination + * blob. This value can specify the time of a completed, aborted, or failed copy attempt. This + * header does not appear if a copy is pending, if this blob has never been the destination in a + * Copy Blob operation, or if this blob has been modified after a concluded Copy Blob operation + * using Set Blob Properties, Put Blob, or Put Block List. + */ + copyCompletionTime?: Date; + /** + * Only appears when x-ms-copy-status is failed or pending. Describes the cause of the last fatal + * or non-fatal copy operation failure. This header does not appear if this blob has never been + * the destination in a Copy Blob operation, or if this blob has been modified after a concluded + * Copy Blob operation using Set Blob Properties, Put Blob, or Put Block List + */ + copyStatusDescription?: string; + /** + * String identifier for this copy operation. Use with Get Blob Properties to check the status of + * this copy operation, or pass to Abort Copy Blob to abort a pending copy. + */ + copyId?: string; + /** + * Contains the number of bytes copied and the total bytes in the source in the last attempted + * Copy Blob operation where this blob was the destination blob. Can show between 0 and + * Content-Length bytes copied. This header does not appear if this blob has never been the + * destination in a Copy Blob operation, or if this blob has been modified after a concluded Copy + * Blob operation using Set Blob Properties, Put Blob, or Put Block List + */ + copyProgress?: string; + /** + * URL up to 2 KB in length that specifies the source blob or file used in the last attempted + * Copy Blob operation where this blob was the destination blob. This header does not appear if + * this blob has never been the destination in a Copy Blob operation, or if this blob has been + * modified after a concluded Copy Blob operation using Set Blob Properties, Put Blob, or Put + * Block List. + */ + copySource?: string; + /** + * State of the copy operation identified by x-ms-copy-id. Possible values include: 'pending', + * 'success', 'aborted', 'failed' + */ + copyStatus?: CopyStatusType; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * A DateTime value returned by the service that uniquely identifies the blob. The value of this + * header indicates the blob version, and may be used in subsequent requests to access this + * version of the blob. + */ + versionId?: string; + /** + * The value of this header indicates whether version of this blob is a current version, see also + * x-ms-version-id header. + */ + isCurrentVersion?: boolean; + /** + * The number of committed blocks present in the blob. This header is returned only for append + * blobs. + */ + blobCommittedBlockCount?: number; + /** + * Returns the name of the encryption scope used to encrypt the blob contents and application + * metadata. Note that the absence of this header implies use of the default account encryption + * scope. + */ + encryptionScope?: string; + /** + * If the blob has a MD5 hash, and if request contains range header (Range or x-ms-range), this + * response header is returned with the value of the whole blob's MD5 value. This value may or + * may not be equal to the value returned in Content-MD5 header, with the latter calculated from + * the requested range + */ + blobContentMD5?: Uint8Array; + /** + * The number of tags associated with the blob + */ + tagCount?: number; + /** + * If this blob has been sealed + */ + isSealed?: boolean; + /** + * UTC date/time value generated by the service that indicates the time at which the blob was + * last read or written to + */ + lastAccessed?: Date; + /** + * UTC date/time value generated by the service that indicates the time at which the blob + * immutability policy will expire. + */ + immutabilityPolicyExpiresOn?: Date; + /** + * Indicates immutability policy mode. + */ + immutabilityPolicyMode?: string; + /** + * Indicates if a legal hold is present on the blob. + */ + legalHold?: boolean; + /** + * The MD5 hash of complete file stored in storage. If the file has a MD5 hash, and if request + * contains range header (Range or x-ms-range), this response header is returned with the value + * of the complete file's MD5 value. This value may or may not be equal to the value returned in + * Content-MD5 header, with the latter calculated from the requested range. + */ + xMsContentMd5?: Uint8Array; + /** + * If the request is to read a specified range and the x-ms-range-get-content-crc64 is set to + * true, then the request returns a crc64 for the range, as long as the range size is less than + * or equal to 4 MB. If both x-ms-range-get-content-crc64 & x-ms-range-get-content-md5 is + * specified in the same request, it will fail with 400(Bad Request) + */ + contentCrc64?: Uint8Array; + errorCode?: string; +} + +/** + * Defines headers for GetProperties operation. + */ +export interface PathGetPropertiesHeaders { + /** + * Indicates that the service supports requests for partial file content. + */ + acceptRanges?: string; + /** + * If the Cache-Control request header has previously been set for the resource, that value is + * returned in this header. + */ + cacheControl?: string; + /** + * If the Content-Disposition request header has previously been set for the resource, that value + * is returned in this header. + */ + contentDisposition?: string; + /** + * If the Content-Encoding request header has previously been set for the resource, that value is + * returned in this header. + */ + contentEncoding?: string; + /** + * If the Content-Language request header has previously been set for the resource, that value is + * returned in this header. + */ + contentLanguage?: string; + /** + * The size of the resource in bytes. + */ + contentLength?: number; + /** + * Indicates the range of bytes returned in the event that the client requested a subset of the + * file by setting the Range request header. + */ + contentRange?: string; + /** + * The content type specified for the resource. If no content type was specified, the default + * content type is application/octet-stream. + */ + contentType?: string; + /** + * The MD5 hash of complete file stored in storage. This header is returned only for + * "GetProperties" operation. If the Content-MD5 header has been set for the file, this response + * header is returned for GetProperties call so that the client can check for message content + * integrity. + */ + contentMD5?: Uint8Array; + /** + * A UTC date/time value generated by the service that indicates the time at which the response + * was initiated. + */ + date?: Date; + /** + * An HTTP entity tag associated with the file or directory. + */ + eTag?: string; + /** + * The data and time the file or directory was last modified. Write operations on the file or + * directory update the last modified time. + */ + lastModified?: Date; + /** + * A server-generated UUID recorded in the analytics logs for troubleshooting and correlation. + */ + requestId?: string; + /** + * The version of the REST protocol used to process the request. + */ + version?: string; + /** + * The type of the resource. The value may be "file" or "directory". If not set, the value is + * "file". + */ + resourceType?: string; + /** + * The user-defined properties associated with the file or directory, in the format of a + * comma-separated list of name and value pairs "n1=v1, n2=v2, ...", where each value is a base64 + * encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 + * character set. + */ + properties?: string; + /** + * The owner of the file or directory. Included in the response if Hierarchical Namespace is + * enabled for the account. + */ + owner?: string; + /** + * The owning group of the file or directory. Included in the response if Hierarchical Namespace + * is enabled for the account. + */ + group?: string; + /** + * The POSIX access permissions for the file owner, the file owning group, and others. Included + * in the response if Hierarchical Namespace is enabled for the account. + */ + permissions?: string; + /** + * The POSIX access control list for the file or directory. Included in the response only if the + * action is "getAccessControl" and Hierarchical Namespace is enabled for the account. + */ + aCL?: string; + /** + * When a resource is leased, specifies whether the lease is of infinite or fixed duration. + * Possible values include: 'infinite', 'fixed' + */ + leaseDuration?: LeaseDurationType; + /** + * Lease state of the resource. Possible values include: 'available', 'leased', 'expired', + * 'breaking', 'broken' + */ + leaseState?: LeaseStateType; + /** + * The lease status of the resource. Possible values include: 'locked', 'unlocked' + */ + leaseStatus?: LeaseStatusType; + metadata?: { [propertyName: string]: string }; + /** + * Returns the date and time the blob was created. + */ + creationTime?: Date; + /** + * Optional. Only valid when Object Replication is enabled for the storage container and on the + * destination blob of the replication. + */ + objectReplicationPolicyId?: string; + objectReplicationRules?: { [propertyName: string]: string }; + /** + * The blob's type. Possible values include: 'BlockBlob', 'PageBlob', 'AppendBlob' + */ + blobType?: BlobType; + /** + * Conclusion time of the last attempted Copy Blob operation where this blob was the destination + * blob. This value can specify the time of a completed, aborted, or failed copy attempt. This + * header does not appear if a copy is pending, if this blob has never been the destination in a + * Copy Blob operation, or if this blob has been modified after a concluded Copy Blob operation + * using Set Blob Properties, Put Blob, or Put Block List. + */ + copyCompletionTime?: Date; + /** + * Only appears when x-ms-copy-status is failed or pending. Describes the cause of the last fatal + * or non-fatal copy operation failure. This header does not appear if this blob has never been + * the destination in a Copy Blob operation, or if this blob has been modified after a concluded + * Copy Blob operation using Set Blob Properties, Put Blob, or Put Block List + */ + copyStatusDescription?: string; + /** + * String identifier for this copy operation. Use with Get Blob Properties to check the status of + * this copy operation, or pass to Abort Copy Blob to abort a pending copy. + */ + copyId?: string; + /** + * Contains the number of bytes copied and the total bytes in the source in the last attempted + * Copy Blob operation where this blob was the destination blob. Can show between 0 and + * Content-Length bytes copied. This header does not appear if this blob has never been the + * destination in a Copy Blob operation, or if this blob has been modified after a concluded Copy + * Blob operation using Set Blob Properties, Put Blob, or Put Block List + */ + copyProgress?: string; + /** + * URL up to 2 KB in length that specifies the source blob or file used in the last attempted + * Copy Blob operation where this blob was the destination blob. This header does not appear if + * this blob has never been the destination in a Copy Blob operation, or if this blob has been + * modified after a concluded Copy Blob operation using Set Blob Properties, Put Blob, or Put + * Block List. + */ + copySource?: string; + /** + * State of the copy operation identified by x-ms-copy-id. Possible values include: 'pending', + * 'success', 'aborted', 'failed' + */ + copyStatus?: CopyStatusType; + /** + * Included if the blob is incremental copy blob. + */ + isIncrementalCopy?: boolean; + /** + * Included if the blob is incremental copy blob or incremental copy snapshot, if + * x-ms-copy-status is success. Snapshot time of the last successful incremental copy snapshot + * for this blob. + */ + destinationSnapshot?: string; + /** + * The current sequence number for a page blob. This header is not returned for block blobs or + * append blobs + */ + blobSequenceNumber?: number; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * The number of committed blocks present in the blob. This header is returned only for append + * blobs. + */ + blobCommittedBlockCount?: number; + /** + * The value of this header is set to true if the blob data and application metadata are + * completely encrypted using the specified algorithm. Otherwise, the value is set to false (when + * the blob is unencrypted, or if only parts of the blob/application metadata are encrypted). + */ + isServerEncrypted?: boolean; + /** + * The SHA-256 hash of the encryption key used to encrypt the metadata. This header is only + * returned when the metadata was encrypted with a customer-provided key. + */ + encryptionKeySha256?: string; + /** + * Returns the name of the encryption scope used to encrypt the blob contents and application + * metadata. Note that the absence of this header implies use of the default account encryption + * scope. + */ + encryptionScope?: string; + /** + * The tier of page blob on a premium storage account or tier of block blob on blob storage LRS + * accounts. For a list of allowed premium page blob tiers, see + * https://docs.microsoft.com/en-us/azure/virtual-machines/windows/premium-storage#features. For + * blob storage LRS accounts, valid values are Hot/Cool/Archive. + */ + accessTier?: string; + /** + * For page blobs on a premium storage account only. If the access tier is not explicitly set on + * the blob, the tier is inferred based on its content length and this header will be returned + * with true value. + */ + accessTierInferred?: boolean; + /** + * For blob storage LRS accounts, valid values are + * rehydrate-pending-to-hot/rehydrate-pending-to-cool. If the blob is being rehydrated and is not + * complete then this header is returned indicating that rehydrate is pending and also tells the + * destination tier. + */ + archiveStatus?: string; + /** + * The time the tier was changed on the object. This is only returned if the tier on the block + * blob was ever set. + */ + accessTierChangeTime?: Date; + /** + * A DateTime value returned by the service that uniquely identifies the blob. The value of this + * header indicates the blob version, and may be used in subsequent requests to access this + * version of the blob. + */ + versionId?: string; + /** + * The value of this header indicates whether version of this blob is a current version, see also + * x-ms-version-id header. + */ + isCurrentVersion?: boolean; + /** + * The number of tags associated with the blob + */ + tagCount?: number; + /** + * The time this blob will expire. + */ + expiresOn?: Date; + /** + * If this blob has been sealed + */ + isSealed?: boolean; + /** + * If an object is in rehydrate pending state then this header is returned with priority of + * rehydrate. Valid values are High and Standard. + */ + rehydratePriority?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the blob was + * last read or written to + */ + lastAccessed?: Date; + /** + * UTC date/time value generated by the service that indicates the time at which the blob + * immutability policy will expire. + */ + immutabilityPolicyExpiresOn?: Date; + /** + * Indicates immutability policy mode. Possible values include: 'Mutable', 'Unlocked', 'Locked' + */ + immutabilityPolicyMode?: BlobImmutabilityPolicyMode; + /** + * Indicates if a legal hold is present on the blob. + */ + legalHold?: boolean; + errorCode?: string; +} + +/** + * Defines headers for Delete operation. + */ +export interface PathDeleteHeaders { + /** + * A UTC date/time value generated by the service that indicates the time at which the response + * was initiated. + */ + date?: Date; + /** + * A server-generated UUID recorded in the analytics logs for troubleshooting and correlation. + */ + requestId?: string; + /** + * The version of the REST protocol used to process the request. + */ + version?: string; + /** + * When deleting a directory, the number of paths that are deleted with each invocation is + * limited. If the number of paths to be deleted exceeds this limit, a continuation token is + * returned in this response header. When a continuation token is returned in the response, it + * must be specified in a subsequent invocation of the delete operation to continue deleting the + * directory. + */ + continuation?: string; + /** + * Returned only for hierarchical namespace space enabled accounts when soft delete is enabled. A + * unique identifier for the entity that can be used to restore it. See the Undelete REST API for + * more information. + */ + deletionId?: string; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + errorCode?: string; +} + +/** + * Defines headers for SetAccessControl operation. + */ +export interface PathSetAccessControlHeaders { + /** + * A UTC date/time value generated by the service that indicates the time at which the response + * was initiated. + */ + date?: Date; + /** + * An HTTP entity tag associated with the file or directory. + */ + eTag?: string; + /** + * The data and time the file or directory was last modified. Write operations on the file or + * directory update the last modified time. + */ + lastModified?: Date; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * A server-generated UUID recorded in the analytics logs for troubleshooting and correlation. + */ + requestId?: string; + /** + * The version of the REST protocol used to process the request. + */ + version?: string; +} + +/** + * Defines headers for SetAccessControlRecursive operation. + */ +export interface PathSetAccessControlRecursiveHeaders { + /** + * A UTC date/time value generated by the service that indicates the time at which the response + * was initiated. + */ + date?: Date; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * When performing setAccessControlRecursive on a directory, the number of paths that are + * processed with each invocation is limited. If the number of paths to be processed exceeds + * this limit, a continuation token is returned in this response header. When a continuation + * token is returned in the response, it must be specified in a subsequent invocation of the + * setAccessControlRecursive operation to continue the setAccessControlRecursive operation on the + * directory. + */ + continuation?: string; + /** + * A server-generated UUID recorded in the analytics logs for troubleshooting and correlation. + */ + requestId?: string; + /** + * The version of the REST protocol used to process the request. + */ + version?: string; +} + +/** + * Defines headers for SetProperties operation. + */ +export interface PathSetPropertiesHeaders { + /** + * A UTC date/time value generated by the service that indicates the time at which the response + * was initiated. + */ + date?: Date; + /** + * An HTTP entity tag associated with the file or directory. + */ + eTag?: string; + /** + * The data and time the file or directory was last modified. Write operations on the file or + * directory update the last modified time. + */ + lastModified?: Date; + /** + * If the Cache-Control request header has previously been set for the resource, that value is + * returned in this header. + */ + cacheControl?: string; + /** + * If the Content-Disposition request header has previously been set for the resource, that value + * is returned in this header. + */ + contentDisposition?: string; + /** + * If the Content-Encoding request header has previously been set for the resource, that value is + * returned in this header. + */ + contentEncoding?: string; + /** + * If the Content-Language request header has previously been set for the resource, that value is + * returned in this header. + */ + contentLanguage?: string; + /** + * The size of the resource in bytes. + */ + contentLength?: number; + /** + * The content type specified for the resource. If no content type was specified, the default + * content type is application/octet-stream. + */ + contentType?: string; + /** + * An MD5 hash of the request content. This header is only returned for "Flush" operation. This + * header is returned so that the client can check for message content integrity. This header + * refers to the content of the request, not actual file content. + */ + contentMD5?: Uint8Array; + /** + * User-defined properties associated with the file or directory, in the format of a + * comma-separated list of name and value pairs "n1=v1, n2=v2, ...", where each value is a base64 + * encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 + * character set. + */ + properties?: string; + /** + * A server-generated UUID recorded in the analytics logs for troubleshooting and correlation. + */ + requestId?: string; + /** + * The version of the REST protocol used to process the request. + */ + version?: string; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; +} + +/** + * Defines headers for FlushData operation. + */ +export interface PathFlushDataHeaders { + /** + * A UTC date/time value generated by the service that indicates the time at which the response + * was initiated. + */ + date?: Date; + /** + * An HTTP entity tag associated with the file or directory. + */ + eTag?: string; + /** + * The data and time the file or directory was last modified. Write operations on the file or + * directory update the last modified time. + */ + lastModified?: Date; + /** + * The size of the resource in bytes. + */ + contentLength?: number; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * A server-generated UUID recorded in the analytics logs for troubleshooting and correlation. + */ + requestId?: string; + /** + * The version of the REST protocol used to process the request. + */ + version?: string; + /** + * The value of this header is set to true if the contents of the request are successfully + * encrypted using the specified algorithm, and false otherwise. + */ + isServerEncrypted?: boolean; + /** + * The SHA-256 hash of the encryption key used to encrypt the blob. This header is only returned + * when the blob was encrypted with a customer-provided key. + */ + encryptionKeySha256?: string; +} + +/** + * Defines headers for AppendData operation. + */ +export interface PathAppendDataHeaders { + /** + * A UTC date/time value generated by the service that indicates the time at which the response + * was initiated. + */ + date?: Date; + /** + * A server-generated UUID recorded in the analytics logs for troubleshooting and correlation. + */ + requestId?: string; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * The version of the REST protocol used to process the request. + */ + version?: string; + /** + * An HTTP entity tag associated with the file or directory. + */ + eTag?: string; + /** + * If the blob has an MD5 hash and this operation is to read the full blob, this response header + * is returned so that the client can check for message content integrity. + */ + contentMD5?: Uint8Array; + /** + * This header is returned so that the client can check for message content integrity. The value + * of this header is computed by the Blob service; it is not necessarily the same value specified + * in the request headers. + */ + xMsContentCrc64?: Uint8Array; + /** + * The value of this header is set to true if the contents of the request are successfully + * encrypted using the specified algorithm, and false otherwise. + */ + isServerEncrypted?: boolean; + /** + * The SHA-256 hash of the encryption key used to encrypt the blob. This header is only returned + * when the blob was encrypted with a customer-provided key. + */ + encryptionKeySha256?: string; +} + +/** + * Defines headers for SetExpiry operation. + */ +export interface PathSetExpiryHeaders { + /** + * The ETag contains a value that you can use to perform operations conditionally. If the request + * version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + eTag?: string; + /** + * Returns the date and time the container was last modified. Any operation that modifies the + * blob, including an update of the blob's metadata or properties, changes the last-modified time + * of the blob. + */ + lastModified?: Date; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated. + */ + date?: Date; + errorCode?: string; +} + +/** + * Defines headers for Undelete operation. + */ +export interface PathUndeleteHeaders { + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * The type of the resource. The value may be "file" or "directory". If not set, the value is + * "file". + */ + resourceType?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated. + */ + date?: Date; + errorCode?: string; +} + +/** + * Defines headers for SetProperties operation. + */ +export interface ServiceSetPropertiesHeaders { + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + errorCode?: string; +} + +/** + * Defines headers for GetProperties operation. + */ +export interface ServiceGetPropertiesHeaders { + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + errorCode?: string; +} + +/** + * Defines headers for GetStatistics operation. + */ +export interface ServiceGetStatisticsHeaders { + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + errorCode?: string; +} + +/** + * Defines headers for ListContainersSegment operation. + */ +export interface ServiceListContainersSegmentHeaders { + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + errorCode?: string; +} + +/** + * Defines headers for GetUserDelegationKey operation. + */ +export interface ServiceGetUserDelegationKeyHeaders { + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + errorCode?: string; +} + +/** + * Defines headers for GetAccountInfo operation. + */ +export interface ServiceGetAccountInfoHeaders { + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + /** + * Identifies the sku name of the account. Possible values include: 'Standard_LRS', + * 'Standard_GRS', 'Standard_RAGRS', 'Standard_ZRS', 'Premium_LRS' + */ + skuName?: SkuName; + /** + * Identifies the account kind. Possible values include: 'Storage', 'BlobStorage', 'StorageV2', + * 'FileStorage', 'BlockBlobStorage' + */ + accountKind?: AccountKind; + /** + * Version 2019-07-07 and newer. Indicates if the account has a hierarchical namespace enabled. + */ + isHierarchicalNamespaceEnabled?: boolean; + errorCode?: string; +} + +/** + * Defines headers for GetAccountInfoWithHead operation. + */ +export interface ServiceGetAccountInfoWithHeadHeaders { + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + /** + * Identifies the sku name of the account. Possible values include: 'Standard_LRS', + * 'Standard_GRS', 'Standard_RAGRS', 'Standard_ZRS', 'Premium_LRS' + */ + skuName?: SkuName; + /** + * Identifies the account kind. Possible values include: 'Storage', 'BlobStorage', 'StorageV2', + * 'FileStorage', 'BlockBlobStorage' + */ + accountKind?: AccountKind; + /** + * Version 2019-07-07 and newer. Indicates if the account has a hierarchical namespace enabled. + */ + isHierarchicalNamespaceEnabled?: boolean; + errorCode?: string; +} + +/** + * Defines headers for SubmitBatch operation. + */ +export interface ServiceSubmitBatchHeaders { + /** + * The media type of the body of the response. For batch requests, this is multipart/mixed; + * boundary=batchresponse_GUID + */ + contentType?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + errorCode?: string; +} + +/** + * Defines headers for FilterBlobs operation. + */ +export interface ServiceFilterBlobsHeaders { + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + errorCode?: string; +} + +/** + * Defines headers for Create operation. + */ +export interface ContainerCreateHeaders { + /** + * The ETag contains a value that you can use to perform operations conditionally. If the request + * version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + eTag?: string; + /** + * Returns the date and time the container was last modified. Any operation that modifies the + * blob, including an update of the blob's metadata or properties, changes the last-modified time + * of the blob. + */ + lastModified?: Date; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + errorCode?: string; +} + +/** + * Defines headers for GetProperties operation. + */ +export interface ContainerGetPropertiesHeaders { + metadata?: { [propertyName: string]: string }; + /** + * The ETag contains a value that you can use to perform operations conditionally. If the request + * version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + eTag?: string; + /** + * Returns the date and time the container was last modified. Any operation that modifies the + * blob, including an update of the blob's metadata or properties, changes the last-modified time + * of the blob. + */ + lastModified?: Date; + /** + * When a blob is leased, specifies whether the lease is of infinite or fixed duration. Possible + * values include: 'infinite', 'fixed' + */ + leaseDuration?: LeaseDurationType; + /** + * Lease state of the blob. Possible values include: 'available', 'leased', 'expired', + * 'breaking', 'broken' + */ + leaseState?: LeaseStateType; + /** + * The current lease status of the blob. Possible values include: 'locked', 'unlocked' + */ + leaseStatus?: LeaseStatusType; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + /** + * Indicated whether data in the container may be accessed publicly and the level of access. + * Possible values include: 'container', 'blob' + */ + blobPublicAccess?: PublicAccessType; + /** + * Indicates whether the container has an immutability policy set on it. + */ + hasImmutabilityPolicy?: boolean; + /** + * Indicates whether the container has a legal hold. + */ + hasLegalHold?: boolean; + /** + * The default encryption scope for the container. + */ + defaultEncryptionScope?: string; + /** + * Indicates whether the container's default encryption scope can be overriden. + */ + denyEncryptionScopeOverride?: boolean; + /** + * Indicates whether version level worm is enabled on a container. + */ + isImmutableStorageWithVersioningEnabled?: boolean; + errorCode?: string; +} + +/** + * Defines headers for GetPropertiesWithHead operation. + */ +export interface ContainerGetPropertiesWithHeadHeaders { + metadata?: { [propertyName: string]: string }; + /** + * The ETag contains a value that you can use to perform operations conditionally. If the request + * version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + eTag?: string; + /** + * Returns the date and time the container was last modified. Any operation that modifies the + * blob, including an update of the blob's metadata or properties, changes the last-modified time + * of the blob. + */ + lastModified?: Date; + /** + * When a blob is leased, specifies whether the lease is of infinite or fixed duration. Possible + * values include: 'infinite', 'fixed' + */ + leaseDuration?: LeaseDurationType; + /** + * Lease state of the blob. Possible values include: 'available', 'leased', 'expired', + * 'breaking', 'broken' + */ + leaseState?: LeaseStateType; + /** + * The current lease status of the blob. Possible values include: 'locked', 'unlocked' + */ + leaseStatus?: LeaseStatusType; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + /** + * Indicated whether data in the container may be accessed publicly and the level of access. + * Possible values include: 'container', 'blob' + */ + blobPublicAccess?: PublicAccessType; + /** + * Indicates whether the container has an immutability policy set on it. + */ + hasImmutabilityPolicy?: boolean; + /** + * Indicates whether the container has a legal hold. + */ + hasLegalHold?: boolean; + /** + * The default encryption scope for the container. + */ + defaultEncryptionScope?: string; + /** + * Indicates whether the container's default encryption scope can be overriden. + */ + denyEncryptionScopeOverride?: boolean; + /** + * Indicates whether version level worm is enabled on a container. + */ + isImmutableStorageWithVersioningEnabled?: boolean; + errorCode?: string; +} + +/** + * Defines headers for Delete operation. + */ +export interface ContainerDeleteHeaders { + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + errorCode?: string; +} + +/** + * Defines headers for SetMetadata operation. + */ +export interface ContainerSetMetadataHeaders { + /** + * The ETag contains a value that you can use to perform operations conditionally. If the request + * version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + eTag?: string; + /** + * Returns the date and time the container was last modified. Any operation that modifies the + * blob, including an update of the blob's metadata or properties, changes the last-modified time + * of the blob. + */ + lastModified?: Date; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + errorCode?: string; +} + +/** + * Defines headers for GetAccessPolicy operation. + */ +export interface ContainerGetAccessPolicyHeaders { + /** + * Indicated whether data in the container may be accessed publicly and the level of access. + * Possible values include: 'container', 'blob' + */ + blobPublicAccess?: PublicAccessType; + /** + * The ETag contains a value that you can use to perform operations conditionally. If the request + * version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + eTag?: string; + /** + * Returns the date and time the container was last modified. Any operation that modifies the + * blob, including an update of the blob's metadata or properties, changes the last-modified time + * of the blob. + */ + lastModified?: Date; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + errorCode?: string; +} + +/** + * Defines headers for SetAccessPolicy operation. + */ +export interface ContainerSetAccessPolicyHeaders { + /** + * The ETag contains a value that you can use to perform operations conditionally. If the request + * version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + eTag?: string; + /** + * Returns the date and time the container was last modified. Any operation that modifies the + * blob, including an update of the blob's metadata or properties, changes the last-modified time + * of the blob. + */ + lastModified?: Date; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + errorCode?: string; +} + +/** + * Defines headers for Restore operation. + */ +export interface ContainerRestoreHeaders { + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + errorCode?: string; +} + +/** + * Defines headers for SubmitBatch operation. + */ +export interface ContainerSubmitBatchHeaders { + /** + * The media type of the body of the response. For batch requests, this is multipart/mixed; + * boundary=batchresponse_GUID + */ + contentType?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + errorCode?: string; +} + +/** + * Defines headers for FilterBlobs operation. + */ +export interface ContainerFilterBlobsHeaders { + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + errorCode?: string; +} + +/** + * Defines headers for AcquireLease operation. + */ +export interface ContainerAcquireLeaseHeaders { + /** + * The ETag contains a value that you can use to perform operations conditionally. If the request + * version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + eTag?: string; + /** + * Returns the date and time the container was last modified. Any operation that modifies the + * blob, including an update of the blob's metadata or properties, changes the last-modified time + * of the blob. + */ + lastModified?: Date; + /** + * Uniquely identifies a container's lease + */ + leaseId?: string; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + errorCode?: string; +} + +/** + * Defines headers for ReleaseLease operation. + */ +export interface ContainerReleaseLeaseHeaders { + /** + * The ETag contains a value that you can use to perform operations conditionally. If the request + * version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + eTag?: string; + /** + * Returns the date and time the container was last modified. Any operation that modifies the + * blob, including an update of the blob's metadata or properties, changes the last-modified time + * of the blob. + */ + lastModified?: Date; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + errorCode?: string; +} + +/** + * Defines headers for RenewLease operation. + */ +export interface ContainerRenewLeaseHeaders { + /** + * The ETag contains a value that you can use to perform operations conditionally. If the request + * version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + eTag?: string; + /** + * Returns the date and time the container was last modified. Any operation that modifies the + * blob, including an update of the blob's metadata or properties, changes the last-modified time + * of the blob. + */ + lastModified?: Date; + /** + * Uniquely identifies a container's lease + */ + leaseId?: string; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + errorCode?: string; +} + +/** + * Defines headers for BreakLease operation. + */ +export interface ContainerBreakLeaseHeaders { + /** + * The ETag contains a value that you can use to perform operations conditionally. If the request + * version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + eTag?: string; + /** + * Returns the date and time the container was last modified. Any operation that modifies the + * blob, including an update of the blob's metadata or properties, changes the last-modified time + * of the blob. + */ + lastModified?: Date; + /** + * Approximate time remaining in the lease period, in seconds. + */ + leaseTime?: number; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + errorCode?: string; +} + +/** + * Defines headers for ChangeLease operation. + */ +export interface ContainerChangeLeaseHeaders { + /** + * The ETag contains a value that you can use to perform operations conditionally. If the request + * version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + eTag?: string; + /** + * Returns the date and time the container was last modified. Any operation that modifies the + * blob, including an update of the blob's metadata or properties, changes the last-modified time + * of the blob. + */ + lastModified?: Date; + /** + * Uniquely identifies a container's lease + */ + leaseId?: string; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + errorCode?: string; +} + +/** + * Defines headers for GetAccountInfo operation. + */ +export interface ContainerGetAccountInfoHeaders { + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + /** + * Identifies the sku name of the account. Possible values include: 'Standard_LRS', + * 'Standard_GRS', 'Standard_RAGRS', 'Standard_ZRS', 'Premium_LRS' + */ + skuName?: SkuName; + /** + * Identifies the account kind. Possible values include: 'Storage', 'BlobStorage', 'StorageV2', + * 'FileStorage', 'BlockBlobStorage' + */ + accountKind?: AccountKind; + errorCode?: string; +} + +/** + * Defines headers for GetAccountInfoWithHead operation. + */ +export interface ContainerGetAccountInfoWithHeadHeaders { + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + /** + * Identifies the sku name of the account. Possible values include: 'Standard_LRS', + * 'Standard_GRS', 'Standard_RAGRS', 'Standard_ZRS', 'Premium_LRS' + */ + skuName?: SkuName; + /** + * Identifies the account kind. Possible values include: 'Storage', 'BlobStorage', 'StorageV2', + * 'FileStorage', 'BlockBlobStorage' + */ + accountKind?: AccountKind; + errorCode?: string; +} + +/** + * Defines headers for Create operation. + */ +export interface PageBlobCreateHeaders { + /** + * The ETag contains a value that you can use to perform operations conditionally. If the request + * version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + eTag?: string; + /** + * Returns the date and time the container was last modified. Any operation that modifies the + * blob, including an update of the blob's metadata or properties, changes the last-modified time + * of the blob. + */ + lastModified?: Date; + /** + * If the blob has an MD5 hash and this operation is to read the full blob, this response header + * is returned so that the client can check for message content integrity. + */ + contentMD5?: Uint8Array; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * A DateTime value returned by the service that uniquely identifies the blob. The value of this + * header indicates the blob version, and may be used in subsequent requests to access this + * version of the blob. + */ + versionId?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + /** + * The value of this header is set to true if the contents of the request are successfully + * encrypted using the specified algorithm, and false otherwise. + */ + isServerEncrypted?: boolean; + /** + * The SHA-256 hash of the encryption key used to encrypt the blob. This header is only returned + * when the blob was encrypted with a customer-provided key. + */ + encryptionKeySha256?: string; + /** + * Returns the name of the encryption scope used to encrypt the blob contents and application + * metadata. Note that the absence of this header implies use of the default account encryption + * scope. + */ + encryptionScope?: string; + errorCode?: string; +} + +/** + * Defines headers for Create operation. + */ +export interface AppendBlobCreateHeaders { + /** + * The ETag contains a value that you can use to perform operations conditionally. If the request + * version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + eTag?: string; + /** + * Returns the date and time the container was last modified. Any operation that modifies the + * blob, including an update of the blob's metadata or properties, changes the last-modified time + * of the blob. + */ + lastModified?: Date; + /** + * If the blob has an MD5 hash and this operation is to read the full blob, this response header + * is returned so that the client can check for message content integrity. + */ + contentMD5?: Uint8Array; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * A DateTime value returned by the service that uniquely identifies the blob. The value of this + * header indicates the blob version, and may be used in subsequent requests to access this + * version of the blob. + */ + versionId?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + /** + * The value of this header is set to true if the contents of the request are successfully + * encrypted using the specified algorithm, and false otherwise. + */ + isServerEncrypted?: boolean; + /** + * The SHA-256 hash of the encryption key used to encrypt the blob. This header is only returned + * when the blob was encrypted with a customer-provided key. + */ + encryptionKeySha256?: string; + /** + * Returns the name of the encryption scope used to encrypt the blob contents and application + * metadata. Note that the absence of this header implies use of the default account encryption + * scope. + */ + encryptionScope?: string; + errorCode?: string; +} + +/** + * Defines headers for Upload operation. + */ +export interface BlockBlobUploadHeaders { + /** + * The ETag contains a value that you can use to perform operations conditionally. If the request + * version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + eTag?: string; + /** + * Returns the date and time the container was last modified. Any operation that modifies the + * blob, including an update of the blob's metadata or properties, changes the last-modified time + * of the blob. + */ + lastModified?: Date; + /** + * If the blob has an MD5 hash and this operation is to read the full blob, this response header + * is returned so that the client can check for message content integrity. + */ + contentMD5?: Uint8Array; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * A DateTime value returned by the service that uniquely identifies the blob. The value of this + * header indicates the blob version, and may be used in subsequent requests to access this + * version of the blob. + */ + versionId?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + /** + * The value of this header is set to true if the contents of the request are successfully + * encrypted using the specified algorithm, and false otherwise. + */ + isServerEncrypted?: boolean; + /** + * The SHA-256 hash of the encryption key used to encrypt the blob. This header is only returned + * when the blob was encrypted with a customer-provided key. + */ + encryptionKeySha256?: string; + /** + * Returns the name of the encryption scope used to encrypt the blob contents and application + * metadata. Note that the absence of this header implies use of the default account encryption + * scope. + */ + encryptionScope?: string; + errorCode?: string; +} + +/** + * Defines headers for PutBlobFromUrl operation. + */ +export interface BlockBlobPutBlobFromUrlHeaders { + /** + * The ETag contains a value that you can use to perform operations conditionally. If the request + * version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + eTag?: string; + /** + * Returns the date and time the container was last modified. Any operation that modifies the + * blob, including an update of the blob's metadata or properties, changes the last-modified time + * of the blob. + */ + lastModified?: Date; + /** + * If the blob has an MD5 hash and this operation is to read the full blob, this response header + * is returned so that the client can check for message content integrity. + */ + contentMD5?: Uint8Array; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * A DateTime value returned by the service that uniquely identifies the blob. The value of this + * header indicates the blob version, and may be used in subsequent requests to access this + * version of the blob. + */ + versionId?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + /** + * The value of this header is set to true if the contents of the request are successfully + * encrypted using the specified algorithm, and false otherwise. + */ + isServerEncrypted?: boolean; + /** + * The SHA-256 hash of the encryption key used to encrypt the blob. This header is only returned + * when the blob was encrypted with a customer-provided key. + */ + encryptionKeySha256?: string; + /** + * Returns the name of the encryption scope used to encrypt the blob contents and application + * metadata. Note that the absence of this header implies use of the default account encryption + * scope. + */ + encryptionScope?: string; + errorCode?: string; +} + +/** + * Defines headers for Undelete operation. + */ +export interface BlobUndeleteHeaders { + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated. + */ + date?: Date; + errorCode?: string; +} + +/** + * Defines headers for SetExpiry operation. + */ +export interface BlobSetExpiryHeaders { + /** + * The ETag contains a value that you can use to perform operations conditionally. If the request + * version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + eTag?: string; + /** + * Returns the date and time the container was last modified. Any operation that modifies the + * blob, including an update of the blob's metadata or properties, changes the last-modified time + * of the blob. + */ + lastModified?: Date; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated. + */ + date?: Date; + errorCode?: string; +} + +/** + * Defines headers for SetHTTPHeaders operation. + */ +export interface BlobSetHTTPHeadersHeaders { + /** + * The ETag contains a value that you can use to perform operations conditionally. If the request + * version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + eTag?: string; + /** + * Returns the date and time the container was last modified. Any operation that modifies the + * blob, including an update of the blob's metadata or properties, changes the last-modified time + * of the blob. + */ + lastModified?: Date; + /** + * The current sequence number for a page blob. This header is not returned for block blobs or + * append blobs + */ + blobSequenceNumber?: number; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + errorCode?: string; +} + +/** + * Defines headers for SetImmutabilityPolicy operation. + */ +export interface BlobSetImmutabilityPolicyHeaders { + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + /** + * Indicates the time the immutability policy will expire. + */ + immutabilityPolicyExpiry?: Date; + /** + * Indicates immutability policy mode. Possible values include: 'Mutable', 'Unlocked', 'Locked' + */ + immutabilityPolicyMode?: BlobImmutabilityPolicyMode; + errorCode?: string; +} + +/** + * Defines headers for DeleteImmutabilityPolicy operation. + */ +export interface BlobDeleteImmutabilityPolicyHeaders { + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + errorCode?: string; +} + +/** + * Defines headers for SetLegalHold operation. + */ +export interface BlobSetLegalHoldHeaders { + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + /** + * Indicates if the blob has a legal hold. + */ + legalHold?: boolean; + errorCode?: string; +} + +/** + * Defines headers for SetMetadata operation. + */ +export interface BlobSetMetadataHeaders { + /** + * The ETag contains a value that you can use to perform operations conditionally. If the request + * version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + eTag?: string; + /** + * Returns the date and time the container was last modified. Any operation that modifies the + * blob, including an update of the blob's metadata or properties, changes the last-modified time + * of the blob. + */ + lastModified?: Date; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * A DateTime value returned by the service that uniquely identifies the blob. The value of this + * header indicates the blob version, and may be used in subsequent requests to access this + * version of the blob. + */ + versionId?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + /** + * The value of this header is set to true if the contents of the request are successfully + * encrypted using the specified algorithm, and false otherwise. + */ + isServerEncrypted?: boolean; + /** + * The SHA-256 hash of the encryption key used to encrypt the metadata. This header is only + * returned when the metadata was encrypted with a customer-provided key. + */ + encryptionKeySha256?: string; + /** + * Returns the name of the encryption scope used to encrypt the blob contents and application + * metadata. Note that the absence of this header implies use of the default account encryption + * scope. + */ + encryptionScope?: string; + errorCode?: string; +} + +/** + * Defines headers for AcquireLease operation. + */ +export interface BlobAcquireLeaseHeaders { + /** + * The ETag contains a value that you can use to perform operations conditionally. If the request + * version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + eTag?: string; + /** + * Returns the date and time the blob was last modified. Any operation that modifies the blob, + * including an update of the blob's metadata or properties, changes the last-modified time of + * the blob. + */ + lastModified?: Date; + /** + * Uniquely identifies a blobs' lease + */ + leaseId?: string; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + errorCode?: string; +} + +/** + * Defines headers for ReleaseLease operation. + */ +export interface BlobReleaseLeaseHeaders { + /** + * The ETag contains a value that you can use to perform operations conditionally. If the request + * version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + eTag?: string; + /** + * Returns the date and time the blob was last modified. Any operation that modifies the blob, + * including an update of the blob's metadata or properties, changes the last-modified time of + * the blob. + */ + lastModified?: Date; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + errorCode?: string; +} + +/** + * Defines headers for RenewLease operation. + */ +export interface BlobRenewLeaseHeaders { + /** + * The ETag contains a value that you can use to perform operations conditionally. If the request + * version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + eTag?: string; + /** + * Returns the date and time the blob was last modified. Any operation that modifies the blob, + * including an update of the blob's metadata or properties, changes the last-modified time of + * the blob. + */ + lastModified?: Date; + /** + * Uniquely identifies a blobs' lease + */ + leaseId?: string; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + errorCode?: string; +} + +/** + * Defines headers for ChangeLease operation. + */ +export interface BlobChangeLeaseHeaders { + /** + * The ETag contains a value that you can use to perform operations conditionally. If the request + * version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + eTag?: string; + /** + * Returns the date and time the blob was last modified. Any operation that modifies the blob, + * including an update of the blob's metadata or properties, changes the last-modified time of + * the blob. + */ + lastModified?: Date; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Uniquely identifies a blobs' lease + */ + leaseId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + errorCode?: string; +} + +/** + * Defines headers for BreakLease operation. + */ +export interface BlobBreakLeaseHeaders { + /** + * The ETag contains a value that you can use to perform operations conditionally. If the request + * version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + eTag?: string; + /** + * Returns the date and time the blob was last modified. Any operation that modifies the blob, + * including an update of the blob's metadata or properties, changes the last-modified time of + * the blob. + */ + lastModified?: Date; + /** + * Approximate time remaining in the lease period, in seconds. + */ + leaseTime?: number; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + errorCode?: string; +} + +/** + * Defines headers for CreateSnapshot operation. + */ +export interface BlobCreateSnapshotHeaders { + /** + * Uniquely identifies the snapshot and indicates the snapshot version. It may be used in + * subsequent requests to access the snapshot + */ + snapshot?: string; + /** + * The ETag contains a value that you can use to perform operations conditionally. If the request + * version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + eTag?: string; + /** + * Returns the date and time the container was last modified. Any operation that modifies the + * blob, including an update of the blob's metadata or properties, changes the last-modified time + * of the blob. + */ + lastModified?: Date; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * A DateTime value returned by the service that uniquely identifies the blob. The value of this + * header indicates the blob version, and may be used in subsequent requests to access this + * version of the blob. + */ + versionId?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + /** + * True if the contents of the request are successfully encrypted using the specified algorithm, + * and false otherwise. For a snapshot request, this header is set to true when metadata was + * provided in the request and encrypted with a customer-provided key. + */ + isServerEncrypted?: boolean; + errorCode?: string; +} + +/** + * Defines headers for StartCopyFromURL operation. + */ +export interface BlobStartCopyFromURLHeaders { + /** + * The ETag contains a value that you can use to perform operations conditionally. If the request + * version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + eTag?: string; + /** + * Returns the date and time the container was last modified. Any operation that modifies the + * blob, including an update of the blob's metadata or properties, changes the last-modified time + * of the blob. + */ + lastModified?: Date; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * A DateTime value returned by the service that uniquely identifies the blob. The value of this + * header indicates the blob version, and may be used in subsequent requests to access this + * version of the blob. + */ + versionId?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + /** + * String identifier for this copy operation. Use with Get Blob Properties to check the status of + * this copy operation, or pass to Abort Copy Blob to abort a pending copy. + */ + copyId?: string; + /** + * State of the copy operation identified by x-ms-copy-id. Possible values include: 'pending', + * 'success', 'aborted', 'failed' + */ + copyStatus?: CopyStatusType; + errorCode?: string; +} + +/** + * Defines headers for CopyFromURL operation. + */ +export interface BlobCopyFromURLHeaders { + /** + * The ETag contains a value that you can use to perform operations conditionally. If the request + * version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + eTag?: string; + /** + * Returns the date and time the container was last modified. Any operation that modifies the + * blob, including an update of the blob's metadata or properties, changes the last-modified time + * of the blob. + */ + lastModified?: Date; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * A DateTime value returned by the service that uniquely identifies the blob. The value of this + * header indicates the blob version, and may be used in subsequent requests to access this + * version of the blob. + */ + versionId?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + /** + * String identifier for this copy operation. + */ + copyId?: string; + /** + * State of the copy operation identified by x-ms-copy-id. Possible values include: 'success' + */ + copyStatus?: SyncCopyStatusType; + /** + * This response header is returned so that the client can check for the integrity of the copied + * content. This header is only returned if the source content MD5 was specified. + */ + contentMD5?: Uint8Array; + /** + * This response header is returned so that the client can check for the integrity of the copied + * content. + */ + xMsContentCrc64?: Uint8Array; + /** + * Returns the name of the encryption scope used to encrypt the blob contents and application + * metadata. Note that the absence of this header implies use of the default account encryption + * scope. + */ + encryptionScope?: string; + errorCode?: string; +} + +/** + * Defines headers for AbortCopyFromURL operation. + */ +export interface BlobAbortCopyFromURLHeaders { + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + errorCode?: string; +} + +/** + * Defines headers for SetTier operation. + */ +export interface BlobSetTierHeaders { + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and newer. + */ + version?: string; + errorCode?: string; +} + +/** + * Defines headers for GetAccountInfo operation. + */ +export interface BlobGetAccountInfoHeaders { + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + /** + * Identifies the sku name of the account. Possible values include: 'Standard_LRS', + * 'Standard_GRS', 'Standard_RAGRS', 'Standard_ZRS', 'Premium_LRS' + */ + skuName?: SkuName; + /** + * Identifies the account kind. Possible values include: 'Storage', 'BlobStorage', 'StorageV2', + * 'FileStorage', 'BlockBlobStorage' + */ + accountKind?: AccountKind; + errorCode?: string; +} + +/** + * Defines headers for GetAccountInfoWithHead operation. + */ +export interface BlobGetAccountInfoWithHeadHeaders { + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + /** + * Identifies the sku name of the account. Possible values include: 'Standard_LRS', + * 'Standard_GRS', 'Standard_RAGRS', 'Standard_ZRS', 'Premium_LRS' + */ + skuName?: SkuName; + /** + * Identifies the account kind. Possible values include: 'Storage', 'BlobStorage', 'StorageV2', + * 'FileStorage', 'BlockBlobStorage' + */ + accountKind?: AccountKind; + errorCode?: string; +} + +/** + * Defines headers for StageBlock operation. + */ +export interface BlockBlobStageBlockHeaders { + /** + * This header is returned so that the client can check for message content integrity. The value + * of this header is computed by the Blob service; it is not necessarily the same value specified + * in the request headers. + */ + contentMD5?: Uint8Array; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + /** + * This header is returned so that the client can check for message content integrity. The value + * of this header is computed by the Blob service; it is not necessarily the same value specified + * in the request headers. + */ + xMsContentCrc64?: Uint8Array; + /** + * The value of this header is set to true if the contents of the request are successfully + * encrypted using the specified algorithm, and false otherwise. + */ + isServerEncrypted?: boolean; + /** + * The SHA-256 hash of the encryption key used to encrypt the block. This header is only returned + * when the block was encrypted with a customer-provided key. + */ + encryptionKeySha256?: string; + /** + * Returns the name of the encryption scope used to encrypt the blob contents and application + * metadata. Note that the absence of this header implies use of the default account encryption + * scope. + */ + encryptionScope?: string; + errorCode?: string; +} + +/** + * Defines headers for StageBlockFromURL operation. + */ +export interface BlockBlobStageBlockFromURLHeaders { + /** + * This header is returned so that the client can check for message content integrity. The value + * of this header is computed by the Blob service; it is not necessarily the same value specified + * in the request headers. + */ + contentMD5?: Uint8Array; + /** + * This header is returned so that the client can check for message content integrity. The value + * of this header is computed by the Blob service; it is not necessarily the same value specified + * in the request headers. + */ + xMsContentCrc64?: Uint8Array; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + /** + * The value of this header is set to true if the contents of the request are successfully + * encrypted using the specified algorithm, and false otherwise. + */ + isServerEncrypted?: boolean; + /** + * The SHA-256 hash of the encryption key used to encrypt the block. This header is only returned + * when the block was encrypted with a customer-provided key. + */ + encryptionKeySha256?: string; + /** + * Returns the name of the encryption scope used to encrypt the blob contents and application + * metadata. Note that the absence of this header implies use of the default account encryption + * scope. + */ + encryptionScope?: string; + errorCode?: string; +} + +/** + * Defines headers for CommitBlockList operation. + */ +export interface BlockBlobCommitBlockListHeaders { + /** + * The ETag contains a value that you can use to perform operations conditionally. If the request + * version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + eTag?: string; + /** + * Returns the date and time the container was last modified. Any operation that modifies the + * blob, including an update of the blob's metadata or properties, changes the last-modified time + * of the blob. + */ + lastModified?: Date; + /** + * This header is returned so that the client can check for message content integrity. This + * header refers to the content of the request, meaning, in this case, the list of blocks, and + * not the content of the blob itself. + */ + contentMD5?: Uint8Array; + /** + * This header is returned so that the client can check for message content integrity. This + * header refers to the content of the request, meaning, in this case, the list of blocks, and + * not the content of the blob itself. + */ + xMsContentCrc64?: Uint8Array; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * A DateTime value returned by the service that uniquely identifies the blob. The value of this + * header indicates the blob version, and may be used in subsequent requests to access this + * version of the blob. + */ + versionId?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + /** + * The value of this header is set to true if the contents of the request are successfully + * encrypted using the specified algorithm, and false otherwise. + */ + isServerEncrypted?: boolean; + /** + * The SHA-256 hash of the encryption key used to encrypt the blob. This header is only returned + * when the blob was encrypted with a customer-provided key. + */ + encryptionKeySha256?: string; + /** + * Returns the name of the encryption scope used to encrypt the blob contents and application + * metadata. Note that the absence of this header implies use of the default account encryption + * scope. + */ + encryptionScope?: string; + errorCode?: string; +} + +/** + * Defines headers for GetBlockList operation. + */ +export interface BlockBlobGetBlockListHeaders { + /** + * Returns the date and time the container was last modified. Any operation that modifies the + * blob, including an update of the blob's metadata or properties, changes the last-modified time + * of the blob. + */ + lastModified?: Date; + /** + * The ETag contains a value that you can use to perform operations conditionally. If the request + * version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + eTag?: string; + /** + * The media type of the body of the response. For Get Block List this is 'application/xml' + */ + contentType?: string; + /** + * The size of the blob in bytes. + */ + blobContentLength?: number; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + errorCode?: string; +} + +/** + * Defines headers for UploadPages operation. + */ +export interface PageBlobUploadPagesHeaders { + /** + * The ETag contains a value that you can use to perform operations conditionally. If the request + * version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + eTag?: string; + /** + * Returns the date and time the container was last modified. Any operation that modifies the + * blob, including an update of the blob's metadata or properties, changes the last-modified time + * of the blob. + */ + lastModified?: Date; + /** + * If the blob has an MD5 hash and this operation is to read the full blob, this response header + * is returned so that the client can check for message content integrity. + */ + contentMD5?: Uint8Array; + /** + * This header is returned so that the client can check for message content integrity. The value + * of this header is computed by the Blob service; it is not necessarily the same value specified + * in the request headers. + */ + xMsContentCrc64?: Uint8Array; + /** + * The current sequence number for the page blob. + */ + blobSequenceNumber?: number; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + /** + * The value of this header is set to true if the contents of the request are successfully + * encrypted using the specified algorithm, and false otherwise. + */ + isServerEncrypted?: boolean; + /** + * The SHA-256 hash of the encryption key used to encrypt the pages. This header is only returned + * when the pages were encrypted with a customer-provided key. + */ + encryptionKeySha256?: string; + /** + * Returns the name of the encryption scope used to encrypt the blob contents and application + * metadata. Note that the absence of this header implies use of the default account encryption + * scope. + */ + encryptionScope?: string; + errorCode?: string; +} + +/** + * Defines headers for ClearPages operation. + */ +export interface PageBlobClearPagesHeaders { + /** + * The ETag contains a value that you can use to perform operations conditionally. If the request + * version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + eTag?: string; + /** + * Returns the date and time the container was last modified. Any operation that modifies the + * blob, including an update of the blob's metadata or properties, changes the last-modified time + * of the blob. + */ + lastModified?: Date; + /** + * If the blob has an MD5 hash and this operation is to read the full blob, this response header + * is returned so that the client can check for message content integrity. + */ + contentMD5?: Uint8Array; + /** + * This header is returned so that the client can check for message content integrity. The value + * of this header is computed by the Blob service; it is not necessarily the same value specified + * in the request headers. + */ + xMsContentCrc64?: Uint8Array; + /** + * The current sequence number for the page blob. + */ + blobSequenceNumber?: number; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + errorCode?: string; +} + +/** + * Defines headers for UploadPagesFromURL operation. + */ +export interface PageBlobUploadPagesFromURLHeaders { + /** + * The ETag contains a value that you can use to perform operations conditionally. If the request + * version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + eTag?: string; + /** + * Returns the date and time the container was last modified. Any operation that modifies the + * blob, including an update of the blob's metadata or properties, changes the last-modified time + * of the blob. + */ + lastModified?: Date; + /** + * If the blob has an MD5 hash and this operation is to read the full blob, this response header + * is returned so that the client can check for message content integrity. + */ + contentMD5?: Uint8Array; + /** + * This header is returned so that the client can check for message content integrity. The value + * of this header is computed by the Blob service; it is not necessarily the same value specified + * in the request headers. + */ + xMsContentCrc64?: Uint8Array; + /** + * The current sequence number for the page blob. + */ + blobSequenceNumber?: number; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + /** + * The value of this header is set to true if the contents of the request are successfully + * encrypted using the specified algorithm, and false otherwise. + */ + isServerEncrypted?: boolean; + /** + * The SHA-256 hash of the encryption key used to encrypt the blob. This header is only returned + * when the blob was encrypted with a customer-provided key. + */ + encryptionKeySha256?: string; + /** + * Returns the name of the encryption scope used to encrypt the blob contents and application + * metadata. Note that the absence of this header implies use of the default account encryption + * scope. + */ + encryptionScope?: string; + errorCode?: string; +} + +/** + * Defines headers for GetPageRanges operation. + */ +export interface PageBlobGetPageRangesHeaders { + /** + * Returns the date and time the container was last modified. Any operation that modifies the + * blob, including an update of the blob's metadata or properties, changes the last-modified time + * of the blob. + */ + lastModified?: Date; + /** + * The ETag contains a value that you can use to perform operations conditionally. If the request + * version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + eTag?: string; + /** + * The size of the blob in bytes. + */ + blobContentLength?: number; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + errorCode?: string; +} + +/** + * Defines headers for GetPageRangesDiff operation. + */ +export interface PageBlobGetPageRangesDiffHeaders { + /** + * Returns the date and time the container was last modified. Any operation that modifies the + * blob, including an update of the blob's metadata or properties, changes the last-modified time + * of the blob. + */ + lastModified?: Date; + /** + * The ETag contains a value that you can use to perform operations conditionally. If the request + * version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + eTag?: string; + /** + * The size of the blob in bytes. + */ + blobContentLength?: number; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + errorCode?: string; +} + +/** + * Defines headers for Resize operation. + */ +export interface PageBlobResizeHeaders { + /** + * The ETag contains a value that you can use to perform operations conditionally. If the request + * version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + eTag?: string; + /** + * Returns the date and time the container was last modified. Any operation that modifies the + * blob, including an update of the blob's metadata or properties, changes the last-modified time + * of the blob. + */ + lastModified?: Date; + /** + * The current sequence number for a page blob. This header is not returned for block blobs or + * append blobs + */ + blobSequenceNumber?: number; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + errorCode?: string; +} + +/** + * Defines headers for UpdateSequenceNumber operation. + */ +export interface PageBlobUpdateSequenceNumberHeaders { + /** + * The ETag contains a value that you can use to perform operations conditionally. If the request + * version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + eTag?: string; + /** + * Returns the date and time the container was last modified. Any operation that modifies the + * blob, including an update of the blob's metadata or properties, changes the last-modified time + * of the blob. + */ + lastModified?: Date; + /** + * The current sequence number for a page blob. This header is not returned for block blobs or + * append blobs + */ + blobSequenceNumber?: number; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + errorCode?: string; +} + +/** + * Defines headers for CopyIncremental operation. + */ +export interface PageBlobCopyIncrementalHeaders { + /** + * The ETag contains a value that you can use to perform operations conditionally. If the request + * version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + eTag?: string; + /** + * Returns the date and time the container was last modified. Any operation that modifies the + * blob, including an update of the blob's metadata or properties, changes the last-modified time + * of the blob. + */ + lastModified?: Date; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + /** + * String identifier for this copy operation. Use with Get Blob Properties to check the status of + * this copy operation, or pass to Abort Copy Blob to abort a pending copy. + */ + copyId?: string; + /** + * State of the copy operation identified by x-ms-copy-id. Possible values include: 'pending', + * 'success', 'aborted', 'failed' + */ + copyStatus?: CopyStatusType; + errorCode?: string; +} + +/** + * Defines headers for AppendBlock operation. + */ +export interface AppendBlobAppendBlockHeaders { + /** + * The ETag contains a value that you can use to perform operations conditionally. If the request + * version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + eTag?: string; + /** + * Returns the date and time the container was last modified. Any operation that modifies the + * blob, including an update of the blob's metadata or properties, changes the last-modified time + * of the blob. + */ + lastModified?: Date; + /** + * If the blob has an MD5 hash and this operation is to read the full blob, this response header + * is returned so that the client can check for message content integrity. + */ + contentMD5?: Uint8Array; + /** + * This header is returned so that the client can check for message content integrity. The value + * of this header is computed by the Blob service; it is not necessarily the same value specified + * in the request headers. + */ + xMsContentCrc64?: Uint8Array; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + /** + * This response header is returned only for append operations. It returns the offset at which + * the block was committed, in bytes. + */ + blobAppendOffset?: string; + /** + * The number of committed blocks present in the blob. This header is returned only for append + * blobs. + */ + blobCommittedBlockCount?: number; + /** + * The value of this header is set to true if the contents of the request are successfully + * encrypted using the specified algorithm, and false otherwise. + */ + isServerEncrypted?: boolean; + /** + * The SHA-256 hash of the encryption key used to encrypt the block. This header is only returned + * when the block was encrypted with a customer-provided key. + */ + encryptionKeySha256?: string; + /** + * Returns the name of the encryption scope used to encrypt the blob contents and application + * metadata. Note that the absence of this header implies use of the default account encryption + * scope. + */ + encryptionScope?: string; + errorCode?: string; +} + +/** + * Defines headers for AppendBlockFromUrl operation. + */ +export interface AppendBlobAppendBlockFromUrlHeaders { + /** + * The ETag contains a value that you can use to perform operations conditionally. If the request + * version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + eTag?: string; + /** + * Returns the date and time the container was last modified. Any operation that modifies the + * blob, including an update of the blob's metadata or properties, changes the last-modified time + * of the blob. + */ + lastModified?: Date; + /** + * If the blob has an MD5 hash and this operation is to read the full blob, this response header + * is returned so that the client can check for message content integrity. + */ + contentMD5?: Uint8Array; + /** + * This header is returned so that the client can check for message content integrity. The value + * of this header is computed by the Blob service; it is not necessarily the same value specified + * in the request headers. + */ + xMsContentCrc64?: Uint8Array; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + /** + * This response header is returned only for append operations. It returns the offset at which + * the block was committed, in bytes. + */ + blobAppendOffset?: string; + /** + * The number of committed blocks present in the blob. This header is returned only for append + * blobs. + */ + blobCommittedBlockCount?: number; + /** + * The SHA-256 hash of the encryption key used to encrypt the block. This header is only returned + * when the block was encrypted with a customer-provided key. + */ + encryptionKeySha256?: string; + /** + * Returns the name of the encryption scope used to encrypt the blob contents and application + * metadata. Note that the absence of this header implies use of the default account encryption + * scope. + */ + encryptionScope?: string; + /** + * The value of this header is set to true if the contents of the request are successfully + * encrypted using the specified algorithm, and false otherwise. + */ + isServerEncrypted?: boolean; + errorCode?: string; +} + +/** + * Defines headers for Seal operation. + */ +export interface AppendBlobSealHeaders { + /** + * The ETag contains a value that you can use to perform operations conditionally. If the request + * version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + eTag?: string; + /** + * Returns the date and time the container was last modified. Any operation that modifies the + * blob, including an update of the blob's metadata or properties, changes the last-modified time + * of the blob. + */ + lastModified?: Date; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + /** + * If this blob has been sealed + */ + isSealed?: boolean; + errorCode?: string; +} + +/** + * Defines headers for Query operation. + */ +export interface BlobQueryHeaders { + /** + * Returns the date and time the container was last modified. Any operation that modifies the + * blob, including an update of the blob's metadata or properties, changes the last-modified time + * of the blob. + */ + lastModified?: Date; + metadata?: { [propertyName: string]: string }; + /** + * The number of bytes present in the response body. + */ + contentLength?: number; + /** + * The media type of the body of the response. For Download Blob this is + * 'application/octet-stream' + */ + contentType?: string; + /** + * Indicates the range of bytes returned in the event that the client requested a subset of the + * blob by setting the 'Range' request header. + */ + contentRange?: string; + /** + * The ETag contains a value that you can use to perform operations conditionally. If the request + * version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + eTag?: string; + /** + * If the blob has an MD5 hash and this operation is to read the full blob, this response header + * is returned so that the client can check for message content integrity. + */ + contentMD5?: Uint8Array; + /** + * This header returns the value that was specified for the Content-Encoding request header + */ + contentEncoding?: string; + /** + * This header is returned if it was previously specified for the blob. + */ + cacheControl?: string; + /** + * This header returns the value that was specified for the 'x-ms-blob-content-disposition' + * header. The Content-Disposition response header field conveys additional information about how + * to process the response payload, and also can be used to attach additional metadata. For + * example, if set to attachment, it indicates that the user-agent should not display the + * response, but instead show a Save As dialog with a filename other than the blob name + * specified. + */ + contentDisposition?: string; + /** + * This header returns the value that was specified for the Content-Language request header. + */ + contentLanguage?: string; + /** + * The current sequence number for a page blob. This header is not returned for block blobs or + * append blobs + */ + blobSequenceNumber?: number; + /** + * The blob's type. Possible values include: 'BlockBlob', 'PageBlob', 'AppendBlob' + */ + blobType?: BlobType; + /** + * Conclusion time of the last attempted Copy Blob operation where this blob was the destination + * blob. This value can specify the time of a completed, aborted, or failed copy attempt. This + * header does not appear if a copy is pending, if this blob has never been the destination in a + * Copy Blob operation, or if this blob has been modified after a concluded Copy Blob operation + * using Set Blob Properties, Put Blob, or Put Block List. + */ + copyCompletionTime?: Date; + /** + * Only appears when x-ms-copy-status is failed or pending. Describes the cause of the last fatal + * or non-fatal copy operation failure. This header does not appear if this blob has never been + * the destination in a Copy Blob operation, or if this blob has been modified after a concluded + * Copy Blob operation using Set Blob Properties, Put Blob, or Put Block List + */ + copyStatusDescription?: string; + /** + * String identifier for this copy operation. Use with Get Blob Properties to check the status of + * this copy operation, or pass to Abort Copy Blob to abort a pending copy. + */ + copyId?: string; + /** + * Contains the number of bytes copied and the total bytes in the source in the last attempted + * Copy Blob operation where this blob was the destination blob. Can show between 0 and + * Content-Length bytes copied. This header does not appear if this blob has never been the + * destination in a Copy Blob operation, or if this blob has been modified after a concluded Copy + * Blob operation using Set Blob Properties, Put Blob, or Put Block List + */ + copyProgress?: string; + /** + * URL up to 2 KB in length that specifies the source blob or file used in the last attempted + * Copy Blob operation where this blob was the destination blob. This header does not appear if + * this blob has never been the destination in a Copy Blob operation, or if this blob has been + * modified after a concluded Copy Blob operation using Set Blob Properties, Put Blob, or Put + * Block List. + */ + copySource?: string; + /** + * State of the copy operation identified by x-ms-copy-id. Possible values include: 'pending', + * 'success', 'aborted', 'failed' + */ + copyStatus?: CopyStatusType; + /** + * When a blob is leased, specifies whether the lease is of infinite or fixed duration. Possible + * values include: 'infinite', 'fixed' + */ + leaseDuration?: LeaseDurationType; + /** + * Lease state of the blob. Possible values include: 'available', 'leased', 'expired', + * 'breaking', 'broken' + */ + leaseState?: LeaseStateType; + /** + * The current lease status of the blob. Possible values include: 'locked', 'unlocked' + */ + leaseStatus?: LeaseStatusType; + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * Indicates that the service supports requests for partial blob content. + */ + acceptRanges?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + /** + * The number of committed blocks present in the blob. This header is returned only for append + * blobs. + */ + blobCommittedBlockCount?: number; + /** + * The value of this header is set to true if the blob data and application metadata are + * completely encrypted using the specified algorithm. Otherwise, the value is set to false (when + * the blob is unencrypted, or if only parts of the blob/application metadata are encrypted). + */ + isServerEncrypted?: boolean; + /** + * The SHA-256 hash of the encryption key used to encrypt the blob. This header is only returned + * when the blob was encrypted with a customer-provided key. + */ + encryptionKeySha256?: string; + /** + * Returns the name of the encryption scope used to encrypt the blob contents and application + * metadata. Note that the absence of this header implies use of the default account encryption + * scope. + */ + encryptionScope?: string; + /** + * If the blob has a MD5 hash, and if request contains range header (Range or x-ms-range), this + * response header is returned with the value of the whole blob's MD5 value. This value may or + * may not be equal to the value returned in Content-MD5 header, with the latter calculated from + * the requested range + */ + blobContentMD5?: Uint8Array; + /** + * If the request is to read a specified range and the x-ms-range-get-content-crc64 is set to + * true, then the request returns a crc64 for the range, as long as the range size is less than + * or equal to 4 MB. If both x-ms-range-get-content-crc64 and x-ms-range-get-content-md5 is + * specified in the same request, it will fail with 400(Bad Request) + */ + contentCrc64?: Uint8Array; + errorCode?: string; +} + +/** + * Defines headers for GetTags operation. + */ +export interface BlobGetTagsHeaders { + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + errorCode?: string; +} + +/** + * Defines headers for SetTags operation. + */ +export interface BlobSetTagsHeaders { + /** + * If a client request id header is sent in the request, this header will be present in the + * response with the same value. + */ + clientRequestId?: string; + /** + * This header uniquely identifies the request that was made and can be used for troubleshooting + * the request. + */ + requestId?: string; + /** + * Indicates the version of the Blob service used to execute the request. This header is returned + * for requests made against version 2009-09-19 and above. + */ + version?: string; + /** + * UTC date/time value generated by the service that indicates the time at which the response was + * initiated + */ + date?: Date; + errorCode?: string; +} + +/** + * Defines values for BlobType. + * Possible values include: 'BlockBlob', 'PageBlob', 'AppendBlob' + * @readonly + * @enum {string} + */ +export enum BlobType { + BlockBlob = 'BlockBlob', + PageBlob = 'PageBlob', + AppendBlob = 'AppendBlob', +} + +/** + * Defines values for LeaseStatusType. + * Possible values include: 'locked', 'unlocked' + * @readonly + * @enum {string} + */ +export enum LeaseStatusType { + Locked = 'locked', + Unlocked = 'unlocked', +} + +/** + * Defines values for LeaseStateType. + * Possible values include: 'available', 'leased', 'expired', 'breaking', 'broken' + * @readonly + * @enum {string} + */ +export enum LeaseStateType { + Available = 'available', + Leased = 'leased', + Expired = 'expired', + Breaking = 'breaking', + Broken = 'broken', +} + +/** + * Defines values for LeaseDurationType. + * Possible values include: 'infinite', 'fixed' + * @readonly + * @enum {string} + */ +export enum LeaseDurationType { + Infinite = 'infinite', + Fixed = 'fixed', +} + +/** + * Defines values for CopyStatusType. + * Possible values include: 'pending', 'success', 'aborted', 'failed' + * @readonly + * @enum {string} + */ +export enum CopyStatusType { + Pending = 'pending', + Success = 'success', + Aborted = 'aborted', + Failed = 'failed', +} + +/** + * Defines values for AccessTier. + * Possible values include: 'P4', 'P6', 'P10', 'P15', 'P20', 'P30', 'P40', 'P50', 'P60', 'P70', + * 'P80', 'Hot', 'Cool', 'Archive', 'Premium' + * @readonly + * @enum {string} + */ +export enum AccessTier { + P4 = 'P4', + P6 = 'P6', + P10 = 'P10', + P15 = 'P15', + P20 = 'P20', + P30 = 'P30', + P40 = 'P40', + P50 = 'P50', + P60 = 'P60', + P70 = 'P70', + P80 = 'P80', + Hot = 'Hot', + Cool = 'Cool', + Archive = 'Archive', + Premium = 'Premium', +} + +/** + * Defines values for ArchiveStatus. + * Possible values include: 'rehydrate-pending-to-hot', 'rehydrate-pending-to-cool' + * @readonly + * @enum {string} + */ +export enum ArchiveStatus { + RehydratePendingToHot = 'rehydrate-pending-to-hot', + RehydratePendingToCool = 'rehydrate-pending-to-cool', +} + +/** + * Defines values for RehydratePriority. + * Possible values include: 'High', 'Standard' + * @readonly + * @enum {string} + */ +export enum RehydratePriority { + High = 'High', + Standard = 'Standard', +} + +/** + * Defines values for BlobImmutabilityPolicyMode. + * Possible values include: 'Mutable', 'Unlocked', 'Locked' + * @readonly + * @enum {string} + */ +export enum BlobImmutabilityPolicyMode { + Mutable = 'Mutable', + Unlocked = 'Unlocked', + Locked = 'Locked', +} + +/** + * Defines values for PublicAccessType. + * Possible values include: 'container', 'blob' + * @readonly + * @enum {string} + */ +export enum PublicAccessType { + Container = 'container', + Blob = 'blob', +} + +/** + * Defines values for StorageErrorCode. + * Possible values include: 'AccountAlreadyExists', 'AccountBeingCreated', 'AccountIsDisabled', + * 'AuthenticationFailed', 'AuthorizationFailure', 'ConditionHeadersNotSupported', + * 'ConditionNotMet', 'EmptyMetadataKey', 'InsufficientAccountPermissions', 'InternalError', + * 'InvalidAuthenticationInfo', 'InvalidHeaderValue', 'InvalidHttpVerb', 'InvalidInput', + * 'InvalidMd5', 'InvalidMetadata', 'InvalidQueryParameterValue', 'InvalidRange', + * 'InvalidResourceName', 'InvalidUri', 'InvalidXmlDocument', 'InvalidXmlNodeValue', 'Md5Mismatch', + * 'MetadataTooLarge', 'MissingContentLengthHeader', 'MissingRequiredQueryParameter', + * 'MissingRequiredHeader', 'MissingRequiredXmlNode', 'MultipleConditionHeadersNotSupported', + * 'OperationTimedOut', 'OutOfRangeInput', 'OutOfRangeQueryParameterValue', 'RequestBodyTooLarge', + * 'ResourceTypeMismatch', 'RequestUrlFailedToParse', 'ResourceAlreadyExists', 'ResourceNotFound', + * 'ServerBusy', 'UnsupportedHeader', 'UnsupportedXmlNode', 'UnsupportedQueryParameter', + * 'UnsupportedHttpVerb', 'AppendPositionConditionNotMet', 'BlobAlreadyExists', + * 'BlobImmutableDueToPolicy', 'BlobNotFound', 'BlobOverwritten', + * 'BlobTierInadequateForContentLength', 'BlobUsesCustomerSpecifiedEncryption', + * 'BlockCountExceedsLimit', 'BlockListTooLong', 'CannotChangeToLowerTier', + * 'CannotVerifyCopySource', 'ContainerAlreadyExists', 'ContainerBeingDeleted', + * 'ContainerDisabled', 'ContainerNotFound', 'ContentLengthLargerThanTierLimit', + * 'CopyAcrossAccountsNotSupported', 'CopyIdMismatch', 'FeatureVersionMismatch', + * 'IncrementalCopyBlobMismatch', 'IncrementalCopyOfEarlierVersionSnapshotNotAllowed', + * 'IncrementalCopySourceMustBeSnapshot', 'InfiniteLeaseDurationRequired', 'InvalidBlobOrBlock', + * 'InvalidBlobTier', 'InvalidBlobType', 'InvalidBlockId', 'InvalidBlockList', 'InvalidOperation', + * 'InvalidPageRange', 'InvalidSourceBlobType', 'InvalidSourceBlobUrl', + * 'InvalidVersionForPageBlobOperation', 'LeaseAlreadyPresent', 'LeaseAlreadyBroken', + * 'LeaseIdMismatchWithBlobOperation', 'LeaseIdMismatchWithContainerOperation', + * 'LeaseIdMismatchWithLeaseOperation', 'LeaseIdMissing', 'LeaseIsBreakingAndCannotBeAcquired', + * 'LeaseIsBreakingAndCannotBeChanged', 'LeaseIsBrokenAndCannotBeRenewed', 'LeaseLost', + * 'LeaseNotPresentWithBlobOperation', 'LeaseNotPresentWithContainerOperation', + * 'LeaseNotPresentWithLeaseOperation', 'MaxBlobSizeConditionNotMet', + * 'NoAuthenticationInformation', 'NoPendingCopyOperation', + * 'OperationNotAllowedOnIncrementalCopyBlob', 'PendingCopyOperation', + * 'PreviousSnapshotCannotBeNewer', 'PreviousSnapshotNotFound', + * 'PreviousSnapshotOperationNotSupported', 'SequenceNumberConditionNotMet', + * 'SequenceNumberIncrementTooLarge', 'SnapshotCountExceeded', 'SnapshotOperationRateExceeded', + * 'SnapshotsPresent', 'SourceConditionNotMet', 'SystemInUse', 'TargetConditionNotMet', + * 'UnauthorizedBlobOverwrite', 'BlobBeingRehydrated', 'BlobArchived', 'BlobNotArchived', + * 'AuthorizationSourceIPMismatch', 'AuthorizationProtocolMismatch', + * 'AuthorizationPermissionMismatch', 'AuthorizationServiceMismatch', + * 'AuthorizationResourceTypeMismatch' + * @readonly + * @enum {string} + */ +export enum StorageErrorCode { + AccountAlreadyExists = 'AccountAlreadyExists', + AccountBeingCreated = 'AccountBeingCreated', + AccountIsDisabled = 'AccountIsDisabled', + AuthenticationFailed = 'AuthenticationFailed', + AuthorizationFailure = 'AuthorizationFailure', + ConditionHeadersNotSupported = 'ConditionHeadersNotSupported', + ConditionNotMet = 'ConditionNotMet', + EmptyMetadataKey = 'EmptyMetadataKey', + InsufficientAccountPermissions = 'InsufficientAccountPermissions', + InternalError = 'InternalError', + InvalidAuthenticationInfo = 'InvalidAuthenticationInfo', + InvalidHeaderValue = 'InvalidHeaderValue', + InvalidHttpVerb = 'InvalidHttpVerb', + InvalidInput = 'InvalidInput', + InvalidMd5 = 'InvalidMd5', + InvalidMetadata = 'InvalidMetadata', + InvalidQueryParameterValue = 'InvalidQueryParameterValue', + InvalidRange = 'InvalidRange', + InvalidResourceName = 'InvalidResourceName', + InvalidUri = 'InvalidUri', + InvalidXmlDocument = 'InvalidXmlDocument', + InvalidXmlNodeValue = 'InvalidXmlNodeValue', + Md5Mismatch = 'Md5Mismatch', + MetadataTooLarge = 'MetadataTooLarge', + MissingContentLengthHeader = 'MissingContentLengthHeader', + MissingRequiredQueryParameter = 'MissingRequiredQueryParameter', + MissingRequiredHeader = 'MissingRequiredHeader', + MissingRequiredXmlNode = 'MissingRequiredXmlNode', + MultipleConditionHeadersNotSupported = 'MultipleConditionHeadersNotSupported', + OperationTimedOut = 'OperationTimedOut', + OutOfRangeInput = 'OutOfRangeInput', + OutOfRangeQueryParameterValue = 'OutOfRangeQueryParameterValue', + RequestBodyTooLarge = 'RequestBodyTooLarge', + ResourceTypeMismatch = 'ResourceTypeMismatch', + RequestUrlFailedToParse = 'RequestUrlFailedToParse', + ResourceAlreadyExists = 'ResourceAlreadyExists', + ResourceNotFound = 'ResourceNotFound', + ServerBusy = 'ServerBusy', + UnsupportedHeader = 'UnsupportedHeader', + UnsupportedXmlNode = 'UnsupportedXmlNode', + UnsupportedQueryParameter = 'UnsupportedQueryParameter', + UnsupportedHttpVerb = 'UnsupportedHttpVerb', + AppendPositionConditionNotMet = 'AppendPositionConditionNotMet', + BlobAlreadyExists = 'BlobAlreadyExists', + BlobImmutableDueToPolicy = 'BlobImmutableDueToPolicy', + BlobNotFound = 'BlobNotFound', + BlobOverwritten = 'BlobOverwritten', + BlobTierInadequateForContentLength = 'BlobTierInadequateForContentLength', + BlobUsesCustomerSpecifiedEncryption = 'BlobUsesCustomerSpecifiedEncryption', + BlockCountExceedsLimit = 'BlockCountExceedsLimit', + BlockListTooLong = 'BlockListTooLong', + CannotChangeToLowerTier = 'CannotChangeToLowerTier', + CannotVerifyCopySource = 'CannotVerifyCopySource', + ContainerAlreadyExists = 'ContainerAlreadyExists', + ContainerBeingDeleted = 'ContainerBeingDeleted', + ContainerDisabled = 'ContainerDisabled', + ContainerNotFound = 'ContainerNotFound', + ContentLengthLargerThanTierLimit = 'ContentLengthLargerThanTierLimit', + CopyAcrossAccountsNotSupported = 'CopyAcrossAccountsNotSupported', + CopyIdMismatch = 'CopyIdMismatch', + FeatureVersionMismatch = 'FeatureVersionMismatch', + IncrementalCopyBlobMismatch = 'IncrementalCopyBlobMismatch', + IncrementalCopyOfEarlierVersionSnapshotNotAllowed = 'IncrementalCopyOfEarlierVersionSnapshotNotAllowed', + IncrementalCopySourceMustBeSnapshot = 'IncrementalCopySourceMustBeSnapshot', + InfiniteLeaseDurationRequired = 'InfiniteLeaseDurationRequired', + InvalidBlobOrBlock = 'InvalidBlobOrBlock', + InvalidBlobTier = 'InvalidBlobTier', + InvalidBlobType = 'InvalidBlobType', + InvalidBlockId = 'InvalidBlockId', + InvalidBlockList = 'InvalidBlockList', + InvalidOperation = 'InvalidOperation', + InvalidPageRange = 'InvalidPageRange', + InvalidSourceBlobType = 'InvalidSourceBlobType', + InvalidSourceBlobUrl = 'InvalidSourceBlobUrl', + InvalidVersionForPageBlobOperation = 'InvalidVersionForPageBlobOperation', + LeaseAlreadyPresent = 'LeaseAlreadyPresent', + LeaseAlreadyBroken = 'LeaseAlreadyBroken', + LeaseIdMismatchWithBlobOperation = 'LeaseIdMismatchWithBlobOperation', + LeaseIdMismatchWithContainerOperation = 'LeaseIdMismatchWithContainerOperation', + LeaseIdMismatchWithLeaseOperation = 'LeaseIdMismatchWithLeaseOperation', + LeaseIdMissing = 'LeaseIdMissing', + LeaseIsBreakingAndCannotBeAcquired = 'LeaseIsBreakingAndCannotBeAcquired', + LeaseIsBreakingAndCannotBeChanged = 'LeaseIsBreakingAndCannotBeChanged', + LeaseIsBrokenAndCannotBeRenewed = 'LeaseIsBrokenAndCannotBeRenewed', + LeaseLost = 'LeaseLost', + LeaseNotPresentWithBlobOperation = 'LeaseNotPresentWithBlobOperation', + LeaseNotPresentWithContainerOperation = 'LeaseNotPresentWithContainerOperation', + LeaseNotPresentWithLeaseOperation = 'LeaseNotPresentWithLeaseOperation', + MaxBlobSizeConditionNotMet = 'MaxBlobSizeConditionNotMet', + NoAuthenticationInformation = 'NoAuthenticationInformation', + NoPendingCopyOperation = 'NoPendingCopyOperation', + OperationNotAllowedOnIncrementalCopyBlob = 'OperationNotAllowedOnIncrementalCopyBlob', + PendingCopyOperation = 'PendingCopyOperation', + PreviousSnapshotCannotBeNewer = 'PreviousSnapshotCannotBeNewer', + PreviousSnapshotNotFound = 'PreviousSnapshotNotFound', + PreviousSnapshotOperationNotSupported = 'PreviousSnapshotOperationNotSupported', + SequenceNumberConditionNotMet = 'SequenceNumberConditionNotMet', + SequenceNumberIncrementTooLarge = 'SequenceNumberIncrementTooLarge', + SnapshotCountExceeded = 'SnapshotCountExceeded', + SnapshotOperationRateExceeded = 'SnapshotOperationRateExceeded', + SnapshotsPresent = 'SnapshotsPresent', + SourceConditionNotMet = 'SourceConditionNotMet', + SystemInUse = 'SystemInUse', + TargetConditionNotMet = 'TargetConditionNotMet', + UnauthorizedBlobOverwrite = 'UnauthorizedBlobOverwrite', + BlobBeingRehydrated = 'BlobBeingRehydrated', + BlobArchived = 'BlobArchived', + BlobNotArchived = 'BlobNotArchived', + AuthorizationSourceIPMismatch = 'AuthorizationSourceIPMismatch', + AuthorizationProtocolMismatch = 'AuthorizationProtocolMismatch', + AuthorizationPermissionMismatch = 'AuthorizationPermissionMismatch', + AuthorizationServiceMismatch = 'AuthorizationServiceMismatch', + AuthorizationResourceTypeMismatch = 'AuthorizationResourceTypeMismatch', +} + +/** + * Defines values for GeoReplicationStatusType. + * Possible values include: 'live', 'bootstrap', 'unavailable' + * @readonly + * @enum {string} + */ +export enum GeoReplicationStatusType { + Live = 'live', + Bootstrap = 'bootstrap', + Unavailable = 'unavailable', +} + +/** + * Defines values for QueryFormatType. + * Possible values include: 'delimited', 'json', 'arrow', 'parquet' + * @readonly + * @enum {string} + */ +export enum QueryFormatType { + Delimited = 'delimited', + Json = 'json', + Arrow = 'arrow', + Parquet = 'parquet', +} + +/** + * Defines values for DeleteSnapshotsOptionType. + * Possible values include: 'include', 'only' + * @readonly + * @enum {string} + */ +export enum DeleteSnapshotsOptionType { + Include = 'include', + Only = 'only', +} + +/** + * Defines values for BlobDeleteType. + * Possible values include: 'Permanent' + * @readonly + * @enum {string} + */ +export enum BlobDeleteType { + Permanent = 'Permanent', +} + +/** + * Defines values for PathSetAccessControlRecursiveMode. + * Possible values include: 'set', 'modify', 'remove' + * @readonly + * @enum {string} + */ +export enum PathSetAccessControlRecursiveMode { + Set = 'set', + Modify = 'modify', + Remove = 'remove', +} + +/** + * Defines values for LeaseAction. + * Possible values include: 'acquire', 'release', 'renew', 'break', 'change', 'auto-renew', + * 'acquire-release' + * @readonly + * @enum {string} + */ +export enum LeaseAction { + Acquire = 'acquire', + Release = 'release', + Renew = 'renew', + Break = 'break', + Change = 'change', + AutoRenew = 'auto-renew', + AcquireRelease = 'acquire-release', +} + +/** + * Defines values for PathExpiryOptions. + * Possible values include: 'NeverExpire', 'RelativeToCreation', 'RelativeToNow', 'Absolute' + * @readonly + * @enum {string} + */ +export enum PathExpiryOptions { + NeverExpire = 'NeverExpire', + RelativeToCreation = 'RelativeToCreation', + RelativeToNow = 'RelativeToNow', + Absolute = 'Absolute', +} + +/** + * Defines values for ListBlobsIncludeItem. + * Possible values include: '', 'copy', 'deleted', 'metadata', 'snapshots', 'uncommittedblobs', + * 'versions', 'tags', 'immutabilitypolicy', 'legalhold', 'deletedwithversions', 'permissions' + * @readonly + * @enum {string} + */ +export enum ListBlobsIncludeItem { + EmptyString = '', + Copy = 'copy', + Deleted = 'deleted', + Metadata = 'metadata', + Snapshots = 'snapshots', + Uncommittedblobs = 'uncommittedblobs', + Versions = 'versions', + Tags = 'tags', + Immutabilitypolicy = 'immutabilitypolicy', + Legalhold = 'legalhold', + Deletedwithversions = 'deletedwithversions', + Permissions = 'permissions', +} + +/** + * Defines values for ListBlobsShowOnly. + * Possible values include: 'deleted' + * @readonly + * @enum {string} + */ +export enum ListBlobsShowOnly { + Deleted = 'deleted', +} + +/** + * Defines values for EncryptionAlgorithmType. + * Possible values include: 'AES256' + * @readonly + * @enum {string} + */ +export enum EncryptionAlgorithmType { + AES256 = 'AES256', +} + +/** + * Defines values for PremiumPageBlobAccessTier. + * Possible values include: 'P4', 'P6', 'P10', 'P15', 'P20', 'P30', 'P40', 'P50', 'P60', 'P70', + * 'P80' + * @readonly + * @enum {string} + */ +export enum PremiumPageBlobAccessTier { + P4 = 'P4', + P6 = 'P6', + P10 = 'P10', + P15 = 'P15', + P20 = 'P20', + P30 = 'P30', + P40 = 'P40', + P50 = 'P50', + P60 = 'P60', + P70 = 'P70', + P80 = 'P80', +} + +/** + * Defines values for BlobExpiryOptions. + * Possible values include: 'NeverExpire', 'RelativeToCreation', 'RelativeToNow', 'Absolute' + * @readonly + * @enum {string} + */ +export enum BlobExpiryOptions { + NeverExpire = 'NeverExpire', + RelativeToCreation = 'RelativeToCreation', + RelativeToNow = 'RelativeToNow', + Absolute = 'Absolute', +} + +/** + * Defines values for BlockListType. + * Possible values include: 'committed', 'uncommitted', 'all' + * @readonly + * @enum {string} + */ +export enum BlockListType { + Committed = 'committed', + Uncommitted = 'uncommitted', + All = 'all', +} + +/** + * Defines values for BlobCopySourceTags. + * Possible values include: 'REPLACE', 'COPY' + * @readonly + * @enum {string} + */ +export enum BlobCopySourceTags { + REPLACE = 'REPLACE', + COPY = 'COPY', +} + +/** + * Defines values for FilterBlobsIncludeItem. + * Possible values include: 'none', 'versions' + * @readonly + * @enum {string} + */ +export enum FilterBlobsIncludeItem { + None = 'none', + Versions = 'versions', +} + +/** + * Defines values for ListContainersIncludeType. + * Possible values include: '', 'metadata', 'deleted', 'system' + * @readonly + * @enum {string} + */ +export enum ListContainersIncludeType { + EmptyString = '', + Metadata = 'metadata', + Deleted = 'deleted', + System = 'system', +} + +/** + * Defines values for SequenceNumberActionType. + * Possible values include: 'max', 'update', 'increment' + * @readonly + * @enum {string} + */ +export enum SequenceNumberActionType { + Max = 'max', + Update = 'update', + Increment = 'increment', +} + +/** + * Defines values for PathResourceType. + * Possible values include: 'directory', 'file' + * @readonly + * @enum {string} + */ +export enum PathResourceType { + Directory = 'directory', + File = 'file', +} + +/** + * Defines values for PathRenameMode. + * Possible values include: 'legacy', 'posix' + * @readonly + * @enum {string} + */ +export enum PathRenameMode { + Legacy = 'legacy', + Posix = 'posix', +} + +/** + * Defines values for PathUpdateAction. + * Possible values include: 'append', 'flush', 'setProperties', 'setAccessControl', + * 'setAccessControlRecursive' + * @readonly + * @enum {string} + */ +export enum PathUpdateAction { + Append = 'append', + Flush = 'flush', + SetProperties = 'setProperties', + SetAccessControl = 'setAccessControl', + SetAccessControlRecursive = 'setAccessControlRecursive', +} + +/** + * Defines values for PathLeaseAction. + * Possible values include: 'acquire', 'break', 'change', 'renew', 'release' + * @readonly + * @enum {string} + */ +export enum PathLeaseAction { + Acquire = 'acquire', + Break = 'break', + Change = 'change', + Renew = 'renew', + Release = 'release', +} + +/** + * Defines values for PathGetPropertiesAction. + * Possible values include: 'getAccessControl', 'getStatus' + * @readonly + * @enum {string} + */ +export enum PathGetPropertiesAction { + GetAccessControl = 'getAccessControl', + GetStatus = 'getStatus', +} + +/** + * Defines values for SkuName. + * Possible values include: 'Standard_LRS', 'Standard_GRS', 'Standard_RAGRS', 'Standard_ZRS', + * 'Premium_LRS' + * @readonly + * @enum {string} + */ +export enum SkuName { + StandardLRS = 'Standard_LRS', + StandardGRS = 'Standard_GRS', + StandardRAGRS = 'Standard_RAGRS', + StandardZRS = 'Standard_ZRS', + PremiumLRS = 'Premium_LRS', +} + +/** + * Defines values for AccountKind. + * Possible values include: 'Storage', 'BlobStorage', 'StorageV2', 'FileStorage', + * 'BlockBlobStorage' + * @readonly + * @enum {string} + */ +export enum AccountKind { + Storage = 'Storage', + BlobStorage = 'BlobStorage', + StorageV2 = 'StorageV2', + FileStorage = 'FileStorage', + BlockBlobStorage = 'BlockBlobStorage', +} + +/** + * Defines values for SyncCopyStatusType. + * Possible values include: 'success' + * @readonly + * @enum {string} + */ +export enum SyncCopyStatusType { + Success = 'success', +} + +/** + * Contains response data for the listFileSystems operation. + */ +export type ServiceListFileSystemsResponse = FileSystemList & ServiceListFileSystemsHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the setProperties operation. + */ +export type ServiceSetPropertiesResponse = ServiceSetPropertiesHeaders & { + /** + * The response status code. + */ + statusCode: 202; +}; + +/** + * Contains response data for the getProperties operation. + */ +export type ServiceGetPropertiesResponse = StorageServiceProperties & ServiceGetPropertiesHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the getStatistics operation. + */ +export type ServiceGetStatisticsResponse = StorageServiceStats & ServiceGetStatisticsHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the listContainersSegment operation. + */ +export type ServiceListContainersSegmentResponse = ListContainersSegmentResponse & ServiceListContainersSegmentHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the getUserDelegationKey operation. + */ +export type ServiceGetUserDelegationKeyResponse = UserDelegationKey & ServiceGetUserDelegationKeyHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the getAccountInfo operation. + */ +export type ServiceGetAccountInfoResponse = ServiceGetAccountInfoHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the getAccountInfoWithHead operation. + */ +export type ServiceGetAccountInfoWithHeadResponse = ServiceGetAccountInfoWithHeadHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the submitBatch operation. + */ +export type ServiceSubmitBatchResponse = ServiceSubmitBatchHeaders & { + /** + * The response body as a node.js Readable stream. + */ + body?: NodeJS.ReadableStream; +} & { + /** + * The response status code. + */ + statusCode: 202; +}; + +/** + * Contains response data for the filterBlobs operation. + */ +export type ServiceFilterBlobsResponse = FilterBlobSegment & ServiceFilterBlobsHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the create operation. + */ +export type FileSystemCreateResponse = FileSystemCreateHeaders & { + /** + * The response status code. + */ + statusCode: 201; +}; + +/** + * Contains response data for the setProperties operation. + */ +export type FileSystemSetPropertiesResponse = FileSystemSetPropertiesHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the getProperties operation. + */ +export type FileSystemGetPropertiesResponse = FileSystemGetPropertiesHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the deleteMethod operation. + */ +export type FileSystemDeleteResponse = FileSystemDeleteHeaders & { + /** + * The response status code. + */ + statusCode: 202; +}; + +/** + * Contains response data for the listPaths operation. + */ +export type FileSystemListPathsResponse = PathList & FileSystemListPathsHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the listBlobFlatSegment operation. + */ +export type FileSystemListBlobFlatSegmentResponse = ListBlobsFlatSegmentResponse & FileSystemListBlobFlatSegmentHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the listBlobHierarchySegment operation. + */ +export type FileSystemListBlobHierarchySegmentResponse = ListBlobsHierarchySegmentResponse & FileSystemListBlobHierarchySegmentHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the create operation. + */ +export type PathCreateResponse = PathCreateHeaders & { + /** + * The response status code. + */ + statusCode: 201; +}; + +/** + * Contains response data for the update operation. + */ +export type PathUpdateResponse = SetAccessControlRecursiveResponse & PathUpdateHeaders & { + /** + * The response status code. + */ + statusCode: 200 | 202; +}; + +/** + * Contains response data for the lease operation. + */ +export type PathLeaseResponse = PathLeaseHeaders & { + /** + * The response status code. + */ + statusCode: 200 | 201 | 202; +}; + +/** + * Contains response data for the read operation. + */ +export type PathReadResponse = PathReadHeaders & { + /** + * The response body as a node.js Readable stream. + */ + body?: NodeJS.ReadableStream; +} & { + /** + * The response status code. + */ + statusCode: 200 | 206; +}; + +/** + * Contains response data for the getProperties operation. + */ +export type PathGetPropertiesResponse = PathGetPropertiesHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the deleteMethod operation. + */ +export type PathDeleteResponse = PathDeleteHeaders & { + /** + * The response status code. + */ + statusCode: 200 | 202; +}; + +/** + * Contains response data for the setAccessControl operation. + */ +export type PathSetAccessControlResponse = PathSetAccessControlHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the setAccessControlRecursive operation. + */ +export type PathSetAccessControlRecursiveResponse = SetAccessControlRecursiveResponse & PathSetAccessControlRecursiveHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the setProperties operation. + */ +export type PathSetPropertiesResponse = PathSetPropertiesHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the flushData operation. + */ +export type PathFlushDataResponse = PathFlushDataHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the appendData operation. + */ +export type PathAppendDataResponse = PathAppendDataHeaders & { + /** + * The response status code. + */ + statusCode: 202; +}; + +/** + * Contains response data for the setExpiry operation. + */ +export type PathSetExpiryResponse = PathSetExpiryHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the undelete operation. + */ +export type PathUndeleteResponse = PathUndeleteHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the create operation. + */ +export type ContainerCreateResponse = ContainerCreateHeaders & { + /** + * The response status code. + */ + statusCode: 201; +}; + +/** + * Contains response data for the getProperties operation. + */ +export type ContainerGetPropertiesResponse = ContainerGetPropertiesHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the getPropertiesWithHead operation. + */ +export type ContainerGetPropertiesWithHeadResponse = ContainerGetPropertiesWithHeadHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the deleteMethod operation. + */ +export type ContainerDeleteResponse = ContainerDeleteHeaders & { + /** + * The response status code. + */ + statusCode: 202; +}; + +/** + * Contains response data for the setMetadata operation. + */ +export type ContainerSetMetadataResponse = ContainerSetMetadataHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the getAccessPolicy operation. + */ +export type ContainerGetAccessPolicyResponse = Array & ContainerGetAccessPolicyHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the setAccessPolicy operation. + */ +export type ContainerSetAccessPolicyResponse = ContainerSetAccessPolicyHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the restore operation. + */ +export type ContainerRestoreResponse = ContainerRestoreHeaders & { + /** + * The response status code. + */ + statusCode: 201; +}; + +/** + * Contains response data for the submitBatch operation. + */ +export type ContainerSubmitBatchResponse = ContainerSubmitBatchHeaders & { + /** + * The response body as a node.js Readable stream. + */ + body?: NodeJS.ReadableStream; +} & { + /** + * The response status code. + */ + statusCode: 202; +}; + +/** + * Contains response data for the filterBlobs operation. + */ +export type ContainerFilterBlobsResponse = FilterBlobSegment & ContainerFilterBlobsHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the acquireLease operation. + */ +export type ContainerAcquireLeaseResponse = ContainerAcquireLeaseHeaders & { + /** + * The response status code. + */ + statusCode: 201; +}; + +/** + * Contains response data for the releaseLease operation. + */ +export type ContainerReleaseLeaseResponse = ContainerReleaseLeaseHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the renewLease operation. + */ +export type ContainerRenewLeaseResponse = ContainerRenewLeaseHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the breakLease operation. + */ +export type ContainerBreakLeaseResponse = ContainerBreakLeaseHeaders & { + /** + * The response status code. + */ + statusCode: 202; +}; + +/** + * Contains response data for the changeLease operation. + */ +export type ContainerChangeLeaseResponse = ContainerChangeLeaseHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the getAccountInfo operation. + */ +export type ContainerGetAccountInfoResponse = ContainerGetAccountInfoHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the getAccountInfoWithHead operation. + */ +export type ContainerGetAccountInfoWithHeadResponse = ContainerGetAccountInfoWithHeadHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the create operation. + */ +export type PageBlobCreateResponse = PageBlobCreateHeaders & { + /** + * The response status code. + */ + statusCode: 201; +}; + +/** + * Contains response data for the uploadPages operation. + */ +export type PageBlobUploadPagesResponse = PageBlobUploadPagesHeaders & { + /** + * The response status code. + */ + statusCode: 201; +}; + +/** + * Contains response data for the clearPages operation. + */ +export type PageBlobClearPagesResponse = PageBlobClearPagesHeaders & { + /** + * The response status code. + */ + statusCode: 201; +}; + +/** + * Contains response data for the uploadPagesFromURL operation. + */ +export type PageBlobUploadPagesFromURLResponse = PageBlobUploadPagesFromURLHeaders & { + /** + * The response status code. + */ + statusCode: 201; +}; + +/** + * Contains response data for the getPageRanges operation. + */ +export type PageBlobGetPageRangesResponse = PageList & PageBlobGetPageRangesHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the getPageRangesDiff operation. + */ +export type PageBlobGetPageRangesDiffResponse = PageList & PageBlobGetPageRangesDiffHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the resize operation. + */ +export type PageBlobResizeResponse = PageBlobResizeHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the updateSequenceNumber operation. + */ +export type PageBlobUpdateSequenceNumberResponse = PageBlobUpdateSequenceNumberHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the copyIncremental operation. + */ +export type PageBlobCopyIncrementalResponse = PageBlobCopyIncrementalHeaders & { + /** + * The response status code. + */ + statusCode: 202; +}; + +/** + * Contains response data for the create operation. + */ +export type AppendBlobCreateResponse = AppendBlobCreateHeaders & { + /** + * The response status code. + */ + statusCode: 201; +}; + +/** + * Contains response data for the appendBlock operation. + */ +export type AppendBlobAppendBlockResponse = AppendBlobAppendBlockHeaders & { + /** + * The response status code. + */ + statusCode: 201; +}; + +/** + * Contains response data for the appendBlockFromUrl operation. + */ +export type AppendBlobAppendBlockFromUrlResponse = AppendBlobAppendBlockFromUrlHeaders & { + /** + * The response status code. + */ + statusCode: 201; +}; + +/** + * Contains response data for the seal operation. + */ +export type AppendBlobSealResponse = AppendBlobSealHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the upload operation. + */ +export type BlockBlobUploadResponse = BlockBlobUploadHeaders & { + /** + * The response status code. + */ + statusCode: 201; +}; + +/** + * Contains response data for the putBlobFromUrl operation. + */ +export type BlockBlobPutBlobFromUrlResponse = BlockBlobPutBlobFromUrlHeaders & { + /** + * The response status code. + */ + statusCode: 201; +}; + +/** + * Contains response data for the stageBlock operation. + */ +export type BlockBlobStageBlockResponse = BlockBlobStageBlockHeaders & { + /** + * The response status code. + */ + statusCode: 201; +}; + +/** + * Contains response data for the stageBlockFromURL operation. + */ +export type BlockBlobStageBlockFromURLResponse = BlockBlobStageBlockFromURLHeaders & { + /** + * The response status code. + */ + statusCode: 201; +}; + +/** + * Contains response data for the commitBlockList operation. + */ +export type BlockBlobCommitBlockListResponse = BlockBlobCommitBlockListHeaders & { + /** + * The response status code. + */ + statusCode: 201; +}; + +/** + * Contains response data for the getBlockList operation. + */ +export type BlockBlobGetBlockListResponse = BlockList & BlockBlobGetBlockListHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the undelete operation. + */ +export type BlobUndeleteResponse = BlobUndeleteHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the setExpiry operation. + */ +export type BlobSetExpiryResponse = BlobSetExpiryHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the setHTTPHeaders operation. + */ +export type BlobSetHTTPHeadersResponse = BlobSetHTTPHeadersHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the setImmutabilityPolicy operation. + */ +export type BlobSetImmutabilityPolicyResponse = BlobSetImmutabilityPolicyHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the deleteImmutabilityPolicy operation. + */ +export type BlobDeleteImmutabilityPolicyResponse = BlobDeleteImmutabilityPolicyHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the setLegalHold operation. + */ +export type BlobSetLegalHoldResponse = BlobSetLegalHoldHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the setMetadata operation. + */ +export type BlobSetMetadataResponse = BlobSetMetadataHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the acquireLease operation. + */ +export type BlobAcquireLeaseResponse = BlobAcquireLeaseHeaders & { + /** + * The response status code. + */ + statusCode: 201; +}; + +/** + * Contains response data for the releaseLease operation. + */ +export type BlobReleaseLeaseResponse = BlobReleaseLeaseHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the renewLease operation. + */ +export type BlobRenewLeaseResponse = BlobRenewLeaseHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the changeLease operation. + */ +export type BlobChangeLeaseResponse = BlobChangeLeaseHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the breakLease operation. + */ +export type BlobBreakLeaseResponse = BlobBreakLeaseHeaders & { + /** + * The response status code. + */ + statusCode: 202; +}; + +/** + * Contains response data for the createSnapshot operation. + */ +export type BlobCreateSnapshotResponse = BlobCreateSnapshotHeaders & { + /** + * The response status code. + */ + statusCode: 201; +}; + +/** + * Contains response data for the startCopyFromURL operation. + */ +export type BlobStartCopyFromURLResponse = BlobStartCopyFromURLHeaders & { + /** + * The response status code. + */ + statusCode: 202; +}; + +/** + * Contains response data for the copyFromURL operation. + */ +export type BlobCopyFromURLResponse = BlobCopyFromURLHeaders & { + /** + * The response status code. + */ + statusCode: 202; +}; + +/** + * Contains response data for the abortCopyFromURL operation. + */ +export type BlobAbortCopyFromURLResponse = BlobAbortCopyFromURLHeaders & { + /** + * The response status code. + */ + statusCode: 204; +}; + +/** + * Contains response data for the setTier operation. + */ +export type BlobSetTierResponse = BlobSetTierHeaders & { + /** + * The response status code. + */ + statusCode: 200 | 202; +}; + +/** + * Contains response data for the getAccountInfo operation. + */ +export type BlobGetAccountInfoResponse = BlobGetAccountInfoHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the getAccountInfoWithHead operation. + */ +export type BlobGetAccountInfoWithHeadResponse = BlobGetAccountInfoWithHeadHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the query operation. + */ +export type BlobQueryResponse = BlobQueryHeaders & { + /** + * The response body as a node.js Readable stream. + */ + body?: NodeJS.ReadableStream; +} & { + /** + * The response status code. + */ + statusCode: 200 | 206; +}; + +/** + * Contains response data for the getTags operation. + */ +export type BlobGetTagsResponse = BlobTags & BlobGetTagsHeaders & { + /** + * The response status code. + */ + statusCode: 200; +}; + +/** + * Contains response data for the setTags operation. + */ +export type BlobSetTagsResponse = BlobSetTagsHeaders & { + /** + * The response status code. + */ + statusCode: 204; +}; diff --git a/src/dfs/generated/artifacts/operation.ts b/src/dfs/generated/artifacts/operation.ts new file mode 100644 index 000000000..1c81c7a33 --- /dev/null +++ b/src/dfs/generated/artifacts/operation.ts @@ -0,0 +1,99 @@ +/* + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + * + * Code generated by Microsoft (R) AutoRest Code Generator. + * Changes may cause incorrect behavior and will be lost if the code is regenerated. + */ + +export enum Operation { + Service_ListFileSystems, + Service_SetProperties, + Service_GetProperties, + Service_GetStatistics, + Service_ListContainersSegment, + Service_GetUserDelegationKey, + Service_GetAccountInfo, + Service_GetAccountInfoWithHead, + Service_SubmitBatch, + Service_FilterBlobs, + FileSystem_Create, + FileSystem_SetProperties, + FileSystem_GetProperties, + FileSystem_Delete, + FileSystem_ListPaths, + FileSystem_ListBlobFlatSegment, + FileSystem_ListBlobHierarchySegment, + Path_Create, + Path_Update, + Path_Lease, + Path_Read, + Path_GetProperties, + Path_Delete, + Path_SetAccessControl, + Path_SetAccessControlRecursive, + Path_SetProperties, + Path_FlushData, + Path_AppendData, + Path_SetExpiry, + Path_Undelete, + Container_Create, + Container_GetProperties, + Container_GetPropertiesWithHead, + Container_Delete, + Container_SetMetadata, + Container_GetAccessPolicy, + Container_SetAccessPolicy, + Container_Restore, + Container_SubmitBatch, + Container_FilterBlobs, + Container_AcquireLease, + Container_ReleaseLease, + Container_RenewLease, + Container_BreakLease, + Container_ChangeLease, + Container_GetAccountInfo, + Container_GetAccountInfoWithHead, + PageBlob_Create, + PageBlob_UploadPages, + PageBlob_ClearPages, + PageBlob_UploadPagesFromURL, + PageBlob_GetPageRanges, + PageBlob_GetPageRangesDiff, + PageBlob_Resize, + PageBlob_UpdateSequenceNumber, + PageBlob_CopyIncremental, + AppendBlob_Create, + AppendBlob_AppendBlock, + AppendBlob_AppendBlockFromUrl, + AppendBlob_Seal, + BlockBlob_Upload, + BlockBlob_PutBlobFromUrl, + BlockBlob_StageBlock, + BlockBlob_StageBlockFromURL, + BlockBlob_CommitBlockList, + BlockBlob_GetBlockList, + Blob_Undelete, + Blob_SetExpiry, + Blob_SetHTTPHeaders, + Blob_SetImmutabilityPolicy, + Blob_DeleteImmutabilityPolicy, + Blob_SetLegalHold, + Blob_SetMetadata, + Blob_AcquireLease, + Blob_ReleaseLease, + Blob_RenewLease, + Blob_ChangeLease, + Blob_BreakLease, + Blob_CreateSnapshot, + Blob_StartCopyFromURL, + Blob_CopyFromURL, + Blob_AbortCopyFromURL, + Blob_SetTier, + Blob_GetAccountInfo, + Blob_GetAccountInfoWithHead, + Blob_Query, + Blob_GetTags, + Blob_SetTags, +} +export default Operation; diff --git a/src/dfs/generated/artifacts/parameters.ts b/src/dfs/generated/artifacts/parameters.ts new file mode 100644 index 000000000..e4ab9ed32 --- /dev/null +++ b/src/dfs/generated/artifacts/parameters.ts @@ -0,0 +1,2305 @@ +/* + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for + * license information. + * + * Code generated by Microsoft (R) AutoRest Code Generator. + * Changes may cause incorrect behavior and will be lost if the code is + * regenerated. + */ + +// tslint:disable:quotemark +// tslint:disable:object-literal-sort-keys + +import * as msRest from "@azure/ms-rest-js"; + +export const access: msRest.OperationParameter = { + parameterPath: [ + "options", + "access" + ], + mapper: { + serializedName: "x-ms-blob-public-access", + type: { + name: "String" + } + } +}; +export const acl: msRest.OperationParameter = { + parameterPath: [ + "options", + "acl" + ], + mapper: { + serializedName: "x-ms-acl", + type: { + name: "String" + } + } +}; +export const action0: msRest.OperationQueryParameter = { + parameterPath: "action", + mapper: { + required: true, + serializedName: "action", + type: { + name: "Enum", + allowedValues: [ + "append", + "flush", + "setProperties", + "setAccessControl", + "setAccessControlRecursive" + ] + } + } +}; +export const action1: msRest.OperationQueryParameter = { + parameterPath: [ + "options", + "action" + ], + mapper: { + serializedName: "action", + type: { + name: "Enum", + allowedValues: [ + "getAccessControl", + "getStatus" + ] + } + } +}; +export const action10: msRest.OperationParameter = { + parameterPath: "action", + mapper: { + required: true, + isConstant: true, + serializedName: "x-ms-lease-action", + defaultValue: 'break', + type: { + name: "String" + } + } +}; +export const action11: msRest.OperationParameter = { + parameterPath: "action", + mapper: { + required: true, + isConstant: true, + serializedName: "x-ms-lease-action", + defaultValue: 'change', + type: { + name: "String" + } + } +}; +export const action2: msRest.OperationQueryParameter = { + parameterPath: "action", + mapper: { + required: true, + isConstant: true, + serializedName: "action", + defaultValue: 'setAccessControl', + type: { + name: "String" + } + } +}; +export const action3: msRest.OperationQueryParameter = { + parameterPath: "action", + mapper: { + required: true, + isConstant: true, + serializedName: "action", + defaultValue: 'setAccessControlRecursive', + type: { + name: "String" + } + } +}; +export const action4: msRest.OperationQueryParameter = { + parameterPath: "action", + mapper: { + required: true, + isConstant: true, + serializedName: "action", + defaultValue: 'setProperties', + type: { + name: "String" + } + } +}; +export const action5: msRest.OperationQueryParameter = { + parameterPath: "action", + mapper: { + required: true, + isConstant: true, + serializedName: "action", + defaultValue: 'flush', + type: { + name: "String" + } + } +}; +export const action6: msRest.OperationQueryParameter = { + parameterPath: "action", + mapper: { + required: true, + isConstant: true, + serializedName: "action", + defaultValue: 'append', + type: { + name: "String" + } + } +}; +export const action7: msRest.OperationParameter = { + parameterPath: "action", + mapper: { + required: true, + isConstant: true, + serializedName: "x-ms-lease-action", + defaultValue: 'acquire', + type: { + name: "String" + } + } +}; +export const action8: msRest.OperationParameter = { + parameterPath: "action", + mapper: { + required: true, + isConstant: true, + serializedName: "x-ms-lease-action", + defaultValue: 'release', + type: { + name: "String" + } + } +}; +export const action9: msRest.OperationParameter = { + parameterPath: "action", + mapper: { + required: true, + isConstant: true, + serializedName: "x-ms-lease-action", + defaultValue: 'renew', + type: { + name: "String" + } + } +}; +export const appendPosition: msRest.OperationParameter = { + parameterPath: [ + "options", + "appendPositionAccessConditions", + "appendPosition" + ], + mapper: { + serializedName: "x-ms-blob-condition-appendpos", + type: { + name: "Number" + } + } +}; +export const blobCacheControl: msRest.OperationParameter = { + parameterPath: [ + "options", + "blobHTTPHeaders", + "blobCacheControl" + ], + mapper: { + serializedName: "x-ms-blob-cache-control", + type: { + name: "String" + } + } +}; +export const blobContentDisposition: msRest.OperationParameter = { + parameterPath: [ + "options", + "blobHTTPHeaders", + "blobContentDisposition" + ], + mapper: { + serializedName: "x-ms-blob-content-disposition", + type: { + name: "String" + } + } +}; +export const blobContentEncoding: msRest.OperationParameter = { + parameterPath: [ + "options", + "blobHTTPHeaders", + "blobContentEncoding" + ], + mapper: { + serializedName: "x-ms-blob-content-encoding", + type: { + name: "String" + } + } +}; +export const blobContentLanguage: msRest.OperationParameter = { + parameterPath: [ + "options", + "blobHTTPHeaders", + "blobContentLanguage" + ], + mapper: { + serializedName: "x-ms-blob-content-language", + type: { + name: "String" + } + } +}; +export const blobContentLength: msRest.OperationParameter = { + parameterPath: "blobContentLength", + mapper: { + required: true, + serializedName: "x-ms-blob-content-length", + type: { + name: "Number" + } + } +}; +export const blobContentMD5: msRest.OperationParameter = { + parameterPath: [ + "options", + "blobHTTPHeaders", + "blobContentMD5" + ], + mapper: { + serializedName: "x-ms-blob-content-md5", + type: { + name: "ByteArray" + } + } +}; +export const blobContentType: msRest.OperationParameter = { + parameterPath: [ + "options", + "blobHTTPHeaders", + "blobContentType" + ], + mapper: { + serializedName: "x-ms-blob-content-type", + type: { + name: "String" + } + } +}; +export const blobDeleteType: msRest.OperationQueryParameter = { + parameterPath: [ + "options", + "blobDeleteType" + ], + mapper: { + serializedName: "deletetype", + type: { + name: "Enum", + allowedValues: [ + "Permanent" + ] + } + } +}; +export const blobSequenceNumber: msRest.OperationParameter = { + parameterPath: [ + "options", + "blobSequenceNumber" + ], + mapper: { + serializedName: "x-ms-blob-sequence-number", + defaultValue: 0, + type: { + name: "Number" + } + } +}; +export const blobTagsString: msRest.OperationParameter = { + parameterPath: [ + "options", + "blobTagsString" + ], + mapper: { + serializedName: "x-ms-tags", + type: { + name: "String" + } + } +}; +export const blobType0: msRest.OperationParameter = { + parameterPath: "blobType", + mapper: { + required: true, + isConstant: true, + serializedName: "x-ms-blob-type", + defaultValue: 'PageBlob', + type: { + name: "String" + } + } +}; +export const blobType1: msRest.OperationParameter = { + parameterPath: "blobType", + mapper: { + required: true, + isConstant: true, + serializedName: "x-ms-blob-type", + defaultValue: 'AppendBlob', + type: { + name: "String" + } + } +}; +export const blobType2: msRest.OperationParameter = { + parameterPath: "blobType", + mapper: { + required: true, + isConstant: true, + serializedName: "x-ms-blob-type", + defaultValue: 'BlockBlob', + type: { + name: "String" + } + } +}; +export const blockId: msRest.OperationQueryParameter = { + parameterPath: "blockId", + mapper: { + required: true, + serializedName: "blockid", + type: { + name: "String" + } + } +}; +export const breakPeriod: msRest.OperationParameter = { + parameterPath: [ + "options", + "breakPeriod" + ], + mapper: { + serializedName: "x-ms-lease-break-period", + type: { + name: "Number" + } + } +}; +export const cacheControl: msRest.OperationParameter = { + parameterPath: [ + "options", + "pathHTTPHeaders", + "cacheControl" + ], + mapper: { + serializedName: "x-ms-cache-control", + type: { + name: "String" + } + } +}; +export const close: msRest.OperationQueryParameter = { + parameterPath: [ + "options", + "close" + ], + mapper: { + serializedName: "close", + type: { + name: "Boolean" + } + } +}; +export const comp0: msRest.OperationQueryParameter = { + parameterPath: "comp", + mapper: { + required: true, + isConstant: true, + serializedName: "comp", + defaultValue: 'properties', + type: { + name: "String" + } + } +}; +export const comp1: msRest.OperationQueryParameter = { + parameterPath: "comp", + mapper: { + required: true, + isConstant: true, + serializedName: "comp", + defaultValue: 'stats', + type: { + name: "String" + } + } +}; +export const comp10: msRest.OperationQueryParameter = { + parameterPath: "comp", + mapper: { + required: true, + isConstant: true, + serializedName: "comp", + defaultValue: 'lease', + type: { + name: "String" + } + } +}; +export const comp11: msRest.OperationQueryParameter = { + parameterPath: "comp", + mapper: { + required: true, + isConstant: true, + serializedName: "comp", + defaultValue: 'page', + type: { + name: "String" + } + } +}; +export const comp12: msRest.OperationQueryParameter = { + parameterPath: "comp", + mapper: { + required: true, + isConstant: true, + serializedName: "comp", + defaultValue: 'pagelist', + type: { + name: "String" + } + } +}; +export const comp13: msRest.OperationQueryParameter = { + parameterPath: "comp", + mapper: { + required: true, + isConstant: true, + serializedName: "comp", + defaultValue: 'incrementalcopy', + type: { + name: "String" + } + } +}; +export const comp14: msRest.OperationQueryParameter = { + parameterPath: "comp", + mapper: { + required: true, + isConstant: true, + serializedName: "comp", + defaultValue: 'appendblock', + type: { + name: "String" + } + } +}; +export const comp15: msRest.OperationQueryParameter = { + parameterPath: "comp", + mapper: { + required: true, + isConstant: true, + serializedName: "comp", + defaultValue: 'seal', + type: { + name: "String" + } + } +}; +export const comp16: msRest.OperationQueryParameter = { + parameterPath: "comp", + mapper: { + required: true, + isConstant: true, + serializedName: "comp", + defaultValue: 'block', + type: { + name: "String" + } + } +}; +export const comp17: msRest.OperationQueryParameter = { + parameterPath: "comp", + mapper: { + required: true, + isConstant: true, + serializedName: "comp", + defaultValue: 'blocklist', + type: { + name: "String" + } + } +}; +export const comp18: msRest.OperationQueryParameter = { + parameterPath: "comp", + mapper: { + required: true, + isConstant: true, + serializedName: "comp", + defaultValue: 'immutabilityPolicies', + type: { + name: "String" + } + } +}; +export const comp19: msRest.OperationQueryParameter = { + parameterPath: "comp", + mapper: { + required: true, + isConstant: true, + serializedName: "comp", + defaultValue: 'legalhold', + type: { + name: "String" + } + } +}; +export const comp2: msRest.OperationQueryParameter = { + parameterPath: "comp", + mapper: { + required: true, + isConstant: true, + serializedName: "comp", + defaultValue: 'list', + type: { + name: "String" + } + } +}; +export const comp20: msRest.OperationQueryParameter = { + parameterPath: "comp", + mapper: { + required: true, + isConstant: true, + serializedName: "comp", + defaultValue: 'snapshot', + type: { + name: "String" + } + } +}; +export const comp21: msRest.OperationQueryParameter = { + parameterPath: "comp", + mapper: { + required: true, + isConstant: true, + serializedName: "comp", + defaultValue: 'copy', + type: { + name: "String" + } + } +}; +export const comp22: msRest.OperationQueryParameter = { + parameterPath: "comp", + mapper: { + required: true, + isConstant: true, + serializedName: "comp", + defaultValue: 'tier', + type: { + name: "String" + } + } +}; +export const comp23: msRest.OperationQueryParameter = { + parameterPath: "comp", + mapper: { + required: true, + isConstant: true, + serializedName: "comp", + defaultValue: 'query', + type: { + name: "String" + } + } +}; +export const comp24: msRest.OperationQueryParameter = { + parameterPath: "comp", + mapper: { + required: true, + isConstant: true, + serializedName: "comp", + defaultValue: 'tags', + type: { + name: "String" + } + } +}; +export const comp3: msRest.OperationQueryParameter = { + parameterPath: "comp", + mapper: { + required: true, + isConstant: true, + serializedName: "comp", + defaultValue: 'userdelegationkey', + type: { + name: "String" + } + } +}; +export const comp4: msRest.OperationQueryParameter = { + parameterPath: "comp", + mapper: { + required: true, + isConstant: true, + serializedName: "comp", + defaultValue: 'batch', + type: { + name: "String" + } + } +}; +export const comp5: msRest.OperationQueryParameter = { + parameterPath: "comp", + mapper: { + required: true, + isConstant: true, + serializedName: "comp", + defaultValue: 'blobs', + type: { + name: "String" + } + } +}; +export const comp6: msRest.OperationQueryParameter = { + parameterPath: "comp", + mapper: { + required: true, + isConstant: true, + serializedName: "comp", + defaultValue: 'expiry', + type: { + name: "String" + } + } +}; +export const comp7: msRest.OperationQueryParameter = { + parameterPath: "comp", + mapper: { + required: true, + isConstant: true, + serializedName: "comp", + defaultValue: 'undelete', + type: { + name: "String" + } + } +}; +export const comp8: msRest.OperationQueryParameter = { + parameterPath: "comp", + mapper: { + required: true, + isConstant: true, + serializedName: "comp", + defaultValue: 'metadata', + type: { + name: "String" + } + } +}; +export const comp9: msRest.OperationQueryParameter = { + parameterPath: "comp", + mapper: { + required: true, + isConstant: true, + serializedName: "comp", + defaultValue: 'acl', + type: { + name: "String" + } + } +}; +export const contentDisposition: msRest.OperationParameter = { + parameterPath: [ + "options", + "pathHTTPHeaders", + "contentDisposition" + ], + mapper: { + serializedName: "x-ms-content-disposition", + type: { + name: "String" + } + } +}; +export const contentEncoding: msRest.OperationParameter = { + parameterPath: [ + "options", + "pathHTTPHeaders", + "contentEncoding" + ], + mapper: { + serializedName: "x-ms-content-encoding", + type: { + name: "String" + } + } +}; +export const contentLanguage: msRest.OperationParameter = { + parameterPath: [ + "options", + "pathHTTPHeaders", + "contentLanguage" + ], + mapper: { + serializedName: "x-ms-content-language", + type: { + name: "String" + } + } +}; +export const contentLength0: msRest.OperationParameter = { + parameterPath: "contentLength", + mapper: { + required: true, + serializedName: "Content-Length", + type: { + name: "Number" + } + } +}; +export const contentLength1: msRest.OperationParameter = { + parameterPath: [ + "options", + "contentLength" + ], + mapper: { + serializedName: "Content-Length", + constraints: { + InclusiveMinimum: 0 + }, + type: { + name: "Number" + } + } +}; +export const contentMD5: msRest.OperationParameter = { + parameterPath: [ + "options", + "pathHTTPHeaders", + "contentMD5" + ], + mapper: { + serializedName: "x-ms-content-md5", + type: { + name: "ByteArray" + } + } +}; +export const contentType: msRest.OperationParameter = { + parameterPath: [ + "options", + "pathHTTPHeaders", + "contentType" + ], + mapper: { + serializedName: "x-ms-content-type", + type: { + name: "String" + } + } +}; +export const continuation: msRest.OperationQueryParameter = { + parameterPath: [ + "options", + "continuation" + ], + mapper: { + serializedName: "continuation", + type: { + name: "String" + } + } +}; +export const copyActionAbortConstant: msRest.OperationParameter = { + parameterPath: "copyActionAbortConstant", + mapper: { + required: true, + isConstant: true, + serializedName: "x-ms-copy-action", + defaultValue: 'abort', + type: { + name: "String" + } + } +}; +export const copyId: msRest.OperationQueryParameter = { + parameterPath: "copyId", + mapper: { + required: true, + serializedName: "copyid", + type: { + name: "String" + } + } +}; +export const copySource: msRest.OperationParameter = { + parameterPath: "copySource", + mapper: { + required: true, + serializedName: "x-ms-copy-source", + type: { + name: "String" + } + } +}; +export const copySourceAuthorization: msRest.OperationParameter = { + parameterPath: [ + "options", + "copySourceAuthorization" + ], + mapper: { + serializedName: "x-ms-copy-source-authorization", + type: { + name: "String" + } + } +}; +export const copySourceBlobProperties: msRest.OperationParameter = { + parameterPath: [ + "options", + "copySourceBlobProperties" + ], + mapper: { + serializedName: "x-ms-copy-source-blob-properties", + type: { + name: "Boolean" + } + } +}; +export const copySourceTags: msRest.OperationParameter = { + parameterPath: [ + "options", + "copySourceTags" + ], + mapper: { + serializedName: "x-ms-copy-source-tag-option", + type: { + name: "Enum", + allowedValues: [ + "REPLACE", + "COPY" + ] + } + } +}; +export const defaultEncryptionScope: msRest.OperationParameter = { + parameterPath: [ + "options", + "containerCpkScopeInfo", + "defaultEncryptionScope" + ], + mapper: { + serializedName: "x-ms-default-encryption-scope", + type: { + name: "String" + } + } +}; +export const deletedContainerName: msRest.OperationParameter = { + parameterPath: [ + "options", + "deletedContainerName" + ], + mapper: { + serializedName: "x-ms-deleted-container-name", + type: { + name: "String" + } + } +}; +export const deletedContainerVersion: msRest.OperationParameter = { + parameterPath: [ + "options", + "deletedContainerVersion" + ], + mapper: { + serializedName: "x-ms-deleted-container-version", + type: { + name: "String" + } + } +}; +export const deleteSnapshots: msRest.OperationParameter = { + parameterPath: [ + "options", + "deleteSnapshots" + ], + mapper: { + serializedName: "x-ms-delete-snapshots", + type: { + name: "Enum", + allowedValues: [ + "include", + "only" + ] + } + } +}; +export const delimiter: msRest.OperationQueryParameter = { + parameterPath: "delimiter", + mapper: { + required: true, + serializedName: "delimiter", + type: { + name: "String" + } + } +}; +export const duration: msRest.OperationParameter = { + parameterPath: [ + "options", + "duration" + ], + mapper: { + serializedName: "x-ms-lease-duration", + type: { + name: "Number" + } + } +}; +export const encryptionAlgorithm: msRest.OperationParameter = { + parameterPath: [ + "options", + "cpkInfo", + "encryptionAlgorithm" + ], + mapper: { + serializedName: "x-ms-encryption-algorithm", + type: { + name: "Enum", + allowedValues: [ + "AES256" + ] + } + } +}; +export const encryptionKey: msRest.OperationParameter = { + parameterPath: [ + "options", + "cpkInfo", + "encryptionKey" + ], + mapper: { + serializedName: "x-ms-encryption-key", + type: { + name: "String" + } + } +}; +export const encryptionKeySha256: msRest.OperationParameter = { + parameterPath: [ + "options", + "cpkInfo", + "encryptionKeySha256" + ], + mapper: { + serializedName: "x-ms-encryption-key-sha256", + type: { + name: "String" + } + } +}; +export const encryptionScope: msRest.OperationParameter = { + parameterPath: [ + "options", + "cpkScopeInfo", + "encryptionScope" + ], + mapper: { + serializedName: "x-ms-encryption-scope", + type: { + name: "String" + } + } +}; +export const expiresOn: msRest.OperationParameter = { + parameterPath: [ + "options", + "expiresOn" + ], + mapper: { + serializedName: "x-ms-expiry-time", + type: { + name: "String" + } + } +}; +export const expiryOptions0: msRest.OperationParameter = { + parameterPath: [ + "options", + "expiryOptions" + ], + mapper: { + serializedName: "x-ms-expiry-option", + type: { + name: "String" + } + } +}; +export const expiryOptions1: msRest.OperationParameter = { + parameterPath: "expiryOptions", + mapper: { + required: true, + serializedName: "x-ms-expiry-option", + type: { + name: "String" + } + } +}; +export const fileSystem: msRest.OperationURLParameter = { + parameterPath: "fileSystem", + mapper: { + required: true, + serializedName: "filesystem", + constraints: { + MaxLength: 63, + MinLength: 3, + Pattern: /^[$a-z0-9](?!.*--)[-a-z0-9]{1,61}[a-z0-9]$/ + }, + type: { + name: "String" + } + } +}; +export const flush: msRest.OperationQueryParameter = { + parameterPath: [ + "options", + "flush" + ], + mapper: { + serializedName: "flush", + type: { + name: "Boolean" + } + } +}; +export const forceFlag: msRest.OperationQueryParameter = { + parameterPath: [ + "options", + "forceFlag" + ], + mapper: { + serializedName: "forceFlag", + type: { + name: "Boolean" + } + } +}; +export const group: msRest.OperationParameter = { + parameterPath: [ + "options", + "group" + ], + mapper: { + serializedName: "x-ms-group", + type: { + name: "String" + } + } +}; +export const ifMatch: msRest.OperationParameter = { + parameterPath: [ + "options", + "modifiedAccessConditions", + "ifMatch" + ], + mapper: { + serializedName: "If-Match", + type: { + name: "String" + } + } +}; +export const ifModifiedSince: msRest.OperationParameter = { + parameterPath: [ + "options", + "modifiedAccessConditions", + "ifModifiedSince" + ], + mapper: { + serializedName: "If-Modified-Since", + type: { + name: "DateTimeRfc1123" + } + } +}; +export const ifNoneMatch: msRest.OperationParameter = { + parameterPath: [ + "options", + "modifiedAccessConditions", + "ifNoneMatch" + ], + mapper: { + serializedName: "If-None-Match", + type: { + name: "String" + } + } +}; +export const ifSequenceNumberEqualTo: msRest.OperationParameter = { + parameterPath: [ + "options", + "sequenceNumberAccessConditions", + "ifSequenceNumberEqualTo" + ], + mapper: { + serializedName: "x-ms-if-sequence-number-eq", + type: { + name: "Number" + } + } +}; +export const ifSequenceNumberLessThan: msRest.OperationParameter = { + parameterPath: [ + "options", + "sequenceNumberAccessConditions", + "ifSequenceNumberLessThan" + ], + mapper: { + serializedName: "x-ms-if-sequence-number-lt", + type: { + name: "Number" + } + } +}; +export const ifSequenceNumberLessThanOrEqualTo: msRest.OperationParameter = { + parameterPath: [ + "options", + "sequenceNumberAccessConditions", + "ifSequenceNumberLessThanOrEqualTo" + ], + mapper: { + serializedName: "x-ms-if-sequence-number-le", + type: { + name: "Number" + } + } +}; +export const ifTags: msRest.OperationParameter = { + parameterPath: [ + "options", + "modifiedAccessConditions", + "ifTags" + ], + mapper: { + serializedName: "x-ms-if-tags", + type: { + name: "String" + } + } +}; +export const ifUnmodifiedSince: msRest.OperationParameter = { + parameterPath: [ + "options", + "modifiedAccessConditions", + "ifUnmodifiedSince" + ], + mapper: { + serializedName: "If-Unmodified-Since", + type: { + name: "DateTimeRfc1123" + } + } +}; +export const immutabilityPolicyExpiry: msRest.OperationParameter = { + parameterPath: [ + "options", + "immutabilityPolicyExpiry" + ], + mapper: { + serializedName: "x-ms-immutability-policy-until-date", + type: { + name: "DateTimeRfc1123" + } + } +}; +export const immutabilityPolicyMode: msRest.OperationParameter = { + parameterPath: [ + "options", + "immutabilityPolicyMode" + ], + mapper: { + serializedName: "x-ms-immutability-policy-mode", + type: { + name: "Enum", + allowedValues: [ + "Mutable", + "Unlocked", + "Locked" + ] + } + } +}; +export const include0: msRest.OperationQueryParameter = { + parameterPath: [ + "options", + "include" + ], + mapper: { + serializedName: "include", + type: { + name: "Sequence", + element: { + type: { + name: "Enum", + allowedValues: [ + "", + "metadata", + "deleted", + "system" + ] + } + } + } + }, + collectionFormat: msRest.QueryCollectionFormat.Csv +}; +export const include1: msRest.OperationQueryParameter = { + parameterPath: [ + "options", + "include" + ], + mapper: { + serializedName: "include", + type: { + name: "Sequence", + element: { + type: { + name: "Enum", + allowedValues: [ + "none", + "versions" + ] + } + } + } + }, + collectionFormat: msRest.QueryCollectionFormat.Csv +}; +export const include2: msRest.OperationQueryParameter = { + parameterPath: [ + "options", + "include" + ], + mapper: { + serializedName: "include", + type: { + name: "Sequence", + element: { + type: { + name: "Enum", + allowedValues: [ + "", + "copy", + "deleted", + "metadata", + "snapshots", + "uncommittedblobs", + "versions", + "tags", + "immutabilitypolicy", + "legalhold", + "deletedwithversions", + "permissions" + ] + } + } + } + }, + collectionFormat: msRest.QueryCollectionFormat.Csv +}; +export const leaseAction: msRest.OperationParameter = { + parameterPath: [ + "options", + "leaseAction" + ], + mapper: { + serializedName: "x-ms-lease-action", + type: { + name: "Enum", + allowedValues: [ + "acquire", + "release", + "renew", + "break", + "change", + "auto-renew", + "acquire-release" + ] + } + } +}; +export const leaseDuration: msRest.OperationParameter = { + parameterPath: [ + "options", + "leaseDuration" + ], + mapper: { + serializedName: "x-ms-lease-duration", + type: { + name: "Number" + } + } +}; +export const leaseId0: msRest.OperationParameter = { + parameterPath: [ + "options", + "leaseAccessConditions", + "leaseId" + ], + mapper: { + serializedName: "x-ms-lease-id", + type: { + name: "String" + } + } +}; +export const leaseId1: msRest.OperationParameter = { + parameterPath: "leaseId", + mapper: { + required: true, + serializedName: "x-ms-lease-id", + type: { + name: "String" + } + } +}; +export const legalHold0: msRest.OperationParameter = { + parameterPath: [ + "options", + "legalHold" + ], + mapper: { + serializedName: "x-ms-legal-hold", + type: { + name: "Boolean" + } + } +}; +export const legalHold1: msRest.OperationParameter = { + parameterPath: "legalHold", + mapper: { + required: true, + serializedName: "x-ms-legal-hold", + type: { + name: "Boolean" + } + } +}; +export const listType: msRest.OperationQueryParameter = { + parameterPath: [ + "options", + "listType" + ], + mapper: { + serializedName: "blocklisttype", + defaultValue: 'committed', + type: { + name: "Enum", + allowedValues: [ + "committed", + "uncommitted", + "all" + ] + } + } +}; +export const marker: msRest.OperationQueryParameter = { + parameterPath: [ + "options", + "marker" + ], + mapper: { + serializedName: "marker", + type: { + name: "String" + } + } +}; +export const maxRecords: msRest.OperationQueryParameter = { + parameterPath: [ + "options", + "maxRecords" + ], + mapper: { + serializedName: "maxRecords", + constraints: { + InclusiveMinimum: 1 + }, + type: { + name: "Number" + } + } +}; +export const maxresults: msRest.OperationQueryParameter = { + parameterPath: [ + "options", + "maxresults" + ], + mapper: { + serializedName: "maxresults", + constraints: { + InclusiveMinimum: 1 + }, + type: { + name: "Number" + } + } +}; +export const maxResults: msRest.OperationQueryParameter = { + parameterPath: [ + "options", + "maxResults" + ], + mapper: { + serializedName: "maxResults", + constraints: { + InclusiveMinimum: 1 + }, + type: { + name: "Number" + } + } +}; +export const maxSize: msRest.OperationParameter = { + parameterPath: [ + "options", + "appendPositionAccessConditions", + "maxSize" + ], + mapper: { + serializedName: "x-ms-blob-condition-maxsize", + type: { + name: "Number" + } + } +}; +export const metadata: msRest.OperationParameter = { + parameterPath: [ + "options", + "metadata" + ], + mapper: { + serializedName: "x-ms-meta", + type: { + name: "Dictionary", + value: { + type: { + name: "String" + } + } + }, + headerCollectionPrefix: "x-ms-meta-" + } +}; +export const mode0: msRest.OperationQueryParameter = { + parameterPath: [ + "options", + "mode" + ], + mapper: { + serializedName: "mode", + type: { + name: "Enum", + allowedValues: [ + "legacy", + "posix" + ] + } + } +}; +export const mode1: msRest.OperationQueryParameter = { + parameterPath: "mode", + mapper: { + required: true, + serializedName: "mode", + type: { + name: "Enum", + allowedValues: [ + "set", + "modify", + "remove" + ] + } + } +}; +export const multipartContentType: msRest.OperationParameter = { + parameterPath: "multipartContentType", + mapper: { + required: true, + serializedName: "Content-Type", + type: { + name: "String" + } + } +}; +export const owner: msRest.OperationParameter = { + parameterPath: [ + "options", + "owner" + ], + mapper: { + serializedName: "x-ms-owner", + type: { + name: "String" + } + } +}; +export const pageWrite0: msRest.OperationParameter = { + parameterPath: "pageWrite", + mapper: { + required: true, + isConstant: true, + serializedName: "x-ms-page-write", + defaultValue: 'update', + type: { + name: "String" + } + } +}; +export const pageWrite1: msRest.OperationParameter = { + parameterPath: "pageWrite", + mapper: { + required: true, + isConstant: true, + serializedName: "x-ms-page-write", + defaultValue: 'clear', + type: { + name: "String" + } + } +}; +export const path0: msRest.OperationQueryParameter = { + parameterPath: [ + "options", + "path" + ], + mapper: { + serializedName: "directory", + type: { + name: "String" + } + } +}; +export const path1: msRest.OperationURLParameter = { + parameterPath: "path", + mapper: { + required: true, + serializedName: "path", + type: { + name: "String" + } + } +}; +export const permissions: msRest.OperationParameter = { + parameterPath: [ + "options", + "permissions" + ], + mapper: { + serializedName: "x-ms-permissions", + type: { + name: "String" + } + } +}; +export const position: msRest.OperationQueryParameter = { + parameterPath: [ + "options", + "position" + ], + mapper: { + serializedName: "position", + type: { + name: "Number" + } + } +}; +export const prefix: msRest.OperationQueryParameter = { + parameterPath: [ + "options", + "prefix" + ], + mapper: { + serializedName: "prefix", + type: { + name: "String" + } + } +}; +export const preventEncryptionScopeOverride: msRest.OperationParameter = { + parameterPath: [ + "options", + "containerCpkScopeInfo", + "preventEncryptionScopeOverride" + ], + mapper: { + serializedName: "x-ms-deny-encryption-scope-override", + type: { + name: "Boolean" + } + } +}; +export const prevsnapshot: msRest.OperationQueryParameter = { + parameterPath: [ + "options", + "prevsnapshot" + ], + mapper: { + serializedName: "prevsnapshot", + type: { + name: "String" + } + } +}; +export const prevSnapshotUrl: msRest.OperationParameter = { + parameterPath: [ + "options", + "prevSnapshotUrl" + ], + mapper: { + serializedName: "x-ms-previous-snapshot-url", + type: { + name: "String" + } + } +}; +export const properties: msRest.OperationParameter = { + parameterPath: [ + "options", + "properties" + ], + mapper: { + serializedName: "x-ms-properties", + type: { + name: "String" + } + } +}; +export const proposedLeaseId0: msRest.OperationParameter = { + parameterPath: [ + "options", + "proposedLeaseId" + ], + mapper: { + serializedName: "x-ms-proposed-lease-id", + type: { + name: "String" + } + } +}; +export const proposedLeaseId1: msRest.OperationParameter = { + parameterPath: "proposedLeaseId", + mapper: { + required: true, + serializedName: "x-ms-proposed-lease-id", + type: { + name: "String" + } + } +}; +export const range0: msRest.OperationParameter = { + parameterPath: [ + "options", + "range" + ], + mapper: { + serializedName: "x-ms-range", + type: { + name: "String" + } + } +}; +export const range1: msRest.OperationParameter = { + parameterPath: "range", + mapper: { + required: true, + serializedName: "x-ms-range", + type: { + name: "String" + } + } +}; +export const rangeGetContentCRC64: msRest.OperationParameter = { + parameterPath: [ + "options", + "rangeGetContentCRC64" + ], + mapper: { + serializedName: "x-ms-range-get-content-crc64", + type: { + name: "Boolean" + } + } +}; +export const rangeGetContentMD5: msRest.OperationParameter = { + parameterPath: [ + "options", + "rangeGetContentMD5" + ], + mapper: { + serializedName: "x-ms-range-get-content-md5", + type: { + name: "Boolean" + } + } +}; +export const recursive0: msRest.OperationQueryParameter = { + parameterPath: "recursive", + mapper: { + required: true, + serializedName: "recursive", + type: { + name: "Boolean" + } + } +}; +export const recursive1: msRest.OperationQueryParameter = { + parameterPath: [ + "options", + "recursive" + ], + mapper: { + serializedName: "recursive", + type: { + name: "Boolean" + } + } +}; +export const rehydratePriority: msRest.OperationParameter = { + parameterPath: [ + "options", + "rehydratePriority" + ], + mapper: { + serializedName: "x-ms-rehydrate-priority", + type: { + name: "String" + } + } +}; +export const renameSource: msRest.OperationParameter = { + parameterPath: [ + "options", + "renameSource" + ], + mapper: { + serializedName: "x-ms-rename-source", + type: { + name: "String" + } + } +}; +export const requestId: msRest.OperationParameter = { + parameterPath: [ + "options", + "requestId" + ], + mapper: { + serializedName: "x-ms-client-request-id", + type: { + name: "String" + } + } +}; +export const resource0: msRest.OperationQueryParameter = { + parameterPath: "resource", + mapper: { + required: true, + isConstant: true, + serializedName: "resource", + defaultValue: 'account', + type: { + name: "String" + } + } +}; +export const resource1: msRest.OperationQueryParameter = { + parameterPath: "resource", + mapper: { + required: true, + isConstant: true, + serializedName: "resource", + defaultValue: 'filesystem', + type: { + name: "String" + } + } +}; +export const resource2: msRest.OperationQueryParameter = { + parameterPath: [ + "options", + "resource" + ], + mapper: { + serializedName: "resource", + type: { + name: "Enum", + allowedValues: [ + "directory", + "file" + ] + } + } +}; +export const restype0: msRest.OperationQueryParameter = { + parameterPath: "restype", + mapper: { + required: true, + isConstant: true, + serializedName: "restype", + defaultValue: 'service', + type: { + name: "String" + } + } +}; +export const restype1: msRest.OperationQueryParameter = { + parameterPath: "restype", + mapper: { + required: true, + isConstant: true, + serializedName: "restype", + defaultValue: 'account', + type: { + name: "String" + } + } +}; +export const restype2: msRest.OperationQueryParameter = { + parameterPath: "restype", + mapper: { + required: true, + isConstant: true, + serializedName: "restype", + defaultValue: 'container', + type: { + name: "String" + } + } +}; +export const retainUncommittedData: msRest.OperationQueryParameter = { + parameterPath: [ + "options", + "retainUncommittedData" + ], + mapper: { + serializedName: "retainUncommittedData", + type: { + name: "Boolean" + } + } +}; +export const sealBlob: msRest.OperationParameter = { + parameterPath: [ + "options", + "sealBlob" + ], + mapper: { + serializedName: "x-ms-seal-blob", + type: { + name: "Boolean" + } + } +}; +export const sequenceNumberAction: msRest.OperationParameter = { + parameterPath: "sequenceNumberAction", + mapper: { + required: true, + serializedName: "x-ms-sequence-number-action", + type: { + name: "Enum", + allowedValues: [ + "max", + "update", + "increment" + ] + } + } +}; +export const showonly: msRest.OperationQueryParameter = { + parameterPath: [ + "options", + "showonly" + ], + mapper: { + serializedName: "showonly", + type: { + name: "Enum", + allowedValues: [ + "deleted" + ] + } + } +}; +export const snapshot: msRest.OperationQueryParameter = { + parameterPath: [ + "options", + "snapshot" + ], + mapper: { + serializedName: "snapshot", + type: { + name: "String" + } + } +}; +export const sourceContentcrc64: msRest.OperationParameter = { + parameterPath: [ + "options", + "sourceContentcrc64" + ], + mapper: { + serializedName: "x-ms-source-content-crc64", + type: { + name: "ByteArray" + } + } +}; +export const sourceContentMD5: msRest.OperationParameter = { + parameterPath: [ + "options", + "sourceContentMD5" + ], + mapper: { + serializedName: "x-ms-source-content-md5", + type: { + name: "ByteArray" + } + } +}; +export const sourceIfMatch: msRest.OperationParameter = { + parameterPath: [ + "options", + "sourceModifiedAccessConditions", + "sourceIfMatch" + ], + mapper: { + serializedName: "x-ms-source-if-match", + type: { + name: "String" + } + } +}; +export const sourceIfModifiedSince: msRest.OperationParameter = { + parameterPath: [ + "options", + "sourceModifiedAccessConditions", + "sourceIfModifiedSince" + ], + mapper: { + serializedName: "x-ms-source-if-modified-since", + type: { + name: "DateTimeRfc1123" + } + } +}; +export const sourceIfNoneMatch: msRest.OperationParameter = { + parameterPath: [ + "options", + "sourceModifiedAccessConditions", + "sourceIfNoneMatch" + ], + mapper: { + serializedName: "x-ms-source-if-none-match", + type: { + name: "String" + } + } +}; +export const sourceIfTags: msRest.OperationParameter = { + parameterPath: [ + "options", + "sourceModifiedAccessConditions", + "sourceIfTags" + ], + mapper: { + serializedName: "x-ms-source-if-tags", + type: { + name: "String" + } + } +}; +export const sourceIfUnmodifiedSince: msRest.OperationParameter = { + parameterPath: [ + "options", + "sourceModifiedAccessConditions", + "sourceIfUnmodifiedSince" + ], + mapper: { + serializedName: "x-ms-source-if-unmodified-since", + type: { + name: "DateTimeRfc1123" + } + } +}; +export const sourceLeaseId: msRest.OperationParameter = { + parameterPath: [ + "options", + "sourceLeaseId" + ], + mapper: { + serializedName: "x-ms-source-lease-id", + type: { + name: "String" + } + } +}; +export const sourceRange0: msRest.OperationParameter = { + parameterPath: "sourceRange", + mapper: { + required: true, + serializedName: "x-ms-source-range", + type: { + name: "String" + } + } +}; +export const sourceRange1: msRest.OperationParameter = { + parameterPath: [ + "options", + "sourceRange" + ], + mapper: { + serializedName: "x-ms-source-range", + type: { + name: "String" + } + } +}; +export const sourceUrl: msRest.OperationParameter = { + parameterPath: "sourceUrl", + mapper: { + required: true, + serializedName: "x-ms-copy-source", + type: { + name: "String" + } + } +}; +export const tier0: msRest.OperationParameter = { + parameterPath: [ + "options", + "tier" + ], + mapper: { + serializedName: "x-ms-access-tier", + type: { + name: "String" + } + } +}; +export const tier1: msRest.OperationParameter = { + parameterPath: "tier", + mapper: { + required: true, + serializedName: "x-ms-access-tier", + type: { + name: "String" + } + } +}; +export const timeout: msRest.OperationQueryParameter = { + parameterPath: [ + "options", + "timeout" + ], + mapper: { + serializedName: "timeout", + constraints: { + InclusiveMinimum: 0 + }, + type: { + name: "Number" + } + } +}; +export const transactionalContentCrc64: msRest.OperationParameter = { + parameterPath: [ + "options", + "transactionalContentCrc64" + ], + mapper: { + serializedName: "x-ms-content-crc64", + type: { + name: "ByteArray" + } + } +}; +export const transactionalContentHash: msRest.OperationParameter = { + parameterPath: [ + "options", + "pathHTTPHeaders", + "transactionalContentHash" + ], + mapper: { + serializedName: "Content-MD5", + type: { + name: "ByteArray" + } + } +}; +export const transactionalContentMD5: msRest.OperationParameter = { + parameterPath: [ + "options", + "transactionalContentMD5" + ], + mapper: { + serializedName: "Content-MD5", + type: { + name: "ByteArray" + } + } +}; +export const umask: msRest.OperationParameter = { + parameterPath: [ + "options", + "umask" + ], + mapper: { + serializedName: "x-ms-umask", + type: { + name: "String" + } + } +}; +export const undeleteSource: msRest.OperationParameter = { + parameterPath: [ + "options", + "undeleteSource" + ], + mapper: { + serializedName: "x-ms-undelete-source", + type: { + name: "String" + } + } +}; +export const upn: msRest.OperationQueryParameter = { + parameterPath: [ + "options", + "upn" + ], + mapper: { + serializedName: "upn", + type: { + name: "Boolean" + } + } +}; +export const url: msRest.OperationURLParameter = { + parameterPath: "url", + mapper: { + required: true, + serializedName: "url", + defaultValue: '', + type: { + name: "String" + } + }, + skipEncoding: true +}; +export const version: msRest.OperationParameter = { + parameterPath: "version", + mapper: { + serializedName: "x-ms-version", + type: { + name: "String" + } + } +}; +export const versionId: msRest.OperationQueryParameter = { + parameterPath: [ + "options", + "versionId" + ], + mapper: { + serializedName: "versionid", + type: { + name: "String" + } + } +}; +export const where: msRest.OperationQueryParameter = { + parameterPath: [ + "options", + "where" + ], + mapper: { + serializedName: "where", + type: { + name: "String" + } + } +}; +export const xMsLeaseAction: msRest.OperationParameter = { + parameterPath: "xMsLeaseAction", + mapper: { + required: true, + serializedName: "x-ms-lease-action", + type: { + name: "Enum", + allowedValues: [ + "acquire", + "break", + "change", + "renew", + "release" + ] + } + } +}; +export const xMsLeaseBreakPeriod: msRest.OperationParameter = { + parameterPath: [ + "options", + "xMsLeaseBreakPeriod" + ], + mapper: { + serializedName: "x-ms-lease-break-period", + type: { + name: "Number" + } + } +}; +export const xMsLeaseDuration: msRest.OperationParameter = { + parameterPath: [ + "options", + "xMsLeaseDuration" + ], + mapper: { + serializedName: "x-ms-lease-duration", + type: { + name: "Number" + } + } +}; +export const xMsRangeGetContentMd5: msRest.OperationParameter = { + parameterPath: [ + "options", + "xMsRangeGetContentMd5" + ], + mapper: { + serializedName: "x-ms-range-get-content-md5", + type: { + name: "Boolean" + } + } +}; +export const xMsRequiresSync: msRest.OperationParameter = { + parameterPath: "xMsRequiresSync", + mapper: { + required: true, + isConstant: true, + serializedName: "x-ms-requires-sync", + defaultValue: 'true', + type: { + name: "String" + } + } +}; diff --git a/src/dfs/generated/artifacts/specifications.ts b/src/dfs/generated/artifacts/specifications.ts new file mode 100644 index 000000000..077fa5f3d --- /dev/null +++ b/src/dfs/generated/artifacts/specifications.ts @@ -0,0 +1,3378 @@ +/* + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for + * license information. + * + * Code generated by Microsoft (R) AutoRest Code Generator. + * Changes may cause incorrect behavior and will be lost if the code is + * regenerated. + */ +// tslint:disable:object-literal-sort-keys + +import * as msRest from "@azure/ms-rest-js"; + +import * as Mappers from "./mappers"; +import { Operation } from "./operation"; +import * as Parameters from "./parameters"; + +const serializer = new msRest.Serializer(Mappers, true); +// specifications for new method group start +const serviceListFileSystemsOperationSpec: msRest.OperationSpec = { + httpMethod: "GET", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.resource0, + Parameters.prefix, + Parameters.continuation, + Parameters.maxResults, + Parameters.timeout + ], + headerParameters: [ + Parameters.requestId, + Parameters.version + ], + responses: { + 200: { + bodyMapper: Mappers.FileSystemList, + headersMapper: Mappers.ServiceListFileSystemsHeaders + }, + default: { + bodyMapper: Mappers.StorageError + } + }, + isXML: true, + serializer +}; + +const serviceSetPropertiesOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.restype0, + Parameters.comp0 + ], + headerParameters: [ + Parameters.version, + Parameters.requestId + ], + requestBody: { + parameterPath: "storageServiceProperties", + mapper: { + ...Mappers.StorageServiceProperties, + required: true + } + }, + contentType: "application/xml; charset=utf-8", + responses: { + 202: { + headersMapper: Mappers.ServiceSetPropertiesHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const serviceGetPropertiesOperationSpec: msRest.OperationSpec = { + httpMethod: "GET", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.restype0, + Parameters.comp0 + ], + headerParameters: [ + Parameters.version, + Parameters.requestId + ], + responses: { + 200: { + bodyMapper: Mappers.StorageServiceProperties, + headersMapper: Mappers.ServiceGetPropertiesHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const serviceGetStatisticsOperationSpec: msRest.OperationSpec = { + httpMethod: "GET", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.restype0, + Parameters.comp1 + ], + headerParameters: [ + Parameters.version, + Parameters.requestId + ], + responses: { + 200: { + bodyMapper: Mappers.StorageServiceStats, + headersMapper: Mappers.ServiceGetStatisticsHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const serviceListContainersSegmentOperationSpec: msRest.OperationSpec = { + httpMethod: "GET", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.prefix, + Parameters.marker, + Parameters.maxresults, + Parameters.include0, + Parameters.timeout, + Parameters.comp2 + ], + headerParameters: [ + Parameters.version, + Parameters.requestId + ], + responses: { + 200: { + bodyMapper: Mappers.ListContainersSegmentResponse, + headersMapper: Mappers.ServiceListContainersSegmentHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const serviceGetUserDelegationKeyOperationSpec: msRest.OperationSpec = { + httpMethod: "POST", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.restype0, + Parameters.comp3 + ], + headerParameters: [ + Parameters.version, + Parameters.requestId + ], + requestBody: { + parameterPath: "keyInfo", + mapper: { + ...Mappers.KeyInfo, + required: true + } + }, + contentType: "application/xml; charset=utf-8", + responses: { + 200: { + bodyMapper: Mappers.UserDelegationKey, + headersMapper: Mappers.ServiceGetUserDelegationKeyHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const serviceGetAccountInfoOperationSpec: msRest.OperationSpec = { + httpMethod: "GET", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.restype1, + Parameters.comp0 + ], + headerParameters: [ + Parameters.version + ], + responses: { + 200: { + headersMapper: Mappers.ServiceGetAccountInfoHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const serviceGetAccountInfoWithHeadOperationSpec: msRest.OperationSpec = { + httpMethod: "HEAD", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.restype1, + Parameters.comp0 + ], + headerParameters: [ + Parameters.version + ], + responses: { + 200: { + headersMapper: Mappers.ServiceGetAccountInfoWithHeadHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const serviceSubmitBatchOperationSpec: msRest.OperationSpec = { + httpMethod: "POST", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.comp4 + ], + headerParameters: [ + Parameters.contentLength0, + Parameters.multipartContentType, + Parameters.version, + Parameters.requestId + ], + requestBody: { + parameterPath: "body", + mapper: { + required: true, + serializedName: "body", + type: { + name: "Stream" + } + } + }, + contentType: "application/xml; charset=utf-8", + responses: { + 202: { + bodyMapper: { + serializedName: "Stream", + type: { + name: "Stream" + } + }, + headersMapper: Mappers.ServiceSubmitBatchHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const serviceFilterBlobsOperationSpec: msRest.OperationSpec = { + httpMethod: "GET", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.where, + Parameters.marker, + Parameters.maxresults, + Parameters.include1, + Parameters.comp5 + ], + headerParameters: [ + Parameters.version, + Parameters.requestId + ], + responses: { + 200: { + bodyMapper: Mappers.FilterBlobSegment, + headersMapper: Mappers.ServiceFilterBlobsHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +// specifications for new method group start +const fileSystemCreateOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{filesystem}", + urlParameters: [ + Parameters.url, + Parameters.fileSystem + ], + queryParameters: [ + Parameters.resource1, + Parameters.timeout + ], + headerParameters: [ + Parameters.properties, + Parameters.requestId, + Parameters.version + ], + responses: { + 201: { + headersMapper: Mappers.FileSystemCreateHeaders + }, + default: { + bodyMapper: Mappers.StorageError + } + }, + isXML: true, + serializer +}; + +const fileSystemSetPropertiesOperationSpec: msRest.OperationSpec = { + httpMethod: "PATCH", + path: "{filesystem}", + urlParameters: [ + Parameters.url, + Parameters.fileSystem + ], + queryParameters: [ + Parameters.resource1, + Parameters.timeout + ], + headerParameters: [ + Parameters.properties, + Parameters.requestId, + Parameters.version, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince + ], + responses: { + 200: { + headersMapper: Mappers.FileSystemSetPropertiesHeaders + }, + default: { + bodyMapper: Mappers.StorageError + } + }, + isXML: true, + serializer +}; + +const fileSystemGetPropertiesOperationSpec: msRest.OperationSpec = { + httpMethod: "HEAD", + path: "{filesystem}", + urlParameters: [ + Parameters.url, + Parameters.fileSystem + ], + queryParameters: [ + Parameters.resource1, + Parameters.timeout + ], + headerParameters: [ + Parameters.requestId, + Parameters.version + ], + responses: { + 200: { + headersMapper: Mappers.FileSystemGetPropertiesHeaders + }, + default: { + bodyMapper: Mappers.StorageError + } + }, + isXML: true, + serializer +}; + +const fileSystemDeleteOperationSpec: msRest.OperationSpec = { + httpMethod: "DELETE", + path: "{filesystem}", + urlParameters: [ + Parameters.url, + Parameters.fileSystem + ], + queryParameters: [ + Parameters.resource1, + Parameters.timeout + ], + headerParameters: [ + Parameters.requestId, + Parameters.version, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince + ], + responses: { + 202: { + headersMapper: Mappers.FileSystemDeleteHeaders + }, + default: { + bodyMapper: Mappers.StorageError + } + }, + isXML: true, + serializer +}; + +const fileSystemListPathsOperationSpec: msRest.OperationSpec = { + httpMethod: "GET", + path: "{filesystem}", + urlParameters: [ + Parameters.url, + Parameters.fileSystem + ], + queryParameters: [ + Parameters.continuation, + Parameters.path0, + Parameters.recursive0, + Parameters.maxResults, + Parameters.upn, + Parameters.resource1, + Parameters.timeout + ], + headerParameters: [ + Parameters.requestId, + Parameters.version + ], + responses: { + 200: { + bodyMapper: Mappers.PathList, + headersMapper: Mappers.FileSystemListPathsHeaders + }, + default: { + bodyMapper: Mappers.StorageError + } + }, + isXML: false, + serializer +}; + +const fileSystemListBlobFlatSegmentOperationSpec: msRest.OperationSpec = { + httpMethod: "GET", + path: "{filesystem}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.prefix, + Parameters.marker, + Parameters.maxResults, + Parameters.maxresults, + Parameters.include2, + Parameters.timeout, + Parameters.restype2, + Parameters.comp2 + ], + headerParameters: [ + Parameters.version, + Parameters.requestId + ], + responses: { + 200: { + bodyMapper: Mappers.ListBlobsFlatSegmentResponse, + headersMapper: Mappers.FileSystemListBlobFlatSegmentHeaders + }, + default: { + bodyMapper: Mappers.StorageError + } + }, + isXML: true, + serializer +}; + +const fileSystemListBlobHierarchySegmentOperationSpec: msRest.OperationSpec = { + httpMethod: "GET", + path: "{filesystem}", + urlParameters: [ + Parameters.url, + Parameters.fileSystem + ], + queryParameters: [ + Parameters.prefix, + Parameters.delimiter, + Parameters.marker, + Parameters.maxResults, + Parameters.maxresults, + Parameters.include2, + Parameters.showonly, + Parameters.timeout, + Parameters.restype2, + Parameters.comp2 + ], + headerParameters: [ + Parameters.version, + Parameters.requestId + ], + responses: { + 200: { + bodyMapper: Mappers.ListBlobsHierarchySegmentResponse, + headersMapper: Mappers.FileSystemListBlobHierarchySegmentHeaders + }, + default: { + bodyMapper: Mappers.StorageError + } + }, + isXML: true, + serializer +}; + +// specifications for new method group start +const pathCreateOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{filesystem}/{path}", + urlParameters: [ + Parameters.url, + Parameters.fileSystem, + Parameters.path1 + ], + queryParameters: [ + Parameters.resource2, + Parameters.continuation, + Parameters.mode0, + Parameters.timeout + ], + headerParameters: [ + Parameters.renameSource, + Parameters.sourceLeaseId, + Parameters.properties, + Parameters.permissions, + Parameters.umask, + Parameters.owner, + Parameters.group, + Parameters.acl, + Parameters.proposedLeaseId0, + Parameters.leaseDuration, + Parameters.expiryOptions0, + Parameters.expiresOn, + Parameters.requestId, + Parameters.version, + Parameters.metadata, + Parameters.cacheControl, + Parameters.contentEncoding, + Parameters.contentLanguage, + Parameters.contentDisposition, + Parameters.contentType, + Parameters.leaseId0, + Parameters.ifMatch, + Parameters.ifNoneMatch, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince, + Parameters.sourceIfMatch, + Parameters.sourceIfNoneMatch, + Parameters.sourceIfModifiedSince, + Parameters.sourceIfUnmodifiedSince, + Parameters.encryptionKey, + Parameters.encryptionKeySha256, + Parameters.encryptionAlgorithm + ], + responses: { + 201: { + headersMapper: Mappers.PathCreateHeaders + }, + default: { + bodyMapper: Mappers.StorageError + } + }, + isXML: true, + serializer +}; + +const pathUpdateOperationSpec: msRest.OperationSpec = { + httpMethod: "PATCH", + path: "{filesystem}/{path}", + urlParameters: [ + Parameters.url, + Parameters.fileSystem, + Parameters.path1 + ], + queryParameters: [ + Parameters.action0, + Parameters.flush, + Parameters.maxRecords, + Parameters.continuation, + Parameters.mode1, + Parameters.forceFlag, + Parameters.position, + Parameters.retainUncommittedData, + Parameters.close, + Parameters.timeout + ], + headerParameters: [ + Parameters.contentLength1, + Parameters.properties, + Parameters.owner, + Parameters.group, + Parameters.permissions, + Parameters.acl, + Parameters.requestId, + Parameters.version, + Parameters.metadata, + Parameters.contentMD5, + Parameters.cacheControl, + Parameters.contentType, + Parameters.contentDisposition, + Parameters.contentEncoding, + Parameters.contentLanguage, + Parameters.leaseId0, + Parameters.ifMatch, + Parameters.ifNoneMatch, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince + ], + requestBody: { + parameterPath: "body", + mapper: { + required: true, + serializedName: "body", + type: { + name: "Stream" + } + } + }, + contentType: "application/octet-stream", + responses: { + 200: { + bodyMapper: Mappers.SetAccessControlRecursiveResponse, + headersMapper: Mappers.PathUpdateHeaders + }, + 202: { + headersMapper: Mappers.PathUpdateHeaders + }, + default: { + bodyMapper: Mappers.StorageError + } + }, + isXML: true, + serializer +}; + +const pathLeaseOperationSpec: msRest.OperationSpec = { + httpMethod: "POST", + path: "{filesystem}/{path}", + urlParameters: [ + Parameters.url, + Parameters.fileSystem, + Parameters.path1 + ], + queryParameters: [ + Parameters.timeout + ], + headerParameters: [ + Parameters.xMsLeaseAction, + Parameters.xMsLeaseDuration, + Parameters.xMsLeaseBreakPeriod, + Parameters.proposedLeaseId0, + Parameters.requestId, + Parameters.version, + Parameters.metadata, + Parameters.leaseId0, + Parameters.ifMatch, + Parameters.ifNoneMatch, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince + ], + responses: { + 200: { + headersMapper: Mappers.PathLeaseHeaders + }, + 201: { + headersMapper: Mappers.PathLeaseHeaders + }, + 202: { + headersMapper: Mappers.PathLeaseHeaders + }, + default: { + bodyMapper: Mappers.StorageError + } + }, + isXML: true, + serializer +}; + +const pathReadOperationSpec: msRest.OperationSpec = { + httpMethod: "GET", + path: "{filesystem}/{path}", + urlParameters: [ + Parameters.url, + Parameters.fileSystem, + Parameters.path1 + ], + queryParameters: [ + Parameters.snapshot, + Parameters.versionId, + Parameters.timeout + ], + headerParameters: [ + Parameters.xMsRangeGetContentMd5, + Parameters.range0, + Parameters.rangeGetContentMD5, + Parameters.rangeGetContentCRC64, + Parameters.version, + Parameters.requestId, + Parameters.metadata, + Parameters.leaseId0, + Parameters.ifMatch, + Parameters.ifNoneMatch, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince, + Parameters.ifTags, + Parameters.encryptionKey, + Parameters.encryptionKeySha256, + Parameters.encryptionAlgorithm + ], + responses: { + 200: { + bodyMapper: { + serializedName: "Stream", + type: { + name: "Stream" + } + }, + headersMapper: Mappers.PathReadHeaders + }, + 206: { + bodyMapper: { + serializedName: "Stream", + type: { + name: "Stream" + } + }, + headersMapper: Mappers.PathReadHeaders + }, + default: { + bodyMapper: Mappers.StorageError + } + }, + isXML: true, + serializer +}; + +const pathGetPropertiesOperationSpec: msRest.OperationSpec = { + httpMethod: "HEAD", + path: "{filesystem}/{path}", + urlParameters: [ + Parameters.url, + Parameters.fileSystem, + Parameters.path1 + ], + queryParameters: [ + Parameters.action1, + Parameters.upn, + Parameters.snapshot, + Parameters.versionId, + Parameters.timeout + ], + headerParameters: [ + Parameters.version, + Parameters.requestId, + Parameters.metadata, + Parameters.leaseId0, + Parameters.ifMatch, + Parameters.ifNoneMatch, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince, + Parameters.ifTags, + Parameters.encryptionKey, + Parameters.encryptionKeySha256, + Parameters.encryptionAlgorithm + ], + responses: { + 200: { + headersMapper: Mappers.PathGetPropertiesHeaders + }, + default: { + bodyMapper: Mappers.StorageError + } + }, + isXML: true, + serializer +}; + +const pathDeleteOperationSpec: msRest.OperationSpec = { + httpMethod: "DELETE", + path: "{filesystem}/{path}", + urlParameters: [ + Parameters.url, + Parameters.fileSystem, + Parameters.path1 + ], + queryParameters: [ + Parameters.recursive1, + Parameters.continuation, + Parameters.snapshot, + Parameters.versionId, + Parameters.timeout, + Parameters.blobDeleteType + ], + headerParameters: [ + Parameters.deleteSnapshots, + Parameters.version, + Parameters.requestId, + Parameters.metadata, + Parameters.leaseId0, + Parameters.ifMatch, + Parameters.ifNoneMatch, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince, + Parameters.ifTags + ], + responses: { + 200: { + headersMapper: Mappers.PathDeleteHeaders + }, + 202: { + headersMapper: Mappers.PathDeleteHeaders + }, + default: { + bodyMapper: Mappers.StorageError + } + }, + isXML: true, + serializer +}; + +const pathSetAccessControlOperationSpec: msRest.OperationSpec = { + httpMethod: "PATCH", + path: "{filesystem}/{path}", + urlParameters: [ + Parameters.url, + Parameters.fileSystem, + Parameters.path1 + ], + queryParameters: [ + Parameters.timeout, + Parameters.action2 + ], + headerParameters: [ + Parameters.owner, + Parameters.group, + Parameters.permissions, + Parameters.acl, + Parameters.requestId, + Parameters.version, + Parameters.leaseId0, + Parameters.ifMatch, + Parameters.ifNoneMatch, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince + ], + responses: { + 200: { + headersMapper: Mappers.PathSetAccessControlHeaders + }, + default: { + bodyMapper: Mappers.StorageError + } + }, + isXML: true, + serializer +}; + +const pathSetAccessControlRecursiveOperationSpec: msRest.OperationSpec = { + httpMethod: "PATCH", + path: "{filesystem}/{path}", + urlParameters: [ + Parameters.url, + Parameters.fileSystem, + Parameters.path1 + ], + queryParameters: [ + Parameters.timeout, + Parameters.continuation, + Parameters.mode1, + Parameters.forceFlag, + Parameters.maxRecords, + Parameters.action3 + ], + headerParameters: [ + Parameters.acl, + Parameters.requestId, + Parameters.version + ], + responses: { + 200: { + bodyMapper: Mappers.SetAccessControlRecursiveResponse, + headersMapper: Mappers.PathSetAccessControlRecursiveHeaders + }, + default: { + bodyMapper: Mappers.StorageError + } + }, + isXML: true, + serializer +}; + +const pathSetPropertiesOperationSpec: msRest.OperationSpec = { + httpMethod: "PATCH", + path: "{filesystem}/{path}", + urlParameters: [ + Parameters.url, + Parameters.fileSystem, + Parameters.path1 + ], + queryParameters: [ + Parameters.action4 + ], + headerParameters: [ + Parameters.properties, + Parameters.permissions, + Parameters.requestId, + Parameters.version, + Parameters.leaseId0, + Parameters.cacheControl, + Parameters.contentType, + Parameters.contentDisposition, + Parameters.contentEncoding, + Parameters.contentLanguage, + Parameters.contentMD5, + Parameters.ifMatch, + Parameters.ifNoneMatch, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince + ], + responses: { + 200: { + headersMapper: Mappers.PathSetPropertiesHeaders + }, + default: { + bodyMapper: Mappers.StorageError + } + }, + isXML: true, + serializer +}; + +const pathFlushDataOperationSpec: msRest.OperationSpec = { + httpMethod: "PATCH", + path: "{filesystem}/{path}", + urlParameters: [ + Parameters.url, + Parameters.fileSystem, + Parameters.path1 + ], + queryParameters: [ + Parameters.timeout, + Parameters.position, + Parameters.retainUncommittedData, + Parameters.close, + Parameters.action5 + ], + headerParameters: [ + Parameters.contentLength1, + Parameters.requestId, + Parameters.version, + Parameters.proposedLeaseId0, + Parameters.xMsLeaseDuration, + Parameters.leaseAction, + Parameters.contentMD5, + Parameters.cacheControl, + Parameters.contentType, + Parameters.contentDisposition, + Parameters.contentEncoding, + Parameters.contentLanguage, + Parameters.leaseId0, + Parameters.ifMatch, + Parameters.ifNoneMatch, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince, + Parameters.encryptionKey, + Parameters.encryptionKeySha256, + Parameters.encryptionAlgorithm + ], + responses: { + 200: { + headersMapper: Mappers.PathFlushDataHeaders + }, + default: { + bodyMapper: Mappers.StorageError + } + }, + isXML: true, + serializer +}; + +const pathAppendDataOperationSpec: msRest.OperationSpec = { + httpMethod: "PATCH", + path: "{filesystem}/{path}", + urlParameters: [ + Parameters.url, + Parameters.fileSystem, + Parameters.path1 + ], + queryParameters: [ + Parameters.position, + Parameters.timeout, + Parameters.flush, + Parameters.action6 + ], + headerParameters: [ + Parameters.contentLength1, + Parameters.transactionalContentCrc64, + Parameters.requestId, + Parameters.version, + Parameters.proposedLeaseId0, + Parameters.xMsLeaseDuration, + Parameters.leaseAction, + Parameters.transactionalContentHash, + Parameters.leaseId0, + Parameters.encryptionKey, + Parameters.encryptionKeySha256, + Parameters.encryptionAlgorithm, + Parameters.ifMatch, + Parameters.ifNoneMatch, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince + ], + requestBody: { + parameterPath: "body", + mapper: { + required: true, + serializedName: "body", + type: { + name: "Stream" + } + } + }, + responses: { + 202: { + headersMapper: Mappers.PathAppendDataHeaders + }, + default: { + bodyMapper: Mappers.StorageError + } + }, + isXML: true, + serializer +}; + +const pathSetExpiryOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{filesystem}/{path}", + urlParameters: [ + Parameters.url, + Parameters.fileSystem, + Parameters.path1 + ], + queryParameters: [ + Parameters.timeout, + Parameters.comp6 + ], + headerParameters: [ + Parameters.version, + Parameters.requestId, + Parameters.expiryOptions1, + Parameters.expiresOn + ], + responses: { + 200: { + headersMapper: Mappers.PathSetExpiryHeaders + }, + default: { + bodyMapper: Mappers.StorageError + } + }, + isXML: true, + serializer +}; + +const pathUndeleteOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{filesystem}/{path}", + urlParameters: [ + Parameters.url, + Parameters.fileSystem, + Parameters.path1 + ], + queryParameters: [ + Parameters.timeout, + Parameters.comp7 + ], + headerParameters: [ + Parameters.undeleteSource, + Parameters.version, + Parameters.requestId + ], + responses: { + 200: { + headersMapper: Mappers.PathUndeleteHeaders + }, + default: { + bodyMapper: Mappers.StorageError + } + }, + isXML: true, + serializer +}; + +// specifications for new method group start +const containerCreateOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{containerName}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.restype2 + ], + headerParameters: [ + Parameters.metadata, + Parameters.access, + Parameters.version, + Parameters.requestId, + Parameters.defaultEncryptionScope, + Parameters.preventEncryptionScopeOverride + ], + responses: { + 201: { + headersMapper: Mappers.ContainerCreateHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const containerGetPropertiesOperationSpec: msRest.OperationSpec = { + httpMethod: "GET", + path: "{containerName}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.restype2 + ], + headerParameters: [ + Parameters.version, + Parameters.requestId, + Parameters.leaseId0 + ], + responses: { + 200: { + headersMapper: Mappers.ContainerGetPropertiesHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const containerGetPropertiesWithHeadOperationSpec: msRest.OperationSpec = { + httpMethod: "HEAD", + path: "{containerName}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.restype2 + ], + headerParameters: [ + Parameters.version, + Parameters.requestId, + Parameters.leaseId0 + ], + responses: { + 200: { + headersMapper: Mappers.ContainerGetPropertiesWithHeadHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const containerDeleteOperationSpec: msRest.OperationSpec = { + httpMethod: "DELETE", + path: "{containerName}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.restype2 + ], + headerParameters: [ + Parameters.version, + Parameters.requestId, + Parameters.leaseId0, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince + ], + responses: { + 202: { + headersMapper: Mappers.ContainerDeleteHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const containerSetMetadataOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{containerName}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.restype2, + Parameters.comp8 + ], + headerParameters: [ + Parameters.metadata, + Parameters.version, + Parameters.requestId, + Parameters.leaseId0, + Parameters.ifModifiedSince + ], + responses: { + 200: { + headersMapper: Mappers.ContainerSetMetadataHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const containerGetAccessPolicyOperationSpec: msRest.OperationSpec = { + httpMethod: "GET", + path: "{containerName}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.restype2, + Parameters.comp9 + ], + headerParameters: [ + Parameters.version, + Parameters.requestId, + Parameters.leaseId0 + ], + responses: { + 200: { + bodyMapper: { + xmlElementName: "SignedIdentifier", + serializedName: "SignedIdentifiers", + type: { + name: "Sequence", + element: { + type: { + name: "Composite", + className: "SignedIdentifier" + } + } + } + }, + headersMapper: Mappers.ContainerGetAccessPolicyHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const containerSetAccessPolicyOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{containerName}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.restype2, + Parameters.comp9 + ], + headerParameters: [ + Parameters.access, + Parameters.version, + Parameters.requestId, + Parameters.leaseId0, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince + ], + requestBody: { + parameterPath: [ + "options", + "containerAcl" + ], + mapper: { + xmlName: "SignedIdentifiers", + xmlElementName: "SignedIdentifier", + serializedName: "containerAcl", + type: { + name: "Sequence", + element: { + type: { + name: "Composite", + className: "SignedIdentifier" + } + } + } + } + }, + contentType: "application/xml; charset=utf-8", + responses: { + 200: { + headersMapper: Mappers.ContainerSetAccessPolicyHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const containerRestoreOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{containerName}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.restype2, + Parameters.comp7 + ], + headerParameters: [ + Parameters.version, + Parameters.requestId, + Parameters.deletedContainerName, + Parameters.deletedContainerVersion + ], + responses: { + 201: { + headersMapper: Mappers.ContainerRestoreHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const containerSubmitBatchOperationSpec: msRest.OperationSpec = { + httpMethod: "POST", + path: "{containerName}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.restype2, + Parameters.comp4 + ], + headerParameters: [ + Parameters.contentLength0, + Parameters.multipartContentType, + Parameters.version, + Parameters.requestId + ], + requestBody: { + parameterPath: "body", + mapper: { + required: true, + serializedName: "body", + type: { + name: "Stream" + } + } + }, + contentType: "application/xml; charset=utf-8", + responses: { + 202: { + bodyMapper: { + serializedName: "Stream", + type: { + name: "Stream" + } + }, + headersMapper: Mappers.ContainerSubmitBatchHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const containerFilterBlobsOperationSpec: msRest.OperationSpec = { + httpMethod: "GET", + path: "{containerName}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.where, + Parameters.marker, + Parameters.maxresults, + Parameters.include1, + Parameters.restype2, + Parameters.comp5 + ], + headerParameters: [ + Parameters.version, + Parameters.requestId + ], + responses: { + 200: { + bodyMapper: Mappers.FilterBlobSegment, + headersMapper: Mappers.ContainerFilterBlobsHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const containerAcquireLeaseOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{containerName}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.comp10, + Parameters.restype2 + ], + headerParameters: [ + Parameters.duration, + Parameters.proposedLeaseId0, + Parameters.version, + Parameters.requestId, + Parameters.action7, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince + ], + responses: { + 201: { + headersMapper: Mappers.ContainerAcquireLeaseHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const containerReleaseLeaseOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{containerName}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.comp10, + Parameters.restype2 + ], + headerParameters: [ + Parameters.leaseId1, + Parameters.version, + Parameters.requestId, + Parameters.action8, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince + ], + responses: { + 200: { + headersMapper: Mappers.ContainerReleaseLeaseHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const containerRenewLeaseOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{containerName}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.comp10, + Parameters.restype2 + ], + headerParameters: [ + Parameters.leaseId1, + Parameters.version, + Parameters.requestId, + Parameters.action9, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince + ], + responses: { + 200: { + headersMapper: Mappers.ContainerRenewLeaseHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const containerBreakLeaseOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{containerName}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.comp10, + Parameters.restype2 + ], + headerParameters: [ + Parameters.breakPeriod, + Parameters.version, + Parameters.requestId, + Parameters.action10, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince + ], + responses: { + 202: { + headersMapper: Mappers.ContainerBreakLeaseHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const containerChangeLeaseOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{containerName}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.comp10, + Parameters.restype2 + ], + headerParameters: [ + Parameters.leaseId1, + Parameters.proposedLeaseId1, + Parameters.version, + Parameters.requestId, + Parameters.action11, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince + ], + responses: { + 200: { + headersMapper: Mappers.ContainerChangeLeaseHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const containerGetAccountInfoOperationSpec: msRest.OperationSpec = { + httpMethod: "GET", + path: "{containerName}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.restype1, + Parameters.comp0 + ], + headerParameters: [ + Parameters.version + ], + responses: { + 200: { + headersMapper: Mappers.ContainerGetAccountInfoHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const containerGetAccountInfoWithHeadOperationSpec: msRest.OperationSpec = { + httpMethod: "HEAD", + path: "{containerName}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.restype1, + Parameters.comp0 + ], + headerParameters: [ + Parameters.version + ], + responses: { + 200: { + headersMapper: Mappers.ContainerGetAccountInfoWithHeadHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +// specifications for new method group start +const pageBlobCreateOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{containerName}/{blob}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout + ], + headerParameters: [ + Parameters.contentLength0, + Parameters.tier0, + Parameters.metadata, + Parameters.blobContentLength, + Parameters.blobSequenceNumber, + Parameters.version, + Parameters.requestId, + Parameters.blobTagsString, + Parameters.immutabilityPolicyExpiry, + Parameters.immutabilityPolicyMode, + Parameters.legalHold0, + Parameters.blobType0, + Parameters.blobContentType, + Parameters.blobContentEncoding, + Parameters.blobContentLanguage, + Parameters.blobContentMD5, + Parameters.blobCacheControl, + Parameters.blobContentDisposition, + Parameters.leaseId0, + Parameters.encryptionKey, + Parameters.encryptionKeySha256, + Parameters.encryptionAlgorithm, + Parameters.encryptionScope, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince, + Parameters.ifMatch, + Parameters.ifNoneMatch, + Parameters.ifTags + ], + responses: { + 201: { + headersMapper: Mappers.PageBlobCreateHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const pageBlobUploadPagesOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{containerName}/{blob}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.comp11 + ], + headerParameters: [ + Parameters.contentLength0, + Parameters.transactionalContentMD5, + Parameters.transactionalContentCrc64, + Parameters.range0, + Parameters.version, + Parameters.requestId, + Parameters.pageWrite0, + Parameters.leaseId0, + Parameters.encryptionKey, + Parameters.encryptionKeySha256, + Parameters.encryptionAlgorithm, + Parameters.encryptionScope, + Parameters.ifSequenceNumberLessThanOrEqualTo, + Parameters.ifSequenceNumberLessThan, + Parameters.ifSequenceNumberEqualTo, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince, + Parameters.ifMatch, + Parameters.ifNoneMatch, + Parameters.ifTags + ], + requestBody: { + parameterPath: "body", + mapper: { + required: true, + serializedName: "body", + type: { + name: "Stream" + } + } + }, + contentType: "application/octet-stream", + responses: { + 201: { + headersMapper: Mappers.PageBlobUploadPagesHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const pageBlobClearPagesOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{containerName}/{blob}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.comp11 + ], + headerParameters: [ + Parameters.contentLength0, + Parameters.range0, + Parameters.version, + Parameters.requestId, + Parameters.pageWrite1, + Parameters.leaseId0, + Parameters.encryptionKey, + Parameters.encryptionKeySha256, + Parameters.encryptionAlgorithm, + Parameters.encryptionScope, + Parameters.ifSequenceNumberLessThanOrEqualTo, + Parameters.ifSequenceNumberLessThan, + Parameters.ifSequenceNumberEqualTo, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince, + Parameters.ifMatch, + Parameters.ifNoneMatch, + Parameters.ifTags + ], + responses: { + 201: { + headersMapper: Mappers.PageBlobClearPagesHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const pageBlobUploadPagesFromURLOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{containerName}/{blob}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.comp11 + ], + headerParameters: [ + Parameters.sourceUrl, + Parameters.sourceRange0, + Parameters.sourceContentMD5, + Parameters.sourceContentcrc64, + Parameters.contentLength0, + Parameters.range1, + Parameters.version, + Parameters.requestId, + Parameters.copySourceAuthorization, + Parameters.pageWrite0, + Parameters.encryptionKey, + Parameters.encryptionKeySha256, + Parameters.encryptionAlgorithm, + Parameters.encryptionScope, + Parameters.leaseId0, + Parameters.ifSequenceNumberLessThanOrEqualTo, + Parameters.ifSequenceNumberLessThan, + Parameters.ifSequenceNumberEqualTo, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince, + Parameters.ifMatch, + Parameters.ifNoneMatch, + Parameters.ifTags, + Parameters.sourceIfModifiedSince, + Parameters.sourceIfUnmodifiedSince, + Parameters.sourceIfMatch, + Parameters.sourceIfNoneMatch + ], + responses: { + 201: { + headersMapper: Mappers.PageBlobUploadPagesFromURLHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const pageBlobGetPageRangesOperationSpec: msRest.OperationSpec = { + httpMethod: "GET", + path: "{containerName}/{blob}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.snapshot, + Parameters.timeout, + Parameters.marker, + Parameters.maxresults, + Parameters.comp12 + ], + headerParameters: [ + Parameters.range0, + Parameters.version, + Parameters.requestId, + Parameters.leaseId0, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince, + Parameters.ifMatch, + Parameters.ifNoneMatch, + Parameters.ifTags + ], + responses: { + 200: { + bodyMapper: Mappers.PageList, + headersMapper: Mappers.PageBlobGetPageRangesHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const pageBlobGetPageRangesDiffOperationSpec: msRest.OperationSpec = { + httpMethod: "GET", + path: "{containerName}/{blob}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.snapshot, + Parameters.timeout, + Parameters.prevsnapshot, + Parameters.marker, + Parameters.maxresults, + Parameters.comp12 + ], + headerParameters: [ + Parameters.prevSnapshotUrl, + Parameters.range0, + Parameters.version, + Parameters.requestId, + Parameters.leaseId0, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince, + Parameters.ifMatch, + Parameters.ifNoneMatch, + Parameters.ifTags + ], + responses: { + 200: { + bodyMapper: Mappers.PageList, + headersMapper: Mappers.PageBlobGetPageRangesDiffHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const pageBlobResizeOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{containerName}/{blob}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.comp0 + ], + headerParameters: [ + Parameters.blobContentLength, + Parameters.version, + Parameters.requestId, + Parameters.leaseId0, + Parameters.encryptionKey, + Parameters.encryptionKeySha256, + Parameters.encryptionAlgorithm, + Parameters.encryptionScope, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince, + Parameters.ifMatch, + Parameters.ifNoneMatch, + Parameters.ifTags + ], + responses: { + 200: { + headersMapper: Mappers.PageBlobResizeHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const pageBlobUpdateSequenceNumberOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{containerName}/{blob}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.comp0 + ], + headerParameters: [ + Parameters.sequenceNumberAction, + Parameters.blobSequenceNumber, + Parameters.version, + Parameters.requestId, + Parameters.leaseId0, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince, + Parameters.ifMatch, + Parameters.ifNoneMatch, + Parameters.ifTags + ], + responses: { + 200: { + headersMapper: Mappers.PageBlobUpdateSequenceNumberHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const pageBlobCopyIncrementalOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{containerName}/{blob}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.comp13 + ], + headerParameters: [ + Parameters.copySource, + Parameters.version, + Parameters.requestId, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince, + Parameters.ifMatch, + Parameters.ifNoneMatch, + Parameters.ifTags + ], + responses: { + 202: { + headersMapper: Mappers.PageBlobCopyIncrementalHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +// specifications for new method group start +const appendBlobCreateOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{containerName}/{blob}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout + ], + headerParameters: [ + Parameters.contentLength0, + Parameters.metadata, + Parameters.version, + Parameters.requestId, + Parameters.blobTagsString, + Parameters.immutabilityPolicyExpiry, + Parameters.immutabilityPolicyMode, + Parameters.legalHold0, + Parameters.blobType1, + Parameters.blobContentType, + Parameters.blobContentEncoding, + Parameters.blobContentLanguage, + Parameters.blobContentMD5, + Parameters.blobCacheControl, + Parameters.blobContentDisposition, + Parameters.leaseId0, + Parameters.encryptionKey, + Parameters.encryptionKeySha256, + Parameters.encryptionAlgorithm, + Parameters.encryptionScope, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince, + Parameters.ifMatch, + Parameters.ifNoneMatch, + Parameters.ifTags + ], + responses: { + 201: { + headersMapper: Mappers.AppendBlobCreateHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const appendBlobAppendBlockOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{containerName}/{blob}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.comp14 + ], + headerParameters: [ + Parameters.contentLength0, + Parameters.transactionalContentMD5, + Parameters.transactionalContentCrc64, + Parameters.version, + Parameters.requestId, + Parameters.leaseId0, + Parameters.maxSize, + Parameters.appendPosition, + Parameters.encryptionKey, + Parameters.encryptionKeySha256, + Parameters.encryptionAlgorithm, + Parameters.encryptionScope, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince, + Parameters.ifMatch, + Parameters.ifNoneMatch, + Parameters.ifTags + ], + requestBody: { + parameterPath: "body", + mapper: { + required: true, + serializedName: "body", + type: { + name: "Stream" + } + } + }, + contentType: "application/octet-stream", + responses: { + 201: { + headersMapper: Mappers.AppendBlobAppendBlockHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const appendBlobAppendBlockFromUrlOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{containerName}/{blob}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.comp14 + ], + headerParameters: [ + Parameters.sourceUrl, + Parameters.sourceRange1, + Parameters.sourceContentMD5, + Parameters.sourceContentcrc64, + Parameters.contentLength0, + Parameters.transactionalContentMD5, + Parameters.version, + Parameters.requestId, + Parameters.copySourceAuthorization, + Parameters.encryptionKey, + Parameters.encryptionKeySha256, + Parameters.encryptionAlgorithm, + Parameters.encryptionScope, + Parameters.leaseId0, + Parameters.maxSize, + Parameters.appendPosition, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince, + Parameters.ifMatch, + Parameters.ifNoneMatch, + Parameters.ifTags, + Parameters.sourceIfModifiedSince, + Parameters.sourceIfUnmodifiedSince, + Parameters.sourceIfMatch, + Parameters.sourceIfNoneMatch + ], + responses: { + 201: { + headersMapper: Mappers.AppendBlobAppendBlockFromUrlHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const appendBlobSealOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{containerName}/{blob}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.comp15 + ], + headerParameters: [ + Parameters.version, + Parameters.requestId, + Parameters.leaseId0, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince, + Parameters.ifMatch, + Parameters.ifNoneMatch, + Parameters.appendPosition + ], + responses: { + 200: { + headersMapper: Mappers.AppendBlobSealHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +// specifications for new method group start +const blockBlobUploadOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{containerName}/{blob}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout + ], + headerParameters: [ + Parameters.transactionalContentMD5, + Parameters.contentLength0, + Parameters.metadata, + Parameters.tier0, + Parameters.version, + Parameters.requestId, + Parameters.blobTagsString, + Parameters.immutabilityPolicyExpiry, + Parameters.immutabilityPolicyMode, + Parameters.legalHold0, + Parameters.transactionalContentCrc64, + Parameters.blobType2, + Parameters.blobContentType, + Parameters.blobContentEncoding, + Parameters.blobContentLanguage, + Parameters.blobContentMD5, + Parameters.blobCacheControl, + Parameters.blobContentDisposition, + Parameters.leaseId0, + Parameters.encryptionKey, + Parameters.encryptionKeySha256, + Parameters.encryptionAlgorithm, + Parameters.encryptionScope, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince, + Parameters.ifMatch, + Parameters.ifNoneMatch, + Parameters.ifTags + ], + requestBody: { + parameterPath: "body", + mapper: { + required: true, + serializedName: "body", + type: { + name: "Stream" + } + } + }, + contentType: "application/octet-stream", + responses: { + 201: { + headersMapper: Mappers.BlockBlobUploadHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const blockBlobPutBlobFromUrlOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{containerName}/{blob}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout + ], + headerParameters: [ + Parameters.transactionalContentMD5, + Parameters.contentLength0, + Parameters.metadata, + Parameters.tier0, + Parameters.version, + Parameters.requestId, + Parameters.sourceContentMD5, + Parameters.blobTagsString, + Parameters.copySource, + Parameters.copySourceBlobProperties, + Parameters.copySourceAuthorization, + Parameters.copySourceTags, + Parameters.blobType2, + Parameters.blobContentType, + Parameters.blobContentEncoding, + Parameters.blobContentLanguage, + Parameters.blobContentMD5, + Parameters.blobCacheControl, + Parameters.blobContentDisposition, + Parameters.leaseId0, + Parameters.encryptionKey, + Parameters.encryptionKeySha256, + Parameters.encryptionAlgorithm, + Parameters.encryptionScope, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince, + Parameters.ifMatch, + Parameters.ifNoneMatch, + Parameters.ifTags, + Parameters.sourceIfModifiedSince, + Parameters.sourceIfUnmodifiedSince, + Parameters.sourceIfMatch, + Parameters.sourceIfNoneMatch, + Parameters.sourceIfTags + ], + responses: { + 201: { + headersMapper: Mappers.BlockBlobPutBlobFromUrlHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const blockBlobStageBlockOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{containerName}/{blob}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.blockId, + Parameters.timeout, + Parameters.comp16 + ], + headerParameters: [ + Parameters.contentLength0, + Parameters.transactionalContentMD5, + Parameters.transactionalContentCrc64, + Parameters.version, + Parameters.requestId, + Parameters.leaseId0, + Parameters.encryptionKey, + Parameters.encryptionKeySha256, + Parameters.encryptionAlgorithm, + Parameters.encryptionScope + ], + requestBody: { + parameterPath: "body", + mapper: { + required: true, + serializedName: "body", + type: { + name: "Stream" + } + } + }, + contentType: "application/octet-stream", + responses: { + 201: { + headersMapper: Mappers.BlockBlobStageBlockHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const blockBlobStageBlockFromURLOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{containerName}/{blob}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.blockId, + Parameters.timeout, + Parameters.comp16 + ], + headerParameters: [ + Parameters.contentLength0, + Parameters.sourceUrl, + Parameters.sourceRange1, + Parameters.sourceContentMD5, + Parameters.sourceContentcrc64, + Parameters.version, + Parameters.requestId, + Parameters.copySourceAuthorization, + Parameters.encryptionKey, + Parameters.encryptionKeySha256, + Parameters.encryptionAlgorithm, + Parameters.encryptionScope, + Parameters.leaseId0, + Parameters.sourceIfModifiedSince, + Parameters.sourceIfUnmodifiedSince, + Parameters.sourceIfMatch, + Parameters.sourceIfNoneMatch + ], + responses: { + 201: { + headersMapper: Mappers.BlockBlobStageBlockFromURLHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const blockBlobCommitBlockListOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{containerName}/{blob}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.comp17 + ], + headerParameters: [ + Parameters.transactionalContentMD5, + Parameters.transactionalContentCrc64, + Parameters.metadata, + Parameters.tier0, + Parameters.version, + Parameters.requestId, + Parameters.blobTagsString, + Parameters.immutabilityPolicyExpiry, + Parameters.immutabilityPolicyMode, + Parameters.legalHold0, + Parameters.blobCacheControl, + Parameters.blobContentType, + Parameters.blobContentEncoding, + Parameters.blobContentLanguage, + Parameters.blobContentMD5, + Parameters.blobContentDisposition, + Parameters.leaseId0, + Parameters.encryptionKey, + Parameters.encryptionKeySha256, + Parameters.encryptionAlgorithm, + Parameters.encryptionScope, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince, + Parameters.ifMatch, + Parameters.ifNoneMatch, + Parameters.ifTags + ], + requestBody: { + parameterPath: "blocks", + mapper: { + ...Mappers.BlockLookupList, + required: true + } + }, + contentType: "application/xml; charset=utf-8", + responses: { + 201: { + headersMapper: Mappers.BlockBlobCommitBlockListHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const blockBlobGetBlockListOperationSpec: msRest.OperationSpec = { + httpMethod: "GET", + path: "{containerName}/{blob}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.snapshot, + Parameters.listType, + Parameters.timeout, + Parameters.comp17 + ], + headerParameters: [ + Parameters.version, + Parameters.requestId, + Parameters.leaseId0, + Parameters.ifTags + ], + responses: { + 200: { + bodyMapper: Mappers.BlockList, + headersMapper: Mappers.BlockBlobGetBlockListHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +// specifications for new method group start +const blobUndeleteOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{containerName}/{blob}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.comp7 + ], + headerParameters: [ + Parameters.version, + Parameters.requestId + ], + responses: { + 200: { + headersMapper: Mappers.BlobUndeleteHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const blobSetExpiryOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{containerName}/{blob}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.comp6 + ], + headerParameters: [ + Parameters.version, + Parameters.requestId, + Parameters.expiryOptions1, + Parameters.expiresOn + ], + responses: { + 200: { + headersMapper: Mappers.BlobSetExpiryHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const blobSetHTTPHeadersOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{containerName}/{blob}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.comp0 + ], + headerParameters: [ + Parameters.version, + Parameters.requestId, + Parameters.blobCacheControl, + Parameters.blobContentType, + Parameters.blobContentMD5, + Parameters.blobContentEncoding, + Parameters.blobContentLanguage, + Parameters.blobContentDisposition, + Parameters.leaseId0, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince, + Parameters.ifMatch, + Parameters.ifNoneMatch, + Parameters.ifTags + ], + responses: { + 200: { + headersMapper: Mappers.BlobSetHTTPHeadersHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const blobSetImmutabilityPolicyOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{containerName}/{blob}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.comp18 + ], + headerParameters: [ + Parameters.version, + Parameters.requestId, + Parameters.immutabilityPolicyExpiry, + Parameters.immutabilityPolicyMode, + Parameters.ifUnmodifiedSince + ], + responses: { + 200: { + headersMapper: Mappers.BlobSetImmutabilityPolicyHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const blobDeleteImmutabilityPolicyOperationSpec: msRest.OperationSpec = { + httpMethod: "DELETE", + path: "{containerName}/{blob}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.comp18 + ], + headerParameters: [ + Parameters.version, + Parameters.requestId + ], + responses: { + 200: { + headersMapper: Mappers.BlobDeleteImmutabilityPolicyHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const blobSetLegalHoldOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{containerName}/{blob}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.comp19 + ], + headerParameters: [ + Parameters.version, + Parameters.requestId, + Parameters.legalHold1 + ], + responses: { + 200: { + headersMapper: Mappers.BlobSetLegalHoldHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const blobSetMetadataOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{containerName}/{blob}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.comp8 + ], + headerParameters: [ + Parameters.metadata, + Parameters.version, + Parameters.requestId, + Parameters.leaseId0, + Parameters.encryptionKey, + Parameters.encryptionKeySha256, + Parameters.encryptionAlgorithm, + Parameters.encryptionScope, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince, + Parameters.ifMatch, + Parameters.ifNoneMatch, + Parameters.ifTags + ], + responses: { + 200: { + headersMapper: Mappers.BlobSetMetadataHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const blobAcquireLeaseOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{containerName}/{blob}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.comp10 + ], + headerParameters: [ + Parameters.duration, + Parameters.proposedLeaseId0, + Parameters.version, + Parameters.requestId, + Parameters.action7, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince, + Parameters.ifMatch, + Parameters.ifNoneMatch, + Parameters.ifTags + ], + responses: { + 201: { + headersMapper: Mappers.BlobAcquireLeaseHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const blobReleaseLeaseOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{containerName}/{blob}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.comp10 + ], + headerParameters: [ + Parameters.leaseId1, + Parameters.version, + Parameters.requestId, + Parameters.action8, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince, + Parameters.ifMatch, + Parameters.ifNoneMatch, + Parameters.ifTags + ], + responses: { + 200: { + headersMapper: Mappers.BlobReleaseLeaseHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const blobRenewLeaseOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{containerName}/{blob}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.comp10 + ], + headerParameters: [ + Parameters.leaseId1, + Parameters.version, + Parameters.requestId, + Parameters.action9, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince, + Parameters.ifMatch, + Parameters.ifNoneMatch, + Parameters.ifTags + ], + responses: { + 200: { + headersMapper: Mappers.BlobRenewLeaseHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const blobChangeLeaseOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{containerName}/{blob}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.comp10 + ], + headerParameters: [ + Parameters.leaseId1, + Parameters.proposedLeaseId1, + Parameters.version, + Parameters.requestId, + Parameters.action11, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince, + Parameters.ifMatch, + Parameters.ifNoneMatch, + Parameters.ifTags + ], + responses: { + 200: { + headersMapper: Mappers.BlobChangeLeaseHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const blobBreakLeaseOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{containerName}/{blob}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.comp10 + ], + headerParameters: [ + Parameters.breakPeriod, + Parameters.version, + Parameters.requestId, + Parameters.action10, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince, + Parameters.ifMatch, + Parameters.ifNoneMatch, + Parameters.ifTags + ], + responses: { + 202: { + headersMapper: Mappers.BlobBreakLeaseHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const blobCreateSnapshotOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{containerName}/{blob}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.comp20 + ], + headerParameters: [ + Parameters.metadata, + Parameters.version, + Parameters.requestId, + Parameters.encryptionKey, + Parameters.encryptionKeySha256, + Parameters.encryptionAlgorithm, + Parameters.encryptionScope, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince, + Parameters.ifMatch, + Parameters.ifNoneMatch, + Parameters.ifTags, + Parameters.leaseId0 + ], + responses: { + 201: { + headersMapper: Mappers.BlobCreateSnapshotHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const blobStartCopyFromURLOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{containerName}/{blob}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout + ], + headerParameters: [ + Parameters.metadata, + Parameters.tier0, + Parameters.rehydratePriority, + Parameters.copySource, + Parameters.version, + Parameters.requestId, + Parameters.blobTagsString, + Parameters.sealBlob, + Parameters.immutabilityPolicyExpiry, + Parameters.immutabilityPolicyMode, + Parameters.legalHold0, + Parameters.sourceIfModifiedSince, + Parameters.sourceIfUnmodifiedSince, + Parameters.sourceIfMatch, + Parameters.sourceIfNoneMatch, + Parameters.sourceIfTags, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince, + Parameters.ifMatch, + Parameters.ifNoneMatch, + Parameters.ifTags, + Parameters.leaseId0 + ], + responses: { + 202: { + headersMapper: Mappers.BlobStartCopyFromURLHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const blobCopyFromURLOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{containerName}/{blob}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout + ], + headerParameters: [ + Parameters.metadata, + Parameters.tier0, + Parameters.copySource, + Parameters.version, + Parameters.requestId, + Parameters.sourceContentMD5, + Parameters.blobTagsString, + Parameters.immutabilityPolicyExpiry, + Parameters.immutabilityPolicyMode, + Parameters.legalHold0, + Parameters.copySourceAuthorization, + Parameters.copySourceTags, + Parameters.xMsRequiresSync, + Parameters.sourceIfModifiedSince, + Parameters.sourceIfUnmodifiedSince, + Parameters.sourceIfMatch, + Parameters.sourceIfNoneMatch, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince, + Parameters.ifMatch, + Parameters.ifNoneMatch, + Parameters.ifTags, + Parameters.leaseId0, + Parameters.encryptionScope + ], + responses: { + 202: { + headersMapper: Mappers.BlobCopyFromURLHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const blobAbortCopyFromURLOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{containerName}/{blob}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.copyId, + Parameters.timeout, + Parameters.comp21 + ], + headerParameters: [ + Parameters.version, + Parameters.requestId, + Parameters.copyActionAbortConstant, + Parameters.leaseId0 + ], + responses: { + 204: { + headersMapper: Mappers.BlobAbortCopyFromURLHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const blobSetTierOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{containerName}/{blob}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.snapshot, + Parameters.versionId, + Parameters.timeout, + Parameters.comp22 + ], + headerParameters: [ + Parameters.tier1, + Parameters.rehydratePriority, + Parameters.version, + Parameters.requestId, + Parameters.leaseId0, + Parameters.ifTags + ], + responses: { + 200: { + headersMapper: Mappers.BlobSetTierHeaders + }, + 202: { + headersMapper: Mappers.BlobSetTierHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const blobGetAccountInfoOperationSpec: msRest.OperationSpec = { + httpMethod: "GET", + path: "{containerName}/{blob}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.restype1, + Parameters.comp0 + ], + headerParameters: [ + Parameters.version + ], + responses: { + 200: { + headersMapper: Mappers.BlobGetAccountInfoHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const blobGetAccountInfoWithHeadOperationSpec: msRest.OperationSpec = { + httpMethod: "HEAD", + path: "{containerName}/{blob}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.restype1, + Parameters.comp0 + ], + headerParameters: [ + Parameters.version + ], + responses: { + 200: { + headersMapper: Mappers.BlobGetAccountInfoWithHeadHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const blobQueryOperationSpec: msRest.OperationSpec = { + httpMethod: "POST", + path: "{containerName}/{blob}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.snapshot, + Parameters.timeout, + Parameters.comp23 + ], + headerParameters: [ + Parameters.version, + Parameters.requestId, + Parameters.leaseId0, + Parameters.encryptionKey, + Parameters.encryptionKeySha256, + Parameters.encryptionAlgorithm, + Parameters.ifModifiedSince, + Parameters.ifUnmodifiedSince, + Parameters.ifMatch, + Parameters.ifNoneMatch, + Parameters.ifTags + ], + requestBody: { + parameterPath: [ + "options", + "queryRequest" + ], + mapper: Mappers.QueryRequest + }, + contentType: "application/xml; charset=utf-8", + responses: { + 200: { + bodyMapper: { + serializedName: "Stream", + type: { + name: "Stream" + } + }, + headersMapper: Mappers.BlobQueryHeaders + }, + 206: { + bodyMapper: { + serializedName: "Stream", + type: { + name: "Stream" + } + }, + headersMapper: Mappers.BlobQueryHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const blobGetTagsOperationSpec: msRest.OperationSpec = { + httpMethod: "GET", + path: "{containerName}/{blob}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.snapshot, + Parameters.versionId, + Parameters.comp24 + ], + headerParameters: [ + Parameters.version, + Parameters.requestId, + Parameters.ifTags, + Parameters.leaseId0 + ], + responses: { + 200: { + bodyMapper: Mappers.BlobTags, + headersMapper: Mappers.BlobGetTagsHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const blobSetTagsOperationSpec: msRest.OperationSpec = { + httpMethod: "PUT", + path: "{containerName}/{blob}", + urlParameters: [ + Parameters.url + ], + queryParameters: [ + Parameters.timeout, + Parameters.versionId, + Parameters.comp24 + ], + headerParameters: [ + Parameters.version, + Parameters.transactionalContentMD5, + Parameters.transactionalContentCrc64, + Parameters.requestId, + Parameters.ifTags, + Parameters.leaseId0 + ], + requestBody: { + parameterPath: [ + "options", + "tags" + ], + mapper: Mappers.BlobTags + }, + contentType: "application/xml; charset=utf-8", + responses: { + 204: { + headersMapper: Mappers.BlobSetTagsHeaders + }, + default: { + bodyMapper: Mappers.StorageErrorBlob + } + }, + isXML: true, + serializer +}; + +const Specifications: { [key: number]: msRest.OperationSpec } = {}; +Specifications[Operation.Service_ListFileSystems] = serviceListFileSystemsOperationSpec; +Specifications[Operation.Service_SetProperties] = serviceSetPropertiesOperationSpec; +Specifications[Operation.Service_GetProperties] = serviceGetPropertiesOperationSpec; +Specifications[Operation.Service_GetStatistics] = serviceGetStatisticsOperationSpec; +Specifications[Operation.Service_ListContainersSegment] = serviceListContainersSegmentOperationSpec; +Specifications[Operation.Service_GetUserDelegationKey] = serviceGetUserDelegationKeyOperationSpec; +Specifications[Operation.Service_GetAccountInfo] = serviceGetAccountInfoOperationSpec; +Specifications[Operation.Service_GetAccountInfoWithHead] = serviceGetAccountInfoWithHeadOperationSpec; +Specifications[Operation.Service_SubmitBatch] = serviceSubmitBatchOperationSpec; +Specifications[Operation.Service_FilterBlobs] = serviceFilterBlobsOperationSpec; +Specifications[Operation.FileSystem_Create] = fileSystemCreateOperationSpec; +Specifications[Operation.FileSystem_SetProperties] = fileSystemSetPropertiesOperationSpec; +Specifications[Operation.FileSystem_GetProperties] = fileSystemGetPropertiesOperationSpec; +Specifications[Operation.FileSystem_Delete] = fileSystemDeleteOperationSpec; +Specifications[Operation.FileSystem_ListPaths] = fileSystemListPathsOperationSpec; +Specifications[Operation.FileSystem_ListBlobFlatSegment] = fileSystemListBlobFlatSegmentOperationSpec; +Specifications[Operation.FileSystem_ListBlobHierarchySegment] = fileSystemListBlobHierarchySegmentOperationSpec; +Specifications[Operation.Path_Create] = pathCreateOperationSpec; +Specifications[Operation.Path_Update] = pathUpdateOperationSpec; +Specifications[Operation.Path_Lease] = pathLeaseOperationSpec; +Specifications[Operation.Path_Read] = pathReadOperationSpec; +Specifications[Operation.Path_GetProperties] = pathGetPropertiesOperationSpec; +Specifications[Operation.Path_Delete] = pathDeleteOperationSpec; +Specifications[Operation.Path_SetAccessControl] = pathSetAccessControlOperationSpec; +Specifications[Operation.Path_SetAccessControlRecursive] = pathSetAccessControlRecursiveOperationSpec; +Specifications[Operation.Path_SetProperties] = pathSetPropertiesOperationSpec; +Specifications[Operation.Path_FlushData] = pathFlushDataOperationSpec; +Specifications[Operation.Path_AppendData] = pathAppendDataOperationSpec; +Specifications[Operation.Path_SetExpiry] = pathSetExpiryOperationSpec; +Specifications[Operation.Path_Undelete] = pathUndeleteOperationSpec; +Specifications[Operation.Container_Create] = containerCreateOperationSpec; +Specifications[Operation.Container_GetProperties] = containerGetPropertiesOperationSpec; +Specifications[Operation.Container_GetPropertiesWithHead] = containerGetPropertiesWithHeadOperationSpec; +Specifications[Operation.Container_Delete] = containerDeleteOperationSpec; +Specifications[Operation.Container_SetMetadata] = containerSetMetadataOperationSpec; +Specifications[Operation.Container_GetAccessPolicy] = containerGetAccessPolicyOperationSpec; +Specifications[Operation.Container_SetAccessPolicy] = containerSetAccessPolicyOperationSpec; +Specifications[Operation.Container_Restore] = containerRestoreOperationSpec; +Specifications[Operation.Container_SubmitBatch] = containerSubmitBatchOperationSpec; +Specifications[Operation.Container_FilterBlobs] = containerFilterBlobsOperationSpec; +Specifications[Operation.Container_AcquireLease] = containerAcquireLeaseOperationSpec; +Specifications[Operation.Container_ReleaseLease] = containerReleaseLeaseOperationSpec; +Specifications[Operation.Container_RenewLease] = containerRenewLeaseOperationSpec; +Specifications[Operation.Container_BreakLease] = containerBreakLeaseOperationSpec; +Specifications[Operation.Container_ChangeLease] = containerChangeLeaseOperationSpec; +Specifications[Operation.Container_GetAccountInfo] = containerGetAccountInfoOperationSpec; +Specifications[Operation.Container_GetAccountInfoWithHead] = containerGetAccountInfoWithHeadOperationSpec; +Specifications[Operation.PageBlob_Create] = pageBlobCreateOperationSpec; +Specifications[Operation.PageBlob_UploadPages] = pageBlobUploadPagesOperationSpec; +Specifications[Operation.PageBlob_ClearPages] = pageBlobClearPagesOperationSpec; +Specifications[Operation.PageBlob_UploadPagesFromURL] = pageBlobUploadPagesFromURLOperationSpec; +Specifications[Operation.PageBlob_GetPageRanges] = pageBlobGetPageRangesOperationSpec; +Specifications[Operation.PageBlob_GetPageRangesDiff] = pageBlobGetPageRangesDiffOperationSpec; +Specifications[Operation.PageBlob_Resize] = pageBlobResizeOperationSpec; +Specifications[Operation.PageBlob_UpdateSequenceNumber] = pageBlobUpdateSequenceNumberOperationSpec; +Specifications[Operation.PageBlob_CopyIncremental] = pageBlobCopyIncrementalOperationSpec; +Specifications[Operation.AppendBlob_Create] = appendBlobCreateOperationSpec; +Specifications[Operation.AppendBlob_AppendBlock] = appendBlobAppendBlockOperationSpec; +Specifications[Operation.AppendBlob_AppendBlockFromUrl] = appendBlobAppendBlockFromUrlOperationSpec; +Specifications[Operation.AppendBlob_Seal] = appendBlobSealOperationSpec; +Specifications[Operation.BlockBlob_Upload] = blockBlobUploadOperationSpec; +Specifications[Operation.BlockBlob_PutBlobFromUrl] = blockBlobPutBlobFromUrlOperationSpec; +Specifications[Operation.BlockBlob_StageBlock] = blockBlobStageBlockOperationSpec; +Specifications[Operation.BlockBlob_StageBlockFromURL] = blockBlobStageBlockFromURLOperationSpec; +Specifications[Operation.BlockBlob_CommitBlockList] = blockBlobCommitBlockListOperationSpec; +Specifications[Operation.BlockBlob_GetBlockList] = blockBlobGetBlockListOperationSpec; +Specifications[Operation.Blob_Undelete] = blobUndeleteOperationSpec; +Specifications[Operation.Blob_SetExpiry] = blobSetExpiryOperationSpec; +Specifications[Operation.Blob_SetHTTPHeaders] = blobSetHTTPHeadersOperationSpec; +Specifications[Operation.Blob_SetImmutabilityPolicy] = blobSetImmutabilityPolicyOperationSpec; +Specifications[Operation.Blob_DeleteImmutabilityPolicy] = blobDeleteImmutabilityPolicyOperationSpec; +Specifications[Operation.Blob_SetLegalHold] = blobSetLegalHoldOperationSpec; +Specifications[Operation.Blob_SetMetadata] = blobSetMetadataOperationSpec; +Specifications[Operation.Blob_AcquireLease] = blobAcquireLeaseOperationSpec; +Specifications[Operation.Blob_ReleaseLease] = blobReleaseLeaseOperationSpec; +Specifications[Operation.Blob_RenewLease] = blobRenewLeaseOperationSpec; +Specifications[Operation.Blob_ChangeLease] = blobChangeLeaseOperationSpec; +Specifications[Operation.Blob_BreakLease] = blobBreakLeaseOperationSpec; +Specifications[Operation.Blob_CreateSnapshot] = blobCreateSnapshotOperationSpec; +Specifications[Operation.Blob_StartCopyFromURL] = blobStartCopyFromURLOperationSpec; +Specifications[Operation.Blob_CopyFromURL] = blobCopyFromURLOperationSpec; +Specifications[Operation.Blob_AbortCopyFromURL] = blobAbortCopyFromURLOperationSpec; +Specifications[Operation.Blob_SetTier] = blobSetTierOperationSpec; +Specifications[Operation.Blob_GetAccountInfo] = blobGetAccountInfoOperationSpec; +Specifications[Operation.Blob_GetAccountInfoWithHead] = blobGetAccountInfoWithHeadOperationSpec; +Specifications[Operation.Blob_Query] = blobQueryOperationSpec; +Specifications[Operation.Blob_GetTags] = blobGetTagsOperationSpec; +Specifications[Operation.Blob_SetTags] = blobSetTagsOperationSpec; +export default Specifications; diff --git a/src/dfs/generated/errors/DeserializationError.ts b/src/dfs/generated/errors/DeserializationError.ts new file mode 100644 index 000000000..82a549096 --- /dev/null +++ b/src/dfs/generated/errors/DeserializationError.ts @@ -0,0 +1,8 @@ +import MiddlewareError from './MiddlewareError'; + +export default class DeserializationError extends MiddlewareError { + public constructor(message: string) { + super(400, message); + this.name = "DeserializationError"; + } +} diff --git a/src/dfs/generated/errors/MiddlewareError.ts b/src/dfs/generated/errors/MiddlewareError.ts new file mode 100644 index 000000000..b39bbcd0b --- /dev/null +++ b/src/dfs/generated/errors/MiddlewareError.ts @@ -0,0 +1,29 @@ +import { OutgoingHttpHeaders } from 'http'; + +export default class MiddlewareError extends Error { + /** + * Creates an instance of MiddlewareError. + * + * @param {number} statusCode HTTP response status code + * @param {string} message Error message + * @param {string} [statusMessage] HTTP response status message + * @param {OutgoingHttpHeaders} [headers] HTTP response headers + * @param {string} [body] HTTP response body + * @param {string} [contentType] HTTP contentType + * @memberof MiddlewareError + */ + constructor( + public readonly statusCode: number, + public readonly message: string, + public readonly statusMessage?: string, + public readonly headers?: OutgoingHttpHeaders, + public readonly body?: string, + public readonly contentType?: string + ) { + super(message); + // https://stackoverflow.com/questions/31626231/custom-error-class-in-typescript + Object.setPrototypeOf(this, MiddlewareError.prototype); + + this.name = "MiddlewareError"; + } +} diff --git a/src/dfs/generated/errors/OperationMismatchError.ts b/src/dfs/generated/errors/OperationMismatchError.ts new file mode 100644 index 000000000..65660cea6 --- /dev/null +++ b/src/dfs/generated/errors/OperationMismatchError.ts @@ -0,0 +1,11 @@ +import MiddlewareError from './MiddlewareError'; + +export default class OperationMismatchError extends MiddlewareError { + public constructor() { + super( + 500, + "No operation provided in context, please make sure dispatchMiddleware is properly used." + ); + this.name = "OperationMismatchError"; + } +} diff --git a/src/dfs/generated/errors/UnsupportedRequestError.ts b/src/dfs/generated/errors/UnsupportedRequestError.ts new file mode 100644 index 000000000..1acee87ea --- /dev/null +++ b/src/dfs/generated/errors/UnsupportedRequestError.ts @@ -0,0 +1,11 @@ +import MiddlewareError from './MiddlewareError'; + +export default class UnsupportedRequestError extends MiddlewareError { + public constructor() { + super( + 400, + "Incoming URL doesn't match any of swagger defined request patterns." + ); + this.name = "UnsupportedRequestError"; + } +} diff --git a/src/dfs/generated/handlers/IAppendBlobHandler.ts b/src/dfs/generated/handlers/IAppendBlobHandler.ts new file mode 100644 index 000000000..a945212cb --- /dev/null +++ b/src/dfs/generated/handlers/IAppendBlobHandler.ts @@ -0,0 +1,20 @@ +/* + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for + * license information. + * + * Code generated by Microsoft (R) AutoRest Code Generator. + * Changes may cause incorrect behavior and will be lost if the code is + * regenerated. + */ +// tslint:disable:max-line-length + +import * as Models from "../artifacts/models"; +import Context from "../Context"; + +export default interface IAppendBlobHandler { + create(contentLength: number, options: Models.AppendBlobCreateOptionalParams, context: Context): Promise; + appendBlock(body: NodeJS.ReadableStream, contentLength: number, options: Models.AppendBlobAppendBlockOptionalParams, context: Context): Promise; + appendBlockFromUrl(sourceUrl: string, contentLength: number, options: Models.AppendBlobAppendBlockFromUrlOptionalParams, context: Context): Promise; + seal(options: Models.AppendBlobSealOptionalParams, context: Context): Promise; +} diff --git a/src/dfs/generated/handlers/IBlobHandler.ts b/src/dfs/generated/handlers/IBlobHandler.ts new file mode 100644 index 000000000..f64deebce --- /dev/null +++ b/src/dfs/generated/handlers/IBlobHandler.ts @@ -0,0 +1,38 @@ +/* + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for + * license information. + * + * Code generated by Microsoft (R) AutoRest Code Generator. + * Changes may cause incorrect behavior and will be lost if the code is + * regenerated. + */ +// tslint:disable:max-line-length + +import * as Models from "../artifacts/models"; +import Context from "../Context"; + +export default interface IBlobHandler { + undelete(options: Models.BlobUndeleteOptionalParams, context: Context): Promise; + setExpiry(expiryOptions: Models.BlobExpiryOptions, options: Models.BlobSetExpiryOptionalParams, context: Context): Promise; + setHTTPHeaders(options: Models.BlobSetHTTPHeadersOptionalParams, context: Context): Promise; + setImmutabilityPolicy(options: Models.BlobSetImmutabilityPolicyOptionalParams, context: Context): Promise; + deleteImmutabilityPolicy(options: Models.BlobDeleteImmutabilityPolicyOptionalParams, context: Context): Promise; + setLegalHold(legalHold: boolean, options: Models.BlobSetLegalHoldOptionalParams, context: Context): Promise; + setMetadata(options: Models.BlobSetMetadataOptionalParams, context: Context): Promise; + acquireLease(options: Models.BlobAcquireLeaseOptionalParams, context: Context): Promise; + releaseLease(leaseId: string, options: Models.BlobReleaseLeaseOptionalParams, context: Context): Promise; + renewLease(leaseId: string, options: Models.BlobRenewLeaseOptionalParams, context: Context): Promise; + changeLease(leaseId: string, proposedLeaseId: string, options: Models.BlobChangeLeaseOptionalParams, context: Context): Promise; + breakLease(options: Models.BlobBreakLeaseOptionalParams, context: Context): Promise; + createSnapshot(options: Models.BlobCreateSnapshotOptionalParams, context: Context): Promise; + startCopyFromURL(copySource: string, options: Models.BlobStartCopyFromURLOptionalParams, context: Context): Promise; + copyFromURL(copySource: string, options: Models.BlobCopyFromURLOptionalParams, context: Context): Promise; + abortCopyFromURL(copyId: string, options: Models.BlobAbortCopyFromURLOptionalParams, context: Context): Promise; + setTier(tier: Models.AccessTier, options: Models.BlobSetTierOptionalParams, context: Context): Promise; + getAccountInfo(context: Context): Promise; + getAccountInfoWithHead(context: Context): Promise; + query(options: Models.BlobQueryOptionalParams, context: Context): Promise; + getTags(options: Models.BlobGetTagsOptionalParams, context: Context): Promise; + setTags(options: Models.BlobSetTagsOptionalParams, context: Context): Promise; +} diff --git a/src/dfs/generated/handlers/IBlockBlobHandler.ts b/src/dfs/generated/handlers/IBlockBlobHandler.ts new file mode 100644 index 000000000..44ae0a430 --- /dev/null +++ b/src/dfs/generated/handlers/IBlockBlobHandler.ts @@ -0,0 +1,22 @@ +/* + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for + * license information. + * + * Code generated by Microsoft (R) AutoRest Code Generator. + * Changes may cause incorrect behavior and will be lost if the code is + * regenerated. + */ +// tslint:disable:max-line-length + +import * as Models from "../artifacts/models"; +import Context from "../Context"; + +export default interface IBlockBlobHandler { + upload(body: NodeJS.ReadableStream, contentLength: number, options: Models.BlockBlobUploadOptionalParams, context: Context): Promise; + putBlobFromUrl(contentLength: number, copySource: string, options: Models.BlockBlobPutBlobFromUrlOptionalParams, context: Context): Promise; + stageBlock(blockId: string, contentLength: number, body: NodeJS.ReadableStream, options: Models.BlockBlobStageBlockOptionalParams, context: Context): Promise; + stageBlockFromURL(blockId: string, contentLength: number, sourceUrl: string, options: Models.BlockBlobStageBlockFromURLOptionalParams, context: Context): Promise; + commitBlockList(blocks: Models.BlockLookupList, options: Models.BlockBlobCommitBlockListOptionalParams, context: Context): Promise; + getBlockList(options: Models.BlockBlobGetBlockListOptionalParams, context: Context): Promise; +} diff --git a/src/dfs/generated/handlers/IContainerHandler.ts b/src/dfs/generated/handlers/IContainerHandler.ts new file mode 100644 index 000000000..55a736074 --- /dev/null +++ b/src/dfs/generated/handlers/IContainerHandler.ts @@ -0,0 +1,33 @@ +/* + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for + * license information. + * + * Code generated by Microsoft (R) AutoRest Code Generator. + * Changes may cause incorrect behavior and will be lost if the code is + * regenerated. + */ +// tslint:disable:max-line-length + +import * as Models from "../artifacts/models"; +import Context from "../Context"; + +export default interface IContainerHandler { + create(options: Models.ContainerCreateOptionalParams, context: Context): Promise; + getProperties(options: Models.ContainerGetPropertiesOptionalParams, context: Context): Promise; + getPropertiesWithHead(options: Models.ContainerGetPropertiesWithHeadOptionalParams, context: Context): Promise; + delete(options: Models.ContainerDeleteMethodOptionalParams, context: Context): Promise; + setMetadata(options: Models.ContainerSetMetadataOptionalParams, context: Context): Promise; + getAccessPolicy(options: Models.ContainerGetAccessPolicyOptionalParams, context: Context): Promise; + setAccessPolicy(options: Models.ContainerSetAccessPolicyOptionalParams, context: Context): Promise; + restore(options: Models.ContainerRestoreOptionalParams, context: Context): Promise; + submitBatch(body: NodeJS.ReadableStream, contentLength: number, multipartContentType: string, options: Models.ContainerSubmitBatchOptionalParams, context: Context): Promise; + filterBlobs(options: Models.ContainerFilterBlobsOptionalParams, context: Context): Promise; + acquireLease(options: Models.ContainerAcquireLeaseOptionalParams, context: Context): Promise; + releaseLease(leaseId: string, options: Models.ContainerReleaseLeaseOptionalParams, context: Context): Promise; + renewLease(leaseId: string, options: Models.ContainerRenewLeaseOptionalParams, context: Context): Promise; + breakLease(options: Models.ContainerBreakLeaseOptionalParams, context: Context): Promise; + changeLease(leaseId: string, proposedLeaseId: string, options: Models.ContainerChangeLeaseOptionalParams, context: Context): Promise; + getAccountInfo(context: Context): Promise; + getAccountInfoWithHead(context: Context): Promise; +} diff --git a/src/dfs/generated/handlers/IFileSystemOperationsHandler.ts b/src/dfs/generated/handlers/IFileSystemOperationsHandler.ts new file mode 100644 index 000000000..d2ab2e779 --- /dev/null +++ b/src/dfs/generated/handlers/IFileSystemOperationsHandler.ts @@ -0,0 +1,23 @@ +/* + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for + * license information. + * + * Code generated by Microsoft (R) AutoRest Code Generator. + * Changes may cause incorrect behavior and will be lost if the code is + * regenerated. + */ +// tslint:disable:max-line-length + +import * as Models from "../artifacts/models"; +import Context from "../Context"; + +export default interface IFileSystemOperationsHandler { + create(options: Models.FileSystemCreateOptionalParams, context: Context): Promise; + setProperties(options: Models.FileSystemSetPropertiesOptionalParams, context: Context): Promise; + getProperties(options: Models.FileSystemGetPropertiesOptionalParams, context: Context): Promise; + delete(options: Models.FileSystemDeleteMethodOptionalParams, context: Context): Promise; + listPaths(recursive: boolean, options: Models.FileSystemListPathsOptionalParams, context: Context): Promise; + listBlobFlatSegment(options: Models.FileSystemListBlobFlatSegmentOptionalParams, context: Context): Promise; + listBlobHierarchySegment(delimiter: string, options: Models.FileSystemListBlobHierarchySegmentOptionalParams, context: Context): Promise; +} diff --git a/src/dfs/generated/handlers/IHandlers.ts b/src/dfs/generated/handlers/IHandlers.ts new file mode 100644 index 000000000..c534178d0 --- /dev/null +++ b/src/dfs/generated/handlers/IHandlers.ts @@ -0,0 +1,21 @@ +// tslint:disable:ordered-imports +import IServiceHandler from "./IServiceHandler"; +import IFileSystemOperationsHandler from "./IFileSystemOperationsHandler"; +import IPathOperationsHandler from "./IPathOperationsHandler"; +import IContainerHandler from "./IContainerHandler"; +import IPageBlobHandler from "./IPageBlobHandler"; +import IAppendBlobHandler from "./IAppendBlobHandler"; +import IBlockBlobHandler from "./IBlockBlobHandler"; +import IBlobHandler from "./IBlobHandler"; + +export interface IHandlers { + serviceHandler: IServiceHandler; + fileSystemOperationsHandler: IFileSystemOperationsHandler; + pathOperationsHandler: IPathOperationsHandler; + containerHandler: IContainerHandler; + pageBlobHandler: IPageBlobHandler; + appendBlobHandler: IAppendBlobHandler; + blockBlobHandler: IBlockBlobHandler; + blobHandler: IBlobHandler; +} +export default IHandlers; diff --git a/src/dfs/generated/handlers/IPageBlobHandler.ts b/src/dfs/generated/handlers/IPageBlobHandler.ts new file mode 100644 index 000000000..64162e650 --- /dev/null +++ b/src/dfs/generated/handlers/IPageBlobHandler.ts @@ -0,0 +1,25 @@ +/* + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for + * license information. + * + * Code generated by Microsoft (R) AutoRest Code Generator. + * Changes may cause incorrect behavior and will be lost if the code is + * regenerated. + */ +// tslint:disable:max-line-length + +import * as Models from "../artifacts/models"; +import Context from "../Context"; + +export default interface IPageBlobHandler { + create(contentLength: number, blobContentLength: number, options: Models.PageBlobCreateOptionalParams, context: Context): Promise; + uploadPages(body: NodeJS.ReadableStream, contentLength: number, options: Models.PageBlobUploadPagesOptionalParams, context: Context): Promise; + clearPages(contentLength: number, options: Models.PageBlobClearPagesOptionalParams, context: Context): Promise; + uploadPagesFromURL(sourceUrl: string, sourceRange: string, contentLength: number, range: string, options: Models.PageBlobUploadPagesFromURLOptionalParams, context: Context): Promise; + getPageRanges(options: Models.PageBlobGetPageRangesOptionalParams, context: Context): Promise; + getPageRangesDiff(options: Models.PageBlobGetPageRangesDiffOptionalParams, context: Context): Promise; + resize(blobContentLength: number, options: Models.PageBlobResizeOptionalParams, context: Context): Promise; + updateSequenceNumber(sequenceNumberAction: Models.SequenceNumberActionType, options: Models.PageBlobUpdateSequenceNumberOptionalParams, context: Context): Promise; + copyIncremental(copySource: string, options: Models.PageBlobCopyIncrementalOptionalParams, context: Context): Promise; +} diff --git a/src/dfs/generated/handlers/IPathOperationsHandler.ts b/src/dfs/generated/handlers/IPathOperationsHandler.ts new file mode 100644 index 000000000..ae5f2c649 --- /dev/null +++ b/src/dfs/generated/handlers/IPathOperationsHandler.ts @@ -0,0 +1,29 @@ +/* + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for + * license information. + * + * Code generated by Microsoft (R) AutoRest Code Generator. + * Changes may cause incorrect behavior and will be lost if the code is + * regenerated. + */ +// tslint:disable:max-line-length + +import * as Models from "../artifacts/models"; +import Context from "../Context"; + +export default interface IPathOperationsHandler { + create(options: Models.PathCreateOptionalParams, context: Context): Promise; + update(action: Models.PathUpdateAction, mode: Models.PathSetAccessControlRecursiveMode, body: NodeJS.ReadableStream, options: Models.PathUpdateOptionalParams, context: Context): Promise; + lease(xMsLeaseAction: Models.PathLeaseAction, options: Models.PathLeaseOptionalParams, context: Context): Promise; + read(options: Models.PathReadOptionalParams, context: Context): Promise; + getProperties(options: Models.PathGetPropertiesOptionalParams, context: Context): Promise; + delete(options: Models.PathDeleteMethodOptionalParams, context: Context): Promise; + setAccessControl(options: Models.PathSetAccessControlOptionalParams, context: Context): Promise; + setAccessControlRecursive(mode: Models.PathSetAccessControlRecursiveMode, options: Models.PathSetAccessControlRecursiveOptionalParams, context: Context): Promise; + setProperties(options: Models.PathSetPropertiesOptionalParams, context: Context): Promise; + flushData(options: Models.PathFlushDataOptionalParams, context: Context): Promise; + appendData(body: NodeJS.ReadableStream, options: Models.PathAppendDataOptionalParams, context: Context): Promise; + setExpiry(expiryOptions: Models.PathExpiryOptions, options: Models.PathSetExpiryOptionalParams, context: Context): Promise; + undelete(options: Models.PathUndeleteOptionalParams, context: Context): Promise; +} diff --git a/src/dfs/generated/handlers/IServiceHandler.ts b/src/dfs/generated/handlers/IServiceHandler.ts new file mode 100644 index 000000000..f150a9add --- /dev/null +++ b/src/dfs/generated/handlers/IServiceHandler.ts @@ -0,0 +1,26 @@ +/* + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for + * license information. + * + * Code generated by Microsoft (R) AutoRest Code Generator. + * Changes may cause incorrect behavior and will be lost if the code is + * regenerated. + */ +// tslint:disable:max-line-length + +import * as Models from "../artifacts/models"; +import Context from "../Context"; + +export default interface IServiceHandler { + listFileSystems(options: Models.ServiceListFileSystemsOptionalParams, context: Context): Promise; + setProperties(storageServiceProperties: Models.StorageServiceProperties, options: Models.ServiceSetPropertiesOptionalParams, context: Context): Promise; + getProperties(options: Models.ServiceGetPropertiesOptionalParams, context: Context): Promise; + getStatistics(options: Models.ServiceGetStatisticsOptionalParams, context: Context): Promise; + listContainersSegment(options: Models.ServiceListContainersSegmentOptionalParams, context: Context): Promise; + getUserDelegationKey(keyInfo: Models.KeyInfo, options: Models.ServiceGetUserDelegationKeyOptionalParams, context: Context): Promise; + getAccountInfo(context: Context): Promise; + getAccountInfoWithHead(context: Context): Promise; + submitBatch(body: NodeJS.ReadableStream, contentLength: number, multipartContentType: string, options: Models.ServiceSubmitBatchOptionalParams, context: Context): Promise; + filterBlobs(options: Models.ServiceFilterBlobsOptionalParams, context: Context): Promise; +} diff --git a/src/dfs/generated/handlers/handlerMappers.ts b/src/dfs/generated/handlers/handlerMappers.ts new file mode 100644 index 000000000..7c72462b5 --- /dev/null +++ b/src/dfs/generated/handlers/handlerMappers.ts @@ -0,0 +1,679 @@ +import Operation from "../artifacts/operation"; + +// tslint:disable:one-line + +export interface IHandlerPath { + handler: string; + method: string; + arguments: string[]; +} + +const operationHandlerMapping: {[key: number]: IHandlerPath} = {}; + +operationHandlerMapping[Operation.Service_ListFileSystems] = { + arguments: [ + "options" + ], + handler: "serviceHandler", + method: "listFileSystems" +}; +operationHandlerMapping[Operation.Service_SetProperties] = { + arguments: [ + "storageServiceProperties", + "options" + ], + handler: "serviceHandler", + method: "setProperties" +}; +operationHandlerMapping[Operation.Service_GetProperties] = { + arguments: [ + "options" + ], + handler: "serviceHandler", + method: "getProperties" +}; +operationHandlerMapping[Operation.Service_GetStatistics] = { + arguments: [ + "options" + ], + handler: "serviceHandler", + method: "getStatistics" +}; +operationHandlerMapping[Operation.Service_ListContainersSegment] = { + arguments: [ + "options" + ], + handler: "serviceHandler", + method: "listContainersSegment" +}; +operationHandlerMapping[Operation.Service_GetUserDelegationKey] = { + arguments: [ + "keyInfo", + "options" + ], + handler: "serviceHandler", + method: "getUserDelegationKey" +}; +operationHandlerMapping[Operation.Service_GetAccountInfo] = { + arguments: [], + handler: "serviceHandler", + method: "getAccountInfo" +}; +operationHandlerMapping[Operation.Service_GetAccountInfoWithHead] = { + arguments: [], + handler: "serviceHandler", + method: "getAccountInfoWithHead" +}; +operationHandlerMapping[Operation.Service_SubmitBatch] = { + arguments: [ + "body", + "contentLength", + "multipartContentType", + "options" + ], + handler: "serviceHandler", + method: "submitBatch" +}; +operationHandlerMapping[Operation.Service_FilterBlobs] = { + arguments: [ + "options" + ], + handler: "serviceHandler", + method: "filterBlobs" +}; +operationHandlerMapping[Operation.FileSystem_Create] = { + arguments: [ + "options" + ], + handler: "fileSystemOperationsHandler", + method: "create" +}; +operationHandlerMapping[Operation.FileSystem_SetProperties] = { + arguments: [ + "options" + ], + handler: "fileSystemOperationsHandler", + method: "setProperties" +}; +operationHandlerMapping[Operation.FileSystem_GetProperties] = { + arguments: [ + "options" + ], + handler: "fileSystemOperationsHandler", + method: "getProperties" +}; +operationHandlerMapping[Operation.FileSystem_Delete] = { + arguments: [ + "options" + ], + handler: "fileSystemOperationsHandler", + method: "delete" +}; +operationHandlerMapping[Operation.FileSystem_ListPaths] = { + arguments: [ + "recursive", + "options" + ], + handler: "fileSystemOperationsHandler", + method: "listPaths" +}; +operationHandlerMapping[Operation.FileSystem_ListBlobFlatSegment] = { + arguments: [ + "options" + ], + handler: "fileSystemOperationsHandler", + method: "listBlobFlatSegment" +}; +operationHandlerMapping[Operation.FileSystem_ListBlobHierarchySegment] = { + arguments: [ + "delimiter", + "options" + ], + handler: "fileSystemOperationsHandler", + method: "listBlobHierarchySegment" +}; +operationHandlerMapping[Operation.Path_Create] = { + arguments: [ + "options" + ], + handler: "pathOperationsHandler", + method: "create" +}; +operationHandlerMapping[Operation.Path_Update] = { + arguments: [ + "action", + "mode", + "body", + "options" + ], + handler: "pathOperationsHandler", + method: "update" +}; +operationHandlerMapping[Operation.Path_Lease] = { + arguments: [ + "xMsLeaseAction", + "options" + ], + handler: "pathOperationsHandler", + method: "lease" +}; +operationHandlerMapping[Operation.Path_Read] = { + arguments: [ + "options" + ], + handler: "pathOperationsHandler", + method: "read" +}; +operationHandlerMapping[Operation.Path_GetProperties] = { + arguments: [ + "options" + ], + handler: "pathOperationsHandler", + method: "getProperties" +}; +operationHandlerMapping[Operation.Path_Delete] = { + arguments: [ + "options" + ], + handler: "pathOperationsHandler", + method: "delete" +}; +operationHandlerMapping[Operation.Path_SetAccessControl] = { + arguments: [ + "options" + ], + handler: "pathOperationsHandler", + method: "setAccessControl" +}; +operationHandlerMapping[Operation.Path_SetAccessControlRecursive] = { + arguments: [ + "mode", + "options" + ], + handler: "pathOperationsHandler", + method: "setAccessControlRecursive" +}; +operationHandlerMapping[Operation.Path_SetProperties] = { + arguments: [ + "options" + ], + handler: "pathOperationsHandler", + method: "setProperties" +}; +operationHandlerMapping[Operation.Path_FlushData] = { + arguments: [ + "options" + ], + handler: "pathOperationsHandler", + method: "flushData" +}; +operationHandlerMapping[Operation.Path_AppendData] = { + arguments: [ + "body", + "options" + ], + handler: "pathOperationsHandler", + method: "appendData" +}; +operationHandlerMapping[Operation.Path_SetExpiry] = { + arguments: [ + "expiryOptions", + "options" + ], + handler: "pathOperationsHandler", + method: "setExpiry" +}; +operationHandlerMapping[Operation.Path_Undelete] = { + arguments: [ + "options" + ], + handler: "pathOperationsHandler", + method: "undelete" +}; +operationHandlerMapping[Operation.Container_Create] = { + arguments: [ + "options" + ], + handler: "containerHandler", + method: "create" +}; +operationHandlerMapping[Operation.Container_GetProperties] = { + arguments: [ + "options" + ], + handler: "containerHandler", + method: "getProperties" +}; +operationHandlerMapping[Operation.Container_GetPropertiesWithHead] = { + arguments: [ + "options" + ], + handler: "containerHandler", + method: "getPropertiesWithHead" +}; +operationHandlerMapping[Operation.Container_Delete] = { + arguments: [ + "options" + ], + handler: "containerHandler", + method: "delete" +}; +operationHandlerMapping[Operation.Container_SetMetadata] = { + arguments: [ + "options" + ], + handler: "containerHandler", + method: "setMetadata" +}; +operationHandlerMapping[Operation.Container_GetAccessPolicy] = { + arguments: [ + "options" + ], + handler: "containerHandler", + method: "getAccessPolicy" +}; +operationHandlerMapping[Operation.Container_SetAccessPolicy] = { + arguments: [ + "options" + ], + handler: "containerHandler", + method: "setAccessPolicy" +}; +operationHandlerMapping[Operation.Container_Restore] = { + arguments: [ + "options" + ], + handler: "containerHandler", + method: "restore" +}; +operationHandlerMapping[Operation.Container_SubmitBatch] = { + arguments: [ + "body", + "contentLength", + "multipartContentType", + "options" + ], + handler: "containerHandler", + method: "submitBatch" +}; +operationHandlerMapping[Operation.Container_FilterBlobs] = { + arguments: [ + "options" + ], + handler: "containerHandler", + method: "filterBlobs" +}; +operationHandlerMapping[Operation.Container_AcquireLease] = { + arguments: [ + "options" + ], + handler: "containerHandler", + method: "acquireLease" +}; +operationHandlerMapping[Operation.Container_ReleaseLease] = { + arguments: [ + "leaseId", + "options" + ], + handler: "containerHandler", + method: "releaseLease" +}; +operationHandlerMapping[Operation.Container_RenewLease] = { + arguments: [ + "leaseId", + "options" + ], + handler: "containerHandler", + method: "renewLease" +}; +operationHandlerMapping[Operation.Container_BreakLease] = { + arguments: [ + "options" + ], + handler: "containerHandler", + method: "breakLease" +}; +operationHandlerMapping[Operation.Container_ChangeLease] = { + arguments: [ + "leaseId", + "proposedLeaseId", + "options" + ], + handler: "containerHandler", + method: "changeLease" +}; +operationHandlerMapping[Operation.Container_GetAccountInfo] = { + arguments: [], + handler: "containerHandler", + method: "getAccountInfo" +}; +operationHandlerMapping[Operation.Container_GetAccountInfoWithHead] = { + arguments: [], + handler: "containerHandler", + method: "getAccountInfoWithHead" +}; +operationHandlerMapping[Operation.PageBlob_Create] = { + arguments: [ + "contentLength", + "blobContentLength", + "options" + ], + handler: "pageBlobHandler", + method: "create" +}; +operationHandlerMapping[Operation.PageBlob_UploadPages] = { + arguments: [ + "body", + "contentLength", + "options" + ], + handler: "pageBlobHandler", + method: "uploadPages" +}; +operationHandlerMapping[Operation.PageBlob_ClearPages] = { + arguments: [ + "contentLength", + "options" + ], + handler: "pageBlobHandler", + method: "clearPages" +}; +operationHandlerMapping[Operation.PageBlob_UploadPagesFromURL] = { + arguments: [ + "sourceUrl", + "sourceRange", + "contentLength", + "range", + "options" + ], + handler: "pageBlobHandler", + method: "uploadPagesFromURL" +}; +operationHandlerMapping[Operation.PageBlob_GetPageRanges] = { + arguments: [ + "options" + ], + handler: "pageBlobHandler", + method: "getPageRanges" +}; +operationHandlerMapping[Operation.PageBlob_GetPageRangesDiff] = { + arguments: [ + "options" + ], + handler: "pageBlobHandler", + method: "getPageRangesDiff" +}; +operationHandlerMapping[Operation.PageBlob_Resize] = { + arguments: [ + "blobContentLength", + "options" + ], + handler: "pageBlobHandler", + method: "resize" +}; +operationHandlerMapping[Operation.PageBlob_UpdateSequenceNumber] = { + arguments: [ + "sequenceNumberAction", + "options" + ], + handler: "pageBlobHandler", + method: "updateSequenceNumber" +}; +operationHandlerMapping[Operation.PageBlob_CopyIncremental] = { + arguments: [ + "copySource", + "options" + ], + handler: "pageBlobHandler", + method: "copyIncremental" +}; +operationHandlerMapping[Operation.AppendBlob_Create] = { + arguments: [ + "contentLength", + "options" + ], + handler: "appendBlobHandler", + method: "create" +}; +operationHandlerMapping[Operation.AppendBlob_AppendBlock] = { + arguments: [ + "body", + "contentLength", + "options" + ], + handler: "appendBlobHandler", + method: "appendBlock" +}; +operationHandlerMapping[Operation.AppendBlob_AppendBlockFromUrl] = { + arguments: [ + "sourceUrl", + "contentLength", + "options" + ], + handler: "appendBlobHandler", + method: "appendBlockFromUrl" +}; +operationHandlerMapping[Operation.AppendBlob_Seal] = { + arguments: [ + "options" + ], + handler: "appendBlobHandler", + method: "seal" +}; +operationHandlerMapping[Operation.BlockBlob_Upload] = { + arguments: [ + "body", + "contentLength", + "options" + ], + handler: "blockBlobHandler", + method: "upload" +}; +operationHandlerMapping[Operation.BlockBlob_PutBlobFromUrl] = { + arguments: [ + "contentLength", + "copySource", + "options" + ], + handler: "blockBlobHandler", + method: "putBlobFromUrl" +}; +operationHandlerMapping[Operation.BlockBlob_StageBlock] = { + arguments: [ + "blockId", + "contentLength", + "body", + "options" + ], + handler: "blockBlobHandler", + method: "stageBlock" +}; +operationHandlerMapping[Operation.BlockBlob_StageBlockFromURL] = { + arguments: [ + "blockId", + "contentLength", + "sourceUrl", + "options" + ], + handler: "blockBlobHandler", + method: "stageBlockFromURL" +}; +operationHandlerMapping[Operation.BlockBlob_CommitBlockList] = { + arguments: [ + "blocks", + "options" + ], + handler: "blockBlobHandler", + method: "commitBlockList" +}; +operationHandlerMapping[Operation.BlockBlob_GetBlockList] = { + arguments: [ + "options" + ], + handler: "blockBlobHandler", + method: "getBlockList" +}; +operationHandlerMapping[Operation.Blob_Undelete] = { + arguments: [ + "options" + ], + handler: "blobHandler", + method: "undelete" +}; +operationHandlerMapping[Operation.Blob_SetExpiry] = { + arguments: [ + "expiryOptions", + "options" + ], + handler: "blobHandler", + method: "setExpiry" +}; +operationHandlerMapping[Operation.Blob_SetHTTPHeaders] = { + arguments: [ + "options" + ], + handler: "blobHandler", + method: "setHTTPHeaders" +}; +operationHandlerMapping[Operation.Blob_SetImmutabilityPolicy] = { + arguments: [ + "options" + ], + handler: "blobHandler", + method: "setImmutabilityPolicy" +}; +operationHandlerMapping[Operation.Blob_DeleteImmutabilityPolicy] = { + arguments: [ + "options" + ], + handler: "blobHandler", + method: "deleteImmutabilityPolicy" +}; +operationHandlerMapping[Operation.Blob_SetLegalHold] = { + arguments: [ + "legalHold", + "options" + ], + handler: "blobHandler", + method: "setLegalHold" +}; +operationHandlerMapping[Operation.Blob_SetMetadata] = { + arguments: [ + "options" + ], + handler: "blobHandler", + method: "setMetadata" +}; +operationHandlerMapping[Operation.Blob_AcquireLease] = { + arguments: [ + "options" + ], + handler: "blobHandler", + method: "acquireLease" +}; +operationHandlerMapping[Operation.Blob_ReleaseLease] = { + arguments: [ + "leaseId", + "options" + ], + handler: "blobHandler", + method: "releaseLease" +}; +operationHandlerMapping[Operation.Blob_RenewLease] = { + arguments: [ + "leaseId", + "options" + ], + handler: "blobHandler", + method: "renewLease" +}; +operationHandlerMapping[Operation.Blob_ChangeLease] = { + arguments: [ + "leaseId", + "proposedLeaseId", + "options" + ], + handler: "blobHandler", + method: "changeLease" +}; +operationHandlerMapping[Operation.Blob_BreakLease] = { + arguments: [ + "options" + ], + handler: "blobHandler", + method: "breakLease" +}; +operationHandlerMapping[Operation.Blob_CreateSnapshot] = { + arguments: [ + "options" + ], + handler: "blobHandler", + method: "createSnapshot" +}; +operationHandlerMapping[Operation.Blob_StartCopyFromURL] = { + arguments: [ + "copySource", + "options" + ], + handler: "blobHandler", + method: "startCopyFromURL" +}; +operationHandlerMapping[Operation.Blob_CopyFromURL] = { + arguments: [ + "copySource", + "options" + ], + handler: "blobHandler", + method: "copyFromURL" +}; +operationHandlerMapping[Operation.Blob_AbortCopyFromURL] = { + arguments: [ + "copyId", + "options" + ], + handler: "blobHandler", + method: "abortCopyFromURL" +}; +operationHandlerMapping[Operation.Blob_SetTier] = { + arguments: [ + "tier", + "options" + ], + handler: "blobHandler", + method: "setTier" +}; +operationHandlerMapping[Operation.Blob_GetAccountInfo] = { + arguments: [], + handler: "blobHandler", + method: "getAccountInfo" +}; +operationHandlerMapping[Operation.Blob_GetAccountInfoWithHead] = { + arguments: [], + handler: "blobHandler", + method: "getAccountInfoWithHead" +}; +operationHandlerMapping[Operation.Blob_Query] = { + arguments: [ + "options" + ], + handler: "blobHandler", + method: "query" +}; +operationHandlerMapping[Operation.Blob_GetTags] = { + arguments: [ + "options" + ], + handler: "blobHandler", + method: "getTags" +}; +operationHandlerMapping[Operation.Blob_SetTags] = { + arguments: [ + "options" + ], + handler: "blobHandler", + method: "setTags" +}; +function getHandlerByOperation(operation: Operation): IHandlerPath | undefined { + return operationHandlerMapping[operation]; +} +export default getHandlerByOperation; diff --git a/src/dfs/generated/middleware/HandlerMiddlewareFactory.ts b/src/dfs/generated/middleware/HandlerMiddlewareFactory.ts new file mode 100644 index 000000000..60a8b1a63 --- /dev/null +++ b/src/dfs/generated/middleware/HandlerMiddlewareFactory.ts @@ -0,0 +1,90 @@ +import Operation from '../artifacts/operation'; +import Specifications from '../artifacts/specifications'; +import Context from '../Context'; +import OperationMismatchError from '../errors/OperationMismatchError'; +import getHandlerByOperation from '../handlers/handlerMappers'; +import IHandlers from '../handlers/IHandlers'; +import { NextFunction } from '../MiddlewareFactory'; +import ILogger from '../utils/ILogger'; + +/** + * Auto generated. HandlerMiddlewareFactory will accept handlers and create handler middleware. + * + * @export + * @class HandlerMiddlewareFactory + */ +export default class HandlerMiddlewareFactory { + /** + * Creates an instance of HandlerMiddlewareFactory. + * Accept handlers and create handler middleware. + * + * @param {IHandlers} handlers Handlers implemented handler interfaces + * @param {ILogger} logger A valid logger + * @memberof HandlerMiddlewareFactory + */ + constructor( + private readonly handlers: IHandlers, + private readonly logger: ILogger + ) {} + + /** + * Creates a handler middleware from input handlers. + * + * @memberof HandlerMiddlewareFactory + */ + public createHandlerMiddleware(): ( + context: Context, + next: NextFunction + ) => void { + return (context: Context, next: NextFunction) => { + this.logger.info( + `HandlerMiddleware: DeserializedParameters=${JSON.stringify( + context.handlerParameters, + (key, value) => { + if (key === "body") { + return "ReadableStream"; + } + return value; + } + )}`, + context.contextId + ); + + if (context.operation === undefined) { + const handlerError = new OperationMismatchError(); + this.logger.error( + `HandlerMiddleware: ${handlerError.message}`, + context.contextId + ); + return next(handlerError); + } + + if (Specifications[context.operation] === undefined) { + this.logger.warn( + `HandlerMiddleware: cannot find handler for operation ${ + Operation[context.operation] + }` + ); + } + + // We assume handlerPath always exists for every generated operation in generated code + const handlerPath = getHandlerByOperation(context.operation)!; + + const args = []; + for (const arg of handlerPath.arguments) { + args.push(context.handlerParameters![arg]); + } + args.push(context); + + const handler = (this.handlers as any)[handlerPath.handler]; + const handlerMethod = handler[handlerPath.method] as () => Promise; + handlerMethod + .apply(handler, args as any) + .then((response: any) => { + context.handlerResponses = response; + }) + .then(next) + .catch(next); + }; + } +} diff --git a/src/dfs/generated/middleware/deserializer.middleware.ts b/src/dfs/generated/middleware/deserializer.middleware.ts new file mode 100644 index 000000000..88813a165 --- /dev/null +++ b/src/dfs/generated/middleware/deserializer.middleware.ts @@ -0,0 +1,59 @@ +import Operation from '../artifacts/operation'; +import Specifications from '../artifacts/specifications'; +import Context from '../Context'; +import DeserializationError from '../errors/DeserializationError'; +import OperationMismatchError from '../errors/OperationMismatchError'; +import IRequest from '../IRequest'; +import { NextFunction } from '../MiddlewareFactory'; +import ILogger from '../utils/ILogger'; +import { deserialize } from '../utils/serializer'; + +/** + * Deserializer Middleware. Deserialize incoming HTTP request into models. + * + * @export + * @param {Context} context + * @param {IRequest} req An IRequest object + * @param {NextFunction} next An next callback or promise + * @param {ILogger} logger A valid logger + * @returns {void} + */ +export default function deserializerMiddleware( + context: Context, + req: IRequest, + next: NextFunction, + logger: ILogger +): void { + logger.verbose( + `DeserializerMiddleware: Start deserializing...`, + context.contextId + ); + + if (context.operation === undefined) { + const handlerError = new OperationMismatchError(); + logger.error( + `DeserializerMiddleware: ${handlerError.message}`, + context.contextId + ); + return next(handlerError); + } + + if (Specifications[context.operation] === undefined) { + logger.warn( + `DeserializerMiddleware: Cannot find deserializer for operation ${ + Operation[context.operation] + }` + ); + } + + deserialize(context, req, Specifications[context.operation], logger) + .then(parameters => { + context.handlerParameters = parameters; + }) + .then(next) + .catch(err => { + const deserializationError = new DeserializationError(err.message); + deserializationError.stack = err.stack; + next(deserializationError); + }); +} diff --git a/src/dfs/generated/middleware/dispatch.middleware.ts b/src/dfs/generated/middleware/dispatch.middleware.ts new file mode 100644 index 000000000..99ac2dffb --- /dev/null +++ b/src/dfs/generated/middleware/dispatch.middleware.ts @@ -0,0 +1,189 @@ +import * as msRest from '@azure/ms-rest-js'; + +import Operation from '../artifacts/operation'; +import Specifications from '../artifacts/specifications'; +import Context from '../Context'; +import UnsupportedRequestError from '../errors/UnsupportedRequestError'; +import IRequest from '../IRequest'; +import { NextFunction } from '../MiddlewareFactory'; +import ILogger from '../utils/ILogger'; +import { isURITemplateMatch } from '../utils/utils'; + +/** + * Dispatch Middleware will try to find out which operation of current HTTP request belongs to, + * by going through request specifications. Operation enum will be assigned to context object. + * Make sure dispatchMiddleware is triggered before other generated middleware. + * + * TODO: Add support for API priorities to deal with both matched APIs + * + * @export + * @param {Context} context Context object + * @param {IRequest} req An request object + * @param {NextFunction} next A callback + * @param {ILogger} logger A valid logger + * @returns {void} + */ +export default function dispatchMiddleware( + context: Context, + req: IRequest, + next: NextFunction, + logger: ILogger +): void { + logger.verbose( + `DispatchMiddleware: Dispatching request...`, + context.contextId + ); + + // Sometimes, more than one operations specifications are all valid against current request + // Such as a SetContainerMetadata request will fit both CreateContainer and SetContainerMetadata specifications + // We need to avoid this kind of situation when define swagger + // However, following code will try to find most suitable operation by selecting operation which + // have most required conditions met + let conditionsMet: number = -1; + + for (const key in Operation) { + if (Operation.hasOwnProperty(key)) { + const operation = parseInt(key, 10); + const res = isRequestAgainstOperation( + req, + Specifications[operation], + context.dispatchPattern + ); + if (res[0] && res[1] > conditionsMet) { + context.operation = operation; + conditionsMet = res[1]; + } + } + } + + if (context.operation === undefined) { + const handlerError = new UnsupportedRequestError(); + logger.error( + `DispatchMiddleware: ${handlerError.message}`, + context.contextId + ); + return next(handlerError); + } + + logger.info( + `DispatchMiddleware: Operation=${Operation[context.operation]}`, + context.contextId + ); + + next(); +} + +/** + * Validation whether current request meets request operation specification. + * + * @param {IRequest} req + * @param {msRest.OperationSpec} spec + * @returns {[boolean, number]} Tuple includes validation result and number of met required conditions + */ +function isRequestAgainstOperation( + req: IRequest, + spec: msRest.OperationSpec, + dispatchPathPattern?: string +): [boolean, number] { + let metConditionsNum = 0; + if (req === undefined || spec === undefined) { + return [false, metConditionsNum]; + } + + const xHttpMethod = req.getHeader("X-HTTP-Method-Override") || req.getHeader("X-HTTP-Method"); + let method = req.getMethod(); + if (xHttpMethod && xHttpMethod.length > 0) { + const value = xHttpMethod.trim(); + if ( + value === "GET" || + value === "MERGE" || + value === "PATCH" || + value === "DELETE" + ) { + method = value; + } + } + + // Validate HTTP method + if (method !== spec.httpMethod) { + return [false, metConditionsNum++]; + } + // Validate URL path + const path = spec.path + ? spec.path.startsWith("/") + ? spec.path + : `/${spec.path}` + : "/"; + if ( + !isURITemplateMatch( + // Use dispatch path with priority + dispatchPathPattern !== undefined ? dispatchPathPattern : req.getPath(), + path + ) + ) { + return [false, metConditionsNum++]; + } + + // Validate required queryParameters + for (const queryParameter of spec.queryParameters || []) { + if (queryParameter.mapper.required) { + const queryValue = req.getQuery( + queryParameter.mapper.serializedName || "" + ); + if (queryValue === undefined) { + return [false, metConditionsNum]; + } + + if ( + queryParameter.mapper.type.name === "Enum" && + queryParameter.mapper.type.allowedValues.findIndex((val) => { + return val === queryValue; + }) < 0 + ) { + return [false, metConditionsNum]; + } + + if ( + queryParameter.mapper.isConstant && + queryParameter.mapper.defaultValue !== queryValue + ) { + return [false, metConditionsNum]; + } + + metConditionsNum++; + } + } + + // Validate required header parameters + for (const headerParameter of spec.headerParameters || []) { + if (headerParameter.mapper.required) { + const headerValue = req.getHeader( + headerParameter.mapper.serializedName || "" + ); + if (headerValue === undefined) { + return [false, metConditionsNum]; + } + + if ( + headerParameter.mapper.type.name === "Enum" && + headerParameter.mapper.type.allowedValues.findIndex((val) => { + return val === headerValue; + }) < 0 + ) { + return [false, metConditionsNum]; + } + + if ( + headerParameter.mapper.isConstant && + `${headerParameter.mapper.defaultValue || ""}`.toLowerCase() !== + headerValue.toLowerCase() + ) { + return [false, metConditionsNum]; + } + + metConditionsNum++; + } + } + + return [true, metConditionsNum]; +} diff --git a/src/dfs/generated/middleware/end.middleware.ts b/src/dfs/generated/middleware/end.middleware.ts new file mode 100644 index 000000000..91692225c --- /dev/null +++ b/src/dfs/generated/middleware/end.middleware.ts @@ -0,0 +1,32 @@ +import Context from '../Context'; +import IResponse from '../IResponse'; +import ILogger from '../utils/ILogger'; + +/** + * End middleware is used to send out final HTTP response. + * + * @export + * @param {Context} context + * @param {Request} req An express compatible Request object + * @param {Response} res An express compatible Response object + * @param {ILogger} logger A valid logger + */ +export default function endMiddleware( + context: Context, + res: IResponse, + logger: ILogger, +): void { + const totalTimeInMS = context.startTime + ? new Date().getTime() - context.startTime.getTime() + : undefined; + + logger.info( + // tslint:disable-next-line:max-line-length + `EndMiddleware: End response. TotalTimeInMS=${totalTimeInMS} StatusCode=${res.getStatusCode()} StatusMessage=${res.getStatusMessage()} Headers=${JSON.stringify( + res.getHeaders() + )}`, + context.contextId + ); + + res.getBodyStream().end(); +} diff --git a/src/dfs/generated/middleware/error.middleware.ts b/src/dfs/generated/middleware/error.middleware.ts new file mode 100644 index 000000000..080dd7250 --- /dev/null +++ b/src/dfs/generated/middleware/error.middleware.ts @@ -0,0 +1,134 @@ +import Context from '../Context'; +import MiddlewareError from '../errors/MiddlewareError'; +import IRequest from '../IRequest'; +import IResponse from '../IResponse'; +import { NextFunction } from '../MiddlewareFactory'; +import ILogger from '../utils/ILogger'; + +/** + * ErrorMiddleware handles following 2 kinds of errors thrown from previous middleware or handlers: + * + * 1. MiddlewareError will be serialized. + * This includes most of expected errors, such as 4XX or some 5xx errors are MiddlewareError. + * + * 2. Other unexpected errors will be serialized to 500 Internal Server error directly. + * Every this kind of error should be carefully checked, and consider to handle it as a MiddlewareError. + * + * @export + * @param {Context} context + * @param {(MiddlewareError | Error)} err A MiddlewareError or Error object + * @param {Request} req An express compatible Request object + * @param {Response} res An express compatible Response object + * @param {NextFunction} next An express middleware next callback + * @param {ILogger} logger A valid logger + * @returns {void} + */ +export default function errorMiddleware( + context: Context, + err: MiddlewareError | Error, + req: IRequest, + res: IResponse, + next: NextFunction, + logger: ILogger +): void { + if (res.headersSent()) { + logger.warn( + `Error middleware received an error, but response.headersSent is true, pass error to next middleware`, + context.contextId + ); + return next(err); + } + + // Only handle ServerError, for other customized error types hand over to + // other error handlers. + if (err instanceof MiddlewareError) { + logger.error( + `ErrorMiddleware: Received a MiddlewareError, fill error information to HTTP response`, + context.contextId + ); + + logger.error( + `ErrorMiddleware: ErrorName=${err.name} ErrorMessage=${ + err.message + } ErrorHTTPStatusCode=${err.statusCode} ErrorHTTPStatusMessage=${ + err.statusMessage + } ErrorHTTPHeaders=${JSON.stringify( + err.headers + )} ErrorHTTPBody=${JSON.stringify(err.body)} ErrorStack=${JSON.stringify( + err.stack + )}`, + context.contextId + ); + + logger.error( + `ErrorMiddleware: Set HTTP code: ${err.statusCode}`, + context.contextId + ); + + res.setStatusCode(err.statusCode); + if (err.statusMessage) { + logger.error( + `ErrorMiddleware: Set HTTP status message: ${err.statusMessage}`, + context.contextId + ); + res.setStatusMessage(err.statusMessage); + } + + if (err.headers) { + for (const key in err.headers) { + if (err.headers.hasOwnProperty(key)) { + const value = err.headers[key]; + if (value) { + logger.error( + `ErrorMiddleware: Set HTTP Header: ${key}=${value}`, + context.contextId + ); + res.setHeader(key, value); + } + } + } + } + + if (err.contentType && req.getMethod() !== "HEAD") { + logger.error( + `ErrorMiddleware: Set content type: ${err.contentType}`, + context.contextId + ); + res.setContentType(err.contentType); + } + + logger.error( + `ErrorMiddleware: Set HTTP body: ${JSON.stringify(err.body)}`, + context.contextId + ); + if (err.body && req.getMethod() !== "HEAD") { + res.getBodyStream().write(err.body); + } + } else if (err instanceof Error) { + logger.error( + `ErrorMiddleware: Received an error, fill error information to HTTP response`, + context.contextId + ); + logger.error( + `ErrorMiddleware: ErrorName=${err.name} ErrorMessage=${ + err.message + } ErrorStack=${JSON.stringify(err.stack)}`, + context.contextId + ); + logger.error(`ErrorMiddleware: Set HTTP code: ${500}`, context.contextId); + res.setStatusCode(500); + + // logger.error( + // `ErrorMiddleware: Set error message: ${err.message}`, + // context.contextID + // ); + // res.getBodyStream().write(err.message); + } else { + logger.warn( + `ErrorMiddleware: Received unhandled error object`, + context.contextId + ); + } + + next(); +} diff --git a/src/dfs/generated/middleware/serializer.middleware.ts b/src/dfs/generated/middleware/serializer.middleware.ts new file mode 100644 index 000000000..66460cf10 --- /dev/null +++ b/src/dfs/generated/middleware/serializer.middleware.ts @@ -0,0 +1,57 @@ +import Operation from '../artifacts/operation'; +import Specifications from '../artifacts/specifications'; +import Context from '../Context'; +import OperationMismatchError from '../errors/OperationMismatchError'; +import IResponse from '../IResponse'; +import { NextFunction } from '../MiddlewareFactory'; +import ILogger from '../utils/ILogger'; +import { serialize } from '../utils/serializer'; + +/** + * SerializerMiddleware will serialize models into HTTP responses. + * + * @export + * @param {Response} res + * @param {NextFunction} next + * @param {ILogger} logger + * @param {Context} context + */ +export default function serializerMiddleware( + context: Context, + res: IResponse, + next: NextFunction, + logger: ILogger +): void { + logger.verbose( + `SerializerMiddleware: Start serializing...`, + context.contextId + ); + + if (context.operation === undefined) { + const handlerError = new OperationMismatchError(); + logger.error( + `SerializerMiddleware: ${handlerError.message}`, + context.contextId + ); + return next(handlerError); + } + + if (Specifications[context.operation] === undefined) { + logger.warn( + `SerializerMiddleware: Cannot find serializer for operation ${ + Operation[context.operation] + }`, + context.contextId + ); + } + + serialize( + context, + res, + Specifications[context.operation], + context.handlerResponses, + logger + ) + .then(next) + .catch(next); +} diff --git a/src/dfs/generated/utils/ILogger.ts b/src/dfs/generated/utils/ILogger.ts new file mode 100644 index 000000000..cef94893d --- /dev/null +++ b/src/dfs/generated/utils/ILogger.ts @@ -0,0 +1,13 @@ +/** + * An interface of logger used by generated code. + * + * @export + * @interface ILogger + */ +export default interface ILogger { + error(message: string, contextID?: string): void; + warn(message: string, contextID?: string): void; + info(message: string, contextID?: string): void; + verbose(message: string, contextID?: string): void; + debug(message: string, contextID?: string): void; +} diff --git a/src/dfs/generated/utils/serializer.ts b/src/dfs/generated/utils/serializer.ts new file mode 100644 index 000000000..8689cbff4 --- /dev/null +++ b/src/dfs/generated/utils/serializer.ts @@ -0,0 +1,392 @@ +import * as msRest from '@azure/ms-rest-js'; + +import * as Mappers from '../artifacts/mappers'; +import Context, { IHandlerParameters } from '../Context'; +import IRequest from '../IRequest'; +import IResponse from '../IResponse'; +import ILogger from './ILogger'; +import { parseXML, stringifyXML } from './xml'; + +export declare type ParameterPath = + | string + | string[] + | { + [propertyName: string]: ParameterPath; + }; + +export async function deserialize( + context: Context, + req: IRequest, + spec: msRest.OperationSpec, + logger: ILogger +): Promise { + const parameters: IHandlerParameters = {}; + + // Deserialize query parameters + for (const queryParameter of spec.queryParameters || []) { + if (!queryParameter.mapper.serializedName) { + throw new TypeError( + `QueryParameter mapper doesn't include valid "serializedName"` + ); + } + const queryKey = queryParameter.mapper.serializedName; + let queryValueOriginal: string | string[] | undefined = req.getQuery( + queryKey + ); + + if ( + queryValueOriginal !== undefined && + queryParameter.collectionFormat !== undefined && + queryParameter.mapper.type.name === "Sequence" + ) { + queryValueOriginal = `${queryValueOriginal}`.split( + queryParameter.collectionFormat + ); + } + + const queryValue = spec.serializer.deserialize( + queryParameter.mapper, + queryValueOriginal, + queryKey + ); + + // TODO: Currently validation is only in serialize method, + // remove when adding validateConstraints to deserialize() + // TODO: Make serialize return ServerError according to different validations? + spec.serializer.serialize(queryParameter.mapper, queryValue); + + setParametersValue(parameters, queryParameter.parameterPath, queryValue); + } + + // Deserialize header parameters + for (const headerParameter of spec.headerParameters || []) { + if (!headerParameter.mapper.serializedName) { + throw new TypeError( + `HeaderParameter mapper doesn't include valid "serializedName"` + ); + } + + const headerCollectionPrefix: + | string + | undefined = (headerParameter.mapper as msRest.DictionaryMapper) + .headerCollectionPrefix; + if (headerCollectionPrefix) { + const dictionary: any = {}; + const headers = req.getHeaders(); + for (const headerKey of Object.keys(headers)) { + if ( + headerKey + .toLowerCase() + .startsWith(headerCollectionPrefix.toLocaleLowerCase()) + ) { + // TODO: Validate collection type by serializer + dictionary[ + headerKey.substring(headerCollectionPrefix.length) + ] = spec.serializer.serialize( + (headerParameter.mapper as msRest.DictionaryMapper).type.value, + headers[headerKey], + headerKey + ); + } + } + setParametersValue(parameters, headerParameter.parameterPath, dictionary); + } else { + const headerKey = headerParameter.mapper.serializedName; + const headerValueOriginal = req.getHeader(headerKey); + const headerValue = spec.serializer.deserialize( + headerParameter.mapper, + headerValueOriginal, + headerKey + ); + + // TODO: Currently validation is only in serialize method, + // remove when adding validateConstraints to deserialize() + spec.serializer.serialize(headerParameter.mapper, headerValue); + + setParametersValue( + parameters, + headerParameter.parameterPath, + headerValue + ); + } + } + + // Deserialize body + const bodyParameter = spec.requestBody; + + if (bodyParameter && bodyParameter.mapper.type.name === "Stream") { + setParametersValue(parameters, "body", req.getBodyStream()); + } else if (bodyParameter) { + const jsonContentTypes = ["application/json", "text/json"]; + const xmlContentTypes = ["application/xml", "application/atom+xml"]; + const contentType = req.getHeader("content-type") || ""; + const contentComponents = !contentType + ? [] + : contentType.split(";").map(component => component.toLowerCase()); + + const isRequestWithJSON = contentComponents.some( + component => jsonContentTypes.indexOf(component) !== -1 + ); // TODO + const isRequestWithXML = + spec.isXML || + contentComponents.some( + component => xmlContentTypes.indexOf(component) !== -1 + ); + // const isRequestWithStream = false; + + const body = await readRequestIntoText(req); + logger.debug( + `deserialize(): Raw request body string is (removed all empty characters) ${body.replace( + /\s/g, + "" + )}`, + context.contextId + ); + + req.setBody(body); + let parsedBody: object = {}; + if (isRequestWithJSON) { + // read body + parsedBody = JSON.parse(body); + } else if (isRequestWithXML) { + parsedBody = (await parseXML(body)) || {}; + } + + let valueToDeserialize: any = parsedBody; + if ( + spec.isXML && + bodyParameter.mapper.type.name === msRest.MapperType.Sequence + ) { + valueToDeserialize = + typeof valueToDeserialize === "object" + ? valueToDeserialize[bodyParameter.mapper.xmlElementName!] + : []; + } + + parsedBody = spec.serializer.deserialize( + bodyParameter.mapper, + valueToDeserialize, + bodyParameter.mapper.serializedName! + ); + + // Validation purpose only, because only serialize supports validation + // TODO: Inject convenience layer error into deserialize; Drop @azure/ms-rest-js, move logic into generated code + spec.serializer.serialize(bodyParameter.mapper, parsedBody); + + setParametersValue(parameters, bodyParameter.parameterPath, parsedBody); + setParametersValue(parameters, "body", req.getBody()); + } + + return parameters; +} + +async function readRequestIntoText(req: IRequest): Promise { + return new Promise((resolve, reject) => { + const segments: string[] = []; + const bodyStream = req.getBodyStream(); + bodyStream.on("data", buffer => { + segments.push(buffer); + }); + bodyStream.on("error", reject); + bodyStream.on("end", () => { + const joined = segments.join(""); + resolve(joined); + }); + }); +} + +function setParametersValue( + parameters: IHandlerParameters, + parameterPath: ParameterPath, + parameterValue: any +) { + if (typeof parameterPath === "string") { + parameters[parameterPath] = parameterValue; + } else if (Array.isArray(parameterPath)) { + let leafParent = parameters; + for (let i = 0; i < parameterPath.length - 1; i++) { + const currentPropertyName = parameterPath[i]; + if (!leafParent[currentPropertyName]) { + leafParent[currentPropertyName] = {}; + } + leafParent = leafParent[currentPropertyName]; + } + + const lastPropertyName = parameterPath[parameterPath.length - 1]; + leafParent[lastPropertyName] = parameterValue; + } else { + throw new TypeError(`parameterPath is not string or string[]`); + } +} + +export async function serialize( + context: Context, + res: IResponse, + spec: msRest.OperationSpec, + handlerResponse: any, + logger: ILogger +): Promise { + const statusCodeInResponse: number = handlerResponse.statusCode; + res.setStatusCode(statusCodeInResponse); + + const responseSpec = spec.responses[statusCodeInResponse]; + if (!responseSpec) { + throw new TypeError( + `Request specification doesn't include provided response status code` + ); + } + + // Serialize headers + const headerSerializer = new msRest.Serializer(Mappers); + const headersMapper = responseSpec.headersMapper; + if (headersMapper && headersMapper.type.name === "Composite") { + const mappersForAllHeaders = headersMapper.type.modelProperties || {}; + + // Handle headerMapper one by one + for (const key in mappersForAllHeaders) { + if (mappersForAllHeaders.hasOwnProperty(key)) { + const headerMapper = mappersForAllHeaders[key]; + const headerName = headerMapper.serializedName; + const headerValueOriginal = handlerResponse[key]; + const headerValueSerialized = headerSerializer.serialize( + headerMapper, + headerValueOriginal + ); + + // Handle collection of headers starting with same prefix, such as x-ms-meta prefix + const headerCollectionPrefix = (headerMapper as msRest.DictionaryMapper) + .headerCollectionPrefix; + if ( + headerCollectionPrefix !== undefined && + headerValueOriginal !== undefined + ) { + for (const collectionHeaderPartialName in headerValueSerialized) { + if ( + headerValueSerialized.hasOwnProperty(collectionHeaderPartialName) + ) { + const collectionHeaderValueSerialized = + headerValueSerialized[collectionHeaderPartialName]; + const collectionHeaderName = `${headerCollectionPrefix}${collectionHeaderPartialName}`; + if ( + collectionHeaderName && + collectionHeaderValueSerialized !== undefined + ) { + res.setHeader( + collectionHeaderName, + collectionHeaderValueSerialized + ); + } + } + } + } else { + if (headerName && headerValueSerialized !== undefined) { + res.setHeader(headerName, headerValueSerialized); + } + } + } + } + } + + // Serialize XML bodies + if ( + spec.isXML && + responseSpec.bodyMapper && + responseSpec.bodyMapper.type.name !== "Stream" + ) { + let body = spec.serializer.serialize( + responseSpec.bodyMapper!, + handlerResponse + ); + + // When root element is sequence type, should wrap with because serialize() doesn't do that + if (responseSpec.bodyMapper!.type.name === "Sequence") { + const sequenceElementName = responseSpec.bodyMapper!.xmlElementName; + if (sequenceElementName !== undefined) { + const newBody = {} as any; + newBody[sequenceElementName] = body; + body = newBody; + } + } + + const xmlBody = stringifyXML(body, { + rootName: + responseSpec.bodyMapper!.xmlName || + responseSpec.bodyMapper!.serializedName + }); + res.setContentType(`application/xml`); + + // TODO: Should send response in a serializer? + res.getBodyStream().write(xmlBody); + logger.debug( + `Serializer: Raw response body string is ${xmlBody}`, + context.contextId + ); + logger.info(`Serializer: Start returning stream body.`, context.contextId); + } + + // Serialize JSON bodies + if ( + !spec.isXML && + responseSpec.bodyMapper && + responseSpec.bodyMapper.type.name !== "Stream" + ) { + let body = spec.serializer.serialize( + responseSpec.bodyMapper!, + handlerResponse + ); + + // When root element is sequence type, should wrap with because serialize() doesn't do that + if (responseSpec.bodyMapper!.type.name === "Sequence") { + const sequenceElementName = responseSpec.bodyMapper!.xmlElementName; + if (sequenceElementName !== undefined) { + const newBody = {} as any; + newBody[sequenceElementName] = body; + body = newBody; + } + } + + if (!res.getHeader("content-type")) { + res.setContentType("application/json"); + } + + const jsonBody = JSON.stringify(body); + + // TODO: Should send response in a serializer? + res.getBodyStream().write(jsonBody); + logger.debug( + `Serializer: Raw response body string is ${jsonBody}`, + context.contextId + ); + logger.info(`Serializer: Start returning stream body.`, context.contextId); + } + + // Serialize stream body + // TODO: Move to end middleware for end tracking + if ( + handlerResponse.body && + responseSpec.bodyMapper && + responseSpec.bodyMapper.type.name === "Stream" + ) { + logger.info(`Serializer: Start returning stream body.`, context.contextId); + + await new Promise((resolve, reject) => { + (handlerResponse.body as NodeJS.ReadableStream) + .on("error", reject) + .pipe(res.getBodyStream()) + .on("error", reject) + .on("close", resolve); + }); + + // const totalTimeInMS = context.startTime + // ? new Date().getTime() - context.startTime.getTime() + // : undefined; + + // logger.info( + // tslint:disable-next-line:max-line-length + // `Serializer: End response. TotalTimeInMS=${totalTimeInMS} StatusCode=${res.getStatusCode()} StatusMessage=${res.getStatusMessage()} Headers=${JSON.stringify( + // res.getHeaders() + // )}`, + // context.contextID + // ); + } +} diff --git a/src/dfs/generated/utils/utils.ts b/src/dfs/generated/utils/utils.ts new file mode 100644 index 000000000..8e39e4dd9 --- /dev/null +++ b/src/dfs/generated/utils/utils.ts @@ -0,0 +1,20 @@ +import URITemplate from 'uri-templates'; + +export function isURITemplateMatch(url: string, template: string): boolean { + const uriTemplate = URITemplate(template); + // TODO: Fixing $ parsing issue such as $logs container cannot work in strict mode issue + const result = (uriTemplate.fromUri as any)(url, { strict: true }); + if (result === undefined) { + return false; + } + + for (const key in result) { + if (result.hasOwnProperty(key)) { + const element = result[key]; + if (element === "") { + return false; + } + } + } + return true; +} \ No newline at end of file diff --git a/src/dfs/generated/utils/xml.ts b/src/dfs/generated/utils/xml.ts new file mode 100644 index 000000000..057eaac91 --- /dev/null +++ b/src/dfs/generated/utils/xml.ts @@ -0,0 +1,41 @@ +import * as xml2js from 'xml2js'; + +export function stringifyXML(obj: any, opts?: { rootName?: string }) { + const builder = new xml2js.Builder({ + explicitArray: false, + explicitCharkey: false, + renderOpts: { + pretty: false + }, + rootName: (opts || {}).rootName + }); + return builder.buildObject(obj); +} + +export function parseXML( + str: string, + explicitChildrenWithOrder: boolean = false +): Promise { + const xmlParser = new xml2js.Parser({ + explicitArray: false, + explicitCharkey: false, + explicitRoot: false, + preserveChildrenOrder: explicitChildrenWithOrder, + explicitChildren: explicitChildrenWithOrder, + emptyTag: undefined + }); + return new Promise((resolve, reject) => { + xmlParser.parseString(str, (err?: Error, res?: any) => { + if (err) { + reject(err); + } else { + resolve(res); + } + }); + }); +} + +export function jsonToXML(json: any): string { + const build = new xml2js.Builder(); + return build.buildObject(json); +} diff --git a/src/dfs/handlers/AppendBlobHandler.ts b/src/dfs/handlers/AppendBlobHandler.ts new file mode 100644 index 000000000..353ec5cfd --- /dev/null +++ b/src/dfs/handlers/AppendBlobHandler.ts @@ -0,0 +1,244 @@ +import { convertRawHeadersToMetadata } from "../../common/utils/utils"; +import { getMD5FromStream, newEtag } from "../../common/utils/utils"; +import { + DEFAULT_FILE_PERMISSIONS, + DEFAULT_GROUP, + DEFAULT_OWNER +} from "../../dfs/utils/constants"; +import DataLakeContext from "../context/DataLakeContext"; +import NotImplementedError from "../errors/NotImplementedError"; +import StorageErrorFactory from "../errors/StorageErrorFactory"; +import * as Models from "../generated/artifacts/models"; +import Context from "../generated/Context"; +import IAppendBlobHandler from "../generated/handlers/IAppendBlobHandler"; +import { BlobModel } from "../persistence/IDataLakeMetadataStore"; +import { toPermissions } from "../storage-file-datalake/transforms"; +import { + BLOB_API_VERSION, + HeaderConstants, + MAX_APPEND_BLOB_BLOCK_COUNT, + MAX_APPEND_BLOB_BLOCK_SIZE +} from "../utils/constants"; +import BaseHandler from "./BaseHandler"; + +export default class AppendBlobHandler + extends BaseHandler + implements IAppendBlobHandler +{ + public async create( + contentLength: number, + options: Models.AppendBlobCreateOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const accountName = blobCtx.account!; + const containerName = blobCtx.container!; + const blobName = blobCtx.blob!; + const date = blobCtx.startTime!; + const etag = newEtag(); + + if (contentLength !== 0 && !this.loose) { + throw StorageErrorFactory.getInvalidOperation( + blobCtx, + "Content-Length must be 0 for Create Append Blob request." + ); + } + + options.blobHTTPHeaders = options.blobHTTPHeaders || {}; + const contentType = + options.blobHTTPHeaders.blobContentType || + context.request!.getHeader("content-type") || + "application/octet-stream"; + + // Preserve metadata key case + const metadata = convertRawHeadersToMetadata( + blobCtx.request!.getRawHeaders() + ); + + const blob: BlobModel = { + deleted: false, + metadata, + accountName, + containerName, + name: blobName, + properties: { + creationTime: date, + lastModified: date, + etag, + contentLength: 0, + contentType, + contentEncoding: options.blobHTTPHeaders.blobContentEncoding, + contentLanguage: options.blobHTTPHeaders.blobContentLanguage, + contentMD5: options.blobHTTPHeaders.blobContentMD5, + contentDisposition: options.blobHTTPHeaders.blobContentDisposition, + cacheControl: options.blobHTTPHeaders.blobCacheControl, + blobType: Models.BlobType.AppendBlob, + leaseStatus: Models.LeaseStatusType.Unlocked, + leaseState: Models.LeaseStateType.Available, + serverEncrypted: true + }, + snapshot: "", + isCommitted: true, + isDirectory: false, + permissions: toPermissions(DEFAULT_FILE_PERMISSIONS)!, + acl: [], + owner: DEFAULT_OWNER, + group: DEFAULT_GROUP, + committedBlocksInOrder: [] + }; + + await this.metadataStore.createBlob( + context, + blob, + options.leaseAccessConditions, + options.modifiedAccessConditions + ); + + const response: Models.AppendBlobCreateResponse = { + statusCode: 201, + eTag: etag, + lastModified: blob.properties.lastModified, + contentMD5: blob.properties.contentMD5, + requestId: context.contextId, + version: BLOB_API_VERSION, + date, + isServerEncrypted: true, + clientRequestId: options.requestId + }; + + return response; + } + + public async appendBlock( + body: NodeJS.ReadableStream, + contentLength: number, + options: Models.AppendBlobAppendBlockOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const accountName = blobCtx.account!; + const containerName = blobCtx.container!; + const blobName = blobCtx.blob!; + const date = blobCtx.startTime!; + + if (contentLength > MAX_APPEND_BLOB_BLOCK_SIZE) { + throw StorageErrorFactory.getRequestEntityTooLarge(blobCtx); + } + + if (contentLength === 0) { + throw StorageErrorFactory.getInvalidHeaderValue(blobCtx, { + HeaderName: HeaderConstants.CONTENT_LENGTH, + HeaderValue: "0" + }); + } + + // TODO: Optimize with cache + const blob = await this.metadataStore.downloadBlob( + blobCtx, + accountName, + containerName, + blobName, + undefined + ); + + if (blob.properties.blobType !== Models.BlobType.AppendBlob) { + throw StorageErrorFactory.getBlobInvalidBlobType(blobCtx); + } + + const committedBlockCount = (blob.committedBlocksInOrder || []).length; + if (committedBlockCount >= MAX_APPEND_BLOB_BLOCK_COUNT) { + throw StorageErrorFactory.getBlockCountExceedsLimit(blobCtx); + } + + // Persist content + const extent = await this.extentStore.appendExtent(body, blobCtx.contextId); + if (extent.count !== contentLength) { + throw StorageErrorFactory.getInvalidOperation( + blobCtx, + `The size of the request body ${extent.count} mismatches the content-length ${contentLength}.` + ); + } + + // MD5 + const contentMD5 = blobCtx.request!.getHeader(HeaderConstants.CONTENT_MD5); + let contentMD5Buffer; + let contentMD5String; + + if (contentMD5 !== undefined) { + contentMD5Buffer = + typeof contentMD5 === "string" + ? Buffer.from(contentMD5, "base64") + : contentMD5; + contentMD5String = + typeof contentMD5 === "string" + ? contentMD5 + : contentMD5Buffer.toString("base64"); + + const stream = await this.extentStore.readExtent( + extent, + blobCtx.contextId + ); + const calculatedContentMD5Buffer = await getMD5FromStream(stream); + const calculatedContentMD5String = Buffer.from( + calculatedContentMD5Buffer + ).toString("base64"); + + if (contentMD5String !== calculatedContentMD5String) { + throw StorageErrorFactory.getMd5Mismatch( + context, + contentMD5String, + calculatedContentMD5String + ); + } + } + + const originOffset = blob.properties.contentLength; + + const properties = await this.metadataStore.appendBlock( + blobCtx, + { + accountName, + containerName, + blobName, + isCommitted: true, + name: "", // No block ID for append block + size: extent.count, + persistency: extent + }, + options.leaseAccessConditions, + options.modifiedAccessConditions, + options.appendPositionAccessConditions + ); + + const response: Models.AppendBlobAppendBlockResponse = { + statusCode: 201, + eTag: properties.etag, + lastModified: properties.lastModified, + contentMD5: contentMD5Buffer, + xMsContentCrc64: undefined, + clientRequestId: options.requestId, + version: BLOB_API_VERSION, + date, + blobAppendOffset: `${originOffset}`, + blobCommittedBlockCount: committedBlockCount + 1, + isServerEncrypted: true + }; + + return response; + } + + public appendBlockFromUrl( + sourceUrl: string, + contentLength: number, + options: Models.AppendBlobAppendBlockFromUrlOptionalParams, + context: Context + ): Promise { + throw new NotImplementedError(context); + } + public async seal( + options: Models.AppendBlobSealOptionalParams, + context: Context + ): Promise { + throw new NotImplementedError(context); + } +} diff --git a/src/dfs/handlers/BaseHandler.ts b/src/dfs/handlers/BaseHandler.ts new file mode 100644 index 000000000..e848d915a --- /dev/null +++ b/src/dfs/handlers/BaseHandler.ts @@ -0,0 +1,20 @@ +import IExtentStore from "../../common/persistence/IExtentStore"; +import ILogger from "../generated/utils/ILogger"; +import IDataLakeMetadataStore from "../persistence/IDataLakeMetadataStore"; + +/** + * BaseHandler class should maintain a singleton to persistency layer, such as maintain a database connection pool. + * So every inherited classes instances can reuse the persistency layer connection. + * + * @export + * @class SimpleHandler + * @implements {IHandler} + */ +export default class BaseHandler { + constructor( + protected readonly metadataStore: IDataLakeMetadataStore, + protected readonly extentStore: IExtentStore, + protected readonly logger: ILogger, + protected readonly loose: boolean + ) {} +} diff --git a/src/dfs/handlers/BlobBatchHandler.ts b/src/dfs/handlers/BlobBatchHandler.ts new file mode 100644 index 000000000..a97368d71 --- /dev/null +++ b/src/dfs/handlers/BlobBatchHandler.ts @@ -0,0 +1,674 @@ +import { URLBuilder } from "@azure/ms-rest-js"; +import IAccountDataStore from "../../common/IAccountDataStore"; +import { OAuthLevel } from "../../common/models"; +import IExtentStore from "../../common/persistence/IExtentStore"; +import AccountSASAuthenticator from "../authentication/AccountSASAuthenticator"; +import BlobSASAuthenticator from "../authentication/BlobSASAuthenticator"; +import BlobSharedKeyAuthenticator from "../authentication/BlobSharedKeyAuthenticator"; +import BlobTokenAuthenticator from "../authentication/BlobTokenAuthenticator"; +import IAuthenticator from "../authentication/IAuthenticator"; +import PublicAccessAuthenticator from "../authentication/PublicAccessAuthenticator"; +import DataLakeContext from "../context/DataLakeContext"; +import StorageErrorFactory from "../errors/StorageErrorFactory"; +import Operation from "../generated/artifacts/operation"; +import Context from "../generated/Context"; +import MiddlewareError from "../generated/errors/MiddlewareError"; +import IHandlers from "../generated/handlers/IHandlers"; +import IRequest, { HttpMethod } from "../generated/IRequest"; +import IResponse from "../generated/IResponse"; +import deserializerMiddleware from "../generated/middleware/deserializer.middleware"; +import dispatchMiddleware from "../generated/middleware/dispatch.middleware"; +import endMiddleware from "../generated/middleware/end.middleware"; +import errorMiddleware from "../generated/middleware/error.middleware"; +import HandlerMiddlewareFactory from "../generated/middleware/HandlerMiddlewareFactory"; +import serializerMiddleware from "../generated/middleware/serializer.middleware"; +import ILogger from "../generated/utils/ILogger"; +import AuthenticationMiddlewareFactory from "../middlewares/AuthenticationMiddlewareFactory"; +import { internnalBlobStorageContextMiddleware } from "../middlewares/blobStorageContext.middleware"; +import IDataLakeMetadataStore from "../persistence/IDataLakeMetadataStore"; +import { + DEFAULT_CONTEXT_PATH, + HTTP_HEADER_DELIMITER, + HTTP_LINE_ENDING +} from "../utils/constants"; +import AppendBlobHandler from "./AppendBlobHandler"; +import { BlobBatchSubRequest } from "./BlobBatchSubRequest"; +import { BlobBatchSubResponse } from "./BlobBatchSubResponse"; +import BlobHandler from "./BlobHandler"; +import BlockBlobHandler from "./BlockBlobHandler"; +import ContainerHandler from "./ContainerHandler"; +import PageBlobHandler from "./PageBlobHandler"; +import PageBlobRangesManager from "./PageBlobRangesManager"; +import ServiceHandler from "./ServiceHandler"; +import FileSystemOperationsHandler from "./FileSystemOperationsHandler"; +import PathOperationsHandler from "./PathOperationsHandler"; +import DataLakeError from "../errors/DataLakeError"; + +type SubRequestNextFunction = (err?: any) => void; +type SubRequestHandler = ( + req: IRequest, + res: IResponse, + locals: any, + next: SubRequestNextFunction +) => any; +type SubRequestErrorHandler = ( + err: any, + req: IRequest, + res: IResponse, + locals: any, + next: SubRequestNextFunction +) => any; + +export class BlobBatchHandler { + private handlers: IHandlers; + private authenticators: IAuthenticator[]; + + private handlePipeline: SubRequestHandler[]; + private errorHandler: SubRequestErrorHandler; + private operationFinder: SubRequestHandler[]; + + constructor( + private readonly accountDataStore: IAccountDataStore, + private readonly oauth: OAuthLevel | undefined, + private readonly metadataStore: IDataLakeMetadataStore, + private readonly extentStore: IExtentStore, + private readonly logger: ILogger, + private readonly loose: boolean + ) { + const subRequestContextMiddleware = ( + req: IRequest, + res: IResponse, + locals: any, + next: SubRequestNextFunction + ) => { + const urlbuilder = URLBuilder.parse(req.getUrl()); + internnalBlobStorageContextMiddleware( + new DataLakeContext(locals, DEFAULT_CONTEXT_PATH), + req, + res, + urlbuilder.getHost()!, + urlbuilder.getPath()!, + next, + true, + true, + this.loose + ); + }; + + const subRequestDispatchMiddleware = ( + req: IRequest, + res: IResponse, + locals: any, + next: SubRequestNextFunction + ) => { + dispatchMiddleware( + new Context(locals, DEFAULT_CONTEXT_PATH, req, res), + req, + next, + this.logger + ); + }; + + // AuthN middleware, like shared key auth or SAS auth + const authenticationMiddlewareFactory = new AuthenticationMiddlewareFactory( + this.logger + ); + + this.authenticators = [ + new PublicAccessAuthenticator(this.metadataStore, logger), + new BlobSharedKeyAuthenticator(this.accountDataStore, logger), + new AccountSASAuthenticator( + this.accountDataStore, + this.metadataStore, + logger + ), + new BlobSASAuthenticator( + this.accountDataStore, + this.metadataStore, + logger + ) + ]; + + if (this.oauth !== undefined) { + this.authenticators.push( + new BlobTokenAuthenticator(this.accountDataStore, this.oauth, logger) + ); + } + + const subRequestAuthenticationMiddleware = ( + req: IRequest, + res: IResponse, + locals: any, + next: SubRequestNextFunction + ) => { + const context = new DataLakeContext(locals, DEFAULT_CONTEXT_PATH); + authenticationMiddlewareFactory + .authenticate(context, req, res, this.authenticators) + .then((pass) => { + // TODO: To support public access, we need to modify here to reject request later in handler + if (pass) { + next(); + } else { + next(StorageErrorFactory.getAuthorizationFailure(context)); + } + }) + .catch((errorInfo) => next(errorInfo)); + }; + + const subRequestDeserializeMiddleware = ( + req: IRequest, + res: IResponse, + locals: any, + next: SubRequestNextFunction + ) => { + deserializerMiddleware( + new Context(locals, DEFAULT_CONTEXT_PATH, req, res), + req, + next, + this.logger + ); + }; + + this.handlers = { + appendBlobHandler: new AppendBlobHandler( + this.metadataStore, + this.extentStore, + this.logger, + this.loose + ), + blobHandler: new BlobHandler( + this.metadataStore, + this.extentStore, + this.logger, + this.loose + ), + blockBlobHandler: new BlockBlobHandler( + this.metadataStore, + this.extentStore, + this.logger, + this.loose + ), + containerHandler: new ContainerHandler( + this.accountDataStore, + this.oauth, + this.metadataStore, + this.extentStore, + this.logger, + this.loose + ), + pageBlobHandler: new PageBlobHandler( + this.metadataStore, + this.extentStore, + this.logger, + this.loose, + new PageBlobRangesManager() + ), + serviceHandler: new ServiceHandler( + this.accountDataStore, + this.oauth, + this.metadataStore, + this.extentStore, + this.logger, + this.loose + ), + fileSystemOperationsHandler: new FileSystemOperationsHandler( + this.accountDataStore, + this.oauth, + this.metadataStore, + this.extentStore, + logger, + loose + ), + pathOperationsHandler: new PathOperationsHandler( + this.metadataStore, + this.extentStore, + logger, + loose, + new PageBlobRangesManager() + ) + }; + + const handlerMiddlewareFactory = new HandlerMiddlewareFactory( + this.handlers, + this.logger + ); + + const subRequestHandlerMiddleware = ( + req: IRequest, + res: IResponse, + locals: any, + next: SubRequestNextFunction + ) => { + handlerMiddlewareFactory.createHandlerMiddleware()( + new Context(locals, DEFAULT_CONTEXT_PATH, req, res), + next + ); + }; + + const subRequestSerializeMiddleWare = ( + req: IRequest, + res: IResponse, + locals: any, + next: SubRequestNextFunction + ) => { + serializerMiddleware( + new Context(locals, DEFAULT_CONTEXT_PATH, req, res), + res, + next, + this.logger + ); + }; + + const subRequestErrorMiddleWare = ( + err: any, + req: IRequest, + res: IResponse, + locals: any, + next: SubRequestNextFunction + ) => { + errorMiddleware( + new Context(locals, DEFAULT_CONTEXT_PATH, req, res), + err, + req, + res, + next, + this.logger + ); + }; + + const subRequestEndMiddleWare = ( + req: IRequest, + res: IResponse, + locals: any, + next: SubRequestNextFunction + ) => { + endMiddleware( + new Context(locals, DEFAULT_CONTEXT_PATH, req, res), + res, + this.logger + ); + next(); + }; + + this.handlePipeline = [ + subRequestContextMiddleware, + subRequestDispatchMiddleware, + subRequestAuthenticationMiddleware, + subRequestDeserializeMiddleware, + subRequestHandlerMiddleware, + subRequestSerializeMiddleWare, + subRequestEndMiddleWare + ]; + + this.operationFinder = [ + subRequestContextMiddleware, + subRequestDispatchMiddleware + ]; + + this.errorHandler = subRequestErrorMiddleWare; + } + + private async streamToBuffer2( + stream: NodeJS.ReadableStream, + buffer: Buffer, + encoding?: BufferEncoding + ): Promise { + let pos = 0; // Position in stream + const bufferSize = buffer.length; + + return new Promise((resolve, reject) => { + stream.on("readable", () => { + let chunk = stream.read(); + if (!chunk) { + return; + } + if (typeof chunk === "string") { + chunk = Buffer.from(chunk, encoding); + } + + if (pos + chunk.length > bufferSize) { + reject( + new Error(`Stream exceeds buffer size. Buffer size: ${bufferSize}`) + ); + return; + } + + buffer.fill(chunk, pos, pos + chunk.length); + pos += chunk.length; + }); + + stream.on("end", () => { + resolve(pos); + }); + + stream.on("error", reject); + }); + } + + private async requestBodyToString( + body: NodeJS.ReadableStream + ): Promise { + let buffer = Buffer.alloc(4 * 1024 * 1024); + + const responseLength = await this.streamToBuffer2(body, buffer); + + // Slice the buffer to trim the empty ending. + buffer = buffer.slice(0, responseLength); + + return buffer.toString(); + } + + private async getSubRequestOperation(request: IRequest): Promise { + const subRequestHandlePipeline = this.operationFinder; + const fakeResponse = new BlobBatchSubResponse(0, "HTTP/1.1"); + return new Promise((resolve, reject) => { + const locals: any = {}; + let i = 0; + const next = (error: any) => { + if (error) { + reject(error); + } else { + ++i; + if (i < subRequestHandlePipeline.length) { + subRequestHandlePipeline[i](request, fakeResponse, locals, next); + } else { + resolve( + new Context(locals, DEFAULT_CONTEXT_PATH, request, fakeResponse) + .operation! + ); + } + } + }; + + subRequestHandlePipeline[i](request, fakeResponse, locals, next); + }); + } + + private async parseSubRequests( + commonRequestId: string, + perRequestPrefix: string, + batchRequestEnding: string, + subRequestPathPrefix: string, + request: IRequest, + body: string, + context: Context + ): Promise { + const requestAll = body.split(batchRequestEnding); + const response1 = requestAll[0]; // string after ending is useless + const response2 = response1.split(perRequestPrefix); + const subRequests = response2.slice(1); + + const blobBatchSubRequests: BlobBatchSubRequest[] = []; + + let previousOperation: Operation | undefined; + + for (const subRequest of subRequests) { + const requestLines = subRequest.split(`${HTTP_LINE_ENDING}`); + + // Content-Encoding + // Content-Type + // Content-ID + // empty line + // Operation infos + if (requestLines.length < 5) throw new Error("Bad request"); + + // Get Content_ID + let lineIndex = 0; + let content_id: number | undefined; + + while (lineIndex < requestLines.length) { + if (requestLines[lineIndex] === "") break; + const header = requestLines[lineIndex].split(HTTP_HEADER_DELIMITER); + + if (header.length !== 2) throw new Error("Bad Request"); + + if (header[0].toLocaleLowerCase() === "content-id") { + content_id = parseInt(header[1], 10); + } + ++lineIndex; + } + + if (content_id === undefined) throw new Error("Bad request"); + + // "DELETE /container166063791875402779/blob0 HTTP/1.1" + ++lineIndex; + const operationInfos = requestLines[lineIndex].split(" "); + if (operationInfos.length < 3) throw new Error("Bad request"); + + const requestPath = operationInfos[1].startsWith("/") + ? operationInfos[1] + : "/" + operationInfos[1]; + + if (!requestPath.startsWith(subRequestPathPrefix)) { + throw new Error("Request from a different container"); + } + + const url = `${request.getEndpoint()}${requestPath}`; + const method = operationInfos[0] as HttpMethod; + const blobBatchSubRequest = new BlobBatchSubRequest( + content_id!, + url, + method, + operationInfos[2], + {} + ); + + ++lineIndex; + while (lineIndex < requestLines.length) { + if (requestLines[lineIndex] === "") break; // Last line + const header = requestLines[lineIndex].split(HTTP_HEADER_DELIMITER); + + if (header.length !== 2) throw new Error("Bad Request"); + blobBatchSubRequest.setHeader(header[0], header[1]); + ++lineIndex; + } + const operation = await this.getSubRequestOperation(blobBatchSubRequest); + if ( + operation !== Operation.Path_Delete && + operation !== Operation.Blob_SetTier + ) { + throw new Error("Not supported operation"); + } + + if (previousOperation === undefined) { + previousOperation = operation; + } else if (operation !== previousOperation!) { + context.contextId = commonRequestId; + throw new DataLakeError( + 400, + "AllBatchSubRequestsShouldBeSameApi", + "All batch subrequests should be the same api.", + "AllBatchSubRequestsShouldBeSameApi", + "All batch subrequests should be the same api.", + context + ); + } + + blobBatchSubRequests.push(blobBatchSubRequest); + } + + if (blobBatchSubRequests.length === 0) { + throw new Error("Bad Request"); + } + + return blobBatchSubRequests; + } + + private serializeSubResponse( + subResponsePrefix: string, + responseEnding: string, + subResponses: BlobBatchSubResponse[] + ): string { + let responseBody = ""; + subResponses.forEach((subResponse) => { + (responseBody += subResponsePrefix), + (responseBody += "Content-Type: application/http" + HTTP_LINE_ENDING); + if (subResponse.content_id !== undefined) { + responseBody += + "Content-ID" + + HTTP_HEADER_DELIMITER + + subResponse.content_id.toString() + + HTTP_LINE_ENDING; + } + responseBody += HTTP_LINE_ENDING; + + responseBody += + subResponse.protocolWithVersion + + " " + + subResponse.getStatusCode().toString() + + " " + + subResponse.getStatusMessage() + + HTTP_LINE_ENDING; + + const headers = subResponse.getHeaders(); + for (const key of Object.keys(headers)) { + responseBody += + key + HTTP_HEADER_DELIMITER + headers[key] + HTTP_LINE_ENDING; + } + + const bodyContent = subResponse.getBodyContent(); + if (bodyContent !== "") { + responseBody += HTTP_LINE_ENDING + bodyContent + HTTP_LINE_ENDING; + } + responseBody += HTTP_LINE_ENDING; + }); + + responseBody += responseEnding; + return responseBody; + } + + public async submitBatch( + body: NodeJS.ReadableStream, + requestBatchBoundary: string, + subRequestPathPrefix: string, + batchRequest: IRequest, + context: Context + ): Promise { + const perRequestPrefix = `--${requestBatchBoundary}${HTTP_LINE_ENDING}`; + const batchRequestEnding = `--${requestBatchBoundary}--`; + + const requestBody = await this.requestBodyToString(body); + let subRequests: BlobBatchSubRequest[] | undefined; + let error: any | undefined; + try { + subRequests = await this.parseSubRequests( + context.contextId!, + perRequestPrefix, + batchRequestEnding, + subRequestPathPrefix, + batchRequest, + requestBody, + context + ); + } catch (err) { + if ( + err instanceof MiddlewareError && + err.hasOwnProperty("storageErrorCode") && + err.hasOwnProperty("storageErrorMessage") && + err.hasOwnProperty("storageRequestID") + ) { + error = err; + } else { + error = new DataLakeError( + 400, + "InvalidInput", + "One of the request inputs is not valid.", + "InvalidInput", + "One of the request inputs is not valid.", + context + ); + } + } + + const subResponses: BlobBatchSubResponse[] = []; + if (subRequests && subRequests.length > 256) { + error = new DataLakeError( + 400, + "ExceedsMaxBatchRequestCount", + "The batch operation exceeds maximum number of allowed subrequests.", + "ExceedsMaxBatchRequestCount", + "The batch operation exceeds maximum number of allowed subrequests.", + context + ); + } + + if (error) { + this.logger.error( + `BlobBatchHandler: ${error.message}`, + context.contextId + ); + const errorResponse = new BlobBatchSubResponse(undefined, "HTTP/1.1"); + await this.HandleOneFailedRequest(error, batchRequest, errorResponse); + subResponses.push(errorResponse); + } else { + for (const subRequest of subRequests!) { + this.logger.info( + `BlobBatchHandler: starting on subrequest ${subRequest.content_id}`, + context.contextId + ); + const subResponse = new BlobBatchSubResponse( + subRequest.content_id, + subRequest.protocolWithVersion + ); + await this.HandleOneSubRequest(subRequest, subResponse); + subResponses.push(subResponse); + this.logger.info( + `BlobBatchHandler: completed on subrequest ${ + subRequest.content_id + } ${subResponse.getHeader("x-ms-request-id")}`, + context.contextId + ); + } + } + + return this.serializeSubResponse( + perRequestPrefix, + batchRequestEnding, + subResponses + ); + } + + private HandleOneSubRequest( + request: IRequest, + response: IResponse + ): Promise { + const subRequestHandlePipeline = this.handlePipeline; + const subRequestErrorHandler = this.errorHandler; + let completed: boolean = false; + return new Promise((resolve, reject) => { + const locals: any = {}; + let i = 0; + const next = (error: any) => { + if (completed) { + resolve(); + return; + } + + if (error) { + subRequestErrorHandler(error, request, response, locals, next); + completed = true; + } else { + ++i; + if (i < subRequestHandlePipeline.length) { + subRequestHandlePipeline[i](request, response, locals, next); + } else { + resolve(); + } + } + }; + + subRequestHandlePipeline[i](request, response, locals, next); + }); + } + + private HandleOneFailedRequest( + err: any, + request: IRequest, + response: IResponse + ): Promise { + const subRequestErrorHandler = this.errorHandler; + return new Promise((resolve, reject) => { + subRequestErrorHandler(err, request, response, {}, resolve); + }); + } +} diff --git a/src/dfs/handlers/BlobBatchSubRequest.ts b/src/dfs/handlers/BlobBatchSubRequest.ts new file mode 100644 index 000000000..f9a35bc0e --- /dev/null +++ b/src/dfs/handlers/BlobBatchSubRequest.ts @@ -0,0 +1,85 @@ +import { URLBuilder } from "@azure/ms-rest-js"; +import { Readable } from "stream"; +import IRequest, { HttpMethod } from "../generated/IRequest"; + +export class BlobBatchSubRequest implements IRequest { + private readonly urlbuilder: URLBuilder; + + public constructor( + public readonly content_id: number, + private readonly url: string, + private readonly method: HttpMethod, + public readonly protocolWithVersion: string, + private readonly headers: { + [header: string]: string | string[] | undefined; + } + ) { + this.urlbuilder = URLBuilder.parse(this.url); + } + + public getMethod(): HttpMethod { + return this.method; + } + + public getUrl(): string { + return this.url; + } + + public getEndpoint(): string { + const urlbuilder = URLBuilder.parse(this.url); + return `${urlbuilder.getScheme()}://${urlbuilder.getHost()}`; + } + + public getPath(): string { + const path = this.urlbuilder.getPath(); + if (path) return path; + return ""; + } + + public getBodyStream(): NodeJS.ReadableStream { + return Readable.from([]); + } + + public getBody(): string | undefined { + return undefined; + } + + public setBody(body: string | undefined): BlobBatchSubRequest { + return this; + } + + public getHeader(field: string): string | undefined { + if (!field) { + throw new TypeError("field argument is required to getHeader"); + } + + if (typeof field !== "string") { + throw new TypeError("name must be a string to getHeader"); + } + + const lc = field.toLowerCase(); + + return this.headers[lc] as string | undefined; + } + + public getHeaders(): { [header: string]: string | string[] | undefined } { + return this.headers; + } + + public getRawHeaders(): string[] { + return []; + } + + public getQuery(key: string): string | undefined { + const queryValue = this.urlbuilder.getQueryParameterValue(key); + return queryValue === undefined ? undefined : queryValue.toString(); + } + + public getProtocol(): string { + return this.urlbuilder.getScheme()!; + } + + public setHeader(key: string, value: string | string[] | undefined) { + this.headers[key.toLowerCase()] = value; + } +} diff --git a/src/dfs/handlers/BlobBatchSubResponse.ts b/src/dfs/handlers/BlobBatchSubResponse.ts new file mode 100644 index 000000000..891c40834 --- /dev/null +++ b/src/dfs/handlers/BlobBatchSubResponse.ts @@ -0,0 +1,84 @@ +import { OutgoingHttpHeaders, STATUS_CODES } from "http"; +import IResponse from "../generated/IResponse"; +import { SubResponseTextBodyStream } from "./SubResponseTextBodyStream"; + +export class BlobBatchSubResponse implements IResponse { + private statusCode: number | undefined; + private statusMessage: string | undefined; + private headers: { [header: string]: string | string[] | undefined }; + private bodyStream: SubResponseTextBodyStream; + + public constructor( + public readonly content_id: number | undefined, + public readonly protocolWithVersion: string + ) { + this.headers = {}; + this.bodyStream = new SubResponseTextBodyStream(this); + } + + public setStatusCode(code: number): IResponse { + this.statusCode = code; + return this; + } + + public getStatusCode(): number { + return this.statusCode!; + } + + public setStatusMessage(message: string): IResponse { + this.statusMessage = message; + return this; + } + + public getStatusMessage(): string { + return this.statusMessage || ""; + } + + public setHeader( + field: string, + value?: string | string[] | undefined | number | boolean + ): IResponse { + if (typeof value === "number") { + value = `${value}`; + } + + if (typeof value === "boolean") { + value = `${value}`; + } + + // Cannot remove if block because of a potential TypeScript bug + if (typeof value === "string" || value instanceof Array) { + this.headers[field] = value; + } + return this; + } + + public getHeader(field: string): number | string | string[] | undefined { + return this.headers[field]; + } + + public getHeaders(): OutgoingHttpHeaders { + return this.headers; + } + + public headersSent(): boolean { + return false; + } + + public setContentType(value: string): IResponse { + this.setHeader("content-type", value); + return this; + } + + public getBodyStream(): NodeJS.WritableStream { + return this.bodyStream; + } + + public getBodyContent(): string { + return this.bodyStream.getBodyContent(); + } + + public end(): void { + this.setStatusMessage(STATUS_CODES[this.statusCode!] || "unknown"); + } +} diff --git a/src/dfs/handlers/BlobHandler.ts b/src/dfs/handlers/BlobHandler.ts new file mode 100644 index 000000000..5184841ac --- /dev/null +++ b/src/dfs/handlers/BlobHandler.ts @@ -0,0 +1,903 @@ +import { URLBuilder } from "@azure/ms-rest-js"; +import axios, { AxiosResponse } from "axios"; +import { URL } from "url"; + +import IExtentStore from "../../common/persistence/IExtentStore"; +import { convertRawHeadersToMetadata } from "../../common/utils/utils"; +import DataLakeContext from "../context/DataLakeContext"; +import NotImplementedError from "../errors/NotImplementedError"; +import StorageErrorFactory from "../errors/StorageErrorFactory"; +import * as Models from "../generated/artifacts/models"; +import Context from "../generated/Context"; +import IBlobHandler from "../generated/handlers/IBlobHandler"; +import ILogger from "../generated/utils/ILogger"; +import { parseXML } from "../generated/utils/xml"; +import { extractStoragePartsFromPath } from "../middlewares/blobStorageContext.middleware"; +import IDataLakeMetadataStore from "../persistence/IDataLakeMetadataStore"; +import { + BLOB_API_VERSION, + EMULATOR_ACCOUNT_KIND, + EMULATOR_ACCOUNT_SKUNAME, + HeaderConstants +} from "../utils/constants"; +import BaseHandler from "./BaseHandler"; +/** + * BlobHandler handles Azure Storage Blob related requests. + * + * @export + * @class BlobHandler + * @extends {BaseHandler} + * @implements {IBlobHandler} + */ +export default class BlobHandler extends BaseHandler implements IBlobHandler { + constructor( + metadataStore: IDataLakeMetadataStore, + extentStore: IExtentStore, + logger: ILogger, + loose: boolean + ) { + super(metadataStore, extentStore, logger, loose); + } + + /** + * Undelete blob. + * + * @param {Models.BlobUndeleteOptionalParams} options + * @param {Context} context + * @returns {Promise} + * @memberof BlobHandler + */ + public async undelete( + options: Models.BlobUndeleteOptionalParams, + context: Context + ): Promise { + throw new NotImplementedError(context); + } + + public async setExpiry( + expiryOptions: Models.BlobExpiryOptions, + options: Models.BlobSetExpiryOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const accountName = blobCtx.account!; + const containerName = blobCtx.container!; + const blobName = blobCtx.blob!; + const date = context.startTime!; + + const blobModel = await this.metadataStore.downloadBlob( + context, + accountName, + containerName, + blobName, + "" + ); + + if ( + options.expiresOn === undefined && + expiryOptions !== Models.BlobExpiryOptions.NeverExpire + ) { + throw StorageErrorFactory.getMissingRequestHeader(context); + } + let expiresOn: Date | undefined; + let timeToExpireInMs; + let startDate = date; + switch (expiryOptions) { + case Models.BlobExpiryOptions.NeverExpire: + expiresOn = undefined; + break; + case Models.BlobExpiryOptions.RelativeToCreation: + startDate = blobModel.properties.creationTime!; + case Models.BlobExpiryOptions.RelativeToNow: + timeToExpireInMs = parseInt(options.expiresOn!); + if (isNaN(timeToExpireInMs)) { + throw StorageErrorFactory.getInvalidHeaderValue(context); + } + expiresOn = new Date(startDate.getTime() + timeToExpireInMs); + break; + case Models.BlobExpiryOptions.Absolute: + expiresOn = new Date(options.expiresOn!); + break; + } + + blobModel.properties.expiresOn = expiresOn; + await this.metadataStore.createBlob(context, blobModel); + + const response: Models.BlobSetExpiryResponse = { + statusCode: 200, + clientRequestId: options.requestId, + requestId: context.contextId, + date, + eTag: blobModel.properties.etag, + lastModified: blobModel.properties.lastModified + }; + + return response; + } + + /** + * Set HTTP Headers. + * see also https://docs.microsoft.com/en-us/rest/api/storageservices/set-blob-properties + * + * @param {Models.BlobSetHTTPHeadersOptionalParams} options + * @param {Context} context + * @returns {Promise} + * @memberof BlobHandler + */ + public async setHTTPHeaders( + options: Models.BlobSetHTTPHeadersOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const account = blobCtx.account!; + const container = blobCtx.container!; + const blob = blobCtx.blob!; + + let res; + + // Workaround for https://github.com/Azure/Azurite/issues/332 + const sequenceNumberAction = context.request!.getHeader( + HeaderConstants.X_MS_SEQUENCE_NUMBER_ACTION + ); + const sequenceNumber = context.request!.getHeader( + HeaderConstants.X_MS_BLOB_SEQUENCE_NUMBER + ); + if (sequenceNumberAction !== undefined) { + this.logger.verbose( + "BlobHandler:setHTTPHeaders() Redirect to updateSequenceNumber...", + context.contextId + ); + res = await this.metadataStore.updateSequenceNumber( + context, + account, + container, + blob, + sequenceNumberAction.toLowerCase() as Models.SequenceNumberActionType, + sequenceNumber === undefined ? undefined : parseInt(sequenceNumber, 10), + options.leaseAccessConditions, + options.modifiedAccessConditions + ); + } else { + res = await this.metadataStore.setBlobHTTPHeaders( + context, + account, + container, + blob, + options.leaseAccessConditions, + options.blobHTTPHeaders, + options.modifiedAccessConditions + ); + } + + const response: Models.BlobSetHTTPHeadersResponse = { + statusCode: 200, + eTag: res.etag, + lastModified: res.lastModified, + blobSequenceNumber: res.blobSequenceNumber, + requestId: context.contextId, + version: BLOB_API_VERSION, + date: context.startTime, + clientRequestId: options.requestId + }; + + return response; + } + + public async setImmutabilityPolicy( + options: Models.BlobSetImmutabilityPolicyOptionalParams, + context: Context + ): Promise { + throw new NotImplementedError(context); + } + + public async deleteImmutabilityPolicy( + options: Models.BlobDeleteImmutabilityPolicyOptionalParams, + context: Context + ): Promise { + throw new NotImplementedError(context); + } + + public async setLegalHold( + legalHold: boolean, + options: Models.BlobSetLegalHoldOptionalParams, + context: Context + ): Promise { + throw new NotImplementedError(context); + } + + /** + * Set Metadata. + * + * @param {Models.BlobSetMetadataOptionalParams} options + * @param {Context} context + * @returns {Promise} + * @memberof BlobHandler + */ + public async setMetadata( + options: Models.BlobSetMetadataOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const account = blobCtx.account!; + const container = blobCtx.container!; + const blob = blobCtx.blob!; + + // Preserve metadata key case + const metadata = convertRawHeadersToMetadata( + blobCtx.request!.getRawHeaders() + ); + + const res = await this.metadataStore.setBlobMetadata( + context, + account, + container, + blob, + options.leaseAccessConditions, + metadata, + options.modifiedAccessConditions + ); + + // ToDo: return correct headers and test for these. + const response: Models.BlobSetMetadataResponse = { + statusCode: 200, + eTag: res.etag, + lastModified: res.lastModified, + isServerEncrypted: true, + requestId: context.contextId, + date: context.startTime, + version: BLOB_API_VERSION, + clientRequestId: options.requestId + }; + + return response; + } + + /** + * Acquire Blob Lease. + * + * @param {Models.BlobAcquireLeaseOptionalParams} options + * @param {Context} context + * @returns {Promise} + * @memberof BlobHandler + */ + public async acquireLease( + options: Models.BlobAcquireLeaseOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const account = blobCtx.account!; + const container = blobCtx.container!; + const blob = blobCtx.blob!; + const snapshot = blobCtx.request!.getQuery("snapshot"); + + if (snapshot !== undefined && snapshot !== "") { + throw StorageErrorFactory.getInvalidOperation( + context, + "A lease cannot be granted for a blob snapshot" + ); + } + + const res = await this.metadataStore.acquireBlobLease( + context, + account, + container, + blob, + options.duration!, + options.proposedLeaseId, + options + ); + + const response: Models.BlobAcquireLeaseResponse = { + date: blobCtx.startTime!, + eTag: res.properties.etag, + lastModified: res.properties.lastModified, + leaseId: res.leaseId, + requestId: context.contextId, + version: BLOB_API_VERSION, + statusCode: 201, + clientRequestId: options.requestId + }; + + return response; + } + + /** + * release blob lease + * + * @param {string} leaseId + * @param {Models.BlobReleaseLeaseOptionalParams} options + * @param {Context} context + * @returns {Promise} + * @memberof BlobHandler + */ + public async releaseLease( + leaseId: string, + options: Models.BlobReleaseLeaseOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const account = blobCtx.account!; + const container = blobCtx.container!; + const blob = blobCtx.blob!; + const res = await this.metadataStore.releaseBlobLease( + context, + account, + container, + blob, + leaseId, + options + ); + + const response: Models.BlobReleaseLeaseResponse = { + date: blobCtx.startTime!, + eTag: res.etag, + lastModified: res.lastModified, + requestId: context.contextId, + version: BLOB_API_VERSION, + statusCode: 200, + clientRequestId: options.requestId + }; + + return response; + } + + /** + * Renew blob lease + * + * @param {string} leaseId + * @param {Models.BlobRenewLeaseOptionalParams} options + * @param {Context} context + * @returns {Promise} + * @memberof BlobHandler + */ + public async renewLease( + leaseId: string, + options: Models.BlobRenewLeaseOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const account = blobCtx.account!; + const container = blobCtx.container!; + const blob = blobCtx.blob!; + const res = await this.metadataStore.renewBlobLease( + context, + account, + container, + blob, + leaseId, + options + ); + + const response: Models.BlobRenewLeaseResponse = { + date: blobCtx.startTime!, + eTag: res.properties.etag, + lastModified: res.properties.lastModified, + leaseId: res.leaseId, + requestId: context.contextId, + version: BLOB_API_VERSION, + statusCode: 200, + clientRequestId: options.requestId + }; + + return response; + } + + /** + * Change lease. + * + * @param {string} leaseId + * @param {string} proposedLeaseId + * @param {Models.BlobChangeLeaseOptionalParams} options + * @param {Context} context + * @returns {Promise} + * @memberof BlobHandler + */ + public async changeLease( + leaseId: string, + proposedLeaseId: string, + options: Models.BlobChangeLeaseOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const account = blobCtx.account!; + const container = blobCtx.container!; + const blob = blobCtx.blob!; + const res = await this.metadataStore.changeBlobLease( + context, + account, + container, + blob, + leaseId, + proposedLeaseId, + options + ); + + const response: Models.BlobChangeLeaseResponse = { + date: blobCtx.startTime!, + eTag: res.properties.etag, + lastModified: res.properties.lastModified, + leaseId: res.leaseId, + requestId: context.contextId, + version: BLOB_API_VERSION, + statusCode: 200, + clientRequestId: options.requestId + }; + + return response; + } + + /** + * Break lease. + * + * @param {Models.BlobBreakLeaseOptionalParams} options + * @param {Context} context + * @returns {Promise} + * @memberof BlobHandler + */ + public async breakLease( + options: Models.BlobBreakLeaseOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const account = blobCtx.account!; + const container = blobCtx.container!; + const blob = blobCtx.blob!; + const res = await this.metadataStore.breakBlobLease( + context, + account, + container, + blob, + options.breakPeriod, + options + ); + + const response: Models.BlobBreakLeaseResponse = { + date: blobCtx.startTime!, + eTag: res.properties.etag, + lastModified: res.properties.lastModified, + leaseTime: res.leaseTime, + requestId: context.contextId, + version: BLOB_API_VERSION, + statusCode: 202, + clientRequestId: options.requestId + }; + + return response; + } + + /** + * Create snapshot. + * + * @see https://docs.microsoft.com/en-us/rest/api/storageservices/snapshot-blob + * + * @param {Models.BlobCreateSnapshotOptionalParams} options + * @param {Context} context + * @returns {Promise} + * @memberof BlobHandler + */ + public async createSnapshot( + options: Models.BlobCreateSnapshotOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const account = blobCtx.account!; + const container = blobCtx.container!; + const blob = blobCtx.blob!; + + // Preserve metadata key case + const metadata = convertRawHeadersToMetadata( + blobCtx.request!.getRawHeaders() + ); + + const res = await this.metadataStore.createSnapshot( + context, + account, + container, + blob, + options.leaseAccessConditions, + !options.metadata || JSON.stringify(options.metadata) === "{}" + ? undefined + : metadata, + options.modifiedAccessConditions + ); + + const response: Models.BlobCreateSnapshotResponse = { + statusCode: 201, + eTag: res.properties.etag, + lastModified: res.properties.lastModified, + requestId: context.contextId, + date: context.startTime!, + version: BLOB_API_VERSION, + snapshot: res.snapshot, + clientRequestId: options.requestId + }; + + return response; + } + + /** + * Start copy from Url. + * + * @param {string} copySource + * @param {Models.BlobStartCopyFromURLOptionalParams} options + * @param {Context} context + * @returns {Promise} + * @memberof BlobHandler + */ + public async startCopyFromURL( + copySource: string, + options: Models.BlobStartCopyFromURLOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const account = blobCtx.account!; + const container = blobCtx.container!; + const blob = blobCtx.blob!; + + // TODO: Check dest Lease status, and set to available if it's expired, see sample in BlobHandler.setMetadata() + const url = new URL(copySource); + const [sourceAccount, sourceContainer, sourceBlob] = + extractStoragePartsFromPath( + url.hostname, + url.pathname, + blobCtx.disableProductStyleUrl + ); + const snapshot = url.searchParams.get("snapshot") || ""; + + if ( + sourceAccount === undefined || + sourceContainer === undefined || + sourceBlob === undefined + ) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + if (sourceAccount !== blobCtx.account) { + await this.validateCopySource(copySource, sourceAccount, context); + } + + // Preserve metadata key case + const metadata = convertRawHeadersToMetadata( + blobCtx.request!.getRawHeaders() + ); + + const res = await this.metadataStore.startCopyFromURL( + context, + { + account: sourceAccount, + container: sourceContainer, + blob: sourceBlob, + snapshot + }, + { account, container, blob }, + copySource, + metadata, + options.tier, + options + ); + + const response: Models.BlobStartCopyFromURLResponse = { + statusCode: 202, + eTag: res.etag, + lastModified: res.lastModified, + requestId: context.contextId, + version: BLOB_API_VERSION, + date: context.startTime, + copyId: res.copyId, + copyStatus: res.copyStatus, + clientRequestId: options.requestId + }; + + return response; + } + + private async validateCopySource( + copySource: string, + sourceAccount: string, + context: Context + ): Promise { + // Currently the only cross-account copy support is from/to the same Azurite instance. In either case access + // is determined by performing a request to the copy source to see if the authentication is valid. + const blobCtx = new DataLakeContext(context); + + const currentServer = blobCtx.request!.getHeader("Host") || ""; + const url = new URL(copySource); + if (currentServer !== url.host) { + this.logger.error( + `BlobHandler:startCopyFromURL() Source account ${url} is not on the same Azurite instance as target account ${blobCtx.account}`, + context.contextId + ); + + throw StorageErrorFactory.getCannotVerifyCopySource( + context, + 404, + "The specified resource does not exist" + ); + } + + this.logger.debug( + `BlobHandler:startCopyFromURL() Validating access to the source account ${sourceAccount}`, + context.contextId + ); + + // In order to retrieve proper error details we make a metadata request to the copy source. If we instead issue + // a HEAD request then the error details are not returned and reporting authentication failures to the caller + // becomes a black box. + const metadataUrl = URLBuilder.parse(copySource); + metadataUrl.setQueryParameter("comp", "metadata"); + const validationResponse: AxiosResponse = await axios.get( + metadataUrl.toString(), + { + // Instructs axios to not throw an error for non-2xx responses + validateStatus: () => true + } + ); + if (validationResponse.status === 200) { + this.logger.debug( + `BlobHandler:startCopyFromURL() Successfully validated access to source account ${sourceAccount}`, + context.contextId + ); + } else { + this.logger.debug( + `BlobHandler:startCopyFromURL() Access denied to source account ${sourceAccount} StatusCode=${validationResponse.status}, AuthenticationErrorDetail=${validationResponse.data}`, + context.contextId + ); + + if (validationResponse.status === 404) { + throw StorageErrorFactory.getCannotVerifyCopySource( + context, + validationResponse.status, + "The specified resource does not exist" + ); + } else { + // For non-successful responses attempt to unwrap the error message from the metadata call. + let message: string = + "Could not verify the copy source within the specified time."; + if ( + validationResponse.headers[HeaderConstants.CONTENT_TYPE] === + "application/xml" + ) { + const authenticationError = await parseXML(validationResponse.data); + if (authenticationError.Message !== undefined) { + message = authenticationError.Message.replace(/\n+/gm, ""); + } + } + + throw StorageErrorFactory.getCannotVerifyCopySource( + context, + validationResponse.status, + message + ); + } + } + } + + /** + * Abort copy from Url. + * + * @param {string} copyId + * @param {Models.BlobAbortCopyFromURLOptionalParams} options + * @param {Context} context + * @returns {Promise} + * @memberof BlobHandler + */ + public async abortCopyFromURL( + copyId: string, + options: Models.BlobAbortCopyFromURLOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const accountName = blobCtx.account!; + const containerName = blobCtx.container!; + const blobName = blobCtx.blob!; + const blob = await this.metadataStore.downloadBlob( + context, + accountName, + containerName, + blobName, + undefined, + options.leaseAccessConditions + ); + + if (blob.properties.copyId !== copyId) { + throw StorageErrorFactory.getCopyIdMismatch(context); + } + + if (blob.properties.copyStatus === Models.CopyStatusType.Success) { + throw StorageErrorFactory.getNoPendingCopyOperation(context); + } + + const response: Models.BlobAbortCopyFromURLResponse = { + statusCode: 204, + requestId: context.contextId, + version: BLOB_API_VERSION, + date: context.startTime, + clientRequestId: options.requestId + }; + + return response; + } + + /** + * Copy from Url. + * + * @param {string} copySource + * @param {Models.BlobStartCopyFromURLOptionalParams} options + * @param {Context} context + * @returns {Promise} + * @memberof BlobHandler + */ + public async copyFromURL( + copySource: string, + options: Models.BlobCopyFromURLOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const account = blobCtx.account!; + const container = blobCtx.container!; + const blob = blobCtx.blob!; + + // TODO: Check dest Lease status, and set to available if it's expired, see sample in BlobHandler.setMetadata() + const url = new URL(copySource); + const [sourceAccount, sourceContainer, sourceBlob, orginalSource] = + extractStoragePartsFromPath( + url.hostname, + url.pathname, + blobCtx.disableProductStyleUrl + ); + + if ( + sourceAccount === undefined || + sourceContainer === undefined || + sourceBlob === undefined || + orginalSource === undefined + ) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + const snapshot = url.searchParams.get("snapshot") || ""; + + if (sourceAccount !== blobCtx.account) { + await this.validateCopySource(copySource, sourceAccount, context); + } + + // Preserve metadata key case + const metadata = convertRawHeadersToMetadata( + blobCtx.request!.getRawHeaders() + ); + + const res = await this.metadataStore.copyFromURL( + context, + { + account: sourceAccount, + container: sourceContainer, + blob: sourceBlob, + snapshot + }, + { account, container, blob }, + copySource, + metadata, + options.tier, + options + ); + + let copyStatus: Models.SyncCopyStatusType | undefined; + if (res.copyStatus !== undefined) { + if (res.copyStatus === Models.CopyStatusType.Success) { + copyStatus = Models.SyncCopyStatusType.Success; + } else { + throw StorageErrorFactory.getUnexpectedSyncCopyStatus( + context, + res.copyStatus + ); + } + } + + const response: Models.BlobCopyFromURLResponse = { + statusCode: 202, + eTag: res.etag, + lastModified: res.lastModified, + requestId: context.contextId, + version: BLOB_API_VERSION, + date: context.startTime, + copyId: res.copyId, + copyStatus, + clientRequestId: options.requestId + }; + + return response; + } + + /** + * Set blob tier. + * + * @param {Models.AccessTier} tier + * @param {Models.BlobSetTierOptionalParams} options + * @param {Context} context + * @returns {Promise} + * @memberof BlobHandler + */ + public async setTier( + tier: Models.AccessTier, + options: Models.BlobSetTierOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const account = blobCtx.account!; + const container = blobCtx.container!; + const blob = blobCtx.blob!; + const res = await this.metadataStore.setTier( + context, + account, + container, + blob, + tier, + options.leaseAccessConditions + ); + + const response: Models.BlobSetTierResponse = { + requestId: context.contextId, + version: BLOB_API_VERSION, + statusCode: res, + clientRequestId: options.requestId + }; + + return response; + } + + /** + * Get account information. + * + * @param {Context} context + * @returns {Promise} + * @memberof BlobHandler + */ + public async getAccountInfo( + context: Context + ): Promise { + const response: Models.BlobGetAccountInfoResponse = { + statusCode: 200, + requestId: context.contextId, + clientRequestId: context.request!.getHeader("x-ms-client-request-id"), + skuName: EMULATOR_ACCOUNT_SKUNAME, + accountKind: EMULATOR_ACCOUNT_KIND, + date: context.startTime!, + version: BLOB_API_VERSION + }; + return response; + } + + /** + * Get account information with headers. + * + * @param {Context} context + * @returns {Promise} + * @memberof BlobHandler + */ + public async getAccountInfoWithHead( + context: Context + ): Promise { + return this.getAccountInfo(context); + } + + public async query( + options: Models.BlobQueryOptionalParams, + context: Context + ): Promise { + throw new NotImplementedError(context); + } + + public async getTags( + options: Models.BlobGetTagsOptionalParams, + context: Context + ): Promise { + throw new NotImplementedError(context); + } + + public async setTags( + options: Models.BlobSetTagsOptionalParams, + context: Context + ): Promise { + throw new NotImplementedError(context); + } +} diff --git a/src/dfs/handlers/BlockBlobHandler.ts b/src/dfs/handlers/BlockBlobHandler.ts new file mode 100644 index 000000000..096418b0a --- /dev/null +++ b/src/dfs/handlers/BlockBlobHandler.ts @@ -0,0 +1,507 @@ +import { convertRawHeadersToMetadata } from "../../common/utils/utils"; +import { + getMD5FromStream, + getMD5FromString, + newEtag +} from "../../common/utils/utils"; +import { + DEFAULT_FILE_PERMISSIONS, + DEFAULT_GROUP, + DEFAULT_OWNER +} from "../../dfs/utils/constants"; +import DataLakeContext from "../context/DataLakeContext"; +import NotImplementedError from "../errors/NotImplementedError"; +import StorageErrorFactory from "../errors/StorageErrorFactory"; +import * as Models from "../generated/artifacts/models"; +import Context from "../generated/Context"; +import IBlockBlobHandler from "../generated/handlers/IBlockBlobHandler"; +import { parseXML } from "../generated/utils/xml"; +import { BlobModel, BlockModel } from "../persistence/IDataLakeMetadataStore"; +import { toPermissions } from "../storage-file-datalake/transforms"; +import { BLOB_API_VERSION } from "../utils/constants"; +import BaseHandler from "./BaseHandler"; + +/** + * BlobHandler handles Azure Storage BlockBlob related requests. + * + * @export + * @class BlockBlobHandler + * @extends {BaseHandler} + * @implements {IBlockBlobHandler} + */ +export default class BlockBlobHandler + extends BaseHandler + implements IBlockBlobHandler +{ + public async upload( + body: NodeJS.ReadableStream, + contentLength: number, + options: Models.BlockBlobUploadOptionalParams, + context: Context + ): Promise { + // TODO: Check Lease status, and set to available if it's expired, see sample in BlobHandler.setMetadata() + const blobCtx = new DataLakeContext(context); + const accountName = blobCtx.account!; + const containerName = blobCtx.container!; + const blobName = blobCtx.blob!; + const date = context.startTime!; + const etag = newEtag(); + options.blobHTTPHeaders = options.blobHTTPHeaders || {}; + const contentType = + options.blobHTTPHeaders.blobContentType || + context.request!.getHeader("content-type") || + "application/octet-stream"; + const contentMD5 = context.request!.getHeader("content-md5") + ? options.blobHTTPHeaders.blobContentMD5 || + context.request!.getHeader("content-md5") + : undefined; + + await this.metadataStore.checkContainerExist( + context, + accountName, + containerName + ); + + const persistency = await this.extentStore.appendExtent( + body, + context.contextId + ); + if (persistency.count !== contentLength) { + throw StorageErrorFactory.getInvalidOperation( + blobCtx, + `The size of the request body ${persistency.count} mismatches the content-length ${contentLength}.` + ); + } + + // Calculate MD5 for validation + const stream = await this.extentStore.readExtent( + persistency, + context.contextId + ); + const calculatedContentMD5 = await getMD5FromStream(stream); + if (contentMD5 !== undefined) { + if (typeof contentMD5 === "string") { + const calculatedContentMD5String = + Buffer.from(calculatedContentMD5).toString("base64"); + if (contentMD5 !== calculatedContentMD5String) { + throw StorageErrorFactory.getInvalidOperation( + context, + "Provided contentMD5 doesn't match." + ); + } + } else { + if (!Buffer.from(contentMD5).equals(calculatedContentMD5)) { + throw StorageErrorFactory.getInvalidOperation( + context, + "Provided contentMD5 doesn't match." + ); + } + } + } + + const blob: BlobModel = { + deleted: false, + // Preserve metadata key case + metadata: convertRawHeadersToMetadata(blobCtx.request!.getRawHeaders()), + accountName, + containerName, + name: blobName, + properties: { + creationTime: date, + lastModified: date, + etag, + contentLength, + contentType, + contentEncoding: options.blobHTTPHeaders.blobContentEncoding, + contentLanguage: options.blobHTTPHeaders.blobContentLanguage, + contentMD5: calculatedContentMD5, + contentDisposition: options.blobHTTPHeaders.blobContentDisposition, + cacheControl: options.blobHTTPHeaders.blobCacheControl, + blobType: Models.BlobType.BlockBlob, + leaseStatus: Models.LeaseStatusType.Unlocked, + leaseState: Models.LeaseStateType.Available, + serverEncrypted: true, + accessTier: Models.AccessTier.Hot, + accessTierInferred: true, + accessTierChangeTime: date + }, + snapshot: "", + isCommitted: true, + persistency, + isDirectory: false, + permissions: toPermissions(DEFAULT_FILE_PERMISSIONS)!, + acl: [], + owner: DEFAULT_OWNER, + group: DEFAULT_GROUP + }; + + if (options.tier !== undefined) { + blob.properties.accessTier = this.parseTier(options.tier); + if (blob.properties.accessTier === undefined) { + throw StorageErrorFactory.getInvalidHeaderValue(context, { + HeaderName: "x-ms-access-tier", + HeaderValue: `${options.tier}` + }); + } + } + // TODO: Need a lock for multi keys including containerName and blobName + // TODO: Provide a specified function. + await this.metadataStore.createBlob( + context, + blob, + options.leaseAccessConditions, + options.modifiedAccessConditions + ); + + const response: Models.BlockBlobUploadResponse = { + statusCode: 201, + eTag: etag, + lastModified: date, + contentMD5: blob.properties.contentMD5, + requestId: blobCtx.contextId, + version: BLOB_API_VERSION, + date, + isServerEncrypted: true, + clientRequestId: options.requestId + }; + + return response; + } + + public async putBlobFromUrl( + contentLength: number, + copySource: string, + options: Models.BlockBlobPutBlobFromUrlOptionalParams, + context: Context + ): Promise { + throw new NotImplementedError(context); + } + + public async stageBlock( + blockId: string, + contentLength: number, + body: NodeJS.ReadableStream, + options: Models.BlockBlobStageBlockOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const accountName = blobCtx.account!; + const containerName = blobCtx.container!; + const blobName = blobCtx.blob!; + const date = blobCtx.startTime!; + + // stageBlock operation doesn't have blobHTTPHeaders + // https://learn.microsoft.com/en-us/rest/api/storageservices/put-block + // options.blobHTTPHeaders = options.blobHTTPHeaders || {}; + const contentMD5 = context.request!.getHeader("content-md5") + ? options.transactionalContentMD5 || + context.request!.getHeader("content-md5") + : undefined; + + this.validateBlockId(blockId, blobCtx); + + await this.metadataStore.checkContainerExist( + context, + accountName, + containerName + ); + + const persistency = await this.extentStore.appendExtent( + body, + context.contextId + ); + if (persistency.count !== contentLength) { + // TODO: Confirm error code + throw StorageErrorFactory.getInvalidOperation( + blobCtx, + `The size of the request body ${persistency.count} mismatches the content-length ${contentLength}.` + ); + } + + // Calculate MD5 for validation + const stream = await this.extentStore.readExtent( + persistency, + context.contextId + ); + const calculatedContentMD5 = await getMD5FromStream(stream); + if (contentMD5 !== undefined) { + if (typeof contentMD5 === "string") { + const calculatedContentMD5String = + Buffer.from(calculatedContentMD5).toString("base64"); + if (contentMD5 !== calculatedContentMD5String) { + throw StorageErrorFactory.getInvalidOperation( + context, + "Provided contentMD5 doesn't match." + ); + } + } else { + if (!Buffer.from(contentMD5).equals(calculatedContentMD5)) { + throw StorageErrorFactory.getInvalidOperation( + context, + "Provided contentMD5 doesn't match." + ); + } + } + } + + const block: BlockModel = { + accountName, + containerName, + blobName, + isCommitted: false, + name: blockId, + size: contentLength, + persistency + }; + + // TODO: Verify it. + await this.metadataStore.stageBlock( + context, + block, + options.leaseAccessConditions + ); + + const response: Models.BlockBlobStageBlockResponse = { + statusCode: 201, + contentMD5: undefined, // TODO: Block content MD5 + requestId: blobCtx.contextId, + version: BLOB_API_VERSION, + date, + isServerEncrypted: true, + clientRequestId: options.requestId + }; + + return response; + } + + public async stageBlockFromURL( + blockId: string, + contentLength: number, + sourceUrl: string, + options: Models.BlockBlobStageBlockFromURLOptionalParams, + context: Context + ): Promise { + throw new NotImplementedError(context); + } + + public async commitBlockList( + blocks: Models.BlockLookupList, + options: Models.BlockBlobCommitBlockListOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const accountName = blobCtx.account!; + const containerName = blobCtx.container!; + const blobName = blobCtx.blob!; + const request = blobCtx.request!; + + options.blobHTTPHeaders = options.blobHTTPHeaders || {}; + const contentType = + options.blobHTTPHeaders.blobContentType || "application/octet-stream"; + + // Here we leveraged generated code utils to parser xml + // Re-parsing request body to get destination blocks + // We don't leverage serialized blocks parameter because it doesn't include sequence + const rawBody = request.getBody(); + const badRequestError = StorageErrorFactory.getInvalidOperation(blobCtx); + if (rawBody === undefined) { + throw badRequestError; + } + const parsed = await parseXML(rawBody, true); + + // Validate selected block list + const commitBlockList = []; + + // $$ is the built-in field of xml2js parsing results when enabling explicitChildrenWithOrder + // TODO: Should make these fields explicit for parseXML method + // TODO: What happens when committedBlocks and uncommittedBlocks contains same block ID? + if (parsed !== undefined && parsed.$$ instanceof Array) { + for (const block of parsed.$$) { + const blockID: string | undefined = block._; + const blockCommitType: string | undefined = block["#name"]; + + if (blockID === undefined || blockCommitType === undefined) { + throw badRequestError; + } + commitBlockList.push({ + blockName: blockID, + blockCommitType + }); + } + } + + const blob: BlobModel = { + accountName, + containerName, + name: blobName, + snapshot: "", + properties: { + lastModified: context.startTime!, + creationTime: context.startTime!, + etag: newEtag() + }, + isCommitted: true, + isDirectory: false, + permissions: toPermissions(DEFAULT_FILE_PERMISSIONS)!, + acl: [], + owner: DEFAULT_OWNER, + group: DEFAULT_GROUP + }; + + blob.properties.blobType = Models.BlobType.BlockBlob; + blob.metadata = convertRawHeadersToMetadata( + // Preserve metadata key case + blobCtx.request!.getRawHeaders() + ); + blob.properties.accessTier = Models.AccessTier.Hot; + blob.properties.cacheControl = options.blobHTTPHeaders.blobCacheControl; + blob.properties.contentType = contentType; + blob.properties.contentMD5 = options.blobHTTPHeaders.blobContentMD5; + blob.properties.contentEncoding = + options.blobHTTPHeaders.blobContentEncoding; + blob.properties.contentLanguage = + options.blobHTTPHeaders.blobContentLanguage; + blob.properties.contentDisposition = + options.blobHTTPHeaders.blobContentDisposition; + + if (options.tier !== undefined) { + blob.properties.accessTier = this.parseTier(options.tier); + if (blob.properties.accessTier === undefined) { + throw StorageErrorFactory.getInvalidHeaderValue(context, { + HeaderName: "x-ms-access-tier", + HeaderValue: `${options.tier}` + }); + } + } else { + blob.properties.accessTier = Models.AccessTier.Hot; + blob.properties.accessTierInferred = true; + } + + await this.metadataStore.commitBlockList( + context, + blob, + commitBlockList, + options.leaseAccessConditions, + options.modifiedAccessConditions + ); + + const contentMD5 = await getMD5FromString(rawBody); + + const response: Models.BlockBlobCommitBlockListResponse = { + statusCode: 201, + eTag: blob.properties.etag, + lastModified: blobCtx.startTime, + contentMD5, + requestId: blobCtx.contextId, + version: BLOB_API_VERSION, + date: blobCtx.startTime, + isServerEncrypted: true, + clientRequestId: options.requestId + }; + return response; + } + + public async getBlockList( + options: Models.BlockBlobGetBlockListOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const accountName = blobCtx.account!; + const containerName = blobCtx.container!; + const blobName = blobCtx.blob!; + const date = blobCtx.startTime!; + + const res = await this.metadataStore.getBlockList( + context, + accountName, + containerName, + blobName, + options.snapshot, + undefined, + options.leaseAccessConditions + ); + + // TODO: Create uncommitted blockblob when stage block + // TODO: Conditional headers support? + + res.properties = res.properties || {}; + const response: Models.BlockBlobGetBlockListResponse = { + statusCode: 200, + lastModified: res.properties.lastModified, + eTag: res.properties.etag, + contentType: res.properties.contentType, + blobContentLength: res.properties.contentLength, + requestId: blobCtx.contextId, + version: BLOB_API_VERSION, + date, + committedBlocks: [], + uncommittedBlocks: [] + }; + + if ( + options.listType !== undefined && + (options.listType.toLowerCase() === + Models.BlockListType.All.toLowerCase() || + options.listType.toLowerCase() === + Models.BlockListType.Uncommitted.toLowerCase()) + ) { + response.uncommittedBlocks = res.uncommittedBlocks; + } + if ( + options.listType === undefined || + options.listType.toLowerCase() === + Models.BlockListType.All.toLowerCase() || + options.listType.toLowerCase() === + Models.BlockListType.Committed.toLowerCase() + ) { + response.committedBlocks = res.committedBlocks; + } + response.clientRequestId = options.requestId; + + return response; + } + + /** + * Get the tier setting from request headers. + * + * @private + * @param {string} tier + * @returns {(Models.AccessTier | undefined)} + * @memberof BlobHandler + */ + private parseTier(tier: string): Models.AccessTier | undefined { + tier = tier.toLowerCase(); + if (tier === Models.AccessTier.Hot.toLowerCase()) { + return Models.AccessTier.Hot; + } + if (tier === Models.AccessTier.Cool.toLowerCase()) { + return Models.AccessTier.Cool; + } + if (tier === Models.AccessTier.Archive.toLowerCase()) { + return Models.AccessTier.Archive; + } + return undefined; + } + + private validateBlockId(blockId: string, context: Context): void { + const rawBlockId = Buffer.from(blockId, "base64"); + + if (blockId !== rawBlockId.toString("base64")) { + throw StorageErrorFactory.getInvalidQueryParameterValue( + context, + "blockid", + blockId, + "Not a valid base64 string." + ); + } + + if (rawBlockId.length > 64) { + throw StorageErrorFactory.getOutOfRangeInput( + context, + "blockid", + blockId, + "Block ID length cannot exceed 64." + ); + } + } +} diff --git a/src/dfs/handlers/ContainerHandler.ts b/src/dfs/handlers/ContainerHandler.ts new file mode 100644 index 000000000..cbea76c4c --- /dev/null +++ b/src/dfs/handlers/ContainerHandler.ts @@ -0,0 +1,626 @@ +import { Readable } from "stream"; +import IAccountDataStore from "../../common/IAccountDataStore"; +import ILogger from "../../common/ILogger"; +import { OAuthLevel } from "../../common/models"; +import IExtentStore from "../../common/persistence/IExtentStore"; +import { convertRawHeadersToMetadata, newEtag } from "../../common/utils/utils"; +import DataLakeContext from "../context/DataLakeContext"; +import NotImplementedError from "../errors/NotImplementedError"; +import * as Models from "../generated/artifacts/models"; +import Context from "../generated/Context"; +import IContainerHandler from "../generated/handlers/IContainerHandler"; +import IDataLakeMetadataStore from "../persistence/IDataLakeMetadataStore"; +import { + BLOB_API_VERSION, + EMULATOR_ACCOUNT_KIND, + EMULATOR_ACCOUNT_SKUNAME +} from "../utils/constants"; +import BaseHandler from "./BaseHandler"; +import { BlobBatchHandler } from "./BlobBatchHandler"; + +/** + * ContainerHandler handles Azure Storage Blob container related requests. + * + * @export + * @class ContainerHandler + * @implements {IHandler} + */ +export default class ContainerHandler + extends BaseHandler + implements IContainerHandler +{ + constructor( + private readonly accountDataStore: IAccountDataStore, + private readonly oauth: OAuthLevel | undefined, + metadataStore: IDataLakeMetadataStore, + extentStore: IExtentStore, + logger: ILogger, + loose: boolean + ) { + super(metadataStore, extentStore, logger, loose); + } + + /** + * Create container. + * + * @param {Models.ContainerCreateOptionalParams} options + * @param {Context} context + * @returns {Promise} + * @memberof ContainerHandler + */ + public async create( + options: Models.ContainerCreateOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const accountName = blobCtx.account!; + const containerName = blobCtx.container!; + const lastModified = blobCtx.startTime!; + const etag = newEtag(); + + // Preserve metadata key case + const metadata = convertRawHeadersToMetadata( + blobCtx.request!.getRawHeaders() + ); + + await this.metadataStore.createContainer(context, { + accountName, + name: containerName, + metadata, + properties: { + etag, + lastModified, + leaseStatus: Models.LeaseStatusType.Unlocked, + leaseState: Models.LeaseStateType.Available, + publicAccess: options.access, + hasImmutabilityPolicy: false, + hasLegalHold: false + } + }); + + const response: Models.ContainerCreateResponse = { + statusCode: 201, + requestId: blobCtx.contextId, + clientRequestId: options.requestId, + eTag: etag, + lastModified, + version: BLOB_API_VERSION + }; + + return response; + } + + /** + * get container properties + * + * @param {Models.ContainerGetPropertiesOptionalParams} options + * @param {Context} context + * @returns {Promise} + * @memberof ContainerHandler + */ + public async getProperties( + options: Models.ContainerGetPropertiesOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const accountName = blobCtx.account!; + const containerName = blobCtx.container!; + + const containerProperties = await this.metadataStore.getContainerProperties( + context, + accountName, + containerName, + options.leaseAccessConditions + ); + + const response: Models.ContainerGetPropertiesResponse = { + statusCode: 200, + requestId: context.contextId, + clientRequestId: options.requestId, + eTag: containerProperties.properties.etag, + ...containerProperties.properties, + blobPublicAccess: containerProperties.properties.publicAccess, + metadata: containerProperties.metadata, + version: BLOB_API_VERSION + }; + + return response; + } + + /** + * get container properties with headers + * + * @param {Models.ContainerGetPropertiesOptionalParams} options + * @param {Context} context + * @returns {Promise} + * @memberof ContainerHandler + */ + public async getPropertiesWithHead( + options: Models.ContainerGetPropertiesOptionalParams, + context: Context + ): Promise { + return this.getProperties(options, context); + } + + /** + * Delete container. + * + * @param {Models.ContainerDeleteMethodOptionalParams} options + * @param {Context} context + * @returns {Promise} + * @memberof ContainerHandler + */ + public async delete( + options: Models.ContainerDeleteMethodOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const accountName = blobCtx.account!; + const containerName = blobCtx.container!; + + // TODO: Mark container as being deleted status, then (mark) delete all blobs async + // When above finishes, execute following delete container operation + // Because following delete container operation will only delete DB metadata for container and + // blobs under the container, but will not clean up blob data in disk + // The current design will directly remove the container and all the blobs belong to it. + await this.metadataStore.deleteContainer( + context, + accountName, + containerName, + options + ); + + const response: Models.ContainerDeleteResponse = { + statusCode: 202, + requestId: context.contextId, + clientRequestId: options.requestId, + date: context.startTime, + version: BLOB_API_VERSION + }; + + return response; + } + + /** + * Set container metadata. + * + * @param {Models.ContainerSetMetadataOptionalParams} options + * @param {Context} context + * @returns {Promise} + * @memberof ContainerHandler + */ + public async setMetadata( + options: Models.ContainerSetMetadataOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const accountName = blobCtx.account!; + const containerName = blobCtx.container!; + const date = blobCtx.startTime!; + const eTag = newEtag(); + + // Preserve metadata key case + const metadata = convertRawHeadersToMetadata( + blobCtx.request!.getRawHeaders() + ); + + await this.metadataStore.setContainerMetadata( + context, + accountName, + containerName, + date, + eTag, + metadata, + options.leaseAccessConditions, + options.modifiedAccessConditions + ); + + const response: Models.ContainerSetMetadataResponse = { + statusCode: 200, + requestId: context.contextId, + clientRequestId: options.requestId, + date, + eTag, + lastModified: date + }; + + return response; + } + + /** + * Get container access policy. + * + * @param {Models.ContainerGetAccessPolicyOptionalParams} options + * @param {Context} context + * @returns {Promise} + * @memberof ContainerHandler + */ + public async getAccessPolicy( + options: Models.ContainerGetAccessPolicyOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const accountName = blobCtx.account!; + const containerName = blobCtx.container!; + + const containerAcl = await this.metadataStore.getContainerACL( + context, + accountName, + containerName, + true, + options.leaseAccessConditions + ); + + const response: any = []; + const responseArray = response as Models.SignedIdentifier[]; + const responseObject = + response as Models.ContainerGetAccessPolicyHeaders & { + statusCode: 200; + }; + if (containerAcl.containerAcl !== undefined) { + responseArray.push(...containerAcl.containerAcl); + } + responseObject.date = containerAcl.properties.lastModified; + responseObject.blobPublicAccess = containerAcl.properties.publicAccess; + responseObject.eTag = containerAcl.properties.etag; + responseObject.lastModified = containerAcl.properties.lastModified; + responseObject.requestId = context.contextId; + responseObject.version = BLOB_API_VERSION; + responseObject.statusCode = 200; + responseObject.clientRequestId = options.requestId; + + // TODO: Need fix generator code since the output containerAcl can't be serialized correctly + // tslint:disable-next-line:max-line-length + // Correct responds： 1232019-04-30T16:00:00.0000000Z2019-12-31T16:00:00.0000000Zr + // tslint:disable-next-line:max-line-length + // Current responds: 1232019-04-30T16:00:00Z2019-12-31T16:00:00Zr" + return response; + } + + /** + * set container access policy + * + * @param {Models.ContainerSetAccessPolicyOptionalParams} options + * @param {Context} context + * @returns {Promise} + * @memberof ContainerHandler + */ + public async setAccessPolicy( + options: Models.ContainerSetAccessPolicyOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const accountName = blobCtx.account!; + const containerName = blobCtx.container!; + + const date = blobCtx.startTime!; + const eTag = newEtag(); + await this.metadataStore.setContainerACL( + context, + accountName, + containerName, + { + lastModified: date, + etag: eTag, + publicAccess: options.access, + containerAcl: options.containerAcl, + leaseAccessConditions: options.leaseAccessConditions, + modifiedAccessConditions: options.modifiedAccessConditions + } + ); + + const response: Models.ContainerSetAccessPolicyResponse = { + date, + eTag, + lastModified: date, + requestId: context.contextId, + version: BLOB_API_VERSION, + statusCode: 200, + clientRequestId: options.requestId + }; + + return response; + } + + public async restore( + options: Models.ContainerRestoreOptionalParams, + context: Context + ): Promise { + throw new NotImplementedError(context); + } + + public async submitBatch( + body: NodeJS.ReadableStream, + contentLength: number, + multipartContentType: string, + options: Models.ContainerSubmitBatchOptionalParams, + context: Context + ): Promise { + const blobServiceCtx = new DataLakeContext(context); + const requestBatchBoundary = blobServiceCtx + .request!.getHeader("content-type")! + .split("=")[1]; + + const blobBatchHandler = new BlobBatchHandler( + this.accountDataStore, + this.oauth, + this.metadataStore, + this.extentStore, + this.logger, + this.loose + ); + + const responseBodyString = await blobBatchHandler.submitBatch( + body, + requestBatchBoundary, + blobServiceCtx.request!.getPath(), + context.request!, + context + ); + + const responseBody = new Readable(); + responseBody.push(responseBodyString); + responseBody.push(null); + + // No client request id defined in batch response, should refine swagger and regenerate from it. + // batch response succeed code should be 202 instead of 200, should refine swagger and regenerate from it. + const response: Models.ContainerSubmitBatchResponse = { + statusCode: 202, + requestId: context.contextId, + version: BLOB_API_VERSION, + contentType: "multipart/mixed; boundary=" + requestBatchBoundary, + body: responseBody + }; + + return response; + } + + public async filterBlobs( + options: Models.ContainerFilterBlobsOptionalParams, + context: Context + ): Promise { + throw new NotImplementedError(context); + } + + /** + * Acquire container lease. + * + * @see https://docs.microsoft.com/en-us/rest/api/storageservices/lease-container + * + * @param {Models.ContainerAcquireLeaseOptionalParams} options + * @param {Context} context + * @returns {Promise} + * @memberof ContainerHandler + */ + public async acquireLease( + options: Models.ContainerAcquireLeaseOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const accountName = blobCtx.account!; + const containerName = blobCtx.container!; + + const res = await this.metadataStore.acquireContainerLease( + context, + accountName, + containerName, + options + ); + + const response: Models.ContainerAcquireLeaseResponse = { + statusCode: 201, + requestId: context.contextId, + clientRequestId: options.requestId, + date: blobCtx.startTime!, + eTag: res.properties.etag, + lastModified: res.properties.lastModified, + leaseId: res.leaseId, + version: BLOB_API_VERSION + }; + + return response; + } + + /** + * Release container lease. + * + * @see https://docs.microsoft.com/en-us/rest/api/storageservices/lease-container + * + * @param {string} leaseId + * @param {Models.ContainerReleaseLeaseOptionalParams} options + * @param {Context} context + * @returns {Promise} + * @memberof ContainerHandler + */ + public async releaseLease( + leaseId: string, + options: Models.ContainerReleaseLeaseOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const accountName = blobCtx.account!; + const containerName = blobCtx.container!; + + const res = await this.metadataStore.releaseContainerLease( + context, + accountName, + containerName, + leaseId, + options + ); + + const response: Models.ContainerReleaseLeaseResponse = { + statusCode: 200, + requestId: context.contextId, + clientRequestId: options.requestId, + date: blobCtx.startTime!, + eTag: res.etag, + lastModified: res.lastModified, + version: BLOB_API_VERSION + }; + + return response; + } + + /** + * Renew container lease. + * + * @see https://docs.microsoft.com/en-us/rest/api/storageservices/lease-container + * + * @param {string} leaseId + * @param {Models.ContainerRenewLeaseOptionalParams} options + * @param {Context} context + * @returns {Promise} + * @memberof ContainerHandler + */ + public async renewLease( + leaseId: string, + options: Models.ContainerRenewLeaseOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const accountName = blobCtx.account!; + const containerName = blobCtx.container!; + + const res = await this.metadataStore.renewContainerLease( + context, + accountName, + containerName, + leaseId, + options + ); + + const response: Models.ContainerRenewLeaseResponse = { + statusCode: 200, + requestId: context.contextId, + clientRequestId: options.requestId, + date: blobCtx.startTime!, + leaseId: res.leaseId, + eTag: res.properties.etag, + lastModified: res.properties.lastModified, + version: BLOB_API_VERSION + }; + + return response; + } + + /** + * Break container lease. + * + * @see https://docs.microsoft.com/en-us/rest/api/storageservices/lease-container + * + * @param {Models.ContainerBreakLeaseOptionalParams} options + * @param {Context} context + * @returns {Promise} + * @memberof ContainerHandler + */ + public async breakLease( + options: Models.ContainerBreakLeaseOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const accountName = blobCtx.account!; + const containerName = blobCtx.container!; + + const res = await this.metadataStore.breakContainerLease( + context, + accountName, + containerName, + options.breakPeriod, + options + ); + + const response: Models.ContainerBreakLeaseResponse = { + statusCode: 202, + requestId: context.contextId, + clientRequestId: options.requestId, + date: blobCtx.startTime!, + eTag: res.properties.etag, + lastModified: res.properties.lastModified, + leaseTime: res.leaseTime, + version: BLOB_API_VERSION + }; + + return response; + } + + /** + * Change container lease. + * + * @see https://docs.microsoft.com/en-us/rest/api/storageservices/lease-container + * + * @param {string} leaseId + * @param {string} proposedLeaseId + * @param {Models.ContainerChangeLeaseOptionalParams} options + * @param {Context} context + * @returns {Promise} + * @memberof ContainerHandler + */ + public async changeLease( + leaseId: string, + proposedLeaseId: string, + options: Models.ContainerChangeLeaseOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const accountName = blobCtx.account!; + const containerName = blobCtx.container!; + + const res = await this.metadataStore.changeContainerLease( + context, + accountName, + containerName, + leaseId, + proposedLeaseId, + options + ); + + const response: Models.ContainerChangeLeaseResponse = { + statusCode: 200, + requestId: context.contextId, + clientRequestId: options.requestId, + date: blobCtx.startTime!, + eTag: res.properties.etag, + lastModified: res.properties.lastModified, + leaseId: res.leaseId, + version: BLOB_API_VERSION + }; + + return response; + } + + /** + * get account info + * + * @param {Context} context + * @returns {Promise} + * @memberof ContainerHandler + */ + public async getAccountInfo( + context: Context + ): Promise { + const response: Models.ContainerGetAccountInfoResponse = { + statusCode: 200, + requestId: context.contextId, + clientRequestId: context.request!.getHeader("x-ms-client-request-id"), + skuName: EMULATOR_ACCOUNT_SKUNAME, + accountKind: EMULATOR_ACCOUNT_KIND, + date: context.startTime!, + version: BLOB_API_VERSION + }; + return response; + } + + /** + * get account info with headers + * + * @param {Context} context + * @returns {Promise} + * @memberof ContainerHandler + */ + public async getAccountInfoWithHead( + context: Context + ): Promise { + return this.getAccountInfo(context); + } +} diff --git a/src/dfs/handlers/FileSystemOperationsHandler.ts b/src/dfs/handlers/FileSystemOperationsHandler.ts new file mode 100644 index 000000000..35f88cfc7 --- /dev/null +++ b/src/dfs/handlers/FileSystemOperationsHandler.ts @@ -0,0 +1,415 @@ +import BaseHandler from "./BaseHandler"; +import IFileSystemOperationsHandler from "../generated/handlers/IFileSystemOperationsHandler"; +import * as Models from "../generated/artifacts/models"; +import { + DATA_LAKE_API_VERSION, + DEFAULT_LIST_BLOBS_MAX_RESULTS +} from "../utils/constants"; +import IAccountDataStore from "../../common/IAccountDataStore"; +import ILogger from "../../common/ILogger"; +import { OAuthLevel } from "../../common/models"; +import IExtentStore from "../../common/persistence/IExtentStore"; +import IDataLakeMetaDataStore from "../persistence/IDataLakeMetadataStore"; +import { convertRawHeadersToMetadata, newEtag } from "../../common/utils/utils"; +import { removeQuotationFromListBlobEtag } from "../utils/utils"; +import { ContainerModel } from "../persistence/IDataLakeMetadataStore"; +import Context from "../generated/Context"; +import DataLakeContext from "../context/DataLakeContext"; +/** + * FileSystemOperationsHandler handles Azure Storage DataLake Gen2 filesystem + * + * @export + * @class FileSystemOperationsHandler + * @extends {BaseHandler} + * @implements {IBlobHandler} + */ +export default class FileSystemOperationsHandler + extends BaseHandler + implements IFileSystemOperationsHandler +{ + constructor( + accountDataStore: IAccountDataStore, + oauth: OAuthLevel | undefined, + metadataStore: IDataLakeMetaDataStore, + extentStore: IExtentStore, + logger: ILogger, + loose: boolean + ) { + super(metadataStore, extentStore, logger, loose); + } + + async create( + options: Models.FileSystemCreateOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const accountName = blobCtx.account!; + const containerName = blobCtx.container!; + const lastModified = blobCtx.startTime!; + const eTag = newEtag(); + + // Preserve metadata key case + const metadata = convertRawHeadersToMetadata( + blobCtx.request!.getRawHeaders() + ); + + const ContainerModel: ContainerModel = { + accountName, + name: containerName, + metadata, + properties: { + leaseStatus: Models.LeaseStatusType.Unlocked, + leaseState: Models.LeaseStateType.Available, + lastModified, + etag: eTag + } + }; + + await this.metadataStore.createContainer(context, ContainerModel); + + const response: Models.FileSystemCreateResponse = { + statusCode: 201, + clientRequestId: options.requestId, + eTag, + lastModified, + date: context.startTime, + version: DATA_LAKE_API_VERSION, + namespaceEnabled: "true" + }; + + return response; + } + /** + * Set prodperties of a filesystem/container + * + * @param {Models.FileSystemSetPropertiesOptionalParams} options + * @param {Context} context + * @return {*} {Promise} + * @memberof FileSystemOperationsHandler + */ + async setProperties( + options: Models.FileSystemSetPropertiesOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const accountName = blobCtx.account!; + const containerName = blobCtx.container!; + + const res = await this.metadataStore.setContainerProperties( + context, + accountName, + containerName, + options.properties === undefined ? "" : options.properties, + undefined, + options.modifiedAccessConditions + ); + + const response: Models.FileSystemSetPropertiesResponse = { + statusCode: 200, + date: blobCtx.startTime, + eTag: res.properties.etag, + lastModified: res.properties.lastModified, + requestId: options.requestId, + version: DATA_LAKE_API_VERSION + }; + + return response; + } + + async getProperties( + options: Models.FileSystemGetPropertiesOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const accountName = blobCtx.account!; + const containerName = blobCtx.container!; + + const res = await this.metadataStore.getContainerProperties( + context, + accountName, + containerName + ); + + const response: Models.FileSystemGetPropertiesResponse = { + statusCode: 200, + date: blobCtx.startTime, + eTag: res.properties.etag, + lastModified: res.properties.lastModified, + properties: res.fileSystemProperties, + requestId: options.requestId, + version: DATA_LAKE_API_VERSION, + namespaceEnabled: "true" + }; + + return response; + } + + async delete( + options: Models.FileSystemDeleteMethodOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const accountName = blobCtx.account!; + const containerName = blobCtx.container!; + + // TODO: Mark container as being deleted status, then (mark) delete all blobs async + // When above finishes, execute following delete container operation + // Because following delete container operation will only delete DB metadata for container and + // blobs under the container, but will not clean up blob data in disk + // The current design will directly remove the container and all the blobs belong to it. + await this.metadataStore.deleteContainer( + context, + accountName, + containerName, + options + ); + + const response: Models.FileSystemDeleteResponse = { + statusCode: 202, + requestId: context.contextId, + // clientRequestId: options.requestId, + date: context.startTime, + version: DATA_LAKE_API_VERSION + }; + + return response; + } + + async listPaths( + recursive: boolean, + options: Models.FileSystemListPathsOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const accountName = blobCtx.account!; + const containerName = blobCtx.container!; + const directory = options.path || ""; + + const containerRes = await this.getProperties(options, context); + + const [paths, marker] = await this.metadataStore.listPaths( + context, + accountName, + containerName, + directory, + recursive, + options + ); + + const response: Models.FileSystemListPathsResponse = { + statusCode: 200, + date: containerRes.date, + eTag: containerRes.eTag, + lastModified: containerRes.lastModified, + requestId: containerRes.requestId, + version: DATA_LAKE_API_VERSION, + paths, + continuation: marker ? marker : undefined + }; + + return response; + } + + /** + * list blobs flat segments + * + * @param {Models.FileSystemListBlobFlatSegmentOptionalParams} options + * @param {Context} context + * @returns {Promise} + * @memberof ContainerHandler + */ + public async listBlobFlatSegment( + options: Models.FileSystemListBlobFlatSegmentOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const accountName = blobCtx.account!; + const containerName = blobCtx.container!; + await this.metadataStore.checkContainerExist( + context, + accountName, + containerName + ); + + const request = context.request!; + const marker = options.marker; + options.marker = options.marker || ""; + let includeSnapshots: boolean = false; + let includeUncommittedBlobs: boolean = false; + if (options.include !== undefined) { + options.include.forEach((element) => { + if ( + Models.ListBlobsIncludeItem.Snapshots.toLowerCase() === + element.toLowerCase() + ) { + includeSnapshots = true; + } + if ( + Models.ListBlobsIncludeItem.Uncommittedblobs.toLowerCase() === + element.toLowerCase() + ) { + includeUncommittedBlobs = true; + } + }); + } + + let maxResults = options.maxresults || options.maxresults; + if ( + maxResults === undefined || + maxResults > DEFAULT_LIST_BLOBS_MAX_RESULTS + ) { + maxResults = DEFAULT_LIST_BLOBS_MAX_RESULTS; + } + + const [blobs, _prefixes, nextMarker] = await this.metadataStore.listBlobs( + context, + accountName, + containerName, + undefined, + undefined, + options.prefix, + maxResults, + marker, + includeSnapshots, + includeUncommittedBlobs + ); + + const serviceEndpoint = `${request.getEndpoint()}/${accountName}`; + const response: Models.FileSystemListBlobFlatSegmentResponse = { + statusCode: 200, + contentType: "application/xml", + requestId: context.contextId, + version: DATA_LAKE_API_VERSION, + date: context.startTime, + serviceEndpoint, + containerName, + prefix: options.prefix || "", + marker: options.marker, + maxResults, + segment: { + blobItems: blobs.map((item) => { + return { + ...item, + deleted: item.deleted !== true ? undefined : true, + snapshot: item.snapshot || undefined, + properties: { + ...item.properties, + etag: removeQuotationFromListBlobEtag(item.properties.etag), + accessTierInferred: + item.properties.accessTierInferred === true ? true : undefined + } + }; + }) + }, + clientRequestId: options.requestId, + nextMarker: `${nextMarker || ""}` + }; + + return response; + } + + /** + * List blobs hierarchy. + * + * @param {string} delimiter + * @param {Models.ContainerListBlobHierarchySegmentOptionalParams} options + * @param {Context} context + * @returns {Promise} + * @memberof ContainerHandler + */ + async listBlobHierarchySegment( + delimiter: string, + options: Models.FileSystemListBlobHierarchySegmentOptionalParams, + context: Context + ): Promise { + // TODO: Need update list out blobs lease properties with BlobHandler.updateLeaseAttributes() + const blobCtx = new DataLakeContext(context); + const accountName = blobCtx.account!; + const containerName = blobCtx.container!; + await this.metadataStore.checkContainerExist( + context, + accountName, + containerName + ); + + const request = context.request!; + const marker = options.marker; + options.prefix = options.prefix || ""; + options.marker = options.marker || ""; + let includeSnapshots: boolean = false; + let includeUncommittedBlobs: boolean = false; + if (options.include !== undefined) { + options.include.forEach((element) => { + if ( + Models.ListBlobsIncludeItem.Snapshots.toLowerCase() === + element.toLowerCase() + ) { + includeSnapshots = true; + } + if ( + Models.ListBlobsIncludeItem.Uncommittedblobs.toLowerCase() === + element.toLowerCase() + ) { + includeUncommittedBlobs = true; + } + }); + } + + let maxResults = options.maxResults || options.maxresults; + if ( + maxResults === undefined || + maxResults > DEFAULT_LIST_BLOBS_MAX_RESULTS + ) { + maxResults = DEFAULT_LIST_BLOBS_MAX_RESULTS; + } + + const [blobItems, blobPrefixes, nextMarker] = await this.metadataStore.list( + false, //check + context, + accountName, + containerName, + delimiter === "" ? undefined : delimiter, + undefined, + options.prefix, + maxResults, + marker, + includeSnapshots, + includeUncommittedBlobs + ); + + const serviceEndpoint = `${request.getEndpoint()}/${accountName}`; + const response: Models.FileSystemListBlobHierarchySegmentResponse = { + statusCode: 200, + contentType: "application/xml", + requestId: context.contextId, + version: DATA_LAKE_API_VERSION, + date: context.startTime, + serviceEndpoint, + containerName, + prefix: options.prefix, + marker: options.marker, + maxResults, + delimiter, + segment: { + blobPrefixes, + blobItems: blobItems.map((item) => { + const newBlobItem: Models.BlobItemInternal = { + ...item, + properties: { + ...item.properties, + etag: removeQuotationFromListBlobEtag(item.properties.etag), + accessTierInferred: + item.properties.accessTierInferred === true ? true : undefined + } + }; + + return newBlobItem; + }) + }, + clientRequestId: options.requestId, + nextMarker: `${nextMarker || ""}` + }; + + return response; + } +} diff --git a/src/dfs/handlers/IPageBlobRangesManager.ts b/src/dfs/handlers/IPageBlobRangesManager.ts new file mode 100644 index 000000000..739f3643e --- /dev/null +++ b/src/dfs/handlers/IPageBlobRangesManager.ts @@ -0,0 +1,18 @@ +import { PageRange } from "../generated/artifacts/models"; +import { PersistencyPageRange } from "../persistence/IDataLakeMetadataStore"; + +export default interface IPageBlobRangesManager { + mergeRange(ranges: PersistencyPageRange[], range: PersistencyPageRange): void; + + clearRange(ranges: PersistencyPageRange[], range: PageRange): void; + + cutRanges( + ranges: PersistencyPageRange[], + range: PageRange + ): PersistencyPageRange[]; + + fillZeroRanges( + ranges: PersistencyPageRange[], + range: PageRange + ): PersistencyPageRange[]; +} diff --git a/src/dfs/handlers/PageBlobHandler.ts b/src/dfs/handlers/PageBlobHandler.ts new file mode 100644 index 000000000..32cc57c8a --- /dev/null +++ b/src/dfs/handlers/PageBlobHandler.ts @@ -0,0 +1,505 @@ +import IExtentStore from "../../common/persistence/IExtentStore"; +import { convertRawHeadersToMetadata, newEtag } from "../../common/utils/utils"; +import { + DEFAULT_FILE_PERMISSIONS, + DEFAULT_GROUP, + DEFAULT_OWNER +} from "../../dfs/utils/constants"; +import DataLakeContext from "../context/DataLakeContext"; +import NotImplementedError from "../errors/NotImplementedError"; +import StorageErrorFactory from "../errors/StorageErrorFactory"; +import * as Models from "../generated/artifacts/models"; +import Context from "../generated/Context"; +import IPageBlobHandler from "../generated/handlers/IPageBlobHandler"; +import ILogger from "../generated/utils/ILogger"; +import BlobLeaseAdapter from "../lease/BlobLeaseAdapter"; +import BlobWriteLeaseValidator from "../lease/BlobWriteLeaseValidator"; +import IDataLakeMetadataStore, { + BlobModel +} from "../persistence/IDataLakeMetadataStore"; +import { toPermissions } from "../storage-file-datalake/transforms"; +import { BLOB_API_VERSION } from "../utils/constants"; +import { deserializePageBlobRangeHeader } from "../utils/utils"; +import BaseHandler from "./BaseHandler"; +import IPageBlobRangesManager from "./IPageBlobRangesManager"; + +/** + * PageBlobHandler handles Azure Storage PageBlob related requests. + * + * @export + * @class PageBlobHandler + * @extends {BaseHandler} + * @implements {IPageBlobHandler} + */ +export default class PageBlobHandler + extends BaseHandler + implements IPageBlobHandler +{ + constructor( + metadataStore: IDataLakeMetadataStore, + extentStore: IExtentStore, + logger: ILogger, + loose: boolean, + private readonly rangesManager: IPageBlobRangesManager + ) { + super(metadataStore, extentStore, logger, loose); + } + + public async uploadPagesFromURL( + sourceUrl: string, + sourceRange: string, + contentLength: number, + range: string, + options: Models.PageBlobUploadPagesFromURLOptionalParams, + context: Context + ): Promise { + throw new NotImplementedError(context); + } + + public async create( + contentLength: number, + blobContentLength: number, + options: Models.PageBlobCreateOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const accountName = blobCtx.account!; + const containerName = blobCtx.container!; + const blobName = blobCtx.blob!; + const date = blobCtx.startTime!; + + if (options.tier !== undefined) { + throw StorageErrorFactory.getAccessTierNotSupportedForBlobType(context); + } + + if (contentLength !== 0) { + throw StorageErrorFactory.getInvalidOperation( + blobCtx, + "Content-Length must be 0 for Create Page Blob request." + ); + } + + if (blobContentLength % 512 !== 0) { + throw StorageErrorFactory.getInvalidOperation( + blobCtx, + "x-ms-content-length must be aligned to a 512-byte boundary." + ); + } + + options.blobHTTPHeaders = options.blobHTTPHeaders || {}; + const contentType = + options.blobHTTPHeaders.blobContentType || + context.request!.getHeader("content-type") || + "application/octet-stream"; + + // const accessTierInferred = options.pageBlobAccessTier === undefined; + + // Check Blob size match tier + // if ( + // !accessTierInferred && + // blobContentLength > PageBlobAccessTierThreshold.get(tier)! + // ) { + // throw StorageErrorFactory.getBlobBlobTierInadequateForContentLength( + // blobCtx + // ); + // } + + // Preserve metadata key case + const metadata = convertRawHeadersToMetadata( + blobCtx.request!.getRawHeaders() + ); + + const etag = newEtag(); + const blob: BlobModel = { + deleted: false, + metadata, + accountName, + containerName, + name: blobName, + properties: { + creationTime: date, + lastModified: date, + etag, + contentLength: blobContentLength, + contentType, + contentEncoding: options.blobHTTPHeaders.blobContentEncoding, + contentLanguage: options.blobHTTPHeaders.blobContentLanguage, + contentMD5: options.blobHTTPHeaders.blobContentMD5, + contentDisposition: options.blobHTTPHeaders.blobContentDisposition, + cacheControl: options.blobHTTPHeaders.blobCacheControl, + blobSequenceNumber: options.blobSequenceNumber + ? options.blobSequenceNumber + : 0, + blobType: Models.BlobType.PageBlob, + leaseStatus: Models.LeaseStatusType.Unlocked, + leaseState: Models.LeaseStateType.Available, + serverEncrypted: true + // TODO: May support setting this part for a premium storage account. + // accessTier: accessTierInferred + // ? ((options.pageBlobAccessTier as any) as Models.AccessTier) + // : Models.AccessTier.P4, // TODO: Infer tier from size + // accessTierInferred + }, + snapshot: "", + isCommitted: true, + isDirectory: false, + permissions: toPermissions(DEFAULT_FILE_PERMISSIONS)!, + acl: [], + owner: DEFAULT_OWNER, + group: DEFAULT_GROUP, + pageRangesInOrder: [] + }; + + // TODO: What's happens when create page blob right before commit block list? Or should we lock + // Should we check if there is an uncommitted blob? + await this.metadataStore.createBlob( + context, + blob, + options.leaseAccessConditions, + options.modifiedAccessConditions + ); + + const response: Models.PageBlobCreateResponse = { + statusCode: 201, + eTag: etag, + lastModified: blob.properties.lastModified, + contentMD5: blob.properties.contentMD5, + requestId: context.contextId, + version: BLOB_API_VERSION, + date, + isServerEncrypted: true, + clientRequestId: options.requestId + }; + + return response; + } + + public async uploadPages( + body: NodeJS.ReadableStream, + contentLength: number, + options: Models.PageBlobUploadPagesOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const accountName = blobCtx.account!; + const containerName = blobCtx.container!; + const blobName = blobCtx.blob!; + const date = blobCtx.startTime!; + + if (contentLength % 512 !== 0) { + throw StorageErrorFactory.getInvalidOperation( + blobCtx, + "content-length or x-ms-content-length must be aligned to a 512-byte boundary." + ); + } + + const blob = await this.metadataStore.downloadBlob( + context, + accountName, + containerName, + blobName, + undefined, + options.leaseAccessConditions + ); + + if (blob.properties.blobType !== Models.BlobType.PageBlob) { + throw StorageErrorFactory.getBlobInvalidBlobType(blobCtx); + } + + // Check Lease status + new BlobWriteLeaseValidator(options.leaseAccessConditions).validate( + new BlobLeaseAdapter(blob), + context + ); + + let ranges; + try { + ranges = deserializePageBlobRangeHeader( + blobCtx.request!.getHeader("range"), + blobCtx.request!.getHeader("x-ms-range"), + true + ); + } catch (err) { + throw StorageErrorFactory.getInvalidPageRange(blobCtx); + } + + const start = ranges[0]; + const end = ranges[1]; // Inclusive + if (end - start + 1 !== contentLength) { + throw StorageErrorFactory.getInvalidPageRange(blobCtx); + } + + // Start Range is bigger than blob length + if (start >= blob.properties.contentLength!) { + throw StorageErrorFactory.getInvalidPageRange(blobCtx); + } + + const persistency = await this.extentStore.appendExtent( + body, + context.contextId + ); + if (persistency.count !== contentLength) { + // TODO: Confirm status code + throw StorageErrorFactory.getInvalidOperation( + blobCtx, + `The size of the request body ${persistency.count} mismatches the content-length ${contentLength}.` + ); + } + + const res = await this.metadataStore.uploadPages( + context, + blob, + start, + end, + persistency, + options.leaseAccessConditions, + options.modifiedAccessConditions, + options.sequenceNumberAccessConditions + ); + + const response: Models.PageBlobUploadPagesResponse = { + statusCode: 201, + eTag: res.etag, + lastModified: date, + contentMD5: undefined, // TODO + blobSequenceNumber: res.blobSequenceNumber, + requestId: blobCtx.contextId, + version: BLOB_API_VERSION, + date, + isServerEncrypted: true, + clientRequestId: options.requestId + }; + + return response; + } + + public async clearPages( + contentLength: number, + options: Models.PageBlobClearPagesOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const accountName = blobCtx.account!; + const containerName = blobCtx.container!; + const blobName = blobCtx.blob!; + const date = blobCtx.startTime!; + + if (contentLength !== 0) { + throw StorageErrorFactory.getInvalidOperation( + blobCtx, + "content-length or x-ms-content-length must be 0 for clear pages operation." + ); + } + + const blob = await this.metadataStore.downloadBlob( + context, + accountName, + containerName, + blobName, + undefined, + options.leaseAccessConditions + ); + + if (blob.properties.blobType !== Models.BlobType.PageBlob) { + throw StorageErrorFactory.getBlobInvalidBlobType(blobCtx); + } + + let ranges; + try { + ranges = deserializePageBlobRangeHeader( + blobCtx.request!.getHeader("range"), + blobCtx.request!.getHeader("x-ms-range"), + true + ); + } catch (err) { + throw StorageErrorFactory.getInvalidPageRange(blobCtx); + } + + const start = ranges[0]; + const end = ranges[1]; + + // Start Range is bigger than blob length + if (start >= blob.properties.contentLength!) { + throw StorageErrorFactory.getInvalidPageRange(blobCtx); + } + + const res = await this.metadataStore.clearRange( + context, + blob, + start, + end, + options.leaseAccessConditions, + options.modifiedAccessConditions, + options.sequenceNumberAccessConditions + ); + + const response: Models.PageBlobClearPagesResponse = { + statusCode: 201, + eTag: res.etag, + lastModified: date, + contentMD5: undefined, // TODO + blobSequenceNumber: res.blobSequenceNumber, + requestId: blobCtx.contextId, + version: BLOB_API_VERSION, + clientRequestId: options.requestId, + date + }; + + return response; + } + + public async getPageRanges( + options: Models.PageBlobGetPageRangesOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const accountName = blobCtx.account!; + const containerName = blobCtx.container!; + const blobName = blobCtx.blob!; + const date = blobCtx.startTime!; + + const blob = await this.metadataStore.getPageRanges( + context, + accountName, + containerName, + blobName, + options.snapshot, + options.leaseAccessConditions, + options.modifiedAccessConditions + ); + + if (blob.properties.blobType !== Models.BlobType.PageBlob) { + throw StorageErrorFactory.getBlobInvalidBlobType(blobCtx); + } + + let ranges = deserializePageBlobRangeHeader( + blobCtx.request!.getHeader("range"), + blobCtx.request!.getHeader("x-ms-range"), + false + ); + if (!ranges) { + ranges = [0, blob.properties.contentLength! - 1]; + } + + // Start Range is bigger than blob length + if (ranges[0] >= blob.properties.contentLength!) { + throw StorageErrorFactory.getInvalidPageRange(blobCtx); + } + + blob.pageRangesInOrder = blob.pageRangesInOrder || []; + const impactedRanges = this.rangesManager.cutRanges( + blob.pageRangesInOrder, + { + start: ranges[0], + end: ranges[1] + } + ); + + const response: Models.PageBlobGetPageRangesResponse = { + statusCode: 200, + pageRange: impactedRanges, + eTag: blob.properties.etag, + blobContentLength: blob.properties.contentLength, + lastModified: date, + requestId: blobCtx.contextId, + version: BLOB_API_VERSION, + clientRequestId: options.requestId, + date + }; + + return response; + } + + public async getPageRangesDiff( + options: Models.PageBlobGetPageRangesDiffOptionalParams, + context: Context + ): Promise { + throw new NotImplementedError(context); + } + + public async resize( + blobContentLength: number, + options: Models.PageBlobResizeOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const accountName = blobCtx.account!; + const containerName = blobCtx.container!; + const blobName = blobCtx.blob!; + const date = blobCtx.startTime!; + + if (blobContentLength % 512 !== 0) { + throw StorageErrorFactory.getInvalidOperation( + blobCtx, + "x-ms-blob-content-length must be aligned to a 512-byte boundary for Page Blob Resize request." + ); + } + + const res = await this.metadataStore.resizePageBlob( + context, + accountName, + containerName, + blobName, + blobContentLength, + options.leaseAccessConditions, + options.modifiedAccessConditions + ); + + const response: Models.PageBlobResizeResponse = { + statusCode: 200, + eTag: res.etag, + lastModified: res.lastModified, + blobSequenceNumber: res.blobSequenceNumber, + requestId: blobCtx.contextId, + version: BLOB_API_VERSION, + clientRequestId: options.requestId, + date + }; + + return response; + } + + public async updateSequenceNumber( + sequenceNumberAction: Models.SequenceNumberActionType, + options: Models.PageBlobUpdateSequenceNumberOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const accountName = blobCtx.account!; + const containerName = blobCtx.container!; + const blobName = blobCtx.blob!; + const date = blobCtx.startTime!; + + const res = await this.metadataStore.updateSequenceNumber( + context, + accountName, + containerName, + blobName, + sequenceNumberAction, + options.blobSequenceNumber, + options.leaseAccessConditions, + options.modifiedAccessConditions + ); + + const response: Models.PageBlobUpdateSequenceNumberResponse = { + statusCode: 200, + eTag: res.etag, + lastModified: res.lastModified, + blobSequenceNumber: res.blobSequenceNumber, + requestId: blobCtx.contextId, + version: BLOB_API_VERSION, + clientRequestId: options.requestId, + date + }; + + return response; + } + + public async copyIncremental( + copySource: string, + options: Models.PageBlobCopyIncrementalOptionalParams, + context: Context + ): Promise { + throw new NotImplementedError(context); + } +} diff --git a/src/dfs/handlers/PageBlobRangesManager.ts b/src/dfs/handlers/PageBlobRangesManager.ts new file mode 100644 index 000000000..7c06c00c9 --- /dev/null +++ b/src/dfs/handlers/PageBlobRangesManager.ts @@ -0,0 +1,481 @@ +import { PageRange } from "../generated/artifacts/models"; +import { + PersistencyPageRange, + ZERO_EXTENT_ID +} from "../persistence/IDataLakeMetadataStore"; +import IPageBlobRangesManager from "./IPageBlobRangesManager"; + +/********** Ranges Merging Strategy ************/ +// +// Example: +// Existing ranges: |---| |------| |---------| |-----| +// New range : |----------------------| +// +// 4 existing ranges, and one new request range. +// Above 4 existing ranges, 3 existing ranges are get impacted. +// +// We have 2 merging strategies: +// #1 Merge all impacted ranges +// : |---| |**------------------------****| +// #2 Split first and last impacted existing ranges: +// : |---| |**|----------------------|****| +// +// Question: +// Every range has a pointer to a chunk in one persistency layer extent. +// When implementing Get Page Ranges Diff request, we will assume 2 (sub)ranges +// are same if they pointing to same range of an extent. (Or we can comparing the +// persistency layer payload, but it's not efficiency.) +// +// For #1 strategy, Get Page Ranges Diff will return a larger range scope than +// the actual changed. Above ranges marked as * will point to new allocated extent, +// while in previous snapshot, they still point to old extent. +// One potential workaround is to update these ranges in snapshot blob to pointing +// to same new allocated extent. But this will make implementation of update ranges +// more complex, and also makes snapshot mutable. +// Another concern is that, the * ranges may have large footprint, which is +// time consuming. +// +// For #2 strategy, there will be more and more small ranges. See the |**| and |****|. +// But it's able . We can make the "merging" work to future GC. +// +// We choose #2 strategy, and leave the merging work to future GC, which will merging +// small ranges in background. +// +// TODO for #2 strategy: +// * Resize API: Should remove additional ranges +// * Download Page Blob API after resize (shift): Should taking resize into consideration +// * Clean Ranges API: Same strategy like update ranges +// * Page Blob GC for Ranges Merger: Merge small ranges and extent chunks for page blob and snapshots +// * GC for un-referred extents +export default class PageBlobRangesManager implements IPageBlobRangesManager { + public mergeRange( + ranges: PersistencyPageRange[], + range: PersistencyPageRange + ): void { + const start = range.start; // Inclusive + const end = range.end; // Inclusive + const persistency = range.persistency; + + const impactedScope = this.selectImpactedRanges(ranges, start, end); + + // Find out first and last impacted range index + const impactedStartIndex = impactedScope[0]; + const impactedEndIndex = impactedScope[1]; + const impactedRangesCount = + impactedStartIndex > impactedEndIndex // No impacted ranges + ? 0 + : impactedEndIndex - impactedStartIndex + 1; + + // New created range for this request payload + const newRange: PersistencyPageRange = { start, end, persistency }; + + if (impactedRangesCount === 0) { + // If there is no existing impacted range, just insert the new range + ranges.splice(impactedStartIndex, 0, newRange); + } else { + // Otherwise, try to split the first and last impacted ranges + const firstImpactedRange = ranges[impactedStartIndex]; + const lastImpactedRange = ranges[impactedEndIndex]; + + // Ranges to be inserted + const newRanges = []; + + // If first range needs to be split, push the split range + if (firstImpactedRange.end >= start && firstImpactedRange.start < start) { + newRanges.push({ + start: firstImpactedRange.start, + end: start - 1, + persistency: { + id: firstImpactedRange.persistency.id, + offset: firstImpactedRange.persistency.offset, + count: start - firstImpactedRange.start + } + }); + } + + newRanges.push(newRange); + + // If last impacted range needs to be split, push the split range + if (end >= lastImpactedRange.start && end < lastImpactedRange.end) { + newRanges.push({ + start: end + 1, + end: lastImpactedRange.end, + persistency: { + id: lastImpactedRange.persistency.id, + offset: + lastImpactedRange.persistency.offset + + (end + 1 - lastImpactedRange.start), + count: lastImpactedRange.end - end + } + }); + } + + ranges.splice(impactedStartIndex, impactedRangesCount, ...newRanges); + } + } + + public clearRange(ranges: PersistencyPageRange[], range: PageRange): void { + const start = range.start; // Inclusive + const end = range.end; // Inclusive + + // Find out existing impacted ranges list + const impactedScope = this.selectImpactedRanges(ranges, start, end); + + // Find out first and last impacted range index + const impactedStartIndex = impactedScope[0]; + const impactedEndIndex = impactedScope[1]; + const impactedRangesCount = + impactedStartIndex > impactedEndIndex // No impacted ranges + ? 0 + : impactedEndIndex - impactedStartIndex + 1; + + if (impactedRangesCount > 0) { + // Try to split the first and last impacted ranges + const firstImpactedRange = ranges[impactedStartIndex]; + const lastImpactedRange = ranges[impactedEndIndex]; + + // Ranges to be inserted + const newRanges = []; + + // If first range needs to be split, push the split range + if (firstImpactedRange.end >= start && firstImpactedRange.start < start) { + newRanges.push({ + start: firstImpactedRange.start, + end: start - 1, + persistency: { + id: firstImpactedRange.persistency.id, + offset: firstImpactedRange.persistency.offset, + count: start - firstImpactedRange.start + } + }); + } + + // If last impacted range needs to be split, push the split range + if (end >= lastImpactedRange.start && end < lastImpactedRange.end) { + newRanges.push({ + start: end + 1, + end: lastImpactedRange.end, + persistency: { + id: lastImpactedRange.persistency.id, + offset: + lastImpactedRange.persistency.offset + + (end + 1 - lastImpactedRange.start), + count: lastImpactedRange.end - end + } + }); + } + + ranges.splice(impactedStartIndex, impactedRangesCount, ...newRanges); + } + } + + /** + * This method will not modify values of parameter ranges. + * + * @param {PersistencyPageRange[]} ranges + * @param {PageRange} range + * @returns {PersistencyPageRange[]} + * @memberof PageBlobRangesManager + */ + public cutRanges( + ranges: PersistencyPageRange[], + range: PageRange + ): PersistencyPageRange[] { + const start = range.start; // Inclusive + const end = range.end; // Inclusive + + const impactedScope = this.selectImpactedRanges(ranges, start, end); + + // Find out first and last impacted range index + const impactedStartIndex = impactedScope[0]; + const impactedEndIndex = impactedScope[1]; + const impactedRangesCount = + impactedStartIndex > impactedEndIndex + ? 0 // No impacted ranges + : impactedEndIndex - impactedStartIndex + 1; + + let impactedRanges: PersistencyPageRange[] = []; + + if (impactedRangesCount > 0) { + impactedRanges = ranges.slice(impactedStartIndex, impactedEndIndex + 1); + + // If first range needs to be split + const firstImpactedRange = impactedRanges[0]; + if (firstImpactedRange.end >= start && firstImpactedRange.start < start) { + impactedRanges[0] = { + start, + end: firstImpactedRange.end, + persistency: { + offset: + start - + firstImpactedRange.start + + firstImpactedRange.persistency.offset, + count: + firstImpactedRange.persistency.count - + (start - firstImpactedRange.start), + id: firstImpactedRange.persistency.id + } + }; + } + + const lastImpactedRange = impactedRanges[impactedRanges.length - 1]; + // If last impacted range needs to be split + if (end >= lastImpactedRange.start && end < lastImpactedRange.end) { + impactedRanges[impactedRanges.length - 1] = { + start: lastImpactedRange.start, + end, + persistency: { + offset: lastImpactedRange.persistency.offset, + count: + lastImpactedRange.persistency.count - + (lastImpactedRange.end - end), + id: lastImpactedRange.persistency.id + } + }; + } + } + + return impactedRanges; + } + + public fillZeroRanges( + ranges: PersistencyPageRange[], + range: PageRange + ): PersistencyPageRange[] { + ranges = this.cutRanges(ranges, range); + const filledRanges: PersistencyPageRange[] = []; + + if (ranges.length === 0) { + return [ + { + start: range.start, + end: range.end, + persistency: { + offset: 0, + count: range.end + 1 - range.start, + id: ZERO_EXTENT_ID + } + } + ]; + } + + const firstRange = ranges[0]; + const lastRange = ranges[ranges.length - 1]; + + if (range.start < firstRange.start) { + filledRanges.push({ + start: range.start, + end: firstRange.start - 1, + persistency: { + offset: 0, + count: firstRange.start - range.start, + id: ZERO_EXTENT_ID + } + }); + } + + // TODO: fill in zero ranges in the middle + for (let i = 0; i < ranges.length - 1; i++) { + const nextRange = ranges[i + 1]; + const currentRange = ranges[i]; + + filledRanges.push(currentRange); + + const gap = nextRange.start - 1 - currentRange.end; + if (gap > 0) { + filledRanges.push({ + start: currentRange.end + 1, + end: nextRange.start - 1, + persistency: { + offset: 0, + count: gap, + id: ZERO_EXTENT_ID + } + }); + } + } + + filledRanges.push(ranges[ranges.length - 1]); + + if (lastRange.end < range.end) { + filledRanges.push({ + start: lastRange.end + 1, + end: range.end, + persistency: { + offset: 0, + count: range.end - lastRange.end, + id: ZERO_EXTENT_ID + } + }); + } + + return filledRanges; + } + + /** + * Select overlapped ranges based on binary search. + * + * @public + * @param {PersistencyPageRange[]} ranges Existing ranges + * @param {number} start Start offset of the new range, inclusive + * @param {number} end End offset of the new range, inclusive + * @returns {[number, number]} Impacted start and end ranges indexes tuple + * When no impacted ranges found, will return indexes + * for 2 closet existing ranges + * [FirstLargerRangeIndex, FirstSmallerRangeIndex] + * @memberof PageBlobRangesManager + */ + public selectImpactedRanges( + ranges: PersistencyPageRange[], + start: number, + end: number + ): [number, number] { + if (ranges.length === 0) { + return [Infinity, -1]; + } + + if (start > end || start < 0) { + throw new RangeError( + // tslint:disable-next-line:max-line-length + "PageBlobRangesManager:selectImpactedRanges() start must less equal than end parameter, start must larger equal than 0." + ); + } + + const impactedRangeStartIndex = this.locateFirstImpactedRange( + ranges, + 0, + ranges.length, + start + ); + + const impactedRangesEndIndex = this.locateLastImpactedRange( + ranges, + 0, + ranges.length, + end + ); + + return [impactedRangeStartIndex, impactedRangesEndIndex]; + } + + /** + * Locate first impacted range for a given position. + * + * @public + * @param {PersistencyPageRange[]} ranges + * @param {number} searchStart Index of start range in ranges array, inclusive + * @param {number} searchEnd Index of end range in ranges array, exclusive + * @param {number} position First range index covers or larger than position will be returned + * @returns {number} Index of first impacted range or Infinity for no results + * @memberof PageBlobHandler + */ + public locateFirstImpactedRange( + ranges: PersistencyPageRange[], + searchStart: number, + searchEnd: number, + position: number + ): number { + searchStart = searchStart < 0 ? 0 : searchStart; + searchEnd = searchEnd > ranges.length ? searchEnd : searchEnd; + if (ranges.length === 0 || searchStart >= searchEnd) { + return Infinity; + } + + // Only last element to check + if (searchStart === searchEnd - 1) { + return this.positionInRange(ranges[searchStart], position) || + position < ranges[searchStart].start + ? searchStart + : Infinity; + } + + // 2 or more elements left + const searchMid = Math.floor((searchStart + searchEnd) / 2); + const indexInLeft = this.locateFirstImpactedRange( + ranges, + searchStart, + searchMid, + position + ); + if (indexInLeft !== Infinity) { + return indexInLeft; + } + if ( + this.positionInRange(ranges[searchMid], position) || + position < ranges[searchMid].start + ) { + return searchMid; + } else { + return this.locateFirstImpactedRange( + ranges, + searchMid + 1, // Remove searchMid range from searching scope + searchEnd, + position + ); + } + } + + /** + * Locate last impacted range for a given position. + * + * @public + * @param {PersistencyPageRange[]} ranges + * @param {number} searchStart Index of start range in ranges array, inclusive + * @param {number} searchEnd Index of end range in ranges array, exclusive + * @param {number} position Last range index covers or less than position will be returned + * @returns {number} Index of first impacted range or -1 for no results + * @memberof PageBlobHandler + */ + public locateLastImpactedRange( + ranges: PersistencyPageRange[], + searchStart: number, + searchEnd: number, + position: number + ): number { + searchStart = searchStart < 0 ? 0 : searchStart; + searchEnd = searchEnd > ranges.length ? searchEnd : searchEnd; + if (ranges.length === 0 || searchStart >= searchEnd) { + return -1; + } + + // Only last element to check + if (searchStart === searchEnd - 1) { + return this.positionInRange(ranges[searchStart], position) || + position > ranges[searchStart].end + ? searchStart + : -1; + } + + // 2 or more elements left + const searchMid = Math.floor((searchStart + searchEnd) / 2); + const indexInRight = this.locateLastImpactedRange( + ranges, + searchMid + 1, // Remove searchMid range from searching scope + searchEnd, + position + ); + if (indexInRight > -1) { + return indexInRight; + } + if ( + this.positionInRange(ranges[searchMid], position) || + position > ranges[searchMid].end + ) { + return searchMid; + } else { + return this.locateLastImpactedRange( + ranges, + searchStart, + searchMid, + position + ); + } + } + + public positionInRange( + range: PersistencyPageRange, + position: number + ): boolean { + return position >= range.start && position <= range.end; + } +} diff --git a/src/dfs/handlers/PathOperationsHandler.ts b/src/dfs/handlers/PathOperationsHandler.ts new file mode 100644 index 000000000..761a0ea23 --- /dev/null +++ b/src/dfs/handlers/PathOperationsHandler.ts @@ -0,0 +1,1648 @@ +import url from "url"; + +import ILogger from "../../common/ILogger"; +import IExtentStore from "../../common/persistence/IExtentStore"; +import { + getMD5FromStream, + getUniqueName, + newEtag +} from "../../common/utils/utils"; +import DataLakeContext from "../context/DataLakeContext"; +import NotImplementedError from "../errors/NotImplementedError"; +import StorageErrorFactory from "../errors/StorageErrorFactory"; +import * as Models from "../generated/artifacts/models"; +import Context from "../generated/Context"; +import IPathOperationsHandler from "../generated/handlers/IPathOperationsHandler"; +import IDataLakeMetaDataStore, { + BlobModel +} from "../persistence/IDataLakeMetadataStore"; +import { + DATA_LAKE_API_VERSION, + DEFAULT_DIR_PERMISSIONS, + DEFAULT_FILE_PERMISSIONS, + DEFAULT_GROUP, + DEFAULT_OWNER, + DEFAULT_UMMASK, + HeaderConstants, + MAX_APPEND_BLOB_BLOCK_COUNT, + MAX_APPEND_BLOB_BLOCK_SIZE +} from "../utils/constants"; +import { + deserializePageBlobRangeHeader, + deserializeRangeHeader, + removeSlash +} from "../utils/utils"; +import BaseHandler from "./BaseHandler"; +import IPageBlobRangesManager from "./IPageBlobRangesManager"; +import { + toPermissions, + toPermissionsString, + toAclString, + toAcl, + permissionsStringToAclString +} from "../storage-file-datalake/transforms"; + +/** + * PathOperationsHandler handles Azure Storage DataLake Gen2 path + * + * @export + * @class PathOperationsHandler + * @extends {BaseHandler} + * @implements {IBlobHandler} + */ +export default class PathOperationsHandler + extends BaseHandler + implements IPathOperationsHandler +{ + constructor( + metadataStore: IDataLakeMetaDataStore, + extentStore: IExtentStore, + logger: ILogger, + loose: boolean, + private readonly rangesManager: IPageBlobRangesManager + ) { + super(metadataStore, extentStore, logger, loose); + } + + async create( + options: Models.PathCreateOptionalParams, + context: Context + ): Promise { + if (options.properties) { + const metadata: { [propertyName: string]: string } = + options.metadata || {}; + const metaDataValues = options.properties.split(","); + metaDataValues.forEach((pair) => { + const idx = pair.indexOf("="); + const name = pair.substring(0, idx); + const value = Buffer.from(pair.substring(idx + 1), "base64").toString(); + + metadata[name] = value; + }); + + options.metadata = metadata; + } + + let response: Models.PathCreateResponse; + if (options.resource === "file") { + response = await this.createBlob(options, context); + } else if (options.resource === "directory") { + response = await this.createDirectory(options, context); + } else if (options.renameSource) { + response = await this.rename(options, context); + } else { + throw StorageErrorFactory.getInvalidQueryParameterValue( + context, + "resource", + options.resource, + "resource must be set to either 'file' or 'directory' or renameSource is set" + ); + } + + return response; + } + + async update( + action: Models.PathUpdateAction, + mode: Models.PathSetAccessControlRecursiveMode, + body: NodeJS.ReadableStream, + options: Models.PathUpdateOptionalParams, + context: Context + ): Promise { + switch (action) { + case Models.PathUpdateAction.Append: + return await this.appendData(body, options, context); + case Models.PathUpdateAction.Flush: + return await this.flushData(options, context); + case Models.PathUpdateAction.SetAccessControl: + return await this.setAccessControl(options, context); + case Models.PathUpdateAction.SetAccessControlRecursive: + return await this.setAccessControlRecursive(mode, options, context); + case Models.PathUpdateAction.SetProperties: + return await this.setProperties(options, context); + } + } + + async lease( + xMsLeaseAction: Models.PathLeaseAction, + options: Models.PathLeaseOptionalParams, + context: Context + ): Promise { + let response: Models.PathLeaseResponse; + switch (xMsLeaseAction) { + case Models.PathLeaseAction.Acquire: + await this.aquireLease(options, context); + response = { statusCode: 201, leaseId: options.proposedLeaseId }; + break; + case Models.PathLeaseAction.Break: + await this.breakLease(options, context); + response = { + statusCode: 202, + leaseTime: `${options.xMsLeaseBreakPeriod}` + }; + case Models.PathLeaseAction.Change: + await this.changeLease(options, context); + response = { statusCode: 200, leaseId: options.proposedLeaseId }; + break; + case Models.PathLeaseAction.Renew: + await this.renewLease(options, context); + response = { statusCode: 200, leaseId: options.proposedLeaseId }; + break; + case Models.PathLeaseAction.Release: + await this.releaseLease(options, context, false); + response = { statusCode: 200, leaseId: options.proposedLeaseId }; + break; + } + + const blobCtx = new DataLakeContext(context); + const account = blobCtx.account!; + const container = blobCtx.container!; + const blob = blobCtx.blob!; + const blobModel: BlobModel = await this.metadataStore.downloadBlob( + context, + account, + container, + blob, + "" + ); + response.date = context.startTime; + response.requestId = context.contextId; + response.eTag = blobModel?.properties.etag; + return response; + } + + async read( + options: Models.PathReadOptionalParams, + context: Context + ): Promise { + if (options.rangeGetContentCRC64 && options.rangeGetContentMD5) { + throw StorageErrorFactory.getInvalidInput( + context, + "rangeGetContentCRC64 and rangeGetContentCRC64 can't be both set at the same time" + ); + } + + const blobCtx = new DataLakeContext(context); + const account = blobCtx.account!; + const container = blobCtx.container!; + const blobName = removeSlash(blobCtx.blob!); + const blob = await this.metadataStore.downloadBlob( + context, + account, + container, + blobName, + options.snapshot, + options.leaseAccessConditions, + options.modifiedAccessConditions + ); + + if (blob.properties.blobType === Models.BlobType.BlockBlob) { + return this.downloadBlockBlobOrAppendBlob(options, context, blob); + } else if (blob.properties.blobType === Models.BlobType.PageBlob) { + return this.downloadPageBlob(options, context, blob); + } else if (blob.properties.blobType === Models.BlobType.AppendBlob) { + return this.downloadBlockBlobOrAppendBlob(options, context, blob); + } else { + //DataLakeApi + return this.downloadBlockBlobOrAppendBlob(options, context, blob); + } + } + + /** + * Download block blob or append blob. + * + * @private + * @param {Models.PathReadOptionalParams} options + * @param {Context} context + * @param {BlobModel} blob + * @returns {Promise} + * @memberof BlobHandler + */ + private async downloadBlockBlobOrAppendBlob( + options: Models.PathReadOptionalParams, + context: Context, + blob: BlobModel + ): Promise { + if (blob.isCommitted === false) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + // Deserializer doesn't handle range header currently, manually parse range headers here + const rangesParts = deserializeRangeHeader( + context.request!.getHeader("range"), + context.request!.getHeader("x-ms-range") + ); + const rangeStart = rangesParts[0]; + let rangeEnd = rangesParts[1]; + + // Start Range is bigger than blob length + if (rangeStart > blob.properties.contentLength!) { + throw StorageErrorFactory.getInvalidPageRange(context); + } + + // Will automatically shift request with longer data end than blob size to blob size + if (rangeEnd + 1 >= blob.properties.contentLength!) { + rangeEnd = blob.properties.contentLength! - 1; + } + + const contentLength = rangeEnd - rangeStart + 1; + const partialRead = contentLength !== blob.properties.contentLength!; + + this.logger.info( + // tslint:disable-next-line:max-line-length + `BlobHandler:downloadBlockBlobOrAppendBlob() NormalizedDownloadRange=bytes=${rangeStart}-${rangeEnd} RequiredContentLength=${contentLength}`, + context.contextId + ); + + let bodyGetter: () => Promise; + const blocks = blob.committedBlocksInOrder; + if (blocks === undefined || blocks.length === 0) { + bodyGetter = async () => { + if (blob.persistency === undefined) { + return this.extentStore.readExtent(undefined, context.contextId); + } + return this.extentStore.readExtent( + { + id: blob.persistency.id, + offset: blob.persistency.offset + rangeStart, + count: Math.min(blob.persistency.count, contentLength) + }, + context.contextId + ); + }; + } else { + bodyGetter = async () => { + return this.extentStore.readExtents( + blocks.map((block) => block.persistency), + rangeStart, + rangeEnd + 1 - rangeStart, + context.contextId + ); + }; + } + + let contentRange: string | undefined; + if ( + context.request!.getHeader("range") || + context.request!.getHeader("x-ms-range") + ) { + contentRange = `bytes ${rangeStart}-${rangeEnd}/${blob.properties + .contentLength!}`; + } + + let body: NodeJS.ReadableStream | undefined = await bodyGetter(); + let contentMD5: Uint8Array | undefined; + if (!partialRead) { + contentMD5 = blob.properties.contentMD5; + } + if ( + contentLength <= 4 * 1024 * 1024 && + contentMD5 === undefined && + body !== undefined + ) { + contentMD5 = await getMD5FromStream(body); + body = await bodyGetter(); + } + + const response: Models.PathReadResponse = { + statusCode: contentRange ? 206 : 200, + body, + metadata: blob.metadata, + eTag: blob.properties.etag, + requestId: context.contextId, + date: context.startTime!, + version: DATA_LAKE_API_VERSION, + ...blob.properties, + cacheControl: + context.request!.getQuery("rscc") ?? blob.properties.cacheControl, + contentDisposition: + context.request!.getQuery("rscd") ?? blob.properties.contentDisposition, + contentEncoding: + context.request!.getQuery("rsce") ?? blob.properties.contentEncoding, + contentLanguage: + context.request!.getQuery("rscl") ?? blob.properties.contentLanguage, + contentType: + context.request!.getQuery("rsct") ?? blob.properties.contentType, + blobContentMD5: blob.properties.contentMD5, + acceptRanges: "bytes", + contentLength, + contentRange, + contentMD5, + isServerEncrypted: true, + clientRequestId: options.requestId, + blobCommittedBlockCount: + blob.properties.blobType === Models.BlobType.AppendBlob + ? (blob.committedBlocksInOrder || []).length + : undefined, + resourceType: "file" + // metadata: blob.metadata, + }; + + return response; + } + + /** + * Download page blob. + * + * @private + * @param {Models.PathReadOptionalParams} options + * @param {Context} context + * @param {BlobModel} blob + * @returns {Promise} + * @memberof BlobHandler + */ + private async downloadPageBlob( + options: Models.PathReadOptionalParams, + context: Context, + blob: BlobModel + ): Promise { + // Deserializer doesn't handle range header currently, manually parse range headers here + const rangesParts = deserializePageBlobRangeHeader( + context.request!.getHeader("range"), + context.request!.getHeader("x-ms-range"), + false + ); + const rangeStart = rangesParts[0]; + let rangeEnd = rangesParts[1]; + + // Start Range is bigger than blob length + if (rangeStart > blob.properties.contentLength!) { + throw StorageErrorFactory.getInvalidPageRange(context); + } + + // Will automatically shift request with longer data end than blob size to blob size + if (rangeEnd + 1 >= blob.properties.contentLength!) { + rangeEnd = blob.properties.contentLength! - 1; + } + + const contentLength = rangeEnd - rangeStart + 1; + const partialRead = contentLength !== blob.properties.contentLength!; + + this.logger.info( + // tslint:disable-next-line:max-line-length + `BlobHandler:downloadPageBlob() NormalizedDownloadRange=bytes=${rangeStart}-${rangeEnd} RequiredContentLength=${contentLength}`, + context.contextId + ); + + // if (contentLength <= 0) { + // return { + // statusCode: 200, + // body: undefined, + // metadata: blob.metadata, + // eTag: blob.properties.etag, + // requestId: context, + // date: context.startTime!, + // version: BLOB_API_VERSION, + // ...blob.properties, + // contentLength, + // contentMD5: undefined + // }; + // } + + blob.pageRangesInOrder = blob.pageRangesInOrder || []; + const ranges = + contentLength <= 0 + ? [] + : this.rangesManager.fillZeroRanges(blob.pageRangesInOrder, { + start: rangeStart, + end: rangeEnd + }); + + const bodyGetter = async () => { + return this.extentStore.readExtents( + ranges.map((value) => value.persistency), + 0, + contentLength, + context.contextId + ); + }; + + let body: NodeJS.ReadableStream | undefined = await bodyGetter(); + let contentMD5: Uint8Array | undefined; + if (!partialRead) { + contentMD5 = blob.properties.contentMD5; + } + if ( + contentLength <= 4 * 1024 * 1024 && + contentMD5 === undefined && + body !== undefined + ) { + contentMD5 = await getMD5FromStream(body); + body = await bodyGetter(); + } + + let contentRange: string | undefined; + if ( + context.request!.getHeader("range") || + context.request!.getHeader("x-ms-range") + ) { + contentRange = `bytes ${rangeStart}-${rangeEnd}/${blob.properties + .contentLength!}`; + } + + const response: Models.PathReadResponse = { + statusCode: + rangesParts[1] === Infinity && rangesParts[0] === 0 ? 200 : 206, + body, + metadata: blob.metadata, + eTag: blob.properties.etag, + requestId: context.contextId, + date: context.startTime!, + version: DATA_LAKE_API_VERSION, + ...blob.properties, + cacheControl: + context.request!.getQuery("rscc") ?? blob.properties.cacheControl, + contentDisposition: + context.request!.getQuery("rscd") ?? blob.properties.contentDisposition, + contentEncoding: + context.request!.getQuery("rsce") ?? blob.properties.contentEncoding, + contentLanguage: + context.request!.getQuery("rscl") ?? blob.properties.contentLanguage, + contentType: + context.request!.getQuery("rsct") ?? blob.properties.contentType, + contentLength, + contentRange, + contentMD5, + blobContentMD5: blob.properties.contentMD5, + isServerEncrypted: true, + creationTime: blob.properties.creationTime, + clientRequestId: options.requestId, + resourceType: "file" + // metadata: blob.metadata, + }; + + return response; + } + + async setProperties( + options: Models.PathSetPropertiesOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const account = blobCtx.account!; + const container = blobCtx.container!; + const blobName = removeSlash(blobCtx.blob!); + const model = await this.metadataStore.downloadBlob( + context, + account, + container, + blobName, + undefined, + options.leaseAccessConditions, + options.modifiedAccessConditions + ); + model.properties.cacheControl = options.pathHTTPHeaders?.cacheControl; + model.properties.contentDisposition = + options.pathHTTPHeaders?.contentDisposition; + model.properties.contentEncoding = options.pathHTTPHeaders?.contentEncoding; + model.properties.contentLanguage = options.pathHTTPHeaders?.contentLanguage; + model.properties.contentType = options.pathHTTPHeaders?.contentType; + model.properties.contentMD5 = options.pathHTTPHeaders?.contentMD5; + model.properties.properties = options.properties; + if (options.permissions) { + const permissions = toPermissions(options.permissions); + + if (permissions === undefined) { + throw StorageErrorFactory.getInvalidHeaderValue(context); + } + + model.permissions = permissions; + } + + this.metadataStore.createBlob( + context, + model, + options.leaseAccessConditions, + options.modifiedAccessConditions + ); + + const response: Models.PathSetPropertiesResponse = { + statusCode: 200, + cacheControl: options.pathHTTPHeaders?.cacheControl, + contentDisposition: options.pathHTTPHeaders?.contentDisposition, + contentEncoding: options.pathHTTPHeaders?.contentEncoding, + contentLanguage: options.pathHTTPHeaders?.contentLanguage, + contentType: options.pathHTTPHeaders?.contentType, + contentMD5: options.pathHTTPHeaders?.contentMD5, + properties: options.properties, + requestId: context.contextId, + clientRequestId: options.requestId, + date: context.startTime, + eTag: model.properties.etag, + lastModified: model.properties.lastModified, + contentLength: model.properties.contentLength, + version: DATA_LAKE_API_VERSION + }; + + return response; + } + + async getProperties( + options: Models.PathGetPropertiesOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const account = blobCtx.account!; + const container = blobCtx.container!; + const blobName = removeSlash(blobCtx.blob!); + const model = await this.metadataStore.downloadBlob( + context, + account, + container, + blobName, + options.snapshot, + options.leaseAccessConditions, + options.modifiedAccessConditions + ); // TODO: Create get metadata specific request in swagger + const againstMetadata = context.request!.getQuery("comp") === "metadata"; + const response: Models.PathGetPropertiesResponse = againstMetadata + ? { + statusCode: 200, + eTag: model.properties.etag, + requestId: context.contextId, + version: DATA_LAKE_API_VERSION, + date: context.startTime, + clientRequestId: options.requestId, + contentLength: model.properties.contentLength, + lastModified: model.properties.lastModified + } + : { + statusCode: 200, + isIncrementalCopy: model.properties.incrementalCopy, + eTag: model.properties.etag, + requestId: context.contextId, + version: DATA_LAKE_API_VERSION, + date: context.startTime, + acceptRanges: "bytes", + isServerEncrypted: true, + clientRequestId: options.requestId, + ...model.properties, + cacheControl: + context.request!.getQuery("rscc") ?? model.properties.cacheControl, + contentDisposition: + context.request!.getQuery("rscd") ?? + model.properties.contentDisposition, + contentEncoding: + context.request!.getQuery("rsce") ?? + model.properties.contentEncoding, + contentLanguage: + context.request!.getQuery("rscl") ?? + model.properties.contentLanguage, + contentType: + context.request!.getQuery("rsct") ?? model.properties.contentType + }; + + response.metadata = model.metadata; + if (!model.isDirectory) { + response.resourceType = "file"; + response.blobCommittedBlockCount = + model.properties.blobType === Models.BlobType.AppendBlob + ? ((model as BlobModel).committedBlocksInOrder || []).length + : undefined; + } else { + response.resourceType = "directory"; + if (response.metadata) { + response.metadata.hdi_isfolder = "true"; + } else { + response.metadata = { hdi_isfolder: "true" }; + } + } + + if (options.action === Models.PathGetPropertiesAction.GetAccessControl) { + response.owner = model.owner; + response.group = model.group; + response.permissions = toPermissionsString(model.permissions); + if (response.permissions || !model.acl) { + response.aCL = permissionsStringToAclString(response.permissions); + } else { + response.aCL = toAclString(model.acl); + } + } + + return response; + } + + async delete( + options: Models.PathDeleteMethodOptionalParams, + context: Context + ): Promise { + const recursive = options.recursive || false; + const blobCtx = new DataLakeContext(context); + const accountName = blobCtx.account!; + const containerName = blobCtx.container!; + const blobName = removeSlash(blobCtx.blob!); + + await this.metadataStore.checkContainerExist( + context, + accountName, + containerName + ); + + const model = await this.metadataStore.getModel( + context, + accountName, + containerName, + blobName, + false, + options.leaseAccessConditions, + options.modifiedAccessConditions, + false + ); + + if (model === undefined) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + if (!model.isDirectory) { + await this.metadataStore.deleteBlob( + context, + accountName, + containerName, + blobName, + options + ); + } else { + await this.metadataStore.deleteDirectory( + context, + accountName, + containerName, + blobName, + recursive, + options + ); + } + + //hack since Blob_Delete return 202 but Path_Delete return 200 + const accept = context.request?.getHeader("Accept"); + const statusCode = + accept !== undefined && accept.toLowerCase().includes("xml") ? 202 : 200; + const response: Models.PathDeleteResponse = { + statusCode, + requestId: context.contextId, + clientRequestId: options.requestId, + version: DATA_LAKE_API_VERSION, + date: context.startTime + }; + + return response; + } + + async setAccessControl( + options: Models.PathSetAccessControlOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const account = blobCtx.account!; + const container = blobCtx.container!; + const blobName = removeSlash(blobCtx.blob!); + const model = await this.metadataStore.downloadBlob( + context, + account, + container, + blobName, + undefined, + options.leaseAccessConditions, + options.modifiedAccessConditions + ); + if (options.owner) model.owner = options.owner; + if (options.group) model.group = options.group; + + checkPermissionAclConflict(context, options); + + if (options.permissions) { + const permissions = toPermissions(options.permissions); + if (permissions === undefined) { + throw StorageErrorFactory.getInvalidHeaderValue(context); + } + model.permissions = permissions; + } + + if (options.acl) { + const acl = toAcl(options.acl); + if (acl === undefined) { + throw StorageErrorFactory.getInvalidHeaderValue(context); + } + + model.acl = acl; + } + + await this.metadataStore.createBlob( + context, + model, + options.leaseAccessConditions, + options.modifiedAccessConditions + ); + + const response: Models.PathSetAccessControlResponse = { + statusCode: 200, + requestId: context.contextId, + clientRequestId: options.requestId, + date: context.startTime, + eTag: model.properties.etag, + lastModified: model.properties.lastModified, + version: DATA_LAKE_API_VERSION + }; + + return response; + } + + async setAccessControlRecursive( + mode: Models.PathSetAccessControlRecursiveMode, + options: Models.PathSetAccessControlRecursiveOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const account = blobCtx.account!; + const container = blobCtx.container!; + const blobName = removeSlash(blobCtx.blob!); + let directoriesSuccessful = 0; + let failedEntries: Models.AclFailedEntry[] = []; + let filesSuccessful = 0; + let failureCount = 0; + switch (mode) { + case Models.PathSetAccessControlRecursiveMode.Set: + const [paths] = await this.metadataStore.listPaths( + context, + account, + container, + blobName, + true, + options + ); + + paths.forEach(async (path) => { + try { + await this.setAccessControl(options, blobCtx); + path.isDirectory ? directoriesSuccessful++ : filesSuccessful++; + } catch (err) { + failedEntries.push({ + name: path.name, + type: err.errorCode, + errorMessage: err.errorMessage + }); + failureCount++; + } + }); + break; + case Models.PathSetAccessControlRecursiveMode.Modify: + case Models.PathSetAccessControlRecursiveMode.Remove: + throw new NotImplementedError(context); + } + + const response: Models.PathSetAccessControlRecursiveResponse = { + statusCode: 200, + clientRequestId: options.requestId, + requestId: context.contextId, + date: context.startTime, + version: DATA_LAKE_API_VERSION, + directoriesSuccessful, + failedEntries, + failureCount, + filesSuccessful + }; + return response; + } + + private uncommittedBlocks: Map = new Map< + string, + string[] + >(); + + async flushData( + options: Models.PathFlushDataOptionalParams, + context: Context + ): Promise { + await this.aquireLease(options, context); + await this.renewLease(options, context); + + const blobCtx = new DataLakeContext(context); + const account = blobCtx.account!; + const container = blobCtx.container!; + const blobName = removeSlash(blobCtx.blob!); + const model = await this.metadataStore.downloadBlob( + context, + account, + container, + blobName, + undefined, + options.leaseAccessConditions, + options.modifiedAccessConditions + ); + model.properties = { + ...model.properties, + cacheControl: + options.pathHTTPHeaders?.cacheControl || model.properties.cacheControl, + contentDisposition: + options.pathHTTPHeaders?.contentDisposition || + model.properties.contentDisposition, + contentEncoding: + options.pathHTTPHeaders?.contentEncoding || + model.properties.contentEncoding, + contentLanguage: + options.pathHTTPHeaders?.contentLanguage || + model.properties.contentLanguage, + //TODO: need to be validated first + // contentMD5: options.pathHTTPHeaders?.contentMD5 || blobModel.properties.contentMD5, + contentType: + options.pathHTTPHeaders?.contentType || model.properties.contentType + }; + + this.metadataStore.createBlob( + context, + model, + options.leaseAccessConditions, + options.modifiedAccessConditions + ); + + const key = `${model.accountName}/${model.containerName}/${model.name}/block`; + let blocks: { blockName: string; blockCommitType: string }[]; + if (this.uncommittedBlocks.has(key)) { + blocks = this.uncommittedBlocks.get(key)!.map((blockName) => { + return { + blockName, + blockCommitType: "uncommitted" + }; + }); + + await this.metadataStore.commitBlockList( + context, + model, + blocks!, + options.leaseAccessConditions, + options.modifiedAccessConditions + ); + + this.uncommittedBlocks.delete(key); + } + + await this.releaseLease(options, context, false); + return { + statusCode: 200, + version: DATA_LAKE_API_VERSION, + clientRequestId: options.requestId, + requestId: context.contextId, + date: context.startTime, + contentLength: options.contentLength, + eTag: model.properties.etag, + lastModified: model.properties.lastModified, + isServerEncrypted: model.properties.serverEncrypted + }; + } + + async appendData( + body: NodeJS.ReadableStream, + options: Models.PathAppendDataOptionalParams, + context: Context + ): Promise { + await this.aquireLease(options, context); + await this.renewLease(options, context); + + const contentLength = options.contentLength!; + if (contentLength > MAX_APPEND_BLOB_BLOCK_SIZE) { + throw StorageErrorFactory.getRequestEntityTooLarge(context); + } + + if (contentLength === 0) { + throw StorageErrorFactory.getInvalidHeaderValue(context, { + HeaderName: HeaderConstants.CONTENT_LENGTH, + HeaderValue: "0" + }); + } + + const blobCtx = new DataLakeContext(context); + const account = blobCtx.account!; + const container = blobCtx.container!; + const blobName = removeSlash(blobCtx.blob!); + const blob = await this.metadataStore.downloadBlob( + context, + account, + container, + blobName, + undefined, + options.leaseAccessConditions, + options.modifiedAccessConditions + ); + + const committedBlockCount = (blob.committedBlocksInOrder || []).length; + if (committedBlockCount >= MAX_APPEND_BLOB_BLOCK_COUNT) { + throw StorageErrorFactory.getBlockCountExceedsLimit(context); + } + + // Persist content + const extent = await this.extentStore.appendExtent(body, context.contextId); + if (extent.count !== contentLength) { + throw StorageErrorFactory.getInvalidOperation( + context, + `The size of the request body ${extent.count} mismatches the content-length ${contentLength}.` + ); + } + + // MD5 + const contentMD5 = context.request!.getHeader(HeaderConstants.CONTENT_MD5); + let contentMD5Buffer; + let contentMD5String; + + if (contentMD5 !== undefined) { + contentMD5Buffer = + typeof contentMD5 === "string" + ? Buffer.from(contentMD5, "base64") + : contentMD5; + contentMD5String = + typeof contentMD5 === "string" + ? contentMD5 + : contentMD5Buffer.toString("base64"); + + const stream = await this.extentStore.readExtent( + extent, + context.contextId + ); + const calculatedContentMD5Buffer = await getMD5FromStream(stream); + const calculatedContentMD5String = Buffer.from( + calculatedContentMD5Buffer + ).toString("base64"); + + if (contentMD5String !== calculatedContentMD5String) { + throw StorageErrorFactory.getMd5Mismatch( + context, + contentMD5String, + calculatedContentMD5String + ); + } + } + + const key = `${blob.accountName}/${blob.containerName}/${blob.name}/block`; + const blockName = getUniqueName(key); + await this.metadataStore.stageBlock( + context, + { + accountName: blob.accountName, + containerName: blob.containerName, + blobName: blob.name, + isCommitted: false, + name: blockName, + size: extent.count, + persistency: extent + }, + options.leaseAccessConditions, + true + ); + + if (!this.uncommittedBlocks.has(key)) { + this.uncommittedBlocks.set(key, [blockName]); + } else { + this.uncommittedBlocks.get(key)?.push(blockName); + } + + const response: Models.PathAppendDataResponse = { + statusCode: 202, + eTag: blob.properties.etag, + // lastModified: properties.lastModified, + contentMD5: contentMD5Buffer, + xMsContentCrc64: undefined, + clientRequestId: options.requestId, + version: DATA_LAKE_API_VERSION, + date: context.startTime, + requestId: context.contextId, + // blobAppendOffset: `${originOffset}`, + // blobCommittedBlockCount: committedBlockCount + 1, + isServerEncrypted: true + }; + + if (options.flush) { + await this.flushData(options, context); + } else { + await this.releaseLease(options, context, true); + } + return response; + } + + async setExpiry( + expiryOptions: Models.PathExpiryOptions, + options: Models.PathSetExpiryOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const accountName = blobCtx.account!; + const containerName = blobCtx.container!; + const blobName = blobCtx.blob!; + const date = context.startTime!; + + const blobModel = await this.metadataStore.downloadBlob( + context, + accountName, + containerName, + blobName, + "" + ); + + if ( + options.expiresOn === undefined && + expiryOptions !== Models.PathExpiryOptions.NeverExpire + ) { + throw StorageErrorFactory.getMissingRequestHeader(context); + } + let expiresOn: Date | undefined; + let timeToExpireInMs; + let startDate = date; + switch (expiryOptions) { + case Models.PathExpiryOptions.NeverExpire: + expiresOn = undefined; + break; + case Models.PathExpiryOptions.RelativeToCreation: + startDate = new Date(blobModel.properties.creationTime!); + case Models.PathExpiryOptions.RelativeToNow: + timeToExpireInMs = parseInt(options.expiresOn!); + if (isNaN(timeToExpireInMs)) { + throw StorageErrorFactory.getInvalidHeaderValue(context); + } + expiresOn = new Date(startDate.getTime() + timeToExpireInMs); + break; + case Models.PathExpiryOptions.Absolute: + expiresOn = new Date(options.expiresOn!); + break; + } + + blobModel.properties.expiresOn = expiresOn; + await this.metadataStore.createBlob(context, blobModel); + + const response: Models.PathSetExpiryResponse = { + statusCode: 200, + clientRequestId: options.requestId, + requestId: context.contextId, + date, + eTag: blobModel.properties.etag, + lastModified: blobModel.properties.lastModified + }; + + return response; + } + + undelete( + options: Models.PathUndeleteOptionalParams, + context: Context + ): Promise { + throw new NotImplementedError(context); + } + + private async createBlob( + options: Models.PathCreateOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const accountName = blobCtx.account!; + const containerName = blobCtx.container!; + const blobName = blobCtx.blob!; + const date = context.startTime!; + const etag = newEtag(); + const contentLength = parseInt( + context.request!.getHeader("content-length") || "-1" + ); + + if (contentLength !== 0 && !this.loose) { + throw StorageErrorFactory.getInvalidOperation( + context, + "Content-Length must be 0 for Create Append Blob request." + ); + } + + checkPermissionAclConflict(context, options); + + const permissions = toPermissions( + options.permissions || DEFAULT_FILE_PERMISSIONS, + options.umask || DEFAULT_UMMASK + ); + + if (permissions === undefined) { + throw StorageErrorFactory.getInvalidHeaderValue(context); + } + + const acl = toAcl(options.acl); + + if (acl === undefined) { + throw StorageErrorFactory.getInvalidHeaderValue(context); + } + + let expiresOn: Date | undefined; + if (options.expiresOn) { + const timeToExpireInMs = parseInt(options.expiresOn); + if (isNaN(timeToExpireInMs)) { + expiresOn = new Date(options.expiresOn); + expiresOn.setMilliseconds(0); + } else { + expiresOn = new Date(date.getTime() + timeToExpireInMs); + } + } + + const contentType = + options.pathHTTPHeaders?.contentType || + context.request!.getHeader("content-type") || + "application/octet-stream"; + + const blob: BlobModel = { + deleted: false, + metadata: options.metadata, + accountName, + containerName, + name: blobName, + properties: { + creationTime: date, + lastModified: date, + etag, + contentLength: 0, + contentType, + expiresOn, + contentEncoding: options.pathHTTPHeaders?.contentEncoding, + contentLanguage: options.pathHTTPHeaders?.contentLanguage, + contentMD5: options.pathHTTPHeaders?.contentMD5, + contentDisposition: options.pathHTTPHeaders?.contentDisposition, + cacheControl: options.pathHTTPHeaders?.cacheControl, + accessTier: Models.AccessTier.Hot, + accessTierInferred: true, + blobType: Models.BlobType.AppendBlob, + leaseStatus: Models.LeaseStatusType.Unlocked, + leaseState: Models.LeaseStateType.Available, + serverEncrypted: true + }, + snapshot: "", + isCommitted: true, + isDirectory: false, + permissions, + acl, + owner: options.owner || DEFAULT_OWNER, + group: options.group || DEFAULT_GROUP, + committedBlocksInOrder: [] + }; + + this.setAdvancedOptions(blob, options, context); + + await this.metadataStore.createBlob( + context, + blob, + options.leaseAccessConditions, + options.modifiedAccessConditions + ); + + const response: Models.PathCreateResponse = { + statusCode: 201, + eTag: etag, + lastModified: blob.properties.lastModified, + // contentMD5: blob.properties.contentMD5, + requestId: context.contextId, + version: DATA_LAKE_API_VERSION, + date, + isServerEncrypted: true, + contentLength: 0 + // clientRequestId: options.requestId + }; + + const originalBlob = blobCtx.originalBlob!; + const idx = originalBlob.lastIndexOf("/"); + if (idx < 0) return response; + const dir = originalBlob.substring(0, idx); + options.modifiedAccessConditions = {}; + await this.createSpecificDirectory(options, context, dir, true); + + return response; + } + + private async createDirectory( + options: Models.PathCreateOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const blob = blobCtx.blob!; + const recursive: boolean = + context.request?.getQuery("recursive") === "true" || true; + return await this.createSpecificDirectory( + options, + context, + blob, + recursive, + options.metadata + ); + } + private async createSpecificDirectory( + options: Models.PathCreateOptionalParams, + context: Context, + dir: string, + recursive: boolean, + metadata?: { [propertyName: string]: string } + ): Promise { + if (options.expiresOn) { + throw StorageErrorFactory.getInvalidInput( + context, + "Set Expiry is not supported for a directory" + ); + } + + const blobCtx = new DataLakeContext(context); + const account = blobCtx.account!; + const container = blobCtx.container!; + const date = context.startTime!; + const etag = newEtag(); + let leaseAccessConditions = options.leaseAccessConditions; + let modifiedAccessConditions = options.modifiedAccessConditions; + await this.metadataStore.checkContainerExist(context, account, container); + + let curDir = dir.endsWith("/") ? dir.substring(0, dir.length - 1) : dir; + let parentDir; + do { + const curDirDecoded = decodeURIComponent(curDir); + const idx = curDir.lastIndexOf("/"); + parentDir = idx < 0 ? "" : curDir.substring(0, idx); + const dirModel = await this.metadataStore.getModel( + context, + account, + container, + curDirDecoded, + false, + leaseAccessConditions, + modifiedAccessConditions, + false + ); + + if ( + dirModel && + modifiedAccessConditions && + modifiedAccessConditions.ifNoneMatch === "*" + ) { + throw StorageErrorFactory.getBlobAlreadyExists(context); + } + + const permissions = toPermissions( + options.permissions || DEFAULT_DIR_PERMISSIONS, + options.umask || DEFAULT_UMMASK + ); + + if (permissions === undefined) { + throw StorageErrorFactory.getInvalidHeaderValue(context); + } + + const acl = toAcl(options.acl); + + if (acl === undefined) { + throw StorageErrorFactory.getInvalidHeaderValue(context); + } + + const newDirModel: BlobModel = { + deleted: false, + accountName: account, + containerName: container, + name: curDirDecoded, + properties: { + creationTime: date, + lastModified: date, + etag, + blobType: Models.BlobType.BlockBlob, + serverEncrypted: true, + accessTier: Models.AccessTier.Hot, + accessTierInferred: true, + accessTierChangeTime: date, + cacheControl: options.pathHTTPHeaders?.cacheControl, + contentEncoding: options.pathHTTPHeaders?.contentEncoding, + contentLanguage: options.pathHTTPHeaders?.contentLanguage, + contentDisposition: options.pathHTTPHeaders?.contentDisposition, + contentType: options.pathHTTPHeaders?.contentType + }, + isCommitted: true, + isDirectory: true, + snapshot: "", + metadata, + owner: options.owner || DEFAULT_OWNER, + group: options.group || DEFAULT_OWNER, + permissions, + acl + }; + + this.setAdvancedOptions(newDirModel, options, context); + + await this.metadataStore.createBlob( + context, + newDirModel, + leaseAccessConditions, + modifiedAccessConditions + ); + + if (parentDir === "") break; + curDir = parentDir; + //Conditions are only valid for base directory + modifiedAccessConditions = {}; + leaseAccessConditions = {}; + } while (recursive); + + const response: Models.PathCreateResponse = { + statusCode: 201, + date, + eTag: etag, + lastModified: date, + requestId: options.requestId, + version: DATA_LAKE_API_VERSION, + contentLength: 0, + isServerEncrypted: true + }; + + return response; + } + + private async rename( + options: Models.PathCreateOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const accountName = blobCtx.account!; + const targetContainer = blobCtx.container!; + const target = blobCtx.blob!; + + let renameSource = url.parse(options.renameSource!).pathname!; + if (renameSource.startsWith("/")) renameSource = renameSource.substring(1); + if (renameSource.startsWith(accountName + "/")) { + renameSource = renameSource.substring(accountName.length + 1); + } + const idx = renameSource.indexOf("/"); + const renameSourceContainer = renameSource.substring(0, idx); + renameSource = decodeURIComponent(renameSource.substring(idx + 1)); + + const model = await this.metadataStore.getModel( + context, + accountName, + renameSourceContainer, + renameSource, + true, + undefined, //we don't send options.leaseAccessConditions since they are for target not source + undefined, //we don't send options.modifiedAccessConditions since they are for target not source + false + ); + const renameFunc = model.isDirectory + ? this.metadataStore.renameDirectory + : this.metadataStore.renameBlob; + const newModel = await renameFunc.call( + this.metadataStore, + context, + accountName, + renameSourceContainer, + renameSource, + targetContainer, + target, + options + ); + + const response: Models.PathCreateResponse = { + statusCode: 201, + eTag: newModel.properties.etag, + lastModified: newModel.properties.lastModified, + requestId: context.contextId, + version: DATA_LAKE_API_VERSION, + contentLength: 0, // we don't sent a body so we must send contentLength to 0 + isServerEncrypted: newModel.properties.serverEncrypted, + date: context.startTime + }; + + return response; + } + + private setAdvancedOptions( + model: BlobModel, + options: Models.PathCreateOptionalParams, + context: Context + ): void { + if (options.proposedLeaseId) { + model.leaseId = options.proposedLeaseId; + model.properties.leaseStatus = Models.LeaseStatusType.Locked; + model.properties.leaseState = Models.LeaseStateType.Leased; + if (options.leaseDuration === -1) { + model.properties.leaseDuration = Models.LeaseDurationType.Infinite; + } else if (options.leaseDuration! < 15 || options.leaseDuration! > 60) { + throw StorageErrorFactory.getInvalidHeaderValue(context); + } else { + model.properties.leaseDuration = Models.LeaseDurationType.Fixed; + model.leaseDurationSeconds = options.leaseDuration; + model.leaseExpireTime = new Date( + context.startTime!.getTime() + options.leaseDuration! * 1000 + ); + } + } + } + + private async aquireLease( + options: + | Models.PathAppendDataOptionalParams + | Models.PathFlushDataOptionalParams + | Models.PathLeaseOptionalParams, + context: Context + ): Promise { + if ( + "leaseAction" in options && + options.leaseAction !== Models.LeaseAction.Acquire && + options.leaseAction !== Models.LeaseAction.AcquireRelease + ) + return; + + if (options.proposedLeaseId === undefined) { + throw StorageErrorFactory.getLeaseNotPresentWithLeaseOperation(context); + } + options.leaseAccessConditions! ||= {}; + + if (options.leaseAccessConditions.leaseId !== undefined) { + throw StorageErrorFactory.getInvalidHeaderValue(context); + } + + options.leaseAccessConditions.leaseId = options.proposedLeaseId; + + if (options.xMsLeaseDuration === undefined) { + throw StorageErrorFactory.getMissingRequestHeader(context); + } + + if (options.leaseAccessConditions.leaseId === undefined) { + throw StorageErrorFactory.getLeaseNotPresentWithLeaseOperation(context); + } + + const blobCtx = new DataLakeContext(context); + const account = blobCtx.account!; + const container = blobCtx.container!; + const blob = blobCtx.blob!; + const snapshot = blobCtx.request!.getQuery("snapshot"); + + if (snapshot !== undefined && snapshot !== "") { + throw StorageErrorFactory.getInvalidOperation( + context, + "A lease cannot be granted for a blob snapshot" + ); + } + + await this.metadataStore.acquireBlobLease( + context, + account, + container, + blob, + options.xMsLeaseDuration, + options.leaseAccessConditions.leaseId, + options + ); + } + + private async releaseLease( + options: + | Models.PathAppendDataOptionalParams + | Models.PathFlushDataOptionalParams + | Models.PathLeaseOptionalParams, + context: Context, + isAppend: boolean + ): Promise { + if ( + "leaseAction" in options && + options.leaseAction !== Models.LeaseAction.Release && + options.leaseAction !== Models.LeaseAction.AcquireRelease + ) + return; + + //https://learn.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/path/update + //Starting with version 2020-08-04 ... 'Release' action is only supported in flush operation. + //but need it in case of skipApiVersion or loose + // if (isAppend && options.leaseAction === Models.LeaseAction.Release) { + // throw StorageErrorFactory.getInvalidHeaderValue(context); + // } + + options.leaseAccessConditions! ||= {}; + if (options.leaseAccessConditions.leaseId === undefined) { + throw StorageErrorFactory.getMissingRequestHeader(context); + } + + const blobCtx = new DataLakeContext(context); + const account = blobCtx.account!; + const container = blobCtx.container!; + const blob = blobCtx.blob!; + await this.metadataStore.releaseBlobLease( + context, + account, + container, + blob, + options.leaseAccessConditions.leaseId, + options + ); + } + + private async renewLease( + options: + | Models.PathAppendDataOptionalParams + | Models.PathFlushDataOptionalParams + | Models.PathLeaseOptionalParams, + context: Context + ): Promise { + if ( + "leaseAction" in options && + options.leaseAction !== Models.LeaseAction.Renew && + options.leaseAction !== Models.LeaseAction.AutoRenew + ) + return; + + options.leaseAccessConditions! ||= {}; + if (options.leaseAccessConditions.leaseId === undefined) { + throw StorageErrorFactory.getLeaseNotPresentWithLeaseOperation(context); + } + + const blobCtx = new DataLakeContext(context); + const account = blobCtx.account!; + const container = blobCtx.container!; + const blob = blobCtx.blob!; + const snapshot = blobCtx.request!.getQuery("snapshot"); + + if (snapshot !== undefined && snapshot !== "") { + throw StorageErrorFactory.getInvalidOperation( + context, + "A lease cannot be granted for a blob snapshot" + ); + } + + await this.metadataStore.renewBlobLease( + context, + account, + container, + blob, + options.leaseAccessConditions.leaseId, + options + ); + } + + async breakLease(options: Models.PathLeaseOptionalParams, context: Context) { + options.leaseAccessConditions! ||= {}; + + if (options.xMsLeaseBreakPeriod === undefined) { + throw StorageErrorFactory.getMissingRequestHeader(context); + } + + if (options.leaseAccessConditions.leaseId === undefined) { + throw StorageErrorFactory.getLeaseNotPresentWithLeaseOperation(context); + } + + const blobCtx = new DataLakeContext(context); + const account = blobCtx.account!; + const container = blobCtx.container!; + const blob = blobCtx.blob!; + const snapshot = blobCtx.request!.getQuery("snapshot"); + + if (snapshot !== undefined && snapshot !== "") { + throw StorageErrorFactory.getInvalidOperation( + context, + "A lease cannot be granted for a blob snapshot" + ); + } + + await this.metadataStore.breakBlobLease( + context, + account, + container, + blob, + options.xMsLeaseBreakPeriod, + options + ); + } + + async changeLease(options: Models.PathLeaseOptionalParams, context: Context) { + if (options.proposedLeaseId === undefined) { + throw StorageErrorFactory.getLeaseNotPresentWithLeaseOperation(context); + } + options.leaseAccessConditions! ||= {}; + if (options.xMsLeaseDuration === undefined) { + throw StorageErrorFactory.getMissingRequestHeader(context); + } + + if (options.leaseAccessConditions.leaseId === undefined) { + throw StorageErrorFactory.getLeaseNotPresentWithLeaseOperation(context); + } + + const blobCtx = new DataLakeContext(context); + const account = blobCtx.account!; + const container = blobCtx.container!; + const blob = blobCtx.blob!; + const snapshot = blobCtx.request!.getQuery("snapshot"); + + if (snapshot !== undefined && snapshot !== "") { + throw StorageErrorFactory.getInvalidOperation( + context, + "A lease cannot be granted for a blob snapshot" + ); + } + + await this.metadataStore.changeBlobLease( + context, + account, + container, + blob, + options.leaseAccessConditions.leaseId, + options.proposedLeaseId, + options + ); + } +} + +function checkPermissionAclConflict( + context: Context, + options: Models.PathSetAccessControlOptionalParams +) { + if (options.permissions && options.acl) { + throw StorageErrorFactory.getInvalidInput( + context, + "Permissions and Acl can't be both set at the same time" + ); + } +} diff --git a/src/dfs/handlers/ServiceHandler.ts b/src/dfs/handlers/ServiceHandler.ts new file mode 100644 index 000000000..e156a9911 --- /dev/null +++ b/src/dfs/handlers/ServiceHandler.ts @@ -0,0 +1,411 @@ +import DataLakeContext from "../context/DataLakeContext"; +import StorageErrorFactory from "../errors/StorageErrorFactory"; +import * as Models from "../generated/artifacts/models"; +import Context from "../generated/Context"; +import IServiceHandler from "../generated/handlers/IServiceHandler"; +import { parseXML } from "../generated/utils/xml"; +import { + BLOB_API_VERSION, + DATA_LAKE_API_VERSION, + DEFAULT_LIST_CONTAINERS_MAX_RESULTS, + EMULATOR_ACCOUNT_KIND, + EMULATOR_ACCOUNT_SKUNAME, + HeaderConstants +} from "../utils/constants"; +import BaseHandler from "./BaseHandler"; +import IAccountDataStore from "../../common/IAccountDataStore"; +import IExtentStore from "../../common/persistence/IExtentStore"; +import IDataLakeMetadataStore from "../persistence/IDataLakeMetadataStore"; +import ILogger from "../../common/ILogger"; +import { BlobBatchHandler } from "./BlobBatchHandler"; +import { Readable } from "stream"; +import { OAuthLevel } from "../../common/models"; +import { BEARER_TOKEN_PREFIX } from "../../common/utils/constants"; +import { decode } from "jsonwebtoken"; +import { getUserDelegationKeyValue } from "../utils/utils"; +import NotImplementedError from "../errors/NotImplementedError"; + +/** + * ServiceHandler handles Azure Storage Blob service related requests. + * + * @export + * @class ServiceHandler + * @implements {IHandler} + */ +export default class ServiceHandler + extends BaseHandler + implements IServiceHandler +{ + constructor( + private readonly accountDataStore: IAccountDataStore, + private readonly oauth: OAuthLevel | undefined, + metadataStore: IDataLakeMetadataStore, + extentStore: IExtentStore, + logger: ILogger, + loose: boolean + ) { + super(metadataStore, extentStore, logger, loose); + } + /** + * Default service properties. + * + * @private + * @memberof ServiceHandler + */ + private readonly defaultServiceProperties = { + cors: [], + defaultServiceVersion: BLOB_API_VERSION, + hourMetrics: { + enabled: false, + retentionPolicy: { + enabled: false + }, + version: "1.0" + }, + logging: { + deleteProperty: true, + read: true, + retentionPolicy: { + enabled: false + }, + version: "1.0", + write: true + }, + minuteMetrics: { + enabled: false, + retentionPolicy: { + enabled: false + }, + version: "1.0" + }, + staticWebsite: { + enabled: false + } + }; + + /** + * List filesystems aka containers. + * + * @param {Models.ServiceListFileSystemsOptionalParams} options + * @param {Context} context + * @returns {Promise} + * @memberof ServiceHandler + */ + async listFileSystems( + options: Models.ServiceListFileSystemsOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const accountName = blobCtx.account!; + + options.maxResults = + options.maxResults || DEFAULT_LIST_CONTAINERS_MAX_RESULTS; + options.prefix = options.prefix || ""; + + const marker = options.continuation || ""; + + const [filesystems, continuation] = await this.metadataStore.listContainers( + context, + accountName, + options.prefix, + options.maxResults, + marker + ); + + const res: Models.ServiceListFileSystemsResponse = { + filesystems, + continuation, + statusCode: 200, + requestId: context.contextId, + version: DATA_LAKE_API_VERSION + }; + + return res; + } + + public async getUserDelegationKey( + keyInfo: Models.KeyInfo, + options: Models.ServiceGetUserDelegationKeyOptionalParams, + context: Context + ): Promise { + const blobContext = new DataLakeContext(context); + const request = blobContext.request!; + const authHeaderValue = request.getHeader(HeaderConstants.AUTHORIZATION); + const token = authHeaderValue!.substr(BEARER_TOKEN_PREFIX.length + 1); + const decodedToken = decode(token) as { [key: string]: any }; + const keyValue = getUserDelegationKeyValue( + decodedToken.oid, + decodedToken.tid, + keyInfo.start, + keyInfo.expiry, + BLOB_API_VERSION + ); + + const response: Models.ServiceGetUserDelegationKeyResponse = { + statusCode: 200, + signedOid: decodedToken.oid, + signedTid: decodedToken.tid, + signedService: "b", + signedVersion: BLOB_API_VERSION, + signedStart: keyInfo.start, + signedExpiry: keyInfo.expiry, + value: keyValue + }; + + return response; + } + + public async submitBatch( + body: NodeJS.ReadableStream, + contentLength: number, + multipartContentType: string, + options: Models.ServiceSubmitBatchOptionalParams, + context: Context + ): Promise { + const blobServiceCtx = new DataLakeContext(context); + const requestBatchBoundary = blobServiceCtx + .request!.getHeader("content-type")! + .split("=")[1]; + + const blobBatchHandler = new BlobBatchHandler( + this.accountDataStore, + this.oauth, + this.metadataStore, + this.extentStore, + this.logger, + this.loose + ); + + const responseBodyString = await blobBatchHandler.submitBatch( + body, + requestBatchBoundary, + "", + context.request!, + context + ); + + const responseBody = new Readable(); + responseBody.push(responseBodyString); + responseBody.push(null); + + // No client request id defined in batch response, should refine swagger and regenerate from it. + // batch response succeed code should be 202 instead of 200, should refine swagger and regenerate from it. + const response: Models.ServiceSubmitBatchResponse = { + statusCode: 202, + requestId: context.contextId, + version: BLOB_API_VERSION, + contentType: "multipart/mixed; boundary=" + requestBatchBoundary, + body: responseBody + }; + + return response; + } + + /** + * Set blob service properties. + * + * @param {Models.StorageServiceProperties} storageServiceProperties + * @param {Models.ServiceSetPropertiesOptionalParams} options + * @param {Context} context + * @returns {Promise} + * @memberof ServiceHandler + */ + public async setProperties( + storageServiceProperties: Models.StorageServiceProperties, + options: Models.ServiceSetPropertiesOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const accountName = blobCtx.account!; + + // TODO: deserializor has a bug that when cors is undefined, + // it will serialize it to empty array instead of undefined + const body = blobCtx.request!.getBody(); + const parsedBody = await parseXML(body || ""); + if ( + !Object.hasOwnProperty.bind(parsedBody)("cors") && + !Object.hasOwnProperty.bind(parsedBody)("Cors") + ) { + storageServiceProperties.cors = undefined; + } + + // Azure Storage allows allowedHeaders and exposedHeaders to be empty, + // Azurite will set to empty string for this scenario + for (const cors of storageServiceProperties.cors || []) { + cors.allowedHeaders = cors.allowedHeaders || ""; + cors.exposedHeaders = cors.exposedHeaders || ""; + } + + await this.metadataStore.setServiceProperties(context, { + ...storageServiceProperties, + accountName + }); + + const response: Models.ServiceSetPropertiesResponse = { + requestId: context.contextId, + statusCode: 202, + version: BLOB_API_VERSION, + clientRequestId: options.requestId + }; + return response; + } + + /** + * Get blob service properties. + * + * @param {Models.ServiceGetPropertiesOptionalParams} options + * @param {Context} context + * @returns {Promise} + * @memberof ServiceHandler + */ + public async getProperties( + options: Models.ServiceGetPropertiesOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const accountName = blobCtx.account!; + + let properties = await this.metadataStore.getServiceProperties( + context, + accountName + ); + if (!properties) { + properties = { ...this.defaultServiceProperties, accountName }; + } + + if (properties.cors === undefined) { + properties.cors = []; + } + + if (properties.cors === undefined) { + properties.cors = []; + } + + if (properties.hourMetrics === undefined) { + properties.hourMetrics = this.defaultServiceProperties.hourMetrics; + } + + if (properties.logging === undefined) { + properties.logging = this.defaultServiceProperties.logging; + } + + if (properties.minuteMetrics === undefined) { + properties.minuteMetrics = this.defaultServiceProperties.minuteMetrics; + } + + if (properties.defaultServiceVersion === undefined) { + properties.defaultServiceVersion = + this.defaultServiceProperties.defaultServiceVersion; + } + + if (properties.staticWebsite === undefined) { + properties.staticWebsite = this.defaultServiceProperties.staticWebsite; + } + + const response: Models.ServiceGetPropertiesResponse = { + ...properties, + requestId: context.contextId, + statusCode: 200, + version: BLOB_API_VERSION, + clientRequestId: options.requestId + }; + return response; + } + + public async getStatistics( + options: Models.ServiceGetStatisticsOptionalParams, + context: Context + ): Promise { + if (!context.context.isSecondary) { + throw StorageErrorFactory.getInvalidQueryParameterValue(context); + } + + const response: Models.ServiceGetStatisticsResponse = { + statusCode: 200, + requestId: context.contextId, + version: BLOB_API_VERSION, + date: context.startTime, + clientRequestId: options.requestId, + geoReplication: { + status: Models.GeoReplicationStatusType.Live, + lastSyncTime: context.startTime! + } + }; + return response; + } + + /** + * List containers. + * + * @param {Models.ServiceListContainersSegmentOptionalParams} options + * @param {Context} context + * @returns {Promise} + * @memberof ServiceHandler + */ + public async listContainersSegment( + options: Models.ServiceListContainersSegmentOptionalParams, + context: Context + ): Promise { + const blobCtx = new DataLakeContext(context); + const request = blobCtx.request!; + const accountName = blobCtx.account!; + + options.maxresults = + options.maxresults || DEFAULT_LIST_CONTAINERS_MAX_RESULTS; + options.prefix = options.prefix || ""; + + const marker = options.marker || ""; + + const containers = await this.metadataStore.listContainers( + context, + accountName, + options.prefix, + options.maxresults, + marker + ); + + // TODO: Need update list out container lease properties with ContainerHandler.updateLeaseAttributes() + const serviceEndpoint = `${request.getEndpoint()}/${accountName}`; + const res: Models.ServiceListContainersSegmentResponse = { + containerItems: containers[0], + maxResults: options.maxresults, + nextMarker: `${containers[1] || ""}`, + prefix: options.prefix, + serviceEndpoint, + statusCode: 200, + requestId: context.contextId, + version: BLOB_API_VERSION, + clientRequestId: options.requestId + }; + + return res; + } + + public async getAccountInfo( + context: Context + ): Promise { + const response: Models.ContainerGetAccountInfoResponse = { + statusCode: 200, + requestId: context.contextId, + clientRequestId: context.request!.getHeader("x-ms-client-request-id"), + skuName: EMULATOR_ACCOUNT_SKUNAME, + accountKind: EMULATOR_ACCOUNT_KIND, + date: context.startTime!, + version: BLOB_API_VERSION + }; + return response; + } + + public async getAccountInfoWithHead( + context: Context + ): Promise { + return this.getAccountInfo(context); + } + + public filterBlobs( + options: Models.ServiceFilterBlobsOptionalParams, + context: Context + ): Promise { + throw new NotImplementedError(context); + } +} diff --git a/src/dfs/handlers/SubResponseTextBodyStream.ts b/src/dfs/handlers/SubResponseTextBodyStream.ts new file mode 100644 index 000000000..136e4bde2 --- /dev/null +++ b/src/dfs/handlers/SubResponseTextBodyStream.ts @@ -0,0 +1,35 @@ +import { Writable } from "stream"; +import { BlobBatchSubResponse } from "./BlobBatchSubResponse"; + +export class SubResponseTextBodyStream extends Writable { + private bodyText: string; + public constructor(private readonly subResponse: BlobBatchSubResponse) { + super(); + this.bodyText = ""; + } + + public _write( + chunk: any, + encoding: BufferEncoding, + callback: (error?: Error | null | undefined) => void + ): void { + this.bodyText += chunk.toString(); + callback(); + } + + public end(cb?: (() => void) | undefined): void; + public end(chunk: any, cb?: (() => void) | undefined): void; + public end( + chunk: any, + encoding: BufferEncoding, + cb?: (() => void) | undefined + ): void; + public end(chunk?: unknown, encoding?: unknown, cb?: unknown): void { + if (chunk) this.bodyText += (chunk! as any).toString(); + this.subResponse.end(); + } + + public getBodyContent(): string { + return this.bodyText; + } +} diff --git a/src/dfs/lease/BlobLeaseAdapter.ts b/src/dfs/lease/BlobLeaseAdapter.ts new file mode 100644 index 000000000..49837e9b3 --- /dev/null +++ b/src/dfs/lease/BlobLeaseAdapter.ts @@ -0,0 +1,53 @@ +import { + LeaseDurationType, + LeaseStateType, + LeaseStatusType +} from "../generated/artifacts/models"; +import { BlobModel } from "../persistence/IDataLakeMetadataStore"; +import { ILease } from "./ILeaseState"; + +export default class BlobLeaseAdapter implements ILease { + public leaseId?: string | undefined; + public leaseState: LeaseStateType; + public leaseStatus: LeaseStatusType; + public leaseDurationType?: LeaseDurationType | undefined; + public leaseDurationSeconds?: number | undefined; + public leaseExpireTime?: Date | undefined; + public leaseBreakTime?: Date | undefined; + + public constructor(blob: BlobModel) { + if (blob.properties.leaseState === undefined) { + blob.properties.leaseState = LeaseStateType.Available; + // throw RangeError( + // `BlobLeaseAdapter:constructor() blob leaseState cannot be undefined.` + // ); + } + + if (blob.properties.leaseStatus === undefined) { + blob.properties.leaseStatus = LeaseStatusType.Unlocked; + // throw RangeError( + // `BlobLeaseAdapter:constructor() blob leaseStatus cannot be undefined.` + // ); + } + + this.leaseId = blob.leaseId; + this.leaseState = blob.properties.leaseState; + this.leaseStatus = blob.properties.leaseStatus; + this.leaseDurationType = blob.properties.leaseDuration; + this.leaseDurationSeconds = blob.leaseDurationSeconds; + this.leaseExpireTime = blob.leaseExpireTime; + this.leaseBreakTime = blob.leaseBreakTime; + } + + public toString(): string { + return JSON.stringify({ + leaseId: this.leaseId, + leaseState: this.leaseState, + leaseStatus: this.leaseStatus, + leaseDurationType: this.leaseDurationType, + leaseDurationSeconds: this.leaseDurationSeconds, + leaseExpireTime: this.leaseExpireTime, + leaseBreakTime: this.leaseBreakTime + }); + } +} diff --git a/src/dfs/lease/BlobLeaseSyncer.ts b/src/dfs/lease/BlobLeaseSyncer.ts new file mode 100644 index 000000000..face2d11c --- /dev/null +++ b/src/dfs/lease/BlobLeaseSyncer.ts @@ -0,0 +1,17 @@ +import { BlobModel } from "../persistence/IDataLakeMetadataStore"; +import { ILease, ILeaseSyncer } from "./ILeaseState"; + +export default class BlobLeaseSyncer implements ILeaseSyncer { + public constructor(public readonly blob: BlobModel) {} + + public sync(lease: ILease): BlobModel { + this.blob.leaseId = lease.leaseId; + this.blob.leaseExpireTime = lease.leaseExpireTime; + this.blob.leaseDurationSeconds = lease.leaseDurationSeconds; + this.blob.leaseBreakTime = lease.leaseBreakTime; + this.blob.properties.leaseDuration = lease.leaseDurationType; + this.blob.properties.leaseState = lease.leaseState; + this.blob.properties.leaseStatus = lease.leaseStatus; + return this.blob; + } +} diff --git a/src/dfs/lease/BlobReadLeaseValidator.ts b/src/dfs/lease/BlobReadLeaseValidator.ts new file mode 100644 index 000000000..4503e1b99 --- /dev/null +++ b/src/dfs/lease/BlobReadLeaseValidator.ts @@ -0,0 +1,36 @@ +import StorageErrorFactory from "../errors/StorageErrorFactory"; +import { + LeaseAccessConditions, + LeaseStatusType +} from "../generated/artifacts/models"; +import Context from "../generated/Context"; +import { ILease, ILeaseValidator } from "./ILeaseState"; + +export default class BlobReadLeaseValidator implements ILeaseValidator { + public constructor( + public readonly leaseAccessConditions?: LeaseAccessConditions + ) {} + + public validate(lease: ILease, context: Context) { + // Check only when input Leased Id is not empty + if ( + this.leaseAccessConditions !== undefined && + this.leaseAccessConditions.leaseId !== undefined && + this.leaseAccessConditions.leaseId !== "" + ) { + // Return error when lease is unlocked + if (lease.leaseStatus === LeaseStatusType.Unlocked) { + throw StorageErrorFactory.getBlobLeaseLost(context); + } else if ( + lease.leaseId !== undefined && + this.leaseAccessConditions.leaseId.toLowerCase() !== + lease.leaseId.toLowerCase() + ) { + // Return error when lease is locked but lease ID not match + throw StorageErrorFactory.getBlobLeaseIdMismatchWithBlobOperation( + context + ); + } + } + } +} diff --git a/src/dfs/lease/BlobWriteLeaseSyncer.ts b/src/dfs/lease/BlobWriteLeaseSyncer.ts new file mode 100644 index 000000000..52cf5a94c --- /dev/null +++ b/src/dfs/lease/BlobWriteLeaseSyncer.ts @@ -0,0 +1,50 @@ +import { LeaseStateType, LeaseStatusType } from "../generated/artifacts/models"; +import { BlobModel } from "../persistence/IDataLakeMetadataStore"; +import { ILease, ILeaseSyncer } from "./ILeaseState"; + +/** + * TODO: Update expire blob lease status to available on write blob operations. + * + * Need run the function on: PutBlob, SetBlobMetadata, SetBlobProperties, + * DeleteBlob, PutBlock, PutBlockList, PutPage, AppendBlock, CopyBlob(dest) + * + * @export + * @class BlobWriteLeaseSyncer + * @implements {ILeaseSyncer} + */ +export default class BlobWriteLeaseSyncer implements ILeaseSyncer { + public constructor(public readonly blob: BlobModel) {} + + public sync(lease: ILease): BlobModel { + this.blob.leaseId = lease.leaseId; + this.blob.leaseExpireTime = lease.leaseExpireTime; + this.blob.leaseDurationSeconds = lease.leaseDurationSeconds; + this.blob.leaseBreakTime = lease.leaseBreakTime; + this.blob.properties.leaseDuration = lease.leaseDurationType; + this.blob.properties.leaseState = lease.leaseState; + this.blob.properties.leaseStatus = lease.leaseStatus; + + if ( + lease.leaseState === LeaseStateType.Expired || + lease.leaseState === LeaseStateType.Broken + ) { + this.blob.properties.leaseState = LeaseStateType.Available; + this.blob.properties.leaseStatus = LeaseStatusType.Unlocked; + this.blob.properties.leaseDuration = undefined; + this.blob.leaseDurationSeconds = undefined; + this.blob.leaseId = undefined; + this.blob.leaseExpireTime = undefined; + this.blob.leaseBreakTime = undefined; + } else { + this.blob.leaseId = lease.leaseId; + this.blob.leaseExpireTime = lease.leaseExpireTime; + this.blob.leaseDurationSeconds = lease.leaseDurationSeconds; + this.blob.leaseBreakTime = lease.leaseBreakTime; + this.blob.properties.leaseDuration = lease.leaseDurationType; + this.blob.properties.leaseState = lease.leaseState; + this.blob.properties.leaseStatus = lease.leaseStatus; + } + + return this.blob; + } +} diff --git a/src/dfs/lease/BlobWriteLeaseValidator.ts b/src/dfs/lease/BlobWriteLeaseValidator.ts new file mode 100644 index 000000000..668ab05d3 --- /dev/null +++ b/src/dfs/lease/BlobWriteLeaseValidator.ts @@ -0,0 +1,40 @@ +import StorageErrorFactory from "../errors/StorageErrorFactory"; +import { + LeaseAccessConditions, + LeaseStatusType +} from "../generated/artifacts/models"; +import Context from "../generated/Context"; +import { ILease, ILeaseValidator } from "./ILeaseState"; + +export default class BlobWriteLeaseValidator implements ILeaseValidator { + public constructor( + public readonly leaseAccessConditions?: LeaseAccessConditions + ) {} + + public validate(lease: ILease, context: Context) { + // check Leased -> Expired + if (lease.leaseStatus === LeaseStatusType.Locked) { + if ( + this.leaseAccessConditions === undefined || + this.leaseAccessConditions.leaseId === undefined || + this.leaseAccessConditions.leaseId === "" + ) { + throw StorageErrorFactory.getBlobLeaseIdMissing(context); + } else if ( + lease.leaseId !== undefined && + this.leaseAccessConditions.leaseId.toLowerCase() !== + lease.leaseId.toLowerCase() + ) { + throw StorageErrorFactory.getBlobLeaseIdMismatchWithBlobOperation( + context + ); + } + } else if ( + this.leaseAccessConditions !== undefined && + this.leaseAccessConditions.leaseId !== undefined && + this.leaseAccessConditions.leaseId !== "" + ) { + throw StorageErrorFactory.getBlobLeaseLost(context); + } + } +} diff --git a/src/dfs/lease/ContainerDeleteLeaseValidator.ts b/src/dfs/lease/ContainerDeleteLeaseValidator.ts new file mode 100644 index 000000000..abf589203 --- /dev/null +++ b/src/dfs/lease/ContainerDeleteLeaseValidator.ts @@ -0,0 +1,41 @@ +import StorageErrorFactory from "../errors/StorageErrorFactory"; +import { + LeaseAccessConditions, + LeaseStatusType +} from "../generated/artifacts/models"; +import Context from "../generated/Context"; +import { ILease, ILeaseValidator } from "./ILeaseState"; + +export default class ContainerDeleteLeaseValidator implements ILeaseValidator { + public constructor( + public readonly leaseAccessConditions?: LeaseAccessConditions + ) {} + + public validate(lease: ILease, context: Context) { + // Check Lease status + if (lease.leaseStatus === LeaseStatusType.Locked) { + if ( + this.leaseAccessConditions === undefined || + this.leaseAccessConditions.leaseId === undefined || + this.leaseAccessConditions.leaseId === null + ) { + throw StorageErrorFactory.getContainerLeaseIdMissing(context); + } else if ( + lease.leaseId !== undefined && + this.leaseAccessConditions.leaseId.toLowerCase() !== + lease.leaseId.toLowerCase() + ) { + throw StorageErrorFactory.getContainerLeaseIdMismatchWithContainerOperation( + context + ); + } + } else if ( + this.leaseAccessConditions !== undefined && + this.leaseAccessConditions.leaseId !== undefined && + this.leaseAccessConditions.leaseId !== null && + this.leaseAccessConditions.leaseId !== "" + ) { + throw StorageErrorFactory.getContainerLeaseLost(context); + } + } +} diff --git a/src/dfs/lease/ContainerLeaseAdapter.ts b/src/dfs/lease/ContainerLeaseAdapter.ts new file mode 100644 index 000000000..ea624343c --- /dev/null +++ b/src/dfs/lease/ContainerLeaseAdapter.ts @@ -0,0 +1,51 @@ +import { + LeaseDurationType, + LeaseStateType, + LeaseStatusType +} from "../generated/artifacts/models"; +import { ContainerModel } from "../persistence/IDataLakeMetadataStore"; +import { ILease } from "./ILeaseState"; + +export default class ContainerLeaseAdapter implements ILease { + public leaseId?: string | undefined; + public leaseState: LeaseStateType; + public leaseStatus: LeaseStatusType; + public leaseDurationType?: LeaseDurationType | undefined; + public leaseDurationSeconds?: number | undefined; + public leaseExpireTime?: Date | undefined; + public leaseBreakTime?: Date | undefined; + + public constructor(container: ContainerModel) { + if (container.properties.leaseState === undefined) { + throw RangeError( + `ContainerLeaseAdapter:constructor() container leaseState cannot be undefined.` + ); + } + + if (container.properties.leaseStatus === undefined) { + throw RangeError( + `ContainerLeaseAdapter:constructor() container leaseStatus cannot be undefined.` + ); + } + + this.leaseId = container.leaseId; + this.leaseState = container.properties.leaseState; + this.leaseStatus = container.properties.leaseStatus; + this.leaseDurationType = container.properties.leaseDuration; + this.leaseDurationSeconds = container.leaseDurationSeconds; + this.leaseExpireTime = container.leaseExpireTime; + this.leaseBreakTime = container.leaseBreakTime; + } + + public toString(): string { + return JSON.stringify({ + leaseId: this.leaseId, + leaseState: this.leaseState, + leaseStatus: this.leaseStatus, + leaseDurationType: this.leaseDurationType, + leaseDurationSeconds: this.leaseDurationSeconds, + leaseExpireTime: this.leaseExpireTime, + leaseBreakTime: this.leaseBreakTime + }); + } +} diff --git a/src/dfs/lease/ContainerLeaseSyncer.ts b/src/dfs/lease/ContainerLeaseSyncer.ts new file mode 100644 index 000000000..c5536a2b5 --- /dev/null +++ b/src/dfs/lease/ContainerLeaseSyncer.ts @@ -0,0 +1,19 @@ +import { ContainerModel } from "../persistence/IDataLakeMetadataStore"; +import { ILease, ILeaseSyncer } from "./ILeaseState"; + +export default class ContainerLeaseSyncer + implements ILeaseSyncer +{ + public constructor(public readonly container: ContainerModel) {} + + public sync(lease: ILease): ContainerModel { + this.container.leaseId = lease.leaseId; + this.container.leaseExpireTime = lease.leaseExpireTime; + this.container.leaseDurationSeconds = lease.leaseDurationSeconds; + this.container.leaseBreakTime = lease.leaseBreakTime; + this.container.properties.leaseDuration = lease.leaseDurationType; + this.container.properties.leaseState = lease.leaseState; + this.container.properties.leaseStatus = lease.leaseStatus; + return this.container; + } +} diff --git a/src/dfs/lease/ContainerReadLeaseValidator.ts b/src/dfs/lease/ContainerReadLeaseValidator.ts new file mode 100644 index 000000000..4a4a6a56a --- /dev/null +++ b/src/dfs/lease/ContainerReadLeaseValidator.ts @@ -0,0 +1,36 @@ +import StorageErrorFactory from "../errors/StorageErrorFactory"; +import { + LeaseAccessConditions, + LeaseStatusType +} from "../generated/artifacts/models"; +import Context from "../generated/Context"; +import { ILease, ILeaseValidator } from "./ILeaseState"; + +export default class ContainerReadLeaseValidator implements ILeaseValidator { + public constructor( + public readonly leaseAccessConditions?: LeaseAccessConditions + ) {} + + public validate(lease: ILease, context: Context) { + // check only when input Leased Id is not empty + if ( + this.leaseAccessConditions !== undefined && + this.leaseAccessConditions.leaseId !== undefined && + this.leaseAccessConditions.leaseId !== "" + ) { + // return error when lease is unlocked + if (lease.leaseStatus === LeaseStatusType.Unlocked) { + throw StorageErrorFactory.getContainerLeaseLost(context); + } else if ( + lease.leaseId !== undefined && + this.leaseAccessConditions.leaseId.toLowerCase() !== + lease.leaseId.toLowerCase() + ) { + // return error when lease is locked but lease ID not match + throw StorageErrorFactory.getContainerLeaseIdMismatchWithContainerOperation( + context + ); + } + } + } +} diff --git a/src/dfs/lease/ILeaseState.ts b/src/dfs/lease/ILeaseState.ts new file mode 100644 index 000000000..4c5dedc3b --- /dev/null +++ b/src/dfs/lease/ILeaseState.ts @@ -0,0 +1,31 @@ +import * as Models from "../generated/artifacts/models"; +import Context from "../generated/Context"; + +export interface ILease { + leaseId?: string; + leaseState?: Models.LeaseStateType; // undefined, Available, Leased, Breaking, Broken, Expired + leaseStatus?: Models.LeaseStatusType; // undefined, Locked, Unlocked + leaseDurationType?: Models.LeaseDurationType; // Fixed, Infinity, undefined + leaseDurationSeconds?: number; // number, undefined, -1 + leaseExpireTime?: Date; + leaseBreakTime?: Date; +} + +export interface ILeaseValidator { + validate(lease: ILease, context: Context): void; +} + +export interface ILeaseSyncer { + sync(lease: ILease): T; +} + +export interface ILeaseState { + lease: ILease; + acquire(duration: number, proposedLeaseId?: string): ILeaseState; + break(breakPeriod?: number): ILeaseState; + renew(leaseId: string): ILeaseState; + change(leaseId: string, proposedLeaseId: string): ILeaseState; + release(leaseId: string): ILeaseState; + validate(validator: ILeaseValidator): ILeaseState; + sync(syncer: ILeaseSyncer): T; +} diff --git a/src/dfs/lease/LeaseAvailableState.ts b/src/dfs/lease/LeaseAvailableState.ts new file mode 100644 index 000000000..f0f7f1284 --- /dev/null +++ b/src/dfs/lease/LeaseAvailableState.ts @@ -0,0 +1,160 @@ +import uuid = require("uuid"); + +import StorageErrorFactory from "../errors/StorageErrorFactory"; +import { + LeaseDurationType, + LeaseStateType, + LeaseStatusType +} from "../generated/artifacts/models"; +import Context from "../generated/Context"; +import { ILease, ILeaseState } from "./ILeaseState"; +import LeaseLeasedState from "./LeaseLeasedState"; +import LeaseStateBase from "./LeaseStateBase"; + +/** + * Available lease state. + * + * @export + * @class LeaseAvailableState + */ +export default class LeaseAvailableState extends LeaseStateBase { + public constructor(lease: ILease, context: Context) { + /* + * LeaseState: Available + * LeaseStatus: Unlocked + * LeaseDurationType: undefined + * LeaseExpireTime: undefined + * LeaseDurationSeconds: undefined + * LeaseBreakTime: undefined + * LeaseId: undefined + */ + if (context.startTime === undefined) { + throw RangeError( + `LeaseAvailableState:constructor() error, context.startTime is undefined.` + ); + } + + if (lease.leaseState === undefined) { + super( + { + leaseId: undefined, + leaseState: undefined, + leaseStatus: undefined, + leaseDurationType: undefined, + leaseDurationSeconds: undefined, + leaseExpireTime: undefined, + leaseBreakTime: undefined + }, + context + ); + return; + } + + if (lease.leaseState !== LeaseStateType.Available) { + throw RangeError( + `LeaseAvailableState:constructor() error, incoming lease state ${lease.leaseState} is not ${LeaseStateType.Available}.` + ); + } + + if (lease.leaseStatus !== LeaseStatusType.Unlocked) { + throw RangeError( + `LeaseAvailableState:constructor() error, incoming lease status ${lease.leaseStatus} is not ${LeaseStatusType.Unlocked}.` + ); + } + + if (lease.leaseId !== undefined) { + throw RangeError( + `LeaseAvailableState:constructor() error, incoming leaseId ${lease.leaseId} is not undefined.` + ); + } + + if (lease.leaseExpireTime !== undefined) { + throw RangeError( + `LeaseAvailableState:constructor() error, incoming leaseExpireTime ${lease.leaseExpireTime} is not undefined.` + ); + } + + if (lease.leaseDurationSeconds !== undefined) { + throw RangeError( + `LeaseAvailableState:constructor() error, incoming leaseDurationSeconds ${lease.leaseDurationSeconds} is not undefined.` + ); + } + + if (lease.leaseDurationType !== undefined) { + throw RangeError( + `LeaseAvailableState:constructor() error, incoming leaseDurationType ${lease.leaseDurationType} is not undefined.` + ); + } + + if (lease.leaseBreakTime !== undefined) { + throw RangeError( + `LeaseAvailableState:constructor() error, incoming leaseBreakTime ${lease.leaseBreakTime} is not undefined.` + ); + } + + // Deep copy + super({ ...lease }, context); + } + + public acquire(duration: number, proposedLeaseId: string = ""): ILeaseState { + if ((duration < 15 || duration > 60) && duration !== -1) { + throw StorageErrorFactory.getInvalidLeaseDuration(this.context); + } + + // TODO: Validate proposedLeaseId follows GUID format + + if (duration === -1) { + return new LeaseLeasedState( + { + leaseId: proposedLeaseId || uuid(), + leaseState: LeaseStateType.Leased, + leaseStatus: LeaseStatusType.Locked, + leaseDurationType: LeaseDurationType.Infinite, + leaseDurationSeconds: undefined, + leaseExpireTime: undefined, + leaseBreakTime: undefined + }, + this.context + ); + } else { + return new LeaseLeasedState( + { + leaseId: proposedLeaseId || uuid(), + leaseState: LeaseStateType.Leased, + leaseStatus: LeaseStatusType.Locked, + leaseDurationType: LeaseDurationType.Fixed, + leaseDurationSeconds: duration, + leaseExpireTime: new Date( + this.context.startTime!.getTime() + duration * 1000 + ), + leaseBreakTime: undefined + }, + this.context + ); + } + } + + public break(): ILeaseState { + throw StorageErrorFactory.getLeaseNotPresentWithLeaseOperation( + this.context + ); + } + + public renew(leaseId: string): ILeaseState { + throw StorageErrorFactory.getLeaseIdMismatchWithLeaseOperation( + this.context + ); + } + + public change(): ILeaseState { + throw StorageErrorFactory.getLeaseNotPresentWithLeaseOperation( + this.context + ); + } + + public release(leaseId: string): ILeaseState { + throw StorageErrorFactory.getLeaseIdMismatchWithLeaseOperation( + this.context + ); + } +} diff --git a/src/dfs/lease/LeaseBreakingState.ts b/src/dfs/lease/LeaseBreakingState.ts new file mode 100644 index 000000000..1d78daea3 --- /dev/null +++ b/src/dfs/lease/LeaseBreakingState.ts @@ -0,0 +1,171 @@ +import { minDate } from "../../common/utils/utils"; +import StorageErrorFactory from "../errors/StorageErrorFactory"; +import { LeaseStateType, LeaseStatusType } from "../generated/artifacts/models"; +import Context from "../generated/Context"; +import { ILease, ILeaseState } from "./ILeaseState"; +import LeaseAvailableState from "./LeaseAvailableState"; +import LeaseBrokenState from "./LeaseBrokenState"; +import LeaseStateBase from "./LeaseStateBase"; + +export default class LeaseBreakingState extends LeaseStateBase { + public constructor(lease: ILease, context: Context) { + /* + * LeaseState: Breaking + * LeaseStatus: Locked + * LeaseDurationType: undefined + * LeaseExpireTime: undefined + * LeaseDurationSeconds: undefined + * LeaseBreakTime: Now < timestamp + * LeaseId: uuid + */ + if (context.startTime === undefined) { + throw RangeError( + `LeaseLeasedState:constructor() error, context.startTime is undefined.` + ); + } + + if (lease.leaseState !== LeaseStateType.Breaking) { + throw RangeError( + `LeaseLeasedState:constructor() error, incoming lease state ${lease.leaseState} is not ${LeaseStateType.Breaking}.` + ); + } + + if (lease.leaseStatus !== LeaseStatusType.Locked) { + throw RangeError( + `LeaseLeasedState:constructor() error, incoming lease status ${lease.leaseStatus} is not ${LeaseStatusType.Locked}.` + ); + } + + if (lease.leaseId === undefined) { + throw RangeError( + `LeaseLeasedState:constructor() error, incoming leaseId ${lease.leaseId} should not be undefined.` + ); + } + + if (lease.leaseExpireTime !== undefined) { + throw RangeError( + `LeaseLeasedState:constructor() error, incoming leaseExpireTime ${lease.leaseExpireTime} is not undefined.` + ); + } + + if (lease.leaseDurationSeconds !== undefined) { + throw RangeError( + `LeaseLeasedState:constructor() error, incoming leaseDurationSeconds ${lease.leaseDurationSeconds} is not undefined.` + ); + } + + if (lease.leaseDurationType !== undefined) { + throw RangeError( + `LeaseLeasedState:constructor() error, incoming leaseDurationType ${lease.leaseDurationType} is not undefined.` + ); + } + + if ( + lease.leaseBreakTime === undefined || + context.startTime >= lease.leaseBreakTime // Current time should be less than break time + ) { + throw RangeError( + `LeaseLeasedState:constructor() error, incoming leaseBreakTime ${lease.leaseBreakTime} is undefined, or less than current time ${context.startTime}.` + ); + } + + // Deep copy + super({ ...lease }, context); + } + + public acquire(duration: number, proposedLeaseId: string = ""): ILeaseState { + if (proposedLeaseId === this.lease.leaseId) { + throw StorageErrorFactory.getLeaseIsBreakingAndCannotBeAcquired( + this.context + ); + } else { + throw StorageErrorFactory.getLeaseAlreadyPresent(this.context); + } + } + + public break(breakPeriod?: number): ILeaseState { + if (breakPeriod === undefined) { + return this; + } + + if (breakPeriod === 0) { + return new LeaseBrokenState( + { + leaseId: this.lease.leaseId, + leaseState: LeaseStateType.Broken, + leaseStatus: LeaseStatusType.Unlocked, + leaseDurationType: undefined, + leaseDurationSeconds: undefined, + leaseExpireTime: undefined, + leaseBreakTime: undefined + }, + this.context + ); + } + + if (breakPeriod > 0 && breakPeriod <= 60) { + const breakTime = new Date( + this.context.startTime!.getTime() + breakPeriod * 1000 + ); + return new LeaseBreakingState( + { + leaseId: this.lease.leaseId, + leaseState: LeaseStateType.Breaking, + leaseStatus: LeaseStatusType.Locked, + leaseDurationType: undefined, + leaseDurationSeconds: undefined, + leaseExpireTime: undefined, + leaseBreakTime: minDate(this.lease.leaseBreakTime!, breakTime) + }, + this.context + ); + } + + throw StorageErrorFactory.getInvalidLeaseBreakPeriod(this.context); + } + + public renew(proposedLeaseId: string): ILeaseState { + if (proposedLeaseId === this.lease.leaseId) { + throw StorageErrorFactory.getLeaseIsBrokenAndCannotBeRenewed( + this.context + ); + } else { + throw StorageErrorFactory.getLeaseIdMismatchWithLeaseOperation( + this.context + ); + } + } + + public change(proposedLeaseId: string): ILeaseState { + if (proposedLeaseId === this.lease.leaseId) { + throw StorageErrorFactory.getLeaseIsBreakingAndCannotBeChanged( + this.context + ); + } else { + throw StorageErrorFactory.getLeaseIdMismatchWithLeaseOperation( + this.context + ); + } + } + + public release(leaseId: string): ILeaseState { + if (this.lease.leaseId !== leaseId) { + throw StorageErrorFactory.getLeaseIdMismatchWithLeaseOperation( + this.context + ); + } + + return new LeaseAvailableState( + { + leaseId: undefined, + leaseState: LeaseStateType.Available, + leaseStatus: LeaseStatusType.Unlocked, + leaseDurationType: undefined, + leaseDurationSeconds: undefined, + leaseExpireTime: undefined, + leaseBreakTime: undefined + }, + this.context + ); + } +} diff --git a/src/dfs/lease/LeaseBrokenState.ts b/src/dfs/lease/LeaseBrokenState.ts new file mode 100644 index 000000000..a733a3250 --- /dev/null +++ b/src/dfs/lease/LeaseBrokenState.ts @@ -0,0 +1,219 @@ +import uuid = require("uuid"); + +import StorageErrorFactory from "../errors/StorageErrorFactory"; +import { + LeaseDurationType, + LeaseStateType, + LeaseStatusType +} from "../generated/artifacts/models"; +import Context from "../generated/Context"; +import { ILease, ILeaseState } from "./ILeaseState"; +import LeaseAvailableState from "./LeaseAvailableState"; +import LeaseLeasedState from "./LeaseLeasedState"; +import LeaseStateBase from "./LeaseStateBase"; + +export default class LeaseBrokenState extends LeaseStateBase { + public constructor(lease: ILease, context: Context) { + if (context.startTime === undefined) { + throw RangeError( + `LeaseBrokenState:constructor() error, context.startTime is undefined.` + ); + } + + if (lease.leaseState === LeaseStateType.Broken) { + /* + * LeaseState: Broken + * LeaseStatus: Unlocked + * LeaseDurationType: undefined + * LeaseExpireTime: undefined + * LeaseDurationSeconds: undefined + * LeaseBreakTime: undefined + * LeaseId: uuid + */ + if (lease.leaseStatus !== LeaseStatusType.Unlocked) { + throw RangeError( + `LeaseBrokenState:constructor() error, incoming lease status ${lease.leaseStatus} is not ${LeaseStatusType.Unlocked}.` + ); + } + + if (lease.leaseId === undefined) { + throw RangeError( + `LeaseBrokenState:constructor() error, incoming leaseId ${lease.leaseId} should not be undefined.` + ); + } + + if (lease.leaseExpireTime !== undefined) { + throw RangeError( + `LeaseBrokenState:constructor() error, incoming leaseExpireTime ${lease.leaseExpireTime} is not undefined.` + ); + } + + if (lease.leaseDurationSeconds !== undefined) { + throw RangeError( + `LeaseBrokenState:constructor() error, incoming leaseDurationSeconds ${lease.leaseDurationSeconds} is not undefined.` + ); + } + + if (lease.leaseDurationType !== undefined) { + throw RangeError( + `LeaseBrokenState:constructor() error, incoming leaseDurationType ${lease.leaseDurationType} is not undefined.` + ); + } + + if (lease.leaseBreakTime !== undefined) { + throw RangeError( + `LeaseBrokenState:constructor() error, incoming leaseBreakTime ${lease.leaseBreakTime} is not undefined.` + ); + } + + // Deep copy + super({ ...lease }, context); + } else if (lease.leaseState === LeaseStateType.Breaking) { + /* + * LeaseState: Breaking + * LeaseStatus: Locked + * LeaseDurationType: undefined + * LeaseExpireTime: undefined + * LeaseDurationSeconds: undefined + * LeaseBreakTime: now >= timestamp + * LeaseId: uuid + */ + if (lease.leaseStatus !== LeaseStatusType.Locked) { + throw RangeError( + `LeaseBrokenState:constructor() error, incoming lease status ${lease.leaseStatus} is not ${LeaseStatusType.Locked}.` + ); + } + + if (lease.leaseId === undefined) { + throw RangeError( + `LeaseBrokenState:constructor() error, incoming leaseId ${lease.leaseId} should not be undefined.` + ); + } + + if (lease.leaseExpireTime !== undefined) { + throw RangeError( + `LeaseBrokenState:constructor() error, incoming leaseExpireTime ${lease.leaseExpireTime} is not undefined.` + ); + } + + if (lease.leaseDurationSeconds !== undefined) { + throw RangeError( + `LeaseBrokenState:constructor() error, incoming leaseDurationSeconds ${lease.leaseDurationSeconds} is not undefined.` + ); + } + + if (lease.leaseDurationType !== undefined) { + throw RangeError( + `LeaseBrokenState:constructor() error, incoming leaseDurationType ${lease.leaseDurationType} is not undefined.` + ); + } + + if ( + lease.leaseBreakTime === undefined || + context.startTime < lease.leaseBreakTime + ) { + throw RangeError( + `LeaseBrokenState:constructor() error, incoming leaseBreakTime ${lease.leaseBreakTime} is undefined, or larger than current time ${context.startTime}.` + ); + } + + super( + { + leaseId: lease.leaseId, + leaseState: LeaseStateType.Broken, + leaseStatus: LeaseStatusType.Unlocked, + leaseDurationType: undefined, + leaseDurationSeconds: undefined, + leaseExpireTime: undefined, + leaseBreakTime: undefined + }, + context + ); + } else { + throw RangeError( + `LeaseBrokenState:constructor() error, incoming lease state ${lease.leaseState} is neither ${LeaseStateType.Broken} or ${LeaseStateType.Breaking}.` + ); + } + } + + public acquire(duration: number, proposedLeaseId: string = ""): ILeaseState { + if ((duration < 15 || duration > 60) && duration !== -1) { + throw StorageErrorFactory.getInvalidLeaseDuration(this.context); + } + + // TODO: Validate proposedLeaseId follows GUID format + + if (duration === -1) { + return new LeaseLeasedState( + { + leaseId: proposedLeaseId || uuid(), + leaseState: LeaseStateType.Leased, + leaseStatus: LeaseStatusType.Locked, + leaseDurationType: LeaseDurationType.Infinite, + leaseDurationSeconds: undefined, + leaseExpireTime: undefined, + leaseBreakTime: undefined + }, + this.context + ); + } else { + return new LeaseLeasedState( + { + leaseId: proposedLeaseId || uuid(), + leaseState: LeaseStateType.Leased, + leaseStatus: LeaseStatusType.Locked, + leaseDurationType: LeaseDurationType.Fixed, + leaseDurationSeconds: duration, + leaseExpireTime: new Date( + this.context.startTime!.getTime() + duration * 1000 + ), + leaseBreakTime: undefined + }, + this.context + ); + } + } + + public break(breakPeriod?: number): ILeaseState { + return this; + } + + public renew(proposedLeaseId: string): ILeaseState { + if (proposedLeaseId === this.lease.leaseId) { + throw StorageErrorFactory.getLeaseIsBrokenAndCannotBeRenewed( + this.context + ); + } else { + throw StorageErrorFactory.getLeaseIdMismatchWithLeaseOperation( + this.context + ); + } + } + + public change(): ILeaseState { + throw StorageErrorFactory.getLeaseNotPresentWithLeaseOperation( + this.context + ); + } + + public release(leaseId: string): ILeaseState { + if (this.lease.leaseId !== leaseId) { + throw StorageErrorFactory.getLeaseIdMismatchWithLeaseOperation( + this.context + ); + } + + return new LeaseAvailableState( + { + leaseId: undefined, + leaseState: LeaseStateType.Available, + leaseStatus: LeaseStatusType.Unlocked, + leaseDurationType: undefined, + leaseDurationSeconds: undefined, + leaseExpireTime: undefined, + leaseBreakTime: undefined + }, + this.context + ); + } +} diff --git a/src/dfs/lease/LeaseExpiredState.ts b/src/dfs/lease/LeaseExpiredState.ts new file mode 100644 index 000000000..f0ac40abd --- /dev/null +++ b/src/dfs/lease/LeaseExpiredState.ts @@ -0,0 +1,243 @@ +import uuid = require("uuid"); + +import StorageErrorFactory from "../errors/StorageErrorFactory"; +import { + LeaseDurationType, + LeaseStateType, + LeaseStatusType +} from "../generated/artifacts/models"; +import Context from "../generated/Context"; +import { ILease, ILeaseState } from "./ILeaseState"; +import LeaseAvailableState from "./LeaseAvailableState"; +import LeaseBrokenState from "./LeaseBrokenState"; +import LeaseLeasedState from "./LeaseLeasedState"; +import LeaseStateBase from "./LeaseStateBase"; + +export default class LeaseExpiredState extends LeaseStateBase { + public constructor(lease: ILease, context: Context) { + if (context.startTime === undefined) { + throw RangeError( + `LeaseExpiredState:constructor() error, context.startTime is undefined.` + ); + } + + if (lease.leaseState === LeaseStateType.Expired) { + /* + * LeaseState: Expired + * LeaseStatus: Unlocked + * LeaseDurationType: undefined + * LeaseExpireTime: undefined + * LeaseDurationSeconds: number + * LeaseBreakTime: undefined + * LeaseId: uuid + */ + if (lease.leaseStatus !== LeaseStatusType.Unlocked) { + throw RangeError( + `LeaseExpiredState:constructor() error, incoming lease status ${lease.leaseStatus} is not ${LeaseStatusType.Unlocked}.` + ); + } + + if (lease.leaseId === undefined) { + throw RangeError( + `LeaseExpiredState:constructor() error, incoming leaseId ${lease.leaseId} should not be undefined.` + ); + } + + if (lease.leaseExpireTime !== undefined) { + throw RangeError( + `LeaseExpiredState:constructor() error, incoming leaseExpireTime ${lease.leaseExpireTime} is undefined.` + ); + } + + if ( + lease.leaseDurationSeconds === undefined || + lease.leaseDurationSeconds === -1 + ) { + throw RangeError( + `LeaseExpiredState:constructor() error, incoming leaseDurationSeconds ${lease.leaseDurationSeconds} is undefined or -1 (infinite).` + ); + } + + if (lease.leaseDurationType !== undefined) { + throw RangeError( + `LeaseExpiredState:constructor() error, incoming leaseDurationType ${lease.leaseDurationType} is not undefined.` + ); + } + + if (lease.leaseBreakTime !== undefined) { + throw RangeError( + `LeaseExpiredState:constructor() error, incoming leaseBreakTime ${lease.leaseBreakTime} is not undefined.` + ); + } + + // Deep copy + super({ ...lease }, context); + } else if (lease.leaseState === LeaseStateType.Leased) { + /* + * LeaseState: Leased + * LeaseStatus: Locked + * LeaseDurationType: Fixed + * LeaseExpireTime: now >= timestamp + * LeaseDurationSeconds: number (not -1) + * LeaseBreakTime: undefined + * LeaseId: uuid + */ + if (lease.leaseStatus !== LeaseStatusType.Locked) { + throw RangeError( + `LeaseExpiredState:constructor() error, incoming lease status ${lease.leaseStatus} is not ${LeaseStatusType.Locked}.` + ); + } + + if (lease.leaseId === undefined) { + throw RangeError( + `LeaseExpiredState:constructor() error, incoming leaseId ${lease.leaseId} should not be undefined.` + ); + } + + if ( + lease.leaseExpireTime === undefined || + context.startTime < lease.leaseExpireTime + ) { + throw RangeError( + `LeaseExpiredState:constructor() error, incoming leaseExpireTime ${lease.leaseExpireTime} is undefined, or larger than current time ${context.startTime}.` + ); + } + + if ( + lease.leaseDurationSeconds === undefined || + lease.leaseDurationSeconds === -1 + ) { + throw RangeError( + `LeaseExpiredState:constructor() error, incoming leaseDurationSeconds ${lease.leaseDurationSeconds} is undefined or -1 (infinite).` + ); + } + + if (lease.leaseDurationType !== LeaseDurationType.Fixed) { + throw RangeError( + `LeaseExpiredState:constructor() error, incoming leaseDurationType ${lease.leaseDurationType} is not ${LeaseDurationType.Fixed}.` + ); + } + + if (lease.leaseBreakTime !== undefined) { + throw RangeError( + `LeaseExpiredState:constructor() error, incoming leaseBreakTime ${lease.leaseBreakTime} is not undefined.` + ); + } + + super( + { + leaseId: lease.leaseId, + leaseState: LeaseStateType.Expired, + leaseStatus: LeaseStatusType.Unlocked, + leaseDurationType: undefined, + leaseDurationSeconds: lease.leaseDurationSeconds, + leaseExpireTime: undefined, + leaseBreakTime: undefined + }, + context + ); + } else { + throw RangeError( + `LeaseExpiredState:constructor() error, incoming lease state ${lease.leaseState} is neither ${LeaseStateType.Expired} or ${LeaseStateType.Leased}.` + ); + } + } + + public acquire(duration: number, proposedLeaseId: string = ""): ILeaseState { + if ((duration < 15 || duration > 60) && duration !== -1) { + throw StorageErrorFactory.getInvalidLeaseDuration(this.context); + } + + // TODO: Validate proposedLeaseId follows GUID format + + if (duration === -1) { + return new LeaseLeasedState( + { + leaseId: proposedLeaseId || uuid(), + leaseState: LeaseStateType.Leased, + leaseStatus: LeaseStatusType.Locked, + leaseDurationType: LeaseDurationType.Infinite, + leaseDurationSeconds: undefined, + leaseExpireTime: undefined, + leaseBreakTime: undefined + }, + this.context + ); + } else { + return new LeaseLeasedState( + { + leaseId: proposedLeaseId || uuid(), + leaseState: LeaseStateType.Leased, + leaseStatus: LeaseStatusType.Locked, + leaseDurationType: LeaseDurationType.Fixed, + leaseDurationSeconds: duration, + leaseExpireTime: new Date( + this.context.startTime!.getTime() + duration * 1000 + ), + leaseBreakTime: undefined + }, + this.context + ); + } + } + + public break(breakPeriod?: number): ILeaseState { + return new LeaseBrokenState( + { + leaseId: this.lease.leaseId, + leaseState: LeaseStateType.Broken, + leaseStatus: LeaseStatusType.Unlocked, + leaseDurationType: undefined, + leaseDurationSeconds: undefined, + leaseExpireTime: undefined, + leaseBreakTime: undefined + }, + this.context + ); + } + + public renew(): ILeaseState { + return new LeaseLeasedState( + { + leaseId: this.lease.leaseId, + leaseState: LeaseStateType.Leased, + leaseStatus: LeaseStatusType.Locked, + leaseDurationType: LeaseDurationType.Fixed, + leaseDurationSeconds: this.lease.leaseDurationSeconds, + leaseExpireTime: new Date( + this.context.startTime!.getTime() + + this.lease.leaseDurationSeconds! * 1000 + ), + leaseBreakTime: undefined + }, + this.context + ); + } + + public change(): ILeaseState { + throw StorageErrorFactory.getLeaseNotPresentWithLeaseOperation( + this.context + ); + } + + public release(leaseId: string): ILeaseState { + if (this.lease.leaseId !== leaseId) { + throw StorageErrorFactory.getLeaseIdMismatchWithLeaseOperation( + this.context + ); + } + + return new LeaseAvailableState( + { + leaseId: undefined, + leaseState: LeaseStateType.Available, + leaseStatus: LeaseStatusType.Unlocked, + leaseDurationType: undefined, + leaseDurationSeconds: undefined, + leaseExpireTime: undefined, + leaseBreakTime: undefined + }, + this.context + ); + } +} diff --git a/src/dfs/lease/LeaseFactory.ts b/src/dfs/lease/LeaseFactory.ts new file mode 100644 index 000000000..2fddbe1b8 --- /dev/null +++ b/src/dfs/lease/LeaseFactory.ts @@ -0,0 +1,63 @@ +import { LeaseStateType } from "../generated/artifacts/models"; +import Context from "../generated/Context"; +import { ILease, ILeaseState } from "./ILeaseState"; +import LeaseAvailableState from "./LeaseAvailableState"; +import LeaseBreakingState from "./LeaseBreakingState"; +import LeaseBrokenState from "./LeaseBrokenState"; +import LeaseExpiredState from "./LeaseExpiredState"; +import LeaseLeasedState from "./LeaseLeasedState"; + +export default class LeaseFactory { + public static createLeaseState(lease: ILease, context: Context): ILeaseState { + if (context.startTime === undefined) { + throw new RangeError( + `LeaseFactory:createLeaseState() context.startTime should not be undefined.` + ); + } + + if ( + lease.leaseState === LeaseStateType.Available || + lease.leaseState === undefined + ) { + return new LeaseAvailableState(lease, context); + } + + if (lease.leaseState === LeaseStateType.Leased) { + if ( + lease.leaseExpireTime === undefined || + context.startTime < lease.leaseExpireTime + ) { + return new LeaseLeasedState(lease, context); + } else { + return new LeaseExpiredState(lease, context); + } + } + + if (lease.leaseState === LeaseStateType.Expired) { + return new LeaseExpiredState(lease, context); + } + + if (lease.leaseState === LeaseStateType.Breaking) { + if (lease.leaseBreakTime === undefined) { + throw new RangeError( + `LeaseFactory:createLeaseState() leaseBreakTime should not be undefined when leaseState is ${LeaseStateType.Breaking}.` + ); + } + if (context.startTime < lease.leaseBreakTime) { + return new LeaseBreakingState(lease, context); + } else { + return new LeaseBrokenState(lease, context); + } + } + + if (lease.leaseState === LeaseStateType.Broken) { + return new LeaseBrokenState(lease, context); + } + + throw new RangeError( + `LeaseFactory:createLeaseState() Cannot create LeaseState instance from lease ${JSON.stringify( + lease + )}` + ); + } +} diff --git a/src/dfs/lease/LeaseLeasedState.ts b/src/dfs/lease/LeaseLeasedState.ts new file mode 100644 index 000000000..a30ff5964 --- /dev/null +++ b/src/dfs/lease/LeaseLeasedState.ts @@ -0,0 +1,293 @@ +import uuid = require("uuid"); + +import { minDate } from "../../common/utils/utils"; +import StorageErrorFactory from "../errors/StorageErrorFactory"; +import { + LeaseDurationType, + LeaseStateType, + LeaseStatusType +} from "../generated/artifacts/models"; +import Context from "../generated/Context"; +import { ILease, ILeaseState } from "./ILeaseState"; +import LeaseAvailableState from "./LeaseAvailableState"; +import LeaseBreakingState from "./LeaseBreakingState"; +import LeaseBrokenState from "./LeaseBrokenState"; +import LeaseStateBase from "./LeaseStateBase"; + +export default class LeaseLeasedState extends LeaseStateBase { + public constructor(lease: ILease, context: Context) { + /* + * LeaseState: Leased + * LeaseStatus: Locked + * LeaseDurationType: Fixed || Infinite + * LeaseExpireTime: now < timestamp || undefined + * LeaseDurationSeconds: number || -1 + * LeaseBreakTime: undefined + * LeaseId: uuid + */ + if (context.startTime === undefined) { + throw RangeError( + `LeaseLeasedState:constructor() error, context.startTime is undefined.` + ); + } + + if (lease.leaseState !== LeaseStateType.Leased) { + throw RangeError( + `LeaseLeasedState:constructor() error, incoming lease state ${lease.leaseState} is not ${LeaseStateType.Leased}.` + ); + } + + if (lease.leaseStatus !== LeaseStatusType.Locked) { + throw RangeError( + `LeaseLeasedState:constructor() error, incoming lease status ${lease.leaseStatus} is not ${LeaseStatusType.Locked}.` + ); + } + + if (lease.leaseId === undefined) { + throw RangeError( + `LeaseLeasedState:constructor() error, incoming leaseId ${lease.leaseId} should not be undefined.` + ); + } + + if ( + lease.leaseExpireTime !== undefined && + context.startTime >= lease.leaseExpireTime + ) { + throw RangeError( + `LeaseLeasedState:constructor() error, incoming leaseExpireTime ${lease.leaseExpireTime} is not undefined, and smaller than current time ${context.startTime}.` + ); + } + + // leaseDurationSeconds could be number or undefined + // if ( + // lease.leaseDurationSeconds !== undefined && + // lease.leaseDurationSeconds !== -1 + // ) { + // throw RangeError( + // tslint:disable-next-line:max-line-length + // `LeaseLeasedState:constructor() error, incoming leaseDurationSeconds ${lease.leaseDurationSeconds} is not undefined, and not equal to -1.` + // ); + // } + + if (lease.leaseDurationType === undefined) { + throw RangeError( + `LeaseLeasedState:constructor() error, incoming leaseDurationType ${lease.leaseDurationType} is undefined.` + ); + } + + if (lease.leaseBreakTime !== undefined) { + throw RangeError( + `LeaseLeasedState:constructor() error, incoming leaseBreakTime ${lease.leaseBreakTime} is not undefined.` + ); + } + + // Deep copy + super({ ...lease }, context); + } + + public acquire(duration: number, proposedLeaseId: string = ""): ILeaseState { + if (proposedLeaseId !== this.lease.leaseId) { + // TODO: Check error message + throw StorageErrorFactory.getLeaseAlreadyPresent(this.context); + } + + if ((duration < 15 || duration > 60) && duration !== -1) { + throw StorageErrorFactory.getInvalidLeaseDuration(this.context); + } + + // TODO: Validate proposedLeaseId follows GUID format + + if (duration === -1) { + return new LeaseLeasedState( + { + leaseId: proposedLeaseId || uuid(), + leaseState: LeaseStateType.Leased, + leaseStatus: LeaseStatusType.Locked, + leaseDurationType: LeaseDurationType.Infinite, + leaseDurationSeconds: undefined, + leaseExpireTime: undefined, + leaseBreakTime: undefined + }, + this.context + ); + } else { + return new LeaseLeasedState( + { + leaseId: proposedLeaseId || uuid(), + leaseState: LeaseStateType.Leased, + leaseStatus: LeaseStatusType.Locked, + leaseDurationType: LeaseDurationType.Fixed, + leaseDurationSeconds: duration, + leaseExpireTime: new Date( + this.context.startTime!.getTime() + duration * 1000 + ), + leaseBreakTime: undefined + }, + this.context + ); + } + } + + public break(breakPeriod?: number): ILeaseState { + if (this.lease.leaseDurationType === LeaseDurationType.Infinite) { + if (breakPeriod === 0 || breakPeriod === undefined) { + return new LeaseBrokenState( + { + leaseId: this.lease.leaseId, + leaseState: LeaseStateType.Broken, + leaseStatus: LeaseStatusType.Unlocked, + leaseDurationType: undefined, + leaseDurationSeconds: undefined, + leaseExpireTime: undefined, + leaseBreakTime: undefined + }, + this.context + ); + } else { + return new LeaseBreakingState( + { + leaseId: this.lease.leaseId, + leaseState: LeaseStateType.Breaking, + leaseStatus: LeaseStatusType.Locked, + leaseDurationType: undefined, + leaseDurationSeconds: undefined, + leaseExpireTime: undefined, + leaseBreakTime: new Date( + this.context.startTime!.getTime() + breakPeriod * 1000 + ) + }, + this.context + ); + } + } + + // Following only cares about this.lease.leaseDurationType === LeaseDurationType.Fixed + + if (breakPeriod === 0) { + return new LeaseBrokenState( + { + leaseId: this.lease.leaseId, + leaseState: LeaseStateType.Broken, + leaseStatus: LeaseStatusType.Unlocked, + leaseDurationType: undefined, + leaseDurationSeconds: undefined, + leaseExpireTime: undefined, + leaseBreakTime: undefined + }, + this.context + ); + } + + if (breakPeriod === undefined) { + return new LeaseBreakingState( + { + leaseId: this.lease.leaseId, + leaseState: LeaseStateType.Breaking, + leaseStatus: LeaseStatusType.Locked, + leaseDurationType: undefined, + leaseDurationSeconds: undefined, + leaseExpireTime: undefined, + leaseBreakTime: this.lease.leaseExpireTime + }, + this.context + ); + } + + if (breakPeriod < 0 || breakPeriod > 60) { + throw StorageErrorFactory.getInvalidLeaseBreakPeriod(this.context); + } + + // Following only cares about breakPeriod between (0, 60] + + const breakTime = new Date( + this.context.startTime!.getTime() + breakPeriod * 1000 + ); + + return new LeaseBreakingState( + { + leaseId: this.lease.leaseId, + leaseState: LeaseStateType.Breaking, + leaseStatus: LeaseStatusType.Locked, + leaseDurationType: undefined, + leaseDurationSeconds: undefined, + leaseExpireTime: undefined, + leaseBreakTime: minDate(this.lease.leaseExpireTime!, breakTime) + }, + this.context + ); + } + + public renew(leaseId: string): ILeaseState { + if (this.lease.leaseId !== leaseId) { + throw StorageErrorFactory.getLeaseIdMismatchWithLeaseOperation( + this.context + ); + } + + if (this.lease.leaseDurationType === LeaseDurationType.Infinite) { + return this; + } + + // Renew a fixed lease + return new LeaseLeasedState( + { + leaseId: this.lease.leaseId, + leaseState: LeaseStateType.Leased, + leaseStatus: LeaseStatusType.Locked, + leaseDurationType: LeaseDurationType.Fixed, + leaseDurationSeconds: this.lease.leaseDurationSeconds, + leaseExpireTime: new Date( + this.context.startTime!.getTime() + + this.lease.leaseDurationSeconds! * 1000 + ), + leaseBreakTime: undefined + }, + this.context + ); + } + + public change(leaseId: string, proposedLeaseId: string): ILeaseState { + if ( + this.lease.leaseId !== leaseId && + this.lease.leaseId !== proposedLeaseId + ) { + throw StorageErrorFactory.getLeaseIdMismatchWithLeaseOperation( + this.context + ); + } + + return new LeaseLeasedState( + { + leaseId: proposedLeaseId, + leaseState: LeaseStateType.Leased, + leaseStatus: LeaseStatusType.Locked, + leaseDurationType: this.lease.leaseDurationType, + leaseDurationSeconds: this.lease.leaseDurationSeconds, + leaseExpireTime: this.lease.leaseExpireTime, + leaseBreakTime: undefined + }, + this.context + ); + } + + public release(leaseId: string): ILeaseState { + if (this.lease.leaseId !== leaseId) { + throw StorageErrorFactory.getLeaseIdMismatchWithLeaseOperation( + this.context + ); + } + + return new LeaseAvailableState( + { + leaseId: undefined, + leaseState: LeaseStateType.Available, + leaseStatus: LeaseStatusType.Unlocked, + leaseDurationType: undefined, + leaseDurationSeconds: undefined, + leaseExpireTime: undefined, + leaseBreakTime: undefined + }, + this.context + ); + } +} diff --git a/src/dfs/lease/LeaseStateBase.ts b/src/dfs/lease/LeaseStateBase.ts new file mode 100644 index 000000000..6f92ae40a --- /dev/null +++ b/src/dfs/lease/LeaseStateBase.ts @@ -0,0 +1,29 @@ +import Context from "../generated/Context"; +import { + ILease, + ILeaseState, + ILeaseSyncer, + ILeaseValidator +} from "./ILeaseState"; + +export default abstract class LeaseStateBase implements ILeaseState { + public constructor( + public readonly lease: ILease, + protected readonly context: Context + ) {} + public abstract acquire( + duration: number, + proposedLeaseId?: string | undefined + ): ILeaseState; + public abstract break(breakPeriod?: number | undefined): ILeaseState; + public abstract renew(leaseId: string): ILeaseState; + public abstract change(leaseId: string, proposedLeaseId: string): ILeaseState; + public abstract release(leaseId: string): ILeaseState; + public sync(syncer: ILeaseSyncer): T { + return syncer.sync(this.lease); + } + public validate(validator: ILeaseValidator): ILeaseState { + validator.validate(this.lease, this.context); + return this; + } +} diff --git a/src/dfs/main.ts b/src/dfs/main.ts new file mode 100644 index 000000000..26d006ebb --- /dev/null +++ b/src/dfs/main.ts @@ -0,0 +1,56 @@ +#!/usr/bin/env node +import * as Logger from "../common/Logger"; +import DataLakeServer from "./DataLakeServer"; +import { DataLakeServerFactory } from "./DataLakeServerFactory"; +import SqlDataLakeServer from "./SqlDataLakeServer"; + +// tslint:disable:no-console + +function shutdown(server: DataLakeServer | SqlDataLakeServer) { + const beforeCloseMessage = `Azurite DataLake service is closing...`; + const afterCloseMessage = `Azurite DataLake service successfully closed`; + + console.log(beforeCloseMessage); + server.close().then(() => { + console.log(afterCloseMessage); + }); +} + +/** + * Entry for Azurite DataLake service. + */ +async function main() { + const blobServerFactory = new DataLakeServerFactory(); + const server = await blobServerFactory.createServer(); + const config = server.config; + + // We use logger singleton as global debugger logger to track detailed outputs cross layers + // Note that, debug log is different from access log which is only available in request handler layer to + // track every request. Access log is not singleton, and initialized in specific RequestHandlerFactory implementations + // Enable debug log by default before first release for debugging purpose + Logger.configLogger(config.enableDebugLog, config.debugLogFilePath); + + // Start server + console.log( + `Azurite DataLake service is starting on ${config.host}:${config.port}` + ); + await server.start(); + console.log( + `Azurite DataLake service successfully listens on ${server.getHttpServerAddress()}` + ); + + // Handle close event + process + .once("message", (msg) => { + if (msg === "shutdown") { + shutdown(server); + } + }) + .once("SIGINT", () => shutdown(server)) + .once("SIGTERM", () => shutdown(server)); +} + +main().catch((err) => { + console.error(`Exit due to unhandled error: ${err.message}`); + process.exit(1); +}); diff --git a/src/dfs/middlewares/AuthenticationMiddlewareFactory.ts b/src/dfs/middlewares/AuthenticationMiddlewareFactory.ts new file mode 100644 index 000000000..dd5aed637 --- /dev/null +++ b/src/dfs/middlewares/AuthenticationMiddlewareFactory.ts @@ -0,0 +1,56 @@ +import { NextFunction, Request, RequestHandler, Response } from "express"; + +import ILogger from "../../common/ILogger"; +import IAuthenticator from "../authentication/IAuthenticator"; +import DataLakeContext from "../context/DataLakeContext"; +import StorageErrorFactory from "../errors/StorageErrorFactory"; +import ExpressRequestAdapter from "../generated/ExpressRequestAdapter"; +import ExpressResponseAdapter from "../generated/ExpressResponseAdapter"; +import IRequest from "../generated/IRequest"; +import IResponse from "../generated/IResponse"; +import { DEFAULT_CONTEXT_PATH } from "../utils/constants"; + +export default class AuthenticationMiddlewareFactory { + constructor(private readonly logger: ILogger) {} + + public createAuthenticationMiddleware( + authenticators: IAuthenticator[] + ): RequestHandler { + return (req: Request, res: Response, next: NextFunction) => { + const request = new ExpressRequestAdapter(req); + const response = new ExpressResponseAdapter(res); + const context = new DataLakeContext(res.locals, DEFAULT_CONTEXT_PATH); + this.authenticate(context, request, response, authenticators) + .then((pass) => { + // TODO: To support public access, we need to modify here to reject request later in handler + if (pass) { + next(); + } else { + next(StorageErrorFactory.getAuthorizationFailure(context)); + } + }) + .catch(next); + }; + } + + public async authenticate( + context: DataLakeContext, + req: IRequest, + res: IResponse, + authenticators: IAuthenticator[] + ): Promise { + this.logger.verbose( + `AuthenticationMiddlewareFactory:createAuthenticationMiddleware() Validating authentications.`, + context.contextId + ); + + let pass: boolean | undefined = false; + for (const authenticator of authenticators) { + pass = await authenticator.validate(req, context); + if (pass === true) { + return true; + } + } + return false; + } +} diff --git a/src/dfs/middlewares/PreflightMiddlewareFactory.ts b/src/dfs/middlewares/PreflightMiddlewareFactory.ts new file mode 100644 index 000000000..1d2fa7626 --- /dev/null +++ b/src/dfs/middlewares/PreflightMiddlewareFactory.ts @@ -0,0 +1,461 @@ +import * as msRest from "@azure/ms-rest-js"; +import { + ErrorRequestHandler, + NextFunction, + Request, + RequestHandler, + Response +} from "express"; + +import glob from "glob-to-regexp"; +import ILogger from "../../common/ILogger"; +import DataLakeContext from "../context/DataLakeContext"; +import StorageErrorFactory from "../errors/StorageErrorFactory"; +import * as Mappers from "../generated/artifacts/mappers"; +import Specifications from "../generated/artifacts/specifications"; +import MiddlewareError from "../generated/errors/MiddlewareError"; +import IDataLakeMetadataStore from "../persistence/IDataLakeMetadataStore"; +import { + DEFAULT_CONTEXT_PATH, + HeaderConstants, + MethodConstants +} from "../utils/constants"; + +export default class PreflightMiddlewareFactory { + constructor(private readonly logger: ILogger) {} + + public createOptionsHandlerMiddleware( + metadataStore: IDataLakeMetadataStore + ): ErrorRequestHandler { + return ( + err: MiddlewareError | Error, + req: Request, + res: Response, + next: NextFunction + ) => { + if (req.method.toUpperCase() === MethodConstants.OPTIONS) { + const context = new DataLakeContext(res.locals, DEFAULT_CONTEXT_PATH); + + const requestId = context.contextId; + const account = context.account!; + + this.logger.info( + `PreflightMiddlewareFactory.createOptionsHandlerMiddleware(): OPTIONS request.`, + requestId + ); + + const origin = req.header(HeaderConstants.ORIGIN); + if (origin === undefined || typeof origin !== "string") { + return next( + StorageErrorFactory.getInvalidCorsHeaderValue(context, { + MessageDetails: `Invalid required CORS header Origin ${JSON.stringify( + origin + )}` + }) + ); + } + + const requestMethod = req.header( + HeaderConstants.ACCESS_CONTROL_REQUEST_METHOD + ); + if (requestMethod === undefined || typeof requestMethod !== "string") { + return next( + StorageErrorFactory.getInvalidCorsHeaderValue(context, { + MessageDetails: `Invalid required CORS header Access-Control-Request-Method ${JSON.stringify( + requestMethod + )}` + }) + ); + } + + const requestHeaders = req.headers[ + HeaderConstants.ACCESS_CONTROL_REQUEST_HEADERS + ] as string; + + metadataStore + .getServiceProperties(context, account) + .then((properties) => { + if (properties === undefined || properties.cors === undefined) { + return next( + StorageErrorFactory.corsPreflightFailure(context, { + MessageDetails: "No CORS rules matches this request" + }) + ); + } + + const corsSet = properties.cors; + for (const cors of corsSet) { + if ( + !this.checkOrigin(origin, cors.allowedOrigins) || + !this.checkMethod(requestMethod, cors.allowedMethods) + ) { + continue; + } + if ( + requestHeaders !== undefined && + !this.checkHeaders(requestHeaders, cors.allowedHeaders || "") + ) { + continue; + } + + res.setHeader( + HeaderConstants.ACCESS_CONTROL_ALLOW_ORIGIN, + origin + ); + res.setHeader( + HeaderConstants.ACCESS_CONTROL_ALLOW_METHODS, + requestMethod + ); + if (requestHeaders !== undefined) { + res.setHeader( + HeaderConstants.ACCESS_CONTROL_ALLOW_HEADERS, + requestHeaders + ); + } + res.setHeader( + HeaderConstants.ACCESS_CONTROL_MAX_AGE, + cors.maxAgeInSeconds + ); + res.setHeader( + HeaderConstants.ACCESS_CONTROL_ALLOW_CREDENTIALS, + "true" + ); + + return next(); + } + return next( + StorageErrorFactory.corsPreflightFailure(context, { + MessageDetails: "No CORS rules matches this request" + }) + ); + }) + .catch(next); + } else { + next(err); + } + }; + } + + public createCorsRequestMiddleware( + metadataStore: IDataLakeMetadataStore, + blockErrorRequest: boolean = false + ): ErrorRequestHandler | RequestHandler { + const internalMethod = ( + err: MiddlewareError | Error | undefined, + req: Request, + res: Response, + next: NextFunction + ) => { + if (req.method.toUpperCase() === MethodConstants.OPTIONS) { + return next(err); + } + + const context = new DataLakeContext(res.locals, DEFAULT_CONTEXT_PATH); + + const account = context.account!; + + const origin = req.headers[HeaderConstants.ORIGIN] as string | undefined; + if (origin === undefined) { + return next(err); + } + + const method = req.method; + if (method === undefined || typeof method !== "string") { + return next(err); + } + + metadataStore + .getServiceProperties(context, account) + .then((properties) => { + if (properties === undefined || properties.cors === undefined) { + return next(err); + } + const corsSet = properties.cors; + const resHeaders = this.getResponseHeaders( + res, + err instanceof MiddlewareError ? err : undefined + ); + + // Here we will match CORS settings in order and select first matched CORS + for (const cors of corsSet) { + if ( + this.checkOrigin(origin, cors.allowedOrigins) && + this.checkMethod(method, cors.allowedMethods) + ) { + const exposedHeaders = this.getExposedHeaders( + resHeaders, + cors.exposedHeaders || "" + ); + + res.setHeader( + HeaderConstants.ACCESS_CONTROL_EXPOSE_HEADERS, + exposedHeaders + ); + + res.setHeader( + HeaderConstants.ACCESS_CONTROL_ALLOW_ORIGIN, + cors.allowedOrigins === "*" ? "*" : origin! // origin is not undefined as checked in checkOrigin() + ); + + if (cors.allowedOrigins !== "*") { + res.setHeader(HeaderConstants.VARY, "Origin"); + res.setHeader( + HeaderConstants.ACCESS_CONTROL_ALLOW_CREDENTIALS, + "true" + ); + } + + return next(err); + } + } + if (corsSet.length > 0) { + res.setHeader(HeaderConstants.VARY, "Origin"); + } + return next(err); + }) + .catch(next); + }; + + if (blockErrorRequest) { + return internalMethod; + } else { + return (req: Request, res: Response, next: NextFunction) => { + internalMethod(undefined, req, res, next); + }; + } + } + + private checkOrigin( + origin: string | undefined, + allowedOrigin: string + ): boolean { + if (allowedOrigin === "*") { + return true; + } + + if (origin === undefined) { + return false; + } + + const allowedOriginArray = allowedOrigin.split(","); + for (const corsOrigin of allowedOriginArray) { + if (corsOrigin.includes("*")) { + return glob(corsOrigin.trim().toLowerCase()).test( + origin.trim().toLowerCase() + ); + } + + if (origin.trim().toLowerCase() === corsOrigin.trim().toLowerCase()) { + return true; + } + } + return false; + } + + private checkMethod(method: string, allowedMethod: string): boolean { + const allowedMethodArray = allowedMethod.split(","); + for (const corsMethod of allowedMethodArray) { + if (method.trim().toLowerCase() === corsMethod.trim().toLowerCase()) { + return true; + } + } + return false; + } + + private checkHeaders(headers: string, allowedHeaders: string): boolean { + const headersArray = headers.split(","); + const allowedHeadersArray = allowedHeaders.split(","); + for (const header of headersArray) { + let flag = false; + const trimmedHeader = header.trim().toLowerCase(); + + for (const allowedHeader of allowedHeadersArray) { + // TODO: Should remove the wrapping blank when set CORS through set properties for service. + const trimmedAllowedHeader = allowedHeader.trim().toLowerCase(); + if ( + trimmedHeader === trimmedAllowedHeader || + (trimmedAllowedHeader[trimmedAllowedHeader.length - 1] === "*" && + trimmedHeader.startsWith( + trimmedAllowedHeader.substr(0, trimmedAllowedHeader.length - 1) + )) + ) { + flag = true; + break; + } + } + + if (flag === false) { + return false; + } + } + + return true; + } + + private getResponseHeaders(res: Response, err?: MiddlewareError): string[] { + const responseHeaderSet = []; + + const context = new DataLakeContext(res.locals, DEFAULT_CONTEXT_PATH); + const handlerResponse = context.handlerResponses; + + if (handlerResponse && context.operation) { + const statusCodeInResponse: number = handlerResponse.statusCode; + const spec = Specifications[context.operation]; + const responseSpec = spec.responses[statusCodeInResponse]; + if (!responseSpec) { + throw new TypeError( + `Request specification doesn't include provided response status code` + ); + } + + // Serialize headers + const headerSerializer = new msRest.Serializer(Mappers); + const headersMapper = responseSpec.headersMapper; + + if (headersMapper && headersMapper.type.name === "Composite") { + const mappersForAllHeaders = headersMapper.type.modelProperties || {}; + + // Handle headerMapper one by one + for (const key in mappersForAllHeaders) { + if (mappersForAllHeaders.hasOwnProperty(key)) { + const headerMapper = mappersForAllHeaders[key]; + const headerName = headerMapper.serializedName; + const headerValueOriginal = handlerResponse[key]; + const headerValueSerialized = headerSerializer.serialize( + headerMapper, + headerValueOriginal + ); + + // Handle collection of headers starting with same prefix, such as x-ms-meta prefix + const headerCollectionPrefix = ( + headerMapper as msRest.DictionaryMapper + ).headerCollectionPrefix; + if ( + headerCollectionPrefix !== undefined && + headerValueOriginal !== undefined + ) { + for (const collectionHeaderPartialName in headerValueSerialized) { + if ( + headerValueSerialized.hasOwnProperty( + collectionHeaderPartialName + ) + ) { + const collectionHeaderValueSerialized = + headerValueSerialized[collectionHeaderPartialName]; + const collectionHeaderName = `${headerCollectionPrefix}${collectionHeaderPartialName}`; + if ( + collectionHeaderName && + collectionHeaderValueSerialized !== undefined + ) { + responseHeaderSet.push(collectionHeaderName); + } + } + } + } else { + if (headerName && headerValueSerialized !== undefined) { + responseHeaderSet.push(headerName); + } + } + } + } + } + + if ( + spec.isXML && + responseSpec.bodyMapper && + responseSpec.bodyMapper.type.name !== "Stream" + ) { + responseHeaderSet.push("content-type"); + responseHeaderSet.push("content-length"); + } else if ( + handlerResponse.body && + responseSpec.bodyMapper && + responseSpec.bodyMapper.type.name === "Stream" + ) { + responseHeaderSet.push("content-length"); + } + } + + const headers = res.getHeaders(); + for (const header in headers) { + if (typeof header === "string") { + responseHeaderSet.push(header); + } + } + + if (err) { + for (const key in err.headers) { + if (err.headers.hasOwnProperty(key)) { + responseHeaderSet.push(key); + } + } + } + + // TODO: Should extract the header by some policy. + // or apply a referred list indicates the related headers. + responseHeaderSet.push("Date"); + responseHeaderSet.push("Connection"); + responseHeaderSet.push("Transfer-Encoding"); + + return responseHeaderSet; + } + + private getExposedHeaders( + responseHeaders: any, + exposedHeaders: string + ): string { + const exposedHeaderRules = exposedHeaders.split(","); + const prefixRules = []; + const simpleHeaders = []; + for (let i = 0; i < exposedHeaderRules.length; i++) { + exposedHeaderRules[i] = exposedHeaderRules[i].trim(); + if (exposedHeaderRules[i].endsWith("*")) { + prefixRules.push( + exposedHeaderRules[i] + .substr(0, exposedHeaderRules[i].length - 1) + .toLowerCase() + ); + } else { + simpleHeaders.push(exposedHeaderRules[i]); + } + } + + const resExposedHeaders: string[] = []; + for (const header of responseHeaders) { + let isMatch = false; + for (const rule of prefixRules) { + if (header.toLowerCase().startsWith(rule)) { + isMatch = true; + break; + } + } + if (!isMatch) { + for (const simpleHeader of simpleHeaders) { + if (header.toLowerCase() === simpleHeader.toLowerCase()) { + isMatch = true; + break; + } + } + } + + if (isMatch) { + resExposedHeaders.push(header); + } + } + + for (const simpleHeader of simpleHeaders) { + let isMatch = false; + for (const header of resExposedHeaders) { + if (simpleHeader.toLowerCase() === header.toLowerCase()) { + isMatch = true; + break; + } + } + if (!isMatch) { + resExposedHeaders.push(simpleHeader); + } + } + + return resExposedHeaders.join(","); + } +} diff --git a/src/dfs/middlewares/StrictModelMiddlewareFactory.ts b/src/dfs/middlewares/StrictModelMiddlewareFactory.ts new file mode 100644 index 000000000..7026fcd34 --- /dev/null +++ b/src/dfs/middlewares/StrictModelMiddlewareFactory.ts @@ -0,0 +1,71 @@ +import { NextFunction, Request, RequestHandler, Response } from "express"; + +import ILogger from "../../common/ILogger"; +import DataLakeContext from "../context/DataLakeContext"; +import StrictModelNotSupportedError from "../errors/StrictModelNotSupportedError"; +import Context from "../generated/Context"; +import ExpressRequestAdapter from "../generated/ExpressRequestAdapter"; +import IRequest from "../generated/IRequest"; +import { DEFAULT_CONTEXT_PATH, HeaderConstants } from "../utils/constants"; + +export type StrictModelRequestValidator = ( + req: IRequest, + context: Context, + logger: ILogger +) => Promise; + +export const UnsupportedHeadersBlocker: StrictModelRequestValidator = async ( + req: IRequest, + context: Context, + logger: ILogger +): Promise => { + const UnsupportedHeaderKeys = [ + HeaderConstants.X_MS_CONTENT_CRC64, + HeaderConstants.X_MS_RANGE_GET_CONTENT_CRC64, + HeaderConstants.X_MS_ENCRYPTION_KEY, + HeaderConstants.X_MS_ENCRYPTION_KEY_SHA256, + HeaderConstants.X_MS_ENCRYPTION_ALGORITHM + ]; + + for (const headerKey of UnsupportedHeaderKeys) { + const value = req.getHeader(headerKey); + if (typeof value === "string") { + throw new StrictModelNotSupportedError(headerKey, context); + } + } +}; + +export const UnsupportedParametersBlocker: StrictModelRequestValidator = async ( + req: IRequest, + context: Context, + logger: ILogger +): Promise => { + const UnsupportedParameterKeys: string[] = []; + + for (const parameterKey of UnsupportedParameterKeys) { + const value = req.getQuery(parameterKey); + if (typeof value === "string") { + throw new StrictModelNotSupportedError(parameterKey, context); + } + } +}; + +export default class StrictModelMiddlewareFactory { + constructor( + private readonly logger: ILogger, + private readonly validators: StrictModelRequestValidator[] + ) {} + + public createStrictModelMiddleware(): RequestHandler { + return (req: Request, res: Response, next: NextFunction) => { + this.validate(req, res).then(next).catch(next); + }; + } + + private async validate(req: Request, res: Response): Promise { + const context = new DataLakeContext(res.locals, DEFAULT_CONTEXT_PATH); + for (const validator of this.validators) { + await validator(new ExpressRequestAdapter(req), context, this.logger); + } + } +} diff --git a/src/dfs/middlewares/blobStorageContext.middleware.ts b/src/dfs/middlewares/blobStorageContext.middleware.ts new file mode 100644 index 000000000..16bf85945 --- /dev/null +++ b/src/dfs/middlewares/blobStorageContext.middleware.ts @@ -0,0 +1,282 @@ +import { NextFunction, Request, RequestHandler, Response } from "express"; +import uuid from "uuid/v4"; + +import logger from "../../common/Logger"; +import { IP_REGEX } from "../../common/utils/constants"; +import { NO_ACCOUNT_HOST_NAMES } from "../../common/utils/constants"; +import DataLakeContext from "../context/DataLakeContext"; +import StorageErrorFactory from "../errors/StorageErrorFactory"; +import IRequest from "../generated/IRequest"; +import IResponse from "../generated/IResponse"; +import { + DEFAULT_CONTEXT_PATH, + HeaderConstants, + SECONDARY_SUFFIX, + ValidAPIVersions, + VERSION +} from "../utils/constants"; +import { checkApiVersion, validateContainerName } from "../utils/utils"; + +export default function createStorageBlobContextMiddleware( + skipApiVersionCheck?: boolean, + disableProductStyleUrl?: boolean, + loose?: boolean +): RequestHandler { + return (req: Request, res: Response, next: NextFunction) => { + return blobStorageContextMiddleware( + req, + res, + next, + skipApiVersionCheck, + disableProductStyleUrl, + loose + ); + }; +} + +/** + * A middleware extract related blob service context. + * + * @export + * @param {Request} req An express compatible Request object + * @param {Response} res An express compatible Response object + * @param {NextFunction} next An express middleware next callback + */ +export function internnalBlobStorageContextMiddleware( + blobContext: DataLakeContext, + req: IRequest, + res: IResponse, + reqHost: string, + reqPath: string, + next: NextFunction, + skipApiVersionCheck?: boolean, + disableProductStyleUrl?: boolean, + loose?: boolean +): void { + // Set server header in every Azurite response + res.setHeader(HeaderConstants.SERVER, `Azurite-DataLake/${VERSION}`); + const requestID = uuid(); + + if (!skipApiVersionCheck) { + const apiVersion = req.getHeader(HeaderConstants.X_MS_VERSION); + if (apiVersion !== undefined) { + checkApiVersion(apiVersion, ValidAPIVersions, blobContext); + } + } + + blobContext.startTime = new Date(); + blobContext.disableProductStyleUrl = disableProductStyleUrl; + blobContext.loose = loose; + + blobContext.xMsRequestID = requestID; + + logger.info( + `BlobStorageContextMiddleware: RequestMethod=${req.getMethod()} RequestURL=${req.getUrl()} RequestHeaders:${JSON.stringify( + req.getHeaders() + )} ClientIP=${req.getEndpoint()} Protocol=${req.getProtocol()} HTTPVersion=version`, + requestID + ); + + const [account, container, blob, originalBlob, isSecondary] = + extractStoragePartsFromPath(reqHost, reqPath, disableProductStyleUrl); + + blobContext.account = account; + blobContext.container = container; + blobContext.blob = blob; + blobContext.originalBlob = originalBlob; + blobContext.isSecondary = isSecondary; + + // Emulator's URL pattern is like http://hostname[:port]/account/container + // (or, alternatively, http[s]://account.localhost[:port]/container) + // Create a router to exclude account name from req.path, as url path in swagger doesn't include account + // Exclude account name from req.path for dispatchMiddleware + blobContext.dispatchPattern = container + ? blob + ? `/container/blob` + : `/container` + : "/"; + + blobContext.authenticationPath = reqPath; + if (isSecondary) { + const pos = blobContext.authenticationPath!.search(SECONDARY_SUFFIX); + blobContext.authenticationPath = + blobContext.authenticationPath!.substr(0, pos) + + blobContext.authenticationPath!.substr(pos + SECONDARY_SUFFIX.length); + } + + if (!account) { + const handlerError = + StorageErrorFactory.getInvalidQueryParameterValue(blobContext); + + logger.error( + `BlobStorageContextMiddleware: BlobStorageContextMiddleware: ${handlerError.message}`, + requestID + ); + + return next(handlerError); + } + + // validate conatainer name, when container name has value (not undefined or empty string) + // skip validate system container + if (container && !container.startsWith("$")) { + validateContainerName(blobContext, container); + } + + logger.info( + `BlobStorageContextMiddleware: Account=${account} Container=${container} Blob=${blob}`, + requestID + ); + next(); +} + +/** + * A middleware extract related blob service context. + * + * @export + * @param {Request} req An express compatible Request object + * @param {Response} res An express compatible Response object + * @param {NextFunction} next An express middleware next callback + */ +export function blobStorageContextMiddleware( + req: Request, + res: Response, + next: NextFunction, + skipApiVersionCheck?: boolean, + disableProductStyleUrl?: boolean, + loose?: boolean +): void { + // Set server header in every Azurite response + res.setHeader(HeaderConstants.SERVER, `Azurite-DataLake/${VERSION}`); + const requestID = uuid(); + const blobContext = new DataLakeContext(res.locals, DEFAULT_CONTEXT_PATH); + blobContext.startTime = new Date(); + blobContext.disableProductStyleUrl = disableProductStyleUrl; + blobContext.loose = loose; + blobContext.xMsRequestID = requestID; + + if (!skipApiVersionCheck) { + const apiVersion = req.header(HeaderConstants.X_MS_VERSION); + if (apiVersion !== undefined) { + checkApiVersion(apiVersion, ValidAPIVersions, blobContext); + } + } + + logger.info( + `BlobStorageContextMiddleware: RequestMethod=${req.method} RequestURL=${ + req.protocol + }://${req.hostname}${req.url} RequestHeaders:${JSON.stringify( + req.headers + )} ClientIP=${req.ip} Protocol=${req.protocol} HTTPVersion=${ + req.httpVersion + }`, + requestID + ); + + const [account, container, blob, originalBlob, isSecondary] = + extractStoragePartsFromPath(req.hostname, req.path, disableProductStyleUrl); + + blobContext.account = account; + blobContext.container = container; + blobContext.blob = blob; + blobContext.originalBlob = originalBlob; + blobContext.isSecondary = isSecondary; + + // Emulator's URL pattern is like http://hostname[:port]/account/container + // (or, alternatively, http[s]://account.localhost[:port]/container) + // Create a router to exclude account name from req.path, as url path in swagger doesn't include account + // Exclude account name from req.path for dispatchMiddleware + blobContext.dispatchPattern = container + ? blob + ? `/container/blob` + : `/container` + : "/"; + + blobContext.authenticationPath = req.path; + if (isSecondary) { + const pos = blobContext.authenticationPath.search(SECONDARY_SUFFIX); + blobContext.authenticationPath = + blobContext.authenticationPath.substr(0, pos) + + blobContext.authenticationPath.substr(pos + SECONDARY_SUFFIX.length); + } + + if (!account) { + const handlerError = + StorageErrorFactory.getInvalidQueryParameterValue(blobContext); + + logger.error( + `BlobStorageContextMiddleware: BlobStorageContextMiddleware: ${handlerError.message}`, + requestID + ); + + return next(handlerError); + } + + // validate conatainer name, when container name has value (not undefined or empty string) + // skip validate system container + if (container && !container.startsWith("$")) { + validateContainerName(blobContext, container); + } + + logger.info( + `BlobStorageContextMiddleware: Account=${account} Container=${container} Blob=${blob}`, + requestID + ); + next(); +} + +/** + * Extract storage account, container, and blob from URL path. + * + * @param {string} path + * @returns {([string | undefined, string | undefined, string | undefined, boolean | undefined])} + */ +export function extractStoragePartsFromPath( + hostname: string, + path: string, + disableProductStyleUrl?: boolean +): [ + string | undefined, + string | undefined, + string | undefined, + string | undefined, + boolean | undefined +] { + let account; + let container; + let blob; + let isSecondary = false; + + const normalizedPath = path.startsWith("/") ? path.substring(1) : path; // Remove starting "/" + + const parts = normalizedPath.split("/"); + + let urlPartIndex = 0; + const isIPAddress = IP_REGEX.test(hostname); + const isNoAccountHostName = NO_ACCOUNT_HOST_NAMES.has(hostname.toLowerCase()); + const firstDotIndex = hostname.indexOf("."); + // If hostname is not an IP address or a known host name, and has a dot inside, + // we assume user wants to access emulator with a production-like URL. + if ( + !disableProductStyleUrl && + !isIPAddress && + !isNoAccountHostName && + firstDotIndex > 0 + ) { + account = hostname.substring(0, firstDotIndex); + } else { + account = parts[urlPartIndex++]; + } + container = parts[urlPartIndex++]; + if (container === account) container = parts[urlPartIndex++]; + const originalBlob = parts.slice(urlPartIndex++).join("/"); + blob = decodeURIComponent(originalBlob).replace(/\\/g, "/"); // Azure Storage Server will replace "\" with "/" in the blob names + + if (account.endsWith(SECONDARY_SUFFIX)) { + account = account.substring(0, account.length - SECONDARY_SUFFIX.length); + isSecondary = true; + } + + if (account !== undefined) account = decodeURIComponent(account); + if (container !== undefined) container = decodeURIComponent(container); + return [account, container, blob, originalBlob, isSecondary]; +} diff --git a/src/dfs/persistence/BlobReferredExtentsAsyncIterator.ts b/src/dfs/persistence/BlobReferredExtentsAsyncIterator.ts new file mode 100644 index 000000000..831f8cfa9 --- /dev/null +++ b/src/dfs/persistence/BlobReferredExtentsAsyncIterator.ts @@ -0,0 +1,109 @@ +import ILogger from "../../common/ILogger"; +import { Logger } from "../../common/Logger"; +import NoLoggerStrategy from "../../common/NoLoggerStrategy"; +import IDataLakeMetadataStore from "./IDataLakeMetadataStore"; + +enum State { + LISTING_EXTENTS_IN_BLOBS, + LISTING_EXTENTS_IN_BLOCKS, + DONE +} + +/** + * An async iterator which enumerates all extents being used. + * + * @export + * @class BlobReferredExtentsAsyncIterator + * @implements {AsyncIterator} + */ +export default class BlobReferredExtentsAsyncIterator + implements AsyncIterator +{ + private state: State = State.LISTING_EXTENTS_IN_BLOBS; + + private blobListingMarker: string | undefined; + private blockListingMarker: string | undefined; + + constructor( + private readonly blobMetadataStore: IDataLakeMetadataStore, + private readonly logger: ILogger = new Logger(new NoLoggerStrategy()) + ) {} + + public async next(): Promise> { + if (this.state === State.LISTING_EXTENTS_IN_BLOBS) { + const [blobs, marker] = await this.blobMetadataStore.listAllBlobs( + undefined, + this.blobListingMarker, + true, // includeSnapshots + true // includeUncommitedBlobs + ); + this.blobListingMarker = marker; + if (marker === undefined) { + this.state = State.LISTING_EXTENTS_IN_BLOCKS; + } + + const extents = []; + for (const blob of blobs) { + this.logger.debug( + `BlobReferredExtentsAsyncIterator:next() Handle blob ${ + blob.accountName + } ${blob.containerName} ${blob.name} ${blob.snapshot} Blocks: ${ + (blob.committedBlocksInOrder || []).length + } PageRanges: ${(blob.pageRangesInOrder || []).length} Persistency: ${ + blob.persistency ? blob.persistency.id : "" + }` + ); + + // this._logger.debug( + // `BlobReferredExtentsAsyncIterator:next() committedBlocksInOrder:${JSON.stringify( + // blob.committedBlocksInOrder + // )}` + // ); + // this._logger.debug( + // `BlobReferredExtentsAsyncIterator:next() pageRangesInOrder:${JSON.stringify( + // blob.pageRangesInOrder + // )}` + // ); + + for (const block of blob.committedBlocksInOrder || []) { + extents.push(block.persistency.id); + } + for (const range of blob.pageRangesInOrder || []) { + extents.push(range.persistency.id); + } + if (blob.persistency) { + extents.push(blob.persistency.id); + } + } + return { + done: false, + value: extents + }; + } else if (this.state === State.LISTING_EXTENTS_IN_BLOCKS) { + const [blocks, marker] = + await this.blobMetadataStore.listUncommittedBlockPersistencyChunks( + this.blockListingMarker + ); + this.blockListingMarker = marker; + if (marker === undefined) { + this.state = State.DONE; + } + + // this._logger.debug( + // `BlobReferredExtentsAsyncIterator:next() Handle uncommitted blocks ${ + // blocks.length + // } ${JSON.stringify(blocks)}` + // ); + + return { + done: false, + value: blocks.map((block) => block.id) + }; + } else { + return { + done: true, + value: [] + }; + } + } +} diff --git a/src/dfs/persistence/IDataLakeMetadataStore.ts b/src/dfs/persistence/IDataLakeMetadataStore.ts new file mode 100644 index 000000000..e002b5eab --- /dev/null +++ b/src/dfs/persistence/IDataLakeMetadataStore.ts @@ -0,0 +1,1288 @@ +import ICleaner from "../../common/ICleaner"; +import IDataStore from "../../common/IDataStore"; +import IGCExtentProvider from "../../common/IGCExtentProvider"; +import * as Models from "../generated/artifacts/models"; +import Context from "../generated/Context"; +import { + PathAccessControlItem, + PathPermissions +} from "../storage-file-datalake/models"; + +/** + * This model describes a chunk inside a persistency extent for a given extent ID. + * A chunk points to a sub-range of an extent. + * + * @export + * @interface IPersistencyChunk + */ +export interface IExtentChunk { + id: string; // The persistency layer storage extent ID where the chunk belongs to + offset: number; // Chunk offset inside the extent where chunk starts in bytes + count: number; // Chunk length in bytes +} + +export const ZERO_EXTENT_ID = "*ZERO*"; + +/** MODELS FOR SERVICE */ +interface IServiceAdditionalProperties { + accountName: string; +} + +export type ServicePropertiesModel = Models.StorageServiceProperties & + IServiceAdditionalProperties; + +/** MODELS FOR CONTAINER */ +interface IContainerAdditionalProperties { + accountName: string; + fileSystemProperties?: string; + leaseDurationSeconds?: number; + leaseId?: string; + leaseExpireTime?: Date; + leaseBreakTime?: Date; + containerAcl?: Models.SignedIdentifier[]; +} + +export type ContainerModel = Models.ContainerItem & + IContainerAdditionalProperties; + +export interface IContainerMetadata { + [propertyName: string]: string; +} + +// The response for getContainerAccessPolicy. +interface IGetContainerAccessPolicyResponse { + properties: Models.ContainerProperties; + containerAcl?: Models.SignedIdentifier[]; +} +export type GetContainerAccessPolicyResponse = + IGetContainerAccessPolicyResponse; + +// The params for setContainerAccessPolicy. +interface ISetContainerAccessPolicyOptions { + lastModified: Date; + etag: string; + containerAcl?: Models.SignedIdentifier[]; + publicAccess?: Models.PublicAccessType; + leaseAccessConditions?: Models.LeaseAccessConditions; + modifiedAccessConditions?: Models.ModifiedAccessConditions; +} +export type SetContainerAccessPolicyOptions = ISetContainerAccessPolicyOptions; + +// The response model for each lease-related request. +interface IContainerLeaseResponse { + properties: Models.ContainerProperties; + leaseId?: string; + leaseTime?: number; +} +export type AcquireContainerLeaseResponse = IContainerLeaseResponse; +export type ReleaseContainerLeaseResponse = Models.ContainerProperties; +export type RenewContainerLeaseResponse = IContainerLeaseResponse; +export type BreakContainerLeaseResponse = IContainerLeaseResponse; +export type ChangeContainerLeaseResponse = IContainerLeaseResponse; + +/** MODELS FOR BLOBS */ +interface IPersistencyPropertiesRequired { + /** + * A reference to persistency layer chunk of data. + * + * @type {IExtentChunk} + * @memberof IPersistencyProperties + */ + persistency: IExtentChunk; +} + +interface IPersistencyPropertiesOptional { + /** + * A reference to persistency layer chunk of data. + * + * @type {IExtentChunk} + * @memberof IPersistencyProperties + */ + persistency?: IExtentChunk; +} + +interface IBlockBlobAdditionalProperties { + /** + * False for uncommitted block blob, otherwise true. + * + * @type {boolean} + * @memberof IBlobAdditionalProperties + */ + isCommitted: boolean; + + /** + * Committed blocks for block blob. + * + * @type {PersistencyBlockModel[]} + * @memberof IBlobAdditionalProperties + */ + committedBlocksInOrder?: PersistencyBlockModel[]; +} + +/** + * PageRange model with pointers to persistency chunk. + */ +export type PersistencyPageRange = IPersistencyPropertiesRequired & + Models.PageRange; + +interface IPageBlobAdditionalProperties { + pageRangesInOrder?: PersistencyPageRange[]; +} + +interface IBlobAdditionalProperties { + accountName: string; + containerName: string; + leaseDurationSeconds?: number; + leaseId?: string; + leaseExpireTime?: Date; + leaseBreakTime?: Date; + isDirectory: boolean; + permissions: PathPermissions; + owner: string; + group: string; + acl: PathAccessControlItem[]; +} + +export type BlobModel = IBlobAdditionalProperties & + IPageBlobAdditionalProperties & + IBlockBlobAdditionalProperties & + Models.BlobItemInternal & + IPersistencyPropertiesOptional; + +export type BlobPrefixModel = IPersistencyPropertiesOptional & + Models.BlobPrefix; + +// The response model for getContainerProperties. +interface IGetBlobPropertiesRes { + properties: Models.BlobPropertiesInternal; + metadata?: Models.BlobMetadata; + blobCommittedBlockCount?: number; // AppendBlobOnly +} +export type GetBlobPropertiesRes = IGetBlobPropertiesRes; + +// The response model for each lease-related request. +interface IBlobLeaseResponse { + properties: Models.BlobPropertiesInternal; + leaseId?: string; + leaseTime?: number; +} +export type AcquireBlobLeaseResponse = IBlobLeaseResponse; +export type ReleaseBlobLeaseResponse = Models.ContainerProperties; +export type RenewBlobLeaseResponse = IBlobLeaseResponse; +export type BreakBlobLeaseResponse = IBlobLeaseResponse; +export type ChangeBlobLeaseResponse = IBlobLeaseResponse; + +// The response model for create snapshot. +interface ICreateSnapshotResponse { + properties: Models.BlobPropertiesInternal; + snapshot: string; +} +export type CreateSnapshotResponse = ICreateSnapshotResponse; + +// The model contain account name, container name, blob name and snapshot for blob. +interface IBlobId { + account: string; + container: string; + blob: string; + snapshot?: string; +} +export type BlobId = IBlobId; + +// The model contain required attributes of pageblob for request getPageRanges. +interface IGetPageRangeResponse { + pageRangesInOrder?: PersistencyPageRange[]; + properties: Models.BlobPropertiesInternal; +} +export type GetPageRangeResponse = IGetPageRangeResponse; + +/** MODELS FOR BLOCKS */ +interface IBlockAdditionalProperties { + accountName: string; + containerName: string; + blobName: string; + isCommitted: boolean; +} + +export type PersistencyBlockModel = Models.Block & + IPersistencyPropertiesRequired; + +export type BlockModel = IBlockAdditionalProperties & PersistencyBlockModel; + +/** + * Persistency layer metadata storage interface. + * + * TODO: Integrate cache layer to cache account, container & blob metadata. + * + * @export + * @interface IDataLakeMetadataStore + * @extends {IDataStore} + */ +export interface IDataLakeMetadataStore + extends IGCExtentProvider, + IDataStore, + ICleaner { + /** + * Update blob service properties. Create service properties if not exists in persistency layer. + * + * TODO: Account's service property should be created when storage account is created or metadata + * storage initialization. This method should only be responsible for updating existing record. + * In this way, we can reduce one I/O call to get account properties. + * + * @param {ServicePropertiesModel} serviceProperties + * @returns {Promise} undefined properties will be ignored during properties setup + * @memberof IBlobMetadataStore + */ + setServiceProperties( + context: Context, + serviceProperties: ServicePropertiesModel + ): Promise; + + /** + * Get service properties for specific storage account. + * + * @param {string} account + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + getServiceProperties( + context: Context, + account: string + ): Promise; + + /** + * List containers with query conditions specified. + * + * @param {string} account + * @param {string} [prefix] + * @param {number} [maxResults] + * @param {number} [marker] + * @returns {(Promise<[ContainerModel[], number | undefined]>)} A tuple including containers and next marker + * @memberof IBlobMetadataStore + */ + listContainers( + context: Context, + account: string, + prefix?: string, + maxResults?: number, + marker?: string + ): Promise<[ContainerModel[], string | undefined]>; + + /** + * Create a container. + * + * @param {ContainerModel} container + * @param {Context} [context] + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + createContainer( + context: Context, + container: ContainerModel + ): Promise; + + /** + * Set container properties. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} [properties] + * @param {Models.LeaseAccessConditions} [leaseAccessConditions] + * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + setContainerProperties( + context: Context, + account: string, + container: string, + properties: string, + leaseAccessConditions?: Models.LeaseAccessConditions, + modifiedAccessConditions?: Models.ModifiedAccessConditions + ): Promise; + + /** + * Get container properties. + * + * @param {string} account + * @param {string} container + * @param {Context} context + * @param {Models.LeaseAccessConditions} [leaseAccessConditions] + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + getContainerProperties( + context: Context, + account: string, + container: string, + leaseAccessConditions?: Models.LeaseAccessConditions + ): Promise; + + /** + * Delete container item if exists from persistency layer. + * Note that this method will mark the specific container with "deleting" tag. Container item + * will be removed only if all blobs under that container has been removed with GC. During + * "deleting" status, container and blobs under that container cannot be accessed. + * + * TODO: Make sure all metadata interface implementation follow up above assumption. + * TODO: GC for async container deletion. + * + * @param {string} account + * @param {string} container + * @param {Context} context + * @param {Models.ContainerDeleteMethodOptionalParams} [options] + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + deleteContainer( + context: Context, + account: string, + container: string, + options?: Models.ContainerDeleteMethodOptionalParams + ): Promise; + + /** + * Set container metadata. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {Date} lastModified + * @param {string} etag + * @param {IContainerMetadata} [metadata] + * @param {Models.LeaseAccessConditions} [leaseAccessConditions] + * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + setContainerMetadata( + context: Context, + account: string, + container: string, + lastModified: Date, + etag: string, + metadata?: IContainerMetadata, + leaseAccessConditions?: Models.LeaseAccessConditions, + modifiedAccessConditions?: Models.ModifiedAccessConditions + ): Promise; + + /** + * Get container access policy. + * + * @param {string} account + * @param {string} container + * @param {Context} context + * @param {Models.LeaseAccessConditions} [leaseAccessConditions] + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + getContainerACL( + context: Context, + account: string, + container: string, + forceExist?: boolean, + leaseAccessConditions?: Models.LeaseAccessConditions + ): Promise; + + getContainerACL( + context: Context, + account: string, + container: string, + forceExist?: true, + leaseAccessConditions?: Models.LeaseAccessConditions + ): Promise; + + getContainerACL( + context: Context, + account: string, + container: string, + forceExist: false, + leaseAccessConditions?: Models.LeaseAccessConditions + ): Promise; + + /** + * Set container access policy. + * + * @param {string} account + * @param {string} container + * @param {SetContainerAccessPolicyOptions} setAclModel + * @param {Context} context + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + setContainerACL( + context: Context, + account: string, + container: string, + setAclModel: SetContainerAccessPolicyOptions + ): Promise; + + /** + * Acquire container lease + * + * @param {string} account + * @param {string} container + * @param {Models.ContainerAcquireLeaseOptionalParams} [options] + * @param {Context} context + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + acquireContainerLease( + context: Context, + account: string, + container: string, + options?: Models.ContainerAcquireLeaseOptionalParams + ): Promise; + + /** + * Release container lease. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} leaseId + * @param {Models.ContainerReleaseLeaseOptionalParams} [options] + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + releaseContainerLease( + context: Context, + account: string, + container: string, + leaseId: string, + options?: Models.ContainerReleaseLeaseOptionalParams + ): Promise; + + /** + * Renew container lease. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} leaseId + * @param {Models.ContainerRenewLeaseOptionalParams} [options] + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + renewContainerLease( + context: Context, + account: string, + container: string, + leaseId: string, + options?: Models.ContainerRenewLeaseOptionalParams + ): Promise; + + /** + * Break container lease. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {(number | undefined)} breakPeriod + * @param {Models.ContainerBreakLeaseOptionalParams} [options] + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + breakContainerLease( + context: Context, + account: string, + container: string, + breakPeriod: number | undefined, + options?: Models.ContainerBreakLeaseOptionalParams + ): Promise; + + /** + * Change container lease. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} leaseId + * @param {string} proposedLeaseId + * @param {Models.ContainerChangeLeaseOptionalParams} [options] + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + changeContainerLease( + context: Context, + account: string, + container: string, + leaseId: string, + proposedLeaseId: string, + options?: Models.ContainerChangeLeaseOptionalParams + ): Promise; + + /** + * Check the existence of a container. + * + * @param {string} account + * @param {string} container + * @param {Context} [context] + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + checkContainerExist( + context: Context, + account: string, + container: string + ): Promise; + + list( + lisDirectories: boolean | undefined, + context: Context, + account: string, + container: string, + delimiter?: string, + blob?: string, + prefix?: string, + maxResults?: number, + marker?: string, + includeSnapshots?: boolean, + includeUncommittedBlobs?: boolean + ): Promise<[BlobModel[], BlobPrefixModel[], string | undefined]>; + + listBlobs( + context: Context, + account: string, + container: string, + delimiter?: string, + blob?: string, + prefix?: string, + maxResults?: number, + marker?: string, + includeSnapshots?: boolean, + includeUncommittedBlobs?: boolean + ): Promise<[BlobModel[], BlobPrefixModel[], string | undefined]>; + + listAllBlobs( + maxResults?: number, + marker?: string, + includeSnapshots?: boolean, + includeUncommittedBlobs?: boolean + ): Promise<[BlobModel[], string | undefined]>; + + /** + * Create blob item in persistency layer. Will replace if blob exists. + * + * @param {Context} context + * @param {BlobModel} blob + * @param {Models.LeaseAccessConditions} [leaseAccessConditions] Optional. Will validate lease if provided + * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + createBlob( + context: Context, + blob: BlobModel, + leaseAccessConditions?: Models.LeaseAccessConditions, + modifiedAccessConditions?: Models.ModifiedAccessConditions + ): Promise; + + /** + * Create snapshot. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {Models.LeaseAccessConditions} [leaseAccessConditions] Optional. Will validate lease if provided + * @param {Models.BlobMetadata} [metadata] + * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + createSnapshot( + context: Context, + account: string, + container: string, + blob: string, + leaseAccessConditions?: Models.LeaseAccessConditions, + metadata?: Models.BlobMetadata, + modifiedAccessConditions?: Models.ModifiedAccessConditions + ): Promise; + + /** + * Gets a blob item from metadata store by account name, container name and blob name. + * Will return block list or page list as well for downloading. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {(string | undefined)} snapshot + * @param {Models.LeaseAccessConditions} [leaseAccessConditions] Optional. Will validate lease if provided + * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + downloadBlob( + context: Context, + account: string, + container: string, + blob: string, + snapshot: string | undefined, + leaseAccessConditions?: Models.LeaseAccessConditions, + modifiedAccessConditions?: Models.ModifiedAccessConditions + ): Promise; + + /** + * Get blob properties. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {(string | undefined)} snapshot + * @param {(Models.LeaseAccessConditions | undefined)} leaseAccessConditions + * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + getBlobProperties( + context: Context, + account: string, + container: string, + blob: string, + snapshot: string | undefined, + leaseAccessConditions: Models.LeaseAccessConditions | undefined, + modifiedAccessConditions?: Models.ModifiedAccessConditions + ): Promise; + + /** + * Delete blob or its snapshots. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {Models.PathDeleteMethodOptionalParams} options + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + deleteBlob( + context: Context, + account: string, + container: string, + blob: string, + options: Models.PathDeleteMethodOptionalParams + ): Promise; + + /** + * Set blob HTTP headers. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {(Models.LeaseAccessConditions | undefined)} leaseAccessConditions + * @param {(Models.BlobHTTPHeaders | undefined)} blobHTTPHeaders + * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + setBlobHTTPHeaders( + context: Context, + account: string, + container: string, + blob: string, + leaseAccessConditions: Models.LeaseAccessConditions | undefined, + blobHTTPHeaders: Models.BlobHTTPHeaders | undefined, + modifiedAccessConditions?: Models.ModifiedAccessConditions + ): Promise; + + /** + * Set blob metadata. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {(Models.LeaseAccessConditions | undefined)} leaseAccessConditions + * @param {(Models.BlobMetadata | undefined)} metadata + * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + setBlobMetadata( + context: Context, + account: string, + container: string, + blob: string, + leaseAccessConditions: Models.LeaseAccessConditions | undefined, + metadata: Models.BlobMetadata | undefined, + modifiedAccessConditions?: Models.ModifiedAccessConditions + ): Promise; + + /** + * Acquire blob lease. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {number} duration + * @param {string} [proposedLeaseId] + * @param {Models.BlobAcquireLeaseOptionalParams} [options] + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + acquireBlobLease( + context: Context, + account: string, + container: string, + blob: string, + duration: number, + proposedLeaseId?: string, + options?: Models.BlobAcquireLeaseOptionalParams + ): Promise; + + /** + * Release blob. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {string} leaseId + * @param {Models.BlobReleaseLeaseOptionalParams} [options] + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + releaseBlobLease( + context: Context, + account: string, + container: string, + blob: string, + leaseId: string, + options?: Models.BlobReleaseLeaseOptionalParams + ): Promise; + + /** + * Renew blob lease. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {string} leaseId + * @param {Models.BlobRenewLeaseOptionalParams} [options] + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + renewBlobLease( + context: Context, + account: string, + container: string, + blob: string, + leaseId: string, + options?: Models.BlobRenewLeaseOptionalParams + ): Promise; + + /** + * Change blob lease. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {string} leaseId + * @param {string} proposedLeaseId + * @param {Models.BlobChangeLeaseOptionalParams} [option] + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + changeBlobLease( + context: Context, + account: string, + container: string, + blob: string, + leaseId: string, + proposedLeaseId: string, + option?: + | Models.BlobChangeLeaseOptionalParams + | Models.PathLeaseOptionalParams + ): Promise; + + /** + * Break blob lease. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {number} [breakPeriod] + * @param {Models.BlobBreakLeaseOptionalParams} [option] + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + breakBlobLease( + context: Context, + account: string, + container: string, + blob: string, + breakPeriod?: number, + option?: + | Models.BlobBreakLeaseOptionalParams + | Models.PathLeaseOptionalParams + ): Promise; + + /** + * Check the existence of a blob. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {string} [snapshot] + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + checkBlobExist( + context: Context, + account: string, + container: string, + blob: string, + snapshot?: string + ): Promise; + + /** + * Get blobType and committed status for SAS authentication. + * + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {string} [snapshot] + * @returns {(Promise< + * { blobType: Models.BlobType | undefined; isCommitted: boolean } | undefined + * >)} + * @memberof IBlobMetadataStore + */ + getBlobType( + account: string, + container: string, + blob: string, + snapshot?: string + ): Promise< + { blobType: Models.BlobType | undefined; isCommitted: boolean } | undefined + >; + + /** + * Start copy from Url. + * + * @param {Context} context + * @param {BlobId} source + * @param {BlobId} destination + * @param {string} copySource + * @param {(Models.BlobMetadata | undefined)} metadata + * @param {(Models.AccessTier | undefined)} tier + * @param {Models.BlobStartCopyFromURLOptionalParams} [leaseAccessConditions] + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + startCopyFromURL( + context: Context, + source: BlobId, + destination: BlobId, + copySource: string, + metadata: Models.BlobMetadata | undefined, + tier: Models.AccessTier | undefined, + leaseAccessConditions?: Models.BlobStartCopyFromURLOptionalParams + ): Promise; + + /** + * Sync copy from Url. + * + * @param {Context} context + * @param {BlobId} source + * @param {BlobId} destination + * @param {string} copySource + * @param {(Models.BlobMetadata | undefined)} metadata + * @param {(Models.AccessTier | undefined)} tier + * @param {Models.BlobCopyFromURLOptionalParams} [leaseAccessConditions] + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + copyFromURL( + context: Context, + source: BlobId, + destination: BlobId, + copySource: string, + metadata: Models.BlobMetadata | undefined, + tier: Models.AccessTier | undefined, + leaseAccessConditions?: Models.BlobCopyFromURLOptionalParams + ): Promise; + + /** + * Update Tier for a blob. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {Models.AccessTier} tier + * @param {(Models.LeaseAccessConditions | undefined)} leaseAccessConditions + * @returns {(Promise<200 | 202>)} + * @memberof IBlobMetadataStore + */ + setTier( + context: Context, + account: string, + container: string, + blob: string, + tier: Models.AccessTier, + leaseAccessConditions: Models.LeaseAccessConditions | undefined + ): Promise<200 | 202>; + + /** + * Update blob block item in persistency layer. Will create if block doesn't exist. + * Will also create a uncommitted block blob. + * + * @param {BlockModel} block + * @param {Context} context + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + stageBlock( + context: Context, + block: BlockModel, + leaseAccessConditions: Models.LeaseAccessConditions | undefined, + append?: boolean + ): Promise; + + /** + * Append block to an append blob. + * + * @param {Context} context + * @param {BlockModel} block + * @param {Models.LeaseAccessConditions} [leaseAccessConditions] + * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] + * @param {Models.AppendPositionAccessConditions} [appendPositionAccessConditions] + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + appendBlock( + context: Context, + block: BlockModel, + leaseAccessConditions?: Models.LeaseAccessConditions, + modifiedAccessConditions?: Models.ModifiedAccessConditions, + appendPositionAccessConditions?: Models.AppendPositionAccessConditions + ): Promise; + + /** + * Commit block list for a blob. + * + * @param {Context} context + * @param {BlobModel} blob + * @param {{ blockName: string; blockCommitType: string }[]} blockList + * @param {Models.LeaseAccessConditions} [leaseAccessConditions] + * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + commitBlockList( + context: Context, + blob: BlobModel, + blockList: { blockName: string; blockCommitType: string }[], + leaseAccessConditions?: Models.LeaseAccessConditions, + modifiedAccessConditions?: Models.ModifiedAccessConditions + ): Promise; + + /** + * Gets blocks list for a blob from persistency layer by account, container and blob names. + * + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {string} [snapshot] + * @param {(boolean | undefined)} isCommitted + * @param {Context} context + * @returns {Promise<{ + * properties: Models.BlobProperties; + * uncommittedBlocks: Models.Block[]; + * committedBlocks: Models.Block[]; + * }>} + * @memberof IBlobMetadataStore + */ + getBlockList( + context: Context, + account: string, + container: string, + blob: string, + snapshot: string | undefined, + isCommitted: boolean | undefined, + leaseAccessConditions: Models.LeaseAccessConditions | undefined + ): Promise<{ + properties: Models.BlobPropertiesInternal; + uncommittedBlocks: Models.Block[]; + committedBlocks: Models.Block[]; + }>; + + /** + * Upload new pages for page blob. + * + * @param {Context} context + * @param {BlobModel} blob + * @param {number} start + * @param {number} end + * @param {IExtentChunk} persistency + * @param {Models.LeaseAccessConditions} [leaseAccessConditions] + * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] + * @param {Models.SequenceNumberAccessConditions} [sequenceNumberAccessConditions] + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + uploadPages( + context: Context, + blob: BlobModel, + start: number, + end: number, + persistency: IExtentChunk, + leaseAccessConditions?: Models.LeaseAccessConditions, + modifiedAccessConditions?: Models.ModifiedAccessConditions, + sequenceNumberAccessConditions?: Models.SequenceNumberAccessConditions + ): Promise; + + /** + * Clear range for a page blob. + * + * @param {Context} context + * @param {BlobModel} blob + * @param {number} start + * @param {number} end + * @param {Models.LeaseAccessConditions} [leaseAccessConditions] + * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] + * @param {Models.SequenceNumberAccessConditions} [sequenceNumberAccessConditions] + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + clearRange( + context: Context, + blob: BlobModel, + start: number, + end: number, + leaseAccessConditions?: Models.LeaseAccessConditions, + modifiedAccessConditions?: Models.ModifiedAccessConditions, + sequenceNumberAccessConditions?: Models.SequenceNumberAccessConditions + ): Promise; + + /** + * Returns the list of valid page ranges for a page blob or snapshot of a page blob. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {string} [snapshot] + * @param {Models.LeaseAccessConditions} [leaseAccessConditions] + * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + getPageRanges( + context: Context, + account: string, + container: string, + blob: string, + snapshot?: string, + leaseAccessConditions?: Models.LeaseAccessConditions, + modifiedAccessConditions?: Models.ModifiedAccessConditions + ): Promise; + + /** + * Resize a page blob. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {number} blobContentLength + * @param {Models.LeaseAccessConditions} [leaseAccessConditions] + * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + resizePageBlob( + context: Context, + account: string, + container: string, + blob: string, + blobContentLength: number, + leaseAccessConditions?: Models.LeaseAccessConditions, + modifiedAccessConditions?: Models.ModifiedAccessConditions + ): Promise; + + /** + * Update the sequence number of a page blob. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {Models.SequenceNumberActionType} sequenceNumberAction + * @param {(number | undefined)} blobSequenceNumber + * @param {Models.LeaseAccessConditions} [leaseAccessConditions] + * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + updateSequenceNumber( + context: Context, + account: string, + container: string, + blob: string, + sequenceNumberAction: Models.SequenceNumberActionType, + blobSequenceNumber: number | undefined, + leaseAccessConditions?: Models.LeaseAccessConditions, + modifiedAccessConditions?: Models.ModifiedAccessConditions + ): Promise; + + /** + * Gets uncommitted blocks list for a blob from persistency layer. + */ + listUncommittedBlockPersistencyChunks( + marker?: string, + maxResults?: number + ): Promise<[IExtentChunk[], string | undefined]>; + + /*************************************************************************** + * + * New DataLake specific functions + * + ***************************************************************************/ + /** + * Rename Directory + * + * @param {Context} context + * @param {string} account + * @param {string} sourceContainer + * @param {string} sourceDirectory + * @param {string} targetContainer + * @param {string} targetDirectory + * @param {Models.DirectoryRenameOptionalParams} options + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + renameDirectory( + context: Context, + account: string, + sourceContainer: string, + sourceDirectory: string, + targetContainer: string, + targetDirectory: string, + options: Models.PathCreateOptionalParams + ): Promise; + + /** + * List paths in Directory + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} directory + * @param {boolean} recursive + * @param {Models.FileSystemListPathsOptionalParams} options + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + listPaths( + context: Context, + account: string, + container: string, + directory: string, + recursive: boolean, + options: Models.FileSystemListPathsOptionalParams + ): Promise<[Models.Path[], string | undefined]>; + + /** + * Delete Directory + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} directory + * @param {boolean} recursiveDirectoryDelete + * @param {Models.DirectoryDeleteMethodOptionalParams} options + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + deleteDirectory( + context: Context, + account: string, + container: string, + directory: string, + recursiveDirectoryDelete: boolean, + options: Models.PathDeleteMethodOptionalParams + ): Promise; + + /** + * Rename Blob + * + * @param {Context} context + * @param {string} account + * @param {string} sourceContainer + * @param {string} sourceBlob + * @param {string} targetContainer + * @param {string} targetBlob + * @param {Models.PathCreateOptionalParams} options + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + renameBlob( + context: Context, + account: string, + sourceContainer: string, + sourceBlob: string, + targetContainer: string, + targetBlob: string, + options: Models.PathCreateOptionalParams + ): Promise; + + /** + * Gets a blob item from metadata store by account name, container name and blob name. + * Will return block list or page list as well for downloading. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {(string | undefined)} snapshot + * @param {Models.LeaseAccessConditions} [leaseAccessConditions] Optional. Will validate lease if provided + * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + getModel( + context: Context, + account: string, + container: string, + blob: string, + forceExist: true, + leaseAccessConditions?: Models.LeaseAccessConditions, + modifiedAccessConditions?: Models.ModifiedAccessConditions, + validateRead?: boolean + ): Promise; + + getModel( + context: Context, + account: string, + container: string, + blob: string, + forceExist: false | undefined, + leaseAccessConditions?: Models.LeaseAccessConditions, + modifiedAccessConditions?: Models.ModifiedAccessConditions, + validateRead?: boolean + ): Promise; + + getModel( + context: Context, + account: string, + container: string, + blob: string, + forceExist?: boolean, + leaseAccessConditions?: Models.LeaseAccessConditions, + modifiedAccessConditions?: Models.ModifiedAccessConditions, + validateRead?: boolean + ): Promise; +} + +export default IDataLakeMetadataStore; diff --git a/src/dfs/persistence/LokiDataLakeMetadataStore.ts b/src/dfs/persistence/LokiDataLakeMetadataStore.ts new file mode 100644 index 000000000..7c3174a8e --- /dev/null +++ b/src/dfs/persistence/LokiDataLakeMetadataStore.ts @@ -0,0 +1,3893 @@ +import { stat } from "fs"; +import Loki from "lokijs"; +import uuid from "uuid/v4"; + +import IGCExtentProvider from "../../common/IGCExtentProvider"; +import { + convertDateTimeStringMsTo7Digital, + rimrafAsync +} from "../../common/utils/utils"; +import { newEtag } from "../../common/utils/utils"; +import { validateReadConditions } from "../conditions/ReadConditionalHeadersValidator"; +import { + validateSequenceNumberWriteConditions, + validateWriteConditions +} from "../conditions/WriteConditionalHeadersValidator"; +import StorageErrorFactory from "../errors/StorageErrorFactory"; +import * as Models from "../generated/artifacts/models"; +import Context from "../generated/Context"; +import PageBlobRangesManager from "../handlers/PageBlobRangesManager"; +import BlobLeaseAdapter from "../lease/BlobLeaseAdapter"; +import BlobLeaseSyncer from "../lease/BlobLeaseSyncer"; +import BlobReadLeaseValidator from "../lease/BlobReadLeaseValidator"; +import BlobWriteLeaseSyncer from "../lease/BlobWriteLeaseSyncer"; +import BlobWriteLeaseValidator from "../lease/BlobWriteLeaseValidator"; +import ContainerDeleteLeaseValidator from "../lease/ContainerDeleteLeaseValidator"; +import ContainerLeaseAdapter from "../lease/ContainerLeaseAdapter"; +import ContainerLeaseSyncer from "../lease/ContainerLeaseSyncer"; +import ContainerReadLeaseValidator from "../lease/ContainerReadLeaseValidator"; +import { ILease } from "../lease/ILeaseState"; +import LeaseFactory from "../lease/LeaseFactory"; +import { + DEFAULT_GROUP, + DEFAULT_LIST_BLOBS_MAX_RESULTS, + DEFAULT_LIST_CONTAINERS_MAX_RESULTS, + DEFAULT_OWNER, + MAX_APPEND_BLOB_BLOCK_COUNT +} from "../utils/constants"; +import BlobReferredExtentsAsyncIterator from "./BlobReferredExtentsAsyncIterator"; +import IDataLakeMetadataStore, { + AcquireBlobLeaseResponse, + AcquireContainerLeaseResponse, + BlobId, + BlobModel, + BlobPrefixModel, + BlockModel, + BreakBlobLeaseResponse, + BreakContainerLeaseResponse, + ChangeBlobLeaseResponse, + ChangeContainerLeaseResponse, + ContainerModel, + CreateSnapshotResponse, + GetBlobPropertiesRes, + GetContainerAccessPolicyResponse, + GetPageRangeResponse, + IContainerMetadata, + IExtentChunk, + PersistencyBlockModel, + ReleaseBlobLeaseResponse, + ReleaseContainerLeaseResponse, + RenewBlobLeaseResponse, + RenewContainerLeaseResponse, + ServicePropertiesModel, + SetContainerAccessPolicyOptions +} from "./IDataLakeMetadataStore"; +import PageWithDelimiter from "./PageWithDelimiter"; +import { DEFAULT_FILE_PERMISSIONS } from "../utils/constants"; +import { removeSlash } from "../utils/utils"; +import { + toPermissions, + toPermissionsString +} from "../storage-file-datalake/transforms"; +/** + * This is a metadata source implementation for blob based on loki DB. + * + * Notice that, following design is for emulator purpose only, and doesn't design for best performance. + * We may want to optimize the persistency layer performance in the future. Such as by distributing metadata + * into different collections, or make binary payload write as an append-only pattern. + * + * Loki DB includes following collections and documents: + * + * -- SERVICE_PROPERTIES_COLLECTION // Collection contains service properties + * // Default collection name is $SERVICES_COLLECTION$ + * // Each document maps to 1 account blob service + * // Unique document properties: accountName + * -- CONTAINERS_COLLECTION // Collection contains all containers + * // Default collection name is $CONTAINERS_COLLECTION$ + * // Each document maps to 1 container + * // Unique document properties: accountName, (container)name + * -- BLOBS_COLLECTION // Collection contains all blobs + * // Default collection name is $BLOBS_COLLECTION$ + * // Each document maps to a blob + * // Unique document properties: accountName, containerName, (blob)name, snapshot + * -- BLOCKS_COLLECTION // Block blob blocks collection includes all UNCOMMITTED blocks + * // Unique document properties: accountName, containerName, blobName, name, isCommitted + * + * @export + * @class LokiBlobMetadataStore + */ +export default class LokiDataLakeMetadataStore + implements IDataLakeMetadataStore, IGCExtentProvider +{ + protected readonly db: Loki; + + private initialized: boolean = false; + private closed: boolean = true; + + private readonly SERVICES_COLLECTION = "$SERVICES_COLLECTION$"; + private readonly CONTAINERS_COLLECTION = "$CONTAINERS_COLLECTION$"; + protected readonly BLOBS_COLLECTION = "$BLOBS_COLLECTION$"; + private readonly BLOCKS_COLLECTION = "$BLOCKS_COLLECTION$"; + + private readonly pageBlobRangesManager = new PageBlobRangesManager(); + + public constructor(public readonly lokiDBPath: string) { + this.db = new Loki(lokiDBPath, { + autosave: true, + autosaveInterval: 5000 + }); + } + + public isInitialized(): boolean { + return this.initialized; + } + + public isClosed(): boolean { + return this.closed; + } + + public async init(): Promise { + await new Promise((resolve, reject) => { + stat(this.lokiDBPath, (statError, stats) => { + if (!statError) { + this.db.loadDatabase({}, (dbError) => { + if (dbError) { + reject(dbError); + } else { + resolve(); + } + }); + } else { + // when DB file doesn't exist, ignore the error because following will re-create the file + resolve(); + } + }); + }); + + // In loki DB implementation, these operations are all sync. Doesn't need an async lock + + // Create service properties collection if not exists + let servicePropertiesColl = this.db.getCollection(this.SERVICES_COLLECTION); + if (servicePropertiesColl === null) { + servicePropertiesColl = this.db.addCollection(this.SERVICES_COLLECTION, { + unique: ["accountName"] + }); + } + + // Create containers collection if not exists + if (this.db.getCollection(this.CONTAINERS_COLLECTION) === null) { + this.db.addCollection(this.CONTAINERS_COLLECTION, { + // Optimization for indexing and searching + // https://rawgit.com/techfort/LokiJS/master/jsdoc/tutorial-Indexing%20and%20Query%20performance.html + indices: ["accountName", "name"] + }); // Optimize for find operation + } + + // Create containers collection if not exists + if (this.db.getCollection(this.BLOBS_COLLECTION) === null) { + this.db.addCollection(this.BLOBS_COLLECTION, { + indices: ["accountName", "containerName", "name", "snapshot"] // Optimize for find operation + }); + } + + // Create blocks collection if not exists + if (this.db.getCollection(this.BLOCKS_COLLECTION) === null) { + this.db.addCollection(this.BLOCKS_COLLECTION, { + indices: ["accountName", "containerName", "blobName", "name"] // Optimize for find operation + }); + } + + await new Promise((resolve, reject) => { + this.db.saveDatabase((err) => { + if (err) { + reject(err); + } else { + resolve(); + } + }); + }); + + this.initialized = true; + this.closed = false; + } + + /** + * Close loki DB. + * + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async close(): Promise { + await new Promise((resolve, reject) => { + this.db.close((err) => { + if (err) { + reject(err); + } else { + resolve(); + } + }); + }); + + this.closed = true; + } + + /** + * Clean LokiBlobMetadataStore. + * + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async clean(): Promise { + if (this.isClosed()) { + await rimrafAsync(this.lokiDBPath); + + return; + } + throw new Error(`Cannot clean LokiBlobMetadataStore, it's not closed.`); + } + + public iteratorExtents(): AsyncIterator { + return new BlobReferredExtentsAsyncIterator(this); + } + + /** + * Update blob service properties. Create service properties if not exists in persistency layer. + * + * TODO: Account's service property should be created when storage account is created or metadata + * storage initialization. This method should only be responsible for updating existing record. + * In this way, we can reduce one I/O call to get account properties. + * + * @param {ServicePropertiesModel} serviceProperties + * @returns {Promise} undefined properties will be ignored during properties setup + * @memberof LokiBlobMetadataStore + */ + public async setServiceProperties( + context: Context, + serviceProperties: ServicePropertiesModel + ): Promise { + const coll = this.db.getCollection(this.SERVICES_COLLECTION); + const doc = coll.by("accountName", serviceProperties.accountName); + + if (doc) { + doc.cors = + serviceProperties.cors === undefined + ? doc.cors + : serviceProperties.cors; + + doc.hourMetrics = + serviceProperties.hourMetrics === undefined + ? doc.hourMetrics + : serviceProperties.hourMetrics; + + doc.logging = + serviceProperties.logging === undefined + ? doc.logging + : serviceProperties.logging; + + doc.minuteMetrics = + serviceProperties.minuteMetrics === undefined + ? doc.minuteMetrics + : serviceProperties.minuteMetrics; + + doc.defaultServiceVersion = + serviceProperties.defaultServiceVersion === undefined + ? doc.defaultServiceVersion + : serviceProperties.defaultServiceVersion; + + doc.deleteRetentionPolicy = + serviceProperties.deleteRetentionPolicy === undefined + ? doc.deleteRetentionPolicy + : serviceProperties.deleteRetentionPolicy; + + doc.staticWebsite = + serviceProperties.staticWebsite === undefined + ? doc.staticWebsite + : serviceProperties.staticWebsite; + + return coll.update(doc); + } else { + return coll.insert(serviceProperties); + } + } + + /** + * Get service properties for specific storage account. + * + * @param {string} account + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async getServiceProperties( + context: Context, + account: string + ): Promise { + const coll = this.db.getCollection(this.SERVICES_COLLECTION); + const doc = coll.by("accountName", account); + return doc ? doc : undefined; + } + + /** + * List containers with query conditions specified. + * + * @param {string} account + * @param {string} [prefix=""] + * @param {number} [maxResults=5000] + * @param {string} [marker=""] + * @returns {(Promise<[ContainerModel[], string | undefined]>)} + * @memberof LokiBlobMetadataStore + */ + public async listContainers( + context: Context, + account: string, + prefix: string = "", + maxResults: number = DEFAULT_LIST_CONTAINERS_MAX_RESULTS, + marker: string = "" + ): Promise<[ContainerModel[], string | undefined]> { + const coll = this.db.getCollection(this.CONTAINERS_COLLECTION); + + const query = + prefix === "" + ? { name: { $gt: marker }, accountName: account } + : { + name: { $regex: `^${this.escapeRegex(prefix)}`, $gt: marker }, + accountName: account + }; + + // Workaround for loki which will ignore $gt when providing $regex + const query2 = { name: { $gt: marker } }; + + const docs = coll + .chain() + .find(query) + .find(query2) + .simplesort("name") + .limit(maxResults + 1) + .data(); + + if (docs.length <= maxResults) { + return [ + docs.map((doc) => { + return LeaseFactory.createLeaseState( + new ContainerLeaseAdapter(doc), + context + ).sync(new ContainerLeaseSyncer(doc)); + }), + undefined + ]; + } else { + // In this case, the last item is the one we get in addition, should set the Marker before it. + const nextMarker = docs[docs.length - 2].name; + docs.pop(); + return [ + docs.map((doc) => { + return LeaseFactory.createLeaseState( + new ContainerLeaseAdapter(doc), + context + ).sync(new ContainerLeaseSyncer(doc)); + }), + nextMarker + ]; + } + } + + /** + * Create a container. + * + * @param {ContainerModel} container + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async createContainer( + context: Context, + container: ContainerModel + ): Promise { + const coll = this.db.getCollection(this.CONTAINERS_COLLECTION); + const doc = coll.findOne({ + accountName: container.accountName, + name: container.name + }); + + if (doc) { + throw StorageErrorFactory.getContainerAlreadyExists(context); + } + + return coll.insert(container); + } + + /** + * Set container properties. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} properties + * @param {Models.LeaseAccessConditions} [leaseAccessConditions] + * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + async setContainerProperties( + context: Context, + account: string, + container: string, + properties: string, + leaseAccessConditions?: Models.LeaseAccessConditions, + modifiedAccessConditions?: Models.ModifiedAccessConditions + ): Promise { + const coll = this.db.getCollection(this.CONTAINERS_COLLECTION); + const doc = await this.getContainerWithLeaseUpdated( + account, + container, + context, + false + ); + + validateWriteConditions(context, modifiedAccessConditions, doc); + + if (!doc) { + throw StorageErrorFactory.getContainerNotFound(context); + } + + new ContainerReadLeaseValidator(leaseAccessConditions).validate( + new ContainerLeaseAdapter(doc), + context + ); + + doc.fileSystemProperties = properties; + coll.update(doc); + return doc; + } + + /** + * Get container properties. + * + * @param {string} account + * @param {string} container + * @param {Context} context + * @param {Models.LeaseAccessConditions} [leaseAccessConditions] + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async getContainerProperties( + context: Context, + account: string, + container: string, + leaseAccessConditions?: Models.LeaseAccessConditions + ): Promise { + const doc = await this.getContainerWithLeaseUpdated( + account, + container, + context + ); + + new ContainerReadLeaseValidator(leaseAccessConditions).validate( + new ContainerLeaseAdapter(doc), + context + ); + + return doc; + } + + /** + * Delete container item if exists from persistency layer. + * + * Loki based implementation will delete container documents from Containers collection, + * blob documents from Blobs collection, and blocks documents from Blocks collection immediately. + * + * Persisted extents data will be deleted by GC. + * + * @param {string} account + * @param {string} container + * @param {Context} context + * @param {Models.ContainerDeleteMethodOptionalParams} [options] + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async deleteContainer( + context: Context, + account: string, + container: string, + options: Models.ContainerDeleteMethodOptionalParams = {} + ): Promise { + const coll = this.db.getCollection(this.CONTAINERS_COLLECTION); + const doc = await this.getContainerWithLeaseUpdated( + account, + container, + context, + false + ); + + validateWriteConditions(context, options.modifiedAccessConditions, doc); + + if (!doc) { + throw StorageErrorFactory.getContainerNotFound(context); + } + + new ContainerDeleteLeaseValidator(options.leaseAccessConditions).validate( + new ContainerLeaseAdapter(doc), + context + ); + + coll.remove(doc); + + const blobColl = this.db.getCollection(this.BLOBS_COLLECTION); + blobColl.findAndRemove({ + accountName: account, + containerName: container + }); + + const blockColl = this.db.getCollection(this.BLOCKS_COLLECTION); + blockColl.findAndRemove({ + accountName: account, + containerName: container + }); + } + + /** + * Set container metadata. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {Date} lastModified + * @param {string} etag + * @param {IContainerMetadata} [metadata] + * @param {Models.LeaseAccessConditions} [leaseAccessConditions] + * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async setContainerMetadata( + context: Context, + account: string, + container: string, + lastModified: Date, + etag: string, + metadata?: IContainerMetadata, + leaseAccessConditions?: Models.LeaseAccessConditions, + modifiedAccessConditions?: Models.ModifiedAccessConditions + ): Promise { + const coll = this.db.getCollection(this.CONTAINERS_COLLECTION); + const doc = await this.getContainerWithLeaseUpdated( + account, + container, + context, + false + ); + + validateWriteConditions(context, modifiedAccessConditions, doc); + + if (!doc) { + throw StorageErrorFactory.getContainerNotFound(context); + } + + new ContainerReadLeaseValidator(leaseAccessConditions).validate( + new ContainerLeaseAdapter(doc), + context + ); + + doc.properties.lastModified = lastModified; + doc.properties.etag = etag; + doc.metadata = metadata; + + return coll.update(doc); + } + + /** + * Get container access policy. + * + * @param {string} account + * @param {string} container + * @param {Context} context + * @param {Models.LeaseAccessConditions} [leaseAccessConditions] + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + getContainerACL( + context: Context, + account: string, + container: string, + forceExist?: true, + leaseAccessConditions?: Models.LeaseAccessConditions + ): Promise; + + getContainerACL( + context: Context, + account: string, + container: string, + forceExist: false, + leaseAccessConditions?: Models.LeaseAccessConditions + ): Promise; + + public async getContainerACL( + context: Context, + account: string, + container: string, + forceExist?: boolean, + leaseAccessConditions?: Models.LeaseAccessConditions | undefined + ): Promise { + const doc = + forceExist === false + ? await this.getContainerWithLeaseUpdated( + account, + container, + context, + false + ) + : await this.getContainerWithLeaseUpdated( + account, + container, + context, + forceExist + ); + + if (!doc) { + if (forceExist === undefined || forceExist === true) { + throw StorageErrorFactory.getContainerNotFound(context); + } + + return undefined; + } + + new ContainerReadLeaseValidator(leaseAccessConditions).validate( + new ContainerLeaseAdapter(doc), + context + ); + + const res: GetContainerAccessPolicyResponse = { + properties: doc.properties, + containerAcl: doc.containerAcl + }; + + return res; + } + + /** + * Set container access policy. + * + * @param {string} account + * @param {string} container + * @param {SetContainerAccessPolicyOptions} setAclModel + * @param {Context} context + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async setContainerACL( + context: Context, + account: string, + container: string, + setAclModel: SetContainerAccessPolicyOptions + ): Promise { + const coll = this.db.getCollection(this.CONTAINERS_COLLECTION); + const doc = await this.getContainerWithLeaseUpdated( + account, + container, + context, + false + ); + + validateWriteConditions(context, setAclModel.modifiedAccessConditions, doc); + + if (!doc) { + throw StorageErrorFactory.getContainerNotFound(context); + } + + new ContainerReadLeaseValidator(setAclModel.leaseAccessConditions).validate( + new ContainerLeaseAdapter(doc), + context + ); + + doc.properties.publicAccess = setAclModel.publicAccess; + doc.containerAcl = setAclModel.containerAcl; + doc.properties.lastModified = setAclModel.lastModified; + doc.properties.etag = setAclModel.etag; + + return coll.update(doc); + } + + /** + * Acquire container lease. + * + * @param {string} account + * @param {string} container + * @param {Models.ContainerAcquireLeaseOptionalParams} options + * @param {Context} context + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async acquireContainerLease( + context: Context, + account: string, + container: string, + options: Models.ContainerAcquireLeaseOptionalParams + ): Promise { + const coll = this.db.getCollection(this.CONTAINERS_COLLECTION); + const doc = await this.getContainer(account, container, context, false); + + validateWriteConditions(context, options.modifiedAccessConditions, doc); + + if (!doc) { + throw StorageErrorFactory.getContainerNotFound(context); + } + + LeaseFactory.createLeaseState(new ContainerLeaseAdapter(doc), context) + .acquire(options.duration!, options.proposedLeaseId) + .sync(new ContainerLeaseSyncer(doc)); + + coll.update(doc); + + return { properties: doc.properties, leaseId: doc.leaseId }; + } + + /** + * Release container lease. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} leaseId + * @param {Models.ContainerReleaseLeaseOptionalParams} [options={}] + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async releaseContainerLease( + context: Context, + account: string, + container: string, + leaseId: string, + options: Models.ContainerReleaseLeaseOptionalParams = {} + ): Promise { + const coll = this.db.getCollection(this.CONTAINERS_COLLECTION); + const doc = await this.getContainer(account, container, context, false); + + validateWriteConditions(context, options.modifiedAccessConditions, doc); + + if (!doc) { + throw StorageErrorFactory.getContainerNotFound(context); + } + + LeaseFactory.createLeaseState(new ContainerLeaseAdapter(doc), context) + .release(leaseId) + .sync(new ContainerLeaseSyncer(doc)); + + coll.update(doc); + + return doc.properties; + } + + /** + * Renew container lease. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} leaseId + * @param {Models.ContainerRenewLeaseOptionalParams} [options={}] + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async renewContainerLease( + context: Context, + account: string, + container: string, + leaseId: string, + options: Models.ContainerRenewLeaseOptionalParams = {} + ): Promise { + const coll = this.db.getCollection(this.CONTAINERS_COLLECTION); + const doc = await this.getContainer(account, container, context, false); + + validateWriteConditions(context, options.modifiedAccessConditions, doc); + + if (!doc) { + throw StorageErrorFactory.getContainerNotFound(context); + } + + LeaseFactory.createLeaseState(new ContainerLeaseAdapter(doc), context) + .renew(leaseId) + .sync(new ContainerLeaseSyncer(doc)); + + coll.update(doc); + + return { properties: doc.properties, leaseId: doc.leaseId }; + } + + /** + * Break container lease. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {(number | undefined)} breakPeriod + * @param {Models.ContainerBreakLeaseOptionalParams} [options={}] + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async breakContainerLease( + context: Context, + account: string, + container: string, + breakPeriod: number | undefined, + options: Models.ContainerBreakLeaseOptionalParams = {} + ): Promise { + const coll = this.db.getCollection(this.CONTAINERS_COLLECTION); + const doc = await this.getContainer(account, container, context, false); + + validateWriteConditions(context, options.modifiedAccessConditions, doc); + + if (!doc) { + throw StorageErrorFactory.getContainerNotFound(context); + } + + LeaseFactory.createLeaseState(new ContainerLeaseAdapter(doc), context) + .break(breakPeriod) + .sync(new ContainerLeaseSyncer(doc)); + + const leaseTimeSeconds: number = + doc.properties.leaseState === Models.LeaseStateType.Breaking && + doc.leaseBreakTime + ? Math.round( + (doc.leaseBreakTime.getTime() - context.startTime!.getTime()) / 1000 + ) + : 0; + + coll.update(doc); + + return { properties: doc.properties, leaseTime: leaseTimeSeconds }; + } + + /** + * Change container lease. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} leaseId + * @param {string} proposedLeaseId + * @param {Models.ContainerChangeLeaseOptionalParams} [options={}] + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async changeContainerLease( + context: Context, + account: string, + container: string, + leaseId: string, + proposedLeaseId: string, + options: Models.ContainerChangeLeaseOptionalParams = {} + ): Promise { + const coll = this.db.getCollection(this.CONTAINERS_COLLECTION); + const doc = await this.getContainer(account, container, context, false); + + validateWriteConditions(context, options.modifiedAccessConditions, doc); + + if (!doc) { + throw StorageErrorFactory.getContainerNotFound(context); + } + + LeaseFactory.createLeaseState(new ContainerLeaseAdapter(doc), context) + .change(leaseId, proposedLeaseId) + .sync(new ContainerLeaseSyncer(doc)); + + coll.update(doc); + + return { properties: doc.properties, leaseId: doc.leaseId }; + } + + /** + * Check the existence of a container. + * + * @param {string} account + * @param {string} container + * @param {Context} [context] + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async checkContainerExist( + context: Context, + account: string, + container: string + ): Promise { + const coll = this.db.getCollection(this.CONTAINERS_COLLECTION); + const doc = coll.findOne({ accountName: account, name: container }); + if (!doc) { + throw StorageErrorFactory.getContainerNotFound(context); + } + } + + public async listBlobs( + context: Context, + account: string, + container: string, + delimiter?: string, + blob?: string, + prefix: string = "", + maxResults: number = DEFAULT_LIST_BLOBS_MAX_RESULTS, + marker: string = "", + includeSnapshots?: boolean, + includeUncommittedBlobs?: boolean + ): Promise<[BlobModel[], BlobPrefixModel[], string | undefined]> { + return this.list( + false, + context, + account, + container, + delimiter, + blob, + prefix, + maxResults, + marker, + includeSnapshots, + includeUncommittedBlobs + ); + } + + public async list( + lisDirectories: boolean | undefined, + context: Context, + account: string, + container: string, + delimiter?: string, + blob?: string, + prefix: string = "", + maxResults: number = DEFAULT_LIST_BLOBS_MAX_RESULTS, + marker: string = "", + includeSnapshots?: boolean, + includeUncommittedBlobs?: boolean + ): Promise<[BlobModel[], BlobPrefixModel[], string | undefined]> { + const query: any = {}; + if (prefix !== "") { + query.name = { $regex: `^${this.escapeRegex(prefix)}` }; + } + if (blob !== undefined) { + query.name = blob; + } + if (account !== undefined) { + query.accountName = account; + } + if (container !== undefined) { + query.containerName = container; + } + + if (lisDirectories !== undefined) { + query.isDirectory = lisDirectories; + } + + if (lisDirectories !== false) { + includeSnapshots = true; + includeUncommittedBlobs = true; + } + + const coll = this.db.getCollection(this.BLOBS_COLLECTION); + const page = new PageWithDelimiter( + maxResults, + delimiter, + prefix + ); + const readPage = async (offset: number): Promise => { + return coll + .chain() + .find(query) + .where((obj) => { + return obj.name > marker!; + }) + .where((obj) => { + return includeSnapshots ? true : obj.snapshot.length === 0; + }) + .where((obj) => { + return includeUncommittedBlobs ? true : obj.isCommitted; + }) + .sort((obj1, obj2) => { + if (obj1.name === obj2.name) return 0; + if (obj1.name > obj2.name) return 1; + return -1; + }) + .offset(offset) + .limit(maxResults) + .data(); + }; + + const nameItem = (item: BlobModel) => { + return item.name; + }; + + const [blobItems, blobPrefixes, nextMarker] = await page.fill( + readPage, + nameItem + ); + + let blobModels = blobItems.map((doc) => { + doc.properties.contentMD5 = this.restoreUint8Array( + doc.properties.contentMD5 + ); + return LeaseFactory.createLeaseState( + new BlobLeaseAdapter(doc), + context + ).sync(new BlobLeaseSyncer(doc)); + }); + blobModels = blobModels.filter((model) => + this.validateExpireConditions(context, model, false) + ); + return [blobModels, blobPrefixes, nextMarker]; + } + + public async listAllBlobs( + maxResults: number = DEFAULT_LIST_BLOBS_MAX_RESULTS, + marker: string = "", + includeSnapshots?: boolean, + includeUncommittedBlobs?: boolean + ): Promise<[BlobModel[], string | undefined]> { + const coll = this.db.getCollection(this.BLOBS_COLLECTION); + + const docs = coll + .chain() + .where((obj) => { + return obj.name > marker!; + }) + .where((obj) => { + return includeSnapshots ? true : obj.snapshot.length === 0; + }) + .where((obj) => { + return includeUncommittedBlobs ? true : obj.isCommitted; + }) + .simplesort("name") + .limit(maxResults + 1) + .data(); + + const filteredDocs = []; + for (const doc of docs) { + const blobDoc = doc as BlobModel; + if (!this.validateExpireConditions(undefined, blobDoc, false)) continue; + blobDoc.properties.contentMD5 = this.restoreUint8Array( + blobDoc.properties.contentMD5 + ); + + filteredDocs.push(blobDoc); + } + + if (filteredDocs.length <= maxResults) { + return [filteredDocs, undefined]; + } else { + const nextMarker = filteredDocs[filteredDocs.length - 2].name; + filteredDocs.pop(); + return [filteredDocs, nextMarker]; + } + } + + /** + * Create blob item in persistency layer. Will replace if blob exists. + * + * @param {Context} context + * @param {BlobModel} blob + * @param {Models.LeaseAccessConditions} [leaseAccessConditions] + * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async createBlob( + context: Context, + blob: BlobModel, + leaseAccessConditions?: Models.LeaseAccessConditions, + modifiedAccessConditions?: Models.ModifiedAccessConditions + ): Promise { + await this.checkContainerExist( + context, + blob.accountName, + blob.containerName + ); + const coll = this.db.getCollection(this.BLOBS_COLLECTION); + const blobDoc: BlobModel = coll.findOne({ + accountName: blob.accountName, + containerName: blob.containerName, + name: blob.name, + snapshot: blob.snapshot + }); + + const valid = this.validateExpireConditions(context, blobDoc, false); + validateWriteConditions(context, modifiedAccessConditions, blobDoc); + + // Create if not exists + if ( + modifiedAccessConditions && + modifiedAccessConditions.ifNoneMatch === "*" && + blobDoc && + valid + ) { + throw StorageErrorFactory.getBlobAlreadyExists(context); + } + + if (blobDoc && valid) { + LeaseFactory.createLeaseState(new BlobLeaseAdapter(blobDoc), context) + .validate(new BlobWriteLeaseValidator(leaseAccessConditions)) + .sync(new BlobLeaseSyncer(blob)); // Keep original blob lease + + if ( + blobDoc.properties !== undefined && + blobDoc.properties.accessTier === Models.AccessTier.Archive + ) { + throw StorageErrorFactory.getBlobArchived(context); + } + coll.remove(blobDoc); + } + delete (blob as any).$loki; + return coll.insert(blob); + } + + /** + * Create snapshot. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {Models.LeaseAccessConditions} [leaseAccessConditions] Optional. Will validate lease if provided + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async createSnapshot( + context: Context, + account: string, + container: string, + blob: string, + leaseAccessConditions?: Models.LeaseAccessConditions, + metadata?: Models.BlobMetadata, + modifiedAccessConditions?: Models.ModifiedAccessConditions + ): Promise { + const coll = this.db.getCollection(this.BLOBS_COLLECTION); + const doc = await this.getBlobWithLeaseUpdated( + account, + container, + blob, + undefined, + context, + false, + true + ); + + this.validateExpireConditions(context, doc, true); + validateReadConditions(context, modifiedAccessConditions, doc); + + if (!doc) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + new BlobReadLeaseValidator(leaseAccessConditions).validate( + new BlobLeaseAdapter(doc), + context + ); + + const snapshotTime = convertDateTimeStringMsTo7Digital( + context.startTime!.toISOString() + ); + + const snapshotBlob: BlobModel = { + name: doc.name, + deleted: false, + snapshot: snapshotTime, + properties: { ...doc.properties }, + metadata: metadata ? { ...metadata } : { ...doc.metadata }, + accountName: doc.accountName, + containerName: doc.containerName, + pageRangesInOrder: + doc.pageRangesInOrder === undefined + ? undefined + : doc.pageRangesInOrder.slice(), + isCommitted: doc.isCommitted, + committedBlocksInOrder: + doc.committedBlocksInOrder === undefined + ? undefined + : doc.committedBlocksInOrder.slice(), + persistency: + doc.persistency === undefined ? undefined : { ...doc.persistency }, + isDirectory: doc.isDirectory, + permissions: doc.permissions, + acl: doc.acl, + owner: doc.owner, + group: doc.group + }; + + new BlobLeaseSyncer(snapshotBlob).sync({ + leaseId: undefined, + leaseExpireTime: undefined, + leaseDurationSeconds: undefined, + leaseBreakTime: undefined, + leaseDurationType: undefined, + leaseState: undefined, + leaseStatus: undefined + }); + + coll.insert(snapshotBlob); + + return { + properties: snapshotBlob.properties, + snapshot: snapshotTime + }; + } + + /** + * Gets a blob item from persistency layer by container name and blob name. + * Will return block list or page list as well for downloading. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {string} [snapshot=""] + * @param {Models.LeaseAccessConditions} [leaseAccessConditions] + * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async downloadBlob( + context: Context, + account: string, + container: string, + blob: string, + snapshot: string = "", + leaseAccessConditions?: Models.LeaseAccessConditions, + modifiedAccessConditions?: Models.ModifiedAccessConditions + ): Promise { + blob = removeSlash(blob); + const doc = await this.getBlobWithLeaseUpdated( + account, + container, + blob, + snapshot, + context, + false, + true + ); + + this.validateExpireConditions(context, doc, true); + validateReadConditions(context, modifiedAccessConditions, doc); + + if (!doc) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + new BlobReadLeaseValidator(leaseAccessConditions).validate( + new BlobLeaseAdapter(doc), + context + ); + + return doc; + } + + /** + * Get blob properties. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {string} [snapshot=""] + * @param {(Models.LeaseAccessConditions | undefined)} leaseAccessConditions + * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async getBlobProperties( + context: Context, + account: string, + container: string, + blob: string, + snapshot: string = "", + leaseAccessConditions: Models.LeaseAccessConditions | undefined, + modifiedAccessConditions?: Models.ModifiedAccessConditions + ): Promise { + const doc = await this.getBlobWithLeaseUpdated( + account, + container, + blob, + snapshot, + context, + false, + true + ); + + this.validateExpireConditions(context, doc, true); + validateReadConditions(context, modifiedAccessConditions, doc); + + // When block blob don't have commited block, should return 404 + if (!doc) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + new BlobReadLeaseValidator(leaseAccessConditions).validate( + new BlobLeaseAdapter(doc), + context + ); + + return { + properties: doc.properties, + metadata: doc.metadata, + blobCommittedBlockCount: + doc.properties.blobType === Models.BlobType.AppendBlob + ? (doc.committedBlocksInOrder || []).length + : undefined + }; + } + + /** + * Delete blob or its snapshots. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {Models.PathDeleteMethodOptionalParams} options + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async deleteBlob( + context: Context, + account: string, + container: string, + blob: string, + options: Models.PathDeleteMethodOptionalParams + ): Promise { + const coll = this.db.getCollection(this.BLOBS_COLLECTION); + await this.checkContainerExist(context, account, container); + + const doc = await this.getBlobWithLeaseUpdated( + account, + container, + blob, + options.snapshot, + context, + false + ); + + this.validateExpireConditions(context, doc, true); + validateWriteConditions(context, options.modifiedAccessConditions, doc); + + if (!doc) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + const againstBaseBlob = doc.snapshot === ""; + + // Check bad requests + if (!againstBaseBlob && options.deleteSnapshots !== undefined) { + throw StorageErrorFactory.getInvalidOperation( + context, + "Invalid operation against a blob snapshot." + ); + } + + new BlobWriteLeaseValidator(options.leaseAccessConditions).validate( + new BlobLeaseAdapter(doc), + context + ); + + // Scenario: Delete base blob only + if (againstBaseBlob && options.deleteSnapshots === undefined) { + const count = coll.count({ + accountName: account, + containerName: container, + name: blob + }); + if (count > 1) { + throw StorageErrorFactory.getSnapshotsPresent(context); + } else { + coll.findAndRemove({ + accountName: account, + containerName: container, + name: blob + }); + } + } + + // Scenario: Delete one snapshot only + if (!againstBaseBlob) { + coll.findAndRemove({ + accountName: account, + containerName: container, + name: blob, + snapshot: doc.snapshot + }); + } + + // Scenario: Delete base blob and snapshots + if ( + againstBaseBlob && + options.deleteSnapshots === Models.DeleteSnapshotsOptionType.Include + ) { + coll.findAndRemove({ + accountName: account, + containerName: container, + name: blob + }); + } + + // Scenario: Delete all snapshots only + if ( + againstBaseBlob && + options.deleteSnapshots === Models.DeleteSnapshotsOptionType.Only + ) { + const query = { + accountName: account, + containerName: container, + name: blob, + snapshot: { $gt: "" } + }; + coll.findAndRemove(query); + } + } + + /** + * Set blob HTTP headers. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {(Models.LeaseAccessConditions | undefined)} leaseAccessConditions + * @param {(Models.BlobHTTPHeaders | undefined)} blobHTTPHeaders + * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async setBlobHTTPHeaders( + context: Context, + account: string, + container: string, + blob: string, + leaseAccessConditions: Models.LeaseAccessConditions | undefined, + blobHTTPHeaders: Models.BlobHTTPHeaders | undefined, + modifiedAccessConditions?: Models.ModifiedAccessConditions + ): Promise { + const coll = this.db.getCollection(this.BLOBS_COLLECTION); + const doc = await this.getBlobWithLeaseUpdated( + account, + container, + blob, + undefined, + context, + false, + true + ); + + this.validateExpireConditions(context, doc, true); + validateWriteConditions(context, modifiedAccessConditions, doc); + + if (!doc) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + const lease = new BlobLeaseAdapter(doc); + new BlobWriteLeaseValidator(leaseAccessConditions).validate(lease, context); + + const blobHeaders = blobHTTPHeaders; + const blobProps = doc.properties; + // as per https://docs.microsoft.com/en-us/rest/api/storageservices/set-blob-properties#remarks + // If any one or more of the following properties is set in the request, + // then all of these properties are set together. + // If a value is not provided for a given property when at least one + // of the properties listed below is set, then that property will + // be cleared for the blob. + if (blobHeaders !== undefined) { + blobProps.cacheControl = blobHeaders.blobCacheControl; + blobProps.contentType = blobHeaders.blobContentType; + blobProps.contentMD5 = blobHeaders.blobContentMD5; + blobProps.contentEncoding = blobHeaders.blobContentEncoding; + blobProps.contentLanguage = blobHeaders.blobContentLanguage; + blobProps.contentDisposition = blobHeaders.blobContentDisposition; + } + doc.properties = blobProps; + doc.properties.etag = newEtag(); + blobProps.lastModified = context.startTime ? context.startTime : new Date(); + + new BlobWriteLeaseSyncer(doc).sync(lease); + + coll.update(doc); + return doc.properties; + } + + /** + * Set blob metadata. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {(Models.LeaseAccessConditions | undefined)} leaseAccessConditions + * @param {(Models.BlobMetadata | undefined)} metadata + * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async setBlobMetadata( + context: Context, + account: string, + container: string, + blob: string, + leaseAccessConditions: Models.LeaseAccessConditions | undefined, + metadata: Models.BlobMetadata | undefined, + modifiedAccessConditions?: Models.ModifiedAccessConditions + ): Promise { + const coll = this.db.getCollection(this.BLOBS_COLLECTION); + const doc = await this.getBlobWithLeaseUpdated( + account, + container, + blob, + undefined, + context, + false, + true + ); + + this.validateExpireConditions(context, doc, true); + validateWriteConditions(context, modifiedAccessConditions, doc); + + if (!doc) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + const lease = new BlobLeaseAdapter(doc); + new BlobWriteLeaseValidator(leaseAccessConditions).validate(lease, context); + new BlobWriteLeaseSyncer(doc).sync(lease); + doc.metadata = metadata; + doc.properties.etag = newEtag(); + doc.properties.lastModified = context.startTime || new Date(); + coll.update(doc); + return doc.properties; + } + + /** + * Acquire blob lease. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {number} duration + * @param {string} [proposedLeaseId] + * @param {Models.BlobAcquireLeaseOptionalParams} [options={}] + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async acquireBlobLease( + context: Context, + account: string, + container: string, + blob: string, + duration: number, + proposedLeaseId?: string, + options: Models.BlobAcquireLeaseOptionalParams = {} + ): Promise { + const coll = this.db.getCollection(this.BLOBS_COLLECTION); + const doc = await this.getBlobWithLeaseUpdated( + account, + container, + blob, + undefined, + context, + false + ); // This may return an uncommitted blob, or undefined for an unexist blob + + this.validateExpireConditions(context, doc, true); + validateWriteConditions(context, options.modifiedAccessConditions, doc); + + // Azure Storage allows lease for a uncommitted blob + if (!doc) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + if (doc.snapshot !== "") { + throw StorageErrorFactory.getBlobSnapshotsPresent(context); + } + + LeaseFactory.createLeaseState(new BlobLeaseAdapter(doc), context) + .acquire(duration, proposedLeaseId) + .sync(new BlobLeaseSyncer(doc)); + + coll.update(doc); + + return { properties: doc.properties, leaseId: doc.leaseId }; + } + + /** + * Release blob. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {string} leaseId + * @param {Models.BlobReleaseLeaseOptionalParams} [options={}] + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async releaseBlobLease( + context: Context, + account: string, + container: string, + blob: string, + leaseId: string, + options: Models.BlobReleaseLeaseOptionalParams = {} + ): Promise { + const coll = this.db.getCollection(this.BLOBS_COLLECTION); + const doc = await this.getBlobWithLeaseUpdated( + account, + container, + blob, + undefined, + context, + false + ); // This may return an uncommitted blob, or undefined for an unexist blob + + this.validateExpireConditions(context, doc, true); + validateWriteConditions(context, options.modifiedAccessConditions, doc); + + // Azure Storage allows lease for a uncommitted blob + if (!doc) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + if (doc.snapshot !== "") { + throw StorageErrorFactory.getBlobSnapshotsPresent(context); + } + + LeaseFactory.createLeaseState(new BlobLeaseAdapter(doc), context) + .release(leaseId) + .sync(new BlobLeaseSyncer(doc)); + + coll.update(doc); + + return doc.properties; + } + + /** + * Renew blob lease. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {string} leaseId + * @param {Models.BlobRenewLeaseOptionalParams} [options={}] + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async renewBlobLease( + context: Context, + account: string, + container: string, + blob: string, + leaseId: string, + options: Models.BlobRenewLeaseOptionalParams = {} + ): Promise { + const coll = this.db.getCollection(this.BLOBS_COLLECTION); + const doc = await this.getBlobWithLeaseUpdated( + account, + container, + blob, + undefined, + context, + false + ); // This may return an uncommitted blob, or undefined for an unexist blob + + this.validateExpireConditions(context, doc, true); + validateWriteConditions(context, options.modifiedAccessConditions, doc); + + // Azure Storage allows lease for a uncommitted blob + if (!doc) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + if (doc.snapshot !== "") { + throw StorageErrorFactory.getBlobSnapshotsPresent(context); + } + + LeaseFactory.createLeaseState(new BlobLeaseAdapter(doc), context) + .renew(leaseId) + .sync(new BlobLeaseSyncer(doc)); + + coll.update(doc); + + return { properties: doc.properties, leaseId: doc.leaseId }; + } + + /** + * Change blob lease. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {string} leaseId + * @param {string} proposedLeaseId + * @param {Models.BlobChangeLeaseOptionalParams} [option={}] + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async changeBlobLease( + context: Context, + account: string, + container: string, + blob: string, + leaseId: string, + proposedLeaseId: string, + options: + | Models.BlobChangeLeaseOptionalParams + | Models.PathLeaseOptionalParams = {} + ): Promise { + const coll = this.db.getCollection(this.BLOBS_COLLECTION); + const doc = await this.getBlobWithLeaseUpdated( + account, + container, + blob, + undefined, + context, + false + ); // This may return an uncommitted blob, or undefined for an unexist blob + + this.validateExpireConditions(context, doc, true); + validateWriteConditions(context, options.modifiedAccessConditions, doc); + + // Azure Storage allows lease for a uncommitted blob + if (!doc) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + if (doc.snapshot !== "") { + throw StorageErrorFactory.getBlobSnapshotsPresent(context); + } + + LeaseFactory.createLeaseState(new BlobLeaseAdapter(doc), context) + .change(leaseId, proposedLeaseId) + .sync(new BlobLeaseSyncer(doc)); + + coll.update(doc); + + return { properties: doc.properties, leaseId: doc.leaseId }; + } + + /** + * Break blob lease. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {(number | undefined)} breakPeriod + * @param {Models.BlobBreakLeaseOptionalParams} [options={}] + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async breakBlobLease( + context: Context, + account: string, + container: string, + blob: string, + breakPeriod: number | undefined, + options: + | Models.BlobBreakLeaseOptionalParams + | Models.PathLeaseOptionalParams = {} + ): Promise { + const coll = this.db.getCollection(this.BLOBS_COLLECTION); + const doc = await this.getBlobWithLeaseUpdated( + account, + container, + blob, + undefined, + context, + false + ); // This may return an uncommitted blob, or undefined for an unexist blob + + this.validateExpireConditions(context, doc, true); + validateWriteConditions(context, options.modifiedAccessConditions, doc); + + // Azure Storage allows lease for a uncommitted blob + if (!doc) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + if (doc.snapshot !== "") { + throw StorageErrorFactory.getBlobSnapshotsPresent(context); + } + + LeaseFactory.createLeaseState(new BlobLeaseAdapter(doc), context) + .break(breakPeriod) + .sync(new BlobLeaseSyncer(doc)); + + const leaseTimeSeconds: number = + doc.properties.leaseState === Models.LeaseStateType.Breaking && + doc.leaseBreakTime + ? Math.round( + (doc.leaseBreakTime.getTime() - context.startTime!.getTime()) / 1000 + ) + : 0; + + coll.update(doc); + + return { properties: doc.properties, leaseTime: leaseTimeSeconds }; + } + + /** + * Check the existence of a blob. + * + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {string} [snapshot=""] + * @param {Context} [context] + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async checkBlobExist( + context: Context, + account: string, + container: string, + blob: string, + snapshot: string = "" + ): Promise { + await this.checkContainerExist(context, account, container); + + const coll = this.db.getCollection(this.BLOBS_COLLECTION); + const doc = coll.findOne({ + accountName: account, + containerName: container, + name: blob, + snapshot + }); + + this.validateExpireConditions(context, doc, true); + if (!doc) { + throw StorageErrorFactory.getBlobNotFound(context); + } + } + + /** + * Get blobType and committed status for SAS authentication. + * + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {string} [snapshot=""] + * @returns {(Promise< + * { blobType: Models.BlobType | undefined; isCommitted: boolean } | undefined + * >)} + * @memberof LokiBlobMetadataStore + */ + public async getBlobType( + account: string, + container: string, + blob: string, + snapshot: string = "" + ): Promise< + { blobType: Models.BlobType | undefined; isCommitted: boolean } | undefined + > { + const coll = this.db.getCollection(this.BLOBS_COLLECTION); + const doc = coll.findOne({ + accountName: account, + containerName: container, + name: blob, + snapshot + }); + + const valid = this.validateExpireConditions(undefined, doc, true); + if (!doc || !valid) { + return undefined; + } + return { blobType: doc.properties.blobType, isCommitted: doc.isCommitted }; + } + + /** + * Start copy from Url. + * + * @param {Context} context + * @param {BlobId} source + * @param {BlobId} destination + * @param {string} copySource + * @param {(Models.BlobMetadata | undefined)} metadata + * @param {(Models.AccessTier | undefined)} tier + * @param {Models.BlobStartCopyFromURLOptionalParams} [leaseAccessConditions] + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async startCopyFromURL( + context: Context, + source: BlobId, + destination: BlobId, + copySource: string, + metadata: Models.BlobMetadata | undefined, + tier: Models.AccessTier | undefined, + options: Models.BlobStartCopyFromURLOptionalParams = {} + ): Promise { + const coll = this.db.getCollection(this.BLOBS_COLLECTION); + const sourceBlob = await this.getBlobWithLeaseUpdated( + source.account, + source.container, + source.blob, + source.snapshot, + context, + true, + true + ); + + this.validateExpireConditions(context, sourceBlob, true); + options.sourceModifiedAccessConditions = + options.sourceModifiedAccessConditions || {}; + validateReadConditions( + context, + { + ifModifiedSince: + options.sourceModifiedAccessConditions.sourceIfModifiedSince, + ifUnmodifiedSince: + options.sourceModifiedAccessConditions.sourceIfUnmodifiedSince, + ifMatch: options.sourceModifiedAccessConditions.sourceIfMatch, + ifNoneMatch: options.sourceModifiedAccessConditions.sourceIfNoneMatch + }, + sourceBlob + ); + + const destBlob = await this.getBlobWithLeaseUpdated( + destination.account, + destination.container, + destination.blob, + undefined, + context, + false + ); + + validateWriteConditions( + context, + options.modifiedAccessConditions, + destBlob + ); + + // Copy if not exists + if ( + options.modifiedAccessConditions && + options.modifiedAccessConditions.ifNoneMatch === "*" && + destBlob + ) { + throw StorageErrorFactory.getBlobAlreadyExists(context); + } + + if (destBlob) { + new BlobWriteLeaseValidator(options.leaseAccessConditions).validate( + new BlobLeaseAdapter(destBlob), + context + ); + } + + // If source is uncommitted or deleted + if ( + sourceBlob === undefined || + sourceBlob.deleted || + !sourceBlob.isCommitted + ) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + if ( + sourceBlob.properties.accessTier === Models.AccessTier.Archive && + (tier === undefined || source.account !== destination.account) + ) { + throw StorageErrorFactory.getBlobArchived(context); + } + + await this.checkContainerExist( + context, + destination.account, + destination.container + ); + + // Deep clone a copied blob + const copiedBlob: BlobModel = { + name: destination.blob, + deleted: false, + snapshot: "", + properties: { + ...sourceBlob.properties, + creationTime: context.startTime!, + lastModified: context.startTime!, + etag: newEtag(), + leaseStatus: + destBlob !== undefined + ? destBlob.properties.leaseStatus + : Models.LeaseStatusType.Unlocked, + leaseState: + destBlob !== undefined + ? destBlob.properties.leaseState + : Models.LeaseStateType.Available, + leaseDuration: + destBlob !== undefined + ? destBlob.properties.leaseDuration + : undefined, + copyId: uuid(), + copyStatus: Models.CopyStatusType.Success, + copySource, + copyProgress: sourceBlob.properties.contentLength + ? `${sourceBlob.properties.contentLength}/${sourceBlob.properties.contentLength}` + : undefined, + copyCompletionTime: context.startTime, + copyStatusDescription: undefined, + incrementalCopy: false, + destinationSnapshot: undefined, + deletedTime: undefined, + remainingRetentionDays: undefined, + archiveStatus: undefined, + accessTierChangeTime: undefined + }, + metadata: + metadata === undefined || Object.keys(metadata).length === 0 + ? { ...sourceBlob.metadata } + : metadata, + accountName: destination.account, + containerName: destination.container, + pageRangesInOrder: sourceBlob.pageRangesInOrder, + isCommitted: sourceBlob.isCommitted, + leaseDurationSeconds: + destBlob !== undefined ? destBlob.leaseDurationSeconds : undefined, + leaseId: destBlob !== undefined ? destBlob.leaseId : undefined, + leaseExpireTime: + destBlob !== undefined ? destBlob.leaseExpireTime : undefined, + leaseBreakTime: + destBlob !== undefined ? destBlob.leaseBreakTime : undefined, + committedBlocksInOrder: sourceBlob.committedBlocksInOrder, + persistency: sourceBlob.persistency, + isDirectory: sourceBlob.isDirectory, + permissions: sourceBlob.permissions, + acl: sourceBlob.acl, + owner: sourceBlob.owner, + group: sourceBlob.group + }; + + if ( + copiedBlob.properties.blobType === Models.BlobType.BlockBlob && + tier !== undefined + ) { + copiedBlob.properties.accessTier = this.parseTier(tier); + if (copiedBlob.properties.accessTier === undefined) { + throw StorageErrorFactory.getInvalidHeaderValue(context, { + HeaderName: "x-ms-access-tier", + HeaderValue: `${tier}` + }); + } + } + + if ( + copiedBlob.properties.blobType === Models.BlobType.PageBlob && + tier !== undefined + ) { + throw StorageErrorFactory.getInvalidHeaderValue(context, { + HeaderName: "x-ms-access-tier", + HeaderValue: `${tier}` + }); + } + + if (destBlob) { + coll.remove(destBlob); + } + coll.insert(copiedBlob); + return copiedBlob.properties; + } + + /** + * Copy from Url. + * + * @param {Context} context + * @param {BlobId} source + * @param {BlobId} destination + * @param {string} copySource + * @param {(Models.BlobMetadata | undefined)} metadata + * @param {(Models.AccessTier | undefined)} tier + * @param {Models.BlobCopyFromURLOptionalParams} [leaseAccessConditions] + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async copyFromURL( + context: Context, + source: BlobId, + destination: BlobId, + copySource: string, + metadata: Models.BlobMetadata | undefined, + tier: Models.AccessTier | undefined, + options: Models.BlobCopyFromURLOptionalParams = {} + ): Promise { + const coll = this.db.getCollection(this.BLOBS_COLLECTION); + const sourceBlob = await this.getBlobWithLeaseUpdated( + source.account, + source.container, + source.blob, + source.snapshot, + context, + true, + true + ); + + this.validateExpireConditions(context, sourceBlob, true); + options.sourceModifiedAccessConditions = + options.sourceModifiedAccessConditions || {}; + validateReadConditions( + context, + { + ifModifiedSince: + options.sourceModifiedAccessConditions.sourceIfModifiedSince, + ifUnmodifiedSince: + options.sourceModifiedAccessConditions.sourceIfUnmodifiedSince, + ifMatch: options.sourceModifiedAccessConditions.sourceIfMatch, + ifNoneMatch: options.sourceModifiedAccessConditions.sourceIfNoneMatch + }, + sourceBlob + ); + + const destBlob = await this.getBlobWithLeaseUpdated( + destination.account, + destination.container, + destination.blob, + undefined, + context, + false + ); + + validateWriteConditions( + context, + options.modifiedAccessConditions, + destBlob + ); + + // Copy if not exists + if ( + options.modifiedAccessConditions && + options.modifiedAccessConditions.ifNoneMatch === "*" && + destBlob + ) { + throw StorageErrorFactory.getBlobAlreadyExists(context); + } + + if (destBlob) { + const lease = new BlobLeaseAdapter(destBlob); + new BlobWriteLeaseSyncer(destBlob).sync(lease); + new BlobWriteLeaseValidator(options.leaseAccessConditions).validate( + lease, + context + ); + } + + // If source is uncommitted or deleted + if ( + sourceBlob === undefined || + sourceBlob.deleted || + !sourceBlob.isCommitted + ) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + if (sourceBlob.properties.accessTier === Models.AccessTier.Archive) { + throw StorageErrorFactory.getBlobArchived(context); + } + + await this.checkContainerExist( + context, + destination.account, + destination.container + ); + + // Deep clone a copied blob + const copiedBlob: BlobModel = { + name: destination.blob, + deleted: false, + snapshot: "", + properties: { + ...sourceBlob.properties, + creationTime: context.startTime!, + lastModified: context.startTime!, + etag: newEtag(), + leaseStatus: + destBlob !== undefined + ? destBlob.properties.leaseStatus + : Models.LeaseStatusType.Unlocked, + leaseState: + destBlob !== undefined + ? destBlob.properties.leaseState + : Models.LeaseStateType.Available, + leaseDuration: + destBlob !== undefined + ? destBlob.properties.leaseDuration + : undefined, + copyId: uuid(), + copyStatus: Models.CopyStatusType.Success, + copySource, + copyProgress: sourceBlob.properties.contentLength + ? `${sourceBlob.properties.contentLength}/${sourceBlob.properties.contentLength}` + : undefined, + copyCompletionTime: context.startTime, + copyStatusDescription: undefined, + incrementalCopy: false, + destinationSnapshot: undefined, + deletedTime: undefined, + remainingRetentionDays: undefined, + archiveStatus: undefined, + accessTierChangeTime: undefined + }, + metadata: + metadata === undefined || Object.keys(metadata).length === 0 + ? { ...sourceBlob.metadata } + : metadata, + accountName: destination.account, + containerName: destination.container, + pageRangesInOrder: sourceBlob.pageRangesInOrder, + isCommitted: sourceBlob.isCommitted, + leaseDurationSeconds: + destBlob !== undefined ? destBlob.leaseDurationSeconds : undefined, + leaseId: destBlob !== undefined ? destBlob.leaseId : undefined, + leaseExpireTime: + destBlob !== undefined ? destBlob.leaseExpireTime : undefined, + leaseBreakTime: + destBlob !== undefined ? destBlob.leaseBreakTime : undefined, + committedBlocksInOrder: sourceBlob.committedBlocksInOrder, + persistency: sourceBlob.persistency, + isDirectory: sourceBlob.isDirectory, + permissions: sourceBlob.permissions, + acl: sourceBlob.acl, + owner: sourceBlob.owner, + group: sourceBlob.group + }; + + if ( + copiedBlob.properties.blobType === Models.BlobType.BlockBlob && + tier !== undefined + ) { + copiedBlob.properties.accessTier = this.parseTier(tier); + if (copiedBlob.properties.accessTier === undefined) { + throw StorageErrorFactory.getInvalidHeaderValue(context, { + HeaderName: "x-ms-access-tier", + HeaderValue: `${tier}` + }); + } + } + + if ( + copiedBlob.properties.blobType === Models.BlobType.PageBlob && + tier !== undefined + ) { + throw StorageErrorFactory.getInvalidHeaderValue(context, { + HeaderName: "x-ms-access-tier", + HeaderValue: `${tier}` + }); + } + + if (destBlob) { + coll.remove(destBlob); + } + coll.insert(copiedBlob); + return copiedBlob.properties; + } + + /** + * Update Tier for a blob. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {Models.AccessTier} tier + * @param {(Models.LeaseAccessConditions | undefined)} leaseAccessConditions + * @returns {(Promise<200 | 202>)} + * @memberof LokiBlobMetadataStore + */ + public async setTier( + context: Context, + account: string, + container: string, + blob: string, + tier: Models.AccessTier, + leaseAccessConditions: Models.LeaseAccessConditions | undefined + ): Promise<200 | 202> { + const coll = this.db.getCollection(this.BLOBS_COLLECTION); + const doc = await this.getBlobWithLeaseUpdated( + account, + container, + blob, + undefined, + context, + true, + true + ); + + this.validateExpireConditions(context, doc, true); + let responseCode: 200 | 202 = 200; + + // Check the lease action aligned with current lease state. + // API reference doesn't mention there is x-ms-lease-id header supported by this API, + // however, it fails to set tier for a leased blocked blob with LeaseIdMissing + const lease = new BlobLeaseAdapter(doc); + new BlobWriteLeaseValidator(leaseAccessConditions).validate(lease, context); + + // Check Blob is not snapshot + if (doc.snapshot !== "") { + throw StorageErrorFactory.getBlobSnapshotsPresent(context); + } + + // Check BlobTier matches blob type + if ( + (tier === Models.AccessTier.Archive || + tier === Models.AccessTier.Cool || + tier === Models.AccessTier.Hot) && + doc.properties.blobType === Models.BlobType.BlockBlob + ) { + // Block blob + // tslint:disable-next-line:max-line-length + // TODO: check blob is not block blob with snapshot, throw StorageErrorFactory.getBlobSnapshotsPresent_hassnapshot() + + // Archive -> Coo/Hot will return 202 + if ( + doc.properties.accessTier === Models.AccessTier.Archive && + (tier === Models.AccessTier.Cool || tier === Models.AccessTier.Hot) + ) { + responseCode = 202; + } + + doc.properties.accessTier = tier; + doc.properties.accessTierInferred = false; + doc.properties.accessTierChangeTime = context.startTime; + } else { + throw StorageErrorFactory.getAccessTierNotSupportedForBlobType(context); + } + + new BlobWriteLeaseSyncer(doc).sync(lease); + coll.update(doc); + + return responseCode; + } + + /** + * Update blob block item in persistency layer. Will create if block doesn't exist. + * Will also create a uncommitted block blob. + * + * @param {BlockModel} block + * @param {Context} context + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async stageBlock( + context: Context, + block: BlockModel, + leaseAccessConditions?: Models.LeaseAccessConditions, + append: boolean = false + ): Promise { + await this.checkContainerExist( + context, + block.accountName, + block.containerName + ); + + const blobColl = this.db.getCollection(this.BLOBS_COLLECTION); + const blobDoc = blobColl.findOne({ + accountName: block.accountName, + containerName: block.containerName, + name: block.blobName + }); + + const valid = this.validateExpireConditions(context, blobDoc, false); + let blobExist = valid; + + if (!blobDoc || !valid) { + const etag = newEtag(); + const newBlob: BlobModel = { + deleted: false, + accountName: block.accountName, + containerName: block.containerName, + name: block.blobName, + properties: { + creationTime: context.startTime, + lastModified: context.startTime!, + etag, + contentLength: 0, + blobType: Models.BlobType.BlockBlob + }, + snapshot: "", + isCommitted: false, + isDirectory: false, + permissions: toPermissions(DEFAULT_FILE_PERMISSIONS)!, + acl: [], + owner: DEFAULT_OWNER, + group: DEFAULT_GROUP + }; + blobColl.insert(newBlob); + } else { + if ( + blobDoc.properties.blobType !== Models.BlobType.BlockBlob && + (!append || blobDoc.properties.blobType !== Models.BlobType.AppendBlob) + ) { + throw StorageErrorFactory.getBlobInvalidBlobType(context); + } + + LeaseFactory.createLeaseState(new BlobLeaseAdapter(blobDoc), context) + .validate(new BlobWriteLeaseValidator(leaseAccessConditions)) + .sync(new BlobWriteLeaseSyncer(blobDoc)); + blobExist = true; + } + + const coll = this.db.getCollection(this.BLOCKS_COLLECTION); + + // If the new block ID does not have same length with before uncommited block ID, return failure. + if (blobExist && !append) { + const existBlockDoc = coll.findOne({ + accountName: block.accountName, + containerName: block.containerName, + blobName: block.blobName + }); + if (existBlockDoc) { + if ( + Buffer.from(existBlockDoc.name, "base64").length !== + Buffer.from(block.name, "base64").length + ) { + throw StorageErrorFactory.getInvalidBlobOrBlock(context); + } + } + } + + const blockDoc = coll.findOne({ + accountName: block.accountName, + containerName: block.containerName, + blobName: block.blobName, + name: block.name, + isCommitted: block.isCommitted + }); + + if (blockDoc) { + coll.remove(blockDoc); + } + + delete (block as any).$loki; + coll.insert(block); + } + + public async appendBlock( + context: Context, + block: BlockModel, + leaseAccessConditions: Models.LeaseAccessConditions = {}, + modifiedAccessConditions: Models.ModifiedAccessConditions = {}, + appendPositionAccessConditions: Models.AppendPositionAccessConditions = {} + ): Promise { + const coll = this.db.getCollection(this.BLOBS_COLLECTION); + const doc = await this.getBlobWithLeaseUpdated( + block.accountName, + block.containerName, + block.blobName, + undefined, + context, + false, + true + ); + + this.validateExpireConditions(context, doc, true); + validateWriteConditions(context, modifiedAccessConditions, doc); + + if (!doc) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + new BlobWriteLeaseValidator(leaseAccessConditions).validate( + new BlobLeaseAdapter(doc), + context + ); + + if (doc.properties.blobType !== Models.BlobType.AppendBlob) { + throw StorageErrorFactory.getBlobInvalidBlobType(context); + } + + if ( + (doc.committedBlocksInOrder || []).length >= MAX_APPEND_BLOB_BLOCK_COUNT + ) { + throw StorageErrorFactory.getBlockCountExceedsLimit(context); + } + + if (appendPositionAccessConditions.appendPosition !== undefined) { + if ( + (doc.properties.contentLength || 0) !== + appendPositionAccessConditions.appendPosition + ) { + throw StorageErrorFactory.getAppendPositionConditionNotMet(context); + } + } + + if (appendPositionAccessConditions.maxSize !== undefined) { + if ( + (doc.properties.contentLength || 0) + block.size > + appendPositionAccessConditions.maxSize + ) { + throw StorageErrorFactory.getMaxBlobSizeConditionNotMet(context); + } + } + + doc.committedBlocksInOrder = doc.committedBlocksInOrder || []; + doc.committedBlocksInOrder.push(block); + doc.properties.etag = newEtag(); + doc.properties.lastModified = context.startTime || new Date(); + doc.properties.contentLength = + (doc.properties.contentLength || 0) + block.size; + coll.update(doc); + + return doc.properties; + } + + /** + * Commit block list for a blob. + * + * @param {Context} context + * @param {BlobModel} blob + * @param {{ blockName: string; blockCommitType: string }[]} blockList + * @param {Models.LeaseAccessConditions} [leaseAccessConditions] + * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async commitBlockList( + context: Context, + blob: BlobModel, + blockList: { blockName: string; blockCommitType: string }[], + leaseAccessConditions?: Models.LeaseAccessConditions, + modifiedAccessConditions?: Models.ModifiedAccessConditions + ): Promise { + const coll = this.db.getCollection(this.BLOBS_COLLECTION); + const doc = await this.getBlobWithLeaseUpdated( + blob.accountName, + blob.containerName, + blob.name, + blob.snapshot, + context, + // XStore allows commit block list with empty block list to create a block blob without stage block call + // In this case, there will no existing blob doc exists + false + ); + + this.validateExpireConditions(context, doc, true); + validateWriteConditions(context, modifiedAccessConditions, doc); + + // Create if not exists + if ( + modifiedAccessConditions && + modifiedAccessConditions.ifNoneMatch === "*" && + doc && + doc.isCommitted + ) { + throw StorageErrorFactory.getBlobAlreadyExists(context); + } + + let lease: ILease | undefined; + if (doc) { + if ( + doc.properties.blobType !== Models.BlobType.BlockBlob && + doc.properties.blobType !== Models.BlobType.AppendBlob + ) { + throw StorageErrorFactory.getBlobInvalidBlobType(context); + } + + lease = new BlobLeaseAdapter(doc); + new BlobWriteLeaseValidator(leaseAccessConditions).validate( + lease, + context + ); + } + + // Get all blocks in persistency layer + const blockColl = this.db.getCollection(this.BLOCKS_COLLECTION); + const pUncommittedBlocks = blockColl + .chain() + .find({ + accountName: blob.accountName, + containerName: blob.containerName, + blobName: blob.name + }) + .data(); + + const pCommittedBlocksMap: Map = new Map(); // persistencyCommittedBlocksMap + if (doc) { + for (const pBlock of doc.committedBlocksInOrder || []) { + pCommittedBlocksMap.set(pBlock.name, pBlock); + } + } + + const pUncommittedBlocksMap: Map = new Map(); // persistencyUncommittedBlocksMap + for (const pBlock of pUncommittedBlocks) { + if (!pBlock.isCommitted) { + pUncommittedBlocksMap.set(pBlock.name, pBlock); + } + } + + const selectedBlockList: PersistencyBlockModel[] = + doc && + doc.properties.blobType === Models.BlobType.AppendBlob && + doc.committedBlocksInOrder + ? doc.committedBlocksInOrder + : []; + for (const block_1 of blockList) { + switch (block_1.blockCommitType.toLowerCase()) { + case "uncommitted": + const pUncommittedBlock = pUncommittedBlocksMap.get( + block_1.blockName + ); + if (pUncommittedBlock === undefined) { + throw StorageErrorFactory.getInvalidBlockList(context); + } else { + selectedBlockList.push(pUncommittedBlock); + } + break; + case "committed": + const pCommittedBlock = pCommittedBlocksMap.get(block_1.blockName); + if (pCommittedBlock === undefined) { + throw StorageErrorFactory.getInvalidBlockList(context); + } else { + selectedBlockList.push(pCommittedBlock); + } + break; + case "latest": + const pLatestBlock = + pUncommittedBlocksMap.get(block_1.blockName) || + pCommittedBlocksMap.get(block_1.blockName); + if (pLatestBlock === undefined) { + throw StorageErrorFactory.getInvalidBlockList(context); + } else { + selectedBlockList.push(pLatestBlock); + } + break; + default: + throw StorageErrorFactory.getInvalidBlockList(context); + } + } + + if (doc) { + // Commit block list + doc.properties.blobType = blob.properties.blobType; + doc.properties.lastModified = blob.properties.lastModified; + doc.committedBlocksInOrder = selectedBlockList; + doc.isCommitted = true; + doc.metadata = blob.metadata; + doc.properties.accessTier = blob.properties.accessTier; + doc.properties.accessTierInferred = blob.properties.accessTierInferred; + doc.properties.etag = blob.properties.etag; + doc.properties.cacheControl = blob.properties.cacheControl; + doc.properties.contentType = blob.properties.contentType; + doc.properties.contentMD5 = blob.properties.contentMD5; + doc.properties.contentEncoding = blob.properties.contentEncoding; + doc.properties.contentLanguage = blob.properties.contentLanguage; + doc.properties.contentDisposition = blob.properties.contentDisposition; + doc.properties.contentLength = selectedBlockList + .map((block) => block.size) + .reduce((total, val) => { + return total + val; + }, 0); + + // set lease state to available if it's expired + if (lease) { + new BlobWriteLeaseSyncer(doc).sync(lease); + } + + coll.update(doc); + } else { + blob.committedBlocksInOrder = selectedBlockList; + blob.properties.contentLength = selectedBlockList + .map((block) => block.size) + .reduce((total, val) => { + return total + val; + }, 0); + coll.insert(blob); + } + + blockColl.findAndRemove({ + accountName: blob.accountName, + containerName: blob.containerName, + blobName: blob.name + }); + } + + /** + * Gets blocks list for a blob from persistency layer by account, container and blob names. + * + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {(string | undefined)} snapshot + * @param {(boolean | undefined)} isCommitted + * @param {Context} context + * @returns {Promise<{ + * properties: Models.BlobProperties; + * uncommittedBlocks: Models.Block[]; + * committedBlocks: Models.Block[]; + * }>} + * @memberof LokiBlobMetadataStore + */ + public async getBlockList( + context: Context, + account: string, + container: string, + blob: string, + snapshot: string | undefined, + isCommitted: boolean | undefined, + leaseAccessConditions: Models.LeaseAccessConditions | undefined + ): Promise<{ + properties: Models.BlobPropertiesInternal; + uncommittedBlocks: Models.Block[]; + committedBlocks: Models.Block[]; + }> { + const doc = await this.getBlobWithLeaseUpdated( + account, + container, + blob, + snapshot, + context + ); + + this.validateExpireConditions(context, doc, true); + new BlobReadLeaseValidator(leaseAccessConditions).validate( + new BlobLeaseAdapter(doc), + context + ); + + if (doc.properties.blobType !== Models.BlobType.BlockBlob) { + throw StorageErrorFactory.getBlobInvalidBlobType(context); + } + + const res: { + properties: Models.BlobPropertiesInternal; + uncommittedBlocks: Models.Block[]; + committedBlocks: Models.Block[]; + } = { + properties: doc.properties, + uncommittedBlocks: [], + committedBlocks: [] + }; + + if (isCommitted !== false && doc.committedBlocksInOrder !== undefined) { + res.committedBlocks = doc.committedBlocksInOrder; + } + + if (isCommitted !== true) { + const blockColl = this.db.getCollection(this.BLOCKS_COLLECTION); + const blocks = await blockColl + .chain() + .find({ + accountName: account, + containerName: container, + blobName: blob + }) + .simplesort("$loki") + .data(); + + for (const item of blocks) { + res.uncommittedBlocks.push(item); + } + } + + return res; + } + + /** + * Upload new pages for page blob. + * + * @param {Context} context + * @param {BlobModel} blob + * @param {number} start + * @param {number} end + * @param {IExtentChunk} persistency + * @param {Models.LeaseAccessConditions} [leaseAccessConditions] + * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] + * @param {Models.SequenceNumberAccessConditions} [sequenceNumberAccessConditions] + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async uploadPages( + context: Context, + blob: BlobModel, + start: number, + end: number, + persistency: IExtentChunk, + leaseAccessConditions?: Models.LeaseAccessConditions, + modifiedAccessConditions?: Models.ModifiedAccessConditions, + sequenceNumberAccessConditions?: Models.SequenceNumberAccessConditions + ): Promise { + const coll = this.db.getCollection(this.BLOBS_COLLECTION); + const doc = await this.getBlobWithLeaseUpdated( + blob.accountName, + blob.containerName, + blob.name, + blob.snapshot, + context!, + false, + true + ); + + this.validateExpireConditions(context, doc, true); + validateWriteConditions(context, modifiedAccessConditions, doc); + + validateSequenceNumberWriteConditions( + context, + sequenceNumberAccessConditions, + doc + ); + + if (!doc) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + if (doc.properties.blobType !== Models.BlobType.PageBlob) { + throw StorageErrorFactory.getBlobInvalidBlobType(context); + } + + const lease = new BlobLeaseAdapter(doc); + new BlobWriteLeaseValidator(leaseAccessConditions).validate(lease, context); + + this.pageBlobRangesManager.mergeRange(doc.pageRangesInOrder || [], { + start, + end, + persistency + }); + + // set lease state to available if it's expired + new BlobWriteLeaseSyncer(doc).sync(lease); + + doc.properties.etag = newEtag(); + doc.properties.lastModified = context.startTime || new Date(); + + coll.update(doc); + + return doc.properties; + } + + /** + * Clear range for a page blob. + * + * @param {Context} context + * @param {BlobModel} blob + * @param {number} start + * @param {number} end + * @param {Models.LeaseAccessConditions} [leaseAccessConditions] + * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] + * @param {Models.SequenceNumberAccessConditions} [sequenceNumberAccessConditions] + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async clearRange( + context: Context, + blob: BlobModel, + start: number, + end: number, + leaseAccessConditions?: Models.LeaseAccessConditions, + modifiedAccessConditions?: Models.ModifiedAccessConditions, + sequenceNumberAccessConditions?: Models.SequenceNumberAccessConditions + ): Promise { + const coll = this.db.getCollection(this.BLOBS_COLLECTION); + const doc = await this.getBlobWithLeaseUpdated( + blob.accountName, + blob.containerName, + blob.name, + blob.snapshot, + context!, + false, + true + ); + + this.validateExpireConditions(context, doc, true); + validateWriteConditions(context, modifiedAccessConditions, doc); + + validateSequenceNumberWriteConditions( + context, + sequenceNumberAccessConditions, + doc + ); + + if (!doc) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + const lease = new BlobLeaseAdapter(doc); + new BlobWriteLeaseValidator(leaseAccessConditions).validate(lease, context); + + this.pageBlobRangesManager.clearRange(doc.pageRangesInOrder || [], { + start, + end + }); + + // TODO: Check other blob update operations need lease reset or not + // set lease state to available if it's expired + new BlobWriteLeaseSyncer(doc).sync(lease); + + doc.properties.etag = newEtag(); + doc.properties.lastModified = context.startTime || new Date(); + + coll.update(doc); + + return doc.properties; + } + + /** + * Returns the list of valid page ranges for a page blob or snapshot of a page blob. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {string} [snapshot] + * @param {Models.LeaseAccessConditions} [leaseAccessConditions] + * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async getPageRanges( + context: Context, + account: string, + container: string, + blob: string, + snapshot?: string, + leaseAccessConditions?: Models.LeaseAccessConditions, + modifiedAccessConditions?: Models.ModifiedAccessConditions + ): Promise { + const doc = await this.getBlobWithLeaseUpdated( + account, + container, + blob, + snapshot, + context, + false, + true + ); + + this.validateExpireConditions(context, doc, true); + validateReadConditions(context, modifiedAccessConditions, doc); + + if (!doc) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + if (doc.properties.blobType !== Models.BlobType.PageBlob) { + throw StorageErrorFactory.getBlobInvalidBlobType(context); + } + + new BlobReadLeaseValidator(leaseAccessConditions).validate( + new BlobLeaseAdapter(doc), + context + ); + + return { + properties: doc.properties, + pageRangesInOrder: doc.pageRangesInOrder + }; + } + + /** + * Resize a page blob. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {number} blobContentLength + * @param {Models.LeaseAccessConditions} [leaseAccessConditions] + * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async resizePageBlob( + context: Context, + account: string, + container: string, + blob: string, + blobContentLength: number, + leaseAccessConditions?: Models.LeaseAccessConditions, + modifiedAccessConditions?: Models.ModifiedAccessConditions + ): Promise { + const coll = this.db.getCollection(this.BLOBS_COLLECTION); + const doc = await this.getBlobWithLeaseUpdated( + account, + container, + blob, + undefined, + context, + false, + true + ); + + this.validateExpireConditions(context, doc, true); + validateWriteConditions(context, modifiedAccessConditions, doc); + + if (!doc) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + if (doc.properties.blobType !== Models.BlobType.PageBlob) { + throw StorageErrorFactory.getInvalidOperation( + context, + "Resize could only be against a page blob." + ); + } + + const lease = new BlobLeaseAdapter(doc); + new BlobWriteLeaseValidator(leaseAccessConditions).validate(lease, context); + + doc.pageRangesInOrder = doc.pageRangesInOrder || []; + if (doc.properties.contentLength! > blobContentLength) { + const start = blobContentLength; + const end = doc.properties.contentLength! - 1; + this.pageBlobRangesManager.clearRange(doc.pageRangesInOrder || [], { + start, + end + }); + } + + doc.properties.contentLength = blobContentLength; + doc.properties.lastModified = context.startTime || new Date(); + doc.properties.etag = newEtag(); + + new BlobWriteLeaseSyncer(doc).sync(lease); + + coll.update(doc); + return doc.properties; + } + + /** + * Update the sequence number of a page blob. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {Models.SequenceNumberActionType} sequenceNumberAction + * @param {(number | undefined)} blobSequenceNumber + * @param {Models.LeaseAccessConditions} [leaseAccessConditions] + * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async updateSequenceNumber( + context: Context, + account: string, + container: string, + blob: string, + sequenceNumberAction: Models.SequenceNumberActionType, + blobSequenceNumber: number | undefined, + leaseAccessConditions?: Models.LeaseAccessConditions, + modifiedAccessConditions?: Models.ModifiedAccessConditions + ): Promise { + const coll = this.db.getCollection(this.BLOBS_COLLECTION); + const doc = await this.getBlobWithLeaseUpdated( + account, + container, + blob, + undefined, + context, + false, + true + ); + + this.validateExpireConditions(context, doc, true); + validateWriteConditions(context, modifiedAccessConditions, doc); + + if (!doc) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + if (doc.properties.blobType !== Models.BlobType.PageBlob) { + throw StorageErrorFactory.getInvalidOperation( + context, + "Get Page Ranges could only be against a page blob." + ); + } + + const lease = new BlobLeaseAdapter(doc); + new BlobWriteLeaseValidator(leaseAccessConditions).validate(lease, context); + + if (doc.properties.blobSequenceNumber === undefined) { + doc.properties.blobSequenceNumber = 0; + } + + switch (sequenceNumberAction) { + case Models.SequenceNumberActionType.Max: + if (blobSequenceNumber === undefined) { + throw StorageErrorFactory.getInvalidOperation( + context, + "x-ms-blob-sequence-number is required when x-ms-sequence-number-action is set to max." + ); + } + doc.properties.blobSequenceNumber = Math.max( + doc.properties.blobSequenceNumber, + blobSequenceNumber + ); + break; + case Models.SequenceNumberActionType.Increment: + if (blobSequenceNumber !== undefined) { + throw StorageErrorFactory.getInvalidOperation( + context, + "x-ms-blob-sequence-number cannot be provided when x-ms-sequence-number-action is set to increment." + ); + } + doc.properties.blobSequenceNumber++; + break; + case Models.SequenceNumberActionType.Update: + if (blobSequenceNumber === undefined) { + throw StorageErrorFactory.getInvalidOperation( + context, + "x-ms-blob-sequence-number is required when x-ms-sequence-number-action is set to update." + ); + } + doc.properties.blobSequenceNumber = blobSequenceNumber; + break; + default: + throw StorageErrorFactory.getInvalidOperation( + context, + "Unsupported x-ms-sequence-number-action value." + ); + } + + doc.properties.etag = newEtag(); + doc.properties.lastModified = context.startTime!; + new BlobWriteLeaseSyncer(doc).sync(lease); + + coll.update(doc); + return doc.properties; + } + + public async listUncommittedBlockPersistencyChunks( + marker: string = "-1", + maxResults: number = 2000 + ): Promise<[IExtentChunk[], string | undefined]> { + const coll = this.db.getCollection(this.BLOCKS_COLLECTION); + let blockDocs = coll + .chain() + .where((obj) => { + return obj.$loki > parseInt(marker, 10); + }) + .simplesort("$loki") + .limit(maxResults + 1) + .data(); + + blockDocs = blockDocs.filter((model) => + this.validateExpireConditions(undefined, model, false) + ); + if (blockDocs.length <= maxResults) { + return [blockDocs.map((block) => block.persistency), undefined]; + } else { + blockDocs.pop(); + const nextMarker = `${blockDocs[maxResults - 1].$loki}`; + return [blockDocs.map((block) => block.persistency), nextMarker]; + } + } + + /** + * LokiJS will persist Uint8Array into Object. + * This method will restore object to Uint8Array. + * + * @private + * @param {*} obj + * @returns {(Uint8Array | undefined)} + * @memberof LokiBlobMetadataStore + */ + private restoreUint8Array(obj: any): Uint8Array | undefined { + if (typeof obj !== "object") { + return undefined; + } + + if (obj instanceof Uint8Array) { + return obj; + } + + if (obj.type === "Buffer") { + obj = obj.data; + } + + const length = Object.keys(obj).length; + const arr = Buffer.allocUnsafe(length); + + for (let i = 0; i < length; i++) { + if (!obj.hasOwnProperty(i)) { + throw new TypeError( + `Cannot restore loki DB persisted object to Uint8Array. Key ${i} is missing.` + ); + } + + arr[i] = obj[i]; + } + + return arr; + } + + /** + * Escape a string to be used as a regex. + * + * @private + * @param {string} regex + * @returns {string} + * @memberof LokiBlobMetadataStore + */ + private escapeRegex(regex: string): string { + return regex.replace(/[-\/\\^$*+?.()|[\]{}]/g, "\\$&"); + } + + /** + * Get a container document from container collection. + * Updated lease related properties according to current time. + * + * @private + * @param {string} account + * @param {string} container + * @param {Context} context + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + + /** + * Get a container document from container collection. + * Updated lease related properties according to current time. + * Will throw ContainerNotFound storage error if container doesn't exist. + * + * @private + * @param {string} account + * @param {string} container + * @param {Context} context + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + private async getContainerWithLeaseUpdated( + account: string, + container: string, + context: Context, + forceExist?: true + ): Promise; + + /** + * Get a container document from container collection. + * Updated lease related properties according to current time. + * Will NOT throw ContainerNotFound storage error if container doesn't exist. + * + * @private + * @param {string} account + * @param {string} container + * @param {Context} context + * @param {false} forceExist + * @returns {(Promise)} + * @memberof LokiBlobMetadataStore + */ + private async getContainerWithLeaseUpdated( + account: string, + container: string, + context: Context, + forceExist: false + ): Promise; + + private async getContainerWithLeaseUpdated( + account: string, + container: string, + context: Context, + forceExist?: boolean + ): Promise { + const coll = this.db.getCollection(this.CONTAINERS_COLLECTION); + const doc = coll.findOne({ accountName: account, name: container }); + + if (forceExist === undefined || forceExist === true) { + if (!doc) { + throw StorageErrorFactory.getContainerNotFound(context); + } + } + + if (!doc) { + return undefined; + } + + LeaseFactory.createLeaseState(new ContainerLeaseAdapter(doc), context).sync( + new ContainerLeaseSyncer(doc) + ); + + return doc; + } + + /** + * Get a container document from Loki collections. + * Will throw ContainerNotFound error when container doesn't exist. + * + * @private + * @param {string} account + * @param {string} container + * @param {Context} context + * @param {true} [forceExist] + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + private async getContainer( + account: string, + container: string, + context: Context, + forceExist?: true + ): Promise; + + /** + * Get a container document from Loki collections. + * Will NOT throw ContainerNotFound error when container doesn't exist. + * + * @private + * @param {string} account + * @param {string} container + * @param {Context} context + * @param {false} forceExist + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + private async getContainer( + account: string, + container: string, + context: Context, + forceExist: false + ): Promise; + + private async getContainer( + account: string, + container: string, + context: Context, + forceExist?: boolean + ): Promise { + const coll = this.db.getCollection(this.CONTAINERS_COLLECTION); + const doc = coll.findOne({ accountName: account, name: container }); + + if (!doc) { + if (forceExist) { + throw StorageErrorFactory.getContainerNotFound(context); + } else { + return undefined; + } + } + + return doc; + } + + /** + * Get a blob document model from Loki collection. + * Will throw BlobNotFound storage error if blob doesn't exist. + * + * @protected + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {(string | undefined)} snapshot + * @param {Context} context + * @param {undefined} [forceExist] + * @param {boolean} [forceCommitted] If true, will take uncommitted blob as a non-exist blob and throw exception. + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + private async getBlobWithLeaseUpdated( + account: string, + container: string, + blob: string, + snapshot: string | undefined, + context: Context, + forceExist?: true, + forceCommitted?: boolean + ): Promise; + + /** + * Get a blob document model from Loki collection. + * Will NOT throw BlobNotFound storage error if blob doesn't exist. + * + * @protected + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {(string | undefined)} snapshot + * @param {Context} context + * @param {false} forceExist + * @param {boolean} [forceCommitted] If true, will take uncommitted blob as a non-exist blob and return undefined. + * @returns {(Promise)} + * @memberof LokiBlobMetadataStore + */ + private async getBlobWithLeaseUpdated( + account: string, + container: string, + blob: string, + snapshot: string | undefined, + context: Context, + forceExist: false, + forceCommitted?: boolean + ): Promise; + + private async getBlobWithLeaseUpdated( + account: string, + container: string, + blob: string, + snapshot: string = "", + context: Context, + forceExist?: boolean, + forceCommitted?: boolean + ): Promise { + await this.checkContainerExist(context, account, container); + + const coll = this.db.getCollection(this.BLOBS_COLLECTION); + const doc = coll.findOne({ + accountName: account, + containerName: container, + name: blob, + snapshot + }); + + // Force exist if parameter forceExist is undefined or true + if (forceExist === undefined || forceExist === true) { + if (forceCommitted) { + if (!doc || !(doc as BlobModel).isCommitted) { + throw StorageErrorFactory.getBlobNotFound(context); + } + } else { + if (!doc) { + throw StorageErrorFactory.getBlobNotFound(context); + } + } + } else { + if (forceCommitted) { + if (!doc || !(doc as BlobModel).isCommitted) { + return undefined; + } + } else { + if (!doc) { + return undefined; + } + } + } + + if (doc.properties) { + doc.properties.contentMD5 = this.restoreUint8Array( + doc.properties.contentMD5 + ); + } + + // Snapshot doesn't have lease + if (snapshot !== undefined && snapshot !== "") { + new BlobLeaseSyncer(doc).sync({ + leaseId: undefined, + leaseExpireTime: undefined, + leaseDurationSeconds: undefined, + leaseBreakTime: undefined, + leaseDurationType: undefined, + leaseState: Models.LeaseStateType.Available, // TODO: Lease state & status should be undefined for snapshots + leaseStatus: Models.LeaseStatusType.Unlocked // TODO: Lease state & status should be undefined for snapshots + }); + } else { + LeaseFactory.createLeaseState(new BlobLeaseAdapter(doc), context).sync( + new BlobLeaseSyncer(doc) + ); + } + + return doc; + } + + /** + * Get the tier setting from request headers. + * + * @private + * @param {string} tier + * @returns {(Models.AccessTier | undefined)} + * @memberof BlobHandler + */ + private parseTier(tier: string): Models.AccessTier | undefined { + tier = tier.toLowerCase(); + if (tier === Models.AccessTier.Hot.toLowerCase()) { + return Models.AccessTier.Hot; + } + if (tier === Models.AccessTier.Cool.toLowerCase()) { + return Models.AccessTier.Cool; + } + if (tier === Models.AccessTier.Archive.toLowerCase()) { + return Models.AccessTier.Archive; + } + return undefined; + } + + /*************************************************************************** + * + * New DataLake specific functions + * + ***************************************************************************/ + + /** + * List paths in Directory + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} directory + * @param {boolean} recursive + * @param {Models.FileSystemListPathsOptionalParams} options + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + async listPaths( + context: Context, + account: string, + container: string, + directory: string, + recursive: boolean, + options: Models.FileSystemListPathsOptionalParams + ): Promise<[Models.Path[], string | undefined]> { + directory = removeSlash(directory); + const paths: Models.Path[] = []; + + if (directory) + await this.getModel(context, account, container, directory, true); + const [blobs, , nextMarker] = await this.list( + undefined, + context, + account, + container, + recursive ? undefined : "/", + undefined, + directory ? directory + "/" : directory, + options.maxResults, + options.continuation + ); + blobs.forEach((blob) => { + paths.push({ + name: blob.name, + isDirectory: blob.isDirectory ? true : undefined, + lastModified: blob.properties.lastModified, + etag: blob.properties.etag, + contentLength: blob.properties.contentLength, + owner: blob.owner, + group: blob.group, + permissions: toPermissionsString(blob.permissions) + }); + }); + + return [paths, nextMarker]; + } + + /** + * Rename Directory + * + * @param {Context} context + * @param {string} account + * @param {string} sourceContainer + * @param {string} sourceDirectory + * @param {string} targetContainer + * @param {string} targetDirectory + * @param {Models.DirectoryRenameOptionalParams} options + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + public async renameDirectory( + context: Context, + account: string, + sourceContainer: string, + sourceDirectory: string, + targetContainer: string, + targetDirectory: string, + options: Models.PathCreateOptionalParams + ): Promise { + sourceDirectory = removeSlash(sourceDirectory); + targetDirectory = removeSlash(targetDirectory); + + const dirModel = await this.getModel( + context, + account, + targetContainer, + targetDirectory, + false + ); + + if ( + options.modifiedAccessConditions && + options.modifiedAccessConditions.ifNoneMatch === "*" && + dirModel + ) { + throw StorageErrorFactory.getBlobAlreadyExists(context); + } + + const [paths] = await this.listPaths( + context, + account, + sourceContainer, + sourceDirectory, + true, + options + ); + + paths.forEach(async (path) => { + await this.rename( + path.isDirectory!, + context, + account, + sourceContainer, + path.name!, + targetContainer, + targetDirectory + path.name!.substring(sourceDirectory.length), + options + ); + }); + + await this.rename( + true, + context, + account, + sourceContainer, + sourceDirectory, + targetContainer, + targetDirectory, + options + ); + const res = await this.getModel( + context, + account, + targetContainer, + targetDirectory, + true + ); + return res!; + } + + /** + * Delete Directory + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} directory + * @param {boolean} recursiveDirectoryDelete + * @param {Models.DirectoryDeleteMethodOptionalParams} options + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + public async deleteDirectory( + context: Context, + account: string, + container: string, + directory: string, + recursiveDirectoryDelete: boolean, + options: Models.PathDeleteMethodOptionalParams + ): Promise { + directory = removeSlash(directory); + await this.checkContainerExist(context, account, container); + + const dir: BlobModel = await this.db + .getCollection(this.BLOBS_COLLECTION) + .findOne({ + accountName: account, + containerName: container, + name: directory + }); + + validateWriteConditions(context, options.modifiedAccessConditions, dir); + + if (dir === null || dir === undefined) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + const [blobs] = await this.list( + undefined, + context, + account, + container, + undefined, + undefined, + directory ? directory + "/" : directory, + DEFAULT_LIST_BLOBS_MAX_RESULTS, + undefined, + true, + true + ); + + if (blobs.length > 0 && !recursiveDirectoryDelete) { + throw StorageErrorFactory.getInvalidOperation( + context, + "Can't Delete non empty folder with non recursive flag" + ); + } else { + const options: Models.PathDeleteMethodOptionalParams = { + deleteSnapshots: Models.DeleteSnapshotsOptionType.Include + }; + + for (const blob of blobs) { + await this.deleteBlob(context, account, container, blob.name, options); + } + } + + await this.deleteBlob(context, account, container, dir.name, options); + } + + /** + * Rename Blob + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} source + * @param {string} target + * @param {Models.BlobDeleteMethodOptionalParams} options + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async renameBlob( + context: Context, + account: string, + sourceContainer: string, + sourceBlob: string, + targetContainer: string, + targetBlob: string, + options: Models.PathCreateOptionalParams + ): Promise { + return this.rename( + false, + context, + account, + sourceContainer, + sourceBlob, + targetContainer, + targetBlob, + options + ); + } + + public async rename( + isDirectory: boolean, + context: Context, + account: string, + sourceContainer: string, + sourceBlob: string, + targetContainer: string, + targetBlob: string, + options: Models.PathCreateOptionalParams + ): Promise { + const target = await this.getModel( + context, + account, + targetContainer, + targetBlob, + false, + options.leaseAccessConditions, + options.modifiedAccessConditions, + false + ); + + validateWriteConditions(context, options.modifiedAccessConditions, target); + + if ( + options.modifiedAccessConditions && + options.modifiedAccessConditions.ifNoneMatch === "*" && + target + ) { + throw StorageErrorFactory.getBlobAlreadyExists(context); + } + + if (target) { + isDirectory + ? await this.deleteDirectory( + context, + account, + targetContainer, + targetBlob, + true, + options + ) + : await this.deleteBlob( + context, + account, + targetContainer, + targetBlob, + options + ); + } + + const model = await this.getModel( + context, + account, + sourceContainer, + sourceBlob + ); + + const newModel: BlobModel = { + ...model, + containerName: targetContainer, + name: targetBlob + }; + + await this.createBlob( + context, + newModel, + options.leaseAccessConditions, + options.modifiedAccessConditions + ); + + const coll = this.db.getCollection(this.BLOBS_COLLECTION); + coll.findAndRemove({ + accountName: account, + containerName: sourceContainer, + name: sourceBlob + }); + return model; + } + + /** + * Gets a blob item from persistency layer by container name and blob name. + * Will return block list or page list as well for downloading. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {string} [snapshot=""] + * @param {Models.LeaseAccessConditions} [leaseAccessConditions] + * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + getModel( + context: Context, + account: string, + container: string, + blob: string, + forceExist?: true, + leaseAccessConditions?: Models.LeaseAccessConditions, + modifiedAccessConditions?: Models.ModifiedAccessConditions, + validateRead?: boolean + ): Promise; + + getModel( + context: Context, + account: string, + container: string, + blob: string, + forceExist: false, + leaseAccessConditions?: Models.LeaseAccessConditions, + modifiedAccessConditions?: Models.ModifiedAccessConditions, + validateRead?: boolean + ): Promise; + + public async getModel( + context: Context, + account: string, + container: string, + blob: string, + forceExists: boolean = true, + leaseAccessConditions?: Models.LeaseAccessConditions, + modifiedAccessConditions?: Models.ModifiedAccessConditions, + validateRead: boolean = true + ): Promise { + const doc = await this.getBlobWithLeaseUpdated( + account, + container, + blob, + "", + context, + false, + true + ); + + if (validateRead) { + validateReadConditions(context, modifiedAccessConditions, doc); + } + + if (!doc) { + if (!forceExists) return undefined; + throw StorageErrorFactory.getBlobNotFound(context); + } + + new BlobReadLeaseValidator(leaseAccessConditions).validate( + new BlobLeaseAdapter(doc), + context + ); + + return doc; + } + + private validateExpireConditions( + context: Context | undefined, + model: BlobModel | undefined, + throwError: boolean + ): boolean { + const now = new Date(); + if ( + model && + model.properties.expiresOn !== undefined && + model.properties.expiresOn < now + ) { + const coll = this.db.getCollection(this.BLOBS_COLLECTION); + coll.remove(model); + if (throwError && context !== undefined) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + return false; + } + + return true; + } +} diff --git a/src/dfs/persistence/PageWithDelimiter.ts b/src/dfs/persistence/PageWithDelimiter.ts new file mode 100644 index 000000000..aa7df251c --- /dev/null +++ b/src/dfs/persistence/PageWithDelimiter.ts @@ -0,0 +1,199 @@ +import { BlobPrefixModel } from "./IDataLakeMetadataStore"; + +/** + * This implements a page of blob results taking delimiters into account. + * + * When a delimiter is passed to list blobs, items must be squashed into BlobPrefix items. + * When maxResults is smaller than the number of prefixed items in the metadata source, multiple reads from + * the source may be necessary. + * + * @export + * @class PageWithDelimiter + */ +export default class PageWithDelimiter { + readonly delimiter: string | undefined; + readonly maxResults: number; + readonly prefix: string | undefined; + readonly prefixLength: number = 0; + + blobItems: BlobType[] = []; + blobPrefixes: Set = new Set(); + latestMarker: string = ""; + + // isFull indicates we could only (maybe) add a prefix + private isFull: boolean = false; + + // isExhausted indicates nothing more should be added + private isExhausted: boolean = false; + + constructor(maxResults: number, delimiter?: string, prefix?: string) { + this.maxResults = maxResults; + if (delimiter !== undefined) { + this.delimiter = delimiter; + } + if (prefix !== undefined) { + this.prefix = prefix; + this.prefixLength = prefix.length; + } + } + + /** + * Empty the page (useful in unit tests) + * + */ + public reset() { + this.blobItems.splice(0); + this.blobPrefixes.clear(); + this.isFull = false; + this.isExhausted = false; + this.latestMarker = ""; + } + + private updateFull() { + this.isFull = + this.blobItems.length + this.blobPrefixes.size === this.maxResults; + } + + /** + * addItem will add to the blob list if possible and update the full/exhausted state of the page + */ + private addItem(item: BlobType): boolean { + if (this.isExhausted) { + return false; + } + let added: boolean = false; + if (!this.isFull) { + this.blobItems.push(item); + added = true; + } + this.updateFull(); + + // if a blob causes fullness the next item read cannot be squashed only duplicate prefixes can + this.isExhausted = this.isFull; + return added; + } + + /** + * addItem will add to the blob list if possible and update the full/exhausted state of the page + */ + private addPrefix(prefix: string): boolean { + if (this.isExhausted) { + return false; + } + let added: boolean = false; + if (this.isFull) { + // the page is exhausted if this prefix is new, only matching prefixes may be 'added' + this.isExhausted = !this.blobPrefixes.has(prefix); + added = !this.isExhausted; + } else { + this.blobPrefixes.add(prefix); + added = true; + } + this.updateFull(); + return added; + } + + /** + * Add a BlobType item to the appropriate collection, update the marker + * + * If no delimiter is used, items are all treated as blobs until maxResults is reached + * + * If a delimiter is used, the name will be checked for to see if the item should + * be treated as a blob or a BlobPrefix. + * + * When the page becomes full, items may still be added iff the item is existing prefix + * + * Return the number of items added + */ + private add(name: string, item: BlobType): boolean { + if (this.isExhausted) { + return false; + } + if (name < this.latestMarker) { + throw new Error( + "add received unsorted item. add must be called on sorted data" + ); + } + const marker = name > this.latestMarker ? name : this.latestMarker; + let added: boolean = false; + if (this.delimiter !== undefined) { + const delimiterPosAfterPrefix = name.indexOf( + this.delimiter, + this.prefixLength + ); + + if (delimiterPosAfterPrefix < 0) { + added = this.addItem(item); + } else { + const prefix = name.substr(0, delimiterPosAfterPrefix + 1); + added = this.addPrefix(prefix); + } + } else { + added = this.addItem(item); + } + if (added) { + this.latestMarker = marker; + } + return added; + } + + /** + * Iterate over an array blobs read from a source and add them until the page cannot accept new items + */ + private processList( + docs: BlobType[], + nameFn: (item: BlobType) => string + ): number { + let added: number = 0; + for (const item of docs) { + if (this.add(nameFn(item), item)) { + added++; + } + if (this.isExhausted) break; + } + return added; + } + + /** + * Fill the page if possible by using the provided reader function. + * + * For any BlobType, the name is used with delimiter to treat the item as a blob or + * a BlobPrefix for the list blobs result. + * + * This function will use the reader for BlobType to keep reading from a metadata + * data source until the source has no more items or the page cannot add any more items. + * + * Return the contents of the page, blobs, prefixes, and a continuation token if applicable + */ + public async fill( + reader: (offset: number) => Promise, + namer: (item: BlobType) => string + ): Promise<[BlobType[], BlobPrefixModel[], string]> { + let offset: number = 0; + let docs = await reader(offset); + let added: number = 0; + while (docs.length) { + added = this.processList(docs, namer); + offset += added; + if (added < this.maxResults) { + break; + } + docs = await reader(offset); + } + return [ + this.blobItems, + this.prefixes(), + added < docs.length ? this.latestMarker : "" + ]; + } + + private prefixes(): BlobPrefixModel[] { + const prefixes: BlobPrefixModel[] = []; + const iter = this.blobPrefixes.values(); + let val; + while (!(val = iter.next()).done) { + prefixes.push({ name: val.value }); + } + return prefixes; + } +} diff --git a/src/dfs/persistence/SqlDataLakeMetadataStore.ts b/src/dfs/persistence/SqlDataLakeMetadataStore.ts new file mode 100644 index 000000000..2c01a0671 --- /dev/null +++ b/src/dfs/persistence/SqlDataLakeMetadataStore.ts @@ -0,0 +1,4279 @@ +import { + BOOLEAN, + DATE, + INTEGER, + Model, + Op, + Options as SequelizeOptions, + Sequelize, + Transaction, + TEXT, + ModelIndexesOptions +} from "sequelize"; + +import uuid from "uuid/v4"; + +import { + DEFAULT_SQL_CHARSET, + DEFAULT_SQL_COLLATE +} from "../../common/utils/constants"; +import { convertDateTimeStringMsTo7Digital } from "../../common/utils/utils"; +import { newEtag } from "../../common/utils/utils"; +import { validateReadConditions } from "../conditions/ReadConditionalHeadersValidator"; +import { validateWriteConditions } from "../conditions/WriteConditionalHeadersValidator"; +import StorageErrorFactory from "../errors/StorageErrorFactory"; +import * as Models from "../generated/artifacts/models"; +import Context from "../generated/Context"; +import BlobLeaseAdapter from "../lease/BlobLeaseAdapter"; +import BlobLeaseSyncer from "../lease/BlobLeaseSyncer"; +import BlobReadLeaseValidator from "../lease/BlobReadLeaseValidator"; +import BlobWriteLeaseSyncer from "../lease/BlobWriteLeaseSyncer"; +import BlobWriteLeaseValidator from "../lease/BlobWriteLeaseValidator"; +import ContainerDeleteLeaseValidator from "../lease/ContainerDeleteLeaseValidator"; +import ContainerLeaseAdapter from "../lease/ContainerLeaseAdapter"; +import ContainerLeaseSyncer from "../lease/ContainerLeaseSyncer"; +import ContainerReadLeaseValidator from "../lease/ContainerReadLeaseValidator"; +import { ILease } from "../lease/ILeaseState"; +import LeaseFactory from "../lease/LeaseFactory"; +import { + DEFAULT_FILE_PERMISSIONS, + DEFAULT_GROUP, + DEFAULT_LIST_BLOBS_MAX_RESULTS, + DEFAULT_LIST_CONTAINERS_MAX_RESULTS, + DEFAULT_OWNER, + MAX_APPEND_BLOB_BLOCK_COUNT +} from "../utils/constants"; +import BlobReferredExtentsAsyncIterator from "./BlobReferredExtentsAsyncIterator"; +import IDataLakeMetadataStore, { + AcquireBlobLeaseResponse, + AcquireContainerLeaseResponse, + BlobId, + BlobModel, + BlobPrefixModel, + BlockModel, + BreakBlobLeaseResponse, + BreakContainerLeaseResponse, + ChangeBlobLeaseResponse, + ChangeContainerLeaseResponse, + ContainerModel, + CreateSnapshotResponse, + GetBlobPropertiesRes, + GetContainerAccessPolicyResponse, + GetPageRangeResponse, + IContainerMetadata, + IExtentChunk, + PersistencyBlockModel, + ReleaseBlobLeaseResponse, + RenewBlobLeaseResponse, + RenewContainerLeaseResponse, + ServicePropertiesModel, + SetContainerAccessPolicyOptions +} from "./IDataLakeMetadataStore"; +import PageWithDelimiter from "./PageWithDelimiter"; +import { removeSlash } from "../utils/utils"; +import NotImplementedError from "../errors/NotImplementedError"; +import { + toAcl, + toAclString, + toPermissions, + toPermissionsString +} from "../storage-file-datalake/transforms"; + +// tslint:disable: max-classes-per-file +class ServicesModel extends Model {} +class ContainersModel extends Model {} +class BlobsModel extends Model {} +class BlocksModel extends Model {} +// class PagesModel extends Model {} + +/** + * A SQL based Blob metadata storage implementation based on Sequelize. + * Refer to CONTRIBUTION.md for how to setup SQL database environment. + * + * @export + * @class SqlDataLakeMetadataStore + * @implements {IBlobMetadataStore} + */ +export default class SqlDataLakeMetadataStore + implements IDataLakeMetadataStore +{ + private initialized: boolean = false; + private closed: boolean = false; + private readonly sequelize: Sequelize; + private readonly isPostgres: boolean; + + /** + * Creates an instance of SqlBlobMetadataStore. + * + * @param {string} connectionURI For example, "postgres://user:pass@example.com:5432/dbname" + * @param {SequelizeOptions} [sequelizeOptions] + * @memberof SqlBlobMetadataStore + */ + public constructor( + connectionURI: string, + sequelizeOptions?: SequelizeOptions, + private readonly clearDB: boolean = false + ) { + // Enable encrypt connection for SQL Server + if (connectionURI.startsWith("mssql") && sequelizeOptions) { + sequelizeOptions.dialectOptions = sequelizeOptions.dialectOptions || {}; + (sequelizeOptions.dialectOptions as any).options = + (sequelizeOptions.dialectOptions as any).options || {}; + (sequelizeOptions.dialectOptions as any).options.encrypt = true; + } + this.isPostgres = connectionURI.startsWith("postgres"); + this.sequelize = new Sequelize(connectionURI, sequelizeOptions); + } + + public async init(): Promise { + await this.sequelize.authenticate(); + + ServicesModel.init( + { + accountName: { + type: "VARCHAR(255)", + primaryKey: true + }, + defaultServiceVersion: { + type: "VARCHAR(10)" + }, + cors: { + type: "VARCHAR(4095)" + }, + logging: { + type: "VARCHAR(255)" + }, + minuteMetrics: { + type: "VARCHAR(255)" + }, + hourMetrics: { + type: "VARCHAR(255)" + }, + staticWebsite: { + type: "VARCHAR(1023)" + }, + deleteRetentionPolicy: { + type: "VARCHAR(255)" + } + }, + { + sequelize: this.sequelize, + modelName: "Services", + tableName: "Services", + timestamps: false + } + ); + + ContainersModel.init( + { + accountName: { + type: "VARCHAR(255)", + unique: "accountname_containername" + }, + // tslint:disable-next-line:max-line-length + // https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata + containerName: { + type: "VARCHAR(255)", + unique: "accountname_containername" + }, + containerId: { + type: INTEGER.UNSIGNED, + primaryKey: true, + autoIncrement: true + }, + lastModified: { + allowNull: false, + type: DATE(6) + }, + etag: { + allowNull: false, + type: "VARCHAR(127)" + }, + // TODO: Confirm max length of metadata pairs + metadata: { + type: "VARCHAR(4095)" + }, + properties: { + type: "VARCHAR(4095)" + }, + containerAcl: { + type: "VARCHAR(1023)" + }, + publicAccess: { + type: "VARCHAR(31)" + }, + lease: { + type: "VARCHAR(1023)" + }, + hasImmutabilityPolicy: { + type: BOOLEAN + }, + hasLegalHold: { + type: BOOLEAN + } + }, + { + sequelize: this.sequelize, + modelName: "Containers", + tableName: "Containers", + charset: DEFAULT_SQL_CHARSET, + collate: DEFAULT_SQL_COLLATE, + timestamps: false + } + ); + + const blobIndexes: ModelIndexesOptions[] = this.isPostgres + ? [ + { + unique: true, + fields: ["accountName", "containerName", "blobName", "snapshot"] + } + ] + : [ + { + unique: false, + fields: ["accountName", "containerName"] + } + ]; + + BlobsModel.init( + { + accountName: { + type: "VARCHAR(255)", + allowNull: false + }, + containerName: { + type: "VARCHAR(255)", + allowNull: false + }, + blobName: { + type: this.isPostgres ? "VARCHAR(65535)" : TEXT("medium"), + allowNull: false + }, + snapshot: { + type: "VARCHAR(64)", + allowNull: false, + defaultValue: "" + }, + blobId: { + type: INTEGER.UNSIGNED, + primaryKey: true, + autoIncrement: true + }, + lastModified: { + allowNull: false, + type: DATE(6) + }, + creationTime: { + allowNull: false, + type: DATE(6) + }, + committedBlocksInOrder: { + type: this.isPostgres ? "VARCHAR(65535)" : TEXT("medium"), + allowNull: true + }, + isCommitted: { + type: BOOLEAN, + allowNull: false + }, + isDirectory: { + type: BOOLEAN, + allowNull: false + }, + permissions: { + type: "VARCHAR(255)", + allowNull: false + }, + acl: { + type: "VARCHAR(255)", + allowNull: false + }, + owner: { + type: "VARCHAR(255)", + allowNull: false + }, + group: { + type: "VARCHAR(255)", + allowNull: false + }, + properties: { + type: this.isPostgres ? "VARCHAR(65535)" : TEXT("medium"), + allowNull: false + }, + leaseBreakTime: { + allowNull: true, + type: DATE(6) + }, + leaseDurationSeconds: { + allowNull: true, + type: INTEGER.UNSIGNED + }, + leaseExpireTime: { + allowNull: true, + type: DATE(6) + }, + leaseId: { + type: "VARCHAR(255)", + allowNull: true + }, + metadata: { + type: "VARCHAR(2047)", + allowNull: true + }, + //objectReplicationMetadata + //pageRangesInOrder + persistency: { + type: "VARCHAR(255)" + }, + versionId: { + type: "VARCHAR(64)", + allowNull: true + } + }, + { + sequelize: this.sequelize, + modelName: "Blobs", + tableName: "Blobs", + timestamps: false, + charset: DEFAULT_SQL_CHARSET, + collate: DEFAULT_SQL_COLLATE, + indexes: blobIndexes + } + ); + + const blockIndexes: ModelIndexesOptions[] = this.isPostgres + ? [ + { + unique: true, + fields: ["accountName", "containerName", "blobName", "blockName"] + } + ] + : [ + { + unique: false, + fields: ["accountName", "containerName"] + } + ]; + + BlocksModel.init( + { + accountName: { + type: "VARCHAR(255)", + allowNull: false + }, + containerName: { + type: "VARCHAR(255)", + allowNull: false + }, + blobName: { + type: this.isPostgres ? "VARCHAR(65535)" : TEXT("medium"), + allowNull: false + }, + // TODO: Check max block name length + blockName: { + type: this.isPostgres ? "VARCHAR(65535)" : TEXT("medium"), + allowNull: false + }, + size: { + type: INTEGER.UNSIGNED, + allowNull: false + }, + persistency: { + type: "VARCHAR(255)" + } + }, + { + sequelize: this.sequelize, + modelName: "Blocks", + tableName: "Blocks", + timestamps: false, + indexes: blockIndexes + } + ); + + if (this.clearDB) { + await this.sequelize.sync({ force: true }); + } else { + // TODO: sync() is only for development purpose, use migration for production + await this.sequelize.sync(); + } + + this.initialized = true; + } + + public isInitialized(): boolean { + return this.initialized; + } + + public async close(): Promise { + await this.sequelize.close(); + this.closed = true; + } + + public isClosed(): boolean { + return this.closed; + } + + public async clean(): Promise { + // TODO: Implement cleanup in database + } + + public async setServiceProperties( + context: Context, + serviceProperties: ServicePropertiesModel + ): Promise { + return await this.sequelize.transaction(async (t) => { + const findResult = await ServicesModel.findByPk( + serviceProperties.accountName, + { + transaction: t + } + ); + const updateValues = { + defaultServiceVersion: serviceProperties.defaultServiceVersion, + cors: this.serializeModelValue(serviceProperties.cors), + logging: this.serializeModelValue(serviceProperties.logging), + minuteMetrics: this.serializeModelValue( + serviceProperties.minuteMetrics + ), + hourMetrics: this.serializeModelValue(serviceProperties.hourMetrics), + staticWebsite: this.serializeModelValue( + serviceProperties.staticWebsite + ), + deleteRetentionPolicy: this.serializeModelValue( + serviceProperties.deleteRetentionPolicy + ) + }; + if (findResult === null) { + await ServicesModel.create( + { + accountName: serviceProperties.accountName, + ...updateValues + }, + { transaction: t } + ); + } else { + const updateResult = await ServicesModel.update(updateValues, { + transaction: t, + where: { + accountName: serviceProperties.accountName + } + }); + + // Set the exactly equal properties will affect 0 rows. + const updatedRows = updateResult[0]; + if (updatedRows > 1) { + throw Error( + `SqlBlobMetadataStore:updateServiceProperties() failed. Update operation affect ${updatedRows} rows.` + ); + } + } + + return serviceProperties; + }); + } + + public async getServiceProperties( + context: Context, + account: string + ): Promise { + const findResult = await ServicesModel.findByPk(account); + if (findResult === null) { + return undefined; + } + + const logging = this.deserializeModelValue( + findResult, + "logging" + ); + const hourMetrics = this.deserializeModelValue( + findResult, + "hourMetrics" + ); + const minuteMetrics = this.deserializeModelValue( + findResult, + "minuteMetrics" + ); + const cors = this.deserializeModelValue( + findResult, + "cors" + ); + const deleteRetentionPolicy = + this.deserializeModelValue( + findResult, + "deleteRetentionPolicy" + ); + const staticWebsite = this.deserializeModelValue( + findResult, + "staticWebsite" + ); + const defaultServiceVersion = this.getModelValue( + findResult, + "defaultServiceVersion" + ); + + const ret: ServicePropertiesModel = { + accountName: account + }; + + if (logging !== undefined) { + ret.logging = logging; + } + if (hourMetrics !== undefined) { + ret.hourMetrics = hourMetrics; + } + if (minuteMetrics !== undefined) { + ret.minuteMetrics = minuteMetrics; + } + if (cors !== undefined) { + ret.cors = cors; + } + if (deleteRetentionPolicy !== undefined) { + ret.deleteRetentionPolicy = deleteRetentionPolicy; + } + if (staticWebsite !== undefined) { + ret.staticWebsite = staticWebsite; + } + if (defaultServiceVersion !== undefined) { + ret.defaultServiceVersion = defaultServiceVersion; + } + + return ret; + } + + public async listContainers( + context: Context, + account: string, + prefix: string = "", + maxResults: number = DEFAULT_LIST_CONTAINERS_MAX_RESULTS, + marker: string + ): Promise<[ContainerModel[], string | undefined]> { + const whereQuery: any = { accountName: account }; + + if (prefix.length > 0) { + whereQuery.containerName = { + [Op.like]: `${prefix}%` + }; + } + + if (marker !== "") { + if (whereQuery.containerName === undefined) { + whereQuery.containerName = { + [Op.gt]: marker + }; + } else { + whereQuery.containerName[Op.gt] = marker; + } + } + + const findResult = await ContainersModel.findAll({ + limit: maxResults + 1, + where: whereQuery as any, + order: [["containerName", "ASC"]] + }); + + const leaseUpdateMapper = (model: ContainersModel) => { + const containerModel = this.convertDbModelToContainerModel(model); + return LeaseFactory.createLeaseState( + new ContainerLeaseAdapter(containerModel), + context + ).sync(new ContainerLeaseSyncer(containerModel)); + }; + + if (findResult.length <= maxResults) { + return [findResult.map(leaseUpdateMapper), undefined]; + } else { + const tail = findResult[findResult.length - 2]; + findResult.pop(); + const nextMarker = this.getModelValue( + tail, + "containerName", + true + ); + return [findResult.map(leaseUpdateMapper), nextMarker]; + } + } + + public async createContainer( + context: Context, + container: ContainerModel + ): Promise { + try { + await ContainersModel.create( + this.convertContainerModelToDbModel(container) + ); + return container; + } catch (err) { + if (err.name === "SequelizeUniqueConstraintError") { + throw StorageErrorFactory.getContainerAlreadyExists(context); + } + throw err; + } + } + + /** + * Set container properties. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} properties + * @param {Models.LeaseAccessConditions} [leaseAccessConditions] + * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + async setContainerProperties( + context: Context, + account: string, + container: string, + properties: string, + leaseAccessConditions?: Models.LeaseAccessConditions, + modifiedAccessConditions?: Models.ModifiedAccessConditions + ): Promise { + return await this.sequelize.transaction(async (t) => { + /* Transaction starts */ + const findResult = await ContainersModel.findOne({ + where: { + accountName: account, + containerName: container + }, + transaction: t + }); + + if (findResult === null || findResult === undefined) { + throw StorageErrorFactory.getContainerNotFound(context); + } + + const containerModel = this.convertDbModelToContainerModel(findResult); + validateWriteConditions( + context, + modifiedAccessConditions, + containerModel + ); + + LeaseFactory.createLeaseState( + this.convertDbModelToLease(findResult), + context + ).validate(new ContainerReadLeaseValidator(leaseAccessConditions)); + + await ContainersModel.update( + { + properties + }, + { + where: { + accountName: account, + containerName: container + }, + transaction: t + } + ); + + containerModel.fileSystemProperties = properties; + return containerModel; + /* Transaction ends */ + }); + } + + public async getContainerProperties( + context: Context, + account: string, + container: string, + leaseAccessConditions?: Models.LeaseAccessConditions + ): Promise { + const findResult = await this.assertContainerExists( + context, + account, + container, + undefined, + true + ); + const containerModel = this.convertDbModelToContainerModel(findResult); + + return LeaseFactory.createLeaseState( + new ContainerLeaseAdapter(containerModel), + context + ) + .validate(new ContainerReadLeaseValidator(leaseAccessConditions)) + .sync(new ContainerLeaseSyncer(containerModel)); + } + + public async deleteContainer( + context: Context, + account: string, + container: string, + options: Models.ContainerDeleteMethodOptionalParams = {} + ): Promise { + await this.sequelize.transaction(async (t) => { + /* Transaction starts */ + const findResult = await ContainersModel.findOne({ + attributes: [ + "accountName", + "containerName", + "etag", + "lastModified", + "lease" + ], + where: { + accountName: account, + containerName: container + }, + transaction: t + }); + + validateWriteConditions( + context, + options.modifiedAccessConditions, + findResult ? this.convertDbModelToContainerModel(findResult) : undefined + ); + + if (findResult === null || findResult === undefined) { + throw StorageErrorFactory.getContainerNotFound(context); + } + + LeaseFactory.createLeaseState( + this.convertDbModelToLease(findResult), + context + ).validate( + new ContainerDeleteLeaseValidator(options.leaseAccessConditions) + ); + + await ContainersModel.destroy({ + where: { + accountName: account, + containerName: container + }, + transaction: t + }); + + await BlobsModel.destroy({ + where: { + accountName: account, + containerName: container + }, + transaction: t + }); + + await BlocksModel.destroy({ + where: { + accountName: account, + containerName: container + }, + transaction: t + }); + /* Transaction ends */ + }); + } + + public async setContainerMetadata( + context: Context, + account: string, + container: string, + lastModified: Date, + etag: string, + metadata?: IContainerMetadata, + leaseAccessConditions?: Models.LeaseAccessConditions, + modifiedAccessConditions?: Models.ModifiedAccessConditions + ): Promise { + return await this.sequelize.transaction(async (t) => { + /* Transaction starts */ + const findResult = await ContainersModel.findOne({ + attributes: [ + "accountName", + "containerName", + "etag", + "lastModified", + "lease" + ], + where: { + accountName: account, + containerName: container + }, + transaction: t + }); + + validateWriteConditions( + context, + modifiedAccessConditions, + findResult ? this.convertDbModelToContainerModel(findResult) : undefined + ); + + if (findResult === null || findResult === undefined) { + throw StorageErrorFactory.getContainerNotFound(context); + } + + LeaseFactory.createLeaseState( + this.convertDbModelToLease(findResult), + context + ).validate(new ContainerReadLeaseValidator(leaseAccessConditions)); + + await ContainersModel.update( + { + lastModified, + etag, + metadata: this.serializeModelValue(metadata) || null + }, + { + where: { + accountName: account, + containerName: container + }, + transaction: t + } + ); + /* Transaction ends */ + }); + } + + /** + * Get container access policy. + * + * @param {string} account + * @param {string} container + * @param {Context} context + * @param {Models.LeaseAccessConditions} [leaseAccessConditions] + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + getContainerACL( + context: Context, + account: string, + container: string, + forceExist?: true, + leaseAccessConditions?: Models.LeaseAccessConditions + ): Promise; + + getContainerACL( + context: Context, + account: string, + container: string, + forceExist: false, + leaseAccessConditions?: Models.LeaseAccessConditions + ): Promise; + + public async getContainerACL( + context: Context, + account: string, + container: string, + forceExist?: boolean, + leaseAccessConditions?: Models.LeaseAccessConditions | undefined + ): Promise { + const findResult = await ContainersModel.findOne({ + where: { + accountName: account, + containerName: container + } + }); + + if (findResult === null || findResult === undefined) { + if (forceExist === undefined || forceExist === true) { + throw StorageErrorFactory.getContainerNotFound(context); + } + + throw StorageErrorFactory.getContainerNotFound(context); + } + + const containerModel = this.convertDbModelToContainerModel(findResult); + + LeaseFactory.createLeaseState( + new ContainerLeaseAdapter(containerModel), + context + ) + .validate(new ContainerReadLeaseValidator(leaseAccessConditions)) + .sync(new ContainerLeaseSyncer(containerModel)); + + return { + properties: containerModel.properties, + containerAcl: containerModel.containerAcl + }; + } + + public async setContainerACL( + context: Context, + account: string, + container: string, + setAclModel: SetContainerAccessPolicyOptions + ): Promise { + await this.sequelize.transaction(async (t) => { + const findResult = await ContainersModel.findOne({ + attributes: [ + "accountName", + "containerName", + "etag", + "lastModified", + "lease" + ], + where: { + accountName: account, + containerName: container + }, + transaction: t + }); + + validateWriteConditions( + context, + setAclModel.modifiedAccessConditions, + findResult ? this.convertDbModelToContainerModel(findResult) : undefined + ); + + if (findResult === null || findResult === undefined) { + throw StorageErrorFactory.getContainerNotFound(context); + } + + const lease = this.convertDbModelToLease(findResult); + + LeaseFactory.createLeaseState(lease, context).validate( + new ContainerReadLeaseValidator(setAclModel.leaseAccessConditions) + ); + + const updateResult = await ContainersModel.update( + { + lastModified: setAclModel.lastModified, + etag: setAclModel.etag, + containerAcl: + this.serializeModelValue(setAclModel.containerAcl) || null, + publicAccess: this.serializeModelValue(setAclModel.publicAccess) + }, + { + where: { + accountName: account, + containerName: container + }, + transaction: t + } + ); + + if (updateResult[0] === 0) { + throw StorageErrorFactory.getContainerNotFound(context); + } + }); + } + + public async acquireContainerLease( + context: Context, + account: string, + container: string, + options: Models.ContainerAcquireLeaseOptionalParams + ): Promise { + return await this.sequelize.transaction(async (t) => { + /* Transaction starts */ + const findResult = await ContainersModel.findOne({ + where: { + accountName: account, + containerName: container + }, + transaction: t + }); + + validateWriteConditions( + context, + options.modifiedAccessConditions, + findResult ? this.convertDbModelToContainerModel(findResult) : undefined + ); + + if (findResult === null || findResult === undefined) { + throw StorageErrorFactory.getContainerNotFound(context); + } + + const containerModel = this.convertDbModelToContainerModel(findResult); + LeaseFactory.createLeaseState( + new ContainerLeaseAdapter(containerModel), + context + ) + .acquire(options.duration!, options.proposedLeaseId) + .sync(new ContainerLeaseSyncer(containerModel)); + + await ContainersModel.update( + this.convertLeaseToDbModel(new ContainerLeaseAdapter(containerModel)), + { + where: { + accountName: account, + containerName: container + }, + transaction: t + } + ); + + return { + properties: containerModel.properties, + leaseId: containerModel.leaseId + }; + /* Transaction ends */ + }); + } + + public async releaseContainerLease( + context: Context, + account: string, + container: string, + leaseId: string, + options: Models.ContainerReleaseLeaseOptionalParams = {} + ): Promise { + return await this.sequelize.transaction(async (t) => { + /* Transaction starts */ + const findResult = await ContainersModel.findOne({ + where: { + accountName: account, + containerName: container + }, + transaction: t + }); + + validateWriteConditions( + context, + options.modifiedAccessConditions, + findResult ? this.convertDbModelToContainerModel(findResult) : undefined + ); + + if (findResult === null || findResult === undefined) { + throw StorageErrorFactory.getContainerNotFound(context); + } + + const containerModel = this.convertDbModelToContainerModel(findResult); + + LeaseFactory.createLeaseState( + new ContainerLeaseAdapter(containerModel), + context + ) + .release(leaseId) + .sync(new ContainerLeaseSyncer(containerModel)); + + await ContainersModel.update( + this.convertLeaseToDbModel(new ContainerLeaseAdapter(containerModel)), + { + where: { + accountName: account, + containerName: container + }, + transaction: t + } + ); + + return containerModel.properties; + /* Transaction ends */ + }); + } + + public async renewContainerLease( + context: Context, + account: string, + container: string, + leaseId: string, + options: Models.ContainerRenewLeaseOptionalParams = {} + ): Promise { + return await this.sequelize.transaction(async (t) => { + /* Transaction starts */ + // TODO: Filter out unnecessary fields in select query + const findResult = await ContainersModel.findOne({ + where: { + accountName: account, + containerName: container + }, + transaction: t + }); + + validateWriteConditions( + context, + options.modifiedAccessConditions, + findResult ? this.convertDbModelToContainerModel(findResult) : undefined + ); + + if (findResult === null || findResult === undefined) { + throw StorageErrorFactory.getContainerNotFound(context); + } + + const containerModel = this.convertDbModelToContainerModel(findResult); + + LeaseFactory.createLeaseState( + new ContainerLeaseAdapter(containerModel), + context + ) + .renew(leaseId) + .sync(new ContainerLeaseSyncer(containerModel)); + + await ContainersModel.update( + this.convertLeaseToDbModel(new ContainerLeaseAdapter(containerModel)), + { + where: { + accountName: account, + containerName: container + }, + transaction: t + } + ); + + return { + properties: containerModel.properties, + leaseId: containerModel.leaseId + }; + /* Transaction ends */ + }); + } + + public async breakContainerLease( + context: Context, + account: string, + container: string, + breakPeriod: number | undefined, + options: Models.ContainerBreakLeaseOptionalParams = {} + ): Promise { + return await this.sequelize.transaction(async (t) => { + const findResult = await ContainersModel.findOne({ + where: { + accountName: account, + containerName: container + }, + transaction: t + }); + + validateWriteConditions( + context, + options.modifiedAccessConditions, + findResult ? this.convertDbModelToContainerModel(findResult) : undefined + ); + + if (findResult === null || findResult === undefined) { + throw StorageErrorFactory.getContainerNotFound(context); + } + + const containerModel = this.convertDbModelToContainerModel(findResult); + + LeaseFactory.createLeaseState( + new ContainerLeaseAdapter(containerModel), + context + ) + .break(breakPeriod) + .sync(new ContainerLeaseSyncer(containerModel)); + + await ContainersModel.update( + this.convertLeaseToDbModel(new ContainerLeaseAdapter(containerModel)), + { + where: { + accountName: account, + containerName: container + }, + transaction: t + } + ); + + const leaseTimeSeconds: number = + containerModel.properties.leaseState === + Models.LeaseStateType.Breaking && containerModel.leaseBreakTime + ? Math.round( + (containerModel.leaseBreakTime.getTime() - + context.startTime!.getTime()) / + 1000 + ) + : 0; + + return { + properties: containerModel.properties, + leaseTime: leaseTimeSeconds + }; + }); + } + + public async changeContainerLease( + context: Context, + account: string, + container: string, + leaseId: string, + proposedLeaseId: string, + options: Models.ContainerChangeLeaseOptionalParams = {} + ): Promise { + return await this.sequelize.transaction(async (t) => { + const findResult = await ContainersModel.findOne({ + where: { + accountName: account, + containerName: container + }, + transaction: t + }); + + validateWriteConditions( + context, + options.modifiedAccessConditions, + findResult ? this.convertDbModelToContainerModel(findResult) : undefined + ); + + if (findResult === null || findResult === undefined) { + throw StorageErrorFactory.getContainerNotFound(context); + } + + const containerModel = this.convertDbModelToContainerModel(findResult); + + LeaseFactory.createLeaseState( + new ContainerLeaseAdapter(containerModel), + context + ) + .change(leaseId, proposedLeaseId) + .sync(new ContainerLeaseSyncer(containerModel)); + + await ContainersModel.update( + this.convertLeaseToDbModel(new ContainerLeaseAdapter(containerModel)), + { + where: { + accountName: account, + containerName: container + }, + transaction: t + } + ); + + return { + properties: containerModel.properties, + leaseId: containerModel.leaseId + }; + }); + } + + public async checkContainerExist( + context: Context, + account: string, + container: string + ): Promise { + await this.assertContainerExists(context, account, container, undefined); + } + + /** + * List paths in Directory + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} directory + * @param {boolean} recursive + * @param {Models.FileSystemListPathsOptionalParams} options + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + async listPaths( + context: Context, + account: string, + container: string, + directory: string, + recursive: boolean, + options: Models.FileSystemListPathsOptionalParams + ): Promise<[Models.Path[], string | undefined]> { + directory = removeSlash(directory); + const paths: Models.Path[] = []; + + if (directory) { + const model = await this.getModel( + context, + account, + container, + directory, + true + ); + + if (!model.isDirectory) { + throw StorageErrorFactory.getPathConflict(context); + } + } + + const [blobs, , nextMarker] = await this.list( + undefined, + context, + account, + container, + recursive ? undefined : "/", + undefined, + directory ? directory + "/" : undefined, + options.maxResults, + options.continuation + ); + blobs.forEach((blob) => { + paths.push({ + name: blob.name, + isDirectory: blob.isDirectory ? true : undefined, + lastModified: blob.properties.lastModified, + etag: blob.properties.etag, + contentLength: blob.properties.contentLength, + owner: blob.owner, + group: blob.group, + permissions: toPermissionsString(blob.permissions) + }); + }); + + return [paths, nextMarker]; + } + + /** + * Rename Directory + * + * @param {Context} context + * @param {string} account + * @param {string} sourceContainer + * @param {string} sourceDirectory + * @param {string} targetContainer + * @param {string} targetDirectory + * @param {Models.DirectoryRenameOptionalParams} options + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + public async renameDirectory( + context: Context, + account: string, + sourceContainer: string, + sourceDirectory: string, + targetContainer: string, + targetDirectory: string, + options: Models.PathCreateOptionalParams + ): Promise { + sourceDirectory = removeSlash(sourceDirectory); + targetDirectory = removeSlash(targetDirectory); + + const dirModel = await this.getModel( + context, + account, + targetContainer, + targetDirectory, + false + ); + + if ( + options.modifiedAccessConditions && + options.modifiedAccessConditions.ifNoneMatch === "*" && + dirModel + ) { + throw StorageErrorFactory.getBlobAlreadyExists(context); + } + + const [paths] = await this.listPaths( + context, + account, + sourceContainer, + sourceDirectory, + true, + options + ); + + paths.forEach(async (path) => { + await this.rename( + path.isDirectory!, + context, + account, + sourceContainer, + path.name!, + targetContainer, + targetDirectory + path.name!.substring(sourceDirectory.length), + options + ); + }); + + await this.rename( + true, + context, + account, + sourceContainer, + sourceDirectory, + targetContainer, + targetDirectory, + options + ); + const res = await this.getModel( + context, + account, + targetContainer, + targetDirectory, + true + ); + return res!; + } + + /** + * Delete Directory + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} directory + * @param {boolean} recursiveDirectoryDelete + * @param {Models.DirectoryDeleteMethodOptionalParams} options + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + public async deleteDirectory( + context: Context, + account: string, + container: string, + directory: string, + recursiveDirectoryDelete: boolean, + options: Models.PathDeleteMethodOptionalParams + ): Promise { + await this.sequelize.transaction(async (t) => { + await this.assertContainerExists(context, account, container, t); + + const directoryFindResult = await BlobsModel.findOne({ + where: { + accountName: account, + containerName: container, + blobName: directory, + isDirectory: true + }, + transaction: t + }); + + validateWriteConditions( + context, + options.modifiedAccessConditions, + directoryFindResult + ? this.convertDbModelToBlobModel(directoryFindResult) // TODO: Reduce double convert + : undefined + ); + + if (directoryFindResult === null || directoryFindResult === undefined) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + const [blobs] = await this.list( + undefined, + context, + account, + container, + undefined, + undefined, + directory ? directory + "/" : directory, + DEFAULT_LIST_BLOBS_MAX_RESULTS, + undefined, + true, + true + ); + + if (blobs.length > 1 && !recursiveDirectoryDelete) { + throw StorageErrorFactory.getInvalidOperation( + context, + "Can't Delete non empty folder with non recursive flag" + ); + } else { + const options: Models.PathDeleteMethodOptionalParams = { + deleteSnapshots: Models.DeleteSnapshotsOptionType.Include + }; + + for (const blob of blobs) { + await this.deleteBlob( + context, + account, + container, + blob.name, + options + ); + } + } + + await BlobsModel.destroy({ + where: { + accountName: account, + containerName: container, + blobName: directory, + isDirectory: true + }, + transaction: t + }); + }); + } + + /** + * Create blob item in persistency layer. Will replace if blob exists. + * + * @param {Context} context + * @param {BlobModel} blob + * @param {Models.LeaseAccessConditions} [leaseAccessConditions] + * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async createBlob( + context: Context, + blob: BlobModel, + leaseAccessConditions?: Models.LeaseAccessConditions, + modifiedAccessConditions?: Models.ModifiedAccessConditions + ): Promise { + return await this.sequelize.transaction(async (t) => { + await this.assertContainerExists( + context, + blob.accountName, + blob.containerName, + t + ); + + const blobFindResult = await BlobsModel.findOne({ + where: { + accountName: blob.accountName, + containerName: blob.containerName, + blobName: blob.name, + snapshot: blob.snapshot, + isCommitted: true + }, + transaction: t + }); + + const blobModel = blobFindResult + ? this.convertDbModelToBlobModel(blobFindResult) + : undefined; + + await this.validateExpireConditions(context, blobModel, t); + validateWriteConditions(context, modifiedAccessConditions, blobModel); + + // Create if not exists + if ( + modifiedAccessConditions && + modifiedAccessConditions.ifNoneMatch === "*" && + blobFindResult + ) { + throw StorageErrorFactory.getBlobAlreadyExists(context); + } + + if (blobModel) { + LeaseFactory.createLeaseState(new BlobLeaseAdapter(blobModel), context) + .validate(new BlobWriteLeaseValidator(leaseAccessConditions)) + .sync(new BlobLeaseSyncer(blob)); // Keep original blob lease; + + if ( + blobModel.properties !== undefined && + blobModel.properties.accessTier === Models.AccessTier.Archive + ) { + throw StorageErrorFactory.getBlobArchived(context); + } + } + + await this.upsertBlob(blob, t); + }); + } + + public async downloadBlob( + context: Context, + account: string, + container: string, + blob: string, + snapshot: string = "", + leaseAccessConditions?: Models.LeaseAccessConditions, + modifiedAccessConditions?: Models.ModifiedAccessConditions + ): Promise { + return await this.sequelize.transaction(async (t) => { + await this.assertContainerExists(context, account, container, t); + + const blobFindResult = await BlobsModel.findOne({ + where: { + accountName: account, + containerName: container, + blobName: blob, + snapshot, + isCommitted: true + }, + transaction: t + }); + + const blobModel: BlobModel | undefined = blobFindResult + ? this.convertDbModelToBlobModel(blobFindResult) + : undefined; + + await this.validateExpireConditions(context, blobModel, t, true); + validateReadConditions(context, modifiedAccessConditions, blobModel); + + if (blobModel === undefined) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + return LeaseFactory.createLeaseState( + new BlobLeaseAdapter(blobModel), + context + ) + .validate(new BlobReadLeaseValidator(leaseAccessConditions)) + .sync(new BlobLeaseSyncer(blobModel)); + }); + } + + public async listBlobs( + context: Context, + account: string, + container: string, + delimiter?: string, + blob?: string, + prefix: string = "", + maxResults: number = DEFAULT_LIST_BLOBS_MAX_RESULTS, + marker?: string, + includeSnapshots?: boolean, + includeUncommittedBlobs?: boolean + ): Promise<[BlobModel[], BlobPrefixModel[], any | undefined]> { + return this.list( + false, + context, + account, + container, + delimiter, + blob, + prefix, + maxResults, + marker, + includeSnapshots, + includeUncommittedBlobs + ); + } + + public async list( + lisDirectories: boolean | undefined, + context: Context, + account: string, + container: string, + delimiter?: string, + blob?: string, + prefix: string = "", + maxResults: number = DEFAULT_LIST_BLOBS_MAX_RESULTS, + marker?: string, + includeSnapshots?: boolean, + includeUncommittedBlobs?: boolean + ): Promise<[BlobModel[], BlobPrefixModel[], any | undefined]> { + return await this.sequelize.transaction(async (t) => { + await this.assertContainerExists(context, account, container, t); + + const whereQuery: any = { + accountName: account, + containerName: container + }; + + if (blob !== undefined) { + whereQuery.blobName = blob; + } else { + if (prefix.length > 0) { + whereQuery.blobName = { + [Op.like]: `${prefix}%` + }; + } + + if (marker !== undefined) { + if (whereQuery.blobName !== undefined) { + whereQuery.blobName[Op.gt] = marker; + } else { + whereQuery.blobName = { + [Op.gt]: marker + }; + } + } + } + if (lisDirectories !== false) { + includeSnapshots = true; + includeUncommittedBlobs = true; + } + if (lisDirectories !== undefined) { + whereQuery.isDirectory = lisDirectories; + } + if (!includeSnapshots) { + whereQuery.snapshot = ""; + } + if (!includeUncommittedBlobs) { + whereQuery.isCommitted = true; + } + + const leaseUpdateMapper = (model: BlobsModel) => { + const blobModel = this.convertDbModelToBlobModel(model); + return LeaseFactory.createLeaseState( + new BlobLeaseAdapter(blobModel), + context + ).sync(new BlobLeaseSyncer(blobModel)); + }; + + // fill the page by possibly querying multiple times + const page = new PageWithDelimiter( + maxResults, + delimiter, + prefix + ); + + const nameItem = (item: BlobsModel): string => { + return this.getModelValue(item, "blobName", true); + }; + + const readPage = async (off: number): Promise => { + return await BlobsModel.findAll({ + where: whereQuery as any, + order: [["blobName", "ASC"]], + transaction: t, + limit: maxResults, + offset: off + }); + }; + + const [blobItems, blobPrefixes, nextMarker] = await page.fill( + readPage, + nameItem + ); + + const blobModels = blobItems.map(leaseUpdateMapper); + const filteredModels = blobModels.filter( + async (blobModel) => + await this.validateExpireConditions(context, blobModel, t, false) + ); + return [filteredModels, blobPrefixes, nextMarker]; + }); + } + + public async listAllBlobs( + maxResults: number = DEFAULT_LIST_BLOBS_MAX_RESULTS, + marker?: string, + includeSnapshots?: boolean, + includeUncommittedBlobs?: boolean + ): Promise<[BlobModel[], any | undefined]> { + const whereQuery: any = {}; + if (marker !== undefined) { + whereQuery.blobName = { + [Op.gt]: marker + }; + } + if (!includeSnapshots) { + whereQuery.snapshot = ""; + } + if (!includeUncommittedBlobs) { + whereQuery.isCommitted = true; + } + + const blobFindResult = await BlobsModel.findAll({ + limit: maxResults + 1, + where: whereQuery as any, + order: [["blobName", "ASC"]] + }); + + const blobModels = blobFindResult + .map(this.convertDbModelToBlobModel.bind(this)) + .filter( + async (model) => + await this.validateExpireConditions( + undefined, + model, + undefined, + false + ) + ); + if (blobModels.length <= maxResults) { + return [blobModels, undefined]; + } else { + blobModels.pop(); + const tail = blobModels[blobModels.length - 1]; + const nextMarker = tail.name; + return [blobModels, nextMarker]; + } + } + + public async stageBlock( + context: Context, + block: BlockModel, + leaseAccessConditions?: Models.LeaseAccessConditions, + append: boolean = false + ): Promise { + await this.sequelize.transaction(async (t) => { + await this.assertContainerExists( + context, + block.accountName, + block.containerName, + t + ); + + const blobFindResult = await BlobsModel.findOne({ + where: { + accountName: block.accountName, + containerName: block.containerName, + blobName: block.blobName, + snapshot: "" + }, + transaction: t + }); + + const blobModel: BlobModel | undefined = blobFindResult + ? this.convertDbModelToBlobModel(blobFindResult) + : undefined; + const valid = await this.validateExpireConditions( + context, + blobModel, + t, + false + ); + if (blobModel !== undefined && valid) { + if (blobModel.isCommitted === true) { + LeaseFactory.createLeaseState( + new BlobLeaseAdapter(blobModel), + context + ).validate(new BlobWriteLeaseValidator(leaseAccessConditions)); + } + + if (!append) { + // If the new block ID does not have same length with before uncommited block ID, return failure. + const existBlock = await BlocksModel.findOne({ + attributes: ["blockName"], + where: { + accountName: block.accountName, + containerName: block.containerName, + blobName: block.blobName + }, + order: [["id", "ASC"]], + transaction: t + }); + if ( + existBlock && + Buffer.from( + this.getModelValue(existBlock, "blockName", true), + "base64" + ).length !== Buffer.from(block.name, "base64").length + ) { + throw StorageErrorFactory.getInvalidBlobOrBlock(context); + } + } + } else { + const newBlob: BlobModel = { + deleted: false, + accountName: block.accountName, + containerName: block.containerName, + name: block.blobName, + properties: { + creationTime: context.startTime!, + lastModified: context.startTime!, + etag: newEtag(), + contentLength: 0, + blobType: Models.BlobType.BlockBlob + }, + snapshot: "", + isCommitted: false, + isDirectory: false, + permissions: toPermissions(DEFAULT_FILE_PERMISSIONS)!, + acl: [], + owner: DEFAULT_OWNER, + group: DEFAULT_GROUP + }; + await this.upsertBlob(newBlob, t); + } + + await this.upsertBlock(block, t); + }); + } + + public async getBlockList( + context: Context, + account: string, + container: string, + blob: string, + snapshot: string = "", + isCommitted?: boolean, + leaseAccessConditions?: Models.LeaseAccessConditions + ): Promise { + return await this.sequelize.transaction(async (t) => { + await this.assertContainerExists(context, account, container, t); + + const blobFindResult = await BlobsModel.findOne({ + where: { + accountName: account, + containerName: container, + blobName: blob, + snapshot + }, + transaction: t + }); + + if (blobFindResult === null || blobFindResult === undefined) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + const blobModel = this.convertDbModelToBlobModel(blobFindResult); + await this.validateExpireConditions(context, blobModel, t, true); + + LeaseFactory.createLeaseState( + new BlobLeaseAdapter(blobModel), + context + ).validate(new BlobReadLeaseValidator(leaseAccessConditions)); + + const res: { + uncommittedBlocks: Models.Block[]; + committedBlocks: Models.Block[]; + } = { + uncommittedBlocks: [], + committedBlocks: [] + }; + + if (isCommitted !== false) { + res.committedBlocks = blobModel.committedBlocksInOrder || []; + } + + if (isCommitted !== true) { + const blocks = await BlocksModel.findAll({ + attributes: ["blockName", "size"], + where: { + accountName: account, + containerName: container, + blobName: blob + }, + order: [["id", "ASC"]], + transaction: t + }); + for (const item of blocks) { + const block = { + name: this.getModelValue(item, "blockName", true), + size: this.getModelValue(item, "size", true) + }; + res.uncommittedBlocks.push(block); + } + } + return res; + }); + } + + public async commitBlockList( + context: Context, + blob: BlobModel, + blockList: { blockName: string; blockCommitType: string }[], + leaseAccessConditions?: Models.LeaseAccessConditions, + modifiedAccessConditions?: Models.ModifiedAccessConditions + ): Promise { + await this.sequelize.transaction(async (t) => { + await this.assertContainerExists( + context, + blob.accountName, + blob.containerName, + t + ); + + const pCommittedBlocksMap: Map = new Map(); // persistencyCommittedBlocksMap + const pUncommittedBlocksMap: Map = + new Map(); // persistencyUncommittedBlocksMap + + const badRequestError = StorageErrorFactory.getInvalidBlockList(context); + + const blobFindResult = await BlobsModel.findOne({ + where: { + accountName: blob.accountName, + containerName: blob.containerName, + blobName: blob.name, + snapshot: blob.snapshot, + isCommitted: true + }, + transaction: t + }); + + const blobModel: BlobModel | undefined = blobFindResult + ? this.convertDbModelToBlobModel(blobFindResult) + : undefined; + const valid = await this.validateExpireConditions( + context, + blobModel, + t, + false + ); + validateWriteConditions( + context, + modifiedAccessConditions, + valid ? blobModel : undefined + ); + + let creationTime = blob.properties.creationTime || context.startTime; + + if (blobModel !== undefined && valid) { + // Create if not exists + if ( + modifiedAccessConditions && + modifiedAccessConditions.ifNoneMatch === "*" && + blobModel && + blobModel.isCommitted + ) { + throw StorageErrorFactory.getBlobAlreadyExists(context); + } + + creationTime = blobModel.properties.creationTime || creationTime; + + LeaseFactory.createLeaseState( + new BlobLeaseAdapter(blobModel), + context + ).validate(new BlobWriteLeaseValidator(leaseAccessConditions)); + + const committedBlocksInOrder = blobModel.committedBlocksInOrder; + for (const pBlock of committedBlocksInOrder || []) { + pCommittedBlocksMap.set(pBlock.name, pBlock); + } + } + + const blockFindResult = await BlocksModel.findAll({ + where: { + accountName: blob.accountName, + containerName: blob.containerName, + blobName: blob.name + }, + transaction: t + }); + for (const item of blockFindResult) { + const block: PersistencyBlockModel = { + name: this.getModelValue(item, "blockName", true), + size: this.getModelValue(item, "size", true), + persistency: this.deserializeModelValue( + item, + "persistency", + true + ) + }; + pUncommittedBlocksMap.set(block.name, block); + } + + const selectedBlockList: PersistencyBlockModel[] = + blobModel && + blobModel.properties.blobType === Models.BlobType.AppendBlob && + blobModel.committedBlocksInOrder + ? blobModel.committedBlocksInOrder + : []; + for (const block of blockList) { + switch (block.blockCommitType.toLowerCase()) { + case "uncommitted": + const pUncommittedBlock = pUncommittedBlocksMap.get( + block.blockName + ); + if (pUncommittedBlock === undefined) { + throw badRequestError; + } else { + selectedBlockList.push(pUncommittedBlock); + } + break; + case "committed": + const pCommittedBlock = pCommittedBlocksMap.get(block.blockName); + if (pCommittedBlock === undefined) { + throw badRequestError; + } else { + selectedBlockList.push(pCommittedBlock); + } + break; + case "latest": + const pLatestBlock = + pUncommittedBlocksMap.get(block.blockName) || + pCommittedBlocksMap.get(block.blockName); + if (pLatestBlock === undefined) { + throw badRequestError; + } else { + selectedBlockList.push(pLatestBlock); + } + break; + default: + throw badRequestError; + } + } + + const commitBlockBlob: BlobModel = { + ...blob, + deleted: false, + committedBlocksInOrder: selectedBlockList, + properties: { + ...blob.properties, + creationTime, + lastModified: blob.properties.lastModified || context.startTime, + contentLength: selectedBlockList + .map((block) => block.size) + .reduce((total, val) => { + return total + val; + }, 0) + } + }; + + new BlobWriteLeaseSyncer(commitBlockBlob).sync( + new BlobLeaseAdapter(commitBlockBlob) + ); + + await this.upsertBlob(commitBlockBlob, t); + + await BlocksModel.destroy({ + where: { + accountName: blob.accountName, + containerName: blob.containerName, + blobName: blob.name + }, + transaction: t + }); + }); + } + + public async getBlobProperties( + context: Context, + account: string, + container: string, + blob: string, + snapshot: string = "", + leaseAccessConditions?: Models.LeaseAccessConditions, + modifiedAccessConditions?: Models.ModifiedAccessConditions + ): Promise { + return await this.sequelize.transaction(async (t) => { + await this.assertContainerExists(context, account, container, t); + + const blobFindResult = await BlobsModel.findOne({ + where: { + accountName: account, + containerName: container, + blobName: blob, + snapshot, + isCommitted: true + }, + transaction: t + }); + + const blobModel: BlobModel | undefined = blobFindResult + ? this.convertDbModelToBlobModel(blobFindResult) + : undefined; + + await this.validateExpireConditions(context, blobModel, t, true); + validateReadConditions(context, modifiedAccessConditions, blobModel); + + if (blobModel === undefined) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + if (!blobModel.isCommitted) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + // TODO: Return blobCommittedBlockCount for append blob + + return LeaseFactory.createLeaseState( + new BlobLeaseAdapter(blobModel), + context + ) + .validate(new BlobReadLeaseValidator(leaseAccessConditions)) + .sync(new BlobLeaseSyncer(blobModel)); + }); + } + + public async createSnapshot( + context: Context, + account: string, + container: string, + blob: string, + leaseAccessConditions?: Models.LeaseAccessConditions, + metadata?: Models.BlobMetadata, + modifiedAccessConditions?: Models.ModifiedAccessConditions + ): Promise { + return await this.sequelize.transaction(async (t) => { + await this.assertContainerExists(context, account, container, t); + + const blobFindResult = await BlobsModel.findOne({ + where: { + accountName: account, + containerName: container, + blobName: blob, + snapshot: "", + isCommitted: true + }, + transaction: t + }); + + const snapshotBlob: BlobModel | undefined = blobFindResult + ? this.convertDbModelToBlobModel(blobFindResult) // TODO: Reduce double convert + : undefined; + await this.validateExpireConditions(context, snapshotBlob, t, true); + validateReadConditions(context, modifiedAccessConditions, snapshotBlob); + + if (snapshotBlob === undefined) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + LeaseFactory.createLeaseState( + new BlobLeaseAdapter(snapshotBlob), + context + ).validate(new BlobReadLeaseValidator(leaseAccessConditions)); + + const snapshotTime = convertDateTimeStringMsTo7Digital( + context.startTime!.toISOString() + ); + + snapshotBlob.snapshot = snapshotTime; + snapshotBlob.metadata = metadata || snapshotBlob.metadata; + + new BlobLeaseSyncer(snapshotBlob).sync({ + leaseId: undefined, + leaseExpireTime: undefined, + leaseDurationSeconds: undefined, + leaseBreakTime: undefined, + leaseDurationType: undefined, + leaseState: undefined, + leaseStatus: undefined + }); + + await this.upsertBlob(snapshotBlob, t); + + return { + properties: snapshotBlob.properties, + snapshot: snapshotTime + }; + }); + } + + /** + * Rename Blob + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} source + * @param {string} target + * @param {Models.BlobDeleteMethodOptionalParams} options + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async renameBlob( + context: Context, + account: string, + sourceContainer: string, + sourceBlob: string, + targetContainer: string, + targetBlob: string, + options: Models.PathCreateOptionalParams + ): Promise { + return this.rename( + false, + context, + account, + sourceContainer, + sourceBlob, + targetContainer, + targetBlob, + options + ); + } + + public async rename( + isDirectory: boolean, + context: Context, + account: string, + sourceContainer: string, + sourceBlob: string, + targetContainer: string, + targetBlob: string, + options: Models.PathCreateOptionalParams + ): Promise { + return await this.sequelize.transaction(async (t) => { + const target = await this.getModel( + context, + account, + targetContainer, + targetBlob, + false, + options.leaseAccessConditions, + options.modifiedAccessConditions, + false + ); + + validateWriteConditions( + context, + options.modifiedAccessConditions, + target + ); + + if ( + options.modifiedAccessConditions && + options.modifiedAccessConditions.ifNoneMatch === "*" && + target + ) { + throw StorageErrorFactory.getBlobAlreadyExists(context); + } + + if (target) { + isDirectory + ? await this.deleteDirectory( + context, + account, + targetContainer, + targetBlob, + true, + options + ) + : await this.deleteBlob( + context, + account, + targetContainer, + targetBlob, + options + ); + } + + const model = await this.getModel( + context, + account, + sourceContainer, + sourceBlob + ); + + const newModel: BlobModel = { + ...model, + containerName: targetContainer, + name: targetBlob + }; + + await this.createBlob( + context, + newModel, + options.leaseAccessConditions, + options.modifiedAccessConditions + ); + + await BlobsModel.destroy({ + where: { + accountName: account, + containerName: sourceContainer, + blobName: sourceBlob + }, + transaction: t + }); + + return model; + }); + } + + /** + * Delete blob or its snapshots. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {Models.PathDeleteMethodOptionalParams} options + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async deleteBlob( + context: Context, + account: string, + container: string, + blob: string, + options: Models.PathDeleteMethodOptionalParams + ): Promise { + await this.sequelize.transaction(async (t) => { + await this.assertContainerExists(context, account, container, t); + + const blobFindResult = await BlobsModel.findOne({ + where: { + accountName: account, + containerName: container, + blobName: blob, + snapshot: options.snapshot === undefined ? "" : options.snapshot, + isCommitted: true // TODO: Support deleting uncommitted block blob + }, + transaction: t + }); + + const blobModel: BlobModel | undefined = blobFindResult + ? this.convertDbModelToBlobModel(blobFindResult) + : undefined; + await this.validateExpireConditions(context, blobModel, t, true); + validateWriteConditions( + context, + options.modifiedAccessConditions, + blobModel + ); + if (blobModel === undefined) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + const againstBaseBlob = blobModel.snapshot === ""; + + if (againstBaseBlob) { + LeaseFactory.createLeaseState( + new BlobLeaseAdapter(blobModel), + context + ).validate(new BlobWriteLeaseValidator(options.leaseAccessConditions)); + } + + // Check bad requests + if (!againstBaseBlob && options.deleteSnapshots !== undefined) { + throw StorageErrorFactory.getInvalidOperation( + context, + "Invalid operation against a blob snapshot." + ); + } + + // Scenario: Delete base blob only + if (againstBaseBlob && options.deleteSnapshots === undefined) { + const count = await BlobsModel.count({ + where: { + accountName: account, + containerName: container, + blobName: blob + }, + transaction: t + }); + + if (count > 1) { + throw StorageErrorFactory.getSnapshotsPresent(context); + } else { + await BlobsModel.destroy({ + where: { + accountName: account, + containerName: container, + blobName: blob + }, + transaction: t + }); + + await BlocksModel.destroy({ + where: { + accountName: account, + containerName: container, + blobName: blob + }, + transaction: t + }); + } + } + + // Scenario: Delete one snapshot only + if (!againstBaseBlob) { + await BlobsModel.destroy({ + where: { + accountName: account, + containerName: container, + blobName: blob, + snapshot: blobModel.snapshot + }, + transaction: t + }); + } + + // Scenario: Delete base blob and snapshots + if ( + againstBaseBlob && + options.deleteSnapshots === Models.DeleteSnapshotsOptionType.Include + ) { + await BlobsModel.destroy({ + where: { + accountName: account, + containerName: container, + blobName: blob + }, + transaction: t + }); + + await BlocksModel.destroy({ + where: { + accountName: account, + containerName: container, + blobName: blob + }, + transaction: t + }); + } + + // Scenario: Delete all snapshots only + if ( + againstBaseBlob && + options.deleteSnapshots === Models.DeleteSnapshotsOptionType.Only + ) { + await BlobsModel.destroy({ + where: { + accountName: account, + containerName: container, + blobName: blob, + snapshot: { [Op.gt]: "" } + }, + transaction: t + }); + } + }); + } + + /** + * Set blob HTTP headers. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {(Models.LeaseAccessConditions | undefined)} leaseAccessConditions + * @param {(Models.BlobHTTPHeaders | undefined)} blobHTTPHeaders + * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async setBlobHTTPHeaders( + context: Context, + account: string, + container: string, + blob: string, + leaseAccessConditions: Models.LeaseAccessConditions | undefined, + blobHTTPHeaders: Models.BlobHTTPHeaders | undefined, + modifiedAccessConditions?: Models.ModifiedAccessConditions + ): Promise { + return await this.sequelize.transaction(async (t) => { + await this.assertContainerExists(context, account, container, t); + + const blobFindResult = await BlobsModel.findOne({ + where: { + accountName: account, + containerName: container, + blobName: blob, + snapshot: "", + isCommitted: true + }, + transaction: t + }); + + const blobModel: BlobModel | undefined = blobFindResult + ? this.convertDbModelToBlobModel(blobFindResult) // TODO: Reduce double convert + : undefined; + await this.validateExpireConditions(context, blobModel, t, true); + validateWriteConditions(context, modifiedAccessConditions, blobModel); + + if (blobModel === undefined) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + LeaseFactory.createLeaseState(new BlobLeaseAdapter(blobModel), context) + .validate(new BlobWriteLeaseValidator(leaseAccessConditions)) + .sync(new BlobWriteLeaseSyncer(blobModel)); + + if (blobHTTPHeaders !== undefined) { + blobModel.properties.cacheControl = blobHTTPHeaders.blobCacheControl; + blobModel.properties.contentType = blobHTTPHeaders.blobContentType; + blobModel.properties.contentMD5 = blobHTTPHeaders.blobContentMD5; + blobModel.properties.contentEncoding = + blobHTTPHeaders.blobContentEncoding; + blobModel.properties.contentLanguage = + blobHTTPHeaders.blobContentLanguage; + blobModel.properties.contentDisposition = + blobHTTPHeaders.blobContentDisposition; + } + + blobModel.properties.etag = newEtag(); + blobModel.properties.lastModified = context.startTime + ? context.startTime + : new Date(); + + await BlobsModel.update(this.convertBlobModelToDbModel(blobModel), { + where: { + accountName: account, + containerName: container, + blobName: blob, + snapshot: "" + }, + transaction: t + }); + + return blobModel.properties; + }); + } + + public async setBlobMetadata( + context: Context, + account: string, + container: string, + blob: string, + leaseAccessConditions: Models.LeaseAccessConditions | undefined, + metadata: Models.BlobMetadata | undefined, + modifiedAccessConditions?: Models.ModifiedAccessConditions + ): Promise { + return await this.sequelize.transaction(async (t) => { + await this.assertContainerExists(context, account, container, t); + + const blobFindResult = await BlobsModel.findOne({ + where: { + accountName: account, + containerName: container, + blobName: blob, + snapshot: "", + isCommitted: true + }, + transaction: t + }); + + const blobModel: BlobModel | undefined = blobFindResult + ? this.convertDbModelToBlobModel(blobFindResult) // TODO: Reduce double convert + : undefined; + await this.validateExpireConditions(context, blobModel, t, true); + validateWriteConditions(context, modifiedAccessConditions, blobModel); + + if (blobModel === undefined) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + LeaseFactory.createLeaseState(new BlobLeaseAdapter(blobModel), context) + .validate(new BlobWriteLeaseValidator(leaseAccessConditions)) + .sync(new BlobWriteLeaseSyncer(blobModel)); + + const lastModified = context.startTime! || new Date(); + const etag = newEtag(); + + await BlobsModel.update( + { + metadata: this.serializeModelValue(metadata) || null, + lastModified, + etag, + ...this.convertLeaseToDbModel(new BlobLeaseAdapter(blobModel)) + }, + { + where: { + accountName: account, + containerName: container, + blobName: blob, + snapshot: "" + }, + transaction: t + } + ); + + const ret: Models.BlobPropertiesInternal = { + lastModified, + etag, + leaseStatus: blobModel.properties.leaseStatus, + leaseDuration: blobModel.properties.leaseDuration, + leaseState: blobModel.properties.leaseState + }; + + return ret; + }); + } + + public async acquireBlobLease( + context: Context, + account: string, + container: string, + blob: string, + duration: number, + proposedLeaseId?: string, + options: Models.BlobAcquireLeaseOptionalParams = {} + ): Promise { + return await this.sequelize.transaction(async (t) => { + await this.assertContainerExists(context, account, container, t); + + const blobModel = await this.getBlobWithLeaseUpdated( + account, + container, + blob, + undefined, + context, + false, + false, + t + ); + + await this.validateExpireConditions(context, blobModel, t, true); + validateWriteConditions( + context, + options.modifiedAccessConditions, + blobModel + ); + + if (blobModel === null || blobModel === undefined) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + if (blobModel.snapshot !== "") { + throw StorageErrorFactory.getBlobSnapshotsPresent(context); + } + + const lease = LeaseFactory.createLeaseState( + new BlobLeaseAdapter(blobModel), + context + ).acquire(duration, proposedLeaseId).lease; + + await BlobsModel.update( + this.convertLeaseToDbBlobsModel(blobModel.properties, lease), + { + where: { + accountName: account, + containerName: container, + blobName: blob, + snapshot: "" + }, + transaction: t + } + ); + return { properties: blobModel.properties, leaseId: lease.leaseId }; + }); + } + + public async releaseBlobLease( + context: Context, + account: string, + container: string, + blob: string, + leaseId: string, + options: Models.BlobReleaseLeaseOptionalParams = {} + ): Promise { + return await this.sequelize.transaction(async (t) => { + await this.assertContainerExists(context, account, container, t); + + const blobFindResult = await BlobsModel.findOne({ + where: { + accountName: account, + containerName: container, + blobName: blob, + snapshot: "", + isCommitted: true + }, + transaction: t + }); + + const blobModel: BlobModel | undefined = blobFindResult + ? this.convertDbModelToBlobModel(blobFindResult) // TODO: Reduce double convert + : undefined; + await this.validateExpireConditions(context, blobModel, t, true); + validateWriteConditions( + context, + options.modifiedAccessConditions, + blobModel + ); + + if (blobModel === undefined) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + const lease = LeaseFactory.createLeaseState( + new BlobLeaseAdapter(blobModel), + context + ).release(leaseId).lease; + + await BlobsModel.update( + this.convertLeaseToDbBlobsModel(blobModel.properties, lease), + { + where: { + accountName: account, + containerName: container, + blobName: blob, + snapshot: "" + }, + transaction: t + } + ); + return blobModel.properties; + }); + } + + public async renewBlobLease( + context: Context, + account: string, + container: string, + blob: string, + leaseId: string, + options: Models.BlobRenewLeaseOptionalParams = {} + ): Promise { + return await this.sequelize.transaction(async (t) => { + await this.assertContainerExists(context, account, container, t); + + const blobFindResult = await BlobsModel.findOne({ + where: { + accountName: account, + containerName: container, + blobName: blob, + snapshot: "", + isCommitted: true + }, + transaction: t + }); + + const blobModel: BlobModel | undefined = blobFindResult + ? this.convertDbModelToBlobModel(blobFindResult) // TODO: Reduce double convert + : undefined; + await this.validateExpireConditions(context, blobModel, t, true); + validateWriteConditions( + context, + options.modifiedAccessConditions, + blobModel + ); + + if (blobModel === undefined) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + const lease = LeaseFactory.createLeaseState( + new BlobLeaseAdapter(blobModel), + context + ).renew(leaseId).lease; + + await BlobsModel.update( + this.convertLeaseToDbBlobsModel(blobModel.properties, lease), + { + where: { + accountName: account, + containerName: container, + blobName: blob, + snapshot: "" + }, + transaction: t + } + ); + return { properties: blobModel.properties, leaseId: lease.leaseId }; + }); + } + + public async changeBlobLease( + context: Context, + account: string, + container: string, + blob: string, + leaseId: string, + proposedLeaseId: string, + options: + | Models.BlobChangeLeaseOptionalParams + | Models.PathLeaseOptionalParams = {} + ): Promise { + return await this.sequelize.transaction(async (t) => { + await this.assertContainerExists(context, account, container, t); + + const blobFindResult = await BlobsModel.findOne({ + where: { + accountName: account, + containerName: container, + blobName: blob, + snapshot: "", + isCommitted: true + }, + transaction: t + }); + + const blobModel: BlobModel | undefined = blobFindResult + ? this.convertDbModelToBlobModel(blobFindResult) // TODO: Reduce double convert + : undefined; + await this.validateExpireConditions(context, blobModel, t, true); + validateWriteConditions( + context, + options.modifiedAccessConditions, + blobModel + ); + + if (blobModel === undefined) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + const lease = LeaseFactory.createLeaseState( + new BlobLeaseAdapter(blobModel), + context + ).change(leaseId, proposedLeaseId).lease; + + await BlobsModel.update( + this.convertLeaseToDbBlobsModel(blobModel.properties, lease), + { + where: { + accountName: account, + containerName: container, + blobName: blob, + snapshot: "" + }, + transaction: t + } + ); + return { properties: blobModel.properties, leaseId: lease.leaseId }; + }); + } + + public async breakBlobLease( + context: Context, + account: string, + container: string, + blob: string, + breakPeriod: number | undefined, + options: + | Models.BlobBreakLeaseOptionalParams + | Models.PathLeaseOptionalParams = {} + ): Promise { + return await this.sequelize.transaction(async (t) => { + await this.assertContainerExists(context, account, container, t); + + const blobFindResult = await BlobsModel.findOne({ + where: { + accountName: account, + containerName: container, + blobName: blob, + snapshot: "", + isCommitted: true + }, + transaction: t + }); + + const blobModel: BlobModel | undefined = blobFindResult + ? this.convertDbModelToBlobModel(blobFindResult) // TODO: Reduce double convert + : undefined; + await this.validateExpireConditions(context, blobModel, t, true); + validateWriteConditions( + context, + options.modifiedAccessConditions, + blobModel + ); + + if (blobModel === undefined) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + const lease = LeaseFactory.createLeaseState( + new BlobLeaseAdapter(blobModel), + context + ).break(breakPeriod).lease; + + const leaseTimeSeconds: number = + lease.leaseState === Models.LeaseStateType.Breaking && + lease.leaseBreakTime + ? Math.round( + (lease.leaseBreakTime.getTime() - context.startTime!.getTime()) / + 1000 + ) + : 0; + + await BlobsModel.update( + this.convertLeaseToDbBlobsModel(blobModel.properties, lease), + { + where: { + accountName: account, + containerName: container, + blobName: blob, + snapshot: "" + }, + transaction: t + } + ); + return { properties: blobModel.properties, leaseTime: leaseTimeSeconds }; + }); + } + + public async checkBlobExist( + context: Context, + account: string, + container: string, + blob: string, + snapshot?: string | undefined + ): Promise { + await this.sequelize.transaction(async (t) => { + await this.assertContainerExists(context, account, container, t); + + const res = await BlobsModel.findOne({ + where: { + accountName: account, + containerName: container, + blobName: blob, + snapshot: snapshot ? snapshot : "" + }, + transaction: t + }); + + if (res === null || res === undefined) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + const blobModel: BlobModel = this.convertDbModelToBlobModel(res); + await this.validateExpireConditions(context, blobModel, t, true); + }); + } + + public async getBlobType( + account: string, + container: string, + blob: string, + snapshot?: string | undefined + ): Promise< + { blobType: Models.BlobType | undefined; isCommitted: boolean } | undefined + > { + const res = await BlobsModel.findOne({ + where: { + accountName: account, + containerName: container, + blobName: blob, + snapshot: snapshot ? snapshot : "" + } + }); + + if (res === null || res === undefined) { + return undefined; + } + + const blobModel: BlobModel = this.convertDbModelToBlobModel(res); + const valid = await this.validateExpireConditions( + undefined, + blobModel, + undefined, + false + ); + + if (!valid) return undefined; + + const properties = + this.deserializeModelValue( + res, + "properties", + true + ); + const blobType = properties.blobType; + const isCommitted = this.getModelValue(res, "isCommitted", true); + + return { blobType, isCommitted }; + } + + public async startCopyFromURL( + context: Context, + source: BlobId, + destination: BlobId, + copySource: string, + metadata: Models.BlobMetadata | undefined, + tier: Models.AccessTier | undefined, + options: Models.BlobStartCopyFromURLOptionalParams = {} + ): Promise { + return await this.sequelize.transaction(async (t) => { + const sourceBlob = await this.getBlobWithLeaseUpdated( + source.account, + source.container, + source.blob, + source.snapshot, + context, + true, + true, + t + ); + + await this.validateExpireConditions(context, sourceBlob, t, true); + options.sourceModifiedAccessConditions = + options.sourceModifiedAccessConditions || {}; + validateReadConditions( + context, + { + ifModifiedSince: + options.sourceModifiedAccessConditions.sourceIfModifiedSince, + ifUnmodifiedSince: + options.sourceModifiedAccessConditions.sourceIfUnmodifiedSince, + ifMatch: options.sourceModifiedAccessConditions.sourceIfMatch, + ifNoneMatch: options.sourceModifiedAccessConditions.sourceIfNoneMatch + }, + sourceBlob + ); + + const destBlob = await this.getBlobWithLeaseUpdated( + destination.account, + destination.container, + destination.blob, + undefined, + context, + false, + undefined, + t + ); + + validateWriteConditions( + context, + options.modifiedAccessConditions, + destBlob + ); + + if (destBlob) { + new BlobWriteLeaseValidator(options.leaseAccessConditions).validate( + new BlobLeaseAdapter(destBlob), + context + ); + } + + // Copy if not exists + if ( + options.modifiedAccessConditions && + options.modifiedAccessConditions.ifNoneMatch === "*" && + destBlob + ) { + throw StorageErrorFactory.getBlobAlreadyExists(context); + } + + // If source is uncommitted or deleted + if ( + sourceBlob === undefined || + sourceBlob.deleted || + !sourceBlob.isCommitted + ) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + if ( + sourceBlob.properties.accessTier === Models.AccessTier.Archive && + (tier === undefined || source.account !== destination.account) + ) { + throw StorageErrorFactory.getBlobArchived(context); + } + + await this.assertContainerExists( + context, + destination.account, + destination.container, + t + ); + + // Deep clone a copied blob + const copiedBlob: BlobModel = { + name: destination.blob, + deleted: false, + snapshot: "", + properties: { + ...sourceBlob.properties, + creationTime: context.startTime!, + lastModified: context.startTime!, + etag: newEtag(), + leaseStatus: + destBlob !== undefined + ? destBlob.properties.leaseStatus + : Models.LeaseStatusType.Unlocked, + leaseState: + destBlob !== undefined + ? destBlob.properties.leaseState + : Models.LeaseStateType.Available, + leaseDuration: + destBlob !== undefined + ? destBlob.properties.leaseDuration + : undefined, + copyId: uuid(), + copyStatus: Models.CopyStatusType.Success, + copySource, + copyProgress: sourceBlob.properties.contentLength + ? `${sourceBlob.properties.contentLength}/${sourceBlob.properties.contentLength}` + : undefined, + copyCompletionTime: context.startTime, + copyStatusDescription: undefined, + incrementalCopy: false, + destinationSnapshot: undefined, + deletedTime: undefined, + remainingRetentionDays: undefined, + archiveStatus: undefined, + accessTierChangeTime: undefined + }, + metadata: + metadata === undefined || Object.keys(metadata).length === 0 + ? { ...sourceBlob.metadata } + : metadata, + accountName: destination.account, + containerName: destination.container, + pageRangesInOrder: sourceBlob.pageRangesInOrder, + isCommitted: sourceBlob.isCommitted, + leaseDurationSeconds: + destBlob !== undefined ? destBlob.leaseDurationSeconds : undefined, + leaseId: destBlob !== undefined ? destBlob.leaseId : undefined, + leaseExpireTime: + destBlob !== undefined ? destBlob.leaseExpireTime : undefined, + leaseBreakTime: + destBlob !== undefined ? destBlob.leaseBreakTime : undefined, + committedBlocksInOrder: sourceBlob.committedBlocksInOrder, + persistency: sourceBlob.persistency, + isDirectory: sourceBlob.isDirectory, + permissions: sourceBlob.permissions, + acl: sourceBlob.acl, + owner: sourceBlob.owner, + group: sourceBlob.group + }; + + if ( + copiedBlob.properties.blobType === Models.BlobType.BlockBlob && + tier !== undefined + ) { + copiedBlob.properties.accessTier = this.parseTier(tier); + if (copiedBlob.properties.accessTier === undefined) { + throw StorageErrorFactory.getInvalidHeaderValue(context, { + HeaderName: "x-ms-access-tier", + HeaderValue: `${tier}` + }); + } + } + + if ( + copiedBlob.properties.blobType === Models.BlobType.PageBlob && + tier !== undefined + ) { + throw StorageErrorFactory.getInvalidHeaderValue(context, { + HeaderName: "x-ms-access-tier", + HeaderValue: `${tier}` + }); + } + + await this.upsertBlob(copiedBlob, t); + return copiedBlob.properties; + }); + } + + /** + * Copy from Url. + * + * @param {Context} context + * @param {BlobId} source + * @param {BlobId} destination + * @param {string} copySource + * @param {(Models.BlobMetadata | undefined)} metadata + * @param {(Models.AccessTier | undefined)} tier + * @param {Models.BlobCopyFromURLOptionalParams} [leaseAccessConditions] + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + public async copyFromURL( + context: Context, + source: BlobId, + destination: BlobId, + copySource: string, + metadata: Models.BlobMetadata | undefined, + tier: Models.AccessTier | undefined, + options: Models.BlobCopyFromURLOptionalParams = {} + ): Promise { + return await this.sequelize.transaction(async (t) => { + const sourceBlob = await this.getBlobWithLeaseUpdated( + source.account, + source.container, + source.blob, + source.snapshot, + context, + true, + true + ); + + await this.validateExpireConditions(context, sourceBlob, t, true); + options.sourceModifiedAccessConditions = + options.sourceModifiedAccessConditions || {}; + validateReadConditions( + context, + { + ifModifiedSince: + options.sourceModifiedAccessConditions.sourceIfModifiedSince, + ifUnmodifiedSince: + options.sourceModifiedAccessConditions.sourceIfUnmodifiedSince, + ifMatch: options.sourceModifiedAccessConditions.sourceIfMatch, + ifNoneMatch: options.sourceModifiedAccessConditions.sourceIfNoneMatch + }, + sourceBlob + ); + + const destBlob = await this.getBlobWithLeaseUpdated( + destination.account, + destination.container, + destination.blob, + undefined, + context, + false + ); + + validateWriteConditions( + context, + options.modifiedAccessConditions, + destBlob + ); + + // Copy if not exists + if ( + options.modifiedAccessConditions && + options.modifiedAccessConditions.ifNoneMatch === "*" && + destBlob + ) { + throw StorageErrorFactory.getBlobAlreadyExists(context); + } + + if (destBlob) { + const lease = new BlobLeaseAdapter(destBlob); + new BlobWriteLeaseSyncer(destBlob).sync(lease); + new BlobWriteLeaseValidator(options.leaseAccessConditions).validate( + lease, + context + ); + } + + // If source is uncommitted or deleted + if ( + sourceBlob === undefined || + sourceBlob.deleted || + !sourceBlob.isCommitted + ) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + if (sourceBlob.properties.accessTier === Models.AccessTier.Archive) { + throw StorageErrorFactory.getBlobArchived(context); + } + + await this.checkContainerExist( + context, + destination.account, + destination.container + ); + + // Deep clone a copied blob + const copiedBlob: BlobModel = { + name: destination.blob, + deleted: false, + snapshot: "", + properties: { + ...sourceBlob.properties, + creationTime: context.startTime!, + lastModified: context.startTime!, + etag: newEtag(), + leaseStatus: + destBlob !== undefined + ? destBlob.properties.leaseStatus + : Models.LeaseStatusType.Unlocked, + leaseState: + destBlob !== undefined + ? destBlob.properties.leaseState + : Models.LeaseStateType.Available, + leaseDuration: + destBlob !== undefined + ? destBlob.properties.leaseDuration + : undefined, + copyId: uuid(), + copyStatus: Models.CopyStatusType.Success, + copySource, + copyProgress: sourceBlob.properties.contentLength + ? `${sourceBlob.properties.contentLength}/${sourceBlob.properties.contentLength}` + : undefined, + copyCompletionTime: context.startTime, + copyStatusDescription: undefined, + incrementalCopy: false, + destinationSnapshot: undefined, + deletedTime: undefined, + remainingRetentionDays: undefined, + archiveStatus: undefined, + accessTierChangeTime: undefined + }, + metadata: + metadata === undefined || Object.keys(metadata).length === 0 + ? { ...sourceBlob.metadata } + : metadata, + accountName: destination.account, + containerName: destination.container, + pageRangesInOrder: sourceBlob.pageRangesInOrder, + isCommitted: sourceBlob.isCommitted, + leaseDurationSeconds: + destBlob !== undefined ? destBlob.leaseDurationSeconds : undefined, + leaseId: destBlob !== undefined ? destBlob.leaseId : undefined, + leaseExpireTime: + destBlob !== undefined ? destBlob.leaseExpireTime : undefined, + leaseBreakTime: + destBlob !== undefined ? destBlob.leaseBreakTime : undefined, + committedBlocksInOrder: sourceBlob.committedBlocksInOrder, + persistency: sourceBlob.persistency, + isDirectory: sourceBlob.isDirectory, + permissions: sourceBlob.permissions, + acl: sourceBlob.acl, + owner: sourceBlob.owner, + group: sourceBlob.group + }; + + if ( + copiedBlob.properties.blobType === Models.BlobType.BlockBlob && + tier !== undefined + ) { + copiedBlob.properties.accessTier = this.parseTier(tier); + if (copiedBlob.properties.accessTier === undefined) { + throw StorageErrorFactory.getInvalidHeaderValue(context, { + HeaderName: "x-ms-access-tier", + HeaderValue: `${tier}` + }); + } + } + + if ( + copiedBlob.properties.blobType === Models.BlobType.PageBlob && + tier !== undefined + ) { + throw StorageErrorFactory.getInvalidHeaderValue(context, { + HeaderName: "x-ms-access-tier", + HeaderValue: `${tier}` + }); + } + + if (destBlob) { + await BlobsModel.destroy({ + where: { + accountName: destBlob.accountName, + containerName: destBlob.containerName, + blobName: destBlob.name + }, + transaction: t + }); + } + + await this.upsertBlob(copiedBlob, t); + return copiedBlob.properties; + }); + } + + public async setTier( + context: Context, + account: string, + container: string, + blob: string, + tier: Models.AccessTier, + leaseAccessConditions?: Models.LeaseAccessConditions + ): Promise<200 | 202> { + return await this.sequelize.transaction(async (t) => { + await this.assertContainerExists(context, account, container, t); + + const blobFindResult = await BlobsModel.findOne({ + where: { + accountName: account, + containerName: container, + blobName: blob, + snapshot: "", + isCommitted: true + }, + transaction: t + }); + + if (blobFindResult === null || blobFindResult === undefined) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + let responseCode: 200 | 202 = 200; + + // check the lease action aligned with current lease state. + // the API has not lease ID input, but run it on a lease blocked blob will fail with LeaseIdMissing, + // this is aligned with server behavior + + const blobModel: BlobModel = + this.convertDbModelToBlobModel(blobFindResult); + + await this.validateExpireConditions(context, blobModel, t, true); + + LeaseFactory.createLeaseState( + new BlobLeaseAdapter(blobModel), + context + ).validate(new BlobWriteLeaseValidator(leaseAccessConditions)); + + // Check Blob is not snapshot + const snapshot = blobModel.snapshot; + if (snapshot !== "") { + throw StorageErrorFactory.getBlobSnapshotsPresent(context); + } + + // Check BlobTier matches blob type + let accessTier = blobModel.properties.accessTier; + const blobType = blobModel.properties.blobType; + if ( + (tier === Models.AccessTier.Archive || + tier === Models.AccessTier.Cool || + tier === Models.AccessTier.Hot) && + blobType === Models.BlobType.BlockBlob + ) { + // Block blob + // tslint:disable-next-line:max-line-length + // TODO: check blob is not block blob with snapshot, throw StorageErrorFactory.getBlobSnapshotsPresent_hassnapshot() + + // Archive -> Coo/Hot will return 202 + if ( + accessTier === Models.AccessTier.Archive && + (tier === Models.AccessTier.Cool || tier === Models.AccessTier.Hot) + ) { + responseCode = 202; + } + + accessTier = tier; + } else { + throw StorageErrorFactory.getAccessTierNotSupportedForBlobType(context); + } + + blobModel.properties.accessTier = accessTier; + blobModel.properties.accessTierInferred = false; + blobModel.properties.accessTierChangeTime = context.startTime; + await BlobsModel.update( + { + properties: this.serializeModelValue(blobModel.properties) + }, + { + where: { + accountName: account, + containerName: container, + blobName: blob, + snapshot: "" + }, + transaction: t + } + ); + + return responseCode; + }); + } + + public uploadPages( + context: Context, + blob: BlobModel, + start: number, + end: number, + persistency: IExtentChunk, + leaseAccessConditions?: Models.LeaseAccessConditions, + modifiedAccessConditions?: Models.ModifiedAccessConditions + ): Promise { + throw new NotImplementedError(context); + } + + public clearRange( + context: Context, + blob: BlobModel, + start: number, + end: number, + leaseAccessConditions?: Models.LeaseAccessConditions, + modifiedAccessConditions?: Models.ModifiedAccessConditions + ): Promise { + throw new NotImplementedError(context); + } + + public getPageRanges( + context: Context, + account: string, + container: string, + blob: string, + snapshot?: string | undefined, + leaseAccessConditions?: Models.LeaseAccessConditions, + modifiedAccessConditions?: Models.ModifiedAccessConditions + ): Promise { + throw new NotImplementedError(context); + } + + public resizePageBlob( + context: Context, + account: string, + container: string, + blob: string, + blobContentLength: number, + leaseAccessConditions?: Models.LeaseAccessConditions, + modifiedAccessConditions?: Models.ModifiedAccessConditions + ): Promise { + throw new NotImplementedError(context); + } + + public updateSequenceNumber( + context: Context, + account: string, + container: string, + blob: string, + sequenceNumberAction: Models.SequenceNumberActionType, + blobSequenceNumber: number | undefined + ): Promise { + throw new NotImplementedError(context); + } + + public async appendBlock( + context: Context, + block: BlockModel, + leaseAccessConditions: Models.LeaseAccessConditions = {}, + modifiedAccessConditions: Models.ModifiedAccessConditions = {}, + appendPositionAccessConditions: Models.AppendPositionAccessConditions = {} + ): Promise { + return await this.sequelize.transaction(async (t) => { + const doc = await this.getBlobWithLeaseUpdated( + block.accountName, + block.containerName, + block.blobName, + undefined, + context, + false, + true + ); + + await this.validateExpireConditions(context, doc, t, true); + validateWriteConditions(context, modifiedAccessConditions, doc); + + if (!doc) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + new BlobWriteLeaseValidator(leaseAccessConditions).validate( + new BlobLeaseAdapter(doc), + context + ); + + if (doc.properties.blobType !== Models.BlobType.AppendBlob) { + throw StorageErrorFactory.getBlobInvalidBlobType(context); + } + + if ( + (doc.committedBlocksInOrder || []).length >= MAX_APPEND_BLOB_BLOCK_COUNT + ) { + throw StorageErrorFactory.getBlockCountExceedsLimit(context); + } + + if (appendPositionAccessConditions.appendPosition !== undefined) { + if ( + (doc.properties.contentLength || 0) !== + appendPositionAccessConditions.appendPosition + ) { + throw StorageErrorFactory.getAppendPositionConditionNotMet(context); + } + } + + if (appendPositionAccessConditions.maxSize !== undefined) { + if ( + (doc.properties.contentLength || 0) + block.size > + appendPositionAccessConditions.maxSize + ) { + throw StorageErrorFactory.getMaxBlobSizeConditionNotMet(context); + } + } + + doc.committedBlocksInOrder = doc.committedBlocksInOrder || []; + doc.committedBlocksInOrder.push(block); + doc.properties.etag = newEtag(); + doc.properties.lastModified = context.startTime || new Date(); + doc.properties.contentLength = + (doc.properties.contentLength || 0) + block.size; + + await this.upsertBlob(doc, t); + return doc.properties; + }); + } + + public async listUncommittedBlockPersistencyChunks( + marker: string = "-1", + maxResults: number = 2000 + ): Promise<[IExtentChunk[], string | undefined]> { + return BlocksModel.findAll({ + attributes: ["id", "persistency"], + where: { + id: { + [Op.gt]: parseInt(marker, 10) + } + }, + limit: maxResults + 1, + order: [["id", "ASC"]] + }).then((res) => { + if (res.length < maxResults) { + return [ + res.map((obj) => { + return this.deserializeModelValue(obj, "persistency", true); + }), + undefined + ]; + } else { + res.pop(); + const nextMarker = this.getModelValue( + res[res.length - 1], + "id", + true + ); + return [ + res.map((obj) => + this.deserializeModelValue(obj, "persistency", true) + ), + nextMarker + ]; + } + }); + } + + /** + * Gets a blob item from persistency layer by container name and blob name. + * Will return block list or page list as well for downloading. + * + * @param {Context} context + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {string} [snapshot=""] + * @param {Models.LeaseAccessConditions} [leaseAccessConditions] + * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + getModel( + context: Context, + account: string, + container: string, + blob: string, + forceExist?: true, + leaseAccessConditions?: Models.LeaseAccessConditions, + modifiedAccessConditions?: Models.ModifiedAccessConditions, + validateRead?: boolean + ): Promise; + + getModel( + context: Context, + account: string, + container: string, + blob: string, + forceExist: false, + leaseAccessConditions?: Models.LeaseAccessConditions, + modifiedAccessConditions?: Models.ModifiedAccessConditions, + validateRead?: boolean + ): Promise; + + public async getModel( + context: Context, + account: string, + container: string, + blob: string, + forceExists: boolean = true, + leaseAccessConditions?: Models.LeaseAccessConditions, + modifiedAccessConditions?: Models.ModifiedAccessConditions, + validateRead: boolean = true + ): Promise { + return await this.sequelize.transaction(async (t) => { + const doc = await this.getBlobWithLeaseUpdated( + account, + container, + blob, + "", + context, + false, + true, + t + ); + + const valid = this.validateExpireConditions(context, doc, t, false); + if (validateRead) { + validateReadConditions(context, modifiedAccessConditions, doc); + } + + if (!doc || !valid) { + if (!forceExists) return undefined; + throw StorageErrorFactory.getBlobNotFound(context); + } + + new BlobReadLeaseValidator(leaseAccessConditions).validate( + new BlobLeaseAdapter(doc), + context + ); + + return doc; + }); + } + + public iteratorExtents(): AsyncIterator { + return new BlobReferredExtentsAsyncIterator(this); + } + + private async assertContainerExists( + context: Context, + account: string, + container: string, + transaction?: Transaction, + fullResult: boolean = false + ): Promise { + const findResult = await ContainersModel.findOne({ + attributes: fullResult ? undefined : ["accountName"], + where: { + accountName: account, + containerName: container + }, + transaction + }); + if (findResult === undefined || findResult === null) { + throw StorageErrorFactory.getContainerNotFound(context); + } + return findResult; + } + + private getModelValue(model: Model, key: string): T | undefined; + private getModelValue(model: Model, key: string, isRequired: true): T; + private getModelValue( + model: Model, + key: string, + isRequired?: boolean + ): T | undefined { + let value = model.get(key) as T | undefined; + if (value === null) { + value = undefined; + } + if (value === undefined && isRequired === true) { + // tslint:disable-next-line:max-line-length + throw new Error( + `SqlBlobMetadataStore:getModelValue() error. ${key} is required but value from database model is undefined.` + ); + } + return value; + } + + private deserializeModelValue( + model: Model, + key: string, + isRequired?: true + ): T; + + private deserializeModelValue( + model: Model, + key: string, + isRequired: false + ): T | undefined; + + private deserializeModelValue( + model: Model, + key: string, + isRequired: boolean = false + ): T | undefined { + const rawValue = this.getModelValue(model, key); + if (typeof rawValue === "string") { + // TODO: Decouple deserializer + return JSON.parse(rawValue) as T; + } + + if (isRequired) { + throw new Error( + // tslint:disable-next-line:max-line-length + `SqlBlobMetadataStore:deserializeModelValue() error. ${key} is required but value from database model is undefined.` + ); + } + + return undefined; + } + + private serializeModelValue(value: any): string | undefined { + if (value === undefined || value === null) { + return undefined; + } + + return JSON.stringify(value); + } + + /** + * This method will restore object to Uint8Array. + * + * @private + * @param {*} obj + * @returns {(Uint8Array | undefined)} + * @memberof LokiBlobMetadataStore + */ + private restoreUint8Array(obj: any): Uint8Array | undefined { + if (typeof obj !== "object") { + return undefined; + } + + if (obj instanceof Uint8Array) { + return obj; + } + + if (obj.type === "Buffer") { + obj = obj.data; + } + + const length = Object.keys(obj).length; + const arr = Buffer.allocUnsafe(length); + + for (let i = 0; i < length; i++) { + if (!obj.hasOwnProperty(i)) { + throw new TypeError( + `Cannot restore sql DB persisted object to Uint8Array. Key ${i} is missing.` + ); + } + + arr[i] = obj[i]; + } + + return arr; + } + + private convertDbModelToContainerModel( + dbModel: ContainersModel + ): ContainerModel { + const accountName = this.getModelValue( + dbModel, + "accountName", + true + ); + const name = this.getModelValue(dbModel, "containerName", true); + const containerAcl = this.deserializeModelValue( + dbModel, + "containerAcl" + ); + const metadata = this.deserializeModelValue<{ + [propertyName: string]: string; + }>(dbModel, "metadata"); + + const lastModified = this.getModelValue( + dbModel, + "lastModified", + true + ); + const etag = this.getModelValue(dbModel, "etag", true); + const publicAccess = this.deserializeModelValue( + dbModel, + "publicAccess" + ); + const lease = this.convertDbModelToLease(dbModel); + const leaseBreakTime = lease.leaseBreakTime; + const leaseExpireTime = lease.leaseExpireTime; + const leaseId = lease.leaseId; + const leaseDurationSeconds = lease.leaseDurationSeconds; + const leaseStatus = lease.leaseStatus; + const leaseState = lease.leaseState; + const leaseDuration = lease.leaseDurationType; + const hasImmutabilityPolicy = this.getModelValue( + dbModel, + "hasImmutabilityPolicy" + ); + const hasLegalHold = this.getModelValue(dbModel, "hasLegalHold"); + const fileSystemProperties = this.getModelValue( + dbModel, + "properties" + ); + + const ret: ContainerModel = { + accountName, + name, + properties: { + lastModified, + etag, + leaseStatus, + leaseDuration, + leaseState + }, + fileSystemProperties, + leaseId, + leaseBreakTime, + leaseExpireTime, + leaseDurationSeconds + }; + + if (metadata !== undefined) { + ret.metadata = metadata; + } + + if (containerAcl !== undefined) { + ret.containerAcl = containerAcl; + } + + if (publicAccess !== undefined) { + ret.properties.publicAccess = publicAccess; + } + + if (hasImmutabilityPolicy !== undefined) { + ret.properties.hasImmutabilityPolicy = hasImmutabilityPolicy; + } + + if (hasLegalHold !== undefined) { + ret.properties.hasLegalHold = hasLegalHold; + } + + return ret; + } + + private convertContainerModelToDbModel(container: ContainerModel): object { + const lease = new ContainerLeaseAdapter(container).toString(); + return { + accountName: container.accountName, + containerName: container.name, + lastModified: container.properties.lastModified, + properties: container.fileSystemProperties, + etag: container.properties.etag, + metadata: this.serializeModelValue(container.metadata), + containerAcl: this.serializeModelValue(container.containerAcl), + publicAccess: this.serializeModelValue(container.properties.publicAccess), + lease, + hasImmutabilityPolicy: container.properties.hasImmutabilityPolicy, + hasLegalHold: container.properties.hasLegalHold + }; + } + + private convertDbModelToBlobModel(dbModel: BlobsModel): BlobModel { + const blobModel: BlobModel = { + accountName: this.getModelValue(dbModel, "accountName", true), + containerName: this.getModelValue(dbModel, "containerName", true), + name: this.getModelValue(dbModel, "blobName", true), + isDirectory: this.getModelValue(dbModel, "isDirectory", true), + permissions: toPermissions( + this.getModelValue(dbModel, "permissions", true) + )!, + acl: toAcl(this.getModelValue(dbModel, "acl", true))!, + owner: this.getModelValue(dbModel, "owner", true), + group: this.getModelValue(dbModel, "group", true), + snapshot: this.getModelValue(dbModel, "snapshot", true), + isCommitted: this.getModelValue(dbModel, "isCommitted", true), + leaseDurationSeconds: this.getModelValue( + dbModel, + "leaseDurationSeconds" + ), + leaseBreakTime: this.getModelValue(dbModel, "leaseBreakTime"), + leaseExpireTime: this.getModelValue(dbModel, "leaseExpireTime"), + leaseId: this.getModelValue(dbModel, "leaseId"), + versionId: this.getModelValue(dbModel, "versionId"), + persistency: this.deserializeModelValue( + dbModel, + "persistency" + ), + committedBlocksInOrder: this.deserializeModelValue< + PersistencyBlockModel[] + >(dbModel, "committedBlocksInOrder"), + metadata: this.deserializeModelValue( + dbModel, + "metadata" + ), + properties: this.deserializeModelValue( + dbModel, + "properties" + ) + }; + + blobModel.properties.contentMD5 = this.restoreUint8Array( + blobModel.properties.contentMD5 + ); + return blobModel; + } + + private convertBlobModelToDbModel(blob: BlobModel): object { + return { + accountName: blob.accountName, + containerName: blob.containerName, + blobName: blob.name, + snapshot: blob.snapshot, + isCommitted: blob.isCommitted, + isDirectory: blob.isDirectory, + permissions: toPermissionsString(blob.permissions), + acl: toAclString(blob.acl), + owner: blob.owner, + group: blob.group, + lastModified: blob.properties.lastModified, + creationTime: blob.properties.creationTime || null, + leaseBreakTime: blob.leaseBreakTime || null, + leaseExpireTime: blob.leaseExpireTime || null, + leaseId: blob.leaseId || null, + leaseDurationSeconds: blob.leaseDurationSeconds || null, + versionId: blob.versionId || null, + persistency: this.serializeModelValue(blob.persistency) || null, + committedBlocksInOrder: + this.serializeModelValue(blob.committedBlocksInOrder) || null, + metadata: this.serializeModelValue(blob.metadata) || null, + properties: this.serializeModelValue(blob.properties) || null + }; + } + + private convertDbModelToLease(dbModel: ContainersModel | BlobsModel): ILease { + const lease = + (this.deserializeModelValue(dbModel, "lease") as ILease) || {}; + + if (lease.leaseBreakTime && typeof lease.leaseBreakTime === "string") { + lease.leaseBreakTime = new Date(lease.leaseBreakTime); + } + + if (lease.leaseExpireTime && typeof lease.leaseExpireTime === "string") { + lease.leaseExpireTime = new Date(lease.leaseExpireTime); + } + + return lease; + } + + private convertLeaseToDbBlobsModel( + properties: Models.BlobPropertiesInternal, + lease: ILease + ): object { + properties.leaseDuration = lease.leaseDurationType; + properties.leaseState = lease.leaseState; + properties.leaseStatus = lease.leaseStatus; + return { + leaseBreakTime: lease.leaseBreakTime || null, + leaseExpireTime: lease.leaseExpireTime || null, + leaseId: lease.leaseId || null, + leaseDurationSeconds: lease.leaseDurationSeconds || null, + properties: this.serializeModelValue(properties) + }; + } + + private convertLeaseToDbModel(lease: ContainerLeaseAdapter): object { + return { lease: lease.toString() }; + } + + /** + * Get a blob document model from Loki collection. + * Will throw BlobNotFound storage error if blob doesn't exist. + * + * @protected + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {(string | undefined)} snapshot + * @param {Context} context + * @param {undefined} [forceExist] + * @param {boolean} [forceCommitted] If true, will take uncommitted blob as a non-exist blob and throw exception. + * @returns {Promise} + * @memberof LokiBlobMetadataStore + */ + private async getBlobWithLeaseUpdated( + account: string, + container: string, + blob: string, + snapshot: string | undefined, + context: Context, + forceExist?: true, + forceCommitted?: boolean, + transaction?: Transaction + ): Promise; + + /** + * Get a blob document model from Loki collection. + * Will NOT throw BlobNotFound storage error if blob doesn't exist. + * + * @protected + * @param {string} account + * @param {string} container + * @param {string} blob + * @param {(string | undefined)} snapshot + * @param {Context} context + * @param {false} forceExist + * @param {boolean} [forceCommitted] If true, will take uncommitted blob as a non-exist blob and return undefined. + * @returns {(Promise)} + * @memberof LokiBlobMetadataStore + */ + private async getBlobWithLeaseUpdated( + account: string, + container: string, + blob: string, + snapshot: string | undefined, + context: Context, + forceExist: false, + forceCommitted?: boolean, + transaction?: Transaction + ): Promise; + + private async getBlobWithLeaseUpdated( + account: string, + container: string, + blob: string, + snapshot: string = "", + context: Context, + forceExist?: boolean, + forceCommitted?: boolean, + transaction?: Transaction + ): Promise { + await this.checkContainerExist(context, account, container); + + const blobFindResult = await BlobsModel.findOne({ + where: { + accountName: account, + containerName: container, + blobName: blob, + snapshot + }, + transaction + }); + + if (blobFindResult === null || blobFindResult === undefined) { + if (forceExist === false) { + return undefined; + } else { + throw StorageErrorFactory.getBlobNotFound(context); + } + } + + // Force exist if parameter forceExist is undefined or true + const doc = this.convertDbModelToBlobModel(blobFindResult); + if (forceExist === undefined || forceExist === true) { + if (forceCommitted) { + if (!doc || !(doc as BlobModel).isCommitted) { + throw StorageErrorFactory.getBlobNotFound(context); + } + } else { + if (!doc) { + throw StorageErrorFactory.getBlobNotFound(context); + } + } + } else { + if (forceCommitted) { + if (!doc || !(doc as BlobModel).isCommitted) { + return undefined; + } + } else { + if (!doc) { + return undefined; + } + } + } + + // Snapshot doesn't have lease + if (snapshot !== undefined && snapshot !== "") { + new BlobLeaseSyncer(doc).sync({ + leaseId: undefined, + leaseExpireTime: undefined, + leaseDurationSeconds: undefined, + leaseBreakTime: undefined, + leaseDurationType: undefined, + leaseState: Models.LeaseStateType.Available, // TODO: Lease state & status should be undefined for snapshots + leaseStatus: Models.LeaseStatusType.Unlocked // TODO: Lease state & status should be undefined for snapshots + }); + } else { + LeaseFactory.createLeaseState(new BlobLeaseAdapter(doc), context).sync( + new BlobLeaseSyncer(doc) + ); + } + + return doc; + } + + /** + * Get the tier setting from request headers. + * + * @private + * @param {string} tier + * @returns {(Models.AccessTier | undefined)} + * @memberof BlobHandler + */ + private parseTier(tier: string): Models.AccessTier | undefined { + tier = tier.toLowerCase(); + if (tier === Models.AccessTier.Hot.toLowerCase()) { + return Models.AccessTier.Hot; + } + if (tier === Models.AccessTier.Cool.toLowerCase()) { + return Models.AccessTier.Cool; + } + if (tier === Models.AccessTier.Archive.toLowerCase()) { + return Models.AccessTier.Archive; + } + return undefined; + } + + private async upsertBlob( + model: BlobModel, + transaction: Transaction + ): Promise { + if (!this.isPostgres) { + await BlobsModel.destroy({ + where: { + accountName: model.accountName, + containerName: model.containerName, + blobName: model.name, + snapshot: model.snapshot + }, + transaction + }); + } + await BlobsModel.upsert(this.convertBlobModelToDbModel(model), { + transaction + }); + } + + private async upsertBlock( + model: BlockModel, + transaction: Transaction + ): Promise { + if (!this.isPostgres) { + await BlocksModel.destroy({ + where: { + accountName: model.accountName, + containerName: model.containerName, + blobName: model.blobName, + blockName: model.name + }, + transaction + }); + } + + await BlocksModel.upsert( + { + accountName: model.accountName, + containerName: model.containerName, + blobName: model.blobName, + blockName: model.name, + size: model.size, + persistency: this.serializeModelValue(model.persistency) + }, + { transaction } + ); + } + + private async validateExpireConditions( + context: Context | undefined, + model: BlobModel | undefined, + transaction: Transaction | undefined, + throwError: boolean = true + ): Promise { + const now = new Date(); + if ( + model && + model.properties.expiresOn !== undefined && + //date is converted to string in DB so need to restore it + new Date(model.properties.expiresOn) < now + ) { + await BlobsModel.destroy({ + where: { + accountName: model.accountName, + containerName: model.containerName, + blobName: model.name, + snapshot: model.snapshot + }, + transaction + }); + + if (throwError && context !== undefined) { + throw StorageErrorFactory.getBlobNotFound(context); + } + + return false; + } + + return true; + } +} diff --git a/src/dfs/storage-file-datalake/models.ts b/src/dfs/storage-file-datalake/models.ts new file mode 100644 index 000000000..ff71be66f --- /dev/null +++ b/src/dfs/storage-file-datalake/models.ts @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +export declare interface PathAccessControlItem { + /** + * Indicates whether this is the default entry for the ACL. + */ + defaultScope: boolean; + /** + * Specifies which role this entry targets. + */ + accessControlType: AccessControlType; + /** + * Specifies the entity for which this entry applies. + */ + entityId: string; + /** + * Access control permissions. + */ + permissions: RolePermissions; +} + +export declare type AccessControlType = "user" | "group" | "mask" | "other"; + +export declare interface RolePermissions { + read: boolean; + write: boolean; + execute: boolean; +} + +export declare interface PathPermissions { + owner: RolePermissions; + group: RolePermissions; + other: RolePermissions; + stickyBit: boolean; + extendedAcls: boolean; +} + +export declare interface RemovePathAccessControlItem { + /** + * Indicates whether this is the default entry for the ACL. + */ + defaultScope: boolean; + /** + * Specifies which role this entry targets. + */ + accessControlType: AccessControlType; + /** + * Specifies the entity for which this entry applies. + * Must be omitted for types mask or other. It must also be omitted when the user or group is the owner. + */ + entityId?: string; +} diff --git a/src/dfs/storage-file-datalake/transforms.ts b/src/dfs/storage-file-datalake/transforms.ts new file mode 100644 index 000000000..484bdd772 --- /dev/null +++ b/src/dfs/storage-file-datalake/transforms.ts @@ -0,0 +1,288 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +import { + PathAccessControlItem, + PathPermissions, + RemovePathAccessControlItem, + RolePermissions +} from "./models"; + +export function toRolePermissions( + permissionsString: string +): RolePermissions | undefined { + if (permissionsString.length !== 3) { + return undefined; + } + + permissionsString = permissionsString.toLowerCase(); + + let read = false; + if (permissionsString[0] === "r") { + read = true; + } else if (permissionsString[0] !== "-") { + return undefined; + } + + let write = false; + if (permissionsString[1] === "w") { + write = true; + } else if (permissionsString[1] !== "-") { + return undefined; + } + + let execute = false; + if (permissionsString[2] === "x") { + execute = true; + } else if (permissionsString[2] !== "-") { + return undefined; + } + + return { read, write, execute }; +} + +const permissionsMap: Map = new Map(); +permissionsMap.set("0", "---"); +permissionsMap.set("1", "--x"); +permissionsMap.set("2", "-w-"); +permissionsMap.set("3", "-wx"); +permissionsMap.set("4", "r--"); +permissionsMap.set("5", "r-x"); +permissionsMap.set("6", "rw-"); +permissionsMap.set("7", "rwx"); + +function normalizePermissionsString( + permissionsString: string, + umask?: string +): string | undefined { + let permissionsNumber = parseInt(permissionsString, 8); + if (!isNaN(permissionsNumber) && permissionsString.length === 4) { + const umaskNumber = parseInt(umask || "not a number", 8); + if (umask !== undefined && umask.length == 4 && !isNaN(umaskNumber)) { + permissionsNumber = permissionsNumber & ~umaskNumber; + permissionsString = permissionsNumber.toString(8).padStart(4, "0"); + } + permissionsString = + permissionsMap.get(permissionsString[1])! + + permissionsMap.get(permissionsString[2]) + + permissionsMap.get(permissionsString[3]); + } + + if (permissionsString.length !== 9 && permissionsString.length !== 10) { + return undefined; + } + + if (permissionsString[8] === "t") { + const firstPart = permissionsString.substr(0, 8); + const lastPart = permissionsString.substr(9); + permissionsString = firstPart + "x" + lastPart; + } else if (permissionsString[8] === "T") { + const firstPart = permissionsString.substr(0, 8); + const lastPart = permissionsString.substr(9); + permissionsString = firstPart + "-" + lastPart; + } + + // Case insensitive + return permissionsString.toLowerCase(); +} + +export function toPermissions( + permissionsString: string, + umask?: string +): PathPermissions | undefined { + const permissionsStr = normalizePermissionsString(permissionsString, umask); + if (permissionsStr === undefined) return undefined; + permissionsString = permissionsStr; + let extendedAcls = false; + if (permissionsString.length === 10) { + if (permissionsString[9] === "+") { + extendedAcls = true; + } else { + return undefined; + } + } + + const owner = toRolePermissions(permissionsString.substr(0, 3)); + const group = toRolePermissions(permissionsString.substr(3, 3)); + const other = toRolePermissions(permissionsString.substr(6, 3)); + + if (owner === undefined || group === undefined || other === undefined) { + return undefined; + } + + return { + owner, + group, + other, + stickyBit: permissionsString[8] === "t" || permissionsString[8] === "T", + extendedAcls + }; +} + +export function toAccessControlItem( + aclItemString: string +): PathAccessControlItem | undefined { + if (aclItemString === "") return undefined; + + aclItemString = aclItemString.toLowerCase(); + + const parts = aclItemString.split(":"); + if (parts.length < 3 || parts.length > 4) { + return undefined; + } + + let defaultScope = false; + let index = 0; + if (parts.length === 4) { + if (parts[index] !== "default") return undefined; + defaultScope = true; + index++; + } + + const accessControlType = parts[index++]; + if ( + accessControlType !== "user" && + accessControlType !== "group" && + accessControlType !== "mask" && + accessControlType !== "other" + ) { + return undefined; + } + + const entityId = parts[index++]; + const permissions = toRolePermissions(parts[index++]); + + if (permissions === undefined) return undefined; + + return { + defaultScope, + accessControlType, + entityId, + permissions + }; +} + +export function toRemoveAccessControlItem( + aclItemString: string +): RemovePathAccessControlItem | undefined { + if (aclItemString === "") { + return undefined; + } + + aclItemString = aclItemString.toLowerCase(); + + const parts = aclItemString.split(":"); + if (parts.length < 1 || parts.length > 3) { + return undefined; + } + + if (parts.length === 3) { + if (parts[0] !== "default") { + return undefined; + } + } + + let defaultScope = false; + let index = 0; + if (parts[index] === "default") { + defaultScope = true; + index++; + } + + const accessControlType = parts[index++]; + if ( + accessControlType !== "user" && + accessControlType !== "group" && + accessControlType !== "mask" && + accessControlType !== "other" + ) { + return undefined; + } + + const entityId = parts[index++]; + + return { + defaultScope, + accessControlType, + entityId + }; +} + +export function toAcl(aclString?: string): PathAccessControlItem[] | undefined { + if (aclString === undefined || aclString === "" || aclString === null) { + return []; + } + + const acls = []; + const aclParts = aclString.split(","); + for (const aclPart of aclParts) { + const acl = toAccessControlItem(aclPart); + if (acl === undefined) return undefined; + acls.push(acl); + } + + return acls; +} + +export function toRemoveAcl( + aclString?: string +): RemovePathAccessControlItem[] | undefined { + if (aclString === undefined || aclString === "" || aclString === null) { + return []; + } + + const acls = []; + const aclParts = aclString.split(","); + for (const aclPart of aclParts) { + const acl = toRemoveAccessControlItem(aclPart); + if (acl === undefined) return undefined; + acls.push(acl); + } + + return acls; +} + +export function toAccessControlItemString(item: PathAccessControlItem): string { + const entityIdString = item.entityId !== undefined ? `:${item.entityId}` : ""; + const permissionsString = + item.permissions !== undefined + ? `:${toRolePermissionsString(item.permissions)}` + : ""; + return `${item.defaultScope ? "default:" : ""}${ + item.accessControlType + }${entityIdString}${permissionsString}`; +} + +export function toAclString(acl: PathAccessControlItem[]): string { + return acl.map(toAccessControlItemString).join(","); +} + +export function toRolePermissionsString( + p: RolePermissions, + stickyBit: boolean = false +): string { + return `${p.read ? "r" : "-"}${p.write ? "w" : "-"}${ + stickyBit ? (p.execute ? "t" : "T") : p.execute ? "x" : "-" + }`; +} + +export function toPermissionsString(permissions: PathPermissions): string { + return `${toRolePermissionsString( + permissions.owner + )}${toRolePermissionsString(permissions.group)}${toRolePermissionsString( + permissions.other, + permissions.stickyBit + )}${permissions.extendedAcls ? "+" : ""}`; +} + +export function permissionsStringToAclString( + permissions: string +): string | undefined { + const normalizedPermissions = normalizePermissionsString(permissions, "0000"); + if (normalizedPermissions === undefined) return undefined; + + return ( + `user::${normalizedPermissions.substring(0, 3)},` + + `group::${normalizedPermissions.substring(3, 6)},` + + `other::${normalizedPermissions.substring(6, 9)}` + ); +} diff --git a/src/dfs/utils/constants.ts b/src/dfs/utils/constants.ts new file mode 100644 index 000000000..1200130a7 --- /dev/null +++ b/src/dfs/utils/constants.ts @@ -0,0 +1,197 @@ +import { StoreDestinationArray } from "../../common/persistence/IExtentStore"; +import * as Models from "../generated/artifacts/models"; +import { getGroup, getUser } from "./utils"; + +export const KB: number = 1024; +export const MB: number = KB * 1024; +export const GB: number = MB * 1024; +export const TB: number = GB * 1024; + +export const FILE_MAX_SINGLE_UPLOAD_THRESHOLD: number = 100 * MB; +export const FILE_UPLOAD_MAX_CHUNK_SIZE: number = 4000 * MB; +export const MAX_APPEND_BLOB_BLOCK_SIZE = FILE_MAX_SINGLE_UPLOAD_THRESHOLD; +export const MAX_APPEND_BLOB_BLOCK_COUNT = 500; +export const BLOCK_BLOB_MAX_BLOCKS: number = 50000; + +export const VERSION = "3.23.0"; +export const BLOB_API_VERSION = "2022-11-02"; +export const DATA_LAKE_API_VERSION = "2021-4-10"; +export const DEFAULT_DATA_LAKE_SERVER_HOST_NAME = "127.0.0.1"; // Change to 0.0.0.0 when needs external access +export const DEFAULT_DATA_LAKE_LISTENING_PORT = 10003; +export const DEFAULT_CONTEXT_PATH = "azurite_dfs_context"; +//https://learn.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/path/create +//The default permission is 0777 for a directory and 0666 for a file. The default umask is 0027. +export const DEFAULT_FILE_PERMISSIONS = "0666"; +export const DEFAULT_DIR_PERMISSIONS = "0777"; +export const DEFAULT_UMMASK = "0027"; +const user = getUser(); +const group = getGroup(); +export const DEFAULT_OWNER = user; +export const DEFAULT_GROUP = group; +export const DEFAULT_DATA_LAKE_LOKI_DB_PATH = "__azurite_db_datalake__.json"; +export const DEFAULT_DATA_LAKE_PERSISTENCE_PATH = "__datalakestorage__"; +export const DEFAULT_DATA_LAKE_EXTENT_LOKI_DB_PATH = + "__azurite_db_datalake_extent__.json"; + +export const DEFAULT_LIST_BLOBS_MAX_RESULTS = 5000; +export const DEFAULT_LIST_CONTAINERS_MAX_RESULTS = 5000; +export const DEFAULT_DEBUG_LOG_PATH = "./debug.log"; +export const DEFAULT_ENABLE_DEBUG_LOG = true; +export const DEFAULT_ACCESS_LOG_PATH = "./access.log"; +export const DEFAULT_ENABLE_ACCESS_LOG = true; +export const LOGGER_CONFIGS = {}; +export const DEFAULT_GC_INTERVAL_MS = 10 * 60 * 1000; +export const DEFAULT_WRITE_CONCURRENCY_PER_LOCATION = 50; +export const EMULATOR_ACCOUNT_NAME = "devstoreaccount1"; +export const EMULATOR_ACCOUNT_KEY_STR = + "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="; +export const EMULATOR_ACCOUNT_KEY = Buffer.from( + EMULATOR_ACCOUNT_KEY_STR, + "base64" +); + +export const EMULATOR_ACCOUNT_SKUNAME = Models.SkuName.StandardRAGRS; +export const EMULATOR_ACCOUNT_KIND = Models.AccountKind.StorageV2; + +export const HeaderConstants = { + AUTHORIZATION: "authorization", + AUTHORIZATION_SCHEME: "Bearer", + CONTENT_ENCODING: "content-encoding", + CONTENT_LANGUAGE: "content-language", + CONTENT_LENGTH: "content-length", + CONTENT_MD5: "content-md5", + CONTENT_TYPE: "content-type", + COOKIE: "Cookie", + DATE: "date", + IF_MATCH: "if-match", + IF_MODIFIED_SINCE: "if-modified-since", + IF_NONE_MATCH: "if-none-match", + IF_UNMODIFIED_SINCE: "if-unmodified-since", + SOURCE_IF_MATCH: "x-ms-source-if-match", + SOURCE_IF_MODIFIED_SINCE: "x-ms-source-if-modified-since", + SOURCE_IF_NONE_MATCH: "x-ms-source-if-none-match", + SOURCE_IF_UNMODIFIED_SINCE: "x-ms-source-if-unmodified-since", + X_MS_IF_SEQUENCE_NUMBER_LE: "x-ms-if-sequence-number-le", + X_MS_IF_SEQUENCE_NUMBER_LT: "x-ms-if-sequence-number-lt", + X_MS_IF_SEQUENCE_NUMBER_EQ: "x-ms-if-sequence-number-eq", + X_MS_BLOB_CONDITION_MAXSIZE: "x-ms-blob-condition-maxsize", + X_MS_BLOB_CONDITION_APPENDPOS: "x-ms-blob-condition-appendpos", + X_MS_SEQUENCE_NUMBER_ACTION: "x-ms-sequence-number-action", + X_MS_BLOB_SEQUENCE_NUMBER: "x-ms-blob-sequence-number", + X_MS_CONTENT_CRC64: "x-ms-content-crc64", + X_MS_RANGE_GET_CONTENT_CRC64: "x-ms-range-get-content-crc64", + X_MS_ENCRYPTION_KEY: "x-ms-encryption-key", + X_MS_ENCRYPTION_KEY_SHA256: "x-ms-encryption-key-sha256", + X_MS_ENCRYPTION_ALGORITHM: "x-ms-encryption-algorithm", + PREFIX_FOR_STORAGE: "x-ms-", + RANGE: "Range", + USER_AGENT: "User-Agent", + X_MS_CLIENT_REQUEST_ID: "x-ms-client-request-id", + X_MS_DATE: "x-ms-date", + SERVER: "Server", + X_MS_META: "x-ms-meta-", + X_MS_VERSION: "x-ms-version", + ORIGIN: "origin", + VARY: "Vary", + ACCESS_CONTROL_EXPOSE_HEADERS: "Access-Control-Expose-Headers", + ACCESS_CONTROL_ALLOW_ORIGIN: "Access-Control-Allow-Origin", + ACCESS_CONTROL_ALLOW_CREDENTIALS: "Access-Control-Allow-Credentials", + ACCESS_CONTROL_ALLOW_METHODS: "Access-Control-Allow-Methods", + ACCESS_CONTROL_ALLOW_HEADERS: "Access-Control-Allow-Headers", + ACCESS_CONTROL_MAX_AGE: "Access-Control-Max-Age", + ACCESS_CONTROL_REQUEST_METHOD: "access-control-request-method", + ACCESS_CONTROL_REQUEST_HEADERS: "access-control-request-headers" +}; + +export const MethodConstants = { + OPTIONS: "OPTIONS" +}; + +export const SECONDARY_SUFFIX = "-secondary"; + +export const DEFAULT_DATA_LAKE_PERSISTENCE_ARRAY: StoreDestinationArray = [ + { + locationId: "Default", + locationPath: DEFAULT_DATA_LAKE_PERSISTENCE_PATH, + maxConcurrency: DEFAULT_WRITE_CONCURRENCY_PER_LOCATION + } +]; + +export const ValidAPIVersions = [ + "2022-11-02", + "2021-12-02", + "2021-10-04", + "2021-08-06", + "2021-06-08", + "2021-04-10", + "2021-02-12", + "2020-12-06", + "2020-10-02", + "2020-08-04", + "2020-06-12", + "2020-04-08", + "2020-02-10", + "2019-12-12", + "2019-10-10", + "2019-07-07", + "2019-02-02", + "2018-11-09", + "2018-03-28", + "2017-11-09", + "2017-07-29", + "2017-04-17", + "2016-05-31", + "2015-12-11", + "2015-07-08", + "2015-04-05", + "2015-02-21", + "2014-02-14", + "2013-08-15", + "2012-02-12", + "2011-08-18", + "2009-09-19", + "2009-07-17", + "2009-04-14" +]; + +// Validate audience, accept following audience patterns +// https://storage.azure.com +// https://storage.azure.com/ +// e406a681-f3d4-42a8-90b6-c2b029497af1 +// https://*.blob.core.windows.net +// https://*.blob.core.windows.net/ +// https://*.blob.core.chinacloudapi.cn +// https://*.blob.core.chinacloudapi.cn/ +// https://*.blob.core.usgovcloudapi.net +// https://*.blob.core.usgovcloudapi.net/ +// https://*.blob.core.cloudapi.de +// https://*.blob.core.cloudapi.de/ +// https://*.dfs.core.windows.net +// https://*.dfs.core.windows.net/ +// https://*.dfs.core.chinacloudapi.cn +// https://*.dfs.core.chinacloudapi.cn/ +// https://*.dfs.core.usgovcloudapi.net +// https://*.dfs.core.usgovcloudapi.net/ +// https://*.dfs.core.cloudapi.de +// https://*.dfs.core.cloudapi.de/ +export const VALID_DATALAKE_AUDIENCES = [ + /^https:\/\/storage\.azure\.com[\/]?$/, + /^e406a681-f3d4-42a8-90b6-c2b029497af1$/, + /^https:\/\/(.*)\.blob\.core\.windows\.net[\/]?$/, + /^https:\/\/(.*)\.blob\.core\.chinacloudapi\.cn[\/]?$/, + /^https:\/\/(.*)\.blob\.core\.usgovcloudapi\.net[\/]?$/, + /^https:\/\/(.*)\.blob\.core\.cloudapi\.de[\/]?$/, + /^https:\/\/(.*)\.dfs\.core\.windows\.net[\/]?$/, + /^https:\/\/(.*)\.dfs\.core\.chinacloudapi\.cn[\/]?$/, + /^https:\/\/(.*)\.dfs\.core\.usgovcloudapi\.net[\/]?$/, + /^https:\/\/(.*)\.dfs\.core\.cloudapi\.de[\/]?$/ +]; + +export const HTTP_LINE_ENDING = "\r\n"; +export const HTTP_HEADER_DELIMITER = ": "; + +export const USERDELEGATIONKEY_BASIC_KEY = + "I17GKLvcJUossaebtsEDZZ2RJ8GNLwLH4m7hRMxbVbkx6wNIRAABj4Rtw0FBhFuEAgmbL4gFMzUw+AStz9Sqdg=="; + +export const AUTHENTICATION_BEARERTOKEN_REQUIRED = + "Only authentication scheme Bearer is supported"; diff --git a/src/dfs/utils/utils.ts b/src/dfs/utils/utils.ts new file mode 100644 index 000000000..c4ec6def2 --- /dev/null +++ b/src/dfs/utils/utils.ts @@ -0,0 +1,226 @@ +import { createHmac } from "crypto"; +import { createWriteStream, PathLike } from "fs"; +import StorageErrorFactory from "../errors/StorageErrorFactory"; +import { USERDELEGATIONKEY_BASIC_KEY } from "./constants"; +import Context from "../generated/Context"; +import Operation from "../generated/artifacts/operation"; +import IRequest from "../generated/IRequest"; +import os from "os"; +import { execSync, ExecSyncOptionsWithStringEncoding } from "child_process"; + +export function checkApiVersion( + inputApiVersion: string, + validApiVersions: Array, + context: Context +): void { + if (!validApiVersions.includes(inputApiVersion)) { + throw StorageErrorFactory.getInvalidAPIVersion(context, inputApiVersion); + } +} + +export async function streamToLocalFile( + stream: NodeJS.ReadableStream, + path: PathLike +): Promise { + return new Promise((resolve, reject) => { + const writeStream = createWriteStream(path); + stream + .on("error", reject) + // .on("end", resolve) + .pipe(writeStream) + .on("close", resolve) + .on("error", reject); + }); +} + +/** + * Default range value [0, Infinite] will be returned if all parameters not provided. + * + * @export + * @param {string} [rangeHeaderValue] + * @param {string} [xMsRangeHeaderValue] + * @returns {[number, number]} + */ +export function deserializeRangeHeader( + rangeHeaderValue?: string, + xMsRangeHeaderValue?: string +): [number, number] { + const range = xMsRangeHeaderValue || rangeHeaderValue; + if (!range) { + return [0, Infinity]; + } + + let parts = range.split("="); + if (parts === undefined || parts.length !== 2) { + throw new RangeError( + `deserializeRangeHeader: raw range value ${range} is wrong.` + ); + } + + parts = parts[1].split("-"); + if (parts === undefined || parts.length < 1 || parts.length > 2) { + throw new RangeError( + `deserializeRangeHeader: raw range value ${range} is wrong.` + ); + } + + const startInclusive = parseInt(parts[0], 10); + let endInclusive = Infinity; + + if (parts.length > 1 && parts[1] !== "") { + endInclusive = parseInt(parts[1], 10); + } + + if (startInclusive > endInclusive) { + throw new RangeError( + `deserializeRangeHeader: raw range value ${range} is wrong.` + ); + } + + return [startInclusive, endInclusive]; +} + +/** + * Deserialize range header into valid page ranges. + * For example, "bytes=0-1023" will return [0, 1023]. + * + * Default range value [0, Infinite] will be returned if all parameters not provided. + * + * @private + * @param {string} [rangeHeaderValue] + * @param {string} [xMsRangeHeaderValue] + * @returns {([number, number] | undefined)} + */ +export function deserializePageBlobRangeHeader( + rangeHeaderValue?: string, + xMsRangeHeaderValue?: string, + force512boundary = true +): [number, number] { + const ranges = deserializeRangeHeader(rangeHeaderValue, xMsRangeHeaderValue); + const startInclusive = ranges[0]; + const endInclusive = ranges[1]; + + if (force512boundary && startInclusive % 512 !== 0) { + throw new RangeError( + `deserializePageBlobRangeHeader: range start value ${startInclusive} doesn't align with 512 boundary.` + ); + } + + if ( + force512boundary && + endInclusive !== Infinity && + (endInclusive + 1) % 512 !== 0 + ) { + throw new RangeError( + `deserializePageBlobRangeHeader: range end value ${endInclusive} doesn't align with 512 boundary.` + ); + } + + return [startInclusive, endInclusive]; +} + +/** + * Remove double Quotation mark from ListBlob returned Etag, to align with server + * + * @param {string} [inputEtag] + * @returns {string} + */ +export function removeQuotationFromListBlobEtag(inputEtag: string): string { + if (inputEtag === undefined) { + return inputEtag; + } + if (inputEtag[0] === '"' && inputEtag[inputEtag.length - 1] === '"') { + return inputEtag.substring(1, inputEtag.length - 1); + } + return inputEtag; +} + +export function validateContainerName(context: Context, containerName: string) { + if ( + containerName !== "" && + (containerName!.length < 3 || containerName!.length > 63) + ) { + throw StorageErrorFactory.getOutOfRangeName(context); + } + const reg = new RegExp("^[a-z0-9](?!.*--)[a-z0-9-]{1,61}[a-z0-9]$"); + if (!reg.test(containerName!)) { + throw StorageErrorFactory.getInvalidResourceName(context); + } +} + +export function getUserDelegationKeyValue( + signedObjectid: string, + signedTenantid: string, + signedStartsOn: string, + signedExpiresOn: string, + signedVersion: string +): string { + const stringToSign = [ + signedObjectid, + signedTenantid, + signedStartsOn, + signedExpiresOn, + "b", + signedVersion + ].join("\n"); + + return createHmac("sha256", USERDELEGATIONKEY_BASIC_KEY) + .update(stringToSign, "utf8") + .digest("base64"); +} + +const DATA_LAKE_OPERATIONS = [ + Operation.FileSystem_Create, + Operation.FileSystem_SetProperties, + Operation.FileSystem_GetProperties, + Operation.FileSystem_Delete, + Operation.FileSystem_ListPaths, + Operation.Path_Create, + Operation.Path_Update, + Operation.Path_Lease, + Operation.Path_Delete, + Operation.Path_SetAccessControl, + Operation.Path_SetAccessControlRecursive, + Operation.Path_FlushData, + Operation.Path_AppendData, + Operation.Path_SetExpiry, + Operation.Path_Undelete +]; + +const COMMON_OPERATIONS = [Operation.Path_Read, Operation.Path_GetProperties]; + +export function isDataLakeOperation( + context: Context, + request: IRequest | undefined = context.request +): boolean { + const accept = request?.getHeader("Accept"); + return ( + DATA_LAKE_OPERATIONS.includes(context.operation!) || + (COMMON_OPERATIONS.includes(context.operation!) && + accept !== undefined && + accept.includes("json")) + ); +} + +export function removeSlash(path: string): string { + if (!path.endsWith("/")) return path; + + return path.substring(0, path.length - 1); +} + +export function getUser(): string { + return os.userInfo().username; +} + +export function getGroup(): string { + if (os.platform() === "win32") return ""; + const options: ExecSyncOptionsWithStringEncoding = { + encoding: "utf8" + }; + const output = execSync("id -gn", options); + if (output) { + return output.substring(0, output.length - 1); //remove trailing \n + } + + return ""; +} diff --git a/src/extension.ts b/src/extension.ts index 84bf64b26..2e2a847b2 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -4,6 +4,7 @@ import VSCAccessLog from "./common/VSCAccessLog"; import VSCNotification from "./common/VSCNotification"; import VSCProgress from "./common/VSCProgress"; import VSCServerManagerBlob from "./common/VSCServerManagerBlob"; +import VSCServerManagerDataLake from "./common/VSCServerManagerDataLake"; import VSCServerManagerQueue from "./common/VSCServerManagerQueue"; import VSCServerManagerTable from "./common/VSCServerManagerTable"; import VSCStatusBarItem from "./common/VSCStatusBarItem"; @@ -11,6 +12,7 @@ import VSCStatusBarItem from "./common/VSCStatusBarItem"; export function activate(context: ExtensionContext) { // Initialize server managers const blobServerManager = new VSCServerManagerBlob(); + const dataLakeServerManager = new VSCServerManagerDataLake(); const queueServerManager = new VSCServerManagerQueue(); const tableServerManager = new VSCServerManagerTable(); @@ -27,21 +29,28 @@ export function activate(context: ExtensionContext) { tableServerManager, window.createStatusBarItem(StatusBarAlignment.Right, 1002) ); + const vscDataLakeStatusBar = new VSCStatusBarItem( + dataLakeServerManager, + window.createStatusBarItem(StatusBarAlignment.Right, 1003) + ); blobServerManager.addEventListener(vscBlobStatusBar); queueServerManager.addEventListener(vscQueueStatusBar); tableServerManager.addEventListener(vscTableStatusBar); + dataLakeServerManager.addEventListener(vscDataLakeStatusBar); // Hook up notification handlers const notification = new VSCNotification(); blobServerManager.addEventListener(notification); queueServerManager.addEventListener(notification); tableServerManager.addEventListener(notification); + dataLakeServerManager.addEventListener(notification); // Hook up progress handlers blobServerManager.addEventListener(new VSCProgress()); queueServerManager.addEventListener(new VSCProgress()); tableServerManager.addEventListener(new VSCProgress()); + dataLakeServerManager.addEventListener(new VSCProgress()); // Hook up access log handlers blobServerManager.addEventListener( @@ -53,22 +62,28 @@ export function activate(context: ExtensionContext) { tableServerManager.addEventListener( new VSCAccessLog(tableServerManager.accessChannelStream) ); + dataLakeServerManager.addEventListener( + new VSCAccessLog(dataLakeServerManager.accessChannelStream) + ); context.subscriptions.push( commands.registerCommand("azurite.start", () => { blobServerManager.start(); queueServerManager.start(); tableServerManager.start(); + dataLakeServerManager.start(); }), commands.registerCommand("azurite.close", () => { blobServerManager.close(); queueServerManager.close(); tableServerManager.close(); + dataLakeServerManager.close(); }), commands.registerCommand("azurite.clean", () => { blobServerManager.clean(); queueServerManager.clean(); tableServerManager.clean(); + dataLakeServerManager.clean(); }), commands.registerCommand(blobServerManager.getStartCommand(), async () => { @@ -101,9 +116,23 @@ export function activate(context: ExtensionContext) { tableServerManager.clean(); }), + commands.registerCommand( + dataLakeServerManager.getStartCommand(), + async () => { + await dataLakeServerManager.start(); + } + ), + commands.registerCommand(dataLakeServerManager.getCloseCommand(), () => { + dataLakeServerManager.close(); + }), + commands.registerCommand(dataLakeServerManager.getCleanCommand(), () => { + dataLakeServerManager.clean(); + }), + vscBlobStatusBar.statusBarItem, vscQueueStatusBar.statusBarItem, - vscTableStatusBar.statusBarItem + vscTableStatusBar.statusBarItem, + vscDataLakeStatusBar.statusBarItem ); } diff --git a/swagger/blob-storage-2021-10-04-data-lake.json b/swagger/blob-storage-2021-10-04-data-lake.json new file mode 100644 index 000000000..3e316d405 --- /dev/null +++ b/swagger/blob-storage-2021-10-04-data-lake.json @@ -0,0 +1,10916 @@ +{ + "swagger": "2.0", + "info": { + "title": "Azure Blob Storage", + "version": "2021-10-04", + "x-ms-code-generation-settings": { + "header": "MIT", + "strictSpecAdherence": false + } + }, + "x-ms-parameterized-host": { + "hostTemplate": "{url}", + "useSchemePrefix": false, + "positionInOperation": "first", + "parameters": [ + { + "$ref": "#/parameters/Url" + } + ] + }, + "schemes": [ + "https" + ], + "consumes": [ + "application/xml" + ], + "produces": [ + "application/xml" + ], + "paths": {}, + "x-ms-paths": { + "/?restype=service&comp=properties": { + "put": { + "tags": [ + "service" + ], + "operationId": "Service_SetProperties", + "description": "Sets properties for a storage account's Blob service endpoint, including properties for Storage Analytics and CORS (Cross-Origin Resource Sharing) rules", + "parameters": [ + { + "$ref": "#/parameters/StorageServiceProperties" + }, + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "202": { + "description": "Success (Accepted)", + "headers": { + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "get": { + "tags": [ + "service" + ], + "operationId": "Service_GetProperties", + "description": "gets the properties of a storage account's Blob service, including properties for Storage Analytics and CORS (Cross-Origin Resource Sharing) rules.", + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "200": { + "description": "Success.", + "headers": { + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + } + }, + "schema": { + "$ref": "#/definitions/StorageServiceProperties" + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "restype", + "description": "restype", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "service" + ] + }, + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "properties" + ] + } + ] + }, + "/?restype=service&comp=stats": { + "get": { + "tags": [ + "service" + ], + "operationId": "Service_GetStatistics", + "description": "Retrieves statistics related to replication for the Blob service. It is only available on the secondary location endpoint when read-access geo-redundant replication is enabled for the storage account.", + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "200": { + "description": "Success.", + "headers": { + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + } + }, + "schema": { + "$ref": "#/definitions/StorageServiceStats" + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "restype", + "description": "restype", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "service" + ] + }, + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "stats" + ] + } + ] + }, + "/?comp=list": { + "get": { + "tags": [ + "service" + ], + "operationId": "Service_ListContainersSegment", + "description": "The List Containers Segment operation returns a list of the containers under the specified account", + "parameters": [ + { + "$ref": "#/parameters/Prefix" + }, + { + "$ref": "#/parameters/Marker" + }, + { + "$ref": "#/parameters/MaxResultsBlob" + }, + { + "$ref": "#/parameters/ListContainersInclude" + }, + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "200": { + "description": "Success.", + "headers": { + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + } + }, + "schema": { + "$ref": "#/definitions/ListContainersSegmentResponse" + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + }, + "x-ms-pageable": { + "nextLinkName": "NextMarker" + } + }, + "parameters": [ + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "list" + ] + } + ] + }, + "/?restype=service&comp=userdelegationkey": { + "post": { + "tags": [ + "service" + ], + "operationId": "Service_GetUserDelegationKey", + "description": "Retrieves a user delegation key for the Blob service. This is only a valid operation when using bearer token authentication.", + "parameters": [ + { + "$ref": "#/parameters/KeyInfo" + }, + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "200": { + "description": "Success.", + "headers": { + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + } + }, + "schema": { + "$ref": "#/definitions/UserDelegationKey" + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "restype", + "description": "restype", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "service" + ] + }, + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "userdelegationkey" + ] + } + ] + }, + "/?restype=account&comp=properties": { + "get": { + "tags": [ + "service" + ], + "operationId": "Service_GetAccountInfo", + "description": "Returns the sku name and account kind ", + "parameters": [ + { + "$ref": "#/parameters/ApiVersionParameter" + } + ], + "responses": { + "200": { + "description": "Success (OK)", + "headers": { + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + }, + "x-ms-sku-name": { + "x-ms-client-name": "SkuName", + "type": "string", + "enum": [ + "Standard_LRS", + "Standard_GRS", + "Standard_RAGRS", + "Standard_ZRS", + "Premium_LRS" + ], + "x-ms-enum": { + "name": "SkuName", + "modelAsString": false + }, + "description": "Identifies the sku name of the account" + }, + "x-ms-account-kind": { + "x-ms-client-name": "AccountKind", + "type": "string", + "enum": [ + "Storage", + "BlobStorage", + "StorageV2", + "FileStorage", + "BlockBlobStorage" + ], + "x-ms-enum": { + "name": "AccountKind", + "modelAsString": false + }, + "description": "Identifies the account kind" + }, + "x-ms-is-hns-enabled": { + "x-ms-client-name": "IsHierarchicalNamespaceEnabled", + "type": "boolean", + "description": "Version 2019-07-07 and newer. Indicates if the account has a hierarchical namespace enabled." + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "head": { + "tags": [ + "service" + ], + "operationId": "Service_GetAccountInfoWithHead", + "description": "Returns the sku name and account kind ", + "parameters": [ + { + "$ref": "#/parameters/ApiVersionParameter" + } + ], + "responses": { + "200": { + "description": "Success (OK)", + "headers": { + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + }, + "x-ms-sku-name": { + "x-ms-client-name": "SkuName", + "type": "string", + "enum": [ + "Standard_LRS", + "Standard_GRS", + "Standard_RAGRS", + "Standard_ZRS", + "Premium_LRS" + ], + "x-ms-enum": { + "name": "SkuName", + "modelAsString": false + }, + "description": "Identifies the sku name of the account" + }, + "x-ms-account-kind": { + "x-ms-client-name": "AccountKind", + "type": "string", + "enum": [ + "Storage", + "BlobStorage", + "StorageV2", + "FileStorage", + "BlockBlobStorage" + ], + "x-ms-enum": { + "name": "AccountKind", + "modelAsString": false + }, + "description": "Identifies the account kind" + }, + "x-ms-is-hns-enabled": { + "x-ms-client-name": "IsHierarchicalNamespaceEnabled", + "type": "boolean", + "description": "Version 2019-07-07 and newer. Indicates if the account has a hierarchical namespace enabled." + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "restype", + "description": "restype", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "account" + ] + }, + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "properties" + ] + } + ] + }, + "/?comp=batch": { + "post": { + "tags": [ + "service" + ], + "operationId": "Service_SubmitBatch", + "description": "The Batch operation allows multiple API calls to be embedded into a single HTTP request.", + "parameters": [ + { + "$ref": "#/parameters/Body" + }, + { + "$ref": "#/parameters/ContentLengthBlob" + }, + { + "$ref": "#/parameters/MultipartContentType" + }, + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "202": { + "description": "Success.", + "headers": { + "Content-Type": { + "type": "string", + "description": "The media type of the body of the response. For batch requests, this is multipart/mixed; boundary=batchresponse_GUID" + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + } + }, + "schema": { + "type": "object", + "format": "file" + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "batch" + ] + } + ] + }, + "/?comp=blobs": { + "get": { + "tags": [ + "service" + ], + "operationId": "Service_FilterBlobs", + "description": "The Filter Blobs operation enables callers to list blobs across all containers whose tags match a given search expression. Filter blobs searches across all containers within a storage account but can be scoped within the expression to a single container.", + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + }, + { + "$ref": "#/parameters/FilterBlobsWhere" + }, + { + "$ref": "#/parameters/Marker" + }, + { + "$ref": "#/parameters/MaxResultsBlob" + }, + { + "$ref": "#/parameters/FilterBlobsInclude" + } + ], + "responses": { + "200": { + "description": "Success", + "headers": { + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + } + }, + "schema": { + "$ref": "#/definitions/FilterBlobSegment" + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "blobs" + ] + } + ] + }, + "/{containerName}?restype=container": { + "put": { + "tags": [ + "container" + ], + "operationId": "Container_Create", + "description": "creates a new container under the specified account. If the container with the same name already exists, the operation fails", + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/Metadata" + }, + { + "$ref": "#/parameters/BlobPublicAccess" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + }, + { + "$ref": "#/parameters/DefaultEncryptionScope" + }, + { + "$ref": "#/parameters/DenyEncryptionScopeOverride" + } + ], + "responses": { + "201": { + "description": "Success, Container created.", + "headers": { + "ETag": { + "type": "string", + "description": "The ETag contains a value that you can use to perform operations conditionally. If the request version is 2011-08-18 or newer, the ETag value will be in quotes." + }, + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the container was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob." + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "get": { + "tags": [ + "container" + ], + "operationId": "Container_GetProperties", + "description": "returns all user-defined metadata and system properties for the specified container. The data returned does not include the container's list of blobs", + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "200": { + "description": "Success", + "headers": { + "x-ms-meta": { + "type": "string", + "x-ms-client-name": "Metadata", + "x-ms-header-collection-prefix": "x-ms-meta-" + }, + "ETag": { + "type": "string", + "description": "The ETag contains a value that you can use to perform operations conditionally. If the request version is 2011-08-18 or newer, the ETag value will be in quotes." + }, + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the container was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob." + }, + "x-ms-lease-duration": { + "x-ms-client-name": "LeaseDuration", + "description": "When a blob is leased, specifies whether the lease is of infinite or fixed duration.", + "type": "string", + "enum": [ + "infinite", + "fixed" + ], + "x-ms-enum": { + "name": "LeaseDurationType", + "modelAsString": false + } + }, + "x-ms-lease-state": { + "x-ms-client-name": "LeaseState", + "description": "Lease state of the blob.", + "type": "string", + "enum": [ + "available", + "leased", + "expired", + "breaking", + "broken" + ], + "x-ms-enum": { + "name": "LeaseStateType", + "modelAsString": false + } + }, + "x-ms-lease-status": { + "x-ms-client-name": "LeaseStatus", + "description": "The current lease status of the blob.", + "type": "string", + "enum": [ + "locked", + "unlocked" + ], + "x-ms-enum": { + "name": "LeaseStatusType", + "modelAsString": false + } + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + }, + "x-ms-blob-public-access": { + "x-ms-client-name": "BlobPublicAccess", + "description": "Indicated whether data in the container may be accessed publicly and the level of access", + "type": "string", + "enum": [ + "container", + "blob" + ], + "x-ms-enum": { + "name": "PublicAccessType", + "modelAsString": true + } + }, + "x-ms-has-immutability-policy": { + "x-ms-client-name": "HasImmutabilityPolicy", + "description": "Indicates whether the container has an immutability policy set on it.", + "type": "boolean" + }, + "x-ms-has-legal-hold": { + "x-ms-client-name": "HasLegalHold", + "description": "Indicates whether the container has a legal hold.", + "type": "boolean" + }, + "x-ms-default-encryption-scope": { + "x-ms-client-name": "DefaultEncryptionScope", + "description": "The default encryption scope for the container.", + "type": "string" + }, + "x-ms-deny-encryption-scope-override": { + "x-ms-client-name": "DenyEncryptionScopeOverride", + "description": "Indicates whether the container's default encryption scope can be overriden.", + "type": "boolean" + }, + "x-ms-immutable-storage-with-versioning-enabled": { + "x-ms-client-name": "IsImmutableStorageWithVersioningEnabled", + "description": "Indicates whether version level worm is enabled on a container.", + "type": "boolean" + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "head": { + "tags": [ + "container" + ], + "operationId": "Container_GetPropertiesWithHead", + "description": "returns all user-defined metadata and system properties for the specified container. The data returned does not include the container's list of blobs", + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "200": { + "description": "Success", + "headers": { + "x-ms-meta": { + "type": "string", + "x-ms-client-name": "Metadata", + "x-ms-header-collection-prefix": "x-ms-meta-" + }, + "ETag": { + "type": "string", + "description": "The ETag contains a value that you can use to perform operations conditionally. If the request version is 2011-08-18 or newer, the ETag value will be in quotes." + }, + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the container was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob." + }, + "x-ms-lease-duration": { + "x-ms-client-name": "LeaseDuration", + "description": "When a blob is leased, specifies whether the lease is of infinite or fixed duration.", + "type": "string", + "enum": [ + "infinite", + "fixed" + ], + "x-ms-enum": { + "name": "LeaseDurationType", + "modelAsString": false + } + }, + "x-ms-lease-state": { + "x-ms-client-name": "LeaseState", + "description": "Lease state of the blob.", + "type": "string", + "enum": [ + "available", + "leased", + "expired", + "breaking", + "broken" + ], + "x-ms-enum": { + "name": "LeaseStateType", + "modelAsString": false + } + }, + "x-ms-lease-status": { + "x-ms-client-name": "LeaseStatus", + "description": "The current lease status of the blob.", + "type": "string", + "enum": [ + "locked", + "unlocked" + ], + "x-ms-enum": { + "name": "LeaseStatusType", + "modelAsString": false + } + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + }, + "x-ms-blob-public-access": { + "x-ms-client-name": "BlobPublicAccess", + "description": "Indicated whether data in the container may be accessed publicly and the level of access", + "type": "string", + "enum": [ + "container", + "blob" + ], + "x-ms-enum": { + "name": "PublicAccessType", + "modelAsString": true + } + }, + "x-ms-has-immutability-policy": { + "x-ms-client-name": "HasImmutabilityPolicy", + "description": "Indicates whether the container has an immutability policy set on it.", + "type": "boolean" + }, + "x-ms-has-legal-hold": { + "x-ms-client-name": "HasLegalHold", + "description": "Indicates whether the container has a legal hold.", + "type": "boolean" + }, + "x-ms-default-encryption-scope": { + "x-ms-client-name": "DefaultEncryptionScope", + "description": "The default encryption scope for the container.", + "type": "string" + }, + "x-ms-deny-encryption-scope-override": { + "x-ms-client-name": "DenyEncryptionScopeOverride", + "description": "Indicates whether the container's default encryption scope can be overriden.", + "type": "boolean" + }, + "x-ms-immutable-storage-with-versioning-enabled": { + "x-ms-client-name": "IsImmutableStorageWithVersioningEnabled", + "description": "Indicates whether version level worm is enabled on a container.", + "type": "boolean" + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "delete": { + "tags": [ + "container" + ], + "operationId": "Container_Delete", + "description": "operation marks the specified container for deletion. The container and any blobs contained within it are later deleted during garbage collection", + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "202": { + "description": "Accepted", + "headers": { + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "restype", + "description": "restype", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "container" + ] + } + ] + }, + "/{containerName}?restype=container&comp=metadata": { + "put": { + "tags": [ + "container" + ], + "operationId": "Container_SetMetadata", + "description": "operation sets one or more user-defined name-value pairs for the specified container.", + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "$ref": "#/parameters/Metadata" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "200": { + "description": "Success", + "headers": { + "ETag": { + "type": "string", + "description": "The ETag contains a value that you can use to perform operations conditionally. If the request version is 2011-08-18 or newer, the ETag value will be in quotes." + }, + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the container was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob." + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "restype", + "description": "restype", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "container" + ] + }, + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "metadata" + ] + } + ] + }, + "/{containerName}?restype=container&comp=acl": { + "get": { + "tags": [ + "container" + ], + "operationId": "Container_GetAccessPolicy", + "description": "gets the permissions for the specified container. The permissions indicate whether container data may be accessed publicly.", + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "200": { + "description": "Success", + "headers": { + "x-ms-blob-public-access": { + "x-ms-client-name": "BlobPublicAccess", + "description": "Indicated whether data in the container may be accessed publicly and the level of access", + "type": "string", + "enum": [ + "container", + "blob" + ], + "x-ms-enum": { + "name": "PublicAccessType", + "modelAsString": true + } + }, + "ETag": { + "type": "string", + "description": "The ETag contains a value that you can use to perform operations conditionally. If the request version is 2011-08-18 or newer, the ETag value will be in quotes." + }, + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the container was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob." + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + } + }, + "schema": { + "$ref": "#/definitions/SignedIdentifiers" + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "put": { + "tags": [ + "container" + ], + "operationId": "Container_SetAccessPolicy", + "description": "sets the permissions for the specified container. The permissions indicate whether blobs in a container may be accessed publicly.", + "parameters": [ + { + "$ref": "#/parameters/ContainerAcl" + }, + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "$ref": "#/parameters/BlobPublicAccess" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "200": { + "description": "Success.", + "headers": { + "ETag": { + "type": "string", + "description": "The ETag contains a value that you can use to perform operations conditionally. If the request version is 2011-08-18 or newer, the ETag value will be in quotes." + }, + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the container was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob." + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "restype", + "description": "restype", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "container" + ] + }, + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "acl" + ] + } + ] + }, + "/{containerName}?restype=container&comp=undelete": { + "put": { + "tags": [ + "container" + ], + "operationId": "Container_Restore", + "description": "Restores a previously-deleted container.", + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + }, + { + "$ref": "#/parameters/DeletedContainerName" + }, + { + "$ref": "#/parameters/DeletedContainerVersion" + } + ], + "responses": { + "201": { + "description": "Created.", + "headers": { + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "restype", + "description": "restype", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "container" + ] + }, + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "undelete" + ] + } + ] + }, + "/{containerName}?restype=container&comp=batch": { + "post": { + "tags": [ + "container" + ], + "operationId": "Container_SubmitBatch", + "description": "The Batch operation allows multiple API calls to be embedded into a single HTTP request.", + "parameters": [ + { + "$ref": "#/parameters/Body" + }, + { + "$ref": "#/parameters/ContentLengthBlob" + }, + { + "$ref": "#/parameters/MultipartContentType" + }, + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "202": { + "description": "Success.", + "headers": { + "Content-Type": { + "type": "string", + "description": "The media type of the body of the response. For batch requests, this is multipart/mixed; boundary=batchresponse_GUID" + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + } + }, + "schema": { + "type": "object", + "format": "file" + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "restype", + "description": "restype", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "container" + ] + }, + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "batch" + ] + } + ] + }, + "/{containerName}?restype=container&comp=blobs": { + "get": { + "tags": [ + "container" + ], + "operationId": "Container_FilterBlobs", + "description": "The Filter Blobs operation enables callers to list blobs in a container whose tags match a given search expression. Filter blobs searches within the given container.", + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + }, + { + "$ref": "#/parameters/FilterBlobsWhere" + }, + { + "$ref": "#/parameters/Marker" + }, + { + "$ref": "#/parameters/MaxResultsBlob" + }, + { + "$ref": "#/parameters/FilterBlobsInclude" + } + ], + "responses": { + "200": { + "description": "Success", + "headers": { + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + } + }, + "schema": { + "$ref": "#/definitions/FilterBlobSegment" + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "restype", + "description": "restype", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "container" + ] + }, + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "blobs" + ] + } + ] + }, + "/{containerName}?comp=lease&restype=container&acquire": { + "put": { + "tags": [ + "container" + ], + "operationId": "Container_AcquireLease", + "description": "[Update] establishes and manages a lock on a container for delete operations. The lock duration can be 15 to 60 seconds, or can be infinite", + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/LeaseDurationBlob" + }, + { + "$ref": "#/parameters/ProposedLeaseIdOptional" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "201": { + "description": "The Acquire operation completed successfully.", + "headers": { + "ETag": { + "type": "string", + "description": "The ETag contains a value that you can use to perform operations conditionally. If the request version is 2011-08-18 or newer, the ETag value will be in quotes." + }, + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the container was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob." + }, + "x-ms-lease-id": { + "x-ms-client-name": "LeaseId", + "type": "string", + "description": "Uniquely identifies a container's lease" + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "lease" + ] + }, + { + "name": "restype", + "description": "restype", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "container" + ] + }, + { + "name": "x-ms-lease-action", + "x-ms-client-name": "action", + "in": "header", + "required": true, + "type": "string", + "enum": [ + "acquire" + ], + "x-ms-enum": { + "name": "LeaseAction", + "modelAsString": false + }, + "x-ms-parameter-location": "method", + "description": "Describes what lease action to take." + } + ] + }, + "/{containerName}?comp=lease&restype=container&release": { + "put": { + "tags": [ + "container" + ], + "operationId": "Container_ReleaseLease", + "description": "[Update] establishes and manages a lock on a container for delete operations. The lock duration can be 15 to 60 seconds, or can be infinite", + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/LeaseIdRequired" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "200": { + "description": "The Release operation completed successfully.", + "headers": { + "ETag": { + "type": "string", + "description": "The ETag contains a value that you can use to perform operations conditionally. If the request version is 2011-08-18 or newer, the ETag value will be in quotes." + }, + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the container was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob." + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "lease" + ] + }, + { + "name": "restype", + "description": "restype", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "container" + ] + }, + { + "name": "x-ms-lease-action", + "x-ms-client-name": "action", + "in": "header", + "required": true, + "type": "string", + "enum": [ + "release" + ], + "x-ms-enum": { + "name": "LeaseAction", + "modelAsString": false + }, + "x-ms-parameter-location": "method", + "description": "Describes what lease action to take." + } + ] + }, + "/{containerName}?comp=lease&restype=container&renew": { + "put": { + "tags": [ + "container" + ], + "operationId": "Container_RenewLease", + "description": "[Update] establishes and manages a lock on a container for delete operations. The lock duration can be 15 to 60 seconds, or can be infinite", + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/LeaseIdRequired" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "200": { + "description": "The Renew operation completed successfully.", + "headers": { + "ETag": { + "type": "string", + "description": "The ETag contains a value that you can use to perform operations conditionally. If the request version is 2011-08-18 or newer, the ETag value will be in quotes." + }, + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the container was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob." + }, + "x-ms-lease-id": { + "x-ms-client-name": "LeaseId", + "type": "string", + "description": "Uniquely identifies a container's lease" + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "lease" + ] + }, + { + "name": "restype", + "description": "restype", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "container" + ] + }, + { + "name": "x-ms-lease-action", + "x-ms-client-name": "action", + "in": "header", + "required": true, + "type": "string", + "enum": [ + "renew" + ], + "x-ms-enum": { + "name": "LeaseAction", + "modelAsString": false + }, + "x-ms-parameter-location": "method", + "description": "Describes what lease action to take." + } + ] + }, + "/{containerName}?comp=lease&restype=container&break": { + "put": { + "tags": [ + "container" + ], + "operationId": "Container_BreakLease", + "description": "[Update] establishes and manages a lock on a container for delete operations. The lock duration can be 15 to 60 seconds, or can be infinite", + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/LeaseBreakPeriod" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "202": { + "description": "The Break operation completed successfully.", + "headers": { + "ETag": { + "type": "string", + "description": "The ETag contains a value that you can use to perform operations conditionally. If the request version is 2011-08-18 or newer, the ETag value will be in quotes." + }, + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the container was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob." + }, + "x-ms-lease-time": { + "x-ms-client-name": "LeaseTime", + "type": "integer", + "description": "Approximate time remaining in the lease period, in seconds." + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "lease" + ] + }, + { + "name": "restype", + "description": "restype", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "container" + ] + }, + { + "name": "x-ms-lease-action", + "x-ms-client-name": "action", + "in": "header", + "required": true, + "type": "string", + "enum": [ + "break" + ], + "x-ms-enum": { + "name": "LeaseAction", + "modelAsString": false + }, + "x-ms-parameter-location": "method", + "description": "Describes what lease action to take." + } + ] + }, + "/{containerName}?comp=lease&restype=container&change": { + "put": { + "tags": [ + "container" + ], + "operationId": "Container_ChangeLease", + "description": "[Update] establishes and manages a lock on a container for delete operations. The lock duration can be 15 to 60 seconds, or can be infinite", + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/LeaseIdRequired" + }, + { + "$ref": "#/parameters/ProposedLeaseIdRequired" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "200": { + "description": "The Change operation completed successfully.", + "headers": { + "ETag": { + "type": "string", + "description": "The ETag contains a value that you can use to perform operations conditionally. If the request version is 2011-08-18 or newer, the ETag value will be in quotes." + }, + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the container was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob." + }, + "x-ms-lease-id": { + "x-ms-client-name": "LeaseId", + "type": "string", + "description": "Uniquely identifies a container's lease" + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "lease" + ] + }, + { + "name": "restype", + "description": "restype", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "container" + ] + }, + { + "name": "x-ms-lease-action", + "x-ms-client-name": "action", + "in": "header", + "required": true, + "type": "string", + "enum": [ + "change" + ], + "x-ms-enum": { + "name": "LeaseAction", + "modelAsString": false + }, + "x-ms-parameter-location": "method", + "description": "Describes what lease action to take." + } + ] + }, + "/{containerName}?restype=account&comp=properties": { + "get": { + "tags": [ + "container" + ], + "operationId": "Container_GetAccountInfo", + "description": "Returns the sku name and account kind ", + "parameters": [ + { + "$ref": "#/parameters/ApiVersionParameter" + } + ], + "responses": { + "200": { + "description": "Success (OK)", + "headers": { + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + }, + "x-ms-sku-name": { + "x-ms-client-name": "SkuName", + "type": "string", + "enum": [ + "Standard_LRS", + "Standard_GRS", + "Standard_RAGRS", + "Standard_ZRS", + "Premium_LRS" + ], + "x-ms-enum": { + "name": "SkuName", + "modelAsString": false + }, + "description": "Identifies the sku name of the account" + }, + "x-ms-account-kind": { + "x-ms-client-name": "AccountKind", + "type": "string", + "enum": [ + "Storage", + "BlobStorage", + "StorageV2", + "FileStorage", + "BlockBlobStorage" + ], + "x-ms-enum": { + "name": "AccountKind", + "modelAsString": false + }, + "description": "Identifies the account kind" + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "head": { + "tags": [ + "container" + ], + "operationId": "Container_GetAccountInfoWithHead", + "description": "Returns the sku name and account kind ", + "parameters": [ + { + "$ref": "#/parameters/ApiVersionParameter" + } + ], + "responses": { + "200": { + "description": "Success (OK)", + "headers": { + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + }, + "x-ms-sku-name": { + "x-ms-client-name": "SkuName", + "type": "string", + "enum": [ + "Standard_LRS", + "Standard_GRS", + "Standard_RAGRS", + "Standard_ZRS", + "Premium_LRS" + ], + "x-ms-enum": { + "name": "SkuName", + "modelAsString": false + }, + "description": "Identifies the sku name of the account" + }, + "x-ms-account-kind": { + "x-ms-client-name": "AccountKind", + "type": "string", + "enum": [ + "Storage", + "BlobStorage", + "StorageV2", + "FileStorage", + "BlockBlobStorage" + ], + "x-ms-enum": { + "name": "AccountKind", + "modelAsString": false + }, + "description": "Identifies the account kind" + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "restype", + "description": "restype", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "account" + ] + }, + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "properties" + ] + } + ] + }, + "/{containerName}/{blob}?PageBlob": { + "put": { + "tags": [ + "blob" + ], + "operationId": "PageBlob_Create", + "description": "The Create operation creates a new page blob.", + "consumes": [ + "application/octet-stream" + ], + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/ContentLengthBlob" + }, + { + "$ref": "#/parameters/PremiumPageBlobAccessTierOptional" + }, + { + "$ref": "#/parameters/BlobContentType" + }, + { + "$ref": "#/parameters/BlobContentEncoding" + }, + { + "$ref": "#/parameters/BlobContentLanguage" + }, + { + "$ref": "#/parameters/BlobContentMD5" + }, + { + "$ref": "#/parameters/BlobCacheControl" + }, + { + "$ref": "#/parameters/Metadata" + }, + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "$ref": "#/parameters/BlobContentDisposition" + }, + { + "$ref": "#/parameters/EncryptionKey" + }, + { + "$ref": "#/parameters/EncryptionKeySha256" + }, + { + "$ref": "#/parameters/EncryptionAlgorithm" + }, + { + "$ref": "#/parameters/EncryptionScope" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/IfMatch" + }, + { + "$ref": "#/parameters/IfNoneMatch" + }, + { + "$ref": "#/parameters/IfTags" + }, + { + "$ref": "#/parameters/BlobContentLengthRequired" + }, + { + "$ref": "#/parameters/BlobSequenceNumber" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + }, + { + "$ref": "#/parameters/BlobTagsHeader" + }, + { + "$ref": "#/parameters/ImmutabilityPolicyExpiry" + }, + { + "$ref": "#/parameters/ImmutabilityPolicyMode" + }, + { + "$ref": "#/parameters/LegalHoldOptional" + } + ], + "responses": { + "201": { + "description": "The blob was created.", + "headers": { + "ETag": { + "type": "string", + "description": "The ETag contains a value that you can use to perform operations conditionally. If the request version is 2011-08-18 or newer, the ETag value will be in quotes." + }, + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the container was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob." + }, + "Content-MD5": { + "type": "string", + "format": "byte", + "description": "If the blob has an MD5 hash and this operation is to read the full blob, this response header is returned so that the client can check for message content integrity." + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "x-ms-version-id": { + "x-ms-client-name": "VersionId", + "type": "string", + "description": "A DateTime value returned by the service that uniquely identifies the blob. The value of this header indicates the blob version, and may be used in subsequent requests to access this version of the blob." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + }, + "x-ms-request-server-encrypted": { + "x-ms-client-name": "IsServerEncrypted", + "type": "boolean", + "description": "The value of this header is set to true if the contents of the request are successfully encrypted using the specified algorithm, and false otherwise." + }, + "x-ms-encryption-key-sha256": { + "x-ms-client-name": "EncryptionKeySha256", + "type": "string", + "description": "The SHA-256 hash of the encryption key used to encrypt the blob. This header is only returned when the blob was encrypted with a customer-provided key." + }, + "x-ms-encryption-scope": { + "x-ms-client-name": "EncryptionScope", + "type": "string", + "description": "Returns the name of the encryption scope used to encrypt the blob contents and application metadata. Note that the absence of this header implies use of the default account encryption scope." + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "x-ms-blob-type", + "x-ms-client-name": "blobType", + "in": "header", + "required": true, + "x-ms-parameter-location": "method", + "description": "Specifies the type of blob to create: block blob, page blob, or append blob.", + "type": "string", + "enum": [ + "PageBlob" + ], + "x-ms-enum": { + "name": "BlobType", + "modelAsString": false + } + } + ] + }, + "/{containerName}/{blob}?AppendBlob": { + "put": { + "tags": [ + "blob" + ], + "operationId": "AppendBlob_Create", + "description": "The Create Append Blob operation creates a new append blob.", + "consumes": [ + "application/octet-stream" + ], + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/ContentLengthBlob" + }, + { + "$ref": "#/parameters/BlobContentType" + }, + { + "$ref": "#/parameters/BlobContentEncoding" + }, + { + "$ref": "#/parameters/BlobContentLanguage" + }, + { + "$ref": "#/parameters/BlobContentMD5" + }, + { + "$ref": "#/parameters/BlobCacheControl" + }, + { + "$ref": "#/parameters/Metadata" + }, + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "$ref": "#/parameters/BlobContentDisposition" + }, + { + "$ref": "#/parameters/EncryptionKey" + }, + { + "$ref": "#/parameters/EncryptionKeySha256" + }, + { + "$ref": "#/parameters/EncryptionAlgorithm" + }, + { + "$ref": "#/parameters/EncryptionScope" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/IfMatch" + }, + { + "$ref": "#/parameters/IfNoneMatch" + }, + { + "$ref": "#/parameters/IfTags" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + }, + { + "$ref": "#/parameters/BlobTagsHeader" + }, + { + "$ref": "#/parameters/ImmutabilityPolicyExpiry" + }, + { + "$ref": "#/parameters/ImmutabilityPolicyMode" + }, + { + "$ref": "#/parameters/LegalHoldOptional" + } + ], + "responses": { + "201": { + "description": "The blob was created.", + "headers": { + "ETag": { + "type": "string", + "description": "The ETag contains a value that you can use to perform operations conditionally. If the request version is 2011-08-18 or newer, the ETag value will be in quotes." + }, + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the container was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob." + }, + "Content-MD5": { + "type": "string", + "format": "byte", + "description": "If the blob has an MD5 hash and this operation is to read the full blob, this response header is returned so that the client can check for message content integrity." + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "x-ms-version-id": { + "x-ms-client-name": "VersionId", + "type": "string", + "description": "A DateTime value returned by the service that uniquely identifies the blob. The value of this header indicates the blob version, and may be used in subsequent requests to access this version of the blob." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + }, + "x-ms-request-server-encrypted": { + "x-ms-client-name": "IsServerEncrypted", + "type": "boolean", + "description": "The value of this header is set to true if the contents of the request are successfully encrypted using the specified algorithm, and false otherwise." + }, + "x-ms-encryption-key-sha256": { + "x-ms-client-name": "EncryptionKeySha256", + "type": "string", + "description": "The SHA-256 hash of the encryption key used to encrypt the blob. This header is only returned when the blob was encrypted with a customer-provided key." + }, + "x-ms-encryption-scope": { + "x-ms-client-name": "EncryptionScope", + "type": "string", + "description": "Returns the name of the encryption scope used to encrypt the blob contents and application metadata. Note that the absence of this header implies use of the default account encryption scope." + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "x-ms-blob-type", + "x-ms-client-name": "blobType", + "in": "header", + "required": true, + "x-ms-parameter-location": "method", + "description": "Specifies the type of blob to create: block blob, page blob, or append blob.", + "type": "string", + "enum": [ + "AppendBlob" + ], + "x-ms-enum": { + "name": "BlobType", + "modelAsString": false + } + } + ] + }, + "/{containerName}/{blob}?BlockBlob": { + "put": { + "tags": [ + "blob" + ], + "operationId": "BlockBlob_Upload", + "description": "The Upload Block Blob operation updates the content of an existing block blob. Updating an existing block blob overwrites any existing metadata on the blob. Partial updates are not supported with Put Blob; the content of the existing blob is overwritten with the content of the new blob. To perform a partial update of the content of a block blob, use the Put Block List operation.", + "consumes": [ + "application/octet-stream" + ], + "parameters": [ + { + "$ref": "#/parameters/Body" + }, + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/ContentMD5Blob" + }, + { + "$ref": "#/parameters/ContentLengthBlob" + }, + { + "$ref": "#/parameters/BlobContentType" + }, + { + "$ref": "#/parameters/BlobContentEncoding" + }, + { + "$ref": "#/parameters/BlobContentLanguage" + }, + { + "$ref": "#/parameters/BlobContentMD5" + }, + { + "$ref": "#/parameters/BlobCacheControl" + }, + { + "$ref": "#/parameters/Metadata" + }, + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "$ref": "#/parameters/BlobContentDisposition" + }, + { + "$ref": "#/parameters/EncryptionKey" + }, + { + "$ref": "#/parameters/EncryptionKeySha256" + }, + { + "$ref": "#/parameters/EncryptionAlgorithm" + }, + { + "$ref": "#/parameters/EncryptionScope" + }, + { + "$ref": "#/parameters/AccessTierOptional" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/IfMatch" + }, + { + "$ref": "#/parameters/IfNoneMatch" + }, + { + "$ref": "#/parameters/IfTags" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + }, + { + "$ref": "#/parameters/BlobTagsHeader" + }, + { + "$ref": "#/parameters/ImmutabilityPolicyExpiry" + }, + { + "$ref": "#/parameters/ImmutabilityPolicyMode" + }, + { + "$ref": "#/parameters/LegalHoldOptional" + }, + { + "$ref": "#/parameters/ContentCrc64" + } + ], + "responses": { + "201": { + "description": "The blob was updated.", + "headers": { + "ETag": { + "type": "string", + "description": "The ETag contains a value that you can use to perform operations conditionally. If the request version is 2011-08-18 or newer, the ETag value will be in quotes." + }, + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the container was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob." + }, + "Content-MD5": { + "type": "string", + "format": "byte", + "description": "If the blob has an MD5 hash and this operation is to read the full blob, this response header is returned so that the client can check for message content integrity." + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "x-ms-version-id": { + "x-ms-client-name": "VersionId", + "type": "string", + "description": "A DateTime value returned by the service that uniquely identifies the blob. The value of this header indicates the blob version, and may be used in subsequent requests to access this version of the blob." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + }, + "x-ms-request-server-encrypted": { + "x-ms-client-name": "IsServerEncrypted", + "type": "boolean", + "description": "The value of this header is set to true if the contents of the request are successfully encrypted using the specified algorithm, and false otherwise." + }, + "x-ms-encryption-key-sha256": { + "x-ms-client-name": "EncryptionKeySha256", + "type": "string", + "description": "The SHA-256 hash of the encryption key used to encrypt the blob. This header is only returned when the blob was encrypted with a customer-provided key." + }, + "x-ms-encryption-scope": { + "x-ms-client-name": "EncryptionScope", + "type": "string", + "description": "Returns the name of the encryption scope used to encrypt the blob contents and application metadata. Note that the absence of this header implies use of the default account encryption scope." + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "x-ms-blob-type", + "x-ms-client-name": "blobType", + "in": "header", + "required": true, + "x-ms-parameter-location": "method", + "description": "Specifies the type of blob to create: block blob, page blob, or append blob.", + "type": "string", + "enum": [ + "BlockBlob" + ], + "x-ms-enum": { + "name": "BlobType", + "modelAsString": false + } + } + ] + }, + "/{containerName}/{blob}?BlockBlob&fromUrl": { + "put": { + "tags": [ + "blob" + ], + "operationId": "BlockBlob_PutBlobFromUrl", + "description": "The Put Blob from URL operation creates a new Block Blob where the contents of the blob are read from a given URL. This API is supported beginning with the 2020-04-08 version. Partial updates are not supported with Put Blob from URL; the content of an existing blob is overwritten with the content of the new blob. To perform partial updates to a block blob’s contents using a source URL, use the Put Block from URL API in conjunction with Put Block List.", + "consumes": [ + "application/octet-stream" + ], + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/ContentMD5Blob" + }, + { + "$ref": "#/parameters/ContentLengthBlob" + }, + { + "$ref": "#/parameters/BlobContentType" + }, + { + "$ref": "#/parameters/BlobContentEncoding" + }, + { + "$ref": "#/parameters/BlobContentLanguage" + }, + { + "$ref": "#/parameters/BlobContentMD5" + }, + { + "$ref": "#/parameters/BlobCacheControl" + }, + { + "$ref": "#/parameters/Metadata" + }, + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "$ref": "#/parameters/BlobContentDisposition" + }, + { + "$ref": "#/parameters/EncryptionKey" + }, + { + "$ref": "#/parameters/EncryptionKeySha256" + }, + { + "$ref": "#/parameters/EncryptionAlgorithm" + }, + { + "$ref": "#/parameters/EncryptionScope" + }, + { + "$ref": "#/parameters/AccessTierOptional" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/IfMatch" + }, + { + "$ref": "#/parameters/IfNoneMatch" + }, + { + "$ref": "#/parameters/IfTags" + }, + { + "$ref": "#/parameters/SourceIfModifiedSince" + }, + { + "$ref": "#/parameters/SourceIfUnmodifiedSince" + }, + { + "$ref": "#/parameters/SourceIfMatch" + }, + { + "$ref": "#/parameters/SourceIfNoneMatch" + }, + { + "$ref": "#/parameters/SourceIfTags" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + }, + { + "$ref": "#/parameters/SourceContentMD5" + }, + { + "$ref": "#/parameters/BlobTagsHeader" + }, + { + "$ref": "#/parameters/CopySource" + }, + { + "$ref": "#/parameters/CopySourceBlobProperties" + }, + { + "$ref": "#/parameters/CopySourceAuthorization" + }, + { + "$ref": "#/parameters/CopySourceTags" + } + ], + "responses": { + "201": { + "description": "The blob was updated.", + "headers": { + "ETag": { + "type": "string", + "description": "The ETag contains a value that you can use to perform operations conditionally. If the request version is 2011-08-18 or newer, the ETag value will be in quotes." + }, + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the container was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob." + }, + "Content-MD5": { + "type": "string", + "format": "byte", + "description": "If the blob has an MD5 hash and this operation is to read the full blob, this response header is returned so that the client can check for message content integrity." + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "x-ms-version-id": { + "x-ms-client-name": "VersionId", + "type": "string", + "description": "A DateTime value returned by the service that uniquely identifies the blob. The value of this header indicates the blob version, and may be used in subsequent requests to access this version of the blob." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + }, + "x-ms-request-server-encrypted": { + "x-ms-client-name": "IsServerEncrypted", + "type": "boolean", + "description": "The value of this header is set to true if the contents of the request are successfully encrypted using the specified algorithm, and false otherwise." + }, + "x-ms-encryption-key-sha256": { + "x-ms-client-name": "EncryptionKeySha256", + "type": "string", + "description": "The SHA-256 hash of the encryption key used to encrypt the blob. This header is only returned when the blob was encrypted with a customer-provided key." + }, + "x-ms-encryption-scope": { + "x-ms-client-name": "EncryptionScope", + "type": "string", + "description": "Returns the name of the encryption scope used to encrypt the blob contents and application metadata. Note that the absence of this header implies use of the default account encryption scope." + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "x-ms-blob-type", + "x-ms-client-name": "blobType", + "in": "header", + "required": true, + "x-ms-parameter-location": "method", + "description": "Specifies the type of blob to create: block blob, page blob, or append blob.", + "type": "string", + "enum": [ + "BlockBlob" + ], + "x-ms-enum": { + "name": "BlobType", + "modelAsString": false + } + } + ] + }, + "/{containerName}/{blob}?comp=undelete": { + "put": { + "tags": [ + "blob" + ], + "operationId": "Blob_Undelete", + "description": "Undelete a blob that was previously soft deleted", + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "200": { + "description": "The blob was undeleted successfully.", + "headers": { + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated." + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "undelete" + ] + } + ] + }, + "/{containerName}/{blob}?comp=expiry": { + "put": { + "tags": [ + "blob" + ], + "operationId": "Blob_SetExpiry", + "description": "Sets the time a blob will expire and be deleted.", + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + }, + { + "$ref": "#/parameters/BlobExpiryOptions" + }, + { + "$ref": "#/parameters/BlobExpiryTime" + } + ], + "responses": { + "200": { + "description": "The blob expiry was set successfully.", + "headers": { + "ETag": { + "type": "string", + "description": "The ETag contains a value that you can use to perform operations conditionally. If the request version is 2011-08-18 or newer, the ETag value will be in quotes." + }, + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the container was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob." + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated." + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "expiry" + ] + } + ] + }, + "/{containerName}/{blob}?comp=properties&SetHTTPHeaders": { + "put": { + "tags": [ + "blob" + ], + "operationId": "Blob_SetHTTPHeaders", + "description": "The Set HTTP Headers operation sets system properties on the blob", + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/BlobCacheControl" + }, + { + "$ref": "#/parameters/BlobContentType" + }, + { + "$ref": "#/parameters/BlobContentMD5" + }, + { + "$ref": "#/parameters/BlobContentEncoding" + }, + { + "$ref": "#/parameters/BlobContentLanguage" + }, + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/IfMatch" + }, + { + "$ref": "#/parameters/IfNoneMatch" + }, + { + "$ref": "#/parameters/IfTags" + }, + { + "$ref": "#/parameters/BlobContentDisposition" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "200": { + "description": "The properties were set successfully.", + "headers": { + "ETag": { + "type": "string", + "description": "The ETag contains a value that you can use to perform operations conditionally. If the request version is 2011-08-18 or newer, the ETag value will be in quotes." + }, + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the container was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob." + }, + "x-ms-blob-sequence-number": { + "x-ms-client-name": "BlobSequenceNumber", + "type": "integer", + "format": "int64", + "description": "The current sequence number for a page blob. This header is not returned for block blobs or append blobs" + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "properties" + ] + } + ] + }, + "/{containerName}/{blob}?comp=immutabilityPolicies": { + "put": { + "tags": [ + "blob" + ], + "operationId": "Blob_SetImmutabilityPolicy", + "description": "The Set Immutability Policy operation sets the immutability policy on the blob", + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/ImmutabilityPolicyExpiry" + }, + { + "$ref": "#/parameters/ImmutabilityPolicyMode" + } + ], + "responses": { + "200": { + "description": "The immutability policy was successfully set.", + "headers": { + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + }, + "x-ms-immutability-policy-until-date": { + "x-ms-client-name": "ImmutabilityPolicyExpiry", + "type": "string", + "format": "date-time-rfc1123", + "description": "Indicates the time the immutability policy will expire." + }, + "x-ms-immutability-policy-mode": { + "x-ms-client-name": "ImmutabilityPolicyMode", + "type": "string", + "enum": [ + "Mutable", + "Unlocked", + "Locked" + ], + "x-ms-enum": { + "name": "BlobImmutabilityPolicyMode", + "modelAsString": false + }, + "description": "Indicates immutability policy mode." + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "delete": { + "tags": [ + "blob" + ], + "operationId": "Blob_DeleteImmutabilityPolicy", + "description": "The Delete Immutability Policy operation deletes the immutability policy on the blob", + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "200": { + "description": "The delete immutability policy request was accepted and the immutability policy will be deleted.", + "headers": { + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "immutabilityPolicies" + ] + } + ] + }, + "/{containerName}/{blob}?comp=legalhold": { + "put": { + "tags": [ + "blob" + ], + "operationId": "Blob_SetLegalHold", + "description": "The Set Legal Hold operation sets a legal hold on the blob.", + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + }, + { + "$ref": "#/parameters/LegalHoldRequired" + } + ], + "responses": { + "200": { + "description": "The legal hold was successfully set on the blob.", + "headers": { + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + }, + "x-ms-legal-hold": { + "x-ms-client-name": "LegalHold", + "type": "boolean", + "description": "Indicates if the blob has a legal hold." + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "legalhold" + ] + } + ] + }, + "/{containerName}/{blob}?comp=metadata": { + "put": { + "tags": [ + "blob" + ], + "operationId": "Blob_SetMetadata", + "description": "The Set Blob Metadata operation sets user-defined metadata for the specified blob as one or more name-value pairs", + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/Metadata" + }, + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "$ref": "#/parameters/EncryptionKey" + }, + { + "$ref": "#/parameters/EncryptionKeySha256" + }, + { + "$ref": "#/parameters/EncryptionAlgorithm" + }, + { + "$ref": "#/parameters/EncryptionScope" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/IfMatch" + }, + { + "$ref": "#/parameters/IfNoneMatch" + }, + { + "$ref": "#/parameters/IfTags" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "200": { + "description": "The metadata was set successfully.", + "headers": { + "ETag": { + "type": "string", + "description": "The ETag contains a value that you can use to perform operations conditionally. If the request version is 2011-08-18 or newer, the ETag value will be in quotes." + }, + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the container was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob." + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "x-ms-version-id": { + "x-ms-client-name": "VersionId", + "type": "string", + "description": "A DateTime value returned by the service that uniquely identifies the blob. The value of this header indicates the blob version, and may be used in subsequent requests to access this version of the blob." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + }, + "x-ms-request-server-encrypted": { + "x-ms-client-name": "IsServerEncrypted", + "type": "boolean", + "description": "The value of this header is set to true if the contents of the request are successfully encrypted using the specified algorithm, and false otherwise." + }, + "x-ms-encryption-key-sha256": { + "x-ms-client-name": "EncryptionKeySha256", + "type": "string", + "description": "The SHA-256 hash of the encryption key used to encrypt the metadata. This header is only returned when the metadata was encrypted with a customer-provided key." + }, + "x-ms-encryption-scope": { + "x-ms-client-name": "EncryptionScope", + "type": "string", + "description": "Returns the name of the encryption scope used to encrypt the blob contents and application metadata. Note that the absence of this header implies use of the default account encryption scope." + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "metadata" + ] + } + ] + }, + "/{containerName}/{blob}?comp=lease&acquire": { + "put": { + "tags": [ + "blob" + ], + "operationId": "Blob_AcquireLease", + "description": "[Update] The Lease Blob operation establishes and manages a lock on a blob for write and delete operations", + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/LeaseDurationBlob" + }, + { + "$ref": "#/parameters/ProposedLeaseIdOptional" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/IfMatch" + }, + { + "$ref": "#/parameters/IfNoneMatch" + }, + { + "$ref": "#/parameters/IfTags" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "201": { + "description": "The Acquire operation completed successfully.", + "headers": { + "ETag": { + "type": "string", + "description": "The ETag contains a value that you can use to perform operations conditionally. If the request version is 2011-08-18 or newer, the ETag value will be in quotes." + }, + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the blob was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob." + }, + "x-ms-lease-id": { + "x-ms-client-name": "LeaseId", + "type": "string", + "description": "Uniquely identifies a blobs' lease" + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "lease" + ] + }, + { + "name": "x-ms-lease-action", + "x-ms-client-name": "action", + "in": "header", + "required": true, + "type": "string", + "enum": [ + "acquire" + ], + "x-ms-enum": { + "name": "LeaseAction", + "modelAsString": false + }, + "x-ms-parameter-location": "method", + "description": "Describes what lease action to take." + } + ] + }, + "/{containerName}/{blob}?comp=lease&release": { + "put": { + "tags": [ + "blob" + ], + "operationId": "Blob_ReleaseLease", + "description": "[Update] The Lease Blob operation establishes and manages a lock on a blob for write and delete operations", + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/LeaseIdRequired" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/IfMatch" + }, + { + "$ref": "#/parameters/IfNoneMatch" + }, + { + "$ref": "#/parameters/IfTags" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "200": { + "description": "The Release operation completed successfully.", + "headers": { + "ETag": { + "type": "string", + "description": "The ETag contains a value that you can use to perform operations conditionally. If the request version is 2011-08-18 or newer, the ETag value will be in quotes." + }, + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the blob was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob." + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "lease" + ] + }, + { + "name": "x-ms-lease-action", + "x-ms-client-name": "action", + "in": "header", + "required": true, + "type": "string", + "enum": [ + "release" + ], + "x-ms-enum": { + "name": "LeaseAction", + "modelAsString": false + }, + "x-ms-parameter-location": "method", + "description": "Describes what lease action to take." + } + ] + }, + "/{containerName}/{blob}?comp=lease&renew": { + "put": { + "tags": [ + "blob" + ], + "operationId": "Blob_RenewLease", + "description": "[Update] The Lease Blob operation establishes and manages a lock on a blob for write and delete operations", + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/LeaseIdRequired" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/IfMatch" + }, + { + "$ref": "#/parameters/IfNoneMatch" + }, + { + "$ref": "#/parameters/IfTags" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "200": { + "description": "The Renew operation completed successfully.", + "headers": { + "ETag": { + "type": "string", + "description": "The ETag contains a value that you can use to perform operations conditionally. If the request version is 2011-08-18 or newer, the ETag value will be in quotes." + }, + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the blob was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob." + }, + "x-ms-lease-id": { + "x-ms-client-name": "LeaseId", + "type": "string", + "description": "Uniquely identifies a blobs' lease" + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "lease" + ] + }, + { + "name": "x-ms-lease-action", + "x-ms-client-name": "action", + "in": "header", + "required": true, + "type": "string", + "enum": [ + "renew" + ], + "x-ms-enum": { + "name": "LeaseAction", + "modelAsString": false + }, + "x-ms-parameter-location": "method", + "description": "Describes what lease action to take." + } + ] + }, + "/{containerName}/{blob}?comp=lease&change": { + "put": { + "tags": [ + "blob" + ], + "operationId": "Blob_ChangeLease", + "description": "[Update] The Lease Blob operation establishes and manages a lock on a blob for write and delete operations", + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/LeaseIdRequired" + }, + { + "$ref": "#/parameters/ProposedLeaseIdRequired" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/IfMatch" + }, + { + "$ref": "#/parameters/IfNoneMatch" + }, + { + "$ref": "#/parameters/IfTags" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "200": { + "description": "The Change operation completed successfully.", + "headers": { + "ETag": { + "type": "string", + "description": "The ETag contains a value that you can use to perform operations conditionally. If the request version is 2011-08-18 or newer, the ETag value will be in quotes." + }, + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the blob was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob." + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-lease-id": { + "x-ms-client-name": "LeaseId", + "type": "string", + "description": "Uniquely identifies a blobs' lease" + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "lease" + ] + }, + { + "name": "x-ms-lease-action", + "x-ms-client-name": "action", + "in": "header", + "required": true, + "type": "string", + "enum": [ + "change" + ], + "x-ms-enum": { + "name": "LeaseAction", + "modelAsString": false + }, + "x-ms-parameter-location": "method", + "description": "Describes what lease action to take." + } + ] + }, + "/{containerName}/{blob}?comp=lease&break": { + "put": { + "tags": [ + "blob" + ], + "operationId": "Blob_BreakLease", + "description": "[Update] The Lease Blob operation establishes and manages a lock on a blob for write and delete operations", + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/LeaseBreakPeriod" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/IfMatch" + }, + { + "$ref": "#/parameters/IfNoneMatch" + }, + { + "$ref": "#/parameters/IfTags" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "202": { + "description": "The Break operation completed successfully.", + "headers": { + "ETag": { + "type": "string", + "description": "The ETag contains a value that you can use to perform operations conditionally. If the request version is 2011-08-18 or newer, the ETag value will be in quotes." + }, + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the blob was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob." + }, + "x-ms-lease-time": { + "x-ms-client-name": "LeaseTime", + "type": "integer", + "description": "Approximate time remaining in the lease period, in seconds." + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "lease" + ] + }, + { + "name": "x-ms-lease-action", + "x-ms-client-name": "action", + "in": "header", + "required": true, + "type": "string", + "enum": [ + "break" + ], + "x-ms-enum": { + "name": "LeaseAction", + "modelAsString": false + }, + "x-ms-parameter-location": "method", + "description": "Describes what lease action to take." + } + ] + }, + "/{containerName}/{blob}?comp=snapshot": { + "put": { + "tags": [ + "blob" + ], + "operationId": "Blob_CreateSnapshot", + "description": "The Create Snapshot operation creates a read-only snapshot of a blob", + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/Metadata" + }, + { + "$ref": "#/parameters/EncryptionKey" + }, + { + "$ref": "#/parameters/EncryptionKeySha256" + }, + { + "$ref": "#/parameters/EncryptionAlgorithm" + }, + { + "$ref": "#/parameters/EncryptionScope" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/IfMatch" + }, + { + "$ref": "#/parameters/IfNoneMatch" + }, + { + "$ref": "#/parameters/IfTags" + }, + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "201": { + "description": "The snaptshot was taken successfully.", + "headers": { + "x-ms-snapshot": { + "x-ms-client-name": "Snapshot", + "type": "string", + "description": "Uniquely identifies the snapshot and indicates the snapshot version. It may be used in subsequent requests to access the snapshot" + }, + "ETag": { + "type": "string", + "description": "The ETag contains a value that you can use to perform operations conditionally. If the request version is 2011-08-18 or newer, the ETag value will be in quotes." + }, + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the container was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob." + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "x-ms-version-id": { + "x-ms-client-name": "VersionId", + "type": "string", + "description": "A DateTime value returned by the service that uniquely identifies the blob. The value of this header indicates the blob version, and may be used in subsequent requests to access this version of the blob." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + }, + "x-ms-request-server-encrypted": { + "x-ms-client-name": "IsServerEncrypted", + "type": "boolean", + "description": "True if the contents of the request are successfully encrypted using the specified algorithm, and false otherwise. For a snapshot request, this header is set to true when metadata was provided in the request and encrypted with a customer-provided key." + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "snapshot" + ] + } + ] + }, + "/{containerName}/{blob}?comp=copy": { + "put": { + "tags": [ + "blob" + ], + "operationId": "Blob_StartCopyFromURL", + "description": "The Start Copy From URL operation copies a blob or an internet resource to a new blob.", + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/Metadata" + }, + { + "$ref": "#/parameters/AccessTierOptional" + }, + { + "$ref": "#/parameters/RehydratePriority" + }, + { + "$ref": "#/parameters/SourceIfModifiedSince" + }, + { + "$ref": "#/parameters/SourceIfUnmodifiedSince" + }, + { + "$ref": "#/parameters/SourceIfMatch" + }, + { + "$ref": "#/parameters/SourceIfNoneMatch" + }, + { + "$ref": "#/parameters/SourceIfTags" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/IfMatch" + }, + { + "$ref": "#/parameters/IfNoneMatch" + }, + { + "$ref": "#/parameters/IfTags" + }, + { + "$ref": "#/parameters/CopySource" + }, + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + }, + { + "$ref": "#/parameters/BlobTagsHeader" + }, + { + "$ref": "#/parameters/SealBlob" + }, + { + "$ref": "#/parameters/ImmutabilityPolicyExpiry" + }, + { + "$ref": "#/parameters/ImmutabilityPolicyMode" + }, + { + "$ref": "#/parameters/LegalHoldOptional" + } + ], + "responses": { + "202": { + "description": "The copy blob has been accepted with the specified copy status.", + "headers": { + "ETag": { + "type": "string", + "description": "The ETag contains a value that you can use to perform operations conditionally. If the request version is 2011-08-18 or newer, the ETag value will be in quotes." + }, + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the container was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob." + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "x-ms-version-id": { + "x-ms-client-name": "VersionId", + "type": "string", + "description": "A DateTime value returned by the service that uniquely identifies the blob. The value of this header indicates the blob version, and may be used in subsequent requests to access this version of the blob." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + }, + "x-ms-copy-id": { + "x-ms-client-name": "CopyId", + "type": "string", + "description": "String identifier for this copy operation. Use with Get Blob Properties to check the status of this copy operation, or pass to Abort Copy Blob to abort a pending copy." + }, + "x-ms-copy-status": { + "x-ms-client-name": "CopyStatus", + "description": "State of the copy operation identified by x-ms-copy-id.", + "type": "string", + "enum": [ + "pending", + "success", + "aborted", + "failed" + ], + "x-ms-enum": { + "name": "CopyStatusType", + "modelAsString": false + } + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + } + }, + "/{containerName}/{blob}?comp=copy&sync": { + "put": { + "tags": [ + "blob" + ], + "operationId": "Blob_CopyFromURL", + "description": "The Copy From URL operation copies a blob or an internet resource to a new blob. It will not return a response until the copy is complete.", + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/Metadata" + }, + { + "$ref": "#/parameters/AccessTierOptional" + }, + { + "$ref": "#/parameters/SourceIfModifiedSince" + }, + { + "$ref": "#/parameters/SourceIfUnmodifiedSince" + }, + { + "$ref": "#/parameters/SourceIfMatch" + }, + { + "$ref": "#/parameters/SourceIfNoneMatch" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/IfMatch" + }, + { + "$ref": "#/parameters/IfNoneMatch" + }, + { + "$ref": "#/parameters/IfTags" + }, + { + "$ref": "#/parameters/CopySource" + }, + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + }, + { + "$ref": "#/parameters/SourceContentMD5" + }, + { + "$ref": "#/parameters/BlobTagsHeader" + }, + { + "$ref": "#/parameters/ImmutabilityPolicyExpiry" + }, + { + "$ref": "#/parameters/ImmutabilityPolicyMode" + }, + { + "$ref": "#/parameters/LegalHoldOptional" + }, + { + "$ref": "#/parameters/CopySourceAuthorization" + }, + { + "$ref": "#/parameters/EncryptionScope" + }, + { + "$ref": "#/parameters/CopySourceTags" + } + ], + "responses": { + "202": { + "description": "The copy has completed.", + "headers": { + "ETag": { + "type": "string", + "description": "The ETag contains a value that you can use to perform operations conditionally. If the request version is 2011-08-18 or newer, the ETag value will be in quotes." + }, + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the container was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob." + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "x-ms-version-id": { + "x-ms-client-name": "VersionId", + "type": "string", + "description": "A DateTime value returned by the service that uniquely identifies the blob. The value of this header indicates the blob version, and may be used in subsequent requests to access this version of the blob." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + }, + "x-ms-copy-id": { + "x-ms-client-name": "CopyId", + "type": "string", + "description": "String identifier for this copy operation." + }, + "x-ms-copy-status": { + "x-ms-client-name": "CopyStatus", + "description": "State of the copy operation identified by x-ms-copy-id.", + "type": "string", + "enum": [ + "success" + ], + "x-ms-enum": { + "name": "SyncCopyStatusType", + "modelAsString": false + } + }, + "Content-MD5": { + "type": "string", + "format": "byte", + "description": "This response header is returned so that the client can check for the integrity of the copied content. This header is only returned if the source content MD5 was specified." + }, + "x-ms-content-crc64": { + "type": "string", + "format": "byte", + "description": "This response header is returned so that the client can check for the integrity of the copied content." + }, + "x-ms-encryption-scope": { + "x-ms-client-name": "EncryptionScope", + "type": "string", + "description": "Returns the name of the encryption scope used to encrypt the blob contents and application metadata. Note that the absence of this header implies use of the default account encryption scope." + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "x-ms-requires-sync", + "description": "This header indicates that this is a synchronous Copy Blob From URL instead of a Asynchronous Copy Blob.", + "in": "header", + "required": true, + "type": "string", + "enum": [ + "true" + ] + } + ] + }, + "/{containerName}/{blob}?comp=copy©id": { + "put": { + "tags": [ + "blob" + ], + "operationId": "Blob_AbortCopyFromURL", + "description": "The Abort Copy From URL operation aborts a pending Copy From URL operation, and leaves a destination blob with zero length and full metadata.", + "parameters": [ + { + "$ref": "#/parameters/CopyId" + }, + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "204": { + "description": "The delete request was accepted and the blob will be deleted.", + "headers": { + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "copy" + ] + }, + { + "name": "x-ms-copy-action", + "description": "Copy action.", + "x-ms-client-name": "copyActionAbortConstant", + "in": "header", + "required": true, + "type": "string", + "enum": [ + "abort" + ], + "x-ms-parameter-location": "method" + } + ] + }, + "/{containerName}/{blob}?comp=tier": { + "put": { + "tags": [ + "blobs" + ], + "operationId": "Blob_SetTier", + "description": "The Set Tier operation sets the tier on a blob. The operation is allowed on a page blob in a premium storage account and on a block blob in a blob storage account (locally redundant storage only). A premium page blob's tier determines the allowed size, IOPS, and bandwidth of the blob. A block blob's tier determines Hot/Cool/Archive storage type. This operation does not update the blob's ETag.", + "parameters": [ + { + "$ref": "#/parameters/Snapshot" + }, + { + "$ref": "#/parameters/VersionId" + }, + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/AccessTierRequired" + }, + { + "$ref": "#/parameters/RehydratePriority" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + }, + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "$ref": "#/parameters/IfTags" + } + ], + "responses": { + "200": { + "description": "The new tier will take effect immediately.", + "headers": { + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and newer." + } + } + }, + "202": { + "description": "The transition to the new tier is pending.", + "headers": { + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and newer." + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "tier" + ] + } + ] + }, + "/{containerName}/{blob}?restype=account&comp=properties": { + "get": { + "tags": [ + "blob" + ], + "operationId": "Blob_GetAccountInfo", + "description": "Returns the sku name and account kind ", + "parameters": [ + { + "$ref": "#/parameters/ApiVersionParameter" + } + ], + "responses": { + "200": { + "description": "Success (OK)", + "headers": { + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + }, + "x-ms-sku-name": { + "x-ms-client-name": "SkuName", + "type": "string", + "enum": [ + "Standard_LRS", + "Standard_GRS", + "Standard_RAGRS", + "Standard_ZRS", + "Premium_LRS" + ], + "x-ms-enum": { + "name": "SkuName", + "modelAsString": false + }, + "description": "Identifies the sku name of the account" + }, + "x-ms-account-kind": { + "x-ms-client-name": "AccountKind", + "type": "string", + "enum": [ + "Storage", + "BlobStorage", + "StorageV2", + "FileStorage", + "BlockBlobStorage" + ], + "x-ms-enum": { + "name": "AccountKind", + "modelAsString": false + }, + "description": "Identifies the account kind" + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "head": { + "tags": [ + "blob" + ], + "operationId": "Blob_GetAccountInfoWithHead", + "description": "Returns the sku name and account kind ", + "parameters": [ + { + "$ref": "#/parameters/ApiVersionParameter" + } + ], + "responses": { + "200": { + "description": "Success (OK)", + "headers": { + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + }, + "x-ms-sku-name": { + "x-ms-client-name": "SkuName", + "type": "string", + "enum": [ + "Standard_LRS", + "Standard_GRS", + "Standard_RAGRS", + "Standard_ZRS", + "Premium_LRS" + ], + "x-ms-enum": { + "name": "SkuName", + "modelAsString": false + }, + "description": "Identifies the sku name of the account" + }, + "x-ms-account-kind": { + "x-ms-client-name": "AccountKind", + "type": "string", + "enum": [ + "Storage", + "BlobStorage", + "StorageV2", + "FileStorage", + "BlockBlobStorage" + ], + "x-ms-enum": { + "name": "AccountKind", + "modelAsString": false + }, + "description": "Identifies the account kind" + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "restype", + "description": "restype", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "account" + ] + }, + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "properties" + ] + } + ] + }, + "/{containerName}/{blob}?comp=block": { + "put": { + "tags": [ + "blockblob" + ], + "operationId": "BlockBlob_StageBlock", + "description": "The Stage Block operation creates a new block to be committed as part of a blob", + "consumes": [ + "application/octet-stream" + ], + "parameters": [ + { + "$ref": "#/parameters/BlockId" + }, + { + "$ref": "#/parameters/ContentLengthBlob" + }, + { + "$ref": "#/parameters/ContentMD5Blob" + }, + { + "$ref": "#/parameters/ContentCrc64" + }, + { + "$ref": "#/parameters/Body" + }, + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "$ref": "#/parameters/EncryptionKey" + }, + { + "$ref": "#/parameters/EncryptionKeySha256" + }, + { + "$ref": "#/parameters/EncryptionAlgorithm" + }, + { + "$ref": "#/parameters/EncryptionScope" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "201": { + "description": "The block was created.", + "headers": { + "Content-MD5": { + "type": "string", + "format": "byte", + "description": "This header is returned so that the client can check for message content integrity. The value of this header is computed by the Blob service; it is not necessarily the same value specified in the request headers." + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + }, + "x-ms-content-crc64": { + "type": "string", + "format": "byte", + "description": "This header is returned so that the client can check for message content integrity. The value of this header is computed by the Blob service; it is not necessarily the same value specified in the request headers." + }, + "x-ms-request-server-encrypted": { + "x-ms-client-name": "IsServerEncrypted", + "type": "boolean", + "description": "The value of this header is set to true if the contents of the request are successfully encrypted using the specified algorithm, and false otherwise." + }, + "x-ms-encryption-key-sha256": { + "x-ms-client-name": "EncryptionKeySha256", + "type": "string", + "description": "The SHA-256 hash of the encryption key used to encrypt the block. This header is only returned when the block was encrypted with a customer-provided key." + }, + "x-ms-encryption-scope": { + "x-ms-client-name": "EncryptionScope", + "type": "string", + "description": "Returns the name of the encryption scope used to encrypt the blob contents and application metadata. Note that the absence of this header implies use of the default account encryption scope." + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "block" + ] + } + ] + }, + "/{containerName}/{blob}?comp=block&fromURL": { + "put": { + "tags": [ + "blockblob" + ], + "operationId": "BlockBlob_StageBlockFromURL", + "description": "The Stage Block operation creates a new block to be committed as part of a blob where the contents are read from a URL.", + "parameters": [ + { + "$ref": "#/parameters/BlockId" + }, + { + "$ref": "#/parameters/ContentLengthBlob" + }, + { + "$ref": "#/parameters/SourceUrl" + }, + { + "$ref": "#/parameters/SourceRange" + }, + { + "$ref": "#/parameters/SourceContentMD5" + }, + { + "$ref": "#/parameters/SourceContentCRC64" + }, + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/EncryptionKey" + }, + { + "$ref": "#/parameters/EncryptionKeySha256" + }, + { + "$ref": "#/parameters/EncryptionAlgorithm" + }, + { + "$ref": "#/parameters/EncryptionScope" + }, + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "$ref": "#/parameters/SourceIfModifiedSince" + }, + { + "$ref": "#/parameters/SourceIfUnmodifiedSince" + }, + { + "$ref": "#/parameters/SourceIfMatch" + }, + { + "$ref": "#/parameters/SourceIfNoneMatch" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + }, + { + "$ref": "#/parameters/CopySourceAuthorization" + } + ], + "responses": { + "201": { + "description": "The block was created.", + "headers": { + "Content-MD5": { + "type": "string", + "format": "byte", + "description": "This header is returned so that the client can check for message content integrity. The value of this header is computed by the Blob service; it is not necessarily the same value specified in the request headers." + }, + "x-ms-content-crc64": { + "type": "string", + "format": "byte", + "description": "This header is returned so that the client can check for message content integrity. The value of this header is computed by the Blob service; it is not necessarily the same value specified in the request headers." + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + }, + "x-ms-request-server-encrypted": { + "x-ms-client-name": "IsServerEncrypted", + "type": "boolean", + "description": "The value of this header is set to true if the contents of the request are successfully encrypted using the specified algorithm, and false otherwise." + }, + "x-ms-encryption-key-sha256": { + "x-ms-client-name": "EncryptionKeySha256", + "type": "string", + "description": "The SHA-256 hash of the encryption key used to encrypt the block. This header is only returned when the block was encrypted with a customer-provided key." + }, + "x-ms-encryption-scope": { + "x-ms-client-name": "EncryptionScope", + "type": "string", + "description": "Returns the name of the encryption scope used to encrypt the blob contents and application metadata. Note that the absence of this header implies use of the default account encryption scope." + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "block" + ] + } + ] + }, + "/{containerName}/{blob}?comp=blocklist": { + "put": { + "tags": [ + "blockblob" + ], + "operationId": "BlockBlob_CommitBlockList", + "description": "The Commit Block List operation writes a blob by specifying the list of block IDs that make up the blob. In order to be written as part of a blob, a block must have been successfully written to the server in a prior Put Block operation. You can call Put Block List to update a blob by uploading only those blocks that have changed, then committing the new and existing blocks together. You can do this by specifying whether to commit a block from the committed block list or from the uncommitted block list, or to commit the most recently uploaded version of the block, whichever list it may belong to.", + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/BlobCacheControl" + }, + { + "$ref": "#/parameters/BlobContentType" + }, + { + "$ref": "#/parameters/BlobContentEncoding" + }, + { + "$ref": "#/parameters/BlobContentLanguage" + }, + { + "$ref": "#/parameters/BlobContentMD5" + }, + { + "$ref": "#/parameters/ContentMD5Blob" + }, + { + "$ref": "#/parameters/ContentCrc64" + }, + { + "$ref": "#/parameters/Metadata" + }, + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "$ref": "#/parameters/BlobContentDisposition" + }, + { + "$ref": "#/parameters/EncryptionKey" + }, + { + "$ref": "#/parameters/EncryptionKeySha256" + }, + { + "$ref": "#/parameters/EncryptionAlgorithm" + }, + { + "$ref": "#/parameters/EncryptionScope" + }, + { + "$ref": "#/parameters/AccessTierOptional" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/IfMatch" + }, + { + "$ref": "#/parameters/IfNoneMatch" + }, + { + "$ref": "#/parameters/IfTags" + }, + { + "name": "blocks", + "description": "Blob Blocks.", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/BlockLookupList" + } + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + }, + { + "$ref": "#/parameters/BlobTagsHeader" + }, + { + "$ref": "#/parameters/ImmutabilityPolicyExpiry" + }, + { + "$ref": "#/parameters/ImmutabilityPolicyMode" + }, + { + "$ref": "#/parameters/LegalHoldOptional" + } + ], + "responses": { + "201": { + "description": "The block list was recorded.", + "headers": { + "ETag": { + "type": "string", + "description": "The ETag contains a value that you can use to perform operations conditionally. If the request version is 2011-08-18 or newer, the ETag value will be in quotes." + }, + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the container was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob." + }, + "Content-MD5": { + "type": "string", + "format": "byte", + "description": "This header is returned so that the client can check for message content integrity. This header refers to the content of the request, meaning, in this case, the list of blocks, and not the content of the blob itself." + }, + "x-ms-content-crc64": { + "type": "string", + "format": "byte", + "description": "This header is returned so that the client can check for message content integrity. This header refers to the content of the request, meaning, in this case, the list of blocks, and not the content of the blob itself." + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "x-ms-version-id": { + "x-ms-client-name": "VersionId", + "type": "string", + "description": "A DateTime value returned by the service that uniquely identifies the blob. The value of this header indicates the blob version, and may be used in subsequent requests to access this version of the blob." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + }, + "x-ms-request-server-encrypted": { + "x-ms-client-name": "IsServerEncrypted", + "type": "boolean", + "description": "The value of this header is set to true if the contents of the request are successfully encrypted using the specified algorithm, and false otherwise." + }, + "x-ms-encryption-key-sha256": { + "x-ms-client-name": "EncryptionKeySha256", + "type": "string", + "description": "The SHA-256 hash of the encryption key used to encrypt the blob. This header is only returned when the blob was encrypted with a customer-provided key." + }, + "x-ms-encryption-scope": { + "x-ms-client-name": "EncryptionScope", + "type": "string", + "description": "Returns the name of the encryption scope used to encrypt the blob contents and application metadata. Note that the absence of this header implies use of the default account encryption scope." + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "get": { + "tags": [ + "blockblob" + ], + "operationId": "BlockBlob_GetBlockList", + "description": "The Get Block List operation retrieves the list of blocks that have been uploaded as part of a block blob", + "parameters": [ + { + "$ref": "#/parameters/Snapshot" + }, + { + "$ref": "#/parameters/BlockListType" + }, + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "$ref": "#/parameters/IfTags" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "200": { + "description": "The page range was written.", + "headers": { + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the container was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob." + }, + "ETag": { + "type": "string", + "description": "The ETag contains a value that you can use to perform operations conditionally. If the request version is 2011-08-18 or newer, the ETag value will be in quotes." + }, + "Content-Type": { + "type": "string", + "description": "The media type of the body of the response. For Get Block List this is 'application/xml'" + }, + "x-ms-blob-content-length": { + "x-ms-client-name": "BlobContentLength", + "type": "integer", + "format": "int64", + "description": "The size of the blob in bytes." + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + } + }, + "schema": { + "$ref": "#/definitions/BlockList" + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "blocklist" + ] + } + ] + }, + "/{containerName}/{blob}?comp=page&update": { + "put": { + "tags": [ + "pageblob" + ], + "operationId": "PageBlob_UploadPages", + "description": "The Upload Pages operation writes a range of pages to a page blob", + "consumes": [ + "application/octet-stream" + ], + "parameters": [ + { + "$ref": "#/parameters/Body" + }, + { + "$ref": "#/parameters/ContentLengthBlob" + }, + { + "$ref": "#/parameters/ContentMD5Blob" + }, + { + "$ref": "#/parameters/ContentCrc64" + }, + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/Range" + }, + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "$ref": "#/parameters/EncryptionKey" + }, + { + "$ref": "#/parameters/EncryptionKeySha256" + }, + { + "$ref": "#/parameters/EncryptionAlgorithm" + }, + { + "$ref": "#/parameters/EncryptionScope" + }, + { + "$ref": "#/parameters/IfSequenceNumberLessThanOrEqualTo" + }, + { + "$ref": "#/parameters/IfSequenceNumberLessThan" + }, + { + "$ref": "#/parameters/IfSequenceNumberEqualTo" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/IfMatch" + }, + { + "$ref": "#/parameters/IfNoneMatch" + }, + { + "$ref": "#/parameters/IfTags" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "201": { + "description": "The page range was written.", + "headers": { + "ETag": { + "type": "string", + "description": "The ETag contains a value that you can use to perform operations conditionally. If the request version is 2011-08-18 or newer, the ETag value will be in quotes." + }, + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the container was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob." + }, + "Content-MD5": { + "type": "string", + "format": "byte", + "description": "If the blob has an MD5 hash and this operation is to read the full blob, this response header is returned so that the client can check for message content integrity." + }, + "x-ms-content-crc64": { + "type": "string", + "format": "byte", + "description": "This header is returned so that the client can check for message content integrity. The value of this header is computed by the Blob service; it is not necessarily the same value specified in the request headers." + }, + "x-ms-blob-sequence-number": { + "x-ms-client-name": "BlobSequenceNumber", + "type": "integer", + "format": "int64", + "description": "The current sequence number for the page blob." + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + }, + "x-ms-request-server-encrypted": { + "x-ms-client-name": "IsServerEncrypted", + "type": "boolean", + "description": "The value of this header is set to true if the contents of the request are successfully encrypted using the specified algorithm, and false otherwise." + }, + "x-ms-encryption-key-sha256": { + "x-ms-client-name": "EncryptionKeySha256", + "type": "string", + "description": "The SHA-256 hash of the encryption key used to encrypt the pages. This header is only returned when the pages were encrypted with a customer-provided key." + }, + "x-ms-encryption-scope": { + "x-ms-client-name": "EncryptionScope", + "type": "string", + "description": "Returns the name of the encryption scope used to encrypt the blob contents and application metadata. Note that the absence of this header implies use of the default account encryption scope." + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "page" + ] + }, + { + "name": "x-ms-page-write", + "x-ms-client-name": "pageWrite", + "in": "header", + "required": true, + "x-ms-parameter-location": "method", + "description": "Required. You may specify one of the following options:\n - Update: Writes the bytes specified by the request body into the specified range. The Range and Content-Length headers must match to perform the update.\n - Clear: Clears the specified range and releases the space used in storage for that range. To clear a range, set the Content-Length header to zero, and the Range header to a value that indicates the range to clear, up to maximum blob size.", + "type": "string", + "enum": [ + "update" + ], + "x-ms-enum": { + "name": "PageWriteType", + "modelAsString": false + } + } + ] + }, + "/{containerName}/{blob}?comp=page&clear": { + "put": { + "tags": [ + "pageblob" + ], + "operationId": "PageBlob_ClearPages", + "description": "The Clear Pages operation clears a set of pages from a page blob", + "consumes": [ + "application/octet-stream" + ], + "parameters": [ + { + "$ref": "#/parameters/ContentLengthBlob" + }, + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/Range" + }, + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "$ref": "#/parameters/EncryptionKey" + }, + { + "$ref": "#/parameters/EncryptionKeySha256" + }, + { + "$ref": "#/parameters/EncryptionAlgorithm" + }, + { + "$ref": "#/parameters/EncryptionScope" + }, + { + "$ref": "#/parameters/IfSequenceNumberLessThanOrEqualTo" + }, + { + "$ref": "#/parameters/IfSequenceNumberLessThan" + }, + { + "$ref": "#/parameters/IfSequenceNumberEqualTo" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/IfMatch" + }, + { + "$ref": "#/parameters/IfNoneMatch" + }, + { + "$ref": "#/parameters/IfTags" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "201": { + "description": "The page range was cleared.", + "headers": { + "ETag": { + "type": "string", + "description": "The ETag contains a value that you can use to perform operations conditionally. If the request version is 2011-08-18 or newer, the ETag value will be in quotes." + }, + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the container was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob." + }, + "Content-MD5": { + "type": "string", + "format": "byte", + "description": "If the blob has an MD5 hash and this operation is to read the full blob, this response header is returned so that the client can check for message content integrity." + }, + "x-ms-content-crc64": { + "type": "string", + "format": "byte", + "description": "This header is returned so that the client can check for message content integrity. The value of this header is computed by the Blob service; it is not necessarily the same value specified in the request headers." + }, + "x-ms-blob-sequence-number": { + "x-ms-client-name": "BlobSequenceNumber", + "type": "integer", + "format": "int64", + "description": "The current sequence number for the page blob." + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "page" + ] + }, + { + "name": "x-ms-page-write", + "x-ms-client-name": "pageWrite", + "in": "header", + "required": true, + "x-ms-parameter-location": "method", + "description": "Required. You may specify one of the following options:\n - Update: Writes the bytes specified by the request body into the specified range. The Range and Content-Length headers must match to perform the update.\n - Clear: Clears the specified range and releases the space used in storage for that range. To clear a range, set the Content-Length header to zero, and the Range header to a value that indicates the range to clear, up to maximum blob size.", + "type": "string", + "enum": [ + "clear" + ], + "x-ms-enum": { + "name": "PageWriteType", + "modelAsString": false + } + } + ] + }, + "/{containerName}/{blob}?comp=page&update&fromUrl": { + "put": { + "tags": [ + "pageblob" + ], + "operationId": "PageBlob_UploadPagesFromURL", + "description": "The Upload Pages operation writes a range of pages to a page blob where the contents are read from a URL", + "consumes": [ + "application/octet-stream" + ], + "parameters": [ + { + "$ref": "#/parameters/SourceUrl" + }, + { + "$ref": "#/parameters/SourceRangeRequiredPutPageFromUrl" + }, + { + "$ref": "#/parameters/SourceContentMD5" + }, + { + "$ref": "#/parameters/SourceContentCRC64" + }, + { + "$ref": "#/parameters/ContentLengthBlob" + }, + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/RangeRequiredPutPageFromUrl" + }, + { + "$ref": "#/parameters/EncryptionKey" + }, + { + "$ref": "#/parameters/EncryptionKeySha256" + }, + { + "$ref": "#/parameters/EncryptionAlgorithm" + }, + { + "$ref": "#/parameters/EncryptionScope" + }, + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "$ref": "#/parameters/IfSequenceNumberLessThanOrEqualTo" + }, + { + "$ref": "#/parameters/IfSequenceNumberLessThan" + }, + { + "$ref": "#/parameters/IfSequenceNumberEqualTo" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/IfMatch" + }, + { + "$ref": "#/parameters/IfNoneMatch" + }, + { + "$ref": "#/parameters/IfTags" + }, + { + "$ref": "#/parameters/SourceIfModifiedSince" + }, + { + "$ref": "#/parameters/SourceIfUnmodifiedSince" + }, + { + "$ref": "#/parameters/SourceIfMatch" + }, + { + "$ref": "#/parameters/SourceIfNoneMatch" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + }, + { + "$ref": "#/parameters/CopySourceAuthorization" + } + ], + "responses": { + "201": { + "description": "The page range was written.", + "headers": { + "ETag": { + "type": "string", + "description": "The ETag contains a value that you can use to perform operations conditionally. If the request version is 2011-08-18 or newer, the ETag value will be in quotes." + }, + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the container was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob." + }, + "Content-MD5": { + "type": "string", + "format": "byte", + "description": "If the blob has an MD5 hash and this operation is to read the full blob, this response header is returned so that the client can check for message content integrity." + }, + "x-ms-content-crc64": { + "type": "string", + "format": "byte", + "description": "This header is returned so that the client can check for message content integrity. The value of this header is computed by the Blob service; it is not necessarily the same value specified in the request headers." + }, + "x-ms-blob-sequence-number": { + "x-ms-client-name": "BlobSequenceNumber", + "type": "integer", + "format": "int64", + "description": "The current sequence number for the page blob." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + }, + "x-ms-request-server-encrypted": { + "x-ms-client-name": "IsServerEncrypted", + "type": "boolean", + "description": "The value of this header is set to true if the contents of the request are successfully encrypted using the specified algorithm, and false otherwise." + }, + "x-ms-encryption-key-sha256": { + "x-ms-client-name": "EncryptionKeySha256", + "type": "string", + "description": "The SHA-256 hash of the encryption key used to encrypt the blob. This header is only returned when the blob was encrypted with a customer-provided key." + }, + "x-ms-encryption-scope": { + "x-ms-client-name": "EncryptionScope", + "type": "string", + "description": "Returns the name of the encryption scope used to encrypt the blob contents and application metadata. Note that the absence of this header implies use of the default account encryption scope." + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "page" + ] + }, + { + "name": "x-ms-page-write", + "x-ms-client-name": "pageWrite", + "in": "header", + "required": true, + "x-ms-parameter-location": "method", + "description": "Required. You may specify one of the following options:\n - Update: Writes the bytes specified by the request body into the specified range. The Range and Content-Length headers must match to perform the update.\n - Clear: Clears the specified range and releases the space used in storage for that range. To clear a range, set the Content-Length header to zero, and the Range header to a value that indicates the range to clear, up to maximum blob size.", + "type": "string", + "enum": [ + "update" + ], + "x-ms-enum": { + "name": "PageWriteType", + "modelAsString": false + } + } + ] + }, + "/{containerName}/{blob}?comp=pagelist": { + "get": { + "tags": [ + "pageblob" + ], + "operationId": "PageBlob_GetPageRanges", + "description": "The Get Page Ranges operation returns the list of valid page ranges for a page blob or snapshot of a page blob", + "parameters": [ + { + "$ref": "#/parameters/Snapshot" + }, + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/Range" + }, + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/IfMatch" + }, + { + "$ref": "#/parameters/IfNoneMatch" + }, + { + "$ref": "#/parameters/IfTags" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + }, + { + "$ref": "#/parameters/Marker" + }, + { + "$ref": "#/parameters/MaxResultsBlob" + } + ], + "responses": { + "200": { + "description": "Information on the page blob was found.", + "headers": { + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the container was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob." + }, + "ETag": { + "type": "string", + "description": "The ETag contains a value that you can use to perform operations conditionally. If the request version is 2011-08-18 or newer, the ETag value will be in quotes." + }, + "x-ms-blob-content-length": { + "x-ms-client-name": "BlobContentLength", + "type": "integer", + "format": "int64", + "description": "The size of the blob in bytes." + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + } + }, + "schema": { + "$ref": "#/definitions/PageList" + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + }, + "x-ms-pageable": { + "nextLinkName": "NextMarker" + } + }, + "parameters": [ + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "pagelist" + ] + } + ] + }, + "/{containerName}/{blob}?comp=pagelist&diff": { + "get": { + "tags": [ + "pageblob" + ], + "operationId": "PageBlob_GetPageRangesDiff", + "description": "The Get Page Ranges Diff operation returns the list of valid page ranges for a page blob that were changed between target blob and previous snapshot.", + "parameters": [ + { + "$ref": "#/parameters/Snapshot" + }, + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/PrevSnapshot" + }, + { + "$ref": "#/parameters/PrevSnapshotUrl" + }, + { + "$ref": "#/parameters/Range" + }, + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/IfMatch" + }, + { + "$ref": "#/parameters/IfNoneMatch" + }, + { + "$ref": "#/parameters/IfTags" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + }, + { + "$ref": "#/parameters/Marker" + }, + { + "$ref": "#/parameters/MaxResultsBlob" + } + ], + "responses": { + "200": { + "description": "Information on the page blob was found.", + "headers": { + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the container was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob." + }, + "ETag": { + "type": "string", + "description": "The ETag contains a value that you can use to perform operations conditionally. If the request version is 2011-08-18 or newer, the ETag value will be in quotes." + }, + "x-ms-blob-content-length": { + "x-ms-client-name": "BlobContentLength", + "type": "integer", + "format": "int64", + "description": "The size of the blob in bytes." + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + } + }, + "schema": { + "$ref": "#/definitions/PageList" + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + }, + "x-ms-pageable": { + "nextLinkName": "NextMarker" + } + }, + "parameters": [ + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "pagelist" + ] + } + ] + }, + "/{containerName}/{blob}?comp=properties&Resize": { + "put": { + "tags": [ + "pageblob" + ], + "operationId": "PageBlob_Resize", + "description": "Resize the Blob", + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "$ref": "#/parameters/EncryptionKey" + }, + { + "$ref": "#/parameters/EncryptionKeySha256" + }, + { + "$ref": "#/parameters/EncryptionAlgorithm" + }, + { + "$ref": "#/parameters/EncryptionScope" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/IfMatch" + }, + { + "$ref": "#/parameters/IfNoneMatch" + }, + { + "$ref": "#/parameters/IfTags" + }, + { + "$ref": "#/parameters/BlobContentLengthRequired" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "200": { + "description": "The Blob was resized successfully", + "headers": { + "ETag": { + "type": "string", + "description": "The ETag contains a value that you can use to perform operations conditionally. If the request version is 2011-08-18 or newer, the ETag value will be in quotes." + }, + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the container was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob." + }, + "x-ms-blob-sequence-number": { + "x-ms-client-name": "BlobSequenceNumber", + "type": "integer", + "format": "int64", + "description": "The current sequence number for a page blob. This header is not returned for block blobs or append blobs" + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "properties" + ] + } + ] + }, + "/{containerName}/{blob}?comp=properties&UpdateSequenceNumber": { + "put": { + "tags": [ + "pageblob" + ], + "operationId": "PageBlob_UpdateSequenceNumber", + "description": "Update the sequence number of the blob", + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/IfMatch" + }, + { + "$ref": "#/parameters/IfNoneMatch" + }, + { + "$ref": "#/parameters/IfTags" + }, + { + "$ref": "#/parameters/SequenceNumberAction" + }, + { + "$ref": "#/parameters/BlobSequenceNumber" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "200": { + "description": "The sequence numbers were updated successfully.", + "headers": { + "ETag": { + "type": "string", + "description": "The ETag contains a value that you can use to perform operations conditionally. If the request version is 2011-08-18 or newer, the ETag value will be in quotes." + }, + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the container was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob." + }, + "x-ms-blob-sequence-number": { + "x-ms-client-name": "BlobSequenceNumber", + "type": "integer", + "format": "int64", + "description": "The current sequence number for a page blob. This header is not returned for block blobs or append blobs" + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "properties" + ] + } + ] + }, + "/{containerName}/{blob}?comp=incrementalcopy": { + "put": { + "tags": [ + "pageblob" + ], + "operationId": "PageBlob_CopyIncremental", + "description": "The Copy Incremental operation copies a snapshot of the source page blob to a destination page blob. The snapshot is copied such that only the differential changes between the previously copied snapshot are transferred to the destination. The copied snapshots are complete copies of the original snapshot and can be read or copied from as usual. This API is supported since REST version 2016-05-31.", + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/IfMatch" + }, + { + "$ref": "#/parameters/IfNoneMatch" + }, + { + "$ref": "#/parameters/IfTags" + }, + { + "$ref": "#/parameters/CopySource" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "202": { + "description": "The blob was copied.", + "headers": { + "ETag": { + "type": "string", + "description": "The ETag contains a value that you can use to perform operations conditionally. If the request version is 2011-08-18 or newer, the ETag value will be in quotes." + }, + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the container was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob." + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + }, + "x-ms-copy-id": { + "x-ms-client-name": "CopyId", + "type": "string", + "description": "String identifier for this copy operation. Use with Get Blob Properties to check the status of this copy operation, or pass to Abort Copy Blob to abort a pending copy." + }, + "x-ms-copy-status": { + "x-ms-client-name": "CopyStatus", + "description": "State of the copy operation identified by x-ms-copy-id.", + "type": "string", + "enum": [ + "pending", + "success", + "aborted", + "failed" + ], + "x-ms-enum": { + "name": "CopyStatusType", + "modelAsString": false + } + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "incrementalcopy" + ] + } + ] + }, + "/{containerName}/{blob}?comp=appendblock": { + "put": { + "tags": [ + "appendblob" + ], + "consumes": [ + "application/octet-stream" + ], + "operationId": "AppendBlob_AppendBlock", + "description": "The Append Block operation commits a new block of data to the end of an existing append blob. The Append Block operation is permitted only if the blob was created with x-ms-blob-type set to AppendBlob. Append Block is supported only on version 2015-02-21 version or later.", + "parameters": [ + { + "$ref": "#/parameters/Body" + }, + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/ContentLengthBlob" + }, + { + "$ref": "#/parameters/ContentMD5Blob" + }, + { + "$ref": "#/parameters/ContentCrc64" + }, + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "$ref": "#/parameters/BlobConditionMaxSize" + }, + { + "$ref": "#/parameters/BlobConditionAppendPos" + }, + { + "$ref": "#/parameters/EncryptionKey" + }, + { + "$ref": "#/parameters/EncryptionKeySha256" + }, + { + "$ref": "#/parameters/EncryptionAlgorithm" + }, + { + "$ref": "#/parameters/EncryptionScope" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/IfMatch" + }, + { + "$ref": "#/parameters/IfNoneMatch" + }, + { + "$ref": "#/parameters/IfTags" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "201": { + "description": "The block was created.", + "headers": { + "ETag": { + "type": "string", + "description": "The ETag contains a value that you can use to perform operations conditionally. If the request version is 2011-08-18 or newer, the ETag value will be in quotes." + }, + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the container was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob." + }, + "Content-MD5": { + "type": "string", + "format": "byte", + "description": "If the blob has an MD5 hash and this operation is to read the full blob, this response header is returned so that the client can check for message content integrity." + }, + "x-ms-content-crc64": { + "type": "string", + "format": "byte", + "description": "This header is returned so that the client can check for message content integrity. The value of this header is computed by the Blob service; it is not necessarily the same value specified in the request headers." + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + }, + "x-ms-blob-append-offset": { + "x-ms-client-name": "BlobAppendOffset", + "type": "string", + "description": "This response header is returned only for append operations. It returns the offset at which the block was committed, in bytes." + }, + "x-ms-blob-committed-block-count": { + "x-ms-client-name": "BlobCommittedBlockCount", + "type": "integer", + "description": "The number of committed blocks present in the blob. This header is returned only for append blobs." + }, + "x-ms-request-server-encrypted": { + "x-ms-client-name": "IsServerEncrypted", + "type": "boolean", + "description": "The value of this header is set to true if the contents of the request are successfully encrypted using the specified algorithm, and false otherwise." + }, + "x-ms-encryption-key-sha256": { + "x-ms-client-name": "EncryptionKeySha256", + "type": "string", + "description": "The SHA-256 hash of the encryption key used to encrypt the block. This header is only returned when the block was encrypted with a customer-provided key." + }, + "x-ms-encryption-scope": { + "x-ms-client-name": "EncryptionScope", + "type": "string", + "description": "Returns the name of the encryption scope used to encrypt the blob contents and application metadata. Note that the absence of this header implies use of the default account encryption scope." + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "appendblock" + ] + } + ] + }, + "/{containerName}/{blob}?comp=appendblock&fromUrl": { + "put": { + "tags": [ + "appendblob" + ], + "operationId": "AppendBlob_AppendBlockFromUrl", + "description": "The Append Block operation commits a new block of data to the end of an existing append blob where the contents are read from a source url. The Append Block operation is permitted only if the blob was created with x-ms-blob-type set to AppendBlob. Append Block is supported only on version 2015-02-21 version or later.", + "parameters": [ + { + "$ref": "#/parameters/SourceUrl" + }, + { + "$ref": "#/parameters/SourceRange" + }, + { + "$ref": "#/parameters/SourceContentMD5" + }, + { + "$ref": "#/parameters/SourceContentCRC64" + }, + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/ContentLengthBlob" + }, + { + "$ref": "#/parameters/ContentMD5Blob" + }, + { + "$ref": "#/parameters/EncryptionKey" + }, + { + "$ref": "#/parameters/EncryptionKeySha256" + }, + { + "$ref": "#/parameters/EncryptionAlgorithm" + }, + { + "$ref": "#/parameters/EncryptionScope" + }, + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "$ref": "#/parameters/BlobConditionMaxSize" + }, + { + "$ref": "#/parameters/BlobConditionAppendPos" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/IfMatch" + }, + { + "$ref": "#/parameters/IfNoneMatch" + }, + { + "$ref": "#/parameters/IfTags" + }, + { + "$ref": "#/parameters/SourceIfModifiedSince" + }, + { + "$ref": "#/parameters/SourceIfUnmodifiedSince" + }, + { + "$ref": "#/parameters/SourceIfMatch" + }, + { + "$ref": "#/parameters/SourceIfNoneMatch" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + }, + { + "$ref": "#/parameters/CopySourceAuthorization" + } + ], + "responses": { + "201": { + "description": "The block was created.", + "headers": { + "ETag": { + "type": "string", + "description": "The ETag contains a value that you can use to perform operations conditionally. If the request version is 2011-08-18 or newer, the ETag value will be in quotes." + }, + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the container was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob." + }, + "Content-MD5": { + "type": "string", + "format": "byte", + "description": "If the blob has an MD5 hash and this operation is to read the full blob, this response header is returned so that the client can check for message content integrity." + }, + "x-ms-content-crc64": { + "type": "string", + "format": "byte", + "description": "This header is returned so that the client can check for message content integrity. The value of this header is computed by the Blob service; it is not necessarily the same value specified in the request headers." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + }, + "x-ms-blob-append-offset": { + "x-ms-client-name": "BlobAppendOffset", + "type": "string", + "description": "This response header is returned only for append operations. It returns the offset at which the block was committed, in bytes." + }, + "x-ms-blob-committed-block-count": { + "x-ms-client-name": "BlobCommittedBlockCount", + "type": "integer", + "description": "The number of committed blocks present in the blob. This header is returned only for append blobs." + }, + "x-ms-encryption-key-sha256": { + "x-ms-client-name": "EncryptionKeySha256", + "type": "string", + "description": "The SHA-256 hash of the encryption key used to encrypt the block. This header is only returned when the block was encrypted with a customer-provided key." + }, + "x-ms-encryption-scope": { + "x-ms-client-name": "EncryptionScope", + "type": "string", + "description": "Returns the name of the encryption scope used to encrypt the blob contents and application metadata. Note that the absence of this header implies use of the default account encryption scope." + }, + "x-ms-request-server-encrypted": { + "x-ms-client-name": "IsServerEncrypted", + "type": "boolean", + "description": "The value of this header is set to true if the contents of the request are successfully encrypted using the specified algorithm, and false otherwise." + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "appendblock" + ] + } + ] + }, + "/{containerName}/{blob}?comp=seal": { + "put": { + "tags": [ + "appendblob" + ], + "operationId": "AppendBlob_Seal", + "description": "The Seal operation seals the Append Blob to make it read-only. Seal is supported only on version 2019-12-12 version or later.", + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + }, + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/IfMatch" + }, + { + "$ref": "#/parameters/IfNoneMatch" + }, + { + "$ref": "#/parameters/BlobConditionAppendPos" + } + ], + "responses": { + "200": { + "description": "The blob was sealed.", + "headers": { + "ETag": { + "type": "string", + "description": "The ETag contains a value that you can use to perform operations conditionally. If the request version is 2011-08-18 or newer, the ETag value will be in quotes." + }, + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the container was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob." + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + }, + "x-ms-blob-sealed": { + "x-ms-client-name": "IsSealed", + "type": "boolean", + "description": "If this blob has been sealed" + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "seal" + ] + } + ] + }, + "/{containerName}/{blob}?comp=query": { + "post": { + "tags": [ + "blob" + ], + "operationId": "Blob_Query", + "description": "The Query operation enables users to select/project on blob data by providing simple query expressions.", + "parameters": [ + { + "$ref": "#/parameters/QueryRequest" + }, + { + "$ref": "#/parameters/Snapshot" + }, + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "$ref": "#/parameters/EncryptionKey" + }, + { + "$ref": "#/parameters/EncryptionKeySha256" + }, + { + "$ref": "#/parameters/EncryptionAlgorithm" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/IfMatch" + }, + { + "$ref": "#/parameters/IfNoneMatch" + }, + { + "$ref": "#/parameters/IfTags" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "200": { + "description": "Returns the content of the entire blob.", + "headers": { + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the container was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob." + }, + "x-ms-meta": { + "type": "string", + "x-ms-client-name": "Metadata", + "x-ms-header-collection-prefix": "x-ms-meta-" + }, + "Content-Length": { + "type": "integer", + "format": "int64", + "description": "The number of bytes present in the response body." + }, + "Content-Type": { + "type": "string", + "description": "The media type of the body of the response. For Download Blob this is 'application/octet-stream'" + }, + "Content-Range": { + "type": "string", + "description": "Indicates the range of bytes returned in the event that the client requested a subset of the blob by setting the 'Range' request header." + }, + "ETag": { + "type": "string", + "description": "The ETag contains a value that you can use to perform operations conditionally. If the request version is 2011-08-18 or newer, the ETag value will be in quotes." + }, + "Content-MD5": { + "type": "string", + "format": "byte", + "description": "If the blob has an MD5 hash and this operation is to read the full blob, this response header is returned so that the client can check for message content integrity." + }, + "Content-Encoding": { + "type": "string", + "description": "This header returns the value that was specified for the Content-Encoding request header" + }, + "Cache-Control": { + "type": "string", + "description": "This header is returned if it was previously specified for the blob." + }, + "Content-Disposition": { + "type": "string", + "description": "This header returns the value that was specified for the 'x-ms-blob-content-disposition' header. The Content-Disposition response header field conveys additional information about how to process the response payload, and also can be used to attach additional metadata. For example, if set to attachment, it indicates that the user-agent should not display the response, but instead show a Save As dialog with a filename other than the blob name specified." + }, + "Content-Language": { + "type": "string", + "description": "This header returns the value that was specified for the Content-Language request header." + }, + "x-ms-blob-sequence-number": { + "x-ms-client-name": "BlobSequenceNumber", + "type": "integer", + "format": "int64", + "description": "The current sequence number for a page blob. This header is not returned for block blobs or append blobs" + }, + "x-ms-blob-type": { + "x-ms-client-name": "BlobType", + "description": "The blob's type.", + "type": "string", + "enum": [ + "BlockBlob", + "PageBlob", + "AppendBlob" + ], + "x-ms-enum": { + "name": "BlobType", + "modelAsString": false + } + }, + "x-ms-copy-completion-time": { + "x-ms-client-name": "CopyCompletionTime", + "type": "string", + "format": "date-time-rfc1123", + "description": "Conclusion time of the last attempted Copy Blob operation where this blob was the destination blob. This value can specify the time of a completed, aborted, or failed copy attempt. This header does not appear if a copy is pending, if this blob has never been the destination in a Copy Blob operation, or if this blob has been modified after a concluded Copy Blob operation using Set Blob Properties, Put Blob, or Put Block List." + }, + "x-ms-copy-status-description": { + "x-ms-client-name": "CopyStatusDescription", + "type": "string", + "description": "Only appears when x-ms-copy-status is failed or pending. Describes the cause of the last fatal or non-fatal copy operation failure. This header does not appear if this blob has never been the destination in a Copy Blob operation, or if this blob has been modified after a concluded Copy Blob operation using Set Blob Properties, Put Blob, or Put Block List" + }, + "x-ms-copy-id": { + "x-ms-client-name": "CopyId", + "type": "string", + "description": "String identifier for this copy operation. Use with Get Blob Properties to check the status of this copy operation, or pass to Abort Copy Blob to abort a pending copy." + }, + "x-ms-copy-progress": { + "x-ms-client-name": "CopyProgress", + "type": "string", + "description": "Contains the number of bytes copied and the total bytes in the source in the last attempted Copy Blob operation where this blob was the destination blob. Can show between 0 and Content-Length bytes copied. This header does not appear if this blob has never been the destination in a Copy Blob operation, or if this blob has been modified after a concluded Copy Blob operation using Set Blob Properties, Put Blob, or Put Block List" + }, + "x-ms-copy-source": { + "x-ms-client-name": "CopySource", + "type": "string", + "description": "URL up to 2 KB in length that specifies the source blob or file used in the last attempted Copy Blob operation where this blob was the destination blob. This header does not appear if this blob has never been the destination in a Copy Blob operation, or if this blob has been modified after a concluded Copy Blob operation using Set Blob Properties, Put Blob, or Put Block List." + }, + "x-ms-copy-status": { + "x-ms-client-name": "CopyStatus", + "description": "State of the copy operation identified by x-ms-copy-id.", + "type": "string", + "enum": [ + "pending", + "success", + "aborted", + "failed" + ], + "x-ms-enum": { + "name": "CopyStatusType", + "modelAsString": false + } + }, + "x-ms-lease-duration": { + "x-ms-client-name": "LeaseDuration", + "description": "When a blob is leased, specifies whether the lease is of infinite or fixed duration.", + "type": "string", + "enum": [ + "infinite", + "fixed" + ], + "x-ms-enum": { + "name": "LeaseDurationType", + "modelAsString": false + } + }, + "x-ms-lease-state": { + "x-ms-client-name": "LeaseState", + "description": "Lease state of the blob.", + "type": "string", + "enum": [ + "available", + "leased", + "expired", + "breaking", + "broken" + ], + "x-ms-enum": { + "name": "LeaseStateType", + "modelAsString": false + } + }, + "x-ms-lease-status": { + "x-ms-client-name": "LeaseStatus", + "description": "The current lease status of the blob.", + "type": "string", + "enum": [ + "locked", + "unlocked" + ], + "x-ms-enum": { + "name": "LeaseStatusType", + "modelAsString": false + } + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Accept-Ranges": { + "type": "string", + "description": "Indicates that the service supports requests for partial blob content." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + }, + "x-ms-blob-committed-block-count": { + "x-ms-client-name": "BlobCommittedBlockCount", + "type": "integer", + "description": "The number of committed blocks present in the blob. This header is returned only for append blobs." + }, + "x-ms-server-encrypted": { + "x-ms-client-name": "IsServerEncrypted", + "type": "boolean", + "description": "The value of this header is set to true if the blob data and application metadata are completely encrypted using the specified algorithm. Otherwise, the value is set to false (when the blob is unencrypted, or if only parts of the blob/application metadata are encrypted)." + }, + "x-ms-encryption-key-sha256": { + "x-ms-client-name": "EncryptionKeySha256", + "type": "string", + "description": "The SHA-256 hash of the encryption key used to encrypt the blob. This header is only returned when the blob was encrypted with a customer-provided key." + }, + "x-ms-encryption-scope": { + "x-ms-client-name": "EncryptionScope", + "type": "string", + "description": "Returns the name of the encryption scope used to encrypt the blob contents and application metadata. Note that the absence of this header implies use of the default account encryption scope." + }, + "x-ms-blob-content-md5": { + "x-ms-client-name": "BlobContentMD5", + "type": "string", + "format": "byte", + "description": "If the blob has a MD5 hash, and if request contains range header (Range or x-ms-range), this response header is returned with the value of the whole blob's MD5 value. This value may or may not be equal to the value returned in Content-MD5 header, with the latter calculated from the requested range" + } + }, + "schema": { + "type": "object", + "format": "file" + } + }, + "206": { + "description": "Returns the content of a specified range of the blob.", + "headers": { + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the container was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob." + }, + "x-ms-meta": { + "type": "string", + "x-ms-client-name": "Metadata", + "x-ms-header-collection-prefix": "x-ms-meta-" + }, + "Content-Length": { + "type": "integer", + "format": "int64", + "description": "The number of bytes present in the response body." + }, + "Content-Type": { + "type": "string", + "description": "The media type of the body of the response. For Download Blob this is 'application/octet-stream'" + }, + "Content-Range": { + "type": "string", + "description": "Indicates the range of bytes returned in the event that the client requested a subset of the blob by setting the 'Range' request header." + }, + "ETag": { + "type": "string", + "description": "The ETag contains a value that you can use to perform operations conditionally. If the request version is 2011-08-18 or newer, the ETag value will be in quotes." + }, + "Content-MD5": { + "type": "string", + "format": "byte", + "description": "If the blob has an MD5 hash and this operation is to read the full blob, this response header is returned so that the client can check for message content integrity." + }, + "Content-Encoding": { + "type": "string", + "description": "This header returns the value that was specified for the Content-Encoding request header" + }, + "Cache-Control": { + "type": "string", + "description": "This header is returned if it was previously specified for the blob." + }, + "Content-Disposition": { + "type": "string", + "description": "This header returns the value that was specified for the 'x-ms-blob-content-disposition' header. The Content-Disposition response header field conveys additional information about how to process the response payload, and also can be used to attach additional metadata. For example, if set to attachment, it indicates that the user-agent should not display the response, but instead show a Save As dialog with a filename other than the blob name specified." + }, + "Content-Language": { + "type": "string", + "description": "This header returns the value that was specified for the Content-Language request header." + }, + "x-ms-blob-sequence-number": { + "x-ms-client-name": "BlobSequenceNumber", + "type": "integer", + "format": "int64", + "description": "The current sequence number for a page blob. This header is not returned for block blobs or append blobs" + }, + "x-ms-blob-type": { + "x-ms-client-name": "BlobType", + "description": "The blob's type.", + "type": "string", + "enum": [ + "BlockBlob", + "PageBlob", + "AppendBlob" + ], + "x-ms-enum": { + "name": "BlobType", + "modelAsString": false + } + }, + "x-ms-content-crc64": { + "x-ms-client-name": "ContentCrc64", + "type": "string", + "format": "byte", + "description": "If the request is to read a specified range and the x-ms-range-get-content-crc64 is set to true, then the request returns a crc64 for the range, as long as the range size is less than or equal to 4 MB. If both x-ms-range-get-content-crc64 and x-ms-range-get-content-md5 is specified in the same request, it will fail with 400(Bad Request)" + }, + "x-ms-copy-completion-time": { + "x-ms-client-name": "CopyCompletionTime", + "type": "string", + "format": "date-time-rfc1123", + "description": "Conclusion time of the last attempted Copy Blob operation where this blob was the destination blob. This value can specify the time of a completed, aborted, or failed copy attempt. This header does not appear if a copy is pending, if this blob has never been the destination in a Copy Blob operation, or if this blob has been modified after a concluded Copy Blob operation using Set Blob Properties, Put Blob, or Put Block List." + }, + "x-ms-copy-status-description": { + "x-ms-client-name": "CopyStatusDescription", + "type": "string", + "description": "Only appears when x-ms-copy-status is failed or pending. Describes the cause of the last fatal or non-fatal copy operation failure. This header does not appear if this blob has never been the destination in a Copy Blob operation, or if this blob has been modified after a concluded Copy Blob operation using Set Blob Properties, Put Blob, or Put Block List" + }, + "x-ms-copy-id": { + "x-ms-client-name": "CopyId", + "type": "string", + "description": "String identifier for this copy operation. Use with Get Blob Properties to check the status of this copy operation, or pass to Abort Copy Blob to abort a pending copy." + }, + "x-ms-copy-progress": { + "x-ms-client-name": "CopyProgress", + "type": "string", + "description": "Contains the number of bytes copied and the total bytes in the source in the last attempted Copy Blob operation where this blob was the destination blob. Can show between 0 and Content-Length bytes copied. This header does not appear if this blob has never been the destination in a Copy Blob operation, or if this blob has been modified after a concluded Copy Blob operation using Set Blob Properties, Put Blob, or Put Block List" + }, + "x-ms-copy-source": { + "x-ms-client-name": "CopySource", + "type": "string", + "description": "URL up to 2 KB in length that specifies the source blob or file used in the last attempted Copy Blob operation where this blob was the destination blob. This header does not appear if this blob has never been the destination in a Copy Blob operation, or if this blob has been modified after a concluded Copy Blob operation using Set Blob Properties, Put Blob, or Put Block List." + }, + "x-ms-copy-status": { + "x-ms-client-name": "CopyStatus", + "description": "State of the copy operation identified by x-ms-copy-id.", + "type": "string", + "enum": [ + "pending", + "success", + "aborted", + "failed" + ], + "x-ms-enum": { + "name": "CopyStatusType", + "modelAsString": false + } + }, + "x-ms-lease-duration": { + "x-ms-client-name": "LeaseDuration", + "description": "When a blob is leased, specifies whether the lease is of infinite or fixed duration.", + "type": "string", + "enum": [ + "infinite", + "fixed" + ], + "x-ms-enum": { + "name": "LeaseDurationType", + "modelAsString": false + } + }, + "x-ms-lease-state": { + "x-ms-client-name": "LeaseState", + "description": "Lease state of the blob.", + "type": "string", + "enum": [ + "available", + "leased", + "expired", + "breaking", + "broken" + ], + "x-ms-enum": { + "name": "LeaseStateType", + "modelAsString": false + } + }, + "x-ms-lease-status": { + "x-ms-client-name": "LeaseStatus", + "description": "The current lease status of the blob.", + "type": "string", + "enum": [ + "locked", + "unlocked" + ], + "x-ms-enum": { + "name": "LeaseStatusType", + "modelAsString": false + } + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Accept-Ranges": { + "type": "string", + "description": "Indicates that the service supports requests for partial blob content." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + }, + "x-ms-blob-committed-block-count": { + "x-ms-client-name": "BlobCommittedBlockCount", + "type": "integer", + "description": "The number of committed blocks present in the blob. This header is returned only for append blobs." + }, + "x-ms-server-encrypted": { + "x-ms-client-name": "IsServerEncrypted", + "type": "boolean", + "description": "The value of this header is set to true if the blob data and application metadata are completely encrypted using the specified algorithm. Otherwise, the value is set to false (when the blob is unencrypted, or if only parts of the blob/application metadata are encrypted)." + }, + "x-ms-encryption-key-sha256": { + "x-ms-client-name": "EncryptionKeySha256", + "type": "string", + "description": "The SHA-256 hash of the encryption key used to encrypt the blob. This header is only returned when the blob was encrypted with a customer-provided key." + }, + "x-ms-encryption-scope": { + "x-ms-client-name": "EncryptionScope", + "type": "string", + "description": "Returns the name of the encryption scope used to encrypt the blob contents and application metadata. Note that the absence of this header implies use of the default account encryption scope." + }, + "x-ms-blob-content-md5": { + "x-ms-client-name": "BlobContentMD5", + "type": "string", + "format": "byte", + "description": "If the blob has a MD5 hash, and if request contains range header (Range or x-ms-range), this response header is returned with the value of the whole blob's MD5 value. This value may or may not be equal to the value returned in Content-MD5 header, with the latter calculated from the requested range" + } + }, + "schema": { + "type": "object", + "format": "file" + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "query" + ] + } + ] + }, + "/{containerName}/{blob}?comp=tags": { + "get": { + "tags": [ + "blob" + ], + "operationId": "Blob_GetTags", + "description": "The Get Tags operation enables users to get the tags associated with a blob.", + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + }, + { + "$ref": "#/parameters/Snapshot" + }, + { + "$ref": "#/parameters/VersionId" + }, + { + "$ref": "#/parameters/IfTags" + }, + { + "$ref": "#/parameters/LeaseIdOptional" + } + ], + "responses": { + "200": { + "description": "Retrieved blob tags", + "headers": { + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + } + }, + "schema": { + "$ref": "#/definitions/BlobTags" + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "put": { + "tags": [ + "blob" + ], + "operationId": "Blob_SetTags", + "description": "The Set Tags operation enables users to set tags on a blob.", + "parameters": [ + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/VersionId" + }, + { + "$ref": "#/parameters/ContentMD5Blob" + }, + { + "$ref": "#/parameters/ContentCrc64" + }, + { + "$ref": "#/parameters/ClientRequestId" + }, + { + "$ref": "#/parameters/IfTags" + }, + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "$ref": "#/parameters/BlobTagsBody" + } + ], + "responses": { + "204": { + "description": "The tags were applied to the blob", + "headers": { + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageErrorBlob" + } + } + } + }, + "parameters": [ + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": [ + "tags" + ] + } + ] + } + }, + "definitions": { + "KeyInfo": { + "type": "object", + "required": [ + "Start", + "Expiry" + ], + "description": "Key information", + "properties": { + "Start": { + "description": "The date-time the key is active in ISO 8601 UTC time", + "type": "string" + }, + "Expiry": { + "description": "The date-time the key expires in ISO 8601 UTC time", + "type": "string" + } + } + }, + "UserDelegationKey": { + "type": "object", + "required": [ + "SignedOid", + "SignedTid", + "SignedStart", + "SignedExpiry", + "SignedService", + "SignedVersion", + "Value" + ], + "description": "A user delegation key", + "properties": { + "SignedOid": { + "description": "The Azure Active Directory object ID in GUID format.", + "type": "string" + }, + "SignedTid": { + "description": "The Azure Active Directory tenant ID in GUID format", + "type": "string" + }, + "SignedStart": { + "description": "The date-time the key is active", + "type": "string", + "format": "date-time" + }, + "SignedExpiry": { + "description": "The date-time the key expires", + "type": "string", + "format": "date-time" + }, + "SignedService": { + "description": "Abbreviation of the Azure Storage service that accepts the key", + "type": "string" + }, + "SignedVersion": { + "description": "The service version that created the key", + "type": "string" + }, + "Value": { + "description": "The key as a base64 string", + "type": "string" + } + } + }, + "PublicAccessType": { + "type": "string", + "enum": [ + "container", + "blob" + ], + "x-ms-enum": { + "name": "PublicAccessType", + "modelAsString": true + } + }, + "CopyStatus": { + "type": "string", + "enum": [ + "pending", + "success", + "aborted", + "failed" + ], + "x-ms-enum": { + "name": "CopyStatusType", + "modelAsString": false + } + }, + "LeaseDuration": { + "type": "string", + "enum": [ + "infinite", + "fixed" + ], + "x-ms-enum": { + "name": "LeaseDurationType", + "modelAsString": false + } + }, + "LeaseState": { + "type": "string", + "enum": [ + "available", + "leased", + "expired", + "breaking", + "broken" + ], + "x-ms-enum": { + "name": "LeaseStateType", + "modelAsString": false + } + }, + "LeaseStatus": { + "type": "string", + "enum": [ + "locked", + "unlocked" + ], + "x-ms-enum": { + "name": "LeaseStatusType", + "modelAsString": false + } + }, + "StorageErrorBlob": { + "type": "object", + "properties": { + "Message": { + "type": "string" + } + } + }, + "AccessPolicy": { + "type": "object", + "description": "An Access policy", + "properties": { + "Start": { + "description": "the date-time the policy is active", + "type": "string", + "format": "date-time" + }, + "Expiry": { + "description": "the date-time the policy expires", + "type": "string", + "format": "date-time" + }, + "Permission": { + "description": "the permissions for the acl policy", + "type": "string" + } + } + }, + "AccessTier": { + "type": "string", + "enum": [ + "P4", + "P6", + "P10", + "P15", + "P20", + "P30", + "P40", + "P50", + "P60", + "P70", + "P80", + "Hot", + "Cool", + "Archive", + "Premium" + ], + "x-ms-enum": { + "name": "AccessTier", + "modelAsString": true + } + }, + "ArchiveStatus": { + "type": "string", + "enum": [ + "rehydrate-pending-to-hot", + "rehydrate-pending-to-cool" + ], + "x-ms-enum": { + "name": "ArchiveStatus", + "modelAsString": true + } + }, + "BlobItemInternal": { + "xml": { + "name": "Blob" + }, + "description": "An Azure Storage blob", + "type": "object", + "required": [ + "Name", + "Properties" + ], + "properties": { + "Name": { + "type": "string" + }, + "Deleted": { + "type": "boolean" + }, + "Snapshot": { + "type": "string" + }, + "VersionId": { + "type": "string" + }, + "IsCurrentVersion": { + "type": "boolean" + }, + "Properties": { + "$ref": "#/definitions/BlobPropertiesInternal" + }, + "Metadata": { + "$ref": "#/definitions/BlobMetadata" + }, + "BlobTags": { + "$ref": "#/definitions/BlobTags" + }, + "ObjectReplicationMetadata": { + "$ref": "#/definitions/ObjectReplicationMetadata" + }, + "HasVersionsOnly": { + "type": "boolean" + }, + "DeletionId": { + "type": "string" + } + } + }, + "BlobPropertiesInternal": { + "xml": { + "name": "Properties" + }, + "description": "Properties of a blob", + "type": "object", + "required": [ + "Etag", + "Last-Modified" + ], + "properties": { + "Creation-Time": { + "type": "string", + "format": "date-time-rfc1123" + }, + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123" + }, + "Etag": { + "type": "string" + }, + "Content-Length": { + "type": "integer", + "format": "int64", + "description": "Size in bytes" + }, + "Content-Type": { + "type": "string" + }, + "Content-Encoding": { + "type": "string" + }, + "Content-Language": { + "type": "string" + }, + "Content-MD5": { + "type": "string", + "format": "byte" + }, + "Content-Disposition": { + "type": "string" + }, + "Cache-Control": { + "type": "string" + }, + "x-ms-blob-sequence-number": { + "x-ms-client-name": "blobSequenceNumber", + "type": "integer", + "format": "int64" + }, + "BlobType": { + "type": "string", + "enum": [ + "BlockBlob", + "PageBlob", + "AppendBlob" + ], + "x-ms-enum": { + "name": "BlobType", + "modelAsString": false + } + }, + "LeaseStatus": { + "$ref": "#/definitions/LeaseStatus" + }, + "LeaseState": { + "$ref": "#/definitions/LeaseState" + }, + "LeaseDuration": { + "$ref": "#/definitions/LeaseDuration" + }, + "CopyId": { + "type": "string" + }, + "CopyStatus": { + "$ref": "#/definitions/CopyStatus" + }, + "CopySource": { + "type": "string" + }, + "CopyProgress": { + "type": "string" + }, + "CopyCompletionTime": { + "type": "string", + "format": "date-time-rfc1123" + }, + "CopyStatusDescription": { + "type": "string" + }, + "ServerEncrypted": { + "type": "boolean" + }, + "IncrementalCopy": { + "type": "boolean" + }, + "DestinationSnapshot": { + "type": "string" + }, + "DeletedTime": { + "type": "string", + "format": "date-time-rfc1123" + }, + "RemainingRetentionDays": { + "type": "integer" + }, + "AccessTier": { + "$ref": "#/definitions/AccessTier" + }, + "AccessTierInferred": { + "type": "boolean" + }, + "ArchiveStatus": { + "$ref": "#/definitions/ArchiveStatus" + }, + "CustomerProvidedKeySha256": { + "type": "string" + }, + "EncryptionScope": { + "type": "string", + "description": "The name of the encryption scope under which the blob is encrypted." + }, + "AccessTierChangeTime": { + "type": "string", + "format": "date-time-rfc1123" + }, + "TagCount": { + "type": "integer" + }, + "Expiry-Time": { + "x-ms-client-name": "ExpiresOn", + "type": "string", + "format": "date-time-rfc1123" + }, + "Sealed": { + "x-ms-client-name": "IsSealed", + "type": "boolean" + }, + "RehydratePriority": { + "$ref": "#/definitions/RehydratePriority" + }, + "LastAccessTime": { + "x-ms-client-name": "LastAccessedOn", + "type": "string", + "format": "date-time-rfc1123" + }, + "ImmutabilityPolicyUntilDate": { + "x-ms-client-name": "ImmutabilityPolicyExpiresOn", + "type": "string", + "format": "date-time-rfc1123" + }, + "ImmutabilityPolicyMode": { + "type": "string", + "enum": [ + "Mutable", + "Unlocked", + "Locked" + ], + "x-ms-enum": { + "name": "BlobImmutabilityPolicyMode", + "modelAsString": false + } + }, + "LegalHold": { + "type": "boolean" + }, + "DeleteTime": { + "type": "string", + "format": "date-time-rfc1123" + }, + "properties": { + "type": "string" + } + } + }, + "ListBlobsHierarchySegmentResponse": { + "xml": { + "name": "EnumerationResults" + }, + "description": "An enumeration of blobs", + "type": "object", + "required": [ + "ServiceEndpoint", + "ContainerName", + "Segment" + ], + "properties": { + "ServiceEndpoint": { + "type": "string", + "xml": { + "attribute": true + } + }, + "ContainerName": { + "type": "string", + "xml": { + "attribute": true + } + }, + "Prefix": { + "type": "string" + }, + "Marker": { + "type": "string" + }, + "MaxResults": { + "type": "integer" + }, + "Delimiter": { + "type": "string" + }, + "Segment": { + "$ref": "#/definitions/BlobHierarchyListSegment" + }, + "NextMarker": { + "type": "string" + } + } + }, + "BlobHierarchyListSegment": { + "xml": { + "name": "Blobs" + }, + "type": "object", + "required": [ + "BlobItems" + ], + "properties": { + "BlobPrefixes": { + "type": "array", + "items": { + "$ref": "#/definitions/BlobPrefix" + } + }, + "BlobItems": { + "type": "array", + "items": { + "$ref": "#/definitions/BlobItemInternal" + } + } + } + }, + "BlobPrefix": { + "type": "object", + "required": [ + "Name" + ], + "properties": { + "Name": { + "type": "string" + } + } + }, + "BlobName": { + "type": "object", + "properties": { + "Encoded": { + "xml": { + "attribute": true, + "name": "Encoded" + }, + "type": "boolean", + "description": "Indicates if the blob name is encoded." + }, + "content": { + "xml": { + "x-ms-text": true + }, + "type": "string", + "description": "The name of the blob." + } + } + }, + "BlobTag": { + "xml": { + "name": "Tag" + }, + "type": "object", + "required": [ + "Key", + "Value" + ], + "properties": { + "Key": { + "type": "string" + }, + "Value": { + "type": "string" + } + } + }, + "BlobTags": { + "type": "object", + "xml": { + "name": "Tags" + }, + "description": "Blob tags", + "required": [ + "BlobTagSet" + ], + "properties": { + "BlobTagSet": { + "xml": { + "wrapped": true, + "name": "TagSet" + }, + "type": "array", + "items": { + "$ref": "#/definitions/BlobTag" + } + } + } + }, + "Block": { + "type": "object", + "required": [ + "Name", + "Size" + ], + "description": "Represents a single block in a block blob. It describes the block's ID and size.", + "properties": { + "Name": { + "description": "The base64 encoded block ID.", + "type": "string" + }, + "Size": { + "description": "The block size in bytes.", + "type": "integer", + "format": "int64" + } + } + }, + "BlockList": { + "type": "object", + "properties": { + "CommittedBlocks": { + "xml": { + "wrapped": true + }, + "type": "array", + "items": { + "$ref": "#/definitions/Block" + } + }, + "UncommittedBlocks": { + "xml": { + "wrapped": true + }, + "type": "array", + "items": { + "$ref": "#/definitions/Block" + } + } + } + }, + "BlockLookupList": { + "type": "object", + "properties": { + "Committed": { + "type": "array", + "items": { + "type": "string", + "xml": { + "name": "Committed" + } + } + }, + "Uncommitted": { + "type": "array", + "items": { + "type": "string", + "xml": { + "name": "Uncommitted" + } + } + }, + "Latest": { + "type": "array", + "items": { + "type": "string", + "xml": { + "name": "Latest" + } + } + } + }, + "xml": { + "name": "BlockList" + } + }, + "ContainerItem": { + "xml": { + "name": "Container" + }, + "type": "object", + "required": [ + "Name", + "Properties" + ], + "description": "An Azure Storage container", + "properties": { + "Name": { + "type": "string" + }, + "Deleted": { + "type": "boolean" + }, + "Version": { + "type": "string" + }, + "Properties": { + "$ref": "#/definitions/ContainerProperties" + }, + "Metadata": { + "$ref": "#/definitions/ContainerMetadata" + } + } + }, + "ContainerProperties": { + "type": "object", + "required": [ + "Last-Modified", + "Etag" + ], + "description": "Properties of a container", + "properties": { + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123" + }, + "Etag": { + "type": "string" + }, + "LeaseStatus": { + "$ref": "#/definitions/LeaseStatus" + }, + "LeaseState": { + "$ref": "#/definitions/LeaseState" + }, + "LeaseDuration": { + "$ref": "#/definitions/LeaseDuration" + }, + "PublicAccess": { + "$ref": "#/definitions/PublicAccessType" + }, + "HasImmutabilityPolicy": { + "type": "boolean" + }, + "HasLegalHold": { + "type": "boolean" + }, + "DefaultEncryptionScope": { + "type": "string" + }, + "DenyEncryptionScopeOverride": { + "type": "boolean", + "x-ms-client-name": "PreventEncryptionScopeOverride" + }, + "DeletedTime": { + "type": "string", + "format": "date-time-rfc1123" + }, + "RemainingRetentionDays": { + "type": "integer" + }, + "ImmutableStorageWithVersioningEnabled": { + "x-ms-client-name": "IsImmutableStorageWithVersioningEnabled", + "type": "boolean", + "description": "Indicates if version level worm is enabled on this container." + } + } + }, + "DelimitedTextConfiguration": { + "xml": { + "name": "DelimitedTextConfiguration" + }, + "description": "Groups the settings used for interpreting the blob data if the blob is delimited text formatted.", + "type": "object", + "properties": { + "ColumnSeparator": { + "type": "string", + "description": "The string used to separate columns.", + "xml": { + "name": "ColumnSeparator" + } + }, + "FieldQuote": { + "type": "string", + "description": "The string used to quote a specific field.", + "xml": { + "name": "FieldQuote" + } + }, + "RecordSeparator": { + "type": "string", + "description": "The string used to separate records.", + "xml": { + "name": "RecordSeparator" + } + }, + "EscapeChar": { + "type": "string", + "description": "The string used as an escape character.", + "xml": { + "name": "EscapeChar" + } + }, + "HeadersPresent": { + "type": "boolean", + "description": "Represents whether the data has headers.", + "xml": { + "name": "HasHeaders" + } + } + } + }, + "JsonTextConfiguration": { + "xml": { + "name": "JsonTextConfiguration" + }, + "description": "json text configuration", + "type": "object", + "properties": { + "RecordSeparator": { + "type": "string", + "description": "The string used to separate records.", + "xml": { + "name": "RecordSeparator" + } + } + } + }, + "ArrowConfiguration": { + "xml": { + "name": "ArrowConfiguration" + }, + "description": "Groups the settings used for formatting the response if the response should be Arrow formatted.", + "type": "object", + "required": [ + "Schema" + ], + "properties": { + "Schema": { + "type": "array", + "items": { + "$ref": "#/definitions/ArrowField" + }, + "xml": { + "wrapped": true, + "name": "Schema" + } + } + } + }, + "ParquetConfiguration": { + "xml": { + "name": "ParquetTextConfiguration" + }, + "description": "parquet configuration", + "type": "object" + }, + "ArrowField": { + "xml": { + "name": "Field" + }, + "description": "Groups settings regarding specific field of an arrow schema", + "type": "object", + "required": [ + "Type" + ], + "properties": { + "Type": { + "type": "string" + }, + "Name": { + "type": "string" + }, + "Precision": { + "type": "integer" + }, + "Scale": { + "type": "integer" + } + } + }, + "ListContainersSegmentResponse": { + "xml": { + "name": "EnumerationResults" + }, + "description": "An enumeration of containers", + "type": "object", + "required": [ + "ServiceEndpoint", + "ContainerItems" + ], + "properties": { + "ServiceEndpoint": { + "type": "string", + "xml": { + "attribute": true + } + }, + "Prefix": { + "type": "string" + }, + "Marker": { + "type": "string" + }, + "MaxResults": { + "type": "integer" + }, + "ContainerItems": { + "xml": { + "wrapped": true, + "name": "Containers" + }, + "type": "array", + "items": { + "$ref": "#/definitions/ContainerItem" + } + }, + "NextMarker": { + "type": "string" + } + } + }, + "CorsRule": { + "description": "CORS is an HTTP feature that enables a web application running under one domain to access resources in another domain. Web browsers implement a security restriction known as same-origin policy that prevents a web page from calling APIs in a different domain; CORS provides a secure way to allow one domain (the origin domain) to call APIs in another domain", + "type": "object", + "required": [ + "AllowedOrigins", + "AllowedMethods", + "MaxAgeInSeconds" + ], + "properties": { + "AllowedOrigins": { + "description": "The origin domains that are permitted to make a request against the storage service via CORS. The origin domain is the domain from which the request originates. Note that the origin must be an exact case-sensitive match with the origin that the user age sends to the service. You can also use the wildcard character '*' to allow all origin domains to make requests via CORS.", + "type": "string" + }, + "AllowedMethods": { + "description": "The methods (HTTP request verbs) that the origin domain may use for a CORS request. (comma separated)", + "type": "string" + }, + "AllowedHeaders": { + "description": "the request headers that the origin domain may specify on the CORS request.", + "type": "string" + }, + "ExposedHeaders": { + "description": "The response headers that may be sent in the response to the CORS request and exposed by the browser to the request issuer", + "type": "string" + }, + "MaxAgeInSeconds": { + "description": "The maximum amount time that a browser should cache the preflight OPTIONS request.", + "type": "integer", + "minimum": 0 + } + } + }, + "ErrorCode": { + "description": "Error codes returned by the service", + "type": "string", + "enum": [ + "AccountAlreadyExists", + "AccountBeingCreated", + "AccountIsDisabled", + "AuthenticationFailed", + "AuthorizationFailure", + "ConditionHeadersNotSupported", + "ConditionNotMet", + "EmptyMetadataKey", + "InsufficientAccountPermissions", + "InternalError", + "InvalidAuthenticationInfo", + "InvalidHeaderValue", + "InvalidHttpVerb", + "InvalidInput", + "InvalidMd5", + "InvalidMetadata", + "InvalidQueryParameterValue", + "InvalidRange", + "InvalidResourceName", + "InvalidUri", + "InvalidXmlDocument", + "InvalidXmlNodeValue", + "Md5Mismatch", + "MetadataTooLarge", + "MissingContentLengthHeader", + "MissingRequiredQueryParameter", + "MissingRequiredHeader", + "MissingRequiredXmlNode", + "MultipleConditionHeadersNotSupported", + "OperationTimedOut", + "OutOfRangeInput", + "OutOfRangeQueryParameterValue", + "RequestBodyTooLarge", + "ResourceTypeMismatch", + "RequestUrlFailedToParse", + "ResourceAlreadyExists", + "ResourceNotFound", + "ServerBusy", + "UnsupportedHeader", + "UnsupportedXmlNode", + "UnsupportedQueryParameter", + "UnsupportedHttpVerb", + "AppendPositionConditionNotMet", + "BlobAlreadyExists", + "BlobImmutableDueToPolicy", + "BlobNotFound", + "BlobOverwritten", + "BlobTierInadequateForContentLength", + "BlobUsesCustomerSpecifiedEncryption", + "BlockCountExceedsLimit", + "BlockListTooLong", + "CannotChangeToLowerTier", + "CannotVerifyCopySource", + "ContainerAlreadyExists", + "ContainerBeingDeleted", + "ContainerDisabled", + "ContainerNotFound", + "ContentLengthLargerThanTierLimit", + "CopyAcrossAccountsNotSupported", + "CopyIdMismatch", + "FeatureVersionMismatch", + "IncrementalCopyBlobMismatch", + "IncrementalCopyOfEarlierVersionSnapshotNotAllowed", + "IncrementalCopySourceMustBeSnapshot", + "InfiniteLeaseDurationRequired", + "InvalidBlobOrBlock", + "InvalidBlobTier", + "InvalidBlobType", + "InvalidBlockId", + "InvalidBlockList", + "InvalidOperation", + "InvalidPageRange", + "InvalidSourceBlobType", + "InvalidSourceBlobUrl", + "InvalidVersionForPageBlobOperation", + "LeaseAlreadyPresent", + "LeaseAlreadyBroken", + "LeaseIdMismatchWithBlobOperation", + "LeaseIdMismatchWithContainerOperation", + "LeaseIdMismatchWithLeaseOperation", + "LeaseIdMissing", + "LeaseIsBreakingAndCannotBeAcquired", + "LeaseIsBreakingAndCannotBeChanged", + "LeaseIsBrokenAndCannotBeRenewed", + "LeaseLost", + "LeaseNotPresentWithBlobOperation", + "LeaseNotPresentWithContainerOperation", + "LeaseNotPresentWithLeaseOperation", + "MaxBlobSizeConditionNotMet", + "NoAuthenticationInformation", + "NoPendingCopyOperation", + "OperationNotAllowedOnIncrementalCopyBlob", + "PendingCopyOperation", + "PreviousSnapshotCannotBeNewer", + "PreviousSnapshotNotFound", + "PreviousSnapshotOperationNotSupported", + "SequenceNumberConditionNotMet", + "SequenceNumberIncrementTooLarge", + "SnapshotCountExceeded", + "SnapshotOperationRateExceeded", + "SnapshotsPresent", + "SourceConditionNotMet", + "SystemInUse", + "TargetConditionNotMet", + "UnauthorizedBlobOverwrite", + "BlobBeingRehydrated", + "BlobArchived", + "BlobNotArchived", + "AuthorizationSourceIPMismatch", + "AuthorizationProtocolMismatch", + "AuthorizationPermissionMismatch", + "AuthorizationServiceMismatch", + "AuthorizationResourceTypeMismatch" + ], + "x-ms-enum": { + "name": "StorageErrorCode", + "modelAsString": true + } + }, + "FilterBlobItem": { + "xml": { + "name": "Blob" + }, + "description": "Blob info from a Filter Blobs API call", + "type": "object", + "required": [ + "Name", + "ContainerName" + ], + "properties": { + "Name": { + "type": "string" + }, + "ContainerName": { + "type": "string" + }, + "Tags": { + "$ref": "#/definitions/BlobTags" + }, + "VersionId": { + "type": "string" + }, + "IsCurrentVersion": { + "type": "boolean" + } + } + }, + "FilterBlobSegment": { + "description": "The result of a Filter Blobs API call", + "xml": { + "name": "EnumerationResults" + }, + "type": "object", + "required": [ + "ServiceEndpoint", + "Where", + "Blobs" + ], + "properties": { + "ServiceEndpoint": { + "type": "string", + "xml": { + "attribute": true + } + }, + "Where": { + "type": "string" + }, + "Blobs": { + "xml": { + "name": "Blobs", + "wrapped": true + }, + "type": "array", + "items": { + "$ref": "#/definitions/FilterBlobItem" + } + }, + "NextMarker": { + "type": "string" + } + } + }, + "GeoReplication": { + "description": "Geo-Replication information for the Secondary Storage Service", + "type": "object", + "required": [ + "Status", + "LastSyncTime" + ], + "properties": { + "Status": { + "description": "The status of the secondary location", + "type": "string", + "enum": [ + "live", + "bootstrap", + "unavailable" + ], + "x-ms-enum": { + "name": "GeoReplicationStatusType", + "modelAsString": true + } + }, + "LastSyncTime": { + "description": "A GMT date/time value, to the second. All primary writes preceding this value are guaranteed to be available for read operations at the secondary. Primary writes after this point in time may or may not be available for reads.", + "type": "string", + "format": "date-time-rfc1123" + } + } + }, + "Logging": { + "description": "Azure Analytics Logging settings.", + "type": "object", + "required": [ + "Version", + "Delete", + "Read", + "Write", + "RetentionPolicy" + ], + "properties": { + "Version": { + "description": "The version of Storage Analytics to configure.", + "type": "string" + }, + "Delete": { + "description": "Indicates whether all delete requests should be logged.", + "type": "boolean" + }, + "Read": { + "description": "Indicates whether all read requests should be logged.", + "type": "boolean" + }, + "Write": { + "description": "Indicates whether all write requests should be logged.", + "type": "boolean" + }, + "RetentionPolicy": { + "$ref": "#/definitions/RetentionPolicy" + } + } + }, + "ContainerMetadata": { + "type": "object", + "xml": { + "name": "Metadata" + }, + "additionalProperties": { + "type": "string" + } + }, + "BlobMetadata": { + "type": "object", + "xml": { + "name": "Metadata" + }, + "properties": { + "Encrypted": { + "type": "string", + "xml": { + "attribute": true + } + } + }, + "additionalProperties": { + "type": "string" + } + }, + "ObjectReplicationMetadata": { + "type": "object", + "xml": { + "name": "OrMetadata" + }, + "additionalProperties": { + "type": "string" + } + }, + "Metrics": { + "description": "a summary of request statistics grouped by API in hour or minute aggregates for blobs", + "required": [ + "Enabled" + ], + "properties": { + "Version": { + "description": "The version of Storage Analytics to configure.", + "type": "string" + }, + "Enabled": { + "description": "Indicates whether metrics are enabled for the Blob service.", + "type": "boolean" + }, + "IncludeAPIs": { + "description": "Indicates whether metrics should generate summary statistics for called API operations.", + "type": "boolean" + }, + "RetentionPolicy": { + "$ref": "#/definitions/RetentionPolicy" + } + } + }, + "PageList": { + "description": "the list of pages", + "type": "object", + "properties": { + "PageRange": { + "type": "array", + "items": { + "$ref": "#/definitions/PageRange" + } + }, + "ClearRange": { + "type": "array", + "items": { + "$ref": "#/definitions/ClearRange" + } + }, + "NextMarker": { + "type": "string" + } + } + }, + "PageRange": { + "type": "object", + "required": [ + "Start", + "End" + ], + "properties": { + "Start": { + "type": "integer", + "format": "int64", + "xml": { + "name": "Start" + } + }, + "End": { + "type": "integer", + "format": "int64", + "xml": { + "name": "End" + } + } + }, + "xml": { + "name": "PageRange" + } + }, + "ClearRange": { + "type": "object", + "required": [ + "Start", + "End" + ], + "properties": { + "Start": { + "type": "integer", + "format": "int64", + "xml": { + "name": "Start" + } + }, + "End": { + "type": "integer", + "format": "int64", + "xml": { + "name": "End" + } + } + }, + "xml": { + "name": "ClearRange" + } + }, + "QueryRequest": { + "description": "Groups the set of query request settings.", + "type": "object", + "required": [ + "QueryType", + "Expression" + ], + "properties": { + "QueryType": { + "type": "string", + "description": "Required. The type of the provided query expression.", + "xml": { + "name": "QueryType" + }, + "enum": [ + "SQL" + ] + }, + "Expression": { + "type": "string", + "description": "The query expression in SQL. The maximum size of the query expression is 256KiB.", + "xml": { + "name": "Expression" + } + }, + "InputSerialization": { + "$ref": "#/definitions/QuerySerialization", + "xml": { + "name": "InputSerialization" + } + }, + "OutputSerialization": { + "$ref": "#/definitions/QuerySerialization", + "xml": { + "name": "OutputSerialization" + } + } + }, + "xml": { + "name": "QueryRequest" + } + }, + "QueryFormat": { + "type": "object", + "required": [ + "Type" + ], + "properties": { + "Type": { + "$ref": "#/definitions/QueryType" + }, + "DelimitedTextConfiguration": { + "$ref": "#/definitions/DelimitedTextConfiguration" + }, + "JsonTextConfiguration": { + "$ref": "#/definitions/JsonTextConfiguration" + }, + "ArrowConfiguration": { + "$ref": "#/definitions/ArrowConfiguration" + }, + "ParquetTextConfiguration": { + "$ref": "#/definitions/ParquetConfiguration" + } + } + }, + "QuerySerialization": { + "type": "object", + "required": [ + "Format" + ], + "properties": { + "Format": { + "$ref": "#/definitions/QueryFormat", + "xml": { + "name": "Format" + } + } + } + }, + "QueryType": { + "type": "string", + "description": "The quick query format type.", + "enum": [ + "delimited", + "json", + "arrow", + "parquet" + ], + "x-ms-enum": { + "name": "QueryFormatType", + "modelAsString": false + }, + "xml": { + "name": "Type" + } + }, + "RehydratePriority": { + "description": "If an object is in rehydrate pending state then this header is returned with priority of rehydrate. Valid values are High and Standard.", + "type": "string", + "enum": [ + "High", + "Standard" + ], + "x-ms-enum": { + "name": "RehydratePriority", + "modelAsString": true + }, + "xml": { + "name": "RehydratePriority" + } + }, + "RetentionPolicy": { + "description": "the retention policy which determines how long the associated data should persist", + "type": "object", + "required": [ + "Enabled" + ], + "properties": { + "Enabled": { + "description": "Indicates whether a retention policy is enabled for the storage service", + "type": "boolean" + }, + "Days": { + "description": "Indicates the number of days that metrics or logging or soft-deleted data should be retained. All data older than this value will be deleted", + "type": "integer", + "minimum": 1 + }, + "AllowPermanentDelete": { + "description": "Indicates whether permanent delete is allowed on this storage account.", + "type": "boolean" + } + } + }, + "SignedIdentifier": { + "xml": { + "name": "SignedIdentifier" + }, + "description": "signed identifier", + "type": "object", + "required": [ + "Id", + "AccessPolicy" + ], + "properties": { + "Id": { + "type": "string", + "description": "a unique id" + }, + "AccessPolicy": { + "$ref": "#/definitions/AccessPolicy" + } + } + }, + "SignedIdentifiers": { + "description": "a collection of signed identifiers", + "type": "array", + "items": { + "$ref": "#/definitions/SignedIdentifier" + }, + "xml": { + "wrapped": true, + "name": "SignedIdentifiers" + } + }, + "StaticWebsite": { + "description": "The properties that enable an account to host a static website", + "type": "object", + "required": [ + "Enabled" + ], + "properties": { + "Enabled": { + "description": "Indicates whether this account is hosting a static website", + "type": "boolean" + }, + "IndexDocument": { + "description": "The default name of the index page under each directory", + "type": "string" + }, + "ErrorDocument404Path": { + "description": "The absolute path of the custom 404 page", + "type": "string" + }, + "DefaultIndexDocumentPath": { + "description": "Absolute path of the default index page", + "type": "string" + } + } + }, + "StorageServiceProperties": { + "description": "Storage Service Properties.", + "type": "object", + "properties": { + "Logging": { + "$ref": "#/definitions/Logging" + }, + "HourMetrics": { + "$ref": "#/definitions/Metrics" + }, + "MinuteMetrics": { + "$ref": "#/definitions/Metrics" + }, + "Cors": { + "description": "The set of CORS rules.", + "type": "array", + "items": { + "$ref": "#/definitions/CorsRule" + }, + "xml": { + "wrapped": true + } + }, + "DefaultServiceVersion": { + "description": "The default version to use for requests to the Blob service if an incoming request's version is not specified. Possible values include version 2008-10-27 and all more recent versions", + "type": "string" + }, + "DeleteRetentionPolicy": { + "$ref": "#/definitions/RetentionPolicy" + }, + "StaticWebsite": { + "$ref": "#/definitions/StaticWebsite" + } + } + }, + "StorageServiceStats": { + "description": "Stats for the storage service.", + "type": "object", + "properties": { + "GeoReplication": { + "$ref": "#/definitions/GeoReplication" + } + } + } + }, + "parameters": { + "Url": { + "name": "url", + "description": "The URL of the service account, container, or blob that is the target of the desired operation.", + "x-ms-parameter-location": "client", + "required": true, + "type": "string", + "in": "path", + "x-ms-skip-url-encoding": true + }, + "ApiVersionParameter": { + "name": "x-ms-version", + "x-ms-parameter-location": "client", + "x-ms-client-name": "version", + "in": "header", + "required": false, + "type": "string", + "description": "Specifies the version of the operation to use for this request." + }, + "BlobCacheControl": { + "name": "x-ms-blob-cache-control", + "x-ms-client-name": "blobCacheControl", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "blob-HTTP-headers" + }, + "description": "Optional. Sets the blob's cache control. If specified, this property is stored with the blob and returned with a read request." + }, + "BlobConditionAppendPos": { + "name": "x-ms-blob-condition-appendpos", + "x-ms-client-name": "appendPosition", + "in": "header", + "required": false, + "type": "integer", + "format": "int64", + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "append-position-access-conditions" + }, + "description": "Optional conditional header, used only for the Append Block operation. A number indicating the byte offset to compare. Append Block will succeed only if the append position is equal to this number. If it is not, the request will fail with the AppendPositionConditionNotMet error (HTTP status code 412 - Precondition Failed)." + }, + "BlobConditionMaxSize": { + "name": "x-ms-blob-condition-maxsize", + "x-ms-client-name": "maxSize", + "in": "header", + "required": false, + "type": "integer", + "format": "int64", + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "append-position-access-conditions" + }, + "description": "Optional conditional header. The max length in bytes permitted for the append blob. If the Append Block operation would cause the blob to exceed that limit or if the blob size is already greater than the value specified in this header, the request will fail with MaxBlobSizeConditionNotMet error (HTTP status code 412 - Precondition Failed)." + }, + "BlobPublicAccess": { + "name": "x-ms-blob-public-access", + "x-ms-client-name": "access", + "in": "header", + "required": false, + "x-ms-parameter-location": "method", + "description": "Specifies whether data in the container may be accessed publicly and the level of access", + "type": "string", + "enum": [ + "container", + "blob" + ], + "x-ms-enum": { + "name": "PublicAccessType", + "modelAsString": true + } + }, + "BlobTagsBody": { + "name": "Tags", + "in": "body", + "schema": { + "$ref": "#/definitions/BlobTags" + }, + "x-ms-parameter-location": "method", + "description": "Blob tags" + }, + "BlobTagsHeader": { + "name": "x-ms-tags", + "x-ms-client-name": "BlobTagsString", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "description": "Optional. Used to set blob tags in various blob operations." + }, + "AccessTierRequired": { + "name": "x-ms-access-tier", + "x-ms-client-name": "tier", + "in": "header", + "required": true, + "type": "string", + "enum": [ + "P4", + "P6", + "P10", + "P15", + "P20", + "P30", + "P40", + "P50", + "P60", + "P70", + "P80", + "Hot", + "Cool", + "Archive", + "Premium" + ], + "x-ms-enum": { + "name": "AccessTier", + "modelAsString": true + }, + "x-ms-parameter-location": "method", + "description": "Indicates the tier to be set on the blob." + }, + "AccessTierOptional": { + "name": "x-ms-access-tier", + "x-ms-client-name": "tier", + "in": "header", + "required": false, + "type": "string", + "enum": [ + "P4", + "P6", + "P10", + "P15", + "P20", + "P30", + "P40", + "P50", + "P60", + "P70", + "P80", + "Hot", + "Cool", + "Archive", + "Premium" + ], + "x-ms-enum": { + "name": "AccessTier", + "modelAsString": true + }, + "x-ms-parameter-location": "method", + "description": "Optional. Indicates the tier to be set on the blob." + }, + "PremiumPageBlobAccessTierOptional": { + "name": "x-ms-access-tier", + "x-ms-client-name": "tier", + "in": "header", + "required": false, + "type": "string", + "enum": [ + "P4", + "P6", + "P10", + "P15", + "P20", + "P30", + "P40", + "P50", + "P60", + "P70", + "P80" + ], + "x-ms-enum": { + "name": "PremiumPageBlobAccessTier", + "modelAsString": true + }, + "x-ms-parameter-location": "method", + "description": "Optional. Indicates the tier to be set on the page blob." + }, + "RehydratePriority": { + "name": "x-ms-rehydrate-priority", + "x-ms-client-name": "rehydratePriority", + "in": "header", + "required": false, + "type": "string", + "enum": [ + "High", + "Standard" + ], + "x-ms-enum": { + "name": "RehydratePriority", + "modelAsString": true + }, + "x-ms-parameter-location": "method", + "description": "Optional: Indicates the priority with which to rehydrate an archived blob." + }, + "BlobContentDisposition": { + "name": "x-ms-blob-content-disposition", + "x-ms-client-name": "blobContentDisposition", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "blob-HTTP-headers" + }, + "description": "Optional. Sets the blob's Content-Disposition header." + }, + "BlobContentEncoding": { + "name": "x-ms-blob-content-encoding", + "x-ms-client-name": "blobContentEncoding", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "blob-HTTP-headers" + }, + "description": "Optional. Sets the blob's content encoding. If specified, this property is stored with the blob and returned with a read request." + }, + "BlobContentLanguage": { + "name": "x-ms-blob-content-language", + "x-ms-client-name": "blobContentLanguage", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "blob-HTTP-headers" + }, + "description": "Optional. Set the blob's content language. If specified, this property is stored with the blob and returned with a read request." + }, + "BlobContentLengthOptional": { + "name": "x-ms-blob-content-length", + "x-ms-client-name": "blobContentLength", + "in": "header", + "required": false, + "type": "integer", + "format": "int64", + "x-ms-parameter-location": "method", + "description": "This header specifies the maximum size for the page blob, up to 1 TB. The page blob size must be aligned to a 512-byte boundary." + }, + "BlobContentLengthRequired": { + "name": "x-ms-blob-content-length", + "x-ms-client-name": "blobContentLength", + "in": "header", + "required": true, + "type": "integer", + "format": "int64", + "x-ms-parameter-location": "method", + "description": "This header specifies the maximum size for the page blob, up to 1 TB. The page blob size must be aligned to a 512-byte boundary." + }, + "BlobContentMD5": { + "name": "x-ms-blob-content-md5", + "x-ms-client-name": "blobContentMD5", + "in": "header", + "required": false, + "type": "string", + "format": "byte", + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "blob-HTTP-headers" + }, + "description": "Optional. An MD5 hash of the blob content. Note that this hash is not validated, as the hashes for the individual blocks were validated when each was uploaded." + }, + "BlobContentType": { + "name": "x-ms-blob-content-type", + "x-ms-client-name": "blobContentType", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "blob-HTTP-headers" + }, + "description": "Optional. Sets the blob's content type. If specified, this property is stored with the blob and returned with a read request." + }, + "BlobDeleteType": { + "name": "deletetype", + "x-ms-client-name": "blobDeleteType", + "in": "query", + "required": false, + "type": "string", + "enum": [ + "Permanent" + ], + "x-ms-enum": { + "name": "BlobDeleteType", + "modelAsString": false + }, + "x-ms-parameter-location": "method", + "description": "Optional. Only possible value is 'permanent', which specifies to permanently delete a blob if blob soft delete is enabled." + }, + "BlobExpiryOptions": { + "name": "x-ms-expiry-option", + "x-ms-client-name": "ExpiryOptions", + "in": "header", + "required": true, + "type": "string", + "enum": [ + "NeverExpire", + "RelativeToCreation", + "RelativeToNow", + "Absolute" + ], + "x-ms-enum": { + "name": "BlobExpiryOptions", + "modelAsString": true + }, + "x-ms-parameter-location": "method", + "description": "Required. Indicates mode of the expiry time" + }, + "BlobExpiryTime": { + "name": "x-ms-expiry-time", + "x-ms-client-name": "ExpiresOn", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "description": "The time to set the blob to expiry" + }, + "BlobSequenceNumber": { + "name": "x-ms-blob-sequence-number", + "x-ms-client-name": "blobSequenceNumber", + "in": "header", + "required": false, + "type": "integer", + "format": "int64", + "default": 0, + "x-ms-parameter-location": "method", + "description": "Set for page blobs only. The sequence number is a user-controlled value that you can use to track requests. The value of the sequence number must be between 0 and 2^63 - 1." + }, + "BlockId": { + "name": "blockid", + "x-ms-client-name": "blockId", + "in": "query", + "type": "string", + "required": true, + "x-ms-parameter-location": "method", + "description": "A valid Base64 string value that identifies the block. Prior to encoding, the string must be less than or equal to 64 bytes in size. For a given blob, the length of the value specified for the blockid parameter must be the same size for each block." + }, + "BlockListType": { + "name": "blocklisttype", + "x-ms-client-name": "listType", + "in": "query", + "required": false, + "default": "committed", + "x-ms-parameter-location": "method", + "description": "Specifies whether to return the list of committed blocks, the list of uncommitted blocks, or both lists together.", + "type": "string", + "enum": [ + "committed", + "uncommitted", + "all" + ], + "x-ms-enum": { + "name": "BlockListType", + "modelAsString": false + } + }, + "Body": { + "name": "body", + "in": "body", + "required": true, + "schema": { + "type": "object", + "format": "file" + }, + "x-ms-parameter-location": "method", + "description": "Initial data" + }, + "ContainerAcl": { + "name": "containerAcl", + "in": "body", + "schema": { + "$ref": "#/definitions/SignedIdentifiers" + }, + "x-ms-parameter-location": "method", + "description": "the acls for the container" + }, + "CopyId": { + "name": "copyid", + "x-ms-client-name": "copyId", + "in": "query", + "required": true, + "type": "string", + "x-ms-parameter-location": "method", + "description": "The copy identifier provided in the x-ms-copy-id header of the original Copy Blob operation." + }, + "ClientRequestId": { + "name": "x-ms-client-request-id", + "x-ms-client-name": "requestId", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "description": "Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the analytics logs when storage analytics logging is enabled." + }, + "ContentCrc64": { + "name": "x-ms-content-crc64", + "x-ms-client-name": "transactionalContentCrc64", + "in": "header", + "required": false, + "type": "string", + "format": "byte", + "x-ms-parameter-location": "method", + "description": "Specify the transactional crc64 for the body, to be validated by the service." + }, + "ContentLengthBlob": { + "name": "Content-Length", + "in": "header", + "required": true, + "type": "integer", + "format": "int64", + "x-ms-parameter-location": "method", + "description": "The length of the request." + }, + "ContentMD5Blob": { + "name": "Content-MD5", + "x-ms-client-name": "transactionalContentMD5", + "in": "header", + "required": false, + "type": "string", + "format": "byte", + "x-ms-parameter-location": "method", + "description": "Specify the transactional md5 for the body, to be validated by the service." + }, + "CopySource": { + "name": "x-ms-copy-source", + "x-ms-client-name": "copySource", + "in": "header", + "required": true, + "type": "string", + "x-ms-parameter-location": "method", + "description": "Specifies the name of the source page blob snapshot. This value is a URL of up to 2 KB in length that specifies a page blob snapshot. The value should be URL-encoded as it would appear in a request URI. The source blob must either be public or must be authenticated via a shared access signature." + }, + "CopySourceAuthorization": { + "name": "x-ms-copy-source-authorization", + "x-ms-client-name": "copySourceAuthorization", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "description": "Only Bearer type is supported. Credentials should be a valid OAuth access token to copy source." + }, + "CopySourceBlobProperties": { + "name": "x-ms-copy-source-blob-properties", + "x-ms-client-name": "copySourceBlobProperties", + "in": "header", + "required": false, + "type": "boolean", + "x-ms-parameter-location": "method", + "description": "Optional, default is true. Indicates if properties from the source blob should be copied." + }, + "CopySourceTags": { + "name": "x-ms-copy-source-tag-option", + "x-ms-client-name": "copySourceTags", + "in": "header", + "required": false, + "type": "string", + "enum": [ + "REPLACE", + "COPY" + ], + "x-ms-enum": { + "name": "BlobCopySourceTags", + "modelAsString": false + }, + "x-ms-parameter-location": "method", + "description": "Optional, default 'replace'. Indicates if source tags should be copied or replaced with the tags specified by x-ms-tags." + }, + "DeleteSnapshots": { + "name": "x-ms-delete-snapshots", + "x-ms-client-name": "deleteSnapshots", + "description": "Required if the blob has associated snapshots. Specify one of the following two options: include: Delete the base blob and all of its snapshots. only: Delete only the blob's snapshots and not the blob itself", + "x-ms-parameter-location": "method", + "in": "header", + "required": false, + "type": "string", + "enum": [ + "include", + "only" + ], + "x-ms-enum": { + "name": "DeleteSnapshotsOptionType", + "modelAsString": false + } + }, + "EncryptionKey": { + "name": "x-ms-encryption-key", + "x-ms-client-name": "encryptionKey", + "type": "string", + "in": "header", + "required": false, + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "cpk-info" + }, + "description": "Optional. Specifies the encryption key to use to encrypt the data provided in the request. If not specified, encryption is performed with the root account encryption key. For more information, see Encryption at Rest for Azure Storage Services." + }, + "EncryptionKeySha256": { + "name": "x-ms-encryption-key-sha256", + "x-ms-client-name": "encryptionKeySha256", + "type": "string", + "in": "header", + "required": false, + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "cpk-info" + }, + "description": "The SHA-256 hash of the provided encryption key. Must be provided if the x-ms-encryption-key header is provided." + }, + "EncryptionAlgorithm": { + "name": "x-ms-encryption-algorithm", + "x-ms-client-name": "encryptionAlgorithm", + "type": "string", + "in": "header", + "required": false, + "enum": [ + "AES256" + ], + "x-ms-enum": { + "name": "EncryptionAlgorithmType", + "modelAsString": false + }, + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "cpk-info" + }, + "description": "The algorithm used to produce the encryption key hash. Currently, the only accepted value is \"AES256\". Must be provided if the x-ms-encryption-key header is provided." + }, + "EncryptionScope": { + "name": "x-ms-encryption-scope", + "x-ms-client-name": "encryptionScope", + "type": "string", + "in": "header", + "required": false, + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "cpk-scope-info" + }, + "description": "Optional. Version 2019-07-07 and later. Specifies the name of the encryption scope to use to encrypt the data provided in the request. If not specified, encryption is performed with the default account encryption scope. For more information, see Encryption at Rest for Azure Storage Services." + }, + "DefaultEncryptionScope": { + "name": "x-ms-default-encryption-scope", + "x-ms-client-name": "DefaultEncryptionScope", + "type": "string", + "in": "header", + "required": false, + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "container-cpk-scope-info" + }, + "description": "Optional. Version 2019-07-07 and later. Specifies the default encryption scope to set on the container and use for all future writes." + }, + "DeletedContainerName": { + "name": "x-ms-deleted-container-name", + "x-ms-client-name": "DeletedContainerName", + "type": "string", + "in": "header", + "required": false, + "x-ms-parameter-location": "method", + "description": "Optional. Version 2019-12-12 and later. Specifies the name of the deleted container to restore." + }, + "DeletedContainerVersion": { + "name": "x-ms-deleted-container-version", + "x-ms-client-name": "DeletedContainerVersion", + "type": "string", + "in": "header", + "required": false, + "x-ms-parameter-location": "method", + "description": "Optional. Version 2019-12-12 and later. Specifies the version of the deleted container to restore." + }, + "DenyEncryptionScopeOverride": { + "name": "x-ms-deny-encryption-scope-override", + "x-ms-client-name": "PreventEncryptionScopeOverride", + "type": "boolean", + "in": "header", + "required": false, + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "container-cpk-scope-info" + }, + "description": "Optional. Version 2019-07-07 and newer. If true, prevents any request from specifying a different encryption scope than the scope set on the container." + }, + "FilterBlobsInclude": { + "name": "include", + "in": "query", + "required": false, + "type": "array", + "collectionFormat": "csv", + "items": { + "type": "string", + "enum": [ + "none", + "versions" + ], + "x-ms-enum": { + "name": "FilterBlobsIncludeItem", + "modelAsString": false + } + }, + "x-ms-parameter-location": "method", + "description": "Include this parameter to specify one or more datasets to include in the response." + }, + "FilterBlobsWhere": { + "name": "where", + "in": "query", + "required": false, + "type": "string", + "description": "Filters the results to return only to return only blobs whose tags match the specified expression.", + "x-ms-parameter-location": "method" + }, + "IfMatch": { + "name": "If-Match", + "x-ms-client-name": "ifMatch", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "modified-access-conditions" + }, + "description": "Specify an ETag value to operate only on blobs with a matching value." + }, + "IfModifiedSince": { + "name": "If-Modified-Since", + "x-ms-client-name": "ifModifiedSince", + "in": "header", + "required": false, + "type": "string", + "format": "date-time-rfc1123", + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "modified-access-conditions" + }, + "description": "Specify this header value to operate only on a blob if it has been modified since the specified date/time." + }, + "IfNoneMatch": { + "name": "If-None-Match", + "x-ms-client-name": "ifNoneMatch", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "modified-access-conditions" + }, + "description": "Specify an ETag value to operate only on blobs without a matching value." + }, + "IfUnmodifiedSince": { + "name": "If-Unmodified-Since", + "x-ms-client-name": "ifUnmodifiedSince", + "in": "header", + "required": false, + "type": "string", + "format": "date-time-rfc1123", + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "modified-access-conditions" + }, + "description": "Specify this header value to operate only on a blob if it has not been modified since the specified date/time." + }, + "IfSequenceNumberEqualTo": { + "name": "x-ms-if-sequence-number-eq", + "x-ms-client-name": "ifSequenceNumberEqualTo", + "in": "header", + "required": false, + "type": "integer", + "format": "int64", + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "sequence-number-access-conditions" + }, + "description": "Specify this header value to operate only on a blob if it has the specified sequence number." + }, + "IfSequenceNumberLessThan": { + "name": "x-ms-if-sequence-number-lt", + "x-ms-client-name": "ifSequenceNumberLessThan", + "in": "header", + "required": false, + "type": "integer", + "format": "int64", + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "sequence-number-access-conditions" + }, + "description": "Specify this header value to operate only on a blob if it has a sequence number less than the specified." + }, + "IfSequenceNumberLessThanOrEqualTo": { + "name": "x-ms-if-sequence-number-le", + "x-ms-client-name": "ifSequenceNumberLessThanOrEqualTo", + "in": "header", + "required": false, + "type": "integer", + "format": "int64", + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "sequence-number-access-conditions" + }, + "description": "Specify this header value to operate only on a blob if it has a sequence number less than or equal to the specified." + }, + "IfTags": { + "name": "x-ms-if-tags", + "x-ms-client-name": "ifTags", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "modified-access-conditions" + }, + "description": "Specify a SQL where clause on blob tags to operate only on blobs with a matching value." + }, + "ImmutabilityPolicyExpiry": { + "name": "x-ms-immutability-policy-until-date", + "x-ms-client-name": "immutabilityPolicyExpiry", + "in": "header", + "required": false, + "type": "string", + "format": "date-time-rfc1123", + "x-ms-parameter-location": "method", + "description": "Specifies the date time when the blobs immutability policy is set to expire." + }, + "ImmutabilityPolicyMode": { + "name": "x-ms-immutability-policy-mode", + "x-ms-client-name": "immutabilityPolicyMode", + "in": "header", + "required": false, + "type": "string", + "enum": [ + "Mutable", + "Unlocked", + "Locked" + ], + "x-ms-enum": { + "name": "BlobImmutabilityPolicyMode", + "modelAsString": false + }, + "x-ms-parameter-location": "method", + "description": "Specifies the immutability policy mode to set on the blob." + }, + "KeyInfo": { + "description": "Key information", + "name": "KeyInfo", + "in": "body", + "x-ms-parameter-location": "method", + "required": true, + "schema": { + "$ref": "#/definitions/KeyInfo" + } + }, + "ListContainersInclude": { + "name": "include", + "in": "query", + "required": false, + "type": "array", + "collectionFormat": "csv", + "items": { + "type": "string", + "enum": [ + "", + "metadata", + "deleted", + "system" + ], + "x-ms-enum": { + "name": "ListContainersIncludeType", + "modelAsString": false + } + }, + "x-ms-parameter-location": "method", + "description": "Include this parameter to specify that the container's metadata be returned as part of the response body." + }, + "LeaseBreakPeriod": { + "name": "x-ms-lease-break-period", + "x-ms-client-name": "breakPeriod", + "in": "header", + "required": false, + "type": "integer", + "x-ms-parameter-location": "method", + "description": "For a break operation, proposed duration the lease should continue before it is broken, in seconds, between 0 and 60. This break period is only used if it is shorter than the time remaining on the lease. If longer, the time remaining on the lease is used. A new lease will not be available before the break period has expired, but the lease may be held for longer than the break period. If this header does not appear with a break operation, a fixed-duration lease breaks after the remaining lease period elapses, and an infinite lease breaks immediately." + }, + "LeaseDurationBlob": { + "name": "x-ms-lease-duration", + "x-ms-client-name": "duration", + "in": "header", + "required": false, + "type": "integer", + "x-ms-parameter-location": "method", + "description": "Specifies the duration of the lease, in seconds, or negative one (-1) for a lease that never expires. A non-infinite lease can be between 15 and 60 seconds. A lease duration cannot be changed using renew or change." + }, + "LeaseIdOptional": { + "name": "x-ms-lease-id", + "x-ms-client-name": "leaseId", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "lease-access-conditions" + }, + "description": "If specified, the operation only succeeds if the resource's lease is active and matches this ID." + }, + "LeaseIdRequired": { + "name": "x-ms-lease-id", + "x-ms-client-name": "leaseId", + "in": "header", + "required": true, + "type": "string", + "x-ms-parameter-location": "method", + "description": "Specifies the current lease ID on the resource." + }, + "LegalHoldOptional": { + "name": "x-ms-legal-hold", + "x-ms-client-name": "legalHold", + "in": "header", + "required": false, + "type": "boolean", + "x-ms-parameter-location": "method", + "description": "Specified if a legal hold should be set on the blob." + }, + "LegalHoldRequired": { + "name": "x-ms-legal-hold", + "x-ms-client-name": "legalHold", + "in": "header", + "required": true, + "type": "boolean", + "x-ms-parameter-location": "method", + "description": "Specified if a legal hold should be set on the blob." + }, + "Marker": { + "name": "marker", + "in": "query", + "required": false, + "type": "string", + "description": "A string value that identifies the portion of the list of containers to be returned with the next listing operation. The operation returns the NextMarker value within the response body if the listing operation did not return all containers remaining to be listed with the current page. The NextMarker value can be used as the value for the marker parameter in a subsequent call to request the next page of list items. The marker value is opaque to the client.", + "x-ms-parameter-location": "method" + }, + "MaxResultsBlob": { + "name": "maxresults", + "in": "query", + "required": false, + "type": "integer", + "minimum": 1, + "x-ms-parameter-location": "method", + "description": "Specifies the maximum number of containers to return. If the request does not specify maxresults, or specifies a value greater than 5000, the server will return up to 5000 items. Note that if the listing operation crosses a partition boundary, then the service will return a continuation token for retrieving the remainder of the results. For this reason, it is possible that the service will return fewer results than specified by maxresults, or than the default of 5000." + }, + "Metadata": { + "name": "x-ms-meta", + "x-ms-client-name": "metadata", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "description": "Optional. Specifies a user-defined name-value pair associated with the blob. If no name-value pairs are specified, the operation will copy the metadata from the source blob or file to the destination blob. If one or more name-value pairs are specified, the destination blob is created with the specified metadata, and metadata is not copied from the source blob or file. Note that beginning with version 2009-09-19, metadata names must adhere to the naming rules for C# identifiers. See Naming and Referencing Containers, Blobs, and Metadata for more information.", + "x-ms-header-collection-prefix": "x-ms-meta-" + }, + "MultipartContentType": { + "name": "Content-Type", + "x-ms-client-name": "multipartContentType", + "in": "header", + "required": true, + "type": "string", + "x-ms-parameter-location": "method", + "description": "Required. The value of this header must be multipart/mixed with a batch boundary. Example header value: multipart/mixed; boundary=batch_" + }, + "ObjectReplicationPolicyId": { + "name": "x-ms-or-policy-id", + "x-ms-client-name": "objectReplicationPolicyId", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "description": "Optional. Only valid when Object Replication is enabled for the storage container and on the destination blob of the replication." + }, + "ObjectReplicationRules": { + "name": "x-ms-or", + "x-ms-client-name": "ObjectReplicationRules", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "description": "Optional. Only valid when Object Replication is enabled for the storage container and on the source blob of the replication. When retrieving this header, it will return the header with the policy id and rule id (e.g. x-ms-or-policyid_ruleid), and the value will be the status of the replication (e.g. complete, failed).", + "x-ms-header-collection-prefix": "x-ms-or-" + }, + "Prefix": { + "name": "prefix", + "in": "query", + "required": false, + "type": "string", + "description": "Filters results to filesystems within the specified prefix.", + "x-ms-parameter-location": "method" + }, + "PrevSnapshot": { + "name": "prevsnapshot", + "in": "query", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "description": "Optional in version 2015-07-08 and newer. The prevsnapshot parameter is a DateTime value that specifies that the response will contain only pages that were changed between target blob and previous snapshot. Changed pages include both updated and cleared pages. The target blob may be a snapshot, as long as the snapshot specified by prevsnapshot is the older of the two. Note that incremental snapshots are currently supported only for blobs created on or after January 1, 2016." + }, + "PrevSnapshotUrl": { + "name": "x-ms-previous-snapshot-url", + "x-ms-client-name": "prevSnapshotUrl", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "description": "Optional. This header is only supported in service versions 2019-04-19 and after and specifies the URL of a previous snapshot of the target blob. The response will only contain pages that were changed between the target blob and its previous snapshot." + }, + "ProposedLeaseIdOptional": { + "name": "x-ms-proposed-lease-id", + "x-ms-client-name": "proposedLeaseId", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "description": "Proposed lease ID, in a GUID string format. The Blob service returns 400 (Invalid request) if the proposed lease ID is not in the correct format. See Guid Constructor (String) for a list of valid GUID string formats." + }, + "ProposedLeaseIdRequired": { + "name": "x-ms-proposed-lease-id", + "x-ms-client-name": "proposedLeaseId", + "in": "header", + "required": true, + "type": "string", + "x-ms-parameter-location": "method", + "description": "Proposed lease ID, in a GUID string format. The Blob service returns 400 (Invalid request) if the proposed lease ID is not in the correct format. See Guid Constructor (String) for a list of valid GUID string formats." + }, + "QueryRequest": { + "name": "queryRequest", + "in": "body", + "x-ms-parameter-location": "client", + "schema": { + "$ref": "#/definitions/QueryRequest" + }, + "description": "the query request" + }, + "Range": { + "name": "x-ms-range", + "x-ms-client-name": "range", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "description": "The HTTP Range request header specifies one or more byte ranges of the resource to be retrieved." + }, + "RangeRequiredPutPageFromUrl": { + "name": "x-ms-range", + "x-ms-client-name": "range", + "in": "header", + "required": true, + "type": "string", + "x-ms-parameter-location": "method", + "description": "The range of bytes to which the source range would be written. The range should be 512 aligned and range-end is required." + }, + "SequenceNumberAction": { + "name": "x-ms-sequence-number-action", + "x-ms-client-name": "sequenceNumberAction", + "in": "header", + "required": true, + "x-ms-parameter-location": "method", + "description": "Required if the x-ms-blob-sequence-number header is set for the request. This property applies to page blobs only. This property indicates how the service should modify the blob's sequence number", + "type": "string", + "enum": [ + "max", + "update", + "increment" + ], + "x-ms-enum": { + "name": "SequenceNumberActionType", + "modelAsString": false + } + }, + "Snapshot": { + "name": "snapshot", + "in": "query", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "description": "The snapshot parameter is an opaque DateTime value that, when present, specifies the blob snapshot to retrieve. For more information on working with blob snapshots, see Creating a Snapshot of a Blob." + }, + "VersionId": { + "name": "versionid", + "x-ms-client-name": "versionId", + "in": "query", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "description": "The version id parameter is an opaque DateTime value that, when present, specifies the version of the blob to operate on. It's for service version 2019-10-10 and newer." + }, + "SealBlob": { + "name": "x-ms-seal-blob", + "x-ms-client-name": "SealBlob", + "in": "header", + "required": false, + "type": "boolean", + "x-ms-parameter-location": "method", + "description": "Overrides the sealed state of the destination blob. Service version 2019-12-12 and newer." + }, + "SourceContainerName": { + "name": "x-ms-source-container-name", + "x-ms-client-name": "SourceContainerName", + "type": "string", + "in": "header", + "required": true, + "x-ms-parameter-location": "method", + "description": "Required. Specifies the name of the container to rename." + }, + "SourceContentMD5": { + "name": "x-ms-source-content-md5", + "x-ms-client-name": "sourceContentMD5", + "in": "header", + "required": false, + "type": "string", + "format": "byte", + "x-ms-parameter-location": "method", + "description": "Specify the md5 calculated for the range of bytes that must be read from the copy source." + }, + "SourceContentCRC64": { + "name": "x-ms-source-content-crc64", + "x-ms-client-name": "sourceContentcrc64", + "in": "header", + "required": false, + "type": "string", + "format": "byte", + "x-ms-parameter-location": "method", + "description": "Specify the crc64 calculated for the range of bytes that must be read from the copy source." + }, + "SourceRange": { + "name": "x-ms-source-range", + "x-ms-client-name": "sourceRange", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "description": "Bytes of source data in the specified range." + }, + "SourceRangeRequiredPutPageFromUrl": { + "name": "x-ms-source-range", + "x-ms-client-name": "sourceRange", + "in": "header", + "required": true, + "type": "string", + "x-ms-parameter-location": "method", + "description": "Bytes of source data in the specified range. The length of this range should match the ContentLength header and x-ms-range/Range destination range header." + }, + "SourceIfMatch": { + "name": "x-ms-source-if-match", + "x-ms-client-name": "sourceIfMatch", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "source-modified-access-conditions" + }, + "description": "Specify an ETag value to operate only on blobs with a matching value." + }, + "SourceIfModifiedSince": { + "name": "x-ms-source-if-modified-since", + "x-ms-client-name": "sourceIfModifiedSince", + "in": "header", + "required": false, + "type": "string", + "format": "date-time-rfc1123", + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "source-modified-access-conditions" + }, + "description": "Specify this header value to operate only on a blob if it has been modified since the specified date/time." + }, + "SourceIfNoneMatch": { + "name": "x-ms-source-if-none-match", + "x-ms-client-name": "sourceIfNoneMatch", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "source-modified-access-conditions" + }, + "description": "Specify an ETag value to operate only on blobs without a matching value." + }, + "SourceIfUnmodifiedSince": { + "name": "x-ms-source-if-unmodified-since", + "x-ms-client-name": "sourceIfUnmodifiedSince", + "in": "header", + "required": false, + "type": "string", + "format": "date-time-rfc1123", + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "source-modified-access-conditions" + }, + "description": "Specify this header value to operate only on a blob if it has not been modified since the specified date/time." + }, + "SourceLeaseId": { + "name": "x-ms-source-lease-id", + "x-ms-client-name": "sourceLeaseId", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "description": "A lease ID for the source path. If specified, the source path must have an active lease and the lease ID must match." + }, + "SourceIfTags": { + "name": "x-ms-source-if-tags", + "x-ms-client-name": "sourceIfTags", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "source-modified-access-conditions" + }, + "description": "Specify a SQL where clause on blob tags to operate only on blobs with a matching value." + }, + "SourceUrl": { + "name": "x-ms-copy-source", + "x-ms-client-name": "sourceUrl", + "in": "header", + "required": true, + "type": "string", + "x-ms-parameter-location": "method", + "description": "Specify a URL to the copy source." + }, + "StorageServiceProperties": { + "name": "StorageServiceProperties", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/StorageServiceProperties" + }, + "x-ms-parameter-location": "method", + "description": "The StorageService properties." + }, + "Timeout": { + "name": "timeout", + "in": "query", + "required": false, + "type": "integer", + "minimum": 0, + "x-ms-parameter-location": "method", + "description": "The timeout parameter is expressed in seconds. For more information, see Setting Timeouts for Blob Service Operations." + } + } +} \ No newline at end of file diff --git a/swagger/data-lake-storage.json-2021-04-10.json b/swagger/data-lake-storage.json-2021-04-10.json new file mode 100644 index 000000000..5b256a570 --- /dev/null +++ b/swagger/data-lake-storage.json-2021-04-10.json @@ -0,0 +1,4592 @@ +{ + "swagger": "2.0", + "info": { + "description": "Azure Data Lake Storage provides storage for Hadoop and other big data workloads.", + "title": "Azure Data Lake Storage REST API", + "version": "2021-04-10", + "x-ms-code-generation-settings": { + "internalConstructors": true, + "name": "DataLakeStorageClient", + "header": "MIT", + "strictSpecAdherence": false + } + }, + "x-ms-parameterized-host": { + "hostTemplate": "{url}", + "useSchemePrefix": false, + "positionInOperation": "first", + "parameters": [ + { + "$ref": "#/parameters/Url" + } + ] + }, + "schemes": ["https"], + "consumes": ["application/json"], + "produces": ["application/json"], + "paths": {}, + "x-ms-paths": { + "/": { + "get": { + "operationId": "Service_ListFileSystems", + "summary": "List FileSystems", + "description": "List filesystems and their properties in given account.", + "x-ms-pageable": { + "itemName": "filesystems", + "nextLinkName": null + }, + "tags": ["Account Operations"], + "parameters": [ + { + "name": "resource", + "in": "query", + "description": "The value must be \"account\" for all account operations.", + "required": true, + "type": "string", + "enum": ["account"], + "x-ms-enum": { + "name": "AccountResourceType", + "modelAsString": false + } + }, + { + "$ref": "#/parameters/Prefix" + }, + { + "$ref": "#/parameters/Continuation" + }, + { + "$ref": "#/parameters/MaxResults" + }, + { + "$ref": "#/parameters/ClientRequestId" + }, + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + } + ], + "responses": { + "200": { + "description": "OK", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string", + "format": "date-time-rfc1123" + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-continuation": { + "x-ms-client-name": "Continuation", + "description": "If the number of filesystems to be listed exceeds the maxResults limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the list operation to continue listing the filesystems.", + "type": "string" + }, + "Content-Type": { + "description": "The content type of list filesystem response. The default content type is application/json.", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/FileSystemList" + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageError" + } + } + } + } + }, + "/{filesystem}": { + "put": { + "operationId": "FileSystem_Create", + "summary": "Create FileSystem", + "description": "Create a FileSystem rooted at the specified location. If the FileSystem already exists, the operation fails. This operation does not support conditional HTTP requests.", + "tags": ["FileSystem Operations"], + "parameters": [ + { + "$ref": "#/parameters/Properties" + } + ], + "responses": { + "201": { + "description": "Created", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string", + "format": "date-time-rfc1123" + }, + "ETag": { + "description": "An HTTP entity tag associated with the FileSystem.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the filesystem was last modified. Operations on files and directories do not affect the last modified time.", + "format": "date-time-rfc1123", + "type": "string" + }, + "x-ms-request-id": { + "x-ms-client-name": "ClientRequestId", + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-namespace-enabled": { + "x-ms-client-name": "NamespaceEnabled", + "description": "A bool string indicates whether the namespace feature is enabled. If \"true\", the namespace is enabled for the filesystem.", + "type": "string" + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageError" + } + } + } + }, + "patch": { + "operationId": "FileSystem_SetProperties", + "summary": "Set FileSystem Properties", + "description": "Set properties for the FileSystem. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", + "tags": ["FileSystem Operations"], + "parameters": [ + { + "$ref": "#/parameters/Properties" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + } + ], + "responses": { + "200": { + "description": "Ok", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string", + "format": "date-time-rfc1123" + }, + "ETag": { + "description": "An HTTP entity tag associated with the filesystem. Changes to filesystem properties affect the entity tag, but operations on files and directories do not.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the filesystem was last modified. Changes to filesystem properties update the last modified time, but operations on files and directories do not.", + "format": "date-time-rfc1123", + "type": "string" + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "description": "The version of the REST protocol used to process the request.", + "type": "string" + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageError" + } + } + } + }, + "head": { + "operationId": "FileSystem_GetProperties", + "summary": "Get FileSystem Properties.", + "description": "All system and user-defined filesystem properties are specified in the response headers.", + "tags": ["FileSystem Operations"], + "responses": { + "200": { + "description": "Ok", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string", + "format": "date-time-rfc1123" + }, + "ETag": { + "description": "An HTTP entity tag associated with the filesystem. Changes to filesystem properties affect the entity tag, but operations on files and directories do not.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the filesystem was last modified. Changes to filesystem properties update the last modified time, but operations on files and directories do not.", + "format": "date-time-rfc1123", + "type": "string" + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-properties": { + "x-ms-client-name": "Properties", + "description": "The user-defined properties associated with the filesystem. A comma-separated list of name and value pairs in the format \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set.", + "type": "string" + }, + "x-ms-namespace-enabled": { + "x-ms-client-name": "NamespaceEnabled", + "description": "A bool string indicates whether the namespace feature is enabled. If \"true\", the namespace is enabled for the filesystem.", + "type": "string" + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageError" + } + } + } + }, + "delete": { + "operationId": "FileSystem_Delete", + "summary": "Delete FileSystem", + "description": "Marks the FileSystem for deletion. When a FileSystem is deleted, a FileSystem with the same identifier cannot be created for at least 30 seconds. While the filesystem is being deleted, attempts to create a filesystem with the same identifier will fail with status code 409 (Conflict), with the service returning additional error information indicating that the filesystem is being deleted. All other operations, including operations on any files or directories within the filesystem, will fail with status code 404 (Not Found) while the filesystem is being deleted. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", + "tags": ["FileSystem Operations"], + "responses": { + "202": { + "description": "Accepted", + "headers": { + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string", + "format": "date-time-rfc1123" + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageError" + } + } + }, + "parameters": [ + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + } + ] + }, + "parameters": [ + { + "$ref": "#/parameters/FileSystem" + }, + { + "$ref": "#/parameters/FileSystemResource" + }, + { + "$ref": "#/parameters/ClientRequestId" + }, + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + } + ] + }, + "/{filesystem}?resource=filesystem": { + "get": { + "operationId": "FileSystem_ListPaths", + "summary": "List Paths", + "description": "List FileSystem paths and their properties.", + "x-ms-pageable": { + "itemName": "paths", + "nextLinkName": null + }, + "tags": ["FileSystem Operations"], + "parameters": [ + { + "$ref": "#/parameters/Continuation" + }, + { + "$ref": "#/parameters/Directory" + }, + { + "$ref": "#/parameters/RecursiveRequired" + }, + { + "$ref": "#/parameters/MaxResults" + }, + { + "$ref": "#/parameters/Upn" + } + ], + "consumes": ["application/json"], + "produces": ["application/xml"], + "responses": { + "200": { + "description": "Ok", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string", + "format": "date-time-rfc1123" + }, + "ETag": { + "description": "An HTTP entity tag associated with the filesystem. Changes to filesystem properties affect the entity tag, but operations on files and directories do not.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the filesystem was last modified. Changes to filesystem properties update the last modified time, but operations on files and directories do not.", + "format": "date-time-rfc1123", + "type": "string" + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-continuation": { + "x-ms-client-name": "Continuation", + "description": "If the number of paths to be listed exceeds the maxResults limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the list operation to continue listing the paths.", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/PathList" + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageError" + } + } + } + }, + "parameters": [ + { + "$ref": "#/parameters/FileSystem" + }, + { + "$ref": "#/parameters/FileSystemResource" + }, + { + "$ref": "#/parameters/ClientRequestId" + }, + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + } + ] + }, + "/{filesystem}?restype=container&comp=list&flat": { + "get": { + "tags": ["containers"], + "operationId": "FileSystem_ListBlobFlatSegment", + "description": "[Update] The List Blobs operation returns a list of the blobs under the specified container", + "parameters": [ + { + "$ref": "#/parameters/Prefix" + }, + { + "$ref": "#/parameters/Marker" + }, + { + "$ref": "#/parameters/MaxResults" + }, + { + "$ref": "#/parameters/MaxResultsBlob" + }, + { + "$ref": "#/parameters/ListBlobsInclude" + }, + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "200": { + "description": "Success.", + "headers": { + "Content-Type": { + "type": "string", + "description": "The media type of the body of the response. For List Blobs this is 'application/xml'" + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + } + }, + "schema": { + "$ref": "#/definitions/ListBlobsFlatSegmentResponse" + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageError" + } + } + }, + "x-ms-pageable": { + "nextLinkName": "NextMarker" + } + }, + "parameters": [ + { + "name": "restype", + "description": "restype", + "in": "query", + "required": true, + "type": "string", + "enum": ["container"] + }, + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": ["list"] + } + ] + }, + "/{filesystem}?restype=container&comp=list&hierarchy": { + "get": { + "tags": ["containers"], + "produces": ["application/xml"], + "operationId": "FileSystem_ListBlobHierarchySegment", + "description": "The List Blobs operation returns a list of the blobs under the specified container", + "parameters": [ + { + "$ref": "#/parameters/Prefix" + }, + { + "$ref": "#/parameters/Delimiter" + }, + { + "$ref": "#/parameters/Marker" + }, + { + "$ref": "#/parameters/MaxResults" + }, + { + "$ref": "#/parameters/MaxResultsBlob" + }, + { + "$ref": "#/parameters/ListBlobsInclude" + }, + { + "$ref": "#/parameters/ListBlobsShowOnly" + }, + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "200": { + "description": "Success.", + "headers": { + "Content-Type": { + "type": "string", + "description": "The media type of the body of the response. For List Blobs this is 'application/xml'" + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated" + } + }, + "schema": { + "$ref": "#/definitions/ListBlobsHierarchySegmentResponse" + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageError" + } + } + }, + "x-ms-pageable": { + "nextLinkName": "NextMarker" + } + }, + "parameters": [ + { + "$ref": "#/parameters/FileSystem" + }, + { + "name": "restype", + "description": "restype", + "in": "query", + "required": true, + "type": "string", + "enum": ["container"] + }, + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": ["list"] + } + ] + }, + "/{filesystem}/{path}": { + "put": { + "operationId": "Path_Create", + "summary": "Create File | Create Directory | Rename File | Rename Directory", + "description": "Create or rename a file or directory. By default, the destination is overwritten and if the destination already exists and has a lease the lease is broken. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). To fail if the destination already exists, use a conditional request with If-None-Match: \"*\".", + "consumes": ["application/octet-stream"], + "tags": ["File and Directory Operations"], + "parameters": [ + { + "name": "resource", + "in": "query", + "description": "Required only for Create File and Create Directory. The value must be \"file\" or \"directory\".", + "required": false, + "type": "string", + "enum": ["directory", "file"], + "x-ms-enum": { + "name": "PathResourceType", + "modelAsString": false + } + }, + { + "$ref": "#/parameters/Continuation" + }, + { + "name": "mode", + "in": "query", + "description": "Optional. Valid only when namespace is enabled. This parameter determines the behavior of the rename operation. The value must be \"legacy\" or \"posix\", and the default value will be \"posix\".", + "required": false, + "type": "string", + "enum": ["legacy", "posix"], + "x-ms-enum": { + "name": "PathRenameMode", + "modelAsString": false + } + }, + { + "$ref": "#/parameters/CacheControl" + }, + { + "$ref": "#/parameters/ContentEncoding" + }, + { + "$ref": "#/parameters/ContentLanguage" + }, + { + "$ref": "#/parameters/ContentDisposition" + }, + { + "$ref": "#/parameters/ContentType" + }, + { + "$ref": "#/parameters/RenameSource" + }, + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "$ref": "#/parameters/SourceLeaseId" + }, + { + "$ref": "#/parameters/Properties" + }, + { + "$ref": "#/parameters/Permissions" + }, + { + "$ref": "#/parameters/Umask" + }, + { + "$ref": "#/parameters/IfMatch" + }, + { + "$ref": "#/parameters/IfNoneMatch" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/SourceIfMatch" + }, + { + "$ref": "#/parameters/SourceIfNoneMatch" + }, + { + "$ref": "#/parameters/SourceIfModifiedSince" + }, + { + "$ref": "#/parameters/SourceIfUnmodifiedSince" + }, + { + "$ref": "#/parameters/EncryptionKey" + }, + { + "$ref": "#/parameters/EncryptionKeySha256" + }, + { + "$ref": "#/parameters/EncryptionAlgorithm" + }, + { + "$ref": "#/parameters/Owner" + }, + { + "$ref": "#/parameters/Group" + }, + { + "$ref": "#/parameters/Acl" + }, + { + "$ref": "#/parameters/ProposedLeaseIdOptional" + }, + { + "$ref": "#/parameters/LeaseDurationMethod" + }, + { + "$ref": "#/parameters/PathExpiryOptionsOptional" + }, + { + "$ref": "#/parameters/PathExpiryTime" + } + ], + "responses": { + "201": { + "description": "The file or directory was created.", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string", + "format": "date-time-rfc1123" + }, + "ETag": { + "description": "An HTTP entity tag associated with the file or directory.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", + "format": "date-time-rfc1123", + "type": "string" + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-continuation": { + "x-ms-client-name": "Continuation", + "description": "When renaming a directory, the number of paths that are renamed with each invocation is limited. If the number of paths to be renamed exceeds this limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the rename operation to continue renaming the directory.", + "type": "string" + }, + "Content-Length": { + "description": "The size of the resource in bytes.", + "type": "integer", + "format": "int64" + }, + "x-ms-request-server-encrypted": { + "x-ms-client-name": "IsServerEncrypted", + "type": "boolean", + "description": "The value of this header is set to true if the contents of the request are successfully encrypted using the specified algorithm, and false otherwise." + }, + "x-ms-encryption-key-sha256": { + "x-ms-client-name": "EncryptionKeySha256", + "type": "string", + "description": "The SHA-256 hash of the encryption key used to encrypt the blob. This header is only returned when the blob was encrypted with a customer-provided key." + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageError" + } + } + } + }, + "patch": { + "operationId": "Path_Update", + "summary": "Append Data | Flush Data | Set Properties | Set Access Control", + "description": "Uploads data to be appended to a file, flushes (writes) previously uploaded data to a file, sets properties for a file or directory, or sets access control for a file or directory. Data can only be appended to a file. Concurrent writes to the same file using multiple clients are not supported. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", + "consumes": ["application/octet-stream"], + "tags": ["File and Directory Operations"], + "parameters": [ + { + "name": "action", + "in": "query", + "description": "The action must be \"append\" to upload data to be appended to a file, \"flush\" to flush previously uploaded data to a file, \"setProperties\" to set the properties of a file or directory, \"setAccessControl\" to set the owner, group, permissions, or access control list for a file or directory, or \"setAccessControlRecursive\" to set the access control list for a directory recursively. Note that Hierarchical Namespace must be enabled for the account in order to use access control. Also note that the Access Control List (ACL) includes permissions for the owner, owning group, and others, so the x-ms-permissions and x-ms-acl request headers are mutually exclusive.", + "required": true, + "type": "string", + "enum": [ + "append", + "flush", + "setProperties", + "setAccessControl", + "setAccessControlRecursive" + ], + "x-ms-enum": { + "name": "PathUpdateAction", + "modelAsString": false + } + }, + { + "name": "flush", + "in": "query", + "description": "Optional. Valid only for append calls. This parameter allows the caller to flush during an append call. Default value is 'false' , if 'true' the data will be flushed with the append call. Note that when using flush=true, the following headers are not supported - 'x-ms-cache-control', 'x-ms-content-encoding', 'x-ms-content-type', 'x-ms-content-language', 'x-ms-content-md5', 'x-ms-content-disposition'. To set these headers during flush, please use action=flush", + "required": false, + "type": "boolean" + }, + { + "name": "maxRecords", + "in": "query", + "description": "Optional. Valid for \"SetAccessControlRecursive\" operation. It specifies the maximum number of files or directories on which the acl change will be applied. If omitted or greater than 2,000, the request will process up to 2,000 items", + "format": "int32", + "minimum": 1, + "required": false, + "type": "integer" + }, + { + "name": "continuation", + "in": "query", + "description": "Optional. The number of paths processed with each invocation is limited. If the number of paths to be processed exceeds this limit, a continuation token is returned in the response header x-ms-continuation. When a continuation token is returned in the response, it must be percent-encoded and specified in a subsequent invocation of setAccessControlRecursive operation.", + "required": false, + "type": "string" + }, + { + "$ref": "#/parameters/PathSetAccessControlRecursiveMode" + }, + { + "$ref": "#/parameters/ForceFlag" + }, + { + "$ref": "#/parameters/Position" + }, + { + "$ref": "#/parameters/RetainUncommittedData" + }, + { + "$ref": "#/parameters/Close" + }, + { + "$ref": "#/parameters/ContentLength" + }, + { + "$ref": "#/parameters/ContentMD5" + }, + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "$ref": "#/parameters/CacheControl" + }, + { + "$ref": "#/parameters/ContentType" + }, + { + "$ref": "#/parameters/ContentDisposition" + }, + { + "$ref": "#/parameters/ContentEncoding" + }, + { + "$ref": "#/parameters/ContentLanguage" + }, + { + "$ref": "#/parameters/Properties" + }, + { + "$ref": "#/parameters/Owner" + }, + { + "$ref": "#/parameters/Group" + }, + { + "$ref": "#/parameters/Permissions" + }, + { + "$ref": "#/parameters/Acl" + }, + { + "$ref": "#/parameters/IfMatch" + }, + { + "$ref": "#/parameters/IfNoneMatch" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/Body" + } + ], + "responses": { + "200": { + "description": "The data was flushed (written) to the file or the properties were set successfully. Response body is optional and is valid only for \"SetAccessControlRecursive\"", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string", + "format": "date-time-rfc1123" + }, + "ETag": { + "description": "An HTTP entity tag associated with the file or directory.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", + "format": "date-time-rfc1123", + "type": "string" + }, + "Accept-Ranges": { + "description": "Indicates that the service supports requests for partial file content.", + "type": "string" + }, + "Cache-Control": { + "description": "If the Cache-Control request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Disposition": { + "description": "If the Content-Disposition request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Encoding": { + "description": "If the Content-Encoding request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Language": { + "description": "If the Content-Language request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Length": { + "description": "The size of the resource in bytes.", + "type": "integer", + "format": "int64" + }, + "Content-Range": { + "description": "Indicates the range of bytes returned in the event that the client requested a subset of the file by setting the Range request header.", + "type": "string" + }, + "Content-Type": { + "description": "The content type specified for the resource. If no content type was specified, the default content type is application/octet-stream.", + "type": "string" + }, + "Content-MD5": { + "description": "An MD5 hash of the request content. This header is only returned for \"Flush\" operation. This header is returned so that the client can check for message content integrity. This header refers to the content of the request, not actual file content.", + "type": "string", + "format": "byte" + }, + "x-ms-properties": { + "x-ms-client-name": "Properties", + "description": "User-defined properties associated with the file or directory, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set.", + "type": "string" + }, + "x-ms-continuation": { + "description": "When performing setAccessControlRecursive on a directory, the number of paths that are processed with each invocation is limited. If the number of paths to be processed exceeds this limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the setAccessControlRecursive operation to continue the setAccessControlRecursive operation on the directory.", + "type": "string" + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "description": "The version of the REST protocol used to process the request.", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/SetAccessControlRecursiveResponse" + } + }, + "202": { + "description": "The uploaded data was accepted.", + "headers": { + "Content-MD5": { + "description": "An MD5 hash of the request content. This header is only returned for \"Append\" operation. This header is returned so that the client can check for message content integrity. The value of this header is computed by the service; it is not necessarily the same value specified in the request headers.", + "type": "string", + "format": "byte" + }, + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string", + "format": "date-time-rfc1123" + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "description": "The version of the REST protocol used to process the request.", + "type": "string" + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageError" + } + } + } + }, + "post": { + "operationId": "Path_Lease", + "summary": "Lease Path", + "description": "Create and manage a lease to restrict write and delete access to the path. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", + "tags": ["File and Directory Operations"], + "parameters": [ + { + "name": "x-ms-lease-action", + "in": "header", + "description": "There are five lease actions: \"acquire\", \"break\", \"change\", \"renew\", and \"release\". Use \"acquire\" and specify the \"x-ms-proposed-lease-id\" and \"x-ms-lease-duration\" to acquire a new lease. Use \"break\" to break an existing lease. When a lease is broken, the lease break period is allowed to elapse, during which time no lease operation except break and release can be performed on the file. When a lease is successfully broken, the response indicates the interval in seconds until a new lease can be acquired. Use \"change\" and specify the current lease ID in \"x-ms-lease-id\" and the new lease ID in \"x-ms-proposed-lease-id\" to change the lease ID of an active lease. Use \"renew\" and specify the \"x-ms-lease-id\" to renew an existing lease. Use \"release\" and specify the \"x-ms-lease-id\" to release a lease.", + "required": true, + "type": "string", + "enum": ["acquire", "break", "change", "renew", "release"], + "x-ms-enum": { + "name": "PathLeaseAction", + "modelAsString": false + } + }, + { + "$ref": "#/parameters/LeaseDuration" + }, + { + "name": "x-ms-lease-break-period", + "in": "header", + "description": "The lease break period duration is optional to break a lease, and specifies the break period of the lease in seconds. The lease break duration must be between 0 and 60 seconds.", + "format": "int32", + "required": false, + "type": "integer" + }, + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "$ref": "#/parameters/ProposedLeaseIdOptional" + }, + { + "$ref": "#/parameters/IfMatch" + }, + { + "$ref": "#/parameters/IfNoneMatch" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + } + ], + "responses": { + "200": { + "description": "The \"renew\", \"change\" or \"release\" action was successful.", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string", + "format": "date-time-rfc1123" + }, + "ETag": { + "description": "An HTTP entity tag associated with the file.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the file was last modified. Write operations on the file update the last modified time.", + "format": "date-time-rfc1123", + "type": "string" + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-lease-id": { + "x-ms-client-name": "LeaseId", + "description": "A successful \"renew\" action returns the lease ID.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + } + } + }, + "201": { + "description": "A new lease has been created. The \"acquire\" action was successful.", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string", + "format": "date-time-rfc1123" + }, + "ETag": { + "description": "An HTTP entity tag associated with the file or directory.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", + "format": "date-time-rfc1123", + "type": "string" + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-lease-id": { + "x-ms-client-name": "LeaseId", + "description": "A successful \"acquire\" action returns the lease ID.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + } + } + }, + "202": { + "description": "The \"break\" lease action was successful.", + "headers": { + "ETag": { + "description": "An HTTP entity tag associated with the file or directory.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", + "format": "date-time-rfc1123", + "type": "string" + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-lease-time": { + "x-ms-client-name": "LeaseTime", + "description": "The time remaining in the lease period in seconds.", + "type": "string" + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageError" + } + } + } + }, + "get": { + "operationId": "Path_Read", + "summary": "Read File", + "description": "Read the contents of a file. For read operations, range requests are supported. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", + "tags": ["File and Directory Operations"], + "parameters": [ + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "name": "x-ms-range-get-content-md5", + "in": "header", + "description": "Optional. When this header is set to \"true\" and specified together with the Range header, the service returns the MD5 hash for the range, as long as the range is less than or equal to 4MB in size. If this header is specified without the Range header, the service returns status code 400 (Bad Request). If this header is set to true when the range exceeds 4 MB in size, the service returns status code 400 (Bad Request).", + "required": false, + "type": "boolean" + }, + { + "$ref": "#/parameters/IfMatch" + }, + { + "$ref": "#/parameters/IfNoneMatch" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/EncryptionKey" + }, + { + "$ref": "#/parameters/EncryptionKeySha256" + }, + { + "$ref": "#/parameters/EncryptionAlgorithm" + }, + { + "$ref": "#/parameters/Snapshot" + }, + { + "$ref": "#/parameters/VersionId" + }, + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/Range" + }, + { + "$ref": "#/parameters/GetRangeContentMD5" + }, + { + "$ref": "#/parameters/GetRangeContentCRC64" + }, + { + "$ref": "#/parameters/IfTags" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "200": { + "description": "Ok", + "headers": { + "Accept-Ranges": { + "description": "Indicates that the service supports requests for partial file content.", + "type": "string" + }, + "Cache-Control": { + "description": "If the Cache-Control request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Disposition": { + "description": "If the Content-Disposition request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Encoding": { + "description": "If the Content-Encoding request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Language": { + "description": "If the Content-Language request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Length": { + "description": "The size of the resource in bytes.", + "type": "integer", + "format": "int64" + }, + "Content-Range": { + "description": "Indicates the range of bytes returned in the event that the client requested a subset of the file by setting the Range request header.", + "type": "string" + }, + "Content-Type": { + "description": "The content type specified for the resource. If no content type was specified, the default content type is application/octet-stream.", + "type": "string" + }, + "Content-MD5": { + "description": "The MD5 hash of complete file. If the file has an MD5 hash and this read operation is to read the complete file, this response header is returned so that the client can check for message content integrity.", + "type": "string", + "format": "byte" + }, + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string", + "format": "date-time-rfc1123" + }, + "ETag": { + "description": "An HTTP entity tag associated with the file or directory.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", + "format": "date-time-rfc1123", + "type": "string" + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-resource-type": { + "x-ms-client-name": "ResourceType", + "description": "The type of the resource. The value may be \"file\" or \"directory\". If not set, the value is \"file\".", + "type": "string" + }, + "x-ms-properties": { + "x-ms-client-name": "Properties", + "description": "The user-defined properties associated with the file or directory, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set.", + "type": "string" + }, + "x-ms-lease-duration": { + "x-ms-client-name": "LeaseDuration", + "description": "When a resource is leased, specifies whether the lease is of infinite or fixed duration.", + "type": "string", + "enum": ["infinite", "fixed"], + "x-ms-enum": { + "name": "LeaseDurationType", + "modelAsString": false + } + }, + "x-ms-lease-state": { + "x-ms-client-name": "LeaseState", + "description": "Lease state of the resource.", + "type": "string", + "enum": [ + "available", + "leased", + "expired", + "breaking", + "broken" + ], + "x-ms-enum": { + "name": "LeaseStateType", + "modelAsString": false + } + }, + "x-ms-lease-status": { + "x-ms-client-name": "LeaseStatus", + "description": "The lease status of the resource.", + "type": "string", + "enum": ["locked", "unlocked"], + "x-ms-enum": { + "name": "LeaseStatusType", + "modelAsString": false + } + }, + "x-ms-request-server-encrypted": { + "x-ms-client-name": "IsServerEncrypted", + "type": "boolean", + "description": "The value of this header is set to true if the contents of the request are successfully encrypted using the specified algorithm, and false otherwise." + }, + "x-ms-encryption-key-sha256": { + "x-ms-client-name": "EncryptionKeySha256", + "type": "string", + "description": "The SHA-256 hash of the encryption key used to encrypt the blob. This header is only returned when the blob was encrypted with a customer-provided key." + }, + "x-ms-meta": { + "type": "string", + "x-ms-client-name": "Metadata", + "x-ms-header-collection-prefix": "x-ms-meta-" + }, + "x-ms-creation-time": { + "x-ms-client-name": "CreationTime", + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the blob was created." + }, + "x-ms-or-policy-id": { + "x-ms-client-name": "ObjectReplicationPolicyId", + "type": "string", + "description": "Optional. Only valid when Object Replication is enabled for the storage container and on the destination blob of the replication." + }, + "x-ms-or": { + "type": "string", + "x-ms-client-name": "ObjectReplicationRules", + "x-ms-header-collection-prefix": "x-ms-or-", + "description": "Optional. Only valid when Object Replication is enabled for the storage container and on the source blob of the replication. When retrieving this header, it will return the header with the policy id and rule id (e.g. x-ms-or-policyid_ruleid), and the value will be the status of the replication (e.g. complete, failed)." + }, + "x-ms-blob-sequence-number": { + "x-ms-client-name": "BlobSequenceNumber", + "type": "integer", + "format": "int64", + "description": "The current sequence number for a page blob. This header is not returned for block blobs or append blobs" + }, + "x-ms-blob-type": { + "x-ms-client-name": "BlobType", + "description": "The blob's type.", + "type": "string", + "enum": ["BlockBlob", "PageBlob", "AppendBlob"], + "x-ms-enum": { + "name": "BlobType", + "modelAsString": false + } + }, + "x-ms-copy-completion-time": { + "x-ms-client-name": "CopyCompletionTime", + "type": "string", + "format": "date-time-rfc1123", + "description": "Conclusion time of the last attempted Copy Blob operation where this blob was the destination blob. This value can specify the time of a completed, aborted, or failed copy attempt. This header does not appear if a copy is pending, if this blob has never been the destination in a Copy Blob operation, or if this blob has been modified after a concluded Copy Blob operation using Set Blob Properties, Put Blob, or Put Block List." + }, + "x-ms-copy-status-description": { + "x-ms-client-name": "CopyStatusDescription", + "type": "string", + "description": "Only appears when x-ms-copy-status is failed or pending. Describes the cause of the last fatal or non-fatal copy operation failure. This header does not appear if this blob has never been the destination in a Copy Blob operation, or if this blob has been modified after a concluded Copy Blob operation using Set Blob Properties, Put Blob, or Put Block List" + }, + "x-ms-copy-id": { + "x-ms-client-name": "CopyId", + "type": "string", + "description": "String identifier for this copy operation. Use with Get Blob Properties to check the status of this copy operation, or pass to Abort Copy Blob to abort a pending copy." + }, + "x-ms-copy-progress": { + "x-ms-client-name": "CopyProgress", + "type": "string", + "description": "Contains the number of bytes copied and the total bytes in the source in the last attempted Copy Blob operation where this blob was the destination blob. Can show between 0 and Content-Length bytes copied. This header does not appear if this blob has never been the destination in a Copy Blob operation, or if this blob has been modified after a concluded Copy Blob operation using Set Blob Properties, Put Blob, or Put Block List" + }, + "x-ms-copy-source": { + "x-ms-client-name": "CopySource", + "type": "string", + "description": "URL up to 2 KB in length that specifies the source blob or file used in the last attempted Copy Blob operation where this blob was the destination blob. This header does not appear if this blob has never been the destination in a Copy Blob operation, or if this blob has been modified after a concluded Copy Blob operation using Set Blob Properties, Put Blob, or Put Block List." + }, + "x-ms-copy-status": { + "x-ms-client-name": "CopyStatus", + "description": "State of the copy operation identified by x-ms-copy-id.", + "type": "string", + "enum": ["pending", "success", "aborted", "failed"], + "x-ms-enum": { + "name": "CopyStatusType", + "modelAsString": false + } + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-version-id": { + "x-ms-client-name": "VersionId", + "type": "string", + "description": "A DateTime value returned by the service that uniquely identifies the blob. The value of this header indicates the blob version, and may be used in subsequent requests to access this version of the blob." + }, + "x-ms-is-current-version": { + "x-ms-client-name": "IsCurrentVersion", + "type": "boolean", + "description": "The value of this header indicates whether version of this blob is a current version, see also x-ms-version-id header." + }, + "x-ms-blob-committed-block-count": { + "x-ms-client-name": "BlobCommittedBlockCount", + "type": "integer", + "description": "The number of committed blocks present in the blob. This header is returned only for append blobs." + }, + "x-ms-server-encrypted": { + "x-ms-client-name": "IsServerEncrypted", + "type": "boolean", + "description": "The value of this header is set to true if the blob data and application metadata are completely encrypted using the specified algorithm. Otherwise, the value is set to false (when the blob is unencrypted, or if only parts of the blob/application metadata are encrypted)." + }, + "x-ms-encryption-scope": { + "x-ms-client-name": "EncryptionScope", + "type": "string", + "description": "Returns the name of the encryption scope used to encrypt the blob contents and application metadata. Note that the absence of this header implies use of the default account encryption scope." + }, + "x-ms-blob-content-md5": { + "x-ms-client-name": "BlobContentMD5", + "type": "string", + "format": "byte", + "description": "If the blob has a MD5 hash, and if request contains range header (Range or x-ms-range), this response header is returned with the value of the whole blob's MD5 value. This value may or may not be equal to the value returned in Content-MD5 header, with the latter calculated from the requested range" + }, + "x-ms-tag-count": { + "x-ms-client-name": "TagCount", + "type": "integer", + "format": "int64", + "description": "The number of tags associated with the blob" + }, + "x-ms-blob-sealed": { + "x-ms-client-name": "IsSealed", + "type": "boolean", + "description": "If this blob has been sealed" + }, + "x-ms-last-access-time": { + "x-ms-client-name": "LastAccessed", + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the blob was last read or written to" + }, + "x-ms-immutability-policy-until-date": { + "x-ms-client-name": "ImmutabilityPolicyExpiresOn", + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the blob immutability policy will expire." + }, + "x-ms-immutability-policy-mode": { + "x-ms-client-name": "ImmutabilityPolicyMode", + "type": "string", + "enum": ["Mutable", "Unlocked", "Locked"], + "x-ms-enum": { + "name": "BlobImmutabilityPolicyMode", + "modelAsString": false + }, + "description": "Indicates immutability policy mode." + }, + "x-ms-legal-hold": { + "x-ms-client-name": "LegalHold", + "type": "boolean", + "description": "Indicates if a legal hold is present on the blob." + } + }, + "schema": { + "type": "object", + "format": "file" + } + }, + "206": { + "description": "Partial content", + "headers": { + "Accept-Ranges": { + "description": "Indicates that the service supports requests for partial file content.", + "type": "string" + }, + "Cache-Control": { + "description": "If the Cache-Control request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Disposition": { + "description": "If the Content-Disposition request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Encoding": { + "description": "If the Content-Encoding request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Language": { + "description": "If the Content-Language request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Length": { + "description": "The size of the resource in bytes.", + "type": "integer", + "format": "int64" + }, + "Content-Range": { + "description": "Indicates the range of bytes returned in the event that the client requested a subset of the file by setting the Range request header.", + "type": "string" + }, + "Content-Type": { + "description": "The content type specified for the resource. If no content type was specified, the default content type is application/octet-stream.", + "type": "string" + }, + "Content-MD5": { + "description": "The MD5 hash of read range. If the request is to read a specified range and the \"x-ms-range-get-content-md5\" is set to true, then the request returns an MD5 hash for the range, as long as the range size is less than or equal to 4 MB.", + "type": "string", + "format": "byte" + }, + "x-ms-content-md5": { + "description": "The MD5 hash of complete file stored in storage. If the file has a MD5 hash, and if request contains range header (Range or x-ms-range), this response header is returned with the value of the complete file's MD5 value. This value may or may not be equal to the value returned in Content-MD5 header, with the latter calculated from the requested range.", + "type": "string", + "format": "byte" + }, + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string", + "format": "date-time-rfc1123" + }, + "ETag": { + "description": "An HTTP entity tag associated with the file or directory.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", + "format": "date-time-rfc1123", + "type": "string" + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-resource-type": { + "x-ms-client-name": "ResourceType", + "description": "The type of the resource. The value may be \"file\" or \"directory\". If not set, the value is \"file\".", + "type": "string" + }, + "x-ms-properties": { + "x-ms-client-name": "Properties", + "description": "The user-defined properties associated with the file or directory, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set.", + "type": "string" + }, + "x-ms-lease-duration": { + "x-ms-client-name": "LeaseDuration", + "description": "When a resource is leased, specifies whether the lease is of infinite or fixed duration.", + "type": "string", + "enum": ["infinite", "fixed"], + "x-ms-enum": { + "name": "LeaseDurationType", + "modelAsString": false + } + }, + "x-ms-lease-state": { + "x-ms-client-name": "LeaseState", + "description": "Lease state of the resource. ", + "type": "string", + "enum": [ + "available", + "leased", + "expired", + "breaking", + "broken" + ], + "x-ms-enum": { + "name": "LeaseStateType", + "modelAsString": false + } + }, + "x-ms-lease-status": { + "x-ms-client-name": "LeaseStatus", + "description": "The lease status of the resource.", + "type": "string", + "enum": ["locked", "unlocked"], + "x-ms-enum": { + "name": "LeaseStatusType", + "modelAsString": false + } + }, + "x-ms-request-server-encrypted": { + "x-ms-client-name": "IsServerEncrypted", + "type": "boolean", + "description": "The value of this header is set to true if the contents of the request are successfully encrypted using the specified algorithm, and false otherwise." + }, + "x-ms-encryption-key-sha256": { + "x-ms-client-name": "EncryptionKeySha256", + "type": "string", + "description": "The SHA-256 hash of the encryption key used to encrypt the blob. This header is only returned when the blob was encrypted with a customer-provided key." + }, + "x-ms-meta": { + "type": "string", + "x-ms-client-name": "Metadata", + "x-ms-header-collection-prefix": "x-ms-meta-" + }, + "x-ms-or-policy-id": { + "x-ms-client-name": "ObjectReplicationPolicyId", + "type": "string", + "description": "Optional. Only valid when Object Replication is enabled for the storage container and on the destination blob of the replication." + }, + "x-ms-or": { + "type": "string", + "x-ms-client-name": "ObjectReplicationRules", + "x-ms-header-collection-prefix": "x-ms-or-", + "description": "Optional. Only valid when Object Replication is enabled for the storage container and on the source blob of the replication. When retrieving this header, it will return the header with the policy id and rule id (e.g. x-ms-or-policyid_ruleid), and the value will be the status of the replication (e.g. complete, failed)." + }, + "x-ms-blob-sequence-number": { + "x-ms-client-name": "BlobSequenceNumber", + "type": "integer", + "format": "int64", + "description": "The current sequence number for a page blob. This header is not returned for block blobs or append blobs" + }, + "x-ms-blob-type": { + "x-ms-client-name": "BlobType", + "description": "The blob's type.", + "type": "string", + "enum": ["BlockBlob", "PageBlob", "AppendBlob"], + "x-ms-enum": { + "name": "BlobType", + "modelAsString": false + } + }, + "x-ms-content-crc64": { + "x-ms-client-name": "ContentCrc64", + "type": "string", + "format": "byte", + "description": "If the request is to read a specified range and the x-ms-range-get-content-crc64 is set to true, then the request returns a crc64 for the range, as long as the range size is less than or equal to 4 MB. If both x-ms-range-get-content-crc64 & x-ms-range-get-content-md5 is specified in the same request, it will fail with 400(Bad Request)" + }, + "x-ms-copy-completion-time": { + "x-ms-client-name": "CopyCompletionTime", + "type": "string", + "format": "date-time-rfc1123", + "description": "Conclusion time of the last attempted Copy Blob operation where this blob was the destination blob. This value can specify the time of a completed, aborted, or failed copy attempt. This header does not appear if a copy is pending, if this blob has never been the destination in a Copy Blob operation, or if this blob has been modified after a concluded Copy Blob operation using Set Blob Properties, Put Blob, or Put Block List." + }, + "x-ms-copy-status-description": { + "x-ms-client-name": "CopyStatusDescription", + "type": "string", + "description": "Only appears when x-ms-copy-status is failed or pending. Describes the cause of the last fatal or non-fatal copy operation failure. This header does not appear if this blob has never been the destination in a Copy Blob operation, or if this blob has been modified after a concluded Copy Blob operation using Set Blob Properties, Put Blob, or Put Block List" + }, + "x-ms-copy-id": { + "x-ms-client-name": "CopyId", + "type": "string", + "description": "String identifier for this copy operation. Use with Get Blob Properties to check the status of this copy operation, or pass to Abort Copy Blob to abort a pending copy." + }, + "x-ms-copy-progress": { + "x-ms-client-name": "CopyProgress", + "type": "string", + "description": "Contains the number of bytes copied and the total bytes in the source in the last attempted Copy Blob operation where this blob was the destination blob. Can show between 0 and Content-Length bytes copied. This header does not appear if this blob has never been the destination in a Copy Blob operation, or if this blob has been modified after a concluded Copy Blob operation using Set Blob Properties, Put Blob, or Put Block List" + }, + "x-ms-copy-source": { + "x-ms-client-name": "CopySource", + "type": "string", + "description": "URL up to 2 KB in length that specifies the source blob or file used in the last attempted Copy Blob operation where this blob was the destination blob. This header does not appear if this blob has never been the destination in a Copy Blob operation, or if this blob has been modified after a concluded Copy Blob operation using Set Blob Properties, Put Blob, or Put Block List." + }, + "x-ms-copy-status": { + "x-ms-client-name": "CopyStatus", + "description": "State of the copy operation identified by x-ms-copy-id.", + "type": "string", + "enum": ["pending", "success", "aborted", "failed"], + "x-ms-enum": { + "name": "CopyStatusType", + "modelAsString": false + } + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-version-id": { + "x-ms-client-name": "VersionId", + "type": "string", + "description": "A DateTime value returned by the service that uniquely identifies the blob. The value of this header indicates the blob version, and may be used in subsequent requests to access this version of the blob." + }, + "x-ms-is-current-version": { + "x-ms-client-name": "IsCurrentVersion", + "type": "boolean", + "description": "The value of this header indicates whether version of this blob is a current version, see also x-ms-version-id header." + }, + "x-ms-blob-committed-block-count": { + "x-ms-client-name": "BlobCommittedBlockCount", + "type": "integer", + "description": "The number of committed blocks present in the blob. This header is returned only for append blobs." + }, + "x-ms-server-encrypted": { + "x-ms-client-name": "IsServerEncrypted", + "type": "boolean", + "description": "The value of this header is set to true if the blob data and application metadata are completely encrypted using the specified algorithm. Otherwise, the value is set to false (when the blob is unencrypted, or if only parts of the blob/application metadata are encrypted)." + }, + "x-ms-encryption-scope": { + "x-ms-client-name": "EncryptionScope", + "type": "string", + "description": "Returns the name of the encryption scope used to encrypt the blob contents and application metadata. Note that the absence of this header implies use of the default account encryption scope." + }, + "x-ms-blob-content-md5": { + "x-ms-client-name": "BlobContentMD5", + "type": "string", + "format": "byte", + "description": "If the blob has a MD5 hash, and if request contains range header (Range or x-ms-range), this response header is returned with the value of the whole blob's MD5 value. This value may or may not be equal to the value returned in Content-MD5 header, with the latter calculated from the requested range" + }, + "x-ms-tag-count": { + "x-ms-client-name": "TagCount", + "type": "integer", + "format": "int64", + "description": "The number of tags associated with the blob" + }, + "x-ms-blob-sealed": { + "x-ms-client-name": "IsSealed", + "type": "boolean", + "description": "If this blob has been sealed" + }, + "x-ms-last-access-time": { + "x-ms-client-name": "LastAccessed", + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the blob was last read or written to" + }, + "x-ms-immutability-policy-until-date": { + "x-ms-client-name": "ImmutabilityPolicyExpiresOn", + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the blob immutability policy will expire." + }, + "x-ms-immutability-policy-mode": { + "x-ms-client-name": "ImmutabilityPolicyMode", + "type": "string", + "description": "Indicates immutability policy mode." + }, + "x-ms-legal-hold": { + "x-ms-client-name": "LegalHold", + "type": "boolean", + "description": "Indicates if a legal hold is present on the blob." + } + }, + "schema": { + "type": "object", + "format": "file" + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageError" + } + } + } + }, + "head": { + "operationId": "Path_GetProperties", + "summary": "Get Properties | Get Status | Get Access Control List", + "description": "Get Properties returns all system and user defined properties for a path. Get Status returns all system defined properties for a path. Get Access Control List returns the access control list for a path. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", + "tags": ["File and Directory Operations"], + "parameters": [ + { + "name": "action", + "in": "query", + "description": "Optional. If the value is \"getStatus\" only the system defined properties for the path are returned. If the value is \"getAccessControl\" the access control list is returned in the response headers (Hierarchical Namespace must be enabled for the account), otherwise the properties are returned.", + "required": false, + "type": "string", + "enum": ["getAccessControl", "getStatus"], + "x-ms-enum": { + "name": "PathGetPropertiesAction", + "modelAsString": false + } + }, + { + "$ref": "#/parameters/Upn" + }, + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "$ref": "#/parameters/IfMatch" + }, + { + "$ref": "#/parameters/IfNoneMatch" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/Snapshot" + }, + { + "$ref": "#/parameters/VersionId" + }, + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/EncryptionKey" + }, + { + "$ref": "#/parameters/EncryptionKeySha256" + }, + { + "$ref": "#/parameters/EncryptionAlgorithm" + }, + { + "$ref": "#/parameters/IfTags" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "200": { + "description": "Returns all properties for the file or directory.", + "headers": { + "Accept-Ranges": { + "description": "Indicates that the service supports requests for partial file content.", + "type": "string" + }, + "Cache-Control": { + "description": "If the Cache-Control request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Disposition": { + "description": "If the Content-Disposition request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Encoding": { + "description": "If the Content-Encoding request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Language": { + "description": "If the Content-Language request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Length": { + "description": "The size of the resource in bytes.", + "type": "integer", + "format": "int64" + }, + "Content-Range": { + "description": "Indicates the range of bytes returned in the event that the client requested a subset of the file by setting the Range request header.", + "type": "string" + }, + "Content-Type": { + "description": "The content type specified for the resource. If no content type was specified, the default content type is application/octet-stream.", + "type": "string" + }, + "Content-MD5": { + "description": "The MD5 hash of complete file stored in storage. This header is returned only for \"GetProperties\" operation. If the Content-MD5 header has been set for the file, this response header is returned for GetProperties call so that the client can check for message content integrity.", + "type": "string", + "format": "byte" + }, + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string", + "format": "date-time-rfc1123" + }, + "ETag": { + "description": "An HTTP entity tag associated with the file or directory.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", + "format": "date-time-rfc1123", + "type": "string" + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-resource-type": { + "x-ms-client-name": "ResourceType", + "description": "The type of the resource. The value may be \"file\" or \"directory\". If not set, the value is \"file\".", + "type": "string" + }, + "x-ms-properties": { + "x-ms-client-name": "Properties", + "description": "The user-defined properties associated with the file or directory, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set.", + "type": "string" + }, + "x-ms-owner": { + "x-ms-client-name": "Owner", + "description": "The owner of the file or directory. Included in the response if Hierarchical Namespace is enabled for the account.", + "type": "string" + }, + "x-ms-group": { + "x-ms-client-name": "Group", + "description": "The owning group of the file or directory. Included in the response if Hierarchical Namespace is enabled for the account.", + "type": "string" + }, + "x-ms-permissions": { + "x-ms-client-name": "Permissions", + "description": "The POSIX access permissions for the file owner, the file owning group, and others. Included in the response if Hierarchical Namespace is enabled for the account.", + "type": "string" + }, + "x-ms-acl": { + "x-ms-client-name": "ACL", + "description": "The POSIX access control list for the file or directory. Included in the response only if the action is \"getAccessControl\" and Hierarchical Namespace is enabled for the account.", + "type": "string" + }, + "x-ms-lease-duration": { + "x-ms-client-name": "LeaseDuration", + "description": "When a resource is leased, specifies whether the lease is of infinite or fixed duration.", + "type": "string", + "enum": ["infinite", "fixed"], + "x-ms-enum": { + "name": "LeaseDurationType", + "modelAsString": false + } + }, + "x-ms-lease-state": { + "x-ms-client-name": "LeaseState", + "description": "Lease state of the resource.", + "type": "string", + "enum": [ + "available", + "leased", + "expired", + "breaking", + "broken" + ], + "x-ms-enum": { + "name": "LeaseStateType", + "modelAsString": false + } + }, + "x-ms-lease-status": { + "x-ms-client-name": "LeaseStatus", + "description": "The lease status of the resource.", + "type": "string", + "enum": ["locked", "unlocked"], + "x-ms-enum": { + "name": "LeaseStatusType", + "modelAsString": false + } + }, + "x-ms-meta": { + "type": "string", + "x-ms-client-name": "Metadata", + "x-ms-header-collection-prefix": "x-ms-meta-" + }, + "x-ms-creation-time": { + "x-ms-client-name": "CreationTime", + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the blob was created." + }, + "x-ms-or-policy-id": { + "x-ms-client-name": "ObjectReplicationPolicyId", + "type": "string", + "description": "Optional. Only valid when Object Replication is enabled for the storage container and on the destination blob of the replication." + }, + "x-ms-or": { + "type": "string", + "x-ms-client-name": "ObjectReplicationRules", + "x-ms-header-collection-prefix": "x-ms-or-", + "description": "Optional. Only valid when Object Replication is enabled for the storage container and on the source blob of the replication. When retrieving this header, it will return the header with the policy id and rule id (e.g. x-ms-or-policyid_ruleid), and the value will be the status of the replication (e.g. complete, failed)." + }, + "x-ms-blob-type": { + "x-ms-client-name": "BlobType", + "description": "The blob's type.", + "type": "string", + "enum": ["BlockBlob", "PageBlob", "AppendBlob"], + "x-ms-enum": { + "name": "BlobType", + "modelAsString": false + } + }, + "x-ms-copy-completion-time": { + "x-ms-client-name": "CopyCompletionTime", + "type": "string", + "format": "date-time-rfc1123", + "description": "Conclusion time of the last attempted Copy Blob operation where this blob was the destination blob. This value can specify the time of a completed, aborted, or failed copy attempt. This header does not appear if a copy is pending, if this blob has never been the destination in a Copy Blob operation, or if this blob has been modified after a concluded Copy Blob operation using Set Blob Properties, Put Blob, or Put Block List." + }, + "x-ms-copy-status-description": { + "x-ms-client-name": "CopyStatusDescription", + "type": "string", + "description": "Only appears when x-ms-copy-status is failed or pending. Describes the cause of the last fatal or non-fatal copy operation failure. This header does not appear if this blob has never been the destination in a Copy Blob operation, or if this blob has been modified after a concluded Copy Blob operation using Set Blob Properties, Put Blob, or Put Block List" + }, + "x-ms-copy-id": { + "x-ms-client-name": "CopyId", + "type": "string", + "description": "String identifier for this copy operation. Use with Get Blob Properties to check the status of this copy operation, or pass to Abort Copy Blob to abort a pending copy." + }, + "x-ms-copy-progress": { + "x-ms-client-name": "CopyProgress", + "type": "string", + "description": "Contains the number of bytes copied and the total bytes in the source in the last attempted Copy Blob operation where this blob was the destination blob. Can show between 0 and Content-Length bytes copied. This header does not appear if this blob has never been the destination in a Copy Blob operation, or if this blob has been modified after a concluded Copy Blob operation using Set Blob Properties, Put Blob, or Put Block List" + }, + "x-ms-copy-source": { + "x-ms-client-name": "CopySource", + "type": "string", + "description": "URL up to 2 KB in length that specifies the source blob or file used in the last attempted Copy Blob operation where this blob was the destination blob. This header does not appear if this blob has never been the destination in a Copy Blob operation, or if this blob has been modified after a concluded Copy Blob operation using Set Blob Properties, Put Blob, or Put Block List." + }, + "x-ms-copy-status": { + "x-ms-client-name": "CopyStatus", + "description": "State of the copy operation identified by x-ms-copy-id.", + "type": "string", + "enum": ["pending", "success", "aborted", "failed"], + "x-ms-enum": { + "name": "CopyStatusType", + "modelAsString": false + } + }, + "x-ms-incremental-copy": { + "x-ms-client-name": "IsIncrementalCopy", + "type": "boolean", + "description": "Included if the blob is incremental copy blob." + }, + "x-ms-copy-destination-snapshot": { + "x-ms-client-name": "DestinationSnapshot", + "type": "string", + "description": "Included if the blob is incremental copy blob or incremental copy snapshot, if x-ms-copy-status is success. Snapshot time of the last successful incremental copy snapshot for this blob." + }, + "x-ms-blob-sequence-number": { + "x-ms-client-name": "BlobSequenceNumber", + "type": "integer", + "format": "int64", + "description": "The current sequence number for a page blob. This header is not returned for block blobs or append blobs" + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-blob-committed-block-count": { + "x-ms-client-name": "BlobCommittedBlockCount", + "type": "integer", + "description": "The number of committed blocks present in the blob. This header is returned only for append blobs." + }, + "x-ms-server-encrypted": { + "x-ms-client-name": "IsServerEncrypted", + "type": "boolean", + "description": "The value of this header is set to true if the blob data and application metadata are completely encrypted using the specified algorithm. Otherwise, the value is set to false (when the blob is unencrypted, or if only parts of the blob/application metadata are encrypted)." + }, + "x-ms-encryption-key-sha256": { + "x-ms-client-name": "EncryptionKeySha256", + "type": "string", + "description": "The SHA-256 hash of the encryption key used to encrypt the metadata. This header is only returned when the metadata was encrypted with a customer-provided key." + }, + "x-ms-encryption-scope": { + "x-ms-client-name": "EncryptionScope", + "type": "string", + "description": "Returns the name of the encryption scope used to encrypt the blob contents and application metadata. Note that the absence of this header implies use of the default account encryption scope." + }, + "x-ms-access-tier": { + "x-ms-client-name": "AccessTier", + "type": "string", + "description": "The tier of page blob on a premium storage account or tier of block blob on blob storage LRS accounts. For a list of allowed premium page blob tiers, see https://docs.microsoft.com/en-us/azure/virtual-machines/windows/premium-storage#features. For blob storage LRS accounts, valid values are Hot/Cool/Archive." + }, + "x-ms-access-tier-inferred": { + "x-ms-client-name": "AccessTierInferred", + "type": "boolean", + "description": "For page blobs on a premium storage account only. If the access tier is not explicitly set on the blob, the tier is inferred based on its content length and this header will be returned with true value." + }, + "x-ms-archive-status": { + "x-ms-client-name": "ArchiveStatus", + "type": "string", + "description": "For blob storage LRS accounts, valid values are rehydrate-pending-to-hot/rehydrate-pending-to-cool. If the blob is being rehydrated and is not complete then this header is returned indicating that rehydrate is pending and also tells the destination tier." + }, + "x-ms-access-tier-change-time": { + "x-ms-client-name": "AccessTierChangeTime", + "type": "string", + "format": "date-time-rfc1123", + "description": "The time the tier was changed on the object. This is only returned if the tier on the block blob was ever set." + }, + "x-ms-version-id": { + "x-ms-client-name": "VersionId", + "type": "string", + "description": "A DateTime value returned by the service that uniquely identifies the blob. The value of this header indicates the blob version, and may be used in subsequent requests to access this version of the blob." + }, + "x-ms-is-current-version": { + "x-ms-client-name": "IsCurrentVersion", + "type": "boolean", + "description": "The value of this header indicates whether version of this blob is a current version, see also x-ms-version-id header." + }, + "x-ms-tag-count": { + "x-ms-client-name": "TagCount", + "type": "integer", + "format": "int64", + "description": "The number of tags associated with the blob" + }, + "x-ms-expiry-time": { + "x-ms-client-name": "ExpiresOn", + "type": "string", + "format": "date-time-rfc1123", + "description": "The time this blob will expire." + }, + "x-ms-blob-sealed": { + "x-ms-client-name": "IsSealed", + "type": "boolean", + "description": "If this blob has been sealed" + }, + "x-ms-rehydrate-priority": { + "x-ms-client-name": "RehydratePriority", + "description": "If an object is in rehydrate pending state then this header is returned with priority of rehydrate. Valid values are High and Standard.", + "type": "string" + }, + "x-ms-last-access-time": { + "x-ms-client-name": "LastAccessed", + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the blob was last read or written to" + }, + "x-ms-immutability-policy-until-date": { + "x-ms-client-name": "ImmutabilityPolicyExpiresOn", + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the blob immutability policy will expire." + }, + "x-ms-immutability-policy-mode": { + "x-ms-client-name": "ImmutabilityPolicyMode", + "type": "string", + "enum": ["Mutable", "Unlocked", "Locked"], + "x-ms-enum": { + "name": "BlobImmutabilityPolicyMode", + "modelAsString": false + }, + "description": "Indicates immutability policy mode." + }, + "x-ms-legal-hold": { + "x-ms-client-name": "LegalHold", + "type": "boolean", + "description": "Indicates if a legal hold is present on the blob." + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageError" + } + } + } + }, + "delete": { + "operationId": "Path_Delete", + "summary": "Delete File | Delete Directory", + "description": "Delete the file or directory. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", + "tags": ["File and Directory Operations", "blob"], + "parameters": [ + { + "$ref": "#/parameters/RecursiveOptional" + }, + { + "$ref": "#/parameters/Continuation" + }, + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "$ref": "#/parameters/IfMatch" + }, + { + "$ref": "#/parameters/IfNoneMatch" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/Snapshot" + }, + { + "$ref": "#/parameters/VersionId" + }, + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/DeleteSnapshots" + }, + { + "$ref": "#/parameters/IfTags" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + }, + { + "$ref": "#/parameters/BlobDeleteType" + } + ], + "responses": { + "200": { + "description": "The file was deleted.", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string", + "format": "date-time-rfc1123" + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-continuation": { + "x-ms-client-name": "Continuation", + "description": "When deleting a directory, the number of paths that are deleted with each invocation is limited. If the number of paths to be deleted exceeds this limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the delete operation to continue deleting the directory.", + "type": "string" + }, + "x-ms-deletion-id": { + "x-ms-client-name": "DeletionId", + "description": "Returned only for hierarchical namespace space enabled accounts when soft delete is enabled. A unique identifier for the entity that can be used to restore it. See the Undelete REST API for more information.", + "type": "string" + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + } + } + }, + "202": { + "description": "The file was deleted.", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string", + "format": "date-time-rfc1123" + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-continuation": { + "x-ms-client-name": "Continuation", + "description": "When deleting a directory, the number of paths that are deleted with each invocation is limited. If the number of paths to be deleted exceeds this limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the delete operation to continue deleting the directory.", + "type": "string" + }, + "x-ms-deletion-id": { + "x-ms-client-name": "DeletionId", + "description": "Returned only for hierarchical namespace space enabled accounts when soft delete is enabled. A unique identifier for the entity that can be used to restore it. See the Undelete REST API for more information.", + "type": "string" + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageError" + } + } + } + }, + "parameters": [ + { + "$ref": "#/parameters/FileSystem" + }, + { + "$ref": "#/parameters/Path" + }, + { + "$ref": "#/parameters/ClientRequestId" + }, + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/Metadata" + } + ] + }, + "/{filesystem}/{path}?action=setAccessControl": { + "patch": { + "tags": ["directory"], + "operationId": "Path_SetAccessControl", + "description": "Set the owner, group, permissions, or access control list for a path.", + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "$ref": "#/parameters/Owner" + }, + { + "$ref": "#/parameters/Group" + }, + { + "$ref": "#/parameters/Permissions" + }, + { + "$ref": "#/parameters/Acl" + }, + { + "$ref": "#/parameters/IfMatch" + }, + { + "$ref": "#/parameters/IfNoneMatch" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/ClientRequestId" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + } + ], + "responses": { + "200": { + "description": "Set directory access control response.", + "headers": { + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated." + }, + "ETag": { + "type": "string", + "description": "An HTTP entity tag associated with the file or directory." + }, + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time." + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "The version of the REST protocol used to process the request." + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "The version of the REST protocol used to process the request." + } + }, + "schema": { + "$ref": "#/definitions/StorageError" + } + } + } + }, + "parameters": [ + { + "$ref": "#/parameters/FileSystem" + }, + { + "$ref": "#/parameters/Path" + }, + { + "name": "action", + "description": "action", + "in": "query", + "required": true, + "type": "string", + "enum": ["setAccessControl"] + } + ] + }, + "/{filesystem}/{path}?action=setAccessControlRecursive": { + "patch": { + "tags": ["directory"], + "operationId": "Path_SetAccessControlRecursive", + "description": "Set the access control list for a path and sub-paths.", + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/Continuation" + }, + { + "$ref": "#/parameters/PathSetAccessControlRecursiveMode" + }, + { + "$ref": "#/parameters/ForceFlag" + }, + { + "name": "maxRecords", + "in": "query", + "description": "Optional. It specifies the maximum number of files or directories on which the acl change will be applied. If omitted or greater than 2,000, the request will process up to 2,000 items", + "format": "int32", + "minimum": 1, + "required": false, + "type": "integer" + }, + { + "$ref": "#/parameters/Acl" + }, + { + "$ref": "#/parameters/ClientRequestId" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + } + ], + "responses": { + "200": { + "description": "Set directory access control recursive response.", + "headers": { + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated." + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-continuation": { + "x-ms-client-name": "Continuation", + "description": "When performing setAccessControlRecursive on a directory, the number of paths that are processed with each invocation is limited. If the number of paths to be processed exceeds this limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the setAccessControlRecursive operation to continue the setAccessControlRecursive operation on the directory.", + "type": "string" + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "The version of the REST protocol used to process the request." + } + }, + "schema": { + "$ref": "#/definitions/SetAccessControlRecursiveResponse" + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "The version of the REST protocol used to process the request." + } + }, + "schema": { + "$ref": "#/definitions/StorageError" + } + } + } + }, + "parameters": [ + { + "$ref": "#/parameters/FileSystem" + }, + { + "$ref": "#/parameters/Path" + }, + { + "name": "action", + "description": "action", + "in": "query", + "required": true, + "type": "string", + "enum": ["setAccessControlRecursive"] + } + ] + }, + "/{filesystem}/{path}?action=setProperties": { + "patch": { + "operationId": "Path_SetProperties", + "summary": "Set Properties for file or directory", + "description": "Uploads data to be appended to a file, flushes (writes) previously uploaded data to a file, sets properties for a file or directory, or sets access control for a file or directory. Data can only be appended to a file. Concurrent writes to the same file using multiple clients are not supported. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", + "consumes": ["application/octet-stream"], + + "parameters": [ + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "$ref": "#/parameters/CacheControl" + }, + { + "$ref": "#/parameters/ContentType" + }, + { + "$ref": "#/parameters/ContentDisposition" + }, + { + "$ref": "#/parameters/ContentEncoding" + }, + { + "$ref": "#/parameters/ContentLanguage" + }, + { + "$ref": "#/parameters/ContentMD5" + }, + { + "$ref": "#/parameters/Properties" + }, + { + "$ref": "#/parameters/Permissions" + }, + { + "$ref": "#/parameters/IfMatch" + }, + { + "$ref": "#/parameters/IfNoneMatch" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/ClientRequestId" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + } + ], + "responses": { + "200": { + "description": "The properties were set successfully.", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string", + "format": "date-time-rfc1123" + }, + "ETag": { + "description": "An HTTP entity tag associated with the file or directory.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", + "format": "date-time-rfc1123", + "type": "string" + }, + "Cache-Control": { + "description": "If the Cache-Control request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Disposition": { + "description": "If the Content-Disposition request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Encoding": { + "description": "If the Content-Encoding request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Language": { + "description": "If the Content-Language request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Length": { + "description": "The size of the resource in bytes.", + "type": "integer", + "format": "int64" + }, + "Content-Type": { + "description": "The content type specified for the resource. If no content type was specified, the default content type is application/octet-stream.", + "type": "string" + }, + "Content-MD5": { + "description": "An MD5 hash of the request content. This header is only returned for \"Flush\" operation. This header is returned so that the client can check for message content integrity. This header refers to the content of the request, not actual file content.", + "type": "string", + "format": "byte" + }, + "x-ms-properties": { + "x-ms-client-name": "Properties", + "description": "User-defined properties associated with the file or directory, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set.", + "type": "string" + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "description": "The version of the REST protocol used to process the request.", + "type": "string" + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "The version of the REST protocol used to process the request." + } + }, + "schema": { + "$ref": "#/definitions/StorageError" + } + } + } + }, + "parameters": [ + { + "$ref": "#/parameters/FileSystem" + }, + { + "$ref": "#/parameters/Path" + }, + { + "name": "action", + "description": "action", + "in": "query", + "required": true, + "type": "string", + "enum": ["setProperties"] + } + ] + }, + "/{filesystem}/{path}?action=flush": { + "patch": { + "tags": ["directory"], + "operationId": "Path_FlushData", + "description": "Set the owner, group, permissions, or access control list for a path.", + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/Position" + }, + { + "$ref": "#/parameters/RetainUncommittedData" + }, + { + "$ref": "#/parameters/Close" + }, + { + "$ref": "#/parameters/ContentLength" + }, + { + "$ref": "#/parameters/ContentMD5" + }, + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "$ref": "#/parameters/CacheControl" + }, + { + "$ref": "#/parameters/ContentType" + }, + { + "$ref": "#/parameters/ContentDisposition" + }, + { + "$ref": "#/parameters/ContentEncoding" + }, + { + "$ref": "#/parameters/ContentLanguage" + }, + { + "$ref": "#/parameters/IfMatch" + }, + { + "$ref": "#/parameters/IfNoneMatch" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "$ref": "#/parameters/ClientRequestId" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/EncryptionKey" + }, + { + "$ref": "#/parameters/EncryptionKeySha256" + }, + { + "$ref": "#/parameters/EncryptionAlgorithm" + }, + { + "$ref": "#/parameters/ProposedLeaseIdOptional" + }, + { + "$ref": "#/parameters/LeaseDuration" + }, + { + "$ref": "#/parameters/LeaseAction" + } + ], + "responses": { + "200": { + "description": "The data was flushed (written) to the file successfully.", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string", + "format": "date-time-rfc1123" + }, + "ETag": { + "description": "An HTTP entity tag associated with the file or directory.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", + "format": "date-time-rfc1123", + "type": "string" + }, + "Content-Length": { + "description": "The size of the resource in bytes.", + "type": "integer", + "format": "int64" + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-request-server-encrypted": { + "x-ms-client-name": "IsServerEncrypted", + "type": "boolean", + "description": "The value of this header is set to true if the contents of the request are successfully encrypted using the specified algorithm, and false otherwise." + }, + "x-ms-encryption-key-sha256": { + "x-ms-client-name": "EncryptionKeySha256", + "type": "string", + "description": "The SHA-256 hash of the encryption key used to encrypt the blob. This header is only returned when the blob was encrypted with a customer-provided key." + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "The version of the REST protocol used to process the request." + } + }, + "schema": { + "$ref": "#/definitions/StorageError" + } + } + } + }, + "parameters": [ + { + "$ref": "#/parameters/FileSystem" + }, + { + "$ref": "#/parameters/Path" + }, + { + "name": "action", + "description": "action", + "in": "query", + "required": true, + "type": "string", + "enum": ["flush"] + } + ] + }, + "/{filesystem}/{path}?action=append": { + "patch": { + "tags": ["directory"], + "operationId": "Path_AppendData", + "description": "Append data to the file.", + "parameters": [ + { + "$ref": "#/parameters/Position" + }, + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/ContentLength" + }, + { + "$ref": "#/parameters/TransactionalContentMD5" + }, + { + "$ref": "#/parameters/ContentCrc64" + }, + { + "$ref": "#/parameters/LeaseIdOptional" + }, + { + "$ref": "#/parameters/Body" + }, + { + "$ref": "#/parameters/ClientRequestId" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/EncryptionKey" + }, + { + "$ref": "#/parameters/EncryptionKeySha256" + }, + { + "$ref": "#/parameters/EncryptionAlgorithm" + }, + { + "$ref": "#/parameters/ProposedLeaseIdOptional" + }, + { + "$ref": "#/parameters/LeaseDuration" + }, + { + "$ref": "#/parameters/LeaseAction" + }, + { + "$ref": "#/parameters/IfMatch" + }, + { + "$ref": "#/parameters/IfNoneMatch" + }, + { + "$ref": "#/parameters/IfModifiedSince" + }, + { + "$ref": "#/parameters/IfUnmodifiedSince" + }, + { + "name": "flush", + "in": "query", + "description": "Optional. This parameter allows the caller to flush during an append call. Default value is 'false' , if 'true' the data will be flushed with the append call. Note that when using flush=true, the following headers are not supported - 'x-ms-cache-control', 'x-ms-content-encoding', 'x-ms-content-type', 'x-ms-content-language', 'x-ms-content-md5', 'x-ms-content-disposition'. To set these headers during flush, please use action=flush", + "required": false, + "type": "boolean" + } + ], + "responses": { + "202": { + "description": "Append data to file control response.", + "headers": { + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation." + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "The version of the REST protocol used to process the request." + }, + "ETag": { + "description": "An HTTP entity tag associated with the file or directory.", + "type": "string" + }, + "Content-MD5": { + "type": "string", + "format": "byte", + "description": "If the blob has an MD5 hash and this operation is to read the full blob, this response header is returned so that the client can check for message content integrity." + }, + "x-ms-content-crc64": { + "type": "string", + "format": "byte", + "description": "This header is returned so that the client can check for message content integrity. The value of this header is computed by the Blob service; it is not necessarily the same value specified in the request headers." + }, + "x-ms-request-server-encrypted": { + "x-ms-client-name": "IsServerEncrypted", + "type": "boolean", + "description": "The value of this header is set to true if the contents of the request are successfully encrypted using the specified algorithm, and false otherwise." + }, + "x-ms-encryption-key-sha256": { + "x-ms-client-name": "EncryptionKeySha256", + "type": "string", + "description": "The SHA-256 hash of the encryption key used to encrypt the blob. This header is only returned when the blob was encrypted with a customer-provided key." + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "The version of the REST protocol used to process the request." + } + }, + "schema": { + "$ref": "#/definitions/StorageError" + } + } + } + }, + "parameters": [ + { + "$ref": "#/parameters/FileSystem" + }, + { + "$ref": "#/parameters/Path" + }, + { + "name": "action", + "description": "action", + "in": "query", + "required": true, + "type": "string", + "enum": ["append"] + } + ] + }, + "/{filesystem}/{path}?comp=expiry": { + "put": { + "tags": ["blob"], + "operationId": "Path_SetExpiry", + "description": "Sets the time a blob will expire and be deleted.", + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + }, + { + "$ref": "#/parameters/PathExpiryOptions" + }, + { + "$ref": "#/parameters/PathExpiryTime" + } + ], + "responses": { + "200": { + "description": "The blob expiry was set successfully.", + "headers": { + "ETag": { + "type": "string", + "description": "The ETag contains a value that you can use to perform operations conditionally. If the request version is 2011-08-18 or newer, the ETag value will be in quotes." + }, + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123", + "description": "Returns the date and time the container was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob." + }, + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated." + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageError" + } + } + } + }, + "parameters": [ + { + "$ref": "#/parameters/FileSystem" + }, + { + "$ref": "#/parameters/Path" + }, + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": ["expiry"] + } + ] + }, + "/{filesystem}/{path}?comp=undelete": { + "put": { + "tags": ["blob"], + "operationId": "Path_Undelete", + "description": "Undelete a path that was previously soft deleted", + "parameters": [ + { + "$ref": "#/parameters/Timeout" + }, + { + "$ref": "#/parameters/UndeleteSource" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/ClientRequestId" + } + ], + "responses": { + "200": { + "description": "The blob was undeleted successfully.", + "headers": { + "x-ms-client-request-id": { + "x-ms-client-name": "ClientRequestId", + "type": "string", + "description": "If a client request id header is sent in the request, this header will be present in the response with the same value." + }, + "x-ms-request-id": { + "x-ms-client-name": "RequestId", + "type": "string", + "description": "This header uniquely identifies the request that was made and can be used for troubleshooting the request." + }, + "x-ms-resource-type": { + "x-ms-client-name": "ResourceType", + "description": "The type of the resource. The value may be \"file\" or \"directory\". If not set, the value is \"file\".", + "type": "string" + }, + "x-ms-version": { + "x-ms-client-name": "Version", + "type": "string", + "description": "Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above." + }, + "Date": { + "type": "string", + "format": "date-time-rfc1123", + "description": "UTC date/time value generated by the service that indicates the time at which the response was initiated." + } + } + }, + "default": { + "description": "Failure", + "headers": { + "x-ms-error-code": { + "x-ms-client-name": "ErrorCode", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/StorageError" + } + } + } + }, + "parameters": [ + { + "$ref": "#/parameters/FileSystem" + }, + { + "$ref": "#/parameters/Path" + }, + { + "name": "comp", + "description": "comp", + "in": "query", + "required": true, + "type": "string", + "enum": ["undelete"] + } + ] + } + }, + "parameters": { + "Url": { + "name": "url", + "description": "The URL of the service account, container, or blob that is the target of the desired operation.", + "required": true, + "x-ms-parameter-location": "client", + "type": "string", + "in": "path", + "x-ms-skip-url-encoding": true + }, + "FileSystemResource": { + "name": "resource", + "in": "query", + "x-ms-parameter-location": "client", + "description": "The value must be \"filesystem\" for all filesystem operations.", + "required": true, + "type": "string", + "enum": ["filesystem"], + "x-ms-enum": { + "name": "FileSystemResourceType", + "modelAsString": false + } + }, + "ApiVersionParameter": { + "name": "x-ms-version", + "x-ms-parameter-location": "client", + "x-ms-client-name": "version", + "in": "header", + "required": false, + "type": "string", + "description": "Specifies the version of the operation to use for this request." + }, + "accountName": { + "description": "The Azure Storage account name.", + "in": "path", + "name": "accountName", + "required": true, + "type": "string", + "x-ms-skip-url-encoding": true, + "x-ms-parameter-location": "method" + }, + "dnsSuffix": { + "default": "dfs.core.windows.net", + "description": "The DNS suffix for the Azure Data Lake Storage endpoint.", + "in": "path", + "name": "dnsSuffix", + "required": true, + "type": "string", + "x-ms-skip-url-encoding": true, + "x-ms-parameter-location": "method" + }, + "ClientRequestId": { + "name": "x-ms-client-request-id", + "x-ms-client-name": "requestId", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "description": "Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the analytics logs when storage analytics logging is enabled." + }, + "Timeout": { + "name": "timeout", + "in": "query", + "required": false, + "type": "integer", + "minimum": 0, + "x-ms-parameter-location": "method", + "description": "The timeout parameter is expressed in seconds. For more information, see Setting Timeouts for Blob Service Operations." + }, + "Range": { + "name": "x-ms-range", + "x-ms-client-name": "range", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "description": "The HTTP Range request header specifies one or more byte ranges of the resource to be retrieved." + }, + "GetRangeContentMD5": { + "name": "x-ms-range-get-content-md5", + "x-ms-client-name": "rangeGetContentMD5", + "in": "header", + "required": false, + "type": "boolean", + "x-ms-parameter-location": "method", + "description": "Optional. When this header is set to \"true\" and specified together with the Range header, the service returns the MD5 hash for the range, as long as the range is less than or equal to 4MB in size. If this header is specified without the Range header, the service returns status code 400 (Bad Request). If this header is set to true when the range exceeds 4 MB in size, the service returns status code 400 (Bad Request)." + }, + "GetRangeContentCRC64": { + "name": "x-ms-range-get-content-crc64", + "x-ms-client-name": "rangeGetContentCRC64", + "in": "header", + "required": false, + "type": "boolean", + "x-ms-parameter-location": "method", + "description": "When set to true and specified together with the Range, the service returns the CRC64 hash for the range, as long as the range is less than or equal to 4 MB in size." + }, + "DeleteSnapshots": { + "name": "x-ms-delete-snapshots", + "x-ms-client-name": "deleteSnapshots", + "description": "Required if the blob has associated snapshots. Specify one of the following two options: include: Delete the base blob and all of its snapshots. only: Delete only the blob's snapshots and not the blob itself", + "x-ms-parameter-location": "method", + "in": "header", + "required": false, + "type": "string", + "enum": ["include", "only"], + "x-ms-enum": { + "name": "DeleteSnapshotsOptionType", + "modelAsString": false + } + }, + "BlobDeleteType": { + "name": "deletetype", + "x-ms-client-name": "blobDeleteType", + "in": "query", + "required": false, + "type": "string", + "enum": ["Permanent"], + "x-ms-enum": { + "name": "BlobDeleteType", + "modelAsString": false + }, + "x-ms-parameter-location": "method", + "description": "Optional. Only possible value is 'permanent', which specifies to permanently delete a blob if blob soft delete is enabled." + }, + "IfMatch": { + "name": "If-Match", + "x-ms-client-name": "ifMatch", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "modified-access-conditions" + }, + "description": "Specify an ETag value to operate only on blobs with a matching value." + }, + "IfModifiedSince": { + "name": "If-Modified-Since", + "x-ms-client-name": "ifModifiedSince", + "in": "header", + "required": false, + "type": "string", + "format": "date-time-rfc1123", + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "modified-access-conditions" + }, + "description": "Specify this header value to operate only on a blob if it has been modified since the specified date/time." + }, + "IfNoneMatch": { + "name": "If-None-Match", + "x-ms-client-name": "ifNoneMatch", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "modified-access-conditions" + }, + "description": "Specify an ETag value to operate only on blobs without a matching value." + }, + "IfUnmodifiedSince": { + "name": "If-Unmodified-Since", + "x-ms-client-name": "ifUnmodifiedSince", + "in": "header", + "required": false, + "type": "string", + "format": "date-time-rfc1123", + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "modified-access-conditions" + }, + "description": "Specify this header value to operate only on a blob if it has not been modified since the specified date/time." + }, + "IfTags": { + "name": "x-ms-if-tags", + "x-ms-client-name": "ifTags", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "modified-access-conditions" + }, + "description": "Specify a SQL where clause on blob tags to operate only on blobs with a matching value." + }, + "Snapshot": { + "name": "snapshot", + "in": "query", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "description": "The snapshot parameter is an opaque DateTime value that, when present, specifies the blob snapshot to retrieve. For more information on working with blob snapshots, see Creating a Snapshot of a Blob." + }, + "VersionId": { + "name": "versionid", + "x-ms-client-name": "versionId", + "in": "query", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "description": "The version id parameter is an opaque DateTime value that, when present, specifies the version of the blob to operate on. It's for service version 2019-10-10 and newer." + }, + "RecursiveOptional": { + "name": "recursive", + "x-ms-parameter-location": "method", + "in": "query", + "description": "Required", + "required": false, + "type": "boolean" + }, + "RecursiveRequired": { + "name": "recursive", + "x-ms-parameter-location": "method", + "in": "query", + "description": "Required", + "required": true, + "type": "boolean" + }, + "Continuation": { + "name": "continuation", + "x-ms-parameter-location": "method", + "in": "query", + "description": "Optional. When deleting a directory, the number of paths that are deleted with each invocation is limited. If the number of paths to be deleted exceeds this limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the delete operation to continue deleting the directory.", + "required": false, + "type": "string" + }, + "PathSetAccessControlRecursiveMode": { + "name": "mode", + "in": "query", + "x-ms-parameter-location": "method", + "description": "Mode \"set\" sets POSIX access control rights on files and directories, \"modify\" modifies one or more POSIX access control rights that pre-exist on files and directories, \"remove\" removes one or more POSIX access control rights that were present earlier on files and directories", + "required": true, + "type": "string", + "enum": ["set", "modify", "remove"], + "x-ms-enum": { + "name": "PathSetAccessControlRecursiveMode", + "modelAsString": false + } + }, + "ForceFlag": { + "name": "forceFlag", + "x-ms-parameter-location": "method", + "in": "query", + "description": "Optional. Valid for \"SetAccessControlRecursive\" operation. If set to false, the operation will terminate quickly on encountering user errors (4XX). If true, the operation will ignore user errors and proceed with the operation on other sub-entities of the directory. Continuation token will only be returned when forceFlag is true in case of user errors. If not set the default value is false for this.", + "required": false, + "type": "boolean" + }, + "Directory": { + "name": "directory", + "x-ms-client-name": "Path", + "x-ms-parameter-location": "method", + "in": "query", + "description": "Optional. Filters results to paths within the specified directory. An error occurs if the directory does not exist.", + "required": false, + "type": "string" + }, + "LeaseIdOptional": { + "name": "x-ms-lease-id", + "x-ms-client-name": "leaseId", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "lease-access-conditions" + }, + "description": "If specified, the operation only succeeds if the resource's lease is active and matches this ID." + }, + "LeaseIdRequired": { + "name": "x-ms-lease-id", + "x-ms-client-name": "leaseId", + "in": "header", + "required": true, + "type": "string", + "x-ms-parameter-location": "method", + "description": "Specifies the current lease ID on the resource." + }, + "ProposedLeaseIdOptional": { + "name": "x-ms-proposed-lease-id", + "x-ms-client-name": "proposedLeaseId", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "description": "Proposed lease ID, in a GUID string format. The Blob service returns 400 (Invalid request) if the proposed lease ID is not in the correct format. See Guid Constructor (String) for a list of valid GUID string formats." + }, + "LeaseDuration": { + "name": "x-ms-lease-duration", + "in": "header", + "description": "The lease duration is required to acquire a lease, and specifies the duration of the lease in seconds. The lease duration must be between 15 and 60 seconds or -1 for infinite lease.", + "format": "int32", + "required": false, + "type": "integer", + "x-ms-parameter-location": "client" + }, + "LeaseAction": { + "name": "x-ms-lease-action", + "x-ms-client-name": "leaseAction", + "in": "header", + "required": false, + "type": "string", + "enum": [ + "acquire", + "release", + "renew", + "break", + "change", + "auto-renew", + "acquire-release" + ], + "x-ms-enum": { + "name": "LeaseAction", + "modelAsString": false + }, + "x-ms-parameter-location": "method", + "description": "Describes what lease action to take." + }, + "LeaseDurationMethod": { + "name": "x-ms-lease-duration", + "x-ms-client-name": "leaseDuration", + "in": "header", + "description": "The lease duration is required to acquire a lease, and specifies the duration of the lease in seconds. The lease duration must be between 15 and 60 seconds or -1 for infinite lease.", + "format": "int64", + "required": false, + "type": "integer", + "x-ms-parameter-location": "method" + }, + "Prefix": { + "name": "prefix", + "in": "query", + "description": "Filters results to filesystems within the specified prefix.", + "required": false, + "type": "string", + "x-ms-parameter-location": "method" + }, + "MaxResults": { + "name": "maxResults", + "in": "query", + "description": "An optional value that specifies the maximum number of items to return. If omitted or greater than 5,000, the response will include up to 5,000 items.", + "format": "int32", + "minimum": 1, + "required": false, + "type": "integer", + "x-ms-parameter-location": "method" + }, + "MaxResultsBlob": { + "name": "maxresults", + "in": "query", + "required": false, + "type": "integer", + "minimum": 1, + "x-ms-parameter-location": "method", + "description": "Specifies the maximum number of containers to return. If the request does not specify maxresults, or specifies a value greater than 5000, the server will return up to 5000 items. Note that if the listing operation crosses a partition boundary, then the service will return a continuation token for retrieving the remainder of the results. For this reason, it is possible that the service will return fewer results than specified by maxresults, or than the default of 5000." + }, + "Properties": { + "name": "x-ms-properties", + "x-ms-client-name": "properties", + "description": "Optional. User-defined properties to be stored with the filesystem, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set. If the filesystem exists, any properties not included in the list will be removed. All properties are removed if the header is omitted. To merge new and existing properties, first get all existing properties and the current E-Tag, then make a conditional request with the E-Tag and include values for all properties.", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method" + }, + "SourceIfMatch": { + "name": "x-ms-source-if-match", + "x-ms-client-name": "sourceIfMatch", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "source-modified-access-conditions" + }, + "description": "Specify an ETag value to operate only on blobs with a matching value." + }, + "SourceIfModifiedSince": { + "name": "x-ms-source-if-modified-since", + "x-ms-client-name": "sourceIfModifiedSince", + "in": "header", + "required": false, + "type": "string", + "format": "date-time-rfc1123", + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "source-modified-access-conditions" + }, + "description": "Specify this header value to operate only on a blob if it has been modified since the specified date/time." + }, + "SourceIfNoneMatch": { + "name": "x-ms-source-if-none-match", + "x-ms-client-name": "sourceIfNoneMatch", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "source-modified-access-conditions" + }, + "description": "Specify an ETag value to operate only on blobs without a matching value." + }, + "SourceIfUnmodifiedSince": { + "name": "x-ms-source-if-unmodified-since", + "x-ms-client-name": "sourceIfUnmodifiedSince", + "in": "header", + "required": false, + "type": "string", + "format": "date-time-rfc1123", + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "source-modified-access-conditions" + }, + "description": "Specify this header value to operate only on a blob if it has not been modified since the specified date/time." + }, + "SourceLeaseId": { + "name": "x-ms-source-lease-id", + "x-ms-client-name": "sourceLeaseId", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "description": "A lease ID for the source path. If specified, the source path must have an active lease and the lease ID must match." + }, + "FileSystem": { + "name": "filesystem", + "x-ms-parameter-location": "client", + "x-ms-client-name": "fileSystem", + "in": "path", + "description": "The filesystem identifier.", + "pattern": "^[$a-z0-9](?!.*--)[-a-z0-9]{1,61}[a-z0-9]$", + "minLength": 3, + "maxLength": 63, + "required": true, + "type": "string" + }, + "Path": { + "name": "path", + "x-ms-parameter-location": "client", + "in": "path", + "description": "The file or directory path.", + "required": true, + "type": "string" + }, + "CacheControl": { + "name": "x-ms-cache-control", + "x-ms-client-name": "cacheControl", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "path-HTTP-headers" + }, + "description": "Optional. Sets the blob's cache control. If specified, this property is stored with the blob and returned with a read request." + }, + "ContentDisposition": { + "name": "x-ms-content-disposition", + "x-ms-client-name": "contentDisposition", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "path-HTTP-headers" + }, + "description": "Optional. Sets the blob's Content-Disposition header." + }, + "ContentEncoding": { + "name": "x-ms-content-encoding", + "x-ms-client-name": "contentEncoding", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "path-HTTP-headers" + }, + "description": "Optional. Sets the blob's content encoding. If specified, this property is stored with the blob and returned with a read request." + }, + "ContentLanguage": { + "name": "x-ms-content-language", + "x-ms-client-name": "contentLanguage", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "path-HTTP-headers" + }, + "description": "Optional. Set the blob's content language. If specified, this property is stored with the blob and returned with a read request." + }, + "ContentType": { + "name": "x-ms-content-type", + "x-ms-client-name": "contentType", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "path-HTTP-headers" + }, + "description": "Optional. Sets the blob's content type. If specified, this property is stored with the blob and returned with a read request." + }, + "TransactionalContentMD5": { + "name": "Content-MD5", + "x-ms-client-name": "transactionalContentHash", + "in": "header", + "required": false, + "type": "string", + "format": "byte", + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "path-HTTP-headers" + }, + "description": "Specify the transactional md5 for the body, to be validated by the service." + }, + "ContentMD5": { + "name": "x-ms-content-md5", + "x-ms-client-name": "contentMD5", + "in": "header", + "required": false, + "type": "string", + "format": "byte", + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "path-HTTP-headers" + }, + "description": "Specify the transactional md5 for the body, to be validated by the service." + }, + "ContentCrc64": { + "name": "x-ms-content-crc64", + "x-ms-client-name": "transactionalContentCrc64", + "in": "header", + "required": false, + "type": "string", + "format": "byte", + "x-ms-parameter-location": "method", + "description": "Specify the transactional crc64 for the body, to be validated by the service." + }, + "Umask": { + "name": "x-ms-umask", + "x-ms-client-name": "umask", + "description": "Optional and only valid if Hierarchical Namespace is enabled for the account. When creating a file or directory and the parent folder does not have a default ACL, the umask restricts the permissions of the file or directory to be created. The resulting permission is given by p bitwise and not u, where p is the permission and u is the umask. For example, if p is 0777 and u is 0057, then the resulting permission is 0720. The default permission is 0777 for a directory and 0666 for a file. The default umask is 0027. The umask must be specified in 4-digit octal notation (e.g. 0766).", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method" + }, + "Permissions": { + "name": "x-ms-permissions", + "x-ms-client-name": "permissions", + "description": "Optional and only valid if Hierarchical Namespace is enabled for the account. Sets POSIX access permissions for the file owner, the file owning group, and others. Each class may be granted read, write, or execute permission. The sticky bit is also supported. Both symbolic (rwxrw-rw-) and 4-digit octal notation (e.g. 0766) are supported.", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method" + }, + "RenameSource": { + "name": "x-ms-rename-source", + "x-ms-client-name": "renameSource", + "in": "header", + "description": "An optional file or directory to be renamed. The value must have the following format: \"/{filesystem}/{path}\". If \"x-ms-properties\" is specified, the properties will overwrite the existing properties; otherwise, the existing properties will be preserved. This value must be a URL percent-encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set.", + "required": false, + "type": "string", + "x-ms-parameter-location": "method" + }, + "Owner": { + "name": "x-ms-owner", + "x-ms-client-name": "owner", + "in": "header", + "required": false, + "type": "string", + "description": "Optional. The owner of the blob or directory.", + "x-ms-parameter-location": "method" + }, + "Group": { + "name": "x-ms-group", + "x-ms-client-name": "group", + "in": "header", + "required": false, + "type": "string", + "description": "Optional. The owning group of the blob or directory.", + "x-ms-parameter-location": "method" + }, + "Acl": { + "name": "x-ms-acl", + "description": "Sets POSIX access control rights on files and directories. The value is a comma-separated list of access control entries. Each access control entry (ACE) consists of a scope, a type, a user or group identifier, and permissions in the format \"[scope:][type]:[id]:[permissions]\".", + "x-ms-client-name": "acl", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method" + }, + "Body": { + "name": "body", + "in": "body", + "required": true, + "schema": { + "type": "object", + "format": "file" + }, + "x-ms-parameter-location": "method", + "description": "Initial data" + }, + "Upn": { + "name": "upn", + "in": "query", + "description": "Optional. Valid only when Hierarchical Namespace is enabled for the account. If \"true\", the user identity values returned in the x-ms-owner, x-ms-group, and x-ms-acl response headers will be transformed from Azure Active Directory Object IDs to User Principal Names. If \"false\", the values will be returned as Azure Active Directory Object IDs. The default value is false. Note that group and application Object IDs are not translated because they do not have unique friendly names.", + "required": false, + "type": "boolean", + "x-ms-parameter-location": "method" + }, + "Position": { + "name": "position", + "in": "query", + "description": "This parameter allows the caller to upload data in parallel and control the order in which it is appended to the file. It is required when uploading data to be appended to the file and when flushing previously uploaded data to the file. The value must be the position where the data is to be appended. Uploaded data is not immediately flushed, or written, to the file. To flush, the previously uploaded data must be contiguous, the position parameter must be specified and equal to the length of the file after all data has been written, and there must not be a request entity body included with the request.", + "format": "int64", + "required": false, + "type": "integer", + "x-ms-parameter-location": "method" + }, + "RetainUncommittedData": { + "name": "retainUncommittedData", + "in": "query", + "description": "Valid only for flush operations. If \"true\", uncommitted data is retained after the flush operation completes; otherwise, the uncommitted data is deleted after the flush operation. The default is false. Data at offsets less than the specified position are written to the file when flush succeeds, but this optional parameter allows data after the flush position to be retained for a future flush operation.", + "required": false, + "type": "boolean", + "x-ms-parameter-location": "method" + }, + "Close": { + "name": "close", + "in": "query", + "description": "Azure Storage Events allow applications to receive notifications when files change. When Azure Storage Events are enabled, a file changed event is raised. This event has a property indicating whether this is the final change to distinguish the difference between an intermediate flush to a file stream and the final close of a file stream. The close query parameter is valid only when the action is \"flush\" and change notifications are enabled. If the value of close is \"true\" and the flush operation completes successfully, the service raises a file change notification with a property indicating that this is the final update (the file stream has been closed). If \"false\" a change notification is raised indicating the file has changed. The default is false. This query parameter is set to true by the Hadoop ABFS driver to indicate that the file stream has been closed.\"", + "required": false, + "type": "boolean", + "x-ms-parameter-location": "method" + }, + "ContentLength": { + "name": "Content-Length", + "in": "header", + "description": "Required for \"Append Data\" and \"Flush Data\". Must be 0 for \"Flush Data\". Must be the length of the request content in bytes for \"Append Data\".", + "minimum": 0, + "required": false, + "type": "integer", + "format": "int64", + "x-ms-parameter-location": "method" + }, + "PathExpiryOptions": { + "name": "x-ms-expiry-option", + "x-ms-client-name": "ExpiryOptions", + "in": "header", + "required": true, + "type": "string", + "enum": [ + "NeverExpire", + "RelativeToCreation", + "RelativeToNow", + "Absolute" + ], + "x-ms-enum": { + "name": "PathExpiryOptions", + "modelAsString": true + }, + "x-ms-parameter-location": "method", + "description": "Required. Indicates mode of the expiry time" + }, + "PathExpiryOptionsOptional": { + "name": "x-ms-expiry-option", + "x-ms-client-name": "ExpiryOptions", + "in": "header", + "required": false, + "type": "string", + "enum": [ + "NeverExpire", + "RelativeToCreation", + "RelativeToNow", + "Absolute" + ], + "x-ms-enum": { + "name": "PathExpiryOptions", + "modelAsString": true + }, + "x-ms-parameter-location": "method", + "description": "Required. Indicates mode of the expiry time" + }, + "PathExpiryTime": { + "name": "x-ms-expiry-time", + "x-ms-client-name": "ExpiresOn", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "description": "The time to set the blob to expiry" + }, + "Metadata": { + "name": "x-ms-meta", + "x-ms-client-name": "metadata", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "description": "Optional. Specifies a user-defined name-value pair associated with the blob. If no name-value pairs are specified, the operation will copy the metadata from the source blob or file to the destination blob. If one or more name-value pairs are specified, the destination blob is created with the specified metadata, and metadata is not copied from the source blob or file. Note that beginning with version 2009-09-19, metadata names must adhere to the naming rules for C# identifiers. See Naming and Referencing Containers, Blobs, and Metadata for more information.", + "x-ms-header-collection-prefix": "x-ms-meta-" + }, + "UndeleteSource": { + "name": "x-ms-undelete-source", + "x-ms-client-name": "UndeleteSource", + "in": "header", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "description": "Only for hierarchical namespace enabled accounts. Optional. The path of the soft deleted blob to undelete." + }, + "Marker": { + "name": "marker", + "in": "query", + "required": false, + "type": "string", + "description": "A string value that identifies the portion of the list of containers to be returned with the next listing operation. The operation returns the NextMarker value within the response body if the listing operation did not return all containers remaining to be listed with the current page. The NextMarker value can be used as the value for the marker parameter in a subsequent call to request the next page of list items. The marker value is opaque to the client.", + "x-ms-parameter-location": "method" + }, + "ListBlobsInclude": { + "name": "include", + "in": "query", + "required": false, + "type": "array", + "collectionFormat": "csv", + "items": { + "type": "string", + "enum": [ + "", + "copy", + "deleted", + "metadata", + "snapshots", + "uncommittedblobs", + "versions", + "tags", + "immutabilitypolicy", + "legalhold", + "deletedwithversions", + "permissions" + ], + "x-ms-enum": { + "name": "ListBlobsIncludeItem", + "modelAsString": false + } + }, + "x-ms-parameter-location": "method", + "description": "Include this parameter to specify one or more datasets to include in the response." + }, + "ListBlobsShowOnly": { + "name": "showonly", + "in": "query", + "required": false, + "type": "string", + "enum": ["deleted"], + "x-ms-enum": { + "name": "ListBlobsShowOnly", + "modelAsString": false + }, + "x-ms-parameter-location": "method", + "description": "Include this parameter to specify one or more datasets to include in the response." + }, + "Delimiter": { + "name": "delimiter", + "description": "When the request includes this parameter, the operation returns a BlobPrefix element in the response body that acts as a placeholder for all blobs whose names begin with the same substring up to the appearance of the delimiter character. The delimiter may be a single character or a string.", + "type": "string", + "x-ms-parameter-location": "method", + "in": "query", + "required": true + }, + "EncryptionKey": { + "name": "x-ms-encryption-key", + "x-ms-client-name": "encryptionKey", + "type": "string", + "in": "header", + "required": false, + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "cpk-info" + }, + "description": "Optional. Specifies the encryption key to use to encrypt the data provided in the request. If not specified, encryption is performed with the root account encryption key. For more information, see Encryption at Rest for Azure Storage Services." + }, + "EncryptionKeySha256": { + "name": "x-ms-encryption-key-sha256", + "x-ms-client-name": "encryptionKeySha256", + "type": "string", + "in": "header", + "required": false, + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "cpk-info" + }, + "description": "The SHA-256 hash of the provided encryption key. Must be provided if the x-ms-encryption-key header is provided." + }, + "EncryptionAlgorithm": { + "name": "x-ms-encryption-algorithm", + "x-ms-client-name": "encryptionAlgorithm", + "type": "string", + "in": "header", + "required": false, + "enum": ["AES256"], + "x-ms-enum": { + "name": "EncryptionAlgorithmType", + "modelAsString": false + }, + "x-ms-parameter-location": "method", + "x-ms-parameter-grouping": { + "name": "cpk-info" + }, + "description": "The algorithm used to produce the encryption key hash. Currently, the only accepted value is \"AES256\". Must be provided if the x-ms-encryption-key header is provided." + } + }, + "definitions": { + "AclFailedEntry": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "type": { + "type": "string" + }, + "errorMessage": { + "type": "string" + } + } + }, + "SetAccessControlRecursiveResponse": { + "type": "object", + "properties": { + "directoriesSuccessful": { + "type": "integer", + "format": "int32" + }, + "filesSuccessful": { + "type": "integer", + "format": "int32" + }, + "failureCount": { + "type": "integer", + "format": "int32" + }, + "failedEntries": { + "type": "array", + "items": { + "$ref": "#/definitions/AclFailedEntry" + } + } + } + }, + "Path": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "isDirectory": { + "default": false, + "type": "boolean" + }, + "lastModified": { + "type": "string", + "format": "date-time-rfc1123" + }, + "etag": { + "type": "string" + }, + "contentLength": { + "type": "integer", + "format": "int64" + }, + "owner": { + "type": "string" + }, + "group": { + "type": "string" + }, + "permissions": { + "type": "string" + }, + "EncryptionScope": { + "type": "string", + "description": "The name of the encryption scope under which the blob is encrypted." + } + } + }, + "PathList": { + "type": "object", + "xml": { + "name": "paths" + }, + "properties": { + "paths": { + "xml": { + "name": "paths" + }, + "type": "array", + "items": { + "$ref": "#/definitions/Path" + } + } + } + }, + "FileSystem": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "lastModified": { + "type": "string" + }, + "eTag": { + "type": "string" + } + } + }, + "ListBlobsFlatSegmentResponse": { + "xml": { + "name": "EnumerationResults" + }, + "description": "An enumeration of blobs", + "type": "object", + "required": ["ServiceEndpoint", "ContainerName", "Segment"], + "properties": { + "ServiceEndpoint": { + "type": "string", + "xml": { + "attribute": true + } + }, + "ContainerName": { + "type": "string", + "xml": { + "attribute": true + } + }, + "Prefix": { + "type": "string" + }, + "Marker": { + "type": "string" + }, + "MaxResults": { + "type": "integer" + }, + "Segment": { + "$ref": "#/definitions/BlobFlatListSegment" + }, + "NextMarker": { + "type": "string" + } + } + }, + "ListBlobsHierarchySegmentResponse": { + "xml": { + "name": "EnumerationResults" + }, + "description": "An enumeration of blobs", + "type": "object", + "required": ["ServiceEndpoint", "ContainerName", "Segment"], + "properties": { + "ServiceEndpoint": { + "type": "string", + "xml": { + "attribute": true + } + }, + "ContainerName": { + "type": "string", + "xml": { + "attribute": true + } + }, + "Prefix": { + "type": "string" + }, + "Marker": { + "type": "string" + }, + "MaxResults": { + "type": "integer" + }, + "Delimiter": { + "type": "string" + }, + "Segment": { + "$ref": "#/definitions/BlobHierarchyListSegment" + }, + "NextMarker": { + "type": "string" + } + } + }, + "BlobFlatListSegment": { + "xml": { + "name": "Blobs" + }, + "required": ["BlobItems"], + "type": "object", + "properties": { + "BlobItems": { + "type": "array", + "items": { + "$ref": "#/definitions/BlobItemInternal" + } + } + } + }, + "BlobHierarchyListSegment": { + "xml": { + "name": "Blobs" + }, + "type": "object", + "required": ["BlobItems"], + "properties": { + "BlobPrefixes": { + "type": "array", + "items": { + "$ref": "#/definitions/BlobPrefix" + } + }, + "BlobItems": { + "type": "array", + "items": { + "$ref": "#/definitions/BlobItemInternal" + } + } + } + }, + "BlobPrefix": { + "type": "object", + "required": ["Name"], + "properties": { + "Name": { + "type": "string" + } + } + }, + "BlobItemInternal": { + "xml": { + "name": "Blob" + }, + "description": "An Azure Storage blob", + "type": "object", + "required": ["Name", "Properties"], + "properties": { + "Name": { + "type": "string" + }, + "Deleted": { + "type": "boolean" + }, + "Snapshot": { + "type": "string" + }, + "VersionId": { + "type": "string" + }, + "IsCurrentVersion": { + "type": "boolean" + }, + "Properties": { + "$ref": "#/definitions/BlobPropertiesInternal" + }, + "DeletionId": { + "type": "string" + } + } + }, + "BlobPropertiesInternal": { + "xml": { + "name": "Properties" + }, + "description": "Properties of a blob", + "type": "object", + "required": ["Etag", "Last-Modified"], + "properties": { + "Creation-Time": { + "type": "string", + "format": "date-time-rfc1123" + }, + "Last-Modified": { + "type": "string", + "format": "date-time-rfc1123" + }, + "Etag": { + "type": "string" + }, + "Content-Length": { + "type": "integer", + "format": "int64", + "description": "Size in bytes" + }, + "Content-Type": { + "type": "string" + }, + "Content-Encoding": { + "type": "string" + }, + "Content-Language": { + "type": "string" + }, + "Content-MD5": { + "type": "string", + "format": "byte" + }, + "Content-Disposition": { + "type": "string" + }, + "Cache-Control": { + "type": "string" + }, + "x-ms-blob-sequence-number": { + "x-ms-client-name": "blobSequenceNumber", + "type": "integer", + "format": "int64" + }, + "CopyId": { + "type": "string" + }, + "CopySource": { + "type": "string" + }, + "CopyProgress": { + "type": "string" + }, + "CopyCompletionTime": { + "type": "string", + "format": "date-time-rfc1123" + }, + "CopyStatusDescription": { + "type": "string" + }, + "ServerEncrypted": { + "type": "boolean" + }, + "IncrementalCopy": { + "type": "boolean" + }, + "DestinationSnapshot": { + "type": "string" + }, + "DeletedTime": { + "type": "string", + "format": "date-time-rfc1123" + }, + "RemainingRetentionDays": { + "type": "integer" + }, + "AccessTierInferred": { + "type": "boolean" + }, + "CustomerProvidedKeySha256": { + "type": "string" + }, + "EncryptionScope": { + "type": "string", + "description": "The name of the encryption scope under which the blob is encrypted." + }, + "AccessTierChangeTime": { + "type": "string", + "format": "date-time-rfc1123" + }, + "TagCount": { + "type": "integer" + }, + "Expiry-Time": { + "x-ms-client-name": "ExpiresOn", + "type": "string", + "format": "date-time-rfc1123" + }, + "Sealed": { + "x-ms-client-name": "IsSealed", + "type": "boolean" + }, + "LastAccessTime": { + "x-ms-client-name": "LastAccessedOn", + "type": "string", + "format": "date-time-rfc1123" + }, + "DeleteTime": { + "type": "string", + "format": "date-time-rfc1123" + }, + "properties": { + "type": "string" + } + } + }, + "FileSystemList": { + "type": "object", + "properties": { + "filesystems": { + "type": "array", + "items": { + "$ref": "#/definitions/FileSystem" + } + } + } + }, + "StorageError": { + "type": "object", + "properties": { + "Message": { + "description": "The service error message.", + "type": "string" + }, + "error": { + "type": "object", + "description": "The service error response object.", + "properties": { + "Code": { + "description": "The service error code.", + "type": "string" + }, + "Message": { + "description": "The service error message.", + "type": "string" + } + } + } + } + } + } +} diff --git a/swagger/dfs.md b/swagger/dfs.md new file mode 100644 index 000000000..9bd2e122e --- /dev/null +++ b/swagger/dfs.md @@ -0,0 +1,46 @@ +# Azurite Server Blob + +> see https://aka.ms/autorest + +```yaml +package-name: azurite-server-blob +title: AzuriteServerBlob +package-version: 1.0.0 +description: Azurite Server for Blob +enable-xml: true +generate-metadata: false +license-header: MICROSOFT_MIT_NO_VERSION +output-folder: ../src/dfs/generated +input-file: + - data-lake-storage.json-2021-04-10.json + - blob-storage-2021-10-04-data-lake.json +model-date-time-as-string: true +optional-response-headers: true +enum-types: true +``` + +## Changes Made to Client Swagger + +1. Added metadata to all path operations +2. changed contentMd5 to bytes in Path_Read in both responses 200, 206 and in Path_GetProperties +3. added metadata to response of getproperties +4. added LeaseAction, LeaseDuration and proposedLeaseId to Path_Flush, Path_AppendData +5. added LeaseState, LeaseStatus to blob properties +6. added IfMacth, IfNoneMatch, IfModifiedSince, IfUnmodifiedSince, Path_AppendData +7. added XmlName("paths") to pathlist +8. added "format": "date-time-rfc1123" to lastModified +9. merge Blob_GetProperties and Path_GetProperties into Path_GetProperties and remove Blob_GetProperties +10. merge Blob_Delete and Path_Delete into Path_Delete and remove Blob_Delete +11. merge Blob_Download and Path_Read into Path_Read and remove Blob_Download +12. move container listBlobsFalt to dataLake swagger and rename it to filesystem listblobsFlat +13. move container listBlobsHierarchy & file system listBlobsHierarchy and remove container listBlobsHierarchy +14. add auto-renew, acquire-release to values permitted in lease action +15. eTag renamed to etag in Path model (required for hadoop) +16. add flush option to Path_AppendData as per spec +17. added Path_SetProperties from Path_Update spec +18. added expiresOn to Path in listPaths spec + +## Changes in Code: + +1. isxml set to false in listPaths (required for hadoop) +2. add req.getHeader("X-HTTP-Method-Override") to dispatch.middleware.ts (required for hadoop) diff --git a/tests/BlobTestServerFactory.ts b/tests/BlobTestServerFactory.ts index c37c19848..2a9360a04 100644 --- a/tests/BlobTestServerFactory.ts +++ b/tests/BlobTestServerFactory.ts @@ -43,7 +43,9 @@ export default class BlobTestServerFactory { cert, key, undefined, - oauth + oauth, + false, + true ); return new SqlBlobServer(config); diff --git a/tests/blob/apis/blob.test.ts b/tests/blob/apis/blob.test.ts index 17d56d108..9a41dab74 100644 --- a/tests/blob/apis/blob.test.ts +++ b/tests/blob/apis/blob.test.ts @@ -812,7 +812,7 @@ describe("BlobAPIs", () => { ); }); - it("Copy blob should work @loki", async () => { + it("Copy blob should work @loki @sql", async () => { const sourceBlob = getUniqueName("blob"); const destBlob = getUniqueName("blob"); @@ -871,7 +871,7 @@ describe("BlobAPIs", () => { ); }); - it("Copy blob should work to override metadata @loki", async () => { + it("Copy blob should work to override metadata @loki @sql", async () => { const sourceBlob = getUniqueName("blob"); const destBlob = getUniqueName("blob"); @@ -930,7 +930,7 @@ describe("BlobAPIs", () => { assert.deepStrictEqual(result.accessTier, "Hot"); }); - it("Copy blob should not override destination Lease status @loki", async () => { + it("Copy blob should not override destination Lease status @loki @sql", async () => { const sourceBlob = getUniqueName("blob"); const destBlob = getUniqueName("blob"); @@ -965,7 +965,7 @@ describe("BlobAPIs", () => { await destLeaseClient.releaseLease(); }); - it("Copy blob should work for page blob @loki", async () => { + it("Copy blob should work for page blob @loki @sql", async () => { const sourceBlob = getUniqueName("blob"); const destBlob = getUniqueName("blob"); @@ -1024,7 +1024,7 @@ describe("BlobAPIs", () => { ); }); - it("Copy blob should not work for page blob and set tier @loki", async () => { + it("Copy blob should not work for page blob and set tier @loki @sql", async () => { const sourceBlob = getUniqueName("blob"); const destBlob = getUniqueName("blob"); @@ -1062,7 +1062,7 @@ describe("BlobAPIs", () => { assert.deepStrictEqual(err.statusCode, 400); }); - it("Copy blob should not work with ifNoneMatch * when dest exist @loki", async () => { + it("Copy blob should not work with ifNoneMatch * when dest exist @loki @sql", async () => { const sourceBlob = getUniqueName("blob"); const destBlob = getUniqueName("blob"); @@ -1133,7 +1133,7 @@ describe("BlobAPIs", () => { assert.fail(); }); - it("Synchronized copy blob should work @loki", async () => { + it("Synchronized copy blob should work @loki @sql", async () => { const sourceBlob = getUniqueName("blob"); const destBlob = getUniqueName("blob"); @@ -1191,7 +1191,7 @@ describe("BlobAPIs", () => { ); }); - it("Synchronized copy blob should work to override metadata @loki", async () => { + it("Synchronized copy blob should work to override metadata @loki @sql", async () => { const sourceBlob = getUniqueName("blob"); const destBlob = getUniqueName("blob"); @@ -1215,7 +1215,7 @@ describe("BlobAPIs", () => { assert.deepStrictEqual(result.metadata, metadata2); }); - it("Synchronized copy blob should not override destination Lease status @loki", async () => { + it("Synchronized copy blob should not override destination Lease status @loki @sql", async () => { const sourceBlob = getUniqueName("blob"); const destBlob = getUniqueName("blob"); @@ -1250,7 +1250,7 @@ describe("BlobAPIs", () => { await destLeaseClient.releaseLease(); }); - it("Synchronized copy blob should work for page blob @loki", async () => { + it("Synchronized copy blob should work for page blob @loki @sql", async () => { const sourceBlob = getUniqueName("blob"); const destBlob = getUniqueName("blob"); diff --git a/tests/blob/apis/service.test.ts b/tests/blob/apis/service.test.ts index 999062aef..60a661e80 100644 --- a/tests/blob/apis/service.test.ts +++ b/tests/blob/apis/service.test.ts @@ -457,7 +457,7 @@ describe("ServiceAPIs", () => { ); }); - it("Get Blob service stats negative @loki", async () => { + it("Get Blob service stats negative @loki @sql", async () => { await serviceClient.getStatistics() .catch((err) => { assert.strictEqual(err.statusCode, 400); @@ -496,7 +496,7 @@ describe("ServiceAPIs - secondary location endpoint", () => { await server.clean(); }); - it("Get Blob service stats @loki", async () => { + it("Get Blob service stats @loki @sql", async () => { await serviceClient.getStatistics() .then((result) => { diff --git a/tests/blob/blockblob.highlevel.test.ts b/tests/blob/blockblob.highlevel.test.ts index 22bac9fb3..6cb8ca550 100644 --- a/tests/blob/blockblob.highlevel.test.ts +++ b/tests/blob/blockblob.highlevel.test.ts @@ -100,7 +100,7 @@ describe("BlockBlobHighlevel", () => { await server.clean(); }); - it("uploadFile should success when blob >= BLOCK_BLOB_MAX_UPLOAD_BLOB_BYTES @loki @sql", async () => { + it("uploadFile should success when blob >= BLOCK_BLOB_MAX_UPLOAD_BLOB_BYTES @loki", async () => { const result = await blockBlobClient.uploadFile(tempFileLarge, { blockSize: 4 * 1024 * 1024, concurrency: 20 @@ -124,7 +124,7 @@ describe("BlockBlobHighlevel", () => { assert.ok(downloadedData.equals(uploadedData)); }).timeout(timeoutForLargeFileUploadingTest); - it("uploadFile should success when blob < BLOCK_BLOB_MAX_UPLOAD_BLOB_BYTES @loki @sql", async () => { + it("uploadFile should success when blob < BLOCK_BLOB_MAX_UPLOAD_BLOB_BYTES @loki", async () => { await blockBlobClient.uploadFile(tempFileSmall, { blockSize: 4 * 1024 * 1024, concurrency: 20 @@ -145,7 +145,7 @@ describe("BlockBlobHighlevel", () => { }); // tslint:disable-next-line:max-line-length - it("uploadFile should success when blob < BLOCK_BLOB_MAX_UPLOAD_BLOB_BYTES and configured maxSingleShotSize @loki @sql", async () => { + it("uploadFile should success when blob < BLOCK_BLOB_MAX_UPLOAD_BLOB_BYTES and configured maxSingleShotSize @loki", async () => { await blockBlobClient.uploadFile(tempFileSmall, { maxSingleShotSize: 0 }); @@ -165,7 +165,7 @@ describe("BlockBlobHighlevel", () => { }); // tslint:disable-next-line: max-line-length - it("uploadFile should update progress when blob >= BLOCK_BLOB_MAX_UPLOAD_BLOB_BYTES @loki @sql", async () => { + it("uploadFile should update progress when blob >= BLOCK_BLOB_MAX_UPLOAD_BLOB_BYTES @loki", async () => { let eventTriggered = false; const aborter = new AbortController(); @@ -179,12 +179,14 @@ describe("BlockBlobHighlevel", () => { aborter.abort(); } }); - } catch (err) {} + } catch (err) { + /**/ + } assert.ok(eventTriggered); }).timeout(timeoutForLargeFileUploadingTest); // tslint:disable-next-line: max-line-length - it("uploadFile should update progress when blob < BLOCK_BLOB_MAX_UPLOAD_BLOB_BYTES @loki @sql", async () => { + it("uploadFile should update progress when blob < BLOCK_BLOB_MAX_UPLOAD_BLOB_BYTES @loki", async () => { let eventTriggered = false; const aborter = new AbortController(); @@ -198,11 +200,13 @@ describe("BlockBlobHighlevel", () => { aborter.abort(); } }); - } catch (err) {} + } catch (err) { + /**/ + } assert.ok(eventTriggered); }); - it("uploadStream should success @loki @sql", async () => { + it("uploadStream should success @loki", async () => { const rs = fs.createReadStream(tempFileLarge); const result = await blockBlobClient.uploadStream(rs, 4 * 1024 * 1024, 20); assert.equal( @@ -228,7 +232,7 @@ describe("BlockBlobHighlevel", () => { fs.unlinkSync(downloadFilePath); }); - it("uploadStream should success for tiny buffers @loki @sql", async () => { + it("uploadStream should success for tiny buffers @loki", async () => { const buf = Buffer.from([0x62, 0x75, 0x66, 0x66, 0x65, 0x72]); const bufferStream = new PassThrough(); bufferStream.end(buf); @@ -252,7 +256,7 @@ describe("BlockBlobHighlevel", () => { fs.unlinkSync(downloadFilePath); }); - it("uploadStream should abort @loki @sql", async () => { + it("uploadStream should abort @loki", async () => { const rs = fs.createReadStream(tempFileLarge); try { @@ -260,12 +264,12 @@ describe("BlockBlobHighlevel", () => { abortSignal: AbortController.timeout(1) }); assert.fail(); - } catch (err:any) { + } catch (err: any) { assert.ok((err.message as string).toLowerCase().includes("abort")); } }).timeout(timeoutForLargeFileUploadingTest); - it("uploadStream should update progress event @loki @sql", async () => { + it("uploadStream should update progress event @loki", async () => { const rs = fs.createReadStream(tempFileLarge); let eventTriggered = false; @@ -278,7 +282,7 @@ describe("BlockBlobHighlevel", () => { assert.ok(eventTriggered); }).timeout(timeoutForLargeFileUploadingTest); - it("downloadToBuffer should success @loki @sql", async () => { + it("downloadToBuffer should success @loki", async () => { const rs = fs.createReadStream(tempFileLarge); const result = await blockBlobClient.uploadStream(rs, 4 * 1024 * 1024, 20); assert.equal( @@ -297,7 +301,7 @@ describe("BlockBlobHighlevel", () => { assert.ok(localFileContent.equals(buf)); }).timeout(timeoutForLargeFileUploadingTest); - it("downloadToBuffer should update progress event @loki @sql", async () => { + it("downloadToBuffer should update progress event @loki", async () => { const rs = fs.createReadStream(tempFileSmall); await blockBlobClient.uploadStream(rs, 4 * 1024 * 1024, 10); @@ -314,11 +318,13 @@ describe("BlockBlobHighlevel", () => { aborter.abort(); } }); - } catch (err) {} + } catch (err) { + /**/ + } assert.ok(eventTriggered); }).timeout(timeoutForLargeFileUploadingTest); - it("blobclient.download should success when internal stream unexpected ends at the stream end @loki @sql", async () => { + it("blobclient.download should success when internal stream unexpected ends at the stream end @loki", async () => { await blockBlobClient.uploadFile(tempFileSmall, { blockSize: 4 * 1024 * 1024, concurrency: 20 @@ -343,8 +349,9 @@ describe("BlockBlobHighlevel", () => { downloadResponse.clientRequestId ); - retirableReadableStreamOptions = (downloadResponse.readableStreamBody! as any) - .options; + retirableReadableStreamOptions = ( + downloadResponse.readableStreamBody! as any + ).options; const downloadedFile = join(tempFolderPath, getUniqueName("downloadfile.")); await readStreamToLocalFile( @@ -360,7 +367,7 @@ describe("BlockBlobHighlevel", () => { }); // tslint:disable-next-line: max-line-length - it("blobclient.download should download full data successfully when internal stream unexpected ends @loki @sql", async () => { + it("blobclient.download should download full data successfully when internal stream unexpected ends @loki", async () => { await blockBlobClient.uploadFile(tempFileSmall, { blockSize: 4 * 1024 * 1024, concurrency: 20 @@ -382,8 +389,9 @@ describe("BlockBlobHighlevel", () => { } }); - retirableReadableStreamOptions = (downloadResponse.readableStreamBody! as any) - .options; + retirableReadableStreamOptions = ( + downloadResponse.readableStreamBody! as any + ).options; const downloadedFile = join(tempFolderPath, getUniqueName("downloadfile.")); await readStreamToLocalFile( @@ -398,7 +406,7 @@ describe("BlockBlobHighlevel", () => { assert.ok(downloadedData.equals(uploadedData)); }); - it("blobclient.download should download partial data when internal stream unexpected ends @loki @sql", async () => { + it("blobclient.download should download partial data when internal stream unexpected ends @loki", async () => { await blockBlobClient.uploadFile(tempFileSmall, { blockSize: 4 * 1024 * 1024, concurrency: 20 @@ -422,8 +430,9 @@ describe("BlockBlobHighlevel", () => { } }); - retirableReadableStreamOptions = (downloadResponse.readableStreamBody! as any) - .options; + retirableReadableStreamOptions = ( + downloadResponse.readableStreamBody! as any + ).options; const downloadedFile = join(tempFolderPath, getUniqueName("downloadfile.")); await readStreamToLocalFile( @@ -442,7 +451,7 @@ describe("BlockBlobHighlevel", () => { ); }); - it("blobclient.download should download data failed when exceeding max stream retry requests @loki @sql", async () => { + it("blobclient.download should download data failed when exceeding max stream retry requests @loki", async () => { await blockBlobClient.uploadFile(tempFileSmall, { blockSize: 4 * 1024 * 1024, concurrency: 20 @@ -468,8 +477,9 @@ describe("BlockBlobHighlevel", () => { } } }); - retirableReadableStreamOptions = (downloadResponse.readableStreamBody! as any) - .options; + retirableReadableStreamOptions = ( + downloadResponse.readableStreamBody! as any + ).options; await readStreamToLocalFile( downloadResponse.readableStreamBody!, downloadedFile diff --git a/tests/blob/handlers/AppendBlobHandler.test.ts b/tests/blob/handlers/AppendBlobHandler.test.ts index 2c4878860..9e0636de1 100644 --- a/tests/blob/handlers/AppendBlobHandler.test.ts +++ b/tests/blob/handlers/AppendBlobHandler.test.ts @@ -96,7 +96,7 @@ describe("AppendBlobHandler", () => { ).thenResolve(extent); describe("create", () => { - it("accepts requests withContent-Length == 0 @loki", async () => { + it("accepts requests withContent-Length == 0 @loki @sql", async () => { const handler = new AppendBlobHandler( instance(metadataStore), instance(extentStore), @@ -108,7 +108,7 @@ describe("AppendBlobHandler", () => { }); }); - it("accepts requests with Content-Length != 0 in loose mode @loki", async () => { + it("accepts requests with Content-Length != 0 in loose mode @loki @sql", async () => { const handler = new AppendBlobHandler( instance(metadataStore), instance(extentStore), @@ -120,7 +120,7 @@ describe("AppendBlobHandler", () => { }); }); - it("rejects requests with Content-Length != 0 @loki", async () => { + it("rejects requests with Content-Length != 0 @loki @sql", async () => { const handler = new AppendBlobHandler( instance(metadataStore), instance(extentStore), @@ -146,7 +146,7 @@ describe("AppendBlobHandler", () => { bufferStream.end(buffer); }); - it("accepts requests with Content-Length != 0 @loki", async () => { + it("accepts requests with Content-Length != 0 @loki @sql", async () => { const handler = new AppendBlobHandler( instance(metadataStore), instance(extentStore), @@ -158,7 +158,7 @@ describe("AppendBlobHandler", () => { }); }); - it("rejects requests with Content-Length == 0 @loki", async () => { + it("rejects requests with Content-Length == 0 @loki @sql", async () => { const handler = new AppendBlobHandler( instance(metadataStore), instance(extentStore), @@ -176,7 +176,7 @@ describe("AppendBlobHandler", () => { ); }); - it("accepts requests with valid MD5 checksum @loki", async () => { + it("accepts requests with valid MD5 checksum @loki @sql", async () => { when(request.getHeader(HeaderConstants.CONTENT_MD5)).thenReturn( "T0EkOEfaaTpPNWwEhhFLxg==" ); @@ -195,7 +195,7 @@ describe("AppendBlobHandler", () => { }); }); - it("rejects requests with invalid MD5 checksum @loki", async () => { + it("rejects requests with invalid MD5 checksum @loki @sql", async () => { when(request.getHeader(HeaderConstants.CONTENT_MD5)).thenReturn( "d3JvbmdfTUQ1X2NoZWNrc3VtCg==" ); diff --git a/tests/blob/oauth.test.ts b/tests/blob/oauth.test.ts index 6febeafab..772186d62 100644 --- a/tests/blob/oauth.test.ts +++ b/tests/blob/oauth.test.ts @@ -933,7 +933,7 @@ describe("Blob OAuth Basic", () => { } }); - it(`Should not work with HTTP @loki @sql`, async () => { + it(`Should not work with HTTP @loki`, async () => { await server.close(); await server.clean(); diff --git a/tests/blob/pagewithdelimiter.test.ts b/tests/blob/pagewithdelimiter.test.ts index 4416adc5b..6daba2f01 100644 --- a/tests/blob/pagewithdelimiter.test.ts +++ b/tests/blob/pagewithdelimiter.test.ts @@ -36,31 +36,31 @@ describe("PageWithDelimiter", () => { "e/2" ]; - it("handles no blob results @loki", async () => { + it("handles no blob results @Loki @sql", async () => { const page = new PageWithDelimiter(5); const [items, prefixes, marker] = await page.fill(createReader([], 5), namer); checkResult(items, prefixes, marker, 0, 0, ""); }); - it("fills 1 result properly @loki", async () => { + it("fills 1 result properly @Loki @sql", async () => { const page = new PageWithDelimiter(1); const [items, prefixes, marker] = await page.fill(createReader(blobs, 1), namer); checkResult(items, prefixes, marker, 1, 0, "a"); }); - it("fills n results properly @loki", async () => { + it("fills n results properly @Loki @sql", async () => { const page = new PageWithDelimiter(5); const [items, prefixes, marker] = await page.fill(createReader(blobs, 5), namer); checkResult(items, prefixes, marker, 5, 0, "c/sub/1"); }); - it("fills exact count with no continuation @loki", async () => { + it("fills exact count with no continuation @Loki @sql", async () => { const page = new PageWithDelimiter(blobs.length); const [items, prefixes, marker] = await page.fill(createReader(blobs, blobs.length), namer); checkResult(items, prefixes, marker, blobs.length, 0, ""); }); - it("fills smaller than max page with no continuation @loki", async () => { + it("fills smaller than max page with no continuation @Loki @sql", async () => { const page = new PageWithDelimiter(blobs.length+1); const [items, prefixes, marker] = await page.fill(createReader(blobs, blobs.length+1), namer); checkResult(items, prefixes, marker, blobs.length, 0, ""); @@ -71,21 +71,21 @@ describe("PageWithDelimiter", () => { describe("and 1 item page size", () => { - it("handles no blob results @loki", async () => { + it("handles no blob results @Loki @sql", async () => { const blobs: string[] = []; const page = new PageWithDelimiter(1, "/"); const [items, prefixes, marker] = await page.fill(createReader(blobs, 1), namer); checkResult(items, prefixes, marker, 0, 0, ""); }); - it("handles 1 blob results @loki", async () => { + it("handles 1 blob results @Loki @sql", async () => { const blobs = ["a"]; const page = new PageWithDelimiter(1, "/"); const [items, prefixes, marker] = await page.fill(createReader(blobs, 1), namer); checkResult(items, prefixes, marker, 1, 0, ""); }); - it("returns 1 of 2 items with proper continuation @loki", async () => { + it("returns 1 of 2 items with proper continuation @Loki @sql", async () => { const blobs = ["a", "b"]; const page = new PageWithDelimiter(1, "/"); let [items, prefixes, marker] = await page.fill(createReader(blobs, 1), namer); @@ -97,14 +97,14 @@ describe("PageWithDelimiter", () => { checkResult(items, prefixes, marker, 1, 0, ""); }); - it("returns first item when prefixes exist @loki", async () => { + it("returns first item when prefixes exist @Loki @sql", async () => { const blobs = ["a/1", "a/2", "a/3", "a/sub/1"]; const page = new PageWithDelimiter(1, "/", "a/"); const [items, prefixes, marker] = await page.fill(createReader(blobs, 1), namer); checkResult(items, prefixes, marker, 1, 0, "a/1"); }); - it("returns first prefix when blobs exist @loki", async () => { + it("returns first prefix when blobs exist @Loki @sql", async () => { const blobs = ["a/s0/1", "a/s0/2", "a/s0/3", "a/s1/1", "a/s2/2", "a/z"]; const page = new PageWithDelimiter(1, "/", "a/"); const [items, prefixes, marker] = await page.fill(createReader(blobs, 1), namer); @@ -114,21 +114,21 @@ describe("PageWithDelimiter", () => { describe("multiple item page size", () => { - it("squashes prefixes @loki", async () => { + it("squashes prefixes @Loki @sql", async () => { const blobs = ["a/s0/1", "a/s0/2", "a/s0/3", "a/s1/1", "a/s1/2", "a/s2/2", "a/z"]; const page = new PageWithDelimiter(2, "/", "a/"); const [items, prefixes, marker] = await page.fill(createReader(blobs, 2), namer); checkResult(items, prefixes, marker, 0, 2, "a/s1/2"); }); - it("squashes a mix @loki", async () => { + it("squashes a mix @Loki @sql", async () => { const blobs = ["a/a", "a/s0/1", "a/s0/2", "a/s1/1", "a/s1/2", "a/z"]; const page = new PageWithDelimiter(2, "/", "a/"); const [items, prefixes, marker] = await page.fill(createReader(blobs, 2), namer); checkResult(items, prefixes, marker, 1, 1, "a/s0/2"); }); - it("follows squashed pages @loki", async () => { + it("follows squashed pages @Loki @sql", async () => { const blobs = ["a/a", "a/s0/1", "a/s0/2", "a/s1/1", "a/s1/2", "a/z"]; const page = new PageWithDelimiter(2, "/", "a/"); let [items, prefixes, marker] = await page.fill(createReader(blobs, 2), namer); @@ -140,7 +140,7 @@ describe("PageWithDelimiter", () => { checkResult(items, prefixes, marker, 1, 1, ""); }); - it("squashes within one larger page @loki", async () => { + it("squashes within one larger page @Loki @sql", async () => { const blobs = ["a/a", "a/s0/1", "a/s0/2", "a/s1/1", "a/s1/2", "a/z"]; const page = new PageWithDelimiter(4, "/", "a/"); let [items, prefixes, marker] = await page.fill(createReader(blobs, 4), namer); diff --git a/tests/blob/sas.test.ts b/tests/blob/sas.test.ts index ef368492f..9efe47815 100644 --- a/tests/blob/sas.test.ts +++ b/tests/blob/sas.test.ts @@ -83,7 +83,8 @@ describe("Shared Access Signature (SAS) authentication", () => { await server.clean(); }); - it("generateAccountSASQueryParameters should generate correct hashes", async () => { + //Doesn't work and shouldn't hardcode results and this test @azure/storage-blob not Azurite + it.skip("generateAccountSASQueryParameters should generate correct hashes", async () => { const startDate = new Date(2022, 3, 16, 14, 31, 48, 0); const endDate = new Date(2022, 3, 17, 14, 31, 48, 0); @@ -338,7 +339,7 @@ describe("Shared Access Signature (SAS) authentication", () => { assert.ok(error); }); - it("Synchronized copy blob should work with write permission in account SAS to override an existing blob @loki", async () => { + it("Synchronized copy blob should work with write permission in account SAS to override an existing blob @loki @sql", async () => { const tmr = new Date(); tmr.setDate(tmr.getDate() + 1); @@ -386,7 +387,7 @@ describe("Shared Access Signature (SAS) authentication", () => { await blob2.syncCopyFromURL(blob1.url); }); - it("Synchronized copy blob shouldn't work without write permission in account SAS to override an existing blob @loki", async () => { + it("Synchronized copy blob shouldn't work without write permission in account SAS to override an existing blob @loki @sql", async () => { const tmr = new Date(); tmr.setDate(tmr.getDate() + 1); @@ -441,7 +442,7 @@ describe("Shared Access Signature (SAS) authentication", () => { assert.ok(error !== undefined); }); - it("Synchronized copy blob should work without write permission in account SAS to an nonexisting blob @loki", async () => { + it("Synchronized copy blob should work without write permission in account SAS to an nonexisting blob @loki @sql", async () => { const tmr = new Date(); tmr.setDate(tmr.getDate() + 1); @@ -488,7 +489,7 @@ describe("Shared Access Signature (SAS) authentication", () => { await blob2.syncCopyFromURL(blob1.url); }); - it("Copy blob should work with write permission in account SAS to override an existing blob @loki", async () => { + it("Copy blob should work with write permission in account SAS to override an existing blob @loki @sql", async () => { const tmr = new Date(); tmr.setDate(tmr.getDate() + 1); @@ -536,7 +537,7 @@ describe("Shared Access Signature (SAS) authentication", () => { await blob2.beginCopyFromURL(blob1.url); }); - it("Copy blob shouldn't work without write permission in account SAS to override an existing blob @loki", async () => { + it("Copy blob shouldn't work without write permission in account SAS to override an existing blob @loki @sql", async () => { const tmr = new Date(); tmr.setDate(tmr.getDate() + 1); @@ -591,7 +592,7 @@ describe("Shared Access Signature (SAS) authentication", () => { assert.ok(error !== undefined); }); - it("Copy blob should work without write permission in account SAS to an nonexisting blob @loki", async () => { + it("Copy blob should work without write permission in account SAS to an nonexisting blob @loki @sql", async () => { const tmr = new Date(); tmr.setDate(tmr.getDate() + 1); @@ -1135,7 +1136,7 @@ describe("Shared Access Signature (SAS) authentication", () => { await containerClient.delete(); }); - it("Synchronized copy blob should work with write permission in blob SAS to override an existing blob @loki", async () => { + it("Synchronized copy blob should work with write permission in blob SAS to override an existing blob @loki @sql", async () => { const now = new Date(); now.setMinutes(now.getMinutes() - 5); // Skip clock skew with server @@ -1182,7 +1183,7 @@ describe("Shared Access Signature (SAS) authentication", () => { await blob2SAS.syncCopyFromURL(blob1.url); }); - it("Synchronized copy blob shouldn't work without write permission in blob SAS to override an existing blob @loki", async () => { + it("Synchronized copy blob shouldn't work without write permission in blob SAS to override an existing blob @loki @sql", async () => { const now = new Date(); now.setMinutes(now.getMinutes() - 5); // Skip clock skew with server @@ -1236,7 +1237,7 @@ describe("Shared Access Signature (SAS) authentication", () => { assert.ok(error !== undefined); }); - it("Synchronized copy blob should work without write permission in account SAS to an nonexisting blob @loki", async () => { + it("Synchronized copy blob should work without write permission in account SAS to an nonexisting blob @loki @sql", async () => { const now = new Date(); now.setMinutes(now.getMinutes() - 5); // Skip clock skew with server @@ -1730,7 +1731,7 @@ describe("Shared Access Signature (SAS) authentication", () => { assert.equal(properties2.metadata!["baz"], "3"); }); - it("Copy blob across accounts should fail if source is archived @loki @sql", async () => { + it("Copy blob across accounts should fail if source is archived @loki", async () => { const now = new Date(); now.setMinutes(now.getMinutes() - 5); // Skip clock skew with server @@ -1783,7 +1784,7 @@ describe("Shared Access Signature (SAS) authentication", () => { assert.equal(error.details.code, "BlobArchived"); }); - it("Sync Copy blob across accounts should work and honor metadata when provided @loki", async () => { + it("Sync Copy blob across accounts should work and honor metadata when provided @loki @sql", async () => { const now = new Date(); now.setMinutes(now.getMinutes() - 5); // Skip clock skew with server diff --git a/tests/blob/specialnaming.test.ts b/tests/blob/specialnaming.test.ts index 24a6be889..b9a8aa744 100644 --- a/tests/blob/specialnaming.test.ts +++ b/tests/blob/specialnaming.test.ts @@ -448,7 +448,7 @@ describe("SpecialNaming", () => { it(`Should work with production style URL when ${productionStyleHostName} is resolvable`, async () => { await dns.promises.lookup(productionStyleHostName).then( async (lookupAddress) => { - const baseURLProductionStyle = `http://${productionStyleHostName}:${server.config.port}`; + const baseURLProductionStyle = `http://${productionStyleHostName}:${server.config.port}/`; const serviceClientProductionStyle = new BlobServiceClient( baseURLProductionStyle, newPipeline( @@ -463,14 +463,12 @@ describe("SpecialNaming", () => { } ) ); - const containerClientProductionStyle = serviceClientProductionStyle.getContainerClient( - containerName - ); + const containerClientProductionStyle = + serviceClientProductionStyle.getContainerClient(containerName); const blobName: string = getUniqueName("myblob"); - const blockBlobClient = containerClientProductionStyle.getBlockBlobClient( - blobName - ); + const blockBlobClient = + containerClientProductionStyle.getBlockBlobClient(blobName); await blockBlobClient.upload("ABC", 3); const response = ( @@ -508,14 +506,12 @@ describe("SpecialNaming", () => { keepAliveOptions: { enable: false } } ); - const containerClientProductionStyle = serviceClientNoHostName.getContainerClient( - containerName - ); + const containerClientProductionStyle = + serviceClientNoHostName.getContainerClient(containerName); const blobName: string = getUniqueName("myblob"); - const blockBlobClient = containerClientProductionStyle.getBlockBlobClient( - blobName - ); + const blockBlobClient = + containerClientProductionStyle.getBlockBlobClient(blobName); await blockBlobClient.upload("ABC", 3); const response = ( diff --git a/tests/dfs/DataLakeTestServerFactory.ts b/tests/dfs/DataLakeTestServerFactory.ts new file mode 100644 index 000000000..d0fd11848 --- /dev/null +++ b/tests/dfs/DataLakeTestServerFactory.ts @@ -0,0 +1,78 @@ +import { StoreDestinationArray } from "../../src/common/persistence/IExtentStore"; +import { DEFAULT_SQL_OPTIONS } from "../../src/common/utils/constants"; +import DataLakeConfiguration from "../../src/dfs/DataLakeConfiguration"; +import DataLakeServer from "../../src/dfs/DataLakeServer"; +import SqlDataLakeConfiguration from "../../src/dfs/SqlDataLakeConfiguration"; +import SqlDataLakeServer from "../../src/dfs/SqlDataLakeServer"; + +export default class DataLakeTestServerFactory { + public createServer( + loose: boolean = false, + skipApiVersionCheck: boolean = false, + https: boolean = false, + oauth?: string + ): DataLakeServer | SqlDataLakeServer { + let databaseConnectionString = process.env.AZURITE_TEST_DB; + // if (databaseConnectionString?.startsWith("mysql")) { + // databaseConnectionString = "postgres://postgres:postgres@127.0.0.1:5432/azurite_dfs_test"; + // } + const isSQL = databaseConnectionString !== undefined; + + const port = 11003; + const host = "127.0.0.1"; + const persistenceArray: StoreDestinationArray = [ + { + locationId: "test", + locationPath: "__test_blob_extent__", + maxConcurrency: 10 + } + ]; + const cert = https ? "tests/server.cert" : undefined; + const key = https ? "tests/server.key" : undefined; + + if (isSQL) { + const config = new SqlDataLakeConfiguration( + host, + port + 100, + databaseConnectionString!, + DEFAULT_SQL_OPTIONS, + persistenceArray, + false, + undefined, + false, + undefined, + loose, + skipApiVersionCheck, + cert, + key, + undefined, + oauth, + false, + true + ); + + return new SqlDataLakeServer(config); + } else { + const lokiMetadataDBPath = "__test_db_blob__.json"; + const lokiExtentDBPath = "__test_db_blob_extent__.json"; + const config = new DataLakeConfiguration( + host, + port, + lokiMetadataDBPath, + lokiExtentDBPath, + persistenceArray, + false, + undefined, + false, + undefined, + loose, + skipApiVersionCheck, + cert, + key, + undefined, + oauth + ); + return new DataLakeServer(config); + } + } +} diff --git a/tests/dfs/RequestPolicy/OPTIONSRequestPolicyFactory.ts b/tests/dfs/RequestPolicy/OPTIONSRequestPolicyFactory.ts new file mode 100644 index 000000000..57e042fcc --- /dev/null +++ b/tests/dfs/RequestPolicy/OPTIONSRequestPolicyFactory.ts @@ -0,0 +1,72 @@ +import { BaseRequestPolicy, WebResource } from "@azure/storage-blob"; + +// Create a policy factory with create() method provided +// In TypeScript, following factory class needs to implement Azure.RequestPolicyFactory type +export default class OPTIONSRequestPolicyFactory { + // Constructor to accept parameters + private origin: string | undefined; + private requstMethod: string | undefined; + private requestHeaders: string | undefined; + + constructor(origin?: string, requstMethod?: string, requestHeaders?: string) { + this.origin = origin; + this.requstMethod = requstMethod; + this.requestHeaders = requestHeaders; + } + + // create() method needs to create a new RequestIDPolicy object + create(nextPolicy: any, options: any) { + return new OPTIONSRequestPolicy( + nextPolicy, + options, + this.origin, + this.requstMethod, + this.requestHeaders + ); + } +} + +// Create a policy by extending from Azure.BaseRequestPolicy +class OPTIONSRequestPolicy extends BaseRequestPolicy { + private origin: string | undefined; + private requstMethod: string | undefined; + private requestHeaders: string | undefined; + constructor( + nextPolicy: any, + options: any, + origin?: string, + requstMethod?: string, + requestHeaders?: string + ) { + super(nextPolicy, options); + this.origin = origin; + this.requstMethod = requstMethod; + this.requestHeaders = requestHeaders; + } + + // Customize HTTP requests and responses by overriding sendRequest + // Parameter request is Azure.WebResource type + async sendRequest(request: WebResource) { + request.method = "OPTIONS"; + request.headers.set("Origin", `${this.origin}`); + if (this.origin !== undefined) { + request.headers.set("Origin", `${this.origin}`); + } + if (this.requstMethod !== undefined) { + request.headers.set( + "Access-Control-Request-Method", + `${this.requstMethod}` + ); + } + if (this.requestHeaders !== undefined) { + request.headers.set( + "Access-Control-Request-Headers", + `${this.requestHeaders}` + ); + } + + const response = await this._nextPolicy.sendRequest(request); + + return response; + } +} diff --git a/tests/dfs/RequestPolicy/OriginPolicyFactory.ts b/tests/dfs/RequestPolicy/OriginPolicyFactory.ts new file mode 100644 index 000000000..a95efe526 --- /dev/null +++ b/tests/dfs/RequestPolicy/OriginPolicyFactory.ts @@ -0,0 +1,35 @@ +import { BaseRequestPolicy, WebResource } from "@azure/storage-blob"; + +// Create a policy factory with create() method provided +// In TypeScript, following factory class needs to implement Azure.RequestPolicyFactory type +export default class OriginPolicyFactory { + // Constructor to accept parameters + private origin: string; + constructor(origin: any) { + this.origin = origin; + } + + // create() method needs to create a new RequestIDPolicy object + create(nextPolicy: any, options: any) { + return new OriginPolicy(nextPolicy, options, this.origin); + } +} + +// Create a policy by extending from Azure.BaseRequestPolicy +class OriginPolicy extends BaseRequestPolicy { + private origin: string; + constructor(nextPolicy: any, options: any, origin: any) { + super(nextPolicy, options); + this.origin = origin; + } + + // Customize HTTP requests and responses by overriding sendRequest + // Parameter request is Azure.WebResource type + async sendRequest(request: WebResource) { + request.headers.set("Origin", `${this.origin}`); + + const response = await this._nextPolicy.sendRequest(request); + + return response; + } +} diff --git a/tests/dfs/RequestPolicy/QueryRequestPolicyFactory.ts b/tests/dfs/RequestPolicy/QueryRequestPolicyFactory.ts new file mode 100644 index 000000000..17cbac473 --- /dev/null +++ b/tests/dfs/RequestPolicy/QueryRequestPolicyFactory.ts @@ -0,0 +1,52 @@ +import { BaseRequestPolicy, WebResource } from "@azure/storage-blob"; + +// Replace the Query in Uri, from old value to new value. This need run with SAS authentication. +// Create a policy factory with create() method provided +// In TypeScript, following factory class needs to implement Azure.RequestPolicyFactory type +export default class QueryRequestPolicyFactory { + // Constructor to accept parameters + private oldValue: string; + private newValue: string; + + constructor(oldValue: string, newValue: string) { + this.oldValue = oldValue; + this.newValue = newValue; + } + + // create() method needs to create a new RequestIDPolicy object + create(nextPolicy: any, options: any) { + return new QueryRequestPolicy( + nextPolicy, + options, + this.oldValue, + this.newValue + ); + } +} + +// Create a policy by extending from Azure.BaseRequestPolicy +class QueryRequestPolicy extends BaseRequestPolicy { + private oldValue: string; + private newValue: string; + + constructor( + nextPolicy: any, + options: any, + oldValue: string, + newValue: string + ) { + super(nextPolicy, options); + this.oldValue = oldValue; + this.newValue = newValue; + } + + // Customize HTTP requests and responses by overriding sendRequest + // Parameter request is Azure.WebResource type + async sendRequest(request: WebResource) { + request.url = request.url.replace(this.oldValue, this.newValue); + + const response = await this._nextPolicy.sendRequest(request); + + return response; + } +} diff --git a/tests/dfs/apis/appendblob.test.ts b/tests/dfs/apis/appendblob.test.ts new file mode 100644 index 000000000..442d307a4 --- /dev/null +++ b/tests/dfs/apis/appendblob.test.ts @@ -0,0 +1,562 @@ +import { + BlobServiceClient, + newPipeline, + StorageSharedKeyCredential +} from "@azure/storage-blob"; + +import { configLogger } from "../../../src/common/Logger"; +import { getMD5FromString } from "../../../src/common/utils/utils"; +import { BlobType } from "../../../src/dfs/generated/artifacts/models"; +import { + bodyToString, + EMULATOR_ACCOUNT_KEY, + EMULATOR_ACCOUNT_NAME, + getUniqueName, + sleep +} from "../../testutils"; +import DataLakeTestServerFactory from "../DataLakeTestServerFactory"; + +import assert = require("assert"); + +// Set true to enable debug log +configLogger(false); + +describe("AppendBlobAPIs", () => { + const factory = new DataLakeTestServerFactory(); + const server = factory.createServer(); + + const baseURL = `http://${server.config.host}:${server.config.port}/devstoreaccount1`; + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ) + ); + + let containerName: string = getUniqueName("container"); + let containerClient = serviceClient.getContainerClient(containerName); + let blobName: string = getUniqueName("blob"); + let blobClient = containerClient.getBlobClient(blobName); + let appendBlobClient = blobClient.getAppendBlobClient(); + + before(async () => { + await server.start(); + }); + + after(async () => { + await server.close(); + await server.clean(); + }); + + beforeEach(async () => { + containerName = getUniqueName("container"); + containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + blobName = getUniqueName("blob"); + blobClient = containerClient.getBlobClient(blobName); + appendBlobClient = blobClient.getAppendBlobClient(); + }); + + afterEach(async () => { + await containerClient.delete(); + }); + + it("Create append blob should work @loki @sql", async () => { + await appendBlobClient.create(); + const properties = await appendBlobClient.getProperties(); + assert.deepStrictEqual(properties.blobType, "AppendBlob"); + assert.deepStrictEqual(properties.leaseState, "available"); + assert.deepStrictEqual(properties.leaseStatus, "unlocked"); + assert.deepStrictEqual(properties.contentLength, 0); + assert.deepStrictEqual(properties.contentType, "application/octet-stream"); + assert.deepStrictEqual(properties.contentMD5, undefined); + assert.deepStrictEqual(properties.contentEncoding, undefined); + assert.deepStrictEqual(properties.contentDisposition, undefined); + assert.deepStrictEqual(properties.contentLanguage, undefined); + assert.deepStrictEqual(properties.cacheControl, undefined); + assert.deepStrictEqual(properties.blobSequenceNumber, undefined); + assert.deepStrictEqual(properties.blobCommittedBlockCount, 0); + }); + + it("Create append blob override existing pageblob @loki @sql", async () => { + const pageBlobClient = blobClient.getPageBlobClient(); + await pageBlobClient.create(512); + + const md5 = new Uint8Array([1, 2, 3, 4, 5]); + const headers = { + blobCacheControl: "blobCacheControl_", + blobContentType: "blobContentType_", + blobContentMD5: md5, + blobContentEncoding: "blobContentEncoding_", + blobContentLanguage: "blobContentLanguage_", + blobContentDisposition: "blobContentDisposition_" + }; + + const metadata = { + key1: "value1", + key2: "val2" + }; + + await appendBlobClient.create({ + blobHTTPHeaders: headers, + metadata + }); + const properties = await appendBlobClient.getProperties(); + assert.deepStrictEqual(properties.blobType, "AppendBlob"); + assert.deepStrictEqual(properties.leaseState, "available"); + assert.deepStrictEqual(properties.leaseStatus, "unlocked"); + assert.deepStrictEqual(properties.contentLength, 0); + assert.deepStrictEqual(properties.contentType, headers.blobContentType); + assert.deepEqual(properties.contentMD5, md5); + assert.deepStrictEqual( + properties.contentEncoding, + headers.blobContentEncoding + ); + assert.deepStrictEqual( + properties.contentDisposition, + headers.blobContentDisposition + ); + assert.deepStrictEqual( + properties.contentLanguage, + headers.blobContentLanguage + ); + assert.deepStrictEqual(properties.cacheControl, headers.blobCacheControl); + assert.deepStrictEqual(properties.metadata, metadata); + assert.deepStrictEqual(properties.blobSequenceNumber, undefined); + assert.deepStrictEqual(properties.blobCommittedBlockCount, 0); + }); + + it("Delete append blob should work @loki @sql", async () => { + await appendBlobClient.create(); + await appendBlobClient.delete(); + }); + + it("Create append blob snapshot should work @loki @sql", async () => { + await appendBlobClient.create(); + const response = await appendBlobClient.createSnapshot(); + const appendBlobSnapshotClient = appendBlobClient.withSnapshot( + response.snapshot! + ); + + await appendBlobClient.appendBlock("hello", 5); + + let properties = await appendBlobClient.getProperties(); + assert.deepStrictEqual(properties.blobType, "AppendBlob"); + assert.deepStrictEqual(properties.leaseState, "available"); + assert.deepStrictEqual(properties.leaseStatus, "unlocked"); + assert.deepStrictEqual(properties.contentLength, 5); + assert.deepStrictEqual(properties.contentType, "application/octet-stream"); + assert.deepStrictEqual(properties.contentMD5, undefined); + assert.deepStrictEqual(properties.contentEncoding, undefined); + assert.deepStrictEqual(properties.contentDisposition, undefined); + assert.deepStrictEqual(properties.contentLanguage, undefined); + assert.deepStrictEqual(properties.cacheControl, undefined); + assert.deepStrictEqual(properties.blobSequenceNumber, undefined); + assert.deepStrictEqual(properties.blobCommittedBlockCount, 1); + + properties = await appendBlobSnapshotClient.getProperties(); + assert.deepStrictEqual(properties.blobType, "AppendBlob"); + assert.deepStrictEqual(properties.leaseState, "available"); + assert.deepStrictEqual(properties.leaseStatus, "unlocked"); + assert.deepStrictEqual(properties.contentLength, 0); + assert.deepStrictEqual(properties.contentType, "application/octet-stream"); + assert.deepStrictEqual(properties.contentMD5, undefined); + assert.deepStrictEqual(properties.contentEncoding, undefined); + assert.deepStrictEqual(properties.contentDisposition, undefined); + assert.deepStrictEqual(properties.contentLanguage, undefined); + assert.deepStrictEqual(properties.cacheControl, undefined); + assert.deepStrictEqual(properties.blobSequenceNumber, undefined); + assert.deepStrictEqual(properties.blobCommittedBlockCount, 0); + }); + + it("Copy append blob snapshot should work @loki @sql", async () => { + await appendBlobClient.create(); + await appendBlobClient.appendBlock("hello", 5); + + const response = await appendBlobClient.createSnapshot(); + const appendBlobSnapshotClient = appendBlobClient.withSnapshot( + response.snapshot! + ); + + await appendBlobClient.appendBlock("world", 5); + + const destAppendBlobClient = + containerClient.getAppendBlobClient("copiedAppendBlob"); + await destAppendBlobClient.beginCopyFromURL(appendBlobSnapshotClient.url); + + let properties = await appendBlobClient.getProperties(); + assert.deepStrictEqual(properties.contentLength, 10); + assert.deepStrictEqual(properties.blobCommittedBlockCount, 2); + + properties = await appendBlobSnapshotClient.getProperties(); + assert.deepStrictEqual(properties.contentLength, 5); + assert.deepStrictEqual(properties.blobCommittedBlockCount, 1); + + await appendBlobClient.delete({ deleteSnapshots: "include" }); + + properties = await destAppendBlobClient.getProperties(); + assert.deepStrictEqual(properties.contentLength, 5); + assert.deepStrictEqual(properties.blobCommittedBlockCount, 1); + assert.ok(properties.copyId); + assert.ok(properties.copyCompletedOn); + assert.deepStrictEqual(properties.copyProgress, "5/5"); + assert.deepStrictEqual(properties.copySource, appendBlobSnapshotClient.url); + assert.deepStrictEqual(properties.copyStatus, "success"); + }); + + it("Synchronized copy append blob snapshot should work @loki @sql", async () => { + await appendBlobClient.create(); + await appendBlobClient.appendBlock("hello", 5); + + const response = await appendBlobClient.createSnapshot(); + const appendBlobSnapshotClient = appendBlobClient.withSnapshot( + response.snapshot! + ); + + await appendBlobClient.appendBlock("world", 5); + + const destAppendBlobClient = + containerClient.getAppendBlobClient("copiedAppendBlob"); + await destAppendBlobClient.syncCopyFromURL(appendBlobSnapshotClient.url); + + let properties = await appendBlobClient.getProperties(); + assert.deepStrictEqual(properties.contentLength, 10); + assert.deepStrictEqual(properties.blobCommittedBlockCount, 2); + + properties = await appendBlobSnapshotClient.getProperties(); + assert.deepStrictEqual(properties.contentLength, 5); + assert.deepStrictEqual(properties.blobCommittedBlockCount, 1); + + await appendBlobClient.delete({ deleteSnapshots: "include" }); + + properties = await destAppendBlobClient.getProperties(); + assert.deepStrictEqual(properties.contentLength, 5); + assert.deepStrictEqual(properties.blobCommittedBlockCount, 1); + assert.ok(properties.copyId); + assert.ok(properties.copyCompletedOn); + assert.deepStrictEqual(properties.copyProgress, "5/5"); + assert.deepStrictEqual(properties.copySource, appendBlobSnapshotClient.url); + }); + + it("Set append blob metadata should work @loki @sql", async () => { + await appendBlobClient.create(); + + const metadata = { + key1: "value1", + key2: "val2" + }; + await appendBlobClient.setMetadata(metadata); + + const properties = await appendBlobClient.getProperties(); + assert.deepStrictEqual(properties.metadata, metadata); + }); + + it("Set append blob HTTP headers should work @loki @sql", async () => { + await appendBlobClient.create(); + + const md5 = new Uint8Array([1, 2, 3, 4, 5]); + const headers = { + blobCacheControl: "blobCacheControl_", + blobContentType: "blobContentType_", + blobContentMD5: md5, + blobContentEncoding: "blobContentEncoding_", + blobContentLanguage: "blobContentLanguage_", + blobContentDisposition: "blobContentDisposition_" + }; + await appendBlobClient.setHTTPHeaders(headers); + + const properties = await appendBlobClient.getProperties(); + assert.deepStrictEqual(properties.cacheControl, headers.blobCacheControl); + assert.deepStrictEqual(properties.contentType, headers.blobContentType); + assert.deepEqual(properties.contentMD5, headers.blobContentMD5); + assert.deepStrictEqual( + properties.contentEncoding, + headers.blobContentEncoding + ); + assert.deepStrictEqual( + properties.contentLanguage, + headers.blobContentLanguage + ); + assert.deepStrictEqual( + properties.contentDisposition, + headers.blobContentDisposition + ); + }); + + it("Set tier should not work for append blob @loki @sql", async function () { + await appendBlobClient.create(); + try { + await blobClient.setAccessTier("hot"); + } catch (err) { + return; + } + assert.fail(); + }); + + it("Append block should work @loki @sql", async () => { + await appendBlobClient.create(); + let appendBlockResponse = await appendBlobClient.appendBlock("abcdef", 6); + assert.deepStrictEqual(appendBlockResponse.blobAppendOffset, "0"); + + const properties1 = await appendBlobClient.getProperties(); + assert.deepStrictEqual(properties1.blobType, "AppendBlob"); + assert.deepStrictEqual(properties1.leaseState, "available"); + assert.deepStrictEqual(properties1.leaseStatus, "unlocked"); + assert.deepStrictEqual(properties1.contentLength, 6); + assert.deepStrictEqual(properties1.contentType, "application/octet-stream"); + assert.deepStrictEqual(properties1.contentMD5, undefined); + assert.deepStrictEqual(properties1.contentEncoding, undefined); + assert.deepStrictEqual(properties1.contentDisposition, undefined); + assert.deepStrictEqual(properties1.contentLanguage, undefined); + assert.deepStrictEqual(properties1.cacheControl, undefined); + assert.deepStrictEqual(properties1.blobSequenceNumber, undefined); + assert.deepStrictEqual(properties1.blobCommittedBlockCount, 1); + assert.deepStrictEqual(properties1.etag, appendBlockResponse.etag); + + await sleep(1000); // Sleep 1 second to make sure last modified time changed + appendBlockResponse = await appendBlobClient.appendBlock("123456", 6); + assert.deepStrictEqual(appendBlockResponse.blobAppendOffset, "6"); + assert.notDeepStrictEqual(appendBlockResponse.etag, properties1.etag); + appendBlockResponse = await appendBlobClient.appendBlock("T", 1); + assert.deepStrictEqual(appendBlockResponse.blobAppendOffset, "12"); + appendBlockResponse = await appendBlobClient.appendBlock("@", 2); + assert.deepStrictEqual(appendBlockResponse.blobAppendOffset, "13"); + + const properties2 = await appendBlobClient.getProperties(); + assert.deepStrictEqual(properties2.blobType, "AppendBlob"); + assert.deepStrictEqual(properties2.leaseState, "available"); + assert.deepStrictEqual(properties2.leaseStatus, "unlocked"); + assert.deepStrictEqual(properties2.contentLength, 14); + assert.deepStrictEqual(properties2.contentType, "application/octet-stream"); + assert.deepStrictEqual(properties2.contentMD5, undefined); + assert.deepStrictEqual(properties2.contentEncoding, undefined); + assert.deepStrictEqual(properties2.contentDisposition, undefined); + assert.deepStrictEqual(properties2.contentLanguage, undefined); + assert.deepStrictEqual(properties2.cacheControl, undefined); + assert.deepStrictEqual(properties2.blobSequenceNumber, undefined); + assert.deepStrictEqual(properties2.blobCommittedBlockCount, 4); + assert.deepStrictEqual(properties1.createdOn, properties2.createdOn); + assert.notDeepStrictEqual( + properties1.lastModified, + properties2.lastModified + ); + assert.notDeepStrictEqual(properties1.etag, properties2.etag); + + const response = await appendBlobClient.download(0); + const string = await bodyToString(response, response.contentLength); + + assert.deepStrictEqual(string, "abcdef123456T@"); + }); + + it("Download append blob should work @loki @sql", async () => { + await appendBlobClient.create(); + await appendBlobClient.appendBlock("abcdef", 6); + await appendBlobClient.appendBlock("123456", 6); + await appendBlobClient.appendBlock("T", 1); + await appendBlobClient.appendBlock("@", 2); + + const response = await appendBlobClient.download(5, 8); + const string = await bodyToString(response, response.contentLength); + assert.deepStrictEqual(string, "f123456T"); + assert.deepStrictEqual(response.blobCommittedBlockCount, 4); + assert.deepStrictEqual(response.blobType, BlobType.AppendBlob); + assert.deepStrictEqual(response.acceptRanges, "bytes"); + assert.deepStrictEqual(response.contentLength, 8); + assert.deepStrictEqual(response.contentRange, "bytes 5-12/14"); + }); + + it("Download append blob should work for snapshot @loki @sql", async () => { + await appendBlobClient.create(); + await appendBlobClient.appendBlock("abcdef", 6); + + const snapshotResponse = await appendBlobClient.createSnapshot(); + const snapshotAppendBlobURL = appendBlobClient.withSnapshot( + snapshotResponse.snapshot! + ); + + await appendBlobClient.appendBlock("123456", 6); + await appendBlobClient.appendBlock("T", 1); + await appendBlobClient.appendBlock("@", 2); + + const response = await snapshotAppendBlobURL.download(3); + const string = await bodyToString(response); + assert.deepStrictEqual(string, "def"); + assert.deepEqual(response.contentMD5, await getMD5FromString("def")); + }); + + it("Download append blob should work for copied blob @loki @sql", async () => { + await appendBlobClient.create(); + await appendBlobClient.appendBlock("abcdef", 6); + + const copiedAppendBlobClient = + containerClient.getAppendBlobClient("copiedAppendBlob"); + await copiedAppendBlobClient.beginCopyFromURL(appendBlobClient.url); + + await appendBlobClient.delete(); + + const response = await copiedAppendBlobClient.download(3); + const string = await bodyToString(response); + assert.deepStrictEqual(string, "def"); + assert.deepEqual(response.contentMD5, await getMD5FromString("def")); + }); + + it("Append block with invalid blob type should not work @loki @sql", async () => { + const pageBlobClient = appendBlobClient.getPageBlobClient(); + await pageBlobClient.create(512); + + try { + await appendBlobClient.appendBlock("a", 1); + } catch (err) { + assert.deepStrictEqual(err.code, "InvalidBlobType"); + return; + } + assert.fail(); + }); + + it("Append block with content length 0 should not work @loki @sql", async () => { + await appendBlobClient.create(); + + try { + await appendBlobClient.appendBlock("", 0); + } catch (err) { + assert.deepStrictEqual(err.code, "InvalidHeaderValue"); + return; + } + assert.fail(); + }); + + it("Append block append position access condition should work @loki @sql", async () => { + await appendBlobClient.create(); + await appendBlobClient.appendBlock("a", 1, { + conditions: { + maxSize: 1, + appendPosition: 0 + } + }); + + try { + await appendBlobClient.appendBlock("a", 1, { + conditions: { + maxSize: 1 + } + }); + } catch (err) { + assert.deepStrictEqual(err.code, "MaxBlobSizeConditionNotMet"); + assert.deepStrictEqual(err.statusCode, 412); + + await appendBlobClient.appendBlock("a", 1, { + conditions: { + appendPosition: 1 + } + }); + + try { + await appendBlobClient.appendBlock("a", 1, { + conditions: { + appendPosition: 0 + } + }); + } catch (err) { + assert.deepStrictEqual(err.code, "AppendPositionConditionNotMet"); + assert.deepStrictEqual(err.statusCode, 412); + return; + } + assert.fail(); + } + assert.fail(); + }); + + it("Append block md5 validation should work @loki @sql", async () => { + await appendBlobClient.create(); + await appendBlobClient.appendBlock("aEf", 1, { + transactionalContentMD5: await getMD5FromString("aEf") + }); + + try { + await appendBlobClient.appendBlock("aEf", 1, { + transactionalContentMD5: await getMD5FromString("invalid") + }); + } catch (err) { + assert.deepStrictEqual(err.code, "Md5Mismatch"); + assert.deepStrictEqual(err.statusCode, 400); + return; + } + assert.fail(); + }); + + it("Append block access condition should work @loki @sql", async () => { + let response = await appendBlobClient.create(); + response = await appendBlobClient.appendBlock("a", 1, { + conditions: { + ifMatch: response.etag + } + }); + + response = await appendBlobClient.appendBlock("a", 1, { + conditions: { + ifNoneMatch: "xxxx" + } + }); + + response = await appendBlobClient.appendBlock("a", 1, { + conditions: { + ifModifiedSince: new Date("2000/01/01") + } + }); + + response = await appendBlobClient.appendBlock("a", 1, { + conditions: { + ifUnmodifiedSince: response.lastModified + } + }); + + try { + await appendBlobClient.appendBlock("a", 1, { + conditions: { + ifMatch: response.etag + "2" + } + }); + } catch (err) { + assert.deepStrictEqual(err.code, "ConditionNotMet"); + assert.deepStrictEqual(err.statusCode, 412); + return; + } + assert.fail(); + }); + + it("Append block lease condition should work @loki @sql", async () => { + await appendBlobClient.create(); + + const leaseId = "abcdefg"; + const blobLeaseClient = appendBlobClient.getBlobLeaseClient(leaseId); + await blobLeaseClient.acquireLease(20); + + const properties = await appendBlobClient.getProperties(); + assert.deepStrictEqual(properties.leaseDuration, "fixed"); + assert.deepStrictEqual(properties.leaseState, "leased"); + assert.deepStrictEqual(properties.leaseStatus, "locked"); + + await appendBlobClient.appendBlock("a", 1, { + conditions: { + leaseId + } + }); + + try { + await appendBlobClient.appendBlock("c", 1); + } catch (err) { + assert.deepStrictEqual(err.code, "LeaseIdMissing"); + assert.deepStrictEqual(err.statusCode, 412); + return; + } + assert.fail(); + }); +}); diff --git a/tests/dfs/apis/blob.test.ts b/tests/dfs/apis/blob.test.ts new file mode 100644 index 000000000..e4293b1b3 --- /dev/null +++ b/tests/dfs/apis/blob.test.ts @@ -0,0 +1,1345 @@ +import { isNode } from "@azure/ms-rest-js"; +import { + BlobServiceClient, + newPipeline, + StorageSharedKeyCredential +} from "@azure/storage-blob"; + +import { configLogger } from "../../../src/common/Logger"; +import { BlobHTTPHeaders } from "../../../src/dfs/generated/artifacts/models"; +import { + bodyToString, + EMULATOR_ACCOUNT_KEY, + EMULATOR_ACCOUNT_NAME, + getUniqueName, + sleep +} from "../../testutils"; +import DataLakeTestServerFactory from "../DataLakeTestServerFactory"; + +import assert = require("assert"); + +// Set true to enable debug log +configLogger(false); + +describe("BlobAPIs", () => { + const factory = new DataLakeTestServerFactory(); + const server = factory.createServer(); + + const baseURL = `http://${server.config.host}:${server.config.port}/devstoreaccount1`; + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ) + ); + + let containerName: string = getUniqueName("container"); + let containerClient = serviceClient.getContainerClient(containerName); + let blobName: string = getUniqueName("blob"); + let blobClient = containerClient.getBlobClient(blobName); + let blockBlobClient = blobClient.getBlockBlobClient(); + let blobLeaseClient = blobClient.getBlobLeaseClient(); + const content = "Hello World"; + + before(async () => { + await server.start(); + }); + + after(async () => { + await server.close(); + await server.clean(); + }); + + beforeEach(async () => { + containerName = getUniqueName("container"); + containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + blobName = getUniqueName("blob"); + blobClient = containerClient.getBlobClient(blobName); + blockBlobClient = blobClient.getBlockBlobClient(); + blobLeaseClient = blobClient.getBlobLeaseClient(); + await blockBlobClient.upload(content, content.length); + }); + + afterEach(async () => { + await containerClient.delete(); + }); + + it("download with with default parameters @loki @sql", async () => { + const result = await blobClient.download(0); + assert.deepStrictEqual(await bodyToString(result, content.length), content); + assert.equal(result.contentRange, undefined); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + }); + + it("download should work with conditional headers @loki @sql", async () => { + const properties = await blobClient.getProperties(); + const result = await blobClient.download(0, undefined, { + conditions: { + ifMatch: properties.etag, + ifNoneMatch: "invalidetag", + ifModifiedSince: new Date("2018/01/01"), + ifUnmodifiedSince: new Date("2188/01/01") + } + }); + assert.deepStrictEqual(await bodyToString(result, content.length), content); + assert.equal(result.contentRange, undefined); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + }); + + it("download should work with ifMatch value * @loki @sql", async () => { + const result = await blobClient.download(0, undefined, { + conditions: { + ifMatch: "*,abc", + ifNoneMatch: "invalidetag", + ifModifiedSince: new Date("2018/01/01"), + ifUnmodifiedSince: new Date("2188/01/01") + } + }); + assert.deepStrictEqual(await bodyToString(result, content.length), content); + assert.equal(result.contentRange, undefined); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + }); + + it("download should not work with invalid conditional header ifMatch @loki @sql", async () => { + const properties = await blobClient.getProperties(); + try { + await blobClient.download(0, undefined, { + conditions: { + ifMatch: properties.etag + "invalid" + } + }); + } catch (error) { + assert.deepStrictEqual(error.statusCode, 412); + return; + } + assert.fail(); + }); + + it("download should not work with conditional header ifNoneMatch @loki @sql", async () => { + const properties = await blobClient.getProperties(); + try { + await blobClient.download(0, undefined, { + conditions: { + ifNoneMatch: properties.etag + } + }); + } catch (error) { + assert.deepStrictEqual(error.statusCode, 304); + return; + } + assert.fail(); + }); + + it("download should not work with conditional header ifNoneMatch * @loki @sql", async () => { + try { + await blobClient.download(0, undefined, { + conditions: { + ifNoneMatch: "*" + } + }); + } catch (error) { + assert.deepStrictEqual(error.statusCode, 400); + return; + } + assert.fail(); + }); + + it("download should not work with conditional header ifModifiedSince @loki @sql", async () => { + try { + await blobClient.download(0, undefined, { + conditions: { + ifModifiedSince: new Date("2120/01/01") + } + }); + } catch (error) { + assert.deepStrictEqual(error.statusCode, 304); + return; + } + assert.fail(); + }); + + it("download should not work with conditional header ifUnmodifiedSince @loki @sql", async () => { + try { + await blobClient.download(0, undefined, { + conditions: { + ifUnmodifiedSince: new Date("2018/01/01") + } + }); + } catch (error) { + assert.deepStrictEqual(error.statusCode, 412); + return; + } + assert.fail(); + }); + + it("download all parameters set @loki @sql", async () => { + const result = await blobClient.download(0, 1, { + rangeGetContentMD5: true + }); + assert.deepStrictEqual(await bodyToString(result, 1), content[0]); + assert.equal(result.contentRange, `bytes 0-0/${content.length}`); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + }); + + it("download entire with range @loki @sql", async () => { + const result = await blobClient.download(0, content.length); + assert.deepStrictEqual(await bodyToString(result, content.length), content); + assert.equal( + result.contentRange, + `bytes 0-${content.length - 1}/${content.length}` + ); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + }); + + it("download out of range @loki @sql", async () => { + try { + await blobClient.download(content.length + 1, content.length + 10); + } catch (error) { + assert.deepStrictEqual(error.statusCode, 416); + return; + } + assert.fail(); + }); + + it("get properties response should not set content-type @loki @sql", async () => { + const blobURL404 = containerClient.getBlobClient("UN_EXIST_BLOB_"); + try { + await blobURL404.getProperties(); + } catch (err) { + assert.ok(!err.response.headers.get("content-type")); + } + + try { + await blobURL404.download(0, 0); + } catch (err) { + assert.notEqual(err.response.headers.get("content-type"), undefined); + } + }); + + it("delete @loki @sql", async () => { + const result = await blobClient.delete(); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + }); + + it("delete should work for valid ifMatch @loki @sql", async () => { + const properties = await blobClient.getProperties(); + + const result = await blobClient.delete({ + conditions: { + ifMatch: properties.etag + } + }); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + }); + + it("delete should work for * ifMatch @loki @sql", async () => { + const result = await blobClient.delete({ + conditions: { + ifMatch: "*" + } + }); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + }); + + it("delete should not work for invalid ifMatch @loki @sql", async () => { + try { + await blobClient.delete({ + conditions: { + ifMatch: "invalid" + } + }); + } catch (error) { + assert.deepStrictEqual(error.statusCode, 412); + return; + } + assert.fail(); + }); + + it("delete should work for valid ifNoneMatch @loki @sql", async () => { + const result = await blobClient.delete({ + conditions: { + ifNoneMatch: "unmatchetag" + } + }); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + }); + + it("delete should not work for invalid ifNoneMatch @loki @sql", async () => { + const properties = await blobClient.getProperties(); + + try { + await blobClient.delete({ + conditions: { + ifNoneMatch: properties.etag + } + }); + } catch (error) { + assert.deepStrictEqual(error.statusCode, 412); + return; + } + assert.fail(); + }); + + it("delete should work for ifNoneMatch * @loki @sql", async () => { + await blobClient.delete({ + conditions: { + ifNoneMatch: "*" + } + }); + }); + + it("delete should work for valid ifModifiedSince * @loki @sql", async () => { + await blobClient.delete({ + conditions: { + ifModifiedSince: new Date("2018/01/01") + } + }); + }); + + it("delete should not work for invalid ifModifiedSince @loki @sql", async () => { + try { + await blobClient.delete({ + conditions: { + ifModifiedSince: new Date("2118/01/01") + } + }); + } catch (error) { + assert.deepStrictEqual(error.statusCode, 412); + return; + } + assert.fail(); + }); + + it("delete should work for valid ifUnmodifiedSince * @loki @sql", async () => { + await blobClient.delete({ + conditions: { + ifUnmodifiedSince: new Date("2118/01/01") + } + }); + }); + + it("delete should not work for invalid ifUnmodifiedSince @loki @sql", async () => { + try { + await blobClient.delete({ + conditions: { + ifUnmodifiedSince: new Date("2018/01/01") + } + }); + } catch (error) { + assert.deepStrictEqual(error.statusCode, 412); + return; + } + assert.fail(); + }); + + it("should create a snapshot from a blob @loki @sql", async () => { + const result = await blobClient.createSnapshot(); + assert.ok(result.snapshot); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + }); + + it("should create a snapshot with metadata from a blob @loki @sql", async () => { + const metadata = { + meta1: "val1", + meta3: "val3" + }; + const result = await blobClient.createSnapshot({ metadata }); + assert.ok(result.snapshot); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + const result2 = await blobClient + .withSnapshot(result.snapshot!) + .getProperties(); + assert.deepStrictEqual(result2.metadata, metadata); + }); + + it("should not delete base blob without include snapshot header @loki @sql", async () => { + const result = await blobClient.createSnapshot(); + assert.ok(result.snapshot); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + const blobSnapshotURL = blobClient.withSnapshot(result.snapshot!); + await blobSnapshotURL.getProperties(); + + let err; + try { + await blobClient.delete({}); + } catch (error) { + err = error; + } + + assert.deepStrictEqual(err.statusCode, 409); + }); + + it("should delete snapshot @loki @sql", async () => { + const result = await blobClient.createSnapshot(); + assert.ok(result.snapshot); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + const blobSnapshotURL = blobClient.withSnapshot(result.snapshot!); + await blobSnapshotURL.getProperties(); + await blobSnapshotURL.delete(); + await blobClient.delete(); + const result2 = ( + await containerClient + .listBlobsFlat({ + includeSnapshots: true + }) + .byPage() + .next() + ).value; + // Verify that the snapshot is deleted + assert.equal(result2.segment.blobItems!.length, 0); + assert.equal( + result2._response.request.headers.get("x-ms-client-request-id"), + result2.clientRequestId + ); + }); + + it("should also list snapshots @loki @sql", async () => { + const result = await blobClient.createSnapshot(); + assert.ok(result.snapshot); + const result2 = ( + await containerClient + .listBlobsFlat({ includeSnapshots: true }) + .byPage() + .next() + ).value; + assert.strictEqual(result2.segment.blobItems!.length, 2); + }); + + it("should setMetadata with new metadata set @loki @sql", async () => { + const metadata = { + a: "a", + b: "b" + }; + const result_setmeta = await blobClient.setMetadata(metadata); + assert.equal( + result_setmeta._response.request.headers.get("x-ms-client-request-id"), + result_setmeta.clientRequestId + ); + const result = await blobClient.getProperties(); + assert.deepStrictEqual(result.metadata, metadata); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + }); + + it("acquireLease_available_proposedLeaseId_fixed @loki @sql", async () => { + const guid = "ca761232ed4211cebacd00aa0057b223"; + const duration = 30; + blobLeaseClient = await blobClient.getBlobLeaseClient(guid); + const result_acquire = await blobLeaseClient.acquireLease(duration); + assert.equal( + result_acquire._response.request.headers.get("x-ms-client-request-id"), + result_acquire._response.request.requestId + ); + + const result = await blobClient.getProperties(); + assert.equal(result.leaseDuration, "fixed"); + assert.equal(result.leaseState, "leased"); + assert.equal(result.leaseStatus, "locked"); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + + const result_release = await blobLeaseClient.releaseLease(); + assert.equal( + result_release._response.request.headers.get("x-ms-client-request-id"), + result_release._response.request.requestId + ); + }); + + it("acquireLease_available_NoproposedLeaseId_infinite @loki @sql", async () => { + const leaseResult = await blobLeaseClient.acquireLease(-1); + const leaseId = leaseResult.leaseId; + assert.ok(leaseId); + + const result = await blobClient.getProperties(); + assert.equal(result.leaseDuration, "infinite"); + assert.equal(result.leaseState, "leased"); + assert.equal(result.leaseStatus, "locked"); + + await blobLeaseClient.releaseLease(); + }); + + it("releaseLease @loki @sql", async () => { + const guid = "ca761232ed4211cebacd00aa0057b223"; + const duration = -1; + blobLeaseClient = await blobClient.getBlobLeaseClient(guid); + await blobLeaseClient.acquireLease(duration); + + let result = await blobClient.getProperties(); + assert.equal(result.leaseDuration, "infinite"); + assert.equal(result.leaseState, "leased"); + assert.equal(result.leaseStatus, "locked"); + + await blobLeaseClient.releaseLease(); + result = await blobClient.getProperties(); + assert.equal(result.leaseDuration, undefined); + assert.equal(result.leaseState, "available"); + assert.equal(result.leaseStatus, "unlocked"); + }); + + it("renewLease @loki @sql", async () => { + const guid = "ca761232ed4211cebacd00aa0057b223"; + const duration = 15; + blobLeaseClient = blobClient.getBlobLeaseClient(guid); + await blobLeaseClient.acquireLease(duration); + + const result = await blobClient.getProperties(); + assert.equal(result.leaseDuration, "fixed"); + assert.equal(result.leaseState, "leased"); + assert.equal(result.leaseStatus, "locked"); + + await sleep(16 * 1000); + const result2 = await blobClient.getProperties(); + assert.ok(!result2.leaseDuration); + assert.equal(result2.leaseState, "expired"); + assert.equal(result2.leaseStatus, "unlocked"); + + await blobLeaseClient.renewLease(); + + const result3 = await blobClient.getProperties(); + assert.equal(result3.leaseDuration, "fixed"); + assert.equal(result3.leaseState, "leased"); + assert.equal(result3.leaseStatus, "locked"); + + await blobLeaseClient.releaseLease(); + }); + + it("changeLease @loki @sql", async () => { + const guid = "ca761232ed4211cebacd00aa0057b223"; + const duration = 15; + blobLeaseClient = blobClient.getBlobLeaseClient(guid); + await blobLeaseClient.acquireLease(duration); + + const result = await blobClient.getProperties(); + assert.equal(result.leaseDuration, "fixed"); + assert.equal(result.leaseState, "leased"); + assert.equal(result.leaseStatus, "locked"); + + const newGuid = "3c7e72ebb4304526bc53d8ecef03798f"; + const result_change = await blobLeaseClient.changeLease(newGuid); + assert.equal( + result_change._response.request.headers.get("x-ms-client-request-id"), + result_change._response.request.requestId + ); + + await blobClient.getProperties(); + await blobLeaseClient.releaseLease(); + }); + + it("breakLease @loki @sql", async () => { + const guid = "ca761232ed4211cebacd00aa0057b223"; + const duration = 15; + blobLeaseClient = blobClient.getBlobLeaseClient(guid); + await blobLeaseClient.acquireLease(duration); + + const result = await blobClient.getProperties(); + assert.equal(result.leaseDuration, "fixed"); + assert.equal(result.leaseState, "leased"); + assert.equal(result.leaseStatus, "locked"); + + const breakDuration = 3; + let breaklefttime = breakDuration; + while (breaklefttime > 0) { + const breakResult = await blobLeaseClient.breakLease(breakDuration); + assert.equal( + breakResult._response.request.headers.get("x-ms-client-request-id"), + breakResult._response.request.requestId + ); + + assert.equal(breakResult.leaseTime! <= breaklefttime, true); + breaklefttime = breakResult.leaseTime!; + + const result2 = await blobClient.getProperties(); + assert.ok(!result2.leaseDuration); + assert.equal(result2.leaseState, "breaking"); + assert.equal(result2.leaseStatus, "locked"); + + await sleep(500); + } + + const result3 = await blobClient.getProperties(); + assert.ok(!result3.leaseDuration); + assert.equal(result3.leaseState, "broken"); + assert.equal(result3.leaseStatus, "unlocked"); + + await blobLeaseClient.releaseLease(); + const result4 = await blobClient.getProperties(); + assert.equal(result4.leaseDuration, undefined); + assert.equal(result4.leaseState, "available"); + assert.equal(result4.leaseStatus, "unlocked"); + }); + + it("should get the correct headers back when setting metadata @loki @sql", async () => { + const metadata = { + a: "a", + b: "b" + }; + const setResult = await blobClient.setMetadata(metadata); + assert.equal( + setResult._response.request.headers.get("x-ms-client-request-id"), + setResult.clientRequestId + ); + assert.notEqual(setResult.date, undefined); + assert.notEqual(setResult.etag, undefined); + assert.notEqual(setResult.isServerEncrypted, undefined); + assert.notEqual(setResult.lastModified, undefined); + assert.notEqual(setResult.requestId, undefined); + assert.notEqual(setResult.version, undefined); + const result = await blobClient.getProperties(); + assert.deepStrictEqual(result.metadata, metadata); + assert.deepStrictEqual(result.accessTier, "Hot"); + assert.deepStrictEqual(result.acceptRanges, "bytes"); + assert.deepStrictEqual(result.blobType, "BlockBlob"); + }); + + // https://docs.microsoft.com/en-us/rest/api/storageservices/get-blob-properties + // as properties retrieval is implemented, the properties should be added to the tests below + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Header + it("should get the correct properties set based on set HTTP headers @loki @sql", async () => { + const cacheControl = "no-cache"; + const contentType = "text/plain; charset=UTF-8"; + const md5 = new Uint8Array([1, 2, 3, 4, 5]); + const contentEncoding = "identity"; + const contentLanguage = "en-US"; + const contentDisposition = "attachment"; + const headers: BlobHTTPHeaders = { + blobCacheControl: cacheControl, + blobContentType: contentType, + blobContentMD5: md5, + blobContentDisposition: contentDisposition, + blobContentLanguage: contentLanguage, + blobContentEncoding: contentEncoding + }; + const result_set = await blobClient.setHTTPHeaders(headers); + assert.equal( + result_set._response.request.headers.get("x-ms-client-request-id"), + result_set.clientRequestId + ); + const result = await blobClient.getProperties(); + assert.deepStrictEqual(result.cacheControl, cacheControl); + assert.deepStrictEqual(result.contentType, contentType); + assert.deepEqual(result.contentMD5, md5); + assert.deepStrictEqual(result.contentDisposition, contentDisposition); + assert.deepStrictEqual(result.contentLanguage, contentLanguage); + }); + + it("setTier set default to cool @loki @sql", async () => { + // Created Blob should have accessTierInferred as true in Get/list + let properties = await blockBlobClient.getProperties(); + assert.equal(properties.accessTier!.toLowerCase(), "hot"); + assert.equal(true, properties.accessTierInferred); + + let listResult = ( + await containerClient + .listBlobsFlat({ + prefix: blobName + }) + .byPage() + .next() + ).value; + assert.equal( + true, + (await listResult).segment.blobItems[0].properties.accessTierInferred + ); + + const result = await blockBlobClient.setAccessTier("Cool"); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + + // After setTier, Blob should have accessTierInferred as false in Get + properties = await blockBlobClient.getProperties(); + assert.equal(properties.accessTier!.toLowerCase(), "cool"); + assert.equal(false, properties.accessTierInferred); + + // After setTier, Blob should have accessTierInferred as undefined in list + listResult = ( + await containerClient + .listBlobsFlat({ + prefix: blobName + }) + .byPage() + .next() + ).value; + assert.equal( + undefined, + (await listResult).segment.blobItems[0].properties.accessTierInferred + ); + }); + + it("setTier set archive to hot @loki @sql", async () => { + await blockBlobClient.setAccessTier("Archive"); + let properties = await blockBlobClient.getProperties(); + assert.equal(properties.accessTier!.toLowerCase(), "archive"); + + await blockBlobClient.setAccessTier("Hot"); + properties = await blockBlobClient.getProperties(); + if (properties.archiveStatus) { + assert.equal( + properties.archiveStatus.toLowerCase(), + "rehydrate-pending-to-hot" + ); + } + }); + + it("setTier on leased blob @loki @sql", async () => { + const leaseResult = await blobLeaseClient.acquireLease(-1); + const leaseId = leaseResult.leaseId; + assert.ok(leaseId); + + await blockBlobClient.setAccessTier("Hot", { + conditions: { leaseId: leaseId } + }); + + const result = await blobClient.getProperties(); + assert.equal(result.leaseDuration, "infinite"); + assert.equal(result.leaseState, "leased"); + assert.equal(result.leaseStatus, "locked"); + if (result.archiveStatus) { + assert.equal( + result.archiveStatus.toLowerCase(), + "rehydrate-pending-to-hot" + ); + } + + await blobLeaseClient.releaseLease(); + + await blockBlobClient.setAccessTier("Archive"); + let properties = await blockBlobClient.getProperties(); + assert.equal(properties.accessTier!.toLowerCase(), "archive"); + + await blockBlobClient.setAccessTier("Hot"); + properties = await blockBlobClient.getProperties(); + if (properties.archiveStatus) { + assert.equal( + properties.archiveStatus.toLowerCase(), + "rehydrate-pending-to-hot" + ); + } + }); + + it("setHTTPHeaders with default parameters @loki @sql", async () => { + await blobClient.setHTTPHeaders({}); + const result = await blobClient.getProperties(); + + assert.deepStrictEqual(result.blobType, "BlockBlob"); + assert.ok(result.lastModified); + assert.deepStrictEqual(result.metadata, {}); + assert.ok(!result.cacheControl); + assert.ok(!result.contentType); + assert.ok(!result.contentMD5); + assert.ok(!result.contentEncoding); + assert.ok(!result.contentLanguage); + assert.ok(!result.contentDisposition); + }); + + it("setHTTPHeaders with all parameters set @loki @sql", async () => { + const headers = { + blobCacheControl: "blobCacheControl", + blobContentDisposition: "blobContentDisposition", + blobContentEncoding: "blobContentEncoding", + blobContentLanguage: "blobContentLanguage", + blobContentMD5: isNode + ? Buffer.from([1, 2, 3, 4]) + : new Uint8Array([1, 2, 3, 4]), + blobContentType: "blobContentType" + }; + await blobClient.setHTTPHeaders(headers); + const result = await blobClient.getProperties(); + assert.ok(result.date); + assert.deepStrictEqual(result.blobType, "BlockBlob"); + assert.ok(result.lastModified); + assert.deepStrictEqual(result.metadata, {}); + assert.deepStrictEqual(result.cacheControl, headers.blobCacheControl); + assert.deepStrictEqual(result.contentType, headers.blobContentType); + assert.deepStrictEqual(result.contentMD5, headers.blobContentMD5); + assert.deepStrictEqual(result.contentEncoding, headers.blobContentEncoding); + assert.deepStrictEqual(result.contentLanguage, headers.blobContentLanguage); + assert.deepStrictEqual( + result.contentDisposition, + headers.blobContentDisposition + ); + }); + + it("Copy blob should work @loki @sql", async () => { + const sourceBlob = getUniqueName("blob"); + const destBlob = getUniqueName("blob"); + + const sourceBlobClient = containerClient.getBlockBlobClient(sourceBlob); + const destBlobClient = containerClient.getBlockBlobClient(destBlob); + + const metadata = { key: "value" }; + const blobHTTPHeaders = { + blobCacheControl: "blobCacheControl", + blobContentDisposition: "blobContentDisposition", + blobContentEncoding: "blobContentEncoding", + blobContentLanguage: "blobContentLanguage", + blobContentType: "blobContentType" + }; + + const result_upload = await sourceBlobClient.upload("hello", 5, { + metadata, + blobHTTPHeaders + }); + assert.equal( + result_upload._response.request.headers.get("x-ms-client-request-id"), + result_upload.clientRequestId + ); + + const result_startcopy = await destBlobClient.beginCopyFromURL( + sourceBlobClient.url + ); + assert.equal( + result_startcopy + .getResult()! + ._response.request.headers.get("x-ms-client-request-id"), + result_startcopy.getResult()!._response.request.requestId + ); + + const result = await destBlobClient.getProperties(); + assert.ok(result.date); + assert.deepStrictEqual(result.blobType, "BlockBlob"); + assert.ok(result.lastModified); + assert.deepStrictEqual(result.metadata, metadata); + assert.deepStrictEqual( + result.cacheControl, + blobHTTPHeaders.blobCacheControl + ); + assert.deepStrictEqual(result.contentType, blobHTTPHeaders.blobContentType); + assert.deepStrictEqual( + result.contentEncoding, + blobHTTPHeaders.blobContentEncoding + ); + assert.deepStrictEqual( + result.contentLanguage, + blobHTTPHeaders.blobContentLanguage + ); + assert.deepStrictEqual( + result.contentDisposition, + blobHTTPHeaders.blobContentDisposition + ); + }); + + it("Copy blob should work to override metadata @loki @sql", async () => { + const sourceBlob = getUniqueName("blob"); + const destBlob = getUniqueName("blob"); + + const sourceBlobClient = containerClient.getBlockBlobClient(sourceBlob); + const destBlobClient = containerClient.getBlockBlobClient(destBlob); + + const metadata = { key: "value" }; + const metadata2 = { key: "value2" }; + await sourceBlobClient.upload("hello", 5, { + metadata + }); + + await destBlobClient.beginCopyFromURL(sourceBlobClient.url, { + metadata: metadata2 + }); + + const result = await destBlobClient.getProperties(); + assert.ok(result.date); + assert.deepStrictEqual(result.blobType, "BlockBlob"); + assert.ok(result.lastModified); + assert.deepStrictEqual(result.metadata, metadata2); + }); + + it("Copy blob should work with source archive blob and accesstier header @loki, @sql", async () => { + const sourceBlob = getUniqueName("blob"); + const destBlob = getUniqueName("blob"); + + const sourceBlobClient = containerClient.getBlockBlobClient(sourceBlob); + const destBlobClient = containerClient.getBlockBlobClient(destBlob); + + await sourceBlobClient.upload("hello", 5); + await sourceBlobClient.setAccessTier("Archive"); + + // Copy from Archive blob without accesstier will fail + let hasError = false; + try { + await destBlobClient.beginCopyFromURL(sourceBlobClient.url); + } catch (error) { + assert.deepStrictEqual(error.statusCode, 409); + hasError = true; + } + if (!hasError) { + assert.fail(); + } + + // Copy from Archive blob with accesstier will success + await destBlobClient.beginCopyFromURL(sourceBlobClient.url, { + tier: "Hot" + }); + + const result = await destBlobClient.getProperties(); + assert.ok(result.date); + assert.deepStrictEqual(result.blobType, "BlockBlob"); + assert.ok(result.lastModified); + assert.deepStrictEqual(result.accessTier, "Hot"); + }); + + it("Copy blob should not override destination Lease status @loki @sql", async () => { + const sourceBlob = getUniqueName("blob"); + const destBlob = getUniqueName("blob"); + + const sourceBlobClient = containerClient.getBlockBlobClient(sourceBlob); + const destBlobClient = containerClient.getBlockBlobClient(destBlob); + + await sourceBlobClient.upload("hello", 5); + await destBlobClient.upload("hello", 5); + + let destLeaseClient = destBlobClient.getBlobLeaseClient(); + const leaseResult = await destLeaseClient.acquireLease(-1); + const leaseId = leaseResult.leaseId; + assert.ok(leaseId); + + const getResult = await destBlobClient.getProperties(); + assert.equal(getResult.leaseDuration, "infinite"); + assert.equal(getResult.leaseState, "leased"); + assert.equal(getResult.leaseStatus, "locked"); + + await destBlobClient.beginCopyFromURL(sourceBlobClient.url, { + conditions: { leaseId } + }); + + const result = await destBlobClient.getProperties(); + assert.ok(result.date); + assert.deepStrictEqual(result.blobType, "BlockBlob"); + assert.ok(result.lastModified); + assert.equal(getResult.leaseDuration, "infinite"); + assert.equal(getResult.leaseState, "leased"); + assert.equal(getResult.leaseStatus, "locked"); + + await destLeaseClient.releaseLease(); + }); + + it("Copy blob should work for page blob @loki @sql", async () => { + const sourceBlob = getUniqueName("blob"); + const destBlob = getUniqueName("blob"); + + const sourceBlobClient = containerClient.getPageBlobClient(sourceBlob); + const destBlobClient = containerClient.getPageBlobClient(destBlob); + + const metadata = { key: "value" }; + const blobHTTPHeaders = { + blobCacheControl: "blobCacheControl", + blobContentDisposition: "blobContentDisposition", + blobContentEncoding: "blobContentEncoding", + blobContentLanguage: "blobContentLanguage", + blobContentType: "blobContentType" + }; + + const result_upload = await sourceBlobClient.create(512, { + metadata, + blobHTTPHeaders + }); + assert.equal( + result_upload._response.request.headers.get("x-ms-client-request-id"), + result_upload.clientRequestId + ); + + const result_startcopy = await destBlobClient.beginCopyFromURL( + sourceBlobClient.url + ); + assert.equal( + result_startcopy + .getResult()! + ._response.request.headers.get("x-ms-client-request-id"), + result_startcopy.getResult()!._response.request.requestId + ); + + const result = await destBlobClient.getProperties(); + assert.ok(result.date); + assert.deepStrictEqual(result.blobType, "PageBlob"); + assert.ok(result.lastModified); + assert.deepStrictEqual(result.metadata, metadata); + assert.deepStrictEqual( + result.cacheControl, + blobHTTPHeaders.blobCacheControl + ); + assert.deepStrictEqual(result.contentType, blobHTTPHeaders.blobContentType); + assert.deepStrictEqual( + result.contentEncoding, + blobHTTPHeaders.blobContentEncoding + ); + assert.deepStrictEqual( + result.contentLanguage, + blobHTTPHeaders.blobContentLanguage + ); + assert.deepStrictEqual( + result.contentDisposition, + blobHTTPHeaders.blobContentDisposition + ); + }); + + it("Copy blob should not work for page blob and set tier @loki @sql", async () => { + const sourceBlob = getUniqueName("blob"); + const destBlob = getUniqueName("blob"); + + const sourceBlobClient = containerClient.getPageBlobClient(sourceBlob); + const destBlobClient = containerClient.getPageBlobClient(destBlob); + + const metadata = { key: "value" }; + const blobHTTPHeaders = { + blobCacheControl: "blobCacheControl", + blobContentDisposition: "blobContentDisposition", + blobContentEncoding: "blobContentEncoding", + blobContentLanguage: "blobContentLanguage", + blobContentType: "blobContentType" + }; + + const result_upload = await sourceBlobClient.create(512, { + metadata, + blobHTTPHeaders + }); + assert.equal( + result_upload._response.request.headers.get("x-ms-client-request-id"), + result_upload.clientRequestId + ); + + let err; + + try { + await destBlobClient.beginCopyFromURL(sourceBlobClient.url, { + tier: "P10" + }); + } catch (error) { + err = error; + } + + assert.deepStrictEqual(err.statusCode, 400); + }); + + it("Copy blob should not work with ifNoneMatch * when dest exist @loki @sql", async () => { + const sourceBlob = getUniqueName("blob"); + const destBlob = getUniqueName("blob"); + + const sourceBlobClient = containerClient.getBlockBlobClient(sourceBlob); + const destBlobClient = containerClient.getBlockBlobClient(destBlob); + + const metadata = { key: "value" }; + const blobHTTPHeaders = { + blobCacheControl: "blobCacheControl", + blobContentDisposition: "blobContentDisposition", + blobContentEncoding: "blobContentEncoding", + blobContentLanguage: "blobContentLanguage", + blobContentType: "blobContentType" + }; + + // upload source + const result_uploadsrc = await sourceBlobClient.upload("hello", 5, { + metadata, + blobHTTPHeaders + }); + assert.equal( + result_uploadsrc._response.request.headers.get("x-ms-client-request-id"), + result_uploadsrc.clientRequestId + ); + + // upload destination + const result_uploaddest = await destBlobClient.upload("hello", 5, { + metadata, + blobHTTPHeaders + }); + assert.equal( + result_uploaddest._response.request.headers.get("x-ms-client-request-id"), + result_uploaddest.clientRequestId + ); + + // async copy + try { + await destBlobClient.beginCopyFromURL(sourceBlobClient.url, { + conditions: { + ifNoneMatch: "*" + } + }); + } catch (error) { + assert.deepStrictEqual(error.statusCode, 409); + return; + } + assert.fail(); + }); + + it("Synchronized copy blob should work @loki @sql", async () => { + const sourceBlob = getUniqueName("blob"); + const destBlob = getUniqueName("blob"); + + const sourceBlobClient = containerClient.getBlockBlobClient(sourceBlob); + const destBlobClient = containerClient.getBlockBlobClient(destBlob); + + const metadata = { key: "value" }; + const blobHTTPHeaders = { + blobCacheControl: "blobCacheControl", + blobContentDisposition: "blobContentDisposition", + blobContentEncoding: "blobContentEncoding", + blobContentLanguage: "blobContentLanguage", + blobContentType: "blobContentType" + }; + + const result_upload = await sourceBlobClient.upload("hello", 5, { + metadata, + blobHTTPHeaders + }); + assert.equal( + result_upload._response.request.headers.get("x-ms-client-request-id"), + result_upload.clientRequestId + ); + + const result_copy = await destBlobClient.syncCopyFromURL( + sourceBlobClient.url + ); + assert.equal( + result_copy._response.request.headers.get("x-ms-client-request-id"), + result_copy._response.request.requestId + ); + assert.equal(result_copy.copyStatus, "success"); + + const result = await destBlobClient.getProperties(); + assert.ok(result.date); + assert.deepStrictEqual(result.blobType, "BlockBlob"); + assert.ok(result.lastModified); + assert.deepStrictEqual(result.metadata, metadata); + assert.deepStrictEqual( + result.cacheControl, + blobHTTPHeaders.blobCacheControl + ); + assert.deepStrictEqual(result.contentType, blobHTTPHeaders.blobContentType); + assert.deepStrictEqual( + result.contentEncoding, + blobHTTPHeaders.blobContentEncoding + ); + assert.deepStrictEqual( + result.contentLanguage, + blobHTTPHeaders.blobContentLanguage + ); + assert.deepStrictEqual( + result.contentDisposition, + blobHTTPHeaders.blobContentDisposition + ); + }); + + it("Synchronized copy blob should work to override metadata @loki @sql", async () => { + const sourceBlob = getUniqueName("blob"); + const destBlob = getUniqueName("blob"); + + const sourceBlobClient = containerClient.getBlockBlobClient(sourceBlob); + const destBlobClient = containerClient.getBlockBlobClient(destBlob); + + const metadata = { key: "value" }; + const metadata2 = { key: "value2" }; + await sourceBlobClient.upload("hello", 5, { + metadata + }); + + await destBlobClient.syncCopyFromURL(sourceBlobClient.url, { + metadata: metadata2 + }); + + const result = await destBlobClient.getProperties(); + assert.ok(result.date); + assert.deepStrictEqual(result.blobType, "BlockBlob"); + assert.ok(result.lastModified); + assert.deepStrictEqual(result.metadata, metadata2); + }); + + it("Synchronized copy blob should not override destination Lease status @loki @sql", async () => { + const sourceBlob = getUniqueName("blob"); + const destBlob = getUniqueName("blob"); + + const sourceBlobClient = containerClient.getBlockBlobClient(sourceBlob); + const destBlobClient = containerClient.getBlockBlobClient(destBlob); + + await sourceBlobClient.upload("hello", 5); + await destBlobClient.upload("hello", 5); + + let destLeaseClient = destBlobClient.getBlobLeaseClient(); + const leaseResult = await destLeaseClient.acquireLease(-1); + const leaseId = leaseResult.leaseId; + assert.ok(leaseId); + + const getResult = await destBlobClient.getProperties(); + assert.equal(getResult.leaseDuration, "infinite"); + assert.equal(getResult.leaseState, "leased"); + assert.equal(getResult.leaseStatus, "locked"); + + await destBlobClient.syncCopyFromURL(sourceBlobClient.url, { + conditions: { leaseId } + }); + + const result = await destBlobClient.getProperties(); + assert.ok(result.date); + assert.deepStrictEqual(result.blobType, "BlockBlob"); + assert.ok(result.lastModified); + assert.equal(getResult.leaseDuration, "infinite"); + assert.equal(getResult.leaseState, "leased"); + assert.equal(getResult.leaseStatus, "locked"); + + await destLeaseClient.releaseLease(); + }); + + it("Synchronized copy blob should work for page blob @loki @sql", async () => { + const sourceBlob = getUniqueName("blob"); + const destBlob = getUniqueName("blob"); + + const sourceBlobClient = containerClient.getPageBlobClient(sourceBlob); + const destBlobClient = containerClient.getPageBlobClient(destBlob); + + const metadata = { key: "value" }; + const blobHTTPHeaders = { + blobCacheControl: "blobCacheControl", + blobContentDisposition: "blobContentDisposition", + blobContentEncoding: "blobContentEncoding", + blobContentLanguage: "blobContentLanguage", + blobContentType: "blobContentType" + }; + + const result_upload = await sourceBlobClient.create(512, { + metadata, + blobHTTPHeaders + }); + assert.equal( + result_upload._response.request.headers.get("x-ms-client-request-id"), + result_upload.clientRequestId + ); + + const result_copy = await destBlobClient.syncCopyFromURL( + sourceBlobClient.url + ); + assert.equal( + result_copy._response.request.headers.get("x-ms-client-request-id"), + result_copy._response.request.requestId + ); + assert.equal(result_copy.copyStatus, "success"); + + const result = await destBlobClient.getProperties(); + assert.ok(result.date); + assert.deepStrictEqual(result.blobType, "PageBlob"); + assert.ok(result.lastModified); + assert.deepStrictEqual(result.metadata, metadata); + assert.deepStrictEqual( + result.cacheControl, + blobHTTPHeaders.blobCacheControl + ); + assert.deepStrictEqual(result.contentType, blobHTTPHeaders.blobContentType); + assert.deepStrictEqual( + result.contentEncoding, + blobHTTPHeaders.blobContentEncoding + ); + assert.deepStrictEqual( + result.contentLanguage, + blobHTTPHeaders.blobContentLanguage + ); + assert.deepStrictEqual( + result.contentDisposition, + blobHTTPHeaders.blobContentDisposition + ); + }); + + it("Acquire Lease on Breaking Lease status, if LeaseId not match, throw LeaseIdMismatchWithLease error @loki @sql", async () => { + // TODO: implement the case later + }); + + it("Renew Lease on Breaking Lease status, if LeaseId not match, throw LeaseIdMismatchWithLease error @loki @sql", async () => { + // TODO: implement the case later + }); + + it("Change Lease on Breaking Lease status, if LeaseId not match, throw LeaseIdMismatchWithLease error @loki @sql", async () => { + // TODO: implement the case later + }); + + it("Renew: Lease on Breaking Lease status, if LeaseId not match, throw LeaseIdMismatchWithLease error @loki @sql", async () => { + // TODO: implement the case later + }); + + it("Acquire Lease on Broken Lease status, if LeaseId not match, throw LeaseIdMismatchWithLease error @loki @sql", async () => { + // TODO: implement the case later + }); + + it("Break Lease on Infinite Lease, if give valid breakPeriod, should be broken after breadperiod @loki @sql", async () => { + // TODO: implement the case later + }); + + it("Break Lease on Infinite Lease, if not give breakPeriod, should be broken immidiately @loki @sql", async () => { + // TODO: implement the case later + }); + + it("Renew: Lease on Leased status, if LeaseId not match, throw LeaseIdMismatchWithLease error @loki @sql", async () => { + // TODO: implement the case later + }); + + it("Change Lease on Leased status, if input LeaseId not match anyone of leaseID or proposedLeaseId, throw LeaseIdMismatchWithLease error @loki @sql", async () => { + // TODO: implement the case later + }); + + it("Change Lease on Leased status, if input LeaseId matches proposedLeaseId, will change success @loki @sql", async () => { + // TODO: implement the case later + }); + + it("UploadPage on a Leased page blob, if input LeaseId matches, will success @loki @sql", async () => { + // TODO: implement the case later + }); + + it("ClearPage on a Leased page blob, if input LeaseId matches, will success @loki @sql", async () => { + // TODO: implement the case later + }); + + it("Resize a Leased page blob, if input LeaseId matches, will success @loki @sql", async () => { + // TODO: implement the case later + }); + + it("UpdateSequenceNumber a Leased page blob, if input LeaseId matches, will success @loki @sql", async () => { + // TODO: implement the case later + }); +}); diff --git a/tests/dfs/apis/blobbatch.test.ts b/tests/dfs/apis/blobbatch.test.ts new file mode 100644 index 000000000..b726567c0 --- /dev/null +++ b/tests/dfs/apis/blobbatch.test.ts @@ -0,0 +1,421 @@ +import assert from "assert"; + +import { + AccountSASPermissions, + AccountSASResourceTypes, + AnonymousCredential, + BlobClient, + BlobServiceClient, + ContainerClient, + ContainerSASPermissions, + newPipeline, + StorageSharedKeyCredential +} from "@azure/storage-blob"; + +import { configLogger } from "../../../src/common/Logger"; +import { + EMULATOR_ACCOUNT_KEY, + EMULATOR_ACCOUNT_NAME, + getUniqueName +} from "../../testutils"; +import DataLakeTestServerFactory from "../DataLakeTestServerFactory"; + +// Set true to enable debug log +configLogger(false); + +describe("Blob batch API", () => { + const factory = new DataLakeTestServerFactory(); + const server = factory.createServer(); + + const baseURL = `http://${server.config.host}:${server.config.port}/devstoreaccount1`; + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ) + ); + + let containerName: string; + let containerClient: ContainerClient; + const content = "Hello World"; + let blobClients: BlobClient[]; + let blobCount = 3; + + before(async () => { + await server.start(); + }); + + after(async () => { + await server.close(); + await server.clean(); + }); + + beforeEach(async () => { + containerName = getUniqueName("container"); + containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + blobClients = []; + for (let i = 0; i < blobCount; ++i) { + const blobName = getUniqueName("blob"); + const blobClient = containerClient.getBlobClient(blobName); + const blockBlobClient = blobClient.getBlockBlobClient(); + await blockBlobClient.upload(content, content.length); + blobClients.push(blobClient); + } + }); + + afterEach(async () => { + await containerClient.delete(); + }); + + it("SubmitBatch batch deleting @loki @sql", async () => { + const blobBatchClient = serviceClient.getBlobBatchClient(); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const sharedKeyCredential = factories[factories.length - 1]; + + // Submit batch request and verify response. + const urls = blobClients.map((b) => b.url); + const resp = await blobBatchClient.deleteBlobs( + urls, + sharedKeyCredential, + {} + ); + assert.equal(resp.subResponses.length, blobCount); + assert.equal(resp.subResponsesSucceededCount, blobCount); + assert.equal(resp.subResponsesFailedCount, 0); + + for (let i = 0; i < blobCount; i++) { + assert.equal(resp.subResponses[i].errorCode, undefined); + assert.equal(resp.subResponses[i].status, 202); + assert.ok(resp.subResponses[i].statusMessage !== ""); + assert.ok(resp.subResponses[i].headers.contains("x-ms-request-id")); + assert.equal(resp.subResponses[i]._request.url, blobClients[i].url); + } + + // Verify blobs deleted. + const resp2 = ( + await containerClient + .listBlobsFlat({ + includeSnapshots: true + }) + .byPage({ maxPageSize: 1 }) + .next() + ).value; + assert.equal(resp2.segment.blobItems.length, 0); + }); + + it("SubmitBatch within container scope - batch set tier @loki @sql", async () => { + const blobBatchClient = containerClient.getBlobBatchClient(); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const sharedKeyCredential = factories[factories.length - 1]; + + // Submit batch request and verify response. + const urls = blobClients.map((b) => b.url); + const resp = await blobBatchClient.setBlobsAccessTier( + urls, + sharedKeyCredential, + "Archive", + {} + ); + assert.equal(resp.subResponses.length, blobCount); + assert.equal(resp.subResponsesSucceededCount, blobCount); + assert.equal(resp.subResponsesFailedCount, 0); + + for (let i = 0; i < blobCount; i++) { + assert.equal(resp.subResponses[i].errorCode, undefined); + assert.equal(resp.subResponses[i].status, 200); + assert.ok(resp.subResponses[i].statusMessage !== ""); + assert.ok(resp.subResponses[i].headers.contains("x-ms-request-id")); + assert.equal(resp.subResponses[i]._request.url, blobClients[i].url); + } + + for (const blobClient of blobClients) { + // Check blob tier set properly. + const resp2 = await blobClient.getProperties(); + assert.equal(resp2.accessTier, "Archive"); + } + }); + + it("SubmitBatch batch set tier @loki @sql", async () => { + const blobBatchClient = serviceClient.getBlobBatchClient(); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const sharedKeyCredential = factories[factories.length - 1]; + + // Submit batch request and verify response. + const urls = blobClients.map((b) => b.url); + const resp = await blobBatchClient.setBlobsAccessTier( + urls, + sharedKeyCredential, + "Archive", + {} + ); + assert.equal(resp.subResponses.length, blobCount); + assert.equal(resp.subResponsesSucceededCount, blobCount); + assert.equal(resp.subResponsesFailedCount, 0); + + for (let i = 0; i < blobCount; i++) { + assert.equal(resp.subResponses[i].errorCode, undefined); + assert.equal(resp.subResponses[i].status, 200); + assert.ok(resp.subResponses[i].statusMessage !== ""); + assert.ok(resp.subResponses[i].headers.contains("x-ms-request-id")); + assert.equal(resp.subResponses[i]._request.url, blobClients[i].url); + } + + for (const blobClient of blobClients) { + // Check blob tier set properly. + const resp2 = await blobClient.getProperties(); + assert.equal(resp2.accessTier, "Archive"); + } + }); + + it("SubmitBatch within container scope - batch deleting blob in different container @loki @sql", async () => { + const blobBatchClient = containerClient.getBlobBatchClient(); + + const containerClientNew = serviceClient.getContainerClient( + getUniqueName("containernew") + ); + await containerClientNew.create(); + + const blockBlobClientNew = containerClientNew.getBlockBlobClient( + getUniqueName("blob") + ); + blockBlobClientNew.upload(content, content.length); + const blobclientsNew: BlobClient[] = []; + blobclientsNew.push(blockBlobClientNew); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const sharedKeyCredential = factories[factories.length - 1]; + + // Submit batch request and verify response. + const urls = blobclientsNew.map((b) => b.url); + const resp = await blobBatchClient.deleteBlobs( + urls, + sharedKeyCredential, + {} + ); + assert.equal(resp.subResponses.length, 1); + assert.equal(resp.subResponsesSucceededCount, 0); + assert.equal(resp.subResponsesFailedCount, 1); + }); + + it("SubmitBatch with SAS token - batch deleting @loki @sql", async () => { + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + const sasUrl = serviceClient.generateAccountSasUrl( + tmr, + AccountSASPermissions.parse("d"), + AccountSASResourceTypes.parse("o").toString() + ); + + const sasServiceClient = new BlobServiceClient( + sasUrl, + newPipeline(new AnonymousCredential(), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + const blobBatchClient = sasServiceClient.getBlobBatchClient(); + const sasBlobClients: BlobClient[] = []; + + for (const blobClient of blobClients) { + const sasBlobClient = sasServiceClient + .getContainerClient(containerName) + .getBlobClient(blobClient.name); + sasBlobClients.push(sasBlobClient); + } + + // Submit batch request and verify response. + const urls = sasBlobClients.map((b) => b.url); + const resp = await blobBatchClient.deleteBlobs( + urls, + new AnonymousCredential(), + {} + ); + assert.equal(resp.subResponses.length, blobCount); + assert.equal(resp.subResponsesSucceededCount, blobCount); + assert.equal(resp.subResponsesFailedCount, 0); + + for (let i = 0; i < blobCount; i++) { + assert.equal(resp.subResponses[i].errorCode, undefined); + assert.equal(resp.subResponses[i].status, 202); + assert.ok(resp.subResponses[i].statusMessage !== ""); + assert.ok(resp.subResponses[i].headers.contains("x-ms-request-id")); + assert.ok( + resp.subResponses[i]._request.url.startsWith(blobClients[i].url) + ); + } + + // Verify blobs deleted. + const resp2 = ( + await containerClient + .listBlobsFlat({ + includeSnapshots: true + }) + .byPage({ maxPageSize: 1 }) + .next() + ).value; + assert.equal(resp2.segment.blobItems.length, 0); + }); + + it("SubmitBatch batch with SAS token set tier @loki @sql", async () => { + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + const sasUrl = serviceClient.generateAccountSasUrl( + tmr, + AccountSASPermissions.parse("w"), + AccountSASResourceTypes.parse("o").toString() + ); + + const sasServiceClient = new BlobServiceClient( + sasUrl, + newPipeline(new AnonymousCredential(), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + const blobBatchClient = sasServiceClient.getBlobBatchClient(); + const sasBlobClients: BlobClient[] = []; + + for (const blobClient of blobClients) { + const sasBlobClient = sasServiceClient + .getContainerClient(containerName) + .getBlobClient(blobClient.name); + sasBlobClients.push(sasBlobClient); + } + + // Submit batch request and verify response. + const urls = sasBlobClients.map((b) => b.url); + const resp = await blobBatchClient.setBlobsAccessTier( + urls, + new AnonymousCredential(), + "Archive", + {} + ); + assert.equal(resp.subResponses.length, blobCount); + assert.equal(resp.subResponsesSucceededCount, blobCount); + assert.equal(resp.subResponsesFailedCount, 0); + + for (let i = 0; i < blobCount; i++) { + assert.equal(resp.subResponses[i].errorCode, undefined); + assert.equal(resp.subResponses[i].status, 200); + assert.ok(resp.subResponses[i].statusMessage !== ""); + assert.ok(resp.subResponses[i].headers.contains("x-ms-request-id")); + assert.ok( + resp.subResponses[i]._request.url.startsWith(blobClients[i].url) + ); + } + + for (const blobClient of blobClients) { + // Check blob tier set properly. + const resp2 = await blobClient.getProperties(); + assert.equal(resp2.accessTier, "Archive"); + } + }); + + it("SubmitBatch within containerScope - with SAS token - batch deleting @loki @sql", async () => { + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + const sasUrl = await containerClient.generateSasUrl({ + permissions: ContainerSASPermissions.parse("rd"), + expiresOn: tmr + }); + + const sasContainerClient = new ContainerClient(sasUrl); + const blobBatchClient = sasContainerClient.getBlobBatchClient(); + const sasBlobClients: BlobClient[] = []; + + for (const blobClient of blobClients) { + const sasBlobClient = sasContainerClient.getBlobClient(blobClient.name); + sasBlobClients.push(sasBlobClient); + } + + // Submit batch request and verify response. + const urls = sasBlobClients.map((b) => b.url); + const resp = await blobBatchClient.deleteBlobs( + urls, + new AnonymousCredential() + ); + assert.equal(resp.subResponses.length, blobCount); + assert.equal(resp.subResponsesSucceededCount, blobCount); + assert.equal(resp.subResponsesFailedCount, 0); + + for (let i = 0; i < blobCount; i++) { + assert.equal(resp.subResponses[i].errorCode, undefined); + assert.equal(resp.subResponses[i].status, 202); + assert.ok(resp.subResponses[i].statusMessage !== ""); + assert.ok(resp.subResponses[i].headers.contains("x-ms-request-id")); + assert.ok( + resp.subResponses[i]._request.url.startsWith(blobClients[i].url) + ); + } + + // Verify blobs deleted. + const resp2 = ( + await containerClient + .listBlobsFlat({ + includeSnapshots: true + }) + .byPage({ maxPageSize: 1 }) + .next() + ).value; + assert.equal(resp2.segment.blobItems.length, 0); + }); + + it("SubmitBatch batch with different operations @loki @sql", async () => { + const blobBatchClient = serviceClient.getBlobBatchClient(); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const sharedKeyCredential = factories[factories.length - 1]; + + // Submit batch request and verify response. + const urls = blobClients.map((b) => b.url); + const resp = await blobBatchClient.deleteBlobs( + urls, + sharedKeyCredential, + {} + ); + assert.equal(resp.subResponses.length, blobCount); + assert.equal(resp.subResponsesSucceededCount, blobCount); + assert.equal(resp.subResponsesFailedCount, 0); + + for (let i = 0; i < blobCount; i++) { + assert.equal(resp.subResponses[i].errorCode, undefined); + assert.equal(resp.subResponses[i].status, 202); + assert.ok(resp.subResponses[i].statusMessage !== ""); + assert.ok(resp.subResponses[i].headers.contains("x-ms-request-id")); + assert.equal(resp.subResponses[i]._request.url, blobClients[i].url); + } + + // Verify blobs deleted. + const resp2 = ( + await containerClient + .listBlobsFlat({ + includeSnapshots: true + }) + .byPage({ maxPageSize: 1 }) + .next() + ).value; + assert.equal(resp2.segment.blobItems.length, 0); + }); +}); diff --git a/tests/dfs/apis/blockblob.test.ts b/tests/dfs/apis/blockblob.test.ts new file mode 100644 index 000000000..965142901 --- /dev/null +++ b/tests/dfs/apis/blockblob.test.ts @@ -0,0 +1,552 @@ +import { + BlobServiceClient, + newPipeline, + StorageSharedKeyCredential +} from "@azure/storage-blob"; + +import { configLogger } from "../../../src/common/Logger"; +import { + base64encode, + bodyToString, + EMULATOR_ACCOUNT_KEY, + EMULATOR_ACCOUNT_NAME, + getUniqueName +} from "../../testutils"; +import DataLakeTestServerFactory from "../DataLakeTestServerFactory"; + +import assert = require("assert"); +import crypto = require("crypto"); + +// Set true to enable debug log +configLogger(false); + +describe("BlockBlobAPIs", () => { + const factory = new DataLakeTestServerFactory(); + const server = factory.createServer(); + + const baseURL = `http://${server.config.host}:${server.config.port}/devstoreaccount1`; + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ) + ); + + let containerName: string = getUniqueName("container"); + let containerClient = serviceClient.getContainerClient(containerName); + let blobName: string = getUniqueName("blob"); + let blobClient = containerClient.getBlobClient(blobName); + let blockBlobClient = blobClient.getBlockBlobClient(); + + before(async () => { + await server.start(); + }); + + after(async () => { + await server.close(); + await server.clean(); + }); + + beforeEach(async () => { + containerName = getUniqueName("container"); + containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + blobName = getUniqueName("blob"); + blobClient = containerClient.getBlobClient(blobName); + blockBlobClient = blobClient.getBlockBlobClient(); + }); + + afterEach(async () => { + await containerClient.delete(); + }); + + it("upload with string body and default parameters @loki @sql", async () => { + const body: string = getUniqueName("randomstring"); + const result_upload = await blockBlobClient.upload(body, body.length); + assert.equal( + result_upload._response.request.headers.get("x-ms-client-request-id"), + result_upload.clientRequestId + ); + const result = await blobClient.download(0); + assert.deepStrictEqual(await bodyToString(result, body.length), body); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + }); + + it("upload empty blob @loki @sql", async () => { + await blockBlobClient.upload("", 0); + const result = await blobClient.download(0); + assert.deepStrictEqual(await bodyToString(result, 0), ""); + }); + + it("upload with string body and all parameters set @loki @sql", async () => { + const body: string = getUniqueName("randomstring"); + const options = { + blobCacheControl: "blobCacheControl", + blobContentDisposition: "blobContentDisposition", + blobContentEncoding: "blobContentEncoding", + blobContentLanguage: "blobContentLanguage", + blobContentType: "blobContentType", + metadata: { + keya: "vala", + keyb: "valb" + } + }; + const result_upload = await blockBlobClient.upload(body, body.length, { + blobHTTPHeaders: options, + metadata: options.metadata + }); + assert.equal( + result_upload._response.request.headers.get("x-ms-client-request-id"), + result_upload.clientRequestId + ); + const result = await blobClient.download(0); + assert.deepStrictEqual(await bodyToString(result, body.length), body); + assert.deepStrictEqual(result.cacheControl, options.blobCacheControl); + assert.deepStrictEqual( + result.contentDisposition, + options.blobContentDisposition + ); + assert.deepStrictEqual(result.contentEncoding, options.blobContentEncoding); + assert.deepStrictEqual(result.contentLanguage, options.blobContentLanguage); + assert.deepStrictEqual(result.contentType, options.blobContentType); + assert.deepStrictEqual(result.metadata, options.metadata); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + }); + + it("stageBlock @loki @sql", async () => { + const body = "HelloWorld"; + const result_stage = await blockBlobClient.stageBlock( + base64encode("1"), + body, + body.length + ); + assert.equal( + result_stage._response.request.headers.get("x-ms-client-request-id"), + result_stage.clientRequestId + ); + await blockBlobClient.stageBlock(base64encode("2"), body, body.length); + + // TODO: azure/storage-blob 12.9.0 will fail on list uncimmited blob from container, will skip following code until this is fix in SDK or Azurite + // const listBlobResponse = await ( + // await containerClient + // .listBlobsFlat({ includeUncommitedBlobs: true }) + // .byPage() + // .next() + // ).value; + // assert.equal(listBlobResponse.segment.blobItems.length, 1); + // assert.deepStrictEqual( + // listBlobResponse.segment.blobItems[0].properties.contentLength, + // 0 + // ); + + const listResponse = await blockBlobClient.getBlockList("uncommitted"); + assert.equal(listResponse.uncommittedBlocks!.length, 2); + assert.equal(listResponse.uncommittedBlocks![0].name, base64encode("1")); + assert.equal(listResponse.uncommittedBlocks![0].size, body.length); + assert.equal(listResponse.uncommittedBlocks![1].name, base64encode("2")); + assert.equal(listResponse.uncommittedBlocks![1].size, body.length); + assert.equal( + listResponse._response.request.headers.get("x-ms-client-request-id"), + listResponse.clientRequestId + ); + }); + + it("stageBlock with double commit block should work @loki @sql", async () => { + const body = "HelloWorld"; + + await blockBlobClient.stageBlock(base64encode("1"), body, body.length); + + await blockBlobClient.stageBlock(base64encode("1"), body, body.length); + + // TODO: azure/storage-blob 12.9.0 will fail on list uncimmited blob from container, will skip following code until this is fix in SDK or Azurite + // const listBlobResponse = ( + // await containerClient + // .listBlobsFlat({ includeUncommitedBlobs: true }) + // .byPage() + // .next() + // ).value; + // assert.equal(listBlobResponse.segment.blobItems.length, 1); + // assert.deepStrictEqual( + // listBlobResponse.segment.blobItems[0].properties.contentLength, + // 0 + // ); + + const listResponse = await blockBlobClient.getBlockList("uncommitted"); + assert.equal(listResponse.uncommittedBlocks!.length, 1); + assert.equal(listResponse.uncommittedBlocks![0].name, base64encode("1")); + assert.equal(listResponse.uncommittedBlocks![0].size, body.length); + assert.equal( + listResponse._response.request.headers.get("x-ms-client-request-id"), + listResponse.clientRequestId + ); + }); + + it("stageBlock with wrong body should throw md5 mismatch @loki @sql", async () => { + const body = "HelloWorld"; + const md5 = new Uint8Array(Buffer.from("anotherBody")); + const options = { transactionalContentMD5: md5 }; + + try { + await blockBlobClient.stageBlock( + base64encode("1"), + body, + body.length, + options + ); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 400); + assert.equal( + e.details.message.indexOf("Provided contentMD5 doesn't match."), + 0 + ); + return; + } + assert.fail("Did not throw an exception."); + }); + + it("stageBlock with md5 hash check @loki @sql", async () => { + const body = "HelloWorld"; + const md5 = crypto.createHash("md5").update(body, "utf8").digest(); + const options = { + transactionalContentMD5: new Uint8Array(Buffer.from(md5)) + }; + + await blockBlobClient.stageBlock( + base64encode("1"), + body, + body.length, + options + ); + + const listResponse = await blockBlobClient.getBlockList("uncommitted"); + assert.equal(listResponse.uncommittedBlocks!.length, 1); + assert.equal(listResponse.uncommittedBlocks![0].name, base64encode("1")); + assert.equal(listResponse.uncommittedBlocks![0].size, body.length); + }); + + it("commitBlockList @loki @sql", async () => { + const body = "HelloWorld"; + await blockBlobClient.stageBlock(base64encode("1"), body, body.length); + await blockBlobClient.stageBlock(base64encode("2"), body, body.length); + const result_commit = await blockBlobClient.commitBlockList([ + base64encode("1"), + base64encode("2") + ]); + assert.equal( + result_commit._response.request.headers.get("x-ms-client-request-id"), + result_commit.clientRequestId + ); + const listResponse = await blockBlobClient.getBlockList("committed"); + assert.equal(listResponse.committedBlocks!.length, 2); + assert.equal(listResponse.committedBlocks![0].name, base64encode("1")); + assert.equal(listResponse.committedBlocks![0].size, body.length); + assert.equal(listResponse.committedBlocks![1].name, base64encode("2")); + assert.equal(listResponse.committedBlocks![1].size, body.length); + assert.equal( + listResponse._response.request.headers.get("x-ms-client-request-id"), + listResponse.clientRequestId + ); + }); + + it("commitBlockList with previous committed blocks @loki @sql", async () => { + const body = "HelloWorld"; + await blockBlobClient.stageBlock(base64encode("1"), body, body.length); + await blockBlobClient.stageBlock(base64encode("2"), body, body.length); + const result_commit = await blockBlobClient.commitBlockList([ + base64encode("1"), + base64encode("2") + ]); + assert.equal( + result_commit._response.request.headers.get("x-ms-client-request-id"), + result_commit.clientRequestId + ); + + const properties1 = await blockBlobClient.getProperties(); + assert.notDeepStrictEqual(properties1.createdOn, undefined); + + const listResponse = await blockBlobClient.getBlockList("committed"); + assert.equal(listResponse.committedBlocks!.length, 2); + assert.equal(listResponse.committedBlocks![0].name, base64encode("1")); + assert.equal(listResponse.committedBlocks![0].size, body.length); + assert.equal(listResponse.committedBlocks![1].name, base64encode("2")); + assert.equal(listResponse.committedBlocks![1].size, body.length); + assert.equal( + listResponse._response.request.headers.get("x-ms-client-request-id"), + listResponse.clientRequestId + ); + + await blockBlobClient.commitBlockList([base64encode("2")]); + const listResponse2 = await blockBlobClient.getBlockList("committed"); + assert.equal(listResponse2.committedBlocks!.length, 1); + assert.equal(listResponse2.committedBlocks![0].name, base64encode("2")); + assert.equal(listResponse2.committedBlocks![0].size, body.length); + + const properties2 = await blockBlobClient.getProperties(); + assert.notDeepStrictEqual(properties2.createdOn, undefined); + assert.deepStrictEqual(properties1.createdOn, properties2.createdOn); + }); + + it("commitBlockList with empty list should create an empty block blob @loki @sql", async () => { + await blockBlobClient.commitBlockList([]); + + const listResponse = await blockBlobClient.getBlockList("committed"); + assert.equal(listResponse.committedBlocks!.length, 0); + + const result = await blobClient.download(0); + assert.deepStrictEqual(await bodyToString(result, 0), ""); + assert.equal(true, result._response.headers.contains("x-ms-creation-time")); + }); + + it("commitBlockList with empty list should not work with ifNoneMatch=* for existing blob @loki @sql", async () => { + await blockBlobClient.commitBlockList([]); + + try { + await blockBlobClient.commitBlockList([], { + conditions: { + ifNoneMatch: "*" + } + }); + } catch (error) { + assert.deepStrictEqual(error.statusCode, 409); + return; + } + + assert.fail(); + }); + + it("upload should not work with ifNoneMatch=* for existing blob @loki @sql", async () => { + await blockBlobClient.commitBlockList([]); + + try { + await blockBlobClient.upload("hello", 5, { + conditions: { + ifNoneMatch: "*" + } + }); + } catch (error) { + assert.deepStrictEqual(error.statusCode, 409); + return; + } + + assert.fail(); + }); + + it("commitBlockList with all parameters set @loki @sql", async () => { + const body = "HelloWorld"; + await blockBlobClient.stageBlock(base64encode("1"), body, body.length); + await blockBlobClient.stageBlock(base64encode("2"), body, body.length); + + const options = { + blobCacheControl: "blobCacheControl", + blobContentDisposition: "blobContentDisposition", + blobContentEncoding: "blobContentEncoding", + blobContentLanguage: "blobContentLanguage", + blobContentType: "blobContentType", + metadata: { + keya: "vala", + keyb: "valb" + } + }; + await blockBlobClient.commitBlockList( + [base64encode("1"), base64encode("2")], + { + blobHTTPHeaders: options, + metadata: options.metadata + } + ); + + const listResponse = await blockBlobClient.getBlockList("committed"); + assert.equal(listResponse.committedBlocks!.length, 2); + assert.equal(listResponse.committedBlocks![0].name, base64encode("1")); + assert.equal(listResponse.committedBlocks![0].size, body.length); + assert.equal(listResponse.committedBlocks![1].name, base64encode("2")); + assert.equal(listResponse.committedBlocks![1].size, body.length); + + const result = await blobClient.download(0); + assert.deepStrictEqual( + await bodyToString(result, body.repeat(2).length), + body.repeat(2) + ); + assert.deepStrictEqual(result.cacheControl, options.blobCacheControl); + assert.deepStrictEqual( + result.contentDisposition, + options.blobContentDisposition + ); + assert.deepStrictEqual(result.contentEncoding, options.blobContentEncoding); + assert.deepStrictEqual(result.contentLanguage, options.blobContentLanguage); + assert.deepStrictEqual(result.contentType, options.blobContentType); + assert.deepStrictEqual(result.metadata, options.metadata); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + }); + + it("getBlockList @loki @sql", async () => { + const body = "HelloWorld"; + await blockBlobClient.stageBlock(base64encode("1"), body, body.length); + await blockBlobClient.stageBlock(base64encode("2"), body, body.length); + await blockBlobClient.commitBlockList([base64encode("2")]); + const listResponse = await blockBlobClient.getBlockList("all"); + assert.equal(listResponse.committedBlocks!.length, 1); + assert.equal(listResponse.uncommittedBlocks!.length, 0); + assert.equal(listResponse.committedBlocks![0].name, base64encode("2")); + assert.equal(listResponse.committedBlocks![0].size, body.length); + }); + + it("getBlockList_BlockListingFilter @loki @sql", async () => { + const body = "HelloWorld"; + await blockBlobClient.stageBlock(base64encode("1"), body, body.length); + await blockBlobClient.stageBlock(base64encode("2"), body, body.length); + + // Getproperties on a block blob without commited block will return 404 + let err; + try { + await blockBlobClient.getProperties(); + } catch (error) { + err = error; + } + assert.deepStrictEqual(err.statusCode, 404); + + // Stage block with block Id length different than the exist uncommited blocks will fail with 400 + try { + await blockBlobClient.stageBlock(base64encode("123"), body, body.length); + } catch (error) { + err = error; + } + assert.deepStrictEqual(err.statusCode, 400); + + await blockBlobClient.commitBlockList([ + base64encode("1"), + base64encode("2") + ]); + + await blockBlobClient.stageBlock(base64encode("123"), body, body.length); + + let listResponse = await blockBlobClient.getBlockList("committed"); + assert.equal(listResponse.committedBlocks!.length, 2); + assert.equal(listResponse.committedBlocks![0].name, base64encode("1")); + assert.equal(listResponse.committedBlocks![0].size, body.length); + assert.equal(listResponse.committedBlocks![1].name, base64encode("2")); + assert.equal(listResponse.committedBlocks![1].size, body.length); + assert.equal(listResponse.uncommittedBlocks!.length, 0); + + listResponse = await blockBlobClient.getBlockList("uncommitted"); + assert.equal(listResponse.uncommittedBlocks!.length, 1); + assert.equal(listResponse.uncommittedBlocks![0].name, base64encode("123")); + assert.equal(listResponse.uncommittedBlocks![0].size, body.length); + assert.equal(listResponse.committedBlocks!.length, 0); + + listResponse = await blockBlobClient.getBlockList("all"); + assert.equal(listResponse.committedBlocks!.length, 2); + assert.equal(listResponse.committedBlocks![0].name, base64encode("1")); + assert.equal(listResponse.committedBlocks![0].size, body.length); + assert.equal(listResponse.committedBlocks![1].name, base64encode("2")); + assert.equal(listResponse.committedBlocks![1].size, body.length); + assert.equal(listResponse.uncommittedBlocks!.length, 1); + assert.equal(listResponse.uncommittedBlocks![0].name, base64encode("123")); + assert.equal(listResponse.uncommittedBlocks![0].size, body.length); + }); + + it("getBlockList for non-existent blob @loki @sql", async () => { + try { + await blockBlobClient.getBlockList("committed"); + } catch (error) { + assert.deepEqual(404, error.statusCode); + return; + } + assert.fail(); + }); + + it("getBlockList for non-existent container @loki @sql", async () => { + const fakeContainer = getUniqueName("container"); + const fakeContainerClient = serviceClient.getContainerClient(fakeContainer); + const fakeBlobClient = fakeContainerClient.getBlobClient(blobName); + const fakeBlockBlobClient = fakeBlobClient.getBlockBlobClient(); + + try { + await fakeBlockBlobClient.getBlockList("committed"); + } catch (error) { + assert.deepEqual(404, error.statusCode); + return; + } + assert.fail(); + }); + + it("getBlockList from snapshot @loki @sql", async () => { + const body = "HelloWorld"; + await blockBlobClient.stageBlock(base64encode("1"), body, body.length); + await blockBlobClient.stageBlock(base64encode("2"), body, body.length); + await blockBlobClient.commitBlockList([base64encode("1")]); + + // Create blob snapshot + const result = await blobClient.createSnapshot(); + assert.ok(result.snapshot); + const blobSnapshotURL = blockBlobClient.withSnapshot(result.snapshot!); + await blobSnapshotURL.getProperties(); + + // Update base blob + await blockBlobClient.stageBlock(base64encode("3"), body, body.length); + await blockBlobClient.stageBlock(base64encode("4"), body, body.length); + await blockBlobClient.commitBlockList([ + base64encode("3"), + base64encode("4") + ]); + + const listResponse = await blobSnapshotURL.getBlockList("all"); + assert.equal(listResponse.committedBlocks!.length, 1); + assert.equal(listResponse.uncommittedBlocks!.length, 0); + assert.equal(listResponse.committedBlocks![0].name, base64encode("1")); + assert.equal(listResponse.committedBlocks![0].size, body.length); + }); + + it("upload with Readable stream body and default parameters @loki @sql", async () => { + const body: string = getUniqueName("randomstring"); + const bodyBuffer = Buffer.from(body); + + await blockBlobClient.upload(bodyBuffer, body.length); + const result = await blobClient.download(0); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + + const downloadedBody = await new Promise((resolve, reject) => { + const buffer: string[] = []; + result.readableStreamBody!.on("data", (data: Buffer) => { + buffer.push(data.toString()); + }); + result.readableStreamBody!.on("end", () => { + resolve(buffer.join("")); + }); + result.readableStreamBody!.on("error", reject); + }); + + assert.deepStrictEqual(downloadedBody, body); + }); + + it("upload with Chinese string body and default parameters @loki @sql", async () => { + const body: string = getUniqueName("randomstring你好"); + await blockBlobClient.upload(body, Buffer.byteLength(body)); + const result = await blobClient.download(0); + assert.deepStrictEqual( + await bodyToString(result, Buffer.byteLength(body)), + body + ); + }); +}); diff --git a/tests/dfs/apis/container.test.ts b/tests/dfs/apis/container.test.ts new file mode 100644 index 000000000..91a431efd --- /dev/null +++ b/tests/dfs/apis/container.test.ts @@ -0,0 +1,1214 @@ +import { + AccountSASPermissions, + AccountSASResourceTypes, + AccountSASServices, + AnonymousCredential, + BlobServiceClient, + generateAccountSASQueryParameters, + newPipeline, + StorageSharedKeyCredential +} from "@azure/storage-blob"; + +import { configLogger } from "../../../src/common/Logger"; +import StorageErrorFactory from "../../../src/dfs/errors/StorageErrorFactory"; +import Context from "../../../src/dfs/generated/Context"; +import { + base64encode, + EMULATOR_ACCOUNT_KEY, + EMULATOR_ACCOUNT_NAME, + getUniqueName, + sleep +} from "../../testutils"; +import DataLakeTestServerFactory from "../DataLakeTestServerFactory"; +import QueryRequestPolicyFactory from "../RequestPolicy/QueryRequestPolicyFactory"; + +import assert = require("assert"); +const context = { contextId: "" } as Context; + +// Set to true enable debug log +configLogger(false); + +describe("ContainerAPIs", () => { + const factory = new DataLakeTestServerFactory(); + const server = factory.createServer(); + + const baseURL = `http://${server.config.host}:${server.config.port}/devstoreaccount1`; + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ) + ); + + let containerName: string = getUniqueName("container"); + let containerClient = serviceClient.getContainerClient(containerName); + let blobLeaseClient = containerClient.getBlobLeaseClient(); + + before(async () => { + await server.start(); + }); + + after(async () => { + await server.close(); + await server.clean(); + }); + + beforeEach(async () => { + containerName = getUniqueName("container"); + containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + blobLeaseClient = containerClient.getBlobLeaseClient(); + }); + + afterEach(async () => { + await containerClient.delete(); + }); + + it("setMetadata @loki @sql", async () => { + const metadata = { + key0: "val0", + keya: "vala", + keyb: "valb" + }; + await containerClient.setMetadata(metadata); + + const result = await containerClient.getProperties(); + assert.deepEqual(result.metadata, metadata); + }); + + it("setMetadata should work with conditional headers @loki @sql", async () => { + // const properties = await containerClient.getProperties(); + await containerClient.setMetadata( + {}, + { + conditions: { + ifModifiedSince: new Date("2018/01/01") + } + } + ); + }); + + it("setMetadata should not work with invalid conditional headers @loki @sql", async () => { + try { + await containerClient.setMetadata( + {}, + { + conditions: { + ifModifiedSince: new Date("2118/01/01") + } + } + ); + } catch (error) { + assert.deepStrictEqual(error.statusCode, 412); + return; + } + + assert.fail(); + }); + + it("getProperties @loki @sql", async () => { + const result = await containerClient.getProperties(); + assert.ok(result.etag!.length > 0); + assert.ok(result.lastModified); + assert.ok(!result.leaseDuration); + assert.equal(result.leaseState, "available"); + assert.equal(result.leaseStatus, "unlocked"); + assert.ok(result.requestId); + assert.ok(result.version); + assert.ok(result.date); + assert.ok(!result.blobPublicAccess); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + }); + + it("getProperties should return 404 for non existed container @loki @sql", async () => { + const nonExistedContainerURL = + serviceClient.getContainerClient("404container-a"); + let expectedError = false; + try { + await nonExistedContainerURL.getProperties(); + } catch (err) { + if (err.response.status === 404) { + expectedError = true; + } + } + assert.ok(expectedError); + }); + + it("getProperties should return 404 for non existed system container @loki @sql", async () => { + const nonExistedContainerURL = serviceClient.getContainerClient("$logs"); + let expectedError = false; + try { + await nonExistedContainerURL.getProperties(); + } catch (err) { + if (err.response.status === 404) { + expectedError = true; + } + } + assert.ok(expectedError); + }); + + it("create with default parameters @loki @sql", (done) => { + // create() with default parameters has been tested in beforeEach + done(); + }); + + it("create with all parameters configured @loki @sql", async () => { + const cURL = serviceClient.getContainerClient(getUniqueName(containerName)); + const metadata = { key: "value" }; + const access = "container"; + const result_create = await cURL.create({ metadata, access }); + assert.equal( + result_create._response.request.headers.get("x-ms-client-request-id"), + result_create.clientRequestId + ); + const result = await cURL.getProperties(); + assert.deepEqual(result.blobPublicAccess, access); + assert.deepEqual(result.metadata, metadata); + }); + + it("create with invalid container name @loki @sql", async () => { + // invalid name since format not right + let invalidName = ["abc-", "abc--d", "Acontainer", "-abc", "dev_public"]; + let expectedError = StorageErrorFactory.getInvalidResourceName(context); + + for (const item in invalidName) { + try { + let cURL = serviceClient.getContainerClient(invalidName[item]); + const result = await cURL.create(); + assert.deepStrictEqual( + result._response.status, + expectedError.statusCode + ); + } catch (error) { + assert.strictEqual(error.statusCode, expectedError.statusCode); + assert.strictEqual(error.code, expectedError.errorCode); + assert.strictEqual( + error.response.parsedBody.message.includes( + expectedError.errorMessage + ), + true + ); + } + } + + // invalid name since length not right + invalidName = [ + "a", + "ab", + "abcd123456789012345678901234567890123456789012345678901234567890" + ]; + expectedError = StorageErrorFactory.getOutOfRangeName(context); + for (const item in invalidName) { + try { + let cURL = serviceClient.getContainerClient(invalidName[item]); + const result = await cURL.create(); + assert.deepStrictEqual( + result._response.status, + expectedError.statusCode + ); + } catch (error) { + assert.strictEqual(error.statusCode, expectedError.statusCode); + assert.strictEqual(error.code, expectedError.errorCode); + assert.strictEqual( + error.response.parsedBody.message.includes( + expectedError.errorMessage + ), + true + ); + } + } + }); + + it("delete @loki @sql", (done) => { + // delete() with default parameters has been tested in afterEach + done(); + }); + + it("listBlobHierarchySegment with default parameters @loki @sql", async () => { + const blobClients = []; + for (let i = 0; i < 3; i++) { + const blobClient = containerClient.getBlobClient( + getUniqueName(`blockblob${i}/${i}`) + ); + const blockBlobClient = blobClient.getBlockBlobClient(); + await blockBlobClient.upload("", 0); + blobClients.push(blobClient); + } + + const delimiter = "/"; + const result = ( + await containerClient.listBlobsByHierarchy(delimiter).byPage().next() + ).value; + assert.ok(result.serviceEndpoint.length > 0); + assert.ok(containerClient.url.indexOf(result.containerName)); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + assert.deepStrictEqual(result.continuationToken, ""); + assert.deepStrictEqual(result.delimiter, delimiter); + assert.deepStrictEqual( + result.segment.blobPrefixes!.length, + blobClients.length + ); + + for (const blob of blobClients) { + let i = 0; + assert.ok(blob.url.indexOf(result.segment.blobPrefixes![i++].name)); + } + + for (const blob of blobClients) { + await blob.delete(); + } + }); + + it("listBlobHierarchySegment with all parameters configured @loki @sql", async () => { + const blobClients = []; + const prefix = "blockblob"; + const metadata = { + keya: "a", + keyb: "c" + }; + const delimiter = "/"; + for (let i = 0; i < 2; i++) { + const blobClient = containerClient.getBlobClient( + getUniqueName(`${prefix}${i}${delimiter}${i}`) + ); + const blockBlobClient = blobClient.getBlockBlobClient(); + await blockBlobClient.upload("", 0, { + metadata + }); + blobClients.push(blobClient); + } + + const result = ( + await containerClient + .listBlobsByHierarchy(delimiter, { + includeCopy: true, + includeDeleted: true, + includeMetadata: true, + includeSnapshots: true, + includeUncommitedBlobs: true, + prefix + }) + .byPage({ maxPageSize: 1 }) + .next() + ).value; + assert.ok(result.serviceEndpoint.length > 0); + assert.ok(containerClient.url.indexOf(result.containerName)); + assert.deepStrictEqual(result.segment.blobPrefixes!.length, 1); + assert.deepStrictEqual(result.segment.blobItems!.length, 0); + assert.ok(blobClients[0].url.indexOf(result.segment.blobPrefixes![0].name)); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + + const result2 = ( + await containerClient + .listBlobsByHierarchy(delimiter, { + includeCopy: true, + includeDeleted: true, + includeMetadata: true, + includeSnapshots: true, + includeUncommitedBlobs: true, + prefix + }) + .byPage({ + continuationToken: result.continuationToken, + maxPageSize: 2 + }) + .next() + ).value; + assert.ok(result2.serviceEndpoint.length > 0); + assert.ok(containerClient.url.indexOf(result2.containerName)); + assert.deepStrictEqual(result2.segment.blobPrefixes!.length, 1); + assert.deepStrictEqual(result2.segment.blobItems!.length, 0); + assert.ok( + blobClients[0].url.indexOf(result2.segment.blobPrefixes![0].name) + ); + + const result3 = ( + await containerClient + .listBlobsByHierarchy(delimiter, { + includeCopy: true, + includeDeleted: true, + includeMetadata: true, + includeSnapshots: true, + includeUncommitedBlobs: true, + prefix: `${prefix}0${delimiter}` + }) + .byPage({ maxPageSize: 2 }) + .next() + ).value; + assert.ok(result3.serviceEndpoint.length > 0); + assert.ok(containerClient.url.indexOf(result3.containerName)); + assert.deepStrictEqual(result3.continuationToken, ""); + assert.deepStrictEqual(result3.delimiter, delimiter); + assert.deepStrictEqual(result3.segment.blobItems!.length, 1); + assert.deepStrictEqual(result3.segment.blobItems![0].metadata, { + ...metadata + }); + assert.ok(blobClients[0].url.indexOf(result3.segment.blobItems![0].name)); + const getResult = await blobClients[0].getProperties(); + assert.equal( + getResult.etag, + '"' + result3.segment.blobItems![0].properties.etag + '"' + ); + + for (const blob of blobClients) { + await blob.delete(); + } + }); + + it("acquireLease_available_proposedLeaseId_fixed @loki @sql", async () => { + const guid = "ca761232-ed42-11ce-bacd-00aa0057b223"; + const duration = 30; + blobLeaseClient = containerClient.getBlobLeaseClient(guid); + const result_acquire = await blobLeaseClient.acquireLease(duration); + assert.equal( + result_acquire._response.request.headers.get("x-ms-client-request-id"), + result_acquire._response.request.requestId + ); + + const result = await containerClient.getProperties(); + assert.equal(result.leaseDuration, "fixed"); + assert.equal(result.leaseState, "leased"); + assert.equal(result.leaseStatus, "locked"); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + + const result_release = await blobLeaseClient.releaseLease(); + assert.equal( + result_release._response.request.headers.get("x-ms-client-request-id"), + result_release._response.request.requestId + ); + }); + + it("acquireLease_available_NoproposedLeaseId_infinite @loki @sql", async () => { + const leaseResult = await blobLeaseClient.acquireLease(-1); + const leaseId = leaseResult.leaseId; + assert.ok(leaseId); + + const result = await containerClient.getProperties(); + assert.equal(result.leaseDuration, "infinite"); + assert.equal(result.leaseState, "leased"); + assert.equal(result.leaseStatus, "locked"); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + + await blobLeaseClient.releaseLease(); + }); + + it("releaseLease @loki @sql", async () => { + const guid = "ca761232ed4211cebacd00aa0057b223"; + const duration = -1; + blobLeaseClient = containerClient.getBlobLeaseClient(guid); + await blobLeaseClient.acquireLease(duration); + + const result = await containerClient.getProperties(); + assert.equal(result.leaseDuration, "infinite"); + assert.equal(result.leaseState, "leased"); + assert.equal(result.leaseStatus, "locked"); + + await blobLeaseClient.releaseLease(); + }); + + it("renewLease @loki @sql", async () => { + const guid = "ca761232ed4211cebacd00aa0057b223"; + const duration = 15; + blobLeaseClient = containerClient.getBlobLeaseClient(guid); + await blobLeaseClient.acquireLease(duration); + + const result = await containerClient.getProperties(); + assert.equal(result.leaseDuration, "fixed"); + assert.equal(result.leaseState, "leased"); + assert.equal(result.leaseStatus, "locked"); + + await sleep(16 * 1000); + const result2 = await containerClient.getProperties(); + assert.ok(!result2.leaseDuration); + assert.equal(result2.leaseState, "expired"); + assert.equal(result2.leaseStatus, "unlocked"); + + await blobLeaseClient.renewLease(); + const result3 = await containerClient.getProperties(); + assert.equal(result3.leaseDuration, "fixed"); + assert.equal(result3.leaseState, "leased"); + assert.equal(result3.leaseStatus, "locked"); + + await blobLeaseClient.releaseLease(); + }); + + it("changeLease @loki @sql", async () => { + const guid = "ca761232ed4211cebacd00aa0057b223"; + const duration = 15; + blobLeaseClient = containerClient.getBlobLeaseClient(guid); + await blobLeaseClient.acquireLease(duration); + + const result = await containerClient.getProperties(); + assert.equal(result.leaseDuration, "fixed"); + assert.equal(result.leaseState, "leased"); + assert.equal(result.leaseStatus, "locked"); + + const newGuid = "3c7e72ebb4304526bc53d8ecef03798f"; + const result_change = await blobLeaseClient.changeLease(newGuid); + assert.equal( + result_change._response.request.headers.get("x-ms-client-request-id"), + result_change._response.request.requestId + ); + + await containerClient.getProperties(); + await blobLeaseClient.releaseLease(); + }); + + it("breakLease @loki @sql", async () => { + const guid = "ca761232ed4211cebacd00aa0057b223"; + const duration = 15; + blobLeaseClient = containerClient.getBlobLeaseClient(guid); + await blobLeaseClient.acquireLease(duration); + + const result = await containerClient.getProperties(); + assert.equal(result.leaseDuration, "fixed"); + assert.equal(result.leaseState, "leased"); + assert.equal(result.leaseStatus, "locked"); + + const breakDuration = 30; + let breaklefttime = breakDuration; + while (breaklefttime > 0) { + const breakResult = await blobLeaseClient.breakLease(breakDuration); + + assert.equal(breakResult.leaseTime! <= breaklefttime, true); + assert.equal( + breakResult._response.request.headers.get("x-ms-client-request-id"), + breakResult._response.request.requestId + ); + breaklefttime = breakResult.leaseTime!; + + const result2 = await containerClient.getProperties(); + assert.ok(!result2.leaseDuration); + if (breaklefttime !== 0) { + assert.equal(result2.leaseState, "breaking"); + assert.equal(result2.leaseStatus, "locked"); + } + + await sleep(500); + } + + const result3 = await containerClient.getProperties(); + assert.ok(!result3.leaseDuration); + assert.equal(result3.leaseState, "broken"); + assert.equal(result3.leaseStatus, "unlocked"); + }); + + it("should correctly list all blobs in the container using listBlobFlatSegment with default parameters @loki @sql", async () => { + const blobClients = []; + for (let i = 0; i < 3; i++) { + const blobClient = containerClient.getBlobClient( + getUniqueName(`blockblob${i}/${i}`) + ); + const blockBlobClient = blobClient.getBlockBlobClient(); + await blockBlobClient.upload("", 0); + blobClients.push(blobClient); + } + + const inputmarker = undefined; + const result = ( + await containerClient + .listBlobsFlat() + .byPage({ continuationToken: inputmarker }) + .next() + ).value; + assert.ok(result.serviceEndpoint.length > 0); + assert.ok(containerClient.url.indexOf(result.containerName)); + assert.deepStrictEqual(result.continuationToken, ""); + assert.deepStrictEqual( + result.segment.blobItems!.length, + blobClients.length + ); + + let i = 0; + for (const blob of blobClients) { + assert.ok(blob.url.indexOf(result.segment.blobItems![i].name)); + const getResult = await blob.getProperties(); + assert.equal( + getResult.etag, + '"' + result.segment.blobItems![i].properties.etag + '"' + ); + assert.deepStrictEqual(result.segment.blobItems![i].snapshot, undefined); + i++; + } + + for (const blob of blobClients) { + await blob.delete(); + } + }); + + // TODO: azure/storage-blob 12.9.0 will fail on list uncimmited blob from container, will skip the case until this is fix in SDK or Azurite + it.skip("should only show uncommitted blobs in listBlobFlatSegment with uncommittedblobs option @loki @sql", async () => { + const blobClient = containerClient.getBlobClient( + getUniqueName("uncommittedblob") + ); + const blockBlobClient = blobClient.getBlockBlobClient(); + + const body = "HelloWorld"; + await blockBlobClient.stageBlock(base64encode("1"), body, body.length); + + const result1 = ( + await containerClient + .listBlobsFlat({ + includeUncommitedBlobs: true + }) + .byPage() + .next() + ).value; + assert.equal(result1.segment.blobItems.length, 1); + + const result2 = (await containerClient.listBlobsFlat().byPage().next()) + .value; + assert.equal(result2.segment.blobItems.length, 0); + }); + + it("should only show uncommitted blobs in listBlobHierarchySegment with uncommittedblobs option @loki @sql", async () => { + const delimiter = "/"; + const blobClient = containerClient.getBlobClient( + getUniqueName("path/uncommittedblob") + ); + const blockBlobClient = blobClient.getBlockBlobClient(); + + const body = "HelloWorld"; + await blockBlobClient.stageBlock(base64encode("1"), body, body.length); + + const result1 = ( + await containerClient + .listBlobsByHierarchy(delimiter, { + includeUncommitedBlobs: true + }) + .byPage() + .next() + ).value; + assert.equal(result1.segment.blobPrefixes!.length, 1); + + const result2 = ( + await containerClient.listBlobsByHierarchy(delimiter).byPage().next() + ).value; + assert.equal(result2.segment.blobPrefixes!.length, 0); + }); + + it("should correctly order all blobs in the container @loki @sql", async () => { + const blobClients = []; + const blobNames: Array = []; + + for (let i = 1; i < 4; i++) { + const name = `blockblob${i}/abc-00${i}`; + const blobClient = containerClient.getBlobClient(name); + const blockBlobClient = blobClient.getBlockBlobClient(); + await blockBlobClient.upload("", 0); + blobClients.push(blobClient); + blobNames.push(name); + } + + const inputmarker = undefined; + const result = ( + await containerClient + .listBlobsFlat({ + prefix: "blockblob" + }) + .byPage({ continuationToken: inputmarker }) + .next() + ).value; + assert.ok(result.serviceEndpoint.length > 0); + assert.ok(containerClient.url.indexOf(result.containerName)); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + + const gotNames: Array = []; + + for (const item of result.segment.blobItems) { + gotNames.push(item.name); + } + + assert.deepStrictEqual(gotNames, blobNames); + + for (const blob of blobClients) { + await blob.delete(); + } + }); + + it("returns no continuationToken when squashed by delimiter @loki @sql", async () => { + const blobClients = []; + let blobNames: Array = [ + "blockblob/abc-001", + "blockblob/abc-003", + "blockblob/abc-002", + "blockblob/abc-004" + ]; + for (let i = 0; i < blobNames.length; i++) { + const blobClient = containerClient.getBlobClient(blobNames[i]); + const blockBlobClient = blobClient.getBlockBlobClient(); + await blockBlobClient.upload("", 0); + blobClients.push(blobClient); + } + + const inputmarker = undefined; + let result = ( + await containerClient + .listBlobsByHierarchy("/", { + prefix: "" + }) + .byPage({ + continuationToken: inputmarker, + maxPageSize: 2 + }) + .next() + ).value; + assert.ok(result.serviceEndpoint.length > 0); + assert.ok(containerClient.url.indexOf(result.containerName)); + assert.equal(result.continuationToken, ""); + assert.equal(result.segment.blobItems.length, 0); + assert.equal(result.segment.blobPrefixes.length, 1); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + + const gotPrefixes: Array = []; + + for (const item of result.segment.blobPrefixes) { + gotPrefixes.push(item.name); + } + + assert.deepStrictEqual(gotPrefixes, ["blockblob/"]); + + for (const blob of blobClients) { + await blob.delete(); + } + }); + + it("returns a valid, correct continuationToken @loki @sql", async () => { + const blobClients = []; + let blobNames: Array = [ + "blockblob/abc-001", + "blockblob/abc-004", + "blockblob/abc-009", + "blockblob/abc-003", + "blockblob/abc-006", + "blockblob/abc-000", + "blockblob/abc-007", + "blockblob/abc-002", + "blockblob/abc-005", + "blockblob/abc-008" + ]; + + for (let i = 0; i < 10; i++) { + const blobClient = containerClient.getBlobClient(blobNames[i]); + const blockBlobClient = blobClient.getBlockBlobClient(); + await blockBlobClient.upload("", 0); + blobClients.push(blobClient); + } + + // Sort blob names for comparison + blobNames = blobNames.sort(); + + const inputmarker = undefined; + let result = ( + await containerClient + .listBlobsFlat() + .byPage({ continuationToken: inputmarker, maxPageSize: 4 }) + .next() + ).value; + assert.ok(result.serviceEndpoint.length > 0); + assert.ok(containerClient.url.indexOf(result.containerName)); + assert.equal(result.continuationToken, "blockblob/abc-003"); + assert.equal(result.segment.blobItems.length, 4); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + + const gotNames: Array = []; + + for (const item of result.segment.blobItems) { + gotNames.push(item.name); + } + + result = ( + await containerClient + .listBlobsFlat() + .byPage({ + continuationToken: result.continuationToken, + maxPageSize: 4 + }) + .next() + ).value; + assert.ok(result.serviceEndpoint.length > 0); + assert.ok(containerClient.url.indexOf(result.containerName)); + assert.equal(result.continuationToken, "blockblob/abc-007"); + assert.equal(result.segment.blobItems.length, 4); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + + for (const item of result.segment.blobItems) { + gotNames.push(item.name); + } + + result = ( + await containerClient + .listBlobsFlat() + .byPage({ + continuationToken: result.continuationToken, + maxPageSize: 4 + }) + .next() + ).value; + assert.ok(result.serviceEndpoint.length > 0); + assert.ok(containerClient.url.indexOf(result.containerName)); + assert.strictEqual(result.continuationToken, ""); + assert.equal(result.segment.blobItems.length, 2); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + + for (const item of result.segment.blobItems) { + gotNames.push(item.name); + } + + assert.deepStrictEqual(gotNames, blobNames); + + for (const blob of blobClients) { + await blob.delete(); + } + }); + + it("list blobs whose name are all number, continuationToken works @loki @sql", async () => { + const blobClients = []; + + for (let i = 0; i < 20; i++) { + const blobClient = containerClient.getBlobClient(i.toString()); + const blockBlobClient = blobClient.getBlockBlobClient(); + await blockBlobClient.upload("", 0); + blobClients.push(blobClient); + } + + let blobNames: Array = [ + "0", + "1", + "10", + "11", + "12", + "13", + "14", + "15", + "16", + "17", + "18", + "19", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9" + ]; + + const inputmarker = undefined; + let result = ( + await containerClient + .listBlobsFlat() + .byPage({ continuationToken: inputmarker, maxPageSize: 10 }) + .next() + ).value; + assert.ok(result.serviceEndpoint.length > 0); + assert.ok(containerClient.url.indexOf(result.containerName)); + assert.equal(result.continuationToken, blobNames[9]); + assert.equal(result.segment.blobItems.length, 10); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + + const gotNames: Array = []; + + for (const item of result.segment.blobItems) { + gotNames.push(item.name); + } + + result = ( + await containerClient + .listBlobsFlat() + .byPage({ + continuationToken: result.continuationToken, + maxPageSize: 10 + }) + .next() + ).value; + assert.ok(result.serviceEndpoint.length > 0); + assert.ok(containerClient.url.indexOf(result.containerName)); + assert.equal(result.continuationToken, ""); + assert.equal(result.segment.blobItems.length, 10); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + + for (const item of result.segment.blobItems) { + gotNames.push(item.name); + } + + assert.deepStrictEqual(gotNames, blobNames); + + for (const blob of blobClients) { + await blob.delete(); + } + }); + + it("getAccessPolicy @loki @sql", async () => { + const result = await containerClient.getAccessPolicy(); + assert.ok(result.etag!.length > 0); + assert.ok(result.lastModified); + assert.ok(result.requestId); + assert.ok(result.version); + assert.ok(result.date); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + }); + + it("setAccessPolicy_publicAccess @loki @sql", async () => { + const access = "blob"; + const containerAcl = [ + { + accessPolicy: { + expiresOn: new Date("2018-12-31T11:22:33.4567890Z"), + permissions: "rwd", + startsOn: new Date("2017-12-31T11:22:33.4567890Z") + }, + id: "MTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTI=" + } + ]; + await containerClient.setAccessPolicy(access, containerAcl); + const result = await containerClient.getAccessPolicy(); + // assert.deepEqual(result.signedIdentifiers, containerAcl); + assert.deepEqual(result.blobPublicAccess, access); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + }); + + // Skip since getAccessPolicy can't get signedIdentifiers now + it("setAccessPolicy_signedIdentifiers @loki @sql", async () => { + const access = "container"; + const containerAcl = [ + { + accessPolicy: { + expiresOn: new Date("2018-12-31T11:22:33.4567890Z"), + permissions: "rwdl", + startsOn: new Date("2017-12-31T11:22:33.4567890Z") + }, + id: "MTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTI=" + }, + { + accessPolicy: { + expiresOn: new Date("2030-11-31T11:22:33.4567890Z"), + permissions: "w", + startsOn: new Date("2017-12-31T11:22:33.4567890Z") + }, + id: "policy2" + } + ]; + + const result_set = await containerClient.setAccessPolicy( + access, + containerAcl + ); + assert.equal( + result_set._response.request.headers.get("x-ms-client-request-id"), + result_set.clientRequestId + ); + const result = await containerClient.getAccessPolicy(); + assert.deepEqual(result.signedIdentifiers, containerAcl); + assert.deepEqual(result.blobPublicAccess, access); + }); + + it("list container should success with include as empty string or deleted @loki @sql", async () => { + // create account sas + const storageSharedKeyCredential = new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ); + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + const sas = generateAccountSASQueryParameters( + { + expiresOn: tmr, + permissions: AccountSASPermissions.parse("rl"), + resourceTypes: AccountSASResourceTypes.parse("sco").toString(), + services: AccountSASServices.parse("b").toString(), + version: "2020-04-08" + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ).toString(); + + // list with include as empty string + // create service client + let pipeline = newPipeline(new AnonymousCredential(), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }); + pipeline.factories.unshift( + new QueryRequestPolicyFactory("include=metadata", "include=") + ); + let serviceClientForOptions = new BlobServiceClient( + `${baseURL}?${sas}`, + pipeline + ); + + // list containers + let result = ( + await serviceClientForOptions + .listContainers({ + includeMetadata: true + }) + .byPage() + .next() + ).value; + + assert.ok(result); + + // list with include as deleted + // create service client + pipeline = newPipeline(new AnonymousCredential(), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }); + pipeline.factories.unshift( + new QueryRequestPolicyFactory("include=metadata", "include=deleted") + ); + serviceClientForOptions = new BlobServiceClient( + `${baseURL}?${sas}`, + pipeline + ); + + // list containers + result = ( + await serviceClientForOptions + .listContainers({ + includeMetadata: true + }) + .byPage() + .next() + ).value; + + assert.ok(result); + }); + + it("list container should success with different include string @loki @sql", async () => { + // prepare blobs + const blobClients = []; + for (let i = 0; i < 3; i++) { + const blobClient = containerClient.getBlobClient( + getUniqueName(`blockblob${i}/${i}`) + ); + const blockBlobClient = blobClient.getBlockBlobClient(); + await blockBlobClient.upload("", 0); + blobClients.push(blobClient); + } + blobClients[0].createSnapshot(); + + // create account sas + const storageSharedKeyCredential = new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ); + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + const sas = generateAccountSASQueryParameters( + { + expiresOn: tmr, + permissions: AccountSASPermissions.parse("rl"), + resourceTypes: AccountSASResourceTypes.parse("sco").toString(), + services: AccountSASServices.parse("b").toString(), + version: "2020-04-08" + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ).toString(); + + // list with empty include + // create container client for + let pipeline = newPipeline(new AnonymousCredential(), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }); + pipeline.factories.unshift( + new QueryRequestPolicyFactory("include=metadata", "include=") + ); + let serviceClientForOptions = new BlobServiceClient( + `${baseURL}?${sas}`, + pipeline + ); + + let ContainerClientForOptions = + serviceClientForOptions.getContainerClient(containerName); + + // list blob with empty include + let result = ( + await ContainerClientForOptions.listBlobsFlat({ + includeMetadata: true + }) + .byPage() + .next() + ).value; + assert.ok(result); + assert.equal(result.segment.blobItems.length, 3); + + // list with include as upcase Snapshot + // create container client for + pipeline = newPipeline(new AnonymousCredential(), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }); + pipeline.factories.unshift( + new QueryRequestPolicyFactory("include=metadata", "include=Snapshots") + ); + serviceClientForOptions = new BlobServiceClient( + `${baseURL}?${sas}`, + pipeline + ); + + ContainerClientForOptions = + serviceClientForOptions.getContainerClient(containerName); + + // list blob with include as upcase Snapshot + result = ( + await ContainerClientForOptions.listBlobsFlat({ + includeMetadata: true + }) + .byPage() + .next() + ).value; + assert.ok(result); + assert.equal(result.segment.blobItems.length, 4); + + // list with mutiple include + // create container client for + pipeline = newPipeline(new AnonymousCredential(), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }); + pipeline.factories.unshift( + new QueryRequestPolicyFactory( + "include=metadata", + "include=snapshots,metadata,uncommittedblobs,copy,deleted,tags,versions,deletedwithversions,immutabilitypolicy,legalhold,permissions" + ) + ); + serviceClientForOptions = new BlobServiceClient( + `${baseURL}?${sas}`, + pipeline + ); + + ContainerClientForOptions = + serviceClientForOptions.getContainerClient(containerName); + + // list blob with mutiple include + result = ( + await ContainerClientForOptions.listBlobsFlat({ + includeMetadata: true + }) + .byPage() + .next() + ).value; + assert.ok(result); + assert.equal(result.segment.blobItems.length, 4); + }); + + // Skip the case currently since js sdk caculate the stringToSign with "+" in prefix instead of decode to space + it.skip("List blob should success with '+' in query @loki @sql", async () => { + const blobClients = []; + let blobNames: Array = ["block blob/abc-001", "block blob/abc-002"]; + + for (let i = 0; i < 2; i++) { + const blobClient = containerClient.getBlobClient(blobNames[i]); + const blockBlobClient = blobClient.getBlockBlobClient(); + await blockBlobClient.upload("", 0); + blobClients.push(blobClient); + } + + // list with prefix has "+" instead of "%20" for space + // create service client + let pipeline = newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift( + new QueryRequestPolicyFactory("prefix=block%20blob", "prefix=block+blob") + ); + const serviceClientForOptions = new BlobServiceClient( + `${baseURL}`, + pipeline + ); + const containerClientForOptions = + serviceClientForOptions.getContainerClient(containerClient.containerName); + + // List blobs + const inputmarker = undefined; + const result = ( + await containerClientForOptions + .listBlobsFlat({ + prefix: "block blob" + }) + .byPage({ continuationToken: inputmarker }) + .next() + ).value; + assert.ok(result.serviceEndpoint.length > 0); + assert.ok(containerClient.url.indexOf(result.containerName)); + assert.equal(result.segment.blobItems.length, 2); + + // verify list out blob names + const gotNames: Array = []; + for (const item of result.segment.blobItems) { + gotNames.push(item.name); + } + assert.deepStrictEqual(gotNames, blobNames); + + // clean up + for (const blob of blobClients) { + await blob.delete(); + } + }); +}); diff --git a/tests/dfs/apis/pageblob.test.ts b/tests/dfs/apis/pageblob.test.ts new file mode 100644 index 000000000..9857d20e6 --- /dev/null +++ b/tests/dfs/apis/pageblob.test.ts @@ -0,0 +1,1759 @@ +import { + BlobServiceClient, + newPipeline, + StorageSharedKeyCredential +} from "@azure/storage-blob"; + +import { configLogger } from "../../../src/common/Logger"; +import { SequenceNumberActionType } from "../../../src/dfs/generated/artifacts/models"; +import { + bodyToString, + EMULATOR_ACCOUNT_KEY, + EMULATOR_ACCOUNT_NAME, + getUniqueName +} from "../../testutils"; +import DataLakeTestServerFactory from "../DataLakeTestServerFactory"; + +import assert = require("assert"); + +// Set true to enable debug log +configLogger(false); + +describe("PageBlobAPIs", () => { + const factory = new DataLakeTestServerFactory(); + const server = factory.createServer(); + + const baseURL = `http://${server.config.host}:${server.config.port}/devstoreaccount1`; + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ) + ); + + let containerName: string = getUniqueName("container"); + let containerClient = serviceClient.getContainerClient(containerName); + let blobName: string = getUniqueName("blob"); + let blobClient = containerClient.getBlobClient(blobName); + let pageBlobClient = blobClient.getPageBlobClient(); + + before(async () => { + await server.start(); + }); + + after(async () => { + await server.close(); + await server.clean(); + }); + + beforeEach(async () => { + containerName = getUniqueName("container"); + containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + blobName = getUniqueName("blob"); + blobClient = containerClient.getBlobClient(blobName); + pageBlobClient = blobClient.getPageBlobClient(); + }); + + afterEach(async () => { + await containerClient.delete(); + }); + + it("create with default parameters @loki", async () => { + const reuslt_create = await pageBlobClient.create(512); + assert.equal( + reuslt_create._response.request.headers.get("x-ms-client-request-id"), + reuslt_create.clientRequestId + ); + + const result = await blobClient.download(0); + assert.deepStrictEqual( + await bodyToString(result, 512), + "\u0000".repeat(512) + ); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + }); + + it("create with all parameters set @loki", async () => { + const options = { + blobHTTPHeaders: { + blobCacheControl: "blobCacheControl", + blobContentDisposition: "blobContentDisposition", + blobContentEncoding: "blobContentEncoding", + blobContentLanguage: "blobContentLanguage", + blobContentType: "blobContentType" + }, + metadata: { + key1: "vala", + key2: "valb" + } + }; + const result_create = await pageBlobClient.create(512, options); + assert.equal( + result_create._response.request.headers.get("x-ms-client-request-id"), + result_create.clientRequestId + ); + + const result = await blobClient.download(0); + assert.deepStrictEqual( + await bodyToString(result, 512), + "\u0000".repeat(512) + ); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + + const properties = await blobClient.getProperties(); + assert.equal( + properties.cacheControl, + options.blobHTTPHeaders.blobCacheControl + ); + assert.equal( + properties.contentDisposition, + options.blobHTTPHeaders.blobContentDisposition + ); + assert.equal( + properties.contentEncoding, + options.blobHTTPHeaders.blobContentEncoding + ); + assert.equal( + properties.contentLanguage, + options.blobHTTPHeaders.blobContentLanguage + ); + assert.equal( + properties.contentType, + options.blobHTTPHeaders.blobContentType + ); + assert.equal(0, properties.blobSequenceNumber); + assert.equal(properties.metadata!.key1, options.metadata.key1); + assert.equal(properties.metadata!.key2, options.metadata.key2); + assert.equal( + properties._response.request.headers.get("x-ms-client-request-id"), + properties.clientRequestId + ); + }); + + it("download page blob with partial ranges @loki", async () => { + const length = 512 * 10; + await pageBlobClient.create(length); + + const ranges = await pageBlobClient.getPageRanges(0, length); + assert.deepStrictEqual((ranges.pageRange || []).length, 0); + assert.deepStrictEqual((ranges.clearRange || []).length, 0); + assert.equal( + ranges._response.request.headers.get("x-ms-client-request-id"), + ranges.clientRequestId + ); + let result = await blobClient.download(0, 10); + assert.deepStrictEqual(result.contentRange, `bytes 0-9/5120`); + assert.deepStrictEqual( + await bodyToString(result, length), + "\u0000".repeat(10) + ); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + + result = await blobClient.download(1); + assert.deepStrictEqual(result.contentRange, `bytes 1-5119/5120`); + assert.deepStrictEqual(result._response.status, 206); + }); + + it("download page blob with no ranges uploaded @loki", async () => { + const length = 512 * 10; + await pageBlobClient.create(length); + + const ranges = await pageBlobClient.getPageRanges(0, length); + assert.deepStrictEqual((ranges.pageRange || []).length, 0); + assert.deepStrictEqual((ranges.clearRange || []).length, 0); + assert.equal( + ranges._response.request.headers.get("x-ms-client-request-id"), + ranges.clientRequestId + ); + + const result = await blobClient.download(0); + assert.deepStrictEqual( + await bodyToString(result, length), + "\u0000".repeat(length) + ); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + }); + + it("download page blob with no ranges uploaded after resize to bigger size @loki", async () => { + let length = 512 * 10; + await pageBlobClient.create(length); + + let ranges = await pageBlobClient.getPageRanges(0, length); + assert.deepStrictEqual((ranges.pageRange || []).length, 0); + assert.deepStrictEqual((ranges.clearRange || []).length, 0); + assert.equal( + ranges._response.request.headers.get("x-ms-client-request-id"), + ranges.clientRequestId + ); + + let result = await blobClient.download(0); + assert.deepStrictEqual( + await bodyToString(result, length), + "\u0000".repeat(length) + ); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + + length *= 2; + await pageBlobClient.resize(length); + ranges = await pageBlobClient.getPageRanges(0, length); + assert.deepStrictEqual((ranges.pageRange || []).length, 0); + assert.deepStrictEqual((ranges.clearRange || []).length, 0); + assert.equal( + ranges._response.request.headers.get("x-ms-client-request-id"), + ranges.clientRequestId + ); + + result = await blobClient.download(0); + assert.deepStrictEqual( + await bodyToString(result, length), + "\u0000".repeat(length) + ); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + }); + + it("download page blob with no ranges uploaded after resize to smaller size @loki", async () => { + let length = 512 * 10; + await pageBlobClient.create(length); + + let ranges = await pageBlobClient.getPageRanges(0, length); + assert.deepStrictEqual((ranges.pageRange || []).length, 0); + assert.deepStrictEqual((ranges.clearRange || []).length, 0); + + let result = await blobClient.download(0); + assert.deepStrictEqual( + await bodyToString(result, length), + "\u0000".repeat(length) + ); + + length /= 2; + const result_resize = await pageBlobClient.resize(length); + assert.equal( + result_resize._response.request.headers.get("x-ms-client-request-id"), + result_resize.clientRequestId + ); + ranges = await pageBlobClient.getPageRanges(0, length); + assert.deepStrictEqual((ranges.pageRange || []).length, 0); + assert.deepStrictEqual((ranges.clearRange || []).length, 0); + + result = await blobClient.download(0); + assert.deepStrictEqual( + await bodyToString(result, length), + "\u0000".repeat(length) + ); + }); + + it("uploadPages @loki", async () => { + await pageBlobClient.create(1024); + + const result = await blobClient.download(0); + assert.equal(await bodyToString(result, 1024), "\u0000".repeat(1024)); + + await pageBlobClient.uploadPages("a".repeat(512), 0, 512); + const result_upload = await pageBlobClient.uploadPages( + "b".repeat(512), + 512, + 512 + ); + assert.equal( + result_upload._response.request.headers.get("x-ms-client-request-id"), + result_upload.clientRequestId + ); + + const page1 = await pageBlobClient.download(0, 512); + const page2 = await pageBlobClient.download(512, 512); + + assert.equal(await bodyToString(page1, 512), "a".repeat(512)); + assert.equal(await bodyToString(page2, 512), "b".repeat(512)); + }); + + it("uploadPages should work with sequence number conditions @loki", async () => { + await pageBlobClient.create(1024); + + await pageBlobClient.updateSequenceNumber( + SequenceNumberActionType.Update, + 10 + ); + + const result = await blobClient.download(0); + assert.equal(await bodyToString(result, 1024), "\u0000".repeat(1024)); + + await pageBlobClient.uploadPages("a".repeat(512), 0, 512, { + conditions: { + ifSequenceNumberEqualTo: 10, + ifSequenceNumberLessThan: 11, + ifSequenceNumberLessThanOrEqualTo: 10 + } + }); + const result_upload = await pageBlobClient.uploadPages( + "b".repeat(512), + 512, + 512 + ); + assert.equal( + result_upload._response.request.headers.get("x-ms-client-request-id"), + result_upload.clientRequestId + ); + + const page1 = await pageBlobClient.download(0, 512); + const page2 = await pageBlobClient.download(512, 512); + + assert.equal(await bodyToString(page1, 512), "a".repeat(512)); + assert.equal(await bodyToString(page2, 512), "b".repeat(512)); + }); + + it("uploadPages should not work if ifSequenceNumberEqualTo doesn't match @loki", async () => { + await pageBlobClient.create(1024); + + await pageBlobClient.updateSequenceNumber( + SequenceNumberActionType.Update, + 10 + ); + + try { + await pageBlobClient.uploadPages("a".repeat(512), 0, 512, { + conditions: { + ifSequenceNumberEqualTo: 11 + } + }); + } catch (error) { + assert.deepStrictEqual(error.statusCode, 412); + return; + } + + assert.fail(); + }); + + it("uploadPages should not work if ifSequenceNumberLessThan doesn't match @loki", async () => { + await pageBlobClient.create(1024); + + await pageBlobClient.updateSequenceNumber( + SequenceNumberActionType.Update, + 10 + ); + + try { + await pageBlobClient.uploadPages("a".repeat(512), 0, 512, { + conditions: { + ifSequenceNumberLessThan: 10 + } + }); + } catch (error) { + assert.deepStrictEqual(error.statusCode, 412); + return; + } + + try { + await pageBlobClient.uploadPages("a".repeat(512), 0, 512, { + conditions: { + ifSequenceNumberLessThan: 9 + } + }); + } catch (error) { + assert.deepStrictEqual(error.statusCode, 412); + return; + } + + assert.fail(); + }); + + it("uploadPages should not work if ifSequenceNumberLessThanOrEqualTo doesn't match @loki", async () => { + await pageBlobClient.create(1024); + + await pageBlobClient.updateSequenceNumber( + SequenceNumberActionType.Update, + 10 + ); + + await pageBlobClient.uploadPages("a".repeat(512), 0, 512, { + conditions: { + ifSequenceNumberLessThanOrEqualTo: 10 + } + }); + + try { + await pageBlobClient.uploadPages("a".repeat(512), 0, 512, { + conditions: { + ifSequenceNumberLessThanOrEqualTo: 9 + } + }); + } catch (error) { + assert.deepStrictEqual(error.statusCode, 412); + return; + } + + assert.fail(); + }); + + it("uploadPages with sequential pages @loki", async () => { + const length = 512 * 3; + await pageBlobClient.create(length); + + const result = await blobClient.download(0); + assert.equal(await bodyToString(result, length), "\u0000".repeat(length)); + + await pageBlobClient.uploadPages("a".repeat(512), 0, 512); + await pageBlobClient.uploadPages("b".repeat(512), 512, 512); + await pageBlobClient.uploadPages("c".repeat(512), 1024, 512); + + const page1 = await pageBlobClient.download(0, 512); + const page2 = await pageBlobClient.download(512, 512); + const page3 = await pageBlobClient.download(1024, 512); + + assert.equal(await bodyToString(page1, 512), "a".repeat(512)); + assert.equal(await bodyToString(page2, 512), "b".repeat(512)); + assert.equal(await bodyToString(page3, 512), "c".repeat(512)); + + const full = await pageBlobClient.download(0); + assert.equal( + await bodyToString(full, length), + "a".repeat(512) + "b".repeat(512) + "c".repeat(512) + ); + + const ranges = await pageBlobClient.getPageRanges(0, length); + assert.deepStrictEqual((ranges.pageRange || []).length, 3); + assert.deepStrictEqual((ranges.clearRange || []).length, 0); + assert.deepStrictEqual(ranges.pageRange![0], { offset: 0, count: 511 }); + assert.deepStrictEqual(ranges.pageRange![1], { offset: 512, count: 511 }); + assert.deepStrictEqual(ranges.pageRange![2], { offset: 1024, count: 511 }); + }); + + it("uploadPages with one big page range @loki", async () => { + const length = 512 * 3; + await pageBlobClient.create(length); + + const result = await blobClient.download(0); + assert.equal(await bodyToString(result, length), "\u0000".repeat(length)); + + await pageBlobClient.uploadPages( + "a".repeat(512) + "b".repeat(512) + "c".repeat(512), + 0, + length + ); + + const page1 = await pageBlobClient.download(0, 512); + const page2 = await pageBlobClient.download(512, 512); + const page3 = await pageBlobClient.download(1024, 512); + + assert.equal(await bodyToString(page1, 512), "a".repeat(512)); + assert.equal(await bodyToString(page2, 512), "b".repeat(512)); + assert.equal(await bodyToString(page3, 512), "c".repeat(512)); + + const full = await pageBlobClient.download(0); + assert.equal( + await bodyToString(full, length), + "a".repeat(512) + "b".repeat(512) + "c".repeat(512) + ); + + const ranges = await pageBlobClient.getPageRanges(0, length); + assert.deepStrictEqual((ranges.pageRange || []).length, 1); + assert.deepStrictEqual((ranges.clearRange || []).length, 0); + assert.deepStrictEqual(ranges.pageRange![0], { offset: 0, count: 1535 }); + }); + + it("uploadPages with non-sequential pages @loki", async () => { + const length = 512 * 5; + await pageBlobClient.create(length); + + const result = await blobClient.download(0); + assert.equal(await bodyToString(result, length), "\u0000".repeat(length)); + + await pageBlobClient.uploadPages("a".repeat(512), 512, 512); + await pageBlobClient.uploadPages("c".repeat(512), 1536, 512); + + const full = await pageBlobClient.download(0); + assert.equal( + await bodyToString(full, length), + "\u0000".repeat(512) + + "a".repeat(512) + + "\u0000".repeat(512) + + "c".repeat(512) + + "\u0000".repeat(512) + ); + + const page1 = await pageBlobClient.download(0, 512); + const page2 = await pageBlobClient.download(512, 512); + const page3 = await pageBlobClient.download(1024, 512); + const page4 = await pageBlobClient.download(1536, 512); + const page5 = await pageBlobClient.download(2048, 512); + + assert.equal(await bodyToString(page1, 512), "\u0000".repeat(512)); + assert.equal(await bodyToString(page2, 512), "a".repeat(512)); + assert.equal(await bodyToString(page3, 512), "\u0000".repeat(512)); + assert.equal(await bodyToString(page4, 512), "c".repeat(512)); + assert.equal(await bodyToString(page5, 512), "\u0000".repeat(512)); + + const ranges = await pageBlobClient.getPageRanges(0, length); + assert.deepStrictEqual((ranges.pageRange || []).length, 2); + assert.deepStrictEqual((ranges.clearRange || []).length, 0); + assert.deepStrictEqual(ranges.pageRange![0], { offset: 512, count: 511 }); + assert.deepStrictEqual(ranges.pageRange![1], { offset: 1536, count: 511 }); + }); + + it("uploadPages to internally override a sequential range @loki", async () => { + const length = 512 * 3; + await pageBlobClient.create(length); + + const result = await blobClient.download(0); + assert.equal(await bodyToString(result, length), "\u0000".repeat(length)); + + await pageBlobClient.uploadPages( + "a".repeat(512) + "b".repeat(512) + "c".repeat(512), + 0, + length + ); + + await pageBlobClient.uploadPages("d".repeat(512), 512, 512); + + const page1 = await pageBlobClient.download(0, 512); + const page2 = await pageBlobClient.download(512, 512); + const page3 = await pageBlobClient.download(1024, 512); + + assert.equal(await bodyToString(page1, 512), "a".repeat(512)); + assert.equal(await bodyToString(page2, 512), "d".repeat(512)); + assert.equal(await bodyToString(page3, 512), "c".repeat(512)); + + const full = await pageBlobClient.download(0); + assert.equal( + await bodyToString(full, length), + "a".repeat(512) + "d".repeat(512) + "c".repeat(512) + ); + + const ranges = await pageBlobClient.getPageRanges(0, length); + assert.deepStrictEqual((ranges.pageRange || []).length, 3); + assert.deepStrictEqual((ranges.clearRange || []).length, 0); + assert.deepStrictEqual(ranges.pageRange![0], { offset: 0, count: 511 }); + assert.deepStrictEqual(ranges.pageRange![1], { offset: 512, count: 511 }); + assert.deepStrictEqual(ranges.pageRange![2], { offset: 1024, count: 511 }); + }); + + it("uploadPages to internally right align override a sequential range @loki", async () => { + const length = 512 * 3; + await pageBlobClient.create(length); + + const result = await blobClient.download(0); + assert.equal(await bodyToString(result, length), "\u0000".repeat(length)); + + await pageBlobClient.uploadPages( + "a".repeat(512) + "b".repeat(512) + "c".repeat(512), + 0, + length + ); + + await pageBlobClient.uploadPages("d".repeat(512), 1024, 512); + + const page1 = await pageBlobClient.download(0, 512); + const page2 = await pageBlobClient.download(512, 512); + const page3 = await pageBlobClient.download(1024, 512); + + assert.equal(await bodyToString(page1, 512), "a".repeat(512)); + assert.equal(await bodyToString(page2, 512), "b".repeat(512)); + assert.equal(await bodyToString(page3, 512), "d".repeat(512)); + + const full = await pageBlobClient.download(0); + assert.equal( + await bodyToString(full, length), + "a".repeat(512) + "b".repeat(512) + "d".repeat(512) + ); + + const ranges = await pageBlobClient.getPageRanges(0, length); + assert.deepStrictEqual((ranges.pageRange || []).length, 2); + assert.deepStrictEqual((ranges.clearRange || []).length, 0); + assert.deepStrictEqual(ranges.pageRange![0], { offset: 0, count: 1023 }); + assert.deepStrictEqual(ranges.pageRange![1], { offset: 1024, count: 511 }); + }); + + it("uploadPages to internally left align override a sequential range @loki", async () => { + const length = 512 * 3; + await pageBlobClient.create(length); + + const result = await blobClient.download(0); + assert.equal(await bodyToString(result, length), "\u0000".repeat(length)); + + await pageBlobClient.uploadPages( + "a".repeat(512) + "b".repeat(512) + "c".repeat(512), + 0, + length + ); + + await pageBlobClient.uploadPages("d".repeat(512), 0, 512); + + const page1 = await pageBlobClient.download(0, 512); + const page2 = await pageBlobClient.download(512, 512); + const page3 = await pageBlobClient.download(1024, 512); + + assert.equal(await bodyToString(page1, 512), "d".repeat(512)); + assert.equal(await bodyToString(page2, 512), "b".repeat(512)); + assert.equal(await bodyToString(page3, 512), "c".repeat(512)); + + const full = await pageBlobClient.download(0); + assert.equal( + await bodyToString(full, length), + "d".repeat(512) + "b".repeat(512) + "c".repeat(512) + ); + + const ranges = await pageBlobClient.getPageRanges(0, length); + assert.deepStrictEqual((ranges.pageRange || []).length, 2); + assert.deepStrictEqual((ranges.clearRange || []).length, 0); + assert.deepStrictEqual(ranges.pageRange![0], { offset: 0, count: 511 }); + assert.deepStrictEqual(ranges.pageRange![1], { offset: 512, count: 1023 }); + }); + + it("uploadPages to totally override a sequential range @loki", async () => { + const length = 512 * 5; + await pageBlobClient.create(length); + + const result = await blobClient.download(0); + assert.equal(await bodyToString(result, length), "\u0000".repeat(length)); + + await pageBlobClient.uploadPages( + "a".repeat(512) + "b".repeat(512) + "c".repeat(512), + 512, + 512 * 3 + ); + + const page1 = await pageBlobClient.download(0, 512); + const page2 = await pageBlobClient.download(512, 512); + const page3 = await pageBlobClient.download(1024, 512); + const page4 = await pageBlobClient.download(1536, 512); + const page5 = await pageBlobClient.download(2048, 512); + + assert.equal(await bodyToString(page1, 512), "\u0000".repeat(512)); + assert.equal(await bodyToString(page2, 512), "a".repeat(512)); + assert.equal(await bodyToString(page3, 512), "b".repeat(512)); + assert.equal(await bodyToString(page4, 512), "c".repeat(512)); + assert.equal(await bodyToString(page5, 512), "\u0000".repeat(512)); + + let full = await pageBlobClient.download(0); + assert.equal( + await bodyToString(full, length), + "\u0000".repeat(512) + + "a".repeat(512) + + "b".repeat(512) + + "c".repeat(512) + + "\u0000".repeat(512) + ); + + let ranges = await pageBlobClient.getPageRanges(0, length); + assert.deepStrictEqual((ranges.pageRange || []).length, 1); + assert.deepStrictEqual((ranges.clearRange || []).length, 0); + assert.deepStrictEqual(ranges.pageRange![0], { offset: 512, count: 1535 }); + + await pageBlobClient.uploadPages("d".repeat(length), 0, length); + + full = await pageBlobClient.download(0); + assert.equal(await bodyToString(full, length), "d".repeat(length)); + + ranges = await pageBlobClient.getPageRanges(0, length); + assert.deepStrictEqual((ranges.pageRange || []).length, 1); + assert.deepStrictEqual((ranges.clearRange || []).length, 0); + assert.deepStrictEqual(ranges.pageRange![0], { + offset: 0, + count: length - 1 + }); + }); + + it("uploadPages to left override a sequential range @loki", async () => { + const length = 512 * 5; + await pageBlobClient.create(length); + + const result = await blobClient.download(0); + assert.equal(await bodyToString(result, length), "\u0000".repeat(length)); + + await pageBlobClient.uploadPages( + "a".repeat(512) + "b".repeat(512) + "c".repeat(512), + 512, + 512 * 3 + ); + + await pageBlobClient.uploadPages("d".repeat(512 * 2), 0, 512 * 2); + + const page1 = await pageBlobClient.download(0, 512); + const page2 = await pageBlobClient.download(512, 512); + const page3 = await pageBlobClient.download(1024, 512); + const page4 = await pageBlobClient.download(1536, 512); + const page5 = await pageBlobClient.download(2048, 512); + + assert.equal(await bodyToString(page1, 512), "d".repeat(512)); + assert.equal(await bodyToString(page2, 512), "d".repeat(512)); + assert.equal(await bodyToString(page3, 512), "b".repeat(512)); + assert.equal(await bodyToString(page4, 512), "c".repeat(512)); + assert.equal(await bodyToString(page5, 512), "\u0000".repeat(512)); + + const full = await pageBlobClient.download(0); + assert.equal( + await bodyToString(full, length), + "d".repeat(512) + + "d".repeat(512) + + "b".repeat(512) + + "c".repeat(512) + + "\u0000".repeat(512) + ); + + const ranges = await pageBlobClient.getPageRanges(0, length); + assert.deepStrictEqual((ranges.pageRange || []).length, 2); + assert.deepStrictEqual((ranges.clearRange || []).length, 0); + assert.deepStrictEqual(ranges.pageRange![0], { offset: 0, count: 1023 }); + assert.deepStrictEqual(ranges.pageRange![1], { offset: 1024, count: 1023 }); + }); + + it("uploadPages to right override a sequential range @loki", async () => { + const length = 512 * 5; + await pageBlobClient.create(length); + + const result = await blobClient.download(0); + assert.equal(await bodyToString(result, length), "\u0000".repeat(length)); + + await pageBlobClient.uploadPages( + "a".repeat(512) + "b".repeat(512) + "c".repeat(512), + 512, + 512 * 3 + ); + + await pageBlobClient.uploadPages("d".repeat(512 * 2), 512 * 3, 512 * 2); + + const page1 = await pageBlobClient.download(0, 512); + const page2 = await pageBlobClient.download(512, 512); + const page3 = await pageBlobClient.download(1024, 512); + const page4 = await pageBlobClient.download(1536, 512); + const page5 = await pageBlobClient.download(2048, 512); + + assert.equal(await bodyToString(page1, 512), "\u0000".repeat(512)); + assert.equal(await bodyToString(page2, 512), "a".repeat(512)); + assert.equal(await bodyToString(page3, 512), "b".repeat(512)); + assert.equal(await bodyToString(page4, 512), "d".repeat(512)); + assert.equal(await bodyToString(page5, 512), "d".repeat(512)); + + const full = await pageBlobClient.download(0); + assert.equal( + await bodyToString(full, length), + "\u0000".repeat(512) + + "a".repeat(512) + + "b".repeat(512) + + "d".repeat(512) + + "d".repeat(512) + ); + + const ranges = await pageBlobClient.getPageRanges(0, length); + assert.deepStrictEqual((ranges.pageRange || []).length, 2); + assert.deepStrictEqual((ranges.clearRange || []).length, 0); + assert.deepStrictEqual(ranges.pageRange![0], { + offset: 512, + count: 512 * 2 - 1 + }); + assert.deepStrictEqual(ranges.pageRange![1], { + offset: 512 * 3, + count: 512 * 2 - 1 + }); + }); + + it("resize override a sequential range @loki", async () => { + let length = 512 * 3; + await pageBlobClient.create(length); + + const result = await blobClient.download(0); + assert.equal(await bodyToString(result, length), "\u0000".repeat(length)); + + await pageBlobClient.uploadPages( + "a".repeat(512) + "b".repeat(512) + "c".repeat(512), + 0, + length + ); + + length = 512 * 2; + const result_resize = await pageBlobClient.resize(length); + assert.equal( + result_resize._response.request.headers.get("x-ms-client-request-id"), + result_resize.clientRequestId + ); + + const page1 = await pageBlobClient.download(0, 512); + const page2 = await pageBlobClient.download(512, 512); + const page3 = await pageBlobClient.download(1024, 512); + + assert.equal(await bodyToString(page1, 512), "a".repeat(512)); + assert.equal(await bodyToString(page2, 512), "b".repeat(512)); + assert.equal(await bodyToString(page3, 512), ""); + + const full = await pageBlobClient.download(0); + assert.equal( + await bodyToString(full, length), + "a".repeat(512) + "b".repeat(512) + ); + + const ranges = await pageBlobClient.getPageRanges(0, length); + assert.deepStrictEqual((ranges.pageRange || []).length, 1); + assert.deepStrictEqual((ranges.clearRange || []).length, 0); + assert.deepStrictEqual(ranges.pageRange![0], { + offset: 0, + count: length - 1 + }); + }); + + it("uploadPages to internally override a non-sequential range @loki", async () => { + const length = 512 * 5; + await pageBlobClient.create(length); + + const result = await blobClient.download(0); + assert.equal(await bodyToString(result, length), "\u0000".repeat(length)); + + await pageBlobClient.uploadPages("a".repeat(512 * 2), 0, 512 * 2); + + await pageBlobClient.uploadPages("b".repeat(512 * 2), 512 * 3, 512 * 2); + + await pageBlobClient.uploadPages("d".repeat(512 * 3), 512, 512 * 3); + + const page1 = await pageBlobClient.download(0, 512); + const page2 = await pageBlobClient.download(512, 512); + const page3 = await pageBlobClient.download(1024, 512); + const page4 = await pageBlobClient.download(1536, 512); + const page5 = await pageBlobClient.download(2048, 512); + + assert.equal(await bodyToString(page1, 512), "a".repeat(512)); + assert.equal(await bodyToString(page2, 512), "d".repeat(512)); + assert.equal(await bodyToString(page3, 512), "d".repeat(512)); + assert.equal(await bodyToString(page4, 512), "d".repeat(512)); + assert.equal(await bodyToString(page5, 512), "b".repeat(512)); + + const full = await pageBlobClient.download(0); + assert.equal( + await bodyToString(full, length), + "a".repeat(512) + + "d".repeat(512) + + "d".repeat(512) + + "d".repeat(512) + + "b".repeat(512) + ); + + const ranges = await pageBlobClient.getPageRanges(0, length); + assert.deepStrictEqual((ranges.pageRange || []).length, 3); + assert.deepStrictEqual((ranges.clearRange || []).length, 0); + assert.deepStrictEqual(ranges.pageRange![0], { + offset: 0, + count: 512 - 1 + }); + assert.deepStrictEqual(ranges.pageRange![1], { + offset: 512, + count: 512 * 3 - 1 + }); + assert.deepStrictEqual(ranges.pageRange![2], { + offset: 512 * 4, + count: 512 - 1 + }); + }); + + it("uploadPages to internally insert into a non-sequential range @loki", async () => { + const length = 512 * 5; + await pageBlobClient.create(length); + + const result = await blobClient.download(0); + assert.equal(await bodyToString(result, length), "\u0000".repeat(length)); + + await pageBlobClient.uploadPages("a".repeat(512 * 1), 0, 512 * 1); + + await pageBlobClient.uploadPages("b".repeat(512 * 1), 512 * 4, 512 * 1); + + await pageBlobClient.uploadPages("d".repeat(512 * 3), 512, 512 * 3); + + const page1 = await pageBlobClient.download(0, 512); + const page2 = await pageBlobClient.download(512, 512); + const page3 = await pageBlobClient.download(1024, 512); + const page4 = await pageBlobClient.download(1536, 512); + const page5 = await pageBlobClient.download(2048, 512); + + assert.equal(await bodyToString(page1, 512), "a".repeat(512)); + assert.equal(await bodyToString(page2, 512), "d".repeat(512)); + assert.equal(await bodyToString(page3, 512), "d".repeat(512)); + assert.equal(await bodyToString(page4, 512), "d".repeat(512)); + assert.equal(await bodyToString(page5, 512), "b".repeat(512)); + + const full = await pageBlobClient.download(0); + assert.equal( + await bodyToString(full, length), + "a".repeat(512) + + "d".repeat(512) + + "d".repeat(512) + + "d".repeat(512) + + "b".repeat(512) + ); + + const ranges = await pageBlobClient.getPageRanges(0, length); + assert.deepStrictEqual((ranges.pageRange || []).length, 3); + assert.deepStrictEqual((ranges.clearRange || []).length, 0); + assert.deepStrictEqual(ranges.pageRange![0], { + offset: 0, + count: 512 - 1 + }); + assert.deepStrictEqual(ranges.pageRange![1], { + offset: 512, + count: 512 * 3 - 1 + }); + assert.deepStrictEqual(ranges.pageRange![2], { + offset: 512 * 4, + count: 512 - 1 + }); + }); + + it("uploadPages to totally override a non-sequential range @loki", async () => { + const length = 512 * 5; + await pageBlobClient.create(length); + + const result = await blobClient.download(0); + assert.equal(await bodyToString(result, length), "\u0000".repeat(length)); + + await pageBlobClient.uploadPages("a".repeat(512 * 1), 512 * 1, 512 * 1); + + await pageBlobClient.uploadPages("b".repeat(512 * 1), 512 * 3, 512 * 1); + + await pageBlobClient.uploadPages("d".repeat(512 * 3), 512, 512 * 3); + + const page1 = await pageBlobClient.download(0, 512); + const page2 = await pageBlobClient.download(512, 512); + const page3 = await pageBlobClient.download(1024, 512); + const page4 = await pageBlobClient.download(1536, 512); + const page5 = await pageBlobClient.download(2048, 512); + + assert.equal(await bodyToString(page1, 512), "\u0000".repeat(512)); + assert.equal(await bodyToString(page2, 512), "d".repeat(512)); + assert.equal(await bodyToString(page3, 512), "d".repeat(512)); + assert.equal(await bodyToString(page4, 512), "d".repeat(512)); + assert.equal(await bodyToString(page5, 512), "\u0000".repeat(512)); + + const full = await pageBlobClient.download(0); + assert.equal( + await bodyToString(full, length), + "\u0000".repeat(512) + + "d".repeat(512) + + "d".repeat(512) + + "d".repeat(512) + + "\u0000".repeat(512) + ); + + const ranges = await pageBlobClient.getPageRanges(0, length); + assert.deepStrictEqual((ranges.pageRange || []).length, 1); + assert.deepStrictEqual((ranges.clearRange || []).length, 0); + assert.deepStrictEqual(ranges.pageRange![0], { + offset: 512, + count: 512 * 3 - 1 + }); + }); + + it("uploadPages to left override a non-sequential range @loki", async () => { + const length = 512 * 5; + await pageBlobClient.create(length); + + const result = await blobClient.download(0); + assert.equal(await bodyToString(result, length), "\u0000".repeat(length)); + + await pageBlobClient.uploadPages("a".repeat(512 * 1), 512 * 1, 512 * 1); + + await pageBlobClient.uploadPages("b".repeat(512 * 1), 512 * 3, 512 * 1); + + await pageBlobClient.uploadPages("d".repeat(512 * 2), 512, 512 * 2); + + const page1 = await pageBlobClient.download(0, 512); + const page2 = await pageBlobClient.download(512, 512); + const page3 = await pageBlobClient.download(1024, 512); + const page4 = await pageBlobClient.download(1536, 512); + const page5 = await pageBlobClient.download(2048, 512); + + assert.equal(await bodyToString(page1, 512), "\u0000".repeat(512)); + assert.equal(await bodyToString(page2, 512), "d".repeat(512)); + assert.equal(await bodyToString(page3, 512), "d".repeat(512)); + assert.equal(await bodyToString(page4, 512), "b".repeat(512)); + assert.equal(await bodyToString(page5, 512), "\u0000".repeat(512)); + + const full = await pageBlobClient.download(0); + assert.equal( + await bodyToString(full, length), + "\u0000".repeat(512) + + "d".repeat(512) + + "d".repeat(512) + + "b".repeat(512) + + "\u0000".repeat(512) + ); + + const ranges = await pageBlobClient.getPageRanges(0, length); + assert.deepStrictEqual((ranges.pageRange || []).length, 2); + assert.deepStrictEqual((ranges.clearRange || []).length, 0); + assert.deepStrictEqual(ranges.pageRange![0], { + offset: 512, + count: 512 * 2 - 1 + }); + assert.deepStrictEqual(ranges.pageRange![1], { + offset: 512 * 3, + count: 512 - 1 + }); + }); + + it("uploadPages to insert into a non-sequential range @loki", async () => { + const length = 512 * 5; + await pageBlobClient.create(length); + + const result = await blobClient.download(0); + assert.equal(await bodyToString(result, length), "\u0000".repeat(length)); + + await pageBlobClient.uploadPages("a".repeat(512 * 1), 512 * 1, 512 * 1); + + await pageBlobClient.uploadPages("b".repeat(512 * 1), 512 * 3, 512 * 1); + + await pageBlobClient.uploadPages("d".repeat(512 * 1), 512 * 2, 512 * 1); + + const page1 = await pageBlobClient.download(0, 512); + const page2 = await pageBlobClient.download(512, 512); + const page3 = await pageBlobClient.download(1024, 512); + const page4 = await pageBlobClient.download(1536, 512); + const page5 = await pageBlobClient.download(2048, 512); + + assert.equal(await bodyToString(page1, 512), "\u0000".repeat(512)); + assert.equal(await bodyToString(page2, 512), "a".repeat(512)); + assert.equal(await bodyToString(page3, 512), "d".repeat(512)); + assert.equal(await bodyToString(page4, 512), "b".repeat(512)); + assert.equal(await bodyToString(page5, 512), "\u0000".repeat(512)); + + const full = await pageBlobClient.download(0); + assert.equal( + await bodyToString(full, length), + "\u0000".repeat(512) + + "a".repeat(512) + + "d".repeat(512) + + "b".repeat(512) + + "\u0000".repeat(512) + ); + + const ranges = await pageBlobClient.getPageRanges(0, length); + assert.deepStrictEqual((ranges.pageRange || []).length, 3); + assert.deepStrictEqual((ranges.clearRange || []).length, 0); + assert.deepStrictEqual(ranges.pageRange![0], { + offset: 512, + count: 512 - 1 + }); + assert.deepStrictEqual(ranges.pageRange![1], { + offset: 512 * 2, + count: 512 - 1 + }); + assert.deepStrictEqual(ranges.pageRange![2], { + offset: 512 * 3, + count: 512 - 1 + }); + }); + + it("uploadPages to right override a non-sequential range @loki", async () => { + const length = 512 * 5; + await pageBlobClient.create(length); + + const result = await blobClient.download(0); + assert.equal(await bodyToString(result, length), "\u0000".repeat(length)); + + await pageBlobClient.uploadPages("a".repeat(512 * 1), 512 * 1, 512 * 1); + + await pageBlobClient.uploadPages("b".repeat(512 * 1), 512 * 3, 512 * 1); + + await pageBlobClient.uploadPages("d".repeat(512 * 2), 512 * 2, 512 * 2); + + const page1 = await pageBlobClient.download(0, 512); + const page2 = await pageBlobClient.download(512, 512); + const page3 = await pageBlobClient.download(1024, 512); + const page4 = await pageBlobClient.download(1536, 512); + const page5 = await pageBlobClient.download(2048, 512); + + assert.equal(await bodyToString(page1, 512), "\u0000".repeat(512)); + assert.equal(await bodyToString(page2, 512), "a".repeat(512)); + assert.equal(await bodyToString(page3, 512), "d".repeat(512)); + assert.equal(await bodyToString(page4, 512), "d".repeat(512)); + assert.equal(await bodyToString(page5, 512), "\u0000".repeat(512)); + + const full = await pageBlobClient.download(0); + assert.equal( + await bodyToString(full, length), + "\u0000".repeat(512) + + "a".repeat(512) + + "d".repeat(512) + + "d".repeat(512) + + "\u0000".repeat(512) + ); + + const ranges = await pageBlobClient.getPageRanges(0, length); + assert.deepStrictEqual((ranges.pageRange || []).length, 2); + assert.deepStrictEqual((ranges.clearRange || []).length, 0); + assert.deepStrictEqual(ranges.pageRange![0], { + offset: 512, + count: 512 - 1 + }); + assert.deepStrictEqual(ranges.pageRange![1], { + offset: 512 * 2, + count: 512 * 2 - 1 + }); + }); + + it("clearPages @loki", async () => { + await pageBlobClient.create(1024); + let result = await blobClient.download(0); + assert.deepStrictEqual( + await bodyToString(result, 1024), + "\u0000".repeat(1024) + ); + + await pageBlobClient.uploadPages("a".repeat(1024), 0, 1024); + result = await pageBlobClient.download(0, 1024); + assert.deepStrictEqual(await bodyToString(result, 1024), "a".repeat(1024)); + + const result_clear = await pageBlobClient.clearPages(0, 512); + assert.equal( + result_clear._response.request.headers.get("x-ms-client-request-id"), + result_clear.clientRequestId + ); + result = await pageBlobClient.download(0, 512); + assert.deepStrictEqual( + await bodyToString(result, 512), + "\u0000".repeat(512) + ); + }); + + it("clearPages should work with sequence number conditions @loki", async () => { + await pageBlobClient.create(1024); + await pageBlobClient.clearPages(0, 512, { + conditions: { + ifSequenceNumberEqualTo: 0, + ifSequenceNumberLessThan: 1, + ifSequenceNumberLessThanOrEqualTo: 0 + } + }); + }); + + it("clearPages should not work with invalid ifSequenceNumberEqualTo @loki", async () => { + await pageBlobClient.create(1024); + try { + await pageBlobClient.clearPages(0, 512, { + conditions: { + ifSequenceNumberEqualTo: 1 + } + }); + } catch (error) { + assert.deepStrictEqual(error.statusCode, 412); + return; + } + assert.fail(); + }); + + it("clearPages should not work with invalid ifSequenceNumberLessThan @loki", async () => { + await pageBlobClient.create(1024); + await pageBlobClient.updateSequenceNumber( + SequenceNumberActionType.Increment + ); + + await pageBlobClient.clearPages(0, 512, { + conditions: { + ifSequenceNumberLessThan: 2 + } + }); + + try { + await pageBlobClient.clearPages(0, 512, { + conditions: { + ifSequenceNumberLessThan: 1 + } + }); + } catch (error) { + assert.deepStrictEqual(error.statusCode, 412); + return; + } + assert.fail(); + }); + + it("clearPages should not work with invalid ifSequenceNumberLessThanOrEqualTo @loki", async () => { + await pageBlobClient.create(1024); + await pageBlobClient.updateSequenceNumber( + SequenceNumberActionType.Increment + ); + + await pageBlobClient.clearPages(0, 512, { + conditions: { + ifSequenceNumberLessThanOrEqualTo: 1 + } + }); + + try { + await pageBlobClient.clearPages(0, 512, { + conditions: { + ifSequenceNumberLessThanOrEqualTo: 0 + } + }); + } catch (error) { + assert.deepStrictEqual(error.statusCode, 412); + return; + } + assert.fail(); + }); + + it("clearPages to internally override a sequential range @loki", async () => { + const length = 512 * 5; + await pageBlobClient.create(length); + + const result = await blobClient.download(0); + assert.equal(await bodyToString(result, length), "\u0000".repeat(length)); + + await pageBlobClient.uploadPages( + "a".repeat(512) + "b".repeat(512) + "c".repeat(512), + 512, + 512 * 3 + ); + + await pageBlobClient.clearPages(512 * 2, 512); + + const page1 = await pageBlobClient.download(0, 512); + const page2 = await pageBlobClient.download(512, 512); + const page3 = await pageBlobClient.download(1024, 512); + const page4 = await pageBlobClient.download(1536, 512); + const page5 = await pageBlobClient.download(2048, 512); + + assert.equal(await bodyToString(page1, 512), "\u0000".repeat(512)); + assert.equal(await bodyToString(page2, 512), "a".repeat(512)); + assert.equal(await bodyToString(page3, 512), "\u0000".repeat(512)); + assert.equal(await bodyToString(page4, 512), "c".repeat(512)); + assert.equal(await bodyToString(page5, 512), "\u0000".repeat(512)); + + const full = await pageBlobClient.download(0); + assert.equal( + await bodyToString(full, length), + "\u0000".repeat(512) + + "a".repeat(512) + + "\u0000".repeat(512) + + "c".repeat(512) + + "\u0000".repeat(512) + ); + + const ranges = await pageBlobClient.getPageRanges(0, length); + assert.deepStrictEqual((ranges.pageRange || []).length, 2); + assert.deepStrictEqual((ranges.clearRange || []).length, 0); + assert.deepStrictEqual(ranges.pageRange![0], { + offset: 512, + count: 512 - 1 + }); + assert.deepStrictEqual(ranges.pageRange![1], { + offset: 512 * 3, + count: 512 - 1 + }); + }); + + it("clearPages to totally override a sequential range @loki", async () => { + const length = 512 * 5; + await pageBlobClient.create(length); + + const result = await blobClient.download(0); + assert.equal(await bodyToString(result, length), "\u0000".repeat(length)); + + await pageBlobClient.uploadPages( + "a".repeat(512) + "b".repeat(512) + "c".repeat(512), + 512, + 512 * 3 + ); + + await pageBlobClient.clearPages(512, 512 * 3); + + const page1 = await pageBlobClient.download(0, 512); + const page2 = await pageBlobClient.download(512, 512); + const page3 = await pageBlobClient.download(1024, 512); + const page4 = await pageBlobClient.download(1536, 512); + const page5 = await pageBlobClient.download(2048, 512); + + assert.equal(await bodyToString(page1, 512), "\u0000".repeat(512)); + assert.equal(await bodyToString(page2, 512), "\u0000".repeat(512)); + assert.equal(await bodyToString(page3, 512), "\u0000".repeat(512)); + assert.equal(await bodyToString(page4, 512), "\u0000".repeat(512)); + assert.equal(await bodyToString(page5, 512), "\u0000".repeat(512)); + + const full = await pageBlobClient.download(0); + assert.equal( + await bodyToString(full, length), + "\u0000".repeat(512) + + "\u0000".repeat(512) + + "\u0000".repeat(512) + + "\u0000".repeat(512) + + "\u0000".repeat(512) + ); + + const ranges = await pageBlobClient.getPageRanges(0, length); + assert.deepStrictEqual((ranges.pageRange || []).length, 0); + assert.deepStrictEqual((ranges.clearRange || []).length, 0); + }); + + it("clearPages to left override a sequential range @loki", async () => { + const length = 512 * 5; + await pageBlobClient.create(length); + + const result = await blobClient.download(0); + assert.equal(await bodyToString(result, length), "\u0000".repeat(length)); + + await pageBlobClient.uploadPages( + "a".repeat(512) + "b".repeat(512) + "c".repeat(512), + 512, + 512 * 3 + ); + + await pageBlobClient.clearPages(512 * 2, 512 * 3); + + const page1 = await pageBlobClient.download(0, 512); + const page2 = await pageBlobClient.download(512, 512); + const page3 = await pageBlobClient.download(1024, 512); + const page4 = await pageBlobClient.download(1536, 512); + const page5 = await pageBlobClient.download(2048, 512); + + assert.equal(await bodyToString(page1, 512), "\u0000".repeat(512)); + assert.equal(await bodyToString(page2, 512), "a".repeat(512)); + assert.equal(await bodyToString(page3, 512), "\u0000".repeat(512)); + assert.equal(await bodyToString(page4, 512), "\u0000".repeat(512)); + assert.equal(await bodyToString(page5, 512), "\u0000".repeat(512)); + + const full = await pageBlobClient.download(0); + assert.equal( + await bodyToString(full, length), + "\u0000".repeat(512) + + "a".repeat(512) + + "\u0000".repeat(512) + + "\u0000".repeat(512) + + "\u0000".repeat(512) + ); + + const ranges = await pageBlobClient.getPageRanges(0, length); + assert.deepStrictEqual((ranges.pageRange || []).length, 1); + assert.deepStrictEqual((ranges.clearRange || []).length, 0); + assert.deepStrictEqual(ranges.pageRange![0], { + offset: 512, + count: 512 - 1 + }); + }); + + it("clearPages to right override a sequential range @loki", async () => { + const length = 512 * 5; + await pageBlobClient.create(length); + + const result = await blobClient.download(0); + assert.equal(await bodyToString(result, length), "\u0000".repeat(length)); + + await pageBlobClient.uploadPages( + "a".repeat(512) + "b".repeat(512) + "c".repeat(512), + 512, + 512 * 3 + ); + + await pageBlobClient.clearPages(0, 512 * 3); + + const page1 = await pageBlobClient.download(0, 512); + const page2 = await pageBlobClient.download(512, 512); + const page3 = await pageBlobClient.download(1024, 512); + const page4 = await pageBlobClient.download(1536, 512); + const page5 = await pageBlobClient.download(2048, 512); + + assert.equal(await bodyToString(page1, 512), "\u0000".repeat(512)); + assert.equal(await bodyToString(page2, 512), "\u0000".repeat(512)); + assert.equal(await bodyToString(page3, 512), "\u0000".repeat(512)); + assert.equal(await bodyToString(page4, 512), "c".repeat(512)); + assert.equal(await bodyToString(page5, 512), "\u0000".repeat(512)); + + const full = await pageBlobClient.download(0); + assert.equal( + await bodyToString(full, length), + "\u0000".repeat(512) + + "\u0000".repeat(512) + + "\u0000".repeat(512) + + "c".repeat(512) + + "\u0000".repeat(512) + ); + + const ranges = await pageBlobClient.getPageRanges(0, length); + assert.deepStrictEqual((ranges.pageRange || []).length, 1); + assert.deepStrictEqual((ranges.clearRange || []).length, 0); + assert.deepStrictEqual(ranges.pageRange![0], { + offset: 512 * 3, + count: 512 - 1 + }); + }); + + it("clearPages to internally override a non-sequential range @loki", async () => { + const length = 512 * 5; + await pageBlobClient.create(length); + + const result = await blobClient.download(0); + assert.equal(await bodyToString(result, length), "\u0000".repeat(length)); + + await pageBlobClient.uploadPages("a".repeat(512), 0, 512); + await pageBlobClient.uploadPages("b".repeat(512), 512 * 2, 512); + await pageBlobClient.uploadPages("c".repeat(512), 512 * 4, 512); + + await pageBlobClient.clearPages(512, 512 * 3); + + const page1 = await pageBlobClient.download(0, 512); + const page2 = await pageBlobClient.download(512, 512); + const page3 = await pageBlobClient.download(1024, 512); + const page4 = await pageBlobClient.download(1536, 512); + const page5 = await pageBlobClient.download(2048, 512); + + assert.equal(await bodyToString(page1, 512), "a".repeat(512)); + assert.equal(await bodyToString(page2, 512), "\u0000".repeat(512)); + assert.equal(await bodyToString(page3, 512), "\u0000".repeat(512)); + assert.equal(await bodyToString(page4, 512), "\u0000".repeat(512)); + assert.equal(await bodyToString(page5, 512), "c".repeat(512)); + + const full = await pageBlobClient.download(0); + assert.equal( + await bodyToString(full, length), + "a".repeat(512) + + "\u0000".repeat(512) + + "\u0000".repeat(512) + + "\u0000".repeat(512) + + "c".repeat(512) + ); + + const ranges = await pageBlobClient.getPageRanges(0, length); + assert.deepStrictEqual((ranges.pageRange || []).length, 2); + assert.deepStrictEqual((ranges.clearRange || []).length, 0); + assert.deepStrictEqual(ranges.pageRange![0], { + offset: 0, + count: 512 * 1 - 1 + }); + assert.deepStrictEqual(ranges.pageRange![1], { + offset: 512 * 4, + count: 512 - 1 + }); + }); + + it("clearPages to internally insert into a non-sequential range @loki", async () => { + const length = 512 * 5; + await pageBlobClient.create(length); + + const result = await blobClient.download(0); + assert.equal(await bodyToString(result, length), "\u0000".repeat(length)); + + await pageBlobClient.uploadPages("a".repeat(512), 0, 512); + await pageBlobClient.uploadPages("b".repeat(512), 512 * 2, 512); + await pageBlobClient.uploadPages("c".repeat(512), 512 * 4, 512); + + await pageBlobClient.clearPages(512, 512 * 1); + + const page1 = await pageBlobClient.download(0, 512); + const page2 = await pageBlobClient.download(512, 512); + const page3 = await pageBlobClient.download(1024, 512); + const page4 = await pageBlobClient.download(1536, 512); + const page5 = await pageBlobClient.download(2048, 512); + + assert.equal(await bodyToString(page1, 512), "a".repeat(512)); + assert.equal(await bodyToString(page2, 512), "\u0000".repeat(512)); + assert.equal(await bodyToString(page3, 512), "b".repeat(512)); + assert.equal(await bodyToString(page4, 512), "\u0000".repeat(512)); + assert.equal(await bodyToString(page5, 512), "c".repeat(512)); + + const full = await pageBlobClient.download(0); + assert.equal( + await bodyToString(full, length), + "a".repeat(512) + + "\u0000".repeat(512) + + "b".repeat(512) + + "\u0000".repeat(512) + + "c".repeat(512) + ); + + const ranges = await pageBlobClient.getPageRanges(0, length); + assert.deepStrictEqual((ranges.pageRange || []).length, 3); + assert.deepStrictEqual((ranges.clearRange || []).length, 0); + assert.deepStrictEqual(ranges.pageRange![0], { + offset: 0, + count: 512 * 1 - 1 + }); + assert.deepStrictEqual(ranges.pageRange![1], { + offset: 512 * 2, + count: 512 - 1 + }); + assert.deepStrictEqual(ranges.pageRange![2], { + offset: 512 * 4, + count: 512 - 1 + }); + }); + + it("clearPages will fail when start range longer than blob length @loki", async () => { + const length = 512 * 2; + await pageBlobClient.create(length); + + const result = await blobClient.download(0); + assert.equal(await bodyToString(result, length), "\u0000".repeat(length)); + + await pageBlobClient.uploadPages("a".repeat(512), 0, 512); + await pageBlobClient.uploadPages("a".repeat(512), 512 * 1, 512); + + await pageBlobClient.getPageRanges(512 * 2 - 1, 512); + try { + await pageBlobClient.clearPages(512 * 2, 512); + } catch (error) { + assert.deepStrictEqual(error.statusCode, 416); + return; + } + assert.fail(); + }); + + it("GetPageRanges will fail when start range longer than blob length @loki", async () => { + const length = 512 * 2; + await pageBlobClient.create(length); + + const result = await blobClient.download(0); + assert.equal(await bodyToString(result, length), "\u0000".repeat(length)); + + await pageBlobClient.uploadPages("a".repeat(512), 0, 512); + await pageBlobClient.uploadPages("a".repeat(512), 512 * 1, 512); + + await pageBlobClient.getPageRanges(512 * 2 - 1, 512); + try { + await pageBlobClient.getPageRanges(512 * 2, 512); + } catch (error) { + assert.deepStrictEqual(error.statusCode, 416); + return; + } + assert.fail(); + }); + + it("UploadPages will fail when start range longer than blob length @loki", async () => { + const length = 512 * 2; + await pageBlobClient.create(length); + + const result = await blobClient.download(0); + assert.equal(await bodyToString(result, length), "\u0000".repeat(length)); + + await pageBlobClient.uploadPages("a".repeat(512), 0, 512); + await pageBlobClient.uploadPages("a".repeat(512), 512 * 1, 512); + + try { + await pageBlobClient.uploadPages("b".repeat(512), 512 * 2, 512); + } catch (error) { + assert.deepStrictEqual(error.statusCode, 416); + return; + } + assert.fail(); + }); + + it("clearPages to totally override a non-sequential range @loki", async () => { + const length = 512 * 5; + await pageBlobClient.create(length); + + const result = await blobClient.download(0); + assert.equal(await bodyToString(result, length), "\u0000".repeat(length)); + + await pageBlobClient.uploadPages("a".repeat(512), 0, 512); + await pageBlobClient.uploadPages("b".repeat(512), 512 * 2, 512); + await pageBlobClient.uploadPages("c".repeat(512), 512 * 4, 512); + + await pageBlobClient.clearPages(0, 512 * 5); + + const page1 = await pageBlobClient.download(0, 512); + const page2 = await pageBlobClient.download(512, 512); + const page3 = await pageBlobClient.download(1024, 512); + const page4 = await pageBlobClient.download(1536, 512); + const page5 = await pageBlobClient.download(2048, 512); + + assert.equal(await bodyToString(page1, 512), "\u0000".repeat(512)); + assert.equal(await bodyToString(page2, 512), "\u0000".repeat(512)); + assert.equal(await bodyToString(page3, 512), "\u0000".repeat(512)); + assert.equal(await bodyToString(page4, 512), "\u0000".repeat(512)); + assert.equal(await bodyToString(page5, 512), "\u0000".repeat(512)); + + const full = await pageBlobClient.download(0); + assert.equal( + await bodyToString(full, length), + "\u0000".repeat(512) + + "\u0000".repeat(512) + + "\u0000".repeat(512) + + "\u0000".repeat(512) + + "\u0000".repeat(512) + ); + + const ranges = await pageBlobClient.getPageRanges(0, length); + assert.deepStrictEqual((ranges.pageRange || []).length, 0); + assert.deepStrictEqual((ranges.clearRange || []).length, 0); + }); + + it("clearPages to left override a non-sequential range @loki", async () => { + const length = 512 * 5; + await pageBlobClient.create(length); + + const result = await blobClient.download(0); + assert.equal(await bodyToString(result, length), "\u0000".repeat(length)); + + await pageBlobClient.uploadPages("a".repeat(512), 0, 512); + await pageBlobClient.uploadPages("b".repeat(512 * 2), 512 * 2, 512 * 2); + + await pageBlobClient.clearPages(512 * 3, 512 * 2); + + const page1 = await pageBlobClient.download(0, 512); + const page2 = await pageBlobClient.download(512, 512); + const page3 = await pageBlobClient.download(1024, 512); + const page4 = await pageBlobClient.download(1536, 512); + const page5 = await pageBlobClient.download(2048, 512); + + assert.equal(await bodyToString(page1, 512), "a".repeat(512)); + assert.equal(await bodyToString(page2, 512), "\u0000".repeat(512)); + assert.equal(await bodyToString(page3, 512), "b".repeat(512)); + assert.equal(await bodyToString(page4, 512), "\u0000".repeat(512)); + assert.equal(await bodyToString(page5, 512), "\u0000".repeat(512)); + + const full = await pageBlobClient.download(0); + assert.equal( + await bodyToString(full, length), + "a".repeat(512) + + "\u0000".repeat(512) + + "b".repeat(512) + + "\u0000".repeat(512) + + "\u0000".repeat(512) + ); + + const ranges = await pageBlobClient.getPageRanges(0, length); + assert.deepStrictEqual((ranges.pageRange || []).length, 2); + assert.deepStrictEqual((ranges.clearRange || []).length, 0); + assert.deepStrictEqual(ranges.pageRange![0], { + offset: 0, + count: 512 * 1 - 1 + }); + assert.deepStrictEqual(ranges.pageRange![1], { + offset: 512 * 2, + count: 512 - 1 + }); + }); + + it("clearPages to right override a non-sequential range @loki", async () => { + const length = 512 * 5; + await pageBlobClient.create(length); + + const result = await blobClient.download(0); + assert.equal(await bodyToString(result, length), "\u0000".repeat(length)); + + await pageBlobClient.uploadPages("a".repeat(512), 512, 512); + await pageBlobClient.uploadPages("b".repeat(512 * 2), 512 * 3, 512 * 2); + + await pageBlobClient.clearPages(0, 512 * 4); + + const page1 = await pageBlobClient.download(0, 512); + const page2 = await pageBlobClient.download(512, 512); + const page3 = await pageBlobClient.download(1024, 512); + const page4 = await pageBlobClient.download(1536, 512); + const page5 = await pageBlobClient.download(2048, 512); + + assert.equal(await bodyToString(page1, 512), "\u0000".repeat(512)); + assert.equal(await bodyToString(page2, 512), "\u0000".repeat(512)); + assert.equal(await bodyToString(page3, 512), "\u0000".repeat(512)); + assert.equal(await bodyToString(page4, 512), "\u0000".repeat(512)); + assert.equal(await bodyToString(page5, 512), "b".repeat(512)); + + const full = await pageBlobClient.download(0); + assert.equal( + await bodyToString(full, length), + "\u0000".repeat(512) + + "\u0000".repeat(512) + + "\u0000".repeat(512) + + "\u0000".repeat(512) + + "b".repeat(512) + ); + + const ranges = await pageBlobClient.getPageRanges(0, length); + assert.deepStrictEqual((ranges.pageRange || []).length, 1); + assert.deepStrictEqual((ranges.clearRange || []).length, 0); + assert.deepStrictEqual(ranges.pageRange![0], { + offset: 512 * 4, + count: 512 - 1 + }); + }); + + it("getPageRanges @loki", async () => { + await pageBlobClient.create(1024); + + const result = await blobClient.download(0); + assert.deepStrictEqual( + await bodyToString(result, 1024), + "\u0000".repeat(1024) + ); + assert.equal(true, result._response.headers.contains("x-ms-creation-time")); + + await pageBlobClient.uploadPages("a".repeat(512), 0, 512); + await pageBlobClient.uploadPages("b".repeat(512), 512, 512); + + const page1 = await pageBlobClient.getPageRanges(0, 512); + const page2 = await pageBlobClient.getPageRanges(512, 512); + + assert.equal(page1.pageRange![0].count, 511); + assert.equal(page2.pageRange![0].count, 511); + }); + + it("updateSequenceNumber @loki", async () => { + await pageBlobClient.create(1024); + let propertiesResponse = await pageBlobClient.getProperties(); + + const result = await pageBlobClient.updateSequenceNumber("increment"); + propertiesResponse = await pageBlobClient.getProperties(); + assert.equal(propertiesResponse.blobSequenceNumber!, 1); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + + await pageBlobClient.updateSequenceNumber("update", 10); + propertiesResponse = await pageBlobClient.getProperties(); + assert.equal(propertiesResponse.blobSequenceNumber!, 10); + + await pageBlobClient.updateSequenceNumber("max", 100); + propertiesResponse = await pageBlobClient.getProperties(); + assert.equal(propertiesResponse.blobSequenceNumber!, 100); + }); + + // devstoreaccount1 is standard storage account which doesn't support premium page blob tiers + it.skip("setAccessTier for Page blob @loki", async () => { + const length = 512 * 5; + await pageBlobClient.create(length); + let propertiesResponse = await pageBlobClient.getProperties(); + + const result = await pageBlobClient.setAccessTier("P10"); + propertiesResponse = await pageBlobClient.getProperties(); + assert.equal(propertiesResponse.accessTier!, "P10"); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + + await pageBlobClient.setAccessTier("P20"); + propertiesResponse = await pageBlobClient.getProperties(); + assert.equal(propertiesResponse.accessTier!, "P20"); + + await pageBlobClient.setAccessTier("P30"); + propertiesResponse = await pageBlobClient.getProperties(); + assert.equal(propertiesResponse.accessTier!, "P30"); + + await pageBlobClient.setAccessTier("P40"); + propertiesResponse = await pageBlobClient.getProperties(); + assert.equal(propertiesResponse.accessTier!, "P40"); + + await pageBlobClient.setAccessTier("P50"); + propertiesResponse = await pageBlobClient.getProperties(); + assert.equal(propertiesResponse.accessTier!, "P50"); + }); +}); diff --git a/tests/dfs/apis/service.test.ts b/tests/dfs/apis/service.test.ts new file mode 100644 index 000000000..90760b3a6 --- /dev/null +++ b/tests/dfs/apis/service.test.ts @@ -0,0 +1,522 @@ +import * as assert from "assert"; + +import { + AccountSASPermissions, + AccountSASResourceTypes, + AccountSASServices, + BlobServiceClient, + generateAccountSASQueryParameters, + newPipeline, + SASProtocol, + StorageSharedKeyCredential +} from "@azure/storage-blob"; + +import { configLogger } from "../../../src/common/Logger"; +import { + EMULATOR_ACCOUNT_KIND, + EMULATOR_ACCOUNT_SKUNAME +} from "../../../src/dfs/utils/constants"; +import { + EMULATOR_ACCOUNT_KEY, + EMULATOR_ACCOUNT_NAME, + getUniqueName +} from "../../testutils"; +import DataLakeTestServerFactory from "../DataLakeTestServerFactory"; + +// Set true to enable debug log +configLogger(false); + +describe("ServiceAPIs", () => { + const factory = new DataLakeTestServerFactory(); + const server = factory.createServer(); + + const baseURL = `http://${server.config.host}:${server.config.port}/devstoreaccount1`; + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ) + ); + + before(async () => { + await server.start(); + }); + + after(async () => { + await server.close(); + await server.clean(); + }); + + it(`getUserDelegationKey with Key credential should fail @loki @sql`, async () => { + const startTime = new Date(); + startTime.setHours(startTime.getHours() - 1); + const expiryTime = new Date(); + expiryTime.setDate(expiryTime.getDate() + 1); + + try { + await serviceClient.getUserDelegationKey(startTime, expiryTime); + assert.fail( + "Should fail to invoke getUserDelegationKey with account key credentials" + ); + } catch (error) { + assert.strictEqual( + (error as any).details.AuthenticationErrorDetail, + "Only authentication scheme Bearer is supported" + ); + } + }); + + it(`getUserDelegationKey with SAS token credential should fail @loki @sql`, async () => { + const sasTokenStart = new Date(); + sasTokenStart.setHours(sasTokenStart.getHours() - 1); + + const sasTokenExpiry = new Date(); + sasTokenExpiry.setDate(sasTokenExpiry.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const sharedKeyCredential = factories[factories.length - 1]; + + const sas = generateAccountSASQueryParameters( + { + startsOn: sasTokenStart, + expiresOn: sasTokenExpiry, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: AccountSASPermissions.parse("rwdlacup"), + protocol: SASProtocol.HttpsAndHttp, + resourceTypes: AccountSASResourceTypes.parse("sco").toString(), + services: AccountSASServices.parse("btqf").toString() + }, + sharedKeyCredential as StorageSharedKeyCredential + ).toString(); + + const sasClient = `${serviceClient.url}?${sas}`; + const serviceClientWithSAS = new BlobServiceClient( + sasClient, + newPipeline() + ); + + const skStart = new Date(); + skStart.setHours(skStart.getHours() - 1); + + const skExpiry = new Date(); + skExpiry.setDate(skExpiry.getDate() + 1); + + try { + await serviceClientWithSAS.getUserDelegationKey(skStart, skExpiry); + assert.fail( + "Should fail to invoke getUserDelegationKey with SAS token credentials" + ); + } catch (error) { + assert.strictEqual( + (error as any).details.AuthenticationErrorDetail, + "Only authentication scheme Bearer is supported" + ); + } + }); + + it("GetServiceProperties @loki @sql", async () => { + const result = await serviceClient.getProperties(); + + assert.ok(typeof result.requestId); + assert.ok(result.requestId!.length > 0); + assert.ok(typeof result.version); + assert.ok(result.version!.length > 0); + + if (result.cors && result.cors!.length > 0) { + assert.ok(result.cors![0].allowedHeaders.length >= 0); + assert.ok(result.cors![0].allowedMethods.length > 0); + assert.ok(result.cors![0].allowedOrigins.length > 0); + assert.ok(result.cors![0].exposedHeaders.length >= 0); + assert.ok(result.cors![0].maxAgeInSeconds >= 0); + } + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + }); + + it("Set CORS with empty AllowedHeaders, ExposedHeaders @loki @sql", async () => { + const serviceProperties = await serviceClient.getProperties(); + + const newCORS = { + allowedHeaders: "", + allowedMethods: "GET", + allowedOrigins: "example.com", + exposedHeaders: "", + maxAgeInSeconds: 8888 + }; + + serviceProperties.cors = [newCORS]; + + await serviceClient.setProperties(serviceProperties); + + const result = await serviceClient.getProperties(); + assert.deepStrictEqual(result.cors![0], newCORS); + }); + + it("SetServiceProperties @loki @sql", async () => { + const serviceProperties = await serviceClient.getProperties(); + + serviceProperties.blobAnalyticsLogging = { + deleteProperty: true, + read: true, + retentionPolicy: { + days: 5, + enabled: true + }, + version: "1.0", + write: true + }; + + serviceProperties.minuteMetrics = { + enabled: true, + includeAPIs: true, + retentionPolicy: { + days: 4, + enabled: true + }, + version: "1.0" + }; + + serviceProperties.hourMetrics = { + enabled: true, + includeAPIs: true, + retentionPolicy: { + days: 3, + enabled: true + }, + version: "1.0" + }; + + const newCORS = { + allowedHeaders: "*", + allowedMethods: "GET", + allowedOrigins: "example.com", + exposedHeaders: "*", + maxAgeInSeconds: 8888 + }; + if (!serviceProperties.cors) { + serviceProperties.cors = [newCORS]; + } else if (serviceProperties.cors!.length < 5) { + serviceProperties.cors.push(newCORS); + } + + if (!serviceProperties.deleteRetentionPolicy) { + serviceProperties.deleteRetentionPolicy = { + days: 2, + enabled: false + }; + } + + const result_set = await serviceClient.setProperties(serviceProperties); + assert.equal( + result_set._response.request.headers.get("x-ms-client-request-id"), + result_set.clientRequestId + ); + + const result = await serviceClient.getProperties(); + assert.ok(typeof result.requestId); + assert.ok(result.requestId!.length > 0); + assert.ok(typeof result.version); + assert.ok(result.version!.length > 0); + assert.deepEqual(result.hourMetrics, serviceProperties.hourMetrics); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + }); + + it("List containers in sorted order @loki @sql", async () => { + const containerNamePrefix = getUniqueName("container"); + const containerName1 = `${containerNamePrefix}cc`; + const containerName2 = `${containerNamePrefix}aa`; + const containerName3 = `${containerNamePrefix}bb`; + const containerClient1 = serviceClient.getContainerClient(containerName1); + const containerClient2 = serviceClient.getContainerClient(containerName2); + const containerClient3 = serviceClient.getContainerClient(containerName3); + await containerClient1.create(); + await containerClient2.create(); + await containerClient3.create(); + const result = ( + await serviceClient + .listContainers({ + prefix: containerNamePrefix + }) + .byPage() + .next() + ).value; + assert.equal(result.containerItems.length, 3); + assert.ok(result.containerItems[0].name.endsWith("aa")); + assert.ok(result.containerItems[1].name.endsWith("bb")); + assert.ok(result.containerItems[2].name.endsWith("cc")); + await containerClient1.delete(); + await containerClient2.delete(); + await containerClient3.delete(); + }); + + it("List containers with marker @loki @sql", async () => { + const containerNamePrefix = getUniqueName("container"); + const containerName1 = `${containerNamePrefix}cc`; + const containerName2 = `${containerNamePrefix}aa`; + const containerName3 = `${containerNamePrefix}bb`; + const containerClient1 = serviceClient.getContainerClient(containerName1); + const containerClient2 = serviceClient.getContainerClient(containerName2); + const containerClient3 = serviceClient.getContainerClient(containerName3); + await containerClient1.create(); + await containerClient2.create(); + await containerClient3.create(); + const result = ( + await serviceClient + .listContainers({ + prefix: containerNamePrefix + }) + .byPage({ continuationToken: containerName2 }) + .next() + ).value; + assert.equal(result.containerItems.length, 2); + assert.equal(result.containerItems[0].name, containerName3); + assert.equal(result.containerItems[1].name, containerName1); + await containerClient1.delete(); + await containerClient2.delete(); + await containerClient3.delete(); + }); + + it("List containers with marker and max result length less than result size @loki @sql", async () => { + const containerNamePrefix = getUniqueName("container"); + const containerName1 = `${containerNamePrefix}cc`; + const containerName2 = `${containerNamePrefix}aa`; + const containerName3 = `${containerNamePrefix}bb`; + const containerClient1 = serviceClient.getContainerClient(containerName1); + const containerClient2 = serviceClient.getContainerClient(containerName2); + const containerClient3 = serviceClient.getContainerClient(containerName3); + await containerClient1.create(); + await containerClient2.create(); + await containerClient3.create(); + const result1 = ( + await serviceClient + .listContainers({ prefix: containerNamePrefix }) + .byPage({ continuationToken: containerName2, maxPageSize: 1 }) + .next() + ).value; + + assert.equal(result1.containerItems.length, 1); + assert.equal(result1.containerItems[0].name, containerName3); + assert.equal(result1.continuationToken, containerName3); + + const result2 = ( + await serviceClient + .listContainers({ prefix: containerNamePrefix }) + .byPage({ + continuationToken: result1.continuationToken, + maxPageSize: 1 + }) + .next() + ).value; + assert.equal(result2.containerItems.length, 1); + assert.ok(result2.containerItems[0].name, containerName1); + assert.equal(result2.continuationToken, ""); + + await containerClient1.delete(); + await containerClient2.delete(); + await containerClient3.delete(); + }); + + it("ListContainers with default parameters @loki @sql", async () => { + const result = (await serviceClient.listContainers().byPage().next()).value; + assert.ok(typeof result.requestId); + assert.ok(result.requestId!.length > 0); + assert.ok(typeof result.version); + assert.ok(result.version!.length > 0); + + assert.ok(result.serviceEndpoint.length > 0); + assert.ok(result.containerItems!.length >= 0); + + if (result.containerItems!.length > 0) { + const container = result.containerItems![0]; + assert.ok(container.name.length > 0); + assert.ok(container.properties.etag.length > 0); + assert.ok(container.properties.lastModified); + } + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + }); + + it("ListContainers with all parameters configured @loki @sql", async () => { + const containerNamePrefix = getUniqueName("container"); + const containerName1 = `${containerNamePrefix}x1`; + const containerName2 = `${containerNamePrefix}x2`; + const containerClient1 = serviceClient.getContainerClient(containerName1); + const containerClient2 = serviceClient.getContainerClient(containerName2); + await containerClient1.create({ metadata: { key: "val" } }); + await containerClient2.create({ metadata: { key: "val" } }); + + const result1 = ( + await serviceClient + .listContainers({ + includeMetadata: true, + prefix: containerNamePrefix + }) + .byPage({ maxPageSize: 1 }) + .next() + ).value; + + assert.ok(result1.continuationToken); + assert.equal(result1.containerItems!.length, 1); + assert.ok(result1.containerItems![0].name.startsWith(containerNamePrefix)); + assert.ok(result1.containerItems![0].properties.etag.length > 0); + assert.ok(result1.containerItems![0].properties.lastModified); + assert.ok(!result1.containerItems![0].properties.leaseDuration); + assert.ok(!result1.containerItems![0].properties.publicAccess); + assert.deepEqual( + result1.containerItems![0].properties.leaseState, + "available" + ); + assert.deepEqual( + result1.containerItems![0].properties.leaseStatus, + "unlocked" + ); + assert.deepEqual(result1.containerItems![0].metadata!.key, "val"); + assert.equal( + result1._response.request.headers.get("x-ms-client-request-id"), + result1.clientRequestId + ); + + const result2 = ( + await serviceClient + .listContainers({ + includeMetadata: true, + prefix: containerNamePrefix + }) + .byPage({ + continuationToken: result1.continuationToken, + maxPageSize: 1 + }) + .next() + ).value; + + assert.equal(result2.containerItems!.length, 1); + assert.ok(result2.containerItems![0].name.startsWith(containerNamePrefix)); + assert.ok(result2.containerItems![0].properties.etag.length > 0); + assert.ok(result2.containerItems![0].properties.lastModified); + assert.ok(!result2.containerItems![0].properties.leaseDuration); + assert.ok(!result2.containerItems![0].properties.publicAccess); + assert.deepEqual( + result2.containerItems![0].properties.leaseState, + "available" + ); + assert.deepEqual( + result2.containerItems![0].properties.leaseStatus, + "unlocked" + ); + assert.deepEqual(result2.containerItems![0].metadata!.key, "val"); + + await containerClient1.delete(); + await containerClient2.delete(); + }); + + it("get Account info @loki @sql", async () => { + const result = await serviceClient.getAccountInfo(); + assert.equal(result.accountKind, EMULATOR_ACCOUNT_KIND); + assert.equal(result.skuName, EMULATOR_ACCOUNT_SKUNAME); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + }); + + it("Get Account/Service Properties with Uri has suffix '/' after account name @loki @sql", async () => { + const baseURL1 = `http://${server.config.host}:${server.config.port}/devstoreaccount1/`; + const serviceClient1 = new BlobServiceClient( + baseURL1, + newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ) + ); + + let result = await serviceClient1.getAccountInfo(); + assert.equal(result.accountKind, EMULATOR_ACCOUNT_KIND); + assert.equal(result.skuName, EMULATOR_ACCOUNT_SKUNAME); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + + result = await serviceClient1.getProperties(); + assert.ok(typeof result.requestId); + assert.ok(result.requestId!.length > 0); + assert.ok(typeof result.version); + assert.ok(result.version!.length > 0); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + }); + + it("Get Blob service stats negative @loki @sql", async () => { + await serviceClient.getStatistics().catch((err) => { + assert.strictEqual(err.statusCode, 400); + assert.strictEqual(err.code, "InvalidQueryParameterValue"); + assert.ok(err); + }); + }); +}); + +describe("ServiceAPIs - secondary location endpoint", () => { + const factory = new DataLakeTestServerFactory(); + const server = factory.createServer(); + + const baseURL = `http://${server.config.host}:${server.config.port}/devstoreaccount1-secondary`; + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ) + ); + + before(async () => { + await server.start(); + }); + + after(async () => { + await server.close(); + await server.clean(); + }); + + it("Get Blob service stats @loki @sql", async () => { + await serviceClient + .getStatistics() + .then((result) => { + assert.strictEqual(result.geoReplication?.status, "live"); + }) + .catch((err) => { + assert.ifError(err); + }); + }); +}); diff --git a/tests/dfs/authentication.test.ts b/tests/dfs/authentication.test.ts new file mode 100644 index 000000000..92439b9ad --- /dev/null +++ b/tests/dfs/authentication.test.ts @@ -0,0 +1,154 @@ +import * as assert from "assert"; + +import { + AnonymousCredential, + BlobServiceClient, + newPipeline, + StorageSharedKeyCredential +} from "@azure/storage-blob"; + +import { configLogger } from "../../src/common/Logger"; +import { + EMULATOR_ACCOUNT_KEY, + EMULATOR_ACCOUNT_NAME, + getUniqueName +} from "../testutils"; +import DataLakeTestServerFactory from "./DataLakeTestServerFactory"; + +// Set true to enable debug log +configLogger(false); + +describe("Authentication", () => { + const factory = new DataLakeTestServerFactory(); + const server = factory.createServer(); + + const baseURL = `http://${server.config.host}:${server.config.port}/devstoreaccount1`; + + before(async () => { + await server.start(); + }); + + after(async () => { + await server.close(); + await server.clean(); + }); + + it(`Should not work without credential @loki @sql`, async () => { + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline(new AnonymousCredential(), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const containerName: string = getUniqueName("1container-with-dash"); + const containerClient = serviceClient.getContainerClient(containerName); + + let err; + try { + await containerClient.create(); + } catch (error) { + err = error; + } finally { + if (err === undefined) { + try { + await containerClient.delete(); + } catch (error) { + /* Noop */ + } + assert.fail(); + } + } + }); + + it(`Should not work without correct account name @loki @sql`, async () => { + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline( + new StorageSharedKeyCredential("invalid", EMULATOR_ACCOUNT_KEY), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ) + ); + + const containerName: string = getUniqueName("1container-with-dash"); + const containerClient = serviceClient.getContainerClient(containerName); + + let err; + try { + await containerClient.create(); + } catch (error) { + err = error; + } finally { + if (err === undefined) { + try { + await containerClient.delete(); + } catch (error) { + /* Noop */ + } + assert.fail(); + } + } + }); + + it(`Should not work without correct account key @loki @sql`, async () => { + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline( + new StorageSharedKeyCredential(EMULATOR_ACCOUNT_NAME, "invalidkey"), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ) + ); + + const containerName: string = getUniqueName("1container-with-dash"); + const containerClient = serviceClient.getContainerClient(containerName); + + let err; + try { + await containerClient.create(); + } catch (error) { + err = error; + } finally { + if (err === undefined) { + try { + await containerClient.delete(); + } catch (error) { + /* Noop */ + } + assert.fail(); + } + } + }); + + it(`Should work with correct shared key @loki @sql`, async () => { + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ) + ); + + const containerName: string = getUniqueName("1container-with-dash"); + const containerClient = serviceClient.getContainerClient(containerName); + + await containerClient.create(); + await containerClient.delete(); + }); +}); diff --git a/tests/dfs/blobCorsRequest.test.ts b/tests/dfs/blobCorsRequest.test.ts new file mode 100644 index 000000000..e15f1af8b --- /dev/null +++ b/tests/dfs/blobCorsRequest.test.ts @@ -0,0 +1,854 @@ +import * as assert from "assert"; + +import { + BlobServiceClient, + newPipeline, + RestError, + StorageSharedKeyCredential +} from "@azure/storage-blob"; + +import { configLogger } from "../../src/common/Logger"; +import { + EMULATOR_ACCOUNT_KEY, + EMULATOR_ACCOUNT_NAME, + sleep +} from "../testutils"; +import DataLakeTestServerFactory from "./DataLakeTestServerFactory"; +import OPTIONSRequestPolicyFactory from "./RequestPolicy/OPTIONSRequestPolicyFactory"; +import OriginPolicyFactory from "./RequestPolicy/OriginPolicyFactory"; + +// Set true to enable debug log +configLogger(false); + +describe("Blob Cors requests test", () => { + const factory = new DataLakeTestServerFactory(); + const server = factory.createServer(); + + const baseURL = `http://${server.config.host}:${server.config.port}/devstoreaccount1`; + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ) + ); + + before(async () => { + await server.start(); + }); + + after(async () => { + await server.close(); + await server.clean(); + }); + + it("OPTIONS request without cors rules in server should be fail @loki @sql", async () => { + const serviceProperties = await serviceClient.getProperties(); + serviceProperties.cors = []; + await serviceClient.setProperties(serviceProperties); + + const origin = "Origin"; + const requestMethod = "GET"; + + const pipeline = newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift( + new OPTIONSRequestPolicyFactory(origin, requestMethod) + ); + const serviceClientForOptions = new BlobServiceClient(baseURL, pipeline); + + let error; + try { + await serviceClientForOptions.getProperties(); + } catch (err) { + error = err; + } + + assert.ok(error.statusCode === 403); + assert.ok( + error.message.includes( + "CORS not enabled or no matching rule found for this request." + ) + ); + }); + + it("OPTIONS request should not work without matching cors rules @loki @sql", async () => { + const serviceProperties = await serviceClient.getProperties(); + + const newCORS = { + allowedHeaders: "header", + allowedMethods: "GET", + allowedOrigins: "test", + exposedHeaders: "*", + maxAgeInSeconds: 8888 + }; + + serviceProperties.cors = [newCORS]; + + await serviceClient.setProperties(serviceProperties); + + await sleep(100); + + let origin = "Origin"; + let requestMethod = "GET"; + + let pipeline = newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift( + new OPTIONSRequestPolicyFactory(origin, requestMethod) + ); + let serviceClientForOptions = new BlobServiceClient(baseURL, pipeline); + + let error; + try { + await serviceClientForOptions.getProperties(); + } catch (err) { + error = err; + } + + assert.ok(error.statusCode === 403); + assert.ok( + error.message.includes( + "CORS not enabled or no matching rule found for this request." + ) + ); + + origin = "test"; + requestMethod = "GET"; + + pipeline = newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift( + new OPTIONSRequestPolicyFactory(origin, requestMethod) + ); + serviceClientForOptions = new BlobServiceClient(baseURL, pipeline); + + const res = await serviceClientForOptions.getProperties(); + assert.ok(res._response.status === 200); + }); + + it("OPTIONS request should not work without Origin header or matching allowedOrigins @loki @sql", async () => { + const serviceProperties = await serviceClient.getProperties(); + + const newCORS = { + allowedHeaders: "header", + allowedMethods: "GET", + allowedOrigins: "test", + exposedHeaders: "*", + maxAgeInSeconds: 8888 + }; + + serviceProperties.cors = [newCORS]; + + await serviceClient.setProperties(serviceProperties); + + await sleep(100); + + const origin = "Origin"; + const requestMethod = "GET"; + + let pipeline = newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift( + new OPTIONSRequestPolicyFactory(origin, requestMethod) + ); + let serviceClientForOptions = new BlobServiceClient(baseURL, pipeline); + + let error; + try { + await serviceClientForOptions.getProperties(); + } catch (err) { + error = err; + } + + assert.ok(error.statusCode === 403); + assert.ok( + error.message.includes( + "CORS not enabled or no matching rule found for this request." + ) + ); + + pipeline = newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift( + new OPTIONSRequestPolicyFactory(undefined, requestMethod) + ); + serviceClientForOptions = new BlobServiceClient(baseURL, pipeline); + + try { + await serviceClientForOptions.getProperties(); + } catch (err) { + error = err; + } + + assert.ok(error.statusCode === 403); + assert.ok( + error.message.includes( + "CORS not enabled or no matching rule found for this request." + ) + ); + }); + + it("OPTIONS request should not work without requestMethod header or matching allowedMethods @loki @sql", async () => { + const serviceProperties = await serviceClient.getProperties(); + + const newCORS = { + allowedHeaders: "header", + allowedMethods: "GET", + allowedOrigins: "test", + exposedHeaders: "*", + maxAgeInSeconds: 8888 + }; + + serviceProperties.cors = [newCORS]; + + await serviceClient.setProperties(serviceProperties); + + await sleep(100); + + const origin = "test"; + const requestMethod = "PUT"; + + let pipeline = newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift( + new OPTIONSRequestPolicyFactory(origin, requestMethod) + ); + let serviceClientForOptions = new BlobServiceClient(baseURL, pipeline); + + let error; + try { + await serviceClientForOptions.getProperties(); + } catch (err) { + error = err; + } + + assert.ok(error.statusCode === 403); + assert.ok( + error.message.includes( + "CORS not enabled or no matching rule found for this request." + ) + ); + + pipeline = newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift( + new OPTIONSRequestPolicyFactory(origin, undefined) + ); + serviceClientForOptions = new BlobServiceClient(baseURL, pipeline); + + try { + await serviceClientForOptions.getProperties(); + } catch (err) { + error = err; + } + + assert.ok(error.statusCode === 400); + assert.ok(error.message.includes("A required CORS header is not present.")); + }); + + it("OPTIONS request should check the defined requestHeaders @loki @sql", async () => { + const serviceProperties = await serviceClient.getProperties(); + + const newCORS = [ + { + allowedHeaders: "header", + allowedMethods: "GET", + allowedOrigins: "test", + exposedHeaders: "*", + maxAgeInSeconds: 8888 + }, + { + allowedHeaders: "*", + allowedMethods: "PUT", + allowedOrigins: "test", + exposedHeaders: "*", + maxAgeInSeconds: 8888 + }, + { + allowedHeaders: "head*", + allowedMethods: "POST", + allowedOrigins: "test", + exposedHeaders: "*", + maxAgeInSeconds: 8888 + } + ]; + + serviceProperties.cors = newCORS; + + await serviceClient.setProperties(serviceProperties); + + await sleep(100); + + // No match + let origin = "test"; + let requestMethod = "GET"; + let reqestHeaders = "head"; + + let pipeline = newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift( + new OPTIONSRequestPolicyFactory(origin, requestMethod, reqestHeaders) + ); + let serviceClientForOptions = new BlobServiceClient(baseURL, pipeline); + + let error; + try { + await serviceClientForOptions.getProperties(); + } catch (err) { + error = err; + } + + assert.ok(error.statusCode === 403); + assert.ok( + error.message.includes( + "CORS not enabled or no matching rule found for this request." + ) + ); + + // Match first cors. + origin = "test"; + requestMethod = "GET"; + reqestHeaders = "header"; + + pipeline = newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift( + new OPTIONSRequestPolicyFactory(origin, requestMethod, reqestHeaders) + ); + serviceClientForOptions = new BlobServiceClient(baseURL, pipeline); + + let res = await serviceClientForOptions.getProperties(); + assert.ok(res._response.status === 200); + + // Match second cors. + origin = "test"; + requestMethod = "PUT"; + reqestHeaders = "head"; + + pipeline = newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift( + new OPTIONSRequestPolicyFactory(origin, requestMethod, reqestHeaders) + ); + serviceClientForOptions = new BlobServiceClient(baseURL, pipeline); + + res = await serviceClientForOptions.getProperties(); + assert.ok(res._response.status === 200); + + // No match. + origin = "test"; + requestMethod = "POST"; + reqestHeaders = "hea"; + + pipeline = newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift( + new OPTIONSRequestPolicyFactory(origin, requestMethod, reqestHeaders) + ); + serviceClientForOptions = new BlobServiceClient(baseURL, pipeline); + + error; + try { + await serviceClientForOptions.getProperties(); + } catch (err) { + error = err; + } + + assert.ok(error.statusCode === 403); + assert.ok( + error.message.includes( + "CORS not enabled or no matching rule found for this request." + ) + ); + + // Match third cors. + origin = "test"; + requestMethod = "POST"; + reqestHeaders = "headerheader"; + + pipeline = newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift( + new OPTIONSRequestPolicyFactory(origin, requestMethod, reqestHeaders) + ); + serviceClientForOptions = new BlobServiceClient(baseURL, pipeline); + + res = await serviceClientForOptions.getProperties(); + assert.ok(res._response.status === 200); + }); + + it("OPTIONS request should work with matching rule containing Origion * @loki @sql", async () => { + const serviceProperties = await serviceClient.getProperties(); + + const newCORS = { + allowedHeaders: "header", + allowedMethods: "GET", + allowedOrigins: "*", + exposedHeaders: "*", + maxAgeInSeconds: 8888 + }; + + serviceProperties.cors = [newCORS]; + + await serviceClient.setProperties(serviceProperties); + + await sleep(100); + + const origin = "anyOrigin"; + const requestMethod = "GET"; + + const pipeline = newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift( + new OPTIONSRequestPolicyFactory(origin, requestMethod) + ); + const serviceClientForOptions = new BlobServiceClient(baseURL, pipeline); + + const res = await serviceClientForOptions.getProperties(); + assert.ok(res._response.status === 200); + }); + + context( + "OPTIONS request should work with matching rule containing wildcard in Origin @loki @sql", + async () => { + const testCases = [ + { origin: undefined, expected: 403 }, + { origin: "contoso.com", expected: 403 }, + { origin: "bar.notcontoso.com", expected: 403 }, + { origin: "foo.contoso.com", expected: 200 }, + { origin: "foo.bar.baz.contoso.com", expected: 200 }, + { origin: "foo.CONTOSO.com", expected: 200 } + ]; + + testCases.forEach(async (testCase) => { + it(`${testCase.origin}`, async () => { + const serviceProperties = await serviceClient.getProperties(); + + const newCORS = { + allowedHeaders: "header", + allowedMethods: "GET", + allowedOrigins: "*.contoso.com", + exposedHeaders: "*", + maxAgeInSeconds: 8888 + }; + + serviceProperties.cors = [newCORS]; + + await serviceClient.setProperties(serviceProperties); + + await sleep(100); + + const origin = testCase.origin; + const requestMethod = "GET"; + const expectedStatus = testCase.expected; + + const pipeline = newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift( + new OPTIONSRequestPolicyFactory(origin, requestMethod) + ); + const serviceClientForOptions = new BlobServiceClient( + baseURL, + pipeline + ); + + let status: number = 0; + try { + const res = await serviceClientForOptions.getProperties(); + status = res._response.status; + } catch (e: any) { + if (!(e instanceof RestError)) { + throw e; + } + + status = e.response?.status || 0; + } + + assert.ok(status === expectedStatus); + }); + }); + } + ); + + it("Response of request to service without cors rules should not contains cors info @loki @sql", async () => { + const serviceProperties = await serviceClient.getProperties(); + + serviceProperties.cors = []; + + await serviceClient.setProperties(serviceProperties); + + await sleep(100); + + const origin = "anyOrigin"; + const pipeline = newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift(new OriginPolicyFactory(origin)); + const serviceClientWithOrigin = new BlobServiceClient(baseURL, pipeline); + + const res: any = await serviceClientWithOrigin.getProperties(); + + assert.ok(res["access-control-allow-origin"] === undefined); + assert.ok(res["access-control-expose-headers"] === undefined); + assert.ok(res.vary === undefined); + }); + + it("Service with mismatching cors rules should response header Vary @loki @sql", async () => { + const serviceProperties = await serviceClient.getProperties(); + + const newCORS = { + allowedHeaders: "header", + allowedMethods: "GET", + allowedOrigins: "test", + exposedHeaders: "*", + maxAgeInSeconds: 8888 + }; + + serviceProperties.cors = [newCORS]; + + await serviceClient.setProperties(serviceProperties); + + await sleep(100); + + const origin = "anyOrigin"; + const pipeline = newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift(new OriginPolicyFactory(origin)); + const serviceClientWithOrigin = new BlobServiceClient(baseURL, pipeline); + + let res: any = await serviceClientWithOrigin.getProperties(); + assert.ok(res.vary !== undefined); + + res = await serviceClient.getProperties(); + assert.ok(res.vary === undefined); + }); + + it("Request Match rule exists that allows all origins (*) @loki @sql", async () => { + const serviceProperties = await serviceClient.getProperties(); + + const newCORS = { + allowedHeaders: "header", + allowedMethods: "GET", + allowedOrigins: "*", + exposedHeaders: "*", + maxAgeInSeconds: 8888 + }; + + serviceProperties.cors = [newCORS]; + + await serviceClient.setProperties(serviceProperties); + + await sleep(100); + + const origin = "anyOrigin"; + const pipeline = newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift(new OriginPolicyFactory(origin)); + const serviceClientWithOrigin = new BlobServiceClient(baseURL, pipeline); + + let res: any = await serviceClientWithOrigin.getProperties(); + assert.ok(res["access-control-allow-origin"] === "*"); + assert.ok(res.vary === undefined); + assert.ok(res["access-control-expose-headers"] !== undefined); + + res = await serviceClient.getProperties(); + assert.ok(res["access-control-allow-origin"] === undefined); + assert.ok(res.vary === undefined); + assert.ok(res["access-control-expose-headers"] === undefined); + }); + + it("Request Match rule exists for exact origin @loki @sql", async () => { + const serviceProperties = await serviceClient.getProperties(); + + const newCORS = { + allowedHeaders: "header", + allowedMethods: "GET", + allowedOrigins: "exactOrigin", + exposedHeaders: "*", + maxAgeInSeconds: 8888 + }; + + serviceProperties.cors = [newCORS]; + + await serviceClient.setProperties(serviceProperties); + + await sleep(100); + + const origin = "exactOrigin"; + const pipeline = newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift(new OriginPolicyFactory(origin)); + const serviceClientWithOrigin = new BlobServiceClient(baseURL, pipeline); + + const res: any = await serviceClientWithOrigin.getProperties(); + assert.ok(res["access-control-allow-origin"] === origin); + assert.ok(res.vary !== undefined); + assert.ok(res["access-control-expose-headers"] !== undefined); + }); + + it("Requests with error response should apply for CORS @loki @sql", async () => { + const serviceProperties = await serviceClient.getProperties(); + + const newCORS = { + allowedHeaders: "header", + allowedMethods: "GET", + allowedOrigins: "exactOrigin", + exposedHeaders: "*", + maxAgeInSeconds: 8888 + }; + + serviceProperties.cors = [newCORS]; + + await serviceClient.setProperties(serviceProperties); + + await sleep(100); + + const origin = "exactOrigin"; + const pipeline = newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift(new OriginPolicyFactory(origin)); + const serviceClientWithOrigin = new BlobServiceClient(baseURL, pipeline); + + const containerClientWithOrigin = + serviceClientWithOrigin.getContainerClient("notexistcontainer"); + + try { + await containerClientWithOrigin.getProperties(); + } catch (err) { + assert.ok( + err.response.headers._headersMap["access-control-allow-origin"] + .value === origin + ); + assert.ok(err.response.headers._headersMap.vary !== undefined); + assert.ok( + err.response.headers._headersMap["access-control-expose-headers"] !== + undefined + ); + } + }); + + it("Request Match rule in sequence @loki @sql", async () => { + const serviceProperties = await serviceClient.getProperties(); + + const newCORS = [ + { + allowedHeaders: "header", + allowedMethods: "GET", + allowedOrigins: "exactOrigin", + exposedHeaders: "*", + maxAgeInSeconds: 8888 + }, + { + allowedHeaders: "header", + allowedMethods: "GET", + allowedOrigins: "*", + exposedHeaders: "*", + maxAgeInSeconds: 8888 + } + ]; + + serviceProperties.cors = newCORS; + + await serviceClient.setProperties(serviceProperties); + + await sleep(100); + + const origin = "exactOrigin"; + const pipeline = newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift(new OriginPolicyFactory(origin)); + const serviceClientWithOrigin = new BlobServiceClient(baseURL, pipeline); + + const res: any = await serviceClientWithOrigin.getProperties(); + assert.ok(res["access-control-allow-origin"] === origin); + assert.ok(res.vary !== undefined); + assert.ok(res["access-control-expose-headers"] !== undefined); + }); +}); diff --git a/tests/dfs/blockblob.highlevel.test.ts b/tests/dfs/blockblob.highlevel.test.ts new file mode 100644 index 000000000..e411a9ea5 --- /dev/null +++ b/tests/dfs/blockblob.highlevel.test.ts @@ -0,0 +1,494 @@ +import { AbortController } from "@azure/abort-controller"; +import { + BlobServiceClient, + newPipeline, + StorageSharedKeyCredential +} from "@azure/storage-blob"; +import assert = require("assert"); +import * as fs from "fs"; +import { join } from "path"; +import { PassThrough } from "stream"; + +import { configLogger } from "../../src/common/Logger"; +import DataLakeTestServerFactory from "./DataLakeTestServerFactory"; +import { + createRandomLocalFile, + EMULATOR_ACCOUNT_KEY, + EMULATOR_ACCOUNT_NAME, + getUniqueName, + readStreamToLocalFile, + rmRecursive +} from "../testutils"; + +// Set true to enable debug log +configLogger(false); + +// tslint:disable:no-empty +describe("BlockBlobHighlevel", () => { + const factory = new DataLakeTestServerFactory(); + // Loose model to bypass if-match header used by download retry + const server = factory.createServer(true); + + const baseURL = `http://${server.config.host}:${server.config.port}/devstoreaccount1`; + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ) + ); + + let containerName = getUniqueName("container"); + let containerClient = serviceClient.getContainerClient(containerName); + let blobName = getUniqueName("blob"); + let blobClient = containerClient.getBlobClient(blobName); + let blockBlobClient = blobClient.getBlockBlobClient(); + let tempFileSmall: string; + let tempFileSmallLength: number; + let tempFileLarge: string; + let tempFileLargeLength: number; + const tempFolderPath = "temp"; + const timeoutForLargeFileUploadingTest = 20 * 60 * 1000; + + beforeEach(async () => { + containerName = getUniqueName("container"); + containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + blobName = getUniqueName("blob"); + blobClient = containerClient.getBlobClient(blobName); + blockBlobClient = blobClient.getBlockBlobClient(); + }); + + afterEach(async function () { + await containerClient.delete(); + }); + + before(async () => { + await server.start(); + + if (!fs.existsSync(tempFolderPath)) { + fs.mkdirSync(tempFolderPath); + } + tempFileLarge = await createRandomLocalFile( + tempFolderPath, + 257, + 1024 * 1024 + ); + tempFileLargeLength = 257 * 1024 * 1024; + tempFileSmall = await createRandomLocalFile( + tempFolderPath, + 15, + 1024 * 1024 + ); + tempFileSmallLength = 15 * 1024 * 1024; + }); + + after(async () => { + // TODO: Find out reason of slow close + await server.close(); + fs.unlinkSync(tempFileLarge); + fs.unlinkSync(tempFileSmall); + await rmRecursive(tempFolderPath); + // TODO: Find out reason of slow clean up + await server.clean(); + }); + + it("uploadFile should success when blob >= BLOCK_BLOB_MAX_UPLOAD_BLOB_BYTES @loki", async () => { + const result = await blockBlobClient.uploadFile(tempFileLarge, { + blockSize: 4 * 1024 * 1024, + concurrency: 20 + }); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + + const downloadResponse = await blockBlobClient.download(0); + const downloadedFile = join(tempFolderPath, getUniqueName("downloadfile.")); + await readStreamToLocalFile( + downloadResponse.readableStreamBody!, + downloadedFile + ); + + const downloadedData = await fs.readFileSync(downloadedFile); + const uploadedData = await fs.readFileSync(tempFileLarge); + + fs.unlinkSync(downloadedFile); + assert.ok(downloadedData.equals(uploadedData)); + }).timeout(timeoutForLargeFileUploadingTest); + + it("uploadFile should success when blob < BLOCK_BLOB_MAX_UPLOAD_BLOB_BYTES @loki", async () => { + await blockBlobClient.uploadFile(tempFileSmall, { + blockSize: 4 * 1024 * 1024, + concurrency: 20 + }); + + const downloadResponse = await blockBlobClient.download(0); + const downloadedFile = join(tempFolderPath, getUniqueName("downloadfile.")); + await readStreamToLocalFile( + downloadResponse.readableStreamBody!, + downloadedFile + ); + + const downloadedData = await fs.readFileSync(downloadedFile); + const uploadedData = await fs.readFileSync(tempFileSmall); + + fs.unlinkSync(downloadedFile); + assert.ok(downloadedData.equals(uploadedData)); + }); + + // tslint:disable-next-line:max-line-length + it("uploadFile should success when blob < BLOCK_BLOB_MAX_UPLOAD_BLOB_BYTES and configured maxSingleShotSize @loki", async () => { + await blockBlobClient.uploadFile(tempFileSmall, { + maxSingleShotSize: 0 + }); + + const downloadResponse = await blockBlobClient.download(0); + const downloadedFile = join(tempFolderPath, getUniqueName("downloadfile.")); + await readStreamToLocalFile( + downloadResponse.readableStreamBody!, + downloadedFile + ); + + const downloadedData = await fs.readFileSync(downloadedFile); + const uploadedData = await fs.readFileSync(tempFileSmall); + + fs.unlinkSync(downloadedFile); + assert.ok(downloadedData.equals(uploadedData)); + }); + + // tslint:disable-next-line: max-line-length + it("uploadFile should update progress when blob >= BLOCK_BLOB_MAX_UPLOAD_BLOB_BYTES @loki", async () => { + let eventTriggered = false; + const aborter = new AbortController(); + + try { + await blockBlobClient.uploadFile(tempFileLarge, { + blockSize: 4 * 1024 * 1024, + concurrency: 20, + onProgress: (ev: any) => { + assert.ok(ev.loadedBytes); + eventTriggered = true; + aborter.abort(); + } + }); + } catch (err) { + /**/ + } + assert.ok(eventTriggered); + }).timeout(timeoutForLargeFileUploadingTest); + + // tslint:disable-next-line: max-line-length + it("uploadFile should update progress when blob < BLOCK_BLOB_MAX_UPLOAD_BLOB_BYTES @loki", async () => { + let eventTriggered = false; + const aborter = new AbortController(); + + try { + await blockBlobClient.uploadFile(tempFileSmall, { + blockSize: 4 * 1024 * 1024, + concurrency: 20, + onProgress: (ev: any) => { + assert.ok(ev.loadedBytes); + eventTriggered = true; + aborter.abort(); + } + }); + } catch (err) { + /**/ + } + assert.ok(eventTriggered); + }); + + it("uploadStream should success @loki", async () => { + const rs = fs.createReadStream(tempFileLarge); + const result = await blockBlobClient.uploadStream(rs, 4 * 1024 * 1024, 20); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + + const downloadResponse = await blockBlobClient.download(0); + + const downloadFilePath = join( + tempFolderPath, + getUniqueName("downloadFile") + ); + await readStreamToLocalFile( + downloadResponse.readableStreamBody!, + downloadFilePath + ); + + const downloadedBuffer = fs.readFileSync(downloadFilePath); + const uploadedBuffer = fs.readFileSync(tempFileLarge); + assert.ok(uploadedBuffer.equals(downloadedBuffer)); + + fs.unlinkSync(downloadFilePath); + }); + + it("uploadStream should success for tiny buffers @loki", async () => { + const buf = Buffer.from([0x62, 0x75, 0x66, 0x66, 0x65, 0x72]); + const bufferStream = new PassThrough(); + bufferStream.end(buf); + + await blockBlobClient.uploadStream(bufferStream, 4 * 1024 * 1024, 20); + + const downloadResponse = await blockBlobClient.download(0); + + const downloadFilePath = join( + tempFolderPath, + getUniqueName("downloadFile") + ); + await readStreamToLocalFile( + downloadResponse.readableStreamBody!, + downloadFilePath + ); + + const downloadedBuffer = fs.readFileSync(downloadFilePath); + assert.ok(buf.equals(downloadedBuffer)); + + fs.unlinkSync(downloadFilePath); + }); + + it("uploadStream should abort @loki", async () => { + const rs = fs.createReadStream(tempFileLarge); + + try { + await blockBlobClient.uploadStream(rs, 4 * 1024 * 1024, 20, { + abortSignal: AbortController.timeout(1) + }); + assert.fail(); + } catch (err: any) { + assert.ok((err.message as string).toLowerCase().includes("abort")); + } + }).timeout(timeoutForLargeFileUploadingTest); + + it("uploadStream should update progress event @loki", async () => { + const rs = fs.createReadStream(tempFileLarge); + let eventTriggered = false; + + await blockBlobClient.uploadStream(rs, 4 * 1024 * 1024, 20, { + onProgress: (ev: any) => { + assert.ok(ev.loadedBytes); + eventTriggered = true; + } + }); + assert.ok(eventTriggered); + }).timeout(timeoutForLargeFileUploadingTest); + + it("downloadToBuffer should success @loki", async () => { + const rs = fs.createReadStream(tempFileLarge); + const result = await blockBlobClient.uploadStream(rs, 4 * 1024 * 1024, 20); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + + const buf = Buffer.alloc(tempFileLargeLength); + await blockBlobClient.downloadToBuffer(buf, 0, undefined, { + blockSize: 4 * 1024 * 1024, + maxRetryRequestsPerBlock: 5, + concurrency: 20 + }); + + const localFileContent = fs.readFileSync(tempFileLarge); + assert.ok(localFileContent.equals(buf)); + }).timeout(timeoutForLargeFileUploadingTest); + + it("downloadToBuffer should update progress event @loki", async () => { + const rs = fs.createReadStream(tempFileSmall); + await blockBlobClient.uploadStream(rs, 4 * 1024 * 1024, 10); + + let eventTriggered = false; + const buf = Buffer.alloc(tempFileSmallLength); + const aborter = new AbortController(); + try { + await blockBlobClient.downloadToBuffer(buf, 0, undefined, { + blockSize: 1 * 1024, + maxRetryRequestsPerBlock: 5, + concurrency: 1, + onProgress: () => { + eventTriggered = true; + aborter.abort(); + } + }); + } catch (err) { + /**/ + } + assert.ok(eventTriggered); + }).timeout(timeoutForLargeFileUploadingTest); + + it("blobclient.download should success when internal stream unexpected ends at the stream end @loki", async () => { + await blockBlobClient.uploadFile(tempFileSmall, { + blockSize: 4 * 1024 * 1024, + concurrency: 20 + }); + + let retirableReadableStreamOptions: any; + const downloadResponse = await blockBlobClient.download(0, undefined, { + conditions: { + // modifiedAccessConditions: { + // ifMatch: uploadResponse.eTag + // } + }, + maxRetryRequests: 1, + onProgress: (ev) => { + if (ev.loadedBytes >= tempFileSmallLength) { + retirableReadableStreamOptions.doInjectErrorOnce = true; + } + } + }); + assert.equal( + downloadResponse._response.request.headers.get("x-ms-client-request-id"), + downloadResponse.clientRequestId + ); + + retirableReadableStreamOptions = ( + downloadResponse.readableStreamBody! as any + ).options; + + const downloadedFile = join(tempFolderPath, getUniqueName("downloadfile.")); + await readStreamToLocalFile( + downloadResponse.readableStreamBody!, + downloadedFile + ); + + const downloadedData = await fs.readFileSync(downloadedFile); + const uploadedData = await fs.readFileSync(tempFileSmall); + + fs.unlinkSync(downloadedFile); + assert.ok(downloadedData.equals(uploadedData)); + }); + + // tslint:disable-next-line: max-line-length + it("blobclient.download should download full data successfully when internal stream unexpected ends @loki", async () => { + await blockBlobClient.uploadFile(tempFileSmall, { + blockSize: 4 * 1024 * 1024, + concurrency: 20 + }); + + let retirableReadableStreamOptions: any; + let injectedErrors = 0; + const downloadResponse = await blockBlobClient.download(0, undefined, { + conditions: { + // modifiedAccessConditions: { + // ifMatch: uploadResponse.eTag + // } + }, + maxRetryRequests: 3, + onProgress: () => { + if (injectedErrors++ < 3) { + retirableReadableStreamOptions.doInjectErrorOnce = true; + } + } + }); + + retirableReadableStreamOptions = ( + downloadResponse.readableStreamBody! as any + ).options; + + const downloadedFile = join(tempFolderPath, getUniqueName("downloadfile.")); + await readStreamToLocalFile( + downloadResponse.readableStreamBody!, + downloadedFile + ); + + const downloadedData = await fs.readFileSync(downloadedFile); + const uploadedData = await fs.readFileSync(tempFileSmall); + + fs.unlinkSync(downloadedFile); + assert.ok(downloadedData.equals(uploadedData)); + }); + + it("blobclient.download should download partial data when internal stream unexpected ends @loki", async () => { + await blockBlobClient.uploadFile(tempFileSmall, { + blockSize: 4 * 1024 * 1024, + concurrency: 20 + }); + + const partialSize = 500 * 1024; + + let retirableReadableStreamOptions: any; + let injectedErrors = 0; + const downloadResponse = await blockBlobClient.download(0, partialSize, { + conditions: { + // modifiedAccessConditions: { + // ifMatch: uploadResponse.eTag + // } + }, + maxRetryRequests: 3, + onProgress: () => { + if (injectedErrors++ < 3) { + retirableReadableStreamOptions.doInjectErrorOnce = true; + } + } + }); + + retirableReadableStreamOptions = ( + downloadResponse.readableStreamBody! as any + ).options; + + const downloadedFile = join(tempFolderPath, getUniqueName("downloadfile.")); + await readStreamToLocalFile( + downloadResponse.readableStreamBody!, + downloadedFile + ); + + const downloadedData = await fs.readFileSync(downloadedFile); + const uploadedData = await fs.readFileSync(tempFileSmall); + + fs.unlinkSync(downloadedFile); + assert.ok( + downloadedData + .slice(0, partialSize) + .equals(uploadedData.slice(0, partialSize)) + ); + }); + + it("blobclient.download should download data failed when exceeding max stream retry requests @loki", async () => { + await blockBlobClient.uploadFile(tempFileSmall, { + blockSize: 4 * 1024 * 1024, + concurrency: 20 + }); + + const downloadedFile = join(tempFolderPath, getUniqueName("downloadfile.")); + + let retirableReadableStreamOptions: any; + let injectedErrors = 0; + let expectedError = false; + + try { + const downloadResponse = await blockBlobClient.download(0, undefined, { + conditions: { + // modifiedAccessConditions: { + // ifMatch: uploadResponse.eTag + // } + }, + maxRetryRequests: 0, + onProgress: () => { + if (injectedErrors++ < 1) { + retirableReadableStreamOptions.doInjectErrorOnce = true; + } + } + }); + retirableReadableStreamOptions = ( + downloadResponse.readableStreamBody! as any + ).options; + await readStreamToLocalFile( + downloadResponse.readableStreamBody!, + downloadedFile + ); + } catch (error) { + expectedError = true; + } + + assert.ok(expectedError); + rmRecursive(downloadedFile); + }); +}); diff --git a/tests/dfs/conditions.test.ts b/tests/dfs/conditions.test.ts new file mode 100644 index 000000000..c0941c1bb --- /dev/null +++ b/tests/dfs/conditions.test.ts @@ -0,0 +1,876 @@ +import assert = require("assert"); + +import { + BlobModel, + ContainerModel +} from "../../src/blob/persistence/IBlobMetadataStore"; +import ConditionalHeadersAdapter from "../../src/dfs/conditions/ConditionalHeadersAdapter"; +import ConditionResourceAdapter from "../../src/dfs/conditions/ConditionResourceAdapter"; +import ReadConditionalHeadersValidator from "../../src/dfs/conditions/ReadConditionalHeadersValidator"; +import WriteConditionalHeadersValidator from "../../src/dfs/conditions/WriteConditionalHeadersValidator"; +import StorageErrorFactory from "../../src/dfs/errors/StorageErrorFactory"; +import Context from "../../src/dfs/generated/Context"; + +const context = { contextId: "" } as Context; + +describe("ConditionalHeadersAdapter", () => { + it("Should work with undefined values @loki @sql", () => { + const validator = new ConditionalHeadersAdapter(context); + assert.deepStrictEqual(validator.ifModifiedSince, undefined); + assert.deepStrictEqual(validator.ifUnmodifiedSince, undefined); + assert.deepStrictEqual(validator.ifMatch, undefined); + assert.deepStrictEqual(validator.ifNoneMatch, undefined); + }); + + it("Should work with single etags @loki @sql", () => { + const modifiedAccessConditions = { + ifModifiedSince: new Date("2020/01/01"), + ifUnmodifiedSince: new Date("2020/02/01"), + ifMatch: "etag1", + ifNoneMatch: "etag2" + }; + + const validator = new ConditionalHeadersAdapter( + context, + modifiedAccessConditions + ); + assert.deepStrictEqual( + validator.ifModifiedSince, + modifiedAccessConditions.ifModifiedSince + ); + assert.deepStrictEqual( + validator.ifUnmodifiedSince, + modifiedAccessConditions.ifUnmodifiedSince + ); + assert.deepStrictEqual(validator.ifMatch, [ + modifiedAccessConditions.ifMatch + ]); + assert.deepStrictEqual(validator.ifNoneMatch, [ + modifiedAccessConditions.ifNoneMatch + ]); + }); + + it("Should work with multi etags @loki @sql", () => { + const modifiedAccessConditions = { + ifModifiedSince: new Date("2020/01/01"), + ifUnmodifiedSince: new Date("2020/02/01"), + ifMatch: "etag1,etag2", + ifNoneMatch: "etag3,etag4,etag5" + }; + + const validator = new ConditionalHeadersAdapter( + context, + modifiedAccessConditions + ); + assert.deepStrictEqual( + validator.ifModifiedSince, + modifiedAccessConditions.ifModifiedSince + ); + assert.deepStrictEqual( + validator.ifUnmodifiedSince, + modifiedAccessConditions.ifUnmodifiedSince + ); + assert.deepStrictEqual(validator.ifMatch, ["etag1", "etag2"]); + assert.deepStrictEqual(validator.ifNoneMatch, ["etag3", "etag4", "etag5"]); + }); + + it("Should work with etags with quotes @loki @sql", () => { + const modifiedAccessConditions = { + ifModifiedSince: new Date("2020/01/01"), + ifUnmodifiedSince: new Date("2020/02/01"), + ifMatch: '"etag1","etag2"', + ifNoneMatch: 'etag3,"etag4",etag5' + }; + + const validator = new ConditionalHeadersAdapter( + context, + modifiedAccessConditions + ); + assert.deepStrictEqual( + validator.ifModifiedSince, + modifiedAccessConditions.ifModifiedSince + ); + assert.deepStrictEqual( + validator.ifUnmodifiedSince, + modifiedAccessConditions.ifUnmodifiedSince + ); + assert.deepStrictEqual(validator.ifMatch, ["etag1", "etag2"]); + assert.deepStrictEqual(validator.ifNoneMatch, ["etag3", "etag4", "etag5"]); + }); +}); + +describe("ConditionResourceAdapter", () => { + it("Should work with undefined or null resource @loki @sql", () => { + const conditionResourceAdapterUndefined = new ConditionResourceAdapter( + undefined + ); + assert.deepStrictEqual(conditionResourceAdapterUndefined.exist, false); + + const conditionResourceAdapterNull = new ConditionResourceAdapter(null); + assert.deepStrictEqual(conditionResourceAdapterNull.exist, false); + }); + + it("Should work with blob model @loki @sql", () => { + const blobModel = { + properties: { + etag: "etag1", + lastModified: new Date("2018/01/01") + } + } as BlobModel; + + const conditionResourceAdapter = new ConditionResourceAdapter(blobModel); + assert.deepStrictEqual(conditionResourceAdapter.exist, true); + assert.deepStrictEqual(conditionResourceAdapter.etag, "etag1"); + assert.deepStrictEqual( + conditionResourceAdapter.lastModified, + blobModel.properties.lastModified + ); + }); + + it("Should work with container model @loki @sql", () => { + const blobModel = { + properties: { + etag: "etag1", + lastModified: new Date() + } + } as ContainerModel; + + const conditionResourceAdapter = new ConditionResourceAdapter(blobModel); + assert.deepStrictEqual(conditionResourceAdapter.exist, true); + assert.deepStrictEqual(conditionResourceAdapter.etag, "etag1"); + + blobModel.properties.lastModified.setMilliseconds(0); + assert.deepStrictEqual( + conditionResourceAdapter.lastModified, + blobModel.properties.lastModified + ); + }); + + it("Should work with etag with quotes @loki @sql", () => { + const blobModel = { + properties: { + etag: '"etag1"', + lastModified: new Date() + } + } as ContainerModel; + + const conditionResourceAdapter = new ConditionResourceAdapter(blobModel); + assert.deepStrictEqual(conditionResourceAdapter.exist, true); + assert.deepStrictEqual(conditionResourceAdapter.etag, "etag1"); + + blobModel.properties.lastModified.setMilliseconds(0); + assert.deepStrictEqual( + conditionResourceAdapter.lastModified, + blobModel.properties.lastModified + ); + }); +}); + +describe("ReadConditionalHeadersValidator for exist resource", () => { + it("Should return 412 preconditoin failed for failed if-match results @loki @sql", () => { + const validator = new ReadConditionalHeadersValidator(); + const modifiedAccessConditions = { ifMatch: "etag1" }; + const blobModel = { + properties: { + etag: "etag2", + lastModified: new Date() + } + } as BlobModel; + + const expectedError = StorageErrorFactory.getConditionNotMet(context); + try { + validator.validate( + context, + new ConditionalHeadersAdapter(context, modifiedAccessConditions), + new ConditionResourceAdapter(blobModel) + ); + } catch (error) { + assert.deepStrictEqual(error.statusCode, expectedError.statusCode); + assert.deepStrictEqual(error.errorCode, expectedError.errorCode); + assert.deepStrictEqual(error.errorMessage, expectedError.errorMessage); + return; + } + + assert.fail(); + }); + + it("Should not return 412 preconditoin failed for successful if-match results @loki @sql", () => { + const validator = new ReadConditionalHeadersValidator(); + const modifiedAccessConditions = { ifMatch: "etag1" }; + const blobModel = { + properties: { + etag: "etag1", + lastModified: new Date() + } + } as BlobModel; + + validator.validate( + context, + new ConditionalHeadersAdapter(context, modifiedAccessConditions), + new ConditionResourceAdapter(blobModel) + ); + }); + + it("Should return 304 Not Modified for failed if-none-match results @loki @sql", () => { + const validator = new ReadConditionalHeadersValidator(); + const modifiedAccessConditions = { ifNoneMatch: 'etag0,"etag1"' }; + const blobModel = { + properties: { + etag: "etag1", + lastModified: new Date() + } + } as BlobModel; + + const expectedError = StorageErrorFactory.getNotModified(context); + try { + validator.validate( + context, + new ConditionalHeadersAdapter(context, modifiedAccessConditions), + new ConditionResourceAdapter(blobModel) + ); + } catch (error) { + assert.deepStrictEqual(error.statusCode, expectedError.statusCode); + assert.deepStrictEqual(error.errorCode, expectedError.errorCode); + assert.deepStrictEqual(error.errorMessage, expectedError.errorMessage); + return; + } + + assert.fail(); + }); + + it("Should not return 304 Not Modified for successful if-none-match results @loki @sql", () => { + const validator = new ReadConditionalHeadersValidator(); + const modifiedAccessConditions = { ifNoneMatch: 'etag0,"etag3"' }; + const blobModel = { + properties: { + etag: "etag1", + lastModified: new Date() + } + } as BlobModel; + + validator.validate( + context, + new ConditionalHeadersAdapter(context, modifiedAccessConditions), + new ConditionResourceAdapter(blobModel) + ); + }); + + it("Should return 304 Not Modified for failed if-modified-since results @loki @sql", () => { + const validator = new ReadConditionalHeadersValidator(); + const modifiedAccessConditions = { + ifModifiedSince: new Date("2019/01/01") + }; + const blobModel = { + properties: { + etag: "etag1", + lastModified: new Date("2018/01/01") + } + } as BlobModel; + + const expectedError = StorageErrorFactory.getNotModified(context); + try { + validator.validate( + context, + new ConditionalHeadersAdapter(context, modifiedAccessConditions), + new ConditionResourceAdapter(blobModel) + ); + } catch (error) { + assert.deepStrictEqual(error.statusCode, expectedError.statusCode); + assert.deepStrictEqual(error.errorCode, expectedError.errorCode); + assert.deepStrictEqual(error.errorMessage, expectedError.errorMessage); + return; + } + + assert.fail(); + }); + + it("Should return 304 Not Modified when if-modified-since same with lastModified @loki @sql", () => { + const validator = new ReadConditionalHeadersValidator(); + const modifiedAccessConditions = { + ifModifiedSince: new Date("2019/01/01") + }; + const blobModel = { + properties: { + etag: "etag1", + lastModified: new Date("2019/01/01") + } + } as BlobModel; + + const expectedError = StorageErrorFactory.getNotModified(context); + try { + validator.validate( + context, + new ConditionalHeadersAdapter(context, modifiedAccessConditions), + new ConditionResourceAdapter(blobModel) + ); + } catch (error) { + assert.deepStrictEqual(error.statusCode, expectedError.statusCode); + assert.deepStrictEqual(error.errorCode, expectedError.errorCode); + assert.deepStrictEqual(error.errorMessage, expectedError.errorMessage); + return; + } + + assert.fail(); + }); + + it("Should not return 304 Not Modified for successful if-modified-since results @loki @sql", () => { + const validator = new ReadConditionalHeadersValidator(); + const modifiedAccessConditions = { + ifModifiedSince: new Date("2019/01/01") + }; + const blobModel = { + properties: { + etag: "etag1", + lastModified: new Date("2020/01/01") + } + } as BlobModel; + + validator.validate( + context, + new ConditionalHeadersAdapter(context, modifiedAccessConditions), + new ConditionResourceAdapter(blobModel) + ); + }); + + it("Should return 412 preconditoin failed for failed if-unmodified-since results @loki @sql", () => { + const validator = new ReadConditionalHeadersValidator(); + const modifiedAccessConditions = { + ifUnmodifiedSince: new Date("2019/01/01") + }; + const blobModel = { + properties: { + etag: "etag2", + lastModified: new Date("2020/01/01") + } + } as BlobModel; + + const expectedError = StorageErrorFactory.getConditionNotMet(context); + try { + validator.validate( + context, + new ConditionalHeadersAdapter(context, modifiedAccessConditions), + new ConditionResourceAdapter(blobModel) + ); + } catch (error) { + assert.deepStrictEqual(error.statusCode, expectedError.statusCode); + assert.deepStrictEqual(error.errorCode, expectedError.errorCode); + assert.deepStrictEqual(error.errorMessage, expectedError.errorMessage); + return; + } + + assert.fail(); + }); + + it("Should not return 412 preconditoin failed when if-unmodified-since same with lastModified @loki @sql", () => { + const validator = new ReadConditionalHeadersValidator(); + const modifiedAccessConditions = { + ifUnmodifiedSince: new Date("2019/01/01") + }; + const blobModel = { + properties: { + etag: "etag2", + lastModified: new Date("2019/01/01") + } + } as BlobModel; + + validator.validate( + context, + new ConditionalHeadersAdapter(context, modifiedAccessConditions), + new ConditionResourceAdapter(blobModel) + ); + }); + + it("Should not return 412 preconditoin failed for successful if-unmodified-since results @loki @sql", () => { + const validator = new ReadConditionalHeadersValidator(); + const modifiedAccessConditions = { + ifUnmodifiedSince: new Date("2019/01/01") + }; + const blobModel = { + properties: { + etag: "etag2", + lastModified: new Date("2018/01/01") + } + } as BlobModel; + + validator.validate( + context, + new ConditionalHeadersAdapter(context, modifiedAccessConditions), + new ConditionResourceAdapter(blobModel) + ); + }); + + it("Should return 200 OK when all conditions match @loki @sql", () => { + const validator = new ReadConditionalHeadersValidator(); + const modifiedAccessConditions = { + ifMatch: "etag1", + ifNoneMatch: "etag3", + ifModifiedSince: new Date("2018/01/01"), + ifUnmodifiedSince: new Date("2020/01/01") + }; + const blobModel = { + properties: { + etag: "etag1", + lastModified: new Date("2019/01/01") + } + } as BlobModel; + + validator.validate( + context, + new ConditionalHeadersAdapter(context, modifiedAccessConditions), + new ConditionResourceAdapter(blobModel) + ); + }); + + it("Should return 412 Precondition Failed when if-none-match and if-unmodified-since fail among all conditions @loki @sql", () => { + const validator = new ReadConditionalHeadersValidator(); + const modifiedAccessConditions = { + ifMatch: "etag1", + ifNoneMatch: '"etag1"', + ifModifiedSince: new Date("2018/01/01"), + ifUnmodifiedSince: new Date("2020/01/01") + }; + const blobModel = { + properties: { + etag: "etag1", + lastModified: new Date("2021/01/01") + } + } as BlobModel; + + const expectedError = StorageErrorFactory.getConditionNotMet(context); + + try { + validator.validate( + context, + new ConditionalHeadersAdapter(context, modifiedAccessConditions), + new ConditionResourceAdapter(blobModel) + ); + } catch (error) { + assert.deepStrictEqual(error.statusCode, expectedError.statusCode); + assert.deepStrictEqual(error.errorCode, expectedError.errorCode); + assert.deepStrictEqual(error.errorMessage, expectedError.errorMessage); + return; + } + + assert.fail(); + }); + + it("Should return 200 OK when if-none-match fails all conditions @loki @sql", () => { + const validator = new ReadConditionalHeadersValidator(); + const modifiedAccessConditions = { + ifMatch: "etag1", + ifNoneMatch: "etag1", + ifModifiedSince: new Date("2018/01/01"), + ifUnmodifiedSince: new Date("2020/01/01") + }; + const blobModel = { + properties: { + etag: "etag1", + lastModified: new Date("2019/01/01") + } + } as BlobModel; + + validator.validate( + context, + new ConditionalHeadersAdapter(context, modifiedAccessConditions), + new ConditionResourceAdapter(blobModel) + ); + }); + + it("Should return 412 Precondition Failed when if-match and if-modified-since fail among all conditions @loki @sql", () => { + const validator = new ReadConditionalHeadersValidator(); + const modifiedAccessConditions = { + ifMatch: "etag1", + ifNoneMatch: '"etag2"', + ifModifiedSince: new Date("2018/01/01"), + ifUnmodifiedSince: new Date("2020/01/01") + }; + const blobModel = { + properties: { + etag: "etag0", + lastModified: new Date("2017/01/01") + } + } as BlobModel; + + const expectedError = StorageErrorFactory.getConditionNotMet(context); + + try { + validator.validate( + context, + new ConditionalHeadersAdapter(context, modifiedAccessConditions), + new ConditionResourceAdapter(blobModel) + ); + } catch (error) { + assert.deepStrictEqual(error.statusCode, expectedError.statusCode); + assert.deepStrictEqual(error.errorCode, expectedError.errorCode); + assert.deepStrictEqual(error.errorMessage, expectedError.errorMessage); + return; + } + + assert.fail(); + }); + + it("Should return 200 OK when if-modified-since fails all conditions @loki @sql", () => { + const validator = new ReadConditionalHeadersValidator(); + const modifiedAccessConditions = { + ifMatch: "etag1", + ifNoneMatch: "etag3", + ifModifiedSince: new Date("2018/01/01"), + ifUnmodifiedSince: new Date("2020/01/01") + }; + const blobModel = { + properties: { + etag: "etag1", + lastModified: new Date("2017/01/01") + } + } as BlobModel; + + validator.validate( + context, + new ConditionalHeadersAdapter(context, modifiedAccessConditions), + new ConditionResourceAdapter(blobModel) + ); + }); + + it("Should return 304 Not Modified when if-none-match and if-modified-since fail @loki @sql", () => { + const validator = new ReadConditionalHeadersValidator(); + const modifiedAccessConditions = { + ifNoneMatch: '"etag1"', + ifModifiedSince: new Date("2018/01/01") + }; + const blobModel = { + properties: { + etag: "etag1", + lastModified: new Date("2017/01/01") + } + } as BlobModel; + + const expectedError = StorageErrorFactory.getNotModified(context); + + try { + validator.validate( + context, + new ConditionalHeadersAdapter(context, modifiedAccessConditions), + new ConditionResourceAdapter(blobModel) + ); + } catch (error) { + assert.deepStrictEqual(error.statusCode, expectedError.statusCode); + assert.deepStrictEqual(error.errorCode, expectedError.errorCode); + assert.deepStrictEqual(error.errorMessage, expectedError.errorMessage); + return; + } + + assert.fail(); + }); +}); + +describe("ReadConditionalHeadersValidator for unexist resource", () => { + it("Should return 412 Precondition Failed for any ifMatch @loki @sql", () => { + const validator = new ReadConditionalHeadersValidator(); + const modifiedAccessConditions = { + ifMatch: "etag1" + }; + + const expectedError = StorageErrorFactory.getConditionNotMet(context); + + try { + validator.validate( + context, + new ConditionalHeadersAdapter(context, modifiedAccessConditions), + new ConditionResourceAdapter(undefined) + ); + } catch (error) { + assert.deepStrictEqual(error.statusCode, expectedError.statusCode); + assert.deepStrictEqual(error.errorCode, expectedError.errorCode); + assert.deepStrictEqual(error.errorMessage, expectedError.errorMessage); + return; + } + + assert.fail(); + }); + + it("Should return 400 getUnsatisfiableCondition for if none-match value * @loki @sql", () => { + const validator = new ReadConditionalHeadersValidator(); + const modifiedAccessConditions = { + ifNoneMatch: "*" + }; + + const expectedError = + StorageErrorFactory.getUnsatisfiableCondition(context); + + try { + validator.validate( + context, + new ConditionalHeadersAdapter(context, modifiedAccessConditions), + new ConditionResourceAdapter(undefined) + ); + } catch (error) { + assert.deepStrictEqual(error.statusCode, expectedError.statusCode); + assert.deepStrictEqual(error.errorCode, expectedError.errorCode); + assert.deepStrictEqual(error.errorMessage, expectedError.errorMessage); + return; + } + + assert.fail(); + }); +}); + +describe("WriteConditionalHeadersValidator for unexist resource", () => { + it("Should throw 400 Bad Request for invalid combinations conditional headers @loki @sql", () => { + const validator = new WriteConditionalHeadersValidator(); + const modifiedAccessConditions = { + ifMatch: "etag1", + ifNoneMatch: '"etag2"', + ifModifiedSince: new Date("2018/01/01"), + ifUnmodifiedSince: new Date("2020/01/01") + }; + + const expectedError = + StorageErrorFactory.getMultipleConditionHeadersNotSupported(context); + + try { + validator.validate( + context, + new ConditionalHeadersAdapter(context, modifiedAccessConditions), + new ConditionResourceAdapter(undefined) + ); + } catch (error) { + assert.deepStrictEqual(error.statusCode, expectedError.statusCode); + assert.deepStrictEqual(error.errorCode, expectedError.errorCode); + assert.deepStrictEqual(error.errorMessage, expectedError.errorMessage); + return; + } + + assert.fail(); + }); + + it("Should throw 400 Bad Request for multi etags in ifMatch @loki @sql", () => { + const validator = new WriteConditionalHeadersValidator(); + const modifiedAccessConditions = { + ifMatch: "etag1,etag2", + ifNoneMatch: '"etag2"', + ifModifiedSince: new Date("2018/01/01"), + ifUnmodifiedSince: new Date("2020/01/01") + }; + + const expectedError = + StorageErrorFactory.getMultipleConditionHeadersNotSupported(context); + + try { + validator.validate( + context, + new ConditionalHeadersAdapter(context, modifiedAccessConditions), + new ConditionResourceAdapter(undefined) + ); + } catch (error) { + assert.deepStrictEqual(error.statusCode, expectedError.statusCode); + assert.deepStrictEqual(error.errorCode, expectedError.errorCode); + assert.deepStrictEqual(error.errorMessage, expectedError.errorMessage); + return; + } + + assert.fail(); + }); + + it("Should throw 400 Bad Request for multi etags in if-none-match @loki @sql", () => { + const validator = new WriteConditionalHeadersValidator(); + const modifiedAccessConditions = { + ifMatch: "etag2", + ifNoneMatch: '"etag2",*', + ifModifiedSince: new Date("2018/01/01"), + ifUnmodifiedSince: new Date("2020/01/01") + }; + + const expectedError = + StorageErrorFactory.getMultipleConditionHeadersNotSupported(context); + + try { + validator.validate( + context, + new ConditionalHeadersAdapter(context, modifiedAccessConditions), + new ConditionResourceAdapter(undefined) + ); + } catch (error) { + assert.deepStrictEqual(error.statusCode, expectedError.statusCode); + assert.deepStrictEqual(error.errorCode, expectedError.errorCode); + assert.deepStrictEqual(error.errorMessage, expectedError.errorMessage); + return; + } + + assert.fail(); + }); + + it("Should throw 412 Precondition Failed for any values in if-match @loki @sql", () => { + const validator = new WriteConditionalHeadersValidator(); + const modifiedAccessConditions = { + ifMatch: "etag2" + }; + + const expectedError = StorageErrorFactory.getConditionNotMet(context); + + try { + validator.validate( + context, + new ConditionalHeadersAdapter(context, modifiedAccessConditions), + new ConditionResourceAdapter(undefined) + ); + } catch (error) { + assert.deepStrictEqual(error.statusCode, expectedError.statusCode); + assert.deepStrictEqual(error.errorCode, expectedError.errorCode); + assert.deepStrictEqual(error.errorMessage, expectedError.errorMessage); + return; + } + + assert.fail(); + }); +}); + +describe("WriteConditionalHeadersValidator for exist resource", () => { + it("Should return 200 for successful if-none-match and failed if-modified-since @loki @sql", () => { + const validator = new WriteConditionalHeadersValidator(); + const modifiedAccessConditions = { + ifNoneMatch: '"etag2"', + ifModifiedSince: new Date("2018/01/01") + }; + + const blobModel = { + properties: { + etag: "etag1", + lastModified: new Date("2017/01/01") + } + } as BlobModel; + + validator.validate( + context, + new ConditionalHeadersAdapter(context, modifiedAccessConditions), + new ConditionResourceAdapter(blobModel) + ); + }); + + it("Should return 200 for successful if-match and failed if-unmodified-since @loki @sql", () => { + const validator = new WriteConditionalHeadersValidator(); + const modifiedAccessConditions = { + ifMatch: '"etag1"', + ifUnmodifiedSince: new Date("2018/01/01") + }; + + const blobModel = { + properties: { + etag: "etag1", + lastModified: new Date("2019/01/01") + } + } as BlobModel; + + validator.validate( + context, + new ConditionalHeadersAdapter(context, modifiedAccessConditions), + new ConditionResourceAdapter(blobModel) + ); + }); + + it("Should return 200 for if-unmodified-since equal with lastModified @loki @sql", () => { + const validator = new WriteConditionalHeadersValidator(); + const modifiedAccessConditions = { + ifUnmodifiedSince: new Date() + }; + + const blobModel = { + properties: { + etag: "etag1", + lastModified: modifiedAccessConditions.ifUnmodifiedSince + } + } as BlobModel; + + validator.validate( + context, + new ConditionalHeadersAdapter(context, modifiedAccessConditions), + new ConditionResourceAdapter(blobModel) + ); + }); + + it("Should return 412 for if-modified-since equal with lastModifiedSince @loki @sql", () => { + const validator = new WriteConditionalHeadersValidator(); + const modifiedAccessConditions = { + ifModifiedSince: new Date() + }; + const blobModel = { + properties: { + etag: "etag1", + lastModified: modifiedAccessConditions.ifModifiedSince + } + } as BlobModel; + + const expectedError = StorageErrorFactory.getConditionNotMet(context); + try { + validator.validate( + context, + new ConditionalHeadersAdapter(context, modifiedAccessConditions), + new ConditionResourceAdapter(blobModel) + ); + } catch (error) { + assert.deepStrictEqual(error.statusCode, expectedError.statusCode); + assert.deepStrictEqual(error.errorCode, expectedError.errorCode); + assert.deepStrictEqual(error.errorMessage, expectedError.errorMessage); + return; + } + + assert.fail(); + }); + + it("Should return 412 for failed if-modified-since results @loki @sql", () => { + const validator = new WriteConditionalHeadersValidator(); + const modifiedAccessConditions = { + ifModifiedSince: new Date("2019/01/01") + }; + const blobModel = { + properties: { + etag: "etag1", + lastModified: new Date("2018/01/01") + } + } as BlobModel; + + const expectedError = StorageErrorFactory.getConditionNotMet(context); + try { + validator.validate( + context, + new ConditionalHeadersAdapter(context, modifiedAccessConditions), + new ConditionResourceAdapter(blobModel) + ); + } catch (error) { + assert.deepStrictEqual(error.statusCode, expectedError.statusCode); + assert.deepStrictEqual(error.errorCode, expectedError.errorCode); + assert.deepStrictEqual(error.errorMessage, expectedError.errorMessage); + return; + } + + assert.fail(); + }); + + it("Should return 412 for failed if-unmodified-since results @loki @sql", () => { + const validator = new WriteConditionalHeadersValidator(); + const modifiedAccessConditions = { + ifUnmodifiedSince: new Date("2019/01/01") + }; + const blobModel = { + properties: { + etag: "etag1", + lastModified: new Date("2020/01/01") + } + } as BlobModel; + + const expectedError = StorageErrorFactory.getConditionNotMet(context); + try { + validator.validate( + context, + new ConditionalHeadersAdapter(context, modifiedAccessConditions), + new ConditionResourceAdapter(blobModel) + ); + } catch (error) { + assert.deepStrictEqual(error.statusCode, expectedError.statusCode); + assert.deepStrictEqual(error.errorCode, expectedError.errorCode); + assert.deepStrictEqual(error.errorMessage, expectedError.errorMessage); + return; + } + + assert.fail(); + }); +}); diff --git a/tests/dfs/handlers/AppendBlobHandler.test.ts b/tests/dfs/handlers/AppendBlobHandler.test.ts new file mode 100644 index 000000000..dc83a5a47 --- /dev/null +++ b/tests/dfs/handlers/AppendBlobHandler.test.ts @@ -0,0 +1,235 @@ +import assert = require("assert"); +import { PassThrough } from "stream"; +import { + anyOfClass, + anything, + deepEqual, + instance, + mock, + when +} from "ts-mockito"; + +import logger, { configLogger } from "../../../src/common/Logger"; +import IExtentStore, { + IExtentChunk +} from "../../../src/common/persistence/IExtentStore"; +import DataLakeContext from "../../../src/dfs/context/DataLakeContext"; +import * as Models from "../../../src/dfs/generated/artifacts/models"; +import Context from "../../../src/dfs/generated/Context"; +import IRequest from "../../../src/dfs/generated/IRequest"; +import AppendBlobHandler from "../../../src/dfs/handlers/AppendBlobHandler"; +import { + BlockModel, + IDataLakeMetadataStore +} from "../../../src/dfs/persistence/IDataLakeMetadataStore"; +import { + DEFAULT_FILE_PERMISSIONS, + DEFAULT_GROUP, + DEFAULT_OWNER, + HeaderConstants +} from "../../../src/dfs/utils/constants"; +import { getUniqueName } from "../../testutils"; +import { toPermissions } from "../../../src/dfs/storage-file-datalake/transforms"; + +// Set true to enable debug log +configLogger(false); + +describe("AppendBlobHandler", () => { + const request: IRequest = mock(); + when(request.getHeader(HeaderConstants.CONTENT_MD5)).thenReturn(undefined); + when(request.getRawHeaders()).thenReturn([]); + + const blobCtx = new DataLakeContext({ contextId: "" } as Context); + blobCtx.contextId = getUniqueName("contextID"); + blobCtx.account = getUniqueName("account"); + blobCtx.container = getUniqueName("container"); + blobCtx.blob = getUniqueName("blob"); + blobCtx.request = instance(request); + + const buffer = Buffer.from("deadbeef"); + + const properties: Models.BlobPropertiesInternal = { + lastModified: new Date(), + etag: getUniqueName("etag"), + blobType: Models.BlobType.AppendBlob, + contentLength: buffer.length + }; + + const extent: IExtentChunk = { + id: getUniqueName("extentID"), + offset: 0, + count: buffer.length + }; + + const metadataStore: IDataLakeMetadataStore = mock(); + when( + metadataStore.downloadBlob( + anything(), + blobCtx.account, + blobCtx.container, + blobCtx.blob, + undefined + ) + ).thenResolve({ + name: blobCtx.blob, + accountName: blobCtx.account, + containerName: blobCtx.container, + isCommitted: false, + properties, + isDirectory: false, + permissions: toPermissions(DEFAULT_FILE_PERMISSIONS)!, + acl: [], + owner: DEFAULT_OWNER, + group: DEFAULT_GROUP + }); + when( + metadataStore.appendBlock( + anything(), + deepEqual({ + accountName: blobCtx.account, + containerName: blobCtx.container, + blobName: blobCtx.blob, + isCommitted: true, + name: "", + size: extent.count, + persistency: extent + } as BlockModel), + undefined, + undefined, + undefined + ) + ).thenResolve(properties); + + const extentStore: IExtentStore = mock(); + when( + // Need to reset the buffer stream for each test which calls this function + // so we accept any PassThrough stream here. + extentStore.appendExtent(anyOfClass(PassThrough), blobCtx.contextId) + ).thenResolve(extent); + + describe("create", () => { + it("accepts requests withContent-Length == 0 @loki @sql", async () => { + const handler = new AppendBlobHandler( + instance(metadataStore), + instance(extentStore), + logger, + false + ); + await assert.doesNotReject(async () => { + await handler.create(0, {}, blobCtx); + }); + }); + + it("accepts requests with Content-Length != 0 in loose mode @loki @sql", async () => { + const handler = new AppendBlobHandler( + instance(metadataStore), + instance(extentStore), + logger, + true + ); + await assert.doesNotReject(async () => { + await handler.create(buffer.length, {}, blobCtx); + }); + }); + + it("rejects requests with Content-Length != 0 @loki @sql", async () => { + const handler = new AppendBlobHandler( + instance(metadataStore), + instance(extentStore), + logger, + false + ); + await assert.rejects( + async () => { + await handler.create(buffer.length, {}, blobCtx); + }, + { + name: "StorageError", + errorCode: "InvalidOperation" + } + ); + }); + }); + + describe("appendBlock", () => { + let bufferStream: PassThrough; + beforeEach(() => { + bufferStream = new PassThrough(); + bufferStream.end(buffer); + }); + + it("accepts requests with Content-Length != 0 @loki @sql", async () => { + const handler = new AppendBlobHandler( + instance(metadataStore), + instance(extentStore), + logger, + false + ); + await assert.doesNotReject(async () => { + await handler.appendBlock(bufferStream, buffer.length, {}, blobCtx); + }); + }); + + it("rejects requests with Content-Length == 0 @loki @sql", async () => { + const handler = new AppendBlobHandler( + instance(metadataStore), + instance(extentStore), + logger, + false + ); + await assert.rejects( + async () => { + await handler.appendBlock(bufferStream, 0, {}, blobCtx); + }, + { + name: "StorageError", + errorCode: "InvalidHeaderValue" + } + ); + }); + + it("accepts requests with valid MD5 checksum @loki @sql", async () => { + when(request.getHeader(HeaderConstants.CONTENT_MD5)).thenReturn( + "T0EkOEfaaTpPNWwEhhFLxg==" + ); + when(extentStore.readExtent(extent, blobCtx.contextId)).thenResolve( + bufferStream + ); + + const handler = new AppendBlobHandler( + instance(metadataStore), + instance(extentStore), + logger, + false + ); + await assert.doesNotReject(async () => { + await handler.appendBlock(bufferStream, buffer.length, {}, blobCtx); + }); + }); + + it("rejects requests with invalid MD5 checksum @loki @sql", async () => { + when(request.getHeader(HeaderConstants.CONTENT_MD5)).thenReturn( + "d3JvbmdfTUQ1X2NoZWNrc3VtCg==" + ); + when(extentStore.readExtent(extent, blobCtx.contextId)).thenResolve( + bufferStream + ); + + const handler = new AppendBlobHandler( + instance(metadataStore), + instance(extentStore), + logger, + false + ); + await assert.rejects( + async () => { + await handler.appendBlock(bufferStream, buffer.length, {}, blobCtx); + }, + { + name: "StorageError", + errorCode: "Md5Mismatch" + } + ); + }); + }); +}); diff --git a/tests/dfs/handlers/PageBlobRangesManager.test.ts b/tests/dfs/handlers/PageBlobRangesManager.test.ts new file mode 100644 index 000000000..f30bc8f03 --- /dev/null +++ b/tests/dfs/handlers/PageBlobRangesManager.test.ts @@ -0,0 +1,274 @@ +import assert = require("assert"); + +import PageBlobRangesManager from "../../../src/dfs/handlers/PageBlobRangesManager"; + +describe("PageBlobRangesManager", () => { + it("selectImpactedRanges @loki @sql", () => { + const testCases = [ + { + ranges: [], + start: 0, + end: 0, + results: [Infinity, -1] + }, + { + ranges: [ + { + start: 0, + end: 9 + }, + { + start: 10, + end: 19 + }, + { + start: 20, + end: 29 + } + ], + start: 0, + end: 0, + results: [0, 0] + }, + { + ranges: [ + { + start: 0, + end: 9 + }, + { + start: 10, + end: 19 + }, + { + start: 20, + end: 29 + } + ], + start: 0, + end: 9, + results: [0, 0] + }, + { + ranges: [ + { + start: 0, + end: 9 + }, + { + start: 10, + end: 19 + }, + { + start: 20, + end: 29 + } + ], + start: 0, + end: 10, + results: [0, 1] + }, + { + ranges: [ + { + start: 0, + end: 9 + }, + { + start: 10, + end: 19 + }, + { + start: 20, + end: 29 + } + ], + start: 5, + end: 15, + results: [0, 1] + }, + { + ranges: [ + { + start: 0, + end: 9 + }, + { + start: 10, + end: 19 + }, + { + start: 20, + end: 29 + } + ], + start: 10, + end: 15, + results: [1, 1] + }, + { + ranges: [ + { + start: 0, + end: 9 + }, + { + start: 20, + end: 29 + }, + { + start: 40, + end: 49 + } + ], + start: 10, + end: 15, + results: [1, 0] + }, + { + ranges: [ + { + start: 0, + end: 9 + }, + { + start: 20, + end: 29 + }, + { + start: 40, + end: 49 + } + ], + start: 10, + end: 19, + results: [1, 0] + }, + { + ranges: [ + { + start: 0, + end: 9 + }, + { + start: 20, + end: 29 + }, + { + start: 40, + end: 49 + } + ], + start: 10, + end: 20, + results: [1, 1] + }, + { + ranges: [ + { + start: 0, + end: 9 + }, + { + start: 20, + end: 29 + }, + { + start: 40, + end: 49 + } + ], + start: 50, + end: 55, + results: [Infinity, 2] + }, + { + ranges: [ + { + start: 20, + end: 29 + }, + { + start: 40, + end: 49 + } + ], + start: 10, + end: 15, + results: [0, -1] + }, + { + ranges: [ + { + start: 0, + end: 9 + }, + { + start: 20, + end: 29 + }, + { + start: 40, + end: 49 + } + ], + start: 21, + end: 21, + results: [1, 1] + }, + { + ranges: [ + { + start: 0, + end: 9 + }, + { + start: 20, + end: 29 + }, + { + start: 40, + end: 49 + } + ], + start: 8, + end: 40, + results: [0, 2] + }, + { + ranges: [ + { + start: 1, + end: 9 + }, + { + start: 20, + end: 29 + }, + { + start: 40, + end: 49 + } + ], + start: 0, + end: 100, + results: [0, 2] + } + ]; + + const manager = new PageBlobRangesManager(); + + testCases.forEach((testCase) => { + const results = manager.selectImpactedRanges( + testCase.ranges as any, + testCase.start, + testCase.end + ); + assert.deepStrictEqual( + results, + testCase.results, + `testCase: ${JSON.stringify(testCase)}` + ); + }); + }); +}); diff --git a/tests/dfs/https.test.ts b/tests/dfs/https.test.ts new file mode 100644 index 000000000..24bcd8eb8 --- /dev/null +++ b/tests/dfs/https.test.ts @@ -0,0 +1,54 @@ +import { + BlobServiceClient, + newPipeline, + StorageSharedKeyCredential +} from "@azure/storage-blob"; + +import { configLogger } from "../../src/common/Logger"; +import { + EMULATOR_ACCOUNT_KEY, + EMULATOR_ACCOUNT_NAME, + getUniqueName +} from "../testutils"; +import DataLakeTestServerFactory from "./DataLakeTestServerFactory"; + +// Set true to enable debug log +configLogger(false); + +describe("Blob HTTPS", () => { + const factory = new DataLakeTestServerFactory(); + const server = factory.createServer(false, false, true); + const baseURL = `https://${server.config.host}:${server.config.port}/devstoreaccount1`; + + before(async () => { + await server.start(); + }); + + after(async () => { + await server.close(); + await server.clean(); + }); + + it(`Should work with correct shared key using HTTPS endpoint @loki @sql`, async () => { + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ) + ); + + const containerName: string = getUniqueName("1container-with-dash"); + const containerClient = serviceClient.getContainerClient(containerName); + + await containerClient.create(); + await containerClient.delete(); + }); +}); diff --git a/tests/dfs/oauth.test.ts b/tests/dfs/oauth.test.ts new file mode 100644 index 000000000..5a6c52051 --- /dev/null +++ b/tests/dfs/oauth.test.ts @@ -0,0 +1,1029 @@ +import * as assert from "assert"; + +import { + AccountSASPermissions, + AccountSASResourceTypes, + AccountSASServices, + AnonymousCredential, + BlobBatch, + BlobClient, + BlobServiceClient, + ContainerClient, + ContainerSASPermissions, + generateAccountSASQueryParameters, + generateBlobSASQueryParameters, + newPipeline, + SASProtocol, + StorageSharedKeyCredential +} from "@azure/storage-blob"; + +import { configLogger } from "../../src/common/Logger"; +import { SimpleTokenCredential } from "../simpleTokenCredential"; +import { + EMULATOR_ACCOUNT_KEY, + EMULATOR_ACCOUNT_NAME, + generateJWTToken, + getUniqueName +} from "../testutils"; +import DataLakeTestServerFactory from "./DataLakeTestServerFactory"; + +// Set true to enable debug log +configLogger(false); + +describe("Blob OAuth Basic", () => { + const factory = new DataLakeTestServerFactory(); + let server = factory.createServer(false, false, true, "basic"); + const baseURL = `https://${server.config.host}:${server.config.port}/devstoreaccount1`; + + before(async () => { + await server.start(); + }); + + after(async () => { + await server.close(); + await server.clean(); + }); + + it(`Should work with create container @loki @sql`, async () => { + const token = generateJWTToken( + new Date("2019/01/01"), + new Date("2019/01/01"), + new Date("2100/01/01"), + "https://sts.windows-ppe.net/ab1f708d-50f6-404c-a006-d71b2ac7a606/", + "https://storage.azure.com", + "user_impersonation", + "23657296-5cd5-45b0-a809-d972a7f4dfe1", + "dd0d0df1-06c3-436c-8034-4b9a153097ce" + ); + + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline(new SimpleTokenCredential(token), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const containerName: string = getUniqueName("1container-with-dash"); + const containerClient = serviceClient.getContainerClient(containerName); + + await containerClient.create(); + await containerClient.delete(); + }); + + it(`Should work with blob batch deleting @loki @sql`, async () => { + const token = generateJWTToken( + new Date("2019/01/01"), + new Date("2019/01/01"), + new Date("2100/01/01"), + "https://sts.windows-ppe.net/ab1f708d-50f6-404c-a006-d71b2ac7a606/", + "https://storage.azure.com", + "user_impersonation", + "23657296-5cd5-45b0-a809-d972a7f4dfe1", + "dd0d0df1-06c3-436c-8034-4b9a153097ce" + ); + + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline(new SimpleTokenCredential(token), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const containerName: string = getUniqueName("1container-with-dash"); + const containerClient = serviceClient.getContainerClient(containerName); + + await containerClient.create(); + + const blobClients: BlobClient[] = []; + const blobCount = 3; + const content = "Hello World"; + + for (let i = 0; i < blobCount; ++i) { + const blobName = getUniqueName("blob"); + const blobClient = containerClient.getBlobClient(blobName); + const blockBlobClient = blobClient.getBlockBlobClient(); + await blockBlobClient.upload(content, content.length); + blobClients.push(blobClient); + } + + const blobBatchClient = serviceClient.getBlobBatchClient(); + // Assemble batch delete request. + const batchDeleteRequest = new BlobBatch(); + for (let i = 0; i < blobCount; i++) { + await batchDeleteRequest.deleteBlob(blobClients[i]); + } + + // Submit batch request and verify response. + const resp = await blobBatchClient.submitBatch(batchDeleteRequest, {}); + assert.equal(resp.subResponses.length, blobCount); + assert.equal(resp.subResponsesSucceededCount, blobCount); + assert.equal(resp.subResponsesFailedCount, 0); + + for (let i = 0; i < blobCount; i++) { + assert.equal(resp.subResponses[i].errorCode, undefined); + assert.equal(resp.subResponses[i].status, 202); + assert.ok(resp.subResponses[i].statusMessage !== ""); + assert.ok(resp.subResponses[i].headers.contains("x-ms-request-id")); + assert.equal(resp.subResponses[i]._request.url, blobClients[i].url); + } + + // Verify blobs deleted. + const resp2 = ( + await containerClient + .listBlobsFlat({ + includeSnapshots: true + }) + .byPage({ maxPageSize: 1 }) + .next() + ).value; + assert.equal(resp2.segment.blobItems.length, 0); + }); + + it(`Should work with blob batch set tier @loki @sql`, async () => { + const token = generateJWTToken( + new Date("2019/01/01"), + new Date("2019/01/01"), + new Date("2100/01/01"), + "https://sts.windows-ppe.net/ab1f708d-50f6-404c-a006-d71b2ac7a606/", + "https://storage.azure.com", + "user_impersonation", + "23657296-5cd5-45b0-a809-d972a7f4dfe1", + "dd0d0df1-06c3-436c-8034-4b9a153097ce" + ); + + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline(new SimpleTokenCredential(token), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const containerName: string = getUniqueName("1container-with-dash"); + const containerClient = serviceClient.getContainerClient(containerName); + + await containerClient.create(); + + const blobClients: BlobClient[] = []; + const blobCount = 3; + const content = "Hello World"; + + for (let i = 0; i < blobCount; ++i) { + const blobName = getUniqueName("blob"); + const blobClient = containerClient.getBlobClient(blobName); + const blockBlobClient = blobClient.getBlockBlobClient(); + await blockBlobClient.upload(content, content.length); + blobClients.push(blobClient); + } + + const blobBatchClient = serviceClient.getBlobBatchClient(); + // Assemble batch delete request. + const batchRequest = new BlobBatch(); + for (let i = 0; i < blobCount; i++) { + await batchRequest.setBlobAccessTier(blobClients[i], "Archive"); + } + + // Submit batch request and verify response. + const resp = await blobBatchClient.submitBatch(batchRequest, {}); + assert.equal(resp.subResponses.length, blobCount); + assert.equal(resp.subResponsesSucceededCount, blobCount); + assert.equal(resp.subResponsesFailedCount, 0); + + for (let i = 0; i < blobCount; i++) { + assert.equal(resp.subResponses[i].errorCode, undefined); + assert.equal(resp.subResponses[i].status, 200); + assert.ok(resp.subResponses[i].statusMessage !== ""); + assert.ok(resp.subResponses[i].headers.contains("x-ms-request-id")); + assert.equal(resp.subResponses[i]._request.url, blobClients[i].url); + } + + for (const blobClient of blobClients) { + // Check blob tier set properly. + const resp2 = await blobClient.getProperties(); + assert.equal(resp2.accessTier, "Archive"); + } + }); + + it(`Should work with delegation SAS @loki @sql`, async () => { + const token = generateJWTToken( + new Date("2019/01/01"), + new Date("2019/01/01"), + new Date("2100/01/01"), + "https://sts.windows-ppe.net/ab1f708d-50f6-404c-a006-d71b2ac7a606/", + "https://storage.azure.com", + "user_impersonation", + "23657296-5cd5-45b0-a809-d972a7f4dfe1", + "dd0d0df1-06c3-436c-8034-4b9a153097ce" + ); + + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline(new SimpleTokenCredential(token), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const startTime = new Date(); + startTime.setHours(startTime.getHours() - 1); + const expiryTime = new Date(); + expiryTime.setDate(expiryTime.getDate() + 1); + + const userDelegationKey = await serviceClient.getUserDelegationKey( + startTime, + expiryTime + ); + + const containerName: string = getUniqueName("1container-with-dash"); + + const sasExpirytime = new Date(); + sasExpirytime.setHours(sasExpirytime.getHours() + 1); + + const containerSAS = generateBlobSASQueryParameters( + { + containerName: containerName, + expiresOn: sasExpirytime, + permissions: ContainerSASPermissions.parse("racwdl") + }, + userDelegationKey, + "devstoreaccount1" + ); + + const containerClient = new ContainerClient( + `${serviceClient.url}/${containerName}?${containerSAS}`, + newPipeline(new AnonymousCredential()) + ); + await containerClient.create(); + await containerClient.delete(); + }); + + it(`Should work with delegation SAS container client doing blob upload @loki @sql`, async () => { + const token = generateJWTToken( + new Date("2019/01/01"), + new Date("2019/01/01"), + new Date("2100/01/01"), + "https://sts.windows-ppe.net/ab1f708d-50f6-404c-a006-d71b2ac7a606/", + "https://storage.azure.com", + "user_impersonation", + "23657296-5cd5-45b0-a809-d972a7f4dfe1", + "dd0d0df1-06c3-436c-8034-4b9a153097ce" + ); + + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline(new SimpleTokenCredential(token), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const startTime = new Date(); + startTime.setHours(startTime.getHours() - 1); + const expiryTime = new Date(); + expiryTime.setDate(expiryTime.getDate() + 1); + + const userDelegationKey = await serviceClient.getUserDelegationKey( + startTime, + expiryTime + ); + + const containerName: string = getUniqueName("1container-with-dash"); + + const sasExpirytime = new Date(); + sasExpirytime.setHours(sasExpirytime.getHours() + 1); + + const containerSAS = generateBlobSASQueryParameters( + { + containerName: containerName, + expiresOn: sasExpirytime, + permissions: ContainerSASPermissions.parse("racwdl") + }, + userDelegationKey, + "devstoreaccount1" + ); + + const containerClient = new ContainerClient( + `${serviceClient.url}/${containerName}?${containerSAS}`, + newPipeline(new AnonymousCredential()) + ); + await containerClient.create(); + const blobClient = await containerClient.getBlockBlobClient("test"); + const data = "Test Data"; + await blobClient.upload(data, data.length); + await containerClient.delete(); + }); + + it(`Should fail with delegation SAS with invalid time duration @loki @sql`, async () => { + const token = generateJWTToken( + new Date("2019/01/01"), + new Date("2019/01/01"), + new Date("2100/01/01"), + "https://sts.windows-ppe.net/ab1f708d-50f6-404c-a006-d71b2ac7a606/", + "https://storage.azure.com", + "user_impersonation", + "23657296-5cd5-45b0-a809-d972a7f4dfe1", + "dd0d0df1-06c3-436c-8034-4b9a153097ce" + ); + + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline(new SimpleTokenCredential(token), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + const containerName: string = getUniqueName("1container-with-dash"); + + // Later user delegation key start time + let startTime = new Date(); + startTime.setMinutes(startTime.getMinutes() + 20); + let expiryTime = new Date(); + expiryTime.setDate(expiryTime.getDate() + 1); + let userDelegationKey = await serviceClient.getUserDelegationKey( + startTime, + expiryTime + ); + + let sasExpirytime = new Date(); + sasExpirytime.setHours(sasExpirytime.getHours() + 1); + + let containerSAS = generateBlobSASQueryParameters( + { + containerName: containerName, + expiresOn: sasExpirytime, + permissions: ContainerSASPermissions.parse("racwdl") + }, + userDelegationKey, + "devstoreaccount1" + ); + + let containerClient = new ContainerClient( + `${serviceClient.url}/${containerName}?${containerSAS}`, + newPipeline(new AnonymousCredential()) + ); + let failed = false; + try { + await containerClient.create(); + } catch (err) { + failed = true; + assert.equal(err.statusCode, 403); + } + assert.ok(failed); + + // Eearlier user delegation key expirty time + startTime = new Date(); + startTime.setDate(startTime.getDate() - 1); + expiryTime = new Date(); + expiryTime.setMinutes(expiryTime.getMinutes() - 1); + userDelegationKey = await serviceClient.getUserDelegationKey( + startTime, + expiryTime + ); + + sasExpirytime = new Date(); + sasExpirytime.setHours(sasExpirytime.getHours() + 1); + + containerSAS = generateBlobSASQueryParameters( + { + containerName: containerName, + expiresOn: sasExpirytime, + permissions: ContainerSASPermissions.parse("racwdl") + }, + userDelegationKey, + "devstoreaccount1" + ); + + containerClient = new ContainerClient( + `${serviceClient.url}/${containerName}?${containerSAS}`, + newPipeline(new AnonymousCredential()) + ); + + failed = false; + try { + await containerClient.create(); + } catch (err) { + failed = true; + assert.equal(err.statusCode, 403); + } + assert.ok(failed); + }); + + it(`Should fail with delegation SAS with access policy @loki @sql`, async () => { + const token = generateJWTToken( + new Date("2019/01/01"), + new Date("2019/01/01"), + new Date("2100/01/01"), + "https://sts.windows-ppe.net/ab1f708d-50f6-404c-a006-d71b2ac7a606/", + "https://storage.azure.com", + "user_impersonation", + "23657296-5cd5-45b0-a809-d972a7f4dfe1", + "dd0d0df1-06c3-436c-8034-4b9a153097ce" + ); + + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline(new SimpleTokenCredential(token), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const startTime = new Date(); + startTime.setHours(startTime.getHours() - 1); + const expiryTime = new Date(); + expiryTime.setDate(expiryTime.getDate() + 1); + const userDelegationKey = await serviceClient.getUserDelegationKey( + startTime, + expiryTime + ); + + const serviceClientWithAccountKey = new BlobServiceClient( + baseURL, + newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ) + ); + const containerName: string = getUniqueName("1container-with-dash"); + const containerClientWithKey = + serviceClientWithAccountKey.getContainerClient(containerName); + await containerClientWithKey.create(); + await containerClientWithKey.setAccessPolicy(undefined, [ + { + accessPolicy: { + permissions: "racwdl" + }, + id: "MTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTI=" + } + ]); + + const sasExpirytime = new Date(); + sasExpirytime.setHours(sasExpirytime.getHours() + 1); + + const containerSAS = generateBlobSASQueryParameters( + { + containerName: containerName, + expiresOn: sasExpirytime, + permissions: ContainerSASPermissions.parse("racwdl"), + identifier: "MTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTI=" + }, + userDelegationKey, + "devstoreaccount1" + ); + + const containerClient = new ContainerClient( + `${serviceClient.url}/${containerName}?${containerSAS}`, + newPipeline(new AnonymousCredential()) + ); + + let failed = false; + try { + await containerClient.getProperties(); + } catch (err) { + failed = true; + assert.equal(err.statusCode, 403); + } + assert.ok(failed); + }); + + it(`Should not work with invalid JWT token @loki @sql`, async () => { + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline(new SimpleTokenCredential("invalid token"), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const containerName: string = getUniqueName("1container-with-dash"); + const containerClient = serviceClient.getContainerClient(containerName); + + try { + await containerClient.create(); + await containerClient.delete(); + } catch (err) { + assert.deepStrictEqual( + err.message.includes("Server failed to authenticate the request."), + true + ); + return; + } + assert.fail(); + }); + + it(`Should work with valid audiences @loki @sql`, async () => { + const audiences = [ + "https://storage.azure.com", + "https://storage.azure.com/", + "e406a681-f3d4-42a8-90b6-c2b029497af1", + "https://devstoreaccount1.blob.core.windows.net", + "https://devstoreaccount1.blob.core.windows.net/", + "https://devstoreaccount1.blob.core.chinacloudapi.cn", + "https://devstoreaccount1.blob.core.chinacloudapi.cn/", + "https://devstoreaccount1.blob.core.usgovcloudapi.net", + "https://devstoreaccount1.blob.core.usgovcloudapi.net/", + "https://devstoreaccount1.blob.core.cloudapi.de", + "https://devstoreaccount1.blob.core.cloudapi.de/" + ]; + + for (const audience of audiences) { + const token = generateJWTToken( + new Date("2019/01/01"), + new Date("2019/01/01"), + new Date("2100/01/01"), + "https://sts.windows-ppe.net/ab1f708d-50f6-404c-a006-d71b2ac7a606/", + audience, + "user_impersonation", + "23657296-5cd5-45b0-a809-d972a7f4dfe1", + "dd0d0df1-06c3-436c-8034-4b9a153097ce" + ); + + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline(new SimpleTokenCredential(token), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const containerName: string = getUniqueName("1container-with-dash"); + const containerClient = serviceClient.getContainerClient(containerName); + + await containerClient.create(); + await containerClient.delete(); + } + }); + + it(`Should not work with invalid audiences @loki @sql`, async () => { + const token = generateJWTToken( + new Date("2019/01/01"), + new Date("2019/01/01"), + new Date("2100/01/01"), + "https://sts.windows-ppe.net/ab1f708d-50f6-404c-a006-d71b2ac7a606/", + "https://invalidaccount.blob.core.windows.net", + "user_impersonation", + "23657296-5cd5-45b0-a809-d972a7f4dfe1", + "dd0d0df1-06c3-436c-8034-4b9a153097ce" + ); + + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline(new SimpleTokenCredential(token), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const containerName: string = getUniqueName("1container-with-dash"); + const containerClient = serviceClient.getContainerClient(containerName); + + try { + await containerClient.create(); + await containerClient.delete(); + } catch (err) { + assert.deepStrictEqual( + err.message.includes("Server failed to authenticate the request."), + true + ); + assert.deepStrictEqual( + err.details.AuthenticationErrorDetail.includes("audience"), + true + ); + return; + } + assert.fail(); + }); + + it(`Should work with valid issuers @loki @sql`, async () => { + const issuerPrefixes = [ + "https://sts.windows.net/", + "https://sts.microsoftonline.de/", + "https://sts.chinacloudapi.cn/", + "https://sts.windows-ppe.net" + ]; + + for (const issuerPrefix of issuerPrefixes) { + const token = generateJWTToken( + new Date("2019/01/01"), + new Date("2019/01/01"), + new Date("2100/01/01"), + `${issuerPrefix}/ab1f708d-50f6-404c-a006-d71b2ac7a606/`, + "e406a681-f3d4-42a8-90b6-c2b029497af1", + "user_impersonation", + "23657296-5cd5-45b0-a809-d972a7f4dfe1", + "dd0d0df1-06c3-436c-8034-4b9a153097ce" + ); + + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline(new SimpleTokenCredential(token), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const containerName: string = getUniqueName("1container-with-dash"); + const containerClient = serviceClient.getContainerClient(containerName); + + await containerClient.create(); + await containerClient.delete(); + } + }); + + it(`Should not work with invalid issuers @loki @sql`, async () => { + const token = generateJWTToken( + new Date("2019/01/01"), + new Date("2019/01/01"), + new Date("2100/01/01"), + "https://invalidissuer/ab1f708d-50f6-404c-a006-d71b2ac7a606/", + "https://invalidaccount.blob.core.windows.net", + "user_impersonation", + "23657296-5cd5-45b0-a809-d972a7f4dfe1", + "dd0d0df1-06c3-436c-8034-4b9a153097ce" + ); + + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline(new SimpleTokenCredential(token), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const containerName: string = getUniqueName("1container-with-dash"); + const containerClient = serviceClient.getContainerClient(containerName); + + try { + await containerClient.create(); + await containerClient.delete(); + } catch (err) { + assert.deepStrictEqual( + err.message.includes("Server failed to authenticate the request."), + true + ); + assert.deepStrictEqual( + err.details.AuthenticationErrorDetail.includes("issuer"), + true + ); + return; + } + assert.fail(); + }); + + it(`Should not work with invalid nbf @loki @sql`, async () => { + const token = generateJWTToken( + new Date("2119/01/01"), + new Date("2019/01/01"), + new Date("2100/01/01"), + "https://sts.windows-ppe.net/ab1f708d-50f6-404c-a006-d71b2ac7a606/", + "https://devstoreaccount1.blob.core.windows.net", + "user_impersonation", + "23657296-5cd5-45b0-a809-d972a7f4dfe1", + "dd0d0df1-06c3-436c-8034-4b9a153097ce" + ); + + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline(new SimpleTokenCredential(token), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const containerName: string = getUniqueName("1container-with-dash"); + const containerClient = serviceClient.getContainerClient(containerName); + + try { + await containerClient.create(); + await containerClient.delete(); + } catch (err) { + assert.deepStrictEqual( + err.message.includes("Server failed to authenticate the request."), + true + ); + assert.deepStrictEqual( + err.details.AuthenticationErrorDetail.includes("Lifetime"), + true + ); + return; + } + assert.fail(); + }); + + it(`Should not work with invalid exp @loki @sql`, async () => { + const token = generateJWTToken( + new Date("2019/01/01"), + new Date("2019/01/01"), + new Date("2019/01/01"), + "https://sts.windows-ppe.net/ab1f708d-50f6-404c-a006-d71b2ac7a606/", + "https://devstoreaccount1.blob.core.windows.net", + "user_impersonation", + "23657296-5cd5-45b0-a809-d972a7f4dfe1", + "dd0d0df1-06c3-436c-8034-4b9a153097ce" + ); + + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline(new SimpleTokenCredential(token), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const containerName: string = getUniqueName("1container-with-dash"); + const containerClient = serviceClient.getContainerClient(containerName); + + try { + await containerClient.create(); + await containerClient.delete(); + } catch (err) { + assert.deepStrictEqual( + err.message.includes("Server failed to authenticate the request."), + true + ); + assert.deepStrictEqual( + err.details.AuthenticationErrorDetail.includes("expire"), + true + ); + return; + } + assert.fail(); + }); + + it(`Should not work with get container ACL @loki @sql`, async () => { + const token = generateJWTToken( + new Date("2019/01/01"), + new Date("2019/01/01"), + new Date("2100/01/01"), + "https://sts.windows-ppe.net/ab1f708d-50f6-404c-a006-d71b2ac7a606/", + "https://devstoreaccount1.blob.core.windows.net", + "user_impersonation", + "23657296-5cd5-45b0-a809-d972a7f4dfe1", + "dd0d0df1-06c3-436c-8034-4b9a153097ce" + ); + + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline(new SimpleTokenCredential(token), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const containerName: string = getUniqueName("1container-with-dash"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + try { + await containerClient.getAccessPolicy(); + } catch (err) { + assert.deepStrictEqual( + err.message.includes("Server failed to authenticate the request."), + true + ); + await containerClient.delete(); + return; + } + await containerClient.delete(); + assert.fail(); + }); + + it(`Should not work with set container ACL @loki @sql`, async () => { + const token = generateJWTToken( + new Date("2019/01/01"), + new Date("2019/01/01"), + new Date("2100/01/01"), + "https://sts.windows-ppe.net/ab1f708d-50f6-404c-a006-d71b2ac7a606/", + "https://devstoreaccount1.blob.core.windows.net", + "user_impersonation", + "23657296-5cd5-45b0-a809-d972a7f4dfe1", + "dd0d0df1-06c3-436c-8034-4b9a153097ce" + ); + + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline(new SimpleTokenCredential(token), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const containerName: string = getUniqueName("1container-with-dash"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + try { + await containerClient.setAccessPolicy("container"); + } catch (err) { + assert.deepStrictEqual( + err.message.includes("Server failed to authenticate the request."), + true + ); + await containerClient.delete(); + return; + } + await containerClient.delete(); + assert.fail(); + }); + + it("Create container with not exist Account, return 404 @loki @sql", async () => { + const accountNameNotExist = "devstoreaccountnotexist"; + const baseURL = `https://${server.config.host}:${server.config.port}/${accountNameNotExist}`; + const containerName: string = getUniqueName("1container-with-dash"); + + // Shared key + const sharedKeyCredential = new StorageSharedKeyCredential( + accountNameNotExist, + EMULATOR_ACCOUNT_KEY + ); + let serviceClient = new BlobServiceClient( + baseURL, + newPipeline(sharedKeyCredential, { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + let containerClientNotExist = + serviceClient.getContainerClient(containerName); + try { + await containerClientNotExist.create(); + } catch (err) { + if (err.statusCode !== 404 && err.code !== "ResourceNotFound") { + assert.fail( + "Create queue with shared key not fail as expected." + err.toString() + ); + } + } + + // Oauth + const token = generateJWTToken( + new Date("2019/01/01"), + new Date("2019/01/01"), + new Date("2100/01/01"), + "https://sts.windows-ppe.net/ab1f708d-50f6-404c-a006-d71b2ac7a606/", + "https://storage.azure.com", + "user_impersonation", + "23657296-5cd5-45b0-a809-d972a7f4dfe1", + "dd0d0df1-06c3-436c-8034-4b9a153097ce" + ); + + serviceClient = new BlobServiceClient( + baseURL, + newPipeline(new SimpleTokenCredential(token), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + containerClientNotExist = serviceClient.getContainerClient(containerName); + try { + await containerClientNotExist.create(); + } catch (err) { + if (err.statusCode !== 404 && err.code !== "ResourceNotFound") { + assert.fail( + "Create queue with oauth not fail as expected." + err.toString() + ); + } + } + // Account SAS + const now = new Date(); + now.setMinutes(now.getMinutes() - 5); + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + const sas = generateAccountSASQueryParameters( + { + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: AccountSASPermissions.parse("rwdlacup"), + protocol: SASProtocol.HttpsAndHttp, + resourceTypes: AccountSASResourceTypes.parse("sco").toString(), + services: AccountSASServices.parse("btqf").toString(), + startsOn: now, + version: "2016-05-31" + }, + sharedKeyCredential + ).toString(); + let sasURL = `${serviceClient.url}?${sas}`; + serviceClient = new BlobServiceClient( + sasURL, + newPipeline(new AnonymousCredential(), { + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + containerClientNotExist = serviceClient.getContainerClient(containerName); + try { + await containerClientNotExist.create(); + } catch (err) { + if (err.statusCode !== 404 && err.code !== "ResourceNotFound") { + assert.fail( + "Create queue with account sas not fail as expected." + err.toString() + ); + } + } + + // Service SAS + const containerSAS = generateBlobSASQueryParameters( + { + containerName, + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: ContainerSASPermissions.parse("racwdl"), + protocol: SASProtocol.HttpsAndHttp, + startsOn: now, + version: "2016-05-31" + }, + sharedKeyCredential + ); + sasURL = `${serviceClient.url}?${containerSAS}`; + serviceClient = new BlobServiceClient( + sasURL, + newPipeline(new AnonymousCredential(), { + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + containerClientNotExist = serviceClient.getContainerClient(containerName); + try { + await containerClientNotExist.create(); + } catch (err) { + if (err.statusCode !== 404 && err.code !== "ResourceNotFound") { + assert.fail( + "Create queue with service sas not fail as expected." + err.toString() + ); + } + } + }); + + it(`Should not work with HTTP @loki @sql`, async () => { + await server.close(); + await server.clean(); + + server = factory.createServer(false, false, false, "basic"); + await server.start(); + + const httpBaseURL = `http://${server.config.host}:${server.config.port}/devstoreaccount1`; + + const token = generateJWTToken( + new Date("2019/01/01"), + new Date("2019/01/01"), + new Date("2019/01/01"), + "https://sts.windows-ppe.net/ab1f708d-50f6-404c-a006-d71b2ac7a606/", + "https://devstoreaccount1.blob.core.windows.net", + "user_impersonation", + "23657296-5cd5-45b0-a809-d972a7f4dfe1", + "dd0d0df1-06c3-436c-8034-4b9a153097ce" + ); + + const serviceClient = new BlobServiceClient( + httpBaseURL, + newPipeline(new SimpleTokenCredential(token), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const containerName: string = getUniqueName("1container-with-dash"); + const containerClient = serviceClient.getContainerClient(containerName); + + try { + await containerClient.create(); + await containerClient.delete(); + } catch (err) { + assert.deepStrictEqual( + err.message.includes("Bearer token authentication is not permitted"), + true + ); + assert.deepStrictEqual(err.message.includes("non-https"), true); + return; + } + assert.fail(); + }); +}); diff --git a/tests/dfs/pagewithdelimiter.test.ts b/tests/dfs/pagewithdelimiter.test.ts new file mode 100644 index 000000000..1b595906c --- /dev/null +++ b/tests/dfs/pagewithdelimiter.test.ts @@ -0,0 +1,210 @@ +import assert = require("assert"); +import { BlobPrefixModel } from "../../src/dfs/persistence/IDataLakeMetadataStore"; +import PageWithDelimiter from "../../src/dfs/persistence/PageWithDelimiter"; + +describe("PageWithDelimiter", () => { + function checkResult( + items: string[], + prefixes: BlobPrefixModel[], + marker: string, + expected_items_count: number, + expected_prefixes_count: number, + expected_marker: string + ): void { + assert.equal(items.length, expected_items_count); + assert.equal(prefixes.length, expected_prefixes_count); + assert.equal(marker, expected_marker); + } + + // a namer is used by fill, just return the value for testing + const namer = (i: string) => { + return i; + }; + // return a reader for a list + const createReader = ( + items: string[], + maxResults: number + ): ((o: number) => Promise) => { + return (o: number) => { + return Promise.resolve(items.slice(o, o + maxResults)); + }; + }; + + describe("with no delimiter", () => { + const blobs: string[] = [ + "a", + "b", + "c/0", + "c/1", + "c/sub/1", + "d", + "e/1", + "e/2" + ]; + + it("handles no blob results @Loki @sql", async () => { + const page = new PageWithDelimiter(5); + const [items, prefixes, marker] = await page.fill( + createReader([], 5), + namer + ); + checkResult(items, prefixes, marker, 0, 0, ""); + }); + + it("fills 1 result properly @Loki @sql", async () => { + const page = new PageWithDelimiter(1); + const [items, prefixes, marker] = await page.fill( + createReader(blobs, 1), + namer + ); + checkResult(items, prefixes, marker, 1, 0, "a"); + }); + + it("fills n results properly @Loki @sql", async () => { + const page = new PageWithDelimiter(5); + const [items, prefixes, marker] = await page.fill( + createReader(blobs, 5), + namer + ); + checkResult(items, prefixes, marker, 5, 0, "c/sub/1"); + }); + + it("fills exact count with no continuation @Loki @sql", async () => { + const page = new PageWithDelimiter(blobs.length); + const [items, prefixes, marker] = await page.fill( + createReader(blobs, blobs.length), + namer + ); + checkResult(items, prefixes, marker, blobs.length, 0, ""); + }); + + it("fills smaller than max page with no continuation @Loki @sql", async () => { + const page = new PageWithDelimiter(blobs.length + 1); + const [items, prefixes, marker] = await page.fill( + createReader(blobs, blobs.length + 1), + namer + ); + checkResult(items, prefixes, marker, blobs.length, 0, ""); + }); + }); + + describe("with '/' delimiter", () => { + describe("and 1 item page size", () => { + it("handles no blob results @Loki @sql", async () => { + const blobs: string[] = []; + const page = new PageWithDelimiter(1, "/"); + const [items, prefixes, marker] = await page.fill( + createReader(blobs, 1), + namer + ); + checkResult(items, prefixes, marker, 0, 0, ""); + }); + + it("handles 1 blob results @Loki @sql", async () => { + const blobs = ["a"]; + const page = new PageWithDelimiter(1, "/"); + const [items, prefixes, marker] = await page.fill( + createReader(blobs, 1), + namer + ); + checkResult(items, prefixes, marker, 1, 0, ""); + }); + + it("returns 1 of 2 items with proper continuation @Loki @sql", async () => { + const blobs = ["a", "b"]; + const page = new PageWithDelimiter(1, "/"); + let [items, prefixes, marker] = await page.fill( + createReader(blobs, 1), + namer + ); + checkResult(items, prefixes, marker, 1, 0, "a"); + + // now cut off the end of the array and ensure no continuation is returned + page.reset(); + [items, prefixes, marker] = await page.fill( + createReader(blobs.slice(1), 1), + namer + ); + checkResult(items, prefixes, marker, 1, 0, ""); + }); + + it("returns first item when prefixes exist @Loki @sql", async () => { + const blobs = ["a/1", "a/2", "a/3", "a/sub/1"]; + const page = new PageWithDelimiter(1, "/", "a/"); + const [items, prefixes, marker] = await page.fill( + createReader(blobs, 1), + namer + ); + checkResult(items, prefixes, marker, 1, 0, "a/1"); + }); + + it("returns first prefix when blobs exist @Loki @sql", async () => { + const blobs = ["a/s0/1", "a/s0/2", "a/s0/3", "a/s1/1", "a/s2/2", "a/z"]; + const page = new PageWithDelimiter(1, "/", "a/"); + const [items, prefixes, marker] = await page.fill( + createReader(blobs, 1), + namer + ); + checkResult(items, prefixes, marker, 0, 1, "a/s0/3"); + }); + }); + + describe("multiple item page size", () => { + it("squashes prefixes @Loki @sql", async () => { + const blobs = [ + "a/s0/1", + "a/s0/2", + "a/s0/3", + "a/s1/1", + "a/s1/2", + "a/s2/2", + "a/z" + ]; + const page = new PageWithDelimiter(2, "/", "a/"); + const [items, prefixes, marker] = await page.fill( + createReader(blobs, 2), + namer + ); + checkResult(items, prefixes, marker, 0, 2, "a/s1/2"); + }); + + it("squashes a mix @Loki @sql", async () => { + const blobs = ["a/a", "a/s0/1", "a/s0/2", "a/s1/1", "a/s1/2", "a/z"]; + const page = new PageWithDelimiter(2, "/", "a/"); + const [items, prefixes, marker] = await page.fill( + createReader(blobs, 2), + namer + ); + checkResult(items, prefixes, marker, 1, 1, "a/s0/2"); + }); + + it("follows squashed pages @Loki @sql", async () => { + const blobs = ["a/a", "a/s0/1", "a/s0/2", "a/s1/1", "a/s1/2", "a/z"]; + const page = new PageWithDelimiter(2, "/", "a/"); + let [items, prefixes, marker] = await page.fill( + createReader(blobs, 2), + namer + ); + checkResult(items, prefixes, marker, 1, 1, "a/s0/2"); + + // now cut off the end of the array and ensure no continuation is returned + page.reset(); + [items, prefixes, marker] = await page.fill( + createReader(blobs.slice(3), 2), + namer + ); + checkResult(items, prefixes, marker, 1, 1, ""); + }); + + it("squashes within one larger page @Loki @sql", async () => { + const blobs = ["a/a", "a/s0/1", "a/s0/2", "a/s1/1", "a/s1/2", "a/z"]; + const page = new PageWithDelimiter(4, "/", "a/"); + let [items, prefixes, marker] = await page.fill( + createReader(blobs, 4), + namer + ); + checkResult(items, prefixes, marker, 2, 2, ""); + }); + }); + }); +}); diff --git a/tests/dfs/sas.test.ts b/tests/dfs/sas.test.ts new file mode 100644 index 000000000..6479bd33a --- /dev/null +++ b/tests/dfs/sas.test.ts @@ -0,0 +1,1808 @@ +import * as assert from "assert"; + +import { URLBuilder } from "@azure/ms-rest-js"; +import { + AccountSASPermissions, + AccountSASResourceTypes, + AccountSASServices, + AnonymousCredential, + AppendBlobClient, + BlobSASPermissions, + BlobServiceClient, + ContainerClient, + ContainerSASPermissions, + generateAccountSASQueryParameters, + generateBlobSASQueryParameters, + newPipeline, + PageBlobClient, + SASProtocol, + StorageSharedKeyCredential +} from "@azure/storage-blob"; + +import { + EMULATOR_ACCOUNT_KEY_STR, + EMULATOR_ACCOUNT_NAME +} from "../../src/blob/utils/constants"; +import { configLogger } from "../../src/common/Logger"; +import { getUniqueName } from "../testutils"; +import DataLakeTestServerFactory from "./DataLakeTestServerFactory"; + +const EMULATOR_ACCOUNT2_NAME = "devstoreaccount2"; +const EMULATOR_ACCOUNT2_KEY_STR = + "MTAwCjE2NQoyMjUKMTAzCjIxOAoyNDEKNDAKNzgKMTkxCjE3OAoyMTQKMTY5CjIxMwo2MQoyNTIKMTQxCg=="; + +// Set true to enable debug log +configLogger(false); + +describe("Shared Access Signature (SAS) authentication", () => { + // Setup two accounts for validating cross-account copy operations + process.env[ + "AZURITE_ACCOUNTS" + ] = `${EMULATOR_ACCOUNT_NAME}:${EMULATOR_ACCOUNT_KEY_STR};${EMULATOR_ACCOUNT2_NAME}:${EMULATOR_ACCOUNT2_KEY_STR}`; + + const factory = new DataLakeTestServerFactory(); + const server = factory.createServer(); + + const baseURL = `http://${server.config.host}:${server.config.port}/devstoreaccount1`; + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY_STR + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ) + ); + + const baseURL2 = `http://${server.config.host}:${server.config.port}/devstoreaccount2`; + const serviceClient2 = new BlobServiceClient( + baseURL2, + newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT2_NAME, + EMULATOR_ACCOUNT2_KEY_STR + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ) + ); + + before(async () => { + await server.start(); + }); + + after(async () => { + await server.close(); + await server.clean(); + }); + + it("generateAccountSASQueryParameters should work @Loki @sql", async () => { + const now = new Date(); + now.setMinutes(now.getMinutes() - 5); // Skip clock skew with server + + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const storageSharedKeyCredential = factories[factories.length - 1]; + + const sas = generateAccountSASQueryParameters( + { + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: AccountSASPermissions.parse("rwdlacup"), + protocol: SASProtocol.HttpsAndHttp, + resourceTypes: AccountSASResourceTypes.parse("sco").toString(), + services: AccountSASServices.parse("btqf").toString(), + startsOn: now, + version: "2016-05-31" + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ).toString(); + + const sasURL = `${serviceClient.url}?${sas}`; + const serviceClientWithSAS = new BlobServiceClient( + sasURL, + newPipeline(new AnonymousCredential(), { + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + await serviceClientWithSAS.getAccountInfo(); + }); + + it("generateAccountSASQueryParameters should work for set blob tier @Loki @sql", async () => { + const now = new Date(); + now.setMinutes(now.getMinutes() - 5); // Skip clock skew with server + + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const storageSharedKeyCredential = factories[factories.length - 1]; + + const sas = generateAccountSASQueryParameters( + { + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: AccountSASPermissions.parse("w"), + protocol: SASProtocol.HttpsAndHttp, + resourceTypes: AccountSASResourceTypes.parse("sco").toString(), + services: AccountSASServices.parse("btqf").toString(), + startsOn: now, + version: "2016-05-31" + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ).toString(); + + const sasURL = `${serviceClient.url}?${sas}`; + const serviceClientWithSAS = new BlobServiceClient( + sasURL, + newPipeline(new AnonymousCredential(), { + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const containerClientWithSAS = serviceClientWithSAS.getContainerClient( + getUniqueName("con") + ); + await containerClientWithSAS.create(); + + const blockBlobClientWithSAS = containerClientWithSAS.getBlockBlobClient( + getUniqueName("blob") + ); + await blockBlobClientWithSAS.upload("abc", 3); + + await blockBlobClientWithSAS.setAccessTier("Hot"); + }); + + it("generateAccountSASQueryParameters should not work with invalid permission @Loki @sql", async () => { + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const storageSharedKeyCredential = factories[factories.length - 1]; + + const sas = generateAccountSASQueryParameters( + { + expiresOn: tmr, + permissions: AccountSASPermissions.parse("wdlcup"), + resourceTypes: AccountSASResourceTypes.parse("sco").toString(), + services: AccountSASServices.parse("btqf").toString() + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ).toString(); + + const sasURL = `${serviceClient.url}?${sas}`; + const serviceClientWithSAS = new BlobServiceClient( + sasURL, + newPipeline(new AnonymousCredential(), { + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + let error; + try { + await serviceClientWithSAS.getProperties(); + } catch (err) { + error = err; + } + + assert.ok(error); + }); + + it("generateAccountSASQueryParameters should not work with invalid service @Loki @sql", async () => { + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const storageSharedKeyCredential = factories[factories.length - 1]; + + const sas = generateAccountSASQueryParameters( + { + expiresOn: tmr, + permissions: AccountSASPermissions.parse("rwdlacup"), + resourceTypes: AccountSASResourceTypes.parse("sco").toString(), + services: AccountSASServices.parse("tqf").toString() + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ).toString(); + + const sasURL = `${serviceClient.url}?${sas}`; + const serviceClientWithSAS = new BlobServiceClient( + sasURL, + newPipeline(new AnonymousCredential(), { + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + let error; + try { + await serviceClientWithSAS.getProperties(); + } catch (err) { + error = err; + } + + assert.ok(error); + }); + + it("generateAccountSASQueryParameters should not work with invalid resource type @Loki @sql", async () => { + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const storageSharedKeyCredential = factories[factories.length - 1]; + + const sas = generateAccountSASQueryParameters( + { + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: AccountSASPermissions.parse("rwdlacup"), + protocol: SASProtocol.HttpsAndHttp, + resourceTypes: AccountSASResourceTypes.parse("co").toString(), + services: AccountSASServices.parse("btqf").toString(), + version: "2016-05-31" + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ).toString(); + + const sasURL = `${serviceClient.url}?${sas}`; + const serviceClientWithSAS = new BlobServiceClient( + sasURL, + newPipeline(new AnonymousCredential(), { + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + let error; + try { + await serviceClientWithSAS.getProperties(); + } catch (err) { + error = err; + } + + assert.ok(error); + }); + + it("Synchronized copy blob should work with write permission in account SAS to override an existing blob @Loki @sql", async () => { + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const storageSharedKeyCredential = factories[factories.length - 1]; + + const sas = generateAccountSASQueryParameters( + { + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: AccountSASPermissions.parse("rwdlacup"), + protocol: SASProtocol.HttpsAndHttp, + resourceTypes: AccountSASResourceTypes.parse("co").toString(), + services: AccountSASServices.parse("btqf").toString(), + version: "2016-05-31" + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ).toString(); + + const sasURL = `${serviceClient.url}?${sas}`; + const serviceClientWithSAS = new BlobServiceClient( + sasURL, + newPipeline(new AnonymousCredential(), { + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const containerName = getUniqueName("con"); + const containerClient = + serviceClientWithSAS.getContainerClient(containerName); + await containerClient.create(); + + const blobName1 = getUniqueName("blob"); + const blobName2 = getUniqueName("blob"); + const blob1 = containerClient.getBlockBlobClient(blobName1); + const blob2 = containerClient.getBlockBlobClient(blobName2); + + await blob1.upload("hello", 5); + await blob2.syncCopyFromURL(blob1.url); + + // this copy should not throw any errors + await blob2.syncCopyFromURL(blob1.url); + }); + + it("Synchronized copy blob shouldn't work without write permission in account SAS to override an existing blob @Loki @sql", async () => { + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const storageSharedKeyCredential = factories[factories.length - 1]; + + const sas = generateAccountSASQueryParameters( + { + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: AccountSASPermissions.parse("rdlacup"), + protocol: SASProtocol.HttpsAndHttp, + resourceTypes: AccountSASResourceTypes.parse("co").toString(), + services: AccountSASServices.parse("btqf").toString(), + version: "2016-05-31" + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ).toString(); + + const sasURL = `${serviceClient.url}?${sas}`; + const serviceClientWithSAS = new BlobServiceClient( + sasURL, + newPipeline(new AnonymousCredential(), { + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const containerName = getUniqueName("con"); + const containerClient = + serviceClientWithSAS.getContainerClient(containerName); + await containerClient.create(); + + const blobName1 = getUniqueName("blob"); + const blobName2 = getUniqueName("blob"); + const blob1 = containerClient.getBlockBlobClient(blobName1); + const blob2 = containerClient.getBlockBlobClient(blobName2); + + await blob1.upload("hello", 5); + await blob2.syncCopyFromURL(blob1.url); + + // this copy should throw 403 error + let error; + try { + await blob2.syncCopyFromURL(blob1.url); + } catch (err) { + error = err; + } + assert.deepEqual(error.statusCode, 403); + assert.ok(error !== undefined); + }); + + it("Synchronized copy blob should work without write permission in account SAS to an nonexisting blob @Loki @sql", async () => { + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const storageSharedKeyCredential = factories[factories.length - 1]; + + const sas = generateAccountSASQueryParameters( + { + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: AccountSASPermissions.parse("c"), + protocol: SASProtocol.HttpsAndHttp, + resourceTypes: AccountSASResourceTypes.parse("co").toString(), + services: AccountSASServices.parse("btqf").toString(), + version: "2016-05-31" + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ).toString(); + + const sasURL = `${serviceClient.url}?${sas}`; + const serviceClientWithSAS = new BlobServiceClient( + sasURL, + newPipeline(new AnonymousCredential(), { + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const containerName = getUniqueName("con"); + const containerClient = + serviceClientWithSAS.getContainerClient(containerName); + await containerClient.create(); + + const blobName1 = getUniqueName("blob"); + const blobName2 = getUniqueName("blob"); + const blob1 = containerClient.getBlockBlobClient(blobName1); + const blob2 = containerClient.getBlockBlobClient(blobName2); + + await blob1.upload("hello", 5); + + // this copy should work + await blob2.syncCopyFromURL(blob1.url); + }); + + it("Copy blob should work with write permission in account SAS to override an existing blob @Loki @sql", async () => { + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const storageSharedKeyCredential = factories[factories.length - 1]; + + const sas = generateAccountSASQueryParameters( + { + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: AccountSASPermissions.parse("rwdlacup"), + protocol: SASProtocol.HttpsAndHttp, + resourceTypes: AccountSASResourceTypes.parse("co").toString(), + services: AccountSASServices.parse("btqf").toString(), + version: "2016-05-31" + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ).toString(); + + const sasURL = `${serviceClient.url}?${sas}`; + const serviceClientWithSAS = new BlobServiceClient( + sasURL, + newPipeline(new AnonymousCredential(), { + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const containerName = getUniqueName("con"); + const containerClient = + serviceClientWithSAS.getContainerClient(containerName); + await containerClient.create(); + + const blobName1 = getUniqueName("blob"); + const blobName2 = getUniqueName("blob"); + const blob1 = containerClient.getBlockBlobClient(blobName1); + const blob2 = containerClient.getBlockBlobClient(blobName2); + + await blob1.upload("hello", 5); + await blob2.beginCopyFromURL(blob1.url); + + // this copy should not throw any errors + await blob2.beginCopyFromURL(blob1.url); + }); + + it("Copy blob shouldn't work without write permission in account SAS to override an existing blob @Loki @sql", async () => { + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const storageSharedKeyCredential = factories[factories.length - 1]; + + const sas = generateAccountSASQueryParameters( + { + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: AccountSASPermissions.parse("rdlacup"), + protocol: SASProtocol.HttpsAndHttp, + resourceTypes: AccountSASResourceTypes.parse("co").toString(), + services: AccountSASServices.parse("btqf").toString(), + version: "2016-05-31" + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ).toString(); + + const sasURL = `${serviceClient.url}?${sas}`; + const serviceClientWithSAS = new BlobServiceClient( + sasURL, + newPipeline(new AnonymousCredential(), { + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const containerName = getUniqueName("con"); + const containerClient = + serviceClientWithSAS.getContainerClient(containerName); + await containerClient.create(); + + const blobName1 = getUniqueName("blob"); + const blobName2 = getUniqueName("blob"); + const blob1 = containerClient.getBlockBlobClient(blobName1); + const blob2 = containerClient.getBlockBlobClient(blobName2); + + await blob1.upload("hello", 5); + await blob2.beginCopyFromURL(blob1.url); + + // this copy should throw 403 error + let error; + try { + await blob2.beginCopyFromURL(blob1.url); + } catch (err) { + error = err; + } + assert.deepEqual(error.statusCode, 403); + assert.ok(error !== undefined); + }); + + it("Copy blob should work without write permission in account SAS to an nonexisting blob @Loki @sql", async () => { + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const storageSharedKeyCredential = factories[factories.length - 1]; + + const sas = generateAccountSASQueryParameters( + { + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: AccountSASPermissions.parse("c"), + protocol: SASProtocol.HttpsAndHttp, + resourceTypes: AccountSASResourceTypes.parse("co").toString(), + services: AccountSASServices.parse("btqf").toString(), + version: "2016-05-31" + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ).toString(); + + const sasURL = `${serviceClient.url}?${sas}`; + const serviceClientWithSAS = new BlobServiceClient( + sasURL, + newPipeline(new AnonymousCredential(), { + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const containerName = getUniqueName("con"); + const containerClient = + serviceClientWithSAS.getContainerClient(containerName); + await containerClient.create(); + + const blobName1 = getUniqueName("blob"); + const blobName2 = getUniqueName("blob"); + const blob1 = containerClient.getBlockBlobClient(blobName1); + const blob2 = containerClient.getBlockBlobClient(blobName2); + + await blob1.upload("hello", 5); + + // this copy should work + await blob2.beginCopyFromURL(blob1.url); + }); + + it("generateBlobSASQueryParameters should work for container @Loki @sql", async () => { + const now = new Date(); + now.setMinutes(now.getMinutes() - 5); // Skip clock skew with server + + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const storageSharedKeyCredential = factories[factories.length - 1]; + + const containerName = getUniqueName("container"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const containerSAS = generateBlobSASQueryParameters( + { + containerName, + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: ContainerSASPermissions.parse("racwdl"), + protocol: SASProtocol.HttpsAndHttp, + startsOn: now, + version: "2016-05-31" + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ); + + const sasURL = `${containerClient.url}?${containerSAS}`; + const containerClientWithSAS = new ContainerClient( + sasURL, + newPipeline(new AnonymousCredential()) + ); + + await await containerClientWithSAS.listBlobsFlat().byPage(); + await containerClient.delete(); + }); + + it("generateBlobSASQueryParameters should work for page blob with original headers @Loki @sql", async () => { + const now = new Date(); + now.setMinutes(now.getMinutes() - 5); // Skip clock skew with server + + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const storageSharedKeyCredential = factories[factories.length - 1]; + + const containerName = getUniqueName("container"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const blobName = getUniqueName("blob"); + const blobClient = containerClient.getPageBlobClient(blobName); + await blobClient.create(1024, { + blobHTTPHeaders: { + blobCacheControl: "cache-control-original", + blobContentType: "content-type-original", + blobContentDisposition: "content-disposition-original", + blobContentEncoding: "content-encoding-original", + blobContentLanguage: "content-language-original" + } + }); + + const blobSAS = generateBlobSASQueryParameters( + { + blobName, + containerName, + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: BlobSASPermissions.parse("racwd"), + protocol: SASProtocol.HttpsAndHttp, + startsOn: now, + version: "2016-05-31" + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ); + + const sasURL = `${blobClient.url}?${blobSAS}`; + const blobClientWithSAS = new PageBlobClient( + sasURL, + newPipeline(new AnonymousCredential()) + ); + + await blobClientWithSAS.getProperties(); + + const properties = await blobClientWithSAS.getProperties(); + assert.equal(properties.cacheControl, "cache-control-original"); + assert.equal(properties.contentDisposition, "content-disposition-original"); + assert.equal(properties.contentEncoding, "content-encoding-original"); + assert.equal(properties.contentLanguage, "content-language-original"); + assert.equal(properties.contentType, "content-type-original"); + + const downloadResponse = await blobClientWithSAS.download(); + assert.equal(downloadResponse.cacheControl, "cache-control-original"); + assert.equal( + downloadResponse.contentDisposition, + "content-disposition-original" + ); + assert.equal(downloadResponse.contentEncoding, "content-encoding-original"); + assert.equal(downloadResponse.contentLanguage, "content-language-original"); + assert.equal(downloadResponse.contentType, "content-type-original"); + + await containerClient.delete(); + }); + + it("generateBlobSASQueryParameters should work for page blob and rscd arguments for filenames with spaces and special characters @Loki @sql", async () => { + const now = new Date(); + now.setMinutes(now.getMinutes() - 5); // Skip clock skew with server + + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const storageSharedKeyCredential = factories[factories.length - 1]; + + const containerName = getUniqueName("container"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const blobName = "this is a test file Ж 大 仮.jpg"; //filename contains spaces and special characters + const blobClient = containerClient.getPageBlobClient(blobName); + await blobClient.create(1024, { + blobHTTPHeaders: { + blobCacheControl: "cache-control-original", + blobContentType: "content-type-original", + blobContentDisposition: "content-type-disposition", + blobContentEncoding: "content-encoding-original", + blobContentLanguage: "content-language-original" + } + }); + + const escapedblobName = encodeURIComponent(blobName); + const blobSAS = generateBlobSASQueryParameters( + { + blobName, + containerName, + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: BlobSASPermissions.parse("racwd"), + protocol: SASProtocol.HttpsAndHttp, + //https://tools.ietf.org/html/rfc5987 + contentDisposition: `attachment; filename=\"${escapedblobName}\"; filename*=UTF-8''${escapedblobName}`, + startsOn: now, + version: "2016-05-31" + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ); + + const sasURL = `${blobClient.url}?${blobSAS}`; + const blobClientWithSAS = new PageBlobClient( + sasURL, + newPipeline(new AnonymousCredential()) + ); + + await blobClientWithSAS.getProperties(); + + const properties = await blobClientWithSAS.getProperties(); + assert.equal(properties.cacheControl, "cache-control-original"); + assert.equal( + properties.contentDisposition, + `attachment; filename=\"${escapedblobName}\"; filename*=UTF-8''${escapedblobName}` + ); + assert.equal(properties.contentEncoding, "content-encoding-original"); + assert.equal(properties.contentLanguage, "content-language-original"); + assert.equal(properties.contentType, "content-type-original"); + + const downloadResponse = await blobClientWithSAS.download(); + assert.equal(downloadResponse.cacheControl, "cache-control-original"); + assert.equal( + downloadResponse.contentDisposition, + `attachment; filename=\"${escapedblobName}\"; filename*=UTF-8''${escapedblobName}` + ); + assert.equal(downloadResponse.contentEncoding, "content-encoding-original"); + assert.equal(downloadResponse.contentLanguage, "content-language-original"); + assert.equal(downloadResponse.contentType, "content-type-original"); + + await containerClient.delete(); + }); + + it("generateBlobSASQueryParameters should work for page blob and override headers @Loki @sql", async () => { + const now = new Date(); + now.setMinutes(now.getMinutes() - 5); // Skip clock skew with server + + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const storageSharedKeyCredential = factories[factories.length - 1]; + + const containerName = getUniqueName("container"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const blobName = getUniqueName("blob"); + const blobClient = containerClient.getPageBlobClient(blobName); + await blobClient.create(1024, { + blobHTTPHeaders: { + blobContentType: "content-type-original" + } + }); + + const blobSAS = generateBlobSASQueryParameters( + { + blobName, + cacheControl: "cache-control-override", + containerName, + contentDisposition: "content-disposition-override", + contentEncoding: "content-encoding-override", + contentLanguage: "content-language-override", + contentType: "content-type-override", + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: BlobSASPermissions.parse("racwd"), + protocol: SASProtocol.HttpsAndHttp, + startsOn: now, + version: "2016-05-31" + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ); + + const sasURL = `${blobClient.url}?${blobSAS}`; + const blobClientWithSAS = new PageBlobClient( + sasURL, + newPipeline(new AnonymousCredential()) + ); + + await blobClientWithSAS.getProperties(); + + const properties = await blobClientWithSAS.getProperties(); + assert.equal(properties.cacheControl, "cache-control-override"); + assert.equal(properties.contentDisposition, "content-disposition-override"); + assert.equal(properties.contentEncoding, "content-encoding-override"); + assert.equal(properties.contentLanguage, "content-language-override"); + assert.equal(properties.contentType, "content-type-override"); + + const downloadResponse = await blobClientWithSAS.download(); + assert.equal(downloadResponse.cacheControl, "cache-control-override"); + assert.equal( + downloadResponse.contentDisposition, + "content-disposition-override" + ); + assert.equal(downloadResponse.contentEncoding, "content-encoding-override"); + assert.equal(downloadResponse.contentLanguage, "content-language-override"); + assert.equal(downloadResponse.contentType, "content-type-override"); + + await containerClient.delete(); + }); + + it("generateBlobSASQueryParameters should work for append blob with original headers @Loki @sql", async () => { + const now = new Date(); + now.setMinutes(now.getMinutes() - 5); // Skip clock skew with server + + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const storageSharedKeyCredential = factories[factories.length - 1]; + + const containerName = getUniqueName("container"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const blobName = getUniqueName("blob"); + const blobClient = containerClient.getAppendBlobClient(blobName); + await blobClient.create({ + blobHTTPHeaders: { + blobCacheControl: "cache-control-original", + blobContentType: "content-type-original", + blobContentDisposition: "content-disposition-original", + blobContentEncoding: "content-encoding-original", + blobContentLanguage: "content-language-original" + } + }); + + const blobSAS = generateBlobSASQueryParameters( + { + blobName, + containerName, + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: BlobSASPermissions.parse("racwd"), + protocol: SASProtocol.HttpsAndHttp, + startsOn: now, + version: "2016-05-31" + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ); + + const sasURL = `${blobClient.url}?${blobSAS}`; + const blobClientWithSAS = new AppendBlobClient( + sasURL, + newPipeline(new AnonymousCredential()) + ); + + await blobClientWithSAS.getProperties(); + + const properties = await blobClientWithSAS.getProperties(); + assert.equal(properties.cacheControl, "cache-control-original"); + assert.equal(properties.contentDisposition, "content-disposition-original"); + assert.equal(properties.contentEncoding, "content-encoding-original"); + assert.equal(properties.contentLanguage, "content-language-original"); + assert.equal(properties.contentType, "content-type-original"); + + const downloadResponse = await blobClientWithSAS.download(); + assert.equal(downloadResponse.cacheControl, "cache-control-original"); + assert.equal( + downloadResponse.contentDisposition, + "content-disposition-original" + ); + assert.equal(downloadResponse.contentEncoding, "content-encoding-original"); + assert.equal(downloadResponse.contentLanguage, "content-language-original"); + assert.equal(downloadResponse.contentType, "content-type-original"); + + await containerClient.delete(); + }); + + it("generateBlobSASQueryParameters should work for append blob and override headers @Loki @sql", async () => { + const now = new Date(); + now.setMinutes(now.getMinutes() - 5); // Skip clock skew with server + + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const storageSharedKeyCredential = factories[factories.length - 1]; + + const containerName = getUniqueName("container"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const blobName = getUniqueName("blob"); + const blobClient = containerClient.getAppendBlobClient(blobName); + await blobClient.create({ + blobHTTPHeaders: { + blobContentType: "content-type-original" + } + }); + + const blobSAS = generateBlobSASQueryParameters( + { + blobName, + cacheControl: "cache-control-override", + containerName, + contentDisposition: "content-disposition-override", + contentEncoding: "content-encoding-override", + contentLanguage: "content-language-override", + contentType: "content-type-override", + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: BlobSASPermissions.parse("racwd"), + protocol: SASProtocol.HttpsAndHttp, + startsOn: now, + version: "2016-05-31" + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ); + + const sasURL = `${blobClient.url}?${blobSAS}`; + const blobClientWithSAS = new AppendBlobClient( + sasURL, + newPipeline(new AnonymousCredential()) + ); + + await blobClientWithSAS.getProperties(); + + const properties = await blobClientWithSAS.getProperties(); + assert.equal(properties.cacheControl, "cache-control-override"); + assert.equal(properties.contentDisposition, "content-disposition-override"); + assert.equal(properties.contentEncoding, "content-encoding-override"); + assert.equal(properties.contentLanguage, "content-language-override"); + assert.equal(properties.contentType, "content-type-override"); + + const downloadResponse = await blobClientWithSAS.download(); + assert.equal(downloadResponse.cacheControl, "cache-control-override"); + assert.equal( + downloadResponse.contentDisposition, + "content-disposition-override" + ); + assert.equal(downloadResponse.contentEncoding, "content-encoding-override"); + assert.equal(downloadResponse.contentLanguage, "content-language-override"); + assert.equal(downloadResponse.contentType, "content-type-override"); + + await containerClient.delete(); + }); + + it("generateBlobSASQueryParameters should work for blob with special naming @Loki @sql", async () => { + const now = new Date(); + now.setMinutes(now.getMinutes() - 5); // Skip clock skew with server + + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const storageSharedKeyCredential = factories[factories.length - 1]; + + const containerName = getUniqueName("container-with-dash"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const blobName = getUniqueName( + // tslint:disable-next-line:max-line-length + "////Upper/blob/empty /another 汉字 ру́сский язы́к ру́сский язы́к عربي/عربى にっぽんご/にほんご . special ~!@#$%^&*()_+`1234567890-={}|[]\\:\";'<>?,/'" + ); + const blobClient = containerClient.getPageBlobClient(blobName); + await blobClient.create(1024, { + blobHTTPHeaders: { + blobContentType: "content-type-original" + } + }); + + const blobSAS = generateBlobSASQueryParameters( + { + // blobName, + // //Caused an issue + // NOTICE: Azure Storage Server will replace "\" with "/" in the blob names + blobName: blobName.replace(/\\/g, "/"), + cacheControl: "cache-control-override", + containerName, + contentDisposition: "content-disposition-override", + contentEncoding: "content-encoding-override", + contentLanguage: "content-language-override", + contentType: "content-type-override", + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: BlobSASPermissions.parse("racwd"), + protocol: SASProtocol.HttpsAndHttp, + startsOn: now, + version: "2016-05-31" + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ); + + const sasURL = `${blobClient.url}?${blobSAS}`; + const blobClientWithSAS = new PageBlobClient( + sasURL, + newPipeline(new AnonymousCredential()) + ); + + await blobClientWithSAS.getProperties(); + + const properties = await blobClientWithSAS.getProperties(); + assert.equal(properties.cacheControl, "cache-control-override"); + assert.equal(properties.contentDisposition, "content-disposition-override"); + assert.equal(properties.contentEncoding, "content-encoding-override"); + assert.equal(properties.contentLanguage, "content-language-override"); + assert.equal(properties.contentType, "content-type-override"); + + const downloadResponse = await blobClientWithSAS.download(); + assert.equal(downloadResponse.cacheControl, "cache-control-override"); + assert.equal( + downloadResponse.contentDisposition, + "content-disposition-override" + ); + assert.equal(downloadResponse.contentEncoding, "content-encoding-override"); + assert.equal(downloadResponse.contentLanguage, "content-language-override"); + assert.equal(downloadResponse.contentType, "content-type-override"); + + await containerClient.delete(); + }); + + it("generateBlobSASQueryParameters should work for blob with access policy @Loki @sql", async () => { + const now = new Date(); + now.setMinutes(now.getMinutes() - 5); // Skip clock skew with server + + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const storageSharedKeyCredential = factories[factories.length - 1]; + + const containerName = getUniqueName("container"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const blobName = getUniqueName("blob"); + const blobClient = containerClient.getPageBlobClient(blobName); + await blobClient.create(1024); + + const id = "unique-id"; + const result = await containerClient.setAccessPolicy(undefined, [ + { + accessPolicy: { + expiresOn: tmr, + permissions: ContainerSASPermissions.parse("racwdl").toString(), + startsOn: now + }, + id + } + ]); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + + const blobSAS = generateBlobSASQueryParameters( + { + containerName, + identifier: id + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ); + + const sasURL = `${blobClient.url}?${blobSAS}`; + const blobClientWithSAS = new PageBlobClient( + sasURL, + newPipeline(new AnonymousCredential()) + ); + + await blobClientWithSAS.getProperties(); + await containerClient.delete(); + }); + + it("Synchronized copy blob should work with write permission in blob SAS to override an existing blob @Loki @sql", async () => { + const now = new Date(); + now.setMinutes(now.getMinutes() - 5); // Skip clock skew with server + + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const storageSharedKeyCredential = factories[factories.length - 1]; + + const containerName = getUniqueName("container"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const containerSAS = generateBlobSASQueryParameters( + { + containerName, + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: ContainerSASPermissions.parse("w"), + protocol: SASProtocol.HttpsAndHttp, + startsOn: now, + version: "2016-05-31" + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ); + + const sasURL = `${containerClient.url}?${containerSAS}`; + const containerClientWithSAS = new ContainerClient( + sasURL, + newPipeline(new AnonymousCredential()) + ); + + const blobName1 = getUniqueName("blob"); + const blobName2 = getUniqueName("blob"); + const blob1 = containerClient.getBlockBlobClient(blobName1); + const blob2 = containerClient.getBlockBlobClient(blobName2); + const blob2SAS = containerClientWithSAS.getBlockBlobClient(blobName2); + + await blob1.upload("hello", 5); + await blob2.upload("world", 5); + + // this copy should not throw any errors + await blob2SAS.syncCopyFromURL(blob1.url); + }); + + it("Synchronized copy blob shouldn't work without write permission in blob SAS to override an existing blob @Loki @sql", async () => { + const now = new Date(); + now.setMinutes(now.getMinutes() - 5); // Skip clock skew with server + + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const storageSharedKeyCredential = factories[factories.length - 1]; + + const containerName = getUniqueName("container"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const containerSAS = generateBlobSASQueryParameters( + { + containerName, + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: ContainerSASPermissions.parse("c"), + protocol: SASProtocol.HttpsAndHttp, + startsOn: now, + version: "2016-05-31" + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ); + + const sasURL = `${containerClient.url}?${containerSAS}`; + const containerClientWithSAS = new ContainerClient( + sasURL, + newPipeline(new AnonymousCredential()) + ); + + const blobName1 = getUniqueName("blob"); + const blobName2 = getUniqueName("blob"); + const blob1 = containerClient.getBlockBlobClient(blobName1); + const blob2 = containerClient.getBlockBlobClient(blobName2); + const blob2SAS = containerClientWithSAS.getBlockBlobClient(blobName2); + + await blob1.upload("hello", 5); + await blob2.upload("world", 5); + + // this copy should throw 403 error + let error; + try { + await blob2SAS.syncCopyFromURL(blob1.url); + } catch (err) { + error = err; + } + assert.deepEqual(error.statusCode, 403); + assert.ok(error !== undefined); + }); + + it("Synchronized copy blob should work without write permission in account SAS to an nonexisting blob @Loki @sql", async () => { + const now = new Date(); + now.setMinutes(now.getMinutes() - 5); // Skip clock skew with server + + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const storageSharedKeyCredential = factories[factories.length - 1]; + + const containerName = getUniqueName("container"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const containerSAS = generateBlobSASQueryParameters( + { + containerName, + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: ContainerSASPermissions.parse("c"), + protocol: SASProtocol.HttpsAndHttp, + startsOn: now, + version: "2016-05-31" + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ); + + const sasURL = `${containerClient.url}?${containerSAS}`; + const containerClientWithSAS = new ContainerClient( + sasURL, + newPipeline(new AnonymousCredential()) + ); + + const blobName1 = getUniqueName("blob"); + const blobName2 = getUniqueName("blob"); + const blob1 = containerClient.getBlockBlobClient(blobName1); + const blob2SAS = containerClientWithSAS.getBlockBlobClient(blobName2); + + await blob1.upload("hello", 5); + + // this copy should work + await blob2SAS.syncCopyFromURL(blob1.url); + }); + + it("Copy blob should work with write permission in blob SAS to override an existing blob @Loki @sql", async () => { + const now = new Date(); + now.setMinutes(now.getMinutes() - 5); // Skip clock skew with server + + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const storageSharedKeyCredential = factories[factories.length - 1]; + + const containerName = getUniqueName("container"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const containerSAS = generateBlobSASQueryParameters( + { + containerName, + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: ContainerSASPermissions.parse("w"), + protocol: SASProtocol.HttpsAndHttp, + startsOn: now, + version: "2016-05-31" + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ); + + const sasURL = `${containerClient.url}?${containerSAS}`; + const containerClientWithSAS = new ContainerClient( + sasURL, + newPipeline(new AnonymousCredential()) + ); + + const blobName1 = getUniqueName("blob"); + const blobName2 = getUniqueName("blob"); + const blob1 = containerClient.getBlockBlobClient(blobName1); + const blob2 = containerClient.getBlockBlobClient(blobName2); + const blob2SAS = containerClientWithSAS.getBlockBlobClient(blobName2); + + await blob1.upload("hello", 5); + await blob2.upload("world", 5); + + // this copy should not throw any errors + await blob2SAS.beginCopyFromURL(blob1.url); + }); + + it("Copy blob shouldn't work without write permission in blob SAS to override an existing blob @Loki @sql", async () => { + const now = new Date(); + now.setMinutes(now.getMinutes() - 5); // Skip clock skew with server + + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const storageSharedKeyCredential = factories[factories.length - 1]; + + const containerName = getUniqueName("container"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const containerSAS = generateBlobSASQueryParameters( + { + containerName, + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: ContainerSASPermissions.parse("c"), + protocol: SASProtocol.HttpsAndHttp, + startsOn: now, + version: "2016-05-31" + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ); + + const sasURL = `${containerClient.url}?${containerSAS}`; + const containerClientWithSAS = new ContainerClient( + sasURL, + newPipeline(new AnonymousCredential()) + ); + + const blobName1 = getUniqueName("blob"); + const blobName2 = getUniqueName("blob"); + const blob1 = containerClient.getBlockBlobClient(blobName1); + const blob2 = containerClient.getBlockBlobClient(blobName2); + const blob2SAS = containerClientWithSAS.getBlockBlobClient(blobName2); + + await blob1.upload("hello", 5); + await blob2.upload("world", 5); + + // this copy should throw 403 error + let error; + try { + await blob2SAS.beginCopyFromURL(blob1.url); + } catch (err) { + error = err; + } + assert.deepEqual(error.statusCode, 403); + assert.ok(error !== undefined); + }); + + it("Copy blob should work without write permission in account SAS to an nonexisting blob @Loki @sql", async () => { + const now = new Date(); + now.setMinutes(now.getMinutes() - 5); // Skip clock skew with server + + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const storageSharedKeyCredential = factories[factories.length - 1]; + + const containerName = getUniqueName("container"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const containerSAS = generateBlobSASQueryParameters( + { + containerName, + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: ContainerSASPermissions.parse("c"), + protocol: SASProtocol.HttpsAndHttp, + startsOn: now, + version: "2016-05-31" + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ); + + const sasURL = `${containerClient.url}?${containerSAS}`; + const containerClientWithSAS = new ContainerClient( + sasURL, + newPipeline(new AnonymousCredential()) + ); + + const blobName1 = getUniqueName("blob"); + const blobName2 = getUniqueName("blob"); + const blob1 = containerClient.getBlockBlobClient(blobName1); + const blob2SAS = containerClientWithSAS.getBlockBlobClient(blobName2); + + await blob1.upload("hello", 5); + + // this copy should work + await blob2SAS.beginCopyFromURL(blob1.url); + }); + + it("GenerateUserDelegationSAS should work for blob snapshot @Loki @sql", async () => { + const now = new Date(); + now.setMinutes(now.getMinutes() - 5); // Skip clock skew with server + + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const storageSharedKeyCredential = factories[factories.length - 1]; + + const containerName = getUniqueName("container"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const blobName = getUniqueName("blob"); + const blobClient = containerClient.getPageBlobClient(blobName); + await blobClient.create(1024, { + blobHTTPHeaders: { + blobContentType: "content-type-original" + } + }); + const response = await blobClient.createSnapshot(); + + const blobSAS = generateBlobSASQueryParameters( + { + blobName, + cacheControl: "cache-control-override", + containerName, + contentDisposition: "content-disposition-override", + contentEncoding: "content-encoding-override", + contentLanguage: "content-language-override", + contentType: "content-type-override", + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: BlobSASPermissions.parse("racwd"), + protocol: SASProtocol.HttpsAndHttp, + startsOn: now, + snapshotTime: response.snapshot + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ); + + const sasURL = `${ + blobClient.withSnapshot(response.snapshot!).url + }&${blobSAS}`; + const blobClientWithSAS = new PageBlobClient( + sasURL, + newPipeline(new AnonymousCredential()) + ); + + const properties = await blobClientWithSAS.getProperties(); + assert.ok(properties); + assert.equal(properties.cacheControl, "cache-control-override"); + assert.equal(properties.contentDisposition, "content-disposition-override"); + assert.equal(properties.contentEncoding, "content-encoding-override"); + assert.equal(properties.contentLanguage, "content-language-override"); + assert.equal(properties.contentType, "content-type-override"); + + const downloadResponse = await blobClientWithSAS.download(); + assert.equal(downloadResponse.cacheControl, "cache-control-override"); + assert.equal( + downloadResponse.contentDisposition, + "content-disposition-override" + ); + assert.equal(downloadResponse.contentEncoding, "content-encoding-override"); + assert.equal(downloadResponse.contentLanguage, "content-language-override"); + assert.equal(downloadResponse.contentType, "content-type-override"); + + await containerClient.delete(); + }); + + it("Copy blob across accounts should require SAS token @Loki @sql", async () => { + const now = new Date(); + now.setMinutes(now.getMinutes() - 5); // Skip clock skew with server + + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const sourceStorageSharedKeyCredential = factories[factories.length - 1]; + + const containerName = getUniqueName("con"); + const sourceContainerClient = + serviceClient.getContainerClient(containerName); + const targetContainerClient = + serviceClient2.getContainerClient(containerName); + await sourceContainerClient.create(); + await targetContainerClient.create(); + + const blobName = getUniqueName("blob"); + const sas = generateBlobSASQueryParameters( + { + containerName, + blobName, + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: BlobSASPermissions.parse("r"), + protocol: SASProtocol.HttpsAndHttp, + startsOn: now, + version: "2016-05-31" + }, + sourceStorageSharedKeyCredential as StorageSharedKeyCredential + ).toString(); + + const sourceBlob = sourceContainerClient.getBlockBlobClient(blobName); + await sourceBlob.upload("hello", 5); + + const targetBlob = targetContainerClient.getBlockBlobClient(blobName); + + let error; + try { + await targetBlob.beginCopyFromURL(sourceBlob.url); + } catch (err) { + error = err; + } + assert.ok(error !== undefined); + assert.equal(error.statusCode, 403); + assert.equal(error.details.errorCode, "CannotVerifyCopySource"); + assert.equal(error.details.code, "CannotVerifyCopySource"); + + // this copy should work + const operation = await targetBlob.beginCopyFromURL( + `${sourceBlob.url}?${sas}` + ); + const copyResponse = await operation.pollUntilDone(); + assert.equal("success", copyResponse.copyStatus); + const fileBuffer = await targetBlob.downloadToBuffer(); + assert.equal(fileBuffer.toString(), "hello"); + }); + + it("Copy blob across accounts should error if hosts mismatch @Loki @sql", async () => { + const now = new Date(); + now.setMinutes(now.getMinutes() - 5); // Skip clock skew with server + + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + const containerName = getUniqueName("con"); + const sourceContainerClient = + serviceClient.getContainerClient(containerName); + const targetContainerClient = + serviceClient2.getContainerClient(containerName); + await sourceContainerClient.create(); + await targetContainerClient.create(); + + const blobName = getUniqueName("blob"); + const sourceBlob = sourceContainerClient.getBlockBlobClient(blobName); + await sourceBlob.upload("hello", 5); + + const targetBlob = targetContainerClient.getBlockBlobClient(blobName); + const sourceUriBuilder = URLBuilder.parse(sourceBlob.url); + sourceUriBuilder.setHost("somewhereelse"); + + let error; + try { + await targetBlob.beginCopyFromURL(sourceUriBuilder.toString()); + } catch (err) { + error = err; + } + assert.deepEqual(error.statusCode, 404); + assert.ok(error !== undefined); + }); + + it("Copy blob across accounts should succeed for public blob access @Loki @sql", async () => { + const now = new Date(); + now.setMinutes(now.getMinutes() - 5); // Skip clock skew with server + + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + const containerName = getUniqueName("con"); + const sourceContainerClient = + serviceClient.getContainerClient(containerName); + const targetContainerClient = + serviceClient2.getContainerClient(containerName); + await sourceContainerClient.create({ + access: "blob" + }); + await targetContainerClient.create(); + + const blobName = getUniqueName("blob"); + const sourceBlob = sourceContainerClient.getBlockBlobClient(blobName); + await sourceBlob.upload("hello", 5); + + const targetBlob = targetContainerClient.getBlockBlobClient(blobName); + const operation = await targetBlob.beginCopyFromURL(sourceBlob.url); + const copyResponse = await operation.pollUntilDone(); + assert.equal("success", copyResponse.copyStatus); + const fileBuffer = await targetBlob.downloadToBuffer(); + assert.equal(fileBuffer.toString(), "hello"); + }); + + it("Copy blob across accounts should succeed for public container access @Loki @sql", async () => { + const now = new Date(); + now.setMinutes(now.getMinutes() - 5); // Skip clock skew with server + + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + const containerName = getUniqueName("con"); + const sourceContainerClient = + serviceClient.getContainerClient(containerName); + const targetContainerClient = + serviceClient2.getContainerClient(containerName); + await sourceContainerClient.create({ + access: "container" + }); + await targetContainerClient.create(); + + const blobName = getUniqueName("blob"); + const sourceBlob = sourceContainerClient.getBlockBlobClient(blobName); + await sourceBlob.upload("hello", 5); + + const targetBlob = targetContainerClient.getBlockBlobClient(blobName); + const operation = await targetBlob.beginCopyFromURL(sourceBlob.url); + const copyResponse = await operation.pollUntilDone(); + assert.equal("success", copyResponse.copyStatus); + const fileBuffer = await targetBlob.downloadToBuffer(); + assert.equal(fileBuffer.toString(), "hello"); + }); + + it("Copy blob across accounts should honor metadata when provided @Loki @sql", async () => { + const now = new Date(); + now.setMinutes(now.getMinutes() - 5); // Skip clock skew with server + + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const sourceStorageSharedKeyCredential = factories[factories.length - 1]; + + const containerName = getUniqueName("con"); + const sourceContainerClient = + serviceClient.getContainerClient(containerName); + const targetContainerClient = + serviceClient2.getContainerClient(containerName); + await sourceContainerClient.create(); + await targetContainerClient.create(); + + const blobName = getUniqueName("blob"); + const blobName2 = getUniqueName("blob"); + const sas = generateBlobSASQueryParameters( + { + containerName, + blobName, + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: BlobSASPermissions.parse("r"), + protocol: SASProtocol.HttpsAndHttp, + startsOn: now, + version: "2016-05-31" + }, + sourceStorageSharedKeyCredential as StorageSharedKeyCredential + ).toString(); + + const sourceBlob = sourceContainerClient.getBlockBlobClient(blobName); + await sourceBlob.upload("hello", 5, { + metadata: { + foo: "1", + bar: "2" + } + }); + + const targetBlob = targetContainerClient.getBlockBlobClient(blobName); + const operation = await targetBlob.beginCopyFromURL( + `${sourceBlob.url}?${sas}` + ); + const copyResponse = await operation.pollUntilDone(); + assert.equal("success", copyResponse.copyStatus); + const properties = await targetBlob.getProperties(); + assert.equal(properties.metadata!["foo"], "1"); + assert.equal(properties.metadata!["bar"], "2"); + + const targetBlobWithProps = + targetContainerClient.getBlockBlobClient(blobName2); + const operation2 = await targetBlobWithProps.beginCopyFromURL( + `${sourceBlob.url}?${sas}`, + { + metadata: { + baz: "3" + } + } + ); + + const copyResponse2 = await operation2.pollUntilDone(); + assert.equal("success", copyResponse2.copyStatus); + const properties2 = await targetBlobWithProps.getProperties(); + assert.equal(properties2.metadata!["foo"], undefined); + assert.equal(properties2.metadata!["bar"], undefined); + assert.equal(properties2.metadata!["baz"], "3"); + }); + + //flaky in sql + it("Copy blob across accounts should fail if source is archived @Loki", async () => { + const now = new Date(); + now.setMinutes(now.getMinutes() - 5); // Skip clock skew with server + + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const sourceStorageSharedKeyCredential = factories[factories.length - 1]; + + const containerName = getUniqueName("con"); + const sourceContainerClient = + serviceClient.getContainerClient(containerName); + const targetContainerClient = + serviceClient2.getContainerClient(containerName); + await sourceContainerClient.create(); + await targetContainerClient.create(); + + const blobName = getUniqueName("blob"); + const sas = generateBlobSASQueryParameters( + { + containerName, + blobName, + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: BlobSASPermissions.parse("r"), + protocol: SASProtocol.HttpsAndHttp, + startsOn: now, + version: "2016-05-31" + }, + sourceStorageSharedKeyCredential as StorageSharedKeyCredential + ).toString(); + + const sourceBlob = sourceContainerClient.getBlockBlobClient(blobName); + await sourceBlob.upload("hello", 5); + sourceBlob.setAccessTier("Archive"); + + const targetBlob = targetContainerClient.getBlockBlobClient(blobName); + + let error; + try { + await targetBlob.beginCopyFromURL(`${sourceBlob.url}?${sas}`); + } catch (err) { + error = err; + } + assert.ok(error !== undefined); + assert.equal(error.statusCode, 409); + assert.equal(error.details.code, "BlobArchived"); + }); + + it("Sync Copy blob across accounts should work and honor metadata when provided @Loki @sql", async () => { + const now = new Date(); + now.setMinutes(now.getMinutes() - 5); // Skip clock skew with server + + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const sourceStorageSharedKeyCredential = factories[factories.length - 1]; + + const containerName = getUniqueName("con"); + const sourceContainerClient = + serviceClient.getContainerClient(containerName); + const targetContainerClient = + serviceClient2.getContainerClient(containerName); + await sourceContainerClient.create(); + await targetContainerClient.create(); + + const blobName = getUniqueName("blob"); + const blobName2 = getUniqueName("blob"); + const sas = generateBlobSASQueryParameters( + { + containerName, + blobName, + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: BlobSASPermissions.parse("r"), + protocol: SASProtocol.HttpsAndHttp, + startsOn: now, + version: "2016-05-31" + }, + sourceStorageSharedKeyCredential as StorageSharedKeyCredential + ).toString(); + + const sourceBlob = sourceContainerClient.getBlockBlobClient(blobName); + await sourceBlob.upload("hello", 5, { + metadata: { + foo: "1", + bar: "2" + } + }); + + // Copy From Uri + const targetBlob = targetContainerClient.getBlockBlobClient(blobName); + const targetBlobWithProps = + targetContainerClient.getBlockBlobClient(blobName2); + const copyResponse3 = await targetBlob.syncCopyFromURL( + `${sourceBlob.url}?${sas}` + ); + assert.equal("success", copyResponse3.copyStatus); + const properties3 = await targetBlob.getProperties(); + assert.equal(properties3.metadata!["foo"], "1"); + assert.equal(properties3.metadata!["bar"], "2"); + + const copyResponse4 = await targetBlobWithProps.syncCopyFromURL( + `${sourceBlob.url}?${sas}`, + { + metadata: { + baz: "3" + } + } + ); + + assert.equal("success", copyResponse4.copyStatus); + const properties4 = await targetBlobWithProps.getProperties(); + assert.equal(properties4.metadata!["foo"], undefined); + assert.equal(properties4.metadata!["bar"], undefined); + assert.equal(properties4.metadata!["baz"], "3"); + }); +}); diff --git a/tests/dfs/specialnaming.test.ts b/tests/dfs/specialnaming.test.ts new file mode 100644 index 000000000..12bce18de --- /dev/null +++ b/tests/dfs/specialnaming.test.ts @@ -0,0 +1,568 @@ +import dns = require("dns"); + +import { + BlobServiceClient, + BlockBlobClient, + newPipeline, + StorageSharedKeyCredential +} from "@azure/storage-blob"; + +import { configLogger } from "../../src/common/Logger"; +import { + appendToURLPath, + EMULATOR_ACCOUNT_KEY, + EMULATOR_ACCOUNT_NAME, + getUniqueName +} from "../testutils"; +import DataLakeTestServerFactory from "./DataLakeTestServerFactory"; + +import assert = require("assert"); + +// Set true to enable debug log +configLogger(false); + +describe("SpecialNaming", () => { + const factory = new DataLakeTestServerFactory(); + const server = factory.createServer(); + + const baseURL = `http://${server.config.host}:${server.config.port}/devstoreaccount1`; + const productionStyleHostName = "devstoreaccount1.localhost"; // Use hosts file to make this resolve + const noAccountHostName = "host.docker.internal"; + const noAccountHostNameConnectionString = `DefaultEndpointsProtocol=http;AccountName=${EMULATOR_ACCOUNT_NAME};AccountKey=${EMULATOR_ACCOUNT_KEY};BlobEndpoint=http://${noAccountHostName}:${server.config.port}/${EMULATOR_ACCOUNT_NAME};`; + + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ) + ); + + const containerName: string = getUniqueName("1container-with-dash"); + const containerClient = serviceClient.getContainerClient(containerName); + + before(async () => { + await server.start(); + await containerClient.create(); + }); + + after(async () => { + await containerClient.delete(); + await server.close(); + await server.clean(); + }); + + it("Should work with special container and blob names with spaces @loki @sql", async () => { + const blobName: string = getUniqueName("blob empty"); + const blockBlobClient = containerClient.getBlockBlobClient(blobName); + + await blockBlobClient.upload("A", 1); + const response = ( + await containerClient + .listBlobsByHierarchy("$", { + prefix: blobName + }) + .byPage() + .next() + ).value; + assert.notDeepStrictEqual(response.segment.blobItems.length, 0); + assert.deepStrictEqual(response.segment.blobItems[0].name, blobName); + }); + + it("Should work with special container and blob names with unicode @loki @sql", async () => { + const blobName: string = getUniqueName("unicod\u00e9"); + const blockBlobClient = containerClient.getBlockBlobClient(blobName); + + await blockBlobClient.upload("A", 1); + const response = ( + await containerClient + .listBlobsByHierarchy("$", { + prefix: blobName + }) + .byPage() + .next() + ).value; + assert.notDeepStrictEqual(response.segment.blobItems.length, 0); + assert.deepStrictEqual(response.segment.blobItems[0].name, blobName); + assert.deepStrictEqual(response.segment.blobItems[0].name, blobName); + }); + + it("Should work with special container and blob names with spaces in URL string @loki @sql", async () => { + const blobName: string = getUniqueName("blob empty"); + const blockBlobClient = new BlockBlobClient( + appendToURLPath(containerClient.url, blobName), + (containerClient as any).pipeline + ); + + await blockBlobClient.upload("A", 1); + const response = ( + await containerClient + .listBlobsByHierarchy("$", { + prefix: blobName + }) + .byPage() + .next() + ).value; + assert.notDeepStrictEqual(response.segment.blobItems.length, 0); + assert.deepStrictEqual(response.segment.blobItems[0].name, blobName); + }); + + it("Should work with special container and blob names with / @loki @sql", async () => { + const blobName: string = getUniqueName("////blob/empty /another"); + const blockBlobClient = containerClient.getBlockBlobClient(blobName); + + await blockBlobClient.upload("A", 1); + await blockBlobClient.getProperties(); + const response = ( + await containerClient + .listBlobsByHierarchy("$", { + prefix: blobName + }) + .byPage() + .next() + ).value; + assert.notDeepStrictEqual(response.segment.blobItems.length, 0); + assert.deepStrictEqual(response.segment.blobItems[0].name, blobName); + }); + + it("Should work with special container and blob names with / in URL string @loki @sql", async () => { + const blobName: string = getUniqueName("////blob/empty /another"); + const blockBlobClient = new BlockBlobClient( + appendToURLPath(containerClient.url, blobName), + (containerClient as any).pipeline + ); + + await blockBlobClient.upload("A", 1); + await blockBlobClient.getProperties(); + const response = ( + await containerClient + .listBlobsByHierarchy("$", { + prefix: blobName + }) + .byPage() + .next() + ).value; + assert.notDeepStrictEqual(response.segment.blobItems.length, 0); + assert.deepStrictEqual(response.segment.blobItems[0].name, blobName); + }); + + it("Should work with special container and blob names uppercase @loki @sql", async () => { + const blobName: string = getUniqueName("////Upper/blob/empty /another"); + const blockBlobClient = containerClient.getBlockBlobClient(blobName); + + await blockBlobClient.upload("A", 1); + await blockBlobClient.getProperties(); + const response = ( + await containerClient + .listBlobsByHierarchy("$", { + prefix: blobName + }) + .byPage() + .next() + ).value; + assert.notDeepStrictEqual(response.segment.blobItems.length, 0); + assert.deepStrictEqual(response.segment.blobItems[0].name, blobName); + }); + + it("Should work with special container and blob names uppercase in URL string @loki @sql", async () => { + const blobName: string = getUniqueName("////Upper/blob/empty /another"); + const blockBlobClient = new BlockBlobClient( + appendToURLPath(containerClient.url, blobName), + (containerClient as any).pipeline + ); + + await blockBlobClient.upload("A", 1); + await blockBlobClient.getProperties(); + const response = ( + await containerClient + .listBlobsByHierarchy("$", { + prefix: blobName + }) + .byPage() + .next() + ).value; + assert.notDeepStrictEqual(response.segment.blobItems.length, 0); + assert.deepStrictEqual(response.segment.blobItems[0].name, blobName); + }); + + it("Should work with special blob names Chinese characters @loki @sql", async () => { + const blobName: string = getUniqueName( + "////Upper/blob/empty /another 汉字" + ); + const blockBlobClient = containerClient.getBlockBlobClient(blobName); + + await blockBlobClient.upload("A", 1); + await blockBlobClient.getProperties(); + const response = ( + await containerClient + .listBlobsByHierarchy("$", { + prefix: blobName + }) + .byPage() + .next() + ).value; + assert.notDeepStrictEqual(response.segment.blobItems.length, 0); + assert.deepStrictEqual(response.segment.blobItems[0].name, blobName); + }); + + it("Should work with special blob names Chinese characters in URL string @loki @sql", async () => { + const blobName: string = getUniqueName( + "////Upper/blob/empty /another 汉字" + ); + const blockBlobClient = new BlockBlobClient( + appendToURLPath(containerClient.url, blobName), + (containerClient as any).pipeline + ); + + await blockBlobClient.upload("A", 1); + await blockBlobClient.getProperties(); + const response = ( + await containerClient + .listBlobsByHierarchy("$", { + prefix: blobName + }) + .byPage() + .next() + ).value; + assert.notDeepStrictEqual(response.segment.blobItems.length, 0); + assert.deepStrictEqual(response.segment.blobItems[0].name, blobName); + }); + + it("Should work with special blob name characters @loki @sql", async () => { + const blobName: string = getUniqueName( + "汉字. special ~!@#$%^&*()_+`1234567890-={}|[]\\:\";'<>?,/'" + ); + const blockBlobClient = containerClient.getBlockBlobClient(blobName); + + await blockBlobClient.upload("A", 1); + await blockBlobClient.getProperties(); + const response = ( + await containerClient + .listBlobsByHierarchy( + "b", // A char doesn't exist in blob name + { + // NOTICE: Azure Storage Server will replace "\" with "/" in the blob names + prefix: blobName.replace(/\\/g, "/") + } + ) + .byPage() + .next() + ).value; + assert.notDeepStrictEqual(response.segment.blobItems.length, 0); + assert.deepStrictEqual( + response.segment.blobItems[0].name, + blobName.replace(/\\/g, "/") + ); + }); + + it("Should work with special blob name characters in URL string @loki @sql", async () => { + const blobName: string = getUniqueName( + "汉字. special ~!@#$%^&*()_+`1234567890-={}|[]\\:\";'<>?,/'" + ); + const blockBlobClient = new BlockBlobClient( + // There are 2 special cases for a URL string: + // Escape "%" when creating XXXURL object with URL strings + // Escape "?" otherwise string after "?" will be treated as URL parameters + appendToURLPath( + containerClient.url, + blobName.replace(/%/g, "%25").replace(/\?/g, "%3F") + ), + (containerClient as any).pipeline + ); + + await blockBlobClient.upload("A", 1); + await blockBlobClient.getProperties(); + const response = ( + await containerClient + .listBlobsByHierarchy( + "b", // "b" doesn't exist in blob name + { + // NOTICE: Azure Storage Server will replace "\" with "/" in the blob names + prefix: blobName.replace(/\\/g, "/") + } + ) + .byPage() + .next() + ).value; + assert.notDeepStrictEqual(response.segment.blobItems.length, 0); + assert.deepStrictEqual( + response.segment.blobItems[0].name, + blobName.replace(/\\/g, "/") + ); + }); + + it("Should work with special blob name Russian URI encoded @loki @sql", async () => { + const blobName: string = getUniqueName("ру́сский язы́к"); + const blobNameEncoded: string = encodeURIComponent(blobName); + const blockBlobClient = containerClient.getBlockBlobClient(blobNameEncoded); + + await blockBlobClient.upload("A", 1); + await blockBlobClient.getProperties(); + const response = ( + await containerClient + .listBlobsByHierarchy("$", { + prefix: blobNameEncoded + }) + .byPage() + .next() + ).value; + assert.notDeepStrictEqual(response.segment.blobItems.length, 0); + assert.deepStrictEqual(response.segment.blobItems[0].name, blobNameEncoded); + }); + + it("Should work with special blob name Russian @loki @sql", async () => { + const blobName: string = getUniqueName("ру́сский язы́к"); + const blockBlobClient = containerClient.getBlockBlobClient(blobName); + + await blockBlobClient.upload("A", 1); + await blockBlobClient.getProperties(); + const response = ( + await containerClient + .listBlobsByHierarchy("$", { + prefix: blobName + }) + .byPage() + .next() + ).value; + assert.notDeepStrictEqual(response.segment.blobItems.length, 0); + assert.deepStrictEqual(response.segment.blobItems[0].name, blobName); + }); + + it("Should work with special blob name Russian in URL string @loki @sql", async () => { + const blobName: string = getUniqueName("ру́сский язы́к"); + const blockBlobClient = new BlockBlobClient( + appendToURLPath(containerClient.url, blobName), + (containerClient as any).pipeline + ); + + await blockBlobClient.upload("A", 1); + await blockBlobClient.getProperties(); + const response = ( + await containerClient + .listBlobsByHierarchy("$", { + prefix: blobName + }) + .byPage() + .next() + ).value; + assert.notDeepStrictEqual(response.segment.blobItems.length, 0); + assert.deepStrictEqual(response.segment.blobItems[0].name, blobName); + }); + + it("Should work with special blob name Arabic URI encoded @loki @sql", async () => { + const blobName: string = getUniqueName("عربي/عربى"); + const blobNameEncoded: string = encodeURIComponent(blobName); + const blockBlobClient = containerClient.getBlockBlobClient(blobNameEncoded); + + await blockBlobClient.upload("A", 1); + await blockBlobClient.getProperties(); + const response = ( + await containerClient + .listBlobsByHierarchy("$", { + prefix: blobNameEncoded + }) + .byPage() + .next() + ).value; + assert.notDeepStrictEqual(response.segment.blobItems.length, 0); + assert.deepStrictEqual(response.segment.blobItems[0].name, blobNameEncoded); + }); + + it("Should work with special blob name Arabic @loki @sql", async () => { + const blobName: string = getUniqueName("عربي/عربى"); + const blockBlobClient = containerClient.getBlockBlobClient(blobName); + + await blockBlobClient.upload("A", 1); + await blockBlobClient.getProperties(); + const response = ( + await containerClient + .listBlobsByHierarchy("$", { + prefix: blobName + }) + .byPage() + .next() + ).value; + assert.notDeepStrictEqual(response.segment.blobItems.length, 0); + assert.deepStrictEqual(response.segment.blobItems[0].name, blobName); + }); + + it("Should work with special blob name Arabic in URL string @loki @sql", async () => { + const blobName: string = getUniqueName("عربي/عربى"); + const blockBlobClient = new BlockBlobClient( + appendToURLPath(containerClient.url, blobName), + (containerClient as any).pipeline + ); + + await blockBlobClient.upload("A", 1); + await blockBlobClient.getProperties(); + const response = ( + await containerClient + .listBlobsByHierarchy("$", { + prefix: blobName + }) + .byPage() + .next() + ).value; + assert.notDeepStrictEqual(response.segment.blobItems.length, 0); + assert.deepStrictEqual(response.segment.blobItems[0].name, blobName); + }); + + it("Should work with special blob name Japanese URI encoded @loki @sql", async () => { + const blobName: string = getUniqueName("にっぽんご/にほんご"); + const blobNameEncoded: string = encodeURIComponent(blobName); + const blockBlobClient = containerClient.getBlockBlobClient(blobNameEncoded); + + await blockBlobClient.upload("A", 1); + await blockBlobClient.getProperties(); + const response = ( + await containerClient + .listBlobsByHierarchy("$", { + prefix: blobNameEncoded + }) + .byPage() + .next() + ).value; + assert.notDeepStrictEqual(response.segment.blobItems.length, 0); + assert.deepStrictEqual(response.segment.blobItems[0].name, blobNameEncoded); + }); + + it("Should work with special blob name Japanese @loki @sql", async () => { + const blobName: string = getUniqueName("にっぽんご/にほんご"); + const blockBlobClient = containerClient.getBlockBlobClient(blobName); + + await blockBlobClient.upload("A", 1); + await blockBlobClient.getProperties(); + const response = ( + await containerClient + .listBlobsByHierarchy("$", { + prefix: blobName + }) + .byPage() + .next() + ).value; + assert.notDeepStrictEqual(response.segment.blobItems.length, 0); + assert.deepStrictEqual(response.segment.blobItems[0].name, blobName); + }); + + it("Should work with special blob name Japanese in URL string @loki @sql", async () => { + const blobName: string = getUniqueName("にっぽんご/にほんご"); + const blockBlobClient = new BlockBlobClient( + appendToURLPath(containerClient.url, blobName), + (containerClient as any).pipeline + ); + + await blockBlobClient.upload("A", 1); + await blockBlobClient.getProperties(); + const response = ( + await containerClient + .listBlobsByHierarchy("$", { + prefix: blobName + }) + .byPage() + .next() + ).value; + assert.notDeepStrictEqual(response.segment.blobItems.length, 0); + assert.deepStrictEqual(response.segment.blobItems[0].name, blobName); + }); + + it(`Should work with production style URL when ${productionStyleHostName} is resolvable`, async () => { + await dns.promises.lookup(productionStyleHostName).then( + async (lookupAddress) => { + const baseURLProductionStyle = `http://${productionStyleHostName}:${server.config.port}/`; + const serviceClientProductionStyle = new BlobServiceClient( + baseURLProductionStyle, + newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ) + ); + const containerClientProductionStyle = + serviceClientProductionStyle.getContainerClient(containerName); + + const blobName: string = getUniqueName("myblob"); + const blockBlobClient = + containerClientProductionStyle.getBlockBlobClient(blobName); + + await blockBlobClient.upload("ABC", 3); + const response = ( + await containerClientProductionStyle + .listBlobsByHierarchy("$", { + prefix: blobName + }) + .byPage() + .next() + ).value; + assert.notDeepStrictEqual(response.segment.blobItems.length, 0); + assert.deepStrictEqual(response.segment.blobItems[0].name, blobName); + }, + () => { + // Cannot perform this test. We need devstoreaccount1.localhost to resolve to 127.0.0.1. + // On Linux, this should just work, + // On Windows, we can't spoof DNS record for specific process. + // So we have options of running our own DNS server (overkill), + // or editing hosts files (machine global operation; and requires running as admin). + // So skip the test case. + assert.ok( + `Skipping test case - it needs ${productionStyleHostName} to be resolvable` + ); + } + ); + }); + + it(`Should work with no account host name URL when ${noAccountHostName} is resolvable`, async () => { + await dns.promises.lookup(noAccountHostName).then( + async (lookupAddress) => { + const serviceClientNoHostName = BlobServiceClient.fromConnectionString( + noAccountHostNameConnectionString, + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + const containerClientProductionStyle = + serviceClientNoHostName.getContainerClient(containerName); + + const blobName: string = getUniqueName("myblob"); + const blockBlobClient = + containerClientProductionStyle.getBlockBlobClient(blobName); + + await blockBlobClient.upload("ABC", 3); + const response = ( + await containerClientProductionStyle + .listBlobsByHierarchy("$", { + prefix: blobName + }) + .byPage() + .next() + ).value; + assert.notDeepStrictEqual(response.segment.blobItems.length, 0); + assert.deepStrictEqual(response.segment.blobItems[0].name, blobName); + }, + () => { + // Cannot perform this test. We need host.docker.internal to resolve to 127.0.0.1. + // On Windows, we can't spoof DNS record for specific process. + // So we have options of running our own DNS server (overkill), + // or editing hosts files (machine global operation; and requires running as admin). + // So skip the test case. + assert.ok( + `Skipping test case - it needs ${noAccountHostName} to be resolvable` + ); + } + ); + }); +}); diff --git a/tests/dfs/storage-file-datalake/apis/aborter.test.ts b/tests/dfs/storage-file-datalake/apis/aborter.test.ts new file mode 100644 index 000000000..6149adffb --- /dev/null +++ b/tests/dfs/storage-file-datalake/apis/aborter.test.ts @@ -0,0 +1,116 @@ +// Copyright (c) Microsoft Corporation. +import assert from "assert"; +import { Context } from "mocha"; + +// Licensed under the MIT license. +import { AbortController, AbortSignal } from "@azure/abort-controller"; +import { + DataLakeFileSystemClient, + DataLakeServiceClient, + StorageSharedKeyCredential +} from "@azure/storage-file-datalake"; + +import { + EMULATOR_ACCOUNT_KEY, + EMULATOR_ACCOUNT_NAME, + getUniqueName +} from "../../../testutils"; +import DataLakeTestServerFactory from "../../DataLakeTestServerFactory"; + +describe("Aborter", () => { + const factory = new DataLakeTestServerFactory(); + const server = factory.createServer(); + + const baseURL = `http://${server.config.host}:${server.config.port}/devstoreaccount1`; + const serviceClient = new DataLakeServiceClient( + baseURL, + new StorageSharedKeyCredential(EMULATOR_ACCOUNT_NAME, EMULATOR_ACCOUNT_KEY), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + + let fileSystemName: string; + let fileSystemClient: DataLakeFileSystemClient; + + before(async () => { + await server.start(); + }); + + after(async () => { + await server.close(); + await server.clean(); + }); + + beforeEach(async function (this: Context) { + checkIfShouldSkip(this); + fileSystemName = getUniqueName("filesystem"); + fileSystemClient = serviceClient.getFileSystemClient(fileSystemName); + }); + + afterEach(async function () { + await fileSystemClient.deleteIfExists(); + }); + + it("Should abort after aborter timeout @loki @sql", async () => { + try { + await fileSystemClient.create({ + abortSignal: AbortController.timeout(1) + }); + assert.fail(); + } catch (err: any) { + assert.equal(err.name, "AbortError"); + } + }); + + it("Should not abort after calling abort() @loki @sql", async () => { + await fileSystemClient.create({ abortSignal: AbortSignal.none }); + }); + + it("Should abort when calling abort() before request finishes @loki @sql", async () => { + const aborter = new AbortController(); + const response = fileSystemClient.create({ abortSignal: aborter.signal }); + aborter.abort(); + try { + await response; + assert.fail(); + } catch (err: any) { + assert.equal(err.name, "AbortError"); + } + }); + + it("Should not abort when calling abort() after request finishes @loki @sql", async () => { + const aborter = new AbortController(); + await fileSystemClient.create({ abortSignal: aborter.signal }); + aborter.abort(); + }); + + it("Should abort after father aborter calls abort() @loki @sql", async () => { + try { + const aborter = new AbortController(); + const childAborter = new AbortController( + aborter.signal, + AbortController.timeout(10 * 60 * 1000) + ); + const response = fileSystemClient.create({ + abortSignal: childAborter.signal + }); + aborter.abort(); + await response; + assert.fail(); + } catch (err: any) { + assert.equal(err.name, "AbortError"); + } + }); +}); + +function checkIfShouldSkip(context: Context) { + if ( + context.currentTest!.title.indexOf("ExpiryTime") > -1 || + context.test!.parent!.title.indexOf("soft delete") > -1 + ) { + context.skip(); + } +} diff --git a/tests/dfs/storage-file-datalake/apis/file.test.ts b/tests/dfs/storage-file-datalake/apis/file.test.ts new file mode 100644 index 000000000..c60dc66bc --- /dev/null +++ b/tests/dfs/storage-file-datalake/apis/file.test.ts @@ -0,0 +1,1056 @@ +import { isNode } from "@azure/ms-rest-js"; +import { BlobServiceClient } from "@azure/storage-blob"; +import { + DataLakeServiceClient, + FileSystemListPathsResponse, + newPipeline, + StorageSharedKeyCredential +} from "@azure/storage-file-datalake"; + +import { configLogger } from "../../../../src/common/Logger"; +import { BlobHTTPHeaders } from "../../../../src/dfs/generated/artifacts/models"; +import { + bodyToString, + EMULATOR_ACCOUNT_KEY, + EMULATOR_ACCOUNT_NAME, + getUniqueName, + sleep, + upload +} from "../../../testutils"; +import DataLakeTestServerFactory from "../../DataLakeTestServerFactory"; + +import assert = require("assert"); + +// Set true to enable debug log +configLogger(false); + +describe("fileAPIs", () => { + const factory = new DataLakeTestServerFactory(); + const server = factory.createServer(); + + const baseURL = `http://${server.config.host}:${server.config.port}/devstoreaccount1`; + const serviceClient = new DataLakeServiceClient( + baseURL, + newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ) + ); + + const blobServiceClient = new BlobServiceClient( + baseURL, + newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ) + ); + + let fileSytemName: string = getUniqueName("filesystem"); + let fileSystemClient = serviceClient.getFileSystemClient(fileSytemName); + let fileName: string = getUniqueName("file"); + let fileClient = fileSystemClient.getFileClient(fileName); + let fileLeaseClient = fileClient.getDataLakeLeaseClient(); + + let containerClient = blobServiceClient.getContainerClient(fileSytemName); + let blobClient = containerClient.getBlobClient(fileName); + blobClient.accountName; + + const content = "Hello World"; + + before(async () => { + await server.start(); + }); + + after(async () => { + await server.close(); + await server.clean(); + }); + + beforeEach(async () => { + fileSytemName = getUniqueName("filesystem"); + fileSystemClient = serviceClient.getFileSystemClient(fileSytemName); + await fileSystemClient.create(); + fileName = getUniqueName("file"); + fileClient = fileSystemClient.getFileClient(fileName); + fileLeaseClient = fileClient.getDataLakeLeaseClient(); + await upload(fileClient, content); + containerClient = blobServiceClient.getContainerClient(fileSytemName); + blobClient = containerClient.getBlobClient(fileName); + }); + + afterEach(async () => { + await fileSystemClient.delete(); + }); + + it("download with with default parameters @loki @sql", async () => { + const result = await fileClient.read(); + assert.deepStrictEqual(await bodyToString(result, content.length), content); + assert.equal(result.contentRange, undefined); + assert.ok(result.requestId); + }); + + it("download should work with conditional headers @loki @sql", async () => { + const properties = await fileClient.getProperties(); + const result = await fileClient.read(0, undefined, { + conditions: { + ifMatch: properties.etag, + ifNoneMatch: "invalidetag", + ifModifiedSince: new Date("2018/01/01"), + ifUnmodifiedSince: new Date("2188/01/01") + } + }); + assert.deepStrictEqual(await bodyToString(result, content.length), content); + assert.equal(result.contentRange, undefined); + assert.ok(result.requestId); + }); + + it("download should work with ifMatch value * @loki @sql", async () => { + const result = await fileClient.read(0, undefined, { + conditions: { + ifMatch: "*,abc", + ifNoneMatch: "invalidetag", + ifModifiedSince: new Date("2018/01/01"), + ifUnmodifiedSince: new Date("2188/01/01") + } + }); + assert.deepStrictEqual(await bodyToString(result, content.length), content); + assert.equal(result.contentRange, undefined); + assert.ok(result.requestId); + }); + + it("download should not work with invalid conditional header ifMatch @loki @sql", async () => { + const properties = await fileClient.getProperties(); + try { + await fileClient.read(0, undefined, { + conditions: { + ifMatch: properties.etag + "invalid" + } + }); + } catch (error) { + assert.deepStrictEqual(error.statusCode, 412); + return; + } + assert.fail(); + }); + + it("download should not work with conditional header ifNoneMatch @loki @sql", async () => { + const properties = await fileClient.getProperties(); + try { + await fileClient.read(0, undefined, { + conditions: { + ifNoneMatch: properties.etag + } + }); + } catch (error) { + assert.deepStrictEqual(error.statusCode, 304); + return; + } + assert.fail(); + }); + + it("download should not work with conditional header ifNoneMatch * @loki @sql", async () => { + try { + await fileClient.read(0, undefined, { + conditions: { + ifNoneMatch: "*" + } + }); + } catch (error) { + assert.deepStrictEqual(error.statusCode, 400); + return; + } + assert.fail(); + }); + + it("download should not work with conditional header ifModifiedSince @loki @sql", async () => { + try { + await fileClient.read(0, undefined, { + conditions: { + ifModifiedSince: new Date("2120/01/01") + } + }); + } catch (error) { + assert.deepStrictEqual(error.statusCode, 304); + return; + } + assert.fail(); + }); + + it("download should not work with conditional header ifUnmodifiedSince @loki @sql", async () => { + try { + await fileClient.read(0, undefined, { + conditions: { + ifUnmodifiedSince: new Date("2018/01/01") + } + }); + } catch (error) { + assert.deepStrictEqual(error.statusCode, 412); + return; + } + assert.fail(); + }); + + it("download all parameters set @loki @sql", async () => { + const result = await fileClient.read(0, 1, { + rangeGetContentMD5: true + }); + assert.deepStrictEqual(await bodyToString(result, 1), content[0]); + assert.equal(result.contentRange, `bytes 0-0/${content.length}`); + assert.ok(result.requestId); + }); + + it("download entire with range @loki @sql", async () => { + const result = await fileClient.read(0, content.length); + assert.deepStrictEqual(await bodyToString(result, content.length), content); + assert.equal( + result.contentRange, + `bytes 0-${content.length - 1}/${content.length}` + ); + assert.ok(result.requestId); + }); + + it("download out of range @loki @sql", async () => { + try { + await fileClient.read(content.length + 1, content.length + 10); + } catch (error) { + assert.deepStrictEqual(error.statusCode, 416); + return; + } + assert.fail(); + }); + + it("get properties response should not set content-type @loki @sql", async () => { + const fileURL404 = fileSystemClient.getFileClient("UN_EXIST_file_"); + try { + await fileURL404.getProperties(); + } catch (err) { + assert.ok(!err.response.headers.get("content-type")); + } + + try { + await fileURL404.read(0, 0); + } catch (err) { + assert.notEqual(err.response.headers.get("content-type"), undefined); + } + }); + + it("delete @loki @sql", async () => { + const result = await fileClient.delete(); + assert.ok(result.requestId); + }); + + it("delete should work for valid ifMatch @loki @sql", async () => { + const properties = await fileClient.getProperties(); + + const result = await fileClient.delete(false, { + conditions: { + ifMatch: properties.etag + } + }); + assert.ok(result.requestId); + }); + + it("delete should work for * ifMatch @loki @sql", async () => { + const result = await fileClient.delete(false, { + conditions: { + ifMatch: "*" + } + }); + assert.ok(result.requestId); + }); + + it("delete should not work for invalid ifMatch @loki @sql", async () => { + try { + await fileClient.delete(false, { + conditions: { + ifMatch: "invalid" + } + }); + } catch (error) { + assert.deepStrictEqual(error.statusCode, 412); + return; + } + assert.fail(); + }); + + it("delete should work for valid ifNoneMatch @loki @sql", async () => { + const result = await fileClient.delete(false, { + conditions: { + ifNoneMatch: "unmatchetag" + } + }); + assert.ok(result.requestId); + }); + + it("delete should not work for invalid ifNoneMatch @loki @sql", async () => { + const properties = await fileClient.getProperties(); + + try { + await fileClient.delete(false, { + conditions: { + ifNoneMatch: properties.etag + } + }); + } catch (error) { + assert.deepStrictEqual(error.statusCode, 412); + return; + } + assert.fail(); + }); + + it("delete should work for ifNoneMatch * @loki @sql", async () => { + await fileClient.delete(false, { + conditions: { + ifNoneMatch: "*" + } + }); + }); + + it("delete should work for valid ifModifiedSince * @loki @sql", async () => { + await fileClient.delete(false, { + conditions: { + ifModifiedSince: new Date("2018/01/01") + } + }); + }); + + it("delete should not work for invalid ifModifiedSince @loki @sql", async () => { + try { + await fileClient.delete(false, { + conditions: { + ifModifiedSince: new Date("2118/01/01") + } + }); + } catch (error) { + assert.deepStrictEqual(error.statusCode, 412); + return; + } + assert.fail(); + }); + + it("delete should work for valid ifUnmodifiedSince * @loki @sql", async () => { + await fileClient.delete(false, { + conditions: { + ifUnmodifiedSince: new Date("2118/01/01") + } + }); + }); + + it("delete should not work for invalid ifUnmodifiedSince @loki @sql", async () => { + try { + await fileClient.delete(false, { + conditions: { + ifUnmodifiedSince: new Date("2018/01/01") + } + }); + } catch (error) { + assert.deepStrictEqual(error.statusCode, 412); + return; + } + assert.fail(); + }); + + it("should create a snapshot from a file @loki @sql", async () => { + const result = await blobClient.createSnapshot(); + assert.ok(result.snapshot); + assert.ok(result.requestId); + }); + + it("should create a snapshot with metadata from a file @loki @sql", async () => { + const metadata = { + meta1: "val1", + meta3: "val3" + }; + const result = await blobClient.createSnapshot({ metadata }); + assert.ok(result.snapshot); + assert.ok(result.requestId); + const result2 = await blobClient + .withSnapshot(result.snapshot!) + .getProperties(); + assert.deepStrictEqual(result2.metadata, metadata); + }); + + it("should not delete base file without include snapshot header @loki @sql", async () => { + const result = await blobClient.createSnapshot(); + assert.ok(result.snapshot); + assert.ok(result.requestId); + const fileSnapshotURL = blobClient.withSnapshot(result.snapshot!); + await fileSnapshotURL.getProperties(); + + let err; + try { + await fileClient.delete(false, {}); + } catch (error) { + err = error; + } + + assert.deepStrictEqual(err.statusCode, 409); + }); + + it("should delete snapshot @loki @sql", async () => { + const result = await blobClient.createSnapshot(); + assert.ok(result.snapshot); + assert.ok(result.requestId); + const fileSnapshotURL = blobClient.withSnapshot(result.snapshot!); + await fileSnapshotURL.getProperties(); + await fileSnapshotURL.delete(); + await fileClient.delete(); + const result2 = (await fileSystemClient.listPaths().byPage().next()) + .value as FileSystemListPathsResponse; + // Verify that the snapshot is deleted + assert.equal(result2.pathItems!.length, 0); + assert.ok(result2.requestId); + }); + + it("should setMetadata with new metadata set @loki @sql", async () => { + const metadata = { + a: "a", + b: "b" + }; + const result_setmeta = await fileClient.setMetadata(metadata); + assert.equal( + result_setmeta._response.request.headers.get("x-ms-client-request-id"), + result_setmeta.clientRequestId + ); + const result = await fileClient.getProperties(); + assert.deepStrictEqual(result.metadata, metadata); + assert.ok(result.requestId); + }); + + it("acquireLease_available_proposedLeaseId_fixed @loki @sql", async () => { + const guid = "ca761232ed4211cebacd00aa0057b223"; + const duration = 30; + fileLeaseClient = fileClient.getDataLakeLeaseClient(guid); + const result_acquire = await fileLeaseClient.acquireLease(duration); + assert.equal( + result_acquire._response.request.headers.get("x-ms-client-request-id"), + result_acquire._response.request.requestId + ); + + const result = await fileClient.getProperties(); + assert.equal(result.leaseDuration, "fixed"); + assert.equal(result.leaseState, "leased"); + assert.equal(result.leaseStatus, "locked"); + assert.ok(result.requestId); + + const result_release = await fileLeaseClient.releaseLease(); + assert.equal( + result_release._response.request.headers.get("x-ms-client-request-id"), + result_release._response.request.requestId + ); + }); + + it("acquireLease_available_NoproposedLeaseId_infinite @loki @sql", async () => { + const leaseResult = await fileLeaseClient.acquireLease(-1); + const leaseId = leaseResult.leaseId; + assert.ok(leaseId); + + const result = await fileClient.getProperties(); + assert.equal(result.leaseDuration, "infinite"); + assert.equal(result.leaseState, "leased"); + assert.equal(result.leaseStatus, "locked"); + + await fileLeaseClient.releaseLease(); + }); + + it("releaseLease @loki @sql", async () => { + const guid = "ca761232ed4211cebacd00aa0057b223"; + const duration = -1; + fileLeaseClient = await fileClient.getDataLakeLeaseClient(guid); + await fileLeaseClient.acquireLease(duration); + + let result = await fileClient.getProperties(); + assert.equal(result.leaseDuration, "infinite"); + assert.equal(result.leaseState, "leased"); + assert.equal(result.leaseStatus, "locked"); + + await fileLeaseClient.releaseLease(); + result = await fileClient.getProperties(); + assert.equal(result.leaseDuration, undefined); + assert.equal(result.leaseState, "available"); + assert.equal(result.leaseStatus, "unlocked"); + }); + + it("renewLease @loki @sql", async () => { + const guid = "ca761232ed4211cebacd00aa0057b223"; + const duration = 15; + fileLeaseClient = await fileClient.getDataLakeLeaseClient(guid); + await fileLeaseClient.acquireLease(duration); + + const result = await fileClient.getProperties(); + assert.equal(result.leaseDuration, "fixed"); + assert.equal(result.leaseState, "leased"); + assert.equal(result.leaseStatus, "locked"); + + await sleep(16 * 1000); + const result2 = await fileClient.getProperties(); + assert.ok(!result2.leaseDuration); + assert.equal(result2.leaseState, "expired"); + assert.equal(result2.leaseStatus, "unlocked"); + + await fileLeaseClient.renewLease(); + + const result3 = await fileClient.getProperties(); + assert.equal(result3.leaseDuration, "fixed"); + assert.equal(result3.leaseState, "leased"); + assert.equal(result3.leaseStatus, "locked"); + + await fileLeaseClient.releaseLease(); + }); + + it("changeLease @loki @sql", async () => { + const guid = "ca761232ed4211cebacd00aa0057b223"; + const duration = 15; + fileLeaseClient = fileClient.getDataLakeLeaseClient(guid); + await fileLeaseClient.acquireLease(duration); + + const result = await fileClient.getProperties(); + assert.equal(result.leaseDuration, "fixed"); + assert.equal(result.leaseState, "leased"); + assert.equal(result.leaseStatus, "locked"); + + const newGuid = "3c7e72ebb4304526bc53d8ecef03798f"; + const result_change = await fileLeaseClient.changeLease(newGuid); + assert.equal( + result_change._response.request.headers.get("x-ms-client-request-id"), + result_change._response.request.requestId + ); + + await fileClient.getProperties(); + await fileLeaseClient.releaseLease(); + }); + + it("breakLease @loki @sql", async () => { + const guid = "ca761232ed4211cebacd00aa0057b223"; + const duration = 15; + fileLeaseClient = fileClient.getDataLakeLeaseClient(guid); + await fileLeaseClient.acquireLease(duration); + + const result = await fileClient.getProperties(); + assert.equal(result.leaseDuration, "fixed"); + assert.equal(result.leaseState, "leased"); + assert.equal(result.leaseStatus, "locked"); + + const breakDuration = 3; + let breaklefttime = breakDuration; + while (breaklefttime > 0) { + const breakResult = await fileLeaseClient.breakLease(breakDuration); + assert.equal( + breakResult._response.request.headers.get("x-ms-client-request-id"), + breakResult._response.request.requestId + ); + + assert.equal(breakResult.leaseTime! <= breaklefttime, true); + breaklefttime = breakResult.leaseTime!; + + const result2 = await fileClient.getProperties(); + assert.ok(!result2.leaseDuration); + assert.equal(result2.leaseState, "breaking"); + assert.equal(result2.leaseStatus, "locked"); + + await sleep(500); + } + + const result3 = await fileClient.getProperties(); + assert.ok(!result3.leaseDuration); + assert.equal(result3.leaseState, "broken"); + assert.equal(result3.leaseStatus, "unlocked"); + + await fileLeaseClient.releaseLease(); + const result4 = await fileClient.getProperties(); + assert.equal(result4.leaseDuration, undefined); + assert.equal(result4.leaseState, "available"); + assert.equal(result4.leaseStatus, "unlocked"); + }); + + it("should get the correct headers back when setting metadata @loki @sql", async () => { + const metadata = { + a: "a", + b: "b" + }; + const setResult = await fileClient.setMetadata(metadata); + assert.equal( + setResult._response.request.headers.get("x-ms-client-request-id"), + setResult.clientRequestId + ); + assert.notEqual(setResult.date, undefined); + assert.notEqual(setResult.etag, undefined); + assert.notEqual(setResult.isServerEncrypted, undefined); + assert.notEqual(setResult.lastModified, undefined); + assert.notEqual(setResult.requestId, undefined); + assert.notEqual(setResult.version, undefined); + const result = await fileClient.getProperties(); + assert.deepStrictEqual(result.metadata, metadata); + assert.deepStrictEqual(result.accessTier, "Hot"); + assert.deepStrictEqual(result.acceptRanges, "bytes"); + }); + + // https://docs.microsoft.com/en-us/rest/api/storageservices/get-file-properties + // as properties retrieval is implemented, the properties should be added to the tests below + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Header + it("should get the correct properties set based on set HTTP headers @loki @sql", async () => { + const cacheControl = "no-cache"; + const contentType = "text/plain; charset=UTF-8"; + const md5 = new Uint8Array([1, 2, 3, 4, 5]); + const contentEncoding = "identity"; + const contentLanguage = "en-US"; + const contentDisposition = "attachment"; + const headers: BlobHTTPHeaders = { + blobCacheControl: cacheControl, + blobContentType: contentType, + blobContentMD5: md5, + blobContentDisposition: contentDisposition, + blobContentLanguage: contentLanguage, + blobContentEncoding: contentEncoding + }; + const result_set = await blobClient.setHTTPHeaders(headers); + assert.equal( + result_set._response.request.headers.get("x-ms-client-request-id"), + result_set.clientRequestId + ); + const result = await fileClient.getProperties(); + assert.deepStrictEqual(result.cacheControl, cacheControl); + assert.deepStrictEqual(result.contentType, contentType); + assert.deepEqual(result.contentMD5, md5); + assert.deepStrictEqual(result.contentDisposition, contentDisposition); + assert.deepStrictEqual(result.contentLanguage, contentLanguage); + }); + + it("setHTTPHeaders with default parameters @loki @sql", async () => { + await blobClient.setHTTPHeaders({}); + const result = await fileClient.getProperties(); + + assert.ok(result.lastModified); + assert.deepStrictEqual(result.metadata, {}); + assert.ok(!result.cacheControl); + assert.ok(!result.contentType); + assert.ok(!result.contentMD5); + assert.ok(!result.contentEncoding); + assert.ok(!result.contentLanguage); + assert.ok(!result.contentDisposition); + }); + + it("setHTTPHeaders with all parameters set @loki @sql", async () => { + const headers: BlobHTTPHeaders = { + blobCacheControl: "fileCacheControl", + blobContentDisposition: "fileContentDisposition", + blobContentEncoding: "fileContentEncoding", + blobContentLanguage: "fileContentLanguage", + blobContentMD5: isNode + ? Buffer.from([1, 2, 3, 4]) + : new Uint8Array([1, 2, 3, 4]), + blobContentType: "fileContentType" + }; + await blobClient.setHTTPHeaders(headers); + const result = await fileClient.getProperties(); + assert.ok(result.date); + assert.ok(result.lastModified); + assert.deepStrictEqual(result.metadata, {}); + assert.deepStrictEqual(result.cacheControl, headers.blobCacheControl); + assert.deepStrictEqual(result.contentType, headers.blobContentType); + assert.deepStrictEqual(result.contentMD5, headers.blobContentMD5); + assert.deepStrictEqual(result.contentEncoding, headers.blobContentEncoding); + assert.deepStrictEqual(result.contentLanguage, headers.blobContentLanguage); + assert.deepStrictEqual( + result.contentDisposition, + headers.blobContentDisposition + ); + }); + + it("Copy file should work @loki @sql", async () => { + const sourcefile = getUniqueName("file"); + const destfile = getUniqueName("file"); + + const sourceBlobClient = containerClient.getBlobClient(sourcefile); + const destBlobClient = containerClient.getBlobClient(destfile); + const sourceFileClient = fileSystemClient.getFileClient(sourcefile); + const destfileClient = fileSystemClient.getFileClient(destfile); + + const metadata = { key: "value" }; + const pathHttpHeaders = { + cacheControl: "fileCacheControl", + contentDisposition: "fileContentDisposition", + contentEncoding: "fileContentEncoding", + contentLanguage: "fileContentLanguage", + contentType: "fileContentType" + }; + + await upload(sourceFileClient, "hello", { pathHttpHeaders, metadata }); + + const result_startcopy = await destBlobClient.beginCopyFromURL( + sourceBlobClient.url + ); + assert.equal( + result_startcopy + .getResult()! + ._response.request.headers.get("x-ms-client-request-id"), + result_startcopy.getResult()!._response.request.requestId + ); + + const result = await destfileClient.getProperties(); + assert.ok(result.date); + assert.ok(result.lastModified); + assert.deepStrictEqual(result.metadata, metadata); + assert.deepStrictEqual(result.cacheControl, pathHttpHeaders.cacheControl); + assert.deepStrictEqual(result.contentType, pathHttpHeaders.contentType); + assert.deepStrictEqual( + result.contentEncoding, + pathHttpHeaders.contentEncoding + ); + assert.deepStrictEqual( + result.contentLanguage, + pathHttpHeaders.contentLanguage + ); + assert.deepStrictEqual( + result.contentDisposition, + pathHttpHeaders.contentDisposition + ); + }); + + it("Copy file should work to override metadata @loki @sql", async () => { + const sourcefile = getUniqueName("file"); + const destfile = getUniqueName("file"); + + const sourceBlobClient = containerClient.getBlobClient(sourcefile); + const destBlobClient = containerClient.getBlobClient(destfile); + const sourceFileClient = fileSystemClient.getFileClient(sourcefile); + const destfileClient = fileSystemClient.getFileClient(destfile); + + const metadata = { key: "value" }; + const metadata2 = { key: "value2" }; + + await upload(sourceFileClient, "hello", { metadata }); + + await destBlobClient.beginCopyFromURL(sourceBlobClient.url, { + metadata: metadata2 + }); + + const result = await destfileClient.getProperties(); + assert.ok(result.date); + assert.ok(result.lastModified); + assert.deepStrictEqual(result.metadata, metadata2); + }); + + it("Copy file should not override destination Lease status @loki @sql", async () => { + const sourcefile = getUniqueName("file"); + const destfile = getUniqueName("file"); + + const sourceBlobClient = containerClient.getBlobClient(sourcefile); + const destBlobClient = containerClient.getBlobClient(destfile); + const sourceFileClient = fileSystemClient.getFileClient(sourcefile); + const destfileClient = fileSystemClient.getFileClient(destfile); + + await upload(sourceFileClient, "hello"); + await upload(destfileClient, "hello"); + + let destLeaseClient = destfileClient.getDataLakeLeaseClient(); + const leaseResult = await destLeaseClient.acquireLease(-1); + const leaseId = leaseResult.leaseId; + assert.ok(leaseId); + + const getResult = await destfileClient.getProperties(); + assert.equal(getResult.leaseDuration, "infinite"); + assert.equal(getResult.leaseState, "leased"); + assert.equal(getResult.leaseStatus, "locked"); + + await destBlobClient.beginCopyFromURL(sourceBlobClient.url, { + conditions: { leaseId } + }); + + const result = await destfileClient.getProperties(); + assert.ok(result.date); + assert.ok(result.lastModified); + assert.equal(getResult.leaseDuration, "infinite"); + assert.equal(getResult.leaseState, "leased"); + assert.equal(getResult.leaseStatus, "locked"); + + await destLeaseClient.releaseLease(); + }); + + it("Copy file should not work with ifNoneMatch * when dest exist @loki @sql", async () => { + const sourcefile = getUniqueName("file"); + const destfile = getUniqueName("file"); + + const sourceBlobClient = containerClient.getBlobClient(sourcefile); + const destBlobClient = containerClient.getBlobClient(destfile); + const sourceFileClient = fileSystemClient.getFileClient(sourcefile); + const destfileClient = fileSystemClient.getFileClient(destfile); + + const metadata = { key: "value" }; + const pathHttpHeaders = { + cacheControl: "fileCacheControl", + contentDisposition: "fileContentDisposition", + contentEncoding: "fileContentEncoding", + contentLanguage: "fileContentLanguage", + contentType: "fileContentType" + }; + + let uploadResult = await sourceFileClient.create({ + pathHttpHeaders, + metadata + }); + assert.ok(uploadResult.requestId); + await upload(sourceFileClient, "hello"); + await upload(destfileClient, "hello", { pathHttpHeaders, metadata }); + + // async copy + try { + await destBlobClient.beginCopyFromURL(sourceBlobClient.url, { + conditions: { + ifNoneMatch: "*" + } + }); + } catch (error) { + assert.deepStrictEqual(error.statusCode, 409); + return; + } + assert.fail(); + + // Sync copy + try { + await destBlobClient.syncCopyFromURL(sourceBlobClient.url, { + conditions: { + ifNoneMatch: "*" + } + }); + } catch (error) { + assert.deepStrictEqual(error.statusCode, 409); + return; + } + assert.fail(); + }); + + it("Synchronized copy file should work @loki @sql", async () => { + const sourcefile = getUniqueName("file"); + const destfile = getUniqueName("file"); + + const sourceBlobClient = containerClient.getBlobClient(sourcefile); + const destBlobClient = containerClient.getBlobClient(destfile); + const sourceFileClient = fileSystemClient.getFileClient(sourcefile); + const destfileClient = fileSystemClient.getFileClient(destfile); + + const metadata = { key: "value" }; + const pathHttpHeaders = { + cacheControl: "fileCacheControl", + contentDisposition: "fileContentDisposition", + contentEncoding: "fileContentEncoding", + contentLanguage: "fileContentLanguage", + contentType: "fileContentType" + }; + + await upload(sourceFileClient, "hello", { pathHttpHeaders, metadata }); + const result_copy = await destBlobClient.syncCopyFromURL( + sourceBlobClient.url + ); + assert.equal( + result_copy._response.request.headers.get("x-ms-client-request-id"), + result_copy._response.request.requestId + ); + assert.equal(result_copy.copyStatus, "success"); + + const result = await destfileClient.getProperties(); + assert.ok(result.date); + assert.ok(result.lastModified); + assert.deepStrictEqual(result.metadata, metadata); + assert.deepStrictEqual(result.cacheControl, pathHttpHeaders.cacheControl); + assert.deepStrictEqual(result.contentType, pathHttpHeaders.contentType); + assert.deepStrictEqual( + result.contentEncoding, + pathHttpHeaders.contentEncoding + ); + assert.deepStrictEqual( + result.contentLanguage, + pathHttpHeaders.contentLanguage + ); + assert.deepStrictEqual( + result.contentDisposition, + pathHttpHeaders.contentDisposition + ); + }); + + it("Synchronized copy file should work to override metadata @loki @sql", async () => { + const sourcefile = getUniqueName("file"); + const destfile = getUniqueName("file"); + + const sourceBlobClient = containerClient.getBlobClient(sourcefile); + const destBlobClient = containerClient.getBlobClient(destfile); + const sourceFileClient = fileSystemClient.getFileClient(sourcefile); + const destfileClient = fileSystemClient.getFileClient(destfile); + + const metadata = { key: "value" }; + const metadata2 = { key: "value2" }; + + const pathHttpHeaders = { + cacheControl: "fileCacheControl", + contentDisposition: "fileContentDisposition", + contentEncoding: "fileContentEncoding", + contentLanguage: "fileContentLanguage", + contentType: "fileContentType" + }; + + await upload(sourceFileClient, "hello", { pathHttpHeaders, metadata }); + await destBlobClient.syncCopyFromURL(sourceBlobClient.url, { + metadata: metadata2 + }); + + const result = await destfileClient.getProperties(); + assert.ok(result.date); + assert.ok(result.lastModified); + assert.deepStrictEqual(result.metadata, metadata2); + }); + + it("Synchronized copy file should not override destination Lease status @loki @sql", async () => { + const sourcefile = getUniqueName("file"); + const destfile = getUniqueName("file"); + + const sourceBlobClient = containerClient.getBlobClient(sourcefile); + const destBlobClient = containerClient.getBlobClient(destfile); + const sourceFileClient = fileSystemClient.getFileClient(sourcefile); + const destfileClient = fileSystemClient.getFileClient(destfile); + + await upload(sourceFileClient, "hello"); + await upload(destfileClient, "hello"); + + let destLeaseClient = destfileClient.getDataLakeLeaseClient(); + const leaseResult = await destLeaseClient.acquireLease(-1); + const leaseId = leaseResult.leaseId; + assert.ok(leaseId); + + const getResult = await destfileClient.getProperties(); + assert.equal(getResult.leaseDuration, "infinite"); + assert.equal(getResult.leaseState, "leased"); + assert.equal(getResult.leaseStatus, "locked"); + + await destBlobClient.syncCopyFromURL(sourceBlobClient.url, { + conditions: { leaseId } + }); + + const result = await destfileClient.getProperties(); + assert.ok(result.date); + assert.ok(result.lastModified); + assert.equal(getResult.leaseDuration, "infinite"); + assert.equal(getResult.leaseState, "leased"); + assert.equal(getResult.leaseStatus, "locked"); + + await destLeaseClient.releaseLease(); + }); + + it("Synchronized copy file should work for page file @loki @sql", async () => { + const sourcefile = getUniqueName("file"); + const destfile = getUniqueName("file"); + + const sourceBlobClient = containerClient.getBlobClient(sourcefile); + const destBlobClient = containerClient.getBlobClient(destfile); + const sourceFileClient = fileSystemClient.getFileClient(sourcefile); + const destfileClient = fileSystemClient.getFileClient(destfile); + + const metadata = { key: "value" }; + + const pathHttpHeaders = { + cacheControl: "fileCacheControl", + contentDisposition: "fileContentDisposition", + contentEncoding: "fileContentEncoding", + contentLanguage: "fileContentLanguage", + contentType: "fileContentType" + }; + + await upload(sourceFileClient, "hello", { pathHttpHeaders, metadata }); + const result_copy = await destBlobClient.syncCopyFromURL( + sourceBlobClient.url + ); + assert.equal( + result_copy._response.request.headers.get("x-ms-client-request-id"), + result_copy._response.request.requestId + ); + assert.equal(result_copy.copyStatus, "success"); + + const result = await destfileClient.getProperties(); + assert.ok(result.date); + assert.ok(result.lastModified); + assert.deepStrictEqual(result.metadata, metadata); + assert.deepStrictEqual(result.cacheControl, pathHttpHeaders.cacheControl); + assert.deepStrictEqual(result.contentType, pathHttpHeaders.contentType); + assert.deepStrictEqual( + result.contentEncoding, + pathHttpHeaders.contentEncoding + ); + assert.deepStrictEqual( + result.contentLanguage, + pathHttpHeaders.contentLanguage + ); + assert.deepStrictEqual( + result.contentDisposition, + pathHttpHeaders.contentDisposition + ); + }); + + it("Acquire Lease on Breaking Lease status, if LeaseId not match, throw LeaseIdMismatchWithLease error @loki @sql", async () => { + // TODO: implement the case later + }); + + it("Renew Lease on Breaking Lease status, if LeaseId not match, throw LeaseIdMismatchWithLease error @loki @sql", async () => { + // TODO: implement the case later + }); + + it("Change Lease on Breaking Lease status, if LeaseId not match, throw LeaseIdMismatchWithLease error @loki @sql", async () => { + // TODO: implement the case later + }); + + it("Renew: Lease on Breaking Lease status, if LeaseId not match, throw LeaseIdMismatchWithLease error @loki @sql", async () => { + // TODO: implement the case later + }); + + it("Acquire Lease on Broken Lease status, if LeaseId not match, throw LeaseIdMismatchWithLease error @loki @sql", async () => { + // TODO: implement the case later + }); + + it("Break Lease on Infinite Lease, if give valid breakPeriod, should be broken after breadperiod @loki @sql", async () => { + // TODO: implement the case later + }); + + it("Break Lease on Infinite Lease, if not give breakPeriod, should be broken immidiately @loki @sql", async () => { + // TODO: implement the case later + }); + + it("Renew: Lease on Leased status, if LeaseId not match, throw LeaseIdMismatchWithLease error @loki @sql", async () => { + // TODO: implement the case later + }); + + it("Change Lease on Leased status, if input LeaseId not match anyone of leaseID or proposedLeaseId, throw LeaseIdMismatchWithLease error @loki @sql", async () => { + // TODO: implement the case later + }); + + it("Change Lease on Leased status, if input LeaseId matches proposedLeaseId, will change success @loki @sql", async () => { + // TODO: implement the case later + }); + + it("UploadPage on a Leased page file, if input LeaseId matches, will success @loki @sql", async () => { + // TODO: implement the case later + }); + + it("ClearPage on a Leased page file, if input LeaseId matches, will success @loki @sql", async () => { + // TODO: implement the case later + }); + + it("Resize a Leased page file, if input LeaseId matches, will success @loki @sql", async () => { + // TODO: implement the case later + }); + + it("UpdateSequenceNumber a Leased page file, if input LeaseId matches, will success @loki @sql", async () => { + // TODO: implement the case later + }); +}); diff --git a/tests/dfs/storage-file-datalake/apis/filesystem.test.ts b/tests/dfs/storage-file-datalake/apis/filesystem.test.ts new file mode 100644 index 000000000..a4423ee0a --- /dev/null +++ b/tests/dfs/storage-file-datalake/apis/filesystem.test.ts @@ -0,0 +1,1205 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { Context } from "mocha"; + +import { + DataLakeDirectoryClient, + DataLakeFileClient, + DataLakeFileSystemClient, + DataLakeServiceClient, + FileSystemListDeletedPathsResponse, + FileSystemListPathsResponse, + StorageSharedKeyCredential +} from "@azure/storage-file-datalake"; + +import { configLogger } from "../../../../src/common/Logger"; +import { + EMULATOR_ACCOUNT_KEY, + EMULATOR_ACCOUNT_NAME, + getEncryptionScope, + getUniqueName, + getYieldedValue +} from "../../../testutils"; +import DataLakeTestServerFactory from "../../DataLakeTestServerFactory"; + +import assert = require("assert"); + +// Set true to enable debug log +configLogger(false); + +describe("DataLakeFileSystemClient", () => { + const factory = new DataLakeTestServerFactory(); + const server = factory.createServer(); + + const baseURL = `http://${server.config.host}:${server.config.port}/devstoreaccount1`; + const serviceClient = new DataLakeServiceClient( + baseURL, + new StorageSharedKeyCredential(EMULATOR_ACCOUNT_NAME, EMULATOR_ACCOUNT_KEY), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + + let fileSystemName: string; + let fileSystemClient: DataLakeFileSystemClient; + + before(async () => { + await server.start(); + }); + + after(async () => { + await server.close(); + await server.clean(); + }); + + beforeEach(async function (this: Context) { + checkIfShouldSkip(this); + fileSystemName = getUniqueName("filesystem"); + fileSystemClient = serviceClient.getFileSystemClient(fileSystemName); + await fileSystemClient.createIfNotExists(); + }); + + afterEach(async () => { + await fileSystemClient.deleteIfExists(); + }); + + it("setMetadata @loki @sql", async () => { + const metadata = { + key0: "val0", + keya: "vala", + keyb: "valb" + }; + await fileSystemClient.setMetadata(metadata); + + const result = await fileSystemClient.getProperties(); + assert.deepEqual(result.metadata, metadata); + }); + + it("setMetadata with tracing @loki @sql", async () => { + const metadata = { + key0: "val0", + keya: "vala", + keyb: "valb" + }; + await fileSystemClient.setMetadata(metadata); + }); + + it("getProperties @loki @sql", async () => { + const result = await fileSystemClient.getProperties(); + assert.ok(result.etag!.length > 0); + assert.ok(result.lastModified); + assert.ok(!result.leaseDuration); + assert.equal(result.leaseState, "available"); + assert.equal(result.leaseStatus, "unlocked"); + assert.ok(result.requestId); + assert.ok(result.version); + assert.ok(result.date); + assert.ok(!result.publicAccess); + assert.ok(result.clientRequestId); // As default pipeline involves UniqueRequestIDPolicy + }); + + it("create with default parameters @loki @sql", (done) => { + // create() with default parameters has been tested in beforeEach + done(); + }); + + it("create with all parameters configured @loki @sql", async () => { + const cClient = serviceClient.getFileSystemClient( + getUniqueName(fileSystemName) + ); + const metadata = { key: "value" }; + const access = "filesystem"; + await cClient.create({ metadata, access }); + const result = await cClient.getProperties(); + assert.deepEqual(result.publicAccess, access); + assert.deepEqual(result.metadata, metadata); + }); + + it("create with encryption scope @loki @sql", async function (this: Context) { + let encryptionScopeName; + try { + encryptionScopeName = getEncryptionScope(); + } catch { + this.skip(); + } + + const cClient = serviceClient.getFileSystemClient( + getUniqueName(fileSystemName) + ); + await cClient.create({ + fileSystemEncryptionScope: { + defaultEncryptionScope: encryptionScopeName, + preventEncryptionScopeOverride: true + } + }); + const result = await cClient.getProperties(); + assert.equal(result.defaultEncryptionScope, encryptionScopeName); + await cClient.delete(); + }); + + it("create with encryption scope - preventEncryptionScopeOverride : false @loki @sql", async function (this: Context) { + let encryptionScopeName; + try { + encryptionScopeName = getEncryptionScope(); + } catch { + this.skip(); + } + + const cClient = serviceClient.getFileSystemClient( + getUniqueName(fileSystemName) + ); + await cClient.create({ + fileSystemEncryptionScope: { + defaultEncryptionScope: encryptionScopeName, + preventEncryptionScopeOverride: false + } + }); + const result = await cClient.getProperties(); + assert.equal(result.defaultEncryptionScope, encryptionScopeName); + await cClient.delete(); + }); + + it("createIfNotExists @loki @sql", async () => { + const cClient = serviceClient.getFileSystemClient( + getUniqueName(fileSystemName) + ); + const metadata = { key: "value" }; + const access = "filesystem"; + const createRes = await cClient.createIfNotExists({ metadata, access }); + assert.ok(createRes.succeeded); + assert.ok(createRes.etag); + + const createRes2 = await cClient.createIfNotExists({ metadata, access }); + assert.ok(!createRes2.succeeded); + + await cClient.deleteIfExists(); + }); + + it("deleteIfExists @loki @sql", async () => { + const cClient = serviceClient.getFileSystemClient( + getUniqueName(fileSystemName) + ); + const res = await cClient.deleteIfExists(); + assert.ok(!res.succeeded); + + await cClient.create(); + const res2 = await cClient.deleteIfExists(); + assert.ok(res2.succeeded); + }); + + it("delete @loki @sql", (done) => { + // delete() with default parameters has been tested in afterEach + done(); + }); + + it("listPaths with default parameters @loki @sql", async () => { + const recordedNow = new Date(); + + const fileClients = []; + for (let i = 0; i < 3; i++) { + const fileClient = fileSystemClient.getFileClient( + getUniqueName(`file${i}`) + ); + await fileClient.create(); + fileClients.push(fileClient); + } + + const result = (await fileSystemClient.listPaths().byPage().next()) + .value as FileSystemListPathsResponse; + + assert.deepStrictEqual(result.continuation, undefined); + assert.deepStrictEqual(result.pathItems!.length, fileClients.length); + assert.ok(fileClients[0].url.indexOf(result.pathItems![0].name!)); + + // The path is created just now, createdOn should be around but may not be the same to current time. + // assert.equal(result.pathItems![0].createdOn?.getUTCFullYear(), recordedNow.getUTCFullYear()); + // assert.equal(result.pathItems![0].createdOn?.getUTCMonth(), recordedNow.getUTCMonth()); + // assert.equal(result.pathItems![0].createdOn?.getUTCDate(), recordedNow.getUTCDate()); + // assert.equal(result.pathItems![0].createdOn?.getUTCHours(), recordedNow.getUTCHours()); + + assert.equal( + result.pathItems![0].lastModified?.getUTCFullYear(), + recordedNow.getUTCFullYear() + ); + assert.equal( + result.pathItems![0].lastModified?.getUTCMonth(), + recordedNow.getUTCMonth() + ); + assert.equal( + result.pathItems![0].lastModified?.getUTCDate(), + recordedNow.getUTCDate() + ); + assert.equal( + result.pathItems![0].lastModified?.getUTCHours(), + recordedNow.getUTCHours() + ); + + for (const file of fileClients) { + await file.delete(); + } + }); + + it("listPaths - Encryption Scope @loki @sql", async function (this: Context) { + let encryptionScopeName; + try { + encryptionScopeName = getEncryptionScope(); + } catch { + this.skip(); + } + + const cClient = serviceClient.getFileSystemClient( + getUniqueName(fileSystemName) + ); + await cClient.create({ + fileSystemEncryptionScope: { + defaultEncryptionScope: encryptionScopeName, + preventEncryptionScopeOverride: true + } + }); + + const fileClient = cClient.getFileClient(getUniqueName(`file`)); + await fileClient.create(); + + const dirClient = cClient.getFileClient(getUniqueName(`dir`)); + await dirClient.create(); + + const result = (await cClient.listPaths().byPage().next()) + .value as FileSystemListPathsResponse; + + assert.equal(result.pathItems!.length, 2); + assert.equal(result.pathItems![0].encryptionScope, encryptionScopeName); + assert.equal(result.pathItems![1].encryptionScope, encryptionScopeName); + + await fileClient.delete(); + await dirClient.delete(); + }); + + it("listPaths - PagedAsyncIterableIterator with Encryption Scope @loki @sql", async function (this: Context) { + let encryptionScopeName; + try { + encryptionScopeName = getEncryptionScope(); + } catch { + this.skip(); + } + + const cClient = serviceClient.getFileSystemClient( + getUniqueName(fileSystemName) + ); + await cClient.create({ + fileSystemEncryptionScope: { + defaultEncryptionScope: encryptionScopeName, + preventEncryptionScopeOverride: true + } + }); + + const fileClient = cClient.getFileClient(getUniqueName(`file`)); + await fileClient.create(); + const dirClient = cClient.getFileClient(getUniqueName(`dir`)); + await dirClient.create(); + + for await (const listedFile of cClient.listPaths()) { + assert.equal(listedFile.encryptionScope, encryptionScopeName); + } + + await fileClient.delete(); + await dirClient.delete(); + }); + + it("listPaths - ExpiryTime, NeverExpire @loki @sql", async function (this: Context) { + const fileClient = fileSystemClient.getFileClient(getUniqueName(`file`)); + await fileClient.create(); + await fileClient.setExpiry("NeverExpire"); + const result = (await fileSystemClient.listPaths().byPage().next()) + .value as FileSystemListPathsResponse; + + assert.equal(result.pathItems![0].expiresOn, undefined); + await fileClient.delete(); + }); + + it.skip("listPaths - ExpiryTime, Absolute [listPaths don't return expiresOn in the spec and adding it make some other sdks fail] @loki @sql", async function (this: Context) { + const now = new Date(); + const delta = 30 * 1000; + const expiresOn = new Date(now.getTime() + delta); + const fileClient = fileSystemClient.getFileClient(getUniqueName(`file`)); + + const content = "Hello, World"; + await fileClient.create(); + await fileClient.append(content, 0, content.length); + await fileClient.flush(content.length); + // const clock = useFakeTimers(now); + let setExpiryPromise: Promise; + try { + setExpiryPromise = fileClient.setExpiry("Absolute", { expiresOn }); + } finally { + // clock.restore(); + } + await setExpiryPromise; + + const result = (await fileSystemClient.listPaths().byPage().next()) + .value as FileSystemListPathsResponse; + + const recordedExpiresOn = new Date(expiresOn.getTime()); + recordedExpiresOn.setMilliseconds(0); // milliseconds dropped + assert.equal( + result.pathItems![0].expiresOn?.getTime(), + recordedExpiresOn.getTime() + ); + await fileClient.delete(); + }); + + it.skip("listPaths - ExpiryTime, RelativeToNow [listPaths don't return expiresOn in the spec and adding it make some other sdks fail] @loki @sql", async () => { + const delta = 30 * 1000; + const fileClient = fileSystemClient.getFileClient(getUniqueName(`file`)); + + const content = "Hello, World"; + await fileClient.create(); + await fileClient.append(content, 0, content.length); + await fileClient.flush(content.length); + await fileClient.setExpiry("RelativeToNow", { timeToExpireInMs: delta }); + + const result = (await fileSystemClient.listPaths().byPage().next()) + .value as FileSystemListPathsResponse; + + assert.ok(result.pathItems![0].expiresOn); + await fileClient.delete(); + }); + + it.skip("listPaths - ExpiryTime, RelativeToCreation [listPaths don't return expiresOn in the spec and adding it make some other sdks fail] @loki @sql", async () => { + const delta = 1000 * 3600 + 0.12; + const fileClient = fileSystemClient.getFileClient(getUniqueName(`file`)); + + const content = "Hello, World"; + await fileClient.create(); + await fileClient.append(content, 0, content.length); + await fileClient.flush(content.length); + await fileClient.setExpiry("RelativeToCreation", { + timeToExpireInMs: delta + }); + + const result = (await fileSystemClient.listPaths().byPage().next()) + .value as FileSystemListPathsResponse; + assert.equal( + result.pathItems![0].expiresOn?.getTime(), + result.pathItems![0].createdOn!.getTime() + Math.round(delta) + ); + await fileClient.delete(); + }); + + it("listPaths with default parameters - null path shouldn't throw error @loki @sql", async () => { + const fileClients = []; + for (let i = 0; i < 3; i++) { + const fileClient = fileSystemClient.getFileClient( + getUniqueName(`file${i}`) + ); + + await fileClient.create(); + fileClients.push(fileClient); + } + + const result = ( + await fileSystemClient.listPaths({ path: "" }).byPage().next() + ).value; + assert.deepStrictEqual(result.continuation, undefined); + assert.deepStrictEqual(result.pathItems!.length, fileClients.length); + assert.ok(fileClients[0].url.indexOf(result.pathItems![0].name)); + + for (const file of fileClients) { + await file.delete(); + } + }); + + it("listPaths with all parameters configured @loki @sql", async () => { + const fileClients = []; + const prefix = "file"; + const metadata = { + keya: "a", + keyb: "c" + }; + for (let i = 0; i < 2; i++) { + const fileClient = fileSystemClient.getFileClient( + getUniqueName(`${prefix}${i}`) + ); + + await fileClient.create({ + metadata: metadata + }); + fileClients.push(fileClient); + } + + const result = ( + await fileSystemClient + .listPaths({ + userPrincipalName: true, + recursive: true, + path: "" + }) + .byPage({ maxPageSize: 1 }) + .next() + ).value as FileSystemListPathsResponse; + + assert.deepStrictEqual(result.pathItems!.length, 1); + assert.ok(fileClients[0].url.indexOf(result.pathItems![0].name!)); + + const result2 = ( + await fileSystemClient + .listPaths({ + userPrincipalName: true, + recursive: true, + path: "" + }) + .byPage({ continuationToken: result.continuation, maxPageSize: 2 }) + .next() + ).value; + + assert.deepStrictEqual(result2.pathItems!.length, 1); + assert.ok(fileClients[0].url.indexOf(result2.pathItems![0].name)); + + for (const file of fileClients) { + await file.delete(); + } + }); + + it("Verify PagedAsyncIterableIterator for listPaths @loki @sql", async () => { + const fileClients = []; + const prefix = "file"; + const metadata = { + keya: "a", + keyb: "c" + }; + for (let i = 0; i < 4; i++) { + const fileClient = fileSystemClient.getFileClient( + getUniqueName(`${prefix}${i}`) + ); + + await fileClient.create({ + metadata + }); + fileClients.push(fileClient); + } + + let i = 0; + for await (const file of fileSystemClient.listPaths({ + userPrincipalName: true, + recursive: true, + path: "" + })) { + assert.ok(fileClients[i].url.indexOf(file.name!)); + i++; + } + + for (const file of fileClients) { + await file.delete(); + } + }); + + it("Verify PagedAsyncIterableIterator(generator .next() syntax) for listPaths @loki @sql", async () => { + const fileClients = []; + const prefix = "file"; + const metadata = { + keya: "a", + keyb: "c" + }; + for (let i = 0; i < 2; i++) { + const fileClient = fileSystemClient.getFileClient( + getUniqueName(`${prefix}${i}`) + ); + + await fileClient.create({ + metadata + }); + fileClients.push(fileClient); + } + + const iterator = fileSystemClient.listPaths({ + userPrincipalName: true, + recursive: true, + path: "" + }); + + let path = getYieldedValue(await iterator.next()); + assert.ok(fileClients[0].url.indexOf(path.name!)); + + path = getYieldedValue(await iterator.next()); + assert.ok(fileClients[1].url.indexOf(path.name!)); + + for (const file of fileClients) { + await file.delete(); + } + }); + + it("Verify PagedAsyncIterableIterator(byPage()) for listPaths @loki @sql", async () => { + const fileClients = []; + const prefix = "file"; + const metadata = { + keya: "a", + keyb: "c" + }; + for (let i = 0; i < 4; i++) { + const fileClient = fileSystemClient.getFileClient( + getUniqueName(`${prefix}${i}`) + ); + + await fileClient.create({ + metadata + }); + fileClients.push(fileClient); + } + + let i = 0; + for await (const response of fileSystemClient + .listPaths({ + userPrincipalName: true, + recursive: true, + path: "" + }) + .byPage({ maxPageSize: 2 })) { + for (const file of response.pathItems || []) { + assert.ok(fileClients[i].url.indexOf(file.name!)); + i++; + } + } + + for (const file of fileClients) { + await file.delete(); + } + }); + + it("Verify PagedAsyncIterableIterator(byPage() - continuationToken) for listPaths @loki @sql", async () => { + const fileClients = []; + const prefix = "file"; + const metadata = { + keya: "a", + keyb: "c" + }; + for (let i = 0; i < 4; i++) { + const fileClient = fileSystemClient.getFileClient( + getUniqueName(`${prefix}${i}`) + ); + + await fileClient.create({ + metadata + }); + fileClients.push(fileClient); + } + + let i = 0; + let iter = fileSystemClient + .listPaths({ + userPrincipalName: true, + recursive: true, + path: "" + }) + .byPage({ maxPageSize: 2 }); + let response = (await iter.next()).value; + for (const file of response.pathItems) { + assert.ok(fileClients[i].url.indexOf(file.name)); + i++; + } + // Gets next marker + const marker = response.continuationToken; + // Passing next marker as continuationToken + iter = fileSystemClient + .listPaths({ + userPrincipalName: true, + recursive: true, + path: "" + }) + .byPage({ continuationToken: marker, maxPageSize: 2 }); + response = (await iter.next()).value; + // Gets 2 blobs + for (const file of response.pathItems) { + assert.ok(fileClients[i].url.indexOf(file.name)); + i++; + } + + for (const file of fileClients) { + await file.delete(); + } + }); + + it("verify fileSystemName passed to the client @loki @sql", async () => { + const accountName = "myaccount"; + const newClient = new DataLakeFileSystemClient( + `https://${accountName}.dfs.core.windows.net/` + fileSystemName + ); + assert.equal( + newClient.name, + fileSystemName, + "File system name is not the same as the one provided." + ); + assert.equal( + newClient.accountName, + accountName, + "Account name is not the same as the one provided." + ); + }); + + it("exists returns true on an existing file system @loki @sql", async () => { + const result = await fileSystemClient.exists(); + assert.ok( + result, + "exists() should return true for an existing file system" + ); + }); + + it("exists returns false on non-existing file system @loki @sql", async () => { + const newFileSystemClient = serviceClient.getFileSystemClient( + getUniqueName("newfilesystem") + ); + const result = await newFileSystemClient.exists(); + assert.ok( + result === false, + "exists() should returns false on non-existing file system" + ); + }); +}); + +describe("DataLakeFileSystemClient with soft delete", () => { + const factory = new DataLakeTestServerFactory(); + const server = factory.createServer(); + + const baseURL = `http://${server.config.host}:${server.config.port}/devstoreaccount1`; + const serviceClient = new DataLakeServiceClient( + baseURL, + new StorageSharedKeyCredential(EMULATOR_ACCOUNT_NAME, EMULATOR_ACCOUNT_KEY), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + + let fileSystemClient: DataLakeFileSystemClient; + + before(async () => { + await server.start(); + }); + + after(async () => { + await server.close(); + await server.clean(); + }); + + beforeEach(async function (this: Context) { + checkIfShouldSkip(this); + + // try { + // serviceClient = getGenericDataLakeServiceClient(recorder, "DFS_SOFT_DELETE_"); + // } catch (err: any) { + // this.skip(); + // } + + fileSystemClient = serviceClient.getFileSystemClient( + getUniqueName(`filesystem`) + ); + await fileSystemClient.createIfNotExists(); + }); + + afterEach(async function () { + if (fileSystemClient) { + await fileSystemClient.deleteIfExists(); + } + }); + + it("listDeletedPaths with default parameters @loki @sql", async () => { + const fileClients = []; + for (let i = 0; i < 3; i++) { + const fileClient = fileSystemClient.getFileClient( + getUniqueName(`file${i}`) + ); + await fileClient.create(); + fileClients.push(fileClient); + } + + for (const file of fileClients) { + await file.delete(); + } + + const result = (await fileSystemClient.listDeletedPaths().byPage().next()) + .value as FileSystemListDeletedPathsResponse; + + assert.deepStrictEqual(result.continuation, undefined); + assert.deepStrictEqual(result.pathItems!.length, fileClients.length); + assert.ok(fileClients[0].url.indexOf(result.pathItems![0].name)); + + for (const pathItem of result.pathItems!) { + assert.ok(pathItem.deletedOn); + assert.ok(pathItem.deletionId); + assert.ok(pathItem.remainingRetentionDays); + } + }); + + it("listDeletedPaths and listPaths with recreating file after deletion @loki @sql", async () => { + const fileClients = []; + for (let i = 0; i < 3; i++) { + const fileClient = fileSystemClient.getFileClient( + getUniqueName(`file${i}`) + ); + await fileClient.create(); + fileClients.push(fileClient); + } + + for (const file of fileClients) { + await file.delete(); + await file.create(); + } + + const result = (await fileSystemClient.listDeletedPaths().byPage().next()) + .value as FileSystemListDeletedPathsResponse; + + assert.deepStrictEqual(result.continuation, undefined); + assert.deepStrictEqual(result.pathItems!.length, fileClients.length); + assert.ok(fileClients[0].url.indexOf(result.pathItems![0].name)); + + for (const pathItem of result.pathItems!) { + assert.ok(pathItem.deletedOn); + assert.ok(pathItem.deletionId); + assert.ok(pathItem.remainingRetentionDays); + } + + const listPathResult = (await fileSystemClient.listPaths().byPage().next()) + .value as FileSystemListPathsResponse; + + assert.deepStrictEqual(listPathResult.continuation, undefined); + assert.deepStrictEqual( + listPathResult.pathItems!.length, + fileClients.length + ); + assert.ok(fileClients[0].url.indexOf(listPathResult.pathItems![0].name!)); + }); + + it("listDeletedPaths with recreating and deletion again @loki @sql", async () => { + const fileClients = []; + for (let i = 0; i < 3; i++) { + const fileClient = fileSystemClient.getFileClient( + getUniqueName(`file${i}`) + ); + await fileClient.create(); + fileClients.push(fileClient); + } + + for (const file of fileClients) { + await file.delete(); + await file.create(); + await file.delete(); + } + + const result = (await fileSystemClient.listDeletedPaths().byPage().next()) + .value as FileSystemListDeletedPathsResponse; + + assert.deepStrictEqual(result.continuation, undefined); + assert.deepStrictEqual(result.pathItems!.length, 2 * fileClients.length); + assert.ok(fileClients[0].url.indexOf(result.pathItems![0].name)); + + for (const pathItem of result.pathItems!) { + assert.ok(pathItem.deletedOn); + assert.ok(pathItem.deletionId); + assert.ok(pathItem.remainingRetentionDays); + } + }); + + it("listDeletedPaths with default parameters - empty path shouldn't throw error @loki @sql", async () => { + const fileClients = []; + for (let i = 0; i < 3; i++) { + const fileClient = fileSystemClient.getFileClient( + getUniqueName(`file${i}`) + ); + + await fileClient.create(); + fileClients.push(fileClient); + } + + for (const file of fileClients) { + await file.delete(); + } + + const result = ( + await fileSystemClient.listDeletedPaths({ prefix: "" }).byPage().next() + ).value; + + assert.deepStrictEqual(result.continuation, undefined); + assert.deepStrictEqual(result.pathItems!.length, fileClients.length); + assert.ok(fileClients[0].url.indexOf(result.pathItems![0].name)); + + for (const pathItem of result.pathItems!) { + assert.ok(pathItem.deletedOn); + assert.ok(pathItem.deletionId); + assert.ok(pathItem.remainingRetentionDays); + } + }); + + it("listDeletedPaths with all parameters configured and byPage with continuationToken @loki @sql", async () => { + const fileClients = []; + const prefix = "file"; + const metadata = { + keya: "a", + keyb: "c" + }; + for (let i = 0; i < 2; i++) { + const fileClient = fileSystemClient.getFileClient( + getUniqueName(`${prefix}${i}`) + ); + + await fileClient.create({ + metadata: metadata + }); + fileClients.push(fileClient); + } + + for (const file of fileClients) { + await file.delete(); + } + + const result = ( + await fileSystemClient + .listDeletedPaths({ + prefix: "" + }) + .byPage({ maxPageSize: 1 }) + .next() + ).value as FileSystemListDeletedPathsResponse; + + assert.deepStrictEqual(result.pathItems!.length, 1); + assert.ok(fileClients[0].url.indexOf(result.pathItems![0].name)); + for (const pathItem of result.pathItems!) { + assert.ok(pathItem.deletedOn); + assert.ok(pathItem.deletionId); + assert.ok(pathItem.remainingRetentionDays); + } + + const result2 = ( + await fileSystemClient + .listDeletedPaths({ + prefix: "" + }) + .byPage({ continuationToken: result.continuation, maxPageSize: 2 }) + .next() + ).value; + + assert.deepStrictEqual(result2.pathItems!.length, 1); + assert.ok(fileClients[0].url.indexOf(result2.pathItems![0].name)); + }); + + it("Verify PagedAsyncIterableIterator for listDeletedPaths @loki @sql", async () => { + const fileClients = []; + const prefix = "file"; + const metadata = { + keya: "a", + keyb: "c" + }; + for (let i = 0; i < 4; i++) { + const fileClient = fileSystemClient.getFileClient( + getUniqueName(`${prefix}${i}`) + ); + + await fileClient.create({ + metadata + }); + fileClients.push(fileClient); + } + + for (const file of fileClients) { + await file.delete(); + } + + let i = 0; + for await (const file of fileSystemClient.listDeletedPaths({ + prefix: "" + })) { + assert.ok(fileClients[i].url.indexOf(file.name)); + assert.ok(file.deletedOn); + assert.ok(file.deletionId); + assert.ok(file.remainingRetentionDays); + i++; + } + }); + + it("Verify PagedAsyncIterableIterator(generator .next() syntax) for listDeletedPaths @loki @sql", async () => { + const fileClients = []; + const prefix = "file"; + const metadata = { + keya: "a", + keyb: "c" + }; + for (let i = 0; i < 2; i++) { + const fileClient = fileSystemClient.getFileClient( + getUniqueName(`${prefix}${i}`) + ); + + await fileClient.create({ + metadata + }); + fileClients.push(fileClient); + } + + for (const file of fileClients) { + await file.delete(); + } + + const iterator = fileSystemClient.listDeletedPaths({ + prefix: "" + }); + + let path = getYieldedValue(await iterator.next()); + assert.ok(fileClients[0].url.indexOf(path.name)); + assert.ok(path.deletedOn); + assert.ok(path.deletionId); + assert.ok(path.remainingRetentionDays); + + path = getYieldedValue(await iterator.next()); + assert.ok(fileClients[1].url.indexOf(path.name)); + assert.ok(path.deletedOn); + assert.ok(path.deletionId); + assert.ok(path.remainingRetentionDays); + }); + + it("Verify PagedAsyncIterableIterator(byPage()) for listDeletedPaths @loki @sql", async () => { + const fileClients = []; + const prefix = "file"; + const metadata = { + keya: "a", + keyb: "c" + }; + for (let i = 0; i < 4; i++) { + const fileClient = fileSystemClient.getFileClient( + getUniqueName(`${prefix}${i}`) + ); + + await fileClient.create({ + metadata + }); + fileClients.push(fileClient); + } + + for (const file of fileClients) { + await file.delete(); + } + + let i = 0; + for await (const response of fileSystemClient + .listDeletedPaths({ + prefix: "" + }) + .byPage({ maxPageSize: 2 })) { + for (const file of response.pathItems || []) { + assert.ok(fileClients[i].url.indexOf(file.name)); + assert.ok(file.deletedOn); + assert.ok(file.deletionId); + assert.ok(file.remainingRetentionDays); + i++; + } + } + }); + + it("Verify PagedAsyncIterableIterator(byPage() - continuationToken) for listDeletedPaths @loki @sql", async () => { + const fileClients = []; + const prefix = "file"; + const metadata = { + keya: "a", + keyb: "c" + }; + for (let i = 0; i < 4; i++) { + const fileClient = fileSystemClient.getFileClient( + getUniqueName(`${prefix}${i}`) + ); + + await fileClient.create({ + metadata + }); + fileClients.push(fileClient); + } + + for (const file of fileClients) { + await file.delete(); + } + + let i = 0; + let iter = fileSystemClient + .listDeletedPaths({ + prefix: "" + }) + .byPage({ maxPageSize: 2 }); + let response = (await iter.next()).value; + for (const file of response.pathItems) { + assert.ok(fileClients[i].url.indexOf(file.name)); + assert.ok(file.deletedOn); + assert.ok(file.deletionId); + assert.ok(file.remainingRetentionDays); + i++; + } + // Gets next marker + const marker = response.continuationToken; + // Passing next marker as continuationToken + iter = fileSystemClient + .listDeletedPaths({ + prefix: "" + }) + .byPage({ continuationToken: marker, maxPageSize: 2 }); + response = (await iter.next()).value; + // Gets 2 blobs + for (const file of response.pathItems) { + assert.ok(fileClients[i].url.indexOf(file.name)); + assert.ok(file.deletedOn); + assert.ok(file.deletionId); + assert.ok(file.remainingRetentionDays); + i++; + } + }); + + it("Undelete file and directory @loki @sql", async () => { + const fileName = getUniqueName(`file`); + const fileClient = fileSystemClient.getFileClient(fileName); + await fileClient.create(); + const fileDeleteResponse = await fileClient.delete(); + assert.ok(fileDeleteResponse.deletionId); + + const fileundeleteResponse = await fileSystemClient.undeletePath( + fileName, + fileDeleteResponse.deletionId ?? "" + ); + + assert.ok(fileundeleteResponse.pathClient instanceof DataLakeFileClient); + + assert.ok(await fileundeleteResponse.pathClient.exists()); + await fileundeleteResponse.pathClient.delete(); + + const directoryName = getUniqueName(`directory`); + const directoryClient = fileSystemClient.getDirectoryClient(directoryName); + await directoryClient.create(); + const directoryDeleteResponse = await directoryClient.delete(); + assert.ok(directoryDeleteResponse.deletionId); + + const directoryUndeleteResponse = await fileSystemClient.undeletePath( + directoryName, + directoryDeleteResponse.deletionId ?? "" + ); + + assert.ok( + directoryUndeleteResponse.pathClient instanceof DataLakeDirectoryClient + ); + + assert.ok(await directoryUndeleteResponse.pathClient.exists()); + await directoryUndeleteResponse.pathClient.delete(); + }); + + it("Undelete file and directory - recreate and delete path and undelete the path with first deletionid @loki @sql", async () => { + const fileName = getUniqueName(`file`); + const fileClient = fileSystemClient.getFileClient(fileName); + await fileClient.create(); + const fileDeleteResponse = await fileClient.delete(); + assert.ok(fileDeleteResponse.deletionId); + const firstDeletionId = fileDeleteResponse.deletionId; + + await fileClient.create(); + await fileClient.delete(); + + const fileundeleteResponse = await fileSystemClient.undeletePath( + fileName, + firstDeletionId ?? "" + ); + + assert.ok(fileundeleteResponse.pathClient instanceof DataLakeFileClient); + + assert.ok(await fileundeleteResponse.pathClient.exists()); + await fileundeleteResponse.pathClient.delete(); + }); + + it("Undelete file and directory - recreate and delete path and undelete the path twice @loki @sql", async () => { + const fileName = getUniqueName(`file`); + const fileClient = fileSystemClient.getFileClient(fileName); + await fileClient.create(); + const firstDeleteResponse = await fileClient.delete(); + assert.ok(firstDeleteResponse.deletionId); + + await fileClient.create(); + const secondDeleteResponse = await fileClient.delete(); + assert.ok(secondDeleteResponse.deletionId); + + const fileundeleteResponse = await fileSystemClient.undeletePath( + fileName, + secondDeleteResponse.deletionId ?? "" + ); + + assert.ok(fileundeleteResponse.pathClient instanceof DataLakeFileClient); + + assert.ok(await fileundeleteResponse.pathClient.exists()); + await fileundeleteResponse.pathClient.delete(); + + try { + await fileSystemClient.undeletePath( + fileName, + firstDeleteResponse.deletionId ?? "" + ); + assert.fail("Second undeletion should fail"); + } catch (err: any) { + /* empty */ + // The test case here expects an expection, so the exception should not fail the case. + } + }); + + it("Undelete file and directory with deleteIfExists @loki @sql", async () => { + const fileName = getUniqueName(`file`); + const fileClient = fileSystemClient.getFileClient(fileName); + await fileClient.create(); + const fileDeleteResponse = await fileClient.deleteIfExists(); + assert.ok(fileDeleteResponse.deletionId); + + const fileundeleteResponse = await fileSystemClient.undeletePath( + fileName, + fileDeleteResponse.deletionId ?? "" + ); + + assert.ok(fileundeleteResponse.pathClient instanceof DataLakeFileClient); + + assert.ok(await fileundeleteResponse.pathClient.exists()); + + const directoryName = getUniqueName(`directory`); + const directoryClient = fileSystemClient.getDirectoryClient(directoryName); + await directoryClient.create(); + const directoryDeleteResponse = await directoryClient.deleteIfExists(); + assert.ok(directoryDeleteResponse.deletionId); + + const directoryUndeleteResponse = await fileSystemClient.undeletePath( + directoryName, + directoryDeleteResponse.deletionId ?? "" + ); + + assert.ok( + directoryUndeleteResponse.pathClient instanceof DataLakeDirectoryClient + ); + + assert.ok(await directoryUndeleteResponse.pathClient.exists()); + }); + + it("Undelete file and directory special char @loki @sql", async () => { + const fileNames = [ + "!'();[]@&%=+$,#äÄöÖüÜß;", + "%21%27%28%29%3B%5B%5D%40%26%25%3D%2B%24%2C%23äÄöÖüÜß%3B", + " a file or directory " + ]; + + for (const fileName of fileNames) { + const fileClient = fileSystemClient.getFileClient(fileName); + await fileClient.create(); + const fileDeleteResponse = await fileClient.delete(); + assert.ok(fileDeleteResponse.deletionId); + + const fileundeleteResponse = await fileSystemClient.undeletePath( + fileName, + fileDeleteResponse.deletionId ?? "" + ); + + assert.ok(fileundeleteResponse.pathClient instanceof DataLakeFileClient); + assert.ok(await fileundeleteResponse.pathClient.exists()); + } + }); +}); + +function checkIfShouldSkip(context: Context) { + if (context.test!.parent!.title.indexOf("soft delete") > -1) { + context.skip(); + } +} diff --git a/tests/dfs/storage-file-datalake/apis/filesystemclient.test.ts b/tests/dfs/storage-file-datalake/apis/filesystemclient.test.ts new file mode 100644 index 000000000..5b473d2c0 --- /dev/null +++ b/tests/dfs/storage-file-datalake/apis/filesystemclient.test.ts @@ -0,0 +1,188 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { Context } from "mocha"; + +import { TokenCredential } from "@azure/core-auth"; +import { + DataLakeFileSystemClient, + DataLakeServiceClient, + FileSystemSASPermissions, + newPipeline, + PublicAccessType, + StorageSharedKeyCredential +} from "@azure/storage-file-datalake"; + +import { + assertClientUsesTokenCredential, + EMULATOR_ACCOUNT_KEY, + EMULATOR_ACCOUNT_NAME, + getUniqueName +} from "../../../testutils"; +import DataLakeTestServerFactory from "../../DataLakeTestServerFactory"; + +import assert = require("assert"); + +describe("DataLakeFileSystemClient Node.js only", () => { + const factory = new DataLakeTestServerFactory(); + const server = factory.createServer(); + + const baseURL = `http://${server.config.host}:${server.config.port}/devstoreaccount1`; + const serviceClient = new DataLakeServiceClient( + baseURL, + new StorageSharedKeyCredential(EMULATOR_ACCOUNT_NAME, EMULATOR_ACCOUNT_KEY), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + + let fileSystemName: string; + let fileSystemClient: DataLakeFileSystemClient; + + before(async () => { + await server.start(); + }); + + after(async () => { + await server.close(); + await server.clean(); + }); + + beforeEach(async function (this: Context) { + fileSystemName = getUniqueName("filesystem"); + fileSystemClient = serviceClient.getFileSystemClient(fileSystemName); + await fileSystemClient.createIfNotExists(); + }); + + afterEach(async function () { + await fileSystemClient.deleteIfExists(); + }); + + it("getAccessPolicy @loki @sql", async () => { + const result = await fileSystemClient.getAccessPolicy(); + assert.ok(result.etag!.length > 0); + assert.ok(result.lastModified); + assert.ok(result.requestId); + assert.ok(result.clientRequestId); + assert.ok(result.version); + assert.ok(result.date); + }); + + it("setAccessPolicy @loki @sql", async () => { + const access: PublicAccessType = "file"; + const acl = [ + { + accessPolicy: { + expiresOn: new Date("2018-12-31T11:22:33.4567890Z"), + permissions: FileSystemSASPermissions.parse("rwd").toString(), + startsOn: new Date("2017-12-31T11:22:33.4567890Z") + }, + id: "MTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTI=" + } + ]; + + await fileSystemClient.setAccessPolicy(access, acl); + const result = await fileSystemClient.getAccessPolicy(); + assert.deepEqual(result.signedIdentifiers, acl); + assert.deepEqual(result.publicAccess, access); + }); + + it("setAccessPolicy should work when expiry and start undefined @loki @sql", async () => { + const access: PublicAccessType = "file"; + const acl = [ + { + accessPolicy: { + permissions: FileSystemSASPermissions.parse("rwd").toString() + }, + id: "MTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTI=" + } + ]; + + await fileSystemClient.setAccessPolicy(access, acl); + const result = await fileSystemClient.getAccessPolicy(); + assert.deepEqual(result.signedIdentifiers, acl); + assert.deepEqual(result.publicAccess, access); + }); + + it("can be created with a url and a credential @loki @sql", async () => { + const credential = fileSystemClient.credential; + const newClient = new DataLakeFileSystemClient( + fileSystemClient.url, + credential + ); + const result = await newClient.getProperties(); + + assert.ok(result.etag!.length > 0); + assert.ok(result.lastModified); + assert.ok(!result.leaseDuration); + assert.equal(result.leaseState, "available"); + assert.equal(result.leaseStatus, "unlocked"); + assert.ok(result.requestId); + assert.ok(result.version); + assert.ok(result.date); + assert.ok(!result.publicAccess); + }); + + it("can be created with a url and a credential and an option bag @loki @sql", async () => { + const credential = fileSystemClient.credential; + const newClient = new DataLakeFileSystemClient( + fileSystemClient.url, + credential, + { + retryOptions: { + maxTries: 5 + } + } + ); + + const result = await newClient.getProperties(); + + assert.ok(result.etag!.length > 0); + assert.ok(result.lastModified); + assert.ok(!result.leaseDuration); + assert.equal(result.leaseState, "available"); + assert.equal(result.leaseStatus, "unlocked"); + assert.ok(result.requestId); + assert.ok(result.version); + assert.ok(result.date); + assert.ok(!result.publicAccess); + }); + + it("can be created with a url and a TokenCredential @loki @sql", async () => { + const tokenCredential: TokenCredential = { + getToken: () => + Promise.resolve({ + token: "token", + expiresOnTimestamp: 12345 + }) + }; + const newClient = new DataLakeFileSystemClient( + fileSystemClient.url, + tokenCredential + ); + assertClientUsesTokenCredential(newClient); + }); + + it("can be created with a url and a pipeline @loki @sql", async () => { + const credential = fileSystemClient.credential; + const pipeline = newPipeline(credential); + const newClient = new DataLakeFileSystemClient( + fileSystemClient.url, + pipeline + ); + + const result = await newClient.getProperties(); + + assert.ok(result.etag!.length > 0); + assert.ok(result.lastModified); + assert.ok(!result.leaseDuration); + assert.equal(result.leaseState, "available"); + assert.equal(result.leaseStatus, "unlocked"); + assert.ok(result.requestId); + assert.ok(result.version); + assert.ok(result.date); + assert.ok(!result.publicAccess); + }); +}); diff --git a/tests/dfs/storage-file-datalake/apis/leaseclient.test.ts b/tests/dfs/storage-file-datalake/apis/leaseclient.test.ts new file mode 100644 index 000000000..96445784e --- /dev/null +++ b/tests/dfs/storage-file-datalake/apis/leaseclient.test.ts @@ -0,0 +1,453 @@ +// Copyright (c) Microsoft Corporation. +import assert from "assert"; +import { Context } from "mocha"; + +import { delay } from "@azure/ms-rest-js"; +// Licensed under the MIT license. +import { + DataLakeDirectoryClient, + DataLakeFileClient, + DataLakeFileSystemClient, + DataLakeServiceClient, + StorageSharedKeyCredential +} from "@azure/storage-file-datalake"; + +import { + EMULATOR_ACCOUNT_KEY, + EMULATOR_ACCOUNT_NAME, + getUniqueName +} from "../../../testutils"; +import DataLakeTestServerFactory from "../../DataLakeTestServerFactory"; + +describe("LeaseClient from FileSystem", () => { + const factory = new DataLakeTestServerFactory(); + const server = factory.createServer(); + + const baseURL = `http://${server.config.host}:${server.config.port}/devstoreaccount1`; + const serviceClient = new DataLakeServiceClient( + baseURL, + new StorageSharedKeyCredential(EMULATOR_ACCOUNT_NAME, EMULATOR_ACCOUNT_KEY), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + + let fileSystemName: string; + let fileSystemClient: DataLakeFileSystemClient; + + before(async () => { + await server.start(); + }); + + after(async () => { + await server.close(); + await server.clean(); + }); + + beforeEach(async function (this: Context) { + fileSystemName = getUniqueName("filesystem"); + fileSystemClient = serviceClient.getFileSystemClient(fileSystemName); + await fileSystemClient.createIfNotExists(); + }); + + afterEach(async function () { + await fileSystemClient.deleteIfExists(); + }); + + it("acquireLease @loki @sql", async () => { + const guid = "ca761232ed4211cebacd00aa0057b223"; + const duration = 30; + const leaseClient = fileSystemClient.getDataLakeLeaseClient(guid); + await leaseClient.acquireLease(duration); + + const result = await fileSystemClient.getProperties(); + assert.equal(result.leaseDuration, "fixed"); + assert.equal(result.leaseState, "leased"); + assert.equal(result.leaseStatus, "locked"); + + await leaseClient.releaseLease(); + }); + + it("acquireLease without specifying a lease id @loki @sql", async () => { + const duration = 30; + const leaseClient = fileSystemClient.getDataLakeLeaseClient(); + await leaseClient.acquireLease(duration); + + const result = await fileSystemClient.getProperties(); + assert.equal(result.leaseDuration, "fixed"); + assert.equal(result.leaseState, "leased"); + assert.equal(result.leaseStatus, "locked"); + + await leaseClient.releaseLease(); + }); + + it("releaseLease @loki @sql", async () => { + const guid = "ca761232ed4211cebacd00aa0057b223"; + const duration = -1; + const leaseClient = fileSystemClient.getDataLakeLeaseClient(guid); + await leaseClient.acquireLease(duration); + + const result = await fileSystemClient.getProperties(); + assert.equal(result.leaseDuration, "infinite"); + assert.equal(result.leaseState, "leased"); + assert.equal(result.leaseStatus, "locked"); + + await leaseClient.releaseLease(); + }); + + it("renewLease @loki @sql", async () => { + const guid = "ca761232ed4211cebacd00aa0057b223"; + const duration = 15; + const leaseClient = fileSystemClient.getDataLakeLeaseClient(guid); + await leaseClient.acquireLease(duration); + + const result = await fileSystemClient.getProperties(); + assert.equal(result.leaseDuration, "fixed"); + assert.equal(result.leaseState, "leased"); + assert.equal(result.leaseStatus, "locked"); + + await delay(20 * 1000); + const result2 = await fileSystemClient.getProperties(); + assert.ok(!result2.leaseDuration); + assert.equal(result2.leaseState, "expired"); + assert.equal(result2.leaseStatus, "unlocked"); + + await leaseClient.renewLease(); + const result3 = await fileSystemClient.getProperties(); + assert.equal(result3.leaseDuration, "fixed"); + assert.equal(result3.leaseState, "leased"); + assert.equal(result3.leaseStatus, "locked"); + + await leaseClient.releaseLease(); + }); + + it("changeLease @loki @sql", async () => { + const guid = "ca761232ed4211cebacd00aa0057b223"; + const duration = 15; + const leaseClient = fileSystemClient.getDataLakeLeaseClient(guid); + await leaseClient.acquireLease(duration); + + const result = await fileSystemClient.getProperties(); + assert.equal(result.leaseDuration, "fixed"); + assert.equal(result.leaseState, "leased"); + assert.equal(result.leaseStatus, "locked"); + + const newGuid = "3c7e72ebb4304526bc53d8ecef03798f"; + await leaseClient.changeLease(newGuid); + + await fileSystemClient.getProperties(); + await leaseClient.releaseLease(); + }); + + it("breakLease @loki @sql", async () => { + const guid = "ca761232ed4211cebacd00aa0057b223"; + const duration = 15; + const leaseClient = fileSystemClient.getDataLakeLeaseClient(guid); + await leaseClient.acquireLease(duration); + + const result = await fileSystemClient.getProperties(); + assert.equal(result.leaseDuration, "fixed"); + assert.equal(result.leaseState, "leased"); + assert.equal(result.leaseStatus, "locked"); + + await leaseClient.breakLease(3); + + const result2 = await fileSystemClient.getProperties(); + assert.ok(!result2.leaseDuration); + assert.equal(result2.leaseState, "breaking"); + assert.equal(result2.leaseStatus, "locked"); + + await delay(3 * 1000); + + const result3 = await fileSystemClient.getProperties(); + assert.ok(!result3.leaseDuration); + assert.equal(result3.leaseState, "broken"); + assert.equal(result3.leaseStatus, "unlocked"); + }); +}); + +describe("LeaseClient from File", () => { + const factory = new DataLakeTestServerFactory(); + const server = factory.createServer(); + + const baseURL = `http://${server.config.host}:${server.config.port}/devstoreaccount1`; + const serviceClient = new DataLakeServiceClient( + baseURL, + new StorageSharedKeyCredential(EMULATOR_ACCOUNT_NAME, EMULATOR_ACCOUNT_KEY), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + + let fileSystemName: string; + let fileSystemClient: DataLakeFileSystemClient; + let fileName: string; + let fileClient: DataLakeFileClient; + + before(async () => { + await server.start(); + }); + + after(async () => { + await server.close(); + await server.clean(); + }); + + beforeEach(async function (this: Context) { + fileSystemName = getUniqueName("filesystem"); + fileSystemClient = serviceClient.getFileSystemClient(fileSystemName); + await fileSystemClient.createIfNotExists(); + fileName = getUniqueName("file"); + fileClient = fileSystemClient.getFileClient(fileName); + await fileClient.create(); + }); + + afterEach(async function () { + await fileSystemClient.deleteIfExists(); + }); + + it("acquireLease @loki @sql", async () => { + const guid = "ca761232ed4211cebacd00aa0057b223"; + const duration = 30; + const leaseClient = fileClient.getDataLakeLeaseClient(guid); + await leaseClient.acquireLease(duration); + + const result = await fileClient.getProperties(); + assert.equal(result.leaseDuration, "fixed"); + assert.equal(result.leaseState, "leased"); + assert.equal(result.leaseStatus, "locked"); + + await leaseClient.releaseLease(); + }); + + it("releaseLease @loki @sql", async () => { + const guid = "ca761232ed4211cebacd00aa0057b223"; + const duration = -1; + const leaseClient = fileClient.getDataLakeLeaseClient(guid); + await leaseClient.acquireLease(duration); + + const result = await fileClient.getProperties(); + assert.equal(result.leaseDuration, "infinite"); + assert.equal(result.leaseState, "leased"); + assert.equal(result.leaseStatus, "locked"); + + await leaseClient.releaseLease(); + }); + + it("renewLease @loki @sql", async () => { + const guid = "ca761232ed4211cebacd00aa0057b223"; + const duration = 15; + const leaseClient = fileClient.getDataLakeLeaseClient(guid); + await leaseClient.acquireLease(duration); + + const result = await fileClient.getProperties(); + assert.equal(result.leaseDuration, "fixed"); + assert.equal(result.leaseState, "leased"); + assert.equal(result.leaseStatus, "locked"); + + await delay(20 * 1000); + + const result2 = await fileClient.getProperties(); + assert.ok(!result2.leaseDuration); + assert.equal(result2.leaseState, "expired"); + // assert.equal(result2.leaseStatus, "unlocked"); // TODO: Potential bug of server which returns "locked" for "expired" lease + + await leaseClient.renewLease(); + const result3 = await fileClient.getProperties(); + assert.equal(result3.leaseDuration, "fixed"); + assert.equal(result3.leaseState, "leased"); + assert.equal(result3.leaseStatus, "locked"); + + await leaseClient.releaseLease(); + }); + + it("changeLease @loki @sql", async () => { + const guid = "ca761232ed4211cebacd00aa0057b223"; + const duration = 15; + const leaseClient = fileClient.getDataLakeLeaseClient(guid); + await leaseClient.acquireLease(duration); + + const result = await fileClient.getProperties(); + assert.equal(result.leaseDuration, "fixed"); + assert.equal(result.leaseState, "leased"); + assert.equal(result.leaseStatus, "locked"); + + const newGuid = "3c7e72ebb4304526bc53d8ecef03798f"; + await leaseClient.changeLease(newGuid); + + await fileClient.getProperties(); + await leaseClient.releaseLease(); + }); + + it("breakLease @loki @sql", async () => { + const guid = "ca761232ed4211cebacd00aa0057b223"; + const duration = 15; + const leaseClient = fileClient.getDataLakeLeaseClient(guid); + await leaseClient.acquireLease(duration); + + const result = await fileClient.getProperties(); + assert.equal(result.leaseDuration, "fixed"); + assert.equal(result.leaseState, "leased"); + assert.equal(result.leaseStatus, "locked"); + + await leaseClient.breakLease(5); + + const result2 = await fileClient.getProperties(); + assert.ok(!result2.leaseDuration); + assert.equal(result2.leaseState, "breaking"); + assert.equal(result2.leaseStatus, "locked"); + + await delay(5 * 1000); + + const result3 = await fileClient.getProperties(); + assert.ok(!result3.leaseDuration); + assert.equal(result3.leaseState, "broken"); + // assert.equal(result3.leaseStatus, "unlocked"); // TODO: Potential bug of server which returns "locked" for "broken" lease + }); +}); + +describe("LeaseClient from Directory", () => { + const factory = new DataLakeTestServerFactory(); + const server = factory.createServer(); + + const baseURL = `http://${server.config.host}:${server.config.port}/devstoreaccount1`; + const serviceClient = new DataLakeServiceClient( + baseURL, + new StorageSharedKeyCredential(EMULATOR_ACCOUNT_NAME, EMULATOR_ACCOUNT_KEY), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + + let fileSystemName: string; + let fileSystemClient: DataLakeFileSystemClient; + let directoryName: string; + let directoryClient: DataLakeDirectoryClient; + + before(async () => { + await server.start(); + }); + + after(async () => { + await server.close(); + await server.clean(); + }); + + beforeEach(async function (this: Context) { + fileSystemName = getUniqueName("filesystem"); + fileSystemClient = serviceClient.getFileSystemClient(fileSystemName); + await fileSystemClient.createIfNotExists(); + directoryName = getUniqueName("dir"); + directoryClient = fileSystemClient.getDirectoryClient(directoryName); + await directoryClient.create(); + }); + + afterEach(async function () { + await fileSystemClient.deleteIfExists(); + }); + + it("acquireLease @loki @sql", async () => { + const guid = "ca761232ed4211cebacd00aa0057b223"; + const duration = 30; + const leaseClient = directoryClient.getDataLakeLeaseClient(guid); + await leaseClient.acquireLease(duration); + + const result = await directoryClient.getProperties(); + assert.equal(result.leaseDuration, "fixed"); + assert.equal(result.leaseState, "leased"); + assert.equal(result.leaseStatus, "locked"); + + await leaseClient.releaseLease(); + }); + + it("releaseLease @loki @sql", async () => { + const guid = "ca761232ed4211cebacd00aa0057b223"; + const duration = -1; + const leaseClient = directoryClient.getDataLakeLeaseClient(guid); + await leaseClient.acquireLease(duration); + + const result = await directoryClient.getProperties(); + assert.equal(result.leaseDuration, "infinite"); + assert.equal(result.leaseState, "leased"); + assert.equal(result.leaseStatus, "locked"); + + await leaseClient.releaseLease(); + }); + + it("renewLease @loki @sql", async () => { + const guid = "ca761232ed4211cebacd00aa0057b223"; + const duration = 15; + const leaseClient = directoryClient.getDataLakeLeaseClient(guid); + await leaseClient.acquireLease(duration); + + const result = await directoryClient.getProperties(); + assert.equal(result.leaseDuration, "fixed"); + assert.equal(result.leaseState, "leased"); + assert.equal(result.leaseStatus, "locked"); + + await delay(20 * 1000); + + const result2 = await directoryClient.getProperties(); + assert.ok(!result2.leaseDuration); + assert.equal(result2.leaseState, "expired"); + // assert.equal(result2.leaseStatus, "unlocked"); // TODO: Potential bug of server which returns "locked" for "expired" lease + + await leaseClient.renewLease(); + const result3 = await directoryClient.getProperties(); + assert.equal(result3.leaseDuration, "fixed"); + assert.equal(result3.leaseState, "leased"); + assert.equal(result3.leaseStatus, "locked"); + + await leaseClient.releaseLease(); + }); + + it("changeLease @loki @sql", async () => { + const guid = "ca761232ed4211cebacd00aa0057b223"; + const duration = 15; + const leaseClient = directoryClient.getDataLakeLeaseClient(guid); + await leaseClient.acquireLease(duration); + + const result = await directoryClient.getProperties(); + assert.equal(result.leaseDuration, "fixed"); + assert.equal(result.leaseState, "leased"); + assert.equal(result.leaseStatus, "locked"); + + const newGuid = "3c7e72ebb4304526bc53d8ecef03798f"; + await leaseClient.changeLease(newGuid); + + await directoryClient.getProperties(); + await leaseClient.releaseLease(); + }); + + it("breakLease @loki @sql", async () => { + const guid = "ca761232ed4211cebacd00aa0057b223"; + const duration = 15; + const leaseClient = directoryClient.getDataLakeLeaseClient(guid); + await leaseClient.acquireLease(duration); + + const result = await directoryClient.getProperties(); + assert.equal(result.leaseDuration, "fixed"); + assert.equal(result.leaseState, "leased"); + assert.equal(result.leaseStatus, "locked"); + + await leaseClient.breakLease(15); + + const result2 = await directoryClient.getProperties(); + assert.ok(!result2.leaseDuration); + assert.equal(result2.leaseState, "breaking"); + assert.equal(result2.leaseStatus, "locked"); + + await delay(15 * 1000); + + const result3 = await directoryClient.getProperties(); + assert.ok(!result3.leaseDuration); + assert.equal(result3.leaseState, "broken"); + // assert.equal(result3.leaseStatus, "unlocked"); // TODO: Potential bug of server which returns "locked" for "broken" lease + }); +}); diff --git a/tests/dfs/storage-file-datalake/apis/pathclient.test.ts b/tests/dfs/storage-file-datalake/apis/pathclient.test.ts new file mode 100644 index 000000000..a101f1fbf --- /dev/null +++ b/tests/dfs/storage-file-datalake/apis/pathclient.test.ts @@ -0,0 +1,1857 @@ +// Copyright (c) Microsoft Corporation. +import assert from "assert"; +import { Context } from "mocha"; + +import { delay } from "@azure/ms-rest-js"; +// Licensed under the MIT license. +import { + DataLakeDirectoryClient, + DataLakeFileClient, + DataLakeFileSystemClient, + DataLakeServiceClient, + StorageSharedKeyCredential +} from "@azure/storage-file-datalake"; + +import { + bodyToString, + EMULATOR_ACCOUNT_KEY, + EMULATOR_ACCOUNT_NAME, + getEncryptionScope, + getUniqueName, + sleep, + Test_CPK_INFO +} from "../../../testutils"; +import DataLakeTestServerFactory from "../../DataLakeTestServerFactory"; +import { AbortController } from "@azure/abort-controller"; +import { toPermissionsString } from "../../../../src/dfs/storage-file-datalake/transforms"; + +describe("DataLakePathClient", () => { + const factory = new DataLakeTestServerFactory(); + const server = factory.createServer(true); + + const baseURL = `http://${server.config.host}:${server.config.port}/devstoreaccount1`; + const serviceClient = new DataLakeServiceClient( + baseURL, + new StorageSharedKeyCredential(EMULATOR_ACCOUNT_NAME, EMULATOR_ACCOUNT_KEY), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + + let fileSystemName: string; + let fileSystemClient: DataLakeFileSystemClient; + let fileName: string; + let fileClient: DataLakeFileClient; + const content = "Hello World"; + + before(async () => { + await server.start(); + }); + + after(async () => { + await server.close(); + await server.clean(); + }); + + beforeEach(async function () { + fileSystemName = getUniqueName("filesystem"); + fileSystemClient = serviceClient.getFileSystemClient(fileSystemName); + await fileSystemClient.createIfNotExists(); + fileName = getUniqueName("file"); + fileClient = fileSystemClient.getFileClient(fileName); + await fileClient.create(); + await fileClient.append(content, 0, content.length); + await fileClient.flush(content.length); + }); + + afterEach(async function () { + await fileSystemClient.deleteIfExists(); + }); + + it("DataLakeFileClient create with meta data @loki @sql", async () => { + const testFileName = getUniqueName("testfile"); + const testFileClient = fileSystemClient.getFileClient(testFileName); + const metadata = { + a: "a", + b: "b" + }; + + await testFileClient.create({ metadata: metadata }); + const result = await testFileClient.getProperties(); + assert.deepStrictEqual(result.metadata, metadata); + }); + + it("DataLakeFileClient create with permission and umark @loki @sql", async () => { + const testFileName = getUniqueName("testfile"); + const testFileClient = fileSystemClient.getFileClient(testFileName); + const permissionString = "0777"; + const umask = "0057"; + + await testFileClient.create({ + permissions: permissionString, + umask: umask + }); + const result = await testFileClient.getAccessControl(); + const permissions = { + owner: { + read: true, + write: true, + execute: true + }, + group: { + read: false, + write: true, + execute: false + }, + other: { + read: false, + write: false, + execute: false + }, + stickyBit: false, + extendedAcls: false + }; + assert.deepEqual(result.permissions, permissions); + }); + + it("DataLakeFileClient create with headers @loki @sql", async () => { + const testFileName = getUniqueName("testfile"); + const testFileClient = fileSystemClient.getFileClient(testFileName); + const httpHeader = { + cacheControl: "control", + contentEncoding: "encoding", + contentLanguage: "language", + contentDisposition: "disposition", + contentType: "type/subtype" + }; + + await testFileClient.create({ pathHttpHeaders: httpHeader }); + const result = await testFileClient.getProperties(); + assert.equal(result.cacheControl, httpHeader.cacheControl); + assert.equal(result.contentEncoding, httpHeader.contentEncoding); + assert.equal(result.contentLanguage, httpHeader.contentLanguage); + assert.equal(result.contentDisposition, httpHeader.contentDisposition); + assert.equal(result.contentType, httpHeader.contentType); + }); + + it("DataLakeFileClient create with leaseId @loki @sql", async () => { + const testFileName = getUniqueName("testfile"); + const testFileClient = fileSystemClient.getFileClient(testFileName); + const leaseId = "25180729-00c9-42b0-938b-ecabce67a007"; + const leaseDuration = 20; + + await testFileClient.create({ + proposedLeaseId: leaseId, + leaseDuration: leaseDuration + }); + const result = await testFileClient.getProperties(); + assert.equal(result.leaseDuration, "fixed"); + assert.equal(result.leaseState, "leased"); + assert.equal(result.leaseStatus, "locked"); + }); + + it("DataLakeFileClient create with relative expiry @loki @sql", async () => { + const testFileName = getUniqueName("testfile"); + const testFileClient = fileSystemClient.getFileClient(testFileName); + const timeToExpireInMs = 60 * 60 * 1000; // 1hour + await testFileClient.create({ expiresOn: timeToExpireInMs }); + const result = await testFileClient.getProperties(); + assert.equal( + result.createdOn!.getTime() + 1000 * 3600, + result.expiresOn!.getTime() + ); + }); + + it("DataLakeFileClient create with absolute expiry @loki @sql", async () => { + const now = new Date(); + const delta = 2 * 1000; + const expiresOn = new Date(now.getTime() + delta); + + const testFileName = getUniqueName("testfile"); + const testFileClient = fileSystemClient.getFileClient(testFileName); + await testFileClient.create({ expiresOn: expiresOn }); + + const result = await testFileClient.getProperties(); + const recordedExpiresOn = new Date(expiresOn.getTime()); + recordedExpiresOn.setMilliseconds(0); // milliseconds dropped + assert.equal(result.expiresOn?.getTime(), recordedExpiresOn.getTime()); + + await delay(delta); + assert.ok(!(await testFileClient.exists())); + }); + + it("DataLakeFileClient create with all parameters @loki @sql", async () => { + const metadata = { + a: "a", + b: "b" + }; + + const permissionString = "0777"; + const umask = "0057"; + + const httpHeader = { + cacheControl: "control", + contentEncoding: "encoding", + contentLanguage: "language", + contentDisposition: "disposition", + contentType: "type/subtype" + }; + const leaseId = "25180729-00c9-42b0-938b-ecabce67a007"; + const leaseDuration = 20; + + const timeToExpireInMs = 60 * 1000; // 60s + + const testFileName = getUniqueName("testfile"); + const testFileClient = fileSystemClient.getFileClient(testFileName); + await testFileClient.create({ + metadata: metadata, + permissions: permissionString, + umask: umask, + pathHttpHeaders: httpHeader, + proposedLeaseId: leaseId, + leaseDuration: leaseDuration, + expiresOn: timeToExpireInMs + }); + + const result = await testFileClient.getProperties(); + assert.deepStrictEqual(result.metadata, metadata); + assert.equal( + result.createdOn!.getTime() + 1000 * 60, + result.expiresOn!.getTime() + ); + assert.equal(result.leaseDuration, "fixed"); + assert.equal(result.leaseState, "leased"); + assert.equal(result.leaseStatus, "locked"); + assert.equal(result.cacheControl, httpHeader.cacheControl); + assert.equal(result.contentEncoding, httpHeader.contentEncoding); + assert.equal(result.contentLanguage, httpHeader.contentLanguage); + assert.equal(result.contentDisposition, httpHeader.contentDisposition); + assert.equal(result.contentType, httpHeader.contentType); + const aclResult = await testFileClient.getAccessControl(); + const permissions = { + owner: { + read: true, + write: true, + execute: true + }, + group: { + read: false, + write: true, + execute: false + }, + other: { + read: false, + write: false, + execute: false + }, + stickyBit: false, + extendedAcls: false + }; + assert.deepEqual(aclResult.permissions, permissions); + }); + + it("DataLakeFileClient createIfNotExists with default parameters @loki @sql", async () => { + const testFileName = getUniqueName("testfile"); + const testFileClient = fileSystemClient.getFileClient(testFileName); + + await testFileClient.createIfNotExists(); + assert.ok(await testFileClient.exists()); + }); + + it("DataLakeFileClient createIfNotExists with meta data @loki @sql", async () => { + const testFileName = getUniqueName("testfile"); + const testFileClient = fileSystemClient.getFileClient(testFileName); + const metadata = { + a: "a", + b: "b" + }; + + await testFileClient.createIfNotExists({ metadata: metadata }); + const result = await testFileClient.getProperties(); + assert.deepStrictEqual(result.metadata, metadata); + }); + + it("DataLakeFileClient createIfNotExists with permission and umark @loki @sql", async () => { + const testFileName = getUniqueName("testfile"); + const testFileClient = fileSystemClient.getFileClient(testFileName); + const permissionString = "0777"; + const umask = "0057"; + + await testFileClient.createIfNotExists({ + permissions: permissionString, + umask: umask + }); + const result = await testFileClient.getAccessControl(); + const permissions = { + owner: { + read: true, + write: true, + execute: true + }, + group: { + read: false, + write: true, + execute: false + }, + other: { + read: false, + write: false, + execute: false + }, + stickyBit: false, + extendedAcls: false + }; + assert.deepEqual(result.permissions, permissions); + }); + + it("DataLakeFileClient createIfNotExists with headers @loki @sql", async () => { + const testFileName = getUniqueName("testfile"); + const testFileClient = fileSystemClient.getFileClient(testFileName); + const httpHeader = { + cacheControl: "control", + contentEncoding: "encoding", + contentLanguage: "language", + contentDisposition: "disposition", + contentType: "type/subtype" + }; + + await testFileClient.createIfNotExists({ pathHttpHeaders: httpHeader }); + const result = await testFileClient.getProperties(); + assert.equal(result.cacheControl, httpHeader.cacheControl); + assert.equal(result.contentEncoding, httpHeader.contentEncoding); + assert.equal(result.contentLanguage, httpHeader.contentLanguage); + assert.equal(result.contentDisposition, httpHeader.contentDisposition); + assert.equal(result.contentType, httpHeader.contentType); + }); + + it("DataLakeFileClient createIfNotExists with leaseId @loki @sql", async () => { + const testFileName = getUniqueName("testfile"); + const testFileClient = fileSystemClient.getFileClient(testFileName); + const leaseId = "25180729-00c9-42b0-938b-ecabce67a007"; + const leaseDuration = 20; + + await testFileClient.createIfNotExists({ + proposedLeaseId: leaseId, + leaseDuration: leaseDuration + }); + const result = await testFileClient.getProperties(); + assert.equal(result.leaseDuration, "fixed"); + assert.equal(result.leaseState, "leased"); + assert.equal(result.leaseStatus, "locked"); + }); + + it("DataLakeFileClient createIfNotExists with relative expiry @loki @sql", async () => { + const testFileName = getUniqueName("testfile"); + const testFileClient = fileSystemClient.getFileClient(testFileName); + const timeToExpireInMs = 60 * 60 * 1000; // 1hour + await testFileClient.createIfNotExists({ expiresOn: timeToExpireInMs }); + const result = await testFileClient.getProperties(); + assert.equal( + result.createdOn!.getTime() + 1000 * 3600, + result.expiresOn!.getTime() + ); + }); + + it("DataLakeFileClient createIfNotExists with absolute expiry @loki @sql", async () => { + const now = new Date(); + const delta = 2 * 1000; + const expiresOn = new Date(now.getTime() + delta); + + const testFileName = getUniqueName("testfile"); + const testFileClient = fileSystemClient.getFileClient(testFileName); + await testFileClient.createIfNotExists({ expiresOn: expiresOn }); + + const result = await testFileClient.getProperties(); + const recordedExpiresOn = new Date(expiresOn.getTime()); + recordedExpiresOn.setMilliseconds(0); // milliseconds dropped + assert.equal(result.expiresOn?.getTime(), recordedExpiresOn.getTime()); + + await delay(delta); + assert.ok(!(await testFileClient.exists())); + }); + + it("DataLakeDirectoryClient create with default parameters @loki @sql", async () => { + const testDirName = getUniqueName("testdir"); + const testdirClient = fileSystemClient.getDirectoryClient(testDirName); + await testdirClient.create(); + assert.ok(await testdirClient.exists()); + }); + + it("DataLakeDirectoryClient create with meta data @loki @sql", async () => { + const testDirName = getUniqueName("testdir"); + const testDirClient = fileSystemClient.getDirectoryClient(testDirName); + const metadata = { + a: "a", + b: "b" + }; + + await testDirClient.create({ metadata: metadata }); + const result = await testDirClient.getProperties(); + assert.deepStrictEqual(result.metadata, { + ...metadata, + hdi_isfolder: "true" + }); + }); + + it("DataLakeDirectoryClient create with permission and umark @loki @sql", async () => { + const testDirName = getUniqueName("testdir"); + const testDirClient = fileSystemClient.getDirectoryClient(testDirName); + const permissionString = "0777"; + const umask = "0057"; + + await testDirClient.create({ permissions: permissionString, umask: umask }); + const result = await testDirClient.getAccessControl(); + const permissions = { + owner: { + read: true, + write: true, + execute: true + }, + group: { + read: false, + write: true, + execute: false + }, + other: { + read: false, + write: false, + execute: false + }, + stickyBit: false, + extendedAcls: false + }; + assert.deepEqual(result.permissions, permissions); + }); + + it("DataLakeDirectoryClient create with headers @loki @sql", async () => { + const testDirName = getUniqueName("testdir"); + const testDirClient = fileSystemClient.getDirectoryClient(testDirName); + const httpHeader = { + cacheControl: "control", + contentEncoding: "encoding", + contentLanguage: "language", + contentDisposition: "disposition", + contentType: "type/subtype" + }; + + await testDirClient.create({ pathHttpHeaders: httpHeader }); + const result = await testDirClient.getProperties(); + assert.equal(result.cacheControl, httpHeader.cacheControl); + assert.equal(result.contentEncoding, httpHeader.contentEncoding); + assert.equal(result.contentLanguage, httpHeader.contentLanguage); + assert.equal(result.contentDisposition, httpHeader.contentDisposition); + assert.equal(result.contentType, httpHeader.contentType); + }); + + it("DataLakeDirectoryClient create with leaseId @loki @sql", async () => { + const testDirName = getUniqueName("testdir"); + const testDirClient = fileSystemClient.getDirectoryClient(testDirName); + const leaseId = "25180729-00c9-42b0-938b-ecabce67a007"; + const leaseDuration = 20; + + await testDirClient.create({ + proposedLeaseId: leaseId, + leaseDuration: leaseDuration + }); + const result = await testDirClient.getProperties(); + assert.equal(result.leaseDuration, "fixed"); + assert.equal(result.leaseState, "leased"); + assert.equal(result.leaseStatus, "locked"); + }); + + it("DataLakeDirectoryClient create with all parameters @loki @sql", async () => { + const metadata = { + a: "a", + b: "b" + }; + + const permissionString = "0777"; + const umask = "0057"; + + const httpHeader = { + cacheControl: "control", + contentEncoding: "encoding", + contentLanguage: "language", + contentDisposition: "disposition", + contentType: "type/subtype" + }; + const leaseId = "25180729-00c9-42b0-938b-ecabce67a007"; + const leaseDuration = 20; + + const testDirName = getUniqueName("testdir"); + const testDirClient = fileSystemClient.getFileClient(testDirName); + await testDirClient.create({ + metadata: metadata, + permissions: permissionString, + umask: umask, + pathHttpHeaders: httpHeader, + proposedLeaseId: leaseId, + leaseDuration: leaseDuration + }); + + const result = await testDirClient.getProperties(); + assert.deepStrictEqual(result.metadata, metadata); + assert.equal(result.leaseDuration, "fixed"); + assert.equal(result.leaseState, "leased"); + assert.equal(result.leaseStatus, "locked"); + assert.equal(result.cacheControl, httpHeader.cacheControl); + assert.equal(result.contentEncoding, httpHeader.contentEncoding); + assert.equal(result.contentLanguage, httpHeader.contentLanguage); + assert.equal(result.contentDisposition, httpHeader.contentDisposition); + assert.equal(result.contentType, httpHeader.contentType); + const aclResult = await testDirClient.getAccessControl(); + const permissions = { + owner: { + read: true, + write: true, + execute: true + }, + group: { + read: false, + write: true, + execute: false + }, + other: { + read: false, + write: false, + execute: false + }, + stickyBit: false, + extendedAcls: false + }; + assert.deepEqual(aclResult.permissions, permissions); + }); + + it("DataLakeDirectoryClient create with relative expiry @loki @sql", async () => { + const testDirName = getUniqueName("testdir"); + const testDirClient = fileSystemClient.getDirectoryClient(testDirName); + const timeToExpireInMs = 60 * 60 * 1000; // 1hour + try { + await testDirClient.create({ expiresOn: timeToExpireInMs }); + assert.fail("Creating directory with expiry should fail."); + } catch (error) { + assert.ok( + (error as any).message.includes( + "Set Expiry is not supported for a directory" + ) + ); + } + }); + + it("DataLakeDirectoryClient create with absolute expiry @loki @sql", async () => { + const now = new Date(); + const delta = 20 * 1000; + const expiresOn = new Date(now.getTime() + delta); + + const testDirName = getUniqueName("testdir"); + const testDirClient = fileSystemClient.getDirectoryClient(testDirName); + + try { + await testDirClient.create({ expiresOn }); + assert.fail("Creating directory with expiry should fail."); + } catch (error) { + assert.ok( + (error as any).message.includes( + "Set Expiry is not supported for a directory" + ) + ); + } + }); + + it("DataLakeDirectoryClient createIfNotExists with default parameters @loki @sql", async () => { + const testDirName = getUniqueName("testdir"); + const testdirClient = fileSystemClient.getDirectoryClient(testDirName); + await testdirClient.createIfNotExists(); + assert.ok(await testdirClient.exists()); + }); + + it("DataLakeDirectoryClient createIfNotExists with meta data @loki @sql", async () => { + const testDirName = getUniqueName("testdir"); + const testDirClient = fileSystemClient.getDirectoryClient(testDirName); + const metadata = { + a: "a", + b: "b" + }; + + await testDirClient.createIfNotExists({ metadata: metadata }); + const result = await testDirClient.getProperties(); + assert.deepStrictEqual(result.metadata, { + ...metadata, + hdi_isfolder: "true" + }); + }); + + it("DataLakeDirectoryClient createIfNotExists with permission and umark @loki @sql", async () => { + const testDirName = getUniqueName("testdir"); + const testDirClient = fileSystemClient.getDirectoryClient(testDirName); + const permissionString = "0777"; + const umask = "0057"; + + await testDirClient.createIfNotExists({ + permissions: permissionString, + umask: umask + }); + const result = await testDirClient.getAccessControl(); + const permissions = { + owner: { + read: true, + write: true, + execute: true + }, + group: { + read: false, + write: true, + execute: false + }, + other: { + read: false, + write: false, + execute: false + }, + stickyBit: false, + extendedAcls: false + }; + assert.deepEqual(result.permissions, permissions); + }); + + it("DataLakeDirectoryClient createIfNotExists with headers @loki @sql", async () => { + const testDirName = getUniqueName("testdir"); + const testDirClient = fileSystemClient.getDirectoryClient(testDirName); + const httpHeader = { + cacheControl: "control", + contentEncoding: "encoding", + contentLanguage: "language", + contentDisposition: "disposition", + contentType: "type/subtype" + }; + + await testDirClient.createIfNotExists({ pathHttpHeaders: httpHeader }); + const result = await testDirClient.getProperties(); + assert.equal(result.cacheControl, httpHeader.cacheControl); + assert.equal(result.contentEncoding, httpHeader.contentEncoding); + assert.equal(result.contentLanguage, httpHeader.contentLanguage); + assert.equal(result.contentDisposition, httpHeader.contentDisposition); + assert.equal(result.contentType, httpHeader.contentType); + }); + + it("DataLakeDirectoryClient createIfNotExists with leaseId @loki @sql", async () => { + const testDirName = getUniqueName("testdir"); + const testDirClient = fileSystemClient.getDirectoryClient(testDirName); + const leaseId = "25180729-00c9-42b0-938b-ecabce67a007"; + const leaseDuration = 20; + + await testDirClient.createIfNotExists({ + proposedLeaseId: leaseId, + leaseDuration: leaseDuration + }); + const result = await testDirClient.getProperties(); + assert.equal(result.leaseDuration, "fixed"); + assert.equal(result.leaseState, "leased"); + assert.equal(result.leaseStatus, "locked"); + }); + + it("DataLakeDirectoryClient createIfNotExists with relative expiry @loki @sql", async () => { + const testDirName = getUniqueName("testdir"); + const testDirClient = fileSystemClient.getDirectoryClient(testDirName); + const timeToExpireInMs = 60 * 60 * 1000; // 1hour + try { + await testDirClient.createIfNotExists({ expiresOn: timeToExpireInMs }); + assert.fail("Creating directory with expiry should fail."); + } catch (error) { + assert.ok( + (error as any).message.includes( + "Set Expiry is not supported for a directory" + ) + ); + } + }); + + it("DataLakeDirectoryClient createIfNotExists with absolute expiry @loki @sql", async () => { + const now = new Date(); + const delta = 20 * 1000; + const expiresOn = new Date(now.getTime() + delta); + + const testDirName = getUniqueName("testdir"); + const testDirClient = fileSystemClient.getDirectoryClient(testDirName); + + try { + await testDirClient.createIfNotExists({ expiresOn: expiresOn }); + assert.fail("Creating directory with expiry should fail."); + } catch (error) { + assert.ok( + (error as any).message.includes( + "Set Expiry is not supported for a directory" + ) + ); + } + }); + + it("read with with default parameters @loki @sql", async () => { + const result = await fileClient.read(); + const read = await bodyToString(result, content.length); + assert.deepStrictEqual(read, content); + }); + + it("read should not have aborted error after read finishes @loki @sql", async () => { + const aborter = new AbortController(); + const result = await fileClient.read(0, undefined, { + abortSignal: aborter.signal + }); + const read = await bodyToString(result, content.length); + assert.deepStrictEqual(read, content); + aborter.abort(); + }); + + it("read all parameters set @loki @sql", async () => { + // For browser scenario, please ensure CORS settings exposed headers: content-md5,x-ms-content-crc64 + // So JS can get contentCrc64 and contentMD5. + const result1 = await fileClient.read(0, 1, { + rangeGetContentCrc64: true + }); + assert.ok(result1.clientRequestId); + // assert.ok(result1.contentCrc64!); + assert.deepStrictEqual(await bodyToString(result1, 1), content[0]); + assert.ok(result1.clientRequestId); + + const result2 = await fileClient.read(1, 1, { + rangeGetContentMD5: true + }); + assert.ok(result2.clientRequestId); + assert.ok(result2.contentMD5!); + + let exceptionCaught = false; + try { + await fileClient.read(2, 1, { + rangeGetContentMD5: true, + rangeGetContentCrc64: true + }); + } catch (err: any) { + exceptionCaught = true; + } + assert.ok(exceptionCaught); + }); + + it("setMetadata with new metadata set @loki @sql", async () => { + const metadata = { + a: "a", + b: "b" + }; + await fileClient.setMetadata(metadata); + const result = await fileClient.getProperties(); + assert.deepStrictEqual(result.metadata, metadata); + }); + + it("setMetadata with cleaning up metadata @loki @sql", async () => { + const metadata = { + a: "a", + b: "b" + }; + await fileClient.setMetadata(metadata); + const result = await fileClient.getProperties(); + assert.deepStrictEqual(result.metadata, metadata); + + await fileClient.setMetadata(); + const result2 = await fileClient.getProperties(); + assert.deepStrictEqual(result2.metadata, {}); + }); + + it("setHttpHeaders with default parameters @loki @sql", async () => { + await fileClient.setHttpHeaders({}); + const result = await fileClient.getProperties(); + + assert.ok(result.lastModified); + assert.deepStrictEqual(result.metadata, {}); + assert.ok(!result.cacheControl); + assert.ok(!result.contentType); + assert.ok(!result.contentMD5); + assert.ok(!result.contentEncoding); + assert.ok(!result.contentLanguage); + assert.ok(!result.contentDisposition); + }); + + it("setHttpHeaders with all parameters set @loki @sql", async () => { + const headers = { + cacheControl: "cacheControl", + contentDisposition: "contentDisposition", + contentEncoding: "contentEncoding", + contentLanguage: "contentLanguage", + contentMD5: new Uint8Array([1, 2, 3, 4]), + contentType: "contentType" + }; + await fileClient.setHttpHeaders(headers); + const result = await fileClient.getProperties(); + assert.ok(result.date); + + assert.ok(result.lastModified); + assert.deepStrictEqual(result.metadata, {}); + assert.deepStrictEqual(result.cacheControl, headers.cacheControl); + assert.deepStrictEqual(result.contentType, headers.contentType); + assert.deepStrictEqual( + Buffer.from(result.contentMD5!, 0), + Buffer.from(headers.contentMD5, 0) + ); + assert.deepStrictEqual(result.contentEncoding, headers.contentEncoding); + assert.deepStrictEqual(result.contentLanguage, headers.contentLanguage); + assert.deepStrictEqual( + result.contentDisposition, + headers.contentDisposition + ); + }); + + it("delete @loki @sql", async () => { + await fileClient.delete(); + }); + + it("read with default parameters and tracing @loki @sql", async () => { + const result = await fileClient.read(undefined, undefined); + assert.deepStrictEqual(await bodyToString(result, content.length), content); + }); + + it("verify fileName and fileSystemName passed to the client @loki @sql", async () => { + const accountName = "myaccount"; + const path = "file/part/1.txt"; + const newClient = new DataLakeFileClient( + `https://${accountName}.dfs.core.windows.net/` + + fileSystemName + + "/" + + path + ); + assert.equal( + newClient.fileSystemName, + fileSystemName, + "File system name is not the same as the one provided." + ); + assert.equal( + newClient.name, + path, + "File name is not the same as the one provided." + ); + assert.equal( + newClient.accountName, + accountName, + "Account name is not the same as the one provided." + ); + }); + + it("append with acquire lease @loki @sql", async () => { + const body = "HelloWorld"; + + const tempFileName = getUniqueName("tempfile2"); + const tempFileClient = fileSystemClient.getFileClient(tempFileName); + const leaseId = "ca761232ed4211cebacd00aa0057b223"; + + await tempFileClient.create(); + + await tempFileClient.append(body, 0, body.length, { + proposedLeaseId: leaseId, + leaseDurationInSeconds: 15, + leaseAction: "acquire" + }); + + let gotError = false; + try { + await tempFileClient.append(body, body.length, body.length, { + flush: true + }); + } catch (err) { + gotError = true; + assert.ok( + err.message.startsWith( + "There is currently a lease on the resource and no lease ID was specified in the request." + ) + ); + } + assert.ok( + gotError, + "Should throw out an exception to write to a leased file without lease id" + ); + + await tempFileClient.append(body, body.length, body.length, { + conditions: { + leaseId: leaseId + }, + flush: true + }); + + const properties = await tempFileClient.getProperties(); + assert.equal(properties.contentLength, body.length * 2); + assert.equal(properties.leaseState, "leased"); + assert.equal(properties.leaseDuration, "fixed"); + assert.equal(properties.leaseStatus, "locked"); + + await tempFileClient.delete(false, { + conditions: { + leaseId: leaseId + } + }); + }); + + it("append with auto-renew lease @loki @sql", async () => { + const body = "HelloWorld"; + + const tempFileName = getUniqueName("tempfile2"); + const tempFileClient = fileSystemClient.getFileClient(tempFileName); + const leaseId = "ca761232ed4211cebacd00aa0057b223"; + + await tempFileClient.create({ + proposedLeaseId: leaseId, + leaseDuration: 15 + }); + + const properties = await tempFileClient.getProperties(); + assert.equal(properties.leaseState, "leased"); + assert.equal(properties.leaseDuration, "fixed"); + assert.equal(properties.leaseStatus, "locked"); + + await sleep(15); + + await tempFileClient.append(body, 0, body.length, { + conditions: { leaseId: leaseId }, + leaseDurationInSeconds: 15, + leaseAction: "auto-renew" + }); + + let gotError = false; + try { + await tempFileClient.append(body, body.length, body.length, { + flush: true + }); + } catch (err) { + gotError = true; + assert.ok( + err.message.startsWith( + "There is currently a lease on the resource and no lease ID was specified in the request." + ) + ); + } + assert.ok( + gotError, + "Should throw out an exception to write to a leased file without lease id" + ); + + await tempFileClient.delete(false, { + conditions: { + leaseId: leaseId + } + }); + }); + + it("append with release lease @loki @sql", async () => { + const body = "HelloWorld"; + + const tempFileName = getUniqueName("tempfile2"); + const tempFileClient = fileSystemClient.getFileClient(tempFileName); + const leaseId = "ca761232ed4211cebacd00aa0057b223"; + + await tempFileClient.create({ + proposedLeaseId: leaseId, + leaseDuration: 15 + }); + + await tempFileClient.append(body, 0, body.length, { + conditions: { leaseId: leaseId } + }); + + await tempFileClient.append(body, body.length, body.length, { + conditions: { leaseId: leaseId }, + leaseAction: "release", + flush: true + }); + + const properties = await tempFileClient.getProperties(); + assert.equal(properties.leaseState, "available"); + assert.equal(properties.leaseStatus, "unlocked"); + + await tempFileClient.delete(); + }); + + it("flush with acquire lease @loki @sql", async () => { + const body = "HelloWorld"; + + const tempFileName = getUniqueName("tempfile2"); + const tempFileClient = fileSystemClient.getFileClient(tempFileName); + const leaseId = "ca761232ed4211cebacd00aa0057b223"; + + await tempFileClient.create(); + + await tempFileClient.append(body, 0, body.length); + await tempFileClient.append(body, body.length, body.length); + + await tempFileClient.flush(body.length * 2, { + proposedLeaseId: leaseId, + leaseDurationInSeconds: 15, + leaseAction: "acquire" + }); + + let gotError = false; + try { + await tempFileClient.delete(); + } catch (err) { + gotError = true; + assert.ok( + err.message.startsWith( + "There is currently a lease on the resource and no lease ID was specified in the request." + ) + ); + } + assert.ok( + gotError, + "Should throw out an exception to write to a leased file without lease id" + ); + + const properties = await tempFileClient.getProperties(); + assert.equal(properties.contentLength, body.length * 2); + assert.equal(properties.leaseState, "leased"); + assert.equal(properties.leaseDuration, "fixed"); + assert.equal(properties.leaseStatus, "locked"); + + await tempFileClient.delete(false, { + conditions: { + leaseId: leaseId + } + }); + }); + + it("flush with auto-renew lease @loki @sql", async () => { + const body = "HelloWorld"; + + const tempFileName = getUniqueName("tempfile2"); + const tempFileClient = fileSystemClient.getFileClient(tempFileName); + const leaseId = "ca761232ed4211cebacd00aa0057b223"; + + await tempFileClient.create({ + proposedLeaseId: leaseId, + leaseDuration: 15 + }); + + await tempFileClient.append(body, 0, body.length, { + conditions: { leaseId: leaseId } + }); + await tempFileClient.append(body, body.length, body.length, { + conditions: { leaseId: leaseId } + }); + + await sleep(15); + + await tempFileClient.flush(body.length * 2, { + conditions: { leaseId: leaseId }, + leaseDurationInSeconds: 15, + leaseAction: "auto-renew" + }); + + let gotError = false; + try { + await tempFileClient.delete(); + } catch (err) { + gotError = true; + assert.ok( + err.message.startsWith( + "There is currently a lease on the resource and no lease ID was specified in the request." + ) + ); + } + assert.ok( + gotError, + "Should throw out an exception to write to a leased file without lease id" + ); + + const properties = await tempFileClient.getProperties(); + assert.equal(properties.contentLength, body.length * 2); + assert.equal(properties.leaseState, "leased"); + assert.equal(properties.leaseDuration, "fixed"); + assert.equal(properties.leaseStatus, "locked"); + + await tempFileClient.delete(false, { + conditions: { + leaseId: leaseId + } + }); + }); + + it("flush with release lease @loki @sql", async () => { + const body = "HelloWorld"; + + const tempFileName = getUniqueName("tempfile2"); + const tempFileClient = fileSystemClient.getFileClient(tempFileName); + const leaseId = "ca761232ed4211cebacd00aa0057b223"; + + await tempFileClient.create({ + proposedLeaseId: leaseId, + leaseDuration: 15 + }); + + await tempFileClient.append(body, 0, body.length, { + conditions: { leaseId: leaseId } + }); + await tempFileClient.append(body, body.length, body.length, { + conditions: { leaseId: leaseId } + }); + + await tempFileClient.flush(body.length * 2, { + conditions: { leaseId: leaseId }, + leaseAction: "release" + }); + + const properties = await tempFileClient.getProperties(); + assert.equal(properties.leaseState, "available"); + assert.equal(properties.leaseStatus, "unlocked"); + + await tempFileClient.delete(); + }); + + it("append with flush should work @loki @sql", async () => { + const body = "HelloWorld"; + + const tempFileName = getUniqueName("tempfile2"); + const tempFileClient = fileSystemClient.getFileClient(tempFileName); + + await tempFileClient.create(); + + await tempFileClient.append(body, 0, body.length, { + transactionalContentMD5: new Uint8Array([]) + }); + await tempFileClient.append(body, body.length, body.length, { + transactionalContentMD5: new Uint8Array([]) + }); + await tempFileClient.append(body, body.length * 2, body.length, { + transactionalContentMD5: new Uint8Array([]), + flush: true + }); + + const properties = await tempFileClient.getProperties(); + assert.deepStrictEqual(properties.contentLength, body.length * 3); + + await tempFileClient.delete(); + }); + + it("append & flush should work @loki @sql", async () => { + const body = "HelloWorld"; + + const tempFileName = getUniqueName("tempfile2"); + const tempFileClient = fileSystemClient.getFileClient(tempFileName); + + await tempFileClient.create(); + + await tempFileClient.append(body, 0, body.length, { + transactionalContentMD5: new Uint8Array([]) + }); + await tempFileClient.append(body, body.length, body.length, { + transactionalContentMD5: new Uint8Array([]) + }); + await tempFileClient.append(body, body.length * 2, body.length, { + transactionalContentMD5: new Uint8Array([]) + }); + + await tempFileClient.flush(body.length * 3); + + const properties = await tempFileClient.getProperties(); + assert.deepStrictEqual(properties.contentLength, body.length * 3); + + await tempFileClient.delete(); + }); + + it("append & flush should work with all parameters @loki @sql", async () => { + const body = "HelloWorld"; + + const tempFileName = getUniqueName("tempfile2"); + const tempFileClient = fileSystemClient.getFileClient(tempFileName); + + const permissions = { + owner: { read: false, write: false, execute: false }, + group: { read: false, write: false, execute: false }, + other: { read: false, write: false, execute: false }, + stickyBit: false, + extendedAcls: false + }; + const permissionsString = toPermissionsString(permissions); + const metadata = { + a: "val-a", + b: "val-b" + }; + let pathHttpHeaders = { + cacheControl: "cacheControl", + contentEncoding: "contentEncoding", + contentLanguage: "contentLanguage", + contentDisposition: "contentDisposition", + contentType: "contentType" + }; + await tempFileClient.create({ + permissions: permissionsString, + metadata, + umask: "0000", + pathHttpHeaders + }); + + let properties = await tempFileClient.getProperties(); + assert.deepStrictEqual(properties.contentLength, 0); + assert.deepStrictEqual( + properties.cacheControl, + pathHttpHeaders.cacheControl + ); + assert.deepStrictEqual( + properties.contentEncoding, + pathHttpHeaders.contentEncoding + ); + assert.deepStrictEqual( + properties.contentLanguage, + pathHttpHeaders.contentLanguage + ); + assert.deepStrictEqual( + properties.contentDisposition, + pathHttpHeaders.contentDisposition + ); + assert.deepStrictEqual(properties.contentType, pathHttpHeaders.contentType); + assert.deepStrictEqual(properties.metadata, metadata); + + const acl = await tempFileClient.getAccessControl(); + assert.deepStrictEqual(acl.permissions, permissions); + + await tempFileClient.append(body, 0, body.length, { + transactionalContentMD5: new Uint8Array([]) + }); + await tempFileClient.append(body, body.length, body.length, { + transactionalContentMD5: new Uint8Array([]) + }); + await tempFileClient.append(body, body.length * 2, body.length, { + transactionalContentMD5: new Uint8Array([]) + }); + + pathHttpHeaders = { + cacheControl: "cacheControl2", + contentEncoding: "contentEncoding2", + contentLanguage: "contentLanguage2", + contentDisposition: "contentDisposition2", + contentType: "contentType2" + }; + await tempFileClient.flush(body.length * 3, { + retainUncommittedData: true, + close: true, + pathHttpHeaders + }); + + properties = await tempFileClient.getProperties(); + assert.deepStrictEqual(properties.contentLength, body.length * 3); + assert.deepStrictEqual( + properties.cacheControl, + pathHttpHeaders.cacheControl + ); + assert.deepStrictEqual( + properties.contentEncoding, + pathHttpHeaders.contentEncoding + ); + assert.deepStrictEqual( + properties.contentLanguage, + pathHttpHeaders.contentLanguage + ); + assert.deepStrictEqual( + properties.contentDisposition, + pathHttpHeaders.contentDisposition + ); + assert.deepStrictEqual(properties.contentType, pathHttpHeaders.contentType); + assert.deepStrictEqual(properties.metadata, metadata); + + await tempFileClient.delete(); + }); + + it("exists returns true on an existing file @loki @sql", async () => { + const result = await fileClient.exists(); + assert.ok(result, "exists() should return true for an existing file"); + }); + + it("exists returns false on non-existing file or directory @loki @sql", async () => { + const newFileClient = fileSystemClient.getFileClient( + getUniqueName("newFile") + ); + const result = await newFileClient.exists(); + assert.ok( + result === false, + "exists() should return false for a non-existing file" + ); + + const newDirectoryClient = fileSystemClient.getDirectoryClient( + getUniqueName("newDirectory") + ); + const dirResult = await newDirectoryClient.exists(); + assert.ok( + dirResult === false, + "exists() should return false for a non-existing directory" + ); + }); + + it("DataLakeDirectoryClient-createIfNotExists @loki @sql", async () => { + const directoryName = getUniqueName("dir"); + const directoryClient = fileSystemClient.getDirectoryClient(directoryName); + const res = await directoryClient.createIfNotExists(); + assert.ok(res.succeeded); + + const res2 = await directoryClient.createIfNotExists(); + assert.ok(!res2.succeeded); + assert.equal(res2.errorCode, "PathAlreadyExists"); + }); + + it("DataLakeFileClient-createIfNotExists @loki @sql", async () => { + const res = await fileClient.createIfNotExists(); + assert.ok(!res.succeeded); + assert.equal(res.errorCode, "PathAlreadyExists"); + }); + + it("DataLakePathClient-deleteIfExists @loki @sql", async () => { + const directoryName = getUniqueName("dir"); + const directoryClient = fileSystemClient.getDirectoryClient(directoryName); + const res = await directoryClient.deleteIfExists(); + assert.ok(!res.succeeded); + assert.equal(res.errorCode, "PathNotFound"); + + await directoryClient.create(); + const res2 = await directoryClient.deleteIfExists(); + assert.ok(res2.succeeded); + }); + + it("DataLakePathClient-deleteIfExists when parent not exists @loki @sql", async () => { + const directoryName = getUniqueName("dir"); + const directoryClient = fileSystemClient.getDirectoryClient(directoryName); + const newFileClient = directoryClient.getFileClient(fileName); + const res2 = await newFileClient.deleteIfExists(); + assert.ok(!res2.succeeded); + assert.deepStrictEqual(res2.errorCode, "PathNotFound"); + }); + + it("set expiry - NeverExpire @loki @sql", async () => { + await fileClient.setExpiry("NeverExpire"); + const getRes = await fileClient.getProperties(); + assert.equal(getRes.expiresOn, undefined); + }); + + it("set expiry - Absolute @loki @sql", async () => { + const now = new Date(); // Flaky workaround for the recording to work. + const delta = 5 * 1000; + const expiresOn = new Date(now.getTime() + delta); + await fileClient.setExpiry("Absolute", { expiresOn }); + + const getRes = await fileClient.getProperties(); + const recordedExpiresOn = new Date(expiresOn.getTime()); + recordedExpiresOn.setMilliseconds(0); // milliseconds dropped + assert.equal(getRes.expiresOn?.getTime(), recordedExpiresOn.getTime()); + + await delay(delta); + assert.ok(!(await fileClient.exists())); + }); + + it("set expiry - RelativeToNow @loki @sql", async () => { + const delta = 1000; + await fileClient.setExpiry("RelativeToNow", { timeToExpireInMs: delta }); + + await delay(delta); + assert.ok(!(await fileClient.exists())); + }); + + it("set expiry - RelativeToCreation @loki @sql", async () => { + const delta = 1000 * 3600 + 0.12; + await fileClient.setExpiry("RelativeToCreation", { + timeToExpireInMs: delta + }); + + const getRes = await fileClient.getProperties(); + assert.equal( + getRes.expiresOn?.getTime(), + getRes.createdOn!.getTime() + Math.round(delta) + ); + }); + + it("set expiry - override @loki @sql", async () => { + const delta = 1000 * 3600; + await fileClient.setExpiry("RelativeToCreation", { + timeToExpireInMs: delta + }); + + const getRes = await fileClient.getProperties(); + assert.equal( + getRes.expiresOn?.getTime(), + getRes.createdOn!.getTime() + delta + ); + + await fileClient.setExpiry("NeverExpire"); + const getRes2 = await fileClient.getProperties(); + assert.equal(getRes2.expiresOn, undefined); + }); +}); + +describe.skip("DataLakePathClient with CPK", () => { + const factory = new DataLakeTestServerFactory(); + const server = factory.createServer(); + + const baseURL = `http://${server.config.host}:${server.config.port}/devstoreaccount1`; + const serviceClient = new DataLakeServiceClient( + baseURL, + new StorageSharedKeyCredential(EMULATOR_ACCOUNT_NAME, EMULATOR_ACCOUNT_KEY), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + + let fileSystemName: string; + let fileSystemClient: DataLakeFileSystemClient; + let fileName: string; + let dirName: string; + let fileClient: DataLakeFileClient; + let dirClient: DataLakeDirectoryClient; + const content = "Hello World"; + + before(async () => { + await server.start(); + }); + + after(async () => { + await server.close(); + await server.clean(); + }); + + beforeEach(async function () { + fileSystemName = getUniqueName("filesystem"); + fileSystemClient = serviceClient.getFileSystemClient(fileSystemName); + await fileSystemClient.createIfNotExists(); + fileName = getUniqueName("file"); + fileClient = fileSystemClient.getFileClient(fileName); + dirName = getUniqueName("dir"); + dirClient = fileSystemClient.getDirectoryClient(dirName); + }); + + afterEach(async function () { + await fileSystemClient.deleteIfExists(); + }); + + it("file create, append, flush and read with cpk @loki @sql", async () => { + await fileClient.create({ + customerProvidedKey: Test_CPK_INFO + }); + await fileClient.append(content, 0, content.length, { + customerProvidedKey: Test_CPK_INFO + }); + await fileClient.flush(content.length, { + customerProvidedKey: Test_CPK_INFO + }); + + const result = await fileClient.read(0, undefined, { + customerProvidedKey: Test_CPK_INFO + }); + assert.deepStrictEqual(await bodyToString(result, content.length), content); + }); + + it("file getProperties with CPK @loki @sql", async () => { + await fileClient.create({ + customerProvidedKey: Test_CPK_INFO + }); + + const result = await fileClient.getProperties({ + customerProvidedKey: Test_CPK_INFO + }); + + assert.equal(result.contentLength, 0); + }); + + it("file getProperties without CPK on a file with CPK @loki @sql", async () => { + await fileClient.create({ + customerProvidedKey: Test_CPK_INFO + }); + + let gotError = false; + try { + await fileClient.getProperties(); + } catch (err: any) { + gotError = true; + assert.equal((err as any).statusCode, 409); + } + + assert.ok(gotError, "Should got an error"); + }); + + it("file getProperties with CPK on a file without CPK @loki @sql", async () => { + await fileClient.create(); + + let gotError = false; + + try { + await fileClient.getProperties({ + customerProvidedKey: Test_CPK_INFO + }); + } catch (err: any) { + gotError = true; + assert.equal((err as any).statusCode, 409); + } + + assert.ok(gotError, "Should got an error"); + }); + + it("file exists with CPK @loki @sql", async () => { + await fileClient.create({ + customerProvidedKey: Test_CPK_INFO + }); + + assert.ok( + await fileClient.exists({ + customerProvidedKey: Test_CPK_INFO + }) + ); + }); + + it("file exists with CPK on a file without CPK @loki @sql", async () => { + await fileClient.create(); + + assert.ok( + await fileClient.exists({ + customerProvidedKey: Test_CPK_INFO + }) + ); + }); + + it("file exists without CPK on a file with CPK @loki @sql", async () => { + await fileClient.create({ + customerProvidedKey: Test_CPK_INFO + }); + + assert.ok(await fileClient.exists()); + }); + + it("file append with cpk to a file without CPK @loki @sql", async () => { + await fileClient.create(); + + let gotError = false; + + try { + await fileClient.append(content, 0, content.length, { + customerProvidedKey: Test_CPK_INFO + }); + } catch (err: any) { + gotError = true; + assert.equal((err as any).statusCode, 409); + } + + assert.ok(gotError, "Should got an error"); + }); + + it("file append without cpk to a file with CPK @loki @sql", async () => { + await fileClient.create({ + customerProvidedKey: Test_CPK_INFO + }); + + let gotError = false; + try { + await fileClient.append(content, 0, content.length); + } catch (err: any) { + gotError = true; + assert.equal((err as any).statusCode, 409); + } + + assert.ok(gotError, "Should got an error"); + }); + + it("file flush with cpk to a file without CPK @loki @sql", async () => { + await fileClient.create(); + await fileClient.append(content, 0, content.length); + + let gotError = false; + try { + await fileClient.flush(content.length, { + customerProvidedKey: Test_CPK_INFO + }); + } catch (err: any) { + gotError = true; + assert.equal((err as any).statusCode, 409); + } + assert.ok(gotError, "Should got an error"); + }); + + it("file flush without cpk to a file with CPK @loki @sql", async () => { + await fileClient.create({ + customerProvidedKey: Test_CPK_INFO + }); + await fileClient.append(content, 0, content.length, { + customerProvidedKey: Test_CPK_INFO + }); + + let gotError = false; + try { + await fileClient.flush(content.length); + } catch (err: any) { + gotError = true; + assert.equal((err as any).statusCode, 409); + } + assert.ok(gotError, "Should got an error"); + }); + + it("file read without cpk to a file with CPK @loki @sql", async () => { + await fileClient.create({ + customerProvidedKey: Test_CPK_INFO + }); + await fileClient.append(content, 0, content.length, { + customerProvidedKey: Test_CPK_INFO + }); + await fileClient.flush(content.length, { + customerProvidedKey: Test_CPK_INFO + }); + + let gotError = false; + try { + await fileClient.read(0, undefined); + } catch (err: any) { + gotError = true; + assert.equal((err as any).statusCode, 409); + } + assert.ok(gotError, "Should got an error"); + }); + + it("file read with cpk to a file without CPK @loki @sql", async () => { + await fileClient.create(); + await fileClient.append(content, 0, content.length); + await fileClient.flush(content.length); + + let gotError = false; + try { + await fileClient.read(0, undefined, { + customerProvidedKey: Test_CPK_INFO + }); + } catch (err: any) { + gotError = true; + assert.equal((err as any).statusCode, 409); + } + + assert.ok(gotError, "Should got an error"); + }); + + it("file setMetadata with CPK @loki @sql", async () => { + await fileClient.create({ + customerProvidedKey: Test_CPK_INFO + }); + + const metadata = { + a: "a", + b: "b" + }; + await fileClient.setMetadata(metadata, { + customerProvidedKey: Test_CPK_INFO + }); + const result = await fileClient.getProperties({ + customerProvidedKey: Test_CPK_INFO + }); + assert.deepStrictEqual(result.metadata, metadata); + }); + + it("file setMetadata without cpk to a file with CPK @loki @sql", async () => { + await fileClient.create({ + customerProvidedKey: Test_CPK_INFO + }); + + let gotError = false; + try { + await fileClient.setMetadata({}); + } catch (err: any) { + gotError = true; + assert.equal((err as any).statusCode, 409); + } + assert.ok(gotError, "Should got an error"); + }); + + it("file setMetadata with cpk to a file without CPK @loki @sql", async () => { + await fileClient.create(); + + let gotError = false; + try { + await fileClient.setMetadata( + {}, + { + customerProvidedKey: Test_CPK_INFO + } + ); + } catch (err: any) { + gotError = true; + assert.equal((err as any).statusCode, 409); + } + assert.ok(gotError, "Should got an error"); + }); + + it("directory create and getProperties with CPK @loki @sql", async () => { + await dirClient.create({ + customerProvidedKey: Test_CPK_INFO + }); + await dirClient.getProperties({ + customerProvidedKey: Test_CPK_INFO + }); + }); + + it("directory getProperties with CPK on a directory without CPK @loki @sql", async () => { + await dirClient.create(); + + let gotError = false; + try { + await dirClient.getProperties({ + customerProvidedKey: Test_CPK_INFO + }); + } catch (err: any) { + gotError = true; + assert.equal((err as any).statusCode, 409); + } + assert.ok(gotError, "Should got an error"); + }); + + it("directory getProperties without CPK on a directory with CPK @loki @sql", async () => { + await dirClient.create({ + customerProvidedKey: Test_CPK_INFO + }); + + let gotError = false; + try { + await dirClient.getProperties(); + } catch (err: any) { + gotError = true; + assert.equal((err as any).statusCode, 409); + } + assert.ok(gotError, "Should got an error"); + }); + + it("directory exists with CPK @loki @sql", async () => { + await dirClient.create({ + customerProvidedKey: Test_CPK_INFO + }); + assert.ok( + await dirClient.exists({ + customerProvidedKey: Test_CPK_INFO + }) + ); + }); + + it("directory exists with CPK on a directory without CPK @loki @sql", async () => { + await dirClient.create(); + assert.ok( + await dirClient.exists({ + customerProvidedKey: Test_CPK_INFO + }) + ); + }); + + it("directory exists without CPK on a directory with CPK @loki @sql", async () => { + await dirClient.create({ + customerProvidedKey: Test_CPK_INFO + }); + + assert.ok(await dirClient.exists()); + }); + + it("directory setMetadata with CPK @loki @sql", async () => { + await dirClient.create({ + customerProvidedKey: Test_CPK_INFO + }); + + const metadata = { + a: "a", + b: "b" + }; + await dirClient.setMetadata(metadata, { + customerProvidedKey: Test_CPK_INFO + }); + const result = await dirClient.getProperties({ + customerProvidedKey: Test_CPK_INFO + }); + assert.deepStrictEqual(result.metadata, { + ...metadata, + hdi_isfolder: "true" + }); + }); + + it("directory setMetadata without cpk to a directory with CPK @loki @sql", async () => { + await dirClient.create({ + customerProvidedKey: Test_CPK_INFO + }); + + let gotError = false; + try { + await dirClient.setMetadata({}); + } catch (err: any) { + gotError = true; + assert.equal((err as any).statusCode, 409); + } + assert.ok(gotError, "Should got an error"); + }); + + it("directory setMetadata with cpk to a directory without CPK @loki @sql", async () => { + await dirClient.create(); + + let gotError = false; + try { + await dirClient.setMetadata( + {}, + { + customerProvidedKey: Test_CPK_INFO + } + ); + } catch (err: any) { + gotError = true; + assert.equal((err as any).statusCode, 409); + } + assert.ok(gotError, "Should got an error"); + }); +}); + +describe.skip("DataLakePathClient - Encryption Scope", () => { + const factory = new DataLakeTestServerFactory(); + const server = factory.createServer(); + + const baseURL = `http://${server.config.host}:${server.config.port}/devstoreaccount1`; + const serviceClient = new DataLakeServiceClient( + baseURL, + new StorageSharedKeyCredential(EMULATOR_ACCOUNT_NAME, EMULATOR_ACCOUNT_KEY), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + + let fileSystemName: string; + let fileSystemClient: DataLakeFileSystemClient; + let encryptionScopeName: string; + + before(async () => { + await server.start(); + }); + + after(async () => { + await server.close(); + await server.clean(); + }); + + beforeEach(async function (this: Context) { + encryptionScopeName = getEncryptionScope(); + fileSystemName = getUniqueName("filesystem"); + fileSystemClient = serviceClient.getFileSystemClient(fileSystemName); + await fileSystemClient.createIfNotExists({ + fileSystemEncryptionScope: { + defaultEncryptionScope: encryptionScopeName, + preventEncryptionScopeOverride: true + } + }); + }); + + afterEach(async function () { + await fileSystemClient?.deleteIfExists(); + }); + + it("DataLakeFileClient - getProperties should return Encryption Scope @loki @sql", async () => { + const fileName = getUniqueName("file"); + const fileClient = fileSystemClient.getFileClient(fileName); + await fileClient.create(); + const result = await fileClient.getProperties(); + assert.equal(result.encryptionScope, encryptionScopeName); + }); + + it("DataLakeDirectoryClient - getProperties should return Encryption Scope @loki @sql", async () => { + const dirName = getUniqueName("dir"); + const dirClient = fileSystemClient.getDirectoryClient(dirName); + await dirClient.create(); + const result = await dirClient.getProperties(); + assert.equal(result.encryptionScope, encryptionScopeName); + }); +}); diff --git a/tests/dfs/storage-file-datalake/apis/serviceclient.test.ts b/tests/dfs/storage-file-datalake/apis/serviceclient.test.ts new file mode 100644 index 000000000..deccbd2d2 --- /dev/null +++ b/tests/dfs/storage-file-datalake/apis/serviceclient.test.ts @@ -0,0 +1,620 @@ +// Copyright (c) Microsoft Corporation. +import assert from "assert"; +import { Context } from "mocha"; + +import { delay } from "@azure/ms-rest-js"; +// Licensed under the MIT license. +import { + DataLakeServiceClient, + DataLakeServiceProperties, + FileSystemItem, + ServiceListFileSystemsSegmentResponse, + StorageSharedKeyCredential +} from "@azure/storage-file-datalake"; + +import { + EMULATOR_ACCOUNT_KEY, + EMULATOR_ACCOUNT_NAME, + getEncryptionScope, + getUniqueName, + getYieldedValue +} from "../../../testutils"; +import DataLakeTestServerFactory from "../../DataLakeTestServerFactory"; + +describe("DataLakeServiceClient", () => { + const factory = new DataLakeTestServerFactory(); + const server = factory.createServer(); + + const baseURL = `http://${server.config.host}:${server.config.port}/devstoreaccount1`; + const serviceClient = new DataLakeServiceClient( + baseURL, + new StorageSharedKeyCredential(EMULATOR_ACCOUNT_NAME, EMULATOR_ACCOUNT_KEY), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + + before(async () => { + await server.start(); + }); + + after(async () => { + await server.close(); + await server.clean(); + }); + + beforeEach(async function (this: Context) { + shouldSkip(this); + }); + + it("SetProperties and GetProperties @loki @sql", async () => { + const previousProperties = await serviceClient.getProperties(); + + let serviceProperties: DataLakeServiceProperties; + + // Need to determine serviceProperties's type before assigning. + /* eslint-disable-next-line prefer-const */ + serviceProperties = { + blobAnalyticsLogging: { + deleteProperty: true, + read: true, + retentionPolicy: { + days: 5, + enabled: true + }, + version: "1.0", + write: true + }, + minuteMetrics: { + enabled: true, + includeAPIs: true, + retentionPolicy: { + days: 4, + enabled: true + }, + version: "1.0" + }, + hourMetrics: { + enabled: true, + includeAPIs: true, + retentionPolicy: { + days: 3, + enabled: true + }, + version: "1.0" + }, + deleteRetentionPolicy: { + days: 2, + enabled: true + } + }; + + await serviceClient.setProperties(serviceProperties); + await delay(5 * 1000); + + let properties = await serviceClient.getProperties(); + assert.deepStrictEqual( + serviceProperties.blobAnalyticsLogging, + properties.blobAnalyticsLogging + ); + assert.deepStrictEqual( + serviceProperties.hourMetrics, + properties.hourMetrics + ); + assert.deepStrictEqual( + serviceProperties.minuteMetrics, + properties.minuteMetrics + ); + assert.deepStrictEqual( + serviceProperties.deleteRetentionPolicy?.days, + properties.deleteRetentionPolicy?.days + ); + assert.deepStrictEqual( + serviceProperties.deleteRetentionPolicy?.enabled, + properties.deleteRetentionPolicy?.enabled + ); + + // Cleanup + await serviceClient.setProperties(previousProperties); + await delay(5 * 1000); + + properties = await serviceClient.getProperties(); + if (previousProperties.cors !== undefined) { + assert.deepStrictEqual(previousProperties.cors, properties.cors); + } + + if (previousProperties.blobAnalyticsLogging !== undefined) { + assert.deepStrictEqual( + previousProperties.blobAnalyticsLogging, + properties.blobAnalyticsLogging + ); + } + + if (previousProperties.hourMetrics !== undefined) { + assert.deepStrictEqual( + previousProperties.hourMetrics, + properties.hourMetrics + ); + } + + if (previousProperties.minuteMetrics !== undefined) { + assert.deepStrictEqual( + previousProperties.minuteMetrics, + properties.minuteMetrics + ); + } + + if (previousProperties.deleteRetentionPolicy?.days !== undefined) { + assert.deepStrictEqual( + previousProperties.deleteRetentionPolicy?.days, + properties.deleteRetentionPolicy?.days + ); + } + + if (previousProperties.deleteRetentionPolicy?.enabled !== undefined) { + assert.deepStrictEqual( + previousProperties.deleteRetentionPolicy?.enabled, + properties.deleteRetentionPolicy?.enabled + ); + } + }); + + it("ListFileSystems with default parameters @loki @sql", async () => { + const result = (await serviceClient.listFileSystems().byPage().next()) + .value as ServiceListFileSystemsSegmentResponse; + assert.ok(typeof result.requestId); + assert.ok(result.requestId!.length > 0); + assert.ok(typeof result.version); + assert.ok(result.version!.length > 0); + assert.ok(typeof result.clientRequestId); + assert.ok(result.clientRequestId!.length > 0); + + assert.ok(result.serviceEndpoint.length > 0); + assert.ok(result.fileSystemItems.length >= 0); + + if (result.fileSystemItems.length > 0) { + const filesystem = result.fileSystemItems[0]; + assert.ok(filesystem.name.length > 0); + assert.ok(filesystem.properties.etag.length > 0); + assert.ok(filesystem.properties.lastModified); + } + }); + + it("ListFileSystems - returns file system encryption scope info @loki @sql", async function (this: Context) { + let encryptionScopeName: string | undefined; + try { + encryptionScopeName = getEncryptionScope(); + } catch { + this.skip(); + } + + const fileSystemName = getUniqueName("filesystem"); + const cClient = serviceClient.getFileSystemClient(fileSystemName); + await cClient.create({ + fileSystemEncryptionScope: { + defaultEncryptionScope: encryptionScopeName, + preventEncryptionScopeOverride: true + } + }); + + const result = (await serviceClient.listFileSystems().byPage().next()) + .value; + assert.ok(result.fileSystemItems.length >= 0); + + let foundTheOne = false; + result.fileSystemItems.forEach((element: FileSystemItem) => { + if (element.name === fileSystemName) { + foundTheOne = true; + assert.equal( + element.properties.defaultEncryptionScope, + encryptionScopeName + ); + } + }); + + assert.ok(foundTheOne, "Should have found the created file system"); + await cClient.delete(); + }); + + it("ListFileSystems - PagedAsyncIterableIterator returns file system encryption scope info @loki @sql", async function (this: Context) { + let encryptionScopeName: string | undefined; + try { + encryptionScopeName = getEncryptionScope(); + } catch { + this.skip(); + } + + const fileSystemName = getUniqueName("filesystem"); + const cClient = serviceClient.getFileSystemClient(fileSystemName); + await cClient.create({ + fileSystemEncryptionScope: { + defaultEncryptionScope: encryptionScopeName, + preventEncryptionScopeOverride: true + } + }); + + let foundTheOne = false; + + for await (const filesystem of serviceClient.listFileSystems()) { + if (filesystem.name === fileSystemName) { + foundTheOne = true; + assert.equal( + filesystem.properties.defaultEncryptionScope, + encryptionScopeName + ); + } + } + + assert.ok(foundTheOne, "Should have found the created file system"); + await cClient.delete(); + }); + + it("ListFileSystems with default parameters - null prefix shouldn't throw error @loki @sql", async () => { + const result = ( + await serviceClient.listFileSystems({ prefix: "" }).byPage().next() + ).value; + + assert.ok(result.fileSystemItems.length >= 0); + + if (result.fileSystemItems.length > 0) { + const filesystem = result.fileSystemItems[0]; + assert.ok(filesystem.name.length > 0); + assert.ok(filesystem.properties.etag.length > 0); + assert.ok(filesystem.properties.lastModified); + } + }); + + it("ListFileSystems with all parameters configured @loki @sql", async function (this: Context) { + const fileSystemNamePrefix = getUniqueName("filesystem1"); + const fileSystemName1 = `${fileSystemNamePrefix}x1`; + const fileSystemName2 = `${fileSystemNamePrefix}x2`; + const fileSystemClient1 = + serviceClient.getFileSystemClient(fileSystemName1); + const fileSystemClient2 = + serviceClient.getFileSystemClient(fileSystemName2); + await fileSystemClient1.create({ metadata: { key: "val" } }); + await fileSystemClient2.create({ metadata: { key: "val" } }); + + const result1 = ( + await serviceClient + .listFileSystems({ + includeMetadata: true, + prefix: fileSystemNamePrefix + }) + .byPage({ maxPageSize: 1 }) + .next() + ).value as ServiceListFileSystemsSegmentResponse; // TODO: Why no intelligence? + + assert.ok(result1.continuationToken); + assert.equal(result1.fileSystemItems.length, 1); + assert.ok(result1.fileSystemItems[0].name.startsWith(fileSystemNamePrefix)); + assert.ok(result1.fileSystemItems[0].properties.etag.length > 0); + assert.ok(result1.fileSystemItems[0].properties.lastModified); + assert.ok(!result1.fileSystemItems[0].properties.leaseDuration); + assert.ok(!result1.fileSystemItems[0].properties.publicAccess); + assert.deepEqual( + result1.fileSystemItems[0].properties.leaseState, + "available" + ); + assert.deepEqual( + result1.fileSystemItems[0].properties.leaseStatus, + "unlocked" + ); + assert.deepEqual(result1.fileSystemItems[0].metadata!.key, "val"); + + const result2 = ( + await serviceClient + .listFileSystems({ + includeMetadata: true, + prefix: fileSystemNamePrefix + }) + .byPage({ + continuationToken: result1.continuationToken, + maxPageSize: 1 + }) + .next() + ).value as ServiceListFileSystemsSegmentResponse; + + assert.ok(!result2.continuationToken); + assert.equal(result2.fileSystemItems.length, 1); + assert.ok(result2.fileSystemItems[0].name.startsWith(fileSystemNamePrefix)); + assert.ok(result2.fileSystemItems[0].properties.etag.length > 0); + assert.ok(result2.fileSystemItems[0].properties.lastModified); + assert.ok(!result2.fileSystemItems[0].properties.leaseDuration); + assert.ok(!result2.fileSystemItems[0].properties.publicAccess); + assert.deepEqual( + result2.fileSystemItems[0].properties.leaseState, + "available" + ); + assert.deepEqual( + result2.fileSystemItems[0].properties.leaseStatus, + "unlocked" + ); + assert.deepEqual(result2.fileSystemItems[0].metadata!.key, "val"); + + await fileSystemClient1.deleteIfExists(); + await fileSystemClient2.deleteIfExists(); + }); + + it("Verify PagedAsyncIterableIterator for ListFileSystems @loki @sql", async () => { + const fileSystemClients = []; + const fileSystemNamePrefix = getUniqueName("filesystem2"); + + for (let i = 0; i < 4; i++) { + const fileSystemName = `${fileSystemNamePrefix}x${i}`; + const fileSystemClient = + serviceClient.getFileSystemClient(fileSystemName); + await fileSystemClient.create({ metadata: { key: "val" } }); + fileSystemClients.push(fileSystemClient); + } + + for await (const filesystem of serviceClient.listFileSystems({ + includeMetadata: true, + prefix: fileSystemNamePrefix + })) { + assert.ok(filesystem.name.startsWith(fileSystemNamePrefix)); + assert.ok(filesystem.properties.etag.length > 0); + assert.ok(filesystem.properties.lastModified); + assert.ok(!filesystem.properties.leaseDuration); + assert.ok(!filesystem.properties.publicAccess); + assert.deepEqual(filesystem.properties.leaseState, "available"); + assert.deepEqual(filesystem.properties.leaseStatus, "unlocked"); + assert.deepEqual(filesystem.metadata!.key, "val"); + } + + for (const client of fileSystemClients) { + await client.deleteIfExists(); + } + }); + + it("Verify PagedAsyncIterableIterator(generator .next() syntax) for ListFileSystems @loki @sql", async () => { + const fileSystemNamePrefix = getUniqueName("filesystem3"); + const fileSystemName1 = `${fileSystemNamePrefix}x1`; + const fileSystemName2 = `${fileSystemNamePrefix}x2`; + const fileSystemClient1 = + serviceClient.getFileSystemClient(fileSystemName1); + const fileSystemClient2 = + serviceClient.getFileSystemClient(fileSystemName2); + await fileSystemClient1.create({ metadata: { key: "val" } }); + await fileSystemClient2.create({ metadata: { key: "val" } }); + + const iterator = serviceClient.listFileSystems({ + includeMetadata: true, + prefix: fileSystemNamePrefix + }); + + let fileSystemItem = getYieldedValue(await iterator.next()); + assert.ok(fileSystemItem.name.startsWith(fileSystemNamePrefix)); + assert.ok(fileSystemItem.properties.etag.length > 0); + assert.ok(fileSystemItem.properties.lastModified); + assert.ok(!fileSystemItem.properties.leaseDuration); + assert.ok(!fileSystemItem.properties.publicAccess); + assert.deepEqual(fileSystemItem.properties.leaseState, "available"); + assert.deepEqual(fileSystemItem.properties.leaseStatus, "unlocked"); + assert.deepEqual(fileSystemItem.metadata!.key, "val"); + + fileSystemItem = getYieldedValue(await iterator.next()); + assert.ok(fileSystemItem.name.startsWith(fileSystemNamePrefix)); + assert.ok(fileSystemItem.properties.etag.length > 0); + assert.ok(fileSystemItem.properties.lastModified); + assert.ok(!fileSystemItem.properties.leaseDuration); + assert.ok(!fileSystemItem.properties.publicAccess); + assert.deepEqual(fileSystemItem.properties.leaseState, "available"); + assert.deepEqual(fileSystemItem.properties.leaseStatus, "unlocked"); + assert.deepEqual(fileSystemItem.metadata!.key, "val"); + + await fileSystemClient1.deleteIfExists(); + await fileSystemClient2.deleteIfExists(); + }); + + it("Verify PagedAsyncIterableIterator(byPage()) for ListFileSystems @loki @sql", async function (this: Context) { + const fileSystemClients = []; + const fileSystemNamePrefix = getUniqueName("filesystem4"); + + for (let i = 0; i < 4; i++) { + const fileSystemName = `${fileSystemNamePrefix}x${i}`; + const fileSystemClient = + serviceClient.getFileSystemClient(fileSystemName); + await fileSystemClient.create({ metadata: { key: "val" } }); + fileSystemClients.push(fileSystemClient); + } + + for await (const response of serviceClient + .listFileSystems({ + includeMetadata: true, + prefix: fileSystemNamePrefix + }) + .byPage({ maxPageSize: 2 })) { + for (const filesystem of response.fileSystemItems) { + assert.ok(filesystem.name.startsWith(fileSystemNamePrefix)); + assert.ok(filesystem.properties.etag.length > 0); + assert.ok(filesystem.properties.lastModified); + assert.ok(!filesystem.properties.leaseDuration); + assert.ok(!filesystem.properties.publicAccess); + assert.deepEqual(filesystem.properties.leaseState, "available"); + assert.deepEqual(filesystem.properties.leaseStatus, "unlocked"); + assert.deepEqual(filesystem.metadata!.key, "val"); + } + } + + for (const client of fileSystemClients) { + await client.deleteIfExists(); + } + }); + + it("Verify PagedAsyncIterableIterator(byPage() - continuationToken) for ListFileSystems @loki @sql", async function (this: Context) { + const fileSystemClients = []; + const fileSystemNamePrefix = getUniqueName("filesystem5"); + + for (let i = 0; i < 4; i++) { + const fileSystemName = `${fileSystemNamePrefix}x${i}`; + const fileSystemClient = + serviceClient.getFileSystemClient(fileSystemName); + await fileSystemClient.create({ metadata: { key: "val" } }); + fileSystemClients.push(fileSystemClient); + } + + let iter = serviceClient + .listFileSystems({ + includeMetadata: true, + prefix: fileSystemNamePrefix + }) + .byPage({ maxPageSize: 2 }); + let response = (await iter.next()).value; + for (const filesystem of response.fileSystemItems) { + assert.ok(filesystem.name.startsWith(fileSystemNamePrefix)); + assert.ok(filesystem.properties.etag.length > 0); + assert.ok(filesystem.properties.lastModified); + assert.ok(!filesystem.properties.leaseDuration); + assert.ok(!filesystem.properties.publicAccess); + assert.deepEqual(filesystem.properties.leaseState, "available"); + assert.deepEqual(filesystem.properties.leaseStatus, "unlocked"); + assert.deepEqual(filesystem.metadata!.key, "val"); + } + // Gets next marker + const marker = response.continuationToken; + // Passing next marker as continuationToken + iter = serviceClient + .listFileSystems({ + includeMetadata: true, + prefix: fileSystemNamePrefix + }) + .byPage({ continuationToken: marker, maxPageSize: 2 }); + response = (await iter.next()).value; + // Gets 2 containers + for (const filesystem of response.fileSystemItems) { + assert.ok(filesystem.name.startsWith(fileSystemNamePrefix)); + assert.ok(filesystem.properties.etag.length > 0); + assert.ok(filesystem.properties.lastModified); + assert.ok(!filesystem.properties.leaseDuration); + assert.ok(!filesystem.properties.publicAccess); + assert.deepEqual(filesystem.properties.leaseState, "available"); + assert.deepEqual(filesystem.properties.leaseStatus, "unlocked"); + assert.deepEqual(filesystem.metadata!.key, "val"); + } + + for (const client of fileSystemClients) { + await client.deleteIfExists(); + } + }); + + it("createFileSystem and deleteFileSystem @loki @sql", async () => { + const fileSystemName = getUniqueName("filesystem6"); + const fileSystemClient = serviceClient.getFileSystemClient(fileSystemName); + const access = "filesystem"; + const metadata = { key: "value" }; + + await fileSystemClient.create({ + access, + metadata + }); + + const result = await fileSystemClient.getProperties(); + assert.deepEqual(result.publicAccess, access); + assert.deepEqual(result.metadata, metadata); + + await serviceClient.getFileSystemClient(fileSystemName).delete(); + try { + await fileSystemClient.getProperties(); + assert.fail( + "Expecting an error in getting properties from a deleted block blob but didn't get one." + ); + } catch (error) { + assert.ok((error.statusCode as number) === 404); + } + }); + + it("renameFileSystem should work @loki @sql", async function (this: Context) { + const fileSystemName = getUniqueName("filesystem"); + const fileSystemClient = serviceClient.getFileSystemClient(fileSystemName); + await fileSystemClient.create(); + + const newFileSystemName = getUniqueName("newfilesystem"); + // const renameRes = await serviceClient.renameFileSystem(fileSystemName, newFileSystemName); + const renameRes = await serviceClient["renameFileSystem"]( + fileSystemName, + newFileSystemName + ); + + const newFileSystemClient = + serviceClient.getFileSystemClient(newFileSystemName); + assert.deepStrictEqual(newFileSystemClient, renameRes.fileSystemClient); + await newFileSystemClient.getProperties(); + + await newFileSystemClient.delete(); + }); + + it("renameFileSystem should work with source lease @loki @sql", async function (this: Context) { + const fileSystemName = getUniqueName("filesystem"); + const fileSystemClient = serviceClient.getFileSystemClient(fileSystemName); + await fileSystemClient.create(); + + const leaseClient = fileSystemClient.getDataLakeLeaseClient(); + await leaseClient.acquireLease(-1); + + const newFileSystemName = getUniqueName("newfilesystem"); + // const renameRes = await serviceClient.renameFileSystem(fileSystemName, newFileSystemName, { + const renameRes = await serviceClient["renameFileSystem"]( + fileSystemName, + newFileSystemName, + { + sourceCondition: { leaseId: leaseClient.leaseId } + } + ); + + const newFileSystemClient = + serviceClient.getFileSystemClient(newFileSystemName); + assert.deepStrictEqual(newFileSystemClient, renameRes.fileSystemClient); + await newFileSystemClient.getProperties(); + + await newFileSystemClient.deleteIfExists(); + }); + + it("undelete and list deleted file system should work @loki @sql", async function (this: Context) { + const fileSystemName = getUniqueName("filesystem"); + const fileSystemClient = serviceClient.getFileSystemClient(fileSystemName); + await fileSystemClient.create(); + + const metadata = { a: "a" }; + await fileSystemClient.setMetadata(metadata); + + await fileSystemClient.delete(); + await delay(30 * 1000); + + let listed = false; + for await (const fileSystemItem of serviceClient.listFileSystems({ + includeDeleted: true, + includeMetadata: true + })) { + if (fileSystemItem.deleted && fileSystemItem.name === fileSystemName) { + listed = true; + // verify list container response + assert.ok(fileSystemItem.versionId); + assert.ok(fileSystemItem.deleted); + assert.ok(fileSystemItem.properties.deletedOn); + assert.ok(fileSystemItem.properties.remainingRetentionDays); + assert.deepStrictEqual(fileSystemItem.metadata, metadata); + + const restoreRes = await serviceClient.undeleteFileSystem( + fileSystemName, + fileSystemItem.versionId! + ); + assert.equal(restoreRes.fileSystemClient.name, fileSystemName); + await restoreRes.fileSystemClient.delete(); + break; + } + } + assert.ok(listed); + }); +}); + +function shouldSkip(context: Context) { + if ( + context.currentTest!.title.indexOf("undelete") > -1 || + context.currentTest!.title.indexOf("renameFileSystem") > -1 + ) { + context.skip(); + } +} diff --git a/tests/dfs/storage-file-datalake/apis/specialnaming.test.ts b/tests/dfs/storage-file-datalake/apis/specialnaming.test.ts new file mode 100644 index 000000000..713ad0ef4 --- /dev/null +++ b/tests/dfs/storage-file-datalake/apis/specialnaming.test.ts @@ -0,0 +1,320 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import assert from "assert"; +import { Context } from "mocha"; + +import { + DataLakeFileClient, + DataLakeFileSystemClient, + DataLakeServiceClient, + StorageSharedKeyCredential +} from "@azure/storage-file-datalake"; + +import { configLogger } from "../../../../src/common/Logger"; +import { + appendToURLPath, + EMULATOR_ACCOUNT_KEY, + EMULATOR_ACCOUNT_NAME, + getUniqueName +} from "../../../testutils"; +import DataLakeTestServerFactory from "../../DataLakeTestServerFactory"; + +// Set true to enable debug log +configLogger(false); + +describe("Special Naming Tests", () => { + const factory = new DataLakeTestServerFactory(); + const server = factory.createServer(); + + const baseURL = `http://${server.config.host}:${server.config.port}/devstoreaccount1`; + const serviceClient = new DataLakeServiceClient( + baseURL, + new StorageSharedKeyCredential(EMULATOR_ACCOUNT_NAME, EMULATOR_ACCOUNT_KEY), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + + let fileSystemName: string; + let fileSystemClient: DataLakeFileSystemClient; + + before(async () => { + await server.start(); + }); + + after(async () => { + await server.close(); + await server.clean(); + }); + + beforeEach(async function (this: Context) { + fileSystemName = getUniqueName("1container-with-dash"); + fileSystemClient = serviceClient.getFileSystemClient(fileSystemName); + await fileSystemClient.createIfNotExists(); + }); + + afterEach(async function () { + await fileSystemClient.deleteIfExists(); + }); + + it("Should work with special container and blob names with spaces @loki @sql", async () => { + const fileName: string = getUniqueName("blob empty"); + const fileClient = fileSystemClient.getFileClient(fileName); + + await fileClient.create(); + const response = (await fileSystemClient.listPaths().byPage().next()).value; + + assert.deepStrictEqual(response.pathItems.length, 1); + assert.deepStrictEqual(response.pathItems[0].name, fileName); + }); + + it("Should work with special container and blob names with spaces in URL string @loki @sql", async () => { + const fileName: string = getUniqueName("blob empty"); + const fileClient = new DataLakeFileClient( + appendToURLPath(fileSystemClient.url, fileName), + (fileSystemClient as any).pipeline + ); + + await fileClient.create(); + const response = (await fileSystemClient.listPaths().byPage().next()).value; + + assert.deepStrictEqual(response.pathItems.length, 1); + assert.deepStrictEqual(response.pathItems[0].name, fileName); + }); + + it("Should work with special container and blob names uppercase @loki @sql", async () => { + const fileName: string = getUniqueName("Upper blob empty another"); + const fileClient = fileSystemClient.getFileClient(fileName); + + await fileClient.create(); + await fileClient.getProperties(); + const response = (await fileSystemClient.listPaths({}).byPage().next()) + .value; + + assert.deepStrictEqual(response.pathItems.length, 1); + assert.deepStrictEqual(response.pathItems[0].name, fileName); + }); + + it("Should work with special container and blob names uppercase in URL string @loki @sql", async () => { + const fileName: string = getUniqueName("Upper blob empty another"); + const fileClient = new DataLakeFileClient( + appendToURLPath(fileSystemClient.url, fileName), + (fileSystemClient as any).pipeline + ); + + await fileClient.create(); + await fileClient.getProperties(); + const response = (await fileSystemClient.listPaths({}).byPage().next()) + .value; + + assert.deepStrictEqual(response.pathItems.length, 1); + assert.deepStrictEqual(response.pathItems[0].name, fileName); + }); + + it("Should work with special blob names Chinese characters @loki @sql", async () => { + const fileName: string = getUniqueName("Upper blob empty another 汉字"); + const fileClient = fileSystemClient.getFileClient(fileName); + + await fileClient.create(); + await fileClient.getProperties(); + const response = (await fileSystemClient.listPaths({}).byPage().next()) + .value; + + assert.deepStrictEqual(response.pathItems.length, 1); + assert.deepStrictEqual(response.pathItems[0].name, fileName); + }); + + it("Should work with special blob names Chinese characters in URL string @loki @sql", async () => { + const fileName: string = getUniqueName("Upper blob empty another 汉字"); + const fileClient = new DataLakeFileClient( + appendToURLPath(fileSystemClient.url, fileName), + (fileSystemClient as any).pipeline + ); + + await fileClient.create(); + await fileClient.getProperties(); + const response = (await fileSystemClient.listPaths({}).byPage().next()) + .value; + + assert.deepStrictEqual(response.pathItems.length, 1); + assert.deepStrictEqual(response.pathItems[0].name, fileName); + }); + + it("Should work with special blob name characters @loki @sql", async () => { + const specialName = + "汉字. special ~!@#$%^&*()_+`1234567890-={}|[]:\";'<>?,'"; + const fileName: string = getUniqueName(specialName); + const fileClient = fileSystemClient.getFileClient(fileName); + + await fileClient.create(); + await fileClient.getProperties(); + const response = (await fileSystemClient.listPaths({}).byPage().next()) + .value; + + assert.deepStrictEqual(response.pathItems.length, 1); + assert.deepStrictEqual(response.pathItems[0].name, fileName); + }); + + it("Should work with special blob name characters in URL string @loki @sql", async () => { + const specialName = + "汉字. special ~!@#$%^&*()_+`1234567890-={}|[]:\";'<>?,'"; + const fileName: string = getUniqueName(specialName); + const fileClient = new DataLakeFileClient( + // There are 2 special cases for a URL string: + // Escape "%" when creating XxxClient object with URL strings + // Escape "?" otherwise string after "?" will be treated as URL parameters + appendToURLPath( + fileSystemClient.url, + fileName.replace(/%/g, "%25").replace(/\?/g, "%3F") + ), + (fileSystemClient as any).pipeline + ); + + await fileClient.create(); + await fileClient.getProperties(); + const response = ( + await fileSystemClient + .listPaths({ + // NOTICE: Azure Storage Server will replace "\" with "/" in the blob names + // .replace(/\\/g, "/") + }) + .byPage() + .next() + ).value; + + assert.deepStrictEqual(response.pathItems.length, 1); + assert.deepStrictEqual(response.pathItems[0].name, fileName); + }); + + it("Should work with special blob name Russian URI encoded @loki @sql", async () => { + const fileName: string = getUniqueName("ру́сский язы́к"); + const fileNameEncoded: string = encodeURIComponent(fileName); + const fileClient = fileSystemClient.getFileClient(fileNameEncoded); + + await fileClient.create(); + await fileClient.getProperties(); + const response = (await fileSystemClient.listPaths({}).byPage().next()) + .value; + + assert.deepStrictEqual(response.pathItems.length, 1); + assert.deepStrictEqual(response.pathItems[0].name, fileNameEncoded); + }); + + it("Should work with special blob name Russian @loki @sql", async () => { + const fileName: string = getUniqueName("ру́сский язы́к"); + const fileClient = fileSystemClient.getFileClient(fileName); + + await fileClient.create(); + await fileClient.getProperties(); + const response = (await fileSystemClient.listPaths({}).byPage().next()) + .value; + + assert.deepStrictEqual(response.pathItems.length, 1); + assert.deepStrictEqual(response.pathItems[0].name, fileName); + }); + + it("Should work with special blob name Russian in URL string @loki @sql", async () => { + const fileName: string = getUniqueName("ру́сский язы́к"); + const fileClient = new DataLakeFileClient( + appendToURLPath(fileSystemClient.url, fileName), + (fileSystemClient as any).pipeline + ); + + await fileClient.create(); + await fileClient.getProperties(); + const response = (await fileSystemClient.listPaths({}).byPage().next()) + .value; + + assert.deepStrictEqual(response.pathItems.length, 1); + assert.deepStrictEqual(response.pathItems[0].name, fileName); + }); + + it("Should work with special blob name Arabic URI encoded @loki @sql", async () => { + const fileName: string = getUniqueName("عربيعربى"); + const fileNameEncoded: string = encodeURIComponent(fileName); + const fileClient = fileSystemClient.getFileClient(fileNameEncoded); + + await fileClient.create(); + await fileClient.getProperties(); + const response = (await fileSystemClient.listPaths({}).byPage().next()) + .value; + + assert.deepStrictEqual(response.pathItems.length, 1); + assert.deepStrictEqual(response.pathItems[0].name, fileNameEncoded); + }); + + it("Should work with special blob name Arabic @loki @sql", async () => { + const fileName: string = getUniqueName("عربيعربى"); + const fileClient = fileSystemClient.getFileClient(fileName); + + await fileClient.create(); + await fileClient.getProperties(); + const response = (await fileSystemClient.listPaths({}).byPage().next()) + .value; + + assert.deepStrictEqual(response.pathItems.length, 1); + assert.deepStrictEqual(response.pathItems[0].name, fileName); + }); + + it("Should work with special blob name Arabic in URL string @loki @sql", async () => { + const fileName: string = getUniqueName("عربيعربى"); + const fileClient = new DataLakeFileClient( + appendToURLPath(fileSystemClient.url, fileName), + (fileSystemClient as any).pipeline + ); + + await fileClient.create(); + await fileClient.getProperties(); + const response = (await fileSystemClient.listPaths({}).byPage().next()) + .value; + + assert.deepStrictEqual(response.pathItems.length, 1); + assert.deepStrictEqual(response.pathItems[0].name, fileName); + }); + + it("Should work with special blob name Japanese URI encoded @loki @sql", async () => { + const fileName: string = getUniqueName("にっぽんごにほんご"); + const fileNameEncoded: string = encodeURIComponent(fileName); + const fileClient = fileSystemClient.getFileClient(fileNameEncoded); + + await fileClient.create(); + await fileClient.getProperties(); + const response = (await fileSystemClient.listPaths({}).byPage().next()) + .value; + + assert.deepStrictEqual(response.pathItems.length, 1); + assert.deepStrictEqual(response.pathItems[0].name, fileNameEncoded); + }); + + it("Should work with special blob name Japanese @loki @sql", async () => { + const fileName: string = getUniqueName("にっぽんごにほんご"); + const fileClient = fileSystemClient.getFileClient(fileName); + + await fileClient.create(); + await fileClient.getProperties(); + const response = (await fileSystemClient.listPaths({}).byPage().next()) + .value; + + assert.deepStrictEqual(response.pathItems.length, 1); + assert.deepStrictEqual(response.pathItems[0].name, fileName); + }); + + it("Should work with special blob name Japanese in URL string @loki @sql", async () => { + const fileName: string = getUniqueName("にっぽんごにほんご"); + const fileClient = new DataLakeFileClient( + appendToURLPath(fileSystemClient.url, fileName), + (fileSystemClient as any).pipeline + ); + + await fileClient.create(); + await fileClient.getProperties(); + const response = (await fileSystemClient.listPaths({}).byPage().next()) + .value; + + assert.deepStrictEqual(response.pathItems.length, 1); + assert.deepStrictEqual(response.pathItems[0].name, fileName); + }); +}); diff --git a/tests/dfs/storage-file-datalake/authentication.test.ts b/tests/dfs/storage-file-datalake/authentication.test.ts new file mode 100644 index 000000000..9f5988ed8 --- /dev/null +++ b/tests/dfs/storage-file-datalake/authentication.test.ts @@ -0,0 +1,154 @@ +import * as assert from "assert"; + +import { + AnonymousCredential, + DataLakeServiceClient, + newPipeline, + StorageSharedKeyCredential +} from "@azure/storage-file-datalake"; + +import { configLogger } from "../../../src/common/Logger"; +import { + EMULATOR_ACCOUNT_KEY, + EMULATOR_ACCOUNT_NAME, + getUniqueName +} from "../../testutils"; +import DataLakeTestServerFactory from "../DataLakeTestServerFactory"; + +// Set true to enable debug log +configLogger(false); + +describe("Authentication", () => { + const factory = new DataLakeTestServerFactory(); + const server = factory.createServer(); + + const baseURL = `http://${server.config.host}:${server.config.port}/devstoreaccount1`; + + before(async () => { + await server.start(); + }); + + after(async () => { + await server.close(); + await server.clean(); + }); + + it(`Should not work without credential @loki @sql`, async () => { + const serviceClient = new DataLakeServiceClient( + baseURL, + newPipeline(new AnonymousCredential(), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const containerName: string = getUniqueName("1container-with-dash"); + const containerClient = serviceClient.getFileSystemClient(containerName); + + let err; + try { + await containerClient.create(); + } catch (error) { + err = error; + } finally { + if (err === undefined) { + try { + await containerClient.delete(); + } catch (error) { + /* Noop */ + } + assert.fail(); + } + } + }); + + it(`Should not work without correct account name @loki @sql`, async () => { + const serviceClient = new DataLakeServiceClient( + baseURL, + newPipeline( + new StorageSharedKeyCredential("invalid", EMULATOR_ACCOUNT_KEY), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ) + ); + + const containerName: string = getUniqueName("1container-with-dash"); + const containerClient = serviceClient.getFileSystemClient(containerName); + + let err; + try { + await containerClient.create(); + } catch (error) { + err = error; + } finally { + if (err === undefined) { + try { + await containerClient.delete(); + } catch (error) { + /* Noop */ + } + assert.fail(); + } + } + }); + + it(`Should not work without correct account key @loki @sql`, async () => { + const serviceClient = new DataLakeServiceClient( + baseURL, + newPipeline( + new StorageSharedKeyCredential(EMULATOR_ACCOUNT_NAME, "invalidkey"), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ) + ); + + const containerName: string = getUniqueName("1container-with-dash"); + const containerClient = serviceClient.getFileSystemClient(containerName); + + let err; + try { + await containerClient.create(); + } catch (error) { + err = error; + } finally { + if (err === undefined) { + try { + await containerClient.delete(); + } catch (error) { + /* Noop */ + } + assert.fail(); + } + } + }); + + it(`Should work with correct shared key @loki @sql`, async () => { + const serviceClient = new DataLakeServiceClient( + baseURL, + newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ) + ); + + const containerName: string = getUniqueName("1container-with-dash"); + const containerClient = serviceClient.getFileSystemClient(containerName); + + await containerClient.create(); + await containerClient.delete(); + }); +}); diff --git a/tests/dfs/storage-file-datalake/blobCorsRequest.test.ts b/tests/dfs/storage-file-datalake/blobCorsRequest.test.ts new file mode 100644 index 000000000..eedd4653c --- /dev/null +++ b/tests/dfs/storage-file-datalake/blobCorsRequest.test.ts @@ -0,0 +1,878 @@ +import * as assert from "assert"; + +import { + DataLakeServiceClient, + newPipeline, + RestError, + StorageSharedKeyCredential +} from "@azure/storage-file-datalake"; + +import { configLogger } from "../../../src/common/Logger"; +import { + EMULATOR_ACCOUNT_KEY, + EMULATOR_ACCOUNT_NAME, + sleep +} from "../../testutils"; +import DataLakeTestServerFactory from "../DataLakeTestServerFactory"; +import OPTIONSRequestPolicyFactory from "../RequestPolicy/OPTIONSRequestPolicyFactory"; +import OriginPolicyFactory from "../RequestPolicy/OriginPolicyFactory"; + +// Set true to enable debug log +configLogger(false); + +describe("Blob Cors requests test", () => { + const factory = new DataLakeTestServerFactory(); + const server = factory.createServer(); + + const baseURL = `http://${server.config.host}:${server.config.port}/devstoreaccount1`; + const serviceClient = new DataLakeServiceClient( + baseURL, + newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ) + ); + + before(async () => { + await server.start(); + }); + + after(async () => { + await server.close(); + await server.clean(); + }); + + it("OPTIONS request without cors rules in server should be fail @loki @sql", async () => { + const serviceProperties = await serviceClient.getProperties(); + serviceProperties.cors = []; + await serviceClient.setProperties(serviceProperties); + + const origin = "Origin"; + const requestMethod = "GET"; + + const pipeline = newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift( + new OPTIONSRequestPolicyFactory(origin, requestMethod) + ); + const serviceClientForOptions = new DataLakeServiceClient( + baseURL, + pipeline + ); + + let error; + try { + await serviceClientForOptions.getProperties(); + } catch (err) { + error = err; + } + + assert.ok(error.statusCode === 403); + assert.ok( + error.message.includes( + "CORS not enabled or no matching rule found for this request." + ) + ); + }); + + it("OPTIONS request should not work without matching cors rules @loki @sql", async () => { + const serviceProperties = await serviceClient.getProperties(); + + const newCORS = { + allowedHeaders: "header", + allowedMethods: "GET", + allowedOrigins: "test", + exposedHeaders: "*", + maxAgeInSeconds: 8888 + }; + + serviceProperties.cors = [newCORS]; + + await serviceClient.setProperties(serviceProperties); + + await sleep(100); + + let origin = "Origin"; + let requestMethod = "GET"; + + let pipeline = newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift( + new OPTIONSRequestPolicyFactory(origin, requestMethod) + ); + let serviceClientForOptions = new DataLakeServiceClient(baseURL, pipeline); + + let error; + try { + await serviceClientForOptions.getProperties(); + } catch (err) { + error = err; + } + + assert.ok(error.statusCode === 403); + assert.ok( + error.message.includes( + "CORS not enabled or no matching rule found for this request." + ) + ); + + origin = "test"; + requestMethod = "GET"; + + pipeline = newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift( + new OPTIONSRequestPolicyFactory(origin, requestMethod) + ); + serviceClientForOptions = new DataLakeServiceClient(baseURL, pipeline); + + const res = await serviceClientForOptions.getProperties(); + assert.ok(res._response.status === 200); + }); + + it("OPTIONS request should not work without Origin header or matching allowedOrigins @loki @sql", async () => { + const serviceProperties = await serviceClient.getProperties(); + + const newCORS = { + allowedHeaders: "header", + allowedMethods: "GET", + allowedOrigins: "test", + exposedHeaders: "*", + maxAgeInSeconds: 8888 + }; + + serviceProperties.cors = [newCORS]; + + await serviceClient.setProperties(serviceProperties); + + await sleep(100); + + const origin = "Origin"; + const requestMethod = "GET"; + + let pipeline = newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift( + new OPTIONSRequestPolicyFactory(origin, requestMethod) + ); + let serviceClientForOptions = new DataLakeServiceClient(baseURL, pipeline); + + let error; + try { + await serviceClientForOptions.getProperties(); + } catch (err) { + error = err; + } + + assert.ok(error.statusCode === 403); + assert.ok( + error.message.includes( + "CORS not enabled or no matching rule found for this request." + ) + ); + + pipeline = newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift( + new OPTIONSRequestPolicyFactory(undefined, requestMethod) + ); + serviceClientForOptions = new DataLakeServiceClient(baseURL, pipeline); + + try { + await serviceClientForOptions.getProperties(); + } catch (err) { + error = err; + } + + assert.ok(error.statusCode === 403); + assert.ok( + error.message.includes( + "CORS not enabled or no matching rule found for this request." + ) + ); + }); + + it("OPTIONS request should not work without requestMethod header or matching allowedMethods @loki @sql", async () => { + const serviceProperties = await serviceClient.getProperties(); + + const newCORS = { + allowedHeaders: "header", + allowedMethods: "GET", + allowedOrigins: "test", + exposedHeaders: "*", + maxAgeInSeconds: 8888 + }; + + serviceProperties.cors = [newCORS]; + + await serviceClient.setProperties(serviceProperties); + + await sleep(100); + + const origin = "test"; + const requestMethod = "PUT"; + + let pipeline = newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift( + new OPTIONSRequestPolicyFactory(origin, requestMethod) + ); + let serviceClientForOptions = new DataLakeServiceClient(baseURL, pipeline); + + let error; + try { + await serviceClientForOptions.getProperties(); + } catch (err) { + error = err; + } + + assert.ok(error.statusCode === 403); + assert.ok( + error.message.includes( + "CORS not enabled or no matching rule found for this request." + ) + ); + + pipeline = newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift( + new OPTIONSRequestPolicyFactory(origin, undefined) + ); + serviceClientForOptions = new DataLakeServiceClient(baseURL, pipeline); + + try { + await serviceClientForOptions.getProperties(); + } catch (err) { + error = err; + } + + assert.ok(error.statusCode === 400); + assert.ok(error.message.includes("A required CORS header is not present.")); + }); + + it("OPTIONS request should check the defined requestHeaders @loki @sql", async () => { + const serviceProperties = await serviceClient.getProperties(); + + const newCORS = [ + { + allowedHeaders: "header", + allowedMethods: "GET", + allowedOrigins: "test", + exposedHeaders: "*", + maxAgeInSeconds: 8888 + }, + { + allowedHeaders: "*", + allowedMethods: "PUT", + allowedOrigins: "test", + exposedHeaders: "*", + maxAgeInSeconds: 8888 + }, + { + allowedHeaders: "head*", + allowedMethods: "POST", + allowedOrigins: "test", + exposedHeaders: "*", + maxAgeInSeconds: 8888 + } + ]; + + serviceProperties.cors = newCORS; + + await serviceClient.setProperties(serviceProperties); + + await sleep(100); + + // No match + let origin = "test"; + let requestMethod = "GET"; + let reqestHeaders = "head"; + + let pipeline = newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift( + new OPTIONSRequestPolicyFactory(origin, requestMethod, reqestHeaders) + ); + let serviceClientForOptions = new DataLakeServiceClient(baseURL, pipeline); + + let error; + try { + await serviceClientForOptions.getProperties(); + } catch (err) { + error = err; + } + + assert.ok(error.statusCode === 403); + assert.ok( + error.message.includes( + "CORS not enabled or no matching rule found for this request." + ) + ); + + // Match first cors. + origin = "test"; + requestMethod = "GET"; + reqestHeaders = "header"; + + pipeline = newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift( + new OPTIONSRequestPolicyFactory(origin, requestMethod, reqestHeaders) + ); + serviceClientForOptions = new DataLakeServiceClient(baseURL, pipeline); + + let res = await serviceClientForOptions.getProperties(); + assert.ok(res._response.status === 200); + + // Match second cors. + origin = "test"; + requestMethod = "PUT"; + reqestHeaders = "head"; + + pipeline = newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift( + new OPTIONSRequestPolicyFactory(origin, requestMethod, reqestHeaders) + ); + serviceClientForOptions = new DataLakeServiceClient(baseURL, pipeline); + + res = await serviceClientForOptions.getProperties(); + assert.ok(res._response.status === 200); + + // No match. + origin = "test"; + requestMethod = "POST"; + reqestHeaders = "hea"; + + pipeline = newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift( + new OPTIONSRequestPolicyFactory(origin, requestMethod, reqestHeaders) + ); + serviceClientForOptions = new DataLakeServiceClient(baseURL, pipeline); + + error; + try { + await serviceClientForOptions.getProperties(); + } catch (err) { + error = err; + } + + assert.ok(error.statusCode === 403); + assert.ok( + error.message.includes( + "CORS not enabled or no matching rule found for this request." + ) + ); + + // Match third cors. + origin = "test"; + requestMethod = "POST"; + reqestHeaders = "headerheader"; + + pipeline = newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift( + new OPTIONSRequestPolicyFactory(origin, requestMethod, reqestHeaders) + ); + serviceClientForOptions = new DataLakeServiceClient(baseURL, pipeline); + + res = await serviceClientForOptions.getProperties(); + assert.ok(res._response.status === 200); + }); + + it("OPTIONS request should work with matching rule containing Origion * @loki @sql", async () => { + const serviceProperties = await serviceClient.getProperties(); + + const newCORS = { + allowedHeaders: "header", + allowedMethods: "GET", + allowedOrigins: "*", + exposedHeaders: "*", + maxAgeInSeconds: 8888 + }; + + serviceProperties.cors = [newCORS]; + + await serviceClient.setProperties(serviceProperties); + + await sleep(100); + + const origin = "anyOrigin"; + const requestMethod = "GET"; + + const pipeline = newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift( + new OPTIONSRequestPolicyFactory(origin, requestMethod) + ); + const serviceClientForOptions = new DataLakeServiceClient( + baseURL, + pipeline + ); + + const res = await serviceClientForOptions.getProperties(); + assert.ok(res._response.status === 200); + }); + + context( + "OPTIONS request should work with matching rule containing wildcard in Origin @loki @sql", + async () => { + const testCases = [ + { origin: undefined, expected: 403 }, + { origin: "contoso.com", expected: 403 }, + { origin: "bar.notcontoso.com", expected: 403 }, + { origin: "foo.contoso.com", expected: 200 }, + { origin: "foo.bar.baz.contoso.com", expected: 200 }, + { origin: "foo.CONTOSO.com", expected: 200 } + ]; + + testCases.forEach(async (testCase) => { + it(`${testCase.origin}`, async () => { + const serviceProperties = await serviceClient.getProperties(); + + const newCORS = { + allowedHeaders: "header", + allowedMethods: "GET", + allowedOrigins: "*.contoso.com", + exposedHeaders: "*", + maxAgeInSeconds: 8888 + }; + + serviceProperties.cors = [newCORS]; + + await serviceClient.setProperties(serviceProperties); + + await sleep(100); + + const origin = testCase.origin; + const requestMethod = "GET"; + const expectedStatus = testCase.expected; + + const pipeline = newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift( + new OPTIONSRequestPolicyFactory(origin, requestMethod) + ); + const serviceClientForOptions = new DataLakeServiceClient( + baseURL, + pipeline + ); + + let status: number = 0; + try { + const res = await serviceClientForOptions.getProperties(); + status = res._response.status; + } catch (e: any) { + if (!(e instanceof RestError)) { + throw e; + } + + status = e.response?.status || 0; + } + + assert.ok(status === expectedStatus); + }); + }); + } + ); + + it("Response of request to service without cors rules should not contains cors info @loki @sql", async () => { + const serviceProperties = await serviceClient.getProperties(); + + serviceProperties.cors = []; + + await serviceClient.setProperties(serviceProperties); + + await sleep(100); + + const origin = "anyOrigin"; + const pipeline = newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift(new OriginPolicyFactory(origin)); + const serviceClientWithOrigin = new DataLakeServiceClient( + baseURL, + pipeline + ); + + const res: any = await serviceClientWithOrigin.getProperties(); + + assert.ok(res["access-control-allow-origin"] === undefined); + assert.ok(res["access-control-expose-headers"] === undefined); + assert.ok(res.vary === undefined); + }); + + it("Service with mismatching cors rules should response header Vary @loki @sql", async () => { + const serviceProperties = await serviceClient.getProperties(); + + const newCORS = { + allowedHeaders: "header", + allowedMethods: "GET", + allowedOrigins: "test", + exposedHeaders: "*", + maxAgeInSeconds: 8888 + }; + + serviceProperties.cors = [newCORS]; + + await serviceClient.setProperties(serviceProperties); + + await sleep(100); + + const origin = "anyOrigin"; + const pipeline = newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift(new OriginPolicyFactory(origin)); + const serviceClientWithOrigin = new DataLakeServiceClient( + baseURL, + pipeline + ); + + let res: any = await serviceClientWithOrigin.getProperties(); + assert.ok(res.vary !== undefined); + + res = await serviceClient.getProperties(); + assert.ok(res.vary === undefined); + }); + + it("Request Match rule exists that allows all origins (*) @loki @sql", async () => { + const serviceProperties = await serviceClient.getProperties(); + + const newCORS = { + allowedHeaders: "header", + allowedMethods: "GET", + allowedOrigins: "*", + exposedHeaders: "*", + maxAgeInSeconds: 8888 + }; + + serviceProperties.cors = [newCORS]; + + await serviceClient.setProperties(serviceProperties); + + await sleep(100); + + const origin = "anyOrigin"; + const pipeline = newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift(new OriginPolicyFactory(origin)); + const serviceClientWithOrigin = new DataLakeServiceClient( + baseURL, + pipeline + ); + + let res: any = await serviceClientWithOrigin.getProperties(); + assert.ok(res["access-control-allow-origin"] === "*"); + assert.ok(res.vary === undefined); + assert.ok(res["access-control-expose-headers"] !== undefined); + + res = await serviceClient.getProperties(); + assert.ok(res["access-control-allow-origin"] === undefined); + assert.ok(res.vary === undefined); + assert.ok(res["access-control-expose-headers"] === undefined); + }); + + it("Request Match rule exists for exact origin @loki @sql", async () => { + const serviceProperties = await serviceClient.getProperties(); + + const newCORS = { + allowedHeaders: "header", + allowedMethods: "GET", + allowedOrigins: "exactOrigin", + exposedHeaders: "*", + maxAgeInSeconds: 8888 + }; + + serviceProperties.cors = [newCORS]; + + await serviceClient.setProperties(serviceProperties); + + await sleep(100); + + const origin = "exactOrigin"; + const pipeline = newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift(new OriginPolicyFactory(origin)); + const serviceClientWithOrigin = new DataLakeServiceClient( + baseURL, + pipeline + ); + + const res: any = await serviceClientWithOrigin.getProperties(); + assert.ok(res["access-control-allow-origin"] === origin); + assert.ok(res.vary !== undefined); + assert.ok(res["access-control-expose-headers"] !== undefined); + }); + + it("Requests with error response should apply for CORS @loki @sql", async () => { + const serviceProperties = await serviceClient.getProperties(); + + const newCORS = { + allowedHeaders: "header", + allowedMethods: "GET", + allowedOrigins: "exactOrigin", + exposedHeaders: "*", + maxAgeInSeconds: 8888 + }; + + serviceProperties.cors = [newCORS]; + + await serviceClient.setProperties(serviceProperties); + + await sleep(100); + + const origin = "exactOrigin"; + const pipeline = newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift(new OriginPolicyFactory(origin)); + const serviceClientWithOrigin = new DataLakeServiceClient( + baseURL, + pipeline + ); + + const containerClientWithOrigin = + serviceClientWithOrigin.getFileSystemClient("notexistcontainer"); + + try { + await containerClientWithOrigin.getProperties(); + } catch (err) { + assert.ok( + err.response.headers._headersMap["access-control-allow-origin"] + .value === origin + ); + assert.ok(err.response.headers._headersMap.vary !== undefined); + assert.ok( + err.response.headers._headersMap["access-control-expose-headers"] !== + undefined + ); + } + }); + + it("Request Match rule in sequence @loki @sql", async () => { + const serviceProperties = await serviceClient.getProperties(); + + const newCORS = [ + { + allowedHeaders: "header", + allowedMethods: "GET", + allowedOrigins: "exactOrigin", + exposedHeaders: "*", + maxAgeInSeconds: 8888 + }, + { + allowedHeaders: "header", + allowedMethods: "GET", + allowedOrigins: "*", + exposedHeaders: "*", + maxAgeInSeconds: 8888 + } + ]; + + serviceProperties.cors = newCORS; + + await serviceClient.setProperties(serviceProperties); + + await sleep(100); + + const origin = "exactOrigin"; + const pipeline = newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift(new OriginPolicyFactory(origin)); + const serviceClientWithOrigin = new DataLakeServiceClient( + baseURL, + pipeline + ); + + const res: any = await serviceClientWithOrigin.getProperties(); + assert.ok(res["access-control-allow-origin"] === origin); + assert.ok(res.vary !== undefined); + assert.ok(res["access-control-expose-headers"] !== undefined); + }); +}); diff --git a/tests/dfs/storage-file-datalake/bugs.test.ts b/tests/dfs/storage-file-datalake/bugs.test.ts new file mode 100644 index 000000000..494b69e33 --- /dev/null +++ b/tests/dfs/storage-file-datalake/bugs.test.ts @@ -0,0 +1,152 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +import assert from "assert"; +import os from "os"; +import { + DataLakeDirectoryClient, + DataLakeFileClient, + DataLakeFileSystemClient, + DataLakeServiceClient, + FileSystemListPathsResponse, + StorageSharedKeyCredential +} from "@azure/storage-file-datalake"; + +import { + bodyToString, + EMULATOR_ACCOUNT_KEY, + EMULATOR_ACCOUNT_NAME, + getUniqueName +} from "../../testutils"; +import DataLakeTestServerFactory from "../DataLakeTestServerFactory"; + +describe("Bugs", () => { + const factory = new DataLakeTestServerFactory(); + const server = factory.createServer(true); + + const baseURL = `http://${server.config.host}:${server.config.port}/devstoreaccount1`; + const serviceClient = new DataLakeServiceClient( + baseURL, + new StorageSharedKeyCredential(EMULATOR_ACCOUNT_NAME, EMULATOR_ACCOUNT_KEY), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + + let fileSystemName: string; + let fileSystemClient: DataLakeFileSystemClient; + let fileName: string; + let fileClient: DataLakeFileClient; + let directoryName: string; + let directoryClient: DataLakeDirectoryClient; + const content = "Hello World"; + + before(async () => { + await server.start(); + }); + + after(async () => { + await server.close(); + await server.clean(); + }); + + beforeEach(async function () { + fileSystemName = getUniqueName("filesystem"); + fileSystemClient = serviceClient.getFileSystemClient(fileSystemName); + await fileSystemClient.createIfNotExists(); + directoryName = getUniqueName("directory"); + directoryClient = fileSystemClient.getDirectoryClient(directoryName); + await directoryClient.create(); + fileName = getUniqueName("file"); + fileClient = fileSystemClient.getFileClient(fileName); + await fileClient.create(); + await fileClient.append(content, 0, content.length); + await fileClient.flush(content.length); + }); + + afterEach(async function () { + await fileSystemClient.deleteIfExists(); + }); + + it("listPaths should work on root @loki @sql", async () => { + await fileSystemClient + .getDirectoryClient(getUniqueName("directory")) + .create(); + await fileSystemClient + .getDirectoryClient(getUniqueName("directory")) + .create(); + await fileSystemClient + .getDirectoryClient(getUniqueName("directory")) + .create(); + await fileSystemClient.getFileClient(getUniqueName("file")).create(); + + const response = (await fileSystemClient.listPaths().byPage().next()) + .value as FileSystemListPathsResponse; + + assert.strictEqual(response.pathItems?.length, 6); + }); + + it("listPaths should not include folders/files with same prefix @loki @sql", async () => { + await fileSystemClient.getDirectoryClient("abc").create(); + await fileSystemClient.getDirectoryClient("abc123").create(); + await fileSystemClient.getDirectoryClient("abc1234").create(); + await fileSystemClient.getFileClient("abc1").create(); + await fileSystemClient.getFileClient("abc12").create(); + + const response = ( + await fileSystemClient.listPaths({ path: "abc" }).byPage().next() + ).value as FileSystemListPathsResponse; + + assert.strictEqual(response.pathItems?.length, 0); + }); + + it("recursive delete should not delete folders/files with same prefix @loki @sql", async () => { + const directoryClient = fileSystemClient.getDirectoryClient("abc"); + await directoryClient.create(); + const directoryClient2 = fileSystemClient.getDirectoryClient("abc123"); + const directoryClient3 = fileSystemClient.getDirectoryClient("abc1234"); + const fileClient1 = fileSystemClient.getFileClient("abc1"); + const fileClient2 = fileSystemClient.getFileClient("abc2"); + await directoryClient2.create(); + await directoryClient3.create(); + await fileClient1.create(); + await fileClient2.create(); + await directoryClient.delete(true); + assert.strictEqual(await directoryClient2.exists(), true); + assert.strictEqual(await directoryClient3.exists(), true); + assert.strictEqual(await fileClient1.exists(), true); + assert.strictEqual(await fileClient2.exists(), true); + }); + + it("multiple append/flush should not overwrite each other @loki @sql", async () => { + await fileClient.create(); + const len = content.length; + await fileClient.append(content, 0, len, { flush: true }); + await fileClient.append(content, len, len, { flush: true }); + await fileClient.append(content, len * 2, len, { flush: true }); + const response = await fileClient.getProperties(); + assert.strictEqual(response.contentLength, len * 3); + const readResponse = await fileClient.read(); + const read = await bodyToString(readResponse); + assert.strictEqual(read, content + content + content); + }); + + it("file should have default access control @loki @sql", async () => { + assert.ok((await fileClient.getAccessControl()).acl); + assert.ok((await fileClient.getAccessControl()).permissions); + assert.ok((await fileClient.getAccessControl()).owner); + if (os.platform() !== "win32") { + assert.ok((await fileClient.getAccessControl()).group); + } + }); + + it("directory should have default access control @loki @sql", async () => { + assert.ok((await directoryClient.getAccessControl()).acl); + assert.ok((await directoryClient.getAccessControl()).permissions); + assert.ok((await directoryClient.getAccessControl()).owner); + if (os.platform() !== "win32") { + assert.ok((await directoryClient.getAccessControl()).group); + } + }); +}); diff --git a/tests/dfs/storage-file-datalake/https.test.ts b/tests/dfs/storage-file-datalake/https.test.ts new file mode 100644 index 000000000..c6b6219e3 --- /dev/null +++ b/tests/dfs/storage-file-datalake/https.test.ts @@ -0,0 +1,54 @@ +import { + DataLakeServiceClient, + newPipeline, + StorageSharedKeyCredential +} from "@azure/storage-file-datalake"; + +import { configLogger } from "../../../src/common/Logger"; +import { + EMULATOR_ACCOUNT_KEY, + EMULATOR_ACCOUNT_NAME, + getUniqueName +} from "../../testutils"; +import DataLakeTestServerFactory from "../DataLakeTestServerFactory"; + +// Set true to enable debug log +configLogger(false); + +describe("Blob HTTPS", () => { + const factory = new DataLakeTestServerFactory(); + const server = factory.createServer(false, false, true); + const baseURL = `https://${server.config.host}:${server.config.port}/devstoreaccount1`; + + before(async () => { + await server.start(); + }); + + after(async () => { + await server.close(); + await server.clean(); + }); + + it(`Should work with correct shared key using HTTPS endpoint @loki @sql`, async () => { + const serviceClient = new DataLakeServiceClient( + baseURL, + newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ) + ); + + const containerName: string = getUniqueName("1container-with-dash"); + const containerClient = serviceClient.getFileSystemClient(containerName); + + await containerClient.create(); + await containerClient.delete(); + }); +}); diff --git a/tests/dfs/storage-file-datalake/integration/files-dir-mixed-apis.test.ts b/tests/dfs/storage-file-datalake/integration/files-dir-mixed-apis.test.ts new file mode 100644 index 000000000..3a49581ca --- /dev/null +++ b/tests/dfs/storage-file-datalake/integration/files-dir-mixed-apis.test.ts @@ -0,0 +1,271 @@ +import assert, { fail } from "assert"; +import { count } from "console"; + +import { BlobServiceClient } from "@azure/storage-blob"; +import { + DataLakeServiceClient, + newPipeline, + StorageSharedKeyCredential +} from "@azure/storage-file-datalake"; + +import { configLogger } from "../../../../src/common/Logger"; +import { + bodyToString, + EMULATOR_ACCOUNT_KEY, + EMULATOR_ACCOUNT_NAME, + getUniqueName +} from "../../../testutils"; +import DataLakeTestServerFactory from "../../DataLakeTestServerFactory"; + +// Set true to enable debug log +configLogger(false); + +describe("DirectoryAndFileApis", () => { + const factory = new DataLakeTestServerFactory(); + const server = factory.createServer(); + + const baseURL = `http://${server.config.host}:${server.config.port}/devstoreaccount1`; + const serviceClient = new DataLakeServiceClient( + baseURL, + newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ) + ); + + const blobServiceClient = new BlobServiceClient( + baseURL, + newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ) + ); + + const fileSytemName: string = getUniqueName("filesystem"); + const fileSystemClient = serviceClient.getFileSystemClient(fileSytemName); + const parentDirectoryName: string = getUniqueName("parentDirectory"); + const parentDirectoryClient = + fileSystemClient.getDirectoryClient(parentDirectoryName); + const chilDirectoryName: string = getUniqueName("childDirectory"); + const childDirectoryClient = + parentDirectoryClient.getSubdirectoryClient(chilDirectoryName); + const grandChildDirectoryName = getUniqueName("grandChildDirectory"); + const grandChildDirectoryClient = childDirectoryClient.getSubdirectoryClient( + grandChildDirectoryName + ); + const file1Name: string = getUniqueName("file1"); + const file1Client = grandChildDirectoryClient.getFileClient(file1Name); + const file2Name: string = getUniqueName("file2"); + const file2Client = grandChildDirectoryClient.getFileClient(file2Name); + + let containerClient = blobServiceClient.getContainerClient(fileSytemName); + let blob1Client = containerClient.getBlobClient( + `${parentDirectoryName}/${chilDirectoryName}/${grandChildDirectoryName}/${file1Name}` + ); + let blob2Client = containerClient.getBlobClient( + `${parentDirectoryName}/${chilDirectoryName}/${grandChildDirectoryName}/${file2Name}` + ); + + const content = "Hello World"; + + before(async () => { + await server.start(); + }); + + after(async () => { + await fileSystemClient.delete(); + await server.close(); + await server.clean(); + }); + + it("create directorys @loki @sql", async () => { + await fileSystemClient.createIfNotExists(); + await parentDirectoryClient.deleteIfExists(true); + //should create parent as well; + await grandChildDirectoryClient.create(); + assert.strictEqual(await grandChildDirectoryClient.exists(), true); + assert.strictEqual(await childDirectoryClient.exists(), true); + assert.strictEqual(await parentDirectoryClient.exists(), true); + }); + + it("create File (DataLake) @loki @sql", async () => { + await file1Client.create(); + assert.strictEqual(await file1Client.exists(), true); + const properties = await file1Client.getProperties(); + assert.strictEqual(properties.contentLength, 0); + await file1Client.delete(); + assert.strictEqual(await file1Client.exists(), false); + }); + + it("create AppendBlob (Blob) @loki @sql", async () => { + await blob1Client.getAppendBlobClient().create(); + assert.strictEqual(await blob1Client.exists(), true); + const properties = await blob1Client.getProperties(); + assert.strictEqual(properties.contentLength, 0); + }); + + it("Append to File (DataLake) @loki @sql", async () => { + await file1Client.append(content, 0, content.length); + await file1Client.flush(content.length); + const properties = await file1Client.getProperties(); + assert.strictEqual(properties.contentLength, content.length); + }); + + it("Append to File (Blob) @loki @sql", async () => { + await blob1Client + .getAppendBlobClient() + .appendBlock(content, content.length); + const properties = await file1Client.getProperties(); + assert.strictEqual(properties.contentLength, content.length * 2); + }); + + it("Append to Non Existing File should fail (DataLake) @loki @sql", async () => { + const nonExistingFileClient = grandChildDirectoryClient.getFileClient( + getUniqueName("non-existing-file") + ); + let error; + try { + await nonExistingFileClient.append(content, 0, content.length); + } catch (err) { + error = err; + assert.strictEqual(err.code, "PathNotFound"); + assert.strictEqual( + err.message.startsWith("The specified path does not exist."), + true + ); + } + + if (!error) fail(); + }); + + it("Flush to Non Existing File should fail (DataLake) @loki @sql", async () => { + const nonExistingFileClient = grandChildDirectoryClient.getFileClient( + getUniqueName("non-existing-file") + ); + let error; + try { + await nonExistingFileClient.flush(content.length); + } catch (err) { + error = err; + assert.strictEqual(err.code, "PathNotFound"); + assert.strictEqual( + err.message.startsWith("The specified path does not exist."), + true + ); + } + + if (!error) fail(); + }); + + it("Append to Non Existing File should fail (Blob) @loki @sql", async () => { + const nonExistingBlobClient = containerClient.getBlobClient( + getUniqueName("non-existing-file") + ); + let error; + try { + await nonExistingBlobClient + .getAppendBlobClient() + .appendBlock(content, content.length); + } catch (err) { + error = err; + assert.strictEqual(err.code, "BlobNotFound"); + assert.strictEqual( + err.message.startsWith("The specified blob does not exist."), + true + ); + } + + if (!error) fail(); + }); + + it("Read File (DataLake) @loki @sql", async () => { + const readContent = await file1Client.readToBuffer(0, count.length * 2); + assert.deepStrictEqual(readContent, Buffer.from(content + content)); + }); + + it("Read File (Blob) @loki @sql", async () => { + const readContent = await blob1Client.downloadToBuffer(0, count.length * 2); + assert.deepStrictEqual(readContent, Buffer.from(content + content)); + }); + + it("Copy File (DataLake) [Read Then upload no direct copy] @loki @sql", async () => { + assert.strictEqual(await file1Client.exists(), true); + assert.strictEqual(await file2Client.exists(), false); + const readResponse = await file1Client.read(); + const readContent = await bodyToString(readResponse); + await file2Client.create(); + await file2Client.append(readContent, 0, readContent.length, { + flush: true + }); + assert.strictEqual(await file1Client.exists(), true); + assert.strictEqual(await file2Client.exists(), true); + const writtenResponse = await file2Client.read(); + const writtenContent = await bodyToString(writtenResponse); + assert.deepStrictEqual(readContent, writtenContent); + await file2Client.delete(); + }); + + it("Copy File (Blob) @loki @sql", async () => { + assert.strictEqual(await blob1Client.exists(), true); + assert.strictEqual(await blob2Client.exists(), false); + await blob2Client.syncCopyFromURL(blob1Client.url); + assert.strictEqual(await blob1Client.exists(), true); + assert.strictEqual(await blob2Client.exists(), true); + const result1 = await blob1Client.download(); + const result2 = await blob2Client.download(); + const readContent1 = await bodyToString(result1); + const readContent2 = await bodyToString(result2); + assert.deepStrictEqual(readContent1, readContent2); + }); + + it.skip("Move/Rename Directory [in azure-data-lake current version only works with production style url, in normal mode it doesn't send account name in url] (DataLake) @loki @sql", async () => { + const destDirClient = parentDirectoryClient.getSubdirectoryClient( + getUniqueName("sourcedir") + ); + assert.strictEqual(await childDirectoryClient.exists(), true); + assert.strictEqual(await destDirClient.exists(), false); + await childDirectoryClient.move(destDirClient.name); + assert.strictEqual(await childDirectoryClient.exists(), false); + assert.strictEqual(await grandChildDirectoryClient.exists(), false); + assert.strictEqual(await file1Client.exists(), false); + assert.strictEqual(await file2Client.exists(), false); + assert.strictEqual(await destDirClient.exists(), true); + await destDirClient.move(childDirectoryClient.name); + assert.strictEqual(await childDirectoryClient.exists(), true); + assert.strictEqual(await grandChildDirectoryClient.exists(), true); + assert.strictEqual(await file1Client.exists(), true); + assert.strictEqual(await file2Client.exists(), true); + assert.strictEqual(await destDirClient.exists(), false); + }); + + it("Append then read should return empty withoutflush (DataLake) @loki @sql", async () => { + await file1Client.deleteIfExists(); + assert.strictEqual(await file1Client.exists(), false); + await file1Client.create(); + assert.strictEqual(await file1Client.exists(), true); + await file1Client.append(content, 0, content.length); + let readResponse = await file1Client.read(); + let readContent = await bodyToString(readResponse); + assert.strictEqual(readContent.length, 0); + await file1Client.append(content, 0, content.length); + assert.strictEqual(readContent.length, 0); + await file1Client.flush(content.length * 2); + readResponse = await file1Client.read(); + readContent = await bodyToString(readResponse); + assert.deepStrictEqual(readContent, content + content); + }); +}); diff --git a/tests/dfs/storage-file-datalake/oauth.test.ts b/tests/dfs/storage-file-datalake/oauth.test.ts new file mode 100644 index 000000000..7cdf4135c --- /dev/null +++ b/tests/dfs/storage-file-datalake/oauth.test.ts @@ -0,0 +1,891 @@ +import * as assert from "assert"; + +import { + AccountSASPermissions, + AccountSASResourceTypes, + AccountSASServices, + AnonymousCredential, + DataLakeFileSystemClient, + DataLakeServiceClient, + FileSystemSASPermissions, + generateAccountSASQueryParameters, + generateDataLakeSASQueryParameters, + newPipeline, + SASProtocol, + StorageSharedKeyCredential +} from "@azure/storage-file-datalake"; + +import { configLogger } from "../../../src/common/Logger"; +import { SimpleTokenCredential } from "../../simpleTokenCredential"; +import { + EMULATOR_ACCOUNT_KEY, + EMULATOR_ACCOUNT_NAME, + generateJWTToken, + getUniqueName, + upload +} from "../../testutils"; +import DataLakeTestServerFactory from "../DataLakeTestServerFactory"; + +// Set true to enable debug log +configLogger(false); + +describe("Blob OAuth Basic", () => { + const factory = new DataLakeTestServerFactory(); + let server = factory.createServer(false, false, true, "basic"); + const baseURL = `https://${server.config.host}:${server.config.port}/devstoreaccount1`; + + before(async () => { + await server.start(); + }); + + after(async () => { + await server.close(); + await server.clean(); + }); + + it(`Should work with create container @loki @sql`, async () => { + const token = generateJWTToken( + new Date("2019/01/01"), + new Date("2019/01/01"), + new Date("2100/01/01"), + "https://sts.windows-ppe.net/ab1f708d-50f6-404c-a006-d71b2ac7a606/", + "https://storage.azure.com", + "user_impersonation", + "23657296-5cd5-45b0-a809-d972a7f4dfe1", + "dd0d0df1-06c3-436c-8034-4b9a153097ce" + ); + + const serviceClient = new DataLakeServiceClient( + baseURL, + newPipeline(new SimpleTokenCredential(token), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const containerName: string = getUniqueName("1container-with-dash"); + const containerClient = serviceClient.getFileSystemClient(containerName); + + await containerClient.create(); + await containerClient.delete(); + }); + + it(`Should work with delegation SAS @loki @sql`, async () => { + const token = generateJWTToken( + new Date("2019/01/01"), + new Date("2019/01/01"), + new Date("2100/01/01"), + "https://sts.windows-ppe.net/ab1f708d-50f6-404c-a006-d71b2ac7a606/", + "https://storage.azure.com", + "user_impersonation", + "23657296-5cd5-45b0-a809-d972a7f4dfe1", + "dd0d0df1-06c3-436c-8034-4b9a153097ce" + ); + + const serviceClient = new DataLakeServiceClient( + baseURL, + newPipeline(new SimpleTokenCredential(token), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const startTime = new Date(); + startTime.setHours(startTime.getHours() - 1); + const expiryTime = new Date(); + expiryTime.setDate(expiryTime.getDate() + 1); + + const userDelegationKey = await serviceClient.getUserDelegationKey( + startTime, + expiryTime + ); + + const containerName: string = getUniqueName("1container-with-dash"); + + const sasExpirytime = new Date(); + sasExpirytime.setHours(sasExpirytime.getHours() + 1); + + const containerSAS = generateDataLakeSASQueryParameters( + { + fileSystemName: containerName, + expiresOn: sasExpirytime, + permissions: FileSystemSASPermissions.parse("racwdl") + }, + userDelegationKey, + "devstoreaccount1" + ); + + const containerClient = new DataLakeFileSystemClient( + `${serviceClient.url}/${containerName}?${containerSAS}`, + newPipeline(new AnonymousCredential()) + ); + await containerClient.create(); + await containerClient.delete(); + }); + + it(`Should work with delegation SAS container client doing blob upload @loki @sql`, async () => { + const token = generateJWTToken( + new Date("2019/01/01"), + new Date("2019/01/01"), + new Date("2100/01/01"), + "https://sts.windows-ppe.net/ab1f708d-50f6-404c-a006-d71b2ac7a606/", + "https://storage.azure.com", + "user_impersonation", + "23657296-5cd5-45b0-a809-d972a7f4dfe1", + "dd0d0df1-06c3-436c-8034-4b9a153097ce" + ); + + const serviceClient = new DataLakeServiceClient( + baseURL, + newPipeline(new SimpleTokenCredential(token), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const startTime = new Date(); + startTime.setHours(startTime.getHours() - 1); + const expiryTime = new Date(); + expiryTime.setDate(expiryTime.getDate() + 1); + + const userDelegationKey = await serviceClient.getUserDelegationKey( + startTime, + expiryTime + ); + + const containerName: string = getUniqueName("1container-with-dash"); + + const sasExpirytime = new Date(); + sasExpirytime.setHours(sasExpirytime.getHours() + 1); + + const containerSAS = generateDataLakeSASQueryParameters( + { + fileSystemName: containerName, + expiresOn: sasExpirytime, + permissions: FileSystemSASPermissions.parse("racwdl") + }, + userDelegationKey, + "devstoreaccount1" + ); + + const containerClient = new DataLakeFileSystemClient( + `${serviceClient.url}/${containerName}?${containerSAS}`, + newPipeline(new AnonymousCredential()) + ); + await containerClient.create(); + const blobClient = await containerClient.getFileClient("test"); + const data = "Test Data"; + await upload(blobClient, data); + await containerClient.delete(); + }); + + it(`Should fail with delegation SAS with invalid time duration @loki @sql`, async () => { + const token = generateJWTToken( + new Date("2019/01/01"), + new Date("2019/01/01"), + new Date("2100/01/01"), + "https://sts.windows-ppe.net/ab1f708d-50f6-404c-a006-d71b2ac7a606/", + "https://storage.azure.com", + "user_impersonation", + "23657296-5cd5-45b0-a809-d972a7f4dfe1", + "dd0d0df1-06c3-436c-8034-4b9a153097ce" + ); + + const serviceClient = new DataLakeServiceClient( + baseURL, + newPipeline(new SimpleTokenCredential(token), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + const containerName: string = getUniqueName("1container-with-dash"); + + // Later user delegation key start time + let startTime = new Date(); + startTime.setMinutes(startTime.getMinutes() + 20); + let expiryTime = new Date(); + expiryTime.setDate(expiryTime.getDate() + 1); + let userDelegationKey = await serviceClient.getUserDelegationKey( + startTime, + expiryTime + ); + + let sasExpirytime = new Date(); + sasExpirytime.setHours(sasExpirytime.getHours() + 1); + + let containerSAS = generateDataLakeSASQueryParameters( + { + fileSystemName: containerName, + expiresOn: sasExpirytime, + permissions: FileSystemSASPermissions.parse("racwdl") + }, + userDelegationKey, + "devstoreaccount1" + ); + + let containerClient = new DataLakeFileSystemClient( + `${serviceClient.url}/${containerName}?${containerSAS}`, + newPipeline(new AnonymousCredential()) + ); + let failed = false; + try { + await containerClient.create(); + } catch (err) { + failed = true; + assert.equal(err.statusCode, 403); + } + assert.ok(failed); + + // Eearlier user delegation key expirty time + startTime = new Date(); + startTime.setDate(startTime.getDate() - 1); + expiryTime = new Date(); + expiryTime.setMinutes(expiryTime.getMinutes() - 1); + userDelegationKey = await serviceClient.getUserDelegationKey( + startTime, + expiryTime + ); + + sasExpirytime = new Date(); + sasExpirytime.setHours(sasExpirytime.getHours() + 1); + + containerSAS = generateDataLakeSASQueryParameters( + { + fileSystemName: containerName, + expiresOn: sasExpirytime, + permissions: FileSystemSASPermissions.parse("racwdl") + }, + userDelegationKey, + "devstoreaccount1" + ); + + containerClient = new DataLakeFileSystemClient( + `${serviceClient.url}/${containerName}?${containerSAS}`, + newPipeline(new AnonymousCredential()) + ); + + failed = false; + try { + await containerClient.create(); + } catch (err) { + failed = true; + assert.equal(err.statusCode, 403); + } + assert.ok(failed); + }); + + it(`Should fail with delegation SAS with access policy @loki @sql`, async () => { + const token = generateJWTToken( + new Date("2019/01/01"), + new Date("2019/01/01"), + new Date("2100/01/01"), + "https://sts.windows-ppe.net/ab1f708d-50f6-404c-a006-d71b2ac7a606/", + "https://storage.azure.com", + "user_impersonation", + "23657296-5cd5-45b0-a809-d972a7f4dfe1", + "dd0d0df1-06c3-436c-8034-4b9a153097ce" + ); + + const serviceClient = new DataLakeServiceClient( + baseURL, + newPipeline(new SimpleTokenCredential(token), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const startTime = new Date(); + startTime.setHours(startTime.getHours() - 1); + const expiryTime = new Date(); + expiryTime.setDate(expiryTime.getDate() + 1); + const userDelegationKey = await serviceClient.getUserDelegationKey( + startTime, + expiryTime + ); + + const serviceClientWithAccountKey = new DataLakeServiceClient( + baseURL, + newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ) + ); + const containerName: string = getUniqueName("1container-with-dash"); + const containerClientWithKey = + serviceClientWithAccountKey.getFileSystemClient(containerName); + await containerClientWithKey.create(); + await containerClientWithKey.setAccessPolicy(undefined, [ + { + accessPolicy: { + permissions: "racwdl" + }, + id: "MTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTI=" + } + ]); + + const sasExpirytime = new Date(); + sasExpirytime.setHours(sasExpirytime.getHours() + 1); + + const containerSAS = generateDataLakeSASQueryParameters( + { + fileSystemName: containerName, + expiresOn: sasExpirytime, + permissions: FileSystemSASPermissions.parse("racwdl"), + identifier: "MTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTI=" + }, + userDelegationKey, + "devstoreaccount1" + ); + + const containerClient = new DataLakeFileSystemClient( + `${serviceClient.url}/${containerName}?${containerSAS}`, + newPipeline(new AnonymousCredential()) + ); + + let failed = false; + try { + await containerClient.getProperties(); + } catch (err) { + failed = true; + assert.equal(err.statusCode, 403); + } + assert.ok(failed); + }); + + it(`Should not work with invalid JWT token @loki @sql`, async () => { + const serviceClient = new DataLakeServiceClient( + baseURL, + newPipeline(new SimpleTokenCredential("invalid token"), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const containerName: string = getUniqueName("1container-with-dash"); + const containerClient = serviceClient.getFileSystemClient(containerName); + + try { + await containerClient.create(); + await containerClient.delete(); + } catch (err) { + assert.deepStrictEqual( + err.message.includes("Server failed to authenticate the request."), + true + ); + return; + } + assert.fail(); + }); + + it(`Should work with valid audiences @loki @sql`, async () => { + const audiences = [ + "https://storage.azure.com", + "https://storage.azure.com/", + "e406a681-f3d4-42a8-90b6-c2b029497af1", + "https://devstoreaccount1.blob.core.windows.net", + "https://devstoreaccount1.blob.core.windows.net/", + "https://devstoreaccount1.blob.core.chinacloudapi.cn", + "https://devstoreaccount1.blob.core.chinacloudapi.cn/", + "https://devstoreaccount1.blob.core.usgovcloudapi.net", + "https://devstoreaccount1.blob.core.usgovcloudapi.net/", + "https://devstoreaccount1.blob.core.cloudapi.de", + "https://devstoreaccount1.blob.core.cloudapi.de/" + ]; + + for (const audience of audiences) { + const token = generateJWTToken( + new Date("2019/01/01"), + new Date("2019/01/01"), + new Date("2100/01/01"), + "https://sts.windows-ppe.net/ab1f708d-50f6-404c-a006-d71b2ac7a606/", + audience, + "user_impersonation", + "23657296-5cd5-45b0-a809-d972a7f4dfe1", + "dd0d0df1-06c3-436c-8034-4b9a153097ce" + ); + + const serviceClient = new DataLakeServiceClient( + baseURL, + newPipeline(new SimpleTokenCredential(token), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const containerName: string = getUniqueName("1container-with-dash"); + const containerClient = serviceClient.getFileSystemClient(containerName); + + await containerClient.create(); + await containerClient.delete(); + } + }); + + it(`Should not work with invalid audiences @loki @sql`, async () => { + const token = generateJWTToken( + new Date("2019/01/01"), + new Date("2019/01/01"), + new Date("2100/01/01"), + "https://sts.windows-ppe.net/ab1f708d-50f6-404c-a006-d71b2ac7a606/", + "https://invalidaccount.blob.core.windows.net", + "user_impersonation", + "23657296-5cd5-45b0-a809-d972a7f4dfe1", + "dd0d0df1-06c3-436c-8034-4b9a153097ce" + ); + + const serviceClient = new DataLakeServiceClient( + baseURL, + newPipeline(new SimpleTokenCredential(token), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const containerName: string = getUniqueName("1container-with-dash"); + const containerClient = serviceClient.getFileSystemClient(containerName); + + try { + await containerClient.create(); + await containerClient.delete(); + } catch (err) { + assert.deepStrictEqual( + err.message.includes("Server failed to authenticate the request."), + true + ); + assert.deepStrictEqual( + err.details.AuthenticationErrorDetail.includes("audience"), + true + ); + return; + } + assert.fail(); + }); + + it(`Should work with valid issuers @loki @sql`, async () => { + const issuerPrefixes = [ + "https://sts.windows.net/", + "https://sts.microsoftonline.de/", + "https://sts.chinacloudapi.cn/", + "https://sts.windows-ppe.net" + ]; + + for (const issuerPrefix of issuerPrefixes) { + const token = generateJWTToken( + new Date("2019/01/01"), + new Date("2019/01/01"), + new Date("2100/01/01"), + `${issuerPrefix}/ab1f708d-50f6-404c-a006-d71b2ac7a606/`, + "e406a681-f3d4-42a8-90b6-c2b029497af1", + "user_impersonation", + "23657296-5cd5-45b0-a809-d972a7f4dfe1", + "dd0d0df1-06c3-436c-8034-4b9a153097ce" + ); + + const serviceClient = new DataLakeServiceClient( + baseURL, + newPipeline(new SimpleTokenCredential(token), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const containerName: string = getUniqueName("1container-with-dash"); + const containerClient = serviceClient.getFileSystemClient(containerName); + + await containerClient.create(); + await containerClient.delete(); + } + }); + + it(`Should not work with invalid issuers @loki @sql`, async () => { + const token = generateJWTToken( + new Date("2019/01/01"), + new Date("2019/01/01"), + new Date("2100/01/01"), + "https://invalidissuer/ab1f708d-50f6-404c-a006-d71b2ac7a606/", + "https://invalidaccount.blob.core.windows.net", + "user_impersonation", + "23657296-5cd5-45b0-a809-d972a7f4dfe1", + "dd0d0df1-06c3-436c-8034-4b9a153097ce" + ); + + const serviceClient = new DataLakeServiceClient( + baseURL, + newPipeline(new SimpleTokenCredential(token), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const containerName: string = getUniqueName("1container-with-dash"); + const containerClient = serviceClient.getFileSystemClient(containerName); + + try { + await containerClient.create(); + await containerClient.delete(); + } catch (err) { + assert.deepStrictEqual( + err.message.includes("Server failed to authenticate the request."), + true + ); + assert.deepStrictEqual( + err.details.AuthenticationErrorDetail.includes("issuer"), + true + ); + return; + } + assert.fail(); + }); + + it(`Should not work with invalid nbf @loki @sql`, async () => { + const token = generateJWTToken( + new Date("2119/01/01"), + new Date("2019/01/01"), + new Date("2100/01/01"), + "https://sts.windows-ppe.net/ab1f708d-50f6-404c-a006-d71b2ac7a606/", + "https://devstoreaccount1.blob.core.windows.net", + "user_impersonation", + "23657296-5cd5-45b0-a809-d972a7f4dfe1", + "dd0d0df1-06c3-436c-8034-4b9a153097ce" + ); + + const serviceClient = new DataLakeServiceClient( + baseURL, + newPipeline(new SimpleTokenCredential(token), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const containerName: string = getUniqueName("1container-with-dash"); + const containerClient = serviceClient.getFileSystemClient(containerName); + + try { + await containerClient.create(); + await containerClient.delete(); + } catch (err) { + assert.deepStrictEqual( + err.message.includes("Server failed to authenticate the request."), + true + ); + assert.deepStrictEqual( + err.details.AuthenticationErrorDetail.includes("Lifetime"), + true + ); + return; + } + assert.fail(); + }); + + it(`Should not work with invalid exp @loki @sql`, async () => { + const token = generateJWTToken( + new Date("2019/01/01"), + new Date("2019/01/01"), + new Date("2019/01/01"), + "https://sts.windows-ppe.net/ab1f708d-50f6-404c-a006-d71b2ac7a606/", + "https://devstoreaccount1.blob.core.windows.net", + "user_impersonation", + "23657296-5cd5-45b0-a809-d972a7f4dfe1", + "dd0d0df1-06c3-436c-8034-4b9a153097ce" + ); + + const serviceClient = new DataLakeServiceClient( + baseURL, + newPipeline(new SimpleTokenCredential(token), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const containerName: string = getUniqueName("1container-with-dash"); + const containerClient = serviceClient.getFileSystemClient(containerName); + + try { + await containerClient.create(); + await containerClient.delete(); + } catch (err) { + assert.deepStrictEqual( + err.message.includes("Server failed to authenticate the request."), + true + ); + assert.deepStrictEqual( + err.details.AuthenticationErrorDetail.includes("expire"), + true + ); + return; + } + assert.fail(); + }); + + it(`Should not work with get container ACL @loki @sql`, async () => { + const token = generateJWTToken( + new Date("2019/01/01"), + new Date("2019/01/01"), + new Date("2100/01/01"), + "https://sts.windows-ppe.net/ab1f708d-50f6-404c-a006-d71b2ac7a606/", + "https://devstoreaccount1.blob.core.windows.net", + "user_impersonation", + "23657296-5cd5-45b0-a809-d972a7f4dfe1", + "dd0d0df1-06c3-436c-8034-4b9a153097ce" + ); + + const serviceClient = new DataLakeServiceClient( + baseURL, + newPipeline(new SimpleTokenCredential(token), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const containerName: string = getUniqueName("1container-with-dash"); + const containerClient = serviceClient.getFileSystemClient(containerName); + await containerClient.create(); + + try { + await containerClient.getAccessPolicy(); + } catch (err) { + assert.deepStrictEqual( + err.message.includes("Server failed to authenticate the request."), + true + ); + await containerClient.delete(); + return; + } + await containerClient.delete(); + assert.fail(); + }); + + it(`Should not work with set container ACL @loki @sql`, async () => { + const token = generateJWTToken( + new Date("2019/01/01"), + new Date("2019/01/01"), + new Date("2100/01/01"), + "https://sts.windows-ppe.net/ab1f708d-50f6-404c-a006-d71b2ac7a606/", + "https://devstoreaccount1.blob.core.windows.net", + "user_impersonation", + "23657296-5cd5-45b0-a809-d972a7f4dfe1", + "dd0d0df1-06c3-436c-8034-4b9a153097ce" + ); + + const serviceClient = new DataLakeServiceClient( + baseURL, + newPipeline(new SimpleTokenCredential(token), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const containerName: string = getUniqueName("1container-with-dash"); + const containerClient = serviceClient.getFileSystemClient(containerName); + await containerClient.create(); + + try { + await containerClient.setAccessPolicy("filesystem"); + } catch (err) { + assert.deepStrictEqual( + err.message.includes("Server failed to authenticate the request."), + true + ); + await containerClient.delete(); + return; + } + await containerClient.delete(); + assert.fail(); + }); + + it("Create container with not exist Account, return 404 @loki @sql", async () => { + const accountNameNotExist = "devstoreaccountnotexist"; + const baseURL = `https://${server.config.host}:${server.config.port}/${accountNameNotExist}`; + const containerName: string = getUniqueName("1container-with-dash"); + + // Shared key + const sharedKeyCredential = new StorageSharedKeyCredential( + accountNameNotExist, + EMULATOR_ACCOUNT_KEY + ); + let serviceClient = new DataLakeServiceClient( + baseURL, + newPipeline(sharedKeyCredential, { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + let containerClientNotExist = + serviceClient.getFileSystemClient(containerName); + try { + await containerClientNotExist.create(); + } catch (err) { + if (err.statusCode !== 404 && err.code !== "ResourceNotFound") { + assert.fail( + "Create queue with shared key not fail as expected." + err.toString() + ); + } + } + + // Oauth + const token = generateJWTToken( + new Date("2019/01/01"), + new Date("2019/01/01"), + new Date("2100/01/01"), + "https://sts.windows-ppe.net/ab1f708d-50f6-404c-a006-d71b2ac7a606/", + "https://storage.azure.com", + "user_impersonation", + "23657296-5cd5-45b0-a809-d972a7f4dfe1", + "dd0d0df1-06c3-436c-8034-4b9a153097ce" + ); + + serviceClient = new DataLakeServiceClient( + baseURL, + newPipeline(new SimpleTokenCredential(token), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + containerClientNotExist = serviceClient.getFileSystemClient(containerName); + try { + await containerClientNotExist.create(); + } catch (err) { + if (err.statusCode !== 404 && err.code !== "ResourceNotFound") { + assert.fail( + "Create queue with oauth not fail as expected." + err.toString() + ); + } + } + // Account SAS + const now = new Date(); + now.setMinutes(now.getMinutes() - 5); + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + const sas = generateAccountSASQueryParameters( + { + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: AccountSASPermissions.parse("rwdlacup"), + protocol: SASProtocol.HttpsAndHttp, + resourceTypes: AccountSASResourceTypes.parse("sco").toString(), + services: AccountSASServices.parse("btqf").toString(), + startsOn: now, + version: "2016-05-31" + }, + sharedKeyCredential + ).toString(); + let sasURL = `${serviceClient.url}?${sas}`; + serviceClient = new DataLakeServiceClient( + sasURL, + newPipeline(new AnonymousCredential(), { + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + containerClientNotExist = serviceClient.getFileSystemClient(containerName); + try { + await containerClientNotExist.create(); + } catch (err) { + if (err.statusCode !== 404 && err.code !== "ResourceNotFound") { + assert.fail( + "Create queue with account sas not fail as expected." + err.toString() + ); + } + } + + // Service SAS + const containerSAS = generateDataLakeSASQueryParameters( + { + fileSystemName: containerName, + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: FileSystemSASPermissions.parse("racwdl"), + protocol: SASProtocol.HttpsAndHttp, + startsOn: now, + version: "2016-05-31" + }, + sharedKeyCredential + ); + sasURL = `${serviceClient.url}?${containerSAS}`; + serviceClient = new DataLakeServiceClient( + sasURL, + newPipeline(new AnonymousCredential(), { + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + containerClientNotExist = serviceClient.getFileSystemClient(containerName); + try { + await containerClientNotExist.create(); + } catch (err) { + if (err.statusCode !== 404 && err.code !== "ResourceNotFound") { + assert.fail( + "Create queue with service sas not fail as expected." + err.toString() + ); + } + } + }); + + it(`Should not work with HTTP @loki @sql`, async () => { + await server.close(); + await server.clean(); + + server = factory.createServer(false, false, false, "basic"); + await server.start(); + + const httpBaseURL = `http://${server.config.host}:${server.config.port}/devstoreaccount1`; + + const token = generateJWTToken( + new Date("2019/01/01"), + new Date("2019/01/01"), + new Date("2019/01/01"), + "https://sts.windows-ppe.net/ab1f708d-50f6-404c-a006-d71b2ac7a606/", + "https://devstoreaccount1.blob.core.windows.net", + "user_impersonation", + "23657296-5cd5-45b0-a809-d972a7f4dfe1", + "dd0d0df1-06c3-436c-8034-4b9a153097ce" + ); + + const serviceClient = new DataLakeServiceClient( + httpBaseURL, + newPipeline(new SimpleTokenCredential(token), { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const containerName: string = getUniqueName("1container-with-dash"); + const containerClient = serviceClient.getFileSystemClient(containerName); + + try { + await containerClient.create(); + await containerClient.delete(); + } catch (err) { + assert.deepStrictEqual( + err.message.includes("Bearer token authentication is not permitted"), + true + ); + assert.deepStrictEqual(err.message.includes("non-https"), true); + return; + } + assert.fail(); + }); +}); diff --git a/tests/dfs/storage-file-datalake/sas.test.ts b/tests/dfs/storage-file-datalake/sas.test.ts new file mode 100644 index 000000000..63a98afec --- /dev/null +++ b/tests/dfs/storage-file-datalake/sas.test.ts @@ -0,0 +1,851 @@ +import * as assert from "assert"; +import { + AccountSASPermissions, + AccountSASResourceTypes, + AccountSASServices, + AnonymousCredential, + DataLakeFileClient, + DataLakeFileSystemClient, + DataLakeSASPermissions, + DataLakeServiceClient, + FileSystemSASPermissions, + generateAccountSASQueryParameters, + generateDataLakeSASQueryParameters, + newPipeline, + SASProtocol, + StorageSharedKeyCredential +} from "@azure/storage-file-datalake"; + +import { configLogger } from "../../../src/common/Logger"; +import { + EMULATOR_ACCOUNT_KEY_STR, + EMULATOR_ACCOUNT_NAME +} from "../../../src/dfs/utils/constants"; +import { getUniqueName } from "../../testutils"; +import DataLakeTestServerFactory from "../DataLakeTestServerFactory"; + +const EMULATOR_ACCOUNT2_NAME = "devstoreaccount2"; +const EMULATOR_ACCOUNT2_KEY_STR = + "MTAwCjE2NQoyMjUKMTAzCjIxOAoyNDEKNDAKNzgKMTkxCjE3OAoyMTQKMTY5CjIxMwo2MQoyNTIKMTQxCg=="; + +// Set true to enable debug log +configLogger(false); + +describe("Shared Access Signature (SAS) authentication", () => { + // Setup two accounts for validating cross-account copy operations + process.env[ + "AZURITE_ACCOUNTS" + ] = `${EMULATOR_ACCOUNT_NAME}:${EMULATOR_ACCOUNT_KEY_STR};${EMULATOR_ACCOUNT2_NAME}:${EMULATOR_ACCOUNT2_KEY_STR}`; + + const factory = new DataLakeTestServerFactory(); + const server = factory.createServer(); + + const baseURL = `http://${server.config.host}:${server.config.port}/devstoreaccount1`; + const serviceClient = new DataLakeServiceClient( + baseURL, + newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY_STR + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ) + ); + + before(async () => { + await server.start(); + }); + + after(async () => { + await server.close(); + await server.clean(); + }); + + it("generateAccountSASQueryParameters should work @loki @sql", async () => { + const now = new Date(); + now.setMinutes(now.getMinutes() - 5); // Skip clock skew with server + + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const storageSharedKeyCredential = factories[factories.length - 1]; + + const sas = generateAccountSASQueryParameters( + { + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: AccountSASPermissions.parse("rwdlacup"), + protocol: SASProtocol.HttpsAndHttp, + resourceTypes: AccountSASResourceTypes.parse("sco").toString(), + services: AccountSASServices.parse("btqf").toString(), + startsOn: now, + version: "2016-05-31" + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ).toString(); + + const sasURL = `${serviceClient.url}?${sas}`; + const serviceClientWithSAS = new DataLakeServiceClient( + sasURL, + newPipeline(new AnonymousCredential(), { + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + await serviceClientWithSAS.getProperties(); + }); + + it("generateAccountSASQueryParameters should work for set blob tier @loki @sql", async () => { + const now = new Date(); + now.setMinutes(now.getMinutes() - 5); // Skip clock skew with server + + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const storageSharedKeyCredential = factories[factories.length - 1]; + + const sas = generateAccountSASQueryParameters( + { + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: AccountSASPermissions.parse("w"), + protocol: SASProtocol.HttpsAndHttp, + resourceTypes: AccountSASResourceTypes.parse("sco").toString(), + services: AccountSASServices.parse("btqf").toString(), + startsOn: now, + version: "2016-05-31" + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ).toString(); + + const sasURL = `${serviceClient.url}?${sas}`; + const serviceClientWithSAS = new DataLakeServiceClient( + sasURL, + newPipeline(new AnonymousCredential(), { + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const containerClientWithSAS = serviceClientWithSAS.getFileSystemClient( + getUniqueName("con") + ); + await containerClientWithSAS.create(); + + const blockBlobClientWithSAS = containerClientWithSAS.getFileClient( + getUniqueName("blob") + ); + await blockBlobClientWithSAS.upload(Buffer.from("abc")); + + //TODO: Revisit + // await blockBlobClientWithSAS.setAccessTier("Hot"); + }); + + it("generateAccountSASQueryParameters should not work with invalid permission @loki @sql", async () => { + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const storageSharedKeyCredential = factories[factories.length - 1]; + + const sas = generateAccountSASQueryParameters( + { + expiresOn: tmr, + permissions: AccountSASPermissions.parse("wdlcup"), + resourceTypes: AccountSASResourceTypes.parse("sco").toString(), + services: AccountSASServices.parse("btqf").toString() + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ).toString(); + + const sasURL = `${serviceClient.url}?${sas}`; + const serviceClientWithSAS = new DataLakeServiceClient( + sasURL, + newPipeline(new AnonymousCredential(), { + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + let error; + try { + await serviceClientWithSAS.getProperties(); + } catch (err) { + error = err; + } + + assert.ok(error); + }); + + it("generateAccountSASQueryParameters should not work with invalid service @loki @sql", async () => { + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const storageSharedKeyCredential = factories[factories.length - 1]; + + const sas = generateAccountSASQueryParameters( + { + expiresOn: tmr, + permissions: AccountSASPermissions.parse("rwdlacup"), + resourceTypes: AccountSASResourceTypes.parse("sco").toString(), + services: AccountSASServices.parse("tqf").toString() + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ).toString(); + + const sasURL = `${serviceClient.url}?${sas}`; + const serviceClientWithSAS = new DataLakeServiceClient( + sasURL, + newPipeline(new AnonymousCredential(), { + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + let error; + try { + await serviceClientWithSAS.getProperties(); + } catch (err) { + error = err; + } + + assert.ok(error); + }); + + it("generateAccountSASQueryParameters should not work with invalid resource type @loki @sql", async () => { + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const storageSharedKeyCredential = factories[factories.length - 1]; + + const sas = generateAccountSASQueryParameters( + { + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: AccountSASPermissions.parse("rwdlacup"), + protocol: SASProtocol.HttpsAndHttp, + resourceTypes: AccountSASResourceTypes.parse("co").toString(), + services: AccountSASServices.parse("btqf").toString(), + version: "2016-05-31" + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ).toString(); + + const sasURL = `${serviceClient.url}?${sas}`; + const serviceClientWithSAS = new DataLakeServiceClient( + sasURL, + newPipeline(new AnonymousCredential(), { + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + let error; + try { + await serviceClientWithSAS.getProperties(); + } catch (err) { + error = err; + } + + assert.ok(error); + }); + + it("Upload/Create/Append should work with write permission in account SAS to override an existing blob", async () => { + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const storageSharedKeyCredential = factories[factories.length - 1]; + + const sas = generateAccountSASQueryParameters( + { + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: AccountSASPermissions.parse("rwdlacup"), + protocol: SASProtocol.HttpsAndHttp, + resourceTypes: AccountSASResourceTypes.parse("co").toString(), + services: AccountSASServices.parse("btqf").toString(), + version: "2016-05-31" + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ).toString(); + + // const sasURL = `http://${productionStyleHostName}:${server.config.port}?${sas}`; + const sasURL = `${serviceClient.url}/?${sas}`; + const serviceClientWithSAS = new DataLakeServiceClient( + sasURL, + newPipeline(new AnonymousCredential(), { + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const fileSystemName = getUniqueName("con"); + const containerClient = + serviceClientWithSAS.getFileSystemClient(fileSystemName); + await containerClient.create(); + + const blobName1 = getUniqueName("blob"); + const blob1 = containerClient.getFileClient(blobName1); + + await blob1.upload(Buffer.from("hello")); + const buffer = await blob1.readToBuffer(); + assert.deepStrictEqual(buffer, Buffer.from("hello")); + }); + + it("Upload/Create/Append shouldn't work without write permission in account SAS to override an existing blob @loki @sql", async () => { + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const storageSharedKeyCredential = factories[factories.length - 1]; + + const sas = generateAccountSASQueryParameters( + { + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: AccountSASPermissions.parse("rdlacup"), + protocol: SASProtocol.HttpsAndHttp, + resourceTypes: AccountSASResourceTypes.parse("co").toString(), + services: AccountSASServices.parse("btqf").toString(), + version: "2016-05-31" + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ).toString(); + + const sasURL = `${serviceClient.url}?${sas}`; + const serviceClientWithSAS = new DataLakeServiceClient( + sasURL, + newPipeline(new AnonymousCredential(), { + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const fileSystemName = getUniqueName("con"); + const containerClient = + serviceClientWithSAS.getFileSystemClient(fileSystemName); + await containerClient.create(); + + const blobName1 = getUniqueName("blob"); + const blob1 = containerClient.getFileClient(blobName1); + + // this copy should throw 403 error + let error; + try { + await blob1.upload(Buffer.from("hello")); + } catch (err) { + error = err; + } + assert.deepEqual(error.statusCode, 403); + assert.ok(error !== undefined); + }); + + it("Create blob should work without write permission in account SAS to an nonexisting blob @loki @sql", async () => { + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const storageSharedKeyCredential = factories[factories.length - 1]; + + const sas = generateAccountSASQueryParameters( + { + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: AccountSASPermissions.parse("c"), + protocol: SASProtocol.HttpsAndHttp, + resourceTypes: AccountSASResourceTypes.parse("co").toString(), + services: AccountSASServices.parse("btqf").toString(), + version: "2016-05-31" + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ).toString(); + + const sasURL = `${serviceClient.url}?${sas}`; + const serviceClientWithSAS = new DataLakeServiceClient( + sasURL, + newPipeline(new AnonymousCredential(), { + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const fileSystemName = getUniqueName("con"); + const containerClient = + serviceClientWithSAS.getFileSystemClient(fileSystemName); + await containerClient.create(); + + const blobName1 = getUniqueName("blob"); + const blob1 = containerClient.getFileClient(blobName1); + await blob1.create(); + }); + + it("Upload/Create/Append blob should work with write permission in account SAS to override an existing blob @loki @sql", async () => { + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const storageSharedKeyCredential = factories[factories.length - 1]; + + const sas = generateAccountSASQueryParameters( + { + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: AccountSASPermissions.parse("rwdlacup"), + protocol: SASProtocol.HttpsAndHttp, + resourceTypes: AccountSASResourceTypes.parse("co").toString(), + services: AccountSASServices.parse("btqf").toString(), + version: "2016-05-31" + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ).toString(); + + const sasURL = `${serviceClient.url}?${sas}`; + const serviceClientWithSAS = new DataLakeServiceClient( + sasURL, + newPipeline(new AnonymousCredential(), { + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + + const fileSystemName = getUniqueName("con"); + const containerClient = + serviceClientWithSAS.getFileSystemClient(fileSystemName); + await containerClient.create(); + + const blobName1 = getUniqueName("blob"); + const blobName2 = getUniqueName("blob"); + const blob1 = containerClient.getFileClient(blobName1); + const blob2 = containerClient.getFileClient(blobName2); + + await blob1.upload(Buffer.from("hello")); + // this copy should not throw any errors + await blob2.create(); + await blob2.append("hello", 0, 5, { flush: true }); + }); + + it("generateDataLakeSASQueryParameters should work for container @loki @sql", async () => { + const now = new Date(); + now.setMinutes(now.getMinutes() - 5); // Skip clock skew with server + + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const storageSharedKeyCredential = factories[factories.length - 1]; + + const fileSystemName = getUniqueName("container"); + const containerClient = serviceClient.getFileSystemClient(fileSystemName); + await containerClient.create(); + + const containerSAS = generateDataLakeSASQueryParameters( + { + fileSystemName, + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: FileSystemSASPermissions.parse("racwdl"), + protocol: SASProtocol.HttpsAndHttp, + startsOn: now, + version: "2016-05-31" + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ); + + const sasURL = `${containerClient.url}?${containerSAS}`; + const containerClientWithSAS = new DataLakeFileSystemClient( + sasURL, + newPipeline(new AnonymousCredential()) + ); + + await containerClientWithSAS.listPaths().byPage().next(); + await containerClient.delete(); + }); + + it("generateDataLakeSASQueryParameters should work for append blob with original headers @loki @sql", async () => { + const now = new Date(); + now.setMinutes(now.getMinutes() - 5); // Skip clock skew with server + + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const storageSharedKeyCredential = factories[factories.length - 1]; + + const fileSystemName = getUniqueName("container"); + const containerClient = serviceClient.getFileSystemClient(fileSystemName); + await containerClient.create(); + + const blobName = getUniqueName("blob"); + const blobClient = containerClient.getFileClient(blobName); + await blobClient.create({ + pathHttpHeaders: { + cacheControl: "cache-control-original", + contentType: "content-type-original", + contentDisposition: "content-disposition-original", + contentEncoding: "content-encoding-original", + contentLanguage: "content-language-original" + } + }); + + const blobSAS = generateDataLakeSASQueryParameters( + { + pathName: blobName, + fileSystemName, + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: DataLakeSASPermissions.parse("racwd"), + protocol: SASProtocol.HttpsAndHttp, + startsOn: now, + version: "2016-05-31" + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ); + + const sasURL = `${blobClient.url}?${blobSAS}`; + const blobClientWithSAS = new DataLakeFileClient( + sasURL, + newPipeline(new AnonymousCredential()) + ); + + await blobClientWithSAS.getProperties(); + + const properties = await blobClientWithSAS.getProperties(); + assert.equal(properties.cacheControl, "cache-control-original"); + assert.equal(properties.contentDisposition, "content-disposition-original"); + assert.equal(properties.contentEncoding, "content-encoding-original"); + assert.equal(properties.contentLanguage, "content-language-original"); + assert.equal(properties.contentType, "content-type-original"); + + const downloadResponse = await blobClientWithSAS.read(); + assert.equal(downloadResponse.cacheControl, "cache-control-original"); + assert.equal( + downloadResponse.contentDisposition, + "content-disposition-original" + ); + assert.equal(downloadResponse.contentEncoding, "content-encoding-original"); + assert.equal(downloadResponse.contentLanguage, "content-language-original"); + assert.equal(downloadResponse.contentType, "content-type-original"); + + await containerClient.delete(); + }); + + it("generateDataLakeSASQueryParameters should work for append blob and override headers @loki @sql", async () => { + const now = new Date(); + now.setMinutes(now.getMinutes() - 5); // Skip clock skew with server + + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const storageSharedKeyCredential = factories[factories.length - 1]; + + const fileSystemName = getUniqueName("container"); + const containerClient = serviceClient.getFileSystemClient(fileSystemName); + await containerClient.create(); + + const blobName = getUniqueName("blob"); + const blobClient = containerClient.getFileClient(blobName); + await blobClient.create({ + pathHttpHeaders: { + contentType: "content-type-original" + } + }); + + const blobSAS = generateDataLakeSASQueryParameters( + { + pathName: blobName, + cacheControl: "cache-control-override", + fileSystemName, + contentDisposition: "content-disposition-override", + contentEncoding: "content-encoding-override", + contentLanguage: "content-language-override", + contentType: "content-type-override", + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: DataLakeSASPermissions.parse("racwd"), + protocol: SASProtocol.HttpsAndHttp, + startsOn: now, + version: "2016-05-31" + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ); + + const sasURL = `${blobClient.url}?${blobSAS}`; + const blobClientWithSAS = new DataLakeFileClient( + sasURL, + newPipeline(new AnonymousCredential()) + ); + + await blobClientWithSAS.getProperties(); + + const properties = await blobClientWithSAS.getProperties(); + assert.equal(properties.cacheControl, "cache-control-override"); + assert.equal(properties.contentDisposition, "content-disposition-override"); + assert.equal(properties.contentEncoding, "content-encoding-override"); + assert.equal(properties.contentLanguage, "content-language-override"); + assert.equal(properties.contentType, "content-type-override"); + + const downloadResponse = await blobClientWithSAS.read(); + assert.equal(downloadResponse.cacheControl, "cache-control-override"); + assert.equal( + downloadResponse.contentDisposition, + "content-disposition-override" + ); + assert.equal(downloadResponse.contentEncoding, "content-encoding-override"); + assert.equal(downloadResponse.contentLanguage, "content-language-override"); + assert.equal(downloadResponse.contentType, "content-type-override"); + + await containerClient.delete(); + }); + + it("generateDataLakeSASQueryParameters should work for blob with special naming @loki @sql", async () => { + const now = new Date(); + now.setMinutes(now.getMinutes() - 5); // Skip clock skew with server + + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const storageSharedKeyCredential = factories[factories.length - 1]; + + const fileSystemName = getUniqueName("container-with-dash"); + const containerClient = serviceClient.getFileSystemClient(fileSystemName); + await containerClient.create(); + + const blobName = getUniqueName( + // tslint:disable-next-line:max-line-length + "////Upper/blob/empty /another 汉字 ру́сский язы́к ру́сский язы́к عربي/عربى にっぽんご/にほんご . special ~!@#$%^&*()_+`1234567890-={}|[]\\:\";'<>?,/'" + ); + const blobClient = containerClient.getFileClient(blobName); + await blobClient.create({ + pathHttpHeaders: { + contentType: "content-type-original" + } + }); + + const blobSAS = generateDataLakeSASQueryParameters( + { + // NOTICE: Azure Storage Server will replace "\" with "/" in the blob names + pathName: blobName.replace(/\\/g, "/"), + cacheControl: "cache-control-override", + fileSystemName, + contentDisposition: "content-disposition-override", + contentEncoding: "content-encoding-override", + contentLanguage: "content-language-override", + contentType: "content-type-override", + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: DataLakeSASPermissions.parse("racwd"), + protocol: SASProtocol.HttpsAndHttp, + startsOn: now, + version: "2016-05-31" + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ); + + const sasURL = `${blobClient.url}?${blobSAS}`; + const blobClientWithSAS = new DataLakeFileClient( + sasURL, + newPipeline(new AnonymousCredential()) + ); + + await blobClientWithSAS.getProperties(); + + const properties = await blobClientWithSAS.getProperties(); + assert.equal(properties.cacheControl, "cache-control-override"); + assert.equal(properties.contentDisposition, "content-disposition-override"); + assert.equal(properties.contentEncoding, "content-encoding-override"); + assert.equal(properties.contentLanguage, "content-language-override"); + assert.equal(properties.contentType, "content-type-override"); + + const downloadResponse = await blobClientWithSAS.read(); + assert.equal(downloadResponse.cacheControl, "cache-control-override"); + assert.equal( + downloadResponse.contentDisposition, + "content-disposition-override" + ); + assert.equal(downloadResponse.contentEncoding, "content-encoding-override"); + assert.equal(downloadResponse.contentLanguage, "content-language-override"); + assert.equal(downloadResponse.contentType, "content-type-override"); + + await containerClient.delete(); + }); + + it("generateDataLakeSASQueryParameters should work for blob with access policy @loki @sql", async () => { + const now = new Date(); + now.setMinutes(now.getMinutes() - 5); // Skip clock skew with server + + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const storageSharedKeyCredential = factories[factories.length - 1]; + + const fileSystemName = getUniqueName("container"); + const containerClient = serviceClient.getFileSystemClient(fileSystemName); + await containerClient.create(); + + const blobName = getUniqueName("blob"); + const blobClient = containerClient.getFileClient(blobName); + await blobClient.create(); + + const id = "unique-id"; + const result = await containerClient.setAccessPolicy(undefined, [ + { + accessPolicy: { + expiresOn: tmr, + permissions: FileSystemSASPermissions.parse("racwdl").toString(), + startsOn: now + }, + id + } + ]); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + + const blobSAS = generateDataLakeSASQueryParameters( + { + fileSystemName, + identifier: id + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ); + + const sasURL = `${blobClient.url}?${blobSAS}`; + const blobClientWithSAS = new DataLakeFileClient( + sasURL, + newPipeline(new AnonymousCredential()) + ); + + await blobClientWithSAS.getProperties(); + await containerClient.delete(); + }); + + it("Synchronized copy blob should work with write permission in blob SAS to override an existing blob @loki @sql", async () => { + const now = new Date(); + now.setMinutes(now.getMinutes() - 5); // Skip clock skew with server + + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const storageSharedKeyCredential = factories[factories.length - 1]; + + const fileSystemName = getUniqueName("container"); + const containerClient = serviceClient.getFileSystemClient(fileSystemName); + await containerClient.create(); + + const containerSAS = generateDataLakeSASQueryParameters( + { + fileSystemName, + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: FileSystemSASPermissions.parse("w"), + protocol: SASProtocol.HttpsAndHttp, + startsOn: now, + version: "2016-05-31" + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ); + + const sasURL = `${containerClient.url}?${containerSAS}`; + const containerClientWithSAS = new DataLakeFileSystemClient( + sasURL, + newPipeline(new AnonymousCredential()) + ); + + const blobName1 = getUniqueName("blob"); + const blobName2 = getUniqueName("blob"); + const blob1 = containerClient.getFileClient(blobName1); + const blob2 = containerClient.getFileClient(blobName2); + const blob1SAS = containerClientWithSAS.getFileClient(blobName1); + + await blob1.create(); + await blob2.create(); + + await blob1SAS.create(); + await blob1SAS.append("hello", 0, 5, { flush: true }); + }); + + it("Synchronized copy blob shouldn't work without write permission in blob SAS to override an existing blob @loki @sql", async () => { + const now = new Date(); + now.setMinutes(now.getMinutes() - 5); // Skip clock skew with server + + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + // By default, credential is always the last element of pipeline factories + const factories = (serviceClient as any).pipeline.factories; + const storageSharedKeyCredential = factories[factories.length - 1]; + + const fileSystemName = getUniqueName("container"); + const containerClient = serviceClient.getFileSystemClient(fileSystemName); + await containerClient.create(); + + const containerSAS = generateDataLakeSASQueryParameters( + { + fileSystemName, + expiresOn: tmr, + ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, + permissions: FileSystemSASPermissions.parse("c"), + protocol: SASProtocol.HttpsAndHttp, + startsOn: now, + version: "2016-05-31" + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ); + + const sasURL = `${containerClient.url}?${containerSAS}`; + const containerClientWithSAS = new DataLakeFileSystemClient( + sasURL, + newPipeline(new AnonymousCredential()) + ); + + const blobName1 = getUniqueName("blob"); + const blobName2 = getUniqueName("blob"); + const blob1 = containerClient.getFileClient(blobName1); + const blob2 = containerClient.getFileClient(blobName2); + const blob1SAS = containerClientWithSAS.getFileClient(blobName1); + + await blob1.upload(Buffer.from("hello")); + await blob2.upload(Buffer.from("world")); + + // this copy should throw 403 error + let error; + try { + await blob1SAS.create(); + await blob1SAS.append("hello", 0, 5, { flush: true }); + } catch (err) { + error = err; + } + assert.deepEqual(error.statusCode, 403); + assert.ok(error !== undefined); + }); +}); diff --git a/tests/dfs/storage-file-datalake/specialnaming.test.ts b/tests/dfs/storage-file-datalake/specialnaming.test.ts new file mode 100644 index 000000000..57b88037b --- /dev/null +++ b/tests/dfs/storage-file-datalake/specialnaming.test.ts @@ -0,0 +1,299 @@ +import dns = require("dns"); + +import { + DataLakeFileClient, + DataLakeServiceClient, + FileSystemListPathsResponse, + newPipeline, + StorageSharedKeyCredential +} from "@azure/storage-file-datalake"; + +import { configLogger } from "../../../src/common/Logger"; +import { + appendToURLPath, + EMULATOR_ACCOUNT_KEY, + EMULATOR_ACCOUNT_NAME, + getUniqueName +} from "../../testutils"; +import DataLakeTestServerFactory from "../DataLakeTestServerFactory"; + +import assert = require("assert"); + +// Set true to enable debug log +configLogger(false); + +describe("SpecialNaming", () => { + const factory = new DataLakeTestServerFactory(); + const server = factory.createServer(); + + const baseURL = `http://${server.config.host}:${server.config.port}/devstoreaccount1`; + const productionStyleHostName = "devstoreaccount1.localhost"; // Use hosts file to make this resolve + const noAccountHostName = "host.docker.internal"; + const noAccountHostNameConnectionString = `DefaultEndpointsProtocol=http;AccountName=${EMULATOR_ACCOUNT_NAME};AccountKey=${EMULATOR_ACCOUNT_KEY};BlobEndpoint=http://${noAccountHostName}:${server.config.port}/${EMULATOR_ACCOUNT_NAME};`; + + const serviceClient = new DataLakeServiceClient( + baseURL, + newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ) + ); + + const content = "Hello World!"; + const filesystemName: string = getUniqueName("1container-with-dash"); + const fileSystemClient = serviceClient.getFileSystemClient(filesystemName); + const parentDirectoryClient = fileSystemClient.getDirectoryClient( + getUniqueName("dir1") + ); + const childDirectoryClient = parentDirectoryClient.getSubdirectoryClient( + getUniqueName("dir2") + ); + const directoryClient = childDirectoryClient.getSubdirectoryClient( + getUniqueName("dir3") + ); + + before(async () => { + await server.start(); + await fileSystemClient.create(); + }); + + after(async () => { + await fileSystemClient.delete(); + await server.close(); + await server.clean(); + }); + + afterEach(async () => { + await parentDirectoryClient.deleteIfExists(true); + }); + + async function testSpecialFile(name: string, url: boolean = false) { + const fileName: string = getUniqueName(name); + const fileClient = url + ? new DataLakeFileClient( + // There are 2 special cases for a URL string: + // Escape "%" when creating XXXURL object with URL strings + // Escape "?" otherwise string after "?" will be treated as URL parameters + appendToURLPath( + directoryClient.url, + fileName.replace(/%/g, "%25").replace(/\?/g, "%3F") + ), + (directoryClient as any).pipeline + ) + : directoryClient.getFileClient(fileName); + + await fileClient.create(); + assert.strictEqual(await parentDirectoryClient.exists(), true); + assert.strictEqual(await childDirectoryClient.exists(), true); + assert.strictEqual(await directoryClient.exists(), true); + assert.strictEqual(await fileClient.exists(), true); + await fileClient.append(content, 0, content.length, { flush: true }); + await fileClient.getProperties(); + const response: FileSystemListPathsResponse = ( + await fileSystemClient + .listPaths({ + path: `${directoryClient.name}/` + }) + .byPage() + .next() + ).value; + assert.strictEqual(response.pathItems!.length, 1); + assert.strictEqual( + response.pathItems![0].name, + `${directoryClient.name}/${fileName.replace(/\\/g, "/")}` + ); + + await parentDirectoryClient.delete(true); + assert.strictEqual(await parentDirectoryClient.exists(), false); + assert.strictEqual(await childDirectoryClient.exists(), false); + assert.strictEqual(await directoryClient.exists(), false); + assert.strictEqual(await fileClient.exists(), false); + } + + it("Should work with special container and blob names with unicode @loki @sql", async () => { + await testSpecialFile("unicod\u00e9"); + }); + + it("Should work with special container and blob names with unicode in URL string @loki @sql", async () => { + await testSpecialFile("unicod\u00e9", true); + }); + + it("Should work with special container and blob names with spaces @loki @sql", async () => { + await testSpecialFile("blob empty"); + }); + + it("Should work with special container and blob names with spaces in URL string @loki @sql", async () => { + await testSpecialFile("blob empty", true); + }); + + /******************************************************************************************************* + * "/" test cases are Invalid in DataLake since "/" denotes folder structure so can only work encoded + * ********************************************************************************************************/ + + // it("Should work with special container and blob names with / @loki @sql", async () => { + // await testSpecialFile("////blob/empty /another"); + // }); + + // it("Should work with special container and blob names with / in URL string @loki @sql", async () => { + // await testSpecialFile("////blob/empty /another", true); + // }); + + it("Should work with special container and blob names uppercase @loki @sql", async () => { + await testSpecialFile(" Upper blob empty another"); + }); + + it("Should work with special container and blob names uppercase in URL string @loki @sql", async () => { + await testSpecialFile(" Upper blob empty another", true); + }); + + it("Should work with special blob names Chinese characters @loki @sql", async () => { + await testSpecialFile(" Upper blob empty another 汉字"); + }); + + it("Should work with special blob names Chinese characters in URL string @loki @sql", async () => { + await testSpecialFile(" Upper blob empty another 汉字", true); + }); + + it("Should work with special blob name characters @loki @sql", async () => { + await testSpecialFile( + "汉字. special ~!@#$%^&*()_+`1234567890-={}|[]:\";'<>?,'" + ); + }); + + it("Should work with special blob name characters in URL string @loki @sql", async () => { + await testSpecialFile( + "汉字. special ~!@#$%^&*()_+`1234567890-={}|[]:\";'<>?,'", + true + ); + }); + + it("Should work with special blob name Russian URI encoded @loki @sql", async () => { + await testSpecialFile(encodeURIComponent("ру́сский язы́к")); + }); + + it("Should work with special blob name Russian @loki @sql", async () => { + await testSpecialFile("ру́сский язы́к"); + }); + + it("Should work with special blob name Russian in URL string @loki @sql", async () => { + await testSpecialFile("ру́сский язы́к", true); + }); + + it("Should work with special blob name Arabic URI encoded @loki @sql", async () => { + await testSpecialFile(encodeURIComponent("عربي/عربى")); + }); + + it("Should work with special blob name Arabic @loki @sql", async () => { + await testSpecialFile("عربي عربى"); + }); + + it("Should work with special blob name Arabic in URL string @loki @sql", async () => { + await testSpecialFile("عربي عربى", true); + }); + + it("Should work with special blob name Japanese URI encoded @loki @sql", async () => { + await testSpecialFile(encodeURIComponent("にっぽんご/にほんご")); + }); + + it("Should work with special blob name Japanese @loki @sql", async () => { + await testSpecialFile("にっぽんごにほんご"); + }); + + it("Should work with special blob name Japanese in URL string @loki @sql", async () => { + await testSpecialFile("にっぽんごにほんご", true); + }); + + it(`Should work with production style URL when ${productionStyleHostName} is resolvable`, async () => { + await dns.promises.lookup(productionStyleHostName).then( + async (lookupAddress) => { + const baseURLProductionStyle = `http://${productionStyleHostName}:${server.config.port}/`; + const serviceClientProductionStyle = new DataLakeServiceClient( + baseURLProductionStyle, + newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ) + ); + const directoryClientProductionStyle = + serviceClientProductionStyle.getFileSystemClient(filesystemName); + + const fileName: string = getUniqueName("myblob"); + const fileClient = + directoryClientProductionStyle.getFileClient(fileName); + + await fileClient.create(); + await fileClient.append(content, 0, content.length, { flush: true }); + const response: FileSystemListPathsResponse = ( + await directoryClientProductionStyle.listPaths().byPage().next() + ).value; + assert.notDeepStrictEqual(response.pathItems!.length, 0); + assert.deepStrictEqual(response.pathItems![0].name, fileName); + }, + () => { + // Cannot perform this test. We need devstoreaccount1.localhost to resolve to 127.0.0.1. + // On Linux, this should just work, + // On Windows, we can't spoof DNS record for specific process. + // So we have options of running our own DNS server (overkill), + // or editing hosts files (machine global operation; and requires running as admin). + // So skip the test case. + assert.ok( + `Skipping test case - it needs ${productionStyleHostName} to be resolvable` + ); + } + ); + }); + + it(`Should work with no account host name URL when ${noAccountHostName} is resolvable`, async () => { + await dns.promises.lookup(noAccountHostName).then( + async (lookupAddress) => { + const serviceClientNoHostName = + DataLakeServiceClient.fromConnectionString( + noAccountHostNameConnectionString, + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + const directoryClientProductionStyle = + serviceClientNoHostName.getFileSystemClient(filesystemName); + + const fileName: string = getUniqueName("myblob"); + const fileClient = + directoryClientProductionStyle.getFileClient(fileName); + + await fileClient.create(); + await fileClient.append(content, 0, content.length, { flush: true }); + const response: FileSystemListPathsResponse = ( + await directoryClientProductionStyle.listPaths().byPage().next() + ).value; + assert.notDeepStrictEqual(response.pathItems!.length, 0); + assert.deepStrictEqual(response.pathItems![0].name, fileName); + }, + () => { + // Cannot perform this test. We need host.docker.internal to resolve to 127.0.0.1. + // On Windows, we can't spoof DNS record for specific process. + // So we have options of running our own DNS server (overkill), + // or editing hosts files (machine global operation; and requires running as admin). + // So skip the test case. + assert.ok( + `Skipping test case - it needs ${noAccountHostName} to be resolvable` + ); + } + ); + }); +}); diff --git a/tests/dfs/utils.test.ts b/tests/dfs/utils.test.ts new file mode 100644 index 000000000..a60c5d38e --- /dev/null +++ b/tests/dfs/utils.test.ts @@ -0,0 +1,59 @@ +import assert = require("assert"); +import { convertRawHeadersToMetadata } from "../../src/common/utils/utils"; + +describe("Utils", () => { + it("convertRawHeadersToMetadata should work @loki @sql", () => { + // upper case, lower case keys/values + const metadata = convertRawHeadersToMetadata([ + "x-ms-meta-Name1", + "Value", + "x-ms-meta-name2", + "234", + "x-ms-meta-name1", + "Value", + "X-Ms-Meta-Name3", + "Value" + ]); + assert.deepStrictEqual(metadata, { + Name1: "Value", + name2: "234", + name1: "Value", + Name3: "Value" + }); + }); + + it("convertRawHeadersToMetadata should work with duplicated metadata @loki @sql", () => { + const metadata = convertRawHeadersToMetadata([ + "x-ms-meta-name1", + "Value", + "x-ms-meta-name1", + "234" + ]); + assert.deepStrictEqual(metadata, { + name1: "Value,234" + }); + }); + + it("convertRawHeadersToMetadata should work with empty metadata @loki @sql", () => { + const metadata = convertRawHeadersToMetadata([ + "x-ms-meta-Name1", + "", + "x-ms-meta-name1", + "234" + ]); + assert.deepStrictEqual(metadata, { + Name1: "", + name1: "234" + }); + }); + + it("convertRawHeadersToMetadata should work with empty raw headers @loki @sql", () => { + const metadata = convertRawHeadersToMetadata(); + assert.deepStrictEqual(metadata, undefined); + }); + + it("convertRawHeadersToMetadata should work with empty raw headers array @loki @sql", () => { + const metadata = convertRawHeadersToMetadata([]); + assert.deepStrictEqual(metadata, undefined); + }); +}); diff --git a/tests/exe.test.ts b/tests/exe.test.ts index e1e70387b..cd8d7ebcb 100644 --- a/tests/exe.test.ts +++ b/tests/exe.test.ts @@ -15,6 +15,11 @@ import { QueueServiceClient, StorageSharedKeyCredential as queueStorageSharedKeyCredential } from "@azure/storage-queue"; +import { + DataLakeServiceClient, + newPipeline as datalakeNewPipeline, + StorageSharedKeyCredential as datalakeStorageSharedKeyCredential +} from "@azure/storage-file-datalake"; import { configLogger } from "../src/common/Logger"; import { @@ -36,6 +41,7 @@ import { overrideRequest, restoreBuildRequestOptions } from "./testutils"; +import DataLakeTestServerFactory from "./dfs/DataLakeTestServerFactory"; // server address used for testing. Note that Azurite.exe has // server address of http://127.0.0.1:10000 and so on by default @@ -43,6 +49,7 @@ import { const blobAddress = "http://127.0.0.1:11000"; const queueAddress = "http://127.0.0.1:11001"; const tableAddress = "http://127.0.0.1:11002"; +const datalakeAddress = "http://127.0.0.1:11003"; // Set true to enable debug log configLogger(false); @@ -70,7 +77,12 @@ describe("exe test", () => { tableName = getUniqueName("table"); const child = execFile( ".\\release\\azurite.exe", - ["--blobPort 11000", "--queuePort 11001", "--tablePort 11002"], + [ + "--blobPort 11000", + "--queuePort 11001", + "--tablePort 11002", + "--datalakePort 11003" + ], { cwd: process.cwd(), shell: true, env: {} } ); @@ -89,6 +101,10 @@ describe("exe test", () => { tableAddress + "\nAzurite Table service is successfully listening at " + tableAddress + + "\nAzurite DataLake service is starting at " + + datalakeAddress + + "\nAzurite DataLake service is successfully listening at " + + datalakeAddress + "\n"; let messageReceived: string = ""; @@ -305,6 +321,82 @@ describe("exe test", () => { }); }); + describe("datalake test", () => { + const factory = new DataLakeTestServerFactory(); + const datalakeServer = factory.createServer(); + + const datalakeBaseURL = `http://${datalakeServer.config.host}:${datalakeServer.config.port}/devstoreaccount1`; + const datalakeServiceClient = new DataLakeServiceClient( + datalakeBaseURL, + datalakeNewPipeline( + new datalakeStorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ) + ); + + let filesystemName: string = getUniqueName("filesystem"); + let filesystemClient = + datalakeServiceClient.getFileSystemClient(filesystemName); + let fileName: string = getUniqueName("file"); + let fileClient = filesystemClient.getFileClient(fileName); + const content = "Hello World"; + + beforeEach(async () => { + filesystemName = getUniqueName("filesystem"); + filesystemClient = + datalakeServiceClient.getFileSystemClient(filesystemName); + await filesystemClient.create(); + fileName = getUniqueName("file"); + fileClient = filesystemClient.getFileClient(fileName); + await fileClient.create(); + await fileClient.append(content, 0, content.length, { flush: true }); + }); + + afterEach(async () => { + await filesystemClient.delete(); + }); + it("download with with default parameters @loki @sql", async () => { + const result = await fileClient.read(); + assert.deepStrictEqual( + await bodyToString(result, content.length), + content + ); + assert.equal(result.contentRange, undefined); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + }); + + it("download should work with conditional headers @loki @sql", async () => { + const properties = await fileClient.getProperties(); + const result = await fileClient.read(0, undefined, { + conditions: { + ifMatch: properties.etag, + ifNoneMatch: "invalidetag", + ifModifiedSince: new Date("2018/01/01"), + ifUnmodifiedSince: new Date("2188/01/01") + } + }); + assert.deepStrictEqual( + await bodyToString(result, content.length), + content + ); + assert.equal(result.contentRange, undefined); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + }); + }); + describe("queue test", () => { // TODO: Create a server factory as tests utils const host = "127.0.0.1"; diff --git a/tests/linuxbinary.test.ts b/tests/linuxbinary.test.ts index 4a888f27a..e82418546 100644 --- a/tests/linuxbinary.test.ts +++ b/tests/linuxbinary.test.ts @@ -1,35 +1,55 @@ // run "EXE Mocha TS File - Loki" in VS Code to run this test -import * as assert from 'assert'; -import * as Azure from 'azure-storage'; -import { execFile } from 'child_process'; -import find from 'find-process'; +import * as assert from "assert"; +import * as Azure from "azure-storage"; +import { execFile } from "child_process"; +import find from "find-process"; import { - BlobServiceClient, newPipeline as blobNewPipeline, + BlobServiceClient, + newPipeline as blobNewPipeline, StorageSharedKeyCredential as blobStorageSharedKeyCredential -} from '@azure/storage-blob'; +} from "@azure/storage-blob"; import { - newPipeline as queueNewPipeline, QueueClient, QueueServiceClient, + DataLakeServiceClient, + newPipeline as datalakeNewPipeline, + StorageSharedKeyCredential as datalakeStorageSharedKeyCredential +} from "@azure/storage-file-datalake"; +import { + newPipeline as queueNewPipeline, + QueueClient, + QueueServiceClient, StorageSharedKeyCredential as queueStorageSharedKeyCredential -} from '@azure/storage-queue'; +} from "@azure/storage-queue"; -import { configLogger } from '../src/common/Logger'; -import { HeaderConstants, TABLE_API_VERSION } from '../src/table/utils/constants'; -import BlobTestServerFactory from './BlobTestServerFactory'; +import { configLogger } from "../src/common/Logger"; +import { + HeaderConstants, + TABLE_API_VERSION +} from "../src/table/utils/constants"; +import BlobTestServerFactory from "./BlobTestServerFactory"; import { - createConnectionStringForTest, HOST, PORT, PROTOCOL -} from './table/utils/table.entity.test.utils'; + createConnectionStringForTest, + HOST, + PORT, + PROTOCOL +} from "./table/utils/table.entity.test.utils"; import { - bodyToString, EMULATOR_ACCOUNT_KEY, EMULATOR_ACCOUNT_NAME, getUniqueName, overrideRequest, + bodyToString, + EMULATOR_ACCOUNT_KEY, + EMULATOR_ACCOUNT_NAME, + getUniqueName, + overrideRequest, restoreBuildRequestOptions -} from './testutils'; +} from "./testutils"; +import DataLakeTestServerFactory from "./dfs/DataLakeTestServerFactory"; -// server address used for testing. Note that Azuritelinux has -// server address of http://127.0.0.1:10000 and so on by default +// server address used for testing. Note that Azuritelinux has +// server address of http://127.0.0.1:10000 and so on by default // and we need to configure them when starting azuritelinux const blobAddress = "http://127.0.0.1:11000"; const queueAddress = "http://127.0.0.1:11001"; const tableAddress = "http://127.0.0.1:11002"; +const datalakeAddress = "http://127.0.0.1:11003"; // Set true to enable debug log configLogger(false); @@ -41,7 +61,6 @@ configLogger(false); const testLocalAzuriteInstance = true; describe("linux binary test", () => { - const tableService = Azure.createTableService( createConnectionStringForTest(testLocalAzuriteInstance) ); @@ -56,19 +75,43 @@ describe("linux binary test", () => { before(async () => { overrideRequest(requestOverride, tableService); tableName = getUniqueName("table"); - const child = execFile("./release/azuritelinux", ["--blobPort 11000", "--queuePort 11001", "--tablePort 11002"], { cwd: process.cwd(), shell: true, env: {} }); + const child = execFile( + "./release/azuritelinux", + [ + "--blobPort 11000", + "--queuePort 11001", + "--tablePort 11002", + "--datalakePort 11003" + ], + { cwd: process.cwd(), shell: true, env: {} } + ); childPid = child.pid; - const fullSuccessMessage = "Azurite Blob service is starting at " + blobAddress + "\nAzurite Blob service is successfully listening at " + blobAddress + - "\nAzurite Queue service is starting at " + queueAddress + "\nAzurite Queue service is successfully listening at " + queueAddress + - "\nAzurite Table service is starting at " + tableAddress + "\nAzurite Table service is successfully listening at " + tableAddress + "\n"; + const fullSuccessMessage = + "Azurite Blob service is starting at " + + blobAddress + + "\nAzurite Blob service is successfully listening at " + + blobAddress + + "\nAzurite Queue service is starting at " + + queueAddress + + "\nAzurite Queue service is successfully listening at " + + queueAddress + + "\nAzurite Table service is starting at " + + tableAddress + + "\nAzurite Table service is successfully listening at " + + tableAddress + + "\nAzurite DataLake service is starting at " + + datalakeAddress + + "\nAzurite DataLake service is successfully listening at " + + datalakeAddress + + "\n"; let messageReceived: string = ""; function stdoutOn() { - return new Promise(resolve => { + return new Promise((resolve) => { // exclamation mark suppresses the TS error that "child.stdout is possibly null" - child.stdout!.on('data', function (data: any) { + child.stdout!.on("data", function (data: any) { messageReceived += data.toString(); if (messageReceived == fullSuccessMessage) { resolve("resolveMessage"); @@ -83,12 +126,12 @@ describe("linux binary test", () => { after(async () => { // TO DO // Currently, the mocha test does not quit unless "--exit" is added to the mocha command - // The current fix is to have "--exit" added but the issue causing mocha to be unable to + // The current fix is to have "--exit" added but the issue causing mocha to be unable to // quit has not been identified restoreBuildRequestOptions(tableService); tableService.removeAllListeners(); - await find('name', 'azuritelinux', true).then((list: any) => { + await find("name", "azuritelinux", true).then((list: any) => { process.kill(list[0].pid); }); @@ -131,20 +174,23 @@ describe("linux binary test", () => { accept: "application/json;odata=minimalmetadata" }; - tableService.listTablesSegmented(null as any, (error, result, response) => { - if (!error) { - assert.strictEqual(response.statusCode, 200); - const headers = response.headers!; - assert.strictEqual(headers["x-ms-version"], TABLE_API_VERSION); - const bodies = response.body! as any; - assert.deepStrictEqual( - bodies["odata.metadata"], - `${PROTOCOL}://${HOST}:${PORT}/${EMULATOR_ACCOUNT_NAME}/$metadata#Tables` - ); - assert.ok(bodies.value[0].TableName); + tableService.listTablesSegmented( + null as any, + (error, result, response) => { + if (!error) { + assert.strictEqual(response.statusCode, 200); + const headers = response.headers!; + assert.strictEqual(headers["x-ms-version"], TABLE_API_VERSION); + const bodies = response.body! as any; + assert.deepStrictEqual( + bodies["odata.metadata"], + `${PROTOCOL}://${HOST}:${PORT}/${EMULATOR_ACCOUNT_NAME}/$metadata#Tables` + ); + assert.ok(bodies.value[0].TableName); + } + done(); } - done(); - }); + ); }); it("deleteTable that exists, @loki", (done) => { @@ -157,15 +203,18 @@ describe("linux binary test", () => { tableService.createTable(tableToDelete, (error, result, response) => { if (!error) { - tableService.deleteTable(tableToDelete, (deleteError, deleteResult) => { - if (!deleteError) { - // no body expected, we expect 204 no content on successful deletion - assert.strictEqual(deleteResult.statusCode, 204); - } else { - assert.ifError(deleteError); + tableService.deleteTable( + tableToDelete, + (deleteError, deleteResult) => { + if (!deleteError) { + // no body expected, we expect 204 no content on successful deletion + assert.strictEqual(deleteResult.statusCode, 204); + } else { + assert.ifError(deleteError); + } + done(); } - done(); - }); + ); } else { assert.fail("Test failed to create the table"); done(); @@ -239,7 +288,10 @@ describe("linux binary test", () => { }); it("download with with default parameters @loki @sql", async () => { const result = await blobClient.download(0); - assert.deepStrictEqual(await bodyToString(result, content.length), content); + assert.deepStrictEqual( + await bodyToString(result, content.length), + content + ); assert.equal(result.contentRange, undefined); assert.equal( result._response.request.headers.get("x-ms-client-request-id"), @@ -257,7 +309,86 @@ describe("linux binary test", () => { ifUnmodifiedSince: new Date("2188/01/01") } }); - assert.deepStrictEqual(await bodyToString(result, content.length), content); + assert.deepStrictEqual( + await bodyToString(result, content.length), + content + ); + assert.equal(result.contentRange, undefined); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + }); + }); + + describe("datalake test", () => { + const factory = new DataLakeTestServerFactory(); + const datalakeServer = factory.createServer(); + + const datalakeBaseURL = `http://${datalakeServer.config.host}:${datalakeServer.config.port}/devstoreaccount1`; + const datalakeServiceClient = new DataLakeServiceClient( + datalakeBaseURL, + datalakeNewPipeline( + new datalakeStorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ) + ); + + let filesystemName: string = getUniqueName("filesystem"); + let filesystemClient = + datalakeServiceClient.getFileSystemClient(filesystemName); + let fileName: string = getUniqueName("file"); + let fileClient = filesystemClient.getFileClient(fileName); + const content = "Hello World"; + + beforeEach(async () => { + filesystemName = getUniqueName("filesystem"); + filesystemClient = + datalakeServiceClient.getFileSystemClient(filesystemName); + await filesystemClient.create(); + fileName = getUniqueName("file"); + fileClient = filesystemClient.getFileClient(fileName); + await fileClient.create(); + await fileClient.append(content, 0, content.length, { flush: true }); + }); + + afterEach(async () => { + await filesystemClient.delete(); + }); + it("download with with default parameters @loki @sql", async () => { + const result = await fileClient.read(); + assert.deepStrictEqual( + await bodyToString(result, content.length), + content + ); + assert.equal(result.contentRange, undefined); + assert.equal( + result._response.request.headers.get("x-ms-client-request-id"), + result.clientRequestId + ); + }); + + it("download should work with conditional headers @loki @sql", async () => { + const properties = await fileClient.getProperties(); + const result = await fileClient.read(0, undefined, { + conditions: { + ifMatch: properties.etag, + ifNoneMatch: "invalidetag", + ifModifiedSince: new Date("2018/01/01"), + ifUnmodifiedSince: new Date("2188/01/01") + } + }); + assert.deepStrictEqual( + await bodyToString(result, content.length), + content + ); assert.equal(result.contentRange, undefined); assert.equal( result._response.request.headers.get("x-ms-client-request-id"), @@ -361,4 +492,3 @@ describe("linux binary test", () => { }); }); }); - diff --git a/tests/testutils.ts b/tests/testutils.ts index 560d7b39a..eb96875e5 100644 --- a/tests/testutils.ts +++ b/tests/testutils.ts @@ -1,6 +1,13 @@ +import { isTokenCredential } from "@azure/core-auth"; +import { + CpkInfo, + DataLakeFileClient, + DataLakeFileSystemClient +} from "@azure/storage-file-datalake"; +import assert from "assert"; import { StorageServiceClient } from "azure-storage"; import { randomBytes } from "crypto"; -import { createWriteStream, readFileSync } from "fs"; +import * as fs from "fs"; import { sign } from "jsonwebtoken"; import { join } from "path"; import rimraf from "rimraf"; @@ -75,7 +82,7 @@ export async function bodyToString( blobBody?: Promise; contentLength?: number; }, - length?: number + length: number = response.contentLength! ): Promise { if (response.contentLength === 0) { return ""; @@ -84,9 +91,12 @@ export async function bodyToString( return new Promise((resolve, reject) => { response.readableStreamBody!.on("readable", () => { let chunk; - chunk = response.readableStreamBody!.read(length); - if (chunk) { - resolve(chunk.toString()); + let result: string | undefined; + while ((chunk = response.readableStreamBody!.read(length))) { + result = result ? result + chunk.toString() : chunk.toString(); + } + if (result) { + resolve(result.toString()); } }); @@ -134,7 +144,7 @@ export async function createRandomLocalFile( ): Promise { return new Promise((resolve, reject) => { const destFile = join(folder, getUniqueName("tempfile.")); - const ws = createWriteStream(destFile); + const ws = fs.createWriteStream(destFile); let offsetInMB = 0; function randomValueHex(len = blockSize) { @@ -145,7 +155,9 @@ export async function createRandomLocalFile( ws.on("open", () => { // tslint:disable-next-line:no-empty - while (offsetInMB++ < blockNumber && ws.write(randomValueHex())) {} + while (offsetInMB++ < blockNumber && ws.write(randomValueHex())) { + // + } if (offsetInMB >= blockNumber) { ws.end(); } @@ -153,7 +165,9 @@ export async function createRandomLocalFile( ws.on("drain", () => { // tslint:disable-next-line:no-empty - while (offsetInMB++ < blockNumber && ws.write(randomValueHex())) {} + while (offsetInMB++ < blockNumber && ws.write(randomValueHex())) { + // + } if (offsetInMB >= blockNumber) { ws.end(); } @@ -168,7 +182,7 @@ export async function readStreamToLocalFile( file: string ) { return new Promise((resolve, reject) => { - const ws = createWriteStream(file); + const ws = fs.createWriteStream(file); rs.pipe(ws); rs.on("error", reject); ws.on("error", reject); @@ -184,9 +198,9 @@ export function generateJWTToken( aud: string, scp: string, oid: string, - tid: string, + tid: string ) { - const privateKey = readFileSync("./tests/server.key"); + const privateKey = fs.readFileSync("./tests/server.key"); const token = sign( { nbf: Math.floor(nbf.getTime() / 1000), @@ -207,7 +221,9 @@ export function generateJWTToken( export function restoreBuildRequestOptions(service: any) { if ((service as any).__proto__.__proto__.__original_buildRequestOptions) { // tslint:disable-next-line: max-line-length - (service as any).__proto__.__proto__._buildRequestOptions = (service as any).__proto__.__proto__.__original_buildRequestOptions; + (service as any).__proto__.__proto__._buildRequestOptions = ( + service as any + ).__proto__.__proto__.__original_buildRequestOptions; } } export function overrideRequest( @@ -224,7 +240,8 @@ export function overrideRequest( : (service as any).__proto__.__proto__._buildRequestOptions; if (!hasOriginal) { - (service as any).__proto__.__proto__.__original_buildRequestOptions = original; + (service as any).__proto__.__proto__.__original_buildRequestOptions = + original; } const _buildRequestOptions = original.bind(service); @@ -251,3 +268,161 @@ export function overrideRequest( ); }; } + +export function getEncryptionScope(): string { + // return "ENCRYPTION_SCOPE"; + throw new Error("encryption scope not implemented yet."); +} + +export function getYieldedValue( + iteratorResult: IteratorResult +): YT { + if (iteratorResult.done) { + assert.fail(`Expected an item but did not get any`); + } + return iteratorResult.value; +} + +export const Test_CPK_INFO: CpkInfo = { + encryptionKey: "MDEyMzQ1NjcwMTIzNDU2NzAxMjM0NTY3MDEyMzQ1Njc=", // [SuppressMessage("Microsoft.Security", "CS001:SecretInline", Justification="This is a fake secret")] + encryptionKeySha256: "3QFFFpRA5+XANHqwwbT4yXDmrT/2JaLt/FKHjzhOdoE=" // [SuppressMessage("Microsoft.Security", "CS001:SecretInline", Justification="This is a fake secret")] +}; + +export function assertClientUsesTokenCredential( + client: DataLakeFileSystemClient +): void { + assert.ok(isTokenCredential(client.credential)); +} + +/** + * ONLY AVAILABLE IN NODE.JS RUNTIME. + * + * Writes the content of a readstream to a local file. Returns a Promise which is completed after the file handle is closed. + * If Promise is rejected, the reason will be set to the first error raised by either the + * ReadableStream or the fs.WriteStream. + * + * @param rs - The read stream. + * @param file - Destination file path. + */ +export async function readStreamToLocalFileWithLogs( + rs: NodeJS.ReadableStream, + file: string +): Promise { + return new Promise((resolve, reject) => { + const ws = fs.createWriteStream(file); + + // Set STREAM_DEBUG env var to log stream events while running tests + // if (process.env.STREAM_DEBUG) { + rs.on("close", () => console.log("rs.close")); + rs.on("data", () => console.log("rs.data")); + rs.on("end", () => console.log("rs.end")); + rs.on("error", () => console.log("rs.error")); + + ws.on("close", () => console.log("ws.close")); + ws.on("drain", () => console.log("ws.drain")); + ws.on("error", () => console.log("ws.error")); + ws.on("finish", () => console.log("ws.finish")); + ws.on("pipe", () => console.log("ws.pipe")); + ws.on("unpipe", () => console.log("ws.unpipe")); + // } + + let error: Error; + + rs.on("error", (err: Error) => { + // First error wins + if (!error) { + error = err; + } + + // When rs.error is raised, rs.end will never be raised automatically, so it must be raised manually + // to ensure ws.close is eventually raised. + rs.emit("end"); + }); + + ws.on("error", (err: Error) => { + // First error wins + if (!error) { + error = err; + } + }); + + ws.on("close", () => { + if (error) { + reject(error); + } else { + resolve(); + } + }); + + rs.pipe(ws); + }); +} + +/** + * Reads a readable stream into buffer entirely. + * + * @param stream - A Node.js Readable stream + * @param buffer - Buffer to be filled, length must greater than or equal to offset + * @param encoding - Encoding of the Readable stream + * @returns with the count of bytes read. + * @throws `RangeError` If buffer size is not big enough. + */ +export async function streamToBuffer2( + stream: NodeJS.ReadableStream, + buffer: Buffer, + encoding?: BufferEncoding +): Promise { + let pos = 0; // Position in stream + const bufferSize = buffer.length; + + return new Promise((resolve, reject) => { + stream.on("readable", () => { + let chunk = stream.read(); + if (!chunk) { + return; + } + if (typeof chunk === "string") { + chunk = Buffer.from(chunk, encoding); + } + + if (pos + chunk.length > bufferSize) { + reject( + new Error(`Stream exceeds buffer size. Buffer size: ${bufferSize}`) + ); + return; + } + + buffer.fill(chunk, pos, pos + chunk.length); + pos += chunk.length; + console.log(pos); + if (pos > bufferSize - 65 * 1024) { + console.log(pos); + } + }); + + stream.on("end", () => { + resolve(pos); + }); + + stream.on("close", () => { + resolve(pos); + }); + + stream.on("error", reject); + }); +} + +export async function upload( + fileClient: DataLakeFileClient, + data: string, + metadata: any = undefined +) { + let uploadResult = await (metadata + ? fileClient.create(metadata) + : fileClient.create()); + assert.ok(uploadResult.requestId); + uploadResult = await fileClient.append(data, 0, data.length); + assert.ok(uploadResult.requestId); + uploadResult = await fileClient.flush(data.length); + assert.ok(uploadResult.requestId); +}