diff --git a/ChangeLog.md b/ChangeLog.md index 41a9fb7da..1ffab02ff 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -28,6 +28,7 @@ General: Blob: - Fixed blob operations hanging when a client disconnects before the operation queue processes the request. (issue #2575) +- Implement `PutBlobFromUrl` (`Put Blob From URL`), which previously returned 501. The source is fetched over loopback, as `PutBlockFromURL` already does, so that SAS authentication and the `x-ms-source-if-*` conditions are enforced by the existing download path. Standard blob properties are copied from the source unless `x-ms-copy-source-blob-properties` is false, request blob content headers override them either way, request metadata replaces the source's rather than adding to it, and `x-ms-copy-source-tag-option: COPY` reads the source's tags over that same authorized path. An `x-ms-source-content-md5`, `x-ms-blob-content-md5`, `Content-MD5`, or `x-ms-content-crc64` header is checked against the copied content, and the response reports the MD5 and CRC64 of that content. A SAS needs Create or Write to create the blob, Write to overwrite it, and Tag as well when the request sets tags with `x-ms-tags` or copies the source's. As with `CopyBlobFromURL`, only sources on the same Azurite instance are supported. Table: diff --git a/README.md b/README.md index 5930392f5..35366126a 100644 --- a/README.md +++ b/README.md @@ -1086,6 +1086,7 @@ Detailed support matrix: - Abort Copy Blob (Only supports copy within same Azurite instance) - Copy Blob From URL (Only supports copy within same Azurite instance, only on Loki) - Put Block From URL (Only supports source within same Azurite instance) + - Put Blob From URL (Only supports source within same Azurite instance) - Access control based on conditional headers - Following features or REST APIs are NOT supported or limited supported in this release (will support more features per customers feedback in future releases) - SharedKey Lite @@ -1099,7 +1100,6 @@ Detailed support matrix: - Concurrent Append - Blob Expiry - Object Replication Service - - Put Blob From URL - Version Level Worm - Sync copy blob by access source with oauth - Encryption Scope diff --git a/src/blob/authentication/AccountSASAuthenticator.ts b/src/blob/authentication/AccountSASAuthenticator.ts index 48b850b07..39b4b5ab1 100644 --- a/src/blob/authentication/AccountSASAuthenticator.ts +++ b/src/blob/authentication/AccountSASAuthenticator.ts @@ -1,7 +1,7 @@ import IAccountDataStore from "../../common/IAccountDataStore"; import ILogger from "../../common/ILogger"; import StorageErrorFactory from "../errors/StorageErrorFactory"; -import { BlobType } from "../generated/artifacts/models"; +import { BlobCopySourceTags, BlobType } from "../generated/artifacts/models"; import Operation from "../generated/artifacts/operation"; import Context from "../generated/Context"; import IRequest from "../generated/IRequest"; @@ -14,7 +14,10 @@ import { import IAuthenticator from "./IAuthenticator"; import OPERATION_ACCOUNT_SAS_PERMISSIONS from "./OperationAccountSASPermission"; import StrictModelNotSupportedError from "../errors/StrictModelNotSupportedError"; -import { AUTHENTICATION_BEARERTOKEN_REQUIRED } from "../utils/constants"; +import { + AUTHENTICATION_BEARERTOKEN_REQUIRED, + HeaderConstants +} from "../utils/constants"; export default class AccountSASAuthenticator implements IAuthenticator { public constructor( @@ -256,6 +259,7 @@ export default class AccountSASAuthenticator implements IAuthenticator { // If copy destination blob exists, then permission must be Write only if ( operation === Operation.BlockBlob_Upload || + operation === Operation.BlockBlob_PutBlobFromUrl || operation === Operation.PageBlob_Create || operation === Operation.AppendBlob_Create || operation === Operation.Blob_StartCopyFromURL || @@ -280,6 +284,26 @@ export default class AccountSASAuthenticator implements IAuthenticator { } } + // Put Blob From URL sets tags on the destination when the request names + // them in x-ms-tags or asks for the source's to be copied, and Azure + // holds either to the Set Blob Tags permission on top of the write. A + // request that sets no tags takes only Create or Write. + if ( + operation === Operation.BlockBlob_PutBlobFromUrl && + (req.getHeader(HeaderConstants.X_MS_TAGS) !== undefined || + req.getHeader(HeaderConstants.X_MS_COPY_SOURCE_TAG_OPTION) === + BlobCopySourceTags.COPY) && + !values.permissions.toString().includes(AccountSASPermission.Tag) + ) { + this.logger.info( + `AccountSASAuthenticator:validate() For ${Operation[operation]}, setting tags on the destination requires the Tag permission.`, + context.contextId + ); + throw StorageErrorFactory.getAuthorizationPermissionMismatch( + context.contextId! + ); + } + this.logger.info( `AccountSASAuthenticator:validate() Account SAS validation successfully.`, context.contextId diff --git a/src/blob/authentication/BlobSASAuthenticator.ts b/src/blob/authentication/BlobSASAuthenticator.ts index ac848b0f8..db58186e5 100644 --- a/src/blob/authentication/BlobSASAuthenticator.ts +++ b/src/blob/authentication/BlobSASAuthenticator.ts @@ -3,12 +3,19 @@ import ILogger from "../../common/ILogger"; import BlobStorageContext from "../context/BlobStorageContext"; import StorageErrorFactory from "../errors/StorageErrorFactory"; import StrictModelNotSupportedError from "../errors/StrictModelNotSupportedError"; -import { AccessPolicy, BlobType } from "../generated/artifacts/models"; +import { + AccessPolicy, + BlobCopySourceTags, + BlobType +} from "../generated/artifacts/models"; import Operation from "../generated/artifacts/operation"; import Context from "../generated/Context"; import IRequest from "../generated/IRequest"; import IBlobMetadataStore from "../persistence/IBlobMetadataStore"; -import { AUTHENTICATION_BEARERTOKEN_REQUIRED } from "../utils/constants"; +import { + AUTHENTICATION_BEARERTOKEN_REQUIRED, + HeaderConstants +} from "../utils/constants"; import { getUserDelegationKeyValue } from "../utils/utils"; import { BlobSASPermission } from "./BlobSASPermissions"; import { BlobSASResourceType } from "./BlobSASResourceType"; @@ -418,6 +425,7 @@ export default class BlobSASAuthenticator implements IAuthenticator { // If copy destination blob exists, then permission must be Write only if ( operation === Operation.BlockBlob_Upload || + operation === Operation.BlockBlob_PutBlobFromUrl || operation === Operation.PageBlob_Create || operation === Operation.AppendBlob_Create || operation === Operation.Blob_StartCopyFromURL || @@ -442,6 +450,26 @@ export default class BlobSASAuthenticator implements IAuthenticator { } } + // Put Blob From URL sets tags on the destination when the request names + // them in x-ms-tags or asks for the source's to be copied, and Azure + // holds either to the Set Blob Tags permission on top of the write. A + // request that sets no tags takes only Create or Write. + if ( + operation === Operation.BlockBlob_PutBlobFromUrl && + (req.getHeader(HeaderConstants.X_MS_TAGS) !== undefined || + req.getHeader(HeaderConstants.X_MS_COPY_SOURCE_TAG_OPTION) === + BlobCopySourceTags.COPY) && + !values.permissions!.toString().includes(BlobSASPermission.Tag) + ) { + this.logger.info( + `BlobSASAuthenticator:validate() For ${Operation[operation]}, setting tags on the destination requires the Tag permission.`, + context.contextId + ); + throw StorageErrorFactory.getAuthorizationPermissionMismatch( + context.contextId! + ); + } + this.logger.info( `BlobSASAuthenticator:validate() Blob service SAS validation successfully.`, context.contextId diff --git a/src/blob/authentication/OperationAccountSASPermission.ts b/src/blob/authentication/OperationAccountSASPermission.ts index 56cd123f8..32ff60e47 100644 --- a/src/blob/authentication/OperationAccountSASPermission.ts +++ b/src/blob/authentication/OperationAccountSASPermission.ts @@ -363,6 +363,17 @@ OPERATION_ACCOUNT_SAS_PERMISSIONS.set( ) ); +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.BlockBlob_PutBlobFromUrl, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Object, + // Create or Write creates the blob. Overwriting an existing one takes + // Write alone, which the authenticator checks separately. + AccountSASPermission.Write + AccountSASPermission.Create + ) +); + OPERATION_ACCOUNT_SAS_PERMISSIONS.set( Operation.PageBlob_Create, new OperationAccountSASPermission( diff --git a/src/blob/authentication/OperationBlobSASPermission.ts b/src/blob/authentication/OperationBlobSASPermission.ts index ddf21c746..3a27e5e19 100644 --- a/src/blob/authentication/OperationBlobSASPermission.ts +++ b/src/blob/authentication/OperationBlobSASPermission.ts @@ -258,6 +258,14 @@ OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( BlobSASPermission.Write + BlobSASPermission.Create ) ); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.BlockBlob_PutBlobFromUrl, + // Create or Write creates the blob. Overwriting an existing one takes + // Write alone, which the authenticator checks separately. + new OperationBlobSASPermission( + BlobSASPermission.Write + BlobSASPermission.Create + ) +); OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( Operation.BlockBlob_StageBlock, new OperationBlobSASPermission(BlobSASPermission.Write) @@ -522,6 +530,14 @@ OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( BlobSASPermission.Write + BlobSASPermission.Create ) ); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.BlockBlob_PutBlobFromUrl, + // Create or Write creates the blob. Overwriting an existing one takes + // Write alone, which the authenticator checks separately. + new OperationBlobSASPermission( + BlobSASPermission.Write + BlobSASPermission.Create + ) +); OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( Operation.BlockBlob_StageBlock, new OperationBlobSASPermission(BlobSASPermission.Write) diff --git a/src/blob/generated/artifacts/mappers.ts b/src/blob/generated/artifacts/mappers.ts index 6f41aeffe..bbed870fc 100644 --- a/src/blob/generated/artifacts/mappers.ts +++ b/src/blob/generated/artifacts/mappers.ts @@ -4839,6 +4839,12 @@ export const BlockBlobPutBlobFromUrlHeaders: msRest.CompositeMapper = { name: "ByteArray" } }, + xMsContentCrc64: { + serializedName: "x-ms-content-crc64", + type: { + name: "ByteArray" + } + }, clientRequestId: { serializedName: "x-ms-client-request-id", type: { diff --git a/src/blob/generated/artifacts/models.ts b/src/blob/generated/artifacts/models.ts index e7f65d7a3..8a9292e21 100644 --- a/src/blob/generated/artifacts/models.ts +++ b/src/blob/generated/artifacts/models.ts @@ -3077,6 +3077,10 @@ export interface BlockBlobPutBlobFromUrlOptionalParams { * Specify the transactional md5 for the body, to be validated by the service. */ transactionalContentMD5?: Uint8Array; + /** + * Specify the transactional crc64 for the body, to be validated by the service. + */ + transactionalContentCrc64?: Uint8Array; /** * Optional. Specifies a user-defined name-value pair associated with the blob. If no name-value * pairs are specified, the operation will copy the metadata from the source blob or file to the @@ -5090,6 +5094,12 @@ export interface BlockBlobPutBlobFromUrlHeaders { * is returned so that the client can check for message content integrity. */ contentMD5?: Uint8Array; + /** + * This header is returned so that the client can check for message content integrity. The value + * of this header is computed by the Blob service; it is not necessarily the same value specified + * in the request headers. + */ + xMsContentCrc64?: Uint8Array; /** * If a client request id header is sent in the request, this header will be present in the * response with the same value. diff --git a/src/blob/generated/artifacts/specifications.ts b/src/blob/generated/artifacts/specifications.ts index c0b1f17e2..eeebcb4c3 100644 --- a/src/blob/generated/artifacts/specifications.ts +++ b/src/blob/generated/artifacts/specifications.ts @@ -2518,6 +2518,7 @@ const blockBlobPutBlobFromUrlOperationSpec: msRest.OperationSpec = { ], headerParameters: [ Parameters.transactionalContentMD5, + Parameters.transactionalContentCrc64, Parameters.contentLength, Parameters.metadata, Parameters.tier0, diff --git a/src/blob/handlers/BlockBlobHandler.ts b/src/blob/handlers/BlockBlobHandler.ts index 3886564e2..e03c09e5e 100644 --- a/src/blob/handlers/BlockBlobHandler.ts +++ b/src/blob/handlers/BlockBlobHandler.ts @@ -12,18 +12,18 @@ import { newEtag } from "../../common/utils/utils"; import BlobStorageContext from "../context/BlobStorageContext"; -import NotImplementedError from "../errors/NotImplementedError"; import StorageErrorFactory from "../errors/StorageErrorFactory"; import * as Models from "../generated/artifacts/models"; import Context from "../generated/Context"; import IBlockBlobHandler from "../generated/handlers/IBlockBlobHandler"; import { parseXML } from "../generated/utils/xml"; import { BlobModel, BlockModel } from "../persistence/IBlobMetadataStore"; -import { BLOB_API_VERSION } from "../utils/constants"; +import { BLOB_API_VERSION, HeaderConstants } from "../utils/constants"; import BaseHandler from "./BaseHandler"; import { computeAndValidateTransactionalChecksums, - getTagsFromString + getTagsFromString, + validateTransactionalChecksumHeaders } from "../utils/utils"; /** @@ -208,9 +208,229 @@ export default class BlockBlobHandler return response; } - public async putBlobFromUrl(contentLength: number, copySource: string, options: Models.BlockBlobPutBlobFromUrlOptionalParams, context: Context + public async putBlobFromUrl( + contentLength: number, + copySource: string, + options: Models.BlockBlobPutBlobFromUrlOptionalParams, + context: Context ): Promise { - throw new NotImplementedError(context.contextId); + const blobCtx = new BlobStorageContext(context); + const accountName = blobCtx.account!; + const containerName = blobCtx.container!; + const blobName = blobCtx.blob!; + const date = blobCtx.startTime!; + const etag = newEtag(); + + // Put Blob From URL carries no request body. + if (contentLength !== 0) { + throw StorageErrorFactory.getInvalidHeaderValue(context.contextId, { + HeaderName: "Content-Length", + HeaderValue: contentLength.toString() + }); + } + + // Three request headers can carry an MD5. x-ms-source-content-md5 is + // this operation's own integrity check over the bytes that arrive from + // the source. x-ms-blob-content-md5 and Content-MD5 get the treatment + // Put Blob gives them, since Put Blob From URL follows Put Blob for the + // custom properties and this request has no body of its own for + // Content-MD5 to describe. All three describe the same bytes, so one is + // compared: the operation's own header first, then x-ms-blob-content-md5 + // over Content-MD5, the order Put Blob uses. x-ms-content-crc64 describes + // those bytes as well, and as on Put Blob it cannot be sent alongside an + // MD5. The header shapes are checked here, before fetching anything. + const blobHTTPHeaders = options.blobHTTPHeaders || {}; + const { md5: expectedContentMD5, crc64: expectedContentCRC64 } = + validateTransactionalChecksumHeaders( + [ + options.sourceContentMD5, + blobHTTPHeaders.blobContentMD5, + options.transactionalContentMD5 + ], + options.transactionalContentCrc64, + context.contextId + ); + + // The destination's tags are either the source's or the request's, never + // both. + const copySourceTags = + options.copySourceTags === Models.BlobCopySourceTags.COPY; + if (copySourceTags && options.blobTagsString !== undefined) { + throw StorageErrorFactory.getBothUserTagsAndSourceTagsCopyPresentException( + context.contextId! + ); + } + + await this.metadataStore.checkContainerExist( + context, + accountName, + containerName + ); + + // Put Blob From URL always copies the whole source, so no range rides + // along with the conditions. + const sourceResponse = await this.readCopySource( + context, + "putBlobFromUrl", + copySource, + BlockBlobHandler.sourceConditionHeaders( + options.sourceModifiedAccessConditions + ) + ); + + // The status was only the response headers arriving; the body can still + // fail midway (socket error, connection reset). Map that to the same + // error as a transport failure rather than letting it escape as a + // bodiless 500, and release the source stream on the way out. + let persistency: IExtentChunk; + try { + persistency = await this.extentStore.appendExtent( + sourceResponse.data, + context.contextId + ); + } catch (err) { + sourceResponse.data.destroy(); + this.logger.error( + `BlockBlobHandler:putBlobFromUrl() Failed to read the copy source body: ${err}`, + context.contextId + ); + throw StorageErrorFactory.getCannotVerifyCopySource( + context.contextId!, + 500, + "Could not verify the copy source within the specified time." + ); + } + + // The response always carries an MD5 and a CRC64 of what was copied, so + // both are always computed, whether or not the request sent one to + // compare them with. Destroy the stream regardless, so a mismatch cannot + // leave the extent handle open. + const stream = await this.extentStore.readExtent( + persistency, + context.contextId + ); + let calculatedContentMD5: Uint8Array | undefined; + let calculatedContentCRC64: Uint8Array | undefined; + try { + ({ md5: calculatedContentMD5, crc64: calculatedContentCRC64 } = + await computeAndValidateTransactionalChecksums( + stream, + { md5: expectedContentMD5, crc64: expectedContentCRC64 }, + context.contextId, + { md5: true, crc64: true } + )); + } finally { + (stream as Readable).destroy?.(); + } + + // COPY reads the source's tags over the same authorized path the content + // came over, so a source that the caller may read but not tag refuses + // the copy rather than leaking them. + const blobTags = copySourceTags + ? await this.readCopySourceTags(context, copySource) + : options.blobTagsString === undefined + ? undefined + : getTagsFromString(options.blobTagsString, context.contextId!); + + // The standard properties are copied from the source unless the request + // turns that off, and a blob content header on the request sets that one + // property either way. The request's own Content-Type is only a + // fallback: a client sends one on a bodiless request without meaning to + // retype the copy. + const copyProperties = options.copySourceBlobProperties !== false; + const sourceProperty = (name: string): string | undefined => + copyProperties ? sourceResponse.headers[name] : undefined; + const contentType = + blobHTTPHeaders.blobContentType || + sourceProperty("content-type") || + context.request!.getHeader("content-type") || + "application/octet-stream"; + + // Metadata named on the request replaces the source's rather than adding + // to it, and naming none copies the source's. Both are read from raw + // headers, which preserve the case of the names. + const metadata = + convertRawHeadersToMetadata( + blobCtx.request!.getRawHeaders(), + context.contextId! + ) ?? + convertRawHeadersToMetadata( + (sourceResponse.data as IncomingMessage).rawHeaders, + context.contextId! + ); + + const blob: BlobModel = { + deleted: false, + metadata, + accountName, + containerName, + name: blobName, + properties: { + creationTime: date, + lastModified: date, + etag, + // The destination's length is the source's, not the Content-Length + // of this bodiless request. + contentLength: persistency.count, + contentType, + contentEncoding: + blobHTTPHeaders.blobContentEncoding || + sourceProperty("content-encoding"), + contentLanguage: + blobHTTPHeaders.blobContentLanguage || + sourceProperty("content-language"), + contentMD5: calculatedContentMD5, + contentDisposition: + blobHTTPHeaders.blobContentDisposition || + sourceProperty("content-disposition"), + cacheControl: + blobHTTPHeaders.blobCacheControl || sourceProperty("cache-control"), + blobType: Models.BlobType.BlockBlob, + leaseStatus: Models.LeaseStatusType.Unlocked, + leaseState: Models.LeaseStateType.Available, + serverEncrypted: true, + accessTier: Models.AccessTier.Hot, + accessTierInferred: true, + accessTierChangeTime: date + }, + snapshot: "", + isCommitted: true, + persistency, + blobTags + }; + + if (options.tier !== undefined) { + blob.properties.accessTier = this.parseTier(options.tier); + if (blob.properties.accessTier === undefined) { + throw StorageErrorFactory.getInvalidHeaderValue(context.contextId, { + HeaderName: "x-ms-access-tier", + HeaderValue: `${options.tier}` + }); + } + blob.properties.accessTierInferred = false; + } + + await this.metadataStore.createBlob( + context, + blob, + options.leaseAccessConditions, + options.modifiedAccessConditions + ); + + const response: Models.BlockBlobPutBlobFromUrlResponse = { + statusCode: 201, + eTag: etag, + lastModified: date, + contentMD5: blob.properties.contentMD5, + xMsContentCrc64: calculatedContentCRC64, + requestId: blobCtx.contextId, + version: BLOB_API_VERSION, + date, + isServerEncrypted: true, + clientRequestId: options.requestId + }; + + return response; } public async stageBlock( @@ -327,33 +547,14 @@ export default class BlockBlobHandler this.validateBlockId(blockId, blobCtx); - // Reject malformed source checksum headers before fetching anything. The - // shared validator would catch these too, but it reports the names of the - // transactional headers, and its errors would surface only after the - // source had already been read and staged. - if ( - options.sourceContentMD5 !== undefined && - options.sourceContentcrc64 !== undefined - ) { - throw StorageErrorFactory.getBothCrc64AndMd5HeaderPresent( - context.contextId - ); - } - if ( - options.sourceContentMD5 !== undefined && - options.sourceContentMD5.length !== 16 - ) { - throw StorageErrorFactory.getInvalidMd5(context.contextId); - } - if ( - options.sourceContentcrc64 !== undefined && - options.sourceContentcrc64.length < 8 - ) { - throw StorageErrorFactory.getInvalidHeaderValue(context.contextId, { - HeaderName: "x-ms-source-content-crc64", - HeaderValue: Buffer.from(options.sourceContentcrc64).toString("base64") - }); - } + // Reject malformed source checksum headers before fetching anything, + // reporting a bad CRC64 under the header this operation carries it in. + validateTransactionalChecksumHeaders( + [options.sourceContentMD5], + options.sourceContentcrc64, + context.contextId, + HeaderConstants.X_MS_SOURCE_CONTENT_CRC64 + ); await this.metadataStore.checkContainerExist( context, @@ -361,70 +562,7 @@ export default class BlockBlobHandler containerName ); - let url: URL; - try { - url = new URL(sourceUrl); - } catch { - throw StorageErrorFactory.getInvalidHeaderValue(context.contextId, { - HeaderName: "x-ms-copy-source", - HeaderValue: sourceUrl - }); - } - - // Only sources within the same Azurite instance are supported, as with - // copyFromURL. - // Hostnames compare case-insensitively and new URL() lowercases its - // host, so normalize the client-supplied header before comparing. - const currentServer = (blobCtx.request!.getHeader("Host") || "") - .toLowerCase(); - if (currentServer !== url.host) { - this.logger.error( - `BlockBlobHandler:stageBlockFromURL() Source ${url} is not on the same Azurite instance as target account ${accountName}`, - context.contextId - ); - throw StorageErrorFactory.getCannotVerifyCopySource( - context.contextId!, - 404, - "The specified resource does not exist" - ); - } - - // The Host header above is client-controlled, so never fetch the - // caller-supplied URL directly; pin the outbound request to the - // loopback address and port this server is actually bound to, keeping - // only the caller's path and query. - const rawRequest = blobCtx.request!.getBodyStream(); - if (!(rawRequest instanceof IncomingMessage) || - rawRequest.socket.localPort === undefined) { - throw StorageErrorFactory.getCannotVerifyCopySource( - context.contextId!, - 404, - "The specified resource does not exist" - ); - } - const scheme = "encrypted" in rawRequest.socket ? "https" : "http"; - // Use the local address this request arrived on rather than a - // hard-coded loopback so non-loopback --blobHost binds keep working; - // IPv6 literals need brackets in URLs. - const localAddress = rawRequest.socket.localAddress || "127.0.0.1"; - const localHost = localAddress.includes(":") ? - `[${localAddress}]` : localAddress; - const pinnedUrl = - `${scheme}://${localHost}:${rawRequest.socket.localPort}` + - `${url.pathname}${url.search}`; - - // Fetch the source range over loopback so that SAS authentication, - // range handling, and source conditions reuse the download path. - // Preserve the source URL's host so product-style source URLs still - // resolve their account from the Host header; the connection itself - // stays pinned to this server's bound address. - // A block must be staged as the bytes the source actually stores, so ask - // for the body verbatim rather than letting anything in the path apply - // transfer compression. - const headers: { [key: string]: string } = { - host: url.host, - "accept-encoding": "identity" - }; + const headers: { [key: string]: string } = {}; if (options.sourceRange !== undefined) { // The download path ignores malformed Range headers, which would // silently stage the entire source blob; reject them up front. @@ -441,83 +579,24 @@ export default class BlockBlobHandler } headers.range = options.sourceRange; } - const sourceConditions = options.sourceModifiedAccessConditions || {}; - if (sourceConditions.sourceIfMatch !== undefined) { - headers["if-match"] = sourceConditions.sourceIfMatch; - } - if (sourceConditions.sourceIfNoneMatch !== undefined) { - headers["if-none-match"] = sourceConditions.sourceIfNoneMatch; - } - if (sourceConditions.sourceIfModifiedSince !== undefined) { - headers["if-modified-since"] = new Date( - sourceConditions.sourceIfModifiedSince - ).toUTCString(); - } - if (sourceConditions.sourceIfUnmodifiedSince !== undefined) { - headers["if-unmodified-since"] = new Date( - sourceConditions.sourceIfUnmodifiedSince - ).toUTCString(); - } // Note: unlike the Copy Blob operations, Put Block From URL has no // x-ms-source-if-tags condition; the generated operation spec does not // deserialize one. + Object.assign( + headers, + BlockBlobHandler.sourceConditionHeaders( + options.sourceModifiedAccessConditions + ) + ); - let sourceResponse: AxiosResponse; - try { - sourceResponse = await axios.get(pinnedUrl, { - headers, - responseType: "stream", - validateStatus: () => true, - // Never decompress. A source blob carries Content-Encoding as a - // stored property, so the download echoes it back even though the - // bytes on the wire are the raw stored ones. Decompressing here - // would stage the decoded content instead of what the source holds, - // and would fail outright when the property does not match the - // bytes. - decompress: false, - // Pin trust to the certificate this server itself presents; see - // getLoopbackHttpsAgent(). - httpsAgent: scheme === "https" - ? getLoopbackHttpsAgent(rawRequest.socket as TLSSocket) - : undefined - }); - } catch (err) { - // Transport-level failures (TLS, connection reset, socket errors) throw - // rather than returning a status. Without this they would escape as a - // bodiless 500 instead of an Azure-shaped error. - this.logger.error( - `BlockBlobHandler:stageBlockFromURL() Failed to read the copy source: ${err}`, - context.contextId - ); - throw StorageErrorFactory.getCannotVerifyCopySource( - context.contextId!, - 500, - "Could not verify the copy source within the specified time." - ); - } - - if (sourceResponse.status === 304 || sourceResponse.status === 412) { - sourceResponse.data.destroy(); - throw StorageErrorFactory.getSourceConditionNotMet(context.contextId!); - } - if (sourceResponse.status === 404) { - sourceResponse.data.destroy(); - throw StorageErrorFactory.getCannotVerifyCopySource( - context.contextId!, - 404, - "The specified resource does not exist" - ); - } - if (sourceResponse.status !== 200 && sourceResponse.status !== 206) { - sourceResponse.data.destroy(); - throw StorageErrorFactory.getCannotVerifyCopySource( - context.contextId!, - sourceResponse.status, - "Could not verify the copy source within the specified time." - ); - } + const sourceResponse = await this.readCopySource( + context, + "stageBlockFromURL", + sourceUrl, + headers + ); - // The status above only means the response headers arrived; the body can + // The status was only the response headers arriving; the body can // still fail midway (socket error, connection reset). Map that to the // same error as a transport failure rather than letting it escape as a // bodiless 500, and release the source stream on the way out. @@ -829,4 +908,252 @@ export default class BlockBlobHandler ); } } + + /** + * Restate the conditions a copy request names for its source as the + * conditional headers of a read, so that the download path answers them + * the way it answers a client reading the source itself. + * + * x-ms-if-tags only ever appears for Put Blob From URL: the Put Block From + * URL specification deserializes no source tag condition. + * + * @private + * @param {Models.SourceModifiedAccessConditions} [conditions] + * @returns {{ [key: string]: string }} + * @memberof BlockBlobHandler + */ + private static sourceConditionHeaders( + conditions: Models.SourceModifiedAccessConditions = {} + ): { [key: string]: string } { + const headers: { [key: string]: string } = {}; + if (conditions.sourceIfMatch !== undefined) { + headers["if-match"] = conditions.sourceIfMatch; + } + if (conditions.sourceIfNoneMatch !== undefined) { + headers["if-none-match"] = conditions.sourceIfNoneMatch; + } + if (conditions.sourceIfModifiedSince !== undefined) { + headers["if-modified-since"] = new Date( + conditions.sourceIfModifiedSince + ).toUTCString(); + } + if (conditions.sourceIfUnmodifiedSince !== undefined) { + headers["if-unmodified-since"] = new Date( + conditions.sourceIfUnmodifiedSince + ).toUTCString(); + } + if (conditions.sourceIfTags !== undefined) { + headers["x-ms-if-tags"] = conditions.sourceIfTags; + } + return headers; + } + + /** + * Read a copy source with a loopback self-request, so that SAS + * authentication, ranges, and source conditions are answered by the + * download path rather than reimplemented against the store. + * + * Only sources within the same Azurite instance are supported, as with + * copyFromURL. The Host header that decides this is the caller's to + * choose, so the request is never made to the URL they supplied: it is + * pinned to the address and port this server is bound to and keeps only + * their path and query, with the source's own host along as a header so + * that product-style source URLs still resolve their account from it. + * + * @private + * @param {Context} context + * @param {string} operation Handler method name, for log messages + * @param {string} copySource The source URL the request named + * @param {{ [key: string]: string }} headers Conditions, ranges + * @param {string} [subresource] A query to append, such as "comp=tags" + * @returns {Promise} A response whose body is a stream + * @memberof BlockBlobHandler + */ + private async readCopySource( + context: Context, + operation: string, + copySource: string, + headers: { [key: string]: string }, + subresource?: string + ): Promise { + const blobCtx = new BlobStorageContext(context); + + let url: URL; + try { + url = new URL(copySource); + } catch { + throw StorageErrorFactory.getInvalidHeaderValue(context.contextId, { + HeaderName: "x-ms-copy-source", + HeaderValue: copySource + }); + } + + // Hostnames compare case-insensitively and new URL() lowercases its + // host, so normalize the client-supplied header before comparing. + const currentServer = (blobCtx.request!.getHeader("Host") || "") + .toLowerCase(); + if (currentServer !== url.host) { + this.logger.error( + `BlockBlobHandler:${operation}() Source ${url} is not on the same Azurite instance as target account ${blobCtx.account}`, + context.contextId + ); + throw StorageErrorFactory.getCannotVerifyCopySource( + context.contextId!, + 404, + "The specified resource does not exist" + ); + } + + const rawRequest = blobCtx.request!.getBodyStream(); + if (!(rawRequest instanceof IncomingMessage) || + rawRequest.socket.localPort === undefined) { + throw StorageErrorFactory.getCannotVerifyCopySource( + context.contextId!, + 404, + "The specified resource does not exist" + ); + } + const scheme = "encrypted" in rawRequest.socket ? "https" : "http"; + // Use the local address this request arrived on rather than a + // hard-coded loopback so non-loopback --blobHost binds keep working; + // IPv6 literals need brackets in URLs. + const localAddress = rawRequest.socket.localAddress || "127.0.0.1"; + const localHost = localAddress.includes(":") ? + `[${localAddress}]` : localAddress; + // Append rather than rebuild the query: a shared access signature signs + // the exact encoding it arrived in, which re-encoding could disturb. + const query = subresource === undefined + ? url.search + : `${url.search}${url.search === "" ? "?" : "&"}${subresource}`; + const pinnedUrl = + `${scheme}://${localHost}:${rawRequest.socket.localPort}` + + `${url.pathname}${query}`; + + let sourceResponse: AxiosResponse; + try { + sourceResponse = await axios.get(pinnedUrl, { + headers: { + host: url.host, + // A copy must carry the bytes the source actually stores, so ask + // for the body verbatim rather than letting anything in the path + // apply transfer compression. + "accept-encoding": "identity", + ...headers + }, + responseType: "stream", + validateStatus: () => true, + // Never decompress. A source blob carries Content-Encoding as a + // stored property, so the download echoes it back even though the + // bytes on the wire are the raw stored ones. Decompressing here + // would copy the decoded content instead of what the source holds, + // and would fail outright when the property does not match the + // bytes. + decompress: false, + // Pin trust to the certificate this server itself presents; see + // getLoopbackHttpsAgent(). + httpsAgent: scheme === "https" + ? getLoopbackHttpsAgent(rawRequest.socket as TLSSocket) + : undefined + }); + } catch (err) { + // Transport-level failures (TLS, connection reset, socket errors) throw + // rather than returning a status. Without this they would escape as a + // bodiless 500 instead of an Azure-shaped error. + this.logger.error( + `BlockBlobHandler:${operation}() Failed to read the copy source: ${err}`, + context.contextId + ); + throw StorageErrorFactory.getCannotVerifyCopySource( + context.contextId!, + 500, + "Could not verify the copy source within the specified time." + ); + } + + if (sourceResponse.status === 304 || sourceResponse.status === 412) { + sourceResponse.data.destroy(); + throw StorageErrorFactory.getSourceConditionNotMet(context.contextId!); + } + if (sourceResponse.status === 404) { + sourceResponse.data.destroy(); + throw StorageErrorFactory.getCannotVerifyCopySource( + context.contextId!, + 404, + "The specified resource does not exist" + ); + } + if (sourceResponse.status !== 200 && sourceResponse.status !== 206) { + sourceResponse.data.destroy(); + throw StorageErrorFactory.getCannotVerifyCopySource( + context.contextId!, + sourceResponse.status, + "Could not verify the copy source within the specified time." + ); + } + + return sourceResponse; + } + + /** + * Read the tags of a copy source, for the copy that asks to carry them + * over. Real Azure charges this to the caller as its own Get Blob Tags + * request against the source, and so does this: the authorization the + * source URL carries has to allow reading them. + * + * @private + * @param {Context} context + * @param {string} copySource The source URL the request named + * @returns {Promise} + * @memberof BlockBlobHandler + */ + private async readCopySourceTags( + context: Context, + copySource: string + ): Promise { + const response = await this.readCopySource( + context, + "putBlobFromUrl", + copySource, + {}, + "comp=tags" + ); + + // As with the content, the status was only the headers arriving: the + // body can still fail midway, and what arrives has to parse. Map either + // to the error a transport failure gets rather than letting it escape + // as a bodiless 500, and release the stream on the way out. + let parsed: any; + try { + const chunks: Buffer[] = []; + for await (const chunk of response.data as IncomingMessage) { + chunks.push(Buffer.from(chunk)); + } + parsed = await parseXML(Buffer.concat(chunks).toString()); + } catch (err) { + response.data.destroy(); + this.logger.error( + `BlockBlobHandler:putBlobFromUrl() Failed to read the copy source tags: ${err}`, + context.contextId + ); + throw StorageErrorFactory.getCannotVerifyCopySource( + context.contextId!, + 500, + "Could not verify the copy source within the specified time." + ); + } + + // parseXML collapses a single element out of its array, and leaves a + // tagless source with no TagSet at all. + const tagSet = parsed.TagSet; + if (tagSet === undefined || tagSet === "" || tagSet.Tag === undefined) { + return undefined; + } + const tags = Array.isArray(tagSet.Tag) ? tagSet.Tag : [tagSet.Tag]; + return { + blobTagSet: tags.map((tag: { Key: string; Value: string }) => ({ + key: tag.Key, + value: tag.Value + })) + }; + } } diff --git a/src/blob/utils/constants.ts b/src/blob/utils/constants.ts index 55933d0aa..649fe8342 100644 --- a/src/blob/utils/constants.ts +++ b/src/blob/utils/constants.ts @@ -59,6 +59,9 @@ export const HeaderConstants = { X_MS_SEQUENCE_NUMBER_ACTION: "x-ms-sequence-number-action", X_MS_BLOB_SEQUENCE_NUMBER: "x-ms-blob-sequence-number", X_MS_CONTENT_CRC64: "x-ms-content-crc64", + X_MS_SOURCE_CONTENT_CRC64: "x-ms-source-content-crc64", + X_MS_TAGS: "x-ms-tags", + X_MS_COPY_SOURCE_TAG_OPTION: "x-ms-copy-source-tag-option", X_MS_RANGE_GET_CONTENT_CRC64: "x-ms-range-get-content-crc64", X_MS_ENCRYPTION_KEY: "x-ms-encryption-key", X_MS_ENCRYPTION_KEY_SHA256: "x-ms-encryption-key-sha256", diff --git a/src/blob/utils/utils.ts b/src/blob/utils/utils.ts index 53e00e1a4..eb7a1c4a9 100644 --- a/src/blob/utils/utils.ts +++ b/src/blob/utils/utils.ts @@ -38,6 +38,14 @@ function decodeBase64HeaderValue(value: string): Buffer | undefined { return decoded; } +function decodeChecksumHeader( + value: Uint8Array | string +): Buffer | undefined { + return typeof value === "string" + ? decodeBase64HeaderValue(value) + : Buffer.from(value); +} + /** * Decodes an MD5 header value (base64 string or raw Uint8Array) and returns * whether the result is exactly 16 bytes - the only shape real Azure accepts. @@ -45,21 +53,62 @@ function decodeBase64HeaderValue(value: string): Buffer | undefined { * x-ms-blob-content-md5 are all rejected with InvalidMd5 (verified live). */ export function isValidMd5Header(value: Uint8Array | string): boolean { - const bytes = - typeof value === "string" - ? decodeBase64HeaderValue(value) - : Buffer.from(value); + const bytes = decodeChecksumHeader(value); return bytes !== undefined && bytes.length === 16; } +/** + * Checks the shape of the checksum headers a request carries and picks the + * pair its bytes are compared with, before anything is read. Every MD5 + * candidate that is present must be well formed, and the first one present + * is the one compared, so callers list them in precedence order. An MD5 and + * a CRC64 cannot be sent together. A malformed CRC64 is reported under + * `crc64HeaderName`, since Put Block From URL carries it as + * x-ms-source-content-crc64 rather than x-ms-content-crc64. + * + * Verified against real Azure for CRC64: fewer than 8 bytes is rejected as + * InvalidHeaderValue; 8 or more bytes pass this check and surface as + * Crc64Mismatch if they do not match. + */ +export function validateTransactionalChecksumHeaders( + md5Candidates: Array, + crc64: Uint8Array | string | undefined, + contextId: string | undefined, + crc64HeaderName: string = HeaderConstants.X_MS_CONTENT_CRC64 +): { md5?: Uint8Array | string; crc64?: Uint8Array | string } { + const md5 = md5Candidates.find((candidate) => candidate !== undefined); + if (md5 !== undefined && crc64 !== undefined) { + throw StorageErrorFactory.getBothCrc64AndMd5HeaderPresent(contextId); + } + for (const candidate of md5Candidates) { + if (candidate !== undefined && !isValidMd5Header(candidate)) { + throw StorageErrorFactory.getInvalidMd5(contextId); + } + } + if (crc64 !== undefined) { + const bytes = decodeChecksumHeader(crc64); + if (bytes === undefined || bytes.length < 8) { + throw StorageErrorFactory.getInvalidHeaderValue(contextId, { + HeaderName: crc64HeaderName, + HeaderValue: + typeof crc64 === "string" + ? crc64 + : Buffer.from(crc64).toString("base64") + }); + } + } + return { md5, crc64 }; +} + /** * Computes MD5 and/or CRC-64/NVME from a stream in a single pass and validates * against the request-supplied values. Throws Md5Mismatch / Crc64Mismatch * (HTTP 400) on mismatch - the documented Azure Storage error codes for * transactional integrity failures. * - * Rejects requests that supply both checksums with `BothCrc64AndMd5HeaderPresent` - * (HTTP 400), matching the real Azure service contract. + * The header shapes are checked by validateTransactionalChecksumHeaders + * first, so a request that supplies both checksums or a malformed one is + * rejected before the stream is read. * * A checksum is computed when its `expected` value is provided, OR when the * corresponding `force` flag is set (for callers that need the value for @@ -71,39 +120,11 @@ export async function computeAndValidateTransactionalChecksums( contextId: string | undefined, force?: { md5?: boolean; crc64?: boolean } ): Promise<{ md5?: Uint8Array; crc64?: Uint8Array }> { - if (expected.md5 !== undefined && expected.crc64 !== undefined) { - throw StorageErrorFactory.getBothCrc64AndMd5HeaderPresent(contextId); - } - if (expected.md5 !== undefined && !isValidMd5Header(expected.md5)) { - throw StorageErrorFactory.getInvalidMd5(contextId); - } - const expectedCrc64RawHeader = - typeof expected.crc64 === "string" - ? expected.crc64 - : expected.crc64 !== undefined - ? Buffer.from(expected.crc64).toString("base64") - : undefined; - - const expectedCrc64Bytes = - expected.crc64 === undefined - ? undefined - : typeof expected.crc64 === "string" - ? decodeBase64HeaderValue(expected.crc64) - : Buffer.from(expected.crc64); - - if ( - expected.crc64 !== undefined && - (expectedCrc64Bytes === undefined || expectedCrc64Bytes.length < 8) - ) { - // CRC-64/NVME is a 64-bit value; the wire format is base64-encoded bytes. - // Verified against real Azure: <8 bytes is rejected as InvalidHeaderValue; - // >=8 bytes is accepted at header-validation and falls through to a value - // comparison (which then surfaces as Crc64Mismatch if it doesn't match). - throw StorageErrorFactory.getInvalidHeaderValue(contextId, { - HeaderName: HeaderConstants.X_MS_CONTENT_CRC64, - HeaderValue: expectedCrc64RawHeader ?? "" - }); - } + validateTransactionalChecksumHeaders( + [expected.md5], + expected.crc64, + contextId + ); const calculated = await computeTransactionalChecksums( stream, expected, @@ -111,13 +132,8 @@ export async function computeAndValidateTransactionalChecksums( ); if (expected.md5 !== undefined) { - const expectedMd5Bytes = - typeof expected.md5 === "string" - ? decodeBase64HeaderValue(expected.md5)! - : Buffer.from(expected.md5); - const calculatedMd5Bytes = Buffer.from(calculated.md5!); - const expectedMd5 = expectedMd5Bytes.toString("base64"); - const calculatedMd5 = calculatedMd5Bytes.toString("base64"); + const expectedMd5 = decodeChecksumHeader(expected.md5)!.toString("base64"); + const calculatedMd5 = Buffer.from(calculated.md5!).toString("base64"); if (expectedMd5 !== calculatedMd5) { throw StorageErrorFactory.getMd5Mismatch( contextId, @@ -126,10 +142,11 @@ export async function computeAndValidateTransactionalChecksums( ); } } - if (expectedCrc64Bytes !== undefined) { - const calculatedCrc64Bytes = Buffer.from(calculated.crc64!); - const expectedCrc64 = expectedCrc64Bytes.toString("base64"); - const calculatedCrc64 = calculatedCrc64Bytes.toString("base64"); + if (expected.crc64 !== undefined) { + const expectedCrc64 = decodeChecksumHeader(expected.crc64)!.toString( + "base64" + ); + const calculatedCrc64 = Buffer.from(calculated.crc64!).toString("base64"); if (expectedCrc64 !== calculatedCrc64) { throw StorageErrorFactory.getCrc64Mismatch( contextId, diff --git a/swagger/blob-storage-2021-10-04.json b/swagger/blob-storage-2021-10-04.json index 43ce6da2b..731b4ddb1 100644 --- a/swagger/blob-storage-2021-10-04.json +++ b/swagger/blob-storage-2021-10-04.json @@ -4543,6 +4543,9 @@ { "$ref": "#/parameters/ContentMD5" }, + { + "$ref": "#/parameters/ContentCrc64" + }, { "$ref": "#/parameters/ContentLength" }, @@ -4658,6 +4661,11 @@ "format": "byte", "description": "If the blob has an MD5 hash and this operation is to read the full blob, this response header is returned so that the client can check for message content integrity." }, + "x-ms-content-crc64": { + "type": "string", + "format": "byte", + "description": "This header is returned so that the client can check for message content integrity. The value of this header is computed by the Blob service; it is not necessarily the same value specified in the request headers." + }, "x-ms-client-request-id": { "x-ms-client-name": "ClientRequestId", "type": "string", diff --git a/swagger/blob.md b/swagger/blob.md index 02025dddf..36b9d9650 100644 --- a/swagger/blob.md +++ b/swagger/blob.md @@ -66,3 +66,5 @@ enum-types: true 16. Add "Cold" to "AccessTier", "AccessTierRequired", "AccessTierOptional"; and add "rehydrate-pending-to-cold" to "ArchiveStatus". (can be removed when upgrade to new API version.) 17. Remove default value setting parameter "BlobSequenceNumber" for header "x-ms-blob-sequence-number" + +18. Add "ContentCrc64" parameter and "x-ms-content-crc64" response header to "BlockBlob_PutBlobFromUrl". The REST reference lists both for this operation; the client swagger carries only the response header. diff --git a/tests/blob/apis/blockblob.test.ts b/tests/blob/apis/blockblob.test.ts index 1cd19661d..745dc1d33 100644 --- a/tests/blob/apis/blockblob.test.ts +++ b/tests/blob/apis/blockblob.test.ts @@ -1035,6 +1035,852 @@ describe("BlockBlobAPIs", () => { assert.fail("Did not throw an exception."); }); + it("putBlobFromUrl copies the source blob @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const result = await blockBlobClient.syncUploadFromURL(sourceUrl); + // The response echoes the MD5 the service computed over what it copied. + assert.deepStrictEqual( + Buffer.from(result.contentMD5!), + Buffer.from(await getMD5FromString(content)) + ); + + const download = await blobClient.download(0); + assert.equal(await bodyToString(download, content.length), content); + // The destination's length is the source's, not the Content-Length of + // the bodiless request that created it. + assert.equal(download.contentLength, content.length); + }); + + it("putBlobFromUrl overwrites an existing destination @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + await blockBlobClient.upload("overwritten", "overwritten".length); + + await blockBlobClient.syncUploadFromURL(sourceUrl); + + const download = await blobClient.download(0); + assert.equal(await bodyToString(download, content.length), content); + }); + + it("putBlobFromUrl copies the source's properties and metadata @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const properties = { + blobCacheControl: "max-age=3600", + blobContentDisposition: "attachment; filename=source.txt", + blobContentEncoding: "identity", + blobContentLanguage: "en", + blobContentType: "text/plain" + }; + const metadata = { keya: "vala", keyb: "valb" }; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length, { + blobHTTPHeaders: properties, + metadata + }); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + await blockBlobClient.syncUploadFromURL(sourceUrl); + + const download = await blobClient.download(0); + assert.equal(download.cacheControl, properties.blobCacheControl); + assert.equal( + download.contentDisposition, + properties.blobContentDisposition + ); + assert.equal(download.contentEncoding, properties.blobContentEncoding); + assert.equal(download.contentLanguage, properties.blobContentLanguage); + assert.equal(download.contentType, properties.blobContentType); + assert.deepStrictEqual(download.metadata, metadata); + }); + + it("putBlobFromUrl with copySourceBlobProperties false leaves the source's properties @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const metadata = { keya: "vala" }; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length, { + blobHTTPHeaders: { + blobCacheControl: "max-age=3600", + blobContentDisposition: "attachment; filename=source.txt", + blobContentEncoding: "identity", + blobContentLanguage: "en", + blobContentType: "text/plain" + }, + metadata + }); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + await blockBlobClient.syncUploadFromURL(sourceUrl, { + copySourceBlobProperties: false + }); + + const download = await blobClient.download(0); + assert.equal(download.cacheControl, undefined); + assert.equal(download.contentDisposition, undefined); + assert.equal(download.contentEncoding, undefined); + assert.equal(download.contentLanguage, undefined); + assert.equal(download.contentType, "application/octet-stream"); + // Metadata answers to its own rule rather than to this header: the + // request named none, so the source's carries over. + assert.deepStrictEqual(download.metadata, metadata); + }); + + it("putBlobFromUrl request headers override the copied properties @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length, { + blobHTTPHeaders: { + blobCacheControl: "max-age=3600", + blobContentType: "text/plain" + } + }); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + await blockBlobClient.syncUploadFromURL(sourceUrl, { + blobHTTPHeaders: { blobContentType: "application/json" } + }); + + const download = await blobClient.download(0); + assert.equal(download.contentType, "application/json"); + // A property the request did not name still comes from the source. + assert.equal(download.cacheControl, "max-age=3600"); + }); + + it("putBlobFromUrl metadata replaces the source's @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length, { + metadata: { keya: "vala" } + }); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + await blockBlobClient.syncUploadFromURL(sourceUrl, { + metadata: { keyc: "valc" } + }); + + const download = await blobClient.download(0); + assert.deepStrictEqual(download.metadata, { keyc: "valc" }); + }); + + it("putBlobFromUrl answers the source conditions @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + const upload = await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + try { + await blockBlobClient.syncUploadFromURL(sourceUrl, { + sourceConditions: { ifMatch: '"0x0000000000000000"' } + }); + assert.fail("Did not throw an exception."); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 412); + assert.equal(e.code, "SourceConditionNotMet"); + } + + // The same condition naming the source's own ETag admits the copy. + await blockBlobClient.syncUploadFromURL(sourceUrl, { + sourceConditions: { ifMatch: upload.etag } + }); + const download = await blobClient.download(0); + assert.equal(await bodyToString(download, content.length), content); + }); + + it("putBlobFromUrl answers the destination conditions @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + await blockBlobClient.upload("existing", "existing".length); + + try { + await blockBlobClient.syncUploadFromURL(sourceUrl, { + conditions: { ifNoneMatch: "*" } + }); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 409); + assert.equal(e.code, "BlobAlreadyExists"); + // The destination the condition protected is untouched. + const download = await blobClient.download(0); + assert.equal(await bodyToString(download, 8), "existing"); + return; + } + assert.fail("Did not throw an exception."); + }); + + it("putBlobFromUrl rejects a request body @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + const destinationUrl = await blockBlobClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("rw"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + // @azure/storage-blob sends no body for this operation, so issue the + // request directly. + const response = await axios.put(destinationUrl, "unexpected body", { + headers: { + "x-ms-copy-source": sourceUrl, + "x-ms-blob-type": "BlockBlob" + }, + validateStatus: () => true + }); + assert.deepStrictEqual(response.status, 400); + assert.ok(response.data.includes("InvalidHeaderValue")); + }); + + it("putBlobFromUrl from a missing source returns 404 @loki @sql", async () => { + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + try { + await blockBlobClient.syncUploadFromURL(sourceUrl); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 404); + assert.equal(e.code, "CannotVerifyCopySource"); + return; + } + assert.fail("Did not throw an exception."); + }); + + it("putBlobFromUrl copies the stored bytes when the source declares Content-Encoding: gzip @loki @sql", async () => { + // A blob's Content-Encoding is stored metadata, not a description of how + // the body is framed on the wire, so the download echoes it back over the + // raw stored bytes. The copy must carry those bytes verbatim rather than + // decoding them (see issue #646 for the same hazard on copy). + const raw = zlib.gzipSync(Buffer.from("HelloWorldFromSourceBlob")); + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(raw, raw.length, { + blobHTTPHeaders: { blobContentEncoding: "gzip" } + }); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + await blockBlobClient.syncUploadFromURL(sourceUrl); + + const download = await blockBlobClient.download(0); + const chunks: Buffer[] = []; + for await (const chunk of download.readableStreamBody!) { + chunks.push(Buffer.from(chunk)); + } + assert.deepStrictEqual( + Buffer.concat(chunks), + raw, + "The copy must be the source's stored bytes, not the decoded ones" + ); + }); + + it("putBlobFromUrl with matching sourceContentMD5 @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const md5 = crypto.createHash("md5").update(content, "utf8").digest(); + const result = await blockBlobClient.syncUploadFromURL(sourceUrl, { + sourceContentMD5: new Uint8Array(md5) + }); + assert.deepStrictEqual(Buffer.from(result.contentMD5!), md5); + }); + + it("putBlobFromUrl with wrong sourceContentMD5 should throw md5 mismatch @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const md5 = crypto.createHash("md5").update("WrongContent", "utf8").digest(); + try { + await blockBlobClient.syncUploadFromURL(sourceUrl, { + sourceContentMD5: new Uint8Array(md5) + }); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 400); + assert.equal(e.code, "Md5Mismatch"); + // The rejected copy left no blob behind. + assert.strictEqual(await blockBlobClient.exists(), false); + return; + } + assert.fail("Did not throw an exception."); + }); + + it("putBlobFromUrl with wrong-length sourceContentMD5 should be rejected @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const targetClient = getBlockBlobClientWithRawHeaders( + containerName, + getUniqueName("target"), + [{ key: "x-ms-source-content-md5", value: Buffer.from("short").toString("base64") }] + ); + + try { + await targetClient.syncUploadFromURL(sourceUrl); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 400); + assert.equal(e.code, "InvalidMd5"); + return; + } + assert.fail("Did not throw an exception."); + }); + + it("putBlobFromUrl with matching Content-MD5 @loki @sql", async () => { + // The REST reference does not list Content-MD5 among this operation's + // request headers, but the swagger carries it and the request has no + // body of its own, so it is checked against the copied content the way + // Put Blob checks it against the body. The SDK does not expose it for + // this operation, so inject the raw header. + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const md5 = crypto.createHash("md5").update(content, "utf8").digest(); + const targetClient = getBlockBlobClientWithRawHeaders( + containerName, + blobName, + [{ key: "content-md5", value: md5.toString("base64") }] + ); + const result = await targetClient.syncUploadFromURL(sourceUrl); + assert.deepStrictEqual(Buffer.from(result.contentMD5!), md5); + + const download = await blobClient.download(0); + assert.equal(await bodyToString(download, content.length), content); + }); + + it("putBlobFromUrl with wrong Content-MD5 should throw md5 mismatch @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const md5 = crypto.createHash("md5").update("WrongContent", "utf8").digest(); + const targetClient = getBlockBlobClientWithRawHeaders( + containerName, + blobName, + [{ key: "content-md5", value: md5.toString("base64") }] + ); + try { + await targetClient.syncUploadFromURL(sourceUrl); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 400); + assert.equal(e.code, "Md5Mismatch"); + // The rejected copy left no blob behind. + assert.strictEqual(await blockBlobClient.exists(), false); + return; + } + assert.fail("Did not throw an exception."); + }); + + it("putBlobFromUrl with wrong-length Content-MD5 should be rejected @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const targetClient = getBlockBlobClientWithRawHeaders( + containerName, + blobName, + [{ key: "content-md5", value: Buffer.from("short").toString("base64") }] + ); + try { + await targetClient.syncUploadFromURL(sourceUrl); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 400); + assert.equal(e.code, "InvalidMd5"); + return; + } + assert.fail("Did not throw an exception."); + }); + + it("putBlobFromUrl rejects a malformed Content-MD5 alongside a matching sourceContentMD5 @loki @sql", async () => { + // Only one of the request's MD5 headers is compared with the copied + // content, but every one of them has to be well formed. + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const md5 = crypto.createHash("md5").update(content, "utf8").digest(); + const targetClient = getBlockBlobClientWithRawHeaders( + containerName, + blobName, + [{ key: "content-md5", value: Buffer.from("short").toString("base64") }] + ); + try { + await targetClient.syncUploadFromURL(sourceUrl, { + sourceContentMD5: new Uint8Array(md5) + }); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 400); + assert.equal(e.code, "InvalidMd5"); + return; + } + assert.fail("Did not throw an exception."); + }); + + it("putBlobFromUrl x-ms-blob-content-md5 takes precedence over Content-MD5 @loki @sql", async () => { + // Put Blob From URL follows Put Blob for these headers, and Put Blob + // compares x-ms-blob-content-md5 when both are sent (see the upload test + // of the same name). + // - Content-MD5 wrong + x-ms-blob-content-md5 correct -> success + // - Content-MD5 correct + x-ms-blob-content-md5 wrong -> Md5Mismatch + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const correctMd5 = crypto.createHash("md5").update(content, "utf8").digest(); + const wrongMd5 = crypto.createHash("md5").update("WrongContent", "utf8").digest(); + + const clientWithWrongContentMd5 = getBlockBlobClientWithRawHeaders( + containerName, + blobName, + [{ key: "content-md5", value: wrongMd5.toString("base64") }] + ); + await clientWithWrongContentMd5.syncUploadFromURL(sourceUrl, { + blobHTTPHeaders: { blobContentMD5: new Uint8Array(correctMd5) } + }); + + const clientWithCorrectContentMd5 = getBlockBlobClientWithRawHeaders( + containerName, + blobName, + [{ key: "content-md5", value: correctMd5.toString("base64") }] + ); + try { + await clientWithCorrectContentMd5.syncUploadFromURL(sourceUrl, { + blobHTTPHeaders: { blobContentMD5: new Uint8Array(wrongMd5) } + }); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 400); + assert.equal(e.code, "Md5Mismatch"); + return; + } + assert.fail("Did not throw an exception."); + }); + + it("putBlobFromUrl returns the CRC64 of the copied content @loki @sql", async () => { + // The response always carries x-ms-content-crc64, computed by the + // service over the copied content, whether or not the request sent a + // checksum. The SDK does not surface the header for this operation, so + // read it from the raw response. + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + const crc64 = Buffer.from(getCRC64FromString(content)).toString("base64"); + + const result = await blockBlobClient.syncUploadFromURL(sourceUrl); + assert.equal(result._response.headers.get("x-ms-content-crc64"), crc64); + + // Still returned when the request checked an MD5 instead. + const md5 = crypto.createHash("md5").update(content, "utf8").digest(); + const resultWithMd5 = await blockBlobClient.syncUploadFromURL(sourceUrl, { + sourceContentMD5: new Uint8Array(md5) + }); + assert.equal( + resultWithMd5._response.headers.get("x-ms-content-crc64"), + crc64 + ); + }); + + it("putBlobFromUrl with matching x-ms-content-crc64 @loki @sql", async () => { + // The SDK does not expose x-ms-content-crc64 for this operation, so + // inject the raw header. + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const crc64 = Buffer.from(getCRC64FromString(content)).toString("base64"); + const targetClient = getBlockBlobClientWithRawHeaders( + containerName, + blobName, + [{ key: "x-ms-content-crc64", value: crc64 }] + ); + const result = await targetClient.syncUploadFromURL(sourceUrl); + assert.equal(result._response.status, 201); + assert.equal(result._response.headers.get("x-ms-content-crc64"), crc64); + + const download = await blobClient.download(0); + assert.equal(await bodyToString(download, content.length), content); + }); + + it("putBlobFromUrl with wrong x-ms-content-crc64 should throw crc64 mismatch @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + // A valid 8-byte CRC64 of a *different* body, to exercise the mismatch + // path rather than the malformed-header path. + const crc64 = Buffer.from(getCRC64FromString("WrongContent")).toString( + "base64" + ); + const targetClient = getBlockBlobClientWithRawHeaders( + containerName, + blobName, + [{ key: "x-ms-content-crc64", value: crc64 }] + ); + try { + await targetClient.syncUploadFromURL(sourceUrl); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 400); + assert.equal(e.code, "Crc64Mismatch"); + // The rejected copy left no blob behind. + assert.strictEqual(await blockBlobClient.exists(), false); + return; + } + assert.fail("Did not throw an exception."); + }); + + it("putBlobFromUrl with wrong-length x-ms-content-crc64 should be rejected @loki @sql", async () => { + // x-ms-content-crc64 must decode to at least 8 bytes (CRC-64 is 64-bit). + // Shorter values are rejected as InvalidHeaderValue, as on Put Block. + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const targetClient = getBlockBlobClientWithRawHeaders( + containerName, + blobName, + [ + { + key: "x-ms-content-crc64", + value: Buffer.from([1, 2, 3, 4]).toString("base64") + } + ] + ); + try { + await targetClient.syncUploadFromURL(sourceUrl); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 400); + assert.equal(e.code, "InvalidHeaderValue"); + assert.equal( + /([^<]*) { + // Put Blob rejects a request that carries both a CRC64 and an MD5, and + // Put Blob From URL follows it: Content-MD5, the header the REST + // reference names, counts, and so does the operation's own + // x-ms-source-content-md5. Both checksums are correct for the source + // content; supplying the two together is rejected regardless. + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const md5 = crypto.createHash("md5").update(content, "utf8").digest(); + const crc64 = Buffer.from(getCRC64FromString(content)).toString("base64"); + const rejection = { + name: "RestError", + statusCode: 400, + code: "BothCrc64AndMd5HeaderPresent" + }; + + const clientWithCrc64AndContentMd5 = getBlockBlobClientWithRawHeaders( + containerName, + blobName, + [ + { key: "x-ms-content-crc64", value: crc64 }, + { key: "content-md5", value: md5.toString("base64") } + ] + ); + await assert.rejects( + clientWithCrc64AndContentMd5.syncUploadFromURL(sourceUrl), + rejection + ); + + const clientWithCrc64 = getBlockBlobClientWithRawHeaders( + containerName, + blobName, + [{ key: "x-ms-content-crc64", value: crc64 }] + ); + await assert.rejects( + clientWithCrc64.syncUploadFromURL(sourceUrl, { + sourceContentMD5: new Uint8Array(md5) + }), + rejection + ); + }); + + it("putBlobFromUrl sets the tags the request names @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length, { + tags: { sourcetag: "sourcevalue" } + }); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("rt"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const tags: Tags = { tag1: "val1", tag2: "val2" }; + await blockBlobClient.syncUploadFromURL(sourceUrl, { tags }); + + // Tags are not copied from the source unless asked for, so the request's + // stand alone. + const result = await blockBlobClient.getTags(); + assert.deepStrictEqual(result.tags, tags); + }); + + it("putBlobFromUrl copies the source's tags when asked @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const tags: Tags = { sourcetag: "sourcevalue", other: "value" }; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length, { tags }); + const sourceUrl = await sourceClient.generateSasUrl({ + // Reading the source's tags is its own permission, as on the service. + permissions: BlobSASPermissions.parse("rt"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + await blockBlobClient.syncUploadFromURL(sourceUrl, { + copySourceTags: "COPY" + }); + + const result = await blockBlobClient.getTags(); + assert.deepStrictEqual(result.tags, tags); + }); + + it("putBlobFromUrl copying the tags of an untagged source @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("rt"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + await blockBlobClient.syncUploadFromURL(sourceUrl, { + copySourceTags: "COPY" + }); + + const result = await blockBlobClient.getTags(); + assert.deepStrictEqual(result.tags, {}); + }); + + it("putBlobFromUrl cannot copy tags the source URL may not read @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length, { + tags: { sourcetag: "sourcevalue" } + }); + // Read permission alone does not extend to the source's tags. + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + try { + await blockBlobClient.syncUploadFromURL(sourceUrl, { + copySourceTags: "COPY" + }); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 403); + assert.equal(e.code, "CannotVerifyCopySource"); + return; + } + assert.fail("Did not throw an exception."); + }); + + it("putBlobFromUrl rejects tags alongside copySourceTags COPY @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("rt"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + try { + await blockBlobClient.syncUploadFromURL(sourceUrl, { + copySourceTags: "COPY", + tags: { tag1: "val1" } + }); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 400); + assert.equal(e.code, "BothUserTagsAndSourceTagsCopyPresentException"); + return; + } + assert.fail("Did not throw an exception."); + }); + + it("putBlobFromUrl sets the access tier @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + await blockBlobClient.syncUploadFromURL(sourceUrl, { tier: "Cool" }); + + const properties = await blockBlobClient.getProperties(); + assert.equal(properties.accessTier, "Cool"); + }); + it("stageBlock with double commit block should work @loki @sql", async () => { const body = "HelloWorld"; diff --git a/tests/blob/sas.test.ts b/tests/blob/sas.test.ts index 3f5c6f6d9..d51786b52 100644 --- a/tests/blob/sas.test.ts +++ b/tests/blob/sas.test.ts @@ -16,7 +16,8 @@ import { AppendBlobClient, BlobBatch, Tags, - BlobClient + BlobClient, + BlockBlobClient } from "@azure/storage-blob"; import * as assert from "assert"; @@ -547,6 +548,206 @@ describe("Shared Access Signature (SAS) authentication", () => { await blob2.syncCopyFromURL(blob1.url); }); + it("Put blob from URL with write permission in account SAS should create and override a blob @loki @sql", async () => { + // Azure grants Put Blob From URL on a new block blob to Create (c) or + // Write (w), and on an existing one to Write (w) alone, the split Put + // Blob has. Write alone therefore creates as well as overrides. + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + const containerName = getUniqueName("container"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const sourceBlob = containerClient.getBlockBlobClient( + getUniqueName("blob") + ); + await sourceBlob.upload("hello", 5); + const sourceUrl = await sourceBlob.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: tmr + }); + + const serviceClientWithSAS = new BlobServiceClient( + serviceClient.generateAccountSasUrl( + tmr, + AccountSASPermissions.parse("w"), + "co" + ), + newPipeline(new AnonymousCredential(), { + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + const blobWithSAS = serviceClientWithSAS + .getContainerClient(containerName) + .getBlockBlobClient(getUniqueName("blob")); + + // Neither the copy that creates the blob nor the one that overrides it + // should throw any errors + await blobWithSAS.syncUploadFromURL(sourceUrl); + await blobWithSAS.syncUploadFromURL(sourceUrl); + }); + + it("Put blob from URL with create permission in account SAS should create but not override a blob @loki @sql", async () => { + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + const containerName = getUniqueName("container"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const sourceBlob = containerClient.getBlockBlobClient( + getUniqueName("blob") + ); + await sourceBlob.upload("hello", 5); + const sourceUrl = await sourceBlob.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: tmr + }); + + const serviceClientWithSAS = new BlobServiceClient( + serviceClient.generateAccountSasUrl( + tmr, + AccountSASPermissions.parse("c"), + "co" + ), + newPipeline(new AnonymousCredential(), { + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + const blobWithSAS = serviceClientWithSAS + .getContainerClient(containerName) + .getBlockBlobClient(getUniqueName("blob")); + + // this copy creates the blob and should not throw any errors + await blobWithSAS.syncUploadFromURL(sourceUrl); + + // overriding it needs Write, so this copy should throw 403 error + let error; + try { + await blobWithSAS.syncUploadFromURL(sourceUrl); + } catch (err) { + error = err; + } + assert.ok(error !== undefined); + assert.deepEqual(error.statusCode, 403); + assert.deepEqual(error.code, "AuthorizationPermissionMismatch"); + }); + + it("Put blob from URL without create or write permission in account SAS should fail @loki @sql", async () => { + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + const containerName = getUniqueName("container"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const sourceBlob = containerClient.getBlockBlobClient( + getUniqueName("blob") + ); + await sourceBlob.upload("hello", 5); + const sourceUrl = await sourceBlob.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: tmr + }); + + const serviceClientWithSAS = new BlobServiceClient( + serviceClient.generateAccountSasUrl( + tmr, + AccountSASPermissions.parse("r"), + "co" + ), + newPipeline(new AnonymousCredential(), { + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + const blobWithSAS = serviceClientWithSAS + .getContainerClient(containerName) + .getBlockBlobClient(getUniqueName("blob")); + + // Read alone grants neither creating the blob nor overriding it, so this + // copy should throw 403 error + let error; + try { + await blobWithSAS.syncUploadFromURL(sourceUrl); + } catch (err) { + error = err; + } + assert.ok(error !== undefined); + assert.deepEqual(error.statusCode, 403); + assert.deepEqual(error.code, "AuthorizationPermissionMismatch"); + }); + + it("Put blob from URL setting tags needs the tag permission in account SAS @loki @sql", async () => { + // Azure holds a request that sets tags on the destination, whether from + // x-ms-tags or by copying the source's, to the Set Blob Tags permission + // on top of the write. The source SAS carries Tag so that only the + // destination SAS decides the outcome. + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + const containerName = getUniqueName("container"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const sourceBlob = containerClient.getBlockBlobClient( + getUniqueName("blob") + ); + await sourceBlob.upload("hello", 5); + await sourceBlob.setTags({ origin: "source" }); + const sourceUrl = await sourceBlob.generateSasUrl({ + permissions: BlobSASPermissions.parse("rt"), + expiresOn: tmr + }); + const blobName = getUniqueName("blob"); + const tags = { key: "value" }; + const destination = (permissions: string) => + new BlobServiceClient( + serviceClient.generateAccountSasUrl( + tmr, + AccountSASPermissions.parse(permissions), + "co" + ), + newPipeline(new AnonymousCredential(), { + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ) + .getContainerClient(containerName) + .getBlockBlobClient(blobName); + + // Write alone does not set tags, so both copies should throw 403 error + const writeOnly = await destination("w"); + for (const options of [{ tags }, { copySourceTags: "COPY" as const }]) { + let error; + try { + await writeOnly.syncUploadFromURL(sourceUrl, options); + } catch (err) { + error = err; + } + assert.ok(error !== undefined); + assert.deepEqual(error.statusCode, 403); + assert.deepEqual(error.code, "AuthorizationPermissionMismatch"); + } + + // With Tag as well, the request's tags are set on the destination and + // the source's tags can be copied to it + const writeAndTag = await destination("wt"); + await writeAndTag.syncUploadFromURL(sourceUrl, { tags }); + assert.deepStrictEqual( + (await containerClient.getBlockBlobClient(blobName).getTags()).tags, + tags + ); + await writeAndTag.syncUploadFromURL(sourceUrl, { copySourceTags: "COPY" }); + assert.deepStrictEqual( + (await containerClient.getBlockBlobClient(blobName).getTags()).tags, + { origin: "source" } + ); + }); + it("Copy blob should work with write permission in account SAS to override an existing blob @loki", async () => { const tmr = new Date(); tmr.setDate(tmr.getDate() + 1); @@ -1688,6 +1889,356 @@ describe("Shared Access Signature (SAS) authentication", () => { await blob2SAS.syncCopyFromURL(blob1.url); }); + it("Put blob from URL with write permission in container SAS should create and override a blob @loki @sql", async () => { + // Azure grants Put Blob From URL on a new block blob to Create (c) or + // Write (w), and on an existing one to Write (w) alone, the split Put + // Blob has. Write alone therefore creates as well as overrides. + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + const containerName = getUniqueName("container"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const sourceBlob = containerClient.getBlockBlobClient( + getUniqueName("blob") + ); + await sourceBlob.upload("hello", 5); + const sourceUrl = await sourceBlob.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: tmr + }); + + const containerClientWithSAS = new ContainerClient( + await containerClient.generateSasUrl({ + permissions: ContainerSASPermissions.parse("w"), + expiresOn: tmr + }), + newPipeline(new AnonymousCredential()) + ); + const blobWithSAS = containerClientWithSAS.getBlockBlobClient( + getUniqueName("blob") + ); + + // Neither the copy that creates the blob nor the one that overrides it + // should throw any errors + await blobWithSAS.syncUploadFromURL(sourceUrl); + await blobWithSAS.syncUploadFromURL(sourceUrl); + }); + + it("Put blob from URL with create permission in container SAS should create but not override a blob @loki @sql", async () => { + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + const containerName = getUniqueName("container"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const sourceBlob = containerClient.getBlockBlobClient( + getUniqueName("blob") + ); + await sourceBlob.upload("hello", 5); + const sourceUrl = await sourceBlob.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: tmr + }); + + const containerClientWithSAS = new ContainerClient( + await containerClient.generateSasUrl({ + permissions: ContainerSASPermissions.parse("c"), + expiresOn: tmr + }), + newPipeline(new AnonymousCredential()) + ); + const blobWithSAS = containerClientWithSAS.getBlockBlobClient( + getUniqueName("blob") + ); + + // this copy creates the blob and should not throw any errors + await blobWithSAS.syncUploadFromURL(sourceUrl); + + // overriding it needs Write, so this copy should throw 403 error + let error; + try { + await blobWithSAS.syncUploadFromURL(sourceUrl); + } catch (err) { + error = err; + } + assert.ok(error !== undefined); + assert.deepEqual(error.statusCode, 403); + assert.deepEqual(error.code, "AuthorizationPermissionMismatch"); + }); + + it("Put blob from URL without create or write permission in container SAS should fail @loki @sql", async () => { + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + const containerName = getUniqueName("container"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const sourceBlob = containerClient.getBlockBlobClient( + getUniqueName("blob") + ); + await sourceBlob.upload("hello", 5); + const sourceUrl = await sourceBlob.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: tmr + }); + + const containerClientWithSAS = new ContainerClient( + await containerClient.generateSasUrl({ + permissions: ContainerSASPermissions.parse("r"), + expiresOn: tmr + }), + newPipeline(new AnonymousCredential()) + ); + const blobWithSAS = containerClientWithSAS.getBlockBlobClient(getUniqueName("blob")); + + // Read alone grants neither creating the blob nor overriding it, so this + // copy should throw 403 error + let error; + try { + await blobWithSAS.syncUploadFromURL(sourceUrl); + } catch (err) { + error = err; + } + assert.ok(error !== undefined); + assert.deepEqual(error.statusCode, 403); + assert.deepEqual(error.code, "AuthorizationPermissionMismatch"); + }); + + it("Put blob from URL with write permission in blob SAS should create and override a blob @loki @sql", async () => { + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + const containerName = getUniqueName("container"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const sourceBlob = containerClient.getBlockBlobClient( + getUniqueName("blob") + ); + await sourceBlob.upload("hello", 5); + const sourceUrl = await sourceBlob.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: tmr + }); + + const blobWithSAS = new BlockBlobClient( + await containerClient.getBlockBlobClient(getUniqueName("blob")).generateSasUrl({ + permissions: BlobSASPermissions.parse("w"), + expiresOn: tmr + }), + newPipeline(new AnonymousCredential()) + ); + + // Neither the copy that creates the blob nor the one that overrides it + // should throw any errors + await blobWithSAS.syncUploadFromURL(sourceUrl); + await blobWithSAS.syncUploadFromURL(sourceUrl); + }); + + it("Put blob from URL with create permission in blob SAS should create but not override a blob @loki @sql", async () => { + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + const containerName = getUniqueName("container"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const sourceBlob = containerClient.getBlockBlobClient( + getUniqueName("blob") + ); + await sourceBlob.upload("hello", 5); + const sourceUrl = await sourceBlob.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: tmr + }); + + const blobWithSAS = new BlockBlobClient( + await containerClient.getBlockBlobClient(getUniqueName("blob")).generateSasUrl({ + permissions: BlobSASPermissions.parse("c"), + expiresOn: tmr + }), + newPipeline(new AnonymousCredential()) + ); + + // this copy creates the blob and should not throw any errors + await blobWithSAS.syncUploadFromURL(sourceUrl); + + // overriding it needs Write, so this copy should throw 403 error + let error; + try { + await blobWithSAS.syncUploadFromURL(sourceUrl); + } catch (err) { + error = err; + } + assert.ok(error !== undefined); + assert.deepEqual(error.statusCode, 403); + assert.deepEqual(error.code, "AuthorizationPermissionMismatch"); + }); + + it("Put blob from URL without create or write permission in blob SAS should fail @loki @sql", async () => { + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + const containerName = getUniqueName("container"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const sourceBlob = containerClient.getBlockBlobClient( + getUniqueName("blob") + ); + await sourceBlob.upload("hello", 5); + const sourceUrl = await sourceBlob.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: tmr + }); + + const blobWithSAS = new BlockBlobClient( + await containerClient.getBlockBlobClient(getUniqueName("blob")).generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: tmr + }), + newPipeline(new AnonymousCredential()) + ); + + // Read alone grants neither creating the blob nor overriding it, so this + // copy should throw 403 error + let error; + try { + await blobWithSAS.syncUploadFromURL(sourceUrl); + } catch (err) { + error = err; + } + assert.ok(error !== undefined); + assert.deepEqual(error.statusCode, 403); + assert.deepEqual(error.code, "AuthorizationPermissionMismatch"); + }); + + it("Put blob from URL setting tags needs the tag permission in container SAS @loki @sql", async () => { + // Azure holds a request that sets tags on the destination, whether from + // x-ms-tags or by copying the source's, to the Set Blob Tags permission + // on top of the write. The source SAS carries Tag so that only the + // destination SAS decides the outcome. + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + const containerName = getUniqueName("container"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const sourceBlob = containerClient.getBlockBlobClient( + getUniqueName("blob") + ); + await sourceBlob.upload("hello", 5); + await sourceBlob.setTags({ origin: "source" }); + const sourceUrl = await sourceBlob.generateSasUrl({ + permissions: BlobSASPermissions.parse("rt"), + expiresOn: tmr + }); + const blobName = getUniqueName("blob"); + const tags = { key: "value" }; + const destination = async (permissions: string) => + new ContainerClient( + await containerClient.generateSasUrl({ + permissions: ContainerSASPermissions.parse(permissions), + expiresOn: tmr + }), + newPipeline(new AnonymousCredential()) + ).getBlockBlobClient(blobName); + + // Write alone does not set tags, so both copies should throw 403 error + const writeOnly = await destination("w"); + for (const options of [{ tags }, { copySourceTags: "COPY" as const }]) { + let error; + try { + await writeOnly.syncUploadFromURL(sourceUrl, options); + } catch (err) { + error = err; + } + assert.ok(error !== undefined); + assert.deepEqual(error.statusCode, 403); + assert.deepEqual(error.code, "AuthorizationPermissionMismatch"); + } + + // With Tag as well, the request's tags are set on the destination and + // the source's tags can be copied to it + const writeAndTag = await destination("wt"); + await writeAndTag.syncUploadFromURL(sourceUrl, { tags }); + assert.deepStrictEqual( + (await containerClient.getBlockBlobClient(blobName).getTags()).tags, + tags + ); + await writeAndTag.syncUploadFromURL(sourceUrl, { copySourceTags: "COPY" }); + assert.deepStrictEqual( + (await containerClient.getBlockBlobClient(blobName).getTags()).tags, + { origin: "source" } + ); + }); + + it("Put blob from URL setting tags needs the tag permission in blob SAS @loki @sql", async () => { + // Azure holds a request that sets tags on the destination, whether from + // x-ms-tags or by copying the source's, to the Set Blob Tags permission + // on top of the write. The source SAS carries Tag so that only the + // destination SAS decides the outcome. + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + const containerName = getUniqueName("container"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const sourceBlob = containerClient.getBlockBlobClient( + getUniqueName("blob") + ); + await sourceBlob.upload("hello", 5); + await sourceBlob.setTags({ origin: "source" }); + const sourceUrl = await sourceBlob.generateSasUrl({ + permissions: BlobSASPermissions.parse("rt"), + expiresOn: tmr + }); + const blobName = getUniqueName("blob"); + const tags = { key: "value" }; + const destination = async (permissions: string) => + new BlockBlobClient( + await containerClient.getBlockBlobClient(blobName).generateSasUrl({ + permissions: BlobSASPermissions.parse(permissions), + expiresOn: tmr + }), + newPipeline(new AnonymousCredential()) + ); + + // Write alone does not set tags, so both copies should throw 403 error + const writeOnly = await destination("w"); + for (const options of [{ tags }, { copySourceTags: "COPY" as const }]) { + let error; + try { + await writeOnly.syncUploadFromURL(sourceUrl, options); + } catch (err) { + error = err; + } + assert.ok(error !== undefined); + assert.deepEqual(error.statusCode, 403); + assert.deepEqual(error.code, "AuthorizationPermissionMismatch"); + } + + // With Tag as well, the request's tags are set on the destination and + // the source's tags can be copied to it + const writeAndTag = await destination("wt"); + await writeAndTag.syncUploadFromURL(sourceUrl, { tags }); + assert.deepStrictEqual( + (await containerClient.getBlockBlobClient(blobName).getTags()).tags, + tags + ); + await writeAndTag.syncUploadFromURL(sourceUrl, { copySourceTags: "COPY" }); + assert.deepStrictEqual( + (await containerClient.getBlockBlobClient(blobName).getTags()).tags, + { origin: "source" } + ); + }); + it("Copy blob should work with write permission in blob SAS to override an existing blob @loki @sql", async () => { const now = new Date(); now.setMinutes(now.getMinutes() - 5); // Skip clock skew with server