From adc07da56d3176103b571c6ea3c0fe980d9df58d Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Tue, 25 Aug 2026 09:06:53 -0700 Subject: [PATCH 1/9] Implement Put Blob From URL Put Blob From URL answered 501, so a client copying a blob in one request had to fall back to the asynchronous Copy Blob and then wait out a copy it had asked to have done by the time it was answered. Read the source the way Put Block From URL already does -- a loopback self-request pinned to the address and port this server is bound to, carrying the caller's path, query and source conditions -- and commit what comes back as a new block blob. The two operations now share that fetch, so authentication, conditions, and the refusal to decompress a source that merely declares an encoding are answered the same way for both, in one place. What the destination ends up with follows the operation's contract. The source's standard properties are copied unless x-ms-copy-source-blob-properties says otherwise, and a blob content header on the request sets that one property either way. Metadata answers to its own rule: named on the request it replaces the source's, named nowhere it is copied. Tags are the request's, or the source's under x-ms-copy-source-tag-option: COPY, which reads them over the same authorized path the content came over -- the extra Get Blob Tags call against the source that the operation is documented to make, so a URL allowed to read the source but not its tags is refused rather than obeyed. Asking for both at once is refused as it already is for Copy Blob From URL. The operation writes a blob, so it joins Put Blob in the shared access signature tables and in the rule that an existing destination demands write permission; an operation missing from those tables fails the request with an internal error rather than a permission one. Co-Authored-By: Claude Opus 5 (1M context) --- ChangeLog.md | 1 + README.md | 2 +- .../authentication/AccountSASAuthenticator.ts | 1 + .../authentication/BlobSASAuthenticator.ts | 1 + .../OperationAccountSASPermission.ts | 10 + .../OperationBlobSASPermission.ts | 14 + src/blob/handlers/BlockBlobHandler.ts | 598 ++++++++++++++---- tests/blob/apis/blockblob.test.ts | 503 +++++++++++++++ 8 files changed, 990 insertions(+), 140 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index 41a9fb7da..8a8085ff6 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -28,6 +28,7 @@ General: Blob: - Fixed blob operations hanging when a client disconnects before the operation queue processes the request. (issue #2575) +- Implement `PutBlobFromUrl` (`Put Blob From URL`), which previously returned 501. The source is fetched over loopback, as `PutBlockFromURL` already does, so that SAS authentication and the `x-ms-source-if-*` conditions are enforced by the existing download path. Standard blob properties are copied from the source unless `x-ms-copy-source-blob-properties` is false, request blob content headers override them either way, request metadata replaces the source's rather than adding to it, and `x-ms-copy-source-tag-option: COPY` reads the source's tags over that same authorized path. As with `CopyBlobFromURL`, only sources on the same Azurite instance are supported. Table: diff --git a/README.md b/README.md index 5930392f5..35366126a 100644 --- a/README.md +++ b/README.md @@ -1086,6 +1086,7 @@ Detailed support matrix: - Abort Copy Blob (Only supports copy within same Azurite instance) - Copy Blob From URL (Only supports copy within same Azurite instance, only on Loki) - Put Block From URL (Only supports source within same Azurite instance) + - Put Blob From URL (Only supports source within same Azurite instance) - Access control based on conditional headers - Following features or REST APIs are NOT supported or limited supported in this release (will support more features per customers feedback in future releases) - SharedKey Lite @@ -1099,7 +1100,6 @@ Detailed support matrix: - Concurrent Append - Blob Expiry - Object Replication Service - - Put Blob From URL - Version Level Worm - Sync copy blob by access source with oauth - Encryption Scope diff --git a/src/blob/authentication/AccountSASAuthenticator.ts b/src/blob/authentication/AccountSASAuthenticator.ts index 48b850b07..7cbb4247a 100644 --- a/src/blob/authentication/AccountSASAuthenticator.ts +++ b/src/blob/authentication/AccountSASAuthenticator.ts @@ -256,6 +256,7 @@ export default class AccountSASAuthenticator implements IAuthenticator { // If copy destination blob exists, then permission must be Write only if ( operation === Operation.BlockBlob_Upload || + operation === Operation.BlockBlob_PutBlobFromUrl || operation === Operation.PageBlob_Create || operation === Operation.AppendBlob_Create || operation === Operation.Blob_StartCopyFromURL || diff --git a/src/blob/authentication/BlobSASAuthenticator.ts b/src/blob/authentication/BlobSASAuthenticator.ts index ac848b0f8..e0d575b61 100644 --- a/src/blob/authentication/BlobSASAuthenticator.ts +++ b/src/blob/authentication/BlobSASAuthenticator.ts @@ -418,6 +418,7 @@ export default class BlobSASAuthenticator implements IAuthenticator { // If copy destination blob exists, then permission must be Write only if ( operation === Operation.BlockBlob_Upload || + operation === Operation.BlockBlob_PutBlobFromUrl || operation === Operation.PageBlob_Create || operation === Operation.AppendBlob_Create || operation === Operation.Blob_StartCopyFromURL || diff --git a/src/blob/authentication/OperationAccountSASPermission.ts b/src/blob/authentication/OperationAccountSASPermission.ts index 56cd123f8..979668078 100644 --- a/src/blob/authentication/OperationAccountSASPermission.ts +++ b/src/blob/authentication/OperationAccountSASPermission.ts @@ -363,6 +363,16 @@ OPERATION_ACCOUNT_SAS_PERMISSIONS.set( ) ); +OPERATION_ACCOUNT_SAS_PERMISSIONS.set( + Operation.BlockBlob_PutBlobFromUrl, + new OperationAccountSASPermission( + AccountSASService.Blob, + AccountSASResourceType.Object, + // Create permission is only available for nonexistent block blob. Handle this scenario separately + AccountSASPermission.Write + AccountSASPermission.Create + ) +); + OPERATION_ACCOUNT_SAS_PERMISSIONS.set( Operation.PageBlob_Create, new OperationAccountSASPermission( diff --git a/src/blob/authentication/OperationBlobSASPermission.ts b/src/blob/authentication/OperationBlobSASPermission.ts index ddf21c746..cced5f88f 100644 --- a/src/blob/authentication/OperationBlobSASPermission.ts +++ b/src/blob/authentication/OperationBlobSASPermission.ts @@ -258,6 +258,13 @@ OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( BlobSASPermission.Write + BlobSASPermission.Create ) ); +OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( + Operation.BlockBlob_PutBlobFromUrl, + // TODO: When destination blob doesn't exist, needs create permission + new OperationBlobSASPermission( + BlobSASPermission.Write + BlobSASPermission.Create + ) +); OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( Operation.BlockBlob_StageBlock, new OperationBlobSASPermission(BlobSASPermission.Write) @@ -522,6 +529,13 @@ OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( BlobSASPermission.Write + BlobSASPermission.Create ) ); +OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( + Operation.BlockBlob_PutBlobFromUrl, + // Create a new blob, must be write + new OperationBlobSASPermission( + BlobSASPermission.Write + BlobSASPermission.Create + ) +); OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( Operation.BlockBlob_StageBlock, new OperationBlobSASPermission(BlobSASPermission.Write) diff --git a/src/blob/handlers/BlockBlobHandler.ts b/src/blob/handlers/BlockBlobHandler.ts index 3886564e2..6e716076b 100644 --- a/src/blob/handlers/BlockBlobHandler.ts +++ b/src/blob/handlers/BlockBlobHandler.ts @@ -12,7 +12,6 @@ import { newEtag } from "../../common/utils/utils"; import BlobStorageContext from "../context/BlobStorageContext"; -import NotImplementedError from "../errors/NotImplementedError"; import StorageErrorFactory from "../errors/StorageErrorFactory"; import * as Models from "../generated/artifacts/models"; import Context from "../generated/Context"; @@ -208,9 +207,222 @@ export default class BlockBlobHandler return response; } - public async putBlobFromUrl(contentLength: number, copySource: string, options: Models.BlockBlobPutBlobFromUrlOptionalParams, context: Context + public async putBlobFromUrl( + contentLength: number, + copySource: string, + options: Models.BlockBlobPutBlobFromUrlOptionalParams, + context: Context ): Promise { - throw new NotImplementedError(context.contextId); + const blobCtx = new BlobStorageContext(context); + const accountName = blobCtx.account!; + const containerName = blobCtx.container!; + const blobName = blobCtx.blob!; + const date = blobCtx.startTime!; + const etag = newEtag(); + + // Put Blob From URL carries no request body. + if (contentLength !== 0) { + throw StorageErrorFactory.getInvalidHeaderValue(context.contextId, { + HeaderName: "Content-Length", + HeaderValue: contentLength.toString() + }); + } + + // Reject a malformed source checksum before fetching anything. The + // shared validator would catch it too, but only once the source had + // already been read. + if ( + options.sourceContentMD5 !== undefined && + options.sourceContentMD5.length !== 16 + ) { + throw StorageErrorFactory.getInvalidMd5(context.contextId); + } + + // The destination's tags are either the source's or the request's, never + // both. + const copySourceTags = + options.copySourceTags === Models.BlobCopySourceTags.COPY; + if (copySourceTags && options.blobTagsString !== undefined) { + throw StorageErrorFactory.getBothUserTagsAndSourceTagsCopyPresentException( + context.contextId! + ); + } + + await this.metadataStore.checkContainerExist( + context, + accountName, + containerName + ); + + // Put Blob From URL always copies the whole source, so no range rides + // along with the conditions. + const sourceResponse = await this.readCopySource( + context, + "putBlobFromUrl", + copySource, + BlockBlobHandler.sourceConditionHeaders( + options.sourceModifiedAccessConditions + ) + ); + + // The status was only the response headers arriving; the body can still + // fail midway (socket error, connection reset). Map that to the same + // error as a transport failure rather than letting it escape as a + // bodiless 500, and release the source stream on the way out. + let persistency: IExtentChunk; + try { + persistency = await this.extentStore.appendExtent( + sourceResponse.data, + context.contextId + ); + } catch (err) { + sourceResponse.data.destroy(); + this.logger.error( + `BlockBlobHandler:putBlobFromUrl() Failed to read the copy source body: ${err}`, + context.contextId + ); + throw StorageErrorFactory.getCannotVerifyCopySource( + context.contextId!, + 500, + "Could not verify the copy source within the specified time." + ); + } + + // The response always echoes an MD5 of what was copied, so it is always + // computed. x-ms-source-content-md5 is this operation's integrity check + // over the bytes that arrived; x-ms-blob-content-md5 gets the same + // treatment Put Blob gives it, since Put Blob From URL follows Put Blob + // for the custom properties. Destroy the stream regardless, so a + // mismatch cannot leave the extent handle open. + const stream = await this.extentStore.readExtent( + persistency, + context.contextId + ); + let calculatedContentMD5: Uint8Array | undefined; + try { + ({ md5: calculatedContentMD5 } = + await computeAndValidateTransactionalChecksums( + stream, + { + md5: + options.sourceContentMD5 ?? + (options.blobHTTPHeaders || {}).blobContentMD5 + }, + context.contextId, + { md5: true } + )); + } finally { + (stream as Readable).destroy?.(); + } + + // COPY reads the source's tags over the same authorized path the content + // came over, so a source that the caller may read but not tag refuses + // the copy rather than leaking them. + const blobTags = copySourceTags + ? await this.readCopySourceTags(context, copySource) + : options.blobTagsString === undefined + ? undefined + : getTagsFromString(options.blobTagsString, context.contextId!); + + // The standard properties are copied from the source unless the request + // turns that off, and a blob content header on the request sets that one + // property either way. The request's own Content-Type is only a + // fallback: a client sends one on a bodiless request without meaning to + // retype the copy. + const copyProperties = options.copySourceBlobProperties !== false; + const sourceProperty = (name: string): string | undefined => + copyProperties ? sourceResponse.headers[name] : undefined; + const blobHTTPHeaders = options.blobHTTPHeaders || {}; + const contentType = + blobHTTPHeaders.blobContentType || + sourceProperty("content-type") || + context.request!.getHeader("content-type") || + "application/octet-stream"; + + // Metadata named on the request replaces the source's rather than adding + // to it, and naming none copies the source's. Both are read from raw + // headers, which preserve the case of the names. + const metadata = + convertRawHeadersToMetadata( + blobCtx.request!.getRawHeaders(), + context.contextId! + ) ?? + convertRawHeadersToMetadata( + (sourceResponse.data as IncomingMessage).rawHeaders, + context.contextId! + ); + + const blob: BlobModel = { + deleted: false, + metadata, + accountName, + containerName, + name: blobName, + properties: { + creationTime: date, + lastModified: date, + etag, + // The destination's length is the source's, not the Content-Length + // of this bodiless request. + contentLength: persistency.count, + contentType, + contentEncoding: + blobHTTPHeaders.blobContentEncoding || + sourceProperty("content-encoding"), + contentLanguage: + blobHTTPHeaders.blobContentLanguage || + sourceProperty("content-language"), + contentMD5: calculatedContentMD5, + contentDisposition: + blobHTTPHeaders.blobContentDisposition || + sourceProperty("content-disposition"), + cacheControl: + blobHTTPHeaders.blobCacheControl || sourceProperty("cache-control"), + blobType: Models.BlobType.BlockBlob, + leaseStatus: Models.LeaseStatusType.Unlocked, + leaseState: Models.LeaseStateType.Available, + serverEncrypted: true, + accessTier: Models.AccessTier.Hot, + accessTierInferred: true, + accessTierChangeTime: date + }, + snapshot: "", + isCommitted: true, + persistency, + blobTags + }; + + if (options.tier !== undefined) { + blob.properties.accessTier = this.parseTier(options.tier); + if (blob.properties.accessTier === undefined) { + throw StorageErrorFactory.getInvalidHeaderValue(context.contextId, { + HeaderName: "x-ms-access-tier", + HeaderValue: `${options.tier}` + }); + } + blob.properties.accessTierInferred = false; + } + + await this.metadataStore.createBlob( + context, + blob, + options.leaseAccessConditions, + options.modifiedAccessConditions + ); + + const response: Models.BlockBlobPutBlobFromUrlResponse = { + statusCode: 201, + eTag: etag, + lastModified: date, + contentMD5: blob.properties.contentMD5, + requestId: blobCtx.contextId, + version: BLOB_API_VERSION, + date, + isServerEncrypted: true, + clientRequestId: options.requestId + }; + + return response; } public async stageBlock( @@ -361,70 +573,7 @@ export default class BlockBlobHandler containerName ); - let url: URL; - try { - url = new URL(sourceUrl); - } catch { - throw StorageErrorFactory.getInvalidHeaderValue(context.contextId, { - HeaderName: "x-ms-copy-source", - HeaderValue: sourceUrl - }); - } - - // Only sources within the same Azurite instance are supported, as with - // copyFromURL. - // Hostnames compare case-insensitively and new URL() lowercases its - // host, so normalize the client-supplied header before comparing. - const currentServer = (blobCtx.request!.getHeader("Host") || "") - .toLowerCase(); - if (currentServer !== url.host) { - this.logger.error( - `BlockBlobHandler:stageBlockFromURL() Source ${url} is not on the same Azurite instance as target account ${accountName}`, - context.contextId - ); - throw StorageErrorFactory.getCannotVerifyCopySource( - context.contextId!, - 404, - "The specified resource does not exist" - ); - } - - // The Host header above is client-controlled, so never fetch the - // caller-supplied URL directly; pin the outbound request to the - // loopback address and port this server is actually bound to, keeping - // only the caller's path and query. - const rawRequest = blobCtx.request!.getBodyStream(); - if (!(rawRequest instanceof IncomingMessage) || - rawRequest.socket.localPort === undefined) { - throw StorageErrorFactory.getCannotVerifyCopySource( - context.contextId!, - 404, - "The specified resource does not exist" - ); - } - const scheme = "encrypted" in rawRequest.socket ? "https" : "http"; - // Use the local address this request arrived on rather than a - // hard-coded loopback so non-loopback --blobHost binds keep working; - // IPv6 literals need brackets in URLs. - const localAddress = rawRequest.socket.localAddress || "127.0.0.1"; - const localHost = localAddress.includes(":") ? - `[${localAddress}]` : localAddress; - const pinnedUrl = - `${scheme}://${localHost}:${rawRequest.socket.localPort}` + - `${url.pathname}${url.search}`; - - // Fetch the source range over loopback so that SAS authentication, - // range handling, and source conditions reuse the download path. - // Preserve the source URL's host so product-style source URLs still - // resolve their account from the Host header; the connection itself - // stays pinned to this server's bound address. - // A block must be staged as the bytes the source actually stores, so ask - // for the body verbatim rather than letting anything in the path apply - // transfer compression. - const headers: { [key: string]: string } = { - host: url.host, - "accept-encoding": "identity" - }; + const headers: { [key: string]: string } = {}; if (options.sourceRange !== undefined) { // The download path ignores malformed Range headers, which would // silently stage the entire source blob; reject them up front. @@ -441,83 +590,24 @@ export default class BlockBlobHandler } headers.range = options.sourceRange; } - const sourceConditions = options.sourceModifiedAccessConditions || {}; - if (sourceConditions.sourceIfMatch !== undefined) { - headers["if-match"] = sourceConditions.sourceIfMatch; - } - if (sourceConditions.sourceIfNoneMatch !== undefined) { - headers["if-none-match"] = sourceConditions.sourceIfNoneMatch; - } - if (sourceConditions.sourceIfModifiedSince !== undefined) { - headers["if-modified-since"] = new Date( - sourceConditions.sourceIfModifiedSince - ).toUTCString(); - } - if (sourceConditions.sourceIfUnmodifiedSince !== undefined) { - headers["if-unmodified-since"] = new Date( - sourceConditions.sourceIfUnmodifiedSince - ).toUTCString(); - } // Note: unlike the Copy Blob operations, Put Block From URL has no // x-ms-source-if-tags condition; the generated operation spec does not // deserialize one. + Object.assign( + headers, + BlockBlobHandler.sourceConditionHeaders( + options.sourceModifiedAccessConditions + ) + ); - let sourceResponse: AxiosResponse; - try { - sourceResponse = await axios.get(pinnedUrl, { - headers, - responseType: "stream", - validateStatus: () => true, - // Never decompress. A source blob carries Content-Encoding as a - // stored property, so the download echoes it back even though the - // bytes on the wire are the raw stored ones. Decompressing here - // would stage the decoded content instead of what the source holds, - // and would fail outright when the property does not match the - // bytes. - decompress: false, - // Pin trust to the certificate this server itself presents; see - // getLoopbackHttpsAgent(). - httpsAgent: scheme === "https" - ? getLoopbackHttpsAgent(rawRequest.socket as TLSSocket) - : undefined - }); - } catch (err) { - // Transport-level failures (TLS, connection reset, socket errors) throw - // rather than returning a status. Without this they would escape as a - // bodiless 500 instead of an Azure-shaped error. - this.logger.error( - `BlockBlobHandler:stageBlockFromURL() Failed to read the copy source: ${err}`, - context.contextId - ); - throw StorageErrorFactory.getCannotVerifyCopySource( - context.contextId!, - 500, - "Could not verify the copy source within the specified time." - ); - } - - if (sourceResponse.status === 304 || sourceResponse.status === 412) { - sourceResponse.data.destroy(); - throw StorageErrorFactory.getSourceConditionNotMet(context.contextId!); - } - if (sourceResponse.status === 404) { - sourceResponse.data.destroy(); - throw StorageErrorFactory.getCannotVerifyCopySource( - context.contextId!, - 404, - "The specified resource does not exist" - ); - } - if (sourceResponse.status !== 200 && sourceResponse.status !== 206) { - sourceResponse.data.destroy(); - throw StorageErrorFactory.getCannotVerifyCopySource( - context.contextId!, - sourceResponse.status, - "Could not verify the copy source within the specified time." - ); - } + const sourceResponse = await this.readCopySource( + context, + "stageBlockFromURL", + sourceUrl, + headers + ); - // The status above only means the response headers arrived; the body can + // The status was only the response headers arriving; the body can // still fail midway (socket error, connection reset). Map that to the // same error as a transport failure rather than letting it escape as a // bodiless 500, and release the source stream on the way out. @@ -829,4 +919,234 @@ export default class BlockBlobHandler ); } } + + /** + * Restate the conditions a copy request names for its source as the + * conditional headers of a read, so that the download path answers them + * the way it answers a client reading the source itself. + * + * x-ms-if-tags only ever appears for Put Blob From URL: the Put Block From + * URL specification deserializes no source tag condition. + * + * @private + * @param {Models.SourceModifiedAccessConditions} [conditions] + * @returns {{ [key: string]: string }} + * @memberof BlockBlobHandler + */ + private static sourceConditionHeaders( + conditions: Models.SourceModifiedAccessConditions = {} + ): { [key: string]: string } { + const headers: { [key: string]: string } = {}; + if (conditions.sourceIfMatch !== undefined) { + headers["if-match"] = conditions.sourceIfMatch; + } + if (conditions.sourceIfNoneMatch !== undefined) { + headers["if-none-match"] = conditions.sourceIfNoneMatch; + } + if (conditions.sourceIfModifiedSince !== undefined) { + headers["if-modified-since"] = new Date( + conditions.sourceIfModifiedSince + ).toUTCString(); + } + if (conditions.sourceIfUnmodifiedSince !== undefined) { + headers["if-unmodified-since"] = new Date( + conditions.sourceIfUnmodifiedSince + ).toUTCString(); + } + if (conditions.sourceIfTags !== undefined) { + headers["x-ms-if-tags"] = conditions.sourceIfTags; + } + return headers; + } + + /** + * Read a copy source with a loopback self-request, so that SAS + * authentication, ranges, and source conditions are answered by the + * download path rather than reimplemented against the store. + * + * Only sources within the same Azurite instance are supported, as with + * copyFromURL. The Host header that decides this is the caller's to + * choose, so the request is never made to the URL they supplied: it is + * pinned to the address and port this server is bound to and keeps only + * their path and query, with the source's own host along as a header so + * that product-style source URLs still resolve their account from it. + * + * @private + * @param {Context} context + * @param {string} operation Handler method name, for log messages + * @param {string} copySource The source URL the request named + * @param {{ [key: string]: string }} headers Conditions, ranges + * @param {string} [subresource] A query to append, such as "comp=tags" + * @returns {Promise} A response whose body is a stream + * @memberof BlockBlobHandler + */ + private async readCopySource( + context: Context, + operation: string, + copySource: string, + headers: { [key: string]: string }, + subresource?: string + ): Promise { + const blobCtx = new BlobStorageContext(context); + + let url: URL; + try { + url = new URL(copySource); + } catch { + throw StorageErrorFactory.getInvalidHeaderValue(context.contextId, { + HeaderName: "x-ms-copy-source", + HeaderValue: copySource + }); + } + + // Hostnames compare case-insensitively and new URL() lowercases its + // host, so normalize the client-supplied header before comparing. + const currentServer = (blobCtx.request!.getHeader("Host") || "") + .toLowerCase(); + if (currentServer !== url.host) { + this.logger.error( + `BlockBlobHandler:${operation}() Source ${url} is not on the same Azurite instance as target account ${blobCtx.account}`, + context.contextId + ); + throw StorageErrorFactory.getCannotVerifyCopySource( + context.contextId!, + 404, + "The specified resource does not exist" + ); + } + + const rawRequest = blobCtx.request!.getBodyStream(); + if (!(rawRequest instanceof IncomingMessage) || + rawRequest.socket.localPort === undefined) { + throw StorageErrorFactory.getCannotVerifyCopySource( + context.contextId!, + 404, + "The specified resource does not exist" + ); + } + const scheme = "encrypted" in rawRequest.socket ? "https" : "http"; + // Use the local address this request arrived on rather than a + // hard-coded loopback so non-loopback --blobHost binds keep working; + // IPv6 literals need brackets in URLs. + const localAddress = rawRequest.socket.localAddress || "127.0.0.1"; + const localHost = localAddress.includes(":") ? + `[${localAddress}]` : localAddress; + // Append rather than rebuild the query: a shared access signature signs + // the exact encoding it arrived in, which re-encoding could disturb. + const query = subresource === undefined + ? url.search + : `${url.search}${url.search === "" ? "?" : "&"}${subresource}`; + const pinnedUrl = + `${scheme}://${localHost}:${rawRequest.socket.localPort}` + + `${url.pathname}${query}`; + + let sourceResponse: AxiosResponse; + try { + sourceResponse = await axios.get(pinnedUrl, { + headers: { + host: url.host, + // A copy must carry the bytes the source actually stores, so ask + // for the body verbatim rather than letting anything in the path + // apply transfer compression. + "accept-encoding": "identity", + ...headers + }, + responseType: "stream", + validateStatus: () => true, + // Never decompress. A source blob carries Content-Encoding as a + // stored property, so the download echoes it back even though the + // bytes on the wire are the raw stored ones. Decompressing here + // would copy the decoded content instead of what the source holds, + // and would fail outright when the property does not match the + // bytes. + decompress: false, + // Pin trust to the certificate this server itself presents; see + // getLoopbackHttpsAgent(). + httpsAgent: scheme === "https" + ? getLoopbackHttpsAgent(rawRequest.socket as TLSSocket) + : undefined + }); + } catch (err) { + // Transport-level failures (TLS, connection reset, socket errors) throw + // rather than returning a status. Without this they would escape as a + // bodiless 500 instead of an Azure-shaped error. + this.logger.error( + `BlockBlobHandler:${operation}() Failed to read the copy source: ${err}`, + context.contextId + ); + throw StorageErrorFactory.getCannotVerifyCopySource( + context.contextId!, + 500, + "Could not verify the copy source within the specified time." + ); + } + + if (sourceResponse.status === 304 || sourceResponse.status === 412) { + sourceResponse.data.destroy(); + throw StorageErrorFactory.getSourceConditionNotMet(context.contextId!); + } + if (sourceResponse.status === 404) { + sourceResponse.data.destroy(); + throw StorageErrorFactory.getCannotVerifyCopySource( + context.contextId!, + 404, + "The specified resource does not exist" + ); + } + if (sourceResponse.status !== 200 && sourceResponse.status !== 206) { + sourceResponse.data.destroy(); + throw StorageErrorFactory.getCannotVerifyCopySource( + context.contextId!, + sourceResponse.status, + "Could not verify the copy source within the specified time." + ); + } + + return sourceResponse; + } + + /** + * Read the tags of a copy source, for the copy that asks to carry them + * over. Real Azure charges this to the caller as its own Get Blob Tags + * request against the source, and so does this: the authorization the + * source URL carries has to allow reading them. + * + * @private + * @param {Context} context + * @param {string} copySource The source URL the request named + * @returns {Promise} + * @memberof BlockBlobHandler + */ + private async readCopySourceTags( + context: Context, + copySource: string + ): Promise { + const response = await this.readCopySource( + context, + "putBlobFromUrl", + copySource, + {}, + "comp=tags" + ); + + const chunks: Buffer[] = []; + for await (const chunk of response.data as IncomingMessage) { + chunks.push(Buffer.from(chunk)); + } + const parsed = await parseXML(Buffer.concat(chunks).toString()); + + // parseXML collapses a single element out of its array, and leaves a + // tagless source with no TagSet at all. + const tagSet = parsed.TagSet; + if (tagSet === undefined || tagSet === "" || tagSet.Tag === undefined) { + return undefined; + } + const tags = Array.isArray(tagSet.Tag) ? tagSet.Tag : [tagSet.Tag]; + return { + blobTagSet: tags.map((tag: { Key: string; Value: string }) => ({ + key: tag.Key, + value: tag.Value + })) + }; + } } diff --git a/tests/blob/apis/blockblob.test.ts b/tests/blob/apis/blockblob.test.ts index 1cd19661d..212accf12 100644 --- a/tests/blob/apis/blockblob.test.ts +++ b/tests/blob/apis/blockblob.test.ts @@ -1035,6 +1035,509 @@ describe("BlockBlobAPIs", () => { assert.fail("Did not throw an exception."); }); + it("putBlobFromUrl copies the source blob @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const result = await blockBlobClient.syncUploadFromURL(sourceUrl); + // The response echoes the MD5 the service computed over what it copied. + assert.deepStrictEqual( + Buffer.from(result.contentMD5!), + Buffer.from(await getMD5FromString(content)) + ); + + const download = await blobClient.download(0); + assert.equal(await bodyToString(download, content.length), content); + // The destination's length is the source's, not the Content-Length of + // the bodiless request that created it. + assert.equal(download.contentLength, content.length); + }); + + it("putBlobFromUrl overwrites an existing destination @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + await blockBlobClient.upload("overwritten", "overwritten".length); + + await blockBlobClient.syncUploadFromURL(sourceUrl); + + const download = await blobClient.download(0); + assert.equal(await bodyToString(download, content.length), content); + }); + + it("putBlobFromUrl copies the source's properties and metadata @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const properties = { + blobCacheControl: "max-age=3600", + blobContentDisposition: "attachment; filename=source.txt", + blobContentEncoding: "identity", + blobContentLanguage: "en", + blobContentType: "text/plain" + }; + const metadata = { keya: "vala", keyb: "valb" }; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length, { + blobHTTPHeaders: properties, + metadata + }); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + await blockBlobClient.syncUploadFromURL(sourceUrl); + + const download = await blobClient.download(0); + assert.equal(download.cacheControl, properties.blobCacheControl); + assert.equal( + download.contentDisposition, + properties.blobContentDisposition + ); + assert.equal(download.contentEncoding, properties.blobContentEncoding); + assert.equal(download.contentLanguage, properties.blobContentLanguage); + assert.equal(download.contentType, properties.blobContentType); + assert.deepStrictEqual(download.metadata, metadata); + }); + + it("putBlobFromUrl with copySourceBlobProperties false leaves the source's properties @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const metadata = { keya: "vala" }; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length, { + blobHTTPHeaders: { + blobCacheControl: "max-age=3600", + blobContentDisposition: "attachment; filename=source.txt", + blobContentEncoding: "identity", + blobContentLanguage: "en", + blobContentType: "text/plain" + }, + metadata + }); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + await blockBlobClient.syncUploadFromURL(sourceUrl, { + copySourceBlobProperties: false + }); + + const download = await blobClient.download(0); + assert.equal(download.cacheControl, undefined); + assert.equal(download.contentDisposition, undefined); + assert.equal(download.contentEncoding, undefined); + assert.equal(download.contentLanguage, undefined); + assert.equal(download.contentType, "application/octet-stream"); + // Metadata answers to its own rule rather than to this header: the + // request named none, so the source's carries over. + assert.deepStrictEqual(download.metadata, metadata); + }); + + it("putBlobFromUrl request headers override the copied properties @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length, { + blobHTTPHeaders: { + blobCacheControl: "max-age=3600", + blobContentType: "text/plain" + } + }); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + await blockBlobClient.syncUploadFromURL(sourceUrl, { + blobHTTPHeaders: { blobContentType: "application/json" } + }); + + const download = await blobClient.download(0); + assert.equal(download.contentType, "application/json"); + // A property the request did not name still comes from the source. + assert.equal(download.cacheControl, "max-age=3600"); + }); + + it("putBlobFromUrl metadata replaces the source's @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length, { + metadata: { keya: "vala" } + }); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + await blockBlobClient.syncUploadFromURL(sourceUrl, { + metadata: { keyc: "valc" } + }); + + const download = await blobClient.download(0); + assert.deepStrictEqual(download.metadata, { keyc: "valc" }); + }); + + it("putBlobFromUrl answers the source conditions @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + const upload = await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + try { + await blockBlobClient.syncUploadFromURL(sourceUrl, { + sourceConditions: { ifMatch: '"0x0000000000000000"' } + }); + assert.fail("Did not throw an exception."); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 412); + assert.equal(e.code, "SourceConditionNotMet"); + } + + // The same condition naming the source's own ETag admits the copy. + await blockBlobClient.syncUploadFromURL(sourceUrl, { + sourceConditions: { ifMatch: upload.etag } + }); + const download = await blobClient.download(0); + assert.equal(await bodyToString(download, content.length), content); + }); + + it("putBlobFromUrl answers the destination conditions @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + await blockBlobClient.upload("existing", "existing".length); + + try { + await blockBlobClient.syncUploadFromURL(sourceUrl, { + conditions: { ifNoneMatch: "*" } + }); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 409); + assert.equal(e.code, "BlobAlreadyExists"); + // The destination the condition protected is untouched. + const download = await blobClient.download(0); + assert.equal(await bodyToString(download, 8), "existing"); + return; + } + assert.fail("Did not throw an exception."); + }); + + it("putBlobFromUrl rejects a request body @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + const destinationUrl = await blockBlobClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("rw"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + // @azure/storage-blob sends no body for this operation, so issue the + // request directly. + const response = await axios.put(destinationUrl, "unexpected body", { + headers: { + "x-ms-copy-source": sourceUrl, + "x-ms-blob-type": "BlockBlob" + }, + validateStatus: () => true + }); + assert.deepStrictEqual(response.status, 400); + assert.ok(response.data.includes("InvalidHeaderValue")); + }); + + it("putBlobFromUrl from a missing source returns 404 @loki @sql", async () => { + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + try { + await blockBlobClient.syncUploadFromURL(sourceUrl); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 404); + assert.equal(e.code, "CannotVerifyCopySource"); + return; + } + assert.fail("Did not throw an exception."); + }); + + it("putBlobFromUrl copies the stored bytes when the source declares Content-Encoding: gzip @loki @sql", async () => { + // A blob's Content-Encoding is stored metadata, not a description of how + // the body is framed on the wire, so the download echoes it back over the + // raw stored bytes. The copy must carry those bytes verbatim rather than + // decoding them (see issue #646 for the same hazard on copy). + const raw = zlib.gzipSync(Buffer.from("HelloWorldFromSourceBlob")); + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(raw, raw.length, { + blobHTTPHeaders: { blobContentEncoding: "gzip" } + }); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + await blockBlobClient.syncUploadFromURL(sourceUrl); + + const download = await blockBlobClient.download(0); + const chunks: Buffer[] = []; + for await (const chunk of download.readableStreamBody!) { + chunks.push(Buffer.from(chunk)); + } + assert.deepStrictEqual( + Buffer.concat(chunks), + raw, + "The copy must be the source's stored bytes, not the decoded ones" + ); + }); + + it("putBlobFromUrl with matching sourceContentMD5 @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const md5 = crypto.createHash("md5").update(content, "utf8").digest(); + const result = await blockBlobClient.syncUploadFromURL(sourceUrl, { + sourceContentMD5: new Uint8Array(md5) + }); + assert.deepStrictEqual(Buffer.from(result.contentMD5!), md5); + }); + + it("putBlobFromUrl with wrong sourceContentMD5 should throw md5 mismatch @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const md5 = crypto.createHash("md5").update("WrongContent", "utf8").digest(); + try { + await blockBlobClient.syncUploadFromURL(sourceUrl, { + sourceContentMD5: new Uint8Array(md5) + }); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 400); + assert.equal(e.code, "Md5Mismatch"); + // The rejected copy left no blob behind. + assert.strictEqual(await blockBlobClient.exists(), false); + return; + } + assert.fail("Did not throw an exception."); + }); + + it("putBlobFromUrl with wrong-length sourceContentMD5 should be rejected @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const targetClient = getBlockBlobClientWithRawHeaders( + containerName, + getUniqueName("target"), + [{ key: "x-ms-source-content-md5", value: Buffer.from("short").toString("base64") }] + ); + + try { + await targetClient.syncUploadFromURL(sourceUrl); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 400); + assert.equal(e.code, "InvalidMd5"); + return; + } + assert.fail("Did not throw an exception."); + }); + + it("putBlobFromUrl sets the tags the request names @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length, { + tags: { sourcetag: "sourcevalue" } + }); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("rt"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const tags: Tags = { tag1: "val1", tag2: "val2" }; + await blockBlobClient.syncUploadFromURL(sourceUrl, { tags }); + + // Tags are not copied from the source unless asked for, so the request's + // stand alone. + const result = await blockBlobClient.getTags(); + assert.deepStrictEqual(result.tags, tags); + }); + + it("putBlobFromUrl copies the source's tags when asked @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const tags: Tags = { sourcetag: "sourcevalue", other: "value" }; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length, { tags }); + const sourceUrl = await sourceClient.generateSasUrl({ + // Reading the source's tags is its own permission, as on the service. + permissions: BlobSASPermissions.parse("rt"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + await blockBlobClient.syncUploadFromURL(sourceUrl, { + copySourceTags: "COPY" + }); + + const result = await blockBlobClient.getTags(); + assert.deepStrictEqual(result.tags, tags); + }); + + it("putBlobFromUrl copying the tags of an untagged source @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("rt"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + await blockBlobClient.syncUploadFromURL(sourceUrl, { + copySourceTags: "COPY" + }); + + const result = await blockBlobClient.getTags(); + assert.deepStrictEqual(result.tags, {}); + }); + + it("putBlobFromUrl cannot copy tags the source URL may not read @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length, { + tags: { sourcetag: "sourcevalue" } + }); + // Read permission alone does not extend to the source's tags. + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + try { + await blockBlobClient.syncUploadFromURL(sourceUrl, { + copySourceTags: "COPY" + }); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 403); + assert.equal(e.code, "CannotVerifyCopySource"); + return; + } + assert.fail("Did not throw an exception."); + }); + + it("putBlobFromUrl rejects tags alongside copySourceTags COPY @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("rt"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + try { + await blockBlobClient.syncUploadFromURL(sourceUrl, { + copySourceTags: "COPY", + tags: { tag1: "val1" } + }); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 400); + assert.equal(e.code, "BothUserTagsAndSourceTagsCopyPresentException"); + return; + } + assert.fail("Did not throw an exception."); + }); + + it("putBlobFromUrl sets the access tier @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + await blockBlobClient.syncUploadFromURL(sourceUrl, { tier: "Cool" }); + + const properties = await blockBlobClient.getProperties(); + assert.equal(properties.accessTier, "Cool"); + }); + it("stageBlock with double commit block should work @loki @sql", async () => { const body = "HelloWorld"; From d7f10d7a0a552615ce1b4db77f24228df654f79a Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Mon, 7 Sep 2026 11:18:10 -0700 Subject: [PATCH 2/9] Check Content-MD5 on Put Blob From URL Put Blob From URL compared the copied content with x-ms-source-content-md5 or x-ms-blob-content-md5 but ignored Content-MD5, which the swagger carries for the operation and which Put Blob, whose semantics the operation follows for these headers, validates. Compare it too, ranked below the other two as Put Blob ranks it below x-ms-blob-content-md5, and reject a malformed value in any of the three before the source is fetched rather than only in the one that would have been compared. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01H7wFx5XWMdmqezhwP3YvZQ --- ChangeLog.md | 2 +- src/blob/handlers/BlockBlobHandler.ts | 48 ++++---- tests/blob/apis/blockblob.test.ts | 164 ++++++++++++++++++++++++++ 3 files changed, 193 insertions(+), 21 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index 8a8085ff6..f896e8372 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -28,7 +28,7 @@ General: Blob: - Fixed blob operations hanging when a client disconnects before the operation queue processes the request. (issue #2575) -- Implement `PutBlobFromUrl` (`Put Blob From URL`), which previously returned 501. The source is fetched over loopback, as `PutBlockFromURL` already does, so that SAS authentication and the `x-ms-source-if-*` conditions are enforced by the existing download path. Standard blob properties are copied from the source unless `x-ms-copy-source-blob-properties` is false, request blob content headers override them either way, request metadata replaces the source's rather than adding to it, and `x-ms-copy-source-tag-option: COPY` reads the source's tags over that same authorized path. As with `CopyBlobFromURL`, only sources on the same Azurite instance are supported. +- Implement `PutBlobFromUrl` (`Put Blob From URL`), which previously returned 501. The source is fetched over loopback, as `PutBlockFromURL` already does, so that SAS authentication and the `x-ms-source-if-*` conditions are enforced by the existing download path. Standard blob properties are copied from the source unless `x-ms-copy-source-blob-properties` is false, request blob content headers override them either way, request metadata replaces the source's rather than adding to it, and `x-ms-copy-source-tag-option: COPY` reads the source's tags over that same authorized path. An `x-ms-source-content-md5`, `x-ms-blob-content-md5`, or `Content-MD5` header is checked against the copied content. As with `CopyBlobFromURL`, only sources on the same Azurite instance are supported. Table: diff --git a/src/blob/handlers/BlockBlobHandler.ts b/src/blob/handlers/BlockBlobHandler.ts index 6e716076b..834cc6753 100644 --- a/src/blob/handlers/BlockBlobHandler.ts +++ b/src/blob/handlers/BlockBlobHandler.ts @@ -22,7 +22,8 @@ import { BLOB_API_VERSION } from "../utils/constants"; import BaseHandler from "./BaseHandler"; import { computeAndValidateTransactionalChecksums, - getTagsFromString + getTagsFromString, + isValidMd5Header } from "../utils/utils"; /** @@ -228,15 +229,29 @@ export default class BlockBlobHandler }); } - // Reject a malformed source checksum before fetching anything. The - // shared validator would catch it too, but only once the source had - // already been read. - if ( - options.sourceContentMD5 !== undefined && - options.sourceContentMD5.length !== 16 - ) { - throw StorageErrorFactory.getInvalidMd5(context.contextId); + // Three request headers can carry an MD5. x-ms-source-content-md5 is + // this operation's own integrity check over the bytes that arrive from + // the source. x-ms-blob-content-md5 and Content-MD5 get the treatment + // Put Blob gives them, since Put Blob From URL follows Put Blob for the + // custom properties and this request has no body of its own for + // Content-MD5 to describe. All three describe the same bytes, so one is + // compared: the operation's own header first, then x-ms-blob-content-md5 + // over Content-MD5, the order Put Blob uses. A malformed value in any of + // them is rejected before fetching anything; the shared validator would + // catch the compared one too, but only once the source had already been + // read, and never an outranked one. + const blobHTTPHeaders = options.blobHTTPHeaders || {}; + const requestMD5s = [ + options.sourceContentMD5, + blobHTTPHeaders.blobContentMD5, + options.transactionalContentMD5 + ]; + for (const md5 of requestMD5s) { + if (md5 !== undefined && !isValidMd5Header(md5)) { + throw StorageErrorFactory.getInvalidMd5(context.contextId); + } } + const expectedContentMD5 = requestMD5s.find((md5) => md5 !== undefined); // The destination's tags are either the source's or the request's, never // both. @@ -289,11 +304,9 @@ export default class BlockBlobHandler } // The response always echoes an MD5 of what was copied, so it is always - // computed. x-ms-source-content-md5 is this operation's integrity check - // over the bytes that arrived; x-ms-blob-content-md5 gets the same - // treatment Put Blob gives it, since Put Blob From URL follows Put Blob - // for the custom properties. Destroy the stream regardless, so a - // mismatch cannot leave the extent handle open. + // computed, whether or not the request sent one to compare it with. + // Destroy the stream regardless, so a mismatch cannot leave the extent + // handle open. const stream = await this.extentStore.readExtent( persistency, context.contextId @@ -303,11 +316,7 @@ export default class BlockBlobHandler ({ md5: calculatedContentMD5 } = await computeAndValidateTransactionalChecksums( stream, - { - md5: - options.sourceContentMD5 ?? - (options.blobHTTPHeaders || {}).blobContentMD5 - }, + { md5: expectedContentMD5 }, context.contextId, { md5: true } )); @@ -332,7 +341,6 @@ export default class BlockBlobHandler const copyProperties = options.copySourceBlobProperties !== false; const sourceProperty = (name: string): string | undefined => copyProperties ? sourceResponse.headers[name] : undefined; - const blobHTTPHeaders = options.blobHTTPHeaders || {}; const contentType = blobHTTPHeaders.blobContentType || sourceProperty("content-type") || diff --git a/tests/blob/apis/blockblob.test.ts b/tests/blob/apis/blockblob.test.ts index 212accf12..18133065d 100644 --- a/tests/blob/apis/blockblob.test.ts +++ b/tests/blob/apis/blockblob.test.ts @@ -1407,6 +1407,170 @@ describe("BlockBlobAPIs", () => { assert.fail("Did not throw an exception."); }); + it("putBlobFromUrl with matching Content-MD5 @loki @sql", async () => { + // The REST reference does not list Content-MD5 among this operation's + // request headers, but the swagger carries it and the request has no + // body of its own, so it is checked against the copied content the way + // Put Blob checks it against the body. The SDK does not expose it for + // this operation, so inject the raw header. + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const md5 = crypto.createHash("md5").update(content, "utf8").digest(); + const targetClient = getBlockBlobClientWithRawHeaders( + containerName, + blobName, + [{ key: "content-md5", value: md5.toString("base64") }] + ); + const result = await targetClient.syncUploadFromURL(sourceUrl); + assert.deepStrictEqual(Buffer.from(result.contentMD5!), md5); + + const download = await blobClient.download(0); + assert.equal(await bodyToString(download, content.length), content); + }); + + it("putBlobFromUrl with wrong Content-MD5 should throw md5 mismatch @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const md5 = crypto.createHash("md5").update("WrongContent", "utf8").digest(); + const targetClient = getBlockBlobClientWithRawHeaders( + containerName, + blobName, + [{ key: "content-md5", value: md5.toString("base64") }] + ); + try { + await targetClient.syncUploadFromURL(sourceUrl); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 400); + assert.equal(e.code, "Md5Mismatch"); + // The rejected copy left no blob behind. + assert.strictEqual(await blockBlobClient.exists(), false); + return; + } + assert.fail("Did not throw an exception."); + }); + + it("putBlobFromUrl with wrong-length Content-MD5 should be rejected @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const targetClient = getBlockBlobClientWithRawHeaders( + containerName, + blobName, + [{ key: "content-md5", value: Buffer.from("short").toString("base64") }] + ); + try { + await targetClient.syncUploadFromURL(sourceUrl); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 400); + assert.equal(e.code, "InvalidMd5"); + return; + } + assert.fail("Did not throw an exception."); + }); + + it("putBlobFromUrl rejects a malformed Content-MD5 alongside a matching sourceContentMD5 @loki @sql", async () => { + // Only one of the request's MD5 headers is compared with the copied + // content, but every one of them has to be well formed. + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const md5 = crypto.createHash("md5").update(content, "utf8").digest(); + const targetClient = getBlockBlobClientWithRawHeaders( + containerName, + blobName, + [{ key: "content-md5", value: Buffer.from("short").toString("base64") }] + ); + try { + await targetClient.syncUploadFromURL(sourceUrl, { + sourceContentMD5: new Uint8Array(md5) + }); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 400); + assert.equal(e.code, "InvalidMd5"); + return; + } + assert.fail("Did not throw an exception."); + }); + + it("putBlobFromUrl x-ms-blob-content-md5 takes precedence over Content-MD5 @loki @sql", async () => { + // Put Blob From URL follows Put Blob for these headers, and Put Blob + // compares x-ms-blob-content-md5 when both are sent (see the upload test + // of the same name). + // - Content-MD5 wrong + x-ms-blob-content-md5 correct -> success + // - Content-MD5 correct + x-ms-blob-content-md5 wrong -> Md5Mismatch + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const correctMd5 = crypto.createHash("md5").update(content, "utf8").digest(); + const wrongMd5 = crypto.createHash("md5").update("WrongContent", "utf8").digest(); + + const clientWithWrongContentMd5 = getBlockBlobClientWithRawHeaders( + containerName, + blobName, + [{ key: "content-md5", value: wrongMd5.toString("base64") }] + ); + await clientWithWrongContentMd5.syncUploadFromURL(sourceUrl, { + blobHTTPHeaders: { blobContentMD5: new Uint8Array(correctMd5) } + }); + + const clientWithCorrectContentMd5 = getBlockBlobClientWithRawHeaders( + containerName, + blobName, + [{ key: "content-md5", value: correctMd5.toString("base64") }] + ); + try { + await clientWithCorrectContentMd5.syncUploadFromURL(sourceUrl, { + blobHTTPHeaders: { blobContentMD5: new Uint8Array(wrongMd5) } + }); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 400); + assert.equal(e.code, "Md5Mismatch"); + return; + } + assert.fail("Did not throw an exception."); + }); + it("putBlobFromUrl sets the tags the request names @loki @sql", async () => { const content = "HelloWorldFromSourceBlob"; const sourceClient = containerClient.getBlockBlobClient( From a0585f74ba998d0873baf2327ba440bb964e6c9c Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Mon, 7 Sep 2026 11:32:40 -0700 Subject: [PATCH 3/9] Pin the SAS permissions Put Blob From URL answers to Azure grants Put Blob From URL on a new block blob to Create or Write and on an existing one to Write alone, the split Put Blob has, and the authenticators already answer it that way: the permission table accepts either, and an existing destination then demands Write. A review read the table's any-of check as letting Write create a blob it should not, which is the documented behaviour, so pin all four cells of the table for account and blob SAS rather than change them. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01H7wFx5XWMdmqezhwP3YvZQ --- tests/blob/sas.test.ts | 168 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 168 insertions(+) diff --git a/tests/blob/sas.test.ts b/tests/blob/sas.test.ts index 3f5c6f6d9..0a579b589 100644 --- a/tests/blob/sas.test.ts +++ b/tests/blob/sas.test.ts @@ -547,6 +547,94 @@ describe("Shared Access Signature (SAS) authentication", () => { await blob2.syncCopyFromURL(blob1.url); }); + it("Put blob from URL with write permission in account SAS should create and override a blob @loki @sql", async () => { + // Azure grants Put Blob From URL on a new block blob to Create (c) or + // Write (w), and on an existing one to Write (w) alone, the split Put + // Blob has. Write alone therefore creates as well as overrides. + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + const containerName = getUniqueName("container"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const sourceBlob = containerClient.getBlockBlobClient( + getUniqueName("blob") + ); + await sourceBlob.upload("hello", 5); + const sourceUrl = await sourceBlob.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: tmr + }); + + const serviceClientWithSAS = new BlobServiceClient( + serviceClient.generateAccountSasUrl( + tmr, + AccountSASPermissions.parse("w"), + "co" + ), + newPipeline(new AnonymousCredential(), { + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + const blobWithSAS = serviceClientWithSAS + .getContainerClient(containerName) + .getBlockBlobClient(getUniqueName("blob")); + + // Neither the copy that creates the blob nor the one that overrides it + // should throw any errors + await blobWithSAS.syncUploadFromURL(sourceUrl); + await blobWithSAS.syncUploadFromURL(sourceUrl); + }); + + it("Put blob from URL with create permission in account SAS should create but not override a blob @loki @sql", async () => { + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + const containerName = getUniqueName("container"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const sourceBlob = containerClient.getBlockBlobClient( + getUniqueName("blob") + ); + await sourceBlob.upload("hello", 5); + const sourceUrl = await sourceBlob.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: tmr + }); + + const serviceClientWithSAS = new BlobServiceClient( + serviceClient.generateAccountSasUrl( + tmr, + AccountSASPermissions.parse("c"), + "co" + ), + newPipeline(new AnonymousCredential(), { + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + const blobWithSAS = serviceClientWithSAS + .getContainerClient(containerName) + .getBlockBlobClient(getUniqueName("blob")); + + // this copy creates the blob and should not throw any errors + await blobWithSAS.syncUploadFromURL(sourceUrl); + + // overriding it needs Write, so this copy should throw 403 error + let error; + try { + await blobWithSAS.syncUploadFromURL(sourceUrl); + } catch (err) { + error = err; + } + assert.ok(error !== undefined); + assert.deepEqual(error.statusCode, 403); + assert.deepEqual(error.code, "AuthorizationPermissionMismatch"); + }); + it("Copy blob should work with write permission in account SAS to override an existing blob @loki", async () => { const tmr = new Date(); tmr.setDate(tmr.getDate() + 1); @@ -1688,6 +1776,86 @@ describe("Shared Access Signature (SAS) authentication", () => { await blob2SAS.syncCopyFromURL(blob1.url); }); + it("Put blob from URL with write permission in blob SAS should create and override a blob @loki @sql", async () => { + // Azure grants Put Blob From URL on a new block blob to Create (c) or + // Write (w), and on an existing one to Write (w) alone, the split Put + // Blob has. Write alone therefore creates as well as overrides. + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + const containerName = getUniqueName("container"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const sourceBlob = containerClient.getBlockBlobClient( + getUniqueName("blob") + ); + await sourceBlob.upload("hello", 5); + const sourceUrl = await sourceBlob.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: tmr + }); + + const containerClientWithSAS = new ContainerClient( + await containerClient.generateSasUrl({ + permissions: ContainerSASPermissions.parse("w"), + expiresOn: tmr + }), + newPipeline(new AnonymousCredential()) + ); + const blobWithSAS = containerClientWithSAS.getBlockBlobClient( + getUniqueName("blob") + ); + + // Neither the copy that creates the blob nor the one that overrides it + // should throw any errors + await blobWithSAS.syncUploadFromURL(sourceUrl); + await blobWithSAS.syncUploadFromURL(sourceUrl); + }); + + it("Put blob from URL with create permission in blob SAS should create but not override a blob @loki @sql", async () => { + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + const containerName = getUniqueName("container"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const sourceBlob = containerClient.getBlockBlobClient( + getUniqueName("blob") + ); + await sourceBlob.upload("hello", 5); + const sourceUrl = await sourceBlob.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: tmr + }); + + const containerClientWithSAS = new ContainerClient( + await containerClient.generateSasUrl({ + permissions: ContainerSASPermissions.parse("c"), + expiresOn: tmr + }), + newPipeline(new AnonymousCredential()) + ); + const blobWithSAS = containerClientWithSAS.getBlockBlobClient( + getUniqueName("blob") + ); + + // this copy creates the blob and should not throw any errors + await blobWithSAS.syncUploadFromURL(sourceUrl); + + // overriding it needs Write, so this copy should throw 403 error + let error; + try { + await blobWithSAS.syncUploadFromURL(sourceUrl); + } catch (err) { + error = err; + } + assert.ok(error !== undefined); + assert.deepEqual(error.statusCode, 403); + assert.deepEqual(error.code, "AuthorizationPermissionMismatch"); + }); + it("Copy blob should work with write permission in blob SAS to override an existing blob @loki @sql", async () => { const now = new Date(); now.setMinutes(now.getMinutes() - 5); // Skip clock skew with server From b919539cccab3726550db5a3e00d00afcdef455e Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Mon, 7 Sep 2026 11:50:16 -0700 Subject: [PATCH 4/9] Say what the Put Blob From URL SAS entries allow The comments on the operation's permission-table entries were copied from Put Blob's and read as though Create alone could create the blob, or Write alone had to. Either creates it; only overwriting an existing blob takes Write alone, which the authenticators check separately. Say that. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01H7wFx5XWMdmqezhwP3YvZQ --- src/blob/authentication/OperationAccountSASPermission.ts | 3 ++- src/blob/authentication/OperationBlobSASPermission.ts | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/blob/authentication/OperationAccountSASPermission.ts b/src/blob/authentication/OperationAccountSASPermission.ts index 979668078..32ff60e47 100644 --- a/src/blob/authentication/OperationAccountSASPermission.ts +++ b/src/blob/authentication/OperationAccountSASPermission.ts @@ -368,7 +368,8 @@ OPERATION_ACCOUNT_SAS_PERMISSIONS.set( new OperationAccountSASPermission( AccountSASService.Blob, AccountSASResourceType.Object, - // Create permission is only available for nonexistent block blob. Handle this scenario separately + // Create or Write creates the blob. Overwriting an existing one takes + // Write alone, which the authenticator checks separately. AccountSASPermission.Write + AccountSASPermission.Create ) ); diff --git a/src/blob/authentication/OperationBlobSASPermission.ts b/src/blob/authentication/OperationBlobSASPermission.ts index cced5f88f..3a27e5e19 100644 --- a/src/blob/authentication/OperationBlobSASPermission.ts +++ b/src/blob/authentication/OperationBlobSASPermission.ts @@ -260,7 +260,8 @@ OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( ); OPERATION_BLOB_SAS_BLOB_PERMISSIONS.set( Operation.BlockBlob_PutBlobFromUrl, - // TODO: When destination blob doesn't exist, needs create permission + // Create or Write creates the blob. Overwriting an existing one takes + // Write alone, which the authenticator checks separately. new OperationBlobSASPermission( BlobSASPermission.Write + BlobSASPermission.Create ) @@ -531,7 +532,8 @@ OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( ); OPERATION_BLOB_SAS_CONTAINER_PERMISSIONS.set( Operation.BlockBlob_PutBlobFromUrl, - // Create a new blob, must be write + // Create or Write creates the blob. Overwriting an existing one takes + // Write alone, which the authenticator checks separately. new OperationBlobSASPermission( BlobSASPermission.Write + BlobSASPermission.Create ) From b071c4363a4ebe5c39ddfe2739dfe02945e825f7 Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Mon, 7 Sep 2026 11:59:33 -0700 Subject: [PATCH 5/9] Answer a failed tag read the way a failed content read is answered Put Blob From URL maps a copy source body that fails midway to CannotVerifyCopySource, but the tag read it makes under x-ms-copy-source-tag-option: COPY read and parsed its body bare, so a reset after the headers or a body that does not parse escaped as a bodiless 500. Give it the same answer, and release the stream. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01H7wFx5XWMdmqezhwP3YvZQ --- src/blob/handlers/BlockBlobHandler.ts | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/src/blob/handlers/BlockBlobHandler.ts b/src/blob/handlers/BlockBlobHandler.ts index 834cc6753..6e4232933 100644 --- a/src/blob/handlers/BlockBlobHandler.ts +++ b/src/blob/handlers/BlockBlobHandler.ts @@ -1137,11 +1137,29 @@ export default class BlockBlobHandler "comp=tags" ); - const chunks: Buffer[] = []; - for await (const chunk of response.data as IncomingMessage) { - chunks.push(Buffer.from(chunk)); + // As with the content, the status was only the headers arriving: the + // body can still fail midway, and what arrives has to parse. Map either + // to the error a transport failure gets rather than letting it escape + // as a bodiless 500, and release the stream on the way out. + let parsed: any; + try { + const chunks: Buffer[] = []; + for await (const chunk of response.data as IncomingMessage) { + chunks.push(Buffer.from(chunk)); + } + parsed = await parseXML(Buffer.concat(chunks).toString()); + } catch (err) { + response.data.destroy(); + this.logger.error( + `BlockBlobHandler:putBlobFromUrl() Failed to read the copy source tags: ${err}`, + context.contextId + ); + throw StorageErrorFactory.getCannotVerifyCopySource( + context.contextId!, + 500, + "Could not verify the copy source within the specified time." + ); } - const parsed = await parseXML(Buffer.concat(chunks).toString()); // parseXML collapses a single element out of its array, and leaves a // tagless source with no TagSet at all. From 5725668ddd51ba5e07f270e925619fc30c401d7b Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Tue, 8 Sep 2026 09:07:43 -0700 Subject: [PATCH 6/9] Check and return x-ms-content-crc64 on Put Blob From URL The REST reference lists x-ms-content-crc64 among this operation's request headers and says the response always carries the CRC64 the service computed. The checked-in swagger had neither, so the request header was never deserialized and the response never carried one. Add the ContentCrc64 parameter and the x-ms-content-crc64 response header to the operation's swagger entry, mirror them into the generated models, mapper and operation spec, and have the handler compare a supplied value with the copied content, reject one sent alongside an MD5 as Put Blob does, and report the computed CRC64 on every success. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01UgrmeBoe86wCnWC8vXR4D4 --- ChangeLog.md | 2 +- src/blob/generated/artifacts/mappers.ts | 6 + src/blob/generated/artifacts/models.ts | 10 + .../generated/artifacts/specifications.ts | 1 + src/blob/handlers/BlockBlobHandler.ts | 43 +++-- swagger/blob-storage-2021-10-04.json | 8 + swagger/blob.md | 2 + tests/blob/apis/blockblob.test.ts | 179 ++++++++++++++++++ 8 files changed, 238 insertions(+), 13 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index f896e8372..f41492357 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -28,7 +28,7 @@ General: Blob: - Fixed blob operations hanging when a client disconnects before the operation queue processes the request. (issue #2575) -- Implement `PutBlobFromUrl` (`Put Blob From URL`), which previously returned 501. The source is fetched over loopback, as `PutBlockFromURL` already does, so that SAS authentication and the `x-ms-source-if-*` conditions are enforced by the existing download path. Standard blob properties are copied from the source unless `x-ms-copy-source-blob-properties` is false, request blob content headers override them either way, request metadata replaces the source's rather than adding to it, and `x-ms-copy-source-tag-option: COPY` reads the source's tags over that same authorized path. An `x-ms-source-content-md5`, `x-ms-blob-content-md5`, or `Content-MD5` header is checked against the copied content. As with `CopyBlobFromURL`, only sources on the same Azurite instance are supported. +- Implement `PutBlobFromUrl` (`Put Blob From URL`), which previously returned 501. The source is fetched over loopback, as `PutBlockFromURL` already does, so that SAS authentication and the `x-ms-source-if-*` conditions are enforced by the existing download path. Standard blob properties are copied from the source unless `x-ms-copy-source-blob-properties` is false, request blob content headers override them either way, request metadata replaces the source's rather than adding to it, and `x-ms-copy-source-tag-option: COPY` reads the source's tags over that same authorized path. An `x-ms-source-content-md5`, `x-ms-blob-content-md5`, `Content-MD5`, or `x-ms-content-crc64` header is checked against the copied content, and the response reports the MD5 and CRC64 of that content. As with `CopyBlobFromURL`, only sources on the same Azurite instance are supported. Table: diff --git a/src/blob/generated/artifacts/mappers.ts b/src/blob/generated/artifacts/mappers.ts index 6f41aeffe..bbed870fc 100644 --- a/src/blob/generated/artifacts/mappers.ts +++ b/src/blob/generated/artifacts/mappers.ts @@ -4839,6 +4839,12 @@ export const BlockBlobPutBlobFromUrlHeaders: msRest.CompositeMapper = { name: "ByteArray" } }, + xMsContentCrc64: { + serializedName: "x-ms-content-crc64", + type: { + name: "ByteArray" + } + }, clientRequestId: { serializedName: "x-ms-client-request-id", type: { diff --git a/src/blob/generated/artifacts/models.ts b/src/blob/generated/artifacts/models.ts index e7f65d7a3..8a9292e21 100644 --- a/src/blob/generated/artifacts/models.ts +++ b/src/blob/generated/artifacts/models.ts @@ -3077,6 +3077,10 @@ export interface BlockBlobPutBlobFromUrlOptionalParams { * Specify the transactional md5 for the body, to be validated by the service. */ transactionalContentMD5?: Uint8Array; + /** + * Specify the transactional crc64 for the body, to be validated by the service. + */ + transactionalContentCrc64?: Uint8Array; /** * Optional. Specifies a user-defined name-value pair associated with the blob. If no name-value * pairs are specified, the operation will copy the metadata from the source blob or file to the @@ -5090,6 +5094,12 @@ export interface BlockBlobPutBlobFromUrlHeaders { * is returned so that the client can check for message content integrity. */ contentMD5?: Uint8Array; + /** + * This header is returned so that the client can check for message content integrity. The value + * of this header is computed by the Blob service; it is not necessarily the same value specified + * in the request headers. + */ + xMsContentCrc64?: Uint8Array; /** * If a client request id header is sent in the request, this header will be present in the * response with the same value. diff --git a/src/blob/generated/artifacts/specifications.ts b/src/blob/generated/artifacts/specifications.ts index c0b1f17e2..eeebcb4c3 100644 --- a/src/blob/generated/artifacts/specifications.ts +++ b/src/blob/generated/artifacts/specifications.ts @@ -2518,6 +2518,7 @@ const blockBlobPutBlobFromUrlOperationSpec: msRest.OperationSpec = { ], headerParameters: [ Parameters.transactionalContentMD5, + Parameters.transactionalContentCrc64, Parameters.contentLength, Parameters.metadata, Parameters.tier0, diff --git a/src/blob/handlers/BlockBlobHandler.ts b/src/blob/handlers/BlockBlobHandler.ts index 6e4232933..d7d1e4615 100644 --- a/src/blob/handlers/BlockBlobHandler.ts +++ b/src/blob/handlers/BlockBlobHandler.ts @@ -236,22 +236,39 @@ export default class BlockBlobHandler // custom properties and this request has no body of its own for // Content-MD5 to describe. All three describe the same bytes, so one is // compared: the operation's own header first, then x-ms-blob-content-md5 - // over Content-MD5, the order Put Blob uses. A malformed value in any of - // them is rejected before fetching anything; the shared validator would - // catch the compared one too, but only once the source had already been - // read, and never an outranked one. + // over Content-MD5, the order Put Blob uses. x-ms-content-crc64 describes + // those bytes as well, and as on Put Blob it cannot be sent alongside an + // MD5. A malformed value in any of these headers is rejected before + // fetching anything; the shared validator would catch the compared ones + // too, but only once the source had already been read, and never an + // outranked MD5. const blobHTTPHeaders = options.blobHTTPHeaders || {}; const requestMD5s = [ options.sourceContentMD5, blobHTTPHeaders.blobContentMD5, options.transactionalContentMD5 ]; + const expectedContentMD5 = requestMD5s.find((md5) => md5 !== undefined); + const expectedContentCRC64 = options.transactionalContentCrc64; + if ( + expectedContentMD5 !== undefined && + expectedContentCRC64 !== undefined + ) { + throw StorageErrorFactory.getBothCrc64AndMd5HeaderPresent( + context.contextId + ); + } for (const md5 of requestMD5s) { if (md5 !== undefined && !isValidMd5Header(md5)) { throw StorageErrorFactory.getInvalidMd5(context.contextId); } } - const expectedContentMD5 = requestMD5s.find((md5) => md5 !== undefined); + if (expectedContentCRC64 !== undefined && expectedContentCRC64.length < 8) { + throw StorageErrorFactory.getInvalidHeaderValue(context.contextId, { + HeaderName: "x-ms-content-crc64", + HeaderValue: Buffer.from(expectedContentCRC64).toString("base64") + }); + } // The destination's tags are either the source's or the request's, never // both. @@ -303,22 +320,23 @@ export default class BlockBlobHandler ); } - // The response always echoes an MD5 of what was copied, so it is always - // computed, whether or not the request sent one to compare it with. - // Destroy the stream regardless, so a mismatch cannot leave the extent - // handle open. + // The response always carries an MD5 and a CRC64 of what was copied, so + // both are always computed, whether or not the request sent one to + // compare them with. Destroy the stream regardless, so a mismatch cannot + // leave the extent handle open. const stream = await this.extentStore.readExtent( persistency, context.contextId ); let calculatedContentMD5: Uint8Array | undefined; + let calculatedContentCRC64: Uint8Array | undefined; try { - ({ md5: calculatedContentMD5 } = + ({ md5: calculatedContentMD5, crc64: calculatedContentCRC64 } = await computeAndValidateTransactionalChecksums( stream, - { md5: expectedContentMD5 }, + { md5: expectedContentMD5, crc64: expectedContentCRC64 }, context.contextId, - { md5: true } + { md5: true, crc64: true } )); } finally { (stream as Readable).destroy?.(); @@ -423,6 +441,7 @@ export default class BlockBlobHandler eTag: etag, lastModified: date, contentMD5: blob.properties.contentMD5, + xMsContentCrc64: calculatedContentCRC64, requestId: blobCtx.contextId, version: BLOB_API_VERSION, date, diff --git a/swagger/blob-storage-2021-10-04.json b/swagger/blob-storage-2021-10-04.json index 43ce6da2b..731b4ddb1 100644 --- a/swagger/blob-storage-2021-10-04.json +++ b/swagger/blob-storage-2021-10-04.json @@ -4543,6 +4543,9 @@ { "$ref": "#/parameters/ContentMD5" }, + { + "$ref": "#/parameters/ContentCrc64" + }, { "$ref": "#/parameters/ContentLength" }, @@ -4658,6 +4661,11 @@ "format": "byte", "description": "If the blob has an MD5 hash and this operation is to read the full blob, this response header is returned so that the client can check for message content integrity." }, + "x-ms-content-crc64": { + "type": "string", + "format": "byte", + "description": "This header is returned so that the client can check for message content integrity. The value of this header is computed by the Blob service; it is not necessarily the same value specified in the request headers." + }, "x-ms-client-request-id": { "x-ms-client-name": "ClientRequestId", "type": "string", diff --git a/swagger/blob.md b/swagger/blob.md index 02025dddf..36b9d9650 100644 --- a/swagger/blob.md +++ b/swagger/blob.md @@ -66,3 +66,5 @@ enum-types: true 16. Add "Cold" to "AccessTier", "AccessTierRequired", "AccessTierOptional"; and add "rehydrate-pending-to-cold" to "ArchiveStatus". (can be removed when upgrade to new API version.) 17. Remove default value setting parameter "BlobSequenceNumber" for header "x-ms-blob-sequence-number" + +18. Add "ContentCrc64" parameter and "x-ms-content-crc64" response header to "BlockBlob_PutBlobFromUrl". The REST reference lists both for this operation; the client swagger carries only the response header. diff --git a/tests/blob/apis/blockblob.test.ts b/tests/blob/apis/blockblob.test.ts index 18133065d..745dc1d33 100644 --- a/tests/blob/apis/blockblob.test.ts +++ b/tests/blob/apis/blockblob.test.ts @@ -1571,6 +1571,185 @@ describe("BlockBlobAPIs", () => { assert.fail("Did not throw an exception."); }); + it("putBlobFromUrl returns the CRC64 of the copied content @loki @sql", async () => { + // The response always carries x-ms-content-crc64, computed by the + // service over the copied content, whether or not the request sent a + // checksum. The SDK does not surface the header for this operation, so + // read it from the raw response. + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + const crc64 = Buffer.from(getCRC64FromString(content)).toString("base64"); + + const result = await blockBlobClient.syncUploadFromURL(sourceUrl); + assert.equal(result._response.headers.get("x-ms-content-crc64"), crc64); + + // Still returned when the request checked an MD5 instead. + const md5 = crypto.createHash("md5").update(content, "utf8").digest(); + const resultWithMd5 = await blockBlobClient.syncUploadFromURL(sourceUrl, { + sourceContentMD5: new Uint8Array(md5) + }); + assert.equal( + resultWithMd5._response.headers.get("x-ms-content-crc64"), + crc64 + ); + }); + + it("putBlobFromUrl with matching x-ms-content-crc64 @loki @sql", async () => { + // The SDK does not expose x-ms-content-crc64 for this operation, so + // inject the raw header. + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const crc64 = Buffer.from(getCRC64FromString(content)).toString("base64"); + const targetClient = getBlockBlobClientWithRawHeaders( + containerName, + blobName, + [{ key: "x-ms-content-crc64", value: crc64 }] + ); + const result = await targetClient.syncUploadFromURL(sourceUrl); + assert.equal(result._response.status, 201); + assert.equal(result._response.headers.get("x-ms-content-crc64"), crc64); + + const download = await blobClient.download(0); + assert.equal(await bodyToString(download, content.length), content); + }); + + it("putBlobFromUrl with wrong x-ms-content-crc64 should throw crc64 mismatch @loki @sql", async () => { + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + // A valid 8-byte CRC64 of a *different* body, to exercise the mismatch + // path rather than the malformed-header path. + const crc64 = Buffer.from(getCRC64FromString("WrongContent")).toString( + "base64" + ); + const targetClient = getBlockBlobClientWithRawHeaders( + containerName, + blobName, + [{ key: "x-ms-content-crc64", value: crc64 }] + ); + try { + await targetClient.syncUploadFromURL(sourceUrl); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 400); + assert.equal(e.code, "Crc64Mismatch"); + // The rejected copy left no blob behind. + assert.strictEqual(await blockBlobClient.exists(), false); + return; + } + assert.fail("Did not throw an exception."); + }); + + it("putBlobFromUrl with wrong-length x-ms-content-crc64 should be rejected @loki @sql", async () => { + // x-ms-content-crc64 must decode to at least 8 bytes (CRC-64 is 64-bit). + // Shorter values are rejected as InvalidHeaderValue, as on Put Block. + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const targetClient = getBlockBlobClientWithRawHeaders( + containerName, + blobName, + [ + { + key: "x-ms-content-crc64", + value: Buffer.from([1, 2, 3, 4]).toString("base64") + } + ] + ); + try { + await targetClient.syncUploadFromURL(sourceUrl); + } catch (e) { + assert.equal(e.name, "RestError"); + assert.equal(e.statusCode, 400); + assert.equal(e.code, "InvalidHeaderValue"); + assert.equal( + /([^<]*) { + // Put Blob rejects a request that carries both a CRC64 and an MD5, and + // Put Blob From URL follows it: Content-MD5, the header the REST + // reference names, counts, and so does the operation's own + // x-ms-source-content-md5. Both checksums are correct for the source + // content; supplying the two together is rejected regardless. + const content = "HelloWorldFromSourceBlob"; + const sourceClient = containerClient.getBlockBlobClient( + getUniqueName("source") + ); + await sourceClient.upload(content, content.length); + const sourceUrl = await sourceClient.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: new Date(Date.now() + 60 * 60 * 1000) + }); + + const md5 = crypto.createHash("md5").update(content, "utf8").digest(); + const crc64 = Buffer.from(getCRC64FromString(content)).toString("base64"); + const rejection = { + name: "RestError", + statusCode: 400, + code: "BothCrc64AndMd5HeaderPresent" + }; + + const clientWithCrc64AndContentMd5 = getBlockBlobClientWithRawHeaders( + containerName, + blobName, + [ + { key: "x-ms-content-crc64", value: crc64 }, + { key: "content-md5", value: md5.toString("base64") } + ] + ); + await assert.rejects( + clientWithCrc64AndContentMd5.syncUploadFromURL(sourceUrl), + rejection + ); + + const clientWithCrc64 = getBlockBlobClientWithRawHeaders( + containerName, + blobName, + [{ key: "x-ms-content-crc64", value: crc64 }] + ); + await assert.rejects( + clientWithCrc64.syncUploadFromURL(sourceUrl, { + sourceContentMD5: new Uint8Array(md5) + }), + rejection + ); + }); + it("putBlobFromUrl sets the tags the request names @loki @sql", async () => { const content = "HelloWorldFromSourceBlob"; const sourceClient = containerClient.getBlockBlobClient( From b8e148ecd0e2db7a1f2e5d45c641dba2445218f7 Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Thu, 10 Sep 2026 10:39:55 -0700 Subject: [PATCH 7/9] Check the checksum headers of both URL operations in one place Put Blob From URL and Put Block From URL each kept their own copy of the shape checks the shared validator makes, so that a malformed header is rejected before the source is fetched rather than after it has been read. Move those checks into validateTransactionalChecksumHeaders, which takes the MD5 candidates in precedence order and the name of the CRC64 header to report, and have the shared validator and both handlers call it. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01H7wFx5XWMdmqezhwP3YvZQ --- src/blob/handlers/BlockBlobHandler.ts | 76 +++++------------ src/blob/utils/constants.ts | 1 + src/blob/utils/utils.ts | 117 +++++++++++++++----------- 3 files changed, 87 insertions(+), 107 deletions(-) diff --git a/src/blob/handlers/BlockBlobHandler.ts b/src/blob/handlers/BlockBlobHandler.ts index d7d1e4615..e03c09e5e 100644 --- a/src/blob/handlers/BlockBlobHandler.ts +++ b/src/blob/handlers/BlockBlobHandler.ts @@ -18,12 +18,12 @@ import Context from "../generated/Context"; import IBlockBlobHandler from "../generated/handlers/IBlockBlobHandler"; import { parseXML } from "../generated/utils/xml"; import { BlobModel, BlockModel } from "../persistence/IBlobMetadataStore"; -import { BLOB_API_VERSION } from "../utils/constants"; +import { BLOB_API_VERSION, HeaderConstants } from "../utils/constants"; import BaseHandler from "./BaseHandler"; import { computeAndValidateTransactionalChecksums, getTagsFromString, - isValidMd5Header + validateTransactionalChecksumHeaders } from "../utils/utils"; /** @@ -238,37 +238,18 @@ export default class BlockBlobHandler // compared: the operation's own header first, then x-ms-blob-content-md5 // over Content-MD5, the order Put Blob uses. x-ms-content-crc64 describes // those bytes as well, and as on Put Blob it cannot be sent alongside an - // MD5. A malformed value in any of these headers is rejected before - // fetching anything; the shared validator would catch the compared ones - // too, but only once the source had already been read, and never an - // outranked MD5. + // MD5. The header shapes are checked here, before fetching anything. const blobHTTPHeaders = options.blobHTTPHeaders || {}; - const requestMD5s = [ - options.sourceContentMD5, - blobHTTPHeaders.blobContentMD5, - options.transactionalContentMD5 - ]; - const expectedContentMD5 = requestMD5s.find((md5) => md5 !== undefined); - const expectedContentCRC64 = options.transactionalContentCrc64; - if ( - expectedContentMD5 !== undefined && - expectedContentCRC64 !== undefined - ) { - throw StorageErrorFactory.getBothCrc64AndMd5HeaderPresent( + const { md5: expectedContentMD5, crc64: expectedContentCRC64 } = + validateTransactionalChecksumHeaders( + [ + options.sourceContentMD5, + blobHTTPHeaders.blobContentMD5, + options.transactionalContentMD5 + ], + options.transactionalContentCrc64, context.contextId ); - } - for (const md5 of requestMD5s) { - if (md5 !== undefined && !isValidMd5Header(md5)) { - throw StorageErrorFactory.getInvalidMd5(context.contextId); - } - } - if (expectedContentCRC64 !== undefined && expectedContentCRC64.length < 8) { - throw StorageErrorFactory.getInvalidHeaderValue(context.contextId, { - HeaderName: "x-ms-content-crc64", - HeaderValue: Buffer.from(expectedContentCRC64).toString("base64") - }); - } // The destination's tags are either the source's or the request's, never // both. @@ -566,33 +547,14 @@ export default class BlockBlobHandler this.validateBlockId(blockId, blobCtx); - // Reject malformed source checksum headers before fetching anything. The - // shared validator would catch these too, but it reports the names of the - // transactional headers, and its errors would surface only after the - // source had already been read and staged. - if ( - options.sourceContentMD5 !== undefined && - options.sourceContentcrc64 !== undefined - ) { - throw StorageErrorFactory.getBothCrc64AndMd5HeaderPresent( - context.contextId - ); - } - if ( - options.sourceContentMD5 !== undefined && - options.sourceContentMD5.length !== 16 - ) { - throw StorageErrorFactory.getInvalidMd5(context.contextId); - } - if ( - options.sourceContentcrc64 !== undefined && - options.sourceContentcrc64.length < 8 - ) { - throw StorageErrorFactory.getInvalidHeaderValue(context.contextId, { - HeaderName: "x-ms-source-content-crc64", - HeaderValue: Buffer.from(options.sourceContentcrc64).toString("base64") - }); - } + // Reject malformed source checksum headers before fetching anything, + // reporting a bad CRC64 under the header this operation carries it in. + validateTransactionalChecksumHeaders( + [options.sourceContentMD5], + options.sourceContentcrc64, + context.contextId, + HeaderConstants.X_MS_SOURCE_CONTENT_CRC64 + ); await this.metadataStore.checkContainerExist( context, diff --git a/src/blob/utils/constants.ts b/src/blob/utils/constants.ts index 55933d0aa..c827f12c1 100644 --- a/src/blob/utils/constants.ts +++ b/src/blob/utils/constants.ts @@ -59,6 +59,7 @@ export const HeaderConstants = { X_MS_SEQUENCE_NUMBER_ACTION: "x-ms-sequence-number-action", X_MS_BLOB_SEQUENCE_NUMBER: "x-ms-blob-sequence-number", X_MS_CONTENT_CRC64: "x-ms-content-crc64", + X_MS_SOURCE_CONTENT_CRC64: "x-ms-source-content-crc64", X_MS_RANGE_GET_CONTENT_CRC64: "x-ms-range-get-content-crc64", X_MS_ENCRYPTION_KEY: "x-ms-encryption-key", X_MS_ENCRYPTION_KEY_SHA256: "x-ms-encryption-key-sha256", diff --git a/src/blob/utils/utils.ts b/src/blob/utils/utils.ts index 53e00e1a4..eb7a1c4a9 100644 --- a/src/blob/utils/utils.ts +++ b/src/blob/utils/utils.ts @@ -38,6 +38,14 @@ function decodeBase64HeaderValue(value: string): Buffer | undefined { return decoded; } +function decodeChecksumHeader( + value: Uint8Array | string +): Buffer | undefined { + return typeof value === "string" + ? decodeBase64HeaderValue(value) + : Buffer.from(value); +} + /** * Decodes an MD5 header value (base64 string or raw Uint8Array) and returns * whether the result is exactly 16 bytes - the only shape real Azure accepts. @@ -45,21 +53,62 @@ function decodeBase64HeaderValue(value: string): Buffer | undefined { * x-ms-blob-content-md5 are all rejected with InvalidMd5 (verified live). */ export function isValidMd5Header(value: Uint8Array | string): boolean { - const bytes = - typeof value === "string" - ? decodeBase64HeaderValue(value) - : Buffer.from(value); + const bytes = decodeChecksumHeader(value); return bytes !== undefined && bytes.length === 16; } +/** + * Checks the shape of the checksum headers a request carries and picks the + * pair its bytes are compared with, before anything is read. Every MD5 + * candidate that is present must be well formed, and the first one present + * is the one compared, so callers list them in precedence order. An MD5 and + * a CRC64 cannot be sent together. A malformed CRC64 is reported under + * `crc64HeaderName`, since Put Block From URL carries it as + * x-ms-source-content-crc64 rather than x-ms-content-crc64. + * + * Verified against real Azure for CRC64: fewer than 8 bytes is rejected as + * InvalidHeaderValue; 8 or more bytes pass this check and surface as + * Crc64Mismatch if they do not match. + */ +export function validateTransactionalChecksumHeaders( + md5Candidates: Array, + crc64: Uint8Array | string | undefined, + contextId: string | undefined, + crc64HeaderName: string = HeaderConstants.X_MS_CONTENT_CRC64 +): { md5?: Uint8Array | string; crc64?: Uint8Array | string } { + const md5 = md5Candidates.find((candidate) => candidate !== undefined); + if (md5 !== undefined && crc64 !== undefined) { + throw StorageErrorFactory.getBothCrc64AndMd5HeaderPresent(contextId); + } + for (const candidate of md5Candidates) { + if (candidate !== undefined && !isValidMd5Header(candidate)) { + throw StorageErrorFactory.getInvalidMd5(contextId); + } + } + if (crc64 !== undefined) { + const bytes = decodeChecksumHeader(crc64); + if (bytes === undefined || bytes.length < 8) { + throw StorageErrorFactory.getInvalidHeaderValue(contextId, { + HeaderName: crc64HeaderName, + HeaderValue: + typeof crc64 === "string" + ? crc64 + : Buffer.from(crc64).toString("base64") + }); + } + } + return { md5, crc64 }; +} + /** * Computes MD5 and/or CRC-64/NVME from a stream in a single pass and validates * against the request-supplied values. Throws Md5Mismatch / Crc64Mismatch * (HTTP 400) on mismatch - the documented Azure Storage error codes for * transactional integrity failures. * - * Rejects requests that supply both checksums with `BothCrc64AndMd5HeaderPresent` - * (HTTP 400), matching the real Azure service contract. + * The header shapes are checked by validateTransactionalChecksumHeaders + * first, so a request that supplies both checksums or a malformed one is + * rejected before the stream is read. * * A checksum is computed when its `expected` value is provided, OR when the * corresponding `force` flag is set (for callers that need the value for @@ -71,39 +120,11 @@ export async function computeAndValidateTransactionalChecksums( contextId: string | undefined, force?: { md5?: boolean; crc64?: boolean } ): Promise<{ md5?: Uint8Array; crc64?: Uint8Array }> { - if (expected.md5 !== undefined && expected.crc64 !== undefined) { - throw StorageErrorFactory.getBothCrc64AndMd5HeaderPresent(contextId); - } - if (expected.md5 !== undefined && !isValidMd5Header(expected.md5)) { - throw StorageErrorFactory.getInvalidMd5(contextId); - } - const expectedCrc64RawHeader = - typeof expected.crc64 === "string" - ? expected.crc64 - : expected.crc64 !== undefined - ? Buffer.from(expected.crc64).toString("base64") - : undefined; - - const expectedCrc64Bytes = - expected.crc64 === undefined - ? undefined - : typeof expected.crc64 === "string" - ? decodeBase64HeaderValue(expected.crc64) - : Buffer.from(expected.crc64); - - if ( - expected.crc64 !== undefined && - (expectedCrc64Bytes === undefined || expectedCrc64Bytes.length < 8) - ) { - // CRC-64/NVME is a 64-bit value; the wire format is base64-encoded bytes. - // Verified against real Azure: <8 bytes is rejected as InvalidHeaderValue; - // >=8 bytes is accepted at header-validation and falls through to a value - // comparison (which then surfaces as Crc64Mismatch if it doesn't match). - throw StorageErrorFactory.getInvalidHeaderValue(contextId, { - HeaderName: HeaderConstants.X_MS_CONTENT_CRC64, - HeaderValue: expectedCrc64RawHeader ?? "" - }); - } + validateTransactionalChecksumHeaders( + [expected.md5], + expected.crc64, + contextId + ); const calculated = await computeTransactionalChecksums( stream, expected, @@ -111,13 +132,8 @@ export async function computeAndValidateTransactionalChecksums( ); if (expected.md5 !== undefined) { - const expectedMd5Bytes = - typeof expected.md5 === "string" - ? decodeBase64HeaderValue(expected.md5)! - : Buffer.from(expected.md5); - const calculatedMd5Bytes = Buffer.from(calculated.md5!); - const expectedMd5 = expectedMd5Bytes.toString("base64"); - const calculatedMd5 = calculatedMd5Bytes.toString("base64"); + const expectedMd5 = decodeChecksumHeader(expected.md5)!.toString("base64"); + const calculatedMd5 = Buffer.from(calculated.md5!).toString("base64"); if (expectedMd5 !== calculatedMd5) { throw StorageErrorFactory.getMd5Mismatch( contextId, @@ -126,10 +142,11 @@ export async function computeAndValidateTransactionalChecksums( ); } } - if (expectedCrc64Bytes !== undefined) { - const calculatedCrc64Bytes = Buffer.from(calculated.crc64!); - const expectedCrc64 = expectedCrc64Bytes.toString("base64"); - const calculatedCrc64 = calculatedCrc64Bytes.toString("base64"); + if (expected.crc64 !== undefined) { + const expectedCrc64 = decodeChecksumHeader(expected.crc64)!.toString( + "base64" + ); + const calculatedCrc64 = Buffer.from(calculated.crc64!).toString("base64"); if (expectedCrc64 !== calculatedCrc64) { throw StorageErrorFactory.getCrc64Mismatch( contextId, From 1c081c768fb208d2e0a8bab3089a357ae59c2fca Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Thu, 10 Sep 2026 10:39:55 -0700 Subject: [PATCH 8/9] Test Put Blob From URL with a blob SAS and with a read-only SAS The tests named for a blob SAS signed the container, so name them for what they sign and add the blob-scoped pair beside them. Also pin that a SAS with neither Create nor Write is refused at each of the three scopes, which the create and overwrite cases alone left open. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01H7wFx5XWMdmqezhwP3YvZQ --- tests/blob/sas.test.ts | 200 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 197 insertions(+), 3 deletions(-) diff --git a/tests/blob/sas.test.ts b/tests/blob/sas.test.ts index 0a579b589..97e0e7558 100644 --- a/tests/blob/sas.test.ts +++ b/tests/blob/sas.test.ts @@ -16,7 +16,8 @@ import { AppendBlobClient, BlobBatch, Tags, - BlobClient + BlobClient, + BlockBlobClient } from "@azure/storage-blob"; import * as assert from "assert"; @@ -635,6 +636,51 @@ describe("Shared Access Signature (SAS) authentication", () => { assert.deepEqual(error.code, "AuthorizationPermissionMismatch"); }); + it("Put blob from URL without create or write permission in account SAS should fail @loki @sql", async () => { + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + const containerName = getUniqueName("container"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const sourceBlob = containerClient.getBlockBlobClient( + getUniqueName("blob") + ); + await sourceBlob.upload("hello", 5); + const sourceUrl = await sourceBlob.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: tmr + }); + + const serviceClientWithSAS = new BlobServiceClient( + serviceClient.generateAccountSasUrl( + tmr, + AccountSASPermissions.parse("r"), + "co" + ), + newPipeline(new AnonymousCredential(), { + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ); + const blobWithSAS = serviceClientWithSAS + .getContainerClient(containerName) + .getBlockBlobClient(getUniqueName("blob")); + + // Read alone grants neither creating the blob nor overriding it, so this + // copy should throw 403 error + let error; + try { + await blobWithSAS.syncUploadFromURL(sourceUrl); + } catch (err) { + error = err; + } + assert.ok(error !== undefined); + assert.deepEqual(error.statusCode, 403); + assert.deepEqual(error.code, "AuthorizationPermissionMismatch"); + }); + it("Copy blob should work with write permission in account SAS to override an existing blob @loki", async () => { const tmr = new Date(); tmr.setDate(tmr.getDate() + 1); @@ -1776,7 +1822,7 @@ describe("Shared Access Signature (SAS) authentication", () => { await blob2SAS.syncCopyFromURL(blob1.url); }); - it("Put blob from URL with write permission in blob SAS should create and override a blob @loki @sql", async () => { + it("Put blob from URL with write permission in container SAS should create and override a blob @loki @sql", async () => { // Azure grants Put Blob From URL on a new block blob to Create (c) or // Write (w), and on an existing one to Write (w) alone, the split Put // Blob has. Write alone therefore creates as well as overrides. @@ -1813,7 +1859,7 @@ describe("Shared Access Signature (SAS) authentication", () => { await blobWithSAS.syncUploadFromURL(sourceUrl); }); - it("Put blob from URL with create permission in blob SAS should create but not override a blob @loki @sql", async () => { + it("Put blob from URL with create permission in container SAS should create but not override a blob @loki @sql", async () => { const tmr = new Date(); tmr.setDate(tmr.getDate() + 1); @@ -1856,6 +1902,154 @@ describe("Shared Access Signature (SAS) authentication", () => { assert.deepEqual(error.code, "AuthorizationPermissionMismatch"); }); + it("Put blob from URL without create or write permission in container SAS should fail @loki @sql", async () => { + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + const containerName = getUniqueName("container"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const sourceBlob = containerClient.getBlockBlobClient( + getUniqueName("blob") + ); + await sourceBlob.upload("hello", 5); + const sourceUrl = await sourceBlob.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: tmr + }); + + const containerClientWithSAS = new ContainerClient( + await containerClient.generateSasUrl({ + permissions: ContainerSASPermissions.parse("r"), + expiresOn: tmr + }), + newPipeline(new AnonymousCredential()) + ); + const blobWithSAS = containerClientWithSAS.getBlockBlobClient(getUniqueName("blob")); + + // Read alone grants neither creating the blob nor overriding it, so this + // copy should throw 403 error + let error; + try { + await blobWithSAS.syncUploadFromURL(sourceUrl); + } catch (err) { + error = err; + } + assert.ok(error !== undefined); + assert.deepEqual(error.statusCode, 403); + assert.deepEqual(error.code, "AuthorizationPermissionMismatch"); + }); + + it("Put blob from URL with write permission in blob SAS should create and override a blob @loki @sql", async () => { + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + const containerName = getUniqueName("container"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const sourceBlob = containerClient.getBlockBlobClient( + getUniqueName("blob") + ); + await sourceBlob.upload("hello", 5); + const sourceUrl = await sourceBlob.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: tmr + }); + + const blobWithSAS = new BlockBlobClient( + await containerClient.getBlockBlobClient(getUniqueName("blob")).generateSasUrl({ + permissions: BlobSASPermissions.parse("w"), + expiresOn: tmr + }), + newPipeline(new AnonymousCredential()) + ); + + // Neither the copy that creates the blob nor the one that overrides it + // should throw any errors + await blobWithSAS.syncUploadFromURL(sourceUrl); + await blobWithSAS.syncUploadFromURL(sourceUrl); + }); + + it("Put blob from URL with create permission in blob SAS should create but not override a blob @loki @sql", async () => { + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + const containerName = getUniqueName("container"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const sourceBlob = containerClient.getBlockBlobClient( + getUniqueName("blob") + ); + await sourceBlob.upload("hello", 5); + const sourceUrl = await sourceBlob.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: tmr + }); + + const blobWithSAS = new BlockBlobClient( + await containerClient.getBlockBlobClient(getUniqueName("blob")).generateSasUrl({ + permissions: BlobSASPermissions.parse("c"), + expiresOn: tmr + }), + newPipeline(new AnonymousCredential()) + ); + + // this copy creates the blob and should not throw any errors + await blobWithSAS.syncUploadFromURL(sourceUrl); + + // overriding it needs Write, so this copy should throw 403 error + let error; + try { + await blobWithSAS.syncUploadFromURL(sourceUrl); + } catch (err) { + error = err; + } + assert.ok(error !== undefined); + assert.deepEqual(error.statusCode, 403); + assert.deepEqual(error.code, "AuthorizationPermissionMismatch"); + }); + + it("Put blob from URL without create or write permission in blob SAS should fail @loki @sql", async () => { + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + const containerName = getUniqueName("container"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const sourceBlob = containerClient.getBlockBlobClient( + getUniqueName("blob") + ); + await sourceBlob.upload("hello", 5); + const sourceUrl = await sourceBlob.generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: tmr + }); + + const blobWithSAS = new BlockBlobClient( + await containerClient.getBlockBlobClient(getUniqueName("blob")).generateSasUrl({ + permissions: BlobSASPermissions.parse("r"), + expiresOn: tmr + }), + newPipeline(new AnonymousCredential()) + ); + + // Read alone grants neither creating the blob nor overriding it, so this + // copy should throw 403 error + let error; + try { + await blobWithSAS.syncUploadFromURL(sourceUrl); + } catch (err) { + error = err; + } + assert.ok(error !== undefined); + assert.deepEqual(error.statusCode, 403); + assert.deepEqual(error.code, "AuthorizationPermissionMismatch"); + }); + it("Copy blob should work with write permission in blob SAS to override an existing blob @loki @sql", async () => { const now = new Date(); now.setMinutes(now.getMinutes() - 5); // Skip clock skew with server From c32e860adb5aa76417d7f6aab445c0822bb6668b Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Thu, 10 Sep 2026 10:39:55 -0700 Subject: [PATCH 9/9] Hold Put Blob From URL tag writes to the Tag permission Azure requires a request that sets tags with x-ms-tags, or copies the source's with x-ms-copy-source-tag-option: COPY, to satisfy Set Blob Tags authorization on the destination on top of the write. Check that in the account and blob SAS authenticators for this operation, so a SAS without Tag is refused with AuthorizationPermissionMismatch, and leave requests that set no tags to Create or Write alone. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01H7wFx5XWMdmqezhwP3YvZQ --- ChangeLog.md | 2 +- .../authentication/AccountSASAuthenticator.ts | 27 ++- .../authentication/BlobSASAuthenticator.ts | 31 ++- src/blob/utils/constants.ts | 2 + tests/blob/sas.test.ts | 189 ++++++++++++++++++ 5 files changed, 246 insertions(+), 5 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index f41492357..1ffab02ff 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -28,7 +28,7 @@ General: Blob: - Fixed blob operations hanging when a client disconnects before the operation queue processes the request. (issue #2575) -- Implement `PutBlobFromUrl` (`Put Blob From URL`), which previously returned 501. The source is fetched over loopback, as `PutBlockFromURL` already does, so that SAS authentication and the `x-ms-source-if-*` conditions are enforced by the existing download path. Standard blob properties are copied from the source unless `x-ms-copy-source-blob-properties` is false, request blob content headers override them either way, request metadata replaces the source's rather than adding to it, and `x-ms-copy-source-tag-option: COPY` reads the source's tags over that same authorized path. An `x-ms-source-content-md5`, `x-ms-blob-content-md5`, `Content-MD5`, or `x-ms-content-crc64` header is checked against the copied content, and the response reports the MD5 and CRC64 of that content. As with `CopyBlobFromURL`, only sources on the same Azurite instance are supported. +- Implement `PutBlobFromUrl` (`Put Blob From URL`), which previously returned 501. The source is fetched over loopback, as `PutBlockFromURL` already does, so that SAS authentication and the `x-ms-source-if-*` conditions are enforced by the existing download path. Standard blob properties are copied from the source unless `x-ms-copy-source-blob-properties` is false, request blob content headers override them either way, request metadata replaces the source's rather than adding to it, and `x-ms-copy-source-tag-option: COPY` reads the source's tags over that same authorized path. An `x-ms-source-content-md5`, `x-ms-blob-content-md5`, `Content-MD5`, or `x-ms-content-crc64` header is checked against the copied content, and the response reports the MD5 and CRC64 of that content. A SAS needs Create or Write to create the blob, Write to overwrite it, and Tag as well when the request sets tags with `x-ms-tags` or copies the source's. As with `CopyBlobFromURL`, only sources on the same Azurite instance are supported. Table: diff --git a/src/blob/authentication/AccountSASAuthenticator.ts b/src/blob/authentication/AccountSASAuthenticator.ts index 7cbb4247a..39b4b5ab1 100644 --- a/src/blob/authentication/AccountSASAuthenticator.ts +++ b/src/blob/authentication/AccountSASAuthenticator.ts @@ -1,7 +1,7 @@ import IAccountDataStore from "../../common/IAccountDataStore"; import ILogger from "../../common/ILogger"; import StorageErrorFactory from "../errors/StorageErrorFactory"; -import { BlobType } from "../generated/artifacts/models"; +import { BlobCopySourceTags, BlobType } from "../generated/artifacts/models"; import Operation from "../generated/artifacts/operation"; import Context from "../generated/Context"; import IRequest from "../generated/IRequest"; @@ -14,7 +14,10 @@ import { import IAuthenticator from "./IAuthenticator"; import OPERATION_ACCOUNT_SAS_PERMISSIONS from "./OperationAccountSASPermission"; import StrictModelNotSupportedError from "../errors/StrictModelNotSupportedError"; -import { AUTHENTICATION_BEARERTOKEN_REQUIRED } from "../utils/constants"; +import { + AUTHENTICATION_BEARERTOKEN_REQUIRED, + HeaderConstants +} from "../utils/constants"; export default class AccountSASAuthenticator implements IAuthenticator { public constructor( @@ -281,6 +284,26 @@ export default class AccountSASAuthenticator implements IAuthenticator { } } + // Put Blob From URL sets tags on the destination when the request names + // them in x-ms-tags or asks for the source's to be copied, and Azure + // holds either to the Set Blob Tags permission on top of the write. A + // request that sets no tags takes only Create or Write. + if ( + operation === Operation.BlockBlob_PutBlobFromUrl && + (req.getHeader(HeaderConstants.X_MS_TAGS) !== undefined || + req.getHeader(HeaderConstants.X_MS_COPY_SOURCE_TAG_OPTION) === + BlobCopySourceTags.COPY) && + !values.permissions.toString().includes(AccountSASPermission.Tag) + ) { + this.logger.info( + `AccountSASAuthenticator:validate() For ${Operation[operation]}, setting tags on the destination requires the Tag permission.`, + context.contextId + ); + throw StorageErrorFactory.getAuthorizationPermissionMismatch( + context.contextId! + ); + } + this.logger.info( `AccountSASAuthenticator:validate() Account SAS validation successfully.`, context.contextId diff --git a/src/blob/authentication/BlobSASAuthenticator.ts b/src/blob/authentication/BlobSASAuthenticator.ts index e0d575b61..db58186e5 100644 --- a/src/blob/authentication/BlobSASAuthenticator.ts +++ b/src/blob/authentication/BlobSASAuthenticator.ts @@ -3,12 +3,19 @@ import ILogger from "../../common/ILogger"; import BlobStorageContext from "../context/BlobStorageContext"; import StorageErrorFactory from "../errors/StorageErrorFactory"; import StrictModelNotSupportedError from "../errors/StrictModelNotSupportedError"; -import { AccessPolicy, BlobType } from "../generated/artifacts/models"; +import { + AccessPolicy, + BlobCopySourceTags, + BlobType +} from "../generated/artifacts/models"; import Operation from "../generated/artifacts/operation"; import Context from "../generated/Context"; import IRequest from "../generated/IRequest"; import IBlobMetadataStore from "../persistence/IBlobMetadataStore"; -import { AUTHENTICATION_BEARERTOKEN_REQUIRED } from "../utils/constants"; +import { + AUTHENTICATION_BEARERTOKEN_REQUIRED, + HeaderConstants +} from "../utils/constants"; import { getUserDelegationKeyValue } from "../utils/utils"; import { BlobSASPermission } from "./BlobSASPermissions"; import { BlobSASResourceType } from "./BlobSASResourceType"; @@ -443,6 +450,26 @@ export default class BlobSASAuthenticator implements IAuthenticator { } } + // Put Blob From URL sets tags on the destination when the request names + // them in x-ms-tags or asks for the source's to be copied, and Azure + // holds either to the Set Blob Tags permission on top of the write. A + // request that sets no tags takes only Create or Write. + if ( + operation === Operation.BlockBlob_PutBlobFromUrl && + (req.getHeader(HeaderConstants.X_MS_TAGS) !== undefined || + req.getHeader(HeaderConstants.X_MS_COPY_SOURCE_TAG_OPTION) === + BlobCopySourceTags.COPY) && + !values.permissions!.toString().includes(BlobSASPermission.Tag) + ) { + this.logger.info( + `BlobSASAuthenticator:validate() For ${Operation[operation]}, setting tags on the destination requires the Tag permission.`, + context.contextId + ); + throw StorageErrorFactory.getAuthorizationPermissionMismatch( + context.contextId! + ); + } + this.logger.info( `BlobSASAuthenticator:validate() Blob service SAS validation successfully.`, context.contextId diff --git a/src/blob/utils/constants.ts b/src/blob/utils/constants.ts index c827f12c1..649fe8342 100644 --- a/src/blob/utils/constants.ts +++ b/src/blob/utils/constants.ts @@ -60,6 +60,8 @@ export const HeaderConstants = { X_MS_BLOB_SEQUENCE_NUMBER: "x-ms-blob-sequence-number", X_MS_CONTENT_CRC64: "x-ms-content-crc64", X_MS_SOURCE_CONTENT_CRC64: "x-ms-source-content-crc64", + X_MS_TAGS: "x-ms-tags", + X_MS_COPY_SOURCE_TAG_OPTION: "x-ms-copy-source-tag-option", X_MS_RANGE_GET_CONTENT_CRC64: "x-ms-range-get-content-crc64", X_MS_ENCRYPTION_KEY: "x-ms-encryption-key", X_MS_ENCRYPTION_KEY_SHA256: "x-ms-encryption-key-sha256", diff --git a/tests/blob/sas.test.ts b/tests/blob/sas.test.ts index 97e0e7558..d51786b52 100644 --- a/tests/blob/sas.test.ts +++ b/tests/blob/sas.test.ts @@ -681,6 +681,73 @@ describe("Shared Access Signature (SAS) authentication", () => { assert.deepEqual(error.code, "AuthorizationPermissionMismatch"); }); + it("Put blob from URL setting tags needs the tag permission in account SAS @loki @sql", async () => { + // Azure holds a request that sets tags on the destination, whether from + // x-ms-tags or by copying the source's, to the Set Blob Tags permission + // on top of the write. The source SAS carries Tag so that only the + // destination SAS decides the outcome. + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + const containerName = getUniqueName("container"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const sourceBlob = containerClient.getBlockBlobClient( + getUniqueName("blob") + ); + await sourceBlob.upload("hello", 5); + await sourceBlob.setTags({ origin: "source" }); + const sourceUrl = await sourceBlob.generateSasUrl({ + permissions: BlobSASPermissions.parse("rt"), + expiresOn: tmr + }); + const blobName = getUniqueName("blob"); + const tags = { key: "value" }; + const destination = (permissions: string) => + new BlobServiceClient( + serviceClient.generateAccountSasUrl( + tmr, + AccountSASPermissions.parse(permissions), + "co" + ), + newPipeline(new AnonymousCredential(), { + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + }) + ) + .getContainerClient(containerName) + .getBlockBlobClient(blobName); + + // Write alone does not set tags, so both copies should throw 403 error + const writeOnly = await destination("w"); + for (const options of [{ tags }, { copySourceTags: "COPY" as const }]) { + let error; + try { + await writeOnly.syncUploadFromURL(sourceUrl, options); + } catch (err) { + error = err; + } + assert.ok(error !== undefined); + assert.deepEqual(error.statusCode, 403); + assert.deepEqual(error.code, "AuthorizationPermissionMismatch"); + } + + // With Tag as well, the request's tags are set on the destination and + // the source's tags can be copied to it + const writeAndTag = await destination("wt"); + await writeAndTag.syncUploadFromURL(sourceUrl, { tags }); + assert.deepStrictEqual( + (await containerClient.getBlockBlobClient(blobName).getTags()).tags, + tags + ); + await writeAndTag.syncUploadFromURL(sourceUrl, { copySourceTags: "COPY" }); + assert.deepStrictEqual( + (await containerClient.getBlockBlobClient(blobName).getTags()).tags, + { origin: "source" } + ); + }); + it("Copy blob should work with write permission in account SAS to override an existing blob @loki", async () => { const tmr = new Date(); tmr.setDate(tmr.getDate() + 1); @@ -2050,6 +2117,128 @@ describe("Shared Access Signature (SAS) authentication", () => { assert.deepEqual(error.code, "AuthorizationPermissionMismatch"); }); + it("Put blob from URL setting tags needs the tag permission in container SAS @loki @sql", async () => { + // Azure holds a request that sets tags on the destination, whether from + // x-ms-tags or by copying the source's, to the Set Blob Tags permission + // on top of the write. The source SAS carries Tag so that only the + // destination SAS decides the outcome. + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + const containerName = getUniqueName("container"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const sourceBlob = containerClient.getBlockBlobClient( + getUniqueName("blob") + ); + await sourceBlob.upload("hello", 5); + await sourceBlob.setTags({ origin: "source" }); + const sourceUrl = await sourceBlob.generateSasUrl({ + permissions: BlobSASPermissions.parse("rt"), + expiresOn: tmr + }); + const blobName = getUniqueName("blob"); + const tags = { key: "value" }; + const destination = async (permissions: string) => + new ContainerClient( + await containerClient.generateSasUrl({ + permissions: ContainerSASPermissions.parse(permissions), + expiresOn: tmr + }), + newPipeline(new AnonymousCredential()) + ).getBlockBlobClient(blobName); + + // Write alone does not set tags, so both copies should throw 403 error + const writeOnly = await destination("w"); + for (const options of [{ tags }, { copySourceTags: "COPY" as const }]) { + let error; + try { + await writeOnly.syncUploadFromURL(sourceUrl, options); + } catch (err) { + error = err; + } + assert.ok(error !== undefined); + assert.deepEqual(error.statusCode, 403); + assert.deepEqual(error.code, "AuthorizationPermissionMismatch"); + } + + // With Tag as well, the request's tags are set on the destination and + // the source's tags can be copied to it + const writeAndTag = await destination("wt"); + await writeAndTag.syncUploadFromURL(sourceUrl, { tags }); + assert.deepStrictEqual( + (await containerClient.getBlockBlobClient(blobName).getTags()).tags, + tags + ); + await writeAndTag.syncUploadFromURL(sourceUrl, { copySourceTags: "COPY" }); + assert.deepStrictEqual( + (await containerClient.getBlockBlobClient(blobName).getTags()).tags, + { origin: "source" } + ); + }); + + it("Put blob from URL setting tags needs the tag permission in blob SAS @loki @sql", async () => { + // Azure holds a request that sets tags on the destination, whether from + // x-ms-tags or by copying the source's, to the Set Blob Tags permission + // on top of the write. The source SAS carries Tag so that only the + // destination SAS decides the outcome. + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + + const containerName = getUniqueName("container"); + const containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + + const sourceBlob = containerClient.getBlockBlobClient( + getUniqueName("blob") + ); + await sourceBlob.upload("hello", 5); + await sourceBlob.setTags({ origin: "source" }); + const sourceUrl = await sourceBlob.generateSasUrl({ + permissions: BlobSASPermissions.parse("rt"), + expiresOn: tmr + }); + const blobName = getUniqueName("blob"); + const tags = { key: "value" }; + const destination = async (permissions: string) => + new BlockBlobClient( + await containerClient.getBlockBlobClient(blobName).generateSasUrl({ + permissions: BlobSASPermissions.parse(permissions), + expiresOn: tmr + }), + newPipeline(new AnonymousCredential()) + ); + + // Write alone does not set tags, so both copies should throw 403 error + const writeOnly = await destination("w"); + for (const options of [{ tags }, { copySourceTags: "COPY" as const }]) { + let error; + try { + await writeOnly.syncUploadFromURL(sourceUrl, options); + } catch (err) { + error = err; + } + assert.ok(error !== undefined); + assert.deepEqual(error.statusCode, 403); + assert.deepEqual(error.code, "AuthorizationPermissionMismatch"); + } + + // With Tag as well, the request's tags are set on the destination and + // the source's tags can be copied to it + const writeAndTag = await destination("wt"); + await writeAndTag.syncUploadFromURL(sourceUrl, { tags }); + assert.deepStrictEqual( + (await containerClient.getBlockBlobClient(blobName).getTags()).tags, + tags + ); + await writeAndTag.syncUploadFromURL(sourceUrl, { copySourceTags: "COPY" }); + assert.deepStrictEqual( + (await containerClient.getBlockBlobClient(blobName).getTags()).tags, + { origin: "source" } + ); + }); + it("Copy blob should work with write permission in blob SAS to override an existing blob @loki @sql", async () => { const now = new Date(); now.setMinutes(now.getMinutes() - 5); // Skip clock skew with server