Skip to content

refactor(storage): dynamically import mime and p-limit with module caching #9230

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
thiyaguk09 wants to merge 4 commits into
base: main
Choose a base branch
from
Draft

refactor(storage): dynamically import mime and p-limit with module caching #9230

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
7c16371
refactor: implement lazy loading for mime and p-limit dependencies to…
thiyaguk09 Sep 1, 2026
60401ce
Merge branch 'main' into feat/7350-dynamic-imports-perf
thiyaguk09 Sep 1, 2026
9d19603
Merge branch 'main' into feat/7350-dynamic-imports-perf
thiyaguk09 Sep 3, 2026
1845423
Merge branch 'main' into feat/7350-dynamic-imports-perf
thiyaguk09 Sep 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
217 changes: 116 additions & 101 deletions handwritten/storage/src/bucket.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,15 @@ import {paginator} from '@google-cloud/paginator';
import {promisifyAll} from '@google-cloud/promisify';
import * as fs from 'fs';
import * as http from 'http';
import mime from 'mime';
import * as path from 'path';
import pLimit from 'p-limit';
import {promisify} from 'util';
import AsyncRetry from 'async-retry';
import {convertObjKeysToSnakeCase, handleContextValidation} from './util.js';
import {
convertObjKeysToSnakeCase,
handleContextValidation,
getMime,
getPLimit,
} from './util.js';

import {Acl, AclMetadata} from './acl.js';
import {Channel} from './channel.js';
Expand Down Expand Up @@ -1733,118 +1736,128 @@ class Bucket extends ServiceObject<Bucket, BucketMetadata> {
const destinationFile = convertToFile(destination);
callback = callback || util.noop;

if (!destinationFile.metadata.contentType) {
const destinationContentType =
mime.getType(destinationFile.name) || undefined;
void (async () => {
try {
if (!destinationFile.metadata.contentType) {
const mime = await getMime();
const destinationContentType =
mime.getType(destinationFile.name) || undefined;

if (destinationContentType) {
destinationFile.metadata.contentType = destinationContentType;
}
}
if (destinationContentType) {
destinationFile.metadata.contentType = destinationContentType;
}
}

let maxRetries = this.storage.retryOptions.maxRetries;
if (
(destinationFile?.instancePreconditionOpts?.ifGenerationMatch ===
undefined &&
options.ifGenerationMatch === undefined &&
this.storage.retryOptions.idempotencyStrategy ===
IdempotencyStrategy.RetryConditional) ||
this.storage.retryOptions.idempotencyStrategy ===
IdempotencyStrategy.RetryNever
) {
maxRetries = 0;
}
let maxRetries = this.storage.retryOptions.maxRetries;
if (
(destinationFile?.instancePreconditionOpts?.ifGenerationMatch ===
undefined &&
options.ifGenerationMatch === undefined &&
this.storage.retryOptions.idempotencyStrategy ===
IdempotencyStrategy.RetryConditional) ||
this.storage.retryOptions.idempotencyStrategy ===
IdempotencyStrategy.RetryNever
) {
maxRetries = 0;
}

const deleteSourceObjects = options.deleteSourceObjects;
const deleteSourceObjects = options.deleteSourceObjects;

const requestQueryObject = Object.assign({}, options);
delete requestQueryObject.deleteSourceObjects;
const requestQueryObject = Object.assign({}, options);
delete requestQueryObject.deleteSourceObjects;

if (requestQueryObject.ifGenerationMatch === undefined) {
Object.assign(
requestQueryObject,
destinationFile.instancePreconditionOpts,
requestQueryObject
);
}
if (requestQueryObject.ifGenerationMatch === undefined) {
Object.assign(
requestQueryObject,
destinationFile.instancePreconditionOpts,
requestQueryObject
);
}

// Make the request from the destination File object.
destinationFile.request(
{
method: 'POST',
uri: '/compose',
maxRetries,
json: {
destination: {
contentType: destinationFile.metadata.contentType,
contentEncoding: destinationFile.metadata.contentEncoding,
contexts:
requestQueryObject.contexts || destinationFile.metadata.contexts,
// Make the request from the destination File object.
destinationFile.request(
{
method: 'POST',
uri: '/compose',
maxRetries,
json: {
destination: {
contentType: destinationFile.metadata.contentType,
contentEncoding: destinationFile.metadata.contentEncoding,
contexts:
requestQueryObject.contexts ||
destinationFile.metadata.contexts,
},
sourceObjects: (sources as File[]).map(source => {
const sourceObject = {
name: source.name,
} as SourceObject;

const generation =
source.generation ?? source.metadata?.generation;
if (generation !== undefined) {
sourceObject.generation = parseInt(generation.toString());
}

return sourceObject;
}),
},
qs: requestQueryObject,
},
sourceObjects: (sources as File[]).map(source => {
const sourceObject = {
name: source.name,
} as SourceObject;

const generation = source.generation ?? source.metadata?.generation;
if (generation !== undefined) {
sourceObject.generation = parseInt(generation.toString());
(err, resp) => {
this.storage.retryOptions.autoRetry = this.instanceRetryValue;
if (err) {
callback!(err, null, resp);
return;
}

return sourceObject;
}),
},
qs: requestQueryObject,
},
(err, resp) => {
this.storage.retryOptions.autoRetry = this.instanceRetryValue;
if (err) {
callback!(err, null, resp);
return;
}

if (deleteSourceObjects) {
const deletePromises = (sources as File[]).map(source => {
const deleteOptions: DeleteOptions = {
ignoreNotFound: true,
userProject: options.userProject,
};
if (deleteSourceObjects) {
const deletePromises = (sources as File[]).map(source => {
const deleteOptions: DeleteOptions = {
ignoreNotFound: true,
userProject: options.userProject,
};

const generation =
source.generation ?? source.metadata?.generation;
if (generation !== undefined) {
deleteOptions.ifGenerationMatch = generation;
}

const generation = source.generation ?? source.metadata?.generation;
if (generation !== undefined) {
deleteOptions.ifGenerationMatch = generation;
}
return source
.delete(deleteOptions)
.catch(deleteErr => deleteErr as Error);
});

return source
.delete(deleteOptions)
.catch(deleteErr => deleteErr as Error);
});
void (async () => {
// eslint-disable-next-line promise/no-promise-in-callback
const results = await Promise.all(deletePromises);
const errors = results.filter(
(res): res is Error => res instanceof Error
);

if (errors.length > 0) {
const cleanupErr = new ComposeCleanupError(
`Compose operation succeeded, but cleaning up source objects failed. Failed to delete ${errors.length} source object(s).`,
errors,
destinationFile,
resp
);
callback!(cleanupErr, destinationFile, resp);
return;
}

void (async () => {
// eslint-disable-next-line promise/no-promise-in-callback
const results = await Promise.all(deletePromises);
const errors = results.filter(
(res): res is Error => res instanceof Error
);

if (errors.length > 0) {
const cleanupErr = new ComposeCleanupError(
`Compose operation succeeded, but cleaning up source objects failed. Failed to delete ${errors.length} source object(s).`,
errors,
destinationFile,
resp
);
callback!(cleanupErr, destinationFile, resp);
return;
callback!(null, destinationFile, resp);
})();
} else {
callback!(null, destinationFile, resp);
}

callback!(null, destinationFile, resp);
})();
} else {
callback!(null, destinationFile, resp);
}
}
);
} catch (err) {
callback!(err as Error, null, null);
}
);
})();
}

createChannel(
Expand Down Expand Up @@ -2288,6 +2301,7 @@ class Bucket extends ServiceObject<Bucket, BucketMetadata> {
void (async () => {
try {
let promises = [];
const pLimit = await getPLimit();
const limit = pLimit(MAX_PARALLEL_LIMIT);
const filesStream = this.getFilesStream(query);

Expand Down Expand Up @@ -4721,6 +4735,7 @@ class Bucket extends ServiceObject<Bucket, BucketMetadata> {
void (async () => {
try {
const [files] = await this.getFiles(options);
const pLimit = await getPLimit();
const limit = pLimit(MAX_PARALLEL_LIMIT);
const promises = files.map(file => {
return limit(() => processFile(file));
Expand Down
Loading
Loading