From 017eacc3417cc8c77f2e7ca1f92cc1be5d54ac1e Mon Sep 17 00:00:00 2001 From: Wish Date: Tue, 27 May 2025 11:49:35 +0800 Subject: [PATCH] Restore keys by using the prefix Signed-off-by: Wish --- dist/restore-only/index.js | 34 +++++++++++++++++++++------------ dist/restore/index.js | 34 +++++++++++++++++++++------------ dist/save-only/index.js | 34 +++++++++++++++++++++------------ dist/save/index.js | 34 +++++++++++++++++++++------------ src/utils/gcsCache.ts | 39 +++++++++++++++++++++++--------------- 5 files changed, 112 insertions(+), 63 deletions(-) diff --git a/dist/restore-only/index.js b/dist/restore-only/index.js index 37c95cba8..20e1c9ffc 100644 --- a/dist/restore-only/index.js +++ b/dist/restore-only/index.js @@ -92559,7 +92559,7 @@ function restoreFromGCS(_paths_1, primaryKey_1) { const archiveFolder = yield utils.createTempDirectory(); const archivePath = path.join(archiveFolder, utils.getCacheFileName(compressionMethod)); const keys = [primaryKey, ...restoreKeys]; - const gcsPath = yield findFileOnGCS(storage, bucket, pathPrefix, keys, compressionMethod); + const gcsPath = yield findLatestFileOnGCS(storage, bucket, pathPrefix, keys, compressionMethod); if (!gcsPath) { core.info(`No matching cache found`); return undefined; @@ -92643,24 +92643,34 @@ function saveToGCS(paths, key) { } }); } -function findFileOnGCS(storage, bucket, pathPrefix, keys, compressionMethod) { +function findLatestFileOnGCS(storage, bucket, pathPrefix, keys, compressionMethod) { return __awaiter(this, void 0, void 0, function* () { + let latestFile = undefined; for (const key of keys) { - const gcsPath = getGCSPath(pathPrefix, key, compressionMethod); - if (yield checkFileExists(storage, bucket, gcsPath)) { - core.info(`Found file on bucket: ${bucket} with key: ${gcsPath}`); - return gcsPath; + const prefix = `${pathPrefix}/${key}`; + const [files] = yield storage.bucket(bucket).getFiles({ prefix }); + for (const file of files) { + if (!file.name.endsWith(utils.getCacheFileName(compressionMethod))) + continue; + core.debug(`Found file: ${file.name} (created: ${file.metadata.timeCreated})`); + const created = file.metadata.timeCreated + ? new Date(file.metadata.timeCreated) + : undefined; + if (!created) { + continue; + } + if (!latestFile || created > latestFile.updated) { + latestFile = { path: file.name, updated: created }; + } } } + if (latestFile) { + core.info(`Use cache: ${latestFile.path} from GCS bucket ${bucket}`); + return latestFile.path; + } return undefined; }); } -function checkFileExists(storage, bucket, path) { - return __awaiter(this, void 0, void 0, function* () { - const [exists] = yield storage.bucket(bucket).file(path).exists(); - return exists; - }); -} /***/ }), diff --git a/dist/restore/index.js b/dist/restore/index.js index ed3cf8745..2f55449ee 100644 --- a/dist/restore/index.js +++ b/dist/restore/index.js @@ -92559,7 +92559,7 @@ function restoreFromGCS(_paths_1, primaryKey_1) { const archiveFolder = yield utils.createTempDirectory(); const archivePath = path.join(archiveFolder, utils.getCacheFileName(compressionMethod)); const keys = [primaryKey, ...restoreKeys]; - const gcsPath = yield findFileOnGCS(storage, bucket, pathPrefix, keys, compressionMethod); + const gcsPath = yield findLatestFileOnGCS(storage, bucket, pathPrefix, keys, compressionMethod); if (!gcsPath) { core.info(`No matching cache found`); return undefined; @@ -92643,24 +92643,34 @@ function saveToGCS(paths, key) { } }); } -function findFileOnGCS(storage, bucket, pathPrefix, keys, compressionMethod) { +function findLatestFileOnGCS(storage, bucket, pathPrefix, keys, compressionMethod) { return __awaiter(this, void 0, void 0, function* () { + let latestFile = undefined; for (const key of keys) { - const gcsPath = getGCSPath(pathPrefix, key, compressionMethod); - if (yield checkFileExists(storage, bucket, gcsPath)) { - core.info(`Found file on bucket: ${bucket} with key: ${gcsPath}`); - return gcsPath; + const prefix = `${pathPrefix}/${key}`; + const [files] = yield storage.bucket(bucket).getFiles({ prefix }); + for (const file of files) { + if (!file.name.endsWith(utils.getCacheFileName(compressionMethod))) + continue; + core.debug(`Found file: ${file.name} (created: ${file.metadata.timeCreated})`); + const created = file.metadata.timeCreated + ? new Date(file.metadata.timeCreated) + : undefined; + if (!created) { + continue; + } + if (!latestFile || created > latestFile.updated) { + latestFile = { path: file.name, updated: created }; + } } } + if (latestFile) { + core.info(`Use cache: ${latestFile.path} from GCS bucket ${bucket}`); + return latestFile.path; + } return undefined; }); } -function checkFileExists(storage, bucket, path) { - return __awaiter(this, void 0, void 0, function* () { - const [exists] = yield storage.bucket(bucket).file(path).exists(); - return exists; - }); -} /***/ }), diff --git a/dist/save-only/index.js b/dist/save-only/index.js index 4af28e44e..7950b6800 100644 --- a/dist/save-only/index.js +++ b/dist/save-only/index.js @@ -92572,7 +92572,7 @@ function restoreFromGCS(_paths_1, primaryKey_1) { const archiveFolder = yield utils.createTempDirectory(); const archivePath = path.join(archiveFolder, utils.getCacheFileName(compressionMethod)); const keys = [primaryKey, ...restoreKeys]; - const gcsPath = yield findFileOnGCS(storage, bucket, pathPrefix, keys, compressionMethod); + const gcsPath = yield findLatestFileOnGCS(storage, bucket, pathPrefix, keys, compressionMethod); if (!gcsPath) { core.info(`No matching cache found`); return undefined; @@ -92656,24 +92656,34 @@ function saveToGCS(paths, key) { } }); } -function findFileOnGCS(storage, bucket, pathPrefix, keys, compressionMethod) { +function findLatestFileOnGCS(storage, bucket, pathPrefix, keys, compressionMethod) { return __awaiter(this, void 0, void 0, function* () { + let latestFile = undefined; for (const key of keys) { - const gcsPath = getGCSPath(pathPrefix, key, compressionMethod); - if (yield checkFileExists(storage, bucket, gcsPath)) { - core.info(`Found file on bucket: ${bucket} with key: ${gcsPath}`); - return gcsPath; + const prefix = `${pathPrefix}/${key}`; + const [files] = yield storage.bucket(bucket).getFiles({ prefix }); + for (const file of files) { + if (!file.name.endsWith(utils.getCacheFileName(compressionMethod))) + continue; + core.debug(`Found file: ${file.name} (created: ${file.metadata.timeCreated})`); + const created = file.metadata.timeCreated + ? new Date(file.metadata.timeCreated) + : undefined; + if (!created) { + continue; + } + if (!latestFile || created > latestFile.updated) { + latestFile = { path: file.name, updated: created }; + } } } + if (latestFile) { + core.info(`Use cache: ${latestFile.path} from GCS bucket ${bucket}`); + return latestFile.path; + } return undefined; }); } -function checkFileExists(storage, bucket, path) { - return __awaiter(this, void 0, void 0, function* () { - const [exists] = yield storage.bucket(bucket).file(path).exists(); - return exists; - }); -} /***/ }), diff --git a/dist/save/index.js b/dist/save/index.js index 75583a251..84844532a 100644 --- a/dist/save/index.js +++ b/dist/save/index.js @@ -92572,7 +92572,7 @@ function restoreFromGCS(_paths_1, primaryKey_1) { const archiveFolder = yield utils.createTempDirectory(); const archivePath = path.join(archiveFolder, utils.getCacheFileName(compressionMethod)); const keys = [primaryKey, ...restoreKeys]; - const gcsPath = yield findFileOnGCS(storage, bucket, pathPrefix, keys, compressionMethod); + const gcsPath = yield findLatestFileOnGCS(storage, bucket, pathPrefix, keys, compressionMethod); if (!gcsPath) { core.info(`No matching cache found`); return undefined; @@ -92656,24 +92656,34 @@ function saveToGCS(paths, key) { } }); } -function findFileOnGCS(storage, bucket, pathPrefix, keys, compressionMethod) { +function findLatestFileOnGCS(storage, bucket, pathPrefix, keys, compressionMethod) { return __awaiter(this, void 0, void 0, function* () { + let latestFile = undefined; for (const key of keys) { - const gcsPath = getGCSPath(pathPrefix, key, compressionMethod); - if (yield checkFileExists(storage, bucket, gcsPath)) { - core.info(`Found file on bucket: ${bucket} with key: ${gcsPath}`); - return gcsPath; + const prefix = `${pathPrefix}/${key}`; + const [files] = yield storage.bucket(bucket).getFiles({ prefix }); + for (const file of files) { + if (!file.name.endsWith(utils.getCacheFileName(compressionMethod))) + continue; + core.debug(`Found file: ${file.name} (created: ${file.metadata.timeCreated})`); + const created = file.metadata.timeCreated + ? new Date(file.metadata.timeCreated) + : undefined; + if (!created) { + continue; + } + if (!latestFile || created > latestFile.updated) { + latestFile = { path: file.name, updated: created }; + } } } + if (latestFile) { + core.info(`Use cache: ${latestFile.path} from GCS bucket ${bucket}`); + return latestFile.path; + } return undefined; }); } -function checkFileExists(storage, bucket, path) { - return __awaiter(this, void 0, void 0, function* () { - const [exists] = yield storage.bucket(bucket).file(path).exists(); - return exists; - }); -} /***/ }), diff --git a/src/utils/gcsCache.ts b/src/utils/gcsCache.ts index 041335ba7..5af8be6c9 100644 --- a/src/utils/gcsCache.ts +++ b/src/utils/gcsCache.ts @@ -128,7 +128,7 @@ async function restoreFromGCS( ); const keys = [primaryKey, ...restoreKeys]; - const gcsPath = await findFileOnGCS( + const gcsPath = await findLatestFileOnGCS( storage, bucket, pathPrefix, @@ -248,28 +248,37 @@ async function saveToGCS( } } -async function findFileOnGCS( +async function findLatestFileOnGCS( storage: Storage, bucket: string, pathPrefix: string, keys: string[], compressionMethod: CompressionMethod ): Promise { + let latestFile: { path: string; updated: Date } | undefined = undefined; for (const key of keys) { - const gcsPath = getGCSPath(pathPrefix, key, compressionMethod); - if (await checkFileExists(storage, bucket, gcsPath)) { - core.info(`Found file on bucket: ${bucket} with key: ${gcsPath}`); - return gcsPath; + const prefix = `${pathPrefix}/${key}`; + const [files] = await storage.bucket(bucket).getFiles({ prefix }); + for (const file of files) { + if (!file.name.endsWith(utils.getCacheFileName(compressionMethod))) + continue; + core.debug( + `Found file: ${file.name} (created: ${file.metadata.timeCreated})` + ); + const created = file.metadata.timeCreated + ? new Date(file.metadata.timeCreated) + : undefined; + if (!created) { + continue; + } + if (!latestFile || created > latestFile.updated) { + latestFile = { path: file.name, updated: created }; + } } } + if (latestFile) { + core.info(`Use cache: ${latestFile.path} from GCS bucket ${bucket}`); + return latestFile.path; + } return undefined; } - -async function checkFileExists( - storage: Storage, - bucket: string, - path: string -): Promise { - const [exists] = await storage.bucket(bucket).file(path).exists(); - return exists; -}