diff --git a/lib/analyze-action.js b/lib/analyze-action.js index 8ecb3977bf..4554cd1838 100644 --- a/lib/analyze-action.js +++ b/lib/analyze-action.js @@ -44531,11 +44531,11 @@ var require_valid = __commonJS({ "node_modules/semver/functions/valid.js"(exports2, module2) { "use strict"; var parse2 = require_parse3(); - var valid3 = (version, options) => { + var valid4 = (version, options) => { const v = parse2(version, options); return v ? v.version : null; }; - module2.exports = valid3; + module2.exports = valid4; } }); @@ -44678,8 +44678,8 @@ var require_rcompare = __commonJS({ "node_modules/semver/functions/rcompare.js"(exports2, module2) { "use strict"; var compare3 = require_compare(); - var rcompare = (a, b, loose) => compare3(b, a, loose); - module2.exports = rcompare; + var rcompare2 = (a, b, loose) => compare3(b, a, loose); + module2.exports = rcompare2; } }); @@ -45895,7 +45895,7 @@ var require_semver2 = __commonJS({ var SemVer = require_semver(); var identifiers = require_identifiers(); var parse2 = require_parse3(); - var valid3 = require_valid(); + var valid4 = require_valid(); var clean3 = require_clean(); var inc = require_inc(); var diff = require_diff(); @@ -45904,7 +45904,7 @@ var require_semver2 = __commonJS({ var patch = require_patch(); var prerelease = require_prerelease(); var compare3 = require_compare(); - var rcompare = require_rcompare(); + var rcompare2 = require_rcompare(); var compareLoose = require_compare_loose(); var compareBuild = require_compare_build(); var sort = require_sort(); @@ -45933,7 +45933,7 @@ var require_semver2 = __commonJS({ var subset = require_subset(); module2.exports = { parse: parse2, - valid: valid3, + valid: valid4, clean: clean3, inc, diff, @@ -45942,7 +45942,7 @@ var require_semver2 = __commonJS({ patch, prerelease, compare: compare3, - rcompare, + rcompare: rcompare2, compareLoose, compareBuild, sort, @@ -47732,16 +47732,16 @@ var require_attribute = __commonJS({ var result = new ValidatorResult(instance, schema2, options, ctx); var self2 = this; schema2.allOf.forEach(function(v, i) { - var valid3 = self2.validateSchema(instance, v, options, ctx); - if (!valid3.valid) { + var valid4 = self2.validateSchema(instance, v, options, ctx); + if (!valid4.valid) { var id = v.$id || v.id; var msg = id || v.title && JSON.stringify(v.title) || v["$ref"] && "<" + v["$ref"] + ">" || "[subschema " + i + "]"; result.addError({ name: "allOf", - argument: { id: msg, length: valid3.errors.length, valid: valid3 }, - message: "does not match allOf schema " + msg + " with " + valid3.errors.length + " error[s]:" + argument: { id: msg, length: valid4.errors.length, valid: valid4 }, + message: "does not match allOf schema " + msg + " with " + valid4.errors.length + " error[s]:" }); - result.importErrors(valid3); + result.importErrors(valid4); } }); return result; @@ -48030,8 +48030,8 @@ var require_attribute = __commonJS({ if (typeof schema2.exclusiveMinimum === "boolean") return; if (!this.types.number(instance)) return; var result = new ValidatorResult(instance, schema2, options, ctx); - var valid3 = instance > schema2.exclusiveMinimum; - if (!valid3) { + var valid4 = instance > schema2.exclusiveMinimum; + if (!valid4) { result.addError({ name: "exclusiveMinimum", argument: schema2.exclusiveMinimum, @@ -48044,8 +48044,8 @@ var require_attribute = __commonJS({ if (typeof schema2.exclusiveMaximum === "boolean") return; if (!this.types.number(instance)) return; var result = new ValidatorResult(instance, schema2, options, ctx); - var valid3 = instance < schema2.exclusiveMaximum; - if (!valid3) { + var valid4 = instance < schema2.exclusiveMaximum; + if (!valid4) { result.addError({ name: "exclusiveMaximum", argument: schema2.exclusiveMaximum, @@ -50828,8 +50828,8 @@ var require_semver3 = __commonJS({ return null; } } - exports2.valid = valid3; - function valid3(version, options) { + exports2.valid = valid4; + function valid4(version, options) { var v = parse2(version, options); return v ? v.version : null; } @@ -51129,8 +51129,8 @@ var require_semver3 = __commonJS({ var versionB = new SemVer(b, loose); return versionA.compare(versionB) || versionA.compareBuild(versionB); } - exports2.rcompare = rcompare; - function rcompare(a, b, loose) { + exports2.rcompare = rcompare2; + function rcompare2(a, b, loose) { return compare3(b, a, loose); } exports2.sort = sort; @@ -51958,7 +51958,7 @@ var require_cacheUtils = __commonJS({ var crypto3 = __importStar2(require("crypto")); var fs20 = __importStar2(require("fs")); var path16 = __importStar2(require("path")); - var semver9 = __importStar2(require_semver3()); + var semver10 = __importStar2(require_semver3()); var util = __importStar2(require("util")); var constants_1 = require_constants12(); var versionSalt = "1.0"; @@ -52051,7 +52051,7 @@ var require_cacheUtils = __commonJS({ function getCompressionMethod() { return __awaiter2(this, void 0, void 0, function* () { const versionOutput = yield getVersion("zstd", ["--quiet"]); - const version = semver9.clean(versionOutput); + const version = semver10.clean(versionOutput); core17.debug(`zstd version: ${version}`); if (versionOutput === "") { return constants_1.CompressionMethod.Gzip; @@ -99334,7 +99334,7 @@ var require_manifest = __commonJS({ exports2._findMatch = _findMatch; exports2._getOsVersion = _getOsVersion; exports2._readLinuxVersionFile = _readLinuxVersionFile; - var semver9 = __importStar2(require_semver2()); + var semver10 = __importStar2(require_semver2()); var core_1 = require_core(); var os5 = require("os"); var cp = require("child_process"); @@ -99348,7 +99348,7 @@ var require_manifest = __commonJS({ for (const candidate of candidates) { const version = candidate.version; (0, core_1.debug)(`check ${version} satisfies ${versionSpec}`); - if (semver9.satisfies(version, versionSpec) && (!stable || candidate.stable === stable)) { + if (semver10.satisfies(version, versionSpec) && (!stable || candidate.stable === stable)) { file = candidate.files.find((item) => { (0, core_1.debug)(`${item.arch}===${archFilter} && ${item.platform}===${platFilter}`); let chk = item.arch === archFilter && item.platform === platFilter; @@ -99357,7 +99357,7 @@ var require_manifest = __commonJS({ if (osVersion === item.platform_version) { chk = true; } else { - chk = semver9.satisfies(osVersion, item.platform_version); + chk = semver10.satisfies(osVersion, item.platform_version); } } return chk; @@ -99617,7 +99617,7 @@ var require_tool_cache = __commonJS({ var os5 = __importStar2(require("os")); var path16 = __importStar2(require("path")); var httpm = __importStar2(require_lib()); - var semver9 = __importStar2(require_semver2()); + var semver10 = __importStar2(require_semver2()); var stream2 = __importStar2(require("stream")); var util = __importStar2(require("util")); var assert_1 = require("assert"); @@ -99890,7 +99890,7 @@ var require_tool_cache = __commonJS({ } function cacheDir(sourceDir, tool, version, arch2) { return __awaiter2(this, void 0, void 0, function* () { - version = semver9.clean(version) || version; + version = semver10.clean(version) || version; arch2 = arch2 || os5.arch(); core17.debug(`Caching tool ${tool} ${version} ${arch2}`); core17.debug(`source dir: ${sourceDir}`); @@ -99908,7 +99908,7 @@ var require_tool_cache = __commonJS({ } function cacheFile(sourceFile, targetFile, tool, version, arch2) { return __awaiter2(this, void 0, void 0, function* () { - version = semver9.clean(version) || version; + version = semver10.clean(version) || version; arch2 = arch2 || os5.arch(); core17.debug(`Caching tool ${tool} ${version} ${arch2}`); core17.debug(`source file: ${sourceFile}`); @@ -99938,7 +99938,7 @@ var require_tool_cache = __commonJS({ } let toolPath = ""; if (versionSpec) { - versionSpec = semver9.clean(versionSpec) || ""; + versionSpec = semver10.clean(versionSpec) || ""; const cachePath = path16.join(_getCacheDirectory(), toolName, versionSpec, arch2); core17.debug(`checking cache: ${cachePath}`); if (fs20.existsSync(cachePath) && fs20.existsSync(`${cachePath}.complete`)) { @@ -100018,7 +100018,7 @@ var require_tool_cache = __commonJS({ } function _createToolPath(tool, version, arch2) { return __awaiter2(this, void 0, void 0, function* () { - const folderPath = path16.join(_getCacheDirectory(), tool, semver9.clean(version) || version, arch2 || ""); + const folderPath = path16.join(_getCacheDirectory(), tool, semver10.clean(version) || version, arch2 || ""); core17.debug(`destination ${folderPath}`); const markerPath = `${folderPath}.complete`; yield io7.rmRF(folderPath); @@ -100028,30 +100028,30 @@ var require_tool_cache = __commonJS({ }); } function _completeToolPath(tool, version, arch2) { - const folderPath = path16.join(_getCacheDirectory(), tool, semver9.clean(version) || version, arch2 || ""); + const folderPath = path16.join(_getCacheDirectory(), tool, semver10.clean(version) || version, arch2 || ""); const markerPath = `${folderPath}.complete`; fs20.writeFileSync(markerPath, ""); core17.debug("finished caching tool"); } function isExplicitVersion(versionSpec) { - const c = semver9.clean(versionSpec) || ""; + const c = semver10.clean(versionSpec) || ""; core17.debug(`isExplicit: ${c}`); - const valid3 = semver9.valid(c) != null; - core17.debug(`explicit? ${valid3}`); - return valid3; + const valid4 = semver10.valid(c) != null; + core17.debug(`explicit? ${valid4}`); + return valid4; } function evaluateVersions(versions, versionSpec) { let version = ""; core17.debug(`evaluating ${versions.length} versions`); versions = versions.sort((a, b) => { - if (semver9.gt(a, b)) { + if (semver10.gt(a, b)) { return 1; } return -1; }); for (let i = versions.length - 1; i >= 0; i--) { const potential = versions[i]; - const satisfied = semver9.satisfies(potential, versionSpec); + const satisfied = semver10.satisfies(potential, versionSpec); if (satisfied) { version = potential; break; @@ -111194,10 +111194,13 @@ async function uploadBundledDatabase(repositoryNwo, language, commitOid, bundled // src/overlay/caching.ts var fs15 = __toESM(require("fs")); var actionsCache4 = __toESM(require_cache5()); +var semver9 = __toESM(require_semver2()); var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 7500; var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1e6; var CACHE_VERSION2 = 1; var CACHE_PREFIX = "codeql-overlay-base-database"; +var CACHE_ENTRY_MAX_AGE_DAYS = 6; +var CACHE_ENTRY_MAX_AGE_MS = CACHE_ENTRY_MAX_AGE_DAYS * 24 * 60 * 60 * 1e3; var MAX_CACHE_OPERATION_MS2 = 6e5; async function checkOverlayBaseDatabase(codeql, config, logger, warningPrefix) { const baseDatabaseOidsFilePath = getBaseDatabaseOidsFilePath(config); @@ -111323,13 +111326,16 @@ async function getCacheSaveKey(config, codeQlVersion, checkoutPath, logger) { return `${restoreKeyPrefix}${sha}-${runId}-${attemptId}`; } async function getCacheRestoreKeyPrefix(config, codeQlVersion) { - const languages = [...config.languages].sort().join("_"); + return `${await getCacheKeyPrefixBase(config.languages)}${codeQlVersion}-`; +} +async function getCacheKeyPrefixBase(parsedLanguages) { + const languagesComponent = [...parsedLanguages].sort().join("_"); const cacheKeyComponents = { automationID: await getAutomationID() // Add more components here as needed in the future }; const componentsHash = createCacheKeyHash(cacheKeyComponents); - return `${CACHE_PREFIX}-${CACHE_VERSION2}-${componentsHash}-${languages}-${codeQlVersion}-`; + return `${CACHE_PREFIX}-${CACHE_VERSION2}-${componentsHash}-${languagesComponent}-`; } // src/status-report.ts diff --git a/lib/init-action.js b/lib/init-action.js index 3ce55149db..499472f2bf 100644 --- a/lib/init-action.js +++ b/lib/init-action.js @@ -44531,11 +44531,11 @@ var require_valid = __commonJS({ "node_modules/semver/functions/valid.js"(exports2, module2) { "use strict"; var parse2 = require_parse3(); - var valid3 = (version, options) => { + var valid4 = (version, options) => { const v = parse2(version, options); return v ? v.version : null; }; - module2.exports = valid3; + module2.exports = valid4; } }); @@ -44678,8 +44678,8 @@ var require_rcompare = __commonJS({ "node_modules/semver/functions/rcompare.js"(exports2, module2) { "use strict"; var compare2 = require_compare(); - var rcompare = (a, b, loose) => compare2(b, a, loose); - module2.exports = rcompare; + var rcompare2 = (a, b, loose) => compare2(b, a, loose); + module2.exports = rcompare2; } }); @@ -45895,7 +45895,7 @@ var require_semver2 = __commonJS({ var SemVer = require_semver(); var identifiers = require_identifiers(); var parse2 = require_parse3(); - var valid3 = require_valid(); + var valid4 = require_valid(); var clean3 = require_clean(); var inc = require_inc(); var diff = require_diff(); @@ -45904,7 +45904,7 @@ var require_semver2 = __commonJS({ var patch = require_patch(); var prerelease = require_prerelease(); var compare2 = require_compare(); - var rcompare = require_rcompare(); + var rcompare2 = require_rcompare(); var compareLoose = require_compare_loose(); var compareBuild = require_compare_build(); var sort = require_sort(); @@ -45933,7 +45933,7 @@ var require_semver2 = __commonJS({ var subset = require_subset(); module2.exports = { parse: parse2, - valid: valid3, + valid: valid4, clean: clean3, inc, diff, @@ -45942,7 +45942,7 @@ var require_semver2 = __commonJS({ patch, prerelease, compare: compare2, - rcompare, + rcompare: rcompare2, compareLoose, compareBuild, sort, @@ -47732,16 +47732,16 @@ var require_attribute = __commonJS({ var result = new ValidatorResult(instance, schema2, options, ctx); var self2 = this; schema2.allOf.forEach(function(v, i) { - var valid3 = self2.validateSchema(instance, v, options, ctx); - if (!valid3.valid) { + var valid4 = self2.validateSchema(instance, v, options, ctx); + if (!valid4.valid) { var id = v.$id || v.id; var msg = id || v.title && JSON.stringify(v.title) || v["$ref"] && "<" + v["$ref"] + ">" || "[subschema " + i + "]"; result.addError({ name: "allOf", - argument: { id: msg, length: valid3.errors.length, valid: valid3 }, - message: "does not match allOf schema " + msg + " with " + valid3.errors.length + " error[s]:" + argument: { id: msg, length: valid4.errors.length, valid: valid4 }, + message: "does not match allOf schema " + msg + " with " + valid4.errors.length + " error[s]:" }); - result.importErrors(valid3); + result.importErrors(valid4); } }); return result; @@ -48030,8 +48030,8 @@ var require_attribute = __commonJS({ if (typeof schema2.exclusiveMinimum === "boolean") return; if (!this.types.number(instance)) return; var result = new ValidatorResult(instance, schema2, options, ctx); - var valid3 = instance > schema2.exclusiveMinimum; - if (!valid3) { + var valid4 = instance > schema2.exclusiveMinimum; + if (!valid4) { result.addError({ name: "exclusiveMinimum", argument: schema2.exclusiveMinimum, @@ -48044,8 +48044,8 @@ var require_attribute = __commonJS({ if (typeof schema2.exclusiveMaximum === "boolean") return; if (!this.types.number(instance)) return; var result = new ValidatorResult(instance, schema2, options, ctx); - var valid3 = instance < schema2.exclusiveMaximum; - if (!valid3) { + var valid4 = instance < schema2.exclusiveMaximum; + if (!valid4) { result.addError({ name: "exclusiveMaximum", argument: schema2.exclusiveMaximum, @@ -50979,8 +50979,8 @@ var require_semver3 = __commonJS({ return null; } } - exports2.valid = valid3; - function valid3(version, options) { + exports2.valid = valid4; + function valid4(version, options) { var v = parse2(version, options); return v ? v.version : null; } @@ -51280,8 +51280,8 @@ var require_semver3 = __commonJS({ var versionB = new SemVer(b, loose); return versionA.compare(versionB) || versionA.compareBuild(versionB); } - exports2.rcompare = rcompare; - function rcompare(a, b, loose) { + exports2.rcompare = rcompare2; + function rcompare2(a, b, loose) { return compare2(b, a, loose); } exports2.sort = sort; @@ -52109,7 +52109,7 @@ var require_cacheUtils = __commonJS({ var crypto3 = __importStar2(require("crypto")); var fs19 = __importStar2(require("fs")); var path18 = __importStar2(require("path")); - var semver10 = __importStar2(require_semver3()); + var semver11 = __importStar2(require_semver3()); var util = __importStar2(require("util")); var constants_1 = require_constants12(); var versionSalt = "1.0"; @@ -52202,7 +52202,7 @@ var require_cacheUtils = __commonJS({ function getCompressionMethod() { return __awaiter2(this, void 0, void 0, function* () { const versionOutput = yield getVersion("zstd", ["--quiet"]); - const version = semver10.clean(versionOutput); + const version = semver11.clean(versionOutput); core16.debug(`zstd version: ${version}`); if (versionOutput === "") { return constants_1.CompressionMethod.Gzip; @@ -99485,7 +99485,7 @@ var require_manifest = __commonJS({ exports2._findMatch = _findMatch; exports2._getOsVersion = _getOsVersion; exports2._readLinuxVersionFile = _readLinuxVersionFile; - var semver10 = __importStar2(require_semver2()); + var semver11 = __importStar2(require_semver2()); var core_1 = require_core(); var os6 = require("os"); var cp = require("child_process"); @@ -99499,7 +99499,7 @@ var require_manifest = __commonJS({ for (const candidate of candidates) { const version = candidate.version; (0, core_1.debug)(`check ${version} satisfies ${versionSpec}`); - if (semver10.satisfies(version, versionSpec) && (!stable || candidate.stable === stable)) { + if (semver11.satisfies(version, versionSpec) && (!stable || candidate.stable === stable)) { file = candidate.files.find((item) => { (0, core_1.debug)(`${item.arch}===${archFilter} && ${item.platform}===${platFilter}`); let chk = item.arch === archFilter && item.platform === platFilter; @@ -99508,7 +99508,7 @@ var require_manifest = __commonJS({ if (osVersion === item.platform_version) { chk = true; } else { - chk = semver10.satisfies(osVersion, item.platform_version); + chk = semver11.satisfies(osVersion, item.platform_version); } } return chk; @@ -99768,7 +99768,7 @@ var require_tool_cache = __commonJS({ var os6 = __importStar2(require("os")); var path18 = __importStar2(require("path")); var httpm = __importStar2(require_lib()); - var semver10 = __importStar2(require_semver2()); + var semver11 = __importStar2(require_semver2()); var stream2 = __importStar2(require("stream")); var util = __importStar2(require("util")); var assert_1 = require("assert"); @@ -100041,7 +100041,7 @@ var require_tool_cache = __commonJS({ } function cacheDir(sourceDir, tool, version, arch2) { return __awaiter2(this, void 0, void 0, function* () { - version = semver10.clean(version) || version; + version = semver11.clean(version) || version; arch2 = arch2 || os6.arch(); core16.debug(`Caching tool ${tool} ${version} ${arch2}`); core16.debug(`source dir: ${sourceDir}`); @@ -100059,7 +100059,7 @@ var require_tool_cache = __commonJS({ } function cacheFile(sourceFile, targetFile, tool, version, arch2) { return __awaiter2(this, void 0, void 0, function* () { - version = semver10.clean(version) || version; + version = semver11.clean(version) || version; arch2 = arch2 || os6.arch(); core16.debug(`Caching tool ${tool} ${version} ${arch2}`); core16.debug(`source file: ${sourceFile}`); @@ -100089,7 +100089,7 @@ var require_tool_cache = __commonJS({ } let toolPath = ""; if (versionSpec) { - versionSpec = semver10.clean(versionSpec) || ""; + versionSpec = semver11.clean(versionSpec) || ""; const cachePath = path18.join(_getCacheDirectory(), toolName, versionSpec, arch2); core16.debug(`checking cache: ${cachePath}`); if (fs19.existsSync(cachePath) && fs19.existsSync(`${cachePath}.complete`)) { @@ -100169,7 +100169,7 @@ var require_tool_cache = __commonJS({ } function _createToolPath(tool, version, arch2) { return __awaiter2(this, void 0, void 0, function* () { - const folderPath = path18.join(_getCacheDirectory(), tool, semver10.clean(version) || version, arch2 || ""); + const folderPath = path18.join(_getCacheDirectory(), tool, semver11.clean(version) || version, arch2 || ""); core16.debug(`destination ${folderPath}`); const markerPath = `${folderPath}.complete`; yield io7.rmRF(folderPath); @@ -100179,30 +100179,30 @@ var require_tool_cache = __commonJS({ }); } function _completeToolPath(tool, version, arch2) { - const folderPath = path18.join(_getCacheDirectory(), tool, semver10.clean(version) || version, arch2 || ""); + const folderPath = path18.join(_getCacheDirectory(), tool, semver11.clean(version) || version, arch2 || ""); const markerPath = `${folderPath}.complete`; fs19.writeFileSync(markerPath, ""); core16.debug("finished caching tool"); } function isExplicitVersion(versionSpec) { - const c = semver10.clean(versionSpec) || ""; + const c = semver11.clean(versionSpec) || ""; core16.debug(`isExplicit: ${c}`); - const valid3 = semver10.valid(c) != null; - core16.debug(`explicit? ${valid3}`); - return valid3; + const valid4 = semver11.valid(c) != null; + core16.debug(`explicit? ${valid4}`); + return valid4; } function evaluateVersions(versions, versionSpec) { let version = ""; core16.debug(`evaluating ${versions.length} versions`); versions = versions.sort((a, b) => { - if (semver10.gt(a, b)) { + if (semver11.gt(a, b)) { return 1; } return -1; }); for (let i = versions.length - 1; i >= 0; i--) { const potential = versions[i]; - const satisfied = semver10.satisfies(potential, versionSpec); + const satisfied = semver11.satisfies(potential, versionSpec); if (satisfied) { version = potential; break; @@ -100812,7 +100812,7 @@ var path17 = __toESM(require("path")); var core15 = __toESM(require_core()); var github3 = __toESM(require_github()); var io6 = __toESM(require_io()); -var semver9 = __toESM(require_semver2()); +var semver10 = __toESM(require_semver2()); // node_modules/uuid/dist-node/stringify.js var byteToHex = []; @@ -109578,10 +109578,13 @@ To opt out of this change, ${envVarOptOut}`; // src/overlay/caching.ts var fs16 = __toESM(require("fs")); var actionsCache4 = __toESM(require_cache5()); +var semver9 = __toESM(require_semver2()); var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 7500; var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1e6; var CACHE_VERSION2 = 1; var CACHE_PREFIX = "codeql-overlay-base-database"; +var CACHE_ENTRY_MAX_AGE_DAYS = 6; +var CACHE_ENTRY_MAX_AGE_MS = CACHE_ENTRY_MAX_AGE_DAYS * 24 * 60 * 60 * 1e3; var MAX_CACHE_OPERATION_MS3 = 6e5; async function checkOverlayBaseDatabase(codeql, config, logger, warningPrefix) { const baseDatabaseOidsFilePath = getBaseDatabaseOidsFilePath(config); @@ -109719,13 +109722,16 @@ async function downloadOverlayBaseDatabaseFromCache(codeql, config, logger) { }; } async function getCacheRestoreKeyPrefix(config, codeQlVersion) { - const languages = [...config.languages].sort().join("_"); + return `${await getCacheKeyPrefixBase(config.languages)}${codeQlVersion}-`; +} +async function getCacheKeyPrefixBase(parsedLanguages) { + const languagesComponent = [...parsedLanguages].sort().join("_"); const cacheKeyComponents = { automationID: await getAutomationID() // Add more components here as needed in the future }; const componentsHash = createCacheKeyHash(cacheKeyComponents); - return `${CACHE_PREFIX}-${CACHE_VERSION2}-${componentsHash}-${languages}-${codeQlVersion}-`; + return `${CACHE_PREFIX}-${CACHE_VERSION2}-${componentsHash}-${languagesComponent}-`; } // src/status-report.ts @@ -110322,12 +110328,12 @@ async function run(startedAt) { const experimental = "2.19.3"; const publicPreview = "2.22.1"; const actualVer = (await codeql.getVersion()).version; - if (semver9.lt(actualVer, experimental)) { + if (semver10.lt(actualVer, experimental)) { throw new ConfigurationError( `Rust analysis is supported by CodeQL CLI version ${experimental} or higher, but found version ${actualVer}` ); } - if (semver9.lt(actualVer, publicPreview)) { + if (semver10.lt(actualVer, publicPreview)) { core15.exportVariable("CODEQL_ENABLE_EXPERIMENTAL_FEATURES" /* EXPERIMENTAL_FEATURES */, "true"); logger.info("Experimental Rust analysis enabled"); } diff --git a/src/api-client.ts b/src/api-client.ts index 4b8cb7b340..8de5058d19 100644 --- a/src/api-client.ts +++ b/src/api-client.ts @@ -249,6 +249,7 @@ export interface ActionsCacheItem { created_at?: string; id?: number; key?: string; + last_accessed_at?: string; size_in_bytes?: number; } diff --git a/src/overlay/caching.test.ts b/src/overlay/caching.test.ts index 9c7abc6bd7..2c60fd7893 100644 --- a/src/overlay/caching.test.ts +++ b/src/overlay/caching.test.ts @@ -7,7 +7,7 @@ import * as sinon from "sinon"; import * as actionsUtil from "../actions-util"; import * as apiClient from "../api-client"; -import { ResolveDatabaseOutput } from "../codeql"; +import { type ResolveDatabaseOutput } from "../codeql"; import * as gitUtils from "../git-utils"; import { BuiltInLanguage } from "../languages"; import { getRunnerLogger } from "../logging"; @@ -23,6 +23,7 @@ import { downloadOverlayBaseDatabaseFromCache, getCacheRestoreKeyPrefix, getCacheSaveKey, + getCodeQlVersionsForOverlayBaseDatabases, } from "./caching"; import { OverlayDatabaseMode } from "./overlay-database-mode"; @@ -285,3 +286,168 @@ test.serial("overlay-base database cache keys remain stable", async (t) => { `Expected save key "${saveKey}" to start with restore key prefix "${restoreKeyPrefix}"`, ); }); + +test.serial( + "getCodeQlVersionsForOverlayBaseDatabases returns unique versions sorted latest first", + async (t) => { + const logger = getRunnerLogger(true); + + sinon.stub(apiClient, "getAutomationID").resolves("test-automation-id/"); + sinon.stub(apiClient, "listActionsCaches").resolves([ + { + key: "codeql-overlay-base-database-1-c5666c509a2d9895-javascript_python-2.23.0-abc123-1-1", + }, + { + key: "codeql-overlay-base-database-1-c5666c509a2d9895-javascript_python-2.24.1-def456-2-1", + }, + { + key: "codeql-overlay-base-database-1-c5666c509a2d9895-javascript_python-2.23.0-ghi789-3-1", + }, + ]); + + const result = await getCodeQlVersionsForOverlayBaseDatabases( + ["javascript", "python"], + logger, + ); + t.deepEqual(result, ["2.24.1", "2.23.0"]); + }, +); + +test.serial( + "getCodeQlVersionsForOverlayBaseDatabases returns empty list when no caches exist", + async (t) => { + const logger = getRunnerLogger(true); + + sinon.stub(apiClient, "getAutomationID").resolves("test-automation-id/"); + sinon.stub(apiClient, "listActionsCaches").resolves([]); + + const result = await getCodeQlVersionsForOverlayBaseDatabases( + ["python"], + logger, + ); + t.deepEqual(result, []); + }, +); + +test.serial( + "getCodeQlVersionsForOverlayBaseDatabases returns empty list when cache keys are unparseable", + async (t) => { + const logger = getRunnerLogger(true); + + sinon.stub(apiClient, "getAutomationID").resolves("test-automation-id/"); + sinon.stub(apiClient, "listActionsCaches").resolves([ + { + key: "codeql-overlay-base-database-1-c5666c509a2d9895-python-malformed", + }, + { key: undefined }, + ]); + + const result = await getCodeQlVersionsForOverlayBaseDatabases( + ["python"], + logger, + ); + t.deepEqual(result, []); + }, +); + +test.serial( + "getCodeQlVersionsForOverlayBaseDatabases returns the single version when only one cache exists", + async (t) => { + const logger = getRunnerLogger(true); + + sinon.stub(apiClient, "getAutomationID").resolves("test-automation-id/"); + sinon.stub(apiClient, "listActionsCaches").resolves([ + { + key: "codeql-overlay-base-database-1-c5666c509a2d9895-cpp-2.25.0-abc123-1-1", + }, + ]); + + const result = await getCodeQlVersionsForOverlayBaseDatabases( + ["cpp"], + logger, + ); + t.deepEqual(result, ["2.25.0"]); + }, +); + +test.serial( + "getCodeQlVersionsForOverlayBaseDatabases resolves language aliases", + async (t) => { + const logger = getRunnerLogger(true); + // The alias `c++` should be resolved to "cpp" and match cache entries keyed with "cpp" + + sinon.stub(apiClient, "getAutomationID").resolves("test-automation-id/"); + sinon.stub(apiClient, "listActionsCaches").resolves([ + { + key: "codeql-overlay-base-database-1-c5666c509a2d9895-cpp-2.25.0-abc123-1-1", + }, + ]); + + const result = await getCodeQlVersionsForOverlayBaseDatabases( + ["c++"], + logger, + ); + t.deepEqual(result, ["2.25.0"]); + }, +); + +test.serial( + "getCodeQlVersionsForOverlayBaseDatabases ignores nightly versions with build metadata", + async (t) => { + const logger = getRunnerLogger(true); + + sinon.stub(apiClient, "getAutomationID").resolves("test-automation-id/"); + sinon.stub(apiClient, "listActionsCaches").resolves([ + { + key: "codeql-overlay-base-database-1-c5666c509a2d9895-python-2.25.0-abc123-1-1", + }, + { + // Nightly release with semver build metadata; should be ignored. + key: "codeql-overlay-base-database-1-c5666c509a2d9895-python-2.26.0+202604211234-def456-2-1", + }, + { + key: "codeql-overlay-base-database-1-c5666c509a2d9895-python-2.24.0-ghi789-3-1", + }, + ]); + + const result = await getCodeQlVersionsForOverlayBaseDatabases( + ["python"], + logger, + ); + t.deepEqual(result, ["2.25.0", "2.24.0"]); + }, +); + +test.serial( + "getCodeQlVersionsForOverlayBaseDatabases ignores cache entries close to eviction", + async (t) => { + const logger = getRunnerLogger(true); + + const now = Date.now(); + const isoDaysAgo = (days: number) => + new Date(now - days * 24 * 60 * 60 * 1000).toISOString(); + + sinon.stub(apiClient, "getAutomationID").resolves("test-automation-id/"); + sinon.stub(apiClient, "listActionsCaches").resolves([ + { + key: "codeql-overlay-base-database-1-c5666c509a2d9895-python-2.25.0-abc123-1-1", + last_accessed_at: isoDaysAgo(1), + }, + { + // Older than the 6-day threshold; close to the 7-day eviction window. + key: "codeql-overlay-base-database-1-c5666c509a2d9895-python-2.26.0-def456-2-1", + last_accessed_at: isoDaysAgo(6.5), + }, + { + key: "codeql-overlay-base-database-1-c5666c509a2d9895-python-2.24.0-ghi789-3-1", + last_accessed_at: isoDaysAgo(3), + }, + ]); + + const result = await getCodeQlVersionsForOverlayBaseDatabases( + ["python"], + logger, + ); + t.deepEqual(result, ["2.25.0", "2.24.0"]); + }, +); diff --git a/src/overlay/caching.ts b/src/overlay/caching.ts index 2dcb7f837b..cdde47b754 100644 --- a/src/overlay/caching.ts +++ b/src/overlay/caching.ts @@ -1,18 +1,24 @@ import * as fs from "fs"; import * as actionsCache from "@actions/cache"; +import * as semver from "semver"; import { getRequiredInput, getWorkflowRunAttempt, getWorkflowRunID, } from "../actions-util"; -import { getAutomationID } from "../api-client"; +import { + type ActionsCacheItem, + getAutomationID, + listActionsCaches, +} from "../api-client"; import { createCacheKeyHash } from "../caching-utils"; import { type CodeQL } from "../codeql"; import { type Config } from "../config-utils"; import { getCommitOid } from "../git-utils"; -import { Logger, withGroupAsync } from "../logging"; +import { type Language, parseBuiltInLanguage } from "../languages"; +import { type Logger, withGroupAsync } from "../logging"; import { CleanupLevel, getBaseDatabaseOidsFilePath, @@ -46,6 +52,12 @@ const OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = const CACHE_VERSION = 1; const CACHE_PREFIX = "codeql-overlay-base-database"; +// The Actions cache evicts entries that have not been accessed in the past 7 +// days. We conservatively set a limit of 6 days to avoid using a cached base DB +// that may be evicted before we can download it. +const CACHE_ENTRY_MAX_AGE_DAYS = 6; +const CACHE_ENTRY_MAX_AGE_MS = CACHE_ENTRY_MAX_AGE_DAYS * 24 * 60 * 60 * 1000; + // The purpose of this ten-minute limit is to guard against the possibility // that the cache service is unresponsive, which would otherwise cause the // entire action to hang. Normally we expect cache operations to complete @@ -404,7 +416,17 @@ export async function getCacheRestoreKeyPrefix( config: Config, codeQlVersion: string, ): Promise { - const languages = [...config.languages].sort().join("_"); + return `${await getCacheKeyPrefixBase(config.languages)}${codeQlVersion}-`; +} + +/** + * Computes the cache key prefix for overlay-base databases, excluding the + * CodeQL version. + */ +async function getCacheKeyPrefixBase( + parsedLanguages: Language[], +): Promise { + const languagesComponent = [...parsedLanguages].sort().join("_"); const cacheKeyComponents = { automationID: await getAutomationID(), @@ -412,17 +434,120 @@ export async function getCacheRestoreKeyPrefix( }; const componentsHash = createCacheKeyHash(cacheKeyComponents); - // For a cached overlay-base database to be considered compatible for overlay - // analysis, all components in the cache restore key must match: - // // CACHE_PREFIX: distinguishes overlay-base databases from other cache objects // CACHE_VERSION: cache format version // componentsHash: hash of additional components (see above for details) - // languages: the languages included in the overlay-base database - // codeQlVersion: CodeQL bundle version + // languagesComponent: the languages included in the overlay-base database // - // Technically we can also include languages and codeQlVersion in the - // componentsHash, but including them explicitly in the cache key makes it - // easier to debug and understand the cache key structure. - return `${CACHE_PREFIX}-${CACHE_VERSION}-${componentsHash}-${languages}-${codeQlVersion}-`; + // Technically we can also include languages in the componentsHash, but + // including them explicitly in the cache key makes it easier to debug and + // understand the cache key structure. + return `${CACHE_PREFIX}-${CACHE_VERSION}-${componentsHash}-${languagesComponent}-`; +} + +/** + * Lists overlay-base database cache entries with the given key prefix, ignoring entries that are + * old enough that they may be evicted by the Actions cache before we attempt to download them. + */ +async function listRecentOverlayBaseDatabaseCaches( + cacheKeyPrefix: string, + logger: Logger, +): Promise { + const allCaches = await listActionsCaches(cacheKeyPrefix); + + if (allCaches.length === 0) { + logger.info("No overlay-base databases found in Actions cache."); + return []; + } + + const cutoffMs = Date.now() - CACHE_ENTRY_MAX_AGE_MS; + const recentCaches = allCaches.filter((cache) => { + if (!cache.last_accessed_at) return true; + const lastAccessedMs = Date.parse(cache.last_accessed_at); + return Number.isNaN(lastAccessedMs) || lastAccessedMs >= cutoffMs; + }); + const numTooOldDatabases = allCaches.length - recentCaches.length; + const tooOldSuffix = + numTooOldDatabases > 0 + ? ` (ignoring ${numTooOldDatabases} that may be evicted soon)` + : ""; + logger.info( + `Found ${allCaches.length} overlay-base ${allCaches.length === 1 ? "database" : "databases"} in the Actions cache${tooOldSuffix}.`, + ); + + return recentCaches; +} + +/** + * Searches the GitHub Actions cache for overlay-base databases matching the given languages, and + * returns all stable CodeQL versions found across matching cache entries. + * + * @returns Unique stable CodeQL versions found in cached overlay-base databases, sorted from latest to + * earliest, or undefined if one of the languages is not a built-in language. + */ +export async function getCodeQlVersionsForOverlayBaseDatabases( + rawLanguages: string[], + logger: Logger, +): Promise { + const languages = rawLanguages.map(parseBuiltInLanguage); + if (languages.includes(undefined)) { + logger.info( + "One or more provided languages are not recognized as built-in languages. " + + "Skipping searching for overlay-base databases in cache.", + ); + return undefined; + } + const cacheKeyPrefix = await getCacheKeyPrefixBase( + languages.filter((l) => l !== undefined), + ); + + logger.debug( + `Searching for overlay-base databases in Actions cache with ` + + `prefix ${cacheKeyPrefix}`, + ); + + const caches = await listRecentOverlayBaseDatabaseCaches( + cacheKeyPrefix, + logger, + ); + + if (caches.length === 0) { + return []; + } + + // Parse CodeQL versions from cache keys, matching only stable releases. + // + // After the prefix, the remaining key format starts with `${codeQlVersion}-`. Nightlies have a + // suffix like `+202604201548` that will prevent a match. + // + // Caveat: this relies on the fact that we haven't released any CodeQL bundles with the + // `x.y.z-` semver format which does not interact well with the current overlay base + // DB cache key format. + const versionRegex = /^([\d.]+)-/; + const versionSet = new Set(); + + for (const cache of caches) { + if (!cache.key) continue; + const suffix = cache.key.substring(cacheKeyPrefix.length); + const match = suffix.match(versionRegex); + if (match && semver.valid(match[1])) { + versionSet.add(match[1]); + } + } + + if (versionSet.size === 0) { + logger.info( + "Could not parse any CodeQL versions from overlay-base database " + + "cache keys.", + ); + return []; + } + + const versions = [...versionSet].sort(semver.rcompare); + + logger.info( + `Found overlay-base databases for the following CodeQL versions in the Actions cache: ${versions.join(", ")}`, + ); + + return versions; }