-
Notifications
You must be signed in to change notification settings - Fork 4
Expand file tree
/
Copy pathcacheInstaller.ts
More file actions
514 lines (457 loc) · 18.8 KB
/
cacheInstaller.ts
File metadata and controls
514 lines (457 loc) · 18.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
import { execFileSync } from 'child_process';
import { createHash } from 'crypto';
import { copyFileSync, existsSync, mkdirSync, writeFileSync } from 'fs';
import { dirname, join, resolve } from 'path';
import type { CdsDependencyCombination } from './types';
import { CdsDependencyGraph, CdsProject } from '../cds/parser/types';
import { DiagnosticSeverity } from '../diagnostics';
import { cdsExtractorLog } from '../logging';
import { resolveCdsVersions } from './versionResolver';
const cacheSubDirName = '.cds-extractor-cache';
/**
* Add a warning diagnostic for dependency version fallback
* @param packageJsonPath Path to the package.json file
* @param warningMessage The warning message
* @param codeqlExePath Path to the CodeQL executable
* @returns True if the diagnostic was added, false otherwise
*/
function addDependencyVersionWarning(
packageJsonPath: string,
warningMessage: string,
codeqlExePath: string,
): boolean {
try {
execFileSync(codeqlExePath, [
'database',
'add-diagnostic',
'--extractor-name=cds',
'--ready-for-status-page',
'--source-id=cds/dependency-version-fallback',
'--source-name=Using fallback versions for SAP CAP CDS dependencies',
`--severity=${DiagnosticSeverity.Warning}`,
`--markdown-message=${warningMessage}`,
`--file-path=${resolve(packageJsonPath)}`,
'--',
`${process.env.CODEQL_EXTRACTOR_CDS_WIP_DATABASE ?? ''}`,
]);
cdsExtractorLog('info', `Added warning diagnostic for dependency fallback: ${packageJsonPath}`);
return true;
} catch (err) {
cdsExtractorLog(
'error',
`Failed to add warning diagnostic for ${packageJsonPath}: ${String(err)}`,
);
return false;
}
}
/**
* Find the nearest `.npmrc` file by searching the given directory and its
* ancestors up to (and including) the filesystem root. npm itself walks the
* directory tree when looking for project-level `.npmrc` files, so we mirror
* that behaviour here.
*
* @param startDir The directory from which to start the upward search.
* @returns The absolute path to the nearest `.npmrc`, or `undefined` if none is found.
*/
export function findNearestNpmrc(startDir: string): string | undefined {
let current = resolve(startDir);
// Walk up the directory tree until we find an .npmrc or reach the root
while (true) {
const candidate = join(current, '.npmrc');
if (existsSync(candidate)) {
return candidate;
}
const parent = dirname(current);
if (parent === current) {
// Reached filesystem root without finding .npmrc
return undefined;
}
current = parent;
}
}
/**
* Copy the project's `.npmrc` file (if any) into the cache directory so that
* `npm install` inside the cache respects custom registry configuration such
* as scoped registries (`@sap:registry=...`), authentication tokens, and
* `strict-ssl` settings.
*
* @param cacheDir The cache directory where dependencies will be installed.
* @param projectDir Absolute path to the project directory whose `.npmrc` should be used.
*/
export function copyNpmrcToCache(cacheDir: string, projectDir: string): void {
const npmrcPath = findNearestNpmrc(projectDir);
if (!npmrcPath) {
return;
}
const dest = join(cacheDir, '.npmrc');
try {
copyFileSync(npmrcPath, dest);
cdsExtractorLog('info', `Copied .npmrc from '${npmrcPath}' to cache directory '${cacheDir}'`);
} catch (err) {
cdsExtractorLog(
'warn',
`Failed to copy .npmrc to cache directory: ${err instanceof Error ? err.message : String(err)}`,
);
}
}
/**
* Install dependencies for CDS projects using a robust cache strategy with fallback logic
* @param dependencyGraph The dependency graph of the project
* @param sourceRoot Source root directory
* @param codeqlExePath Path to the CodeQL executable (optional)
* @returns Map of project directories to their corresponding cache directories
*/
export function cacheInstallDependencies(
dependencyGraph: CdsDependencyGraph,
sourceRoot: string,
codeqlExePath?: string,
): Map<string, string> {
// Sanity check that we found at least one project
if (dependencyGraph.projects.size === 0) {
cdsExtractorLog('info', 'No CDS projects found for dependency installation.');
cdsExtractorLog(
'info',
'This is expected if the source contains no CAP/CDS projects and should be handled by the caller.',
);
return new Map<string, string>();
}
// Extract unique dependency combinations from all projects with version resolution
const dependencyCombinations = extractUniqueDependencyCombinations(dependencyGraph.projects);
if (dependencyCombinations.length === 0) {
cdsExtractorLog(
'error',
'No CDS dependencies found in any project. This means projects were detected but lack proper @sap/cds dependencies.',
);
cdsExtractorLog(
'info',
'Will attempt to use system-installed CDS tools if available, but compilation may fail.',
);
return new Map<string, string>();
}
cdsExtractorLog(
'info',
`Found ${dependencyCombinations.length} unique CDS dependency combination(s).`,
);
// Log each dependency combination for transparency
for (const combination of dependencyCombinations) {
const { cdsVersion, cdsDkVersion, hash, resolvedCdsVersion, resolvedCdsDkVersion, isFallback } =
combination;
const actualCdsVersion = resolvedCdsVersion ?? cdsVersion;
const actualCdsDkVersion = resolvedCdsDkVersion ?? cdsDkVersion;
const fallbackNote = isFallback ? ' (using fallback versions)' : '';
const indexerNote = combination.cdsIndexerVersion
? `, @sap/cds-indexer@${combination.cdsIndexerVersion}`
: '';
cdsExtractorLog(
'info',
`Dependency combination ${hash.substring(0, 8)}: @sap/cds@${actualCdsVersion}, @sap/cds-dk@${actualCdsDkVersion}${indexerNote}${fallbackNote}`,
);
}
// Create a cache directory under the source root directory.
const cacheRootDir = join(sourceRoot, cacheSubDirName);
cdsExtractorLog(
'info',
`Using cache directory '${cacheSubDirName}' within source root directory '${cacheRootDir}'`,
);
if (!existsSync(cacheRootDir)) {
try {
mkdirSync(cacheRootDir, { recursive: true });
cdsExtractorLog('info', `Created cache directory: ${cacheRootDir}`);
} catch (err) {
cdsExtractorLog(
'warn',
`Failed to create cache directory: ${err instanceof Error ? err.message : String(err)}`,
);
cdsExtractorLog('info', 'Skipping dependency installation due to cache directory failure.');
return new Map<string, string>();
}
} else {
cdsExtractorLog('info', `Cache directory already exists: ${cacheRootDir}`);
}
// Map to track which cache directory to use for each project
const projectCacheDirMap = new Map<string, string>();
let successfulInstallations = 0;
// Install each unique dependency combination in its own cache directory
for (const combination of dependencyCombinations) {
const { cdsVersion, cdsDkVersion, hash } = combination;
const { resolvedCdsVersion, resolvedCdsDkVersion } = combination;
const cacheDirName = `cds-${hash}`;
const cacheDir = join(cacheRootDir, cacheDirName);
cdsExtractorLog(
'info',
`Processing dependency combination ${hash.substring(0, 8)} in cache directory: ${cacheDirName}`,
);
// Create the cache directory if it doesn't exist
if (!existsSync(cacheDir)) {
try {
mkdirSync(cacheDir, { recursive: true });
cdsExtractorLog('info', `Created cache subdirectory: ${cacheDirName}`);
} catch (err) {
cdsExtractorLog(
'error',
`Failed to create cache directory for combination ${hash.substring(0, 8)} (${cacheDirName}): ${
err instanceof Error ? err.message : String(err)
}`,
);
continue;
}
// Create a package.json for this dependency combination using resolved versions
const actualCdsVersion = resolvedCdsVersion ?? cdsVersion;
const actualCdsDkVersion = resolvedCdsDkVersion ?? cdsDkVersion;
const cacheDeps: Record<string, string> = {
'@sap/cds': actualCdsVersion,
'@sap/cds-dk': actualCdsDkVersion,
};
// Include @sap/cds-indexer in the cache when a project depends on it.
// This is a best-effort optimization: on systems with access to the
// private npm registry that hosts @sap/cds-indexer, it will be cached
// alongside @sap/cds and @sap/cds-dk. On systems without access, the
// npm install will still succeed for the other packages (it won't fail
// the overall installation — npm install is run with --no-optional
// semantics handled below).
if (combination.cdsIndexerVersion) {
cacheDeps['@sap/cds-indexer'] = combination.cdsIndexerVersion;
cdsExtractorLog(
'info',
`Including @sap/cds-indexer@${combination.cdsIndexerVersion} in cache for combination ${hash.substring(0, 8)}`,
);
}
const packageJson = {
name: `cds-extractor-cache-${hash}`,
version: '1.0.0',
private: true,
dependencies: cacheDeps,
};
try {
writeFileSync(join(cacheDir, 'package.json'), JSON.stringify(packageJson, null, 2));
cdsExtractorLog('info', `Created package.json in cache subdirectory: ${cacheDirName}`);
} catch (err) {
cdsExtractorLog(
'error',
`Failed to create package.json in cache directory ${cacheDirName}: ${
err instanceof Error ? err.message : String(err)
}`,
);
continue;
}
}
// Ensure the cache directory has an .npmrc that reflects the projects' registry configuration
const npmrcProjectDir = Array.from(dependencyGraph.projects.values())
.map(project => project.projectDir)
.find(projectDir => projectDir && existsSync(join(sourceRoot, projectDir, '.npmrc')));
if (npmrcProjectDir) {
copyNpmrcToCache(cacheDir, join(sourceRoot, npmrcProjectDir));
}
// Try to install dependencies in the cache directory
// Get the first project package.json path for diagnostic purposes
const samplePackageJsonPath = Array.from(dependencyGraph.projects.values()).find(
project => project.packageJson,
)?.projectDir;
const packageJsonPath = samplePackageJsonPath
? join(sourceRoot, samplePackageJsonPath, 'package.json')
: undefined;
const installSuccess = installDependenciesInCache(
cacheDir,
combination,
cacheDirName,
packageJsonPath,
codeqlExePath,
);
if (!installSuccess) {
cdsExtractorLog(
'warn',
`Skipping failed dependency combination ${hash.substring(0, 8)} (cache directory: ${cacheDirName})`,
);
continue;
}
successfulInstallations++;
// Associate projects with this dependency combination
for (const [projectDir, project] of Array.from(dependencyGraph.projects.entries())) {
if (!project.packageJson) {
continue;
}
const p_cdsVersion = project.packageJson.dependencies?.['@sap/cds'] ?? 'latest';
const p_cdsDkVersion = project.packageJson.devDependencies?.['@sap/cds-dk'] ?? p_cdsVersion;
const p_cdsIndexerVersion =
project.packageJson.dependencies?.['@sap/cds-indexer'] ??
project.packageJson.devDependencies?.['@sap/cds-indexer'] ??
undefined;
// Resolve the project's versions to match against the combination's resolved versions
const projectResolvedVersions = resolveCdsVersions(p_cdsVersion, p_cdsDkVersion);
const projectActualCdsVersion = projectResolvedVersions.resolvedCdsVersion ?? p_cdsVersion;
const projectActualCdsDkVersion =
projectResolvedVersions.resolvedCdsDkVersion ?? p_cdsDkVersion;
// Match based on resolved versions since that's what the hash is based on
const combinationActualCdsVersion = combination.resolvedCdsVersion ?? combination.cdsVersion;
const combinationActualCdsDkVersion =
combination.resolvedCdsDkVersion ?? combination.cdsDkVersion;
if (
projectActualCdsVersion === combinationActualCdsVersion &&
projectActualCdsDkVersion === combinationActualCdsDkVersion &&
p_cdsIndexerVersion === combination.cdsIndexerVersion
) {
projectCacheDirMap.set(projectDir, cacheDir);
}
}
}
// Log final status
if (successfulInstallations === 0) {
cdsExtractorLog('error', 'Failed to install any dependency combinations.');
if (dependencyCombinations.length > 0) {
cdsExtractorLog(
'error',
`All ${dependencyCombinations.length} dependency combination(s) failed to install. This will likely cause compilation failures.`,
);
}
} else if (successfulInstallations < dependencyCombinations.length) {
cdsExtractorLog(
'warn',
`Successfully installed ${successfulInstallations} out of ${dependencyCombinations.length} dependency combinations.`,
);
} else {
cdsExtractorLog('info', 'All dependency combinations installed successfully.');
}
// Log project-to-cache-directory mappings for transparency.
if (projectCacheDirMap.size > 0) {
cdsExtractorLog('info', `Project to cache directory mappings:`);
for (const [projectDir, cacheDir] of Array.from(projectCacheDirMap.entries())) {
const cacheDirName = join(cacheDir).split('/').pop() ?? 'unknown';
cdsExtractorLog('info', ` ${projectDir} → ${cacheDirName}`);
}
} else {
cdsExtractorLog(
'warn',
'No project to cache directory mappings created. Projects may not have compatible dependencies installed.',
);
}
return projectCacheDirMap;
}
/**
* Extracts unique dependency combinations from the dependency graph.
* @param projects A map of projects from the dependency graph.
* @returns An array of unique dependency combinations.
*/
function extractUniqueDependencyCombinations(
projects: Map<string, CdsProject>,
): CdsDependencyCombination[] {
const combinations = new Map<string, CdsDependencyCombination>();
for (const project of Array.from(projects.values())) {
if (!project.packageJson) {
continue;
}
const cdsVersion = project.packageJson.dependencies?.['@sap/cds'] ?? 'latest';
const cdsDkVersion = project.packageJson.devDependencies?.['@sap/cds-dk'] ?? cdsVersion;
// Detect optional @sap/cds-indexer dependency
const cdsIndexerVersion =
project.packageJson.dependencies?.['@sap/cds-indexer'] ??
project.packageJson.devDependencies?.['@sap/cds-indexer'] ??
undefined;
// Resolve versions first to ensure we cache based on actual resolved versions
cdsExtractorLog(
'info',
`Resolving available dependency versions for project '${project.projectDir}' with dependencies: [@sap/cds@${cdsVersion}, @sap/cds-dk@${cdsDkVersion}]`,
);
const resolvedVersions = resolveCdsVersions(cdsVersion, cdsDkVersion);
const { resolvedCdsVersion, resolvedCdsDkVersion, ...rest } = resolvedVersions;
// Log the resolved CDS dependency versions for the project
if (resolvedCdsVersion && resolvedCdsDkVersion) {
let statusMsg: string;
if (resolvedVersions.cdsExactMatch && resolvedVersions.cdsDkExactMatch) {
statusMsg = ' (exact match)';
} else if (!resolvedVersions.isFallback) {
statusMsg = ' (compatible versions)';
} else {
statusMsg = ' (using fallback versions)';
}
cdsExtractorLog(
'info',
`Resolved to: @sap/cds@${resolvedCdsVersion}, @sap/cds-dk@${resolvedCdsDkVersion}${statusMsg}`,
);
} else {
cdsExtractorLog(
'error',
`Failed to resolve CDS dependencies: @sap/cds@${cdsVersion}, @sap/cds-dk@${cdsDkVersion}`,
);
}
// Calculate hash based on resolved versions to ensure proper cache reuse
const actualCdsVersion = resolvedCdsVersion ?? cdsVersion;
const actualCdsDkVersion = resolvedCdsDkVersion ?? cdsDkVersion;
const hashInput = cdsIndexerVersion
? `${actualCdsVersion}|${actualCdsDkVersion}|${cdsIndexerVersion}`
: `${actualCdsVersion}|${actualCdsDkVersion}`;
const hash = createHash('sha256').update(hashInput).digest('hex');
if (!combinations.has(hash)) {
combinations.set(hash, {
cdsVersion,
cdsDkVersion,
cdsIndexerVersion,
hash,
resolvedCdsVersion: resolvedCdsVersion ?? undefined,
resolvedCdsDkVersion: resolvedCdsDkVersion ?? undefined,
...rest,
});
}
}
return Array.from(combinations.values());
}
/**
* Attempt to install dependencies in a cache directory with fallback logic
* @param cacheDir Cache directory path
* @param combination Dependency combination to install
* @param cacheDirName Name of the cache directory for logging
* @param packageJsonPath Optional package.json path for diagnostics
* @param codeqlExePath Optional CodeQL executable path for diagnostics
* @returns True if installation succeeded, false otherwise
*/
function installDependenciesInCache(
cacheDir: string,
combination: CdsDependencyCombination,
cacheDirName: string,
packageJsonPath?: string,
codeqlExePath?: string,
): boolean {
const { resolvedCdsVersion, resolvedCdsDkVersion, isFallback, warning } = combination;
// Check if node_modules directory already exists in the cache dir
const nodeModulesExists =
existsSync(join(cacheDir, 'node_modules', '@sap', 'cds')) &&
existsSync(join(cacheDir, 'node_modules', '@sap', 'cds-dk'));
if (nodeModulesExists) {
cdsExtractorLog(
'info',
`Using cached dependencies for @sap/cds@${resolvedCdsVersion} and @sap/cds-dk@${resolvedCdsDkVersion} from ${cacheDirName}`,
);
// Add warning diagnostic if using fallback versions
if (isFallback && warning && packageJsonPath && codeqlExePath) {
addDependencyVersionWarning(packageJsonPath, warning, codeqlExePath);
}
return true;
}
if (!resolvedCdsVersion || !resolvedCdsDkVersion) {
cdsExtractorLog('error', 'Cannot install dependencies: no compatible versions found');
return false;
}
// Install dependencies in the cache directory
cdsExtractorLog(
'info',
`Installing @sap/cds@${resolvedCdsVersion} and @sap/cds-dk@${resolvedCdsDkVersion} in cache directory: ${cacheDirName}`,
);
if (isFallback && warning) {
cdsExtractorLog('warn', warning);
}
try {
execFileSync('npm', ['install', '--quiet', '--no-audit', '--no-fund'], {
cwd: cacheDir,
stdio: 'inherit',
shell: true,
});
// Add warning diagnostic if using fallback versions
if (isFallback && warning && packageJsonPath && codeqlExePath) {
addDependencyVersionWarning(packageJsonPath, warning, codeqlExePath);
}
return true;
} catch (err) {
const errorMessage = `Failed to install resolved dependencies in cache directory ${cacheDir}: ${err instanceof Error ? err.message : String(err)}`;
cdsExtractorLog('error', errorMessage);
return false;
}
}