Refactor CDS extractor run modes

Remove "run modes" in an attempt to simplify the overall CDS extractor.

Author: data-douser

extractors/cds/tools/.gitignore

@@ -8,8 +8,3 @@ debug/
 # local environment.
 out/
 
-# Since we expect the build process to be run on the system where the CDS extractor
-# is being run, we do not need/want to check-in our own package-lock.json version
-# when we know it will be different / overwritten on each system.
-package-lock.json
-

extractors/cds/tools/cds-extractor.ts

@@ -3,60 +3,46 @@ import { join } from 'path';
 import { sync as globSync } from 'glob';
 
 import { determineCdsCommand } from './src/cds';
-import { orchestrateCompilation } from './src/cds/compiler/graph';
-import {
-  handleDebugParserMode,
-  handleDebugCompilerMode,
-  isDebugMode,
-  isDebugParserMode,
-  isDebugCompilerMode,
-} from './src/cds/parser/debugUtils';
-import { buildEnhancedCdsProjectDependencyGraph } from './src/cds/parser/graph';
+import { orchestrateCompilation } from './src/cds/compiler';
+import { buildEnhancedCdsProjectDependencyGraph } from './src/cds/parser';
 import { runJavaScriptExtractor } from './src/codeql';
 import { addCompilationDiagnostic } from './src/diagnostics';
 import { configureLgtmIndexFilters, setupAndValidateEnvironment } from './src/environment';
 import {
   cdsExtractorLog,
-  setSourceRootDirectory,
+  endPerformanceTracking,
+  logExtractorStop,
   logExtractorStart,
-  logExtractorEnd,
+  logMemoryUsage,
   logPerformanceMilestone,
+  setSourceRootDirectory,
   startPerformanceTracking,
-  endPerformanceTracking,
-  logMemoryUsage,
 } from './src/logging';
 import { installDependencies } from './src/packageManager';
-import { RunMode } from './src/runMode';
 import { validateArguments } from './src/utils';
 
-// Validate the first argument matches an allowed run mode and validate that
-// the total number of arguments is correct for the specified run mode.
+// Validate the script arguments.
 const validationResult = validateArguments(process.argv);
 if (!validationResult.isValid) {
   console.warn(validationResult.usageMessage);
   // Exit with an error code on invalid use of this script.
   process.exit(1);
 }
 
-// Get the validated and sanitized arguments
-const { runMode, sourceRoot } = validationResult.args!;
+// Get the validated and sanitized arguments.
+const { sourceRoot } = validationResult.args!;
 
-// Initialize the unified logging system with the source root directory
+// Initialize the unified logging system with the source root directory.
 setSourceRootDirectory(sourceRoot);
 
-// Log the start of the CDS extractor session
-logExtractorStart(runMode, sourceRoot);
+// Log the start of the CDS extractor session as a whole.
+logExtractorStart(sourceRoot);
+// We log the memory usage at the start of the extractor to track memory growth.
 logMemoryUsage('Extractor Start');
 
-// Check for autobuild mode
-if (runMode === (RunMode.AUTOBUILD as string)) {
-  cdsExtractorLog('info', 'Autobuild mode is not implemented yet.');
-  logExtractorEnd(false, 'Terminated: Autobuild mode not implemented');
-  process.exit(1);
-}
-
-// Setup the environment and validate all requirements first, before changing directory
-// This ensures we can properly locate the CodeQL tools
+// Setup the environment and validate all requirements first, before changing
+// directory back to the "sourceRoot" directory. This ensures we can properly locate
+// the CodeQL tools.
 startPerformanceTracking('Environment Setup');
 const {
   success: envSetupSuccess,
@@ -76,7 +62,7 @@ if (!envSetupSuccess) {
     )}.`,
   );
   // Exit with an error code when environment setup fails.
-  logExtractorEnd(false, 'Terminated: Environment setup failed');
+  logExtractorStop(false, 'Terminated: Environment setup failed');
   process.exit(1);
 }
 
@@ -86,18 +72,24 @@ process.chdir(sourceRoot);
 
 cdsExtractorLog(
   'info',
-  `CodeQL CDS extractor using run mode '${runMode}' for scan of project source root directory '${sourceRoot}'.`,
+  `CodeQL CDS extractor using autobuild mode for scan of project source root directory '${sourceRoot}'.`,
 );
 
-// Using the enhanced project-aware approach to find CDS projects and their dependencies
 cdsExtractorLog('info', 'Building enhanced CDS project dependency graph...');
 
-// Build the enhanced dependency graph using the new enhanced parser
+// Build the CDS project `dependencyGraph` as the foundation for the extraction process.
+// This graph will contain all discovered CDS projects, their dependencies, the `.cds`
+// files discovered within each project, the expected `.cds.json` files for each project
+// and the compilation status of such `.cds.json` files.
+//
+// The `dependencyGraph` will be updated as CDS extractor phases progress, allowing for
+// a single data structure to be used for planning, execution, retries (i.e. error handling),
+// debugging, and final reporting.
 let dependencyGraph;
 
-startPerformanceTracking('Dependency Graph Build');
 try {
-  dependencyGraph = buildEnhancedCdsProjectDependencyGraph(sourceRoot, runMode, __dirname);
+  startPerformanceTracking('Dependency Graph Build');
+  dependencyGraph = buildEnhancedCdsProjectDependencyGraph(sourceRoot, __dirname);
   endPerformanceTracking('Dependency Graph Build');
 
   logPerformanceMilestone(
@@ -106,13 +98,6 @@ try {
   );
   logMemoryUsage('After Dependency Graph');
 
-  // Handle debug modes early - these modes should exit after completing their specific tasks
-  if (isDebugParserMode(runMode)) {
-    const debugSuccess = handleDebugParserMode(dependencyGraph, sourceRoot, __dirname);
-    logExtractorEnd(debugSuccess, `Debug parser mode completed: ${runMode}`);
-    process.exit(debugSuccess ? 0 : 1);
-  }
-
   // Log details about discovered projects for debugging
   if (dependencyGraph.projects.size > 0) {
     for (const [projectDir, project] of dependencyGraph.projects.entries()) {
@@ -152,40 +137,25 @@ try {
 } catch (error) {
   cdsExtractorLog('error', `Failed to build enhanced dependency graph: ${String(error)}`);
   // Exit with error since we can't continue without a proper dependency graph
-  logExtractorEnd(false, 'Terminated: Dependency graph build failed');
+  logExtractorStop(false, 'Terminated: Dependency graph build failed');
   process.exit(1);
 }
 
-// Install dependencies of discovered CAP/CDS projects
-cdsExtractorLog('info', 'Installing dependencies for discovered CDS projects...');
-
 startPerformanceTracking('Dependency Installation');
 const projectCacheDirMap = installDependencies(dependencyGraph, sourceRoot, codeqlExePath);
 endPerformanceTracking('Dependency Installation');
 
 const cdsFilePathsToProcess: string[] = [];
 
-cdsExtractorLog('info', 'Extracting CDS files from discovered projects...');
-
 // Use the enhanced dependency graph to collect all `.cds` files from each project.
 // We want to "extract" all `.cds` files from all projects so that we have a copy
 // of each `.cds` source file in the CodeQL database.
 for (const project of dependencyGraph.projects.values()) {
   cdsFilePathsToProcess.push(...project.cdsFiles);
 }
 
-cdsExtractorLog('info', 'Processing CDS files to JSON using enhanced compilation orchestration...');
-
-// Check if we're running in debug mode
-if (isDebugMode(runMode)) {
-  cdsExtractorLog(
-    'info',
-    `Running in ${runMode} mode - enhanced debug information will be collected...`,
-  );
-}
-
-// Initialize CDS command cache early to avoid repeated testing during compilation
-// This is a critical optimization that avoids testing commands for every single file
+// Initialize CDS command cache early to avoid repeated testing during compilation.
+// This is a critical optimization that avoids testing commands for every single file.
 startPerformanceTracking('CDS Command Cache Initialization');
 try {
   determineCdsCommand(undefined, sourceRoot);
@@ -194,25 +164,18 @@ try {
 } catch (error) {
   endPerformanceTracking('CDS Command Cache Initialization');
   cdsExtractorLog('warn', `CDS command cache initialization failed: ${String(error)}`);
-  // Continue anyway - individual calls will handle fallbacks
 }
 
+// TODO : Improve logging / debugging of dependencyGraph.statusSummary. Just log the JSON?
 cdsExtractorLog(
   'info',
   `Found ${cdsFilePathsToProcess.length} total CDS files, ${dependencyGraph.statusSummary.totalCdsFiles} CDS files in dependency graph`,
 );
 
 startPerformanceTracking('CDS Compilation');
 try {
-  // Use the new orchestrated compilation approach with debug awareness
-  orchestrateCompilation(dependencyGraph, projectCacheDirMap, codeqlExePath, isDebugMode(runMode));
-
-  // Check if we should exit for debug modes after successful compilation
-  if (isDebugCompilerMode(runMode)) {
-    const debugSuccess = handleDebugCompilerMode(dependencyGraph, runMode);
-    logExtractorEnd(debugSuccess, `Debug mode completed: ${runMode}`);
-    process.exit(debugSuccess ? 0 : 1);
-  }
+  // Use the new orchestrated compilation approach (autobuild mode, no debug)
+  orchestrateCompilation(dependencyGraph, projectCacheDirMap, codeqlExePath, false);
 
   // Handle compilation failures for normal mode
   if (!dependencyGraph.statusSummary.overallSuccess) {
@@ -249,16 +212,17 @@ try {
 // Configure the "LGTM" index filters for proper extraction.
 configureLgtmIndexFilters();
 
-// Run CodeQL's JavaScript extractor to process the compiled JSON files.
+// Run CodeQL's JavaScript extractor to process the .cds source files and
+// the compiled .cds.json files.
 startPerformanceTracking('JavaScript Extraction');
 const extractorResult = runJavaScriptExtractor(sourceRoot, autobuildScriptPath, codeqlExePath);
 endPerformanceTracking('JavaScript Extraction');
 
 if (!extractorResult.success && extractorResult.error) {
   cdsExtractorLog('error', `Error running JavaScript extractor: ${extractorResult.error}`);
-  logExtractorEnd(false, 'JavaScript extractor failed');
+  logExtractorStop(false, 'JavaScript extractor failed');
 } else {
-  logExtractorEnd(true, 'CDS extraction completed successfully');
+  logExtractorStop(true, 'CDS extraction completed successfully');
 }
 
 // Use the `cds-extractor.js` name in the log message as that is the name of the script

extractors/cds/tools/index-files.sh

@@ -2,63 +2,50 @@
 
 set -eu
 
-if [ $# -ne 1 ]
-then
-    echo "Usage:  $0 <response_file_path>"
-    exit 1
-fi
+## This script currently:
+## - ignores any arguments passed to it;
+## - assumes it is run from the root of the project source directory;
+## 
 
 if ! command -v node > /dev/null
 then
     echo "node executable is required (in PATH) to run the 'cds-extractor.js' script. Please install Node.js and try again."
     exit 2
 fi
 
-if ! command -v npm > /dev/null
-then
-    echo "npm executable is required (in PATH) to install the dependencies for the 'cds-extractor.js' script."
-    exit 3
-fi
-
 # Set the _cwd variable to the present working directory (PWD) as the directory
 # from which this script was called, which we assume is the "source root" directory
 # of the project that to be scanned / indexed.
 _cwd="$PWD"
-_response_file_path="$1"
-_run_mode="index-files"
 _script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+_cds_extractor_js_path="${_script_dir}/out/cds-extractor.js"
+_cds_extractor_node_modules_dir="${_script_dir}/node_modules"
 
-echo "Checking response file for CDS files to index"
+if [ ! -f "${_cds_extractor_js_path}" ]; then
+    echo "Error: The 'cds-extractor.js' script does not exist at the expected path: ${_cds_extractor_js_path}"
+    echo "Please ensure that the script has been built and is available in the 'out' directory."
+    exit 3
+fi
 
-# Terminate early if the _response_file_path doesn't exist or is empty,
-# which indicates that no CDS files were selected or found.
-if [ ! -f "$_response_file_path" ] || [ ! -s "$_response_file_path" ]
-then
-    echo "'codeql database cds-extractor --language cds' command terminated early as response file '$_response_file_path' does not exist or is empty. This is because no CDS files were selected or found."
-    # Exit without error to avoid failing any calling (javascript)
-    # extractor, and llow the tool the report the lack of coverage
-    # for CDS files.
-    exit 0
+if [ ! -d "${_cds_extractor_node_modules_dir}" ]; then
+    echo "Error: The 'node_modules' directory does not exist at the expected path: ${_cds_extractor_node_modules_dir}"
+    echo "Please ensure that the dependencies have been installed by running 'npm install' in the 'extractors/cds/tools' directory."
+    exit 4
 fi
 
-# Change to the directory of this script to ensure that npm looks up the
+# Change to the directory of this shell script to ensure that npm looks up the
 # package.json file in the correct directory and installs the dependencies
 # (i.e. node_modules) relative to this directory. This is technically a
 # violation of the assumption that extractor scripts will be run with the
 # current working directory set to the root of the project source, but we
 # also need node_modules to be installed here and not in the project source
 # root, so we make a compromise of:
-#  1. changing to this script's directory;
-#  2. installing node dependencies here;
-#  3. passing the original working directory as a parameter to the
+#  1. changing to this shell script's directory;
+#  2. passing the original working directory as a parameter to the
 #     cds-extractor.js script;
-#  4. expecting the cds-extractor.js script to immediately change back to
+#  3. expecting the cds-extractor.js script to immediately change back to
 #     original working (aka the project source root) directory.
 
 cd "$_script_dir" && \
-echo "Installing node package dependencies" && \
-npm install --quiet --no-audit --no-fund && \
-echo "Building TypeScript code" && \
-npm run build && \
 echo "Running the 'cds-extractor.js' script" && \
-node "$(dirname "$0")/out/cds-extractor.js" "$_run_mode" "$_cwd" "$_response_file_path"
+node "${_cds_extractor_js_path}" "$_cwd"