Another attempted fix for index-files.js cwd

Changes the way the index-files.js script is invoked such that the
original `--source-root` directory is used, where possible, as the
current working directory for any work performed with within the
extractor. Passes the original working directory as a parameter of
the index-files.js script to allow that (child) script to run from
the project/source root while ensuring node package dependencies
are still installed in `extractors/cds/tools/node_modules`.

Author: data-douser

extractors/cds/tools/autobuild.sh

@@ -2,12 +2,6 @@
 
 set -eu
 
-# NOTE: the code below is copied in three places:
-#  - scripts/compile-cds.sh
-#  - extractors/cds/tools/autobuild.sh (here)
-#  - extractors/javascript/tools/pre-finalize.sh
-# Any changes should be synchronized between these three places.
-
 exec "${CODEQL_DIST}/codeql" database index-files \
     --include-extension=.cds \
     --language cds \

extractors/cds/tools/index-files.cmd

@@ -19,20 +19,36 @@ if %ERRORLEVEL% neq 0 (
 
 set "_response_file_path=%~1"
 set "_script_dir=%~dp0"
+REM Set _cwd before changing the working directory to the script directory.
+set "_cwd=%CD%"
 
 echo Checking response file for CDS files to index
 
+REM Terminate early if the _response_file_path doesn't exist or is empty,
+REM which indicates that no CDS files were selected or found.
 if not exist "%_response_file_path%" (
     echo 'codeql database index-files --language cds' command terminated early as response file '%_response_file_path%' does not exist or is empty. This is because no CDS files were selected or found.
     exit /b 0
 )
 
-REM Change to the directory of this script to ensure that npm looks up
-REM the package.json file in the correct directory and installs the
-REM dependencies (i.e. node_modules) relative to this directory.
+REM Change to the directory of this script to ensure that npm looks up the
+REM package.json file in the correct directory and installs the dependencies
+REM (i.e. node_modules) relative to this directory. This is technically a
+REM violation of the assumption that extractor scripts will be run with the
+REM current working directory set to the root of the project source, but we
+REM also need node_modules to be installed here and not in the project source
+REM root, so we make a compromise of:
+REM  1. changing to this script's directory;
+REM  2. installing node dependencies here;
+REM  3. passing the original working directory as a parameter to the
+REM     index-files.js script;
+REM  4. expecting the index-files.js script to immediately change back to
+REM     the original working (aka the project source root) directory.
+
 cd /d "%_script_dir%" && ^
-echo Installing node package dependencies and running the 'index-files.js' script && ^
-npm install --quiet --no-audit --no-fund --no-package-json && ^
-node "%_script_dir%index-files.js" "%_response_file_path%"
+echo Installing node package dependencies && ^
+npm install --quiet --no-audit --no-fund && ^
+echo Running the 'index-files.js' script && ^
+node "%_script_dir%index-files.js" "%_response_file_path%" "%_cwd%"
 
 exit /b %ERRORLEVEL%

extractors/cds/tools/index-files.js

@@ -4,19 +4,29 @@ const { arch, platform } = require('os');
 const { dirname, join, resolve } = require('path');
 const { quote } = require('shell-quote');
 
-console.log('Indexing CDS files');
+// Terminate early if this script is not invoked with the required arguments.
+if (process.argv.length !== 4) {
+    console.warn(`Usage: node index-files.js <response-file> <source-root>`);
+    process.exit(0);
+}
 
 const responseFile = process.argv[2];
+const sourceRoot = process.argv[3];
+
+// Force this script, and any process it spawns, to use the project (source)
+// root directory as the current working directory.
+process.chdir(sourceRoot);
+
+console.log(`Indexing CDS files in project source directory: ${sourceRoot}`);
 
 const osPlatform = platform();
 const osPlatformArch = arch();
 console.log(`Detected OS platform=${osPlatform} : arch=${osPlatformArch}`);
 const codeqlExe = osPlatform === 'win32' ? 'codeql.exe' : 'codeql';
 const codeqlExePath = join(quote([process.env.CODEQL_DIST]), codeqlExe);
-const projectRootDir = resolve(`${dirname(__filename)}/../../..`);
 
-if (!existsSync(projectRootDir)) {
-    console.warn(`'${codeqlExe} database index-files --language cds' terminated early due to internal error: could not find project root directory '${projectRootDir}'.`);
+if (!existsSync(sourceRoot)) {
+    console.warn(`'${codeqlExe} database index-files --language cds' terminated early due to internal error: could not find project root directory '${sourceRoot}'.`);
     process.exit(0);
 }
 
@@ -265,13 +275,15 @@ console.log(
  * 'javascript' extractor.
  *
  * IMPORTANT: The JavaScript extractor autobuild script must be invoked with
- * the current working directory set to the project root directory because it
- * assumes it is running from there. Without the `cwd` property set to the
- * project root directory, the autobuild script will not detect the .cds.json
- * files as being in the project and will not index them.
+ * the current working directory set to the project (source) root directory
+ * because it assumes it is running from there. The JavaScript extractor will
+ * only find the .cds files to index (to the database) if those file are
+ * relative to where the autobuild script is invoked from, which should be the
+ * same as the `--source-root` argument passed to the `codeql database create`
+ * command.
  */
 spawnSync(
     autobuildScriptPath,
     [],
-    { cwd: projectRootDir, env: process.env, shell: true, stdio: 'inherit' }
+    { cwd: sourceRoot, env: process.env, shell: true, stdio: 'inherit' }
 );

extractors/cds/tools/index-files.sh

@@ -20,6 +20,7 @@ then
     exit 3
 fi
 
+_cwd="$PWD"
 _response_file_path="$1"
 _script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
@@ -36,11 +37,22 @@ then
     exit 0
 fi
 
-# Change to the directory of this script to ensure that npm looks up
-# the package.json file in the correct directory and installs the
-# dependencies (i.e. node_modules) relative to this directory.
+# Change to the directory of this script to ensure that npm looks up the
+# package.json file in the correct directory and installs the dependencies
+# (i.e. node_modules) relative to this directory. This is technically a
+# violation of the assumption that extractor scripts will be run with the
+# current working directory set to the root of the project source, but we
+# also need node_modules to be installed here and not in the project source
+# root, so we make a compromise of:
+#  1. changing to this script's directory;
+#  2. installing node dependencies here;
+#  3. passing the original working directory as a parameter to the
+#     index-files.js script;
+#  4. expecting the index-files.js script to immediately change back to
+#     the original working (aka the project source root) directory.
+
 cd "$_script_dir" && \
 echo "Installing node package dependencies" && \
 npm install --quiet --no-audit --no-fund && \
 echo "Running the 'index-files.js' script" && \
-node "$(dirname "$0")/index-files.js" "$_response_file_path"
+node "$(dirname "$0")/index-files.js" "$_response_file_path" "${_cwd}"

extractors/javascript/tools/pre-finalize.sh

@@ -2,14 +2,7 @@
 
 set -eu
 
-# NOTE: the code below is copied in three places:
-#  - scripts/compile-cds.sh
-#  - extractors/cds/tools/autobuild.sh
-#  - extractors/javascript/tools/pre-finalize.sh (here)
-# Any changes should be synchronized between these three places.
-
-# Do not extract CDS files if the
-#  CODEQL_EXTRACTOR_CDS_SKIP_EXTRACTION
+# Do not extract CDS files if the CODEQL_EXTRACTOR_CDS_SKIP_EXTRACTION
 # environment variable is set.
 if [ -z "${CODEQL_EXTRACTOR_CDS_SKIP_EXTRACTION:-}" ]; then
     # Call the index-files command with the CDS extractor