bundle integration; quick bundles as well as correct processing from configuration file.

jsinglet · jsinglet · commit ed856968bb9b · 2024-02-21T12:28:48.000-05:00
diff --git a/src/CodeQLToolkit.Core/Main.cs b/src/CodeQLToolkit.Core/Main.cs
@@ -29,6 +29,7 @@ public static async Task<int> Main(string[] args)
             rootCommand.AddGlobalOption(Globals.BasePathOption);
             rootCommand.AddGlobalOption(Globals.AutomationTypeOption);
             rootCommand.AddGlobalOption(Globals.Development);
+            rootCommand.AddGlobalOption(Globals.UseBundle);
 
             var versionCommand = new Command("version", "Get the current tool version.");
             rootCommand.Add(versionCommand);
diff --git a/src/CodeQLToolkit.Features/Bundle/Lifecycle/BundleLifecycleFeature.cs b/src/CodeQLToolkit.Features/Bundle/Lifecycle/BundleLifecycleFeature.cs
@@ -32,22 +32,22 @@ public override LanguageType[] SupportedLangauges
 
         public void Register(Command parentCommand)
         {
-            Log<BundleLifecycleFeature>.G().LogInformation("Registering lifecycle submodule.");
+            //Log<BundleLifecycleFeature>.G().LogInformation("Registering lifecycle submodule.");
 
             var setCommand = new Command("set", "Functions pertaining to setting variables related to custom CodeQL bundles.");
-            parentCommand.Add(setCommand);
+            //parentCommand.Add(setCommand);
 
             var enableCommand = new Command("enable-custom-bundles", "Enables custom CodeQL Bundles.");            
-            setCommand.Add(enableCommand);
+            //setCommand.Add(enableCommand);
 
             var disableCommand = new Command("disable-custom-bundles", "Disables custom CodeQL Bundles.");
-            setCommand.Add(disableCommand);
+            //setCommand.Add(disableCommand);
 
             var getCommand = new Command("get", "Functions pertaining to getting variables related to CodeQL Bundles.");
-            parentCommand.Add(getCommand);
+            //parentCommand.Add(getCommand);
 
             var getEnabledCommand = new Command("enabled", "Determines if custom CodeQL Bundles are enabled.");
-            getCommand.Add(getEnabledCommand);
+            //getCommand.Add(getEnabledCommand);
 
             {
                 enableCommand.SetHandler((basePath) =>
diff --git a/src/CodeQLToolkit.Features/Bundle/Lifecycle/Targets/GetEnabledCustomCodeQLBundlesLifecycleTarget.cs b/src/CodeQLToolkit.Features/Bundle/Lifecycle/Targets/GetEnabledCustomCodeQLBundlesLifecycleTarget.cs
@@ -22,7 +22,7 @@ override public void Run()
             var config = c.FromFile();
 
             Console.WriteLine($"---------current settings---------");
-            Console.WriteLine($"CodeQL Custom Bundles Enabled: {config.EnableCustomCodeQLBundles}");
+            //Console.WriteLine($"CodeQL Custom Bundles Enabled: {config.EnableCustomCodeQLBundles}");
             Console.WriteLine($"----------------------------------");
             Console.WriteLine("(hint: use `qlt bundle set` to modify these values.)");
 
diff --git a/src/CodeQLToolkit.Features/Bundle/Lifecycle/Targets/SetDisableCustomCodeQLBundlesLifecycleTarget.cs b/src/CodeQLToolkit.Features/Bundle/Lifecycle/Targets/SetDisableCustomCodeQLBundlesLifecycleTarget.cs
@@ -21,7 +21,7 @@ override public void Run()
 
             var config = c.FromFile();
 
-            config.EnableCustomCodeQLBundles = false;
+            //config.EnableCustomCodeQLBundles = false;
 
             config.ToFile();
 
diff --git a/src/CodeQLToolkit.Features/Bundle/Lifecycle/Targets/SetEnableCustomCodeQLBundlesLifecycleTarget.cs b/src/CodeQLToolkit.Features/Bundle/Lifecycle/Targets/SetEnableCustomCodeQLBundlesLifecycleTarget.cs
@@ -21,7 +21,7 @@ override public void Run()
 
             var config = c.FromFile();
 
-            config.EnableCustomCodeQLBundles = true;
+            //config.EnableCustomCodeQLBundles = true;
 
             config.ToFile();
 
diff --git a/src/CodeQLToolkit.Features/CodeQL/Commands/CodeQLCommandFeature.cs b/src/CodeQLToolkit.Features/CodeQL/Commands/CodeQLCommandFeature.cs
@@ -39,26 +39,31 @@ public void Register(Command parentCommand)
             parentCommand.Add(runCommand);
             
             var installCommand = new Command("install", "Installs CodeQL (bundle or release distribution) locally.");
-            //var useCommand = new Command("use", "Switches tooling to use a different CodeQL version and updates the paths accordingly.");
-            //var listCommand = new Command("list", "Lists versions of CodeQL available locally.");
+            var customBundleOption = new Option<bool>("--custom-bundle", () => false, "Build a custom bundle and compile the bundle.") { IsRequired = true};
+            var quickBundleOption = new Option<bool>("--quick-bundle", () => false,  "Build a custom bundle and DO NOT compile the bundle.") { IsRequired = true};
+            var packsOption = new Option<string[]>("--packs", "When creating bundles, this specifies the packs to include, Example `pack1 pack2 pack3`. You may specify also as `--pack pack1 --pack2 --pack3`") { IsRequired = false, AllowMultipleArgumentsPerToken = true };
 
-            runCommand.Add(installCommand);
-            //runCommand.Add(useCommand);
-            //runCommand.Add(listCommand);
+            installCommand.Add(customBundleOption);
+            installCommand.Add(quickBundleOption);
+            installCommand.Add(packsOption);
 
-           
-            installCommand.SetHandler((basePath, automationType) =>
+            runCommand.Add(installCommand);
+            
+            installCommand.SetHandler((basePath, automationType, customBundleOption, quickBundleOption, packs) =>
             {
                 Log<CodeQLCommandFeature>.G().LogInformation("Executing install command...");
 
                 new InstallCommand()
                 {
                     Base = basePath,
                     AutomationTarget = automationType,
+                    CustomBundles = customBundleOption,
+                    QuickBundles = quickBundleOption,
+                    Packs = packs
                 }.Run();
 
 
-            }, Globals.BasePathOption, Globals.AutomationTypeOption);
+            }, Globals.BasePathOption, Globals.AutomationTypeOption, customBundleOption, quickBundleOption, packsOption);
         }
 
         public int Run()
diff --git a/src/CodeQLToolkit.Features/CodeQL/Commands/Targets/InstallCommand.cs b/src/CodeQLToolkit.Features/CodeQL/Commands/Targets/InstallCommand.cs
@@ -11,13 +11,33 @@ namespace CodeQLToolkit.Features.CodeQL.Commands.Targets
 {
     public class InstallCommand : CommandTarget
     {
-
+        public bool CustomBundles { get; set; }
+        public bool QuickBundles { get; set; }
+        public string[] Packs { get; set; }
         public override void Run()
         {
             Log<InstallCommand>.G().LogInformation($"Running Install command");
 
             // First, check if CodeQL is installed.
             var installation = CodeQLInstallation.LoadFromConfig(Base);
+            if(CustomBundles || QuickBundles)
+            {
+                installation.EnableCustomCodeQLBundles = true;
+                if (Packs!=null && Packs.Length > 0)
+                {
+                    Log<InstallCommand>.G().LogInformation($"Overriding Packs on the command line. The following Packs will be packaged:");
+                    installation.ExportedCustomizationPacks = Packs;
+                }
+                else
+                {
+                    Log<InstallCommand>.G().LogInformation($"Using `qlt.conf.json` to build list of packs to bundle. The following Packs will be packaged:");
+                }
+
+                installation.LogPacksToBeBuilt();
+
+
+                installation.QuickBundle = QuickBundles;
+            }
 
             Log<InstallCommand>.G().LogInformation($"Checking for installation...");
 
@@ -32,7 +52,6 @@ public override void Run()
                 Log<InstallCommand>.G().LogInformation($"Installing CodeQL...");
                 installation.Install();
 
-
                 // set the environment variable
                 Log<InstallCommand>.G().LogInformation($"Setting QLT_CODEQL_HOME to {installation.CodeQLHome}...");
                 Log<InstallCommand>.G().LogInformation($"Setting QLT_CODEQL_PATH to {installation.CodeQLToolBinary}...");
diff --git a/src/CodeQLToolkit.Features/Query/Commands/QueryCommandFeature.cs b/src/CodeQLToolkit.Features/Query/Commands/QueryCommandFeature.cs
@@ -30,7 +30,8 @@ public void Register(Command parentCommand)
 
             runCommand.Add(installPacksQueryCommand);
 
-            installPacksQueryCommand.SetHandler((basePath) => new InstallQueryPacksCommandTarget() { Base = basePath }.Run(), Globals.BasePathOption);
+            installPacksQueryCommand.SetHandler(
+                (basePath, useBundle) => new InstallQueryPacksCommandTarget() { Base = basePath, UseBundle = useBundle}.Run(), Globals.BasePathOption, Globals.UseBundle);
 
         }
 
diff --git a/src/CodeQLToolkit.Features/Query/Commands/Targets/InstallQueryPacksCommandTarget.cs b/src/CodeQLToolkit.Features/Query/Commands/Targets/InstallQueryPacksCommandTarget.cs
@@ -1,4 +1,5 @@
 ﻿using CodeQLToolkit.Shared.CodeQL;
+using Microsoft.VisualBasic;
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
@@ -23,6 +24,10 @@ public override void Run()
             var installation = CodeQLInstallation.LoadFromConfig(Base);
 
 
+            installation.EnableCustomCodeQLBundles = UseBundle;
+
+            installation.IsInstalledOrDie();
+
             foreach ( string file in files )
             {
                 Log<InstallQueryPacksCommandTarget>.G().LogInformation($"Installing qlpack {file}...");
diff --git a/src/CodeQLToolkit.Features/Test/Commands/Targets/Actions/ExecuteUnitTestsCommandTarget.cs b/src/CodeQLToolkit.Features/Test/Commands/Targets/Actions/ExecuteUnitTestsCommandTarget.cs
@@ -75,11 +75,9 @@ public override void Run()
                      // Get A Copy of the installation 
                      var installation = CodeQLInstallation.LoadFromConfig(Base);
 
-                     if(!installation.IsInstalled())
-                     {
-                         DieWithError($"Requested CodeQL Version ({CLIVersion}) Not Installed. Run `qlt codeql run install` before running this step.");
-                     }
+                     installation.EnableCustomCodeQLBundles = UseBundle;
 
+                     installation.IsInstalledOrDie();
 
                      using (Process process = new Process())
                      {
diff --git a/src/CodeQLToolkit.Features/Test/Commands/TestCommandFeature.cs b/src/CodeQLToolkit.Features/Test/Commands/TestCommandFeature.cs
@@ -94,7 +94,7 @@ public void Register(Command parentCommand)
             }, Globals.BasePathOption, Globals.AutomationTypeOption, matrixOSVersion);
 
             //stdLibIdent
-            unitTestsCommand.SetHandler((basePath, automationType, numThreads, workDirectory, language, runnerOS, extraArgs) => {
+            unitTestsCommand.SetHandler((basePath, automationType, numThreads, workDirectory, language, runnerOS, extraArgs, useBundle) => {
 
                 Log<TestCommandFeature>.G().LogInformation("Executing execute-unit-tests command...");
 
@@ -125,6 +125,7 @@ public void Register(Command parentCommand)
                 featureTarget.CLIVersion = config.CodeQLCLI;
                 featureTarget.STDLibIdent = config.CodeQLStandardLibraryIdent;
                 featureTarget.ExtraCodeQLArgs = extraArgs;
+                featureTarget.UseBundle = useBundle;
 
                 featureTarget.Run();
 
@@ -134,7 +135,8 @@ public void Register(Command parentCommand)
                workDirectoryOption, 
                languageOption, 
                runnerOSOption, 
-               extraCodeQLOptions
+               extraCodeQLOptions,
+               Globals.UseBundle
             );
 
 
diff --git a/src/CodeQLToolkit.Features/Validation/Commands/Targets/CheckQueriesCommandTarget.cs b/src/CodeQLToolkit.Features/Validation/Commands/Targets/CheckQueriesCommandTarget.cs
@@ -23,6 +23,9 @@ public override void Run()
 
             var installation = CodeQLInstallation.LoadFromConfig(Base);
 
+            installation.EnableCustomCodeQLBundles = UseBundle;
+
+            installation.IsInstalledOrDie();
 
             using (Process process = new Process())
             {
diff --git a/src/CodeQLToolkit.Features/Validation/Commands/ValidationCommandFeature.cs b/src/CodeQLToolkit.Features/Validation/Commands/ValidationCommandFeature.cs
@@ -41,18 +41,19 @@ public void Register(Command parentCommand)
             runCommand.Add(checkQueryQueriesCommand);
 
 
-            checkQueryQueriesCommand.SetHandler((language, basePath, prettyPrint) =>
+            checkQueryQueriesCommand.SetHandler((language, basePath, prettyPrint, useBundle) =>
             {
                 Log<ValidationCommandFeature>.G().LogInformation("Executing check-query-metadata command...");
 
                 new CheckQueriesCommandTarget()
                 {
                     Base = basePath,
                     Language = language,
-                    PrettyPrint = prettyPrint,                    
+                    PrettyPrint = prettyPrint,              
+                    UseBundle = useBundle
                 }.Run();
                 
-            }, languageOption, Globals.BasePathOption, prettyPrintOption);
+            }, languageOption, Globals.BasePathOption, prettyPrintOption, Globals.UseBundle);
         }
 
         public int Run()
diff --git a/src/CodeQLToolkit.Shared/CodeQL/CodeQLInstallation.cs b/src/CodeQLToolkit.Shared/CodeQL/CodeQLInstallation.cs
@@ -19,6 +19,7 @@ public class CodeQLInstallation
         public string StandardLibraryIdent { get; set; }
         public bool EnableCustomCodeQLBundles { get; set; }
         public string[] ExportedCustomizationPacks { get; set; }
+        public bool QuickBundle {  get; set; }
         public string Base { get; set; }
 
         public static CodeQLInstallation LoadFromConfig(string Base)
@@ -38,7 +39,6 @@ public static CodeQLInstallation LoadFromConfig(QLTConfig c)
 
             return new CodeQLInstallation
             {
-                EnableCustomCodeQLBundles = config.EnableCustomCodeQLBundles,
                 CLIVersion = config.CodeQLCLI,
                 CLIBundle = config.CodeQLCLIBundle,
                 StandardLibraryIdent = config.CodeQLStandardLibraryIdent,
@@ -50,6 +50,17 @@ public static CodeQLInstallation LoadFromConfig(QLTConfig c)
 
         }
 
+        public void LogPacksToBeBuilt()
+        {
+            if(ExportedCustomizationPacks != null)
+            {
+                foreach(var p in ExportedCustomizationPacks)
+                {
+                    Log<CodeQLInstallation>.G().LogInformation($"Pack: {p}");
+                }
+            }
+        }
+
         public ArtifactKind Kind 
         {
             get
@@ -280,7 +291,15 @@ private void CustomBundleInstall()
                 process.StartInfo.UseShellExecute = false;
                 process.StartInfo.RedirectStandardOutput = false;
                 process.StartInfo.RedirectStandardError = false;
-                process.StartInfo.Arguments = $"-b {customBundleSource} -o {CustomBundleOutputDirectory} -w {workingDirectory} {packs}";
+                if (QuickBundle)
+                {
+                    Log<CodeQLInstallation>.G().LogInformation($"Note: Quick Bundles enabled and pre-compilation will be disabled...");
+                    process.StartInfo.Arguments = $"-nc -b {customBundleSource} -o {CustomBundleOutputDirectory} -w {workingDirectory} {packs}";
+                }
+                else
+                {
+                    process.StartInfo.Arguments = $"-b {customBundleSource} -o {CustomBundleOutputDirectory} -w {workingDirectory} {packs}";
+                }
 
                 process.Start();
 
@@ -355,12 +374,32 @@ private string GetIdentForPackage(ArtifactKind k)
 
             throw new NotImplementedException();
         }
-        // TODO -- need to check environment variables to see if custom bundle is "installed"
+
+        public bool IsInstalledOrDie()
+        {
+            var installed = IsInstalled();
+
+            if (!installed)
+            {
+                ProcessUtils.DieWithError("CodeQL not installed. Please run `qlt codeql run install --help` for more info.");
+            }
+
+            return true;
+        }
+
         public bool IsInstalled()
         {
             Log<CodeQLInstallation>.G().LogInformation($"Checking if codeql is installed...");
-            Log<CodeQLInstallation>.G().LogInformation($"Requested CLI Version {CLIVersion}, Standard Library Ident: {StandardLibraryIdent}, CLIBundle: {CLIBundle}, Using Custom Bundles: {EnableCustomCodeQLBundles}");
 
+            if (EnableCustomCodeQLBundles)
+            {
+                Log<CodeQLInstallation>.G().LogInformation($"Requested Custom Bundle Based on Bundle {CLIBundle}");
+            }
+            else
+            {
+                Log<CodeQLInstallation>.G().LogInformation($"Requested CLI Version {CLIVersion} with Standard Library Ident: {StandardLibraryIdent}");
+            }
+           
             Log<CodeQLInstallation>.G().LogInformation($"Checking for existance of required directories...");
 
             if (!Directory.Exists(InstallationDirectory))
diff --git a/src/CodeQLToolkit.Shared/CodeQL/RESOLUTION.md b/src/CodeQLToolkit.Shared/CodeQL/RESOLUTION.md
diff --git a/src/CodeQLToolkit.Shared/Options/Globals.cs b/src/CodeQLToolkit.Shared/Options/Globals.cs
diff --git a/src/CodeQLToolkit.Shared/Target/ITarget.cs b/src/CodeQLToolkit.Shared/Target/ITarget.cs
diff --git a/src/CodeQLToolkit.Shared/Utils/QLTConfig.cs b/src/CodeQLToolkit.Shared/Utils/QLTConfig.cs

Original file line number	Diff line number	Diff line change
`@@ -30,7 +30,8 @@ public void Register(Command parentCommand)`
`30`	`30`
`31`	`31`	`runCommand.Add(installPacksQueryCommand);`
`32`	`32`
`33`		`- installPacksQueryCommand.SetHandler((basePath) => new InstallQueryPacksCommandTarget() { Base = basePath }.Run(), Globals.BasePathOption);`
	`33`	`+ installPacksQueryCommand.SetHandler(`
	`34`	`+ (basePath, useBundle) => new InstallQueryPacksCommandTarget() { Base = basePath, UseBundle = useBundle}.Run(), Globals.BasePathOption, Globals.UseBundle);`
`34`	`35`
`35`	`36`	`}`
`36`	`37`