Report job has infra failure to run-service (#4073)

Author: TingluoHuang

src/Runner.Common/RunServer.cs

@@ -30,6 +30,7 @@ Task CompleteJobAsync(
             string environmentUrl,
             IList<Telemetry> telemetry,
             string billingOwnerId,
+            string infrastructureFailureCategory,
             CancellationToken token);
 
         Task<RenewJobResponse> RenewJobAsync(Guid planId, Guid jobId, CancellationToken token);
@@ -80,11 +81,12 @@ public Task CompleteJobAsync(
             string environmentUrl,
             IList<Telemetry> telemetry,
             string billingOwnerId,
+            string infrastructureFailureCategory,
             CancellationToken cancellationToken)
         {
             CheckConnection();
             return RetryRequest(
-                async () => await _runServiceHttpClient.CompleteJobAsync(requestUri, planId, jobId, result, outputs, stepResults, jobAnnotations, environmentUrl, telemetry, billingOwnerId, cancellationToken), cancellationToken,
+                async () => await _runServiceHttpClient.CompleteJobAsync(requestUri, planId, jobId, result, outputs, stepResults, jobAnnotations, environmentUrl, telemetry, billingOwnerId, infrastructureFailureCategory, cancellationToken), cancellationToken,
                 shouldRetry: ex =>
                     ex is not VssUnauthorizedException &&               // HTTP status 401
                     ex is not TaskOrchestrationJobNotFoundException);   // HTTP status 404

src/Runner.Listener/JobDispatcher.cs

@@ -1211,7 +1211,7 @@ private async Task ForceFailJob(IRunServer runServer, Pipelines.AgentJobRequestM
                     jobAnnotations.Add(annotation.Value);
                 }
 
-                await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, TaskResult.Failed, outputs: null, stepResults: null, jobAnnotations: jobAnnotations, environmentUrl: null, telemetry: null, billingOwnerId: message.BillingOwnerId, CancellationToken.None);
+                await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, TaskResult.Failed, outputs: null, stepResults: null, jobAnnotations: jobAnnotations, environmentUrl: null, telemetry: null, billingOwnerId: message.BillingOwnerId, infrastructureFailureCategory: null, CancellationToken.None);
             }
             catch (Exception ex)
             {

src/Runner.Worker/ActionManager.cs

@@ -111,15 +111,15 @@ public sealed class ActionManager : RunnerService, IActionManager
             {
                 // Log the error and fail the PrepareActionsAsync Initialization.
                 Trace.Error($"Caught exception from PrepareActionsAsync Initialization: {ex}");
-                executionContext.InfrastructureError(ex.Message);
+                executionContext.InfrastructureError(ex.Message, category: "resolve_action");
                 executionContext.Result = TaskResult.Failed;
                 throw;
             }
             catch (InvalidActionArchiveException ex)
             {
                 // Log the error and fail the PrepareActionsAsync Initialization.
                 Trace.Error($"Caught exception from PrepareActionsAsync Initialization: {ex}");
-                executionContext.InfrastructureError(ex.Message);
+                executionContext.InfrastructureError(ex.Message, category: "invalid_action_download");
                 executionContext.Result = TaskResult.Failed;
                 throw;
             }
@@ -777,15 +777,15 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont
                 IOUtil.DeleteDirectory(destDirectory, executionContext.CancellationToken);
                 Directory.CreateDirectory(destDirectory);
 
-                if (downloadInfo.PackageDetails != null) 
+                if (downloadInfo.PackageDetails != null)
                 {
                     executionContext.Output($"##[group]Download immutable action package '{downloadInfo.NameWithOwner}@{downloadInfo.Ref}'");
                     executionContext.Output($"Version: {downloadInfo.PackageDetails.Version}");
                     executionContext.Output($"Digest: {downloadInfo.PackageDetails.ManifestDigest}");
                     executionContext.Output($"Source commit SHA: {downloadInfo.ResolvedSha}");
                     executionContext.Output("##[endgroup]");
-                } 
-                else 
+                }
+                else
                 {
                     executionContext.Output($"Download action repository '{downloadInfo.NameWithOwner}@{downloadInfo.Ref}' (SHA:{downloadInfo.ResolvedSha})");
                 }

src/Runner.Worker/ExecutionContext.cs

@@ -522,6 +522,10 @@ public TaskResult Complete(TaskResult? result = null, string currentOperation =
                     if (annotation != null)
                     {
                         stepResult.Annotations.Add(annotation.Value);
+                        if (annotation.Value.IsInfrastructureIssue && string.IsNullOrEmpty(Global.InfrastructureFailureCategory))
+                        {
+                            Global.InfrastructureFailureCategory = issue.Category;
+                        }
                     }
                 });
 
@@ -1335,9 +1339,9 @@ public static void Error(this IExecutionContext context, string message)
         }
 
         // Do not add a format string overload. See comment on ExecutionContext.Write().
-        public static void InfrastructureError(this IExecutionContext context, string message)
+        public static void InfrastructureError(this IExecutionContext context, string message, string category = null)
         {
-            var issue = new Issue() { Type = IssueType.Error, Message = message, IsInfrastructureIssue = true };
+            var issue = new Issue() { Type = IssueType.Error, Message = message, IsInfrastructureIssue = true, Category = category };
             context.AddIssue(issue, ExecutionContextLogOptions.Default);
         }
 

src/Runner.Worker/GlobalContext.cs

@@ -27,6 +27,7 @@ public sealed class GlobalContext
         public StepsContext StepsContext { get; set; }
         public Variables Variables { get; set; }
         public bool WriteDebug { get; set; }
+        public string InfrastructureFailureCategory { get; set; }
         public JObject ContainerHookState { get; set; }
     }
 }

src/Runner.Worker/JobRunner.cs

@@ -321,7 +321,7 @@ private async Task<TaskResult> CompleteJobAsync(IRunServer runServer, IExecution
             {
                 try
                 {
-                    await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, result, jobContext.JobOutputs, jobContext.Global.StepsResult, jobContext.Global.JobAnnotations, environmentUrl, telemetry, billingOwnerId: message.BillingOwnerId, default);
+                    await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, result, jobContext.JobOutputs, jobContext.Global.StepsResult, jobContext.Global.JobAnnotations, environmentUrl, telemetry, billingOwnerId: message.BillingOwnerId, infrastructureFailureCategory: jobContext.Global.InfrastructureFailureCategory, default);
                     return result;
                 }
                 catch (VssUnauthorizedException ex)

src/Sdk/RSWebApi/Contracts/CompleteJobRequest.cs

@@ -35,5 +35,8 @@ public class CompleteJobRequest
 
         [DataMember(Name = "billingOwnerId", EmitDefaultValue = false)]
         public string BillingOwnerId { get; set; }
+
+        [DataMember(Name = "infrastructureFailureCategory", EmitDefaultValue = false)]
+        public string InfrastructureFailureCategory { get; set; }
     }
 }

src/Sdk/RSWebApi/Contracts/IssueExtensions.cs

@@ -42,6 +42,7 @@ public static class IssueExtensions
                 StartColumn = columnNumber,
                 EndColumn = endColumnNumber,
                 StepNumber = stepNumber,
+                IsInfrastructureIssue = issue.IsInfrastructureIssue ?? false
             };
         }
 

src/Sdk/RSWebApi/RunServiceHttpClient.cs

@@ -131,6 +131,7 @@ public async Task CompleteJobAsync(
             string environmentUrl,
             IList<Telemetry> telemetry,
             string billingOwnerId,
+            string infrastructureFailureCategory,
             CancellationToken cancellationToken = default)
         {
             HttpMethod httpMethod = new HttpMethod("POST");
@@ -145,6 +146,7 @@ public async Task CompleteJobAsync(
                 EnvironmentUrl = environmentUrl,
                 Telemetry = telemetry,
                 BillingOwnerId = billingOwnerId,
+                InfrastructureFailureCategory = infrastructureFailureCategory
             };
 
             requestUri = new Uri(requestUri, "completejob");