Improve retry handling based on feedback (#1447)

Author: juliobbv

src/Runner.Common/JobServer.cs

@@ -38,7 +38,8 @@ public sealed class JobServer : RunnerService, IJobServer
         public async Task ConnectAsync(VssConnection jobConnection)
         {
             _connection = jobConnection;
-            int attemptCount = 5;
+            int totalAttempts = 5;
+            int attemptCount = totalAttempts;
             var configurationStore = HostContext.GetService<IConfigurationStore>();
             var runnerSettings = configurationStore.GetSettings();
 
@@ -56,18 +57,21 @@ public async Task ConnectAsync(VssConnection jobConnection)
 
                     if (runnerSettings.IsHostedServer)
                     {
-                        await CheckNetworkEndpointsAsync();
+                        await CheckNetworkEndpointsAsync(attemptCount);
                     }
                 }
 
-                await Task.Delay(100);
+                int attempt = totalAttempts - attemptCount;
+                TimeSpan backoff = BackoffTimerHelper.GetExponentialBackoff(attempt, TimeSpan.FromMilliseconds(100), TimeSpan.FromSeconds(3.2), TimeSpan.FromMilliseconds(100));
+
+                await Task.Delay(backoff);
             }
 
             _taskClient = _connection.GetClient<TaskHttpClient>();
             _hasConnection = true;
         }
 
-        private async Task CheckNetworkEndpointsAsync()
+        private async Task CheckNetworkEndpointsAsync(int attemptsLeft)
         {
             try
             {
@@ -79,8 +83,8 @@ private async Task CheckNetworkEndpointsAsync()
 
                     actionsClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents);
 
-                    // Call the _apis/health endpoint
-                    var response = await actionsClient.GetAsync(new Uri(baseUri, "_apis/health"));
+                    // Call the _apis/health endpoint, and include how many attempts are left as a URL query for easy tracking
+                    var response = await actionsClient.GetAsync(new Uri(baseUri, $"_apis/health?_internalRunnerAttemptsLeft={attemptsLeft}"));
                     Trace.Info($"Actions health status code: {response.StatusCode}");
                 }
             }
@@ -100,8 +104,8 @@ private async Task CheckNetworkEndpointsAsync()
                 {
                     gitHubClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents);
 
-                    // Call the api.github.com endpoint
-                    var response = await gitHubClient.GetAsync("https://api.github.com");
+                    // Call the api.github.com endpoint, and include how many attempts are left as a URL query for easy tracking
+                    var response = await gitHubClient.GetAsync($"https://api.github.com?_internalRunnerAttemptsLeft={attemptsLeft}");
                     Trace.Info($"api.github.com status code: {response.StatusCode}");
                 }
             }