Merge with main

.devcontainer/devcontainer.json

@@ -4,7 +4,7 @@
   "features": {
     "ghcr.io/devcontainers/features/docker-in-docker:1": {},
     "ghcr.io/devcontainers/features/dotnet": {
-      "version": "6.0.419"
+      "version": "6.0.421"
     },
     "ghcr.io/devcontainers/features/node:1": {
       "version": "16"

images/Dockerfile

@@ -4,9 +4,9 @@ FROM mcr.microsoft.com/dotnet/runtime-deps:6.0-jammy as build
 ARG TARGETOS
 ARG TARGETARCH
 ARG RUNNER_VERSION
-ARG RUNNER_CONTAINER_HOOKS_VERSION=0.5.1
-ARG DOCKER_VERSION=25.0.2
-ARG BUILDX_VERSION=0.12.1
+ARG RUNNER_CONTAINER_HOOKS_VERSION=0.6.0
+ARG DOCKER_VERSION=25.0.4
+ARG BUILDX_VERSION=0.13.1
 
 RUN apt update -y && apt install curl unzip -y
 

releaseNote.md

@@ -1,30 +1,20 @@
 ## What's Changed
-* Prepare v2.313.0 Release by @luketomlinson in https://github.com/actions/runner/pull/3137
-* Pass RunnerOS during job acquire. by @TingluoHuang in https://github.com/actions/runner/pull/3140
-* Process `snapshot` tokens by @davidomid in https://github.com/actions/runner/pull/3135
-* Update dotnet sdk to latest version @6.0.419 by @github-actions in https://github.com/actions/runner/pull/3158
-* handle broker run service exception handling by @yaananth in https://github.com/actions/runner/pull/3163
-* Add a retry logic to docker login operation by @enescakir in https://github.com/actions/runner/pull/3089
-* Broker fixes for token refreshes and AccessDeniedException by @luketomlinson in https://github.com/actions/runner/pull/3161
-* Remove USE_BROKER_FLOW by @luketomlinson in https://github.com/actions/runner/pull/3162
-* Refresh Token for BrokerServer by @luketomlinson in https://github.com/actions/runner/pull/3167
-* Better step timeout message. by @TingluoHuang in https://github.com/actions/runner/pull/3166
-
-## New Contributors
-* @davidomid made their first contribution in https://github.com/actions/runner/pull/3135
-* @enescakir made their first contribution in https://github.com/actions/runner/pull/3089
-
-**Full Changelog**: https://github.com/actions/runner/compare/v2.313.0...v2.314.0
+
+- Preserve dates when deserializing job message from Run Service by @ericsciple in https://github.com/actions/runner/pull/3269
+
+**Full Changelog**: https://github.com/actions/runner/compare/v2.316.0...v2.316.1
 
 _Note: Actions Runner follows a progressive release policy, so the latest release might not be available to your enterprise, organization, or repository yet.
 To confirm which version of the Actions Runner you should expect, please view the download instructions for your enterprise, organization, or repository.
 See https://docs.github.com/en/enterprise-cloud@latest/actions/hosting-your-own-runners/adding-self-hosted-runners_
 
 ## Windows x64
+
 We recommend configuring the runner in a root folder of the Windows drive (e.g. "C:\actions-runner"). This will help avoid issues related to service identity folder permissions and long file path restrictions on Windows.
 
 The following snipped needs to be run on `powershell`:
-``` powershell
+
+```powershell
 # Create a folder under the drive root
 mkdir \actions-runner ; cd \actions-runner
 # Download the latest runner package
@@ -35,12 +25,14 @@ Add-Type -AssemblyName System.IO.Compression.FileSystem ;
 ```
 
 ## [Pre-release] Windows arm64
+
 **Warning:** Windows arm64 runners are currently in preview status and use [unofficial versions of nodejs](https://unofficial-builds.nodejs.org/). They are not intended for production workflows.
 
 We recommend configuring the runner in a root folder of the Windows drive (e.g. "C:\actions-runner"). This will help avoid issues related to service identity folder permissions and long file path restrictions on Windows.
 
 The following snipped needs to be run on `powershell`:
-``` powershell
+
+```powershell
 # Create a folder under the drive root
 mkdir \actions-runner ; cd \actions-runner
 # Download the latest runner package
@@ -52,7 +44,7 @@ Add-Type -AssemblyName System.IO.Compression.FileSystem ;
 
 ## OSX x64
 
-``` bash
+```bash
 # Create a folder
 mkdir actions-runner && cd actions-runner
 # Download the latest runner package
@@ -63,7 +55,7 @@ tar xzf ./actions-runner-osx-x64-<RUNNER_VERSION>.tar.gz
 
 ## OSX arm64 (Apple silicon)
 
-``` bash
+```bash
 # Create a folder
 mkdir actions-runner && cd actions-runner
 # Download the latest runner package
@@ -74,7 +66,7 @@ tar xzf ./actions-runner-osx-arm64-<RUNNER_VERSION>.tar.gz
 
 ## Linux x64
 
-``` bash
+```bash
 # Create a folder
 mkdir actions-runner && cd actions-runner
 # Download the latest runner package
@@ -85,7 +77,7 @@ tar xzf ./actions-runner-linux-x64-<RUNNER_VERSION>.tar.gz
 
 ## Linux arm64
 
-``` bash
+```bash
 # Create a folder
 mkdir actions-runner && cd actions-runner
 # Download the latest runner package
@@ -96,7 +88,7 @@ tar xzf ./actions-runner-linux-arm64-<RUNNER_VERSION>.tar.gz
 
 ## Linux arm
 
-``` bash
+```bash
 # Create a folder
 mkdir actions-runner && cd actions-runner
 # Download the latest runner package
@@ -106,6 +98,7 @@ tar xzf ./actions-runner-linux-arm-<RUNNER_VERSION>.tar.gz
 ```
 
 ## Using your self hosted runner
+
 For additional details about configuring, running, or shutting down the runner please check out our [product docs.](https://help.github.com/en/actions/automating-your-workflow-with-github-actions/adding-self-hosted-runners)
 
 ## SHA-256 Checksums

src/Misc/layoutbin/RunnerService.js

@@ -114,6 +114,11 @@ var runService = function () {
             );
             stopping = true;
           }
+        } else if (code === 5) {
+          console.log(
+            "Runner listener exit with Session Conflict error, stop the service, no retry needed."
+          );
+          stopping = true;
         } else {
           var messagePrefix = "Runner listener exit with undefined return code";
           unknownFailureRetryCount++;

src/Misc/layoutroot/run-helper.cmd.template

@@ -49,5 +49,10 @@ if %ERRORLEVEL% EQU 4 (
   exit /b 1
 )
 
+if %ERRORLEVEL% EQU 5 (
+  echo "Runner listener exit with Session Conflict error, stop the service, no retry needed."
+  exit /b 0
+)
+
 echo "Exiting after unknown error code: %ERRORLEVEL%"
 exit /b 0

src/Misc/layoutroot/run-helper.sh.template

@@ -70,6 +70,9 @@ elif [[ $returnCode == 4 ]]; then
         "$DIR"/safe_sleep.sh 1
     done
     exit 2
+elif [[ $returnCode == 5 ]]; then
+    echo "Runner listener exit with Session Conflict error, stop the service, no retry needed."
+    exit 0
 else
     echo "Exiting with unknown error code: ${returnCode}"
     exit 0

src/Misc/layoutroot/run.sh

@@ -38,7 +38,7 @@ runWithManualTrap() {
         cp -f "$DIR"/run-helper.sh.template "$DIR"/run-helper.sh
         "$DIR"/run-helper.sh $* &
         PID=$!
-        wait -f $PID
+        wait $PID
         returnCode=$?
         if [[ $returnCode -eq 2 ]]; then
             echo "Restarting runner..."
@@ -84,4 +84,4 @@ if [[ -z "$RUNNER_MANUALLY_TRAP_SIG" ]]; then
     run $*
 else
     runWithManualTrap $*
-fi
+fi

src/Runner.Common/ActionsRunServer.cs

@@ -20,12 +20,12 @@ public sealed class ActionsRunServer : RunnerService, IActionsRunServer
     {
         private bool _hasConnection;
         private VssConnection _connection;
-        private TaskAgentHttpClient _taskAgentClient;
+        private ActionsRunServerHttpClient _actionsRunServerClient;
 
         public async Task ConnectAsync(Uri serverUrl, VssCredentials credentials)
         {
             _connection = await EstablishVssConnection(serverUrl, credentials, TimeSpan.FromSeconds(100));
-            _taskAgentClient = _connection.GetClient<TaskAgentHttpClient>();
+            _actionsRunServerClient = _connection.GetClient<ActionsRunServerHttpClient>();
             _hasConnection = true;
         }
 
@@ -42,7 +42,7 @@ public Task<AgentJobRequestMessage> GetJobMessageAsync(string id, CancellationTo
             CheckConnection();
             var jobMessage = RetryRequest<AgentJobRequestMessage>(async () =>
                                                     {
-                                                        return await _taskAgentClient.GetJobMessageAsync(id, cancellationToken);
+                                                        return await _actionsRunServerClient.GetJobMessageAsync(id, cancellationToken);
                                                     }, cancellationToken);
 
             return jobMessage;

src/Runner.Common/Constants.cs

@@ -154,6 +154,7 @@ public static class ReturnCode
                 public const int RetryableError = 2;
                 public const int RunnerUpdating = 3;
                 public const int RunOnceRunnerUpdating = 4;
+                public const int SessionConflict = 5;
             }
 
             public static class Features
@@ -181,6 +182,9 @@ public static class Features
             public static readonly string DeprecatedNodeVersion = "node16";
             public static readonly string EnforcedNode12DetectedAfterEndOfLife = "The following actions uses node12 which is deprecated and will be forced to run on node16: {0}. For more info: https://github.blog/changelog/2023-06-13-github-actions-all-actions-will-run-on-node16-instead-of-node12-by-default/";
             public static readonly string EnforcedNode12DetectedAfterEndOfLifeEnvVariable = "Node16ForceActionsWarnings";
+            public static readonly string EnforcedNode16DetectedAfterEndOfLife = "The following actions uses Node.js version which is deprecated and will be forced to run on node20: {0}. For more info: https://github.blog/changelog/2024-03-07-github-actions-all-actions-will-run-on-node20-instead-of-node16-by-default/";
+            public static readonly string EnforcedNode16DetectedAfterEndOfLifeEnvVariable = "Node20ForceActionsWarnings";
+
         }
 
         public static class RunnerEvent
@@ -252,6 +256,7 @@ public static class Actions
                 public static readonly string RunnerDebug = "ACTIONS_RUNNER_DEBUG";
                 public static readonly string StepDebug = "ACTIONS_STEP_DEBUG";
                 public static readonly string AllowActionsUseUnsecureNodeVersion = "ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION";
+                public static readonly string ManualForceActionsToNode20 = "FORCE_JAVASCRIPT_ACTIONS_TO_NODE20";
             }
 
             public static class Agent
@@ -263,6 +268,7 @@ public static class Agent
                 public static readonly string ForcedActionsNodeVersion = "ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION";
                 public static readonly string PrintLogToStdout = "ACTIONS_RUNNER_PRINT_LOG_TO_STDOUT";
                 public static readonly string ActionArchiveCacheDirectory = "ACTIONS_RUNNER_ACTION_ARCHIVE_CACHE";
+                public static readonly string ManualForceActionsToNode20 = "FORCE_JAVASCRIPT_ACTIONS_TO_NODE20";
             }
 
             public static class System

src/Runner.Common/JobServerQueue.cs

@@ -19,7 +19,7 @@ public interface IJobServerQueue : IRunnerService, IThrottlingReporter
         TaskCompletionSource<int> JobRecordUpdated { get; }
         event EventHandler<ThrottlingEventArgs> JobServerQueueThrottling;
         Task ShutdownAsync();
-        void Start(Pipelines.AgentJobRequestMessage jobRequest, bool resultsServiceOnly = false, bool enableTelemetry = false);
+        void Start(Pipelines.AgentJobRequestMessage jobRequest, bool resultsServiceOnly = false);
         void QueueWebConsoleLine(Guid stepRecordId, string line, long? lineNumber = null);
         void QueueFileUpload(Guid timelineId, Guid timelineRecordId, string type, string name, string path, bool deleteSource);
         void QueueResultsUpload(Guid timelineRecordId, string name, string path, string type, bool deleteSource, bool finalize, bool firstBlock, long totalLines);
@@ -74,6 +74,7 @@ public sealed class JobServerQueue : RunnerService, IJobServerQueue
         private readonly List<JobTelemetry> _jobTelemetries = new();
         private bool _queueInProcess = false;
         private bool _resultsServiceOnly = false;
+        private int _resultsServiceExceptionsCount = 0;
         private Stopwatch _resultsUploadTimer = new();
         private Stopwatch _actionsUploadTimer = new();
 
@@ -104,11 +105,10 @@ public override void Initialize(IHostContext hostContext)
             _resultsServer = hostContext.GetService<IResultsServer>();
         }
 
-        public void Start(Pipelines.AgentJobRequestMessage jobRequest, bool resultsServiceOnly = false, bool enableTelemetry = false)
+        public void Start(Pipelines.AgentJobRequestMessage jobRequest, bool resultsServiceOnly = false)
         {
             Trace.Entering();
             _resultsServiceOnly = resultsServiceOnly;
-            _enableTelemetry = enableTelemetry;
 
             var serviceEndPoint = jobRequest.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
 
@@ -139,6 +139,12 @@ public void Start(Pipelines.AgentJobRequestMessage jobRequest, bool resultsServi
                 _resultsClientInitiated = true;
             }
 
+            // Enable telemetry if we have both results service and actions service
+            if (_resultsClientInitiated && !_resultsServiceOnly)
+            {
+                _enableTelemetry = true;
+            }
+
             if (_queueInProcess)
             {
                 Trace.Info("No-opt, all queue process tasks are running.");
@@ -574,9 +580,9 @@ private async Task ProcessResultsUploadQueueAsync(bool runOnce = false)
                             Trace.Info("Catch exception during file upload to results, keep going since the process is best effort.");
                             Trace.Error(ex);
                             errorCount++;
-
+                            _resultsServiceExceptionsCount++;
                             // If we hit any exceptions uploading to Results, let's skip any additional uploads to Results unless Results is serving logs
-                            if (!_resultsServiceOnly)
+                            if (!_resultsServiceOnly && _resultsServiceExceptionsCount > 3)
                             {
                                 _resultsClientInitiated = false;
                                 SendResultsTelemetry(ex);

src/Runner.Listener/BrokerMessageListener.cs

@@ -42,7 +42,7 @@ public override void Initialize(IHostContext hostContext)
             _brokerServer = HostContext.GetService<IBrokerServer>();
         }
 
-        public async Task<Boolean> CreateSessionAsync(CancellationToken token)
+        public async Task<CreateSessionResult> CreateSessionAsync(CancellationToken token)
         {
             Trace.Entering();
 
@@ -99,7 +99,7 @@ public async Task<Boolean> CreateSessionAsync(CancellationToken token)
                         encounteringError = false;
                     }
 
-                    return true;
+                    return CreateSessionResult.Success;
                 }
                 catch (OperationCanceledException) when (token.IsCancellationRequested)
                 {
@@ -123,7 +123,7 @@ public async Task<Boolean> CreateSessionAsync(CancellationToken token)
                         if (string.Equals(vssOAuthEx.Error, "invalid_client", StringComparison.OrdinalIgnoreCase))
                         {
                             _term.WriteError("Failed to create a session. The runner registration has been deleted from the server, please re-configure. Runner registrations are automatically deleted for runners that have not connected to the service recently.");
-                            return false;
+                            return CreateSessionResult.Failure;
                         }
 
                         // Check whether we get 401 because the runner registration already removed by the service.
@@ -134,14 +134,18 @@ public async Task<Boolean> CreateSessionAsync(CancellationToken token)
                         if (string.Equals(authError, "invalid_client", StringComparison.OrdinalIgnoreCase))
                         {
                             _term.WriteError("Failed to create a session. The runner registration has been deleted from the server, please re-configure. Runner registrations are automatically deleted for runners that have not connected to the service recently.");
-                            return false;
+                            return CreateSessionResult.Failure;
                         }
                     }
 
                     if (!IsSessionCreationExceptionRetriable(ex))
                     {
                         _term.WriteError($"Failed to create session. {ex.Message}");
-                        return false;
+                        if (ex is TaskAgentSessionConflictException)
+                        {
+                            return CreateSessionResult.SessionConflict;
+                        }
+                        return CreateSessionResult.Failure;
                     }
 
                     if (!encounteringError) //print the message only on the first error

src/Runner.Listener/JobDispatcher.cs

@@ -1155,18 +1155,13 @@ private async Task LogWorkerProcessUnhandledException(IJobServer jobServer, Pipe
                 TimelineRecord jobRecord = timeline.Records.FirstOrDefault(x => x.Id == message.JobId && x.RecordType == "Job");
                 ArgUtil.NotNull(jobRecord, nameof(jobRecord));
 
-
                 jobRecord.ErrorCount++;
                 jobRecord.Issues.Add(issue);
 
-                if (message.Variables.TryGetValue("DistributedTask.MarkJobAsFailedOnWorkerCrash", out var markJobAsFailedOnWorkerCrash) &&
-                    StringUtil.ConvertToBoolean(markJobAsFailedOnWorkerCrash?.Value))
-                {
-                    Trace.Info("Mark the job as failed since the worker crashed");
-                    jobRecord.Result = TaskResult.Failed;
-                    // mark the job as completed so service will pickup the result
-                    jobRecord.State = TimelineRecordState.Completed;
-                }
+                Trace.Info("Mark the job as failed since the worker crashed");
+                jobRecord.Result = TaskResult.Failed;
+                // mark the job as completed so service will pickup the result
+                jobRecord.State = TimelineRecordState.Completed;
 
                 await jobServer.UpdateTimelineRecordsAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, new TimelineRecord[] { jobRecord }, CancellationToken.None);
             }