From 45a4043dd8fbefe2bf20598859a0304bd45688c6 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Fri, 21 Mar 2025 14:21:48 -0700 Subject: [PATCH 01/16] Fix GrpcWorkerChannel.StartWorkerProcessAsync timeout --- .../Channel/GrpcWorkerChannel.cs | 24 ++++-- .../Extensions/ExceptionExtensions.cs | 21 ++++-- .../Host/IWorkerFunctionMetadataProvider.cs | 4 +- src/WebJobs.Script/Host/ScriptHostState.cs | 6 +- .../Host/WorkerFunctionMetadataProvider.cs | 9 ++- .../ProcessManagement/IWorkerProcess.cs | 6 +- .../ProcessManagement/WorkerProcess.cs | 74 +++++++++++++------ .../Workers/Rpc/RpcWorkerProcess.cs | 6 +- .../Workers/Http/HttpWorkerChannelTests.cs | 6 +- .../Workers/Rpc/GrpcWorkerChannelTests.cs | 6 +- 10 files changed, 111 insertions(+), 51 deletions(-) diff --git a/src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs b/src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs index a0a04b7b16..a8f760f87d 100644 --- a/src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs +++ b/src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs @@ -367,19 +367,31 @@ private void DispatchMessage(InboundGrpcEvent msg) public bool IsChannelReadyForInvocations() { - return !_disposing && !_disposed && _state.HasFlag(RpcWorkerChannelState.InvocationBuffersInitialized | RpcWorkerChannelState.Initialized); + return !_disposing && !_disposed + && _state.HasFlag( + RpcWorkerChannelState.InvocationBuffersInitialized | RpcWorkerChannelState.Initialized); } public async Task StartWorkerProcessAsync(CancellationToken cancellationToken) { - RegisterCallbackForNextGrpcMessage(MsgType.StartStream, _workerConfig.CountOptions.ProcessStartupTimeout, 1, SendWorkerInitRequest, HandleWorkerStartStreamError); - // note: it is important that the ^^^ StartStream is in place *before* we start process the loop, otherwise we get a race condition + RegisterCallbackForNextGrpcMessage( + MsgType.StartStream, + _workerConfig.CountOptions.ProcessStartupTimeout, + count: 1, + SendWorkerInitRequest, + HandleWorkerStartStreamError); + + // note: it is important that the ^^^ StartStream is in place *before* we start process the loop, + // otherwise we get a race condition _ = ProcessInbound(); _workerChannelLogger.LogDebug("Initiating Worker Process start up"); - await _rpcWorkerProcess.StartProcessAsync(); - _state = _state | RpcWorkerChannelState.Initializing; - await _workerInitTask.Task; + await _rpcWorkerProcess.StartProcessAsync(cancellationToken); + _state |= RpcWorkerChannelState.Initializing; + Task winner = await Task.WhenAny( + _workerInitTask.Task, _rpcWorkerProcess.WaitForExitAsync(cancellationToken)) + .WaitAsync(cancellationToken); + await winner; } public async Task GetWorkerStatusAsync() diff --git a/src/WebJobs.Script/Extensions/ExceptionExtensions.cs b/src/WebJobs.Script/Extensions/ExceptionExtensions.cs index d60286aa5f..da1d36e79d 100644 --- a/src/WebJobs.Script/Extensions/ExceptionExtensions.cs +++ b/src/WebJobs.Script/Extensions/ExceptionExtensions.cs @@ -1,7 +1,8 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. -using System.Reflection; +using System.Collections.Generic; +using System.Runtime.ExceptionServices; using System.Runtime.InteropServices; using System.Threading; using Microsoft.Azure.WebJobs.Host.Diagnostics; @@ -32,12 +33,22 @@ or SEHException public static string ToFormattedString(this Exception exception) { - if (exception == null) + ArgumentNullException.ThrowIfNull(exception); + return ExceptionFormatter.GetFormattedException(exception); + } + + public static void ThrowIfErrorsPresent(IList exceptions, string message = null) + { + switch (exceptions) { - throw new ArgumentNullException(nameof(exception)); + case null or []: + return; + case [Exception e]: + ExceptionDispatchInfo.Capture(e).Throw(); + return; + default: + throw new AggregateException(message, exceptions); } - - return ExceptionFormatter.GetFormattedException(exception); } } } diff --git a/src/WebJobs.Script/Host/IWorkerFunctionMetadataProvider.cs b/src/WebJobs.Script/Host/IWorkerFunctionMetadataProvider.cs index ace3c61ed1..b19b890ec8 100644 --- a/src/WebJobs.Script/Host/IWorkerFunctionMetadataProvider.cs +++ b/src/WebJobs.Script/Host/IWorkerFunctionMetadataProvider.cs @@ -9,14 +9,14 @@ namespace Microsoft.Azure.WebJobs.Script { /// - /// Defines an interface for fetching function metadata from Out-of-Proc language workers + /// Defines an interface for fetching function metadata from Out-of-Proc language workers. /// internal interface IWorkerFunctionMetadataProvider { ImmutableDictionary> FunctionErrors { get; } /// - /// Attempts to get function metadata from Out-of-Proc language workers + /// Attempts to get function metadata from Out-of-Proc language workers. /// /// FunctionMetadataResult that either contains the function metadata or indicates that a fall back option for fetching metadata should be used Task GetFunctionMetadataAsync(IEnumerable workerConfigs, bool forceRefresh = false); diff --git a/src/WebJobs.Script/Host/ScriptHostState.cs b/src/WebJobs.Script/Host/ScriptHostState.cs index 2816f4f706..4fe31b9f95 100644 --- a/src/WebJobs.Script/Host/ScriptHostState.cs +++ b/src/WebJobs.Script/Host/ScriptHostState.cs @@ -6,7 +6,7 @@ namespace Microsoft.Azure.WebJobs.Script public enum ScriptHostState { /// - /// The host has not yet been created + /// The host has not yet been created. /// Default, @@ -28,7 +28,7 @@ public enum ScriptHostState Running, /// - /// The host is in an error state + /// The host is in an error state. /// Error, @@ -43,7 +43,7 @@ public enum ScriptHostState Stopped, /// - /// The host is offline + /// The host is offline. /// Offline } diff --git a/src/WebJobs.Script/Host/WorkerFunctionMetadataProvider.cs b/src/WebJobs.Script/Host/WorkerFunctionMetadataProvider.cs index 018a4dc016..f97d149d95 100644 --- a/src/WebJobs.Script/Host/WorkerFunctionMetadataProvider.cs +++ b/src/WebJobs.Script/Host/WorkerFunctionMetadataProvider.cs @@ -77,7 +77,7 @@ public async Task GetFunctionMetadataAsync(IEnumerable GetFunctionMetadataAsync(IEnumerable errors = null; foreach (string workerId in channels.Keys.ToList()) { if (channels.TryGetValue(workerId, out TaskCompletionSource languageWorkerChannelTask)) @@ -129,7 +130,7 @@ public async Task GetFunctionMetadataAsync(IEnumerable ValidateFunctionAppFormat(_scriptOptions.CurrentValue.ScriptPath, _logger, _environment)); @@ -140,9 +141,13 @@ public async Task GetFunctionMetadataAsync(IEnumerable _scriptApplicationHostOptions; - private bool _useStdErrorStreamForErrorsOnly; - private Queue _processStdErrDataQueue = new Queue(3); + private readonly object _syncLock = new(); + private readonly bool _useStdErrorStreamForErrorsOnly; + private Queue _processStdErrDataQueue = new(3); private IHostProcessMonitor _processMonitor; - private object _syncLock = new object(); + private TaskCompletionSource _processExit; // used to hold custom exceptions on non-success exit. internal WorkerProcess(IScriptEventManager eventManager, IProcessRegistry processRegistry, ILogger workerProcessLogger, IWorkerConsoleLogSource consoleLogSource, IMetricsLogger metricsLogger, IServiceProvider serviceProvider, ILoggerFactory loggerFactory, IEnvironment environment, IOptionsMonitor scriptApplicationHostOptions, bool useStdErrStreamForErrorsOnly = false) @@ -69,8 +71,9 @@ internal WorkerProcess(IScriptEventManager eventManager, IProcessRegistry proces internal abstract Process CreateWorkerProcess(); - public Task StartProcessAsync() + public Task StartProcessAsync(CancellationToken cancellationToken = default) { + cancellationToken.ThrowIfCancellationRequested(); using (_metricsLogger.LatencyEvent(MetricEventNames.ProcessStart)) { Process = CreateWorkerProcess(); @@ -79,6 +82,7 @@ public Task StartProcessAsync() AssignUserExecutePermissionsIfNotExists(); } + _processExit = new(); try { Process.ErrorDataReceived += (sender, e) => OnErrorDataReceived(sender, e); @@ -103,12 +107,24 @@ public Task StartProcessAsync() } catch (Exception ex) { + _processExit.TrySetException(ex); _workerProcessLogger.LogError(ex, $"Failed to start Worker Channel. Process fileName: {Process.StartInfo.FileName}"); return Task.FromException(ex); } } } + public Task WaitForExitAsync(CancellationToken cancellationToken = default) + { + if (_processExit is { } tcs) + { + // We use a TaskCompletionSource (and not Process.WaitForExitAsync) so we can propagate our custom exceptions. + return tcs.Task.WaitAsync(cancellationToken); + } + + throw new InvalidOperationException("Process has not been started yet."); + } + private void OnErrorDataReceived(object sender, DataReceivedEventArgs e) { if (e.Data != null) @@ -159,42 +175,58 @@ private void OnProcessExited(object sender, EventArgs e) if (Disposing) { - // No action needed return; } try { - if (Process.ExitCode == WorkerConstants.SuccessExitCode) - { - Process.WaitForExit(); - Process.Close(); - } - else if (Process.ExitCode == WorkerConstants.IntentionalRestartExitCode) + ThrowIfExitError(); + + Process.WaitForExit(); + if (Process.ExitCode == WorkerConstants.IntentionalRestartExitCode) { HandleWorkerProcessRestart(); } - else - { - string exceptionMessage = string.Join(",", _processStdErrDataQueue.Where(s => !string.IsNullOrEmpty(s))); - string sanitizedExceptionMessage = Sanitizer.Sanitize(exceptionMessage); - var processExitEx = new WorkerProcessExitException($"{Process.StartInfo.FileName} exited with code {Process.ExitCode} (0x{Process.ExitCode.ToString("X")})", new Exception(sanitizedExceptionMessage)); - processExitEx.ExitCode = Process.ExitCode; - processExitEx.Pid = Process.Id; - HandleWorkerProcessExitError(processExitEx); - } + } + catch (WorkerProcessExitException processExitEx) + { + _processExit.TrySetException(processExitEx); + HandleWorkerProcessExitError(processExitEx); } catch (Exception exc) { - _workerProcessLogger?.LogDebug(exc, "Exception on worker process exit. Process id: {processId}", Process?.Id); // ignore process is already disposed + _processExit.TrySetException(exc); + _workerProcessLogger?.LogDebug(exc, "Exception on worker process exit. Process id: {processId}", Process?.Id); } finally { + _processExit.TrySetResult(); UnregisterFromProcessMonitor(); + Process.Close(); } } + private void ThrowIfExitError() + { + if (Process.ExitCode is WorkerConstants.SuccessExitCode or WorkerConstants.IntentionalRestartExitCode) + { + return; + } + + string exceptionMessage = string.Join(",", _processStdErrDataQueue.Where(s => !string.IsNullOrEmpty(s))); + string sanitizedExceptionMessage = Sanitizer.Sanitize(exceptionMessage); + WorkerProcessExitException processExitEx = new( + $"{Process.StartInfo.FileName} exited with code {Process.ExitCode} (0x{Process.ExitCode:X})", + new Exception(sanitizedExceptionMessage)) + { + ExitCode = Process.ExitCode, + Pid = Process.Id + }; + + throw processExitEx; + } + private void OnOutputDataReceived(object sender, DataReceivedEventArgs e) { if (e.Data != null) diff --git a/src/WebJobs.Script/Workers/Rpc/RpcWorkerProcess.cs b/src/WebJobs.Script/Workers/Rpc/RpcWorkerProcess.cs index 76a9766503..7e2504ad4f 100644 --- a/src/WebJobs.Script/Workers/Rpc/RpcWorkerProcess.cs +++ b/src/WebJobs.Script/Workers/Rpc/RpcWorkerProcess.cs @@ -74,14 +74,12 @@ internal override Process CreateWorkerProcess() internal override void HandleWorkerProcessExitError(WorkerProcessExitException rpcWorkerProcessExitException) { + ArgumentNullException.ThrowIfNull(rpcWorkerProcessExitException); if (Disposing) { return; } - if (rpcWorkerProcessExitException == null) - { - throw new ArgumentNullException(nameof(rpcWorkerProcessExitException)); - } + // The subscriber of WorkerErrorEvent is expected to Dispose() the errored channel _workerProcessLogger.LogError(rpcWorkerProcessExitException, $"Language Worker Process exited. Pid={rpcWorkerProcessExitException.Pid}.", _workerProcessArguments.ExecutablePath); _eventManager.Publish(new WorkerErrorEvent(_runtime, _workerId, rpcWorkerProcessExitException)); diff --git a/test/WebJobs.Script.Tests/Workers/Http/HttpWorkerChannelTests.cs b/test/WebJobs.Script.Tests/Workers/Http/HttpWorkerChannelTests.cs index 4429e589fc..d102ba42b2 100644 --- a/test/WebJobs.Script.Tests/Workers/Http/HttpWorkerChannelTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Http/HttpWorkerChannelTests.cs @@ -35,7 +35,7 @@ public async Task TestStartWorkerProcess(bool isWorkerReady) Mock httpWorkerService = new Mock(); httpWorkerService.Setup(a => a.IsWorkerReady(It.IsAny())).Returns(Task.FromResult(isWorkerReady)); - workerProcess.Setup(a => a.StartProcessAsync()).Returns(Task.CompletedTask); + workerProcess.Setup(a => a.StartProcessAsync(default)).Returns(Task.CompletedTask); TestLogger logger = new TestLogger("HttpWorkerChannel"); IHttpWorkerChannel testWorkerChannel = new HttpWorkerChannel("RandomWorkerId", _eventManager, workerProcess.Object, httpWorkerService.Object, logger, _metricsLogger, 3); Task resultTask = null; @@ -61,7 +61,7 @@ public async Task TestStartWorkerProcess_WorkerServiceThrowsException() IMetricsLogger metricsLogger = new TestMetricsLogger(); httpWorkerService.Setup(a => a.IsWorkerReady(It.IsAny())).Throws(new Exception("RandomException")); - workerProcess.Setup(a => a.StartProcessAsync()).Returns(Task.CompletedTask); + workerProcess.Setup(a => a.StartProcessAsync(default)).Returns(Task.CompletedTask); TestLogger logger = new TestLogger("HttpWorkerChannel"); IHttpWorkerChannel testWorkerChannel = new HttpWorkerChannel("RandomWorkerId", _eventManager, workerProcess.Object, httpWorkerService.Object, logger, _metricsLogger, 3); Task resultTask = null; @@ -85,7 +85,7 @@ public async Task TestStartWorkerProcess_WorkerProcessThrowsException() IMetricsLogger metricsLogger = new TestMetricsLogger(); httpWorkerService.Setup(a => a.IsWorkerReady(It.IsAny())).Throws(new Exception("RandomException")); - workerProcess.Setup(a => a.StartProcessAsync()).Throws(new Exception("RandomException")); + workerProcess.Setup(a => a.StartProcessAsync(default)).Throws(new Exception("RandomException")); TestLogger logger = new TestLogger("HttpWorkerChannel"); IHttpWorkerChannel testWorkerChannel = new HttpWorkerChannel("RandomWorkerId", _eventManager, workerProcess.Object, httpWorkerService.Object, logger, _metricsLogger, 3); Task resultTask = null; diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/GrpcWorkerChannelTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/GrpcWorkerChannelTests.cs index 4483f8e4ab..938f63891b 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/GrpcWorkerChannelTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/GrpcWorkerChannelTests.cs @@ -76,7 +76,7 @@ public GrpcWorkerChannelTests(ITestOutputHelper testOutput) _testWorkerConfig.CountOptions.InitializationTimeout = TimeSpan.FromSeconds(5); _testWorkerConfig.CountOptions.EnvironmentReloadTimeout = TimeSpan.FromSeconds(5); - _mockrpcWorkerProcess.Setup(m => m.StartProcessAsync()).Returns(Task.CompletedTask); + _mockrpcWorkerProcess.Setup(m => m.StartProcessAsync(default)).Returns(Task.CompletedTask); _mockrpcWorkerProcess.Setup(m => m.Id).Returns(910); _testEnvironment = new TestEnvironment(); _testEnvironment.SetEnvironmentVariable(FunctionDataCacheConstants.FunctionDataCacheEnabledSettingName, "1"); @@ -225,7 +225,7 @@ public async Task WorkerChannel_Dispose_Without_WorkerTerminateCapability() public async Task StartWorkerProcessAsync_Invoked_SetupFunctionBuffers_Verify_ReadyForInvocation() { await CreateDefaultWorkerChannel(); - _mockrpcWorkerProcess.Verify(m => m.StartProcessAsync(), Times.Once); + _mockrpcWorkerProcess.Verify(m => m.StartProcessAsync(default), Times.Once); Assert.False(_workerChannel.IsChannelReadyForInvocations()); _workerChannel.SetupFunctionInvocationBuffers(GetTestFunctionsList("node")); Assert.True(_workerChannel.IsChannelReadyForInvocations()); @@ -288,7 +288,7 @@ public async Task StartWorkerProcessAsync_WorkerProcess_Throws() { // note: uses custom worker channel Mock mockrpcWorkerProcessThatThrows = new Mock(); - mockrpcWorkerProcessThatThrows.Setup(m => m.StartProcessAsync()).Throws(); + mockrpcWorkerProcessThatThrows.Setup(m => m.StartProcessAsync(default)).Throws(); _workerChannel = new GrpcWorkerChannel( _workerId, From cc3410cc519cd054594e7dd2cc565d9ba07eb6f5 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Mon, 14 Apr 2025 10:57:53 -0700 Subject: [PATCH 02/16] Add unit tests for worker exit --- .../Channel/GrpcWorkerChannel.cs | 11 +- .../ProcessManagement/WorkerProcess.cs | 6 +- .../Workers/Rpc/GrpcWorkerChannelTests.cs | 139 +++++++++++------- .../Workers/Rpc/RpcWorkerProcessTests.cs | 62 +++++++- 4 files changed, 153 insertions(+), 65 deletions(-) diff --git a/src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs b/src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs index a8f760f87d..8052a23979 100644 --- a/src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs +++ b/src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs @@ -388,10 +388,15 @@ public async Task StartWorkerProcessAsync(CancellationToken cancellationToken) _workerChannelLogger.LogDebug("Initiating Worker Process start up"); await _rpcWorkerProcess.StartProcessAsync(cancellationToken); _state |= RpcWorkerChannelState.Initializing; - Task winner = await Task.WhenAny( - _workerInitTask.Task, _rpcWorkerProcess.WaitForExitAsync(cancellationToken)) - .WaitAsync(cancellationToken); + Task exited = _rpcWorkerProcess.WaitForExitAsync(cancellationToken); + Task winner = await Task.WhenAny(_workerInitTask.Task, exited).WaitAsync(cancellationToken); await winner; + + if (winner == exited) + { + // process exited without throwing. We need to throw to indicate process is not running. + throw new WorkerProcessExitException("Worker process exited before initializing."); + } } public async Task GetWorkerStatusAsync() diff --git a/src/WebJobs.Script/Workers/ProcessManagement/WorkerProcess.cs b/src/WebJobs.Script/Workers/ProcessManagement/WorkerProcess.cs index f9bbad769b..139055c916 100644 --- a/src/WebJobs.Script/Workers/ProcessManagement/WorkerProcess.cs +++ b/src/WebJobs.Script/Workers/ProcessManagement/WorkerProcess.cs @@ -85,9 +85,9 @@ public Task StartProcessAsync(CancellationToken cancellationToken = default) _processExit = new(); try { - Process.ErrorDataReceived += (sender, e) => OnErrorDataReceived(sender, e); - Process.OutputDataReceived += (sender, e) => OnOutputDataReceived(sender, e); - Process.Exited += (sender, e) => OnProcessExited(sender, e); + Process.ErrorDataReceived += OnErrorDataReceived; + Process.OutputDataReceived += OnOutputDataReceived; + Process.Exited += OnProcessExited; Process.EnableRaisingEvents = true; string sanitizedArguments = Sanitizer.Sanitize(Process.StartInfo.Arguments); diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/GrpcWorkerChannelTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/GrpcWorkerChannelTests.cs index 938f63891b..adcb8ea576 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/GrpcWorkerChannelTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/GrpcWorkerChannelTests.cs @@ -39,9 +39,9 @@ public class GrpcWorkerChannelTests : IDisposable private static string _expectedSystemLogMessage = "Random system log message"; private static string _expectedLoadMsgPartial = "Sending FunctionLoadRequest for "; - private readonly Mock _mockrpcWorkerProcess = new Mock(); + private readonly Mock _mockRpcWorkerProcess = new Mock(); private readonly string _workerId = "testWorkerId"; - private readonly string _scriptRootPath = "c:\testdir"; + private readonly string _scriptRootPath = "c:\\testdir"; private readonly IScriptEventManager _eventManager = new ScriptEventManager(); private readonly Mock _mockScriptHostManager = new Mock(MockBehavior.Strict); private readonly TestMetricsLogger _metricsLogger = new TestMetricsLogger(); @@ -57,7 +57,6 @@ public class GrpcWorkerChannelTests : IDisposable private readonly IOptionsMonitor _hostOptionsMonitor; private readonly IMemoryMappedFileAccessor _mapAccessor; private readonly ISharedMemoryManager _sharedMemoryManager; - private readonly IFunctionDataCache _functionDataCache; private readonly IOptions _workerConcurrencyOptions; private readonly ITestOutputHelper _testOutput; private readonly IOptions _hostingConfigOptions; @@ -76,24 +75,24 @@ public GrpcWorkerChannelTests(ITestOutputHelper testOutput) _testWorkerConfig.CountOptions.InitializationTimeout = TimeSpan.FromSeconds(5); _testWorkerConfig.CountOptions.EnvironmentReloadTimeout = TimeSpan.FromSeconds(5); - _mockrpcWorkerProcess.Setup(m => m.StartProcessAsync(default)).Returns(Task.CompletedTask); - _mockrpcWorkerProcess.Setup(m => m.Id).Returns(910); + _mockRpcWorkerProcess.Setup(m => m.StartProcessAsync(It.IsAny())).Returns(Task.CompletedTask); + _mockRpcWorkerProcess.Setup(m => m.WaitForExitAsync(It.IsAny())).Returns(Task.Delay(Timeout.Infinite)); + _mockRpcWorkerProcess.Setup(m => m.Id).Returns(910); _testEnvironment = new TestEnvironment(); _testEnvironment.SetEnvironmentVariable(FunctionDataCacheConstants.FunctionDataCacheEnabledSettingName, "1"); _workerConcurrencyOptions = Options.Create(new WorkerConcurrencyOptions()); _workerConcurrencyOptions.Value.CheckInterval = TimeSpan.FromSeconds(1); - ILogger mmapAccessorLogger = NullLogger.Instance; + ILogger mMapAccessorLogger = NullLogger.Instance; if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { - _mapAccessor = new MemoryMappedFileAccessorWindows(mmapAccessorLogger); + _mapAccessor = new MemoryMappedFileAccessorWindows(mMapAccessorLogger); } else { - _mapAccessor = new MemoryMappedFileAccessorUnix(mmapAccessorLogger, _testEnvironment); + _mapAccessor = new MemoryMappedFileAccessorUnix(mMapAccessorLogger, _testEnvironment); } _sharedMemoryManager = new SharedMemoryManager(_loggerFactory, _mapAccessor); - _functionDataCache = new FunctionDataCache(_sharedMemoryManager, _loggerFactory, _testEnvironment); var hostOptions = new ScriptApplicationHostOptions { @@ -127,7 +126,7 @@ private Task CreateDefaultWorkerChannel(bool autoStart = true, IDictionary(async () => }); } + [Fact] + public async Task StartWorkerProcessAsync_ProcessExits_Throws() + { + _mockRpcWorkerProcess.Setup(m => m.WaitForExitAsync(It.IsAny())) + .Returns(Task.CompletedTask); + await CreateDefaultWorkerChannel(autoStart: false); + + WorkerProcessExitException ex = await Assert.ThrowsAsync( + () => _workerChannel.StartWorkerProcessAsync(default)) + .WaitAsync(TimeSpan.FromMilliseconds(100)); + Assert.Equal(0, ex.ExitCode); + Assert.Equal("Worker process exited before initializing.", ex.Message); + } + + [Fact] + public async Task StartWorkerProcessAsync_ProcessFaults_Throws() + { + WorkerProcessExitException expected = new("Process has faulted.") { ExitCode = -1 }; + _mockRpcWorkerProcess.Setup(m => m.WaitForExitAsync(It.IsAny())) + .ThrowsAsync(expected); + await CreateDefaultWorkerChannel(autoStart: false); + + WorkerProcessExitException actual = await Assert.ThrowsAsync( + () => _workerChannel.StartWorkerProcessAsync(default)) + .WaitAsync(TimeSpan.FromMilliseconds(100)); + Assert.Equal(expected, actual); + } + + [Fact] + public async Task StartWorkerProcessAsync_TimesOut() + { + await CreateDefaultWorkerChannel(autoStart: false); // suppress for timeout + var initTask = _workerChannel.StartWorkerProcessAsync(CancellationToken.None); + await Assert.ThrowsAsync(async () => await initTask); + } + + [Fact] + public async Task StartWorkerProcessAsync_WorkerProcess_Throws() + { + // note: uses custom worker channel + Mock mockrpcWorkerProcessThatThrows = new Mock(); + mockrpcWorkerProcessThatThrows.Setup(m => m.StartProcessAsync(default)).Throws(); + + _workerChannel = new GrpcWorkerChannel( + _workerId, + _eventManager, + _mockScriptHostManager.Object, + _testWorkerConfig, + mockrpcWorkerProcessThatThrows.Object, + _logger, + _metricsLogger, + 0, + _testEnvironment, + _hostOptionsMonitor, + _sharedMemoryManager, + _workerConcurrencyOptions, + _hostingConfigOptions, + _httpProxyService); + await Assert.ThrowsAsync(async () => await _workerChannel.StartWorkerProcessAsync(CancellationToken.None)); + } + + [Fact] + public async Task StartWorkerProcessAsync_Invoked_SetupFunctionBuffers_Verify_ReadyForInvocation() + { + await CreateDefaultWorkerChannel(); + _mockRpcWorkerProcess.Verify(m => m.StartProcessAsync(default), Times.Once); + Assert.False(_workerChannel.IsChannelReadyForInvocations()); + _workerChannel.SetupFunctionInvocationBuffers(GetTestFunctionsList("node")); + Assert.True(_workerChannel.IsChannelReadyForInvocations()); + } + [Fact] public async Task WorkerChannel_Dispose_With_WorkerTerminateCapability() { @@ -221,16 +291,6 @@ public async Task WorkerChannel_Dispose_Without_WorkerTerminateCapability() Assert.False(traces.Any(m => string.Equals(m.FormattedMessage, expectedLogMsg))); } - [Fact] - public async Task StartWorkerProcessAsync_Invoked_SetupFunctionBuffers_Verify_ReadyForInvocation() - { - await CreateDefaultWorkerChannel(); - _mockrpcWorkerProcess.Verify(m => m.StartProcessAsync(default), Times.Once); - Assert.False(_workerChannel.IsChannelReadyForInvocations()); - _workerChannel.SetupFunctionInvocationBuffers(GetTestFunctionsList("node")); - Assert.True(_workerChannel.IsChannelReadyForInvocations()); - } - [Fact] public async Task DisposingChannel_NotReadyForInvocation() { @@ -259,14 +319,6 @@ public void SetupFunctionBuffers_Verify_ReadyForInvocation_Returns_False() Assert.False(_workerChannel.IsChannelReadyForInvocations()); } - [Fact] - public async Task StartWorkerProcessAsync_TimesOut() - { - await CreateDefaultWorkerChannel(autoStart: false); // suppress for timeout - var initTask = _workerChannel.StartWorkerProcessAsync(CancellationToken.None); - await Assert.ThrowsAsync(async () => await initTask); - } - [Fact] public async Task SendEnvironmentReloadRequest_Generates_ExpectedMetrics() { @@ -283,31 +335,6 @@ public async Task SendEnvironmentReloadRequest_Generates_ExpectedMetrics() Assert.True(_metricsLogger.EventsBegan.Contains(MetricEventNames.SpecializationEnvironmentReloadRequestResponse)); } - [Fact] - public async Task StartWorkerProcessAsync_WorkerProcess_Throws() - { - // note: uses custom worker channel - Mock mockrpcWorkerProcessThatThrows = new Mock(); - mockrpcWorkerProcessThatThrows.Setup(m => m.StartProcessAsync(default)).Throws(); - - _workerChannel = new GrpcWorkerChannel( - _workerId, - _eventManager, - _mockScriptHostManager.Object, - _testWorkerConfig, - mockrpcWorkerProcessThatThrows.Object, - _logger, - _metricsLogger, - 0, - _testEnvironment, - _hostOptionsMonitor, - _sharedMemoryManager, - _workerConcurrencyOptions, - _hostingConfigOptions, - _httpProxyService); - await Assert.ThrowsAsync(async () => await _workerChannel.StartWorkerProcessAsync(CancellationToken.None)); - } - [Fact] public async Task SendWorkerInitRequest_PublishesOutboundEvent() { @@ -547,7 +574,7 @@ public async Task Drain_Verify() _eventManager, _mockScriptHostManager.Object, _testWorkerConfig, - _mockrpcWorkerProcess.Object, + _mockRpcWorkerProcess.Object, _logger, _metricsLogger, 0, @@ -1256,7 +1283,7 @@ public async Task GetLatencies_StartsTimer_WhenDynamicConcurrencyEnabled() _eventManager, _mockScriptHostManager.Object, config, - _mockrpcWorkerProcess.Object, + _mockRpcWorkerProcess.Object, _logger, _metricsLogger, 0, @@ -1297,7 +1324,7 @@ public async Task GetLatencies_DoesNot_StartTimer_WhenDynamicConcurrencyDisabled _eventManager, _mockScriptHostManager.Object, config, - _mockrpcWorkerProcess.Object, + _mockRpcWorkerProcess.Object, _logger, _metricsLogger, 0, diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerProcessTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerProcessTests.cs index 3757a50a9f..02f720a90c 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerProcessTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerProcessTests.cs @@ -2,10 +2,9 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; -using System.Collections; -using System.Collections.Generic; using System.Diagnostics; using System.Linq; +using System.Threading.Tasks; using Microsoft.Azure.WebJobs.Host.Scale; using Microsoft.Azure.WebJobs.Script.Config; using Microsoft.Azure.WebJobs.Script.Eventing; @@ -260,5 +259,62 @@ public void WorkerProcess_WaitForExit_AfterExit_DoesNotThrow() Exception ex = traces.Single().Exception; Assert.IsType(ex); } + + [Fact] + public async Task WorkerProcess_WaitForExit_NotStarted_Throws() + { + using var rpcWorkerProcess = GetRpcWorkerConfigProcess( + TestHelpers.GetTestWorkerConfigsWithExecutableWorkingDirectory().ElementAt(0)); + await Assert.ThrowsAsync(() => rpcWorkerProcess.WaitForExitAsync()); + } + + [Fact] + public async Task WorkerProcess_WaitForExit_Success_TaskCompletes() + { + // arrange + using Process process = GetProcess(exitCode: 0); + _hostProcessMonitorMock.Setup(m => m.RegisterChildProcess(process)); + _workerProcessFactory.Setup(m => m.CreateWorkerProcess(It.IsNotNull())).Returns(process); + using var rpcWorkerProcess = GetRpcWorkerConfigProcess( + TestHelpers.GetTestWorkerConfigsWithExecutableWorkingDirectory().ElementAt(0)); + + // act + await rpcWorkerProcess.StartProcessAsync(); + + // assert + await rpcWorkerProcess.WaitForExitAsync(); + } + + [Fact] + public async Task WorkerProcess_WaitForExit_Error_Rethrows() + { + // arrange + using Process process = GetProcess(exitCode: -1); + _hostProcessMonitorMock.Setup(m => m.RegisterChildProcess(process)); + _workerProcessFactory.Setup(m => m.CreateWorkerProcess(It.IsNotNull())).Returns(process); + using var rpcWorkerProcess = GetRpcWorkerConfigProcess( + TestHelpers.GetTestWorkerConfigsWithExecutableWorkingDirectory().ElementAt(0)); + + // act + await rpcWorkerProcess.StartProcessAsync(); + + // assert + await Assert.ThrowsAnyAsync(() => rpcWorkerProcess.WaitForExitAsync()); + } + + private static Process GetProcess(int exitCode) + { + return new() + { + StartInfo = new() + { + WindowStyle = ProcessWindowStyle.Hidden, + FileName = OperatingSystem.IsWindows() ? "cmd" : "bash", + Arguments = OperatingSystem.IsWindows() ? $"/C exit {exitCode}" : $"-c \"exit {exitCode}\"", + RedirectStandardError = true, + RedirectStandardOutput = true, + } + }; + } } -} \ No newline at end of file +} From 640cfe4ac59f615cddc3108d45794bf2293b2606 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Mon, 14 Apr 2025 13:39:56 -0700 Subject: [PATCH 03/16] Fix expectations --- test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerProcessTests.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerProcessTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerProcessTests.cs index 02f720a90c..88599e1e02 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerProcessTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerProcessTests.cs @@ -274,6 +274,7 @@ public async Task WorkerProcess_WaitForExit_Success_TaskCompletes() // arrange using Process process = GetProcess(exitCode: 0); _hostProcessMonitorMock.Setup(m => m.RegisterChildProcess(process)); + _hostProcessMonitorMock.Setup(m => m.UnregisterChildProcess(process)); _workerProcessFactory.Setup(m => m.CreateWorkerProcess(It.IsNotNull())).Returns(process); using var rpcWorkerProcess = GetRpcWorkerConfigProcess( TestHelpers.GetTestWorkerConfigsWithExecutableWorkingDirectory().ElementAt(0)); @@ -291,6 +292,7 @@ public async Task WorkerProcess_WaitForExit_Error_Rethrows() // arrange using Process process = GetProcess(exitCode: -1); _hostProcessMonitorMock.Setup(m => m.RegisterChildProcess(process)); + _hostProcessMonitorMock.Setup(m => m.UnregisterChildProcess(process)); _workerProcessFactory.Setup(m => m.CreateWorkerProcess(It.IsNotNull())).Returns(process); using var rpcWorkerProcess = GetRpcWorkerConfigProcess( TestHelpers.GetTestWorkerConfigsWithExecutableWorkingDirectory().ElementAt(0)); From 54a51ec124083ddd9c4790a32811341c6f10fd2d Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Thu, 17 Apr 2025 08:43:59 -0700 Subject: [PATCH 04/16] Skip artifact on reruns --- eng/ci/templates/jobs/run-unit-tests.yml | 2 +- .../Configuration/FunctionsHostingConfigOptionsSetupTest.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/ci/templates/jobs/run-unit-tests.yml b/eng/ci/templates/jobs/run-unit-tests.yml index 9367f60e3a..1dc8e21eed 100644 --- a/eng/ci/templates/jobs/run-unit-tests.yml +++ b/eng/ci/templates/jobs/run-unit-tests.yml @@ -16,7 +16,7 @@ jobs: displayName: Publish deps.json path: $(Build.ArtifactStagingDirectory) artifact: WebHost_Deps - condition: failed() + condition: and(failed(), eq(variables['System.JobAttempt'], 1)) # only publish on first attempt steps: - template: /eng/ci/templates/install-dotnet.yml@self diff --git a/test/WebJobs.Script.Tests/Configuration/FunctionsHostingConfigOptionsSetupTest.cs b/test/WebJobs.Script.Tests/Configuration/FunctionsHostingConfigOptionsSetupTest.cs index bac23b01c4..bf5dc64ca8 100644 --- a/test/WebJobs.Script.Tests/Configuration/FunctionsHostingConfigOptionsSetupTest.cs +++ b/test/WebJobs.Script.Tests/Configuration/FunctionsHostingConfigOptionsSetupTest.cs @@ -46,7 +46,7 @@ public void Configure_DoesNotSet_Options_IfFileEmpty() [Fact] public void Configure_DoesNotSet_Options_IfFileDoesNotExist() { - string fileName = Path.Combine("C://settings.txt"); + string fileName = Path.Combine(Path.GetTempPath(), "settings.txt"); IConfiguration configuraton = GetConfiguration(fileName, string.Empty); FunctionsHostingConfigOptionsSetup setup = new FunctionsHostingConfigOptionsSetup(configuraton); FunctionsHostingConfigOptions options = new FunctionsHostingConfigOptions(); From ca6ce423ac6dcb211b3bdcd4569027ff1723e066 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Thu, 17 Apr 2025 10:57:45 -0700 Subject: [PATCH 05/16] Fix tests setting env vars --- .../TestHelpers.cs | 25 ++-- .../DefaultDependencyValidatorTests.cs | 10 +- .../FunctionMetadataProviderTests.cs | 117 +++++++++--------- .../WorkerFunctionMetadataProviderTests.cs | 6 +- 4 files changed, 80 insertions(+), 78 deletions(-) diff --git a/test/WebJobs.Script.Tests.Shared/TestHelpers.cs b/test/WebJobs.Script.Tests.Shared/TestHelpers.cs index 7711c8dc80..25739e9a17 100644 --- a/test/WebJobs.Script.Tests.Shared/TestHelpers.cs +++ b/test/WebJobs.Script.Tests.Shared/TestHelpers.cs @@ -313,23 +313,26 @@ private static async Task ReadAllLinesSafeAsync(string logFile) public static async Task ReadStreamToEnd(Stream stream) { stream.Position = 0; - using (var sr = new StreamReader(stream)) - { - return await sr.ReadToEndAsync(); - } + using var sr = new StreamReader(stream); + return await sr.ReadToEndAsync(); } - public static IList GetTestWorkerConfigs(bool includeDllWorker = false, int processCountValue = 1, - TimeSpan? processStartupInterval = null, TimeSpan? processRestartInterval = null, TimeSpan? processShutdownTimeout = null, bool workerIndexing = false) + public static IList GetTestWorkerConfigs( + bool includeDllWorker = false, + int processCountValue = 1, + TimeSpan? processStartupInterval = null, + TimeSpan? processRestartInterval = null, + TimeSpan? processShutdownTimeout = null, + bool workerIndexing = false) { var defaultCountOptions = new WorkerProcessCountOptions(); TimeSpan startupInterval = processStartupInterval ?? defaultCountOptions.ProcessStartupInterval; TimeSpan restartInterval = processRestartInterval ?? defaultCountOptions.ProcessRestartInterval; TimeSpan shutdownTimeout = processShutdownTimeout ?? defaultCountOptions.ProcessShutdownTimeout; - var workerConfigs = new List - { - new RpcWorkerConfig + List workerConfigs = + [ + new() { Description = GetTestWorkerDescription("node", ".js", workerIndexing), CountOptions = new WorkerProcessCountOptions @@ -340,7 +343,7 @@ public static IList GetTestWorkerConfigs(bool includeDllWorker ProcessShutdownTimeout = shutdownTimeout } }, - new RpcWorkerConfig + new() { Description = GetTestWorkerDescription("java", ".jar", workerIndexing), CountOptions = new WorkerProcessCountOptions @@ -351,7 +354,7 @@ public static IList GetTestWorkerConfigs(bool includeDllWorker ProcessShutdownTimeout = shutdownTimeout } } - }; + ]; // Allow tests to have a worker that claims the .dll extension. if (includeDllWorker) diff --git a/test/WebJobs.Script.Tests/Configuration/DefaultDependencyValidatorTests.cs b/test/WebJobs.Script.Tests/Configuration/DefaultDependencyValidatorTests.cs index 476bbc33af..3de0c8e665 100644 --- a/test/WebJobs.Script.Tests/Configuration/DefaultDependencyValidatorTests.cs +++ b/test/WebJobs.Script.Tests/Configuration/DefaultDependencyValidatorTests.cs @@ -28,7 +28,7 @@ public class DefaultDependencyValidatorTests [Fact] public async Task Validator_AllValid() { - LogMessage invalidServicesMessage = await RunTest(); + LogMessage invalidServicesMessage = await RunTestAsync(); string msg = $"If you have registered new dependencies, make sure to update the DependencyValidator. Invalid Services:{Environment.NewLine}"; Assert.True(invalidServicesMessage == null, msg + invalidServicesMessage?.Exception?.ToString()); @@ -37,7 +37,7 @@ public async Task Validator_AllValid() [Fact] public async Task Validator_InvalidServices_ThrowsException() { - LogMessage invalidServicesMessage = await RunTest(configureJobHost: s => + LogMessage invalidServicesMessage = await RunTestAsync(configureJobHost: s => { s.AddSingleton(); s.AddSingleton(); @@ -61,7 +61,7 @@ public async Task Validator_InvalidServices_ThrowsException() [Fact] public async Task Validator_NoJobHost() { - LogMessage invalidServicesMessage = await RunTest(configureWebHost: s => + LogMessage invalidServicesMessage = await RunTestAsync(configureWebHost: s => { // This will force us to skip host startup (which removes the JobHost) s.AddSingleton(); @@ -71,10 +71,10 @@ public async Task Validator_NoJobHost() Assert.True(invalidServicesMessage == null, msg + invalidServicesMessage?.Exception?.ToString()); } - private async Task RunTest(Action configureWebHost = null, Action configureJobHost = null, bool expectSuccess = true) + private async Task RunTestAsync(Action configureWebHost = null, Action configureJobHost = null, bool expectSuccess = true) { LogMessage invalidServicesMessage = null; - TestLoggerProvider loggerProvider = new TestLoggerProvider(); + TestLoggerProvider loggerProvider = new(); var builder = Program.CreateWebHostBuilder(null) .ConfigureLogging(b => diff --git a/test/WebJobs.Script.Tests/FunctionMetadataProviderTests.cs b/test/WebJobs.Script.Tests/FunctionMetadataProviderTests.cs index 277de03c16..eca068cd5f 100644 --- a/test/WebJobs.Script.Tests/FunctionMetadataProviderTests.cs +++ b/test/WebJobs.Script.Tests/FunctionMetadataProviderTests.cs @@ -1,11 +1,8 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. -using System; using System.Collections.Generic; using System.Collections.Immutable; -using System.IO; -using System.Linq; using System.Threading.Tasks; using Microsoft.Azure.WebJobs.Script.Config; using Microsoft.Azure.WebJobs.Script.Description; @@ -31,95 +28,97 @@ public FunctionMetadataProviderTests() } [Fact] - public void GetFunctionMetadataAsync_WorkerIndexing_HostFallback() + public async Task GetFunctionMetadataAsync_WorkerIndexing_HostFallback() { // Arrange _logger.ClearLogMessages(); + ImmutableArray functionMetadataCollection = GetTestFunctionMetadata(); + IList workerConfigs = TestHelpers.GetTestWorkerConfigs(); + foreach (RpcWorkerConfig config in workerConfigs) + { + config.Description.WorkerIndexing = "true"; + } - var function = GetTestRawFunctionMetadata(useDefaultMetadataIndexing: true); - IEnumerable rawFunctionMetadataCollection = new List() { function }; - var functionMetadataCollection = new List(); - functionMetadataCollection.Add(GetTestFunctionMetadata()); - - var workerConfigs = TestHelpers.GetTestWorkerConfigs().ToImmutableArray(); - workerConfigs.ToList().ForEach(config => config.Description.WorkerIndexing = "true"); - var scriptjobhostoptions = new ScriptJobHostOptions(); - scriptjobhostoptions.RootScriptPath = Path.Combine(Environment.CurrentDirectory, @"..", "..", "..", "..", "sample", "node"); - - var environment = SystemEnvironment.Instance; + TestEnvironment environment = new(); environment.SetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime, "node"); environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags, "EnableWorkerIndexing"); - var defaultProvider = new FunctionMetadataProvider(_logger, _workerFunctionMetadataProvider.Object, _hostFunctionMetadataProvider.Object, new OptionsWrapper(new FunctionsHostingConfigOptions()), SystemEnvironment.Instance); + FunctionMetadataProvider defaultProvider = new( + _logger, + _workerFunctionMetadataProvider.Object, + _hostFunctionMetadataProvider.Object, + new OptionsWrapper(new FunctionsHostingConfigOptions()), + environment); - FunctionMetadataResult result = new FunctionMetadataResult(true, functionMetadataCollection.ToImmutableArray()); - _workerFunctionMetadataProvider.Setup(m => m.GetFunctionMetadataAsync(workerConfigs, false)).Returns(Task.FromResult(result)); - _hostFunctionMetadataProvider.Setup(m => m.GetFunctionMetadataAsync(workerConfigs, false)).Returns(Task.FromResult(functionMetadataCollection.ToImmutableArray())); + FunctionMetadataResult result = new(true, functionMetadataCollection); + _workerFunctionMetadataProvider.Setup(m => m.GetFunctionMetadataAsync(workerConfigs, false)) + .ReturnsAsync(result); + _hostFunctionMetadataProvider.Setup(m => m.GetFunctionMetadataAsync(workerConfigs, false)) + .ReturnsAsync(functionMetadataCollection); // Act - var functions = defaultProvider.GetFunctionMetadataAsync(workerConfigs, false).GetAwaiter().GetResult(); + ImmutableArray functions = await defaultProvider + .GetFunctionMetadataAsync(workerConfigs, false); // Assert Assert.Equal(1, functions.Length); - var traces = _logger.GetLogMessages(); - var functionLoadLogs = traces.Where(m => string.Equals(m.FormattedMessage, "Fallback to host indexing as worker denied indexing")); - Assert.True(functionLoadLogs.Any()); + Assert.Contains( + _logger.GetLogMessages(), + m => string.Equals(m.FormattedMessage, "Fallback to host indexing as worker denied indexing")); } [Fact] - public void GetFunctionMetadataAsync_HostIndexing() + public async Task GetFunctionMetadataAsync_HostIndexing() { // Arrange _logger.ClearLogMessages(); + ImmutableArray functionMetadataCollection = GetTestFunctionMetadata(); + IList workerConfigs = TestHelpers.GetTestWorkerConfigs(); + foreach (RpcWorkerConfig config in workerConfigs) + { + config.Description.WorkerIndexing = "true"; + } - var function = GetTestRawFunctionMetadata(useDefaultMetadataIndexing: true); - IEnumerable rawFunctionMetadataCollection = new List() { function }; - var functionMetadataCollection = new List(); - functionMetadataCollection.Add(GetTestFunctionMetadata()); - - var workerConfigs = TestHelpers.GetTestWorkerConfigs().ToImmutableArray(); - workerConfigs.ToList().ForEach(config => config.Description.WorkerIndexing = "true"); - var scriptjobhostoptions = new ScriptJobHostOptions(); - scriptjobhostoptions.RootScriptPath = Path.Combine(Environment.CurrentDirectory, @"..", "..", "..", "..", "sample", "node"); - - var environment = SystemEnvironment.Instance; + TestEnvironment environment = new(); environment.SetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime, "node"); environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags, string.Empty); - var optionsMonitor = TestHelpers.CreateOptionsMonitor(new FunctionsHostingConfigOptions()); - var workerMetadataProvider = new Mock(); - workerMetadataProvider.Setup(m => m.GetFunctionMetadataAsync(It.IsAny>(), false)).Returns(Task.FromResult(new FunctionMetadataResult(true, ImmutableArray.Empty))); + Mock workerMetadataProvider = new(); + workerMetadataProvider.Setup(m => m.GetFunctionMetadataAsync(It.IsAny>(), false)) + .ReturnsAsync(new FunctionMetadataResult(true, [])); - var defaultProvider = new FunctionMetadataProvider(_logger, workerMetadataProvider.Object, _hostFunctionMetadataProvider.Object, new OptionsWrapper(new FunctionsHostingConfigOptions()), SystemEnvironment.Instance); + FunctionMetadataProvider defaultProvider = new( + _logger, + workerMetadataProvider.Object, + _hostFunctionMetadataProvider.Object, + new OptionsWrapper(new FunctionsHostingConfigOptions()), + environment); - FunctionMetadataResult result = new FunctionMetadataResult(true, functionMetadataCollection.ToImmutableArray()); - _hostFunctionMetadataProvider.Setup(m => m.GetFunctionMetadataAsync(workerConfigs, false)).Returns(Task.FromResult(functionMetadataCollection.ToImmutableArray())); + FunctionMetadataResult result = new(true, functionMetadataCollection); + _hostFunctionMetadataProvider.Setup(m => m.GetFunctionMetadataAsync(workerConfigs, false)) + .ReturnsAsync(functionMetadataCollection); // Act - var functions = defaultProvider.GetFunctionMetadataAsync(workerConfigs, false).GetAwaiter().GetResult(); + ImmutableArray functions = await defaultProvider + .GetFunctionMetadataAsync(workerConfigs, false); // Assert Assert.Equal(1, functions.Length); - var traces = _logger.GetLogMessages(); - var functionLoadLogs = traces.Where(m => string.Equals(m.FormattedMessage, "Fallback to host indexing as worker denied indexing")); - Assert.True(functionLoadLogs.Any()); + Assert.Contains( + _logger.GetLogMessages(), + m => string.Equals(m.FormattedMessage, "Fallback to host indexing as worker denied indexing")); } - private static RawFunctionMetadata GetTestRawFunctionMetadata(bool useDefaultMetadataIndexing) + private static ImmutableArray GetTestFunctionMetadata(string name = "testFunction") { - return new RawFunctionMetadata() - { - UseDefaultMetadataIndexing = useDefaultMetadataIndexing - }; - } - - private static FunctionMetadata GetTestFunctionMetadata(string name = "testFunction") - { - return new FunctionMetadata() - { - Name = name, - Language = "node" - }; + return + [ + new FunctionMetadata() + { + Name = name, + Language = "node" + } + ]; } } } \ No newline at end of file diff --git a/test/WebJobs.Script.Tests/WorkerFunctionMetadataProviderTests.cs b/test/WebJobs.Script.Tests/WorkerFunctionMetadataProviderTests.cs index 7fc0737a8e..86fa090147 100644 --- a/test/WebJobs.Script.Tests/WorkerFunctionMetadataProviderTests.cs +++ b/test/WebJobs.Script.Tests/WorkerFunctionMetadataProviderTests.cs @@ -193,11 +193,11 @@ public async void ValidateFunctionMetadata_Logging() { }); - var environment = SystemEnvironment.Instance; + TestEnvironment environment = new(); environment.SetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime, "node"); - var workerFunctionMetadataProvider = new WorkerFunctionMetadataProvider(optionsMonitor, logger, SystemEnvironment.Instance, - mockWebHostRpcWorkerChannelManager.Object, mockScriptHostManager.Object); + var workerFunctionMetadataProvider = new WorkerFunctionMetadataProvider( + optionsMonitor, logger, environment, mockWebHostRpcWorkerChannelManager.Object, mockScriptHostManager.Object); await workerFunctionMetadataProvider.GetFunctionMetadataAsync(workerConfigs, false); var traces = logger.GetLogMessages(); From a53f92dc626676aba59d725c059351c82a7c7a15 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Thu, 17 Apr 2025 15:46:18 -0700 Subject: [PATCH 06/16] Refactor ScriptStartupTypeDiscovererTests for env isolation --- .../ScriptStartupTypeLocator.cs | 28 +- .../ScriptHostBuilderExtensions.cs | 3 +- .../TempDirectory.cs | 16 - .../ScriptStartupTypeDiscovererTests.cs | 1466 ++++++----------- 4 files changed, 480 insertions(+), 1033 deletions(-) diff --git a/src/WebJobs.Script/DependencyInjection/ScriptStartupTypeLocator.cs b/src/WebJobs.Script/DependencyInjection/ScriptStartupTypeLocator.cs index 953fc56429..55b95f1323 100644 --- a/src/WebJobs.Script/DependencyInjection/ScriptStartupTypeLocator.cs +++ b/src/WebJobs.Script/DependencyInjection/ScriptStartupTypeLocator.cs @@ -36,29 +36,37 @@ public sealed class ScriptStartupTypeLocator : IWebJobsStartupTypeLocator private readonly IExtensionBundleManager _extensionBundleManager; private readonly IFunctionMetadataManager _functionMetadataManager; private readonly IMetricsLogger _metricsLogger; - private readonly Lazy> _startupTypes; + private readonly IEnvironment _environment; private readonly IOptions _extensionRequirementOptions; + private readonly Lazy> _startupTypes; private static string[] _builtinExtensionAssemblies = GetBuiltinExtensionAssemblies(); - public ScriptStartupTypeLocator(string rootScriptPath, ILogger logger, IExtensionBundleManager extensionBundleManager, - IFunctionMetadataManager functionMetadataManager, IMetricsLogger metricsLogger, IOptions extensionRequirementOptions) + public ScriptStartupTypeLocator( + string rootScriptPath, + ILogger logger, + IExtensionBundleManager extensionBundleManager, + IFunctionMetadataManager functionMetadataManager, + IMetricsLogger metricsLogger, + IEnvironment environment, + IOptions extensionRequirementOptions) { _rootScriptPath = rootScriptPath ?? throw new ArgumentNullException(nameof(rootScriptPath)); _extensionBundleManager = extensionBundleManager ?? throw new ArgumentNullException(nameof(extensionBundleManager)); _logger = logger; _functionMetadataManager = functionMetadataManager; _metricsLogger = metricsLogger; - _startupTypes = new Lazy>(() => GetExtensionsStartupTypesAsync().ConfigureAwait(false).GetAwaiter().GetResult()); + _environment = environment; _extensionRequirementOptions = extensionRequirementOptions; + _startupTypes = new Lazy>(() => GetExtensionsStartupTypesAsync().ConfigureAwait(false).GetAwaiter().GetResult()); } private static string[] GetBuiltinExtensionAssemblies() { - return new[] - { + return + [ typeof(WebJobs.Extensions.Http.HttpWebJobsStartup).Assembly.GetName().Name, typeof(WebJobs.Extensions.ExtensionsWebJobsStartup).Assembly.GetName().Name - }; + ]; } public Type[] GetStartupTypes() @@ -102,11 +110,11 @@ public async Task> GetExtensionsStartupTypesAsync() } } - bool isDotnetIsolatedApp = Utility.IsDotnetIsolatedApp(SystemEnvironment.Instance, functionMetadataCollection); + bool isDotnetIsolatedApp = Utility.IsDotnetIsolatedApp(_environment, functionMetadataCollection); bool isDotnetApp = isPrecompiledFunctionApp || isDotnetIsolatedApp; - var isLogicApp = SystemEnvironment.Instance.IsLogicApp(); + var isLogicApp = _environment.IsLogicApp(); - if (SystemEnvironment.Instance.IsPlaceholderModeEnabled()) + if (_environment.IsPlaceholderModeEnabled()) { // Do not move this. // Calling this log statement in the placeholder mode to avoid jitting during specialization diff --git a/src/WebJobs.Script/ScriptHostBuilderExtensions.cs b/src/WebJobs.Script/ScriptHostBuilderExtensions.cs index 851c611a08..7080db33d7 100644 --- a/src/WebJobs.Script/ScriptHostBuilderExtensions.cs +++ b/src/WebJobs.Script/ScriptHostBuilderExtensions.cs @@ -142,12 +142,13 @@ public static IHostBuilder AddScriptHost(this IHostBuilder builder, var bundleManager = new ExtensionBundleManager(extensionBundleOptions, SystemEnvironment.Instance, loggerFactory, configOption, httpClientFactory); var metadataServiceManager = applicationOptions.RootServiceProvider.GetService(); - ScriptStartupTypeLocator locator = new( + var locator = new ScriptStartupTypeLocator( applicationOptions.ScriptPath, loggerFactory.CreateLogger(), bundleManager, metadataServiceManager, metricsLogger, + SystemEnvironment.Instance, extensionRequirementOptions); // The locator (and thus the bundle manager) need to be created now in order to configure app configuration. diff --git a/test/WebJobs.Script.Tests.Shared/TempDirectory.cs b/test/WebJobs.Script.Tests.Shared/TempDirectory.cs index 22acb59fa5..4cb858159b 100644 --- a/test/WebJobs.Script.Tests.Shared/TempDirectory.cs +++ b/test/WebJobs.Script.Tests.Shared/TempDirectory.cs @@ -19,25 +19,9 @@ public TempDirectory(string path) Directory.CreateDirectory(path); } - ~TempDirectory() - { - Dispose(false); - } - public string Path { get; } public void Dispose() - { - GC.SuppressFinalize(this); - Dispose(true); - } - - private void Dispose(bool disposing) - { - DeleteDirectory(); - } - - private void DeleteDirectory() { try { diff --git a/test/WebJobs.Script.Tests/ScriptStartupTypeDiscovererTests.cs b/test/WebJobs.Script.Tests/ScriptStartupTypeDiscovererTests.cs index f43eef9209..fcad06db4b 100644 --- a/test/WebJobs.Script.Tests/ScriptStartupTypeDiscovererTests.cs +++ b/test/WebJobs.Script.Tests/ScriptStartupTypeDiscovererTests.cs @@ -28,39 +28,43 @@ namespace Microsoft.Azure.WebJobs.Script.Tests { - public class ScriptStartupTypeDiscovererTests + public sealed class ScriptStartupTypeDiscovererTests : IDisposable { + private readonly TempDirectory _directory = new(); + private readonly TestMetricsLogger _metricsLogger = new(); + private readonly TestLoggerProvider _loggerProvider = new(); + private readonly TestEnvironment _environment = new(); + private readonly Mock _bundleManager = new(); + private readonly Mock _metadataManager = new(); + + public ScriptStartupTypeDiscovererTests() + { + SetupMetadataManager(null); + } + + public void Dispose() + { + _directory.Dispose(); + } + [Fact] public async Task GetExtensionsStartupTypes_UsesDefaultMinVersion() { - using (var directory = GetTempDirectory()) - { - TestMetricsLogger testMetricsLogger = new TestMetricsLogger(); - TestLoggerProvider testLoggerProvider = new TestLoggerProvider(); - LoggerFactory factory = new LoggerFactory(); - factory.AddProvider(testLoggerProvider); - var testLogger = factory.CreateLogger(); - - var binPath = Path.Combine(directory.Path, "bin"); - - var mockExtensionBundleManager = new Mock(); - mockExtensionBundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(true); - mockExtensionBundleManager.Setup(e => e.GetExtensionBundleBinPathAsync()).Returns(Task.FromResult(binPath)); - mockExtensionBundleManager.Setup(e => e.IsLegacyExtensionBundle()).Returns(false); - mockExtensionBundleManager.Setup(e => e.GetExtensionBundleDetails()).Returns(Task.FromResult(GetBundleDetails("2.1.0"))); - - var languageWorkerOptions = new TestOptionsMonitor(new LanguageWorkerOptions()); - var mockFunctionMetadataManager = GetTestFunctionMetadataManager(languageWorkerOptions); - OptionsWrapper optionsWrapper = new(new ExtensionRequirementOptions()); - var discoverer = new ScriptStartupTypeLocator(directory.Path, testLogger, mockExtensionBundleManager.Object, mockFunctionMetadataManager, testMetricsLogger, optionsWrapper); - - // Act - var exception = await Assert.ThrowsAsync(async () => await discoverer.GetExtensionsStartupTypesAsync()); - var traces = testLoggerProvider.GetAllLogMessages(); - - // Assert - Assert.True(traces.Any(m => string.Equals(m.FormattedMessage, $"Referenced bundle Microsoft.Azure.Functions.ExtensionBundle of version 2.1.0 does not meet the required minimum version of 2.6.1. Update your extension bundle reference in host.json to reference 2.6.1 or later."))); - } + InstallExtensions(ExtensionInstall.Storage(true), ExtensionInstall.SendGrid(true)); + var binPath = Path.Combine(_directory.Path, "bin"); + + _bundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(true); + _bundleManager.Setup(e => e.GetExtensionBundleBinPathAsync()).ReturnsAsync(binPath); + _bundleManager.Setup(e => e.IsLegacyExtensionBundle()).Returns(false); + _bundleManager.Setup(e => e.GetExtensionBundleDetails()).ReturnsAsync(GetBundleDetails("2.1.0")); + + // Act + ScriptStartupTypeLocator discoverer = CreateSystemUnderTest(); + var exception = await Assert.ThrowsAsync(discoverer.GetExtensionsStartupTypesAsync); + var traces = _loggerProvider.GetAllLogMessages(); + + // Assert + Assert.True(traces.Any(m => string.Equals(m.FormattedMessage, $"Referenced bundle Microsoft.Azure.Functions.ExtensionBundle of version 2.1.0 does not meet the required minimum version of 2.6.1. Update your extension bundle reference in host.json to reference 2.6.1 or later."))); } [Theory] @@ -68,46 +72,32 @@ public async Task GetExtensionsStartupTypes_UsesDefaultMinVersion() [InlineData("2.6.1", "2.1.0")] public async Task GetExtensionsStartupTypes_RejectsBundleConfiguredviaHostingEnvConfig(string expectedBundleVersion, string actualBundleVersion) { - using (var directory = GetTempDirectory()) - { - TestMetricsLogger testMetricsLogger = new TestMetricsLogger(); - TestLoggerProvider testLoggerProvider = new TestLoggerProvider(); - LoggerFactory factory = new LoggerFactory(); - factory.AddProvider(testLoggerProvider); - var testLogger = factory.CreateLogger(); + var binPath = Path.Combine(_directory.Path, "bin"); - var binPath = Path.Combine(directory.Path, "bin"); + _bundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(true); + _bundleManager.Setup(e => e.GetExtensionBundleBinPathAsync()).ReturnsAsync(binPath); + _bundleManager.Setup(e => e.IsLegacyExtensionBundle()).Returns(false); + _bundleManager.Setup(e => e.GetExtensionBundleDetails()).ReturnsAsync(GetBundleDetails(actualBundleVersion)); - var mockExtensionBundleManager = new Mock(); - mockExtensionBundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(true); - mockExtensionBundleManager.Setup(e => e.GetExtensionBundleBinPathAsync()).Returns(Task.FromResult(binPath)); - mockExtensionBundleManager.Setup(e => e.IsLegacyExtensionBundle()).Returns(false); - mockExtensionBundleManager.Setup(e => e.GetExtensionBundleDetails()).Returns(Task.FromResult(GetBundleDetails(actualBundleVersion))); - - var languageWorkerOptions = new TestOptionsMonitor(new LanguageWorkerOptions()); - var mockFunctionMetadataManager = GetTestFunctionMetadataManager(languageWorkerOptions); - - var extensionRequirementOptions = new Config.ExtensionRequirementOptions(); - extensionRequirementOptions.Bundles = new List() - { - new BundleRequirement() + ExtensionRequirementOptions extensionRequirementOptions = new() + { + Bundles = + [ + new() { Id = "Microsoft.Azure.Functions.ExtensionBundle", MinimumVersion = expectedBundleVersion } - }; - - OptionsWrapper optionsWrapper = new(extensionRequirementOptions); - - var discoverer = new ScriptStartupTypeLocator(directory.Path, testLogger, mockExtensionBundleManager.Object, mockFunctionMetadataManager, testMetricsLogger, optionsWrapper); + ] + }; - // Act - var exception = await Assert.ThrowsAsync(async () => await discoverer.GetExtensionsStartupTypesAsync()); - var traces = testLoggerProvider.GetAllLogMessages(); + // Act + ScriptStartupTypeLocator discoverer = CreateSystemUnderTest(extensionRequirements: extensionRequirementOptions); + var exception = await Assert.ThrowsAsync(async () => await discoverer.GetExtensionsStartupTypesAsync()); + var traces = _loggerProvider.GetAllLogMessages(); - // Assert - Assert.True(traces.Any(m => string.Equals(m.FormattedMessage, $"Referenced bundle Microsoft.Azure.Functions.ExtensionBundle of version {actualBundleVersion} does not meet the required minimum version of {expectedBundleVersion}. Update your extension bundle reference in host.json to reference {expectedBundleVersion} or later."))); - } + // Assert + Assert.True(traces.Any(m => string.Equals(m.FormattedMessage, $"Referenced bundle Microsoft.Azure.Functions.ExtensionBundle of version {actualBundleVersion} does not meet the required minimum version of {expectedBundleVersion}. Update your extension bundle reference in host.json to reference {expectedBundleVersion} or later."))); } [Theory] @@ -118,72 +108,39 @@ public async Task GetExtensionsStartupTypes_RejectsBundleConfiguredviaHostingEnv public async Task GetExtensionsStartupTypes_AcceptsRequiredBundleVersions(string minBundleVersion, string actualBundleVersion, string minExtensionVersion) { // "TypeName": , Microsoft.Azure.WebJobs.Extensions.Storage, Version=4.0.5.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35", - using (var directory = GetTempDirectory()) - { - TestMetricsLogger testMetricsLogger = new TestMetricsLogger(); - TestLoggerProvider testLoggerProvider = new TestLoggerProvider(); - LoggerFactory factory = new LoggerFactory(); - factory.AddProvider(testLoggerProvider); - var testLogger = factory.CreateLogger(); + string binPath = InstallExtensions(ExtensionInstall.Storage(true), ExtensionInstall.SendGrid(true)); - var binPath = Path.Combine(directory.Path, "bin"); - - var mockExtensionBundleManager = new Mock(); - - if (string.IsNullOrEmpty(actualBundleVersion)) - { - mockExtensionBundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(false); - } - else - { - mockExtensionBundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(true); - } - - mockExtensionBundleManager.Setup(e => e.GetExtensionBundleBinPathAsync()).Returns(Task.FromResult(binPath)); - mockExtensionBundleManager.Setup(e => e.IsLegacyExtensionBundle()).Returns(false); - mockExtensionBundleManager.Setup(e => e.GetExtensionBundleDetails()).Returns(Task.FromResult(GetBundleDetails(actualBundleVersion))); + if (string.IsNullOrEmpty(actualBundleVersion)) + { + _bundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(false); + } + else + { + _bundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(true); + } - var languageWorkerOptions = new TestOptionsMonitor(new LanguageWorkerOptions()); - var mockFunctionMetadataManager = GetTestFunctionMetadataManager(languageWorkerOptions); + _bundleManager.Setup(e => e.GetExtensionBundleBinPathAsync()).ReturnsAsync(binPath); + _bundleManager.Setup(e => e.IsLegacyExtensionBundle()).Returns(false); + _bundleManager.Setup(e => e.GetExtensionBundleDetails()).ReturnsAsync(GetBundleDetails(actualBundleVersion)); - var extensionRequirementOptions = new Config.ExtensionRequirementOptions(); + // Act + ScriptStartupTypeLocator discoverer = CreateSystemUnderTest(extensionRequirements: GetExtensionRequirementOptions(minBundleVersion, minExtensionVersion)); + var types = await discoverer.GetExtensionsStartupTypesAsync(); + var traces = _loggerProvider.GetAllLogMessages(); - IEnumerable bundleRequirment = string.IsNullOrEmpty(minBundleVersion) - ? null - : new List() { new BundleRequirement() { Id = "Microsoft.Azure.Functions.ExtensionBundle", MinimumVersion = minBundleVersion } }; + // Assert - IEnumerable extensionRequirements = string.IsNullOrEmpty(minExtensionVersion) - ? null - : new List() - { - new ExtensionStartupTypeRequirement() - { - Name = "AzureStorageWebJobsStartup", - AssemblyName = "Microsoft.Azure.WebJobs.Extensions.Storage", - MinimumAssemblyVersion = minExtensionVersion - } - }; - - extensionRequirementOptions.Bundles = bundleRequirment; - extensionRequirementOptions.Extensions = extensionRequirements; - - OptionsWrapper optionsWrapper = new(extensionRequirementOptions); - var discoverer = new ScriptStartupTypeLocator(directory.Path, testLogger, mockExtensionBundleManager.Object, mockFunctionMetadataManager, testMetricsLogger, optionsWrapper); - var types = await discoverer.GetExtensionsStartupTypesAsync(); - // Act - var traces = testLoggerProvider.GetAllLogMessages(); - - if (string.IsNullOrEmpty(actualBundleVersion)) - { - Assert.True(traces.Any(m => m.FormattedMessage.Contains($"Extension Bundle not loaded"))); - } - else - { - Assert.True(traces.Any(m => m.FormattedMessage.Contains($"Loading extension bundle"))); - } - Assert.True(traces.Any(m => m.FormattedMessage.Contains($"Loading startup extension 'Storage"))); - AssertNoErrors(traces); + if (string.IsNullOrEmpty(actualBundleVersion)) + { + Assert.True(traces.Any(m => m.FormattedMessage.Contains($"Extension Bundle not loaded"))); } + else + { + Assert.True(traces.Any(m => m.FormattedMessage.Contains($"Loading extension bundle"))); + } + + Assert.True(traces.Any(m => m.FormattedMessage.Contains($"Loading startup extension 'Storage"))); + AssertNoErrors(traces); } [Theory] @@ -193,65 +150,29 @@ public async Task GetExtensionsStartupTypes_AcceptsRequiredBundleVersions(string public async Task GetExtensionsStartupTypes_RejectsRequiredBundleVersions(string minBundleVersion, string actualBundleVersion, string minExtensionVersion) { // "TypeName": , Microsoft.Azure.WebJobs.Extensions.Storage, Version=4.0.5.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35", - using (var directory = GetTempDirectory()) + string binPath = InstallExtensions(ExtensionInstall.Storage(true), ExtensionInstall.SendGrid(true)); + if (string.IsNullOrEmpty(actualBundleVersion)) { - TestMetricsLogger testMetricsLogger = new TestMetricsLogger(); - TestLoggerProvider testLoggerProvider = new TestLoggerProvider(); - LoggerFactory factory = new LoggerFactory(); - factory.AddProvider(testLoggerProvider); - var testLogger = factory.CreateLogger(); - - var binPath = Path.Combine(directory.Path, "bin"); - - var mockExtensionBundleManager = new Mock(); - - if (string.IsNullOrEmpty(actualBundleVersion)) - { - mockExtensionBundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(false); - } - else - { - mockExtensionBundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(true); - } - - mockExtensionBundleManager.Setup(e => e.GetExtensionBundleBinPathAsync()).Returns(Task.FromResult(binPath)); - mockExtensionBundleManager.Setup(e => e.IsLegacyExtensionBundle()).Returns(false); - mockExtensionBundleManager.Setup(e => e.GetExtensionBundleDetails()).Returns(Task.FromResult(GetBundleDetails(actualBundleVersion))); - - var languageWorkerOptions = new TestOptionsMonitor(new LanguageWorkerOptions()); - var mockFunctionMetadataManager = GetTestFunctionMetadataManager(languageWorkerOptions); + _bundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(false); + } + else + { + _bundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(true); + } - var extensionRequirementOptions = new Config.ExtensionRequirementOptions(); + _bundleManager.Setup(e => e.GetExtensionBundleBinPathAsync()).ReturnsAsync(binPath); + _bundleManager.Setup(e => e.IsLegacyExtensionBundle()).Returns(false); + _bundleManager.Setup(e => e.GetExtensionBundleDetails()).ReturnsAsync(GetBundleDetails(actualBundleVersion)); - IEnumerable bundleRequirment = string.IsNullOrEmpty(minBundleVersion) - ? null - : new List() { new BundleRequirement() { Id = "Microsoft.Azure.Functions.ExtensionBundle", MinimumVersion = minBundleVersion } }; + // Act + ScriptStartupTypeLocator discoverer = CreateSystemUnderTest(extensionRequirements: GetExtensionRequirementOptions(minBundleVersion, minExtensionVersion)); + var exception = await Assert.ThrowsAsync(discoverer.GetExtensionsStartupTypesAsync); + var traces = _loggerProvider.GetAllLogMessages(); - IEnumerable extensionRequirements = string.IsNullOrEmpty(minExtensionVersion) - ? null - : new List() - { - new ExtensionStartupTypeRequirement() - { - Name = "AzureStorageWebJobsStartup", - AssemblyName = "Microsoft.Azure.WebJobs.Extensions.Storage", - MinimumAssemblyVersion = minExtensionVersion - } - }; - - extensionRequirementOptions.Bundles = bundleRequirment; - extensionRequirementOptions.Extensions = extensionRequirements; - - OptionsWrapper optionsWrapper = new(extensionRequirementOptions); - var discoverer = new ScriptStartupTypeLocator(directory.Path, testLogger, mockExtensionBundleManager.Object, mockFunctionMetadataManager, testMetricsLogger, optionsWrapper); - var exception = await Assert.ThrowsAsync(async () => await discoverer.GetExtensionsStartupTypesAsync()); - var traces = testLoggerProvider.GetAllLogMessages(); - - if (!string.IsNullOrEmpty(minBundleVersion)) - { - // Assert - Assert.True(traces.Any(m => string.Equals(m.FormattedMessage, $"Referenced bundle Microsoft.Azure.Functions.ExtensionBundle of version 4.9.0 does not meet the required minimum version of 4.12.0. Update your extension bundle reference in host.json to reference 4.12.0 or later."))); - } + // Assert + if (!string.IsNullOrEmpty(minBundleVersion)) + { + Assert.True(traces.Any(m => string.Equals(m.FormattedMessage, $"Referenced bundle Microsoft.Azure.Functions.ExtensionBundle of version 4.9.0 does not meet the required minimum version of 4.12.0. Update your extension bundle reference in host.json to reference 4.12.0 or later."))); } } @@ -263,56 +184,26 @@ public async Task GetExtensionsStartupTypes_RejectsRequiredBundleVersions(string public async Task GetExtensionsStartupTypes_AcceptsRequiredExtensionVersions(string minBundleVersion, bool extensionConfigured, string minExtensionVersion) { // "TypeName": , Microsoft.Azure.WebJobs.Extensions.Storage, Version=4.0.5.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35", - using (var directory = GetTempDirectory(extensionConfigured)) + if (extensionConfigured) { - TestMetricsLogger testMetricsLogger = new TestMetricsLogger(); - TestLoggerProvider testLoggerProvider = new TestLoggerProvider(); - LoggerFactory factory = new LoggerFactory(); - factory.AddProvider(testLoggerProvider); - var testLogger = factory.CreateLogger(); - - var binPath = Path.Combine(directory.Path, "bin"); - - var mockExtensionBundleManager = new Mock(); - - mockExtensionBundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(false); - - var languageWorkerOptions = new TestOptionsMonitor(new LanguageWorkerOptions()); - var mockFunctionMetadataManager = GetTestFunctionMetadataManager(languageWorkerOptions); + InstallExtensions(ExtensionInstall.Storage(true), ExtensionInstall.SendGrid(true)); + } - var extensionRequirementOptions = new Config.ExtensionRequirementOptions(); + _bundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(false); - IEnumerable bundleRequirment = string.IsNullOrEmpty(minBundleVersion) - ? null - : new List() { new BundleRequirement() { Id = "Microsoft.Azure.Functions.ExtensionBundle", MinimumVersion = minBundleVersion } }; + // Act + ScriptStartupTypeLocator discoverer = CreateSystemUnderTest(extensionRequirements: GetExtensionRequirementOptions(minBundleVersion, minExtensionVersion)); + var types = await discoverer.GetExtensionsStartupTypesAsync(); + var traces = _loggerProvider.GetAllLogMessages(); - IEnumerable extensionRequirements = string.IsNullOrEmpty(minExtensionVersion) - ? null - : new List() - { - new ExtensionStartupTypeRequirement() - { - Name = "AzureStorageWebJobsStartup", - AssemblyName = "Microsoft.Azure.WebJobs.Extensions.Storage", - MinimumAssemblyVersion = minExtensionVersion - } - }; - - extensionRequirementOptions.Bundles = bundleRequirment; - extensionRequirementOptions.Extensions = extensionRequirements; - - OptionsWrapper optionsWrapper = new(extensionRequirementOptions); - var discoverer = new ScriptStartupTypeLocator(directory.Path, testLogger, mockExtensionBundleManager.Object, mockFunctionMetadataManager, testMetricsLogger, optionsWrapper); - var types = await discoverer.GetExtensionsStartupTypesAsync(); - // Act - var traces = testLoggerProvider.GetAllLogMessages(); - Assert.True(traces.Any(m => m.FormattedMessage.Contains($"Extension Bundle not loaded"))); - if (extensionConfigured) - { - Assert.True(traces.Any(m => m.FormattedMessage.Contains($"Loading startup extension 'Storage"))); - } - AssertNoErrors(traces); + // Assert + Assert.True(traces.Any(m => m.FormattedMessage.Contains($"Extension Bundle not loaded"))); + if (extensionConfigured) + { + Assert.True(traces.Any(m => m.FormattedMessage.Contains($"Loading startup extension 'Storage"))); } + + AssertNoErrors(traces); } [Theory] @@ -320,235 +211,95 @@ public async Task GetExtensionsStartupTypes_AcceptsRequiredExtensionVersions(str [InlineData("4.12.0", "4.0.6")] public async Task GetExtensionsStartupTypes_RejectsRequiredExtensionVersions(string minBundleVersion, string minExtensionVersion) { - using (var directory = GetTempDirectory()) - { - TestMetricsLogger testMetricsLogger = new TestMetricsLogger(); - TestLoggerProvider testLoggerProvider = new TestLoggerProvider(); - LoggerFactory factory = new LoggerFactory(); - factory.AddProvider(testLoggerProvider); - var testLogger = factory.CreateLogger(); - - var binPath = Path.Combine(directory.Path, "bin"); - - var mockExtensionBundleManager = new Mock(); + string binPath = InstallExtensions(ExtensionInstall.Storage(true), ExtensionInstall.SendGrid(true)); + _bundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(false); - mockExtensionBundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(false); - - var languageWorkerOptions = new TestOptionsMonitor(new LanguageWorkerOptions()); - var mockFunctionMetadataManager = GetTestFunctionMetadataManager(languageWorkerOptions); - - var extensionRequirementOptions = new Config.ExtensionRequirementOptions(); - - IEnumerable bundleRequirment = string.IsNullOrEmpty(minBundleVersion) - ? null - : new List() { new BundleRequirement() { Id = "Microsoft.Azure.Functions.ExtensionBundle", MinimumVersion = minBundleVersion } }; + // Act + ScriptStartupTypeLocator discoverer = CreateSystemUnderTest(extensionRequirements: GetExtensionRequirementOptions(minBundleVersion, minExtensionVersion)); + var exception = await Assert.ThrowsAsync(discoverer.GetExtensionsStartupTypesAsync); - IEnumerable extensionRequirements = string.IsNullOrEmpty(minExtensionVersion) - ? null - : new List() - { - new ExtensionStartupTypeRequirement() - { - Name = "AzureStorageWebJobsStartup", - AssemblyName = "Microsoft.Azure.WebJobs.Extensions.Storage", - MinimumAssemblyVersion = minExtensionVersion - } - }; - - extensionRequirementOptions.Bundles = bundleRequirment; - extensionRequirementOptions.Extensions = extensionRequirements; - - OptionsWrapper optionsWrapper = new(extensionRequirementOptions); - var discoverer = new ScriptStartupTypeLocator(directory.Path, testLogger, mockExtensionBundleManager.Object, mockFunctionMetadataManager, testMetricsLogger, optionsWrapper); - var exception = await Assert.ThrowsAsync(async () => await discoverer.GetExtensionsStartupTypesAsync()); - - // Act - var traces = testLoggerProvider.GetAllLogMessages(); - Assert.True(traces.Any(m => m.FormattedMessage.Contains($"Extension Bundle not loaded"))); - Assert.True(traces.Any(m => m.FormattedMessage.Contains($"ExtensionStartupType AzureStorageWebJobsStartup from assembly 'Microsoft.Azure.WebJobs.Extensions.Storage, Version=4.0.5.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35' does not meet the required minimum version of 4.0.6"))); - } + // Assert + var traces = _loggerProvider.GetAllLogMessages(); + Assert.True(traces.Any(m => m.FormattedMessage.Contains($"Extension Bundle not loaded"))); + Assert.True(traces.Any(m => m.FormattedMessage.Contains($"ExtensionStartupType AzureStorageWebJobsStartup from assembly 'Microsoft.Azure.WebJobs.Extensions.Storage, Version=4.0.5.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35' does not meet the required minimum version of 4.0.6"))); } [Fact] public async Task GetExtensionsStartupTypes_FiltersBuiltinExtensionsAsync() { - var references = new[] - { - new ExtensionReference { Name = "Http", TypeName = typeof(HttpWebJobsStartup).AssemblyQualifiedName }, - new ExtensionReference { Name = "Timers", TypeName = typeof(ExtensionsWebJobsStartup).AssemblyQualifiedName }, - new ExtensionReference { Name = "Storage", TypeName = typeof(AzureStorageWebJobsStartup).AssemblyQualifiedName }, - }; - - var extensions = new JObject - { - { "extensions", JArray.FromObject(references) } - }; - - var mockExtensionBundleManager = new Mock(); - TestMetricsLogger testMetricsLogger = new TestMetricsLogger(); - mockExtensionBundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(false); - - using (var directory = new TempDirectory()) - { - var binPath = Path.Combine(directory.Path, "bin"); - Directory.CreateDirectory(binPath); - - void CopyToBin(string path) - { - File.Copy(path, Path.Combine(binPath, Path.GetFileName(path))); - } - - CopyToBin(typeof(HttpWebJobsStartup).Assembly.Location); - CopyToBin(typeof(ExtensionsWebJobsStartup).Assembly.Location); - CopyToBin(typeof(AzureStorageWebJobsStartup).Assembly.Location); - - File.WriteAllText(Path.Combine(binPath, "extensions.json"), extensions.ToString()); - - TestLoggerProvider testLoggerProvider = new TestLoggerProvider(); - LoggerFactory factory = new LoggerFactory(); - factory.AddProvider(testLoggerProvider); - var testLogger = factory.CreateLogger(); + InstallExtensions(ExtensionInstall.Http(), ExtensionInstall.Timers(), ExtensionInstall.Storage()); + _bundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(false); - var languageWorkerOptions = new TestOptionsMonitor(new LanguageWorkerOptions()); - - var mockFunctionMetadataManager = GetTestFunctionMetadataManager(languageWorkerOptions, ImmutableArray.Empty); - OptionsWrapper optionsWrapper = new(new ExtensionRequirementOptions()); - var discoverer = new ScriptStartupTypeLocator(directory.Path, testLogger, mockExtensionBundleManager.Object, mockFunctionMetadataManager, testMetricsLogger, optionsWrapper); - - // Act - var types = await discoverer.GetExtensionsStartupTypesAsync(); - var traces = testLoggerProvider.GetAllLogMessages(); + // Act + ScriptStartupTypeLocator discoverer = CreateSystemUnderTest(); + var types = await discoverer.GetExtensionsStartupTypesAsync(); + var traces = _loggerProvider.GetAllLogMessages(); - // Assert - AreExpectedMetricsGenerated(testMetricsLogger); - Assert.Single(types); - Assert.Equal(typeof(AzureStorageWebJobsStartup).FullName, types.Single().FullName); - Assert.True(traces.Any(m => string.Equals(m.FormattedMessage, $"The extension startup type '{references[0].TypeName}' belongs to a builtin extension"))); - Assert.True(traces.Any(m => string.Equals(m.FormattedMessage, $"The extension startup type '{references[1].TypeName}' belongs to a builtin extension"))); - AssertNoErrors(traces); - } + // Assert + AreExpectedMetricsGenerated(); + Assert.Single(types); + Assert.Equal(typeof(AzureStorageWebJobsStartup).FullName, types.Single().FullName); + Assert.True(traces.Any(m => string.Equals(m.FormattedMessage, $"The extension startup type '{typeof(HttpWebJobsStartup).AssemblyQualifiedName}' belongs to a builtin extension"))); + Assert.True(traces.Any(m => string.Equals(m.FormattedMessage, $"The extension startup type '{typeof(ExtensionsWebJobsStartup).AssemblyQualifiedName}' belongs to a builtin extension"))); + AssertNoErrors(traces); } [Fact] public async Task GetExtensionsStartupTypes_ExtensionBundleReturnsNullPath_ReturnsNull() { - var mockExtensionBundleManager = new Mock(); - mockExtensionBundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(true); - mockExtensionBundleManager.Setup(e => e.GetExtensionBundlePath()).ReturnsAsync(string.Empty); - mockExtensionBundleManager.Setup(e => e.GetExtensionBundleDetails()).Returns(Task.FromResult(GetBundleDetails())); + _bundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(true); + _bundleManager.Setup(e => e.GetExtensionBundlePath()).ReturnsAsync(string.Empty); + _bundleManager.Setup(e => e.GetExtensionBundleDetails()).ReturnsAsync(GetBundleDetails()); - using (var directory = new TempDirectory()) - { - TestLoggerProvider testLoggerProvider = new TestLoggerProvider(); - LoggerFactory factory = new LoggerFactory(); - TestMetricsLogger testMetricsLogger = new TestMetricsLogger(); - factory.AddProvider(testLoggerProvider); - var testLogger = factory.CreateLogger(); - - var languageWorkerOptions = new TestOptionsMonitor(new LanguageWorkerOptions()); - var mockFunctionMetadataManager = GetTestFunctionMetadataManager(languageWorkerOptions, ImmutableArray.Empty); - OptionsWrapper optionsWrapper = new(new ExtensionRequirementOptions()); - var discoverer = new ScriptStartupTypeLocator(directory.Path, testLogger, mockExtensionBundleManager.Object, mockFunctionMetadataManager, testMetricsLogger, optionsWrapper); - - // Act - var types = await discoverer.GetExtensionsStartupTypesAsync(); - var traces = testLoggerProvider.GetAllLogMessages(); - - // Assert - AreExpectedMetricsGenerated(testMetricsLogger); - Assert.NotNull(types); - Assert.Equal(types.Count(), 0); - Assert.True(traces.Any(m => string.Equals(m.FormattedMessage, $"Unable to find or download extension bundle"))); - } + // Act + ScriptStartupTypeLocator discoverer = CreateSystemUnderTest(); + var types = await discoverer.GetExtensionsStartupTypesAsync(); + var traces = _loggerProvider.GetAllLogMessages(); + + // Assert + AreExpectedMetricsGenerated(); + Assert.NotNull(types); + Assert.Equal(types.Count(), 0); + Assert.True(traces.Any(m => string.Equals(m.FormattedMessage, $"Unable to find or download extension bundle"))); } [Fact] public async Task GetExtensionsStartupTypes_ValidExtensionBundle_FiltersBuiltinExtensionsAsync() { - var references = new[] - { - new ExtensionReference { Name = "Http", TypeName = typeof(HttpWebJobsStartup).AssemblyQualifiedName }, - new ExtensionReference { Name = "Timers", TypeName = typeof(ExtensionsWebJobsStartup).AssemblyQualifiedName }, - new ExtensionReference { Name = "Storage", TypeName = typeof(AzureStorageWebJobsStartup).AssemblyQualifiedName }, - }; - - var extensions = new JObject - { - { "extensions", JArray.FromObject(references) } - }; - - var mockExtensionBundleManager = new Mock(); - mockExtensionBundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(true); - mockExtensionBundleManager.Setup(e => e.GetExtensionBundleDetails()).Returns(Task.FromResult(GetBundleDetails())); - TestMetricsLogger testMetricsLogger = new TestMetricsLogger(); + string binPath = InstallExtensions(ExtensionInstall.Http(), ExtensionInstall.Timers(), ExtensionInstall.Storage()); + _bundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(true); + _bundleManager.Setup(e => e.GetExtensionBundleDetails()).ReturnsAsync(GetBundleDetails()); + _bundleManager.Setup(e => e.GetExtensionBundleBinPathAsync()).ReturnsAsync(binPath); - using (var directory = new TempDirectory()) - { - var binPath = Path.Combine(directory.Path, "bin"); - mockExtensionBundleManager.Setup(e => e.GetExtensionBundleBinPathAsync()).Returns(Task.FromResult(binPath)); - Directory.CreateDirectory(binPath); - - void CopyToBin(string path) - { - File.Copy(path, Path.Combine(binPath, Path.GetFileName(path))); - } + // Act + ScriptStartupTypeLocator discoverer = CreateSystemUnderTest(); + var types = await discoverer.GetExtensionsStartupTypesAsync(); + var traces = _loggerProvider.GetAllLogMessages(); - CopyToBin(typeof(HttpWebJobsStartup).Assembly.Location); - CopyToBin(typeof(ExtensionsWebJobsStartup).Assembly.Location); - CopyToBin(typeof(AzureStorageWebJobsStartup).Assembly.Location); - - File.WriteAllText(Path.Combine(binPath, "extensions.json"), extensions.ToString()); - - TestLoggerProvider testLoggerProvider = new TestLoggerProvider(); - LoggerFactory factory = new LoggerFactory(); - factory.AddProvider(testLoggerProvider); - var testLogger = factory.CreateLogger(); - - var languageWorkerOptions = new TestOptionsMonitor(new LanguageWorkerOptions()); - var mockFunctionMetadataManager = GetTestFunctionMetadataManager(languageWorkerOptions, ImmutableArray.Empty); - OptionsWrapper optionsWrapper = new(new ExtensionRequirementOptions()); - var discoverer = new ScriptStartupTypeLocator(directory.Path, testLogger, mockExtensionBundleManager.Object, mockFunctionMetadataManager, testMetricsLogger, optionsWrapper); - - // Act - var types = await discoverer.GetExtensionsStartupTypesAsync(); - var traces = testLoggerProvider.GetAllLogMessages(); - - // Assert - Assert.Single(types); - AreExpectedMetricsGenerated(testMetricsLogger); - Assert.Equal(typeof(AzureStorageWebJobsStartup).FullName, types.Single().FullName); - Assert.True(traces.Any(m => string.Equals(m.FormattedMessage, $"The extension startup type '{references[0].TypeName}' belongs to a builtin extension"))); - Assert.True(traces.Any(m => string.Equals(m.FormattedMessage, $"The extension startup type '{references[1].TypeName}' belongs to a builtin extension"))); - AssertNoErrors(traces); - } + // Assert + Assert.Single(types); + AreExpectedMetricsGenerated(); + Assert.Equal(typeof(AzureStorageWebJobsStartup).FullName, types.Single().FullName); + Assert.True(traces.Any(m => string.Equals(m.FormattedMessage, $"The extension startup type '{typeof(HttpWebJobsStartup).AssemblyQualifiedName}' belongs to a builtin extension"))); + Assert.True(traces.Any(m => string.Equals(m.FormattedMessage, $"The extension startup type '{typeof(ExtensionsWebJobsStartup).AssemblyQualifiedName}' belongs to a builtin extension"))); + AssertNoErrors(traces); } [Fact] public async Task GetExtensionsStartupTypes_UnableToDownloadExtensionBundle_ReturnsNull() { - var mockExtensionBundleManager = new Mock(); - mockExtensionBundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(true); - mockExtensionBundleManager.Setup(e => e.GetExtensionBundleDetails()).Returns(Task.FromResult(GetBundleDetails())); - mockExtensionBundleManager.Setup(e => e.GetExtensionBundlePath()).ReturnsAsync(string.Empty); - - TestLoggerProvider testLoggerProvider = new TestLoggerProvider(); - LoggerFactory factory = new LoggerFactory(); - TestMetricsLogger testMetricsLogger = new TestMetricsLogger(); - factory.AddProvider(testLoggerProvider); - var testLogger = factory.CreateLogger(); - - var languageWorkerOptions = new TestOptionsMonitor(new LanguageWorkerOptions()); - var mockFunctionMetadataManager = GetTestFunctionMetadataManager(languageWorkerOptions, ImmutableArray.Empty); - OptionsWrapper optionsWrapper = new(new ExtensionRequirementOptions()); - var discoverer = new ScriptStartupTypeLocator(string.Empty, testLogger, mockExtensionBundleManager.Object, mockFunctionMetadataManager, testMetricsLogger, optionsWrapper); + _bundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(true); + _bundleManager.Setup(e => e.GetExtensionBundleDetails()).ReturnsAsync(GetBundleDetails()); + _bundleManager.Setup(e => e.GetExtensionBundlePath()).ReturnsAsync(string.Empty); // Act + ScriptStartupTypeLocator discoverer = CreateSystemUnderTest(string.Empty); var types = await discoverer.GetExtensionsStartupTypesAsync(); - var traces = testLoggerProvider.GetAllLogMessages(); + var traces = _loggerProvider.GetAllLogMessages(); // Assert Assert.True(traces.Any(m => string.Equals(m.FormattedMessage, $"Unable to find or download extension bundle"))); - AreExpectedMetricsGenerated(testMetricsLogger); + AreExpectedMetricsGenerated(); Assert.NotNull(types); Assert.Equal(types.Count(), 0); } @@ -556,164 +307,59 @@ public async Task GetExtensionsStartupTypes_UnableToDownloadExtensionBundle_Retu [Fact] public async Task GetExtensionsStartupTypes_BundlesConfiguredBindingsNotConfigured_LoadsAllExtensions() { - var storageExtensionReference = new ExtensionReference { Name = "Storage", TypeName = typeof(AzureStorageWebJobsStartup).AssemblyQualifiedName }; - var sendGridExtensionReference = new ExtensionReference { Name = "SendGrid", TypeName = typeof(AzureStorageWebJobsStartup).AssemblyQualifiedName }; - var references = new[] { storageExtensionReference, sendGridExtensionReference }; - TestMetricsLogger testMetricsLogger = new TestMetricsLogger(); - - var extensions = new JObject - { - { "extensions", JArray.FromObject(references) } - }; - - using (var directory = new TempDirectory()) - { - var binPath = Path.Combine(directory.Path, "bin"); - Directory.CreateDirectory(binPath); + string binPath = InstallExtensions(ExtensionInstall.Storage(), ExtensionInstall.SendGrid()); + _bundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(true); + _bundleManager.Setup(e => e.GetExtensionBundleBinPathAsync()).ReturnsAsync(binPath); + _bundleManager.Setup(e => e.GetExtensionBundleDetails()).ReturnsAsync(GetBundleDetails()); - void CopyToBin(string path) - { - File.Copy(path, Path.Combine(binPath, Path.GetFileName(path))); - } + // Act + ScriptStartupTypeLocator discoverer = CreateSystemUnderTest(); + var types = await discoverer.GetExtensionsStartupTypesAsync(); + var traces = _loggerProvider.GetAllLogMessages(); - CopyToBin(typeof(AzureStorageWebJobsStartup).Assembly.Location); - - File.WriteAllText(Path.Combine(binPath, "extensions.json"), extensions.ToString()); - - TestLoggerProvider testLoggerProvider = new TestLoggerProvider(); - LoggerFactory factory = new LoggerFactory(); - factory.AddProvider(testLoggerProvider); - var testLogger = factory.CreateLogger(); - - var languageWorkerOptions = new TestOptionsMonitor(new LanguageWorkerOptions()); - var mockFunctionMetadataManager = GetTestFunctionMetadataManager(languageWorkerOptions); - var mockExtensionBundleManager = new Mock(); - mockExtensionBundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(true); - mockExtensionBundleManager.Setup(e => e.GetExtensionBundleBinPathAsync()).Returns(Task.FromResult(binPath)); - mockExtensionBundleManager.Setup(e => e.GetExtensionBundleDetails()).Returns(Task.FromResult(GetBundleDetails())); - OptionsWrapper optionsWrapper = new(new ExtensionRequirementOptions()); - var discoverer = new ScriptStartupTypeLocator(directory.Path, testLogger, mockExtensionBundleManager.Object, mockFunctionMetadataManager, testMetricsLogger, optionsWrapper); - - // Act - var types = await discoverer.GetExtensionsStartupTypesAsync(); - var traces = testLoggerProvider.GetAllLogMessages(); - - // Assert - AreExpectedMetricsGenerated(testMetricsLogger); - Assert.Equal(types.Count(), 2); - Assert.Equal(typeof(AzureStorageWebJobsStartup).FullName, types.FirstOrDefault().FullName); - AssertNoErrors(traces); - } + // Assert + AreExpectedMetricsGenerated(); + Assert.Equal(types.Count(), 2); + Assert.Equal(typeof(AzureStorageWebJobsStartup).FullName, types.FirstOrDefault().FullName); + AssertNoErrors(traces); } [Fact] public async Task GetExtensionsStartupTypes_BundlesNotConfiguredBindingsNotConfigured_LoadsAllExtensions() { - var references = new[] - { - new ExtensionReference { Name = "Storage", TypeName = typeof(AzureStorageWebJobsStartup).AssemblyQualifiedName } - }; - - var extensions = new JObject - { - { "extensions", JArray.FromObject(references) } - }; - - var mockExtensionBundleManager = new Mock(); - TestMetricsLogger testMetricsLogger = new TestMetricsLogger(); - mockExtensionBundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(false); - - using (var directory = new TempDirectory()) - { - var binPath = Path.Combine(directory.Path, "bin"); - Directory.CreateDirectory(binPath); - - void CopyToBin(string path) - { - File.Copy(path, Path.Combine(binPath, Path.GetFileName(path))); - } - - CopyToBin(typeof(AzureStorageWebJobsStartup).Assembly.Location); - - File.WriteAllText(Path.Combine(binPath, "extensions.json"), extensions.ToString()); + InstallExtensions(ExtensionInstall.Storage()); + _bundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(false); - TestLoggerProvider testLoggerProvider = new TestLoggerProvider(); - LoggerFactory factory = new LoggerFactory(); - factory.AddProvider(testLoggerProvider); - var testLogger = factory.CreateLogger(); - - // mock Function metadata - var languageWorkerOptions = new TestOptionsMonitor(new LanguageWorkerOptions()); - var mockFunctionMetadataManager = GetTestFunctionMetadataManager(languageWorkerOptions, ImmutableArray.Empty); - OptionsWrapper optionsWrapper = new(new ExtensionRequirementOptions()); - var discoverer = new ScriptStartupTypeLocator(directory.Path, testLogger, mockExtensionBundleManager.Object, mockFunctionMetadataManager, testMetricsLogger, optionsWrapper); - - // Act - var types = await discoverer.GetExtensionsStartupTypesAsync(); - var traces = testLoggerProvider.GetAllLogMessages(); + // Act + ScriptStartupTypeLocator discoverer = CreateSystemUnderTest(); + var types = await discoverer.GetExtensionsStartupTypesAsync(); + var traces = _loggerProvider.GetAllLogMessages(); - // Assert - AreExpectedMetricsGenerated(testMetricsLogger); - Assert.Single(types); - Assert.Equal(typeof(AzureStorageWebJobsStartup).FullName, types.Single().FullName); - AssertNoErrors(traces); - } + // Assert + AreExpectedMetricsGenerated(); + Assert.Single(types); + Assert.Equal(typeof(AzureStorageWebJobsStartup).FullName, types.Single().FullName); + AssertNoErrors(traces); } [Fact] public async Task GetExtensionsStartupTypes_BundlesConfiguredBindingsConfigured_PerformsSelectiveLoading() { - var storageExtensionReference = new ExtensionReference { Name = "Storage", TypeName = typeof(AzureStorageWebJobsStartup).AssemblyQualifiedName }; - storageExtensionReference.Bindings.Add("blob"); - var sendGridExtensionReference = new ExtensionReference { Name = "SendGrid", TypeName = typeof(AzureStorageWebJobsStartup).AssemblyQualifiedName }; - sendGridExtensionReference.Bindings.Add("sendGrid"); - var references = new[] { storageExtensionReference, sendGridExtensionReference }; - TestMetricsLogger testMetricsLogger = new TestMetricsLogger(); - - var extensions = new JObject - { - { "extensions", JArray.FromObject(references) } - }; - - using (var directory = new TempDirectory()) - { - var binPath = Path.Combine(directory.Path, "bin"); - Directory.CreateDirectory(binPath); - - void CopyToBin(string path) - { - File.Copy(path, Path.Combine(binPath, Path.GetFileName(path))); - } - - CopyToBin(typeof(AzureStorageWebJobsStartup).Assembly.Location); - - File.WriteAllText(Path.Combine(binPath, "extensions.json"), extensions.ToString()); - - TestLoggerProvider testLoggerProvider = new TestLoggerProvider(); - LoggerFactory factory = new LoggerFactory(); - factory.AddProvider(testLoggerProvider); - var testLogger = factory.CreateLogger(); - - var languageWorkerOptions = new TestOptionsMonitor(new LanguageWorkerOptions()); - var mockFunctionMetadataManager = GetTestFunctionMetadataManager(languageWorkerOptions); - - var mockExtensionBundleManager = new Mock(); - mockExtensionBundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(true); - mockExtensionBundleManager.Setup(e => e.GetExtensionBundleBinPathAsync()).Returns(Task.FromResult(binPath)); - mockExtensionBundleManager.Setup(e => e.GetExtensionBundleDetails()).Returns(Task.FromResult(GetBundleDetails())); - OptionsWrapper optionsWrapper = new(new ExtensionRequirementOptions()); - var discoverer = new ScriptStartupTypeLocator(directory.Path, testLogger, mockExtensionBundleManager.Object, mockFunctionMetadataManager, testMetricsLogger, optionsWrapper); + InstallExtensions(ExtensionInstall.Storage(true), ExtensionInstall.SendGrid(true)); + _bundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(true); + _bundleManager.Setup(e => e.GetExtensionBundleBinPathAsync()).ReturnsAsync(Path.Combine(_directory.Path, "bin")); + _bundleManager.Setup(e => e.GetExtensionBundleDetails()).ReturnsAsync(GetBundleDetails()); - // Act - var types = await discoverer.GetExtensionsStartupTypesAsync(); - var traces = testLoggerProvider.GetAllLogMessages(); + // Act + ScriptStartupTypeLocator discoverer = CreateSystemUnderTest(); + var types = await discoverer.GetExtensionsStartupTypesAsync(); + var traces = _loggerProvider.GetAllLogMessages(); - //Assert - AreExpectedMetricsGenerated(testMetricsLogger); - Assert.Single(types); - Assert.Equal(typeof(AzureStorageWebJobsStartup).FullName, types.Single().FullName); - AssertNoErrors(traces); - } + //Assert + AreExpectedMetricsGenerated(); + Assert.Single(types); + Assert.Equal(typeof(AzureStorageWebJobsStartup).FullName, types.Single().FullName); + AssertNoErrors(traces); } [Theory] @@ -721,83 +367,50 @@ void CopyToBin(string path) [InlineData(true)] public async Task GetExtensionsStartupTypes_LegacyBundles_UsesExtensionBundleBinaries(bool hasPrecompiledFunctions) { - using (var directory = GetTempDirectory()) + string binPath = InstallExtensions(ExtensionInstall.Storage(true), ExtensionInstall.SendGrid(true)); + _bundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(true); + _bundleManager.Setup(e => e.GetExtensionBundleBinPathAsync()).ReturnsAsync(binPath); + _bundleManager.Setup(e => e.IsLegacyExtensionBundle()).Returns(true); + _bundleManager.Setup(e => e.GetExtensionBundleDetails()).ReturnsAsync(GetBundleDetails()); + + if (hasPrecompiledFunctions) { - var binPath = Path.Combine(directory.Path, "bin"); - TestMetricsLogger testMetricsLogger = new TestMetricsLogger(); - var testLogger = GetTestLogger(); - - var mockExtensionBundleManager = new Mock(); - mockExtensionBundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(true); - mockExtensionBundleManager.Setup(e => e.GetExtensionBundleBinPathAsync()).Returns(Task.FromResult(binPath)); - mockExtensionBundleManager.Setup(e => e.IsLegacyExtensionBundle()).Returns(true); - mockExtensionBundleManager.Setup(e => e.GetExtensionBundleDetails()).Returns(Task.FromResult(GetBundleDetails())); - - var languageWorkerOptions = new TestOptionsMonitor(new LanguageWorkerOptions()); - var mockFunctionMetadataManager = GetTestFunctionMetadataManager(languageWorkerOptions, hasPrecompiledFunction: hasPrecompiledFunctions); - OptionsWrapper optionsWrapper = new(new ExtensionRequirementOptions()); - var discoverer = new ScriptStartupTypeLocator(directory.Path, testLogger, mockExtensionBundleManager.Object, mockFunctionMetadataManager, testMetricsLogger, optionsWrapper); - - // Act - var types = await discoverer.GetExtensionsStartupTypesAsync(); - - //Assert - AreExpectedMetricsGenerated(testMetricsLogger); - Assert.Single(types); - Assert.Equal(typeof(AzureStorageWebJobsStartup).FullName, types.Single().FullName); + SetupMetadataManager(DotNetScriptTypes.DotNetAssembly); } + + // Act + ScriptStartupTypeLocator discoverer = CreateSystemUnderTest(); + var types = await discoverer.GetExtensionsStartupTypesAsync(); + + //Assert + AreExpectedMetricsGenerated(); + Assert.Single(types); + Assert.Equal(typeof(AzureStorageWebJobsStartup).FullName, types.Single().FullName); } [Fact] public async Task GetExtensionsStartupTypes_WorkerRuntimeNotSetForNodeApp_LoadsExtensionBundle() { - var vars = new Dictionary(); - - using (var directory = GetTempDirectory()) - using (var env = new TestScopedEnvironmentVariable(vars)) - { - var binPath = Path.Combine(directory.Path, "bin"); - TestMetricsLogger testMetricsLogger = new TestMetricsLogger(); - TestLoggerProvider testLoggerProvider = new TestLoggerProvider(); - LoggerFactory factory = new LoggerFactory(); - factory.AddProvider(testLoggerProvider); - var testLogger = factory.CreateLogger(); + string binPath = InstallExtensions(ExtensionInstall.Storage(true), ExtensionInstall.SendGrid(true)); + _bundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(true); + _bundleManager.Setup(e => e.GetExtensionBundleBinPathAsync()).ReturnsAsync(binPath); + _bundleManager.Setup(e => e.IsLegacyExtensionBundle()).Returns(false); + _bundleManager.Setup(e => e.GetExtensionBundleDetails()).ReturnsAsync(GetBundleDetails()); + SetupMetadataManager(RpcWorkerConstants.NodeLanguageWorkerName); - var mockExtensionBundleManager = new Mock(); - mockExtensionBundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(true); - mockExtensionBundleManager.Setup(e => e.GetExtensionBundleBinPathAsync()).Returns(Task.FromResult(binPath)); - mockExtensionBundleManager.Setup(e => e.IsLegacyExtensionBundle()).Returns(false); - mockExtensionBundleManager.Setup(e => e.GetExtensionBundleDetails()).Returns(Task.FromResult(GetBundleDetails())); - - RpcWorkerConfig nodeWorkerConfig = new RpcWorkerConfig() { Description = TestHelpers.GetTestWorkerDescription("node", "none", false) }; - RpcWorkerConfig dotnetIsolatedWorkerConfig = new RpcWorkerConfig() { Description = TestHelpers.GetTestWorkerDescription("dotnet-isolated", "none", false) }; - - var tempOptions = new LanguageWorkerOptions(); - tempOptions.WorkerConfigs = new List(); - tempOptions.WorkerConfigs.Add(nodeWorkerConfig); - tempOptions.WorkerConfigs.Add(dotnetIsolatedWorkerConfig); - - var optionsMonitor = new TestOptionsMonitor(tempOptions); - var mockFunctionMetadataManager = GetTestFunctionMetadataManager(optionsMonitor, hasNodeFunctions: true); - - var languageWorkerOptions = new TestOptionsMonitor(tempOptions); - - OptionsWrapper optionsWrapper = new(new ExtensionRequirementOptions()); - var discoverer = new ScriptStartupTypeLocator(directory.Path, testLogger, mockExtensionBundleManager.Object, mockFunctionMetadataManager, testMetricsLogger, optionsWrapper); - - // Act - var types = await discoverer.GetExtensionsStartupTypesAsync(); + // Act + ScriptStartupTypeLocator discoverer = CreateSystemUnderTest(); + var types = await discoverer.GetExtensionsStartupTypesAsync(); - //Assert - var traces = testLoggerProvider.GetAllLogMessages(); - var traceMessage = traces.FirstOrDefault(val => string.Equals(val.EventId.Name, "ScriptStartNotLoadingExtensionBundle")); - bool loadingExtensionBundle = traceMessage == null; + //Assert + var traces = _loggerProvider.GetAllLogMessages(); + var traceMessage = traces.FirstOrDefault(val => val.EventId.Name.Equals("ScriptStartNotLoadingExtensionBundle")); + bool loadingExtensionBundle = traceMessage == null; - Assert.True(loadingExtensionBundle); - AreExpectedMetricsGenerated(testMetricsLogger); - Assert.Single(types); - Assert.Equal(typeof(AzureStorageWebJobsStartup).FullName, types.Single().FullName); - } + Assert.True(loadingExtensionBundle); + AreExpectedMetricsGenerated(); + Assert.Single(types); + Assert.Equal(typeof(AzureStorageWebJobsStartup).FullName, types.Single().FullName); } [Theory(Skip = "This test is failing on CI and needs to be fixed.")] @@ -807,73 +420,44 @@ public async Task GetExtensionsStartupTypes_WorkerRuntimeNotSetForNodeApp_LoadsE [InlineData(false, false)] public async Task GetExtensionsStartupTypes_DotnetIsolated_ExtensionBundleConfigured(bool isLogicApp, bool workerRuntimeSet) { - var vars = new Dictionary(); - if (isLogicApp) { - vars.Add(EnvironmentSettingNames.AppKind, ScriptConstants.WorkFlowAppKind); + _environment.SetEnvironmentVariable(EnvironmentSettingNames.AppKind, ScriptConstants.WorkFlowAppKind); } if (workerRuntimeSet) { - vars.Add(EnvironmentSettingNames.FunctionWorkerRuntime, "dotnet-isolated"); + _environment.SetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime, "dotnet-isolated"); } - using (var directory = GetTempDirectory()) - using (var env = new TestScopedEnvironmentVariable(vars)) - { - var binPath = Path.Combine(directory.Path, "bin"); - TestMetricsLogger testMetricsLogger = new TestMetricsLogger(); - TestLoggerProvider testLoggerProvider = new TestLoggerProvider(); - LoggerFactory factory = new LoggerFactory(); - factory.AddProvider(testLoggerProvider); - var testLogger = factory.CreateLogger(); - - var mockExtensionBundleManager = new Mock(); - mockExtensionBundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(true); - mockExtensionBundleManager.Setup(e => e.GetExtensionBundleBinPathAsync()).Returns(Task.FromResult(binPath)); - mockExtensionBundleManager.Setup(e => e.IsLegacyExtensionBundle()).Returns(false); - mockExtensionBundleManager.Setup(e => e.GetExtensionBundleDetails()).Returns(Task.FromResult(GetBundleDetails())); - - RpcWorkerConfig workerConfig = new RpcWorkerConfig() { Description = TestHelpers.GetTestWorkerDescription("dotnet-isolated", "none", true) }; - var tempOptions = new LanguageWorkerOptions(); - tempOptions.WorkerConfigs = new List(); - tempOptions.WorkerConfigs.Add(workerConfig); - - RpcWorkerConfig nodeWorkerConfig = new RpcWorkerConfig() { Description = TestHelpers.GetTestWorkerDescription("node", "none", false) }; - tempOptions.WorkerConfigs.Add(nodeWorkerConfig); - - var optionsMonitor = new TestOptionsMonitor(tempOptions); - - var mockFunctionMetadataManager = GetTestFunctionMetadataManager(optionsMonitor, hasDotnetIsolatedFunctions: true); + var binPath = Path.Combine(_directory.Path, "bin"); + _bundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(true); + _bundleManager.Setup(e => e.GetExtensionBundleBinPathAsync()).ReturnsAsync(binPath); + _bundleManager.Setup(e => e.IsLegacyExtensionBundle()).Returns(false); + _bundleManager.Setup(e => e.GetExtensionBundleDetails()).ReturnsAsync(GetBundleDetails()); + SetupMetadataManager(RpcWorkerConstants.DotNetIsolatedLanguageWorkerName); - var languageWorkerOptions = new TestOptionsMonitor(tempOptions); - - OptionsWrapper optionsWrapper = new(new ExtensionRequirementOptions()); - var discoverer = new ScriptStartupTypeLocator(directory.Path, testLogger, mockExtensionBundleManager.Object, mockFunctionMetadataManager, testMetricsLogger, optionsWrapper); - - // Act - var types = await discoverer.GetExtensionsStartupTypesAsync(); - - //Assert - var traces = testLoggerProvider.GetAllLogMessages(); - var traceMessage = traces.FirstOrDefault(val => val.EventId.Name.Equals("ScriptStartNotLoadingExtensionBundle")); - - bool loadingExtensionBundle = traceMessage == null; + // Act + ScriptStartupTypeLocator discoverer = CreateSystemUnderTest(); + var types = await discoverer.GetExtensionsStartupTypesAsync(); - if (isLogicApp) - { - Assert.True(loadingExtensionBundle); - } - else - { - Assert.False(loadingExtensionBundle); - } + //Assert + var traces = _loggerProvider.GetAllLogMessages(); + var traceMessage = traces.FirstOrDefault(val => val.EventId.Name.Equals("ScriptStartNotLoadingExtensionBundle")); + bool loadingExtensionBundle = traceMessage == null; - AreExpectedMetricsGenerated(testMetricsLogger); - Assert.Single(types); - Assert.Equal(typeof(AzureStorageWebJobsStartup).FullName, types.Single().FullName); + if (isLogicApp) + { + Assert.True(loadingExtensionBundle); } + else + { + Assert.False(loadingExtensionBundle); + } + + AreExpectedMetricsGenerated(); + Assert.Single(types); + Assert.Equal(typeof(AzureStorageWebJobsStartup).FullName, types.Single().FullName); } [Theory] @@ -881,141 +465,58 @@ public async Task GetExtensionsStartupTypes_DotnetIsolated_ExtensionBundleConfig [InlineData(true)] public async Task GetExtensionsStartupTypes_NonLegacyBundles_UsesBundlesForNonPrecompiledFunctions(bool hasPrecompiledFunctions) { - using (var directory = GetTempDirectory()) - { - TestMetricsLogger testMetricsLogger = new TestMetricsLogger(); - var testLogger = GetTestLogger(); - - string bundlePath = hasPrecompiledFunctions ? "FakePath" : directory.Path; - var binPath = Path.Combine(directory.Path, "bin"); - - var mockExtensionBundleManager = new Mock(); - mockExtensionBundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(true); - mockExtensionBundleManager.Setup(e => e.GetExtensionBundleBinPathAsync()).Returns(Task.FromResult(binPath)); - mockExtensionBundleManager.Setup(e => e.IsLegacyExtensionBundle()).Returns(false); - mockExtensionBundleManager.Setup(e => e.GetExtensionBundleDetails()).Returns(Task.FromResult(GetBundleDetails())); - - var languageWorkerOptions = new TestOptionsMonitor(new LanguageWorkerOptions()); - var mockFunctionMetadataManager = GetTestFunctionMetadataManager(languageWorkerOptions, hasPrecompiledFunction: hasPrecompiledFunctions); - OptionsWrapper optionsWrapper = new(new ExtensionRequirementOptions()); - var discoverer = new ScriptStartupTypeLocator(directory.Path, testLogger, mockExtensionBundleManager.Object, mockFunctionMetadataManager, testMetricsLogger, optionsWrapper); - - // Act - var types = await discoverer.GetExtensionsStartupTypesAsync(); - - //Assert - AreExpectedMetricsGenerated(testMetricsLogger); - Assert.Single(types); - Assert.Equal(typeof(AzureStorageWebJobsStartup).FullName, types.Single().FullName); - } + InstallExtensions(ExtensionInstall.Storage(true), ExtensionInstall.SendGrid(true)); + string bundlePath = hasPrecompiledFunctions ? "FakePath" : _directory.Path; + var binPath = Path.Combine(_directory.Path, "bin"); + + _bundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(true); + _bundleManager.Setup(e => e.GetExtensionBundleBinPathAsync()).ReturnsAsync(binPath); + _bundleManager.Setup(e => e.IsLegacyExtensionBundle()).Returns(false); + _bundleManager.Setup(e => e.GetExtensionBundleDetails()).ReturnsAsync(GetBundleDetails()); + SetupMetadataManager(DotNetScriptTypes.DotNetAssembly); + + // Act + ScriptStartupTypeLocator discoverer = CreateSystemUnderTest(); + var types = await discoverer.GetExtensionsStartupTypesAsync(); + + //Assert + AreExpectedMetricsGenerated(); + Assert.Single(types); + Assert.Equal(typeof(AzureStorageWebJobsStartup).FullName, types.Single().FullName); } [Fact] public async Task GetExtensionsStartupTypes_BundlesNotConfiguredBindingsConfigured_LoadsAllExtensions() { - var storageExtensionReference = new ExtensionReference { Name = "Storage", TypeName = typeof(AzureStorageWebJobsStartup).AssemblyQualifiedName }; - storageExtensionReference.Bindings.Add("blob"); - var sendGridExtensionReference = new ExtensionReference { Name = "SendGrid", TypeName = typeof(AzureStorageWebJobsStartup).AssemblyQualifiedName }; - sendGridExtensionReference.Bindings.Add("sendGrid"); - var references = new[] { storageExtensionReference, sendGridExtensionReference }; - TestMetricsLogger testMetricsLogger = new TestMetricsLogger(); - - var extensions = new JObject - { - { "extensions", JArray.FromObject(references) } - }; - - using (var directory = new TempDirectory()) - { - var binPath = Path.Combine(directory.Path, "bin"); - Directory.CreateDirectory(binPath); + InstallExtensions(ExtensionInstall.Storage(true), ExtensionInstall.SendGrid(true)); + _bundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(false); - void CopyToBin(string path) - { - File.Copy(path, Path.Combine(binPath, Path.GetFileName(path))); - } - - CopyToBin(typeof(AzureStorageWebJobsStartup).Assembly.Location); - - File.WriteAllText(Path.Combine(binPath, "extensions.json"), extensions.ToString()); - - TestLoggerProvider testLoggerProvider = new TestLoggerProvider(); - LoggerFactory factory = new LoggerFactory(); - factory.AddProvider(testLoggerProvider); - var testLogger = factory.CreateLogger(); - - var mockExtensionBundleManager = new Mock(); - mockExtensionBundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(false); - - var languageWorkerOptions = new TestOptionsMonitor(new LanguageWorkerOptions()); - var mockFunctionMetadataManager = GetTestFunctionMetadataManager(languageWorkerOptions); - OptionsWrapper optionsWrapper = new(new ExtensionRequirementOptions()); - var discoverer = new ScriptStartupTypeLocator(directory.Path, testLogger, mockExtensionBundleManager.Object, mockFunctionMetadataManager, testMetricsLogger, optionsWrapper); - - // Act - var types = await discoverer.GetExtensionsStartupTypesAsync(); + // Act + ScriptStartupTypeLocator discoverer = CreateSystemUnderTest(); + var types = await discoverer.GetExtensionsStartupTypesAsync(); - // Assert - AreExpectedMetricsGenerated(testMetricsLogger); - Assert.Equal(types.Count(), 2); - Assert.Equal(typeof(AzureStorageWebJobsStartup).FullName, types.FirstOrDefault().FullName); - } + // Assert + AreExpectedMetricsGenerated(); + Assert.Equal(types.Count(), 2); + Assert.Equal(typeof(AzureStorageWebJobsStartup).FullName, types.FirstOrDefault().FullName); } [Fact] public async Task GetExtensionsStartupTypes_NoBindings_In_ExtensionJson() { - TestMetricsLogger testMetricsLogger = new TestMetricsLogger(); + ExtensionInstall blobs = new("AzureStorageBlobs", typeof(AzureStorageWebJobsStartup)); + string binPath = InstallExtensions(ExtensionInstall.Storage(true), blobs); - using var directory = new TempDirectory(); - var binPath = Path.Combine(directory.Path, "bin"); - Directory.CreateDirectory(binPath); - - void CopyToBin(string path) - { - File.Copy(path, Path.Combine(binPath, Path.GetFileName(path))); - } - - CopyToBin(typeof(AzureStorageWebJobsStartup).Assembly.Location); - - string extensionJson = $$""" - { - "extensions": [ - { - "name": "Storage", - "typeName": "{{typeof(AzureStorageWebJobsStartup).AssemblyQualifiedName}}", - "hintPath": "Microsoft.Azure.WebJobs.Extensions.Storage.dll" - }, - { - "Name": "AzureStorageBlobs", - "TypeName": "{{typeof(AzureStorageWebJobsStartup).AssemblyQualifiedName}}" - } - ] - } - """; - - File.WriteAllText(Path.Combine(binPath, "extensions.json"), extensionJson); - - TestLoggerProvider testLoggerProvider = new TestLoggerProvider(); - LoggerFactory factory = new LoggerFactory(); - factory.AddProvider(testLoggerProvider); - var testLogger = factory.CreateLogger(); - - var mockExtensionBundleManager = new Mock(); - mockExtensionBundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(true); - mockExtensionBundleManager.Setup(e => e.GetExtensionBundleDetails()).Returns(Task.FromResult(new ExtensionBundleDetails() { Id = "bundleID", Version = "1.0.0" })); - mockExtensionBundleManager.Setup(e => e.GetExtensionBundleBinPathAsync()).Returns(Task.FromResult(binPath)); - - var languageWorkerOptions = new TestOptionsMonitor(new LanguageWorkerOptions()); - var mockFunctionMetadataManager = GetTestFunctionMetadataManager(languageWorkerOptions); - OptionsWrapper optionsWrapper = new(new ExtensionRequirementOptions()); - var discoverer = new ScriptStartupTypeLocator(directory.Path, testLogger, mockExtensionBundleManager.Object, mockFunctionMetadataManager, testMetricsLogger, optionsWrapper); + _bundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(true); + _bundleManager.Setup(e => e.GetExtensionBundleDetails()).Returns(Task.FromResult(new ExtensionBundleDetails() { Id = "bundleID", Version = "1.0.0" })); + _bundleManager.Setup(e => e.GetExtensionBundleBinPathAsync()).Returns(Task.FromResult(binPath)); // Act + ScriptStartupTypeLocator discoverer = CreateSystemUnderTest(); var types = await discoverer.GetExtensionsStartupTypesAsync(); // Assert - AreExpectedMetricsGenerated(testMetricsLogger); + AreExpectedMetricsGenerated(); Assert.Equal(types.Count(), 2); Assert.Equal(typeof(AzureStorageWebJobsStartup).FullName, types.FirstOrDefault().FullName); } @@ -1023,268 +524,221 @@ void CopyToBin(string path) [Fact] public async Task GetExtensionsStartupTypes_RejectsBundleBelowMinimumVersion() { - using (var directory = GetTempDirectory()) - { - TestMetricsLogger testMetricsLogger = new TestMetricsLogger(); - TestLoggerProvider testLoggerProvider = new TestLoggerProvider(); - LoggerFactory factory = new LoggerFactory(); - factory.AddProvider(testLoggerProvider); - var testLogger = factory.CreateLogger(); - - var binPath = Path.Combine(directory.Path, "bin"); - - var mockExtensionBundleManager = new Mock(); - mockExtensionBundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(true); - mockExtensionBundleManager.Setup(e => e.GetExtensionBundleBinPathAsync()).Returns(Task.FromResult(binPath)); - mockExtensionBundleManager.Setup(e => e.IsLegacyExtensionBundle()).Returns(false); - mockExtensionBundleManager.Setup(e => e.GetExtensionBundleDetails()).Returns(Task.FromResult(GetBundleDetails("2.1.0"))); - - var languageWorkerOptions = new TestOptionsMonitor(new LanguageWorkerOptions()); - var mockFunctionMetadataManager = GetTestFunctionMetadataManager(languageWorkerOptions); - OptionsWrapper optionsWrapper = new(new ExtensionRequirementOptions()); - var discoverer = new ScriptStartupTypeLocator(directory.Path, testLogger, mockExtensionBundleManager.Object, mockFunctionMetadataManager, testMetricsLogger, optionsWrapper); - - // Act - var exception = await Assert.ThrowsAsync(async () => await discoverer.GetExtensionsStartupTypesAsync()); - var traces = testLoggerProvider.GetAllLogMessages(); - - // Assert - Assert.True(traces.Any(m => string.Equals(m.FormattedMessage, $"Referenced bundle Microsoft.Azure.Functions.ExtensionBundle of version 2.1.0 does not meet the required minimum version of 2.6.1. Update your extension bundle reference in host.json to reference 2.6.1 or later."))); - } + var binPath = Path.Combine(_directory.Path, "bin"); + + _bundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(true); + _bundleManager.Setup(e => e.GetExtensionBundleBinPathAsync()).ReturnsAsync(binPath); + _bundleManager.Setup(e => e.IsLegacyExtensionBundle()).Returns(false); + _bundleManager.Setup(e => e.GetExtensionBundleDetails()).ReturnsAsync(GetBundleDetails("2.1.0")); + + // Act + ScriptStartupTypeLocator discoverer = CreateSystemUnderTest(); + var exception = await Assert.ThrowsAsync(discoverer.GetExtensionsStartupTypesAsync); + var traces = _loggerProvider.GetAllLogMessages(); + + // Assert + Assert.True(traces.Any(m => string.Equals(m.FormattedMessage, $"Referenced bundle Microsoft.Azure.Functions.ExtensionBundle of version 2.1.0 does not meet the required minimum version of 2.6.1. Update your extension bundle reference in host.json to reference 2.6.1 or later."))); } [Fact] public async Task GetExtensionsStartupTypes_RejectsExtensionsBelowMinimumVersion() { - var mockExtensionBundleManager = new Mock(); - TestMetricsLogger testMetricsLogger = new TestMetricsLogger(); - mockExtensionBundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(false); + _bundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(false); - using (var directory = new TempDirectory()) - { - var binPath = Path.Combine(directory.Path, "bin"); - Directory.CreateDirectory(binPath); + var binPath = Path.Combine(_directory.Path, "bin"); + Directory.CreateDirectory(binPath); - void CopyToBin(string path) - { - File.Copy(path, Path.Combine(binPath, Path.GetFileName(path))); - } + void CopyToBin(string path) + { + File.Copy(path, Path.Combine(binPath, Path.GetFileName(path))); + } - // create a bin folder that has out of date extensions - var extensionBinPath = Path.Combine(Environment.CurrentDirectory, @"TestScripts\OutOfDateExtension\bin"); - foreach (var f in Directory.GetFiles(extensionBinPath)) - { - CopyToBin(f); - } - TestLoggerProvider testLoggerProvider = new TestLoggerProvider(); - LoggerFactory factory = new LoggerFactory(); - factory.AddProvider(testLoggerProvider); - var testLogger = factory.CreateLogger(); - - var languageWorkerOptions = new TestOptionsMonitor(new LanguageWorkerOptions()); - var mockFunctionMetadataManager = GetTestFunctionMetadataManager(languageWorkerOptions, ImmutableArray.Empty); - OptionsWrapper optionsWrapper = new(new ExtensionRequirementOptions()); - var discoverer = new ScriptStartupTypeLocator(directory.Path, testLogger, mockExtensionBundleManager.Object, mockFunctionMetadataManager, testMetricsLogger, optionsWrapper); - - // Act - var exception = await Assert.ThrowsAsync(async () => await discoverer.GetExtensionsStartupTypesAsync()); - var traces = testLoggerProvider.GetAllLogMessages(); - - // Assert - - var storageTrace = traces.FirstOrDefault(m => m.FormattedMessage.StartsWith("ExtensionStartupType AzureStorageWebJobsStartup")); - Assert.NotNull(storageTrace); - Assert.Equal("ExtensionStartupType AzureStorageWebJobsStartup from assembly 'Microsoft.Azure.WebJobs.Extensions.Storage, Version=3.0.10.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35' does not meet the required minimum version of 4.0.4.0. Update your NuGet package reference for Microsoft.Azure.WebJobs.Extensions.Storage to 4.0.4 or later.", - storageTrace.FormattedMessage); + // create a bin folder that has out of date extensions + var extensionBinPath = Path.Combine(Environment.CurrentDirectory, @"TestScripts\OutOfDateExtension\bin"); + foreach (var f in Directory.GetFiles(extensionBinPath)) + { + CopyToBin(f); } + + // Act + ScriptStartupTypeLocator discoverer = CreateSystemUnderTest(); + var exception = await Assert.ThrowsAsync(discoverer.GetExtensionsStartupTypesAsync); + var traces = _loggerProvider.GetAllLogMessages(); + + // Assert + var storageTrace = traces.FirstOrDefault(m => m.FormattedMessage.StartsWith("ExtensionStartupType AzureStorageWebJobsStartup")); + Assert.NotNull(storageTrace); + Assert.Equal("ExtensionStartupType AzureStorageWebJobsStartup from assembly 'Microsoft.Azure.WebJobs.Extensions.Storage, Version=3.0.10.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35' does not meet the required minimum version of 4.0.4.0. Update your NuGet package reference for Microsoft.Azure.WebJobs.Extensions.Storage to 4.0.4 or later.", + storageTrace.FormattedMessage); } [Fact] public async Task GetExtensionsStartupTypes_WorkerIndexing_PerformsSelectiveLoading() { - var storageExtensionReference = new ExtensionReference { Name = "Storage", TypeName = typeof(AzureStorageWebJobsStartup).AssemblyQualifiedName }; - storageExtensionReference.Bindings.Add("blob"); - var sendGridExtensionReference = new ExtensionReference { Name = "SendGrid", TypeName = typeof(AzureStorageWebJobsStartup).AssemblyQualifiedName }; - sendGridExtensionReference.Bindings.Add("sendGrid"); - var references = new[] { storageExtensionReference, sendGridExtensionReference }; - TestMetricsLogger testMetricsLogger = new TestMetricsLogger(); - - var extensions = new JObject - { - { "extensions", JArray.FromObject(references) } - }; + string binPath = InstallExtensions(ExtensionInstall.Storage(true), ExtensionInstall.SendGrid(true)); + _bundleManager.Setup(e => e.GetExtensionBundleDetails()).ReturnsAsync(new ExtensionBundleDetails() { Id = "bundleID", Version = "1.0.0" }); + _bundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(true); + _bundleManager.Setup(e => e.GetExtensionBundleBinPathAsync()).ReturnsAsync(binPath); - using (var directory = new TempDirectory()) - { - var binPath = Path.Combine(directory.Path, "bin"); - Directory.CreateDirectory(binPath); + _environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags, ScriptConstants.FeatureFlagEnableWorkerIndexing); + _environment.SetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime, "python"); - void CopyToBin(string path) - { - File.Copy(path, Path.Combine(binPath, Path.GetFileName(path))); - } + // Act + ScriptStartupTypeLocator discoverer = CreateSystemUnderTest(); + var types = await discoverer.GetExtensionsStartupTypesAsync(); + var traces = _loggerProvider.GetAllLogMessages(); - CopyToBin(typeof(AzureStorageWebJobsStartup).Assembly.Location); - - File.WriteAllText(Path.Combine(binPath, "extensions.json"), extensions.ToString()); - - TestLoggerProvider testLoggerProvider = new TestLoggerProvider(); - LoggerFactory factory = new LoggerFactory(); - factory.AddProvider(testLoggerProvider); - var testLogger = factory.CreateLogger(); - - var mockExtensionBundleManager = new Mock(); - mockExtensionBundleManager.Setup(e => e.GetExtensionBundleDetails()).Returns(Task.FromResult(new ExtensionBundleDetails() { Id = "bundleID", Version = "1.0.0" })); - mockExtensionBundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(true); - mockExtensionBundleManager.Setup(e => e.GetExtensionBundleBinPathAsync()).Returns(Task.FromResult(binPath)); - - // mock worker config and environment variables to make host choose worker indexing - RpcWorkerConfig workerConfig = new RpcWorkerConfig() { Description = TestHelpers.GetTestWorkerDescription("python", "none", true) }; - var tempOptions = new LanguageWorkerOptions(); - tempOptions.WorkerConfigs = new List(); - tempOptions.WorkerConfigs.Add(workerConfig); - var optionsMonitor = new TestOptionsMonitor(tempOptions); - var mockFunctionMetadataManager = GetTestFunctionMetadataManager(optionsMonitor); - - var languageWorkerOptions = new TestOptionsMonitor(tempOptions); - Environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags, ScriptConstants.FeatureFlagEnableWorkerIndexing); - Environment.SetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime, "python"); - OptionsWrapper optionsWrapper = new(new ExtensionRequirementOptions()); - var discoverer = new ScriptStartupTypeLocator(directory.Path, testLogger, mockExtensionBundleManager.Object, mockFunctionMetadataManager, testMetricsLogger, optionsWrapper); - - // Act - var types = await discoverer.GetExtensionsStartupTypesAsync(); - var traces = testLoggerProvider.GetAllLogMessages(); - Environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags, null); - Environment.SetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime, null); - - //Assert that filtering did not take place because of worker indexing - Assert.True(types.Count() == 1); - Assert.Equal(typeof(AzureStorageWebJobsStartup).FullName, types.ElementAt(0).FullName); - AssertNoErrors(traces); - } + //Assert that filtering did not take place because of worker indexing + Assert.True(types.Count() == 1); + Assert.Equal(typeof(AzureStorageWebJobsStartup).FullName, types.ElementAt(0).FullName); + AssertNoErrors(traces); } [Fact] public async Task GetExtensionsStartupTypes_EmptyExtensionsArray() { - TestMetricsLogger testMetricsLogger = new TestMetricsLogger(); - - using var directory = new TempDirectory(); - var binPath = Path.Combine(directory.Path, "bin"); - Directory.CreateDirectory(binPath); - - // extensions.json file with an empty extensions array (simulating extensions.json produced by in-proc app) - string extensionJson = """ - { - "extensions": [] - } - """; - File.WriteAllText(Path.Combine(binPath, "extensions.json"), extensionJson); - - TestLoggerProvider testLoggerProvider = new TestLoggerProvider(); - LoggerFactory factory = new LoggerFactory(); - factory.AddProvider(testLoggerProvider); - var testLogger = factory.CreateLogger(); - - var mockExtensionBundleManager = new Mock(); - mockExtensionBundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(true); - mockExtensionBundleManager.Setup(e => e.GetExtensionBundleDetails()).Returns(Task.FromResult(new ExtensionBundleDetails() { Id = "bundleID", Version = "1.0.0" })); - mockExtensionBundleManager.Setup(e => e.GetExtensionBundleBinPathAsync()).Returns(Task.FromResult(binPath)); - - var languageWorkerOptions = new TestOptionsMonitor(new LanguageWorkerOptions()); - var mockFunctionMetadataManager = GetTestFunctionMetadataManager(languageWorkerOptions); - OptionsWrapper optionsWrapper = new(new ExtensionRequirementOptions()); - var discoverer = new ScriptStartupTypeLocator(directory.Path, testLogger, mockExtensionBundleManager.Object, mockFunctionMetadataManager, testMetricsLogger, optionsWrapper); + string binPath = InstallExtensions(); + _bundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(true); + _bundleManager.Setup(e => e.GetExtensionBundleDetails()).Returns(Task.FromResult(new ExtensionBundleDetails() { Id = "bundleID", Version = "1.0.0" })); + _bundleManager.Setup(e => e.GetExtensionBundleBinPathAsync()).Returns(Task.FromResult(binPath)); + // Act + ScriptStartupTypeLocator discoverer = CreateSystemUnderTest(); var types = await discoverer.GetExtensionsStartupTypesAsync(); - var traces = testLoggerProvider.GetAllLogMessages(); + var traces = _loggerProvider.GetAllLogMessages(); - AreExpectedMetricsGenerated(testMetricsLogger); + AreExpectedMetricsGenerated(); Assert.Empty(types); // Ensure no types are loaded because the extensions array is empty AssertNoErrors(traces); } - private IFunctionMetadataManager GetTestFunctionMetadataManager(IOptionsMonitor options, ICollection metadataCollection = null, bool hasPrecompiledFunction = false, bool hasNodeFunctions = false, bool hasDotnetIsolatedFunctions = false) + private static ExtensionBundleDetails GetBundleDetails(string version = "2.7.0") { - var functionMetadata = new FunctionMetadata(); - functionMetadata.Bindings.Add(new BindingMetadata() { Type = "blob" }); - - if (hasPrecompiledFunction) + return new ExtensionBundleDetails { - functionMetadata.Language = DotNetScriptTypes.DotNetAssembly; - } - if (hasNodeFunctions) + Id = "Microsoft.Azure.Functions.ExtensionBundle", + Version = version + }; + } + + private static ExtensionRequirementOptions GetExtensionRequirementOptions(string minBundleVersion, string minExtensionVersion) + { + ExtensionRequirementOptions extensionRequirementOptions = new(); + IEnumerable bundleRequirment = string.IsNullOrEmpty(minBundleVersion) + ? null + : [new() { Id = "Microsoft.Azure.Functions.ExtensionBundle", MinimumVersion = minBundleVersion }]; + + IEnumerable extensionRequirements = string.IsNullOrEmpty(minExtensionVersion) + ? null : + [ + new() + { + Name = "AzureStorageWebJobsStartup", + AssemblyName = "Microsoft.Azure.WebJobs.Extensions.Storage", + MinimumAssemblyVersion = minExtensionVersion + } + ]; + + extensionRequirementOptions.Bundles = bundleRequirment; + extensionRequirementOptions.Extensions = extensionRequirements; + return extensionRequirementOptions; + } + + private string InstallExtensions(params ExtensionInstall[] extensions) + { + string binPath = Path.Combine(_directory.Path, "bin"); + Directory.CreateDirectory(binPath); + + JArray jArray = []; + foreach (ExtensionInstall e in extensions) { - functionMetadata.Language = RpcWorkerConstants.NodeLanguageWorkerName; + ExtensionReference reference = e.GetReference(); + jArray.Add(JObject.FromObject(reference)); + e.CopyTo(binPath); } - if (hasDotnetIsolatedFunctions) + JObject jObject = new() { - functionMetadata.Language = RpcWorkerConstants.DotNetIsolatedLanguageWorkerName; - } + { "extensions", jArray }, + }; - var functionMetadataCollection = metadataCollection ?? new List() { functionMetadata }; + File.WriteAllText(Path.Combine(binPath, "extensions.json"), jObject.ToString()); + return binPath; + } - var functionMetadataManager = new Mock(); - functionMetadataManager.Setup(e => e.GetFunctionMetadata(true, true, false)).Returns(functionMetadataCollection.ToImmutableArray()); - return functionMetadataManager.Object; + private void SetupMetadataManager(string language) + { + FunctionMetadata functionMetadata = new(); + functionMetadata.Bindings.Add(new BindingMetadata() { Type = "blob" }); + functionMetadata.Language = language; + ImmutableArray result = [functionMetadata]; + _metadataManager.Setup(m => m.GetFunctionMetadata(true, true, false)).Returns(result); } - private bool AreExpectedMetricsGenerated(TestMetricsLogger metricsLogger) + private ScriptStartupTypeLocator CreateSystemUnderTest(string rootPath = null, ExtensionRequirementOptions extensionRequirements = null) { - return metricsLogger.EventsBegan.Contains(MetricEventNames.ParseExtensions) && metricsLogger.EventsEnded.Contains(MetricEventNames.ParseExtensions); + LoggerFactory factory = new(); + factory.AddProvider(_loggerProvider); + OptionsWrapper optionsWrapper = new(extensionRequirements ?? new()); + return new( + rootPath ?? _directory.Path, + factory.CreateLogger(), + _bundleManager.Object, + _metadataManager.Object, + _metricsLogger, + _environment, + optionsWrapper); } - private TempDirectory GetTempDirectory(bool copyExtensionsToBin = true) + private bool AreExpectedMetricsGenerated() { - var directory = new TempDirectory(); + return _metricsLogger.EventsBegan.Contains(MetricEventNames.ParseExtensions) && _metricsLogger.EventsEnded.Contains(MetricEventNames.ParseExtensions); + } - if (copyExtensionsToBin) + private class ExtensionInstall(string name, Type startupType, params string[] bindings) + { + public static ExtensionInstall Storage(bool includeBinding = false) { - var storageExtensionReference = new ExtensionReference { Name = "Storage", TypeName = typeof(AzureStorageWebJobsStartup).AssemblyQualifiedName }; - storageExtensionReference.Bindings.Add("blob"); - var sendGridExtensionReference = new ExtensionReference { Name = "SendGrid", TypeName = typeof(AzureStorageWebJobsStartup).AssemblyQualifiedName }; - sendGridExtensionReference.Bindings.Add("sendGrid"); - var references = new[] { storageExtensionReference, sendGridExtensionReference }; + string[] bindings = includeBinding ? ["blob"] : []; + return new("Storage", typeof(AzureStorageWebJobsStartup), bindings); + } - var extensions = new JObject - { - { "extensions", JArray.FromObject(references) } - }; + public static ExtensionInstall SendGrid(bool includeBinding = false) + { + string[] bindings = includeBinding ? ["sendGrid"] : []; + return new("SendGrid", typeof(AzureStorageWebJobsStartup), bindings); + } - var binPath = Path.Combine(directory.Path, "bin"); - Directory.CreateDirectory(binPath); + public static ExtensionInstall Timers() + { + return new("Timers", typeof(ExtensionsWebJobsStartup)); + } - void CopyToBin(string path) + public static ExtensionInstall Http() + { + return new("Http", typeof(HttpWebJobsStartup)); + } + + public ExtensionReference GetReference() + { + ExtensionReference reference = new() { Name = name, TypeName = startupType.AssemblyQualifiedName }; + foreach (string binding in bindings ?? Enumerable.Empty()) { - File.Copy(path, Path.Combine(binPath, Path.GetFileName(path))); + reference.Bindings.Add(binding); } - CopyToBin(typeof(AzureStorageWebJobsStartup).Assembly.Location); - - File.WriteAllText(Path.Combine(binPath, "extensions.json"), extensions.ToString()); + return reference; } - return directory; - } - - private ExtensionBundleDetails GetBundleDetails(string version = "2.7.0") - { - return new ExtensionBundleDetails + public void CopyTo(string path) { - Id = "Microsoft.Azure.Functions.ExtensionBundle", - Version = version - }; - } - - private ILogger GetTestLogger() - { - TestLoggerProvider testLoggerProvider = new TestLoggerProvider(); - LoggerFactory factory = new LoggerFactory(); - factory.AddProvider(testLoggerProvider); - var testLogger = factory.CreateLogger(); - return testLogger; + string file = startupType.Assembly.Location; + string destination = Path.Combine(path, Path.GetFileName(file)); + if (!File.Exists(destination)) + { + File.Copy(file, destination); + } + } } private static void AssertNoErrors(IList traces) From d3a26101fb345644157624b38539e6cd294e1d9e Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Mon, 21 Apr 2025 13:13:01 -0700 Subject: [PATCH 07/16] Fix more unit tests --- Directory.Build.targets | 4 ++ .../Workers/Rpc/RpcWorkerProcess.cs | 37 +++++----- .../TestFunctionHost.cs | 58 +++++++-------- .../Description/FunctionInvokerBaseTests.cs | 46 +++++------- .../Extensions/EnvironmentExtensionsTests.cs | 70 +++++++++++-------- 5 files changed, 110 insertions(+), 105 deletions(-) diff --git a/Directory.Build.targets b/Directory.Build.targets index 8905fec051..10729ddb49 100644 --- a/Directory.Build.targets +++ b/Directory.Build.targets @@ -2,4 +2,8 @@ + + $(ArtifactsPath)/log/$(ArtifactsProjectName)/tests_$(ArtifactsPivots)/ + + diff --git a/src/WebJobs.Script/Workers/Rpc/RpcWorkerProcess.cs b/src/WebJobs.Script/Workers/Rpc/RpcWorkerProcess.cs index 7e2504ad4f..d1398681e6 100644 --- a/src/WebJobs.Script/Workers/Rpc/RpcWorkerProcess.cs +++ b/src/WebJobs.Script/Workers/Rpc/RpcWorkerProcess.cs @@ -26,22 +26,23 @@ internal class RpcWorkerProcess : WorkerProcess private readonly IOptions _hostingConfigOptions; private readonly IEnvironment _environment; - internal RpcWorkerProcess(string runtime, - string workerId, - string rootScriptPath, - Uri serverUri, - RpcWorkerConfig rpcWorkerConfig, - IScriptEventManager eventManager, - IWorkerProcessFactory processFactory, - IProcessRegistry processRegistry, - ILogger workerProcessLogger, - IWorkerConsoleLogSource consoleLogSource, - IMetricsLogger metricsLogger, - IServiceProvider serviceProvider, - IOptions hostingConfigOptions, - IEnvironment environment, - IOptionsMonitor scriptApplicationHostOptions, - ILoggerFactory loggerFactory) + internal RpcWorkerProcess( + string runtime, + string workerId, + string rootScriptPath, + Uri serverUri, + RpcWorkerConfig rpcWorkerConfig, + IScriptEventManager eventManager, + IWorkerProcessFactory processFactory, + IProcessRegistry processRegistry, + ILogger workerProcessLogger, + IWorkerConsoleLogSource consoleLogSource, + IMetricsLogger metricsLogger, + IServiceProvider serviceProvider, + IOptions hostingConfigOptions, + IEnvironment environment, + IOptionsMonitor scriptApplicationHostOptions, + ILoggerFactory loggerFactory) : base(eventManager, processRegistry, workerProcessLogger, consoleLogSource, metricsLogger, serviceProvider, loggerFactory, environment, scriptApplicationHostOptions, rpcWorkerConfig.Description.UseStdErrorStreamForErrorsOnly) { @@ -81,7 +82,7 @@ internal override void HandleWorkerProcessExitError(WorkerProcessExitException r } // The subscriber of WorkerErrorEvent is expected to Dispose() the errored channel - _workerProcessLogger.LogError(rpcWorkerProcessExitException, $"Language Worker Process exited. Pid={rpcWorkerProcessExitException.Pid}.", _workerProcessArguments.ExecutablePath); + _workerProcessLogger.LogError(rpcWorkerProcessExitException, $"Language Worker Process exited. Pid={rpcWorkerProcessExitException.Pid}.", _workerProcessArguments?.ExecutablePath); _eventManager.Publish(new WorkerErrorEvent(_runtime, _workerId, rpcWorkerProcessExitException)); } @@ -91,4 +92,4 @@ internal override void HandleWorkerProcessRestart() _eventManager.Publish(new WorkerRestartEvent(_runtime, _workerId)); } } -} \ No newline at end of file +} diff --git a/test/WebJobs.Script.Tests.Integration/TestFunctionHost.cs b/test/WebJobs.Script.Tests.Integration/TestFunctionHost.cs index d98c74284b..7aabad818a 100644 --- a/test/WebJobs.Script.Tests.Integration/TestFunctionHost.cs +++ b/test/WebJobs.Script.Tests.Integration/TestFunctionHost.cs @@ -104,35 +104,35 @@ public TestFunctionHost(string scriptPath, string logPath, string testDataPath = .AddFilter("Azure.Core", LogLevel.Warning); }) .ConfigureServices(services => - { - services.Replace(new ServiceDescriptor(typeof(ISecretManagerProvider), new TestSecretManagerProvider(new TestSecretManager()))); - services.Replace(new ServiceDescriptor(typeof(IOptions), sp => - { - _hostOptions.RootServiceProvider = sp; - return new OptionsWrapper(_hostOptions); - }, ServiceLifetime.Singleton)); - services.Replace(new ServiceDescriptor(typeof(IOptionsMonitor), sp => - { - _hostOptions.RootServiceProvider = sp; - return TestHelpers.CreateOptionsMonitor(_hostOptions); - }, ServiceLifetime.Singleton)); - services.Replace(new ServiceDescriptor(typeof(IExtensionBundleManager), new TestExtensionBundleManager())); - services.Replace(new ServiceDescriptor(typeof(IFunctionMetadataManager), sp => - { - var montior = sp.GetService>(); - var scriptManager = sp.GetService(); - var loggerFactory = sp.GetService(); - var environment = sp.GetService(); - - return GetMetadataManager(montior, scriptManager, loggerFactory, environment); - }, ServiceLifetime.Singleton)); - - services.AddSingleton(); - services.SkipDependencyValidation(); - - // Allows us to configure services as the last step, thereby overriding anything - services.AddSingleton(new PostConfigureServices(configureWebHostServices)); - }) + { + services.Replace(new ServiceDescriptor(typeof(ISecretManagerProvider), new TestSecretManagerProvider(new TestSecretManager()))); + services.Replace(new ServiceDescriptor(typeof(IOptions), sp => + { + _hostOptions.RootServiceProvider = sp; + return new OptionsWrapper(_hostOptions); + }, ServiceLifetime.Singleton)); + services.Replace(new ServiceDescriptor(typeof(IOptionsMonitor), sp => + { + _hostOptions.RootServiceProvider = sp; + return TestHelpers.CreateOptionsMonitor(_hostOptions); + }, ServiceLifetime.Singleton)); + services.Replace(new ServiceDescriptor(typeof(IExtensionBundleManager), new TestExtensionBundleManager())); + services.Replace(new ServiceDescriptor(typeof(IFunctionMetadataManager), sp => + { + var montior = sp.GetService>(); + var scriptManager = sp.GetService(); + var loggerFactory = sp.GetService(); + var environment = sp.GetService(); + + return GetMetadataManager(montior, scriptManager, loggerFactory, environment); + }, ServiceLifetime.Singleton)); + + services.AddSingleton(); + services.SkipDependencyValidation(); + + // Allows us to configure services as the last step, thereby overriding anything + services.AddSingleton(new PostConfigureServices(configureWebHostServices)); + }) .ConfigureScriptHostWebJobsBuilder(scriptHostWebJobsBuilder => { /// REVIEW THIS diff --git a/test/WebJobs.Script.Tests/Description/FunctionInvokerBaseTests.cs b/test/WebJobs.Script.Tests/Description/FunctionInvokerBaseTests.cs index ef62d61f7e..b957a0e53c 100644 --- a/test/WebJobs.Script.Tests/Description/FunctionInvokerBaseTests.cs +++ b/test/WebJobs.Script.Tests/Description/FunctionInvokerBaseTests.cs @@ -9,53 +9,49 @@ using Microsoft.Azure.WebJobs.Host.Loggers; using Microsoft.Azure.WebJobs.Script.Description; using Microsoft.Azure.WebJobs.Script.Diagnostics; -using Microsoft.Azure.WebJobs.Script.Eventing; using Microsoft.Azure.WebJobs.Script.Metrics; using Microsoft.Azure.WebJobs.Script.WebHost.Diagnostics; -using Microsoft.Azure.WebJobs.Script.Workers.Rpc; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.WebJobs.Script.Tests; -using Moq; using Newtonsoft.Json.Linq; using Xunit; namespace Microsoft.Azure.WebJobs.Script.Tests { - public class FunctionInvokerBaseTests : IDisposable + public sealed class FunctionInvokerBaseTests : IDisposable { - private MockInvoker _invoker; - private IHost _host; - private ScriptHost _scriptHost; - private TestMetricsLogger _metricsLogger; - private TestLoggerProvider _testLoggerProvider; + private readonly TestMetricsLogger _metricsLogger = new(); + private readonly TestLoggerProvider _testLoggerProvider = new(); + private readonly MockInvoker _invoker; + private readonly IHost _host; + private readonly ScriptHost _scriptHost; public FunctionInvokerBaseTests() { - _metricsLogger = new TestMetricsLogger(); - _testLoggerProvider = new TestLoggerProvider(); - - ILoggerFactory loggerFactory = new LoggerFactory(); + LoggerFactory loggerFactory = new(); loggerFactory.AddProvider(_testLoggerProvider); - var eventManager = new ScriptEventManager(); - - var metadata = new FunctionMetadata + FunctionMetadata metadata = new() { Name = "TestFunction", ScriptFile = "index.js", Language = "node" }; + JObject binding = JObject.FromObject(new { type = "manualTrigger", name = "manual", direction = "in" }); + metadata.Bindings.Add(BindingMetadata.Create(binding)); - var metadataManager = new MockMetadataManager(new[] { metadata }); + MockMetadataManager metadataManager = new([metadata]); + + // TODO: Can we instantiate a ScriptHost directly? _host = new HostBuilder() .ConfigureDefaultTestWebScriptHost() .ConfigureServices(s => @@ -66,10 +62,7 @@ public FunctionInvokerBaseTests() .Build(); _scriptHost = _host.GetScriptHost(); - _scriptHost.InitializeAsync().Wait(); - var hostMetrics = _host.Services.GetService(); - _invoker = new MockInvoker(_scriptHost, _metricsLogger, hostMetrics, metadataManager, metadata, loggerFactory); } @@ -182,17 +175,16 @@ await Assert.ThrowsAsync(async () => Assert.Equal(startLatencyEvent, completedLatencyEvent); } - protected virtual void Dispose(bool disposing) + public void Dispose() { - if (disposing) + try { _host?.Dispose(); } - } - - public void Dispose() - { - Dispose(true); + catch (Exception) + { + // this might throw due to invalid setup. + } } private class MockInvoker : FunctionInvokerBase diff --git a/test/WebJobs.Script.Tests/Extensions/EnvironmentExtensionsTests.cs b/test/WebJobs.Script.Tests/Extensions/EnvironmentExtensionsTests.cs index c7845ef521..88820ac239 100644 --- a/test/WebJobs.Script.Tests/Extensions/EnvironmentExtensionsTests.cs +++ b/test/WebJobs.Script.Tests/Extensions/EnvironmentExtensionsTests.cs @@ -14,16 +14,24 @@ namespace Microsoft.Azure.WebJobs.Script.Tests.Extensions public class EnvironmentExtensionsTests { [Fact] - public void GetEffectiveCoresCount_ReturnsExpectedResult() + public void GetEffectiveCoresCount_NoSku_ReturnsExpectedResult() { - TestEnvironment env = new TestEnvironment(); + TestEnvironment env = new(); Assert.Equal(Environment.ProcessorCount, EnvironmentExtensions.GetEffectiveCoresCount(env)); + } - env.Clear(); + [Fact] + public void GetEffectiveCoresCount_DynamicSku_ReturnsExpectedResult() + { + TestEnvironment env = new(); env.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteSku, ScriptConstants.DynamicSku); Assert.Equal(1, EnvironmentExtensions.GetEffectiveCoresCount(env)); + } - env.Clear(); + [Fact] + public void GetEffectiveCoresCount_DynamicSkuWithInstanceId_ReturnsExpectedResult() + { + TestEnvironment env = new(); env.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteSku, ScriptConstants.DynamicSku); env.SetEnvironmentVariable(EnvironmentSettingNames.RoleInstanceId, "dw0SmallDedicatedWebWorkerRole_hr0HostRole-0-VM-1"); Assert.Equal(Environment.ProcessorCount, EnvironmentExtensions.GetEffectiveCoresCount(env)); @@ -33,7 +41,7 @@ public void GetEffectiveCoresCount_ReturnsExpectedResult() [Trait(TestTraits.Group, TestTraits.AdminIsolationTests)] public void IsAdminIsolationEnabled_ReturnsExpectedResult() { - TestEnvironment env = new TestEnvironment(); + TestEnvironment env = new(); Assert.False(EnvironmentExtensions.IsAdminIsolationEnabled(env)); env.SetEnvironmentVariable(EnvironmentSettingNames.FunctionsAdminIsolationEnabled, "0"); @@ -47,9 +55,9 @@ public void IsAdminIsolationEnabled_ReturnsExpectedResult() [InlineData("dw0SmallDedicatedWebWorkerRole_hr0HostRole-0-VM-1", true)] [InlineData(null, false)] [InlineData("", false)] - public void IsVMSS_RetrunsExpectedResult(string roleInstanceId, bool expected) + public void IsVMSS_ReturnsExpectedResult(string roleInstanceId, bool expected) { - IEnvironment env = new TestEnvironment(); + TestEnvironment env = new(); if (roleInstanceId != null) { env.SetEnvironmentVariable(EnvironmentSettingNames.RoleInstanceId, roleInstanceId); @@ -67,7 +75,7 @@ public void IsVMSS_RetrunsExpectedResult(string roleInstanceId, bool expected) [InlineData("Foo,Bar", false)] public void IsAzureMonitorEnabled_ReturnsExpectedResult(string value, bool expected) { - IEnvironment env = new TestEnvironment(); + TestEnvironment env = new(); if (value != null) { env.SetEnvironmentVariable(EnvironmentSettingNames.AzureMonitorCategories, value); @@ -82,7 +90,7 @@ public void IsAzureMonitorEnabled_ReturnsExpectedResult(string value, bool expec [InlineData(null, "")] public void GetAntaresComputerName_ReturnsExpectedResult(string computerName, string expectedComputerName) { - IEnvironment env = new TestEnvironment(); + TestEnvironment env = new(); if (!string.IsNullOrEmpty(expectedComputerName)) { env.SetEnvironmentVariable(EnvironmentSettingNames.AntaresComputerName, computerName); @@ -102,7 +110,7 @@ public void GetAntaresComputerName_ReturnsExpectedResult(string computerName, st [InlineData(false, "", null, "")] public void GetInstanceId_ReturnsExpectedResult(bool isLinuxConsumption, string containerName, string websiteInstanceId, string expectedValue) { - IEnvironment env = new TestEnvironment(); + TestEnvironment env = new(); if (isLinuxConsumption) { if (!string.IsNullOrEmpty(containerName)) @@ -133,7 +141,7 @@ public void GetInstanceId_ReturnsExpectedResult(bool isLinuxConsumption, string [InlineData(false, false, "89.0.7.73", null, "")] public void GetAntaresVersion_ReturnsExpectedResult(bool isLinuxConsumption, bool isLinuxAppService, string platformVersionLinux, string platformVersionWindows, string expectedValue) { - IEnvironment env = new TestEnvironment(); + TestEnvironment env = new(); if (isLinuxConsumption) { env.SetEnvironmentVariable(EnvironmentSettingNames.ContainerName, "RandomContainerName"); @@ -166,7 +174,7 @@ public void GetAntaresVersion_ReturnsExpectedResult(bool isLinuxConsumption, boo [InlineData(true, true, false, true)] public void IsConsumptionSku_ReturnsExpectedResult(bool isLinuxConsumption, bool isWindowsConsumption, bool isFlexConsumption, bool expectedValue) { - IEnvironment env = new TestEnvironment(); + TestEnvironment env = new(); if (isLinuxConsumption) { env.SetEnvironmentVariable(EnvironmentSettingNames.ContainerName, "RandomContainerName"); @@ -197,7 +205,7 @@ public void IsConsumptionSku_ReturnsExpectedResult(bool isLinuxConsumption, bool [InlineData("", "", "", "", false)] public void IsFlexConsumptionSku_ReturnsExpectedResult(string sku, string websiteInstanceId, string containerName, string legionServiceHost, bool expected) { - IEnvironment env = new TestEnvironment(); + TestEnvironment env = new(); env.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteSku, sku); env.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteInstanceId, websiteInstanceId); env.SetEnvironmentVariable(EnvironmentSettingNames.ContainerName, containerName); @@ -218,7 +226,7 @@ public void IsFlexConsumptionSku_ReturnsExpectedResult(string sku, string websit [InlineData("RandomContainerName", null, null, true, null, false)] // Managed App Environment public void IsAnyLinuxConsumption_ReturnsExpectedResult(string containerName, string podName, string legionServiceHostName, bool isManagedAppEnvironment, string sku, bool expectedValue) { - IEnvironment env = new TestEnvironment(); + TestEnvironment env = new(); if (!string.IsNullOrEmpty(containerName)) { env.SetEnvironmentVariable(ContainerName, containerName); @@ -253,7 +261,7 @@ public void IsAnyLinuxConsumption_ReturnsExpectedResult(string containerName, st [InlineData(false, false, false)] public void IsAnyKubernetesEnvironment_ReturnsExpectedResult(bool isKubernetesManagedHosting, bool isManagedAppEnvironment, bool expectedValue) { - IEnvironment env = new TestEnvironment(); + TestEnvironment env = new(); if (isKubernetesManagedHosting) { env.SetEnvironmentVariable(KubernetesServiceHost, "10.0.0.1"); @@ -273,7 +281,7 @@ public void IsAnyKubernetesEnvironment_ReturnsExpectedResult(bool isKubernetesMa [InlineData(false, false)] public void IsManagedAppEnvironment_ReturnsExpectedResult(bool isManagedAppEnvironment, bool expectedValue) { - IEnvironment env = new TestEnvironment(); + TestEnvironment env = new(); if (isManagedAppEnvironment) { env.SetEnvironmentVariable(EnvironmentSettingNames.ManagedEnvironment, "true"); @@ -295,7 +303,7 @@ public void IsManagedAppEnvironment_ReturnsExpectedResult(bool isManagedAppEnvir [InlineData("RandomPodName", "RandomLegionServiceHostName", ScriptConstants.DynamicSku, null, true)] public void IsLinuxConsumptionOnLegion_ReturnsExpectedResult(string websitePodName, string legionServiceHostName, string websiteSku, string websiteSkuName, bool expectedValue) { - IEnvironment env = new TestEnvironment(); + TestEnvironment env = new(); if (!string.IsNullOrEmpty(websitePodName)) { @@ -330,7 +338,7 @@ public void IsLinuxConsumptionOnLegion_ReturnsExpectedResult(string websitePodNa [InlineData(null, null, false)] public void IsV2CompatMode(string extensionVersion, string compatMode, bool expected) { - IEnvironment env = new TestEnvironment(); + TestEnvironment env = new(); if (extensionVersion != null) { @@ -353,7 +361,7 @@ public void IsV2CompatMode(string extensionVersion, string compatMode, bool expe [InlineData("k8se-apps", "10.0.0.1", true, false)] public void IsKubernetesManagedHosting_ReturnsExpectedResult(string podNamespace, string kubernetesServiceHost, bool isManagedAppEnvironment, bool expected) { - var environment = new TestEnvironment(); + TestEnvironment environment = new(); environment.SetEnvironmentVariable(KubernetesServiceHost, kubernetesServiceHost); environment.SetEnvironmentVariable(PodNamespace, podNamespace); if (isManagedAppEnvironment) @@ -374,7 +382,7 @@ public void IsKubernetesManagedHosting_ReturnsExpectedResult(string podNamespace [InlineData("", "")] public void Returns_WorkerRuntime(string workerRuntime, string expectedWorkerRuntime) { - var environment = new TestEnvironment(); + TestEnvironment environment = new(); environment.SetEnvironmentVariable(FunctionWorkerRuntime, workerRuntime); Assert.Equal(expectedWorkerRuntime, environment.GetFunctionsWorkerRuntime()); } @@ -388,9 +396,9 @@ public void Returns_WorkerRuntime(string workerRuntime, string expectedWorkerRun [InlineData("", "")] public void Returns_FunctionsExtensionVersion(string functionsExtensionVersion, string functionsExtensionVersionExpected) { - var enviroment = new TestEnvironment(); - enviroment.SetEnvironmentVariable(FunctionsExtensionVersion, functionsExtensionVersion); - Assert.Equal(functionsExtensionVersionExpected, enviroment.GetFunctionsExtensionVersion()); + TestEnvironment environment = new(); + environment.SetEnvironmentVariable(FunctionsExtensionVersion, functionsExtensionVersion); + Assert.Equal(functionsExtensionVersionExpected, environment.GetFunctionsExtensionVersion()); } [Theory] @@ -407,7 +415,7 @@ public void Returns_FunctionsExtensionVersion(string functionsExtensionVersion, [InlineData("", false, false, false)] public void Returns_SupportsAzureFileShareMount(string workerRuntime, bool useLowerCase, bool useUpperCase, bool supportsAzureFileShareMount) { - var environment = new TestEnvironment(); + TestEnvironment environment = new(); if (useLowerCase && !useUpperCase) { workerRuntime = workerRuntime.ToLowerInvariant(); @@ -427,7 +435,7 @@ public void Returns_SupportsAzureFileShareMount(string workerRuntime, bool useLo [InlineData("", "")] public void Returns_GetHttpLeaderEndpoint(string httpLeaderEndpoint, string expected) { - var environment = new TestEnvironment(); + TestEnvironment environment = new(); if (!string.IsNullOrEmpty(httpLeaderEndpoint)) { @@ -448,7 +456,7 @@ public void Returns_GetHttpLeaderEndpoint(string httpLeaderEndpoint, string expe [InlineData("10.0.0.1", null, true)] public void IsDrainOnApplicationStopping_ReturnsExpectedResult(string serviceHostValue, string drainOnStoppingValue, bool expected) { - var environment = new TestEnvironment(); + TestEnvironment environment = new(); environment.SetEnvironmentVariable(KubernetesServiceHost, serviceHostValue); environment.SetEnvironmentVariable(DrainOnApplicationStopping, drainOnStoppingValue); Assert.Equal(expected, environment.DrainOnApplicationStoppingEnabled()); @@ -461,7 +469,7 @@ public void IsDrainOnApplicationStopping_ReturnsExpectedResult(string serviceHos [InlineData("true", null, true)] public void IsWorkerDynamicConcurrencyEnabled_ReturnsExpectedResult(string concurrencyEnabledValue, string processCountValue, bool expected) { - var environment = new TestEnvironment(); + TestEnvironment environment = new(); environment.SetEnvironmentVariable(RpcWorkerConstants.FunctionsWorkerDynamicConcurrencyEnabled, concurrencyEnabledValue); environment.SetEnvironmentVariable(RpcWorkerConstants.FunctionsWorkerProcessCountSettingName, processCountValue); Assert.Equal(expected, environment.IsWorkerDynamicConcurrencyEnabled()); @@ -476,7 +484,7 @@ public void IsWorkerDynamicConcurrencyEnabled_ReturnsExpectedResult(string concu [InlineData("node", "", "node")] public void GetLanguageWorkerListToStartInPlaceholder_ReturnsExpectedResult(string workerRuntime, string workerRuntimeList, string expected) { - var environment = new TestEnvironment(); + TestEnvironment environment = new(); environment.SetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime, workerRuntime); environment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerPlaceholderModeListSettingName, workerRuntimeList); var resultSet = environment.GetLanguageWorkerListToStartInPlaceholder(); @@ -496,7 +504,7 @@ public void GetLanguageWorkerListToStartInPlaceholder_ReturnsExpectedResult(stri [InlineData("test", "test", true)] public void AzureFilesAppSettingsExist_ReturnsExpectedResult(string connectionString, string contentShare, bool expected) { - var environment = new TestEnvironment(); + TestEnvironment environment = new(); environment.SetEnvironmentVariable(AzureFilesConnectionString, connectionString); environment.SetEnvironmentVariable(AzureFilesContentShare, contentShare); Assert.Equal(expected, environment.AzureFilesAppSettingsExist()); @@ -512,7 +520,7 @@ public void AzureFilesAppSettingsExist_ReturnsExpectedResult(string connectionSt [InlineData("node", false)] public void IsInProc_ReturnsExpectedResult(string value, bool expected) { - var environment = new TestEnvironment(); + TestEnvironment environment = new(); if (value != null) { environment.SetEnvironmentVariable(FunctionWorkerRuntime, value); @@ -530,7 +538,7 @@ public void IsInProc_ReturnsExpectedResult(string value, bool expected) [InlineData("node", false)] public void IsInProc_WithRuntimeParameter_ReturnsExpectedResult(string value, bool expected) { - var environment = new TestEnvironment(); + TestEnvironment environment = new(); Assert.Equal(expected, environment.IsInProc(value)); } From 0dbd8e27b4c3ed176ee380d28fb7110b3c4db9ba Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Wed, 14 May 2025 15:53:05 -0700 Subject: [PATCH 08/16] Update new test method --- .../ScriptStartupTypeDiscovererTests.cs | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/test/WebJobs.Script.Tests/ScriptStartupTypeDiscovererTests.cs b/test/WebJobs.Script.Tests/ScriptStartupTypeDiscovererTests.cs index fcad06db4b..343d331e4a 100644 --- a/test/WebJobs.Script.Tests/ScriptStartupTypeDiscovererTests.cs +++ b/test/WebJobs.Script.Tests/ScriptStartupTypeDiscovererTests.cs @@ -504,12 +504,16 @@ public async Task GetExtensionsStartupTypes_BundlesNotConfiguredBindingsConfigur [Fact] public async Task GetExtensionsStartupTypes_NoBindings_In_ExtensionJson() { - ExtensionInstall blobs = new("AzureStorageBlobs", typeof(AzureStorageWebJobsStartup)); - string binPath = InstallExtensions(ExtensionInstall.Storage(true), blobs); + ExtensionInstall storage1 = new("AzureStorageBlobs", typeof(AzureStorageWebJobsStartup)); + ExtensionInstall storage2 = new("Storage", typeof(AzureStorageWebJobsStartup)) + { + HintPath = "Microsoft.Azure.WebJobs.Extensions.Storage.dll" + }; + string binPath = InstallExtensions(storage1, storage2); _bundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(true); - _bundleManager.Setup(e => e.GetExtensionBundleDetails()).Returns(Task.FromResult(new ExtensionBundleDetails() { Id = "bundleID", Version = "1.0.0" })); - _bundleManager.Setup(e => e.GetExtensionBundleBinPathAsync()).Returns(Task.FromResult(binPath)); + _bundleManager.Setup(e => e.GetExtensionBundleDetails()).ReturnsAsync(new ExtensionBundleDetails() { Id = "bundleID", Version = "1.0.0" }); + _bundleManager.Setup(e => e.GetExtensionBundleBinPathAsync()).ReturnsAsync(binPath); // Act ScriptStartupTypeLocator discoverer = CreateSystemUnderTest(); @@ -697,6 +701,8 @@ private bool AreExpectedMetricsGenerated() private class ExtensionInstall(string name, Type startupType, params string[] bindings) { + public string HintPath { get; init; } + public static ExtensionInstall Storage(bool includeBinding = false) { string[] bindings = includeBinding ? ["blob"] : []; @@ -721,7 +727,12 @@ public static ExtensionInstall Http() public ExtensionReference GetReference() { - ExtensionReference reference = new() { Name = name, TypeName = startupType.AssemblyQualifiedName }; + ExtensionReference reference = new() + { + Name = name, + TypeName = startupType.AssemblyQualifiedName, + HintPath = HintPath, + }; foreach (string binding in bindings ?? Enumerable.Empty()) { reference.Bindings.Add(binding); From bb16e3a23d1b6419cfc25c6eb4c32b382f128a82 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Mon, 19 May 2025 11:43:14 -0700 Subject: [PATCH 09/16] Update release notes --- release_notes.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/release_notes.md b/release_notes.md index 1dfa7e41cd..de2ee6e324 100644 --- a/release_notes.md +++ b/release_notes.md @@ -5,3 +5,6 @@ --> - Update Python Worker Version to [4.40.2](https://github.com/Azure/azure-functions-python-worker/releases/tag/4.40.2) - Add JitTrace Files for v4.1044 +- Memory allocation optimizations in `ScriptStartupTypeLocator.GetExtensionsStartupTypesAsync` (#11012) +- Fix invocation timeout when incoming request contains "x-ms-invocation-id" header (#10980) +- Throw exception instead of timing out when worker channel exits before initializing gRPC (#10937) From d9d1cd6be17dc4ae33ad30c394ab8650f25ed6db Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Fri, 18 Jul 2025 10:05:36 -0700 Subject: [PATCH 10/16] Swallow Publish event exceptions --- .../Workers/Rpc/RpcWorkerProcess.cs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/WebJobs.Script/Workers/Rpc/RpcWorkerProcess.cs b/src/WebJobs.Script/Workers/Rpc/RpcWorkerProcess.cs index d1398681e6..189cf58341 100644 --- a/src/WebJobs.Script/Workers/Rpc/RpcWorkerProcess.cs +++ b/src/WebJobs.Script/Workers/Rpc/RpcWorkerProcess.cs @@ -83,13 +83,25 @@ internal override void HandleWorkerProcessExitError(WorkerProcessExitException r // The subscriber of WorkerErrorEvent is expected to Dispose() the errored channel _workerProcessLogger.LogError(rpcWorkerProcessExitException, $"Language Worker Process exited. Pid={rpcWorkerProcessExitException.Pid}.", _workerProcessArguments?.ExecutablePath); - _eventManager.Publish(new WorkerErrorEvent(_runtime, _workerId, rpcWorkerProcessExitException)); + PublishNoThrow(new WorkerErrorEvent(_runtime, _workerId, rpcWorkerProcessExitException)); } internal override void HandleWorkerProcessRestart() { _workerProcessLogger?.LogInformation("Language Worker Process exited and needs to be restarted."); - _eventManager.Publish(new WorkerRestartEvent(_runtime, _workerId)); + PublishNoThrow(new WorkerRestartEvent(_runtime, _workerId)); + } + + private void PublishNoThrow(RpcChannelEvent @event) + { + try + { + _eventManager.Publish(@event); + } + catch (Exception ex) + { + _workerProcessLogger.LogWarning(ex, "Failed to publish RpcChannelEvent."); + } } } } From 16650da675f77578ba0ce201c63022d5c71a9040 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Wed, 1 Oct 2025 20:15:46 -0700 Subject: [PATCH 11/16] Fix test warning, add exit code --- src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs | 10 ++++++---- .../Workers/ProcessManagement/IWorkerProcess.cs | 2 +- .../Workers/ProcessManagement/WorkerProcess.cs | 9 +++++---- .../ScriptStartupTypeDiscovererTests.cs | 10 +++++----- .../Workers/Rpc/GrpcWorkerChannelTests.cs | 6 ++++-- 5 files changed, 21 insertions(+), 16 deletions(-) diff --git a/src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs b/src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs index 8052a23979..58b231c928 100644 --- a/src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs +++ b/src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs @@ -23,7 +23,6 @@ using Microsoft.Azure.WebJobs.Script.Diagnostics; using Microsoft.Azure.WebJobs.Script.Diagnostics.OpenTelemetry; using Microsoft.Azure.WebJobs.Script.Eventing; -using Microsoft.Azure.WebJobs.Script.Exceptions; using Microsoft.Azure.WebJobs.Script.Extensions; using Microsoft.Azure.WebJobs.Script.Grpc.Eventing; using Microsoft.Azure.WebJobs.Script.Grpc.Extensions; @@ -388,14 +387,17 @@ public async Task StartWorkerProcessAsync(CancellationToken cancellationToken) _workerChannelLogger.LogDebug("Initiating Worker Process start up"); await _rpcWorkerProcess.StartProcessAsync(cancellationToken); _state |= RpcWorkerChannelState.Initializing; - Task exited = _rpcWorkerProcess.WaitForExitAsync(cancellationToken); + Task exited = _rpcWorkerProcess.WaitForExitAsync(cancellationToken); Task winner = await Task.WhenAny(_workerInitTask.Task, exited).WaitAsync(cancellationToken); await winner; if (winner == exited) { - // process exited without throwing. We need to throw to indicate process is not running. - throw new WorkerProcessExitException("Worker process exited before initializing."); + // Process exited without throwing. We need to throw to indicate process is not running. + throw new WorkerProcessExitException("Worker process exited before initializing.") + { + ExitCode = await exited, + }; } } diff --git a/src/WebJobs.Script/Workers/ProcessManagement/IWorkerProcess.cs b/src/WebJobs.Script/Workers/ProcessManagement/IWorkerProcess.cs index a39dad9cf5..27f6cdf994 100644 --- a/src/WebJobs.Script/Workers/ProcessManagement/IWorkerProcess.cs +++ b/src/WebJobs.Script/Workers/ProcessManagement/IWorkerProcess.cs @@ -15,7 +15,7 @@ public interface IWorkerProcess Task StartProcessAsync(CancellationToken cancellationToken = default); - Task WaitForExitAsync(CancellationToken cancellationToken = default); + Task WaitForExitAsync(CancellationToken cancellationToken = default); void WaitForProcessExitInMilliSeconds(int waitTime); } diff --git a/src/WebJobs.Script/Workers/ProcessManagement/WorkerProcess.cs b/src/WebJobs.Script/Workers/ProcessManagement/WorkerProcess.cs index 139055c916..b535e9761f 100644 --- a/src/WebJobs.Script/Workers/ProcessManagement/WorkerProcess.cs +++ b/src/WebJobs.Script/Workers/ProcessManagement/WorkerProcess.cs @@ -38,7 +38,7 @@ internal abstract class WorkerProcess : IWorkerProcess, IDisposable private readonly bool _useStdErrorStreamForErrorsOnly; private Queue _processStdErrDataQueue = new(3); private IHostProcessMonitor _processMonitor; - private TaskCompletionSource _processExit; // used to hold custom exceptions on non-success exit. + private TaskCompletionSource _processExit; // used to hold custom exceptions on non-success exit. internal WorkerProcess(IScriptEventManager eventManager, IProcessRegistry processRegistry, ILogger workerProcessLogger, IWorkerConsoleLogSource consoleLogSource, IMetricsLogger metricsLogger, IServiceProvider serviceProvider, ILoggerFactory loggerFactory, IEnvironment environment, IOptionsMonitor scriptApplicationHostOptions, bool useStdErrStreamForErrorsOnly = false) @@ -114,8 +114,9 @@ public Task StartProcessAsync(CancellationToken cancellationToken = default) } } - public Task WaitForExitAsync(CancellationToken cancellationToken = default) + public Task WaitForExitAsync(CancellationToken cancellationToken = default) { + ObjectDisposedException.ThrowIf(Disposing, this); if (_processExit is { } tcs) { // We use a TaskCompletionSource (and not Process.WaitForExitAsync) so we can propagate our custom exceptions. @@ -201,7 +202,7 @@ private void OnProcessExited(object sender, EventArgs e) } finally { - _processExit.TrySetResult(); + _processExit.TrySetResult(Process.ExitCode); UnregisterFromProcessMonitor(); Process.Close(); } @@ -375,7 +376,7 @@ private void AssignUserExecutePermissionsIfNotExists() return; } - UnixFileInfo fileInfo = new UnixFileInfo(filePath); + UnixFileInfo fileInfo = new(filePath); if (!fileInfo.FileAccessPermissions.HasFlag(FileAccessPermissions.UserExecute)) { _workerProcessLogger.LogDebug("Assigning execute permissions to file: {filePath}", filePath); diff --git a/test/WebJobs.Script.Tests/ScriptStartupTypeDiscovererTests.cs b/test/WebJobs.Script.Tests/ScriptStartupTypeDiscovererTests.cs index 343d331e4a..dd7f90c872 100644 --- a/test/WebJobs.Script.Tests/ScriptStartupTypeDiscovererTests.cs +++ b/test/WebJobs.Script.Tests/ScriptStartupTypeDiscovererTests.cs @@ -616,6 +616,11 @@ public async Task GetExtensionsStartupTypes_EmptyExtensionsArray() AssertNoErrors(traces); } + private static void AssertNoErrors(IList traces) + { + Assert.False(traces.Any(m => m.Level == LogLevel.Error || m.Level == LogLevel.Critical)); + } + private static ExtensionBundleDetails GetBundleDetails(string version = "2.7.0") { return new ExtensionBundleDetails @@ -751,10 +756,5 @@ public void CopyTo(string path) } } } - - private static void AssertNoErrors(IList traces) - { - Assert.False(traces.Any(m => m.Level == LogLevel.Error || m.Level == LogLevel.Critical)); - } } } diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/GrpcWorkerChannelTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/GrpcWorkerChannelTests.cs index adcb8ea576..1b8211f045 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/GrpcWorkerChannelTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/GrpcWorkerChannelTests.cs @@ -76,7 +76,9 @@ public GrpcWorkerChannelTests(ITestOutputHelper testOutput) _testWorkerConfig.CountOptions.EnvironmentReloadTimeout = TimeSpan.FromSeconds(5); _mockRpcWorkerProcess.Setup(m => m.StartProcessAsync(It.IsAny())).Returns(Task.CompletedTask); - _mockRpcWorkerProcess.Setup(m => m.WaitForExitAsync(It.IsAny())).Returns(Task.Delay(Timeout.Infinite)); + + TaskCompletionSource tcs = new(); + _mockRpcWorkerProcess.Setup(m => m.WaitForExitAsync(It.IsAny())).Returns(tcs.Task); _mockRpcWorkerProcess.Setup(m => m.Id).Returns(910); _testEnvironment = new TestEnvironment(); _testEnvironment.SetEnvironmentVariable(FunctionDataCacheConstants.FunctionDataCacheEnabledSettingName, "1"); @@ -185,7 +187,7 @@ await Assert.ThrowsAsync(async () => public async Task StartWorkerProcessAsync_ProcessExits_Throws() { _mockRpcWorkerProcess.Setup(m => m.WaitForExitAsync(It.IsAny())) - .Returns(Task.CompletedTask); + .ReturnsAsync(0); await CreateDefaultWorkerChannel(autoStart: false); WorkerProcessExitException ex = await Assert.ThrowsAsync( From a816a7fcdd97d5058e090e9be48019b29b02418c Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Thu, 2 Oct 2025 12:37:21 -0700 Subject: [PATCH 12/16] Ensure test process is cleaned up --- .../Workers/Rpc/RpcWorkerProcessTests.cs | 40 ++++++++++++++----- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerProcessTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerProcessTests.cs index 88599e1e02..81441d9a78 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerProcessTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerProcessTests.cs @@ -272,10 +272,11 @@ public async Task WorkerProcess_WaitForExit_NotStarted_Throws() public async Task WorkerProcess_WaitForExit_Success_TaskCompletes() { // arrange - using Process process = GetProcess(exitCode: 0); - _hostProcessMonitorMock.Setup(m => m.RegisterChildProcess(process)); - _hostProcessMonitorMock.Setup(m => m.UnregisterChildProcess(process)); - _workerProcessFactory.Setup(m => m.CreateWorkerProcess(It.IsNotNull())).Returns(process); + await using ProcessWrapper wrapper = new(exitCode: 0); + _hostProcessMonitorMock.Setup(m => m.RegisterChildProcess(wrapper.Process)); + _hostProcessMonitorMock.Setup(m => m.UnregisterChildProcess(wrapper.Process)); + _workerProcessFactory.Setup(m => m.CreateWorkerProcess(It.IsNotNull())) + .Returns(wrapper.Process); using var rpcWorkerProcess = GetRpcWorkerConfigProcess( TestHelpers.GetTestWorkerConfigsWithExecutableWorkingDirectory().ElementAt(0)); @@ -290,10 +291,11 @@ public async Task WorkerProcess_WaitForExit_Success_TaskCompletes() public async Task WorkerProcess_WaitForExit_Error_Rethrows() { // arrange - using Process process = GetProcess(exitCode: -1); - _hostProcessMonitorMock.Setup(m => m.RegisterChildProcess(process)); - _hostProcessMonitorMock.Setup(m => m.UnregisterChildProcess(process)); - _workerProcessFactory.Setup(m => m.CreateWorkerProcess(It.IsNotNull())).Returns(process); + await using ProcessWrapper wrapper = new(exitCode: -1); + _hostProcessMonitorMock.Setup(m => m.RegisterChildProcess(wrapper.Process)); + _hostProcessMonitorMock.Setup(m => m.UnregisterChildProcess(wrapper.Process)); + _workerProcessFactory.Setup(m => m.CreateWorkerProcess(It.IsNotNull())) + .Returns(wrapper.Process); using var rpcWorkerProcess = GetRpcWorkerConfigProcess( TestHelpers.GetTestWorkerConfigsWithExecutableWorkingDirectory().ElementAt(0)); @@ -310,13 +312,33 @@ private static Process GetProcess(int exitCode) { StartInfo = new() { - WindowStyle = ProcessWindowStyle.Hidden, FileName = OperatingSystem.IsWindows() ? "cmd" : "bash", Arguments = OperatingSystem.IsWindows() ? $"/C exit {exitCode}" : $"-c \"exit {exitCode}\"", RedirectStandardError = true, RedirectStandardOutput = true, + CreateNoWindow = true, + ErrorDialog = false, + UseShellExecute = false } }; } + + private class ProcessWrapper(int exitCode) : IAsyncDisposable + { + public Process Process { get; } = GetProcess(exitCode); + + public async ValueTask DisposeAsync() + { + if (!Process.HasExited) + { + // We need to kill the entire process tree to ensure + // CI tests don't hang due to child processes lingering around. + Process.Kill(entireProcessTree: true); + await Process.WaitForExitAsync(); + } + + Process.Dispose(); + } + } } } From 562d5daa9691bb514eb44d2088f113156f8f391a Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Thu, 2 Oct 2025 13:23:24 -0700 Subject: [PATCH 13/16] Revert "Ensure test process is cleaned up" This reverts commit a816a7fcdd97d5058e090e9be48019b29b02418c. --- .../Workers/Rpc/RpcWorkerProcessTests.cs | 40 +++++-------------- 1 file changed, 9 insertions(+), 31 deletions(-) diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerProcessTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerProcessTests.cs index 81441d9a78..88599e1e02 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerProcessTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerProcessTests.cs @@ -272,11 +272,10 @@ public async Task WorkerProcess_WaitForExit_NotStarted_Throws() public async Task WorkerProcess_WaitForExit_Success_TaskCompletes() { // arrange - await using ProcessWrapper wrapper = new(exitCode: 0); - _hostProcessMonitorMock.Setup(m => m.RegisterChildProcess(wrapper.Process)); - _hostProcessMonitorMock.Setup(m => m.UnregisterChildProcess(wrapper.Process)); - _workerProcessFactory.Setup(m => m.CreateWorkerProcess(It.IsNotNull())) - .Returns(wrapper.Process); + using Process process = GetProcess(exitCode: 0); + _hostProcessMonitorMock.Setup(m => m.RegisterChildProcess(process)); + _hostProcessMonitorMock.Setup(m => m.UnregisterChildProcess(process)); + _workerProcessFactory.Setup(m => m.CreateWorkerProcess(It.IsNotNull())).Returns(process); using var rpcWorkerProcess = GetRpcWorkerConfigProcess( TestHelpers.GetTestWorkerConfigsWithExecutableWorkingDirectory().ElementAt(0)); @@ -291,11 +290,10 @@ public async Task WorkerProcess_WaitForExit_Success_TaskCompletes() public async Task WorkerProcess_WaitForExit_Error_Rethrows() { // arrange - await using ProcessWrapper wrapper = new(exitCode: -1); - _hostProcessMonitorMock.Setup(m => m.RegisterChildProcess(wrapper.Process)); - _hostProcessMonitorMock.Setup(m => m.UnregisterChildProcess(wrapper.Process)); - _workerProcessFactory.Setup(m => m.CreateWorkerProcess(It.IsNotNull())) - .Returns(wrapper.Process); + using Process process = GetProcess(exitCode: -1); + _hostProcessMonitorMock.Setup(m => m.RegisterChildProcess(process)); + _hostProcessMonitorMock.Setup(m => m.UnregisterChildProcess(process)); + _workerProcessFactory.Setup(m => m.CreateWorkerProcess(It.IsNotNull())).Returns(process); using var rpcWorkerProcess = GetRpcWorkerConfigProcess( TestHelpers.GetTestWorkerConfigsWithExecutableWorkingDirectory().ElementAt(0)); @@ -312,33 +310,13 @@ private static Process GetProcess(int exitCode) { StartInfo = new() { + WindowStyle = ProcessWindowStyle.Hidden, FileName = OperatingSystem.IsWindows() ? "cmd" : "bash", Arguments = OperatingSystem.IsWindows() ? $"/C exit {exitCode}" : $"-c \"exit {exitCode}\"", RedirectStandardError = true, RedirectStandardOutput = true, - CreateNoWindow = true, - ErrorDialog = false, - UseShellExecute = false } }; } - - private class ProcessWrapper(int exitCode) : IAsyncDisposable - { - public Process Process { get; } = GetProcess(exitCode); - - public async ValueTask DisposeAsync() - { - if (!Process.HasExited) - { - // We need to kill the entire process tree to ensure - // CI tests don't hang due to child processes lingering around. - Process.Kill(entireProcessTree: true); - await Process.WaitForExitAsync(); - } - - Process.Dispose(); - } - } } } From ba560ce77e218e58c7dd46d21976fe01181bee16 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Thu, 2 Oct 2025 13:42:09 -0700 Subject: [PATCH 14/16] Add crash blame for debugging --- eng/ci/templates/official/jobs/run-non-e2e-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/ci/templates/official/jobs/run-non-e2e-tests.yml b/eng/ci/templates/official/jobs/run-non-e2e-tests.yml index 6e4e6e93b8..afef27f22f 100644 --- a/eng/ci/templates/official/jobs/run-non-e2e-tests.yml +++ b/eng/ci/templates/official/jobs/run-non-e2e-tests.yml @@ -86,7 +86,7 @@ jobs: inputs: command: test testRunTitle: Non-E2E integration tests - arguments: '--filter "Category!=E2E" -c release --no-build' + arguments: '--filter "Category!=E2E" -c release --no-build --blame-crash' projects: $(test_projects) env: AzureWebJobsStorage: $(Storage) From de9970f998b3f5c24b4b68d8790e21e3ee7d97dd Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Thu, 2 Oct 2025 14:17:35 -0700 Subject: [PATCH 15/16] Only close process on success --- .../Workers/ProcessManagement/WorkerProcess.cs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/WebJobs.Script/Workers/ProcessManagement/WorkerProcess.cs b/src/WebJobs.Script/Workers/ProcessManagement/WorkerProcess.cs index b535e9761f..35d5fe02cc 100644 --- a/src/WebJobs.Script/Workers/ProcessManagement/WorkerProcess.cs +++ b/src/WebJobs.Script/Workers/ProcessManagement/WorkerProcess.cs @@ -179,12 +179,18 @@ private void OnProcessExited(object sender, EventArgs e) return; } + int exit = 0; try { ThrowIfExitError(); - Process.WaitForExit(); - if (Process.ExitCode == WorkerConstants.IntentionalRestartExitCode) + exit = Process.ExitCode; + if (Process.ExitCode == WorkerConstants.SuccessExitCode) + { + Process.WaitForExit(); + Process.Close(); + } + else if (Process.ExitCode == WorkerConstants.IntentionalRestartExitCode) { HandleWorkerProcessRestart(); } @@ -202,9 +208,8 @@ private void OnProcessExited(object sender, EventArgs e) } finally { - _processExit.TrySetResult(Process.ExitCode); + _processExit.TrySetResult(exit); UnregisterFromProcessMonitor(); - Process.Close(); } } From 003f830b3c7a9722b1d79318a64e99725e0e5df8 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Fri, 3 Oct 2025 14:47:36 -0700 Subject: [PATCH 16/16] Revert --blame-crash change --- eng/ci/templates/official/jobs/run-non-e2e-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/ci/templates/official/jobs/run-non-e2e-tests.yml b/eng/ci/templates/official/jobs/run-non-e2e-tests.yml index afef27f22f..6e4e6e93b8 100644 --- a/eng/ci/templates/official/jobs/run-non-e2e-tests.yml +++ b/eng/ci/templates/official/jobs/run-non-e2e-tests.yml @@ -86,7 +86,7 @@ jobs: inputs: command: test testRunTitle: Non-E2E integration tests - arguments: '--filter "Category!=E2E" -c release --no-build --blame-crash' + arguments: '--filter "Category!=E2E" -c release --no-build' projects: $(test_projects) env: AzureWebJobsStorage: $(Storage)