Skip to content

Commit

Permalink
For node20 fallback on host, check for glibc instead of OS version ch…
Browse files Browse the repository at this point in the history 
…eck (#4524)

* wip

* ci

* wip

* wip

* ci

* fix

* add telemetry for container "NeedsNode16Redirect"
reduce telemetry spam for host

* fix

* fix: only override container node when task targets node20

* use correct warning for containers

* update wording

* update telemetry

* only emit HostNode20to16Fallback once per ExecutionContext

* minor
  • Loading branch information
@merlynomsft
merlynomsft authored Nov 17, 2023
1 parent aaf9fcd commit 4c5dad6
Show file tree
Hide file tree
Showing 7 changed files with 263 additions and 140 deletions.
3 changes: 1 addition & 2 deletions src/Agent.Sdk/ContainerInfo.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,10 +90,9 @@ public ContainerInfo(Pipelines.ContainerResource container, Boolean isJobContain
public string CurrentUserId { get; set; }
public string CurrentGroupName { get; set; }
public string CurrentGroupId { get; set; }
public bool NeedsNode16Redirect { get; set; }

public bool IsJobContainer { get; set; }
public bool MapDockerSocket { get; set; }
public bool NeedsNode16Redirect { get; set; }
public PlatformUtil.OS ImageOS
{
get
Expand Down
45 changes: 0 additions & 45 deletions src/Agent.Sdk/Util/PlatformUtil.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,21 +102,6 @@ public static bool RunningOnRHEL6
}
}

public static bool RunningOnRHEL7
{
get
{
if (!(detectedRHEL7 is null))
{
return (bool)detectedRHEL7;
}

DetectRHEL7();

return (bool)detectedRHEL7;
}
}

public static string GetSystemId()
{
return PlatformUtil.HostOS switch
Expand Down Expand Up @@ -167,34 +152,6 @@ private static void DetectRHEL6()
}
}

private static void DetectRHEL7()
{
lock (detectedRHEL7lock)
{
if (!RunningOnLinux || !File.Exists("/etc/redhat-release"))
{
detectedRHEL7 = false;
}
else
{
detectedRHEL7 = false;
try
{
string redhatVersion = File.ReadAllText("/etc/redhat-release");
if (redhatVersion.StartsWith("CentOS release 7.")
|| redhatVersion.StartsWith("Red Hat Enterprise Linux Server release 7."))
{
detectedRHEL7 = true;
}
}
catch (IOException)
{
// IOException indicates we couldn't read that file; probably not RHEL7
}
}
}
}

private static string GetLinuxId()
{
if (RunningOnLinux && File.Exists("/etc/os-release"))
Expand Down Expand Up @@ -314,8 +271,6 @@ private static string GetWindowsVersion()

private static bool? detectedRHEL6 = null;
private static object detectedRHEL6lock = new object();
private static bool? detectedRHEL7 = null;
private static object detectedRHEL7lock = new object();

public static Architecture HostArchitecture
{
Expand Down
69 changes: 38 additions & 31 deletions src/Agent.Worker/ContainerOperationProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
using Microsoft.VisualStudio.Services.Agent.Util;
using Microsoft.VisualStudio.Services.Agent.Worker.Container;
using Microsoft.VisualStudio.Services.Agent.Worker.Handlers;
using Microsoft.VisualStudio.Services.Agent.Worker.Telemetry;
using Microsoft.VisualStudio.Services.Common;
using Microsoft.Win32;
using Newtonsoft.Json;
Expand Down Expand Up @@ -767,49 +768,37 @@ private async Task StartContainerAsync(IExecutionContext executionContext, Conta
}
}

bool useNode20InUnsupportedSystem = AgentKnobs.UseNode20InUnsupportedSystem.GetValue(executionContext).AsBoolean();

if(!useNode20InUnsupportedSystem)
if (PlatformUtil.RunningOnLinux)
{
var node20 = container.TranslateToContainerPath(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), NodeHandler.Node20_1Folder, "bin", $"node{IOUtil.ExeExtension}"));
bool useNode20InUnsupportedSystem = AgentKnobs.UseNode20InUnsupportedSystem.GetValue(executionContext).AsBoolean();

string node20TestCmd = $"bash -c \"{node20} -v\"";
List<string> nodeInfo = await DockerExec(executionContext, container.ContainerId, node20TestCmd, noExceptionOnError: true);
if (nodeInfo.Count > 0)
if (!useNode20InUnsupportedSystem)
{
foreach(var nodeInfoLine in nodeInfo)
var node20 = container.TranslateToContainerPath(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), NodeHandler.Node20_1Folder, "bin", $"node{IOUtil.ExeExtension}"));

string node20TestCmd = $"bash -c \"{node20} -v\"";
List<string> nodeVersionOutput = await DockerExec(executionContext, container.ContainerId, node20TestCmd, noExceptionOnError: true);

container.NeedsNode16Redirect = WorkerUtilities.IsCommandResultGlibcError(executionContext, nodeVersionOutput, out string nodeInfoLine);

if (container.NeedsNode16Redirect)
{
// detect example error from node 20 attempting to run on Ubuntu18:
// /__a/externals/node20/bin/node: /lib/x86_64-linux-gnu/libm.so.6: version `GLIBC_2.27' not found (required by /__a/externals/node20/bin/node)
// /__a/externals/node20/bin/node: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.28' not found (required by /__a/externals/node20/bin/node)
// /__a/externals/node20/bin/node: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.25' not found (required by /__a/externals/node20/bin/node)
if(nodeInfoLine.Contains("version `GLIBC_2.28' not found")
|| nodeInfoLine.Contains("version `GLIBC_2.25' not found")
|| nodeInfoLine.Contains("version `GLIBC_2.27' not found"))
{
executionContext.Debug($"GLIBC error found executing node -v; setting NeedsNode16Redirect: {nodeInfoLine}");
executionContext.Warning($"The container operating system doesn't support Node20. Using Node16 instead. " +
"Please upgrade the operating system of the container to ensure compatibility with Node20 tasks: " +
"https://github.com/nodesource/distributions");

container.NeedsNode16Redirect = true;
}
PublishTelemetry(
executionContext,
new Dictionary<string, string>
{
{ "ContainerNode20to16Fallback", container.NeedsNode16Redirect.ToString() }
}
);
}
}

}

if (!string.IsNullOrEmpty(containerUserName))
{
container.CurrentUserName = containerUserName;
}

if(!useNode20InUnsupportedSystem)
{
if(container.NeedsNode16Redirect)
{
container.CustomNodePath = container.TranslateToContainerPath(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), NodeHandler.Node16Folder, "bin", $"node{IOUtil.ExeExtension}"));
}
}
}
}
}
Expand Down Expand Up @@ -1039,5 +1028,23 @@ private static void ThrowIfWrongWindowsVersion(IExecutionContext executionContex
throw new ArgumentOutOfRangeException(@"HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion\ReleaseId");
}
}

private void PublishTelemetry(
IExecutionContext executionContext,
object telemetryData,
string feature = nameof(ContainerOperationProvider)
)
{
var cmd = new Command("telemetry", "publish")
{
Data = JsonConvert.SerializeObject(telemetryData, Formatting.None)
};
cmd.Properties.Add("area", "PipelinesTasks");
cmd.Properties.Add("feature", feature);

var publishTelemetryCmd = new TelemetryCommandExtension();
publishTelemetryCmd.Initialize(HostContext);
publishTelemetryCmd.ProcessCommand(executionContext, cmd);
}
}
}
44 changes: 44 additions & 0 deletions src/Agent.Worker/ExecutionContext.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
using Microsoft.TeamFoundation.DistributedTask.WebApi;
using Pipelines = Microsoft.TeamFoundation.DistributedTask.Pipelines;
using Microsoft.VisualStudio.Services.Agent.Util;
using Microsoft.VisualStudio.Services.Agent.Worker.Telemetry;
using Newtonsoft.Json;

namespace Microsoft.VisualStudio.Services.Agent.Worker
{
Expand Down Expand Up @@ -86,6 +88,7 @@ public interface IExecutionContext : IAgentService, IKnobValueContext
/// </summary>
/// <returns></returns>
void CancelForceTaskCompletion();
void EmitHostNode20FallbackTelemetry(bool node20ResultsInGlibCErrorHost);
}

public sealed class ExecutionContext : AgentService, IExecutionContext, IDisposable
Expand Down Expand Up @@ -118,6 +121,7 @@ public sealed class ExecutionContext : AgentService, IExecutionContext, IDisposa
private string _buildLogsFile;
private FileStream _buildLogsData;
private StreamWriter _buildLogsWriter;
private bool emittedHostNode20FallbackTelemetry = false;

// only job level ExecutionContext will track throttling delay.
private long _totalThrottlingDelayInMilliseconds = 0;
Expand Down Expand Up @@ -889,6 +893,46 @@ public void ReInitializeForceCompleted()
this._forceCompleteCancellationTokenSource = new CancellationTokenSource();
}

public void EmitHostNode20FallbackTelemetry(bool node20ResultsInGlibCErrorHost)
{
if (!emittedHostNode20FallbackTelemetry)
{
PublishTelemetry(new Dictionary<string, string>
{
{ "HostNode20to16Fallback", node20ResultsInGlibCErrorHost.ToString() }
});

emittedHostNode20FallbackTelemetry = true;
}
}

// This overload is to handle specific types some other way.
private void PublishTelemetry<T>(
Dictionary<string, T> telemetryData,
string feature = "TaskHandler"
)
{
// JsonConvert.SerializeObject always converts to base object.
PublishTelemetry((object)telemetryData, feature);
}

private void PublishTelemetry(
object telemetryData,
string feature = "TaskHandler"
)
{
var cmd = new Command("telemetry", "publish")
{
Data = JsonConvert.SerializeObject(telemetryData, Formatting.None)
};
cmd.Properties.Add("area", "PipelinesTasks");
cmd.Properties.Add("feature", feature);

var publishTelemetryCmd = new TelemetryCommandExtension();
publishTelemetryCmd.Initialize(HostContext);
publishTelemetryCmd.ProcessCommand(this, cmd);
}

public void Dispose()
{
_cancellationTokenSource?.Dispose();
Expand Down
Loading

0 comments on commit 4c5dad6

Please sign in to comment.