From 8f1fc2df16fdc195d7e64748b3bf0242c702411c Mon Sep 17 00:00:00 2001 From: jas88 Date: Thu, 10 Aug 2023 18:56:01 -0500 Subject: [PATCH] Redo Python handling, find file paths from Registry instead of guessing --- .../Python/Unit/Python2And3InstalledTests.cs | 5 +- .../DataProvider/PythonDataProvider.cs | 196 +++++++++--------- .../Properties/AssemblyInfo.cs | 4 +- 3 files changed, 98 insertions(+), 107 deletions(-) diff --git a/LoadModules.Extensions.Tests/Python/Unit/Python2And3InstalledTests.cs b/LoadModules.Extensions.Tests/Python/Unit/Python2And3InstalledTests.cs index 19fc6c1..5921d00 100644 --- a/LoadModules.Extensions.Tests/Python/Unit/Python2And3InstalledTests.cs +++ b/LoadModules.Extensions.Tests/Python/Unit/Python2And3InstalledTests.cs @@ -31,15 +31,14 @@ public void PythonScript_OverrideExecutablePath_VersionMismatch() provider.Check(new AcceptAllCheckNotifier());// version 3 should now be installed //version 3 executable path is explicit override for executing commands - provider.OverridePythonExecutablePath = new FileInfo(Path.Combine(provider.GetFullPythonInstallDirectory(), "python.exe")); + provider.OverridePythonExecutablePath = new FileInfo(provider.GetFullPythonPath()); provider.Version = PythonVersion.Version2; //so we now know that version 3 is installed, and we have overriden the python path to the .exe explicitly and we are trying to launch with Version2 enum now var ex = Assert.Throws(()=> { provider.Check(ThrowImmediatelyCheckNotifier.Quiet); - //provider.Fetch(MockRepository.GenerateStub(), new GracefulCancellationToken()); }); - StringAssert.Contains(@"which is incompatible with the desired version 2.7.1",ex?.Message); + StringAssert.Contains(@"which is incompatible with the desired version 2.7",ex?.Message); } } \ No newline at end of file diff --git a/Python/LoadModules.Extensions.Python/DataProvider/PythonDataProvider.cs b/Python/LoadModules.Extensions.Python/DataProvider/PythonDataProvider.cs index cbbed6d..7a5356b 100644 --- a/Python/LoadModules.Extensions.Python/DataProvider/PythonDataProvider.cs +++ b/Python/LoadModules.Extensions.Python/DataProvider/PythonDataProvider.cs @@ -3,19 +3,24 @@ using System.Diagnostics; using System.IO; using System.Linq; +using System.Runtime.Versioning; using System.Threading.Tasks; using FAnsi.Discovery; +using Microsoft.Win32; using Rdmp.Core.Curation; using Rdmp.Core.Curation.Data; using Rdmp.Core.DataFlowPipeline; using Rdmp.Core.DataLoad; using Rdmp.Core.DataLoad.Engine.DataProvider; using Rdmp.Core.DataLoad.Engine.Job; +using Rdmp.Core.ReusableLibraryCode.Annotations; using Rdmp.Core.ReusableLibraryCode.Checks; using Rdmp.Core.ReusableLibraryCode.Progress; namespace LoadModules.Extensions.Python.DataProvider; +#nullable enable + public enum PythonVersion { NotSet, @@ -23,26 +28,24 @@ public enum PythonVersion Version3 } -public class PythonDataProvider:IPluginDataProvider +public sealed class PythonDataProvider:IPluginDataProvider { - - [DemandsInitialization("The Python script to run")] - public string FullPathToPythonScriptToRun { get; set; } + public string? FullPathToPythonScriptToRun { get; set; } - [DemandsInitialization("The maximum number of seconds to allow the python script to run for before declaring it a failure, 0 for indefinetly")] + [DemandsInitialization("The maximum number of seconds to allow the python script to run for before declaring it a failure, 0 for indefinitely")] public int MaximumNumberOfSecondsToLetScriptRunFor { get; set; } - [DemandsInitialization("Python version required to run your script")] + [DemandsInitialization("Python version required to run your script")] public PythonVersion Version { get; set; } [DemandsInitialization("Override Python Executable Path")] - public FileInfo OverridePythonExecutablePath { get; set; } + public FileInfo? OverridePythonExecutablePath { get; set; } + - public void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventsListener) { - + } public void Check(ICheckNotifier notifier) @@ -60,14 +63,15 @@ public void Check(ICheckNotifier notifier) { var version = GetPythonVersion(); - if (version.StartsWith(GetExpectedPythonVersion())) + if (version?.StartsWith(GetExpectedPythonVersion(), StringComparison.Ordinal)==true) notifier.OnCheckPerformed( new CheckEventArgs( - $"Found Expected Python version {version} on the host machine at directory {GetFullPythonInstallDirectory()}", CheckResult.Success)); - else if (version.StartsWith(GetCompatiblePythonVersion())) + $"Found Expected Python version {version} on the host machine at {GetPython(Version == PythonVersion.Version2 ? '2' : '3').path}", CheckResult.Success)); + else if (version is not null && ((version[0] == '3' && Version == PythonVersion.Version3) || + (version[0] == '2' && Version == PythonVersion.Version2))) notifier.OnCheckPerformed( new CheckEventArgs( - $"Found Compatible Python version {version} on the host machine at directory {GetFullPythonInstallDirectory()}", CheckResult.Success)); + $"Found Compatible Python version {version} on the host machine at {GetPython(Version == PythonVersion.Version2 ? '2' : '3').path}", CheckResult.Success)); else { notifier.OnCheckPerformed( @@ -101,68 +105,51 @@ public void Check(ICheckNotifier notifier) CheckResult.Warning)); } - public string GetPythonVersion() + public string? GetPythonVersion() { - var info = GetPythonCommand(@"-c ""import sys; print(sys.version)"""); - + const string getVersion = """ + -c "import sys; print(sys.version)" + """; var toMemory = new ToMemoryDataLoadEventListener(true); + var result = ExecuteProcess(toMemory, getVersion, 600); - var result = ExecuteProcess(toMemory, info, 600); - if (result != 0) return null; - + var msg = toMemory.EventsReceivedBySender[this].SingleOrDefault(); if (msg != null) return msg.Message; - throw new Exception($"Call to {info.Arguments} did not return any value but exited with code {result}"); + throw new Exception($"Call to {getVersion} did not return any value but exited with code {result}"); } - private ProcessStartInfo GetPythonCommand(string command) + private string GetPythonCommand() { - string exeFullPath; - if (OverridePythonExecutablePath == null) - { - //e.g. c:\python34 - var installDir = GetFullPythonInstallDirectory(); - exeFullPath = Path.Combine(installDir, "python"); - } - else - { - if (!OverridePythonExecutablePath.Exists) - throw new FileNotFoundException( - $"The specified OverridePythonExecutablePath:{OverridePythonExecutablePath} does not exist"); - if(OverridePythonExecutablePath.Name != "python.exe") - throw new FileNotFoundException( - $"The specified OverridePythonExecutablePath:{OverridePythonExecutablePath} file is not called python.exe... what is going on here?"); - - exeFullPath = OverridePythonExecutablePath.FullName; - } + return GetPython(Version==PythonVersion.Version2?'2':'3').path; - var info = new ProcessStartInfo(exeFullPath) - { - Arguments = command - }; + if (!OverridePythonExecutablePath.Exists) + throw new FileNotFoundException( + $"The specified OverridePythonExecutablePath:{OverridePythonExecutablePath} does not exist"); + if(OverridePythonExecutablePath.Name != "python.exe") + throw new FileNotFoundException( + $"The specified OverridePythonExecutablePath:{OverridePythonExecutablePath} file is not called python.exe... what is going on here?"); - return info; + return OverridePythonExecutablePath.FullName; } public void Initialize(ILoadDirectory hicProjectDirectory, DiscoveredDatabase dbInfo) { - + } public ExitCodeType Fetch(IDataLoadJob job, GracefulCancellationToken cancellationToken) { - var processStartInfo = GetPythonCommand(FullPathToPythonScriptToRun); - int exitCode; try { - exitCode = ExecuteProcess(job, processStartInfo,MaximumNumberOfSecondsToLetScriptRunFor); + exitCode = ExecuteProcess(job, FullPathToPythonScriptToRun, MaximumNumberOfSecondsToLetScriptRunFor); } catch (TimeoutException e) { @@ -176,13 +163,17 @@ public ExitCodeType Fetch(IDataLoadJob job, GracefulCancellationToken cancellati return exitCode == 0 ? ExitCodeType.Success : ExitCodeType.Error; } - private int ExecuteProcess(IDataLoadEventListener listener, ProcessStartInfo processStartInfo, int maximumNumberOfSecondsToLetScriptRunFor) + private int ExecuteProcess(IDataLoadEventListener listener, string script, int maximumNumberOfSecondsToLetScriptRunFor) { - processStartInfo.RedirectStandardOutput = true; - processStartInfo.RedirectStandardError = true; - - processStartInfo.UseShellExecute = false; - processStartInfo.CreateNoWindow = true; + var processStartInfo = new ProcessStartInfo + { + FileName = GetPythonCommand(), + Arguments = script, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + }; Process p; @@ -197,7 +188,7 @@ private int ExecuteProcess(IDataLoadEventListener listener, ProcessStartInfo pro }; p.OutputDataReceived += (s, e) => allOutputDataConsumed = OutputDataReceived(e, listener,false); p.ErrorDataReceived += (s, e) => allErrorDataConsumed = OutputDataReceived(e, listener,true); - + p.Start(); p.BeginErrorReadLine(); p.BeginOutputReadLine(); @@ -208,11 +199,11 @@ private int ExecuteProcess(IDataLoadEventListener listener, ProcessStartInfo pro throw new Exception( $"Failed to launch:{Environment.NewLine}{processStartInfo.FileName}{Environment.NewLine} with Arguments:{processStartInfo.Arguments}",e); } - + // To avoid deadlocks, always read the output stream first and then wait. var startTime = DateTime.Now; - + while (!p.WaitForExit(100))//while process has not exited { if (!TimeoutExpired(startTime)) continue; //if timeout expired @@ -242,21 +233,20 @@ private int ExecuteProcess(IDataLoadEventListener listener, ProcessStartInfo pro lock(this) if (_outputDataReceivedExceptions.Any()) - if (_outputDataReceivedExceptions.Count == 1) - throw _outputDataReceivedExceptions[0]; - else - throw new AggregateException(_outputDataReceivedExceptions); + throw _outputDataReceivedExceptions.Count == 1 + ? _outputDataReceivedExceptions[0] + : new AggregateException(_outputDataReceivedExceptions); return p.ExitCode; } - readonly List _outputDataReceivedExceptions = new(); + private readonly List _outputDataReceivedExceptions = new(); private bool OutputDataReceived(DataReceivedEventArgs e, IDataLoadEventListener listener,bool isErrorStream) { if(e.Data == null) return true; - + lock (this) { try @@ -271,7 +261,7 @@ private bool OutputDataReceived(DataReceivedEventArgs e, IDataLoadEventListener return true; } } - + return false; } private bool TimeoutExpired(DateTime startTime) @@ -283,54 +273,56 @@ private bool TimeoutExpired(DateTime startTime) } - public string GetFullPythonInstallDirectory() + public string GetFullPythonPath() { - return Path.Combine(AppDomain.CurrentDomain.BaseDirectory, GetPythonFolderName()); + return GetPython(Version == PythonVersion.Version2 ? '2' : '3').path; } - private string GetPythonFolderName() + [SupportedOSPlatform("windows")] + private static IEnumerable<(decimal minor, string fullVersion, string path)> GetPythonVersions(RegistryKey? k,char major) { - return Version switch - { - PythonVersion.NotSet => throw new Exception("Python version not set yet"), - PythonVersion.Version2 => "python27", - PythonVersion.Version3 => "python35", - _ => throw new ArgumentOutOfRangeException() - }; - } - - private string GetExpectedPythonVersion() - { - return Version switch - { - PythonVersion.NotSet => throw new Exception("Python version not set yet"), - PythonVersion.Version2 => "2.7.1", - PythonVersion.Version3 => "3.4.3", - _ => throw new ArgumentOutOfRangeException() - }; - } - private string GetCompatiblePythonVersion() - { - return Version switch + if (k is null) yield break; + + foreach (var v in k.GetSubKeyNames()) { - PythonVersion.NotSet => throw new Exception("Python version not set yet"), - PythonVersion.Version2 => "2", - PythonVersion.Version3 => "3", - _ => throw new ArgumentOutOfRangeException() - }; - } - public string GetDescription() - { - throw new NotImplementedException(); + if (v.Length < 3 || v[0] != major || v[1] != '.' || !decimal.TryParse(v[2..], out var minor)) + continue; + + using var details = k.OpenSubKey(v); + if (details is null) continue; + + var fullVersion = details?.GetValue("Version") ?? v; + + using var pathKey = details?.OpenSubKey("InstallPath"); + if (pathKey is null) continue; + + var path = pathKey.GetValue("ExecutablePath")?.ToString() ?? Path.Combine(pathKey?.GetValue(null)?.ToString() ?? "DUMMY","python.exe"); + + if (!path.Contains("DUMMY",StringComparison.Ordinal)) + yield return (minor,fullVersion.ToString()??"0.0.0", path.ToString()??"none"); + } } - public IDataProvider Clone() + private static (decimal minor, string fullVersion, string path) GetPython(char major) { - throw new NotImplementedException(); + if (!OperatingSystem.IsWindows()) throw new InvalidOperationException("This Python plugin is Windows only for now"); + + using var machine = Registry.LocalMachine.OpenSubKey("SOFTWARE\\Python\\PythonCore"); + using var user = Registry.CurrentUser.OpenSubKey("SOFTWARE\\Python\\PythonCore"); + using var machine32 = Registry.LocalMachine.OpenSubKey("SOFTWARE\\WOW6432Node\\Python\\PythonCore"); + using var user32 = Registry.CurrentUser.OpenSubKey("SOFTWARE\\WOW6432Node\\Python\\PythonCore"); + var candidate = GetPythonVersions(machine,major).Union(GetPythonVersions(user,major)).MaxBy(static v=>v.minor); + return candidate; } - public bool Validate(ILoadDirectory destination) + private string GetExpectedPythonVersion() { - return true; + if (Version != PythonVersion.Version2 && Version!=PythonVersion.Version3) + throw new Exception("Python version not set yet or invalid"); + + if (!OperatingSystem.IsWindows()) throw new InvalidOperationException("This Python plugin is Windows only for now"); + + var major = Version == PythonVersion.Version2 ? '2' : '3'; + return GetPython(major).fullVersion; } } \ No newline at end of file diff --git a/Python/LoadModules.Extensions.Python/Properties/AssemblyInfo.cs b/Python/LoadModules.Extensions.Python/Properties/AssemblyInfo.cs index 527c238..baf104b 100644 --- a/Python/LoadModules.Extensions.Python/Properties/AssemblyInfo.cs +++ b/Python/LoadModules.Extensions.Python/Properties/AssemblyInfo.cs @@ -1,7 +1,7 @@ using System.Runtime.InteropServices; -// Setting ComVisible to false makes the types in this assembly not visible -// to COM components. If you need to access a type in this assembly from +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from // COM, set the ComVisible attribute to true on that type. [assembly: ComVisible(false)]