Skip to content

Commit

Permalink
Redo Python handling, find file paths from Registry instead of guessing
Browse files Browse the repository at this point in the history
  • Loading branch information
@jas88
jas88 committed Aug 10, 2023
1 parent ab9eecc commit 8f1fc2d
Show file tree
Hide file tree
Showing 3 changed files with 98 additions and 107 deletions.
5 changes: 2 additions & 3 deletions LoadModules.Extensions.Tests/Python/Unit/Python2And3InstalledTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,15 +31,14 @@ public void PythonScript_OverrideExecutablePath_VersionMismatch()
provider.Check(new AcceptAllCheckNotifier());// version 3 should now be installed

//version 3 executable path is explicit override for executing commands
provider.OverridePythonExecutablePath = new FileInfo(Path.Combine(provider.GetFullPythonInstallDirectory(), "python.exe"));
provider.OverridePythonExecutablePath = new FileInfo(provider.GetFullPythonPath());
provider.Version = PythonVersion.Version2;

//so we now know that version 3 is installed, and we have overriden the python path to the .exe explicitly and we are trying to launch with Version2 enum now
var ex = Assert.Throws<Exception>(()=>
{
provider.Check(ThrowImmediatelyCheckNotifier.Quiet);
//provider.Fetch(MockRepository.GenerateStub<IDataLoadJob>(), new GracefulCancellationToken());
});
StringAssert.Contains(@"which is incompatible with the desired version 2.7.1",ex?.Message);
StringAssert.Contains(@"which is incompatible with the desired version 2.7",ex?.Message);
}
}
196 changes: 94 additions & 102 deletions Python/LoadModules.Extensions.Python/DataProvider/PythonDataProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,46 +3,49 @@
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Runtime.Versioning;
using System.Threading.Tasks;
using FAnsi.Discovery;
using Microsoft.Win32;
using Rdmp.Core.Curation;
using Rdmp.Core.Curation.Data;
using Rdmp.Core.DataFlowPipeline;
using Rdmp.Core.DataLoad;
using Rdmp.Core.DataLoad.Engine.DataProvider;
using Rdmp.Core.DataLoad.Engine.Job;
using Rdmp.Core.ReusableLibraryCode.Annotations;
using Rdmp.Core.ReusableLibraryCode.Checks;
using Rdmp.Core.ReusableLibraryCode.Progress;

namespace LoadModules.Extensions.Python.DataProvider;

#nullable enable

public enum PythonVersion
{
NotSet,
Version2,
Version3
}

public class PythonDataProvider:IPluginDataProvider
public sealed class PythonDataProvider:IPluginDataProvider
{


[DemandsInitialization("The Python script to run")]
public string FullPathToPythonScriptToRun { get; set; }
public string? FullPathToPythonScriptToRun { get; set; }

[DemandsInitialization("The maximum number of seconds to allow the python script to run for before declaring it a failure, 0 for indefinetly")]
[DemandsInitialization("The maximum number of seconds to allow the python script to run for before declaring it a failure, 0 for indefinitely")]
public int MaximumNumberOfSecondsToLetScriptRunFor { get; set; }

[DemandsInitialization("Python version required to run your script")]
[DemandsInitialization("Python version required to run your script")]
public PythonVersion Version { get; set; }

[DemandsInitialization("Override Python Executable Path")]
public FileInfo OverridePythonExecutablePath { get; set; }
public FileInfo? OverridePythonExecutablePath { get; set; }



public void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventsListener)
{

}

public void Check(ICheckNotifier notifier)
Expand All @@ -60,14 +63,15 @@ public void Check(ICheckNotifier notifier)
{
var version = GetPythonVersion();

if (version.StartsWith(GetExpectedPythonVersion()))
if (version?.StartsWith(GetExpectedPythonVersion(), StringComparison.Ordinal)==true)
notifier.OnCheckPerformed(
new CheckEventArgs(
$"Found Expected Python version {version} on the host machine at directory {GetFullPythonInstallDirectory()}", CheckResult.Success));
else if (version.StartsWith(GetCompatiblePythonVersion()))
$"Found Expected Python version {version} on the host machine at {GetPython(Version == PythonVersion.Version2 ? '2' : '3').path}", CheckResult.Success));
else if (version is not null && ((version[0] == '3' && Version == PythonVersion.Version3) ||
(version[0] == '2' && Version == PythonVersion.Version2)))
notifier.OnCheckPerformed(
new CheckEventArgs(
$"Found Compatible Python version {version} on the host machine at directory {GetFullPythonInstallDirectory()}", CheckResult.Success));
$"Found Compatible Python version {version} on the host machine at {GetPython(Version == PythonVersion.Version2 ? '2' : '3').path}", CheckResult.Success));
else
{
notifier.OnCheckPerformed(
Expand Down Expand Up @@ -101,68 +105,51 @@ public void Check(ICheckNotifier notifier)
CheckResult.Warning));
}

public string GetPythonVersion()
public string? GetPythonVersion()
{
var info = GetPythonCommand(@"-c ""import sys; print(sys.version)""");

const string getVersion = """
-c "import sys; print(sys.version)"
""";
var toMemory = new ToMemoryDataLoadEventListener(true);
var result = ExecuteProcess(toMemory, getVersion, 600);

var result = ExecuteProcess(toMemory, info, 600);

if (result != 0)
return null;

var msg = toMemory.EventsReceivedBySender[this].SingleOrDefault();

if (msg != null)
return msg.Message;

throw new Exception($"Call to {info.Arguments} did not return any value but exited with code {result}");
throw new Exception($"Call to {getVersion} did not return any value but exited with code {result}");
}

private ProcessStartInfo GetPythonCommand(string command)
private string GetPythonCommand()
{
string exeFullPath;

if (OverridePythonExecutablePath == null)
{
//e.g. c:\python34
var installDir = GetFullPythonInstallDirectory();
exeFullPath = Path.Combine(installDir, "python");
}
else
{
if (!OverridePythonExecutablePath.Exists)
throw new FileNotFoundException(
$"The specified OverridePythonExecutablePath:{OverridePythonExecutablePath} does not exist");
if(OverridePythonExecutablePath.Name != "python.exe")
throw new FileNotFoundException(
$"The specified OverridePythonExecutablePath:{OverridePythonExecutablePath} file is not called python.exe... what is going on here?");

exeFullPath = OverridePythonExecutablePath.FullName;
}
return GetPython(Version==PythonVersion.Version2?'2':'3').path;

var info = new ProcessStartInfo(exeFullPath)
{
Arguments = command
};
if (!OverridePythonExecutablePath.Exists)
throw new FileNotFoundException(
$"The specified OverridePythonExecutablePath:{OverridePythonExecutablePath} does not exist");
if(OverridePythonExecutablePath.Name != "python.exe")
throw new FileNotFoundException(
$"The specified OverridePythonExecutablePath:{OverridePythonExecutablePath} file is not called python.exe... what is going on here?");

return info;
return OverridePythonExecutablePath.FullName;
}

public void Initialize(ILoadDirectory hicProjectDirectory, DiscoveredDatabase dbInfo)
{

}

public ExitCodeType Fetch(IDataLoadJob job, GracefulCancellationToken cancellationToken)
{
var processStartInfo = GetPythonCommand(FullPathToPythonScriptToRun);

int exitCode;
try
{
exitCode = ExecuteProcess(job, processStartInfo,MaximumNumberOfSecondsToLetScriptRunFor);
exitCode = ExecuteProcess(job, FullPathToPythonScriptToRun, MaximumNumberOfSecondsToLetScriptRunFor);

Check warning on line 152 in Python/LoadModules.Extensions.Python/DataProvider/PythonDataProvider.cs

View workflow job for this annotation

GitHub Actions / package 

Possible null reference argument for parameter 'script' in 'int PythonDataProvider.ExecuteProcess(IDataLoadEventListener listener, string script, int maximumNumberOfSecondsToLetScriptRunFor)'.
}
catch (TimeoutException e)
{
Expand All @@ -176,13 +163,17 @@ public ExitCodeType Fetch(IDataLoadJob job, GracefulCancellationToken cancellati
return exitCode == 0 ? ExitCodeType.Success : ExitCodeType.Error;
}

private int ExecuteProcess(IDataLoadEventListener listener, ProcessStartInfo processStartInfo, int maximumNumberOfSecondsToLetScriptRunFor)
private int ExecuteProcess(IDataLoadEventListener listener, string script, int maximumNumberOfSecondsToLetScriptRunFor)
{
processStartInfo.RedirectStandardOutput = true;
processStartInfo.RedirectStandardError = true;

processStartInfo.UseShellExecute = false;
processStartInfo.CreateNoWindow = true;
var processStartInfo = new ProcessStartInfo
{
FileName = GetPythonCommand(),
Arguments = script,
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
CreateNoWindow = true
};

Process p;

Expand All @@ -197,7 +188,7 @@ private int ExecuteProcess(IDataLoadEventListener listener, ProcessStartInfo pro
};
p.OutputDataReceived += (s, e) => allOutputDataConsumed = OutputDataReceived(e, listener,false);
p.ErrorDataReceived += (s, e) => allErrorDataConsumed = OutputDataReceived(e, listener,true);

p.Start();
p.BeginErrorReadLine();
p.BeginOutputReadLine();
Expand All @@ -208,11 +199,11 @@ private int ExecuteProcess(IDataLoadEventListener listener, ProcessStartInfo pro
throw new Exception(
$"Failed to launch:{Environment.NewLine}{processStartInfo.FileName}{Environment.NewLine} with Arguments:{processStartInfo.Arguments}",e);
}

// To avoid deadlocks, always read the output stream first and then wait.
var startTime = DateTime.Now;


while (!p.WaitForExit(100))//while process has not exited
{
if (!TimeoutExpired(startTime)) continue; //if timeout expired
Expand Down Expand Up @@ -242,21 +233,20 @@ private int ExecuteProcess(IDataLoadEventListener listener, ProcessStartInfo pro

lock(this)
if (_outputDataReceivedExceptions.Any())
if (_outputDataReceivedExceptions.Count == 1)
throw _outputDataReceivedExceptions[0];
else
throw new AggregateException(_outputDataReceivedExceptions);
throw _outputDataReceivedExceptions.Count == 1
? _outputDataReceivedExceptions[0]
: new AggregateException(_outputDataReceivedExceptions);

return p.ExitCode;
}

readonly List<Exception> _outputDataReceivedExceptions = new();
private readonly List<Exception> _outputDataReceivedExceptions = new();

private bool OutputDataReceived(DataReceivedEventArgs e, IDataLoadEventListener listener,bool isErrorStream)
{
if(e.Data == null)
return true;

lock (this)
{
try
Expand All @@ -271,7 +261,7 @@ private bool OutputDataReceived(DataReceivedEventArgs e, IDataLoadEventListener
return true;
}
}

return false;
}
private bool TimeoutExpired(DateTime startTime)
Expand All @@ -283,54 +273,56 @@ private bool TimeoutExpired(DateTime startTime)
}


public string GetFullPythonInstallDirectory()
public string GetFullPythonPath()
{
return Path.Combine(AppDomain.CurrentDomain.BaseDirectory, GetPythonFolderName());
return GetPython(Version == PythonVersion.Version2 ? '2' : '3').path;
}

private string GetPythonFolderName()
[SupportedOSPlatform("windows")]
private static IEnumerable<(decimal minor, string fullVersion, string path)> GetPythonVersions(RegistryKey? k,char major)
{
return Version switch
{
PythonVersion.NotSet => throw new Exception("Python version not set yet"),
PythonVersion.Version2 => "python27",
PythonVersion.Version3 => "python35",
_ => throw new ArgumentOutOfRangeException()
};
}

private string GetExpectedPythonVersion()
{
return Version switch
{
PythonVersion.NotSet => throw new Exception("Python version not set yet"),
PythonVersion.Version2 => "2.7.1",
PythonVersion.Version3 => "3.4.3",
_ => throw new ArgumentOutOfRangeException()
};
}
private string GetCompatiblePythonVersion()
{
return Version switch
if (k is null) yield break;

foreach (var v in k.GetSubKeyNames())
{
PythonVersion.NotSet => throw new Exception("Python version not set yet"),
PythonVersion.Version2 => "2",
PythonVersion.Version3 => "3",
_ => throw new ArgumentOutOfRangeException()
};
}
public string GetDescription()
{
throw new NotImplementedException();
if (v.Length < 3 || v[0] != major || v[1] != '.' || !decimal.TryParse(v[2..], out var minor))
continue;

using var details = k.OpenSubKey(v);
if (details is null) continue;

var fullVersion = details?.GetValue("Version") ?? v;

using var pathKey = details?.OpenSubKey("InstallPath");
if (pathKey is null) continue;

var path = pathKey.GetValue("ExecutablePath")?.ToString() ?? Path.Combine(pathKey?.GetValue(null)?.ToString() ?? "DUMMY","python.exe");

if (!path.Contains("DUMMY",StringComparison.Ordinal))
yield return (minor,fullVersion.ToString()??"0.0.0", path.ToString()??"none");
}
}

public IDataProvider Clone()
private static (decimal minor, string fullVersion, string path) GetPython(char major)
{
throw new NotImplementedException();
if (!OperatingSystem.IsWindows()) throw new InvalidOperationException("This Python plugin is Windows only for now");

using var machine = Registry.LocalMachine.OpenSubKey("SOFTWARE\\Python\\PythonCore");
using var user = Registry.CurrentUser.OpenSubKey("SOFTWARE\\Python\\PythonCore");
using var machine32 = Registry.LocalMachine.OpenSubKey("SOFTWARE\\WOW6432Node\\Python\\PythonCore");
using var user32 = Registry.CurrentUser.OpenSubKey("SOFTWARE\\WOW6432Node\\Python\\PythonCore");
var candidate = GetPythonVersions(machine,major).Union(GetPythonVersions(user,major)).MaxBy(static v=>v.minor);
return candidate;
}

public bool Validate(ILoadDirectory destination)
private string GetExpectedPythonVersion()
{
return true;
if (Version != PythonVersion.Version2 && Version!=PythonVersion.Version3)
throw new Exception("Python version not set yet or invalid");

if (!OperatingSystem.IsWindows()) throw new InvalidOperationException("This Python plugin is Windows only for now");

var major = Version == PythonVersion.Version2 ? '2' : '3';
return GetPython(major).fullVersion;
}
}
4 changes: 2 additions & 2 deletions Python/LoadModules.Extensions.Python/Properties/AssemblyInfo.cs
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
using System.Runtime.InteropServices;

// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]

Expand Down

0 comments on commit 8f1fc2d

Please sign in to comment.