diff --git a/LLama.Examples/Program.cs b/LLama.Examples/Program.cs index b24ef406b..63114120d 100644 --- a/LLama.Examples/Program.cs +++ b/LLama.Examples/Program.cs @@ -1,5 +1,6 @@ using LLama.Native; using Spectre.Console; +using System.Runtime.InteropServices; AnsiConsole.MarkupLineInterpolated( $""" @@ -16,23 +17,24 @@ __ __ ____ __ """); -// Configure native library to use. This must be done before any other llama.cpp methods are called! -NativeLibraryConfig - .Instance - .WithCuda(); - // Configure logging. Change this to `true` to see log messages from llama.cpp var showLLamaCppLogs = false; NativeLibraryConfig - .Instance + .All .WithLogCallback((level, message) => - { - if (showLLamaCppLogs) - Console.WriteLine($"[llama {level}]: {message.TrimEnd('\n')}"); - }); + { + if (showLLamaCppLogs) + Console.WriteLine($"[llama {level}]: {message.TrimEnd('\n')}"); + }); + +// Configure native library to use. This must be done before any other llama.cpp methods are called! +NativeLibraryConfig + .All + .WithCuda() + //.WithAutoDownload() // An experimental feature + .DryRun(out var loadedllamaLibrary, out var loadedLLavaLibrary); // Calling this method forces loading to occur now. NativeApi.llama_empty_call(); -await ExampleRunner.Run(); - +await ExampleRunner.Run(); \ No newline at end of file diff --git a/LLama/Abstractions/INativeLibrary.cs b/LLama/Abstractions/INativeLibrary.cs new file mode 100644 index 000000000..a7e00b753 --- /dev/null +++ b/LLama/Abstractions/INativeLibrary.cs @@ -0,0 +1,29 @@ +using LLama.Native; +using System; +using System.Collections.Generic; +using System.Text; + +namespace LLama.Abstractions +{ + /// + /// Descriptor of a native library. + /// + public interface INativeLibrary + { + /// + /// Metadata of this library. + /// + NativeLibraryMetadata? Metadata { get; } + + /// + /// Prepare the native library file and returns the local path of it. + /// If it's a relative path, LLamaSharp will search the path in the search directies you set. + /// + /// The system information of the current machine. + /// The log callback. + /// + /// The relative paths of the library. You could return multiple paths to try them one by one. If no file is available, please return an empty array. + /// + IEnumerable Prepare(SystemInfo systemInfo, NativeLogConfig.LLamaLogCallback? logCallback = null); + } +} diff --git a/LLama/Abstractions/INativeLibrarySelectingPolicy.cs b/LLama/Abstractions/INativeLibrarySelectingPolicy.cs new file mode 100644 index 000000000..41335202e --- /dev/null +++ b/LLama/Abstractions/INativeLibrarySelectingPolicy.cs @@ -0,0 +1,24 @@ +using LLama.Native; +using System; +using System.Collections.Generic; +using System.Text; + +namespace LLama.Abstractions +{ +#if NET6_0_OR_GREATER + /// + /// Decides the selected native library that should be loaded according to the configurations. + /// + public interface INativeLibrarySelectingPolicy + { + /// + /// Select the native library. + /// + /// + /// The system information of the current machine. + /// The log callback. + /// The information of the selected native library files, in order by priority from the beginning to the end. + IEnumerable Apply(NativeLibraryConfig.Description description, SystemInfo systemInfo, NativeLogConfig.LLamaLogCallback? logCallback = null); + } +#endif +} diff --git a/LLama/LLamaSharp.csproj b/LLama/LLamaSharp.csproj index b6079f5a5..87728cc73 100644 --- a/LLama/LLamaSharp.csproj +++ b/LLama/LLamaSharp.csproj @@ -3,7 +3,7 @@ netstandard2.0;net6.0;net8.0 LLama enable - 10 + 12 AnyCPU;x64;Arm64 True diff --git a/LLama/Native/Load/DefaultNativeLibrarySelectingPolicy.cs b/LLama/Native/Load/DefaultNativeLibrarySelectingPolicy.cs new file mode 100644 index 000000000..5cb3b0c5a --- /dev/null +++ b/LLama/Native/Load/DefaultNativeLibrarySelectingPolicy.cs @@ -0,0 +1,69 @@ +using LLama.Abstractions; +using System.Collections.Generic; +using System.Runtime.InteropServices; + +namespace LLama.Native +{ +#if NET6_0_OR_GREATER + /// + public class DefaultNativeLibrarySelectingPolicy: INativeLibrarySelectingPolicy + { + /// + public IEnumerable Apply(NativeLibraryConfig.Description description, SystemInfo systemInfo, NativeLogConfig.LLamaLogCallback? logCallback) + { + List results = new(); + + // Show the configuration we're working with + Log(description.ToString(), LLamaLogLevel.Info, logCallback); + + // If a specific path is requested, only use it, no fall back. + if (!string.IsNullOrEmpty(description.Path)) + { + yield return new NativeLibraryFromPath(description.Path); + } + else + { + if (description.UseCuda) + { + yield return new NativeLibraryWithCuda(systemInfo.CudaMajorVersion, description.Library, description.SkipCheck); + } + + if(!description.UseCuda || description.AllowFallback) + { + if (description.AllowFallback) + { + // Try all of the AVX levels we can support. + if (description.AvxLevel >= AvxLevel.Avx512) + yield return new NativeLibraryWithAvx(description.Library, AvxLevel.Avx512, description.SkipCheck); + + if (description.AvxLevel >= AvxLevel.Avx2) + yield return new NativeLibraryWithAvx(description.Library, AvxLevel.Avx2, description.SkipCheck); + + if (description.AvxLevel >= AvxLevel.Avx) + yield return new NativeLibraryWithAvx(description.Library, AvxLevel.Avx, description.SkipCheck); + + yield return new NativeLibraryWithAvx(description.Library, AvxLevel.None, description.SkipCheck); + } + else + { + yield return new NativeLibraryWithAvx(description.Library, description.AvxLevel, description.SkipCheck); + } + } + + if(systemInfo.OSPlatform == OSPlatform.OSX || description.AllowFallback) + { + yield return new NativeLibraryWithMacOrFallback(description.Library, description.SkipCheck); + } + } + } + + private void Log(string message, LLamaLogLevel level, NativeLogConfig.LLamaLogCallback? logCallback) + { + if (!message.EndsWith("\n")) + message += "\n"; + + logCallback?.Invoke(level, message); + } + } +#endif +} diff --git a/LLama/Native/Load/NativeLibraryConfig.cs b/LLama/Native/Load/NativeLibraryConfig.cs new file mode 100644 index 000000000..26d05909a --- /dev/null +++ b/LLama/Native/Load/NativeLibraryConfig.cs @@ -0,0 +1,618 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using LLama.Abstractions; +using Microsoft.Extensions.Logging; + +namespace LLama.Native +{ +#if NET6_0_OR_GREATER + /// + /// Allows configuration of the native llama.cpp libraries to load and use. + /// All configuration must be done before using **any** other LLamaSharp methods! + /// + public sealed partial class NativeLibraryConfig + { + private string? _libraryPath; + + private bool _useCuda = true; + private AvxLevel _avxLevel; + private bool _allowFallback = true; + private bool _skipCheck = false; + + /// + /// search directory -> priority level, 0 is the lowest. + /// + private readonly List _searchDirectories = new List(); + + internal INativeLibrarySelectingPolicy SelectingPolicy { get; private set; } = new DefaultNativeLibrarySelectingPolicy(); + + #region configurators + /// + /// Load a specified native library as backend for LLamaSharp. + /// When this method is called, all the other configurations will be ignored. + /// + /// The full path to the native library to load. + /// Thrown if `LibraryHasLoaded` is true. + public NativeLibraryConfig WithLibrary(string? libraryPath) + { + ThrowIfLoaded(); + + _libraryPath = libraryPath; + return this; + } + + /// + /// Configure whether to use cuda backend if possible. Default is true. + /// + /// + /// + /// Thrown if `LibraryHasLoaded` is true. + public NativeLibraryConfig WithCuda(bool enable = true) + { + ThrowIfLoaded(); + + _useCuda = enable; + return this; + } + + /// + /// Configure the prefferred avx support level of the backend. + /// Default value is detected automatically due to your operating system. + /// + /// + /// + /// Thrown if `LibraryHasLoaded` is true. + public NativeLibraryConfig WithAvx(AvxLevel level) + { + ThrowIfLoaded(); + + _avxLevel = level; + return this; + } + + /// + /// Configure whether to allow fallback when there's no match for preferred settings. Default is true. + /// + /// + /// + /// Thrown if `LibraryHasLoaded` is true. + public NativeLibraryConfig WithAutoFallback(bool enable = true) + { + ThrowIfLoaded(); + + _allowFallback = enable; + return this; + } + + /// + /// Whether to skip the check when you don't allow fallback. This option + /// may be useful under some complex conditions. For example, you're sure + /// you have your cublas configured but LLamaSharp take it as invalid by mistake. Default is false; + /// + /// + /// + /// Thrown if `LibraryHasLoaded` is true. + public NativeLibraryConfig SkipCheck(bool enable = true) + { + ThrowIfLoaded(); + + _skipCheck = enable; + return this; + } + + /// + /// Add self-defined search directories. Note that the file structure of the added + /// directories must be the same as the default directory. Besides, the directory + /// won't be used recursively. + /// + /// + /// + public NativeLibraryConfig WithSearchDirectories(IEnumerable directories) + { + ThrowIfLoaded(); + + _searchDirectories.AddRange(directories); + return this; + } + + /// + /// Add self-defined search directories. Note that the file structure of the added + /// directories must be the same as the default directory. Besides, the directory + /// won't be used recursively. + /// + /// + /// + public NativeLibraryConfig WithSearchDirectory(string directory) + { + ThrowIfLoaded(); + + _searchDirectories.Add(directory); + return this; + } + + /// + /// Set the policy which decides how to select the desired native libraries and order them by priority. + /// By default we use . + /// + /// + /// + public NativeLibraryConfig WithSelectingPolicy(INativeLibrarySelectingPolicy policy) + { + ThrowIfLoaded(); + + SelectingPolicy = policy; + return this; + } + + #endregion + + internal Description CheckAndGatherDescription() + { + if (_allowFallback && _skipCheck) + throw new ArgumentException("Cannot skip the check when fallback is allowed."); + + var path = _libraryPath; + + + return new Description( + path, + NativeLibraryName, + _useCuda, + _avxLevel, + _allowFallback, + _skipCheck, + _searchDirectories.Concat(new[] { "./" }).ToArray() + ); + } + + internal static string AvxLevelToString(AvxLevel level) + { + return level switch + { + AvxLevel.None => string.Empty, + AvxLevel.Avx => "avx", + AvxLevel.Avx2 => "avx2", + AvxLevel.Avx512 => "avx512", + _ => throw new ArgumentException($"Unknown AvxLevel '{level}'") + }; + } + + /// + /// Private constructor prevents new instances of this class being created + /// + private NativeLibraryConfig(NativeLibraryName nativeLibraryName) + { + NativeLibraryName = nativeLibraryName; + + // Automatically detect the highest supported AVX level + if (System.Runtime.Intrinsics.X86.Avx.IsSupported) + _avxLevel = AvxLevel.Avx; + if (System.Runtime.Intrinsics.X86.Avx2.IsSupported) + _avxLevel = AvxLevel.Avx2; + + if (CheckAVX512()) + _avxLevel = AvxLevel.Avx512; + } + + private static bool CheckAVX512() + { + if (!System.Runtime.Intrinsics.X86.X86Base.IsSupported) + return false; + + // ReSharper disable UnusedVariable (ebx is used when < NET8) + var (_, ebx, ecx, _) = System.Runtime.Intrinsics.X86.X86Base.CpuId(7, 0); + // ReSharper restore UnusedVariable + + var vnni = (ecx & 0b_1000_0000_0000) != 0; + +#if NET8_0_OR_GREATER + var f = System.Runtime.Intrinsics.X86.Avx512F.IsSupported; + var bw = System.Runtime.Intrinsics.X86.Avx512BW.IsSupported; + var vbmi = System.Runtime.Intrinsics.X86.Avx512Vbmi.IsSupported; +#else + var f = (ebx & (1 << 16)) != 0; + var bw = (ebx & (1 << 30)) != 0; + var vbmi = (ecx & 0b_0000_0000_0010) != 0; +#endif + + return vnni && vbmi && bw && f; + } + + /// + /// The description of the native library configurations that's already specified. + /// + /// + /// + /// + /// + /// + /// + /// + public record Description(string? Path, NativeLibraryName Library, bool UseCuda, AvxLevel AvxLevel, bool AllowFallback, bool SkipCheck, + string[] SearchDirectories) + { + /// + public override string ToString() + { + string avxLevelString = AvxLevel switch + { + AvxLevel.None => "NoAVX", + AvxLevel.Avx => "AVX", + AvxLevel.Avx2 => "AVX2", + AvxLevel.Avx512 => "AVX512", + _ => "Unknown" + }; + + string searchDirectoriesString = "{ " + string.Join(", ", SearchDirectories) + " }"; + + return $"NativeLibraryConfig Description:\n" + + $"- LibraryName: {Library}\n" + + $"- Path: '{Path}'\n" + + $"- PreferCuda: {UseCuda}\n" + + $"- PreferredAvxLevel: {avxLevelString}\n" + + $"- AllowFallback: {AllowFallback}\n" + + $"- SkipCheck: {SkipCheck}\n" + + $"- SearchDirectories and Priorities: {searchDirectoriesString}"; + } + } + } +#endif + + public sealed partial class NativeLibraryConfig + { + /// + /// Set configurations for all the native libraries, including LLama and LLava + /// + [Obsolete("Please use NativeLibraryConfig.All instead, or set configurations for NativeLibraryConfig.LLama and NativeLibraryConfig.LLavaShared respectively.")] + public static NativeLibraryConfigContainer Instance => All; + + /// + /// Set configurations for all the native libraries, including LLama and LLava + /// + public static NativeLibraryConfigContainer All { get; } + + /// + /// Configuration for LLama native library + /// + public static NativeLibraryConfig LLama { get; } + + /// + /// Configuration for LLava native library + /// + public static NativeLibraryConfig LLava { get; } + + + /// + /// The current version. + /// + public static string CurrentVersion => VERSION; // This should be changed before publishing new version. TODO: any better approach? + + private const string COMMIT_HASH = "f7001c"; + private const string VERSION = "master"; + + /// + /// Get the llama.cpp commit hash of the current version. + /// + /// + public static string GetNativeLibraryCommitHash() => COMMIT_HASH; + + static NativeLibraryConfig() + { + LLama = new(NativeLibraryName.LLama); + LLava = new(NativeLibraryName.LLava); + All = new(LLama, LLava); + } + +#if NETSTANDARD2_0 + private NativeLibraryConfig(NativeLibraryName nativeLibraryName) + { + NativeLibraryName = nativeLibraryName; + } +#endif + + /// + /// Check if the native library has already been loaded. Configuration cannot be modified if this is true. + /// + public bool LibraryHasLoaded { get; internal set; } + + internal NativeLibraryName NativeLibraryName { get; } + + internal NativeLogConfig.LLamaLogCallback? LogCallback { get; private set; } = null; + + private void ThrowIfLoaded() + { + if (LibraryHasLoaded) + throw new InvalidOperationException("The library has already loaded, you can't change the configurations. " + + "Please finish the configuration setting before any call to LLamaSharp native APIs." + + "Please use NativeLibraryConfig.DryRun if you want to see whether it's loaded " + + "successfully but still have chance to modify the configurations."); + } + + /// + /// Set the log callback that will be used for all llama.cpp log messages + /// + /// + /// + public NativeLibraryConfig WithLogCallback(NativeLogConfig.LLamaLogCallback? callback) + { + ThrowIfLoaded(); + + LogCallback = callback; + return this; + } + + /// + /// Set the log callback that will be used for all llama.cpp log messages + /// + /// + /// + public NativeLibraryConfig WithLogCallback(ILogger? logger) + { + ThrowIfLoaded(); + + // Redirect to llama_log_set. This will wrap the logger in a delegate and bind that as the log callback instead. + NativeLogConfig.llama_log_set(logger); + + return this; + } + + /// + /// Try to load the native library with the current configurations, + /// but do not actually set it to . + /// + /// You can still modify the configuration after this calling but only before any call from . + /// + /// + /// The loaded livrary. When the loading failed, this will be null. + /// However if you are using .NET standard2.0, this will never return null. + /// + /// Whether the running is successful. + public bool DryRun(out INativeLibrary? loadedLibrary) + { + LogCallback?.Invoke(LLamaLogLevel.Debug, $"Beginning dry run for {this.NativeLibraryName.GetLibraryName()}..."); + return NativeLibraryUtils.TryLoadLibrary(this, out loadedLibrary) != IntPtr.Zero; + } + } + + /// + /// A class to set same configurations to multiple libraries at the same time. + /// + public sealed class NativeLibraryConfigContainer + { + private NativeLibraryConfig[] _configs; + + internal NativeLibraryConfigContainer(params NativeLibraryConfig[] configs) + { + _configs = configs; + } + + #region configurators + +#if NET6_0_OR_GREATER + /// + /// Load a specified native library as backend for LLamaSharp. + /// When this method is called, all the other configurations will be ignored. + /// + /// The full path to the llama library to load. + /// The full path to the llava library to load. + /// Thrown if `LibraryHasLoaded` is true. + public NativeLibraryConfigContainer WithLibrary(string? llamaPath, string? llavaPath) + { + foreach(var config in _configs) + { + if(config.NativeLibraryName == NativeLibraryName.LLama && llamaPath is not null) + { + config.WithLibrary(llamaPath); + } + if(config.NativeLibraryName == NativeLibraryName.LLava && llavaPath is not null) + { + config.WithLibrary(llavaPath); + } + } + + return this; + } + + /// + /// Configure whether to use cuda backend if possible. + /// + /// + /// + /// Thrown if `LibraryHasLoaded` is true. + public NativeLibraryConfigContainer WithCuda(bool enable = true) + { + foreach(var config in _configs) + { + config.WithCuda(enable); + } + return this; + } + + /// + /// Configure the prefferred avx support level of the backend. + /// + /// + /// + /// Thrown if `LibraryHasLoaded` is true. + public NativeLibraryConfigContainer WithAvx(AvxLevel level) + { + foreach (var config in _configs) + { + config.WithAvx(level); + } + return this; + } + + /// + /// Configure whether to allow fallback when there's no match for preferred settings. + /// + /// + /// + /// Thrown if `LibraryHasLoaded` is true. + public NativeLibraryConfigContainer WithAutoFallback(bool enable = true) + { + foreach (var config in _configs) + { + config.WithAutoFallback(enable); + } + return this; + } + + /// + /// Whether to skip the check when you don't allow fallback. This option + /// may be useful under some complex conditions. For example, you're sure + /// you have your cublas configured but LLamaSharp take it as invalid by mistake. + /// + /// + /// + /// Thrown if `LibraryHasLoaded` is true. + public NativeLibraryConfigContainer SkipCheck(bool enable = true) + { + foreach (var config in _configs) + { + config.SkipCheck(enable); + } + return this; + } + + /// + /// Add self-defined search directories. Note that the file structure of the added + /// directories must be the same as the default directory. Besides, the directory + /// won't be used recursively. + /// + /// + /// + public NativeLibraryConfigContainer WithSearchDirectories(IEnumerable directories) + { + foreach (var config in _configs) + { + config.WithSearchDirectories(directories); + } + return this; + } + + /// + /// Add self-defined search directories. Note that the file structure of the added + /// directories must be the same as the default directory. Besides, the directory + /// won't be used recursively. + /// + /// + /// + public NativeLibraryConfigContainer WithSearchDirectory(string directory) + { + foreach (var config in _configs) + { + config.WithSearchDirectory(directory); + } + return this; + } + + /// + /// Set the policy which decides how to select the desired native libraries and order them by priority. + /// By default we use . + /// + /// + /// + public NativeLibraryConfigContainer WithSelectingPolicy(INativeLibrarySelectingPolicy policy) + { + foreach (var config in _configs) + { + config.WithSelectingPolicy(policy); + } + return this; + } +#endif + + /// + /// Set the log callback that will be used for all llama.cpp log messages + /// + /// + /// + public NativeLibraryConfigContainer WithLogCallback(NativeLogConfig.LLamaLogCallback? callback) + { + foreach (var config in _configs) + { + config.WithLogCallback(callback); + } + return this; + } + + /// + /// Set the log callback that will be used for all llama.cpp log messages + /// + /// + /// + public NativeLibraryConfigContainer WithLogCallback(ILogger? logger) + { + foreach (var config in _configs) + { + config.WithLogCallback(logger); + } + return this; + } + + #endregion + + /// + /// Try to load the native library with the current configurations, + /// but do not actually set it to . + /// + /// You can still modify the configuration after this calling but only before any call from . + /// + /// Whether the running is successful. + public bool DryRun(out INativeLibrary? loadedLLamaNativeLibrary, out INativeLibrary? loadedLLavaNativeLibrary) + { + bool success = true; + foreach(var config in _configs) + { + success &= config.DryRun(out var loadedLibrary); + if(config.NativeLibraryName == NativeLibraryName.LLama) + { + loadedLLamaNativeLibrary = loadedLibrary; + } + else if(config.NativeLibraryName == NativeLibraryName.LLava) + { + loadedLLavaNativeLibrary = loadedLibrary; + } + else + { + throw new Exception("Unknown native library config during the dry run."); + } + } + loadedLLamaNativeLibrary = loadedLLavaNativeLibrary = null; + return success; + } + } + + /// + /// The name of the native library + /// + public enum NativeLibraryName + { + /// + /// The native library compiled from llama.cpp. + /// + LLama, + /// + /// The native library compiled from the LLaVA example of llama.cpp. + /// + LLava + } + + internal static class LibraryNameExtensions + { + public static string GetLibraryName(this NativeLibraryName name) + { + switch (name) + { + case NativeLibraryName.LLama: + return NativeApi.libraryName; + case NativeLibraryName.LLava: + return NativeApi.llavaLibraryName; + default: + throw new ArgumentOutOfRangeException(nameof(name), name, null); + } + } + } +} diff --git a/LLama/Native/Load/NativeLibraryFromPath.cs b/LLama/Native/Load/NativeLibraryFromPath.cs new file mode 100644 index 000000000..8cd99c308 --- /dev/null +++ b/LLama/Native/Load/NativeLibraryFromPath.cs @@ -0,0 +1,31 @@ +using LLama.Abstractions; +using System.Collections.Generic; + +namespace LLama.Native +{ + /// + /// A native library specified with a local file path. + /// + public class NativeLibraryFromPath: INativeLibrary + { + private string _path; + + /// + public NativeLibraryMetadata? Metadata => null; + + /// + /// + /// + /// + public NativeLibraryFromPath(string path) + { + _path = path; + } + + /// + public IEnumerable Prepare(SystemInfo systemInfo, NativeLogConfig.LLamaLogCallback? logCallback) + { + return [_path]; + } + } +} diff --git a/LLama/Native/Load/NativeLibraryMetadata.cs b/LLama/Native/Load/NativeLibraryMetadata.cs new file mode 100644 index 000000000..654c9002f --- /dev/null +++ b/LLama/Native/Load/NativeLibraryMetadata.cs @@ -0,0 +1,43 @@ + +namespace LLama.Native +{ + /// + /// Information of a native library file. + /// + /// Which kind of library it is. + /// Whether it's compiled with cublas. + /// Which AvxLevel it's compiled with. + public record class NativeLibraryMetadata(NativeLibraryName NativeLibraryName, bool UseCuda, AvxLevel AvxLevel) + { + public override string ToString() + { + return $"(NativeLibraryName: {NativeLibraryName}, UseCuda: {UseCuda}, AvxLevel: {AvxLevel})"; + } + } + + /// + /// Avx support configuration + /// + public enum AvxLevel + { + /// + /// No AVX + /// + None, + + /// + /// Advanced Vector Extensions (supported by most processors after 2011) + /// + Avx, + + /// + /// AVX2 (supported by most processors after 2013) + /// + Avx2, + + /// + /// AVX512 (supported by some processors after 2016, not widely supported) + /// + Avx512, + } +} diff --git a/LLama/Native/Load/NativeLibraryUtils.cs b/LLama/Native/Load/NativeLibraryUtils.cs new file mode 100644 index 000000000..9dd7c8af1 --- /dev/null +++ b/LLama/Native/Load/NativeLibraryUtils.cs @@ -0,0 +1,160 @@ +using LLama.Abstractions; +using LLama.Exceptions; +using System; +using System.Collections.Generic; +using System.IO; +using System.Runtime.InteropServices; + +namespace LLama.Native +{ + internal static class NativeLibraryUtils + { + /// + /// Try to load libllama/llava_shared, using CPU feature detection to try and load a more specialised DLL if possible + /// + /// The library handle to unload later, or IntPtr.Zero if no library was loaded + internal static IntPtr TryLoadLibrary(NativeLibraryConfig config, out INativeLibrary? loadedLibrary) + { +#if NET6_0_OR_GREATER + var description = config.CheckAndGatherDescription(); + var systemInfo = SystemInfo.Get(); + Log($"Loading library: '{config.NativeLibraryName.GetLibraryName()}'", LLamaLogLevel.Debug, config.LogCallback); + + // Get platform specific parts of the path (e.g. .so/.dll/.dylib, libName prefix or not) + NativeLibraryUtils.GetPlatformPathParts(systemInfo.OSPlatform, out var os, out var ext, out var libPrefix); + Log($"Detected OS Platform: '{systemInfo.OSPlatform}'", LLamaLogLevel.Info, config.LogCallback); + Log($"Detected OS string: '{os}'", LLamaLogLevel.Debug, config.LogCallback); + Log($"Detected extension string: '{ext}'", LLamaLogLevel.Debug, config.LogCallback); + Log($"Detected prefix string: '{libPrefix}'", LLamaLogLevel.Debug, config.LogCallback); + + // Set the flag to ensure this config can no longer be modified + config.LibraryHasLoaded = true; + + // Show the configuration we're working with + Log(description.ToString(), LLamaLogLevel.Info, config.LogCallback); + + // Get the libraries ordered by priority from the selecting policy. + var libraries = config.SelectingPolicy.Apply(description, systemInfo, config.LogCallback); + + foreach (var library in libraries) + { + // Prepare the local library file and get the path. + var paths = library.Prepare(systemInfo, config.LogCallback); + foreach (var path in paths) + { + Log($"Got relative library path '{path}' from local with {library.Metadata}, trying to load it...", LLamaLogLevel.Debug, config.LogCallback); + + var result = TryLoad(path, description.SearchDirectories, config.LogCallback); + if (result != IntPtr.Zero) + { + loadedLibrary = library; + return result; + } + } + } + + // If fallback is allowed, we will make the last try (the default system loading) when calling the native api. + // Otherwise we throw an exception here. + if (!description.AllowFallback) + { + throw new RuntimeError("Failed to load the native library. Please check the log for more information."); + } + loadedLibrary = null; +#else + loadedLibrary = new UnknownNativeLibrary(); +#endif + + Log($"No library was loaded before calling native apis. " + + $"This is not an error under netstandard2.0 but needs attention with net6 or higher.", LLamaLogLevel.Warning, config.LogCallback); + return IntPtr.Zero; + +#if NET6_0_OR_GREATER + // Try to load a DLL from the path. + // Returns null if nothing is loaded. + static IntPtr TryLoad(string path, IEnumerable searchDirectories, NativeLogConfig.LLamaLogCallback? logCallback) + { + var fullPath = TryFindPath(path, searchDirectories); + Log($"Found full path file '{fullPath}' for relative path '{path}'", LLamaLogLevel.Debug, logCallback); + if (NativeLibrary.TryLoad(fullPath, out var handle)) + { + Log($"Successfully loaded '{fullPath}'", LLamaLogLevel.Info, logCallback); + return handle; + } + + Log($"Failed Loading '{fullPath}'", LLamaLogLevel.Info, logCallback); + return IntPtr.Zero; + } +#endif + } + + // Try to find the given file in any of the possible search paths + private static string TryFindPath(string filename, IEnumerable searchDirectories) + { + // Try the configured search directories in the configuration + foreach (var path in searchDirectories) + { + var candidate = Path.Combine(path, filename); + if (File.Exists(candidate)) + return candidate; + } + + // Try a few other possible paths + var possiblePathPrefix = new[] { + AppDomain.CurrentDomain.BaseDirectory, + Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location) ?? "" + }; + + foreach (var path in possiblePathPrefix) + { + var candidate = Path.Combine(path, filename); + if (File.Exists(candidate)) + return candidate; + } + + return filename; + } + + private static void Log(string message, LLamaLogLevel level, NativeLogConfig.LLamaLogCallback? logCallback) + { + if (!message.EndsWith("\n")) + message += "\n"; + + logCallback?.Invoke(level, message); + } + +#if NET6_0_OR_GREATER + public static void GetPlatformPathParts(OSPlatform platform, out string os, out string fileExtension, out string libPrefix) + { + if (platform == OSPlatform.Windows) + { + os = "win-x64"; + fileExtension = ".dll"; + libPrefix = ""; + return; + } + + if (platform == OSPlatform.Linux) + { + os = "linux-x64"; + fileExtension = ".so"; + libPrefix = "lib"; + return; + } + + if (platform == OSPlatform.OSX) + { + fileExtension = ".dylib"; + + os = System.Runtime.Intrinsics.Arm.ArmBase.Arm64.IsSupported + ? "osx-arm64" + : "osx-x64"; + libPrefix = "lib"; + } + else + { + throw new RuntimeError("Your operating system is not supported, please open an issue in LLamaSharp."); + } + } +#endif + } +} diff --git a/LLama/Native/Load/NativeLibraryWithAvx.cs b/LLama/Native/Load/NativeLibraryWithAvx.cs new file mode 100644 index 000000000..7b5421b4d --- /dev/null +++ b/LLama/Native/Load/NativeLibraryWithAvx.cs @@ -0,0 +1,62 @@ +using LLama.Abstractions; +using System.Collections.Generic; +using System.Runtime.InteropServices; + +namespace LLama.Native +{ +#if NET6_0_OR_GREATER + /// + /// A native library compiled with avx support but without cuda/cublas. + /// + public class NativeLibraryWithAvx : INativeLibrary + { + private NativeLibraryName _libraryName; + private AvxLevel _avxLevel; + private bool _skipCheck; + + /// + public NativeLibraryMetadata? Metadata + { + get + { + return new NativeLibraryMetadata(_libraryName, false, _avxLevel); + } + } + + /// + /// + /// + /// + /// + /// + public NativeLibraryWithAvx(NativeLibraryName libraryName, AvxLevel avxLevel, bool skipCheck) + { + _libraryName = libraryName; + _avxLevel = avxLevel; + _skipCheck = skipCheck; + } + + /// + public IEnumerable Prepare(SystemInfo systemInfo, NativeLogConfig.LLamaLogCallback? logCallback) + { + if (systemInfo.OSPlatform != OSPlatform.Windows && systemInfo.OSPlatform != OSPlatform.Linux && !_skipCheck) + { + // Not supported on systems other than Windows and Linux. + return []; + } + var path = GetAvxPath(systemInfo, _avxLevel, logCallback); + return path is null ? [] : [path]; + } + + private string? GetAvxPath(SystemInfo systemInfo, AvxLevel avxLevel, NativeLogConfig.LLamaLogCallback? logCallback) + { + NativeLibraryUtils.GetPlatformPathParts(systemInfo.OSPlatform, out var os, out var fileExtension, out var libPrefix); + var avxStr = NativeLibraryConfig.AvxLevelToString(avxLevel); + if (!string.IsNullOrEmpty(avxStr)) + avxStr += "/"; + var relativePath = $"runtimes/{os}/native/{avxStr}{libPrefix}{_libraryName.GetLibraryName()}{fileExtension}"; + return relativePath; + } + } +#endif +} diff --git a/LLama/Native/Load/NativeLibraryWithCuda.cs b/LLama/Native/Load/NativeLibraryWithCuda.cs new file mode 100644 index 000000000..d3b06b864 --- /dev/null +++ b/LLama/Native/Load/NativeLibraryWithCuda.cs @@ -0,0 +1,79 @@ +using LLama.Abstractions; +using System.Collections.Generic; +using System.Runtime.InteropServices; + +namespace LLama.Native +{ +#if NET6_0_OR_GREATER + /// + /// A native library compiled with cublas/cuda. + /// + public class NativeLibraryWithCuda : INativeLibrary + { + private int _majorCudaVersion; + private NativeLibraryName _libraryName; + private AvxLevel _avxLevel; + private bool _skipCheck; + + /// + public NativeLibraryMetadata? Metadata + { + get + { + return new NativeLibraryMetadata(_libraryName, true, _avxLevel); + } + } + + /// + /// + /// + /// + /// + /// + public NativeLibraryWithCuda(int majorCudaVersion, NativeLibraryName libraryName, bool skipCheck) + { + _majorCudaVersion = majorCudaVersion; + _libraryName = libraryName; + _skipCheck = skipCheck; + } + + /// + public IEnumerable Prepare(SystemInfo systemInfo, NativeLogConfig.LLamaLogCallback? logCallback) + { + // TODO: Avx level is ignored now, needs to be implemented in the future. + if (systemInfo.OSPlatform == OSPlatform.Windows || systemInfo.OSPlatform == OSPlatform.Linux || _skipCheck) + { + if (_majorCudaVersion == -1 && _skipCheck) + { + // Currently only 11 and 12 are supported. + var cuda12LibraryPath = GetCudaPath(systemInfo, 12, logCallback); + if (cuda12LibraryPath is not null) + { + yield return cuda12LibraryPath; + } + var cuda11LibraryPath = GetCudaPath(systemInfo, 11, logCallback); + if (cuda11LibraryPath is not null) + { + yield return cuda11LibraryPath; + } + } + else if (_majorCudaVersion != -1) + { + var cudaLibraryPath = GetCudaPath(systemInfo, _majorCudaVersion, logCallback); + if (cudaLibraryPath is not null) + { + yield return cudaLibraryPath; + } + } + } + } + + private string? GetCudaPath(SystemInfo systemInfo, int cudaVersion, NativeLogConfig.LLamaLogCallback? logCallback) + { + NativeLibraryUtils.GetPlatformPathParts(systemInfo.OSPlatform, out var os, out var fileExtension, out var libPrefix); + var relativePath = $"runtimes/{os}/native/cuda{cudaVersion}/{libPrefix}{_libraryName.GetLibraryName()}{fileExtension}"; + return relativePath; + } + } +#endif +} diff --git a/LLama/Native/Load/NativeLibraryWithMacOrFallback.cs b/LLama/Native/Load/NativeLibraryWithMacOrFallback.cs new file mode 100644 index 000000000..5df339307 --- /dev/null +++ b/LLama/Native/Load/NativeLibraryWithMacOrFallback.cs @@ -0,0 +1,64 @@ +using LLama.Abstractions; +using System.Collections.Generic; +using System.Runtime.InteropServices; + +namespace LLama.Native +{ +#if NET6_0_OR_GREATER + /// + /// A native library compiled on Mac, or fallbacks from all other libraries in the selection. + /// + public class NativeLibraryWithMacOrFallback : INativeLibrary + { + private NativeLibraryName _libraryName; + private bool _skipCheck; + + /// + public NativeLibraryMetadata? Metadata + { + get + { + return new NativeLibraryMetadata(_libraryName, false, AvxLevel.None); + } + } + + /// + /// + /// + /// + /// + public NativeLibraryWithMacOrFallback(NativeLibraryName libraryName, bool skipCheck) + { + _libraryName = libraryName; + _skipCheck = skipCheck; + } + + /// + public IEnumerable Prepare(SystemInfo systemInfo, NativeLogConfig.LLamaLogCallback? logCallback) + { + var path = GetPath(systemInfo, AvxLevel.None, logCallback); + return path is null ?[] : [path]; + } + + private string? GetPath(SystemInfo systemInfo, AvxLevel avxLevel, NativeLogConfig.LLamaLogCallback? logCallback) + { + NativeLibraryUtils.GetPlatformPathParts(systemInfo.OSPlatform, out var os, out var fileExtension, out var libPrefix); + string relativePath; + if (systemInfo.OSPlatform == OSPlatform.OSX) + { + relativePath = $"runtimes/{os}/native/{libPrefix}{_libraryName.GetLibraryName()}{fileExtension}"; + } + else + { + var avxStr = NativeLibraryConfig.AvxLevelToString(AvxLevel.None); + if (!string.IsNullOrEmpty(avxStr)) + avxStr += "/"; + + relativePath = $"runtimes/{os}/native/{avxStr}{libPrefix}{_libraryName.GetLibraryName()}{fileExtension}"; + } + + return relativePath; + } + } +#endif +} diff --git a/LLama/Native/Load/SystemInfo.cs b/LLama/Native/Load/SystemInfo.cs new file mode 100644 index 000000000..0ffc67e91 --- /dev/null +++ b/LLama/Native/Load/SystemInfo.cs @@ -0,0 +1,129 @@ +using System; +using System.IO; +using System.Runtime.InteropServices; +using System.Text.Json; + +namespace LLama.Native +{ + /// + /// Operating system information. + /// + /// + /// + public record class SystemInfo(OSPlatform OSPlatform, int CudaMajorVersion) + { + /// + /// Get the system information of the current machine. + /// + /// + /// + public static SystemInfo Get() + { + OSPlatform platform; + if(RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + platform = OSPlatform.Windows; + } + else if(RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + platform = OSPlatform.Linux; + } + else if(RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + platform = OSPlatform.OSX; + } + else + { + throw new PlatformNotSupportedException(); + } + + return new SystemInfo(platform, GetCudaMajorVersion()); + } + + #region CUDA version + private static int GetCudaMajorVersion() + { + string? cudaPath; + string version = ""; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + cudaPath = Environment.GetEnvironmentVariable("CUDA_PATH"); + if (cudaPath is null) + { + return -1; + } + + //Ensuring cuda bin path is reachable. Especially for MAUI environment. + string cudaBinPath = Path.Combine(cudaPath, "bin"); + + if (Directory.Exists(cudaBinPath)) + { + AddDllDirectory(cudaBinPath); + } + + version = GetCudaVersionFromPath(cudaPath); + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + // Try the default first + cudaPath = "/usr/local/bin/cuda"; + version = GetCudaVersionFromPath(cudaPath); + if (string.IsNullOrEmpty(version)) + { + cudaPath = Environment.GetEnvironmentVariable("LD_LIBRARY_PATH"); + if (cudaPath is null) + { + return -1; + } + foreach (var path in cudaPath.Split(':')) + { + version = GetCudaVersionFromPath(Path.Combine(path, "..")); + if (string.IsNullOrEmpty(version)) + { + break; + } + } + } + } + + if (string.IsNullOrEmpty(version)) + return -1; + + version = version.Split('.')[0]; + if (int.TryParse(version, out var majorVersion)) + return majorVersion; + + return -1; + } + + private static string GetCudaVersionFromPath(string cudaPath) + { + try + { + string json = File.ReadAllText(Path.Combine(cudaPath, cudaVersionFile)); + using (JsonDocument document = JsonDocument.Parse(json)) + { + JsonElement root = document.RootElement; + JsonElement cublasNode = root.GetProperty("libcublas"); + JsonElement versionNode = cublasNode.GetProperty("version"); + if (versionNode.ValueKind == JsonValueKind.Undefined) + { + return string.Empty; + } + return versionNode.GetString() ?? ""; + } + } + catch (Exception) + { + return string.Empty; + } + } + + // Put it here to avoid calling NativeApi when getting the cuda version. + [DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)] + internal static extern int AddDllDirectory(string NewDirectory); + + private const string cudaVersionFile = "version.json"; + #endregion + } +} diff --git a/LLama/Native/Load/UnknownNativeLibrary.cs b/LLama/Native/Load/UnknownNativeLibrary.cs new file mode 100644 index 000000000..fa29ac0d4 --- /dev/null +++ b/LLama/Native/Load/UnknownNativeLibrary.cs @@ -0,0 +1,25 @@ +using LLama.Abstractions; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace LLama.Native +{ + /// + /// When you are using .NET standard2.0, dynamic native library loading is not supported. + /// This class will be returned in . + /// + public class UnknownNativeLibrary: INativeLibrary + { + /// + public NativeLibraryMetadata? Metadata => null; + + /// + public IEnumerable Prepare(SystemInfo systemInfo, NativeLogConfig.LLamaLogCallback? logCallback = null) + { + throw new NotSupportedException("This class is only a placeholder and should not be used to load native library."); + } + } +} diff --git a/LLama/Native/NativeApi.Load.cs b/LLama/Native/NativeApi.Load.cs index 5275023e6..f1bd765e4 100644 --- a/LLama/Native/NativeApi.Load.cs +++ b/LLama/Native/NativeApi.Load.cs @@ -4,6 +4,7 @@ using System.Runtime.InteropServices; using System.Text.Json; using System.Collections.Generic; +using LLama.Abstractions; namespace LLama.Native { @@ -18,7 +19,8 @@ static NativeApi() SetDllImportResolver(); // Set flag to indicate that this point has been passed. No native library config can be done after this point. - NativeLibraryConfig.LibraryHasLoaded = true; + NativeLibraryConfig.LLama.LibraryHasLoaded = true; + NativeLibraryConfig.LLava.LibraryHasLoaded = true; // Immediately make a call which requires loading the llama DLL. This method call // can't fail unless the DLL hasn't been loaded. @@ -38,8 +40,8 @@ static NativeApi() } // Now that the "loaded" flag is set configure logging in llama.cpp - if (NativeLibraryConfig.Instance.LogCallback != null) - NativeLogConfig.llama_log_set(NativeLibraryConfig.Instance.LogCallback); + if (NativeLibraryConfig.LLama.LogCallback != null) + NativeLogConfig.llama_log_set(NativeLibraryConfig.LLama.LogCallback); // Init llama.cpp backend llama_backend_init(); @@ -64,7 +66,7 @@ private static void SetDllImportResolver() return _loadedLlamaHandle; // Try to load a preferred library, based on CPU feature detection - _loadedLlamaHandle = TryLoadLibraries(LibraryName.Llama); + _loadedLlamaHandle = NativeLibraryUtils.TryLoadLibrary(NativeLibraryConfig.LLama, out _loadedLLamaLibrary); return _loadedLlamaHandle; } @@ -75,7 +77,7 @@ private static void SetDllImportResolver() return _loadedLlavaSharedHandle; // Try to load a preferred library, based on CPU feature detection - _loadedLlavaSharedHandle = TryLoadLibraries(LibraryName.LlavaShared); + _loadedLlavaSharedHandle = NativeLibraryUtils.TryLoadLibrary(NativeLibraryConfig.LLava, out _loadedLLavaLibrary); return _loadedLlavaSharedHandle; } @@ -85,343 +87,26 @@ private static void SetDllImportResolver() #endif } - private static void Log(string message, LLamaLogLevel level) - { - if (!message.EndsWith("\n")) - message += "\n"; - - NativeLibraryConfig.Instance.LogCallback?.Invoke(level, message); - } - - #region CUDA version - private static int GetCudaMajorVersion() - { - string? cudaPath; - string version = ""; - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - cudaPath = Environment.GetEnvironmentVariable("CUDA_PATH"); - if (cudaPath is null) - { - return -1; - } - - //Ensuring cuda bin path is reachable. Especially for MAUI environment. - string cudaBinPath = Path.Combine(cudaPath, "bin"); - - if (Directory.Exists(cudaBinPath)) - { - AddDllDirectory(cudaBinPath); - } - - version = GetCudaVersionFromPath(cudaPath); - } - else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) - { - // Try the default first - cudaPath = "/usr/local/bin/cuda"; - version = GetCudaVersionFromPath(cudaPath); - if (string.IsNullOrEmpty(version)) - { - cudaPath = Environment.GetEnvironmentVariable("LD_LIBRARY_PATH"); - if (cudaPath is null) - { - return -1; - } - foreach (var path in cudaPath.Split(':')) - { - version = GetCudaVersionFromPath(Path.Combine(path, "..")); - if (string.IsNullOrEmpty(version)) - { - break; - } - } - } - } - - if (string.IsNullOrEmpty(version)) - return -1; - - version = version.Split('.')[0]; - if (int.TryParse(version, out var majorVersion)) - return majorVersion; - - return -1; - } - - private static string GetCudaVersionFromPath(string cudaPath) - { - try - { - string json = File.ReadAllText(Path.Combine(cudaPath, cudaVersionFile)); - using (JsonDocument document = JsonDocument.Parse(json)) - { - JsonElement root = document.RootElement; - JsonElement cublasNode = root.GetProperty("libcublas"); - JsonElement versionNode = cublasNode.GetProperty("version"); - if (versionNode.ValueKind == JsonValueKind.Undefined) - { - return string.Empty; - } - return versionNode.GetString() ?? ""; - } - } - catch (Exception) - { - return string.Empty; - } - } - #endregion - -#if NET6_0_OR_GREATER - private static IEnumerable GetLibraryTryOrder(NativeLibraryConfig.Description configuration) - { - var loadingName = configuration.Library.GetLibraryName(); - Log($"Loading library: '{loadingName}'", LLamaLogLevel.Debug); - - // Get platform specific parts of the path (e.g. .so/.dll/.dylib, libName prefix or not) - GetPlatformPathParts(out var platform, out var os, out var ext, out var libPrefix); - Log($"Detected OS Platform: '{platform}'", LLamaLogLevel.Info); - Log($"Detected OS string: '{os}'", LLamaLogLevel.Debug); - Log($"Detected extension string: '{ext}'", LLamaLogLevel.Debug); - Log($"Detected prefix string: '{libPrefix}'", LLamaLogLevel.Debug); - - if (configuration.UseCuda && (platform == OSPlatform.Windows || platform == OSPlatform.Linux)) - { - var cudaVersion = GetCudaMajorVersion(); - Log($"Detected cuda major version {cudaVersion}.", LLamaLogLevel.Info); - - if (cudaVersion == -1 && !configuration.AllowFallback) - { - // if check skipped, we just try to load cuda libraries one by one. - if (configuration.SkipCheck) - { - yield return GetCudaLibraryPath(loadingName, "cuda12"); - yield return GetCudaLibraryPath(loadingName, "cuda11"); - } - else - { - throw new RuntimeError("Configured to load a cuda library but no cuda detected on your device."); - } - } - else if (cudaVersion == 11) - { - yield return GetCudaLibraryPath(loadingName, "cuda11"); - } - else if (cudaVersion == 12) - { - yield return GetCudaLibraryPath(loadingName, "cuda12"); - } - else if (cudaVersion > 0) - { - throw new RuntimeError($"Cuda version {cudaVersion} hasn't been supported by LLamaSharp, please open an issue for it."); - } - - // otherwise no cuda detected but allow fallback - } - - // Add the CPU/Metal libraries - if (platform == OSPlatform.OSX) - { - // On Mac it's very simple, there's no AVX to consider. - yield return GetMacLibraryPath(loadingName); - } - else - { - if (configuration.AllowFallback) - { - // Try all of the AVX levels we can support. - if (configuration.AvxLevel >= NativeLibraryConfig.AvxLevel.Avx512) - yield return GetAvxLibraryPath(loadingName, NativeLibraryConfig.AvxLevel.Avx512); - - if (configuration.AvxLevel >= NativeLibraryConfig.AvxLevel.Avx2) - yield return GetAvxLibraryPath(loadingName, NativeLibraryConfig.AvxLevel.Avx2); - - if (configuration.AvxLevel >= NativeLibraryConfig.AvxLevel.Avx) - yield return GetAvxLibraryPath(loadingName, NativeLibraryConfig.AvxLevel.Avx); - - yield return GetAvxLibraryPath(loadingName, NativeLibraryConfig.AvxLevel.None); - } - else - { - // Fallback is not allowed - use the exact specified AVX level - yield return GetAvxLibraryPath(loadingName, configuration.AvxLevel); - } - } - } - - private static string GetMacLibraryPath(string libraryName) - { - GetPlatformPathParts(out _, out var os, out var fileExtension, out var libPrefix); - - return $"runtimes/{os}/native/{libPrefix}{libraryName}{fileExtension}"; - } - - /// - /// Given a CUDA version and some path parts, create a complete path to the library file - /// - /// Library being loaded (e.g. "llama") - /// CUDA version (e.g. "cuda11") - /// - private static string GetCudaLibraryPath(string libraryName, string cuda) - { - GetPlatformPathParts(out _, out var os, out var fileExtension, out var libPrefix); - - return $"runtimes/{os}/native/{cuda}/{libPrefix}{libraryName}{fileExtension}"; - } - /// - /// Given an AVX level and some path parts, create a complete path to the library file + /// Get the loaded native library. If you are using netstandard2.0, it will always return null. /// - /// Library being loaded (e.g. "llama") - /// + /// /// - private static string GetAvxLibraryPath(string libraryName, NativeLibraryConfig.AvxLevel avx) - { - GetPlatformPathParts(out _, out var os, out var fileExtension, out var libPrefix); - - var avxStr = NativeLibraryConfig.AvxLevelToString(avx); - if (!string.IsNullOrEmpty(avxStr)) - avxStr += "/"; - - return $"runtimes/{os}/native/{avxStr}{libPrefix}{libraryName}{fileExtension}"; - } - - private static void GetPlatformPathParts(out OSPlatform platform, out string os, out string fileExtension, out string libPrefix) - { - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - platform = OSPlatform.Windows; - os = "win-x64"; - fileExtension = ".dll"; - libPrefix = ""; - return; - } - - if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) - { - platform = OSPlatform.Linux; - os = "linux-x64"; - fileExtension = ".so"; - libPrefix = "lib"; - return; - } - - if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) - { - platform = OSPlatform.OSX; - fileExtension = ".dylib"; - - os = System.Runtime.Intrinsics.Arm.ArmBase.Arm64.IsSupported - ? "osx-arm64" - : "osx-x64"; - libPrefix = "lib"; - } - else - { - throw new RuntimeError("Your operating system is not supported, please open an issue in LLamaSharp."); - } - } -#endif - - /// - /// Try to load libllama/llava_shared, using CPU feature detection to try and load a more specialised DLL if possible - /// - /// The library handle to unload later, or IntPtr.Zero if no library was loaded - private static IntPtr TryLoadLibraries(LibraryName lib) + /// + public static INativeLibrary? GetLoadedNativeLibrary(NativeLibraryName name) { -#if NET6_0_OR_GREATER - var configuration = NativeLibraryConfig.CheckAndGatherDescription(lib); - - // Set the flag to ensure the NativeLibraryConfig can no longer be modified - NativeLibraryConfig.LibraryHasLoaded = true; - - // Show the configuration we're working with - Log(configuration.ToString(), LLamaLogLevel.Info); - - // If a specific path is requested, load that or immediately fail - if (!string.IsNullOrEmpty(configuration.Path)) + return name switch { - if (!NativeLibrary.TryLoad(configuration.Path, out var handle)) - throw new RuntimeError($"Failed to load the native library [{configuration.Path}] you specified."); - - Log($"Successfully loaded the library [{configuration.Path}] specified by user", LLamaLogLevel.Info); - return handle; - } - - // Get a list of locations to try loading (in order of preference) - var libraryTryLoadOrder = GetLibraryTryOrder(configuration); - - foreach (var libraryPath in libraryTryLoadOrder) - { - var fullPath = TryFindPath(libraryPath); - Log($"Trying '{fullPath}'", LLamaLogLevel.Debug); - - var result = TryLoad(fullPath); - if (result != IntPtr.Zero) - { - Log($"Loaded '{fullPath}'", LLamaLogLevel.Info); - return result; - } - - Log($"Failed Loading '{fullPath}'", LLamaLogLevel.Info); - } - - if (!configuration.AllowFallback) - { - throw new RuntimeError("Failed to load the library that match your rule, please" + - " 1) check your rule." + - " 2) try to allow fallback." + - " 3) or open an issue if it's expected to be successful."); - } -#endif - - Log($"No library was loaded before calling native apis. " + - $"This is not an error under netstandard2.0 but needs attention with net6 or higher.", LLamaLogLevel.Warning); - return IntPtr.Zero; - -#if NET6_0_OR_GREATER - // Try to load a DLL from the path. - // Returns null if nothing is loaded. - static IntPtr TryLoad(string path) - { - if (NativeLibrary.TryLoad(path, out var handle)) - return handle; - - return IntPtr.Zero; - } - - // Try to find the given file in any of the possible search paths - string TryFindPath(string filename) - { - // Try the configured search directories in the configuration - foreach (var path in configuration.SearchDirectories) - { - var candidate = Path.Combine(path, filename); - if (File.Exists(candidate)) - return candidate; - } - - // Try a few other possible paths - var possiblePathPrefix = new[] { - AppDomain.CurrentDomain.BaseDirectory, - Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location) ?? "" - }; - - foreach (var path in possiblePathPrefix) - { - var candidate = Path.Combine(path, filename); - if (File.Exists(candidate)) - return candidate; - } - - return filename; - } -#endif + NativeLibraryName.LLama => _loadedLLamaLibrary, + NativeLibraryName.LLava => _loadedLLavaLibrary, + _ => throw new ArgumentException($"Library name {name} is not found.") + }; } internal const string libraryName = "llama"; - internal const string llavaLibraryName = "llava_shared"; - private const string cudaVersionFile = "version.json"; + internal const string llavaLibraryName = "llava_shared"; + + private static INativeLibrary? _loadedLLamaLibrary = null; + private static INativeLibrary? _loadedLLavaLibrary = null; } } diff --git a/LLama/Native/NativeApi.cs b/LLama/Native/NativeApi.cs index 715225ed2..8a6491776 100644 --- a/LLama/Native/NativeApi.cs +++ b/LLama/Native/NativeApi.cs @@ -19,9 +19,6 @@ public static void llama_empty_call() llama_max_devices(); } - [DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)] - private static extern int AddDllDirectory(string NewDirectory); - /// /// Get the maximum number of devices supported by llama.cpp /// diff --git a/LLama/Native/NativeLibraryConfig.cs b/LLama/Native/NativeLibraryConfig.cs deleted file mode 100644 index f198b179a..000000000 --- a/LLama/Native/NativeLibraryConfig.cs +++ /dev/null @@ -1,332 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using Microsoft.Extensions.Logging; - -namespace LLama.Native -{ -#if NET6_0_OR_GREATER - /// - /// Allows configuration of the native llama.cpp libraries to load and use. - /// All configuration must be done before using **any** other LLamaSharp methods! - /// - public sealed partial class NativeLibraryConfig - { - private string? _libraryPath; - private string? _libraryPathLLava; - - private bool _useCuda = true; - private AvxLevel _avxLevel; - private bool _allowFallback = true; - private bool _skipCheck = false; - - /// - /// search directory -> priority level, 0 is the lowest. - /// - private readonly List _searchDirectories = new List(); - - #region configurators - /// - /// Load a specified native library as backend for LLamaSharp. - /// When this method is called, all the other configurations will be ignored. - /// - /// The full path to the llama library to load. - /// The full path to the llava library to load. - /// Thrown if `LibraryHasLoaded` is true. - public NativeLibraryConfig WithLibrary(string? llamaPath, string? llavaPath) - { - ThrowIfLoaded(); - - _libraryPath = llamaPath; - _libraryPathLLava = llavaPath; - return this; - } - - /// - /// Configure whether to use cuda backend if possible. - /// - /// - /// - /// Thrown if `LibraryHasLoaded` is true. - public NativeLibraryConfig WithCuda(bool enable = true) - { - ThrowIfLoaded(); - - _useCuda = enable; - return this; - } - - /// - /// Configure the prefferred avx support level of the backend. - /// - /// - /// - /// Thrown if `LibraryHasLoaded` is true. - public NativeLibraryConfig WithAvx(AvxLevel level) - { - ThrowIfLoaded(); - - _avxLevel = level; - return this; - } - - /// - /// Configure whether to allow fallback when there's no match for preferred settings. - /// - /// - /// - /// Thrown if `LibraryHasLoaded` is true. - public NativeLibraryConfig WithAutoFallback(bool enable = true) - { - ThrowIfLoaded(); - - _allowFallback = enable; - return this; - } - - /// - /// Whether to skip the check when you don't allow fallback. This option - /// may be useful under some complex conditions. For example, you're sure - /// you have your cublas configured but LLamaSharp take it as invalid by mistake. - /// - /// - /// - /// Thrown if `LibraryHasLoaded` is true. - public NativeLibraryConfig SkipCheck(bool enable = true) - { - ThrowIfLoaded(); - - _skipCheck = enable; - return this; - } - - /// - /// Add self-defined search directories. Note that the file structure of the added - /// directories must be the same as the default directory. Besides, the directory - /// won't be used recursively. - /// - /// - /// - public NativeLibraryConfig WithSearchDirectories(IEnumerable directories) - { - ThrowIfLoaded(); - - _searchDirectories.AddRange(directories); - return this; - } - - /// - /// Add self-defined search directories. Note that the file structure of the added - /// directories must be the same as the default directory. Besides, the directory - /// won't be used recursively. - /// - /// - /// - public NativeLibraryConfig WithSearchDirectory(string directory) - { - ThrowIfLoaded(); - - _searchDirectories.Add(directory); - return this; - } - #endregion - - internal static Description CheckAndGatherDescription(LibraryName library) - { - if (Instance._allowFallback && Instance._skipCheck) - throw new ArgumentException("Cannot skip the check when fallback is allowed."); - - var path = library switch - { - LibraryName.Llama => Instance._libraryPath, - LibraryName.LlavaShared => Instance._libraryPathLLava, - _ => throw new ArgumentException($"Unknown library name '{library}'", nameof(library)), - }; - - return new Description( - path, - library, - Instance._useCuda, - Instance._avxLevel, - Instance._allowFallback, - Instance._skipCheck, - Instance._searchDirectories.Concat(new[] { "./" }).ToArray() - ); - } - - internal static string AvxLevelToString(AvxLevel level) - { - return level switch - { - AvxLevel.None => string.Empty, - AvxLevel.Avx => "avx", - AvxLevel.Avx2 => "avx2", - AvxLevel.Avx512 => "avx512", - _ => throw new ArgumentException($"Unknown AvxLevel '{level}'") - }; - } - - /// - /// Private constructor prevents new instances of this class being created - /// - private NativeLibraryConfig() - { - // Automatically detect the highest supported AVX level - if (System.Runtime.Intrinsics.X86.Avx.IsSupported) - _avxLevel = AvxLevel.Avx; - if (System.Runtime.Intrinsics.X86.Avx2.IsSupported) - _avxLevel = AvxLevel.Avx2; - - if (CheckAVX512()) - _avxLevel = AvxLevel.Avx512; - } - - private static bool CheckAVX512() - { - if (!System.Runtime.Intrinsics.X86.X86Base.IsSupported) - return false; - - // ReSharper disable UnusedVariable (ebx is used when < NET8) - var (_, ebx, ecx, _) = System.Runtime.Intrinsics.X86.X86Base.CpuId(7, 0); - // ReSharper restore UnusedVariable - - var vnni = (ecx & 0b_1000_0000_0000) != 0; - -#if NET8_0_OR_GREATER - var f = System.Runtime.Intrinsics.X86.Avx512F.IsSupported; - var bw = System.Runtime.Intrinsics.X86.Avx512BW.IsSupported; - var vbmi = System.Runtime.Intrinsics.X86.Avx512Vbmi.IsSupported; -#else - var f = (ebx & (1 << 16)) != 0; - var bw = (ebx & (1 << 30)) != 0; - var vbmi = (ecx & 0b_0000_0000_0010) != 0; -#endif - - return vnni && vbmi && bw && f; - } - - /// - /// Avx support configuration - /// - public enum AvxLevel - { - /// - /// No AVX - /// - None, - - /// - /// Advanced Vector Extensions (supported by most processors after 2011) - /// - Avx, - - /// - /// AVX2 (supported by most processors after 2013) - /// - Avx2, - - /// - /// AVX512 (supported by some processors after 2016, not widely supported) - /// - Avx512, - } - - internal record Description(string? Path, LibraryName Library, bool UseCuda, AvxLevel AvxLevel, bool AllowFallback, bool SkipCheck, string[] SearchDirectories) - { - public override string ToString() - { - string avxLevelString = AvxLevel switch - { - AvxLevel.None => "NoAVX", - AvxLevel.Avx => "AVX", - AvxLevel.Avx2 => "AVX2", - AvxLevel.Avx512 => "AVX512", - _ => "Unknown" - }; - - string searchDirectoriesString = "{ " + string.Join(", ", SearchDirectories) + " }"; - - return $"NativeLibraryConfig Description:\n" + - $"- LibraryName: {Library}\n" + - $"- Path: '{Path}'\n" + - $"- PreferCuda: {UseCuda}\n" + - $"- PreferredAvxLevel: {avxLevelString}\n" + - $"- AllowFallback: {AllowFallback}\n" + - $"- SkipCheck: {SkipCheck}\n" + - $"- SearchDirectories and Priorities: {searchDirectoriesString}"; - } - } - } -#endif - - public sealed partial class NativeLibraryConfig - { - /// - /// Get the config instance - /// - public static NativeLibraryConfig Instance { get; } = new(); - - /// - /// Check if the native library has already been loaded. Configuration cannot be modified if this is true. - /// - public static bool LibraryHasLoaded { get; internal set; } - - internal NativeLogConfig.LLamaLogCallback? LogCallback; - - private static void ThrowIfLoaded() - { - if (LibraryHasLoaded) - throw new InvalidOperationException("NativeLibraryConfig must be configured before using **any** other LLamaSharp methods!"); - } - - /// - /// Set the log callback that will be used for all llama.cpp log messages - /// - /// - /// - public NativeLibraryConfig WithLogCallback(NativeLogConfig.LLamaLogCallback? callback) - { - ThrowIfLoaded(); - - LogCallback = callback; - return this; - } - - /// - /// Set the log callback that will be used for all llama.cpp log messages - /// - /// - /// - public NativeLibraryConfig WithLogCallback(ILogger? logger) - { - ThrowIfLoaded(); - - // Redirect to llama_log_set. This will wrap the logger in a delegate and bind that as the log callback instead. - NativeLogConfig.llama_log_set(logger); - - return this; - } - } - - internal enum LibraryName - { - Llama, - LlavaShared - } - - internal static class LibraryNameExtensions - { - public static string GetLibraryName(this LibraryName name) - { - switch (name) - { - case LibraryName.Llama: - return NativeApi.libraryName; - case LibraryName.LlavaShared: - return NativeApi.llavaLibraryName; - default: - throw new ArgumentOutOfRangeException(nameof(name), name, null); - } - } - } -} diff --git a/LLama/Native/NativeLogConfig.cs b/LLama/Native/NativeLogConfig.cs index ebcd23d47..82b097fb3 100644 --- a/LLama/Native/NativeLogConfig.cs +++ b/LLama/Native/NativeLogConfig.cs @@ -37,7 +37,7 @@ public static class NativeLogConfig public static void llama_log_set(LLamaLogCallback? logCallback) #pragma warning restore IDE1006 // Naming Styles { - if (NativeLibraryConfig.LibraryHasLoaded) + if (NativeLibraryConfig.LLama.LibraryHasLoaded) { // The library is loaded, just pass the callback directly to llama.cpp native_llama_log_set(logCallback);