Merge branch 'master' into auto_download

.gitignore

@@ -347,4 +347,6 @@ site/
 /LLama.Unittest/Models/*.gguf
 
 /LLama.Benchmark/Models/*.bin
-/LLama.Benchmark/Models/*.gguf
+/LLama.Benchmark/Models/*.gguf
+
+**/appsettings.Local.json

LLama.Examples/Examples/BatchedExecutorFork.cs

@@ -1,4 +1,4 @@
-using LLama.Batched;
+using LLama.Batched;
 using LLama.Common;
 using LLama.Native;
 using LLama.Sampling;
@@ -67,6 +67,13 @@ await AnsiConsole
                 root.Display(display);
                 AnsiConsole.Write(display);
             });
+
+        // Print some stats
+        var timings = executor.Context.NativeHandle.GetTimings();
+        AnsiConsole.MarkupLine($"Total Tokens Evaluated: {timings.TokensEvaluated}");
+        AnsiConsole.MarkupLine($"Total Tokens Sampled: {timings.TokensSampled}");
+        AnsiConsole.MarkupLine($"Eval Time: {(timings.Eval + timings.PromptEval).TotalMilliseconds}ms");
+        AnsiConsole.MarkupLine($"Sample Time: {timings.Sampling.TotalMilliseconds}ms");
     }
 
     private class Node

LLama.Examples/Examples/BatchedExecutorGuidance.cs

@@ -1,4 +1,4 @@
-using LLama.Batched;
+using LLama.Batched;
 using LLama.Common;
 using LLama.Native;
 using LLama.Sampling;
@@ -105,18 +105,19 @@ public override ISamplingPipeline Clone()
 
         protected override void ProcessLogits(SafeLLamaContextHandle ctx, Span<float> logits, ReadOnlySpan<LLamaToken> lastTokens)
         {
-            if (guidance == null)
-                return;
-
-            // Get the logits generated by the guidance sequences
-            var guidanceLogits = guidance.Sample();
-
-            // Use those logits to guide this sequence
-            NativeApi.llama_sample_apply_guidance(ctx, logits, guidanceLogits, weight);
         }
 
         protected override LLamaToken ProcessTokenDataArray(SafeLLamaContextHandle ctx, LLamaTokenDataArray candidates, ReadOnlySpan<LLamaToken> lastTokens)
         {
+            if (guidance != null)
+            {
+                // Get the logits generated by the guidance sequences
+                var guidanceLogits = guidance.Sample();
+
+                // Modify these logits based on the guidance logits
+                candidates.Guidance(ctx, guidanceLogits, weight);
+            }
+
             candidates.Temperature(ctx, 0.8f);
             candidates.TopK(ctx, 25);
 

LLama.KernelMemory/BuilderExtensions.cs

@@ -1,4 +1,4 @@
-using Microsoft.KernelMemory;
+using Microsoft.KernelMemory;
 using System;
 using System.Collections.Generic;
 using System.Linq;
@@ -81,23 +81,22 @@ public static IKernelMemoryBuilder WithLLamaSharpDefaults(this IKernelMemoryBuil
         {
             var parameters = new ModelParams(config.ModelPath)
             {
-                ContextSize = config?.ContextSize ?? 2048,
-                Seed = config?.Seed ?? 0,
-                GpuLayerCount = config?.GpuLayerCount ?? 20,
+                ContextSize = config.ContextSize ?? 2048,
+                Seed = config.Seed ?? 0,
+                GpuLayerCount = config.GpuLayerCount ?? 20,
                 Embeddings = true,
-                MainGpu = config?.MainGpu ?? 0,
-                SplitMode = config?.SplitMode ?? GPUSplitMode.None,
+                MainGpu = config.MainGpu,
+                SplitMode = config.SplitMode
             };
 
-            if (weights == null)
+            if (weights == null || context == null)
             {
                 weights = LLamaWeights.LoadFromFile(parameters);
                 context = weights.CreateContext(parameters);
             }
 
             var executor = new StatelessExecutor(weights, parameters);
-            var embedder = new LLamaEmbedder(weights, parameters);
-            builder.WithLLamaSharpTextEmbeddingGeneration(new LLamaSharpTextEmbeddingGenerator(embedder));
+            builder.WithLLamaSharpTextEmbeddingGeneration(new LLamaSharpTextEmbeddingGenerator(config, weights));
             builder.WithLLamaSharpTextGeneration(new LlamaSharpTextGenerator(weights, context, executor, config?.DefaultInferenceParams));
             return builder;
         }		

LLama.KernelMemory/LLamaSharp.KernelMemory.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFrameworks>net6.0;net7.0;net8.0</TargetFrameworks>
+    <TargetFrameworks>net6.0;net8.0</TargetFrameworks>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
     <Version>0.12.0</Version>

LLama.KernelMemory/LLamaSharpTextEmbeddingGenerator.cs

@@ -1,5 +1,6 @@
-using LLama;
+using LLama;
 using LLama.Common;
+using LLama.Native;
 using Microsoft.KernelMemory;
 using Microsoft.KernelMemory.AI;
 
@@ -29,6 +30,9 @@ public LLamaSharpTextEmbeddingGenerator(LLamaSharpConfig config)
             this._config = config;
             var @params = new ModelParams(_config.ModelPath)
             {
+                ContextSize = config.ContextSize ?? 2048,
+                Seed = config.Seed ?? 0,
+                GpuLayerCount = config.GpuLayerCount ?? 20,
                 Embeddings = true,
                 MainGpu = _config.MainGpu,
                 SplitMode = _config.SplitMode
@@ -49,6 +53,9 @@ public LLamaSharpTextEmbeddingGenerator(LLamaSharpConfig config, LLamaWeights we
             this._config = config;
             var @params = new ModelParams(_config.ModelPath)
             {
+                ContextSize = config.ContextSize ?? 2048,
+                Seed = config.Seed ?? 0,
+                GpuLayerCount = config.GpuLayerCount ?? 20,
                 Embeddings = true,
                 MainGpu = _config.MainGpu,
                 SplitMode = _config.SplitMode

LLama.SemanticKernel/LLamaSharp.SemanticKernel.csproj

@@ -1,7 +1,7 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
 	<PropertyGroup>
-		<TargetFrameworks>netstandard2.0;net6.0;net7.0</TargetFrameworks>
+		<TargetFrameworks>netstandard2.0;net6.0;net8.0</TargetFrameworks>
 		<RootNamespace>LLamaSharp.SemanticKernel</RootNamespace>
 		<Nullable>enable</Nullable>
 		<LangVersion>10</LangVersion>

LLama.SemanticKernel/README.md

@@ -1,6 +1,6 @@
 # LLamaSharp.SemanticKernel
 
-LLamaSharp.SemanticKernel are connections for [SemanticKernel](https://github.com/microsoft/semantic-kernel): an SDK for integrating various LLM interfaces into a single implementation. With this, you can add local LLaMa queries as another connection point with your existing connections.
+LLamaSharp.SemanticKernel are connections for [SemanticKernel](https://github.com/microsoft/semantic-kernel): an SDK for integrating various LLM interfaces into a single implementation. With this, you can add local LLaMA queries as another connection point with your existing connections.
 
 For reference on how to implement it, view the following examples: 
 

LLama.Web/Common/ISessionConfig.cs

@@ -1,13 +1,23 @@
-namespace LLama.Web.Common
+using System.ComponentModel;
+
+namespace LLama.Web.Common;
+
+public interface ISessionConfig
 {
-    public interface ISessionConfig
-    {
-        string AntiPrompt { get; set; }
-        List<string> AntiPrompts { get; set; }
-        LLamaExecutorType ExecutorType { get; set; }
-        string Model { get; set; }
-        string OutputFilter { get; set; }
-        List<string> OutputFilters { get; set; }
-        string Prompt { get; set; }
-    }
+    string AntiPrompt { get; set; }
+
+    [DisplayName("Anti Prompts")]
+    List<string> AntiPrompts { get; set; }
+
+    [DisplayName("Executor Type")]
+    LLamaExecutorType ExecutorType { get; set; }
+
+    string Model { get; set; }
+
+    [DisplayName("Output Filter")]
+    string OutputFilter { get; set; }
+
+    List<string> OutputFilters { get; set; }
+
+    string Prompt { get; set; }
 }

LLama.Web/Common/LLamaOptions.cs

@@ -1,12 +1,7 @@
-namespace LLama.Web.Common
-{
-    public class LLamaOptions
-    {
-        public ModelLoadType ModelLoadType { get; set; }
-        public List<ModelOptions> Models { get; set; }
+namespace LLama.Web.Common;
 
-        public void Initialize()
-        {
-        }
-    }
+public class LLamaOptions
+{
+    public ModelLoadType ModelLoadType { get; set; }
+    public List<ModelOptions> Models { get; set; }
 }

LLama.Web/Common/SessionConfig.cs

@@ -1,14 +1,13 @@
-namespace LLama.Web.Common
+namespace LLama.Web.Common;
+
+public class SessionConfig : ISessionConfig
 {
-    public class SessionConfig : ISessionConfig
-    {
-        public string Model { get; set; }
-        public string Prompt { get; set; }
+    public string Model { get; set; }
+    public string Prompt { get; set; }
 
-        public string AntiPrompt { get; set; }
-        public List<string> AntiPrompts { get; set; }
-        public string OutputFilter { get; set; }
-        public List<string> OutputFilters { get; set; }
-        public LLamaExecutorType ExecutorType { get; set; }
-    }
+    public string AntiPrompt { get; set; }
+    public List<string> AntiPrompts { get; set; }
+    public string OutputFilter { get; set; }
+    public List<string> OutputFilters { get; set; }
+    public LLamaExecutorType ExecutorType { get; set; }
 }

LLama.Web/Extensions.cs

@@ -1,54 +1,52 @@
-using LLama.Web.Common;
+using LLama.Web.Common;
 
-namespace LLama.Web
+namespace LLama.Web;
+
+public static  class Extensions
 {
-    public static  class Extensions
+    /// <summary>
+    /// Combines the AntiPrompts list and AntiPrompt csv 
+    /// </summary>
+    /// <param name="sessionConfig">The session configuration.</param>
+    /// <returns>Combined AntiPrompts with duplicates removed</returns>
+    public static List<string> GetAntiPrompts(this ISessionConfig sessionConfig)
     {
-        /// <summary>
-        /// Combines the AntiPrompts list and AntiPrompt csv 
-        /// </summary>
-        /// <param name="sessionConfig">The session configuration.</param>
-        /// <returns>Combined AntiPrompts with duplicates removed</returns>
-        public static List<string> GetAntiPrompts(this ISessionConfig sessionConfig)
-        {
-            return CombineCSV(sessionConfig.AntiPrompts, sessionConfig.AntiPrompt);
-        }
-
-        /// <summary>
-        /// Combines the OutputFilters list and OutputFilter csv 
-        /// </summary>
-        /// <param name="sessionConfig">The session configuration.</param>
-        /// <returns>Combined OutputFilters with duplicates removed</returns>
-        public static List<string> GetOutputFilters(this ISessionConfig sessionConfig)
-        {
-            return CombineCSV(sessionConfig.OutputFilters, sessionConfig.OutputFilter);
-        }
+        return CombineCSV(sessionConfig.AntiPrompts, sessionConfig.AntiPrompt);
+    }
 
+    /// <summary>
+    /// Combines the OutputFilters list and OutputFilter csv 
+    /// </summary>
+    /// <param name="sessionConfig">The session configuration.</param>
+    /// <returns>Combined OutputFilters with duplicates removed</returns>
+    public static List<string> GetOutputFilters(this ISessionConfig sessionConfig)
+    {
+        return CombineCSV(sessionConfig.OutputFilters, sessionConfig.OutputFilter);
+    }
 
-        /// <summary>
-        /// Combines a string list and a csv and removes duplicates
-        /// </summary>
-        /// <param name="list">The list.</param>
-        /// <param name="csv">The CSV.</param>
-        /// <returns>Combined list with duplicates removed</returns>
-        private static List<string> CombineCSV(List<string> list, string csv)
-        {
-            var results = list is null || list.Count == 0
-                ? CommaSeparatedToList(csv)
-                : CommaSeparatedToList(csv).Concat(list);
-            return results
-                .Distinct()
-                .ToList();
-        }
+    /// <summary>
+    /// Combines a string list and a csv and removes duplicates
+    /// </summary>
+    /// <param name="list">The list.</param>
+    /// <param name="csv">The CSV.</param>
+    /// <returns>Combined list with duplicates removed</returns>
+    private static List<string> CombineCSV(List<string> list, string csv)
+    {
+        var results = list is null || list.Count == 0
+            ? CommaSeparatedToList(csv)
+            : CommaSeparatedToList(csv).Concat(list);
+        return results
+            .Distinct()
+            .ToList();
+    }
 
-        private static List<string> CommaSeparatedToList(string value)
-        {
-            if (string.IsNullOrEmpty(value))
-                return new List<string>();
+    private static List<string> CommaSeparatedToList(string value)
+    {
+        if (string.IsNullOrEmpty(value))
+            return new List<string>();
 
-            return value.Split(",", StringSplitOptions.RemoveEmptyEntries)
-                 .Select(x => x.Trim())
-                 .ToList();
-        }
+        return value.Split(",", StringSplitOptions.RemoveEmptyEntries)
+             .Select(x => x.Trim())
+             .ToList();
     }
 }