feat: duckduckgo search and websearch retriever (#87)

Co-authored-by: Evgenii Khoroshev <[email protected]>

src/libs/LangChain.Core/LangChain.Core.csproj

@@ -14,10 +14,6 @@
     <None Remove="Chains\QuestionAnswering\**" />
   </ItemGroup>
 
-  <ItemGroup Label="Usings">
-    <Using Remove="System.Net.Http" />
-  </ItemGroup>
-
   <PropertyGroup Label="NuGet">
     <Description>LangChain core classes.</Description>
     <PackageTags>$(PackageTags);core</PackageTags>
@@ -32,6 +28,7 @@
 
   <ItemGroup Condition="'$(TargetFramework)'=='net4.6.2'">
     <PackageReference Include="System.Text.Json" />
+    <PackageReference Include="System.Net.Http" />
   </ItemGroup>
 
   <ItemGroup Condition="'$(TargetFramework)'=='netstandard2.0'">

src/libs/LangChain.Core/Retrievers/WebSearchRetriever.cs

@@ -0,0 +1,32 @@
+using LangChain.Callback;
+using LangChain.Docstore;
+using LangChain.Utilities;
+
+namespace LangChain.Retrievers;
+
+public sealed class WebSearchRetriever : BaseRetriever
+{
+    private readonly IWebSearch _webSearch;
+    private readonly int _k;
+
+    public WebSearchRetriever(IWebSearch webSearch, int k = 10)
+    {
+        _webSearch = webSearch;
+        _k = k;
+    }
+
+    protected override async Task<IEnumerable<Document>> GetRelevantDocumentsCoreAsync(
+        string query,
+        CallbackManagerForRetrieverRun runManager = null)
+    {
+        var searchResult = await _webSearch.ResultsAsync(query, _k);
+
+        return searchResult.Select(v => new Document(
+            v.Body,
+            new Dictionary<string, object>()
+            {
+                ["title"] = v.Title,
+                ["link"] = v.Link
+            }));
+    }
+}