Upgrade to .NET 8

pavlovtech · Oct 19, 2024 · a79f45b · a79f45b
1 parent 988ea8c
commit a79f45b
Show file tree

Hide file tree

Showing 12 changed files with 85 additions and 80 deletions.
diff --git a/Examples/BrownsfashionScraper/BrownsfashionScraper.csproj b/Examples/BrownsfashionScraper/BrownsfashionScraper.csproj
@@ -1,24 +1,24 @@
 <Project Sdk="Microsoft.NET.Sdk.Worker">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <Nullable>enable</Nullable>
     <ImplicitUsings>enable</ImplicitUsings>
     <UserSecretsId>dotnet-BrownsfashionScraper-626F9B50-CA29-4462-A3BC-2B48D45857DC</UserSecretsId>
     <DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Hosting" Version="7.0.1" />
-    <PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.19.5" />
-	<PackageReference Include="Serilog" Version="3.0.1" />
-	<PackageReference Include="serilog.aspnetcore" Version="7.0.0" />
-	<PackageReference Include="Serilog.Enrichers.Environment" Version="2.2.0" />
-	<PackageReference Include="Serilog.Enrichers.Process" Version="2.0.2" />
-	<PackageReference Include="Serilog.Enrichers.Thread" Version="3.1.0" />
-	<PackageReference Include="Serilog.Sinks.Async" Version="1.5.0" />
-	<PackageReference Include="Serilog.Sinks.Console" Version="4.1.0" />
-	<PackageReference Include="Serilog.Sinks.File" Version="5.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.1" />
+    <PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.21.0" />
+	<PackageReference Include="Serilog" Version="4.0.2" />
+	<PackageReference Include="serilog.aspnetcore" Version="8.0.3" />
+	<PackageReference Include="Serilog.Enrichers.Environment" Version="3.0.1" />
+	<PackageReference Include="Serilog.Enrichers.Process" Version="3.0.0" />
+	<PackageReference Include="Serilog.Enrichers.Thread" Version="4.0.0" />
+	<PackageReference Include="Serilog.Sinks.Async" Version="2.0.0" />
+	<PackageReference Include="Serilog.Sinks.Console" Version="6.0.0" />
+	<PackageReference Include="Serilog.Sinks.File" Version="6.0.0" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/Examples/WebReaper.AzureFuncs/WebReaper.AzureFuncs.csproj b/Examples/WebReaper.AzureFuncs/WebReaper.AzureFuncs.csproj
@@ -1,17 +1,17 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <AzureFunctionsVersion>v4</AzureFunctionsVersion>
-    <LangVersion>11</LangVersion>
+    <LangVersion>12</LangVersion>
   </PropertyGroup>
   <ItemGroup>
     <PackageReference Include="Microsoft.Azure.Functions.Extensions" Version="1.1.0" />
     <PackageReference Include="Microsoft.Azure.WebJobs.Extensions.OpenApi" Version="1.5.1" />
-    <PackageReference Include="Microsoft.Azure.WebJobs.Extensions.ServiceBus" Version="5.11.0" />
-    <PackageReference Include="Microsoft.Extensions.Caching.StackExchangeRedis" Version="7.0.10" />
-    <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="7.0.0" />
-    <PackageReference Include="Microsoft.NET.Sdk.Functions" Version="4.2.0" />
-    <PackageReference Include="StackExchange.Redis" Version="2.6.122" />
+    <PackageReference Include="Microsoft.Azure.WebJobs.Extensions.ServiceBus" Version="5.16.4" />
+    <PackageReference Include="Microsoft.Extensions.Caching.StackExchangeRedis" Version="8.0.10" />
+    <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="8.0.1" />
+    <PackageReference Include="Microsoft.NET.Sdk.Functions" Version="4.5.0" />
+    <PackageReference Include="StackExchange.Redis" Version="2.8.16" />
   </ItemGroup>
   <ItemGroup>
     <None Update="host.json">

diff --git a/Examples/WebReaper.ConsoleApplication/WebReaper.ConsoleApplication.csproj b/Examples/WebReaper.ConsoleApplication/WebReaper.ConsoleApplication.csproj
@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
     <LangVersion>11</LangVersion>

diff --git a/...ebReaper.DistributedScraperWorkerService/WebReaper.DistributedScraperWorkerService.csproj b/...ebReaper.DistributedScraperWorkerService/WebReaper.DistributedScraperWorkerService.csproj
@@ -1,27 +1,27 @@
 <Project Sdk="Microsoft.NET.Sdk.Worker">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <Nullable>enable</Nullable>
     <ImplicitUsings>enable</ImplicitUsings>
     <UserSecretsId>dotnet-DistributedScraperWorkerService-BDA4CDE4-6E87-4C55-87B5-568C510A73A4</UserSecretsId>
-    <LangVersion>11</LangVersion>
+    <LangVersion>12</LangVersion>
   </PropertyGroup>
 
   <PropertyGroup Condition=" '$(RunConfiguration)' == 'DistributedScraperWorkerService' ">
     <ExternalConsole>true</ExternalConsole>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Hosting" Version="7.0.1" />
-    <PackageReference Include="Serilog" Version="3.0.1" />
-    <PackageReference Include="serilog.aspnetcore" Version="7.0.0" />
-    <PackageReference Include="Serilog.Enrichers.Environment" Version="2.2.0" />
-    <PackageReference Include="Serilog.Enrichers.Process" Version="2.0.2" />
-    <PackageReference Include="Serilog.Enrichers.Thread" Version="3.1.0" />
-    <PackageReference Include="Serilog.Sinks.Async" Version="1.5.0" />
-    <PackageReference Include="Serilog.Sinks.Console" Version="4.1.0" />
-    <PackageReference Include="Serilog.Sinks.File" Version="5.0.0" />
-    <PackageReference Include="Serilog.Sinks.Seq" Version="5.2.2" />
+    <PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.1" />
+    <PackageReference Include="Serilog" Version="4.0.2" />
+    <PackageReference Include="serilog.aspnetcore" Version="8.0.3" />
+    <PackageReference Include="Serilog.Enrichers.Environment" Version="3.0.1" />
+    <PackageReference Include="Serilog.Enrichers.Process" Version="3.0.0" />
+    <PackageReference Include="Serilog.Enrichers.Thread" Version="4.0.0" />
+    <PackageReference Include="Serilog.Sinks.Async" Version="2.0.0" />
+    <PackageReference Include="Serilog.Sinks.Console" Version="6.0.0" />
+    <PackageReference Include="Serilog.Sinks.File" Version="6.0.0" />
+    <PackageReference Include="Serilog.Sinks.Seq" Version="8.0.0" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/Examples/WebReaper.ScraperWorkerService/WebReaper.ScraperWorkerService.csproj b/Examples/WebReaper.ScraperWorkerService/WebReaper.ScraperWorkerService.csproj
@@ -1,27 +1,27 @@
 <Project Sdk="Microsoft.NET.Sdk.Worker">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <Nullable>enable</Nullable>
     <ImplicitUsings>enable</ImplicitUsings>
     <UserSecretsId>dotnet-ScraperWorkerService-D9156731-5295-4079-A291-FD567AEC1775</UserSecretsId>
-    <LangVersion>11</LangVersion>
+    <LangVersion>12</LangVersion>
   </PropertyGroup>
 
   <PropertyGroup Condition=" '$(RunConfiguration)' == 'ScraperWorkerService' ">
     <ExternalConsole>true</ExternalConsole>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Hosting" Version="7.0.1" />
-    <PackageReference Include="Serilog" Version="3.0.1" />
-    <PackageReference Include="serilog.aspnetcore" Version="7.0.0" />
-    <PackageReference Include="Serilog.Enrichers.Environment" Version="2.2.0" />
-    <PackageReference Include="Serilog.Enrichers.Process" Version="2.0.2" />
-    <PackageReference Include="Serilog.Enrichers.Thread" Version="3.1.0" />
-    <PackageReference Include="Serilog.Sinks.Async" Version="1.5.0" />
-    <PackageReference Include="Serilog.Sinks.Console" Version="4.1.0" />
-    <PackageReference Include="Serilog.Sinks.File" Version="5.0.0" />
-    <PackageReference Include="Serilog.Sinks.Seq" Version="5.2.2" />
+    <PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.1" />
+    <PackageReference Include="Serilog" Version="4.0.2" />
+    <PackageReference Include="serilog.aspnetcore" Version="8.0.3" />
+    <PackageReference Include="Serilog.Enrichers.Environment" Version="3.0.1" />
+    <PackageReference Include="Serilog.Enrichers.Process" Version="3.0.0" />
+    <PackageReference Include="Serilog.Enrichers.Thread" Version="4.0.0" />
+    <PackageReference Include="Serilog.Sinks.Async" Version="2.0.0" />
+    <PackageReference Include="Serilog.Sinks.Console" Version="6.0.0" />
+    <PackageReference Include="Serilog.Sinks.File" Version="6.0.0" />
+    <PackageReference Include="Serilog.Sinks.Seq" Version="8.0.0" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/Misc/WebReaper.ProxyProviders/WebReaper.ProxyProviders.csproj b/Misc/WebReaper.ProxyProviders/WebReaper.ProxyProviders.csproj
@@ -1,10 +1,10 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
-    <LangVersion>11</LangVersion>
+    <LangVersion>12</LangVersion>
   </PropertyGroup>
 
   <ItemGroup>

diff --git a/WebReaper.Tests/WebReaper.IntegrationTests/ScraperTests.cs b/WebReaper.Tests/WebReaper.IntegrationTests/ScraperTests.cs
@@ -108,7 +108,7 @@ public async Task SimpleTestWithSPA()
                 Path = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)
             });
 
-            await browserFetcher.DownloadAsync(BrowserFetcher.DefaultChromiumRevision);
+            await browserFetcher.DownloadAsync();
 
             var result = new List<ParsedData>();
 
@@ -127,7 +127,7 @@ public async Task SimpleTestWithSPA()
 
             _ = engine.RunAsync();
 
-            await Task.Delay(20000);
+            await Task.Delay(40000);
 
             Assert.NotEmpty(result);
         }

diff --git a/WebReaper.Tests/WebReaper.IntegrationTests/WebReaper.IntegrationTests.csproj b/WebReaper.Tests/WebReaper.IntegrationTests/WebReaper.IntegrationTests.csproj
@@ -1,23 +1,23 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
 
     <IsPackable>false</IsPackable>
 
-    <LangVersion>11</LangVersion>
+    <LangVersion>12</LangVersion>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.7.0" />
-    <PackageReference Include="xunit" Version="2.5.0" />
-    <PackageReference Include="xunit.runner.visualstudio" Version="2.5.0">
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
+    <PackageReference Include="xunit" Version="2.9.2" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.8.2">
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
-    <PackageReference Include="coverlet.collector" Version="6.0.0">
+    <PackageReference Include="coverlet.collector" Version="6.0.2">
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>

diff --git a/WebReaper.Tests/WebReaper.UnitTests/WebReaper.UnitTests.csproj b/WebReaper.Tests/WebReaper.UnitTests/WebReaper.UnitTests.csproj
@@ -1,24 +1,24 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
 
     <IsPackable>false</IsPackable>
 
-    <LangVersion>11</LangVersion>
+    <LangVersion>12</LangVersion>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.7.0" />
-    <PackageReference Include="System.Text.Encoding.CodePages" Version="7.0.0" />
-    <PackageReference Include="xunit" Version="2.5.0" />
-    <PackageReference Include="xunit.runner.visualstudio" Version="2.5.0">
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
+    <PackageReference Include="System.Text.Encoding.CodePages" Version="8.0.0" />
+    <PackageReference Include="xunit" Version="2.9.2" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.8.2">
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
-    <PackageReference Include="coverlet.collector" Version="6.0.0">
+    <PackageReference Include="coverlet.collector" Version="6.0.2">
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>

diff --git a/WebReaper/Core/Loaders/Concrete/PuppeteerPageLoader.cs b/WebReaper/Core/Loaders/Concrete/PuppeteerPageLoader.cs
@@ -1,6 +1,7 @@
 using System.Reflection;
 using Microsoft.Extensions.Logging;
 using PuppeteerSharp;
+using PuppeteerSharp.BrowserData;
 using WebReaper.Core.CookieStorage.Abstract;
 using WebReaper.Core.Loaders.Abstract;
 using WebReaper.Domain.PageActions;
@@ -30,10 +31,13 @@ public async Task<string> Load(string url, List<PageAction>? pageActions = null,
         });
 
         await _semaphore.WaitAsync();
+
+        InstalledBrowser installedBrowser;
+
         try
         {
             Logger.LogInformation("{class}.{method}: Downloading browser...", nameof(PuppeteerPageLoader), nameof(Load));
-            await browserFetcher.DownloadAsync(BrowserFetcher.DefaultChromiumRevision);
+            installedBrowser = await browserFetcher.DownloadAsync();
             Logger.LogInformation("{class}.{method}: Browser is downloaded", nameof(PuppeteerPageLoader), nameof(Load));
         }
         finally
@@ -45,7 +49,8 @@ public async Task<string> Load(string url, List<PageAction>? pageActions = null,
         await using var browser = await Puppeteer.LaunchAsync(new LaunchOptions
         {
             Headless = headless,
-            ExecutablePath = browserFetcher.RevisionInfo(BrowserFetcher.DefaultChromiumRevision).ExecutablePath
+            ExecutablePath = browserFetcher.GetExecutablePath(installedBrowser.BuildId),
+            Args = new[] { "--ignore-certificate-errors" }
         });
 
         Logger.LogInformation("{class}.{method}: creating a new page", nameof(PuppeteerPageLoader), nameof(Load));

diff --git a/WebReaper/Core/Loaders/Concrete/PuppeteerPageLoaderWithProxies.cs b/WebReaper/Core/Loaders/Concrete/PuppeteerPageLoaderWithProxies.cs
@@ -3,6 +3,7 @@
 using PuppeteerExtraSharp;
 using PuppeteerExtraSharp.Plugins.ExtraStealth;
 using PuppeteerSharp;
+using PuppeteerSharp.BrowserData;
 using WebReaper.Core.CookieStorage.Abstract;
 using WebReaper.Core.Loaders.Abstract;
 using WebReaper.Domain.PageActions;
@@ -35,9 +36,12 @@ public async Task<string> Load(string url, List<PageAction>? pageActions = null,
         });
 
         await _semaphore.WaitAsync();
+
+        InstalledBrowser installedBrowser;
+
         try
         {
-            await browserFetcher.DownloadAsync(BrowserFetcher.DefaultChromiumRevision);
+            installedBrowser = await browserFetcher.DownloadAsync();
         }
         finally
         {
@@ -52,7 +56,7 @@ public async Task<string> Load(string url, List<PageAction>? pageActions = null,
         await using var browser = await puppeteerExtra.LaunchAsync(new LaunchOptions
         {
             Headless = headless,
-            ExecutablePath = browserFetcher.RevisionInfo(BrowserFetcher.DefaultChromiumRevision).ExecutablePath,
+            ExecutablePath = browserFetcher.GetExecutablePath(installedBrowser.BuildId),
             Args = new[]
             {
                 "--disable-dev-shm-usage",

diff --git a/WebReaper/WebReaper.csproj b/WebReaper/WebReaper.csproj
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
     <PropertyGroup>
-        <TargetFramework>net7.0</TargetFramework>
+        <TargetFramework>net8.0</TargetFramework>
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
         <Authors>Alex Pavlov</Authors>
@@ -19,7 +19,7 @@
         <Version>3.5.1</Version>
         <GenerateDocumentationFile>true</GenerateDocumentationFile>
         <DocumentationFile>$(MSBuildProjectDirectory)\API.xml</DocumentationFile>
-        <LangVersion>11</LangVersion>
+        <LangVersion>12</LangVersion>
         <Title>WebReaper</Title>
     </PropertyGroup>
 
@@ -40,22 +40,18 @@
         <None Remove="Azure.Messaging.ServiceBus" />
     </ItemGroup>
     <ItemGroup>
-        <PackageReference Include="AngleSharp" Version="1.0.4" />
-        <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.35.2" />
-        <PackageReference Include="Microsoft.Extensions.Http" Version="7.0.0" />
-        <PackageReference Include="MongoDB.Driver" Version="2.20.0" />
+        <PackageReference Include="AngleSharp" Version="1.1.2" />
+        <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.44.1" />
+        <PackageReference Include="Microsoft.Extensions.Http" Version="8.0.1" />
+        <PackageReference Include="MongoDB.Driver" Version="3.0.0" />
         <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
-        <PackageReference Include="Polly" Version="7.2.4" />
+        <PackageReference Include="Polly" Version="8.4.2" />
         <PackageReference Include="PuppeteerExtraSharp" Version="2.0.0" />
-        <PackageReference Include="PuppeteerSharp" Version="10.1.2" />
-        <PackageReference Include="StackExchange.Redis" Version="2.6.122" />
-        <PackageReference Include="System.Text.Encoding.CodePages" Version="7.0.0" />
-        <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="7.0.1" />
-        <PackageReference Include="Azure.Messaging.ServiceBus" Version="7.16.0" />
-        <PackageReference Include="Vsxmd" Version="1.4.5">
-            <PrivateAssets>all</PrivateAssets>
-            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
-        </PackageReference>
+        <PackageReference Include="PuppeteerSharp" Version="20.0.2" />
+        <PackageReference Include="StackExchange.Redis" Version="2.8.16" />
+        <PackageReference Include="System.Text.Encoding.CodePages" Version="8.0.0" />
+        <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.2" />
+        <PackageReference Include="Azure.Messaging.ServiceBus" Version="7.18.2" />
     </ItemGroup>
     <ItemGroup>
         <None Include="..\README.md">