diff --git a/Examples/BrownsfashionScraper/BrownsfashionScraper.csproj b/Examples/BrownsfashionScraper/BrownsfashionScraper.csproj index 378dddc..d3f7f14 100644 --- a/Examples/BrownsfashionScraper/BrownsfashionScraper.csproj +++ b/Examples/BrownsfashionScraper/BrownsfashionScraper.csproj @@ -1,7 +1,7 @@  - net7.0 + net8.0 enable enable dotnet-BrownsfashionScraper-626F9B50-CA29-4462-A3BC-2B48D45857DC @@ -9,16 +9,16 @@ - - - - - - - - - - + + + + + + + + + + diff --git a/Examples/WebReaper.AzureFuncs/WebReaper.AzureFuncs.csproj b/Examples/WebReaper.AzureFuncs/WebReaper.AzureFuncs.csproj index 36f6f2e..7648a2c 100644 --- a/Examples/WebReaper.AzureFuncs/WebReaper.AzureFuncs.csproj +++ b/Examples/WebReaper.AzureFuncs/WebReaper.AzureFuncs.csproj @@ -1,17 +1,17 @@  - net7.0 + net8.0 v4 - 11 + 12 - - - - - + + + + + diff --git a/Examples/WebReaper.ConsoleApplication/WebReaper.ConsoleApplication.csproj b/Examples/WebReaper.ConsoleApplication/WebReaper.ConsoleApplication.csproj index 34d73fc..4f565bb 100644 --- a/Examples/WebReaper.ConsoleApplication/WebReaper.ConsoleApplication.csproj +++ b/Examples/WebReaper.ConsoleApplication/WebReaper.ConsoleApplication.csproj @@ -2,7 +2,7 @@ Exe - net7.0 + net8.0 enable enable 11 diff --git a/Examples/WebReaper.DistributedScraperWorkerService/WebReaper.DistributedScraperWorkerService.csproj b/Examples/WebReaper.DistributedScraperWorkerService/WebReaper.DistributedScraperWorkerService.csproj index 5d5537b..8281aa0 100644 --- a/Examples/WebReaper.DistributedScraperWorkerService/WebReaper.DistributedScraperWorkerService.csproj +++ b/Examples/WebReaper.DistributedScraperWorkerService/WebReaper.DistributedScraperWorkerService.csproj @@ -1,27 +1,27 @@  - net7.0 + net8.0 enable enable dotnet-DistributedScraperWorkerService-BDA4CDE4-6E87-4C55-87B5-568C510A73A4 - 11 + 12 true - - - - - - - - - - + + + + + + + + + + diff --git a/Examples/WebReaper.ScraperWorkerService/WebReaper.ScraperWorkerService.csproj b/Examples/WebReaper.ScraperWorkerService/WebReaper.ScraperWorkerService.csproj index f757665..07aa051 100644 --- a/Examples/WebReaper.ScraperWorkerService/WebReaper.ScraperWorkerService.csproj +++ b/Examples/WebReaper.ScraperWorkerService/WebReaper.ScraperWorkerService.csproj @@ -1,27 +1,27 @@  - net7.0 + net8.0 enable enable dotnet-ScraperWorkerService-D9156731-5295-4079-A291-FD567AEC1775 - 11 + 12 true - - - - - - - - - - + + + + + + + + + + diff --git a/Misc/WebReaper.ProxyProviders/WebReaper.ProxyProviders.csproj b/Misc/WebReaper.ProxyProviders/WebReaper.ProxyProviders.csproj index 0f64150..c0b65b1 100644 --- a/Misc/WebReaper.ProxyProviders/WebReaper.ProxyProviders.csproj +++ b/Misc/WebReaper.ProxyProviders/WebReaper.ProxyProviders.csproj @@ -1,10 +1,10 @@  - net7.0 + net8.0 enable enable - 11 + 12 diff --git a/WebReaper.Tests/WebReaper.IntegrationTests/ScraperTests.cs b/WebReaper.Tests/WebReaper.IntegrationTests/ScraperTests.cs index f2860f0..382d4ac 100644 --- a/WebReaper.Tests/WebReaper.IntegrationTests/ScraperTests.cs +++ b/WebReaper.Tests/WebReaper.IntegrationTests/ScraperTests.cs @@ -108,7 +108,7 @@ public async Task SimpleTestWithSPA() Path = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) }); - await browserFetcher.DownloadAsync(BrowserFetcher.DefaultChromiumRevision); + await browserFetcher.DownloadAsync(); var result = new List(); @@ -127,7 +127,7 @@ public async Task SimpleTestWithSPA() _ = engine.RunAsync(); - await Task.Delay(20000); + await Task.Delay(40000); Assert.NotEmpty(result); } diff --git a/WebReaper.Tests/WebReaper.IntegrationTests/WebReaper.IntegrationTests.csproj b/WebReaper.Tests/WebReaper.IntegrationTests/WebReaper.IntegrationTests.csproj index 61044d8..4d908d2 100644 --- a/WebReaper.Tests/WebReaper.IntegrationTests/WebReaper.IntegrationTests.csproj +++ b/WebReaper.Tests/WebReaper.IntegrationTests/WebReaper.IntegrationTests.csproj @@ -1,23 +1,23 @@ - net7.0 + net8.0 enable enable false - 11 + 12 - - - + + + runtime; build; native; contentfiles; analyzers; buildtransitive all - + runtime; build; native; contentfiles; analyzers; buildtransitive all diff --git a/WebReaper.Tests/WebReaper.UnitTests/WebReaper.UnitTests.csproj b/WebReaper.Tests/WebReaper.UnitTests/WebReaper.UnitTests.csproj index fc7719b..4783623 100644 --- a/WebReaper.Tests/WebReaper.UnitTests/WebReaper.UnitTests.csproj +++ b/WebReaper.Tests/WebReaper.UnitTests/WebReaper.UnitTests.csproj @@ -1,24 +1,24 @@  - net7.0 + net8.0 enable enable false - 11 + 12 - - - - + + + + runtime; build; native; contentfiles; analyzers; buildtransitive all - + runtime; build; native; contentfiles; analyzers; buildtransitive all diff --git a/WebReaper/Core/Loaders/Concrete/PuppeteerPageLoader.cs b/WebReaper/Core/Loaders/Concrete/PuppeteerPageLoader.cs index 5b8e54e..1040df9 100644 --- a/WebReaper/Core/Loaders/Concrete/PuppeteerPageLoader.cs +++ b/WebReaper/Core/Loaders/Concrete/PuppeteerPageLoader.cs @@ -1,6 +1,7 @@ using System.Reflection; using Microsoft.Extensions.Logging; using PuppeteerSharp; +using PuppeteerSharp.BrowserData; using WebReaper.Core.CookieStorage.Abstract; using WebReaper.Core.Loaders.Abstract; using WebReaper.Domain.PageActions; @@ -30,10 +31,13 @@ public async Task Load(string url, List? pageActions = null, }); await _semaphore.WaitAsync(); + + InstalledBrowser installedBrowser; + try { Logger.LogInformation("{class}.{method}: Downloading browser...", nameof(PuppeteerPageLoader), nameof(Load)); - await browserFetcher.DownloadAsync(BrowserFetcher.DefaultChromiumRevision); + installedBrowser = await browserFetcher.DownloadAsync(); Logger.LogInformation("{class}.{method}: Browser is downloaded", nameof(PuppeteerPageLoader), nameof(Load)); } finally @@ -45,7 +49,8 @@ public async Task Load(string url, List? pageActions = null, await using var browser = await Puppeteer.LaunchAsync(new LaunchOptions { Headless = headless, - ExecutablePath = browserFetcher.RevisionInfo(BrowserFetcher.DefaultChromiumRevision).ExecutablePath + ExecutablePath = browserFetcher.GetExecutablePath(installedBrowser.BuildId), + Args = new[] { "--ignore-certificate-errors" } }); Logger.LogInformation("{class}.{method}: creating a new page", nameof(PuppeteerPageLoader), nameof(Load)); diff --git a/WebReaper/Core/Loaders/Concrete/PuppeteerPageLoaderWithProxies.cs b/WebReaper/Core/Loaders/Concrete/PuppeteerPageLoaderWithProxies.cs index 9c61e49..d930139 100644 --- a/WebReaper/Core/Loaders/Concrete/PuppeteerPageLoaderWithProxies.cs +++ b/WebReaper/Core/Loaders/Concrete/PuppeteerPageLoaderWithProxies.cs @@ -3,6 +3,7 @@ using PuppeteerExtraSharp; using PuppeteerExtraSharp.Plugins.ExtraStealth; using PuppeteerSharp; +using PuppeteerSharp.BrowserData; using WebReaper.Core.CookieStorage.Abstract; using WebReaper.Core.Loaders.Abstract; using WebReaper.Domain.PageActions; @@ -35,9 +36,12 @@ public async Task Load(string url, List? pageActions = null, }); await _semaphore.WaitAsync(); + + InstalledBrowser installedBrowser; + try { - await browserFetcher.DownloadAsync(BrowserFetcher.DefaultChromiumRevision); + installedBrowser = await browserFetcher.DownloadAsync(); } finally { @@ -52,7 +56,7 @@ public async Task Load(string url, List? pageActions = null, await using var browser = await puppeteerExtra.LaunchAsync(new LaunchOptions { Headless = headless, - ExecutablePath = browserFetcher.RevisionInfo(BrowserFetcher.DefaultChromiumRevision).ExecutablePath, + ExecutablePath = browserFetcher.GetExecutablePath(installedBrowser.BuildId), Args = new[] { "--disable-dev-shm-usage", diff --git a/WebReaper/WebReaper.csproj b/WebReaper/WebReaper.csproj index 349921d..5388e56 100644 --- a/WebReaper/WebReaper.csproj +++ b/WebReaper/WebReaper.csproj @@ -1,7 +1,7 @@  - net7.0 + net8.0 enable enable Alex Pavlov @@ -19,7 +19,7 @@ 3.5.1 true $(MSBuildProjectDirectory)\API.xml - 11 + 12 WebReaper @@ -40,22 +40,18 @@ - - - - + + + + - + - - - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - + + + + +