Merge pull request #24 from justynhunter/feature/allow-contentparser-set

Adds methods to set the IContentParser
pavlovtech · Nov 21, 2023 · 988ea8c · 988ea8c
2 parents c5d4b7c + 0f84f72
commit 988ea8c
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 9 deletions.
diff --git a/WebReaper/Builders/ScraperEngineBuilder.cs b/WebReaper/Builders/ScraperEngineBuilder.cs
@@ -8,6 +8,7 @@
 using WebReaper.Core.CookieStorage.Abstract;
 using WebReaper.Core.LinkTracker.Abstract;
 using WebReaper.Core.LinkTracker.Concrete;
+using WebReaper.Core.Parser.Abstract;
 using WebReaper.Core.Scheduler.Abstract;
 using WebReaper.Core.Scheduler.Concrete;
 using WebReaper.Domain;
@@ -35,9 +36,14 @@ public class ScraperEngineBuilder
 
     private IScheduler Scheduler { get; set; } = new InMemoryScheduler();
     private IScraperConfigStorage? ConfigStorage { get; set; } = new InMemoryScraperConfigStorage();
-
     protected IProxyProvider? ProxyProvider { get; set; }
 
+    public ScraperEngineBuilder WithContentParser(IContentParser contentParser)
+    {
+        SpiderBuilder.WithContentParser(contentParser);
+        return this;
+    }
+
     public ScraperEngineBuilder AddSink(IScraperSink sink)
     {
         SpiderBuilder.AddSink(sink);
@@ -186,7 +192,6 @@ public ScraperEngineBuilder GetWithBrowser(
         ConfigBuilder.GetWithBrowser(startUrls, actionBuilder?.Invoke(new PageActionBuilder()));
         return this;
     }
-
     public ScraperEngineBuilder GetWithBrowser(params string[] startUrls)
     {
         ConfigBuilder.GetWithBrowser(startUrls);
@@ -201,7 +206,7 @@ public ScraperEngineBuilder Follow(string linkSelector)
 
     public ScraperEngineBuilder FollowWithBrowser(
         string linkSelector,
-        Func<PageActionBuilder, 
+        Func<PageActionBuilder,
         List<PageAction>>? actionBuilder = null)
     {
         ConfigBuilder.FollowWithBrowser(linkSelector, actionBuilder?.Invoke(new PageActionBuilder()));
@@ -278,7 +283,6 @@ public ScraperEngineBuilder WithMongoDbCookieStorage(string connectionString, st
             logger);
         return this;
     }
-
     public ScraperEngineBuilder WithFileCookieStorage(string fileName)
     {
         SpiderBuilder.WithFileCookieStorage(fileName);
@@ -335,12 +339,10 @@ public ScraperEngineBuilder WithParallelismDegree(int parallelismDegree)
     public async Task<ScraperEngine> BuildAsync()
     {
         SpiderBuilder.WithConfigStorage(ConfigStorage);
-
         var config = ConfigBuilder.Build();
         var spider = SpiderBuilder.Build();
-
         await ConfigStorage.CreateConfigAsync(config);
 
         return new ScraperEngine(_parallelismDegree, ConfigStorage, Scheduler, spider, Logger);
     }
-}
+}
diff --git a/WebReaper/Builders/SpiderBuilder.cs b/WebReaper/Builders/SpiderBuilder.cs
@@ -51,6 +51,12 @@ public class SpiderBuilder
 
     protected event Action<ParsedData> ScrapedData;
 
+    public SpiderBuilder WithContentParser(IContentParser contentParser)
+    {
+        ContentParser = contentParser;
+        return this;
+    }
+
     public SpiderBuilder WithLogger(ILogger logger)
     {
         Logger = logger;
@@ -166,7 +172,7 @@ public SpiderBuilder WithRedisCookieStorage(string connectionString, string redi
         CookieStorage = new RedisCookieStorage(connectionString, redisKey, Logger);
         return this;
     }
-    
+
     public SpiderBuilder WithFileCookieStorage(string fileName)
     {
         CookieStorage = new FileCookieStorage(fileName, Logger);
@@ -235,4 +241,4 @@ public ISpider Build()
 
         return spider;
     }
-}
+}