Add request-dont-filter

docs/changelog.lisp

@@ -14,6 +14,14 @@
                               "DOWNLOADER"
                               "SCRAPYCL"
                               "HTTP"))
+    (0.2.1 2025-04-10
+           "
+Changed
+=======
+
+Added `request-dont-filter` slot to `scrapycl:request`, allowing repeated scraping of the
+same URL when set to T.
+")
   (0.2.0 2025-02-07
          "
 Changed

src/core.lisp

@@ -18,6 +18,7 @@
            #:response-body
            #:response-headers)
   (:export #:request-url
+           #:request-dont-filter
            #:request)
   (:export #:typed-output)
   (:export #:json-lines

src/engine.lisp

@@ -7,7 +7,8 @@
                 #:start
                 #:process
                 #:enqueue
-                #:request-url)
+                #:request-url
+                #:request-dont-filter)
   (:import-from #:scrapycl/spider)
   (:import-from #:scrapycl/task
                 #:task
@@ -50,7 +51,7 @@
              (setf (gethash url seen-urls)
                    t)))
       (declare (dynamic-extent #'seen-url-p))
-      
+
       (loop for task = (get-next-task spider)
             while task
             do (let* (;; It is important to have this var
@@ -71,9 +72,11 @@
                               ;; We need this block to not visit same URL twice and
                               ;; to break link loops:
                               (scrapycl/core:request
-                               (unless (seen-url-p (request-url object))
-                                 (register-url (request-url object))
-                                 (enqueue spider object)))
+                               (cond ((request-dont-filter object)
+                                      (enqueue spider object))
+                                     ((not (seen-url-p (request-url object)))
+                                      (register-url (request-url object))
+                                      (enqueue spider object))))
                               (t
                                (enqueue spider object)))))
                    (declare (dynamic-extent #'walk))
@@ -138,7 +141,7 @@
         (unless results
           (log:debug "Process didn't return new objects"))
         results)))
-  
+
   (:method ((spider t) (object t))
     (cond
       (*output-func*
@@ -153,7 +156,7 @@
   (bt2:with-lock-held ((scrapycl/spider::%spider-queue-lock spider))
     (clear-queue (scrapycl/spider::%spider-queue spider))
     (values))
-    
+
   (uiop:while-collecting (collect-item)
     (let* ((output-is-function
              (and output

src/request.lisp

@@ -3,6 +3,7 @@
   (:import-from #:scrapycl/core
                 #:request
                 #:request-url
+                #:request-dont-filter
                 #:url))
 (in-package #:scrapycl/request)
 
@@ -12,7 +13,12 @@
         :type url
         :initform (error "Please, provide :URL argument.")
         :reader request-url
-        :documentation "URL to fetch data from.")))
+        :documentation "URL to fetch data from.")
+   (dont-filter :initarg :dont-filter
+                :type (member t nil)
+                :initform nil
+                :reader request-dont-filter
+                :documentation "Exclude request from filtering.")))
 
 
 (defmethod print-object ((obj request) stream)