Revert "Switch to simple built-in FileFetcher on import (#3881)" (#3893)

Author: dafeder

composer.json

@@ -18,6 +18,7 @@
         "fmizzell/maquina": "^1.1.0",
         "getdkan/contracts": "^1.0.0",
         "getdkan/csv-parser": "^1.2.3",
+        "getdkan/file-fetcher" : "^4.1.0",
         "getdkan/harvest": "^1.0.0",
         "getdkan/json-schema-provider": "^0.1.2",
         "getdkan/locker": "^1.1.0",

modules/common/common.services.yml

@@ -24,6 +24,11 @@ services:
     factory: entity_type.manager:getStorage
     arguments: ['node']
 
+  dkan.common.file_fetcher:
+    class: \Drupal\common\FileFetcher\Factory
+    arguments:
+      - '@dkan.common.job_store'
+
   dkan.common.dataset_info:
     class: \Drupal\common\DatasetInfo
     calls:

modules/common/src/FileFetcher/Factory.php

@@ -0,0 +1,56 @@
+<?php
+
+namespace Drupal\common\FileFetcher;
+
+use Contracts\FactoryInterface;
+use Drupal\common\Storage\JobStoreFactory;
+use FileFetcher\FileFetcher;
+
+/**
+ * File fetcher Factory.
+ */
+class Factory implements FactoryInterface {
+
+  /**
+   * Job store factory service.
+   *
+   * @var \Drupal\common\Storage\JobStoreFactory
+   */
+  private $factory;
+
+  /**
+   * Default file fetcher config.
+   *
+   * @var array
+   */
+  private $configDefault = [
+    'keep_original_filename' => TRUE,
+  ];
+
+  /**
+   * Constructor.
+   */
+  public function __construct(JobStoreFactory $factory) {
+    $this->factory = $factory;
+  }
+
+  /**
+   * Inherited.
+   *
+   * @inheritdoc
+   */
+  public function getInstance(string $identifier, array $config = []) {
+    $config = array_merge($this->configDefault, $config);
+    return FileFetcher::get($identifier, $this->getFileFetcherJobStore(), $config);
+  }
+
+  /**
+   * Private.
+   */
+  private function getFileFetcherJobStore() {
+    /** @var \Drupal\common\Storage\JobStoreFactory $jobStoreFactory */
+    $jobStoreFactory = $this->factory;
+    return $jobStoreFactory->getInstance(FileFetcher::class);
+  }
+
+}

modules/common/tests/src/Traits/CleanUp.php

@@ -3,7 +3,7 @@
 namespace Drupal\Tests\common\Traits;
 
 use Drupal\node\Entity\Node;
-use Drupal\datastore\Plugin\QueueWorker\FileFetcherJob;
+use FileFetcher\FileFetcher;
 
 /**
  *
@@ -50,7 +50,7 @@ private function removeAllFileFetchingJobs() {
     $jobStoreFactory = \Drupal::service('dkan.common.job_store');
 
     /** @var \Drupal\common\Storage\JobStore $jobStore */
-    $jobStore = $jobStoreFactory->getInstance(FileFetcherJob::class);
+    $jobStore = $jobStoreFactory->getInstance(FileFetcher::class);
     foreach ($jobStore->retrieveAll() as $id) {
       $jobStore->remove($id);
     }

modules/common/tests/src/Unit/Storage/JobStoreTest.php

@@ -9,9 +9,9 @@
 use Drupal\Core\Database\Schema;
 use Drupal\Core\Database\StatementWrapper;
 use Drupal\common\Storage\JobStore;
-use Drupal\datastore\Plugin\QueueWorker\FileFetcherJob;
 
 use Contracts\Mock\Storage\Memory;
+use FileFetcher\FileFetcher;
 use MockChain\Chain;
 use MockChain\Sequence;
 use PHPUnit\Framework\TestCase;
@@ -30,7 +30,7 @@ public function testConstruction() {
       ->add(Connection::class, "schema", Schema::class)
       ->add(Schema::class, "tableExists", FALSE);
 
-    $jobStore = new JobStore(FileFetcherJob::class, $chain->getMock());
+    $jobStore = new JobStore(FileFetcher::class, $chain->getMock());
     $this->assertTrue(is_object($jobStore));
   }
 
@@ -59,8 +59,8 @@ public function testRetrieve() {
       ->add(Connection::class, 'query', StatementWrapper::class)
       ->add(StatementWrapper::class, 'fetchAll', $fieldInfo);
 
-    $jobStore = new JobStore(FileFetcherJob::class, $chain->getMock());
-    $this->assertEquals($job_data, $jobStore->retrieve("1", FileFetcherJob::class));
+    $jobStore = new JobStore(FileFetcher::class, $chain->getMock());
+    $this->assertEquals($job_data, $jobStore->retrieve("1", FileFetcher::class));
   }
 
   /**
@@ -90,7 +90,7 @@ public function testRetrieveAll() {
       ->add(Connection::class, 'query', StatementWrapper::class)
       ->add(StatementWrapper::class, 'fetchAll', $sequence);
 
-    $jobStore = new JobStore(FileFetcherJob::class, $chain->getMock());
+    $jobStore = new JobStore(FileFetcher::class, $chain->getMock());
     $this->assertTrue(is_array($jobStore->retrieveAll()));
   }
 
@@ -127,7 +127,7 @@ public function testStore() {
       ->add(StatementWrapper::class, 'fetchAll', $fieldInfo)
       ->getMock();
 
-    $jobStore = new JobStore(FileFetcherJob::class, $connection);
+    $jobStore = new JobStore(FileFetcher::class, $connection);
 
     $this->assertEquals("1", $jobStore->store(json_encode($jobObject), "1"));
   }
@@ -151,16 +151,16 @@ public function testRemove() {
       ->add(StatementWrapper::class, 'fetchAll', $fieldInfo)
       ->getMock();
 
-    $jobStore = new JobStore(FileFetcherJob::class, $connection);
+    $jobStore = new JobStore(FileFetcher::class, $connection);
 
-    $this->assertEquals("", $jobStore->remove("1", FileFetcherJob::class));
+    $this->assertEquals("", $jobStore->remove("1", FileFetcher::class));
   }
 
   /**
    * Private.
    */
   private function getFileFetcher() {
-    return FileFetcherJob::get("1", new Memory(), ["filePath" => "file://" . __DIR__ . "/../../data/countries.csv"]);
+    return FileFetcher::get("1", new Memory(), ["filePath" => "file://" . __DIR__ . "/../../data/countries.csv"]);
   }
 
 }

modules/datastore/datastore.services.yml

@@ -17,6 +17,7 @@ services:
     class: \Drupal\datastore\Service\ResourceLocalizer
     arguments:
       - '@dkan.metastore.resource_mapper'
+      - '@dkan.common.file_fetcher'
       - '@dkan.common.drupal_files'
       - '@dkan.common.job_store'
 

modules/datastore/src/Plugin/QueueWorker/FileFetcherJob.php

@@ -3,7 +3,6 @@
 namespace Drupal\datastore;
 
 use Drupal\common\DataResource;
-use Drupal\datastore\Plugin\QueueWorker\FileFetcherJob;
 use Drupal\common\Storage\JobStoreFactory;
 use Procrastinator\Result;
 use Symfony\Component\DependencyInjection\ContainerInterface;
@@ -13,6 +12,7 @@
 use Drupal\datastore\Service\ResourceLocalizer;
 use Drupal\datastore\Service\Factory\ImportFactoryInterface;
 use Drupal\datastore\Service\Info\ImportInfoList;
+use FileFetcher\FileFetcher;
 
 /**
  * Main services for the datastore.
@@ -33,13 +33,6 @@ class Service implements ContainerInjectionInterface {
    */
   private $importServiceFactory;
 
-  /**
-   * Import info list service.
-   *
-   * @var \Drupal\datastore\Service\Info\ImportInfoList
-   */
-  private ImportInfoList $importInfoList;
-
   /**
    * Drupal queue.
    *
@@ -211,7 +204,7 @@ public function drop(string $identifier, ?string $version = NULL, bool $local_re
     if ($local_resource) {
       $this->resourceLocalizer->remove($identifier, $version);
       $this->jobStoreFactory
-        ->getInstance(FileFetcherJob::class)
+        ->getInstance(FileFetcher::class)
         ->remove(substr(str_replace('__', '_', $resource_id), 0, -11));
     }
   }

modules/datastore/src/Service.php

@@ -6,6 +6,7 @@
 use Drupal\common\Storage\JobStoreFactory;
 use Drupal\datastore\Service\Factory\ImportFactoryInterface;
 use Drupal\datastore\Service\ResourceLocalizer;
+use FileFetcher\FileFetcher;
 use Procrastinator\Job\Job;
 
 /**
@@ -37,7 +38,7 @@ class ImportInfo {
   /**
    * FileFetcher service.
    *
-   * @var \Drupal\datastore\Plugin\QueueWorker\FileFetcherJob
+   * @var \FileFetcher\FileFetcher
    */
   private $fileFetcher;