Fix: SQLite "too many SQL variables" error in Importer (#97)

* Optimize batch processing in Importer by chunking items into smaller batches of 500 for improved performance during import operations.

* Enhance batch processing in Importer to track current and total batches, ensuring UpdateCollectionTreeJob is dispatched only after the final batch is processed.

* Keep track of multiple batch IDs

* Chunk items into batches & dispatch `UpdateCollectionTreeJob` once all batches have finished

* Chunks of 500

* Code formatting

---------

Co-authored-by: Duncan McClean <[email protected]>

Author: pixelastronauts

src/Importer.php

@@ -6,6 +6,7 @@
 use Illuminate\Support\Facades\Bus;
 use Illuminate\Support\Facades\Cache;
 use Illuminate\Support\Facades\Schema;
+use Illuminate\Support\LazyCollection;
 use Statamic\Importer\Exceptions\JobBatchesTableMissingException;
 use Statamic\Importer\Imports\Import;
 use Statamic\Importer\Jobs\ImportItemJob;
@@ -19,6 +20,7 @@ class Importer
 
     public static function run(Import $import): void
     {
+        $import->batchIds([])->save();
         Cache::forget("importer.{$import->id}.parents");
 
         if (! Schema::connection(config('queue.batching.database', env('DB_CONNECTION', 'sqlite')))->hasTable(config('queue.batching.table', 'job_batches'))) {
@@ -30,14 +32,18 @@ public static function run(Import $import): void
             'xml' => (new Xml($import))->getItems($import->get('path')),
         };
 
-        Bus::batch($items->map(fn (array $item) => new ImportItemJob($import, $item)))
-            ->before(fn (Batch $batch) => $import->batchId($batch->id)->save())
-            ->finally(function (Batch $batch) use ($import) {
-                if ($import->get('destination.type') === 'entries') {
-                    UpdateCollectionTreeJob::dispatch($import);
-                }
-            })
-            ->dispatch();
+        $items
+            ->chunk(500)
+            ->each(function (LazyCollection $chunk) use ($import) {
+                Bus::batch($chunk->map(fn (array $item) => new ImportItemJob($import, $item)))
+                    ->before(fn (Batch $batch) => $import->batchIds([...$import->batchIds(), $batch->id])->save())
+                    ->finally(function (Batch $batch) use ($import) {
+                        if ($import->allBatchesHaveFinished() && $import->get('destination.type') === 'entries') {
+                            UpdateCollectionTreeJob::dispatch($import);
+                        }
+                    })
+                    ->dispatch();
+            });
     }
 
     public static function getTransformer(string $fieldtype): ?string

src/Imports/Import.php

@@ -2,7 +2,8 @@
 
 namespace Statamic\Importer\Imports;
 
-use Illuminate\Bus\PendingBatch;
+use Illuminate\Bus\Batch;
+use Illuminate\Support\Collection as SupportCollection;
 use Illuminate\Support\Facades\Bus;
 use Statamic\Facades\Collection;
 use Statamic\Facades\Taxonomy;
@@ -20,7 +21,7 @@ class Import
     public $id;
     public $name;
     public $config;
-    public $batchId;
+    public $batchIds;
 
     public function __construct()
     {
@@ -59,26 +60,27 @@ public function get(string $key, ?string $default = null): mixed
         return data_get($this->config, $key, $default);
     }
 
-    public function batchId($batchId = null)
+    public function batchIds($batchIds = null)
     {
-        return $this->fluentlyGetOrSet('batchId')->args(func_get_args());
+        return $this->fluentlyGetOrSet('batchIds')->args(func_get_args());
     }
 
-    public function batch(): ?PendingBatch
+    public function batches(): SupportCollection
     {
-        if (! $this->batchId()) {
-            return null;
-        }
+        return collect($this->batchIds())->map(fn ($id): Batch => Bus::findBatch($id));
+    }
 
-        return Bus::batch($this->batchId());
+    public function allBatchesHaveFinished(): bool
+    {
+        return $this->batches()->every(fn (Batch $batch) => $batch->finished());
     }
 
     public function fileData(): array
     {
         return collect([
             'name' => $this->name(),
             'config' => $this->config()->all(),
-            'batch_id' => $this->batchId(),
+            'batch_ids' => $this->batchIds(),
         ])->filter()->all();
     }
 

src/Imports/ImportRepository.php

@@ -23,7 +23,7 @@ public function all(): Collection
                 return $this->make()
                     ->id($id)
                     ->name(Arr::pull($data, 'name'))
-                    ->batchId(Arr::pull($data, 'batch_id'))
+                    ->batchIds(Arr::pull($data, 'batch_ids'))
                     ->config(Arr::pull($data, 'config'));
             })
             ->sortBy->name()