From 50d3632205701afb4fd7b391dea59d76ee65f70b Mon Sep 17 00:00:00 2001 From: Iwona Just Date: Tue, 25 Jul 2023 10:25:51 +0100 Subject: [PATCH 1/5] index file existing on remote fs --- src/helpers/AssetHelper.php | 101 +++++++++++++++++++++++++++++------- 1 file changed, 83 insertions(+), 18 deletions(-) diff --git a/src/helpers/AssetHelper.php b/src/helpers/AssetHelper.php index 7fe734b7..b1e87700 100644 --- a/src/helpers/AssetHelper.php +++ b/src/helpers/AssetHelper.php @@ -15,6 +15,9 @@ use craft\helpers\Json; use craft\helpers\StringHelper; use craft\helpers\UrlHelper; +use craft\models\AssetIndexData; +use craft\models\FsListing; +use craft\models\Volume; use Throwable; use yii\base\Exception; use yii\base\InvalidArgumentException; @@ -79,6 +82,61 @@ public static function downloadFile($srcName, $dstName, int $chunkSize = 1, bool return $status; } + public static function indexExistingFile($urlFromFeed, $fieldInfo, $feed, $field = null, $element = null, $folderId = null, $newFilename = null): AssetElement|bool + { + $assets = Craft::$app->getAssets(); + + if (!$folderId) { + if (!$field) { + throw new InvalidArgumentException('$folderId and $field cannot both be null.'); + } + $folderId = $field->resolveDynamicPathToFolderId($element); + } + + $folder = $assets->findFolder(['id' => $folderId]); + $volume = $folder->getVolume(); + + $filename = $newFilename ? AssetsHelper::prepareAssetName($newFilename, false) : self::getRemoteUrlFilename($urlFromFeed); + + // get the url/path of the asset we're supposed to end up with + $asset = new AssetElement(); + $asset->setFilename($filename); + $asset->folderId = $folder->id; + $asset->folderPath = $folder->path; + $asset->volumeId = $volume->id; + $targetUrl = \craft\helpers\Assets::generateUrl($volume->getFs(), $asset); + + $rootUrl = $volume->getRootUrl() ?? ''; + $targetPath = str_replace($rootUrl, '', $targetUrl); + + // check if it exists + if (!$volume->fileExists($targetPath)) { + // if it doesn't - proceed with createAsset() + return false; + } + + // if it does - index it + $listing = new FsListing([ + 'dirname' => pathinfo($targetPath, PATHINFO_DIRNAME), + 'basename' => pathinfo($targetPath, PATHINFO_BASENAME), + 'type' => 'file', + 'dateModified' => $volume->getDateModified($targetPath), + 'fileSize' => $volume->getFileSize($targetPath), + ]); + + $indexEntry = new AssetIndexData([ + 'volumeId' => $volume->id, + //'sessionId' => $sessionId, + 'uri' => $listing->getUri(), + 'size' => $listing->getFileSize(), + 'timestamp' => $listing->getDateModified(), + 'isDir' => $listing->getIsDir(), + //'inProgress' => true, + ]); + + return Craft::$app->getAssetIndexer()->indexFileByEntry($indexEntry); + } + /** * @param array $urls * @param $fieldInfo @@ -102,30 +160,37 @@ public static function fetchRemoteImage(array $urls, $fieldInfo, $feed, $field = // user has set to use that instead, so we're good to proceed. foreach ($urls as $url) { try { - $filename = $newFilename ? AssetsHelper::prepareAssetName($newFilename, false) : self::getRemoteUrlFilename($url); + $indexedAsset = self::indexExistingFile($url, $fieldInfo, $feed, $field, $element, $folderId, $newFilename); - $fetchedImage = $tempFeedMePath . $filename; + if ($indexedAsset instanceof AssetElement) { + $uploadedAssets[] = $indexedAsset->id; + } else { - // But also check if we've downloaded this recently, use the copy in the temp directory - $cachedImage = FileHelper::findFiles($tempFeedMePath, [ - 'only' => [$filename], - 'recursive' => false, - ]); + $filename = $newFilename ? AssetsHelper::prepareAssetName($newFilename, false) : self::getRemoteUrlFilename($url); - Plugin::info('Fetching remote image `{i}` - `{j}`', ['i' => $url, 'j' => $filename]); + $fetchedImage = $tempFeedMePath . $filename; - if (!$cachedImage) { - self::downloadFile($url, $fetchedImage, 1, true, $feed['id']); - } else { - $fetchedImage = $cachedImage[0]; - } + // But also check if we've downloaded this recently, use the copy in the temp directory + $cachedImage = FileHelper::findFiles($tempFeedMePath, [ + 'only' => [$filename], + 'recursive' => false, + ]); - $result = self::createAsset($fetchedImage, $filename, $folderId, $field, $element, $conflict, Hash::get($feed, 'updateSearchIndexes')); + Plugin::info('Fetching remote image `{i}` - `{j}`', ['i' => $url, 'j' => $filename]); - if ($result) { - $uploadedAssets[] = $result; - } else { - Plugin::error('Failed to create asset from `{i}`', ['i' => $url]); + if (!$cachedImage) { + self::downloadFile($url, $fetchedImage, 1, true, $feed['id']); + } else { + $fetchedImage = $cachedImage[0]; + } + + $result = self::createAsset($fetchedImage, $filename, $folderId, $field, $element, $conflict, Hash::get($feed, 'updateSearchIndexes')); + + if ($result) { + $uploadedAssets[] = $result; + } else { + Plugin::error('Failed to create asset from `{i}`', ['i' => $url]); + } } } catch (Throwable $e) { if ($field) { From 95cfa75f009b5db5efe37ed6953184ac596da094 Mon Sep 17 00:00:00 2001 From: Iwona Just Date: Mon, 31 Jul 2023 14:31:46 +0100 Subject: [PATCH 2/5] cleanup --- src/helpers/AssetHelper.php | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/src/helpers/AssetHelper.php b/src/helpers/AssetHelper.php index b1e87700..59cad3e8 100644 --- a/src/helpers/AssetHelper.php +++ b/src/helpers/AssetHelper.php @@ -82,7 +82,25 @@ public static function downloadFile($srcName, $dstName, int $chunkSize = 1, bool return $status; } - public static function indexExistingFile($urlFromFeed, $fieldInfo, $feed, $field = null, $element = null, $folderId = null, $newFilename = null): AssetElement|bool + /** + * Check if the file exists in the FS - if it does, index it (like via asset indexing) and return it + * otherwise return false and proceed with creating the asset + * + * @param $urlFromFeed + * @param $fieldInfo + * @param $feed + * @param $field + * @param $element + * @param $folderId + * @param $newFilename + * @return AssetElement|bool + * @throws \craft\errors\AssetDisallowedExtensionException + * @throws \craft\errors\FsException + * @throws \craft\errors\MissingAssetException + * @throws \craft\errors\VolumeException + * @throws \yii\base\InvalidConfigException + */ + public static function indexExistingFile($urlFromFeed, $field = null, $element = null, $folderId = null, $newFilename = null): AssetElement|bool { $assets = Craft::$app->getAssets(); @@ -94,6 +112,10 @@ public static function indexExistingFile($urlFromFeed, $fieldInfo, $feed, $field } $folder = $assets->findFolder(['id' => $folderId]); + if (!$folder) { + throw new InvalidArgumentException('Cannot find folder by ID.'); + } + $volume = $folder->getVolume(); $filename = $newFilename ? AssetsHelper::prepareAssetName($newFilename, false) : self::getRemoteUrlFilename($urlFromFeed); @@ -104,7 +126,7 @@ public static function indexExistingFile($urlFromFeed, $fieldInfo, $feed, $field $asset->folderId = $folder->id; $asset->folderPath = $folder->path; $asset->volumeId = $volume->id; - $targetUrl = \craft\helpers\Assets::generateUrl($volume->getFs(), $asset); + $targetUrl = AssetsHelper::generateUrl($volume->getFs(), $asset); $rootUrl = $volume->getRootUrl() ?? ''; $targetPath = str_replace($rootUrl, '', $targetUrl); @@ -126,12 +148,10 @@ public static function indexExistingFile($urlFromFeed, $fieldInfo, $feed, $field $indexEntry = new AssetIndexData([ 'volumeId' => $volume->id, - //'sessionId' => $sessionId, 'uri' => $listing->getUri(), 'size' => $listing->getFileSize(), 'timestamp' => $listing->getDateModified(), 'isDir' => $listing->getIsDir(), - //'inProgress' => true, ]); return Craft::$app->getAssetIndexer()->indexFileByEntry($indexEntry); @@ -160,12 +180,11 @@ public static function fetchRemoteImage(array $urls, $fieldInfo, $feed, $field = // user has set to use that instead, so we're good to proceed. foreach ($urls as $url) { try { - $indexedAsset = self::indexExistingFile($url, $fieldInfo, $feed, $field, $element, $folderId, $newFilename); + $indexedAsset = self::indexExistingFile($url, $field, $element, $folderId, $newFilename); if ($indexedAsset instanceof AssetElement) { $uploadedAssets[] = $indexedAsset->id; } else { - $filename = $newFilename ? AssetsHelper::prepareAssetName($newFilename, false) : self::getRemoteUrlFilename($url); $fetchedImage = $tempFeedMePath . $filename; From 08281d59830b063960c31b7f3eda9b6de9277832 Mon Sep 17 00:00:00 2001 From: Iwona Just Date: Mon, 31 Jul 2023 14:46:30 +0100 Subject: [PATCH 3/5] dry --- src/helpers/AssetHelper.php | 53 ++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/src/helpers/AssetHelper.php b/src/helpers/AssetHelper.php index 59cad3e8..14864d60 100644 --- a/src/helpers/AssetHelper.php +++ b/src/helpers/AssetHelper.php @@ -18,6 +18,7 @@ use craft\models\AssetIndexData; use craft\models\FsListing; use craft\models\Volume; +use craft\models\VolumeFolder; use Throwable; use yii\base\Exception; use yii\base\InvalidArgumentException; @@ -102,22 +103,8 @@ public static function downloadFile($srcName, $dstName, int $chunkSize = 1, bool */ public static function indexExistingFile($urlFromFeed, $field = null, $element = null, $folderId = null, $newFilename = null): AssetElement|bool { - $assets = Craft::$app->getAssets(); - - if (!$folderId) { - if (!$field) { - throw new InvalidArgumentException('$folderId and $field cannot both be null.'); - } - $folderId = $field->resolveDynamicPathToFolderId($element); - } - - $folder = $assets->findFolder(['id' => $folderId]); - if (!$folder) { - throw new InvalidArgumentException('Cannot find folder by ID.'); - } - + $folder = self::_getAssetFolder($folderId, $field, $element); $volume = $folder->getVolume(); - $filename = $newFilename ? AssetsHelper::prepareAssetName($newFilename, false) : self::getRemoteUrlFilename($urlFromFeed); // get the url/path of the asset we're supposed to end up with @@ -299,15 +286,7 @@ public static function createBase64Image($base64, $fieldInfo, $feed, $field = nu private static function createAsset(string $tempFilePath, string $filename, ?int $folderId, ?Assets $field, ?ElementInterface $element, string $conflict, bool $updateSearchIndexes): bool|int { $assets = Craft::$app->getAssets(); - - if (!$folderId) { - if (!$field) { - throw new InvalidArgumentException('$folderId and $field cannot both be null.'); - } - $folderId = $field->resolveDynamicPathToFolderId($element); - } - - $folder = $assets->findFolder(['id' => $folderId]); + $folder = self::_getAssetFolder($folderId, $field, $element); // Create the new asset (even if we're setting it to replace) $asset = new AssetElement(); @@ -453,4 +432,30 @@ public static function getRemoteUrlExtension($url): string return StringHelper::toLowerCase($extension); } + + /** + * Find and return folder for an asset by folder id or field and element + * + * @param int|null $folderId + * @param Assets|null $field + * @param ElementInterface|null $element + * @return VolumeFolder + */ + private static function _getAssetFolder(?int $folderId, ?Assets $field, ?ElementInterface $element): VolumeFolder + { + if (!$folderId) { + if (!$field) { + throw new InvalidArgumentException('$folderId and $field cannot both be null.'); + } + $folderId = $field->resolveDynamicPathToFolderId($element); + } + + $folder = Craft::$app->getAssets()->findFolder(['id' => $folderId]); + + if (!$folder) { + throw new InvalidArgumentException('Cannot find folder by ID.'); + } + + return $folder; + } } From 017df4c7374adf549d91b21ab7b883afc2082358 Mon Sep 17 00:00:00 2001 From: Iwona Just Date: Mon, 31 Jul 2023 15:16:35 +0100 Subject: [PATCH 4/5] ecs --- src/helpers/AssetHelper.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/helpers/AssetHelper.php b/src/helpers/AssetHelper.php index 14864d60..ad09a533 100644 --- a/src/helpers/AssetHelper.php +++ b/src/helpers/AssetHelper.php @@ -17,7 +17,6 @@ use craft\helpers\UrlHelper; use craft\models\AssetIndexData; use craft\models\FsListing; -use craft\models\Volume; use craft\models\VolumeFolder; use Throwable; use yii\base\Exception; From 27c4beb383a07b1e18a3ae14caa0b0bdaac8785c Mon Sep 17 00:00:00 2001 From: Iwona Just Date: Tue, 1 Aug 2023 09:22:26 +0100 Subject: [PATCH 5/5] respect the conflict strategy --- src/helpers/AssetHelper.php | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/helpers/AssetHelper.php b/src/helpers/AssetHelper.php index ad09a533..e05ea30d 100644 --- a/src/helpers/AssetHelper.php +++ b/src/helpers/AssetHelper.php @@ -100,8 +100,13 @@ public static function downloadFile($srcName, $dstName, int $chunkSize = 1, bool * @throws \craft\errors\VolumeException * @throws \yii\base\InvalidConfigException */ - public static function indexExistingFile($urlFromFeed, $field = null, $element = null, $folderId = null, $newFilename = null): AssetElement|bool + public static function indexExistingFile($urlFromFeed, string $conflict, $field = null, $element = null, $folderId = null, $newFilename = null): AssetElement|bool { + // if the conflict strategy is to replace or create, just bail straight away + // and allow for creation of the asset even the file exists + if ($conflict == AssetElement::SCENARIO_REPLACE || $conflict == AssetElement::SCENARIO_CREATE) { + return false; + } $folder = self::_getAssetFolder($folderId, $field, $element); $volume = $folder->getVolume(); $filename = $newFilename ? AssetsHelper::prepareAssetName($newFilename, false) : self::getRemoteUrlFilename($urlFromFeed); @@ -166,7 +171,7 @@ public static function fetchRemoteImage(array $urls, $fieldInfo, $feed, $field = // user has set to use that instead, so we're good to proceed. foreach ($urls as $url) { try { - $indexedAsset = self::indexExistingFile($url, $field, $element, $folderId, $newFilename); + $indexedAsset = self::indexExistingFile($url, $conflict, $field, $element, $folderId, $newFilename); if ($indexedAsset instanceof AssetElement) { $uploadedAssets[] = $indexedAsset->id;