Merge branch 'v4' into bugfix/749-asset-filename-conflicts

# Conflicts:
#	src/web/assets/feedme/dist/FeedMe.js
#	src/web/assets/feedme/dist/FeedMe.js.map

CHANGELOG.md

@@ -2,7 +2,17 @@
 
 ## Unreleased
 
+- Fixed a bug where you could not override feed settings with `false` if you used a `config/feed-me.php` config file. ([#1352](https://github.com/craftcms/feed-me/issues/1352))
+- Fixed a bug where `queueTtr`, `queueMaxRetry`, and `assetDownloadCurl` settings were being ignored if you used a `config/feed-me.php` config file. ([#1356](https://github.com/craftcms/feed-me/issues/1356))
+
+## 4.7.0 - 2023-07-06
+
+- The “Create if they do not exist” setting on a feed’s mapping now conditionally show/hide the checkbox for Categories, Entries and Users relation fields. ([#1077](https://github.com/craftcms/feed-me/issues/1077))
 - Fixed a bug where an element’s status would not update on a multi-site install if the status was the only thing that changed. ([#1310](https://github.com/craftcms/feed-me/issues/1310))
+- Fixed a bug where the `DataTypes::EVENT_AFTER_FETCH_FEED` event was not being fired for local feeds. ([#1101](https://github.com/craftcms/feed-me/issues/1101))
+- Fixed a PHP error that could happen when importing into a User field. ([#844](https://github.com/craftcms/feed-me/issues/844))
+- Fixed a bug where slugs imported from a feed would not be respected if they contained underscores. ([#603](https://github.com/craftcms/feed-me/issues/603))
+- Fixed a bug where nested subcategories sharing the same slug but with different parents would only import the last subcategory. ([#780](https://github.com/craftcms/feed-me/issues/780))
 
 ## 4.6.3.1 - 2023-05-02
 

src/base/Element.php

@@ -14,6 +14,7 @@
 use craft\feedme\helpers\DateHelper;
 use craft\feedme\models\FeedModel;
 use craft\helpers\Db;
+use craft\helpers\ElementHelper;
 use craft\helpers\StringHelper;
 
 /**
@@ -157,7 +158,11 @@ public function matchExistingElement($data, $settings)
                     continue;
                 }
 
-                $criteria[$handle] = Db::escapeParam($feedValue);
+                if ($handle === 'parent') {
+                    $criteria['descendantOf'] = Db::escapeParam($feedValue);
+                } else {
+                    $criteria[$handle] = Db::escapeParam($feedValue);
+                }
             }
         }
 
@@ -305,6 +310,12 @@ protected function parseSlug($feedData, $fieldInfo)
             $value = $this->_asciiString($value);
         }
 
+        // normalize the slug and check if it's valid;
+        // if it is - use it, otherwise _createSlug()
+        if (is_string($value) && ($value = ElementHelper::normalizeSlug($value)) !== '') {
+            return $value;
+        }
+
         return $this->_createSlug($value);
     }
 

src/fields/Users.php

@@ -80,11 +80,20 @@ public function parseField()
 
         // Get source id's for connecting
         $groupIds = [];
+        $isAdmin = false;
 
         if (is_array($sources)) {
+            // go through sources that start with "group:" and get group uid for those
             foreach ($sources as $source) {
-                [, $uid] = explode(':', $source);
-                $groupIds[] = Db::idByUid('{{%usergroups}}', $uid);
+                if (str_starts_with($source, 'group:')) {
+                    [, $uid] = explode(':', $source);
+                    $groupIds[] = Db::idByUid('{{%usergroups}}', $uid);
+                }
+            }
+
+            // the other source left in Craft 3 can be 'admins' for which we'll need a separate query
+            if (in_array('admins', $sources, true)) {
+                $isAdmin = true;
             }
         } elseif ($sources === '*') {
             $groupIds = null;
@@ -120,22 +129,29 @@ public function parseField()
                 $columnName = Craft::$app->getFields()->oldFieldColumnPrefix . $match;
             }
 
-            $query = UserElement::find();
-
+            $ids = [];
             $criteria['status'] = null;
             $criteria['groupId'] = $groupIds;
             $criteria['limit'] = $limit;
             $criteria['where'] = ['=', $columnName, $dataValue];
 
-            Craft::configure($query, $criteria);
-
-            Plugin::info('Search for existing user with query `{i}`', ['i' => json_encode($criteria)]);
-
-            $ids = $query->ids();
+            // If the only source for the Users field is "admins" we don't have to bother with this query.
+            if (!($isAdmin && empty($groupIds))) {
+                $ids = $this->_findUsers($criteria);
+                $foundElements = array_merge($foundElements, $ids);
+            }
 
-            $foundElements = array_merge($foundElements, $ids);
+            // Previous query would look through selected groups or if "all" was selected
+            // (in which case groupIds would be null, and wouldn't actually limit the query).
+            // So if we haven't found a match with the previous query, and field sources contains "admins",
+            // we have to look for the user among admins too.
+            if ($isAdmin && count($ids) === 0) {
+                unset($criteria['groupId']);
+                $criteria['admin'] = true;
 
-            Plugin::info('Found `{i}` existing users: `{j}`', ['i' => count($foundElements), 'j' => json_encode($foundElements)]);
+                $ids = $this->_findUsers($criteria);
+                $foundElements = array_merge($foundElements, $ids);
+            }
 
             // Check if we should create the element. But only if email is provided (for the moment)
             if ((count($ids) == 0) && $create && $match === 'email') {
@@ -202,4 +218,24 @@ private function _createElement($dataValue, $groupId)
 
         return $element->id;
     }
+
+    /**
+     * Attempt to find User based on search criteria. Return array of found IDs.
+     *
+     * @param $criteria
+     * @return array|int[]
+     */
+    private function _findUsers($criteria): array
+    {
+        $query = UserElement::find();
+        Craft::configure($query, $criteria);
+
+        Plugin::info('Search for existing user with query `{i}`', ['i' => json_encode($criteria)]);
+
+        $ids = $query->ids();
+
+        Plugin::info('Found `{i}` existing users: `{j}`', ['i' => count($ids), 'j' => json_encode($ids)]);
+
+        return $ids;
+    }
 }

src/helpers/AssetHelper.php

@@ -21,11 +21,12 @@ class AssetHelper
      * @param $dstName
      * @param int $chunkSize
      * @param bool $returnbytes
+     * @param null|int $feedId
      * @return bool|int
      */
-    public static function downloadFile($srcName, $dstName, $chunkSize = 1, $returnbytes = true)
+    public static function downloadFile($srcName, $dstName, $chunkSize = 1, $returnbytes = true, $feedId = null)
     {
-        $assetDownloadCurl = Plugin::$plugin->getSettings()->assetDownloadCurl;
+        $assetDownloadCurl = Plugin::$plugin->service->getConfig('assetDownloadCurl', $feedId);
 
         // Provide some legacy support
         if ($assetDownloadCurl) {
@@ -106,7 +107,7 @@ public static function fetchRemoteImage(array $urls, $fieldInfo, $feed, $field =
                 Plugin::info('Fetching remote image `{i}` - `{j}`', ['i' => $url, 'j' => $filename]);
 
                 if (!$cachedImage) {
-                    self::downloadFile($url, $fetchedImage);
+                    self::downloadFile($url, $fetchedImage, 1, true, $feed['id']);
                 } else {
                     $fetchedImage = $cachedImage[0];
                 }

src/queue/jobs/FeedImport.php

@@ -51,15 +51,15 @@ class FeedImport extends BaseJob implements RetryableJobInterface
      */
     public function getTtr()
     {
-        return Plugin::$plugin->getSettings()->queueTtr ?? Plugin::getInstance()->queue->ttr;
+        return Plugin::$plugin->service->getConfig('queueTtr', $this->feed->id) ?? Plugin::getInstance()->queue->ttr;
     }
 
     /**
      * @inheritDoc
      */
     public function canRetry($attempt, $error)
     {
-        $attempts = Plugin::$plugin->getSettings()->queueMaxRetry ?? Plugin::getInstance()->queue->attempts;
+        $attempts = Plugin::$plugin->service->getConfig('queueMaxRetry', $this->feed->id) ?? Plugin::getInstance()->queue->attempts;
         return $attempt < $attempts;
     }
 

src/services/DataTypes.php

@@ -173,7 +173,13 @@ public function getRawData($url, $feedId = null)
             $filepath = realpath($url);
 
             if (!$filepath) {
-                return ['success' => false, 'error' => 'File path cannot be found.'];
+                $response = ['success' => false, 'error' => 'File path cannot be found.'];
+
+                return $this->_triggerEventAfterFetchFeed([
+                    'url' => $url,
+                    'feedId' => $feedId,
+                    'response' => $response,
+                ]);
             }
 
             $data = @file_get_contents($filepath);
@@ -188,7 +194,11 @@ public function getRawData($url, $feedId = null)
                 $response = ['success' => true, 'data' => $data];
             }
 
-            return $response;
+            return $this->_triggerEventAfterFetchFeed([
+                'url' => $url,
+                'feedId' => $feedId,
+                'response' => $response,
+            ]);
         }
 
         try {
@@ -207,15 +217,11 @@ public function getRawData($url, $feedId = null)
             Craft::$app->getErrorHandler()->logException($e);
         }
 
-        $event = new FeedDataEvent([
+        return $this->_triggerEventAfterFetchFeed([
             'url' => $url,
             'feedId' => $feedId,
             'response' => $response,
         ]);
-
-        Event::trigger(static::class, self::EVENT_AFTER_FETCH_FEED, $event);
-
-        return $event->response;
     }
 
     /**
@@ -467,4 +473,19 @@ private function _getCache($url)
     {
         return Craft::$app->cache->get(base64_encode(urlencode($url)));
     }
+
+    /**
+     * Trigger EVENT_AFTER_FETCH_FEED
+     *
+     * @param $data
+     * @return mixed
+     */
+    private function _triggerEventAfterFetchFeed($data)
+    {
+        $event = new FeedDataEvent($data);
+
+        Event::trigger(static::class, self::EVENT_AFTER_FETCH_FEED, $event);
+
+        return $event->response;
+    }
 }

src/services/Service.php

@@ -29,7 +29,7 @@ public function getConfig($key, $feedId = null)
         if ($feedId) {
             $configFeedItem = Hash::get($settings, 'feedOptions.' . $feedId . '.' . $key);
 
-            if ($configFeedItem) {
+            if (isset($configFeedItem)) {
                 $configItem = $configFeedItem;
             }
         }

src/templates/_includes/fields/categories.html

@@ -56,7 +56,7 @@
 
         {{ forms.selectField({
             name: 'options[match]',
-            class: '',
+            class: 'categories-field-match',
             value: hash_get(feed.fieldMapping, optionsPath ~ '.match') ?: '',
             options: matchAttributes,
         }) }}

src/templates/_includes/fields/entries.html

@@ -58,7 +58,7 @@
 
         {{ forms.selectField({
             name: 'options[match]',
-            class: '',
+            class: 'entries-field-match',
             value: hash_get(feed.fieldMapping, optionsPath ~ '.match') ?: '',
             options: matchAttributes,
         }) }}

src/templates/_includes/fields/users.html

@@ -57,7 +57,7 @@
 
         {{ forms.selectField({
             name: 'options[match]',
-            class: '',
+            class: 'users-field-match',
             value: hash_get(feed.fieldMapping, optionsPath ~ '.match') ?: '',
             options: matchAttributes,
         }) }}

src/web/assets/feedme/src/js/feed-me.js

@@ -115,6 +115,35 @@ $(function () {
     }
   }
 
+  // for categories and entries - only show "create if they do not exist" if matching done by "title"
+  // for users - only show "create if they do not exist" if matching done by "email"
+  $(
+    '.categories-field-match select, .entries-field-match select, .users-field-match select'
+  ).on('change', function (e) {
+    var match = 'title';
+    var $selectParent = $(this).parent();
+    var $elementCreateContainer = $(this)
+      .parents('.field-extra-settings')
+      .find('.element-create');
+
+    if ($selectParent.hasClass('users-field-match')) {
+      match = 'email';
+    }
+
+    if ($(this).val() === match) {
+      $elementCreateContainer.show();
+    } else {
+      $elementCreateContainer.find('input').prop('checked', false);
+      $('.field-extra-settings .element-create input').trigger('change');
+      $elementCreateContainer.hide();
+    }
+  });
+
+  // On-load, hide/show element create container
+  $('.categories-field-match select').trigger('change');
+  $('.entries-field-match select').trigger('change');
+  $('.users-field-match select').trigger('change');
+
   // For Assets, only show the upload options if we decide to upload
   $('.assets-uploads input').on('change', function (e) {
     var $options = $(this).parents('.field-extra-settings').find('.select');

src/web/twig/variables/FeedMeVariable.php

@@ -269,6 +269,10 @@ public function fieldCanBeUniqueId($field)
     {
         $type = $field['type'] ?? 'attribute';
 
+        if (isset($field['type']) && $field['handle'] === 'parent') {
+            $type = 'parent';
+        }
+
         if (is_object($field)) {
             $type = get_class($field);
         }
@@ -290,6 +294,7 @@ public function fieldCanBeUniqueId($field)
         $supportedValues = [
             'assets',
             'attribute',
+            'parent',
         ];
 
         $supported = array_merge($supportedFields, $supportedValues);