Merge pull request #198 from contao-estatemanager/feature/import-rework

Import adjustments, bugfixes and new features
contao-estatemanager · Feb 22, 2023 · 92157c0 · 92157c0
2 parents 3004be7 + 035cafc
commit 92157c0
Show file tree

Hide file tree

Showing 18 changed files with 308 additions and 40 deletions.
diff --git a/src/Resources/contao/classes/RealEstateCronImporter.php b/src/Resources/contao/classes/RealEstateCronImporter.php
@@ -87,7 +87,7 @@ protected function sync($objInterface)
 
         foreach ($syncFiles as $syncFile)
         {
-            if (null === $syncFile['synctime'])
+            if ((null === $syncFile['synctime']) || $syncFile['modified'])
             {
                 $files[] = $syncFile;
             }

diff --git a/src/Resources/contao/classes/RealEstateImporter.php b/src/Resources/contao/classes/RealEstateImporter.php
@@ -12,6 +12,8 @@
 
 
 use Contao\BackendTemplate;
+use Contao\Config;
+use Contao\CoreBundle\Monolog\ContaoContext;
 use Contao\Database;
 use Contao\Dbafs;
 use Contao\Files;
@@ -21,6 +23,9 @@
 use Contao\StringUtil;
 use Contao\System;
 use Contao\ZipReader;
+use Exception;
+use Psr\Log\LogLevel;
+use Throwable;
 
 class RealEstateImporter extends \BackendModule
 {
@@ -120,6 +125,11 @@ class RealEstateImporter extends \BackendModule
      */
     public $importMessage = 'File imported.';
 
+    /**
+     * @var string
+     */
+    public $errorMessage = '';
+
     /**
      * @var boolean
      */
@@ -362,9 +372,18 @@ public function startSync($syncFile='')
 
         if (($this->loadData()))
         {
+            try
+            {
+                $this->syncData();
+            }
+            catch (Throwable $e)
+            {
+                $this->handleImportError($e->getMessage());
+            }
+
             //$this->addLog('OpenImmo data loaded', 1, 'success');
 
-            if ($this->syncData())
+            /*if ($this->syncData())
             {
                 //$this->addLog('Import and synchronization was successful', 0, 'success');
                 //\Message::addConfirmation('Import and synchronization was successful');
@@ -373,7 +392,7 @@ public function startSync($syncFile='')
             {
                 //$this->addLog('OpenImmo data could not be synchronized.', 0, 'error');
                 //\Message::addError('OpenImmo data could not be synchronized.');
-            }
+            }*/
         }
         else
         {
@@ -434,6 +453,7 @@ protected function syncData()
                 {
                     $this->importStatus = 2;
                     $this->importMessage = 'File partially imported.';
+                    $this->errorMessage = 'File was partially imported as no provider could be found';
 
                     continue;
                 }
@@ -850,22 +870,30 @@ protected function updateCatalog($contactPersonRecords, $realEstateRecords)
 
         try {
             $tmpFolder = new Folder($this->objImportFolder->path . '/tmp');
-        } catch (\Exception $e) {
+        } catch (Exception $e) {
             return;
         }
 
         // Clear tmp folder
         $tmpFolder->purge();
 
         // Create history entry
-        $objInterfaceHistory = new InterfaceHistoryModel();
-        $objInterfaceHistory->pid = $this->objInterface->id;
-        $objInterfaceHistory->tstamp = time();
-        $objInterfaceHistory->source = $this->originalSyncFile;
-        $objInterfaceHistory->action = '';
+
+        // In case file already existed before but has been skipped due to an error
+        if (null === ($objInterfaceHistory = InterfaceHistoryModel::findBySource($this->originalSyncFile)))
+        {
+            $objInterfaceHistory = new InterfaceHistoryModel();
+            $objInterfaceHistory->pid      = $this->objInterface->id;
+            $objInterfaceHistory->source   = $this->originalSyncFile;
+        }
+
+        $objInterfaceHistory->tstamp   = time();
+        $objInterfaceHistory->mtime    = FilesHelper::fileModTime($this->originalSyncFile);
+        $objInterfaceHistory->action   = '';
         $objInterfaceHistory->username = $this->username;
-        $objInterfaceHistory->text = $this->importMessage;
-        $objInterfaceHistory->status = $this->importStatus;
+        $objInterfaceHistory->text     = $this->importMessage;
+        $objInterfaceHistory->message  = $this->errorMessage;
+        $objInterfaceHistory->status   = $this->importStatus;
         $objInterfaceHistory->save();
 
         return true;
@@ -893,7 +921,7 @@ protected function loadData()
 
         try {
             $this->data = simplexml_load_string($data, 'SimpleXMLElement', LIBXML_NOCDATA);
-        } catch (\Exception $e) {
+        } catch (Exception $e) {
             return false;
         }
 
@@ -911,7 +939,7 @@ public function getSyncFiles($searchForZip=true)
     {
         try {
             $folder = new Folder($this->objImportFolder->path);
-        } catch (\Exception $e) {
+        } catch (Exception $e) {
             return array();
         }
 
@@ -956,14 +984,23 @@ public function getSyncFiles($searchForZip=true)
                 }
             }*/
 
+            // If the file with the same name has been found but was modified
+            $modified =
+                array_key_exists($file, $arrSynced) &&      // Only if it previously existed
+                (0 !== intval($arrSynced[$file]->mtime)) && // Do not reimport pre-update files (< 1.0.29)
+                $mtime !== intval($arrSynced[$file]->mtime) // Only if file has been modified
+            ;
+
             $arrFiles[] = array(
-                "file" => $file,
-                "time" => $mtime,
-                "size" => $size,
-                "user" => array_key_exists($file, $arrSynced) ? $arrSynced[$file]->username : null,
-                "status" => array_key_exists($file, $arrSynced) ? intval($arrSynced[$file]->status) : 0,
+                "file"     => $file,
+                "time"     => $mtime,
+                "size"     => $size,
+                "user"     => array_key_exists($file, $arrSynced) ? $arrSynced[$file]->username : null,
+                "status"   => array_key_exists($file, $arrSynced) ? intval($arrSynced[$file]->status) : 0,
                 "synctime" => array_key_exists($file, $arrSynced) ? intval($arrSynced[$file]->tstamp) : null,
-                "checked" => false
+                "message"  => array_key_exists($file, $arrSynced) ? $arrSynced[$file]->message : null,
+                "modified" => $modified,
+                "checked"  => false
             );
         }
 
@@ -984,7 +1021,7 @@ public function getSyncFile($file)
                 $this->addLog('File is empty.', 0, 'error');
 
                 // Delete empty file
-                try{ unlink(TL_ROOT . '/' . $file); }catch (\Exception $exception){}
+                try{ unlink(TL_ROOT . '/' . $file); }catch (Exception $exception){}
 
                 return false;
             }
@@ -1016,13 +1053,13 @@ public function getSyncFile($file)
      *
      * @param string $path path to zip file
      *
-     * @throws \Exception
+     * @throws Exception
      */
     public function unzipArchive($path)
     {
         try {
             $tmpFolder = new Folder(FilesHelper::fileDirPath($path) . 'tmp');
-        } catch (\Exception $e) {
+        } catch (Exception $e) {
             return;
         }
 
@@ -1333,8 +1370,10 @@ protected function saveImage($interfaceMapping, $tmpGroup, &$value, &$values)
             return false;
         }
 
-        $fileSize = FilesHelper::fileSize($this->objImportFolder->path . '/tmp/' . $value);
-        if ($fileSize > 3000000 || $fileSize === 0)
+        $fileSize  = FilesHelper::fileSize($this->objImportFolder->path . '/tmp/' . $value);
+        $maxUpload = Config::get('estateManagerMaxFileSize') ?? 3000000;
+
+        if ($fileSize > $maxUpload || $fileSize === 0)
         {
             return false;
         }
@@ -1463,6 +1502,32 @@ protected function getTableMetaData($strTable)
         return $arrReturn;
     }
 
+    /**
+     * Handles import errors by logging the error and disabling the file for the next import
+     */
+    private function handleImportError(string $message): void
+    {
+        $errorMsg = 'The real estate file: ' . $this->originalSyncFile . ' could not be imported.';
+        $logger   = static::getContainer()->get('monolog.logger.contao');
+        $logger->log(LogLevel::ERROR, $errorMsg, array('contao' => new ContaoContext(__METHOD__, 'ERROR')));
+
+        if (null === ($objInterfaceHistory = InterfaceHistoryModel::findBySource($this->originalSyncFile)))
+        {
+            $objInterfaceHistory = new InterfaceHistoryModel();
+            $objInterfaceHistory->pid      = $this->objInterface->id;
+            $objInterfaceHistory->source   = $this->originalSyncFile;
+        }
+
+        $objInterfaceHistory->tstamp   = time();
+        $objInterfaceHistory->mtime    = FilesHelper::fileModTime($this->originalSyncFile);
+        $objInterfaceHistory->username = $this->username;
+        $objInterfaceHistory->text     = 'File has been skipped: <a href="/' . $this->originalSyncFile . '" target="_blank">' . $this->originalSyncFile . '</a>';
+        $objInterfaceHistory->message  = $message;
+        $objInterfaceHistory->status   = 2;
+        $objInterfaceHistory->action   = 'ERROR';
+        $objInterfaceHistory->save();
+    }
+
     public function addLog($strMessage, $level=0, $strType='raw', $data=null)
     {
         // ToDo: In Datei auslagern

diff --git a/src/Resources/contao/config/config.php b/src/Resources/contao/config/config.php
@@ -193,3 +193,4 @@
 $GLOBALS['TL_CONFIG']['numberFormatDecimals'] = ',';
 $GLOBALS['TL_CONFIG']['numberFormatThousands'] = '.';
 $GLOBALS['TL_CONFIG']['roomOptions'] = '1,2,3,4,5,6,7,8';
+$GLOBALS['TL_CONFIG']['estateManagerMaxFileSize'] = 3000000;
diff --git a/src/Resources/contao/data/import_field_formats.php b/src/Resources/contao/data/import_field_formats.php
@@ -968,5 +968,13 @@
         (
             array('combine', '', '', ' ', '', 'a:2:{i:0;a:2:{s:5:"field";s:3:"ort";s:6:"remove";s:1:"1";}i:1;a:2:{s:5:"field";s:16:"regionalerZusatz";s:6:"remove";s:1:"1";}}', '')
         )
-    )
-);
+    ),
+    array
+    (
+        'field' => array('sicherheitstechnik', '', '', NULL),
+        'actions' => array
+        (
+            array('unserialize', '', '', ', ', '', NULL, '')
+        )
+    ),
+);
diff --git a/src/Resources/contao/data/import_interface_mappings.php b/src/Resources/contao/data/import_interface_mappings.php
@@ -24,7 +24,7 @@
     array('tl_contact_person', 'vorname',                  'kontaktperson', 'vorname'),
     array('tl_contact_person', 'titel',                    'kontaktperson', 'titel'),
     array('tl_contact_person', 'position',                 'kontaktperson', 'position'),
-    array('tl_contact_person', 'anrede',                   'kontaktperson', 'anrede'),
+    array('tl_contact_person', 'anrede',                   'kontaktperson', 'anrede', array('text', 'lowercase')),
     array('tl_contact_person', 'anrede_brief',             'kontaktperson', 'anrede_brief'),
     array('tl_contact_person', 'firma',                    'kontaktperson', 'firma'),
     array('tl_contact_person', 'strasse',                  'kontaktperson', 'strasse'),

diff --git a/src/Resources/contao/dca/tl_contact_person.php b/src/Resources/contao/dca/tl_contact_person.php
@@ -126,10 +126,10 @@
             'label'                   => &$GLOBALS['TL_LANG']['tl_contact_person']['anrede'],
             'exclude'                 => true,
             'inputType'               => 'select',
-            'options'                 => array('herr','frau'),
+            'options'                 => array('herr', 'frau', 'firma', 'eheleute', 'familie', 'nicht zutreffend', 'divers'),
             'reference'               => &$GLOBALS['TL_LANG']['tl_contact_person'],
             'eval'                    => array('tl_class'=>'w50'),
-            'sql'                     => "varchar(8) NOT NULL default ''"
+            'sql'                     => "varchar(16) NOT NULL default ''"
         ),
         'firma' => array
         (
@@ -563,15 +563,31 @@ public function stringifyContactPerson(array $arrRow): string
     public function generateSalutation(string $varValue, Contao\DataContainer $dc): string
     {
         // Generate salutation if there is none
-        if ($varValue == '')
+        if ('' == $varValue)
         {
-            if($dc->activeRecord->anrede == 'herr'){
-                $salutation = &$GLOBALS['TL_LANG']['tl_contact_person']['salutationMr'][0];
-            }else{
-                $salutation = &$GLOBALS['TL_LANG']['tl_contact_person']['salutationMrs'][0];
+            $prefix  = &$GLOBALS['TL_LANG']['tl_contact_person']['salutationGlobal'][0];
+            $prefix2 = null;
+
+            switch ($salutation = $dc->activeRecord->anrede)
+            {
+                case 'herr':
+                    $prefix  = &$GLOBALS['TL_LANG']['tl_contact_person']['salutationMr'][0];
+                    $prefix2 = &$GLOBALS['TL_LANG']['tl_contact_person'][$salutation][1];
+                    break;
+
+                case 'frau':
+                    $prefix  = &$GLOBALS['TL_LANG']['tl_contact_person']['salutationMrs'][0];
+                    $prefix2 = &$GLOBALS['TL_LANG']['tl_contact_person'][$salutation][1];
+                    break;
+
+                case 'eheleute':
+                case 'familie':
+                case 'firma':
+                    $prefix2 = &$GLOBALS['TL_LANG']['tl_contact_person'][$salutation][1];
+                    break;
             }
 
-            $varValue = $salutation . ' ' . $GLOBALS['TL_LANG']['tl_contact_person'][$dc->activeRecord->anrede][0] . ' ' . ($dc->activeRecord->titel ? $dc->activeRecord->titel . ' ' : '') . $dc->activeRecord->vorname . ' ' . $dc->activeRecord->name;
+            $varValue = $prefix . ($prefix2 ? ' '. $prefix2 : '') . ' ' . ($dc->activeRecord->titel ? $dc->activeRecord->titel . ' ' : '') . $dc->activeRecord->vorname . ' ' . $dc->activeRecord->name;
         }
 
         return $varValue;

diff --git a/src/Resources/contao/dca/tl_interface_history.php b/src/Resources/contao/dca/tl_interface_history.php
@@ -90,6 +90,10 @@
             'flag'                    => 6,
             'sql'                     => "int(10) unsigned NOT NULL default '0'"
         ),
+        'mtime' => array
+        (
+            'sql'                     => "int(10) unsigned NOT NULL default '0'"
+        ),
         'source' => array
         (
             'label'                   => &$GLOBALS['TL_LANG']['tl_interface_history']['source'],
@@ -119,6 +123,12 @@
             'search'                  => true,
             'sql'                     => "text NULL"
         ),
+        'message' => array
+        (
+            'label'                   => &$GLOBALS['TL_LANG']['tl_interface_history']['message'],
+            'search'                  => true,
+            'sql'                     => "text NULL"
+        ),
         'status' => array
         (
             'label'                   => &$GLOBALS['TL_LANG']['tl_interface_history']['username'],

diff --git a/src/Resources/contao/dca/tl_real_estate.php b/src/Resources/contao/dca/tl_real_estate.php
@@ -219,7 +219,7 @@
             'sql'                     => "varchar(64) NOT NULL default ''"
         ),
 
-         // Objektkategorien
+        // Objektkategorien
         'nutzungsart' => array
         (
             'label'                     => &$GLOBALS['TL_LANG']['tl_real_estate']['nutzungsart'],
@@ -2272,7 +2272,7 @@
             'inputType'                 => 'text',
             'search'                    => true,
             'flag'                      => 1,
-            'eval'                      => array('mandatory'=>true, 'maxlength'=>255, 'tl_class'=> 'w50'),
+            'eval'                      => array('maxlength'=>255, 'tl_class'=> 'w50'),
             'sql'                       => "varchar(255) NOT NULL default ''",
         ),
         'objektbeschreibung'  => array
@@ -5238,6 +5238,13 @@ public function generateAlias($varValue, $dc, string $title=''): string
         if (!$varValue)
         {
             $title = $dc->activeRecord !== null ? $dc->activeRecord->objekttitel : $title;
+
+            // Do not generate an alias if no title is given
+            if (empty($title))
+            {
+                return '';
+            }
+
             $varValue = Contao\System::getContainer()->get('contao.slug.generator')->generate($title);
         }
 

diff --git a/src/Resources/contao/dca/tl_real_estate_config.php b/src/Resources/contao/dca/tl_real_estate_config.php
@@ -24,7 +24,7 @@
 	// Palettes
 	'palettes' => array
 	(
-		'default'                     => '{global_legend},estateManagerAdminEmail;{real_estate_list_legend},defaultSorting,statusTokenNewDisplayDuration,defaultNumberOfMainDetails,defaultNumberOfMainAttr,defaultImage;{provider_contact_legend},defaultContactPersonImage,defaultContactPersonFemaleImage,defaultContactPersonMaleImage;{number_legend:hide},numberFormatDecimals,numberFormatThousands;{filter_config_legend:hide},roomOptions'
+		'default'                     => '{global_legend},estateManagerAdminEmail;{real_estate_list_legend},defaultSorting,statusTokenNewDisplayDuration,defaultNumberOfMainDetails,defaultNumberOfMainAttr,defaultImage;{provider_contact_legend},defaultContactPersonImage,defaultContactPersonFemaleImage,defaultContactPersonMaleImage;{number_legend:hide},numberFormatDecimals,numberFormatThousands;{filter_config_legend:hide},roomOptions;{import_legend:hide},estateManagerMaxFileSize'
 	),
 
 	// Fields
@@ -125,5 +125,12 @@
             'inputType'               => 'text',
             'eval'                    => array('mandatory'=>true, 'rgxp'=>'natural', 'tl_class'=>'w50')
         ),
+        'estateManagerMaxFileSize' => array
+        (
+            'label'                   => &$GLOBALS['TL_LANG']['tl_real_estate_config']['maxFileSize'],
+            'default'                 => '3000000',
+            'inputType'               => 'text',
+            'eval'                    => array('mandatory'=>true, 'rgxp'=>'natural', 'nospace'=>true, 'tl_class'=>'w50')
+        ),
 	)
 );