Merge pull request #49 from geo2france/dev

Merge dev 0.2.2

Author: jbdesbas

plugin/idg/gui/dlg_settings.py

@@ -135,8 +135,9 @@ def apply(self):
         # dump new settings into QgsSettings
         self.plg_settings.save_from_object(settings) #Les variables globales ne sont peut être pas MAJ ici
 
-        iface.mainWindow().findChildren(QWidget, 'Browser')[0].refresh() # refresh browser (supprimer et recreer le registre IDG plutôt ?)
-
+        registry = QgsApplication.dataItemProviderRegistry()
+        provider = registry.provider('IDG Provider')
+        provider.root.repopulate()
         if __debug__:
             self.log(
                 message="DEBUG - Settings successfully saved.",

plugin/idg/metadata.txt

@@ -7,8 +7,8 @@ description[fr]=Plugin fournissant un accès simple aux données de différentes
 about=Plugin providing easy access to data from different Spatial Data Infrastructures
 about[fr]=Plugin fournissant un accès simple aux données de différentes Infrastructures de Données Géographiques DataGrandEst, Géo2France, GeoBretagne et OPenIG
 icon=resources/images/layers-svgrepo-com.svg
-tags=opendata,sdi,DataGrandEst,Géo2France,GéoBretagne,OPenIG
-tags[fr]=opendata,idg,DataGrandEst,Géo2France,GéoBretagne,OPenIG
+tags=opendata,sdi,DataGrandEst,Géo2France,GéoBretagne,OPenIG, crige
+tags[fr]=opendata,idg,DataGrandEst,Géo2France,GéoBretagne,OPenIG,crige
 
 # credits and contact
 author=Benjamin CHARTIER, Jean-Baptiste DESBAS
@@ -24,5 +24,5 @@ qgisMinimumVersion=3.00
 qgisMaximumVersion=3.99
 
 # versioning
-version=0.2.1
-changelog=
+version=0.2.2
+changelog=Amélioration performance et stabilité

plugin/idg/plugin_main.py

@@ -23,7 +23,7 @@
 from idg.gui.dock import DockWidget
 from idg.gui.about_box import AboutBox
 from idg.gui.param_box import ParamBox
-from idg.toolbelt.tree_node_factory import TreeNodeFactory, download_tree_config_file, download_all_config_files, download_default_idg_list
+from idg.toolbelt.tree_node_factory import TreeNodeFactory, download_tree_config_file, download_all_config_files, download_default_idg_list, DownloadAllConfigFilesAsync
 
 import os
 import json
@@ -57,15 +57,19 @@ def __init__(self, iface: QgisInterface):
 
         config_struct = None
         config_string = ""
-        
-        # Download the config if needed
-        #if self.need_download_tree_config_file():
-        #    download_tree_config_file(PluginGlobals.instance().CONFIG_FILE_URLS[0])
-        
-        # Read the resources tree file and update the GUI
-        #self.ressources_tree = TreeNodeFactory(PluginGlobals.instance().config_file_path).root_node # dev
-        download_default_idg_list()
-        download_all_config_files(RemotePlatforms().stock_idgs)
+
+        self.registry = QgsApplication.dataItemProviderRegistry()
+        self.provider = IdgProvider(self.iface)
+
+        self.iface.initializationCompleted.connect(self.post_ui_init)
+
+
+    def post_ui_init(self):
+        """Run after plugin's UI has been initialized."""
+        download_default_idg_list() # TODO a passer en asynchrone aussi ?
+        self.task = DownloadAllConfigFilesAsync(RemotePlatforms().stock_idgs)
+        self.task.finished.connect(self.populate_browser)
+        self.task.start()
 
     def need_download_tree_config_file(self):
         """
@@ -112,15 +116,17 @@ def initGui(self):
     # Create a menu
         self.createPluginMenu()
 
+        # Add browser IDG provider
+        self.registry.addProvider(self.provider)
+
         # Create a dockable panel with a tree of resources
         #self.dock = DockWidget()
         #self.dock.set_tree_content(self.ressources_tree)
         #self.iface.addDockWidget(Qt.RightDockWidgetArea, self.dock) # dev
 
-    # Add browser provider
-        self.registry = QgsApplication.dataItemProviderRegistry()       
-        self.provider = IdgProvider(self.iface)
-        self.registry.addProvider(self.provider)
+
+    def populate_browser(self):
+        self.provider.root.repopulate()
 
     def unload(self):
         """Cleans up when plugin is disabled/uninstalled."""

plugin/idg/toolbelt/__init__.py

@@ -7,3 +7,4 @@
 from .singleton import Singleton
 from .browser import IdgProvider
 from .remote_platforms import RemotePlatforms
+from .network_manager import NetworkRequestsManager  # noqa: F401

plugin/idg/toolbelt/browser.py

@@ -1,4 +1,4 @@
-from qgis.core import QgsDataItemProvider, QgsDataCollectionItem, QgsDataItem, QgsDataProvider, QgsProject, \
+from qgis.core import Qgis, QgsDataItemProvider, QgsDataCollectionItem, QgsDataItem, QgsDataProvider, QgsProject, \
     QgsLayerTreeLayer, QgsLayerTreeGroup, QgsMimeDataUtils, QgsAbstractMetadataBase, QgsApplication, QgsIconUtils
 from qgis.gui import QgisInterface
 from qgis.PyQt.QtGui import QIcon
@@ -8,9 +8,7 @@
 from idg.__about__ import __title__
 from qgis.PyQt.QtWidgets import QAction, QMenu
 from qgis.utils import iface
-
 import os.path
-import json
 import webbrowser
 
 
@@ -40,15 +38,16 @@ def capabilities(self):
         return QgsDataProvider.Net
 
     def createDataItem(self, path, parentItem):
-        root = RootCollection(self.iface)
-        return root
+        self.root = RootCollection(self.iface)
+        return self.root
 
 
 class RootCollection(QgsDataCollectionItem):
     def __init__(self, iface: QgisInterface):
         self.iface = iface
         QgsDataCollectionItem.__init__(self, None, "IDG", "/IDG")
         self.setIcon(QIcon(PluginGlobals.instance().plugin_path+'/resources/images/layers-svgrepo-com.svg'))
+        self.setState(Qgis.BrowserItemState.Populating)
 
     def actions(self, parent):
         actions = list()
@@ -71,15 +70,15 @@ def menus(self, parent):
         menu.addSeparator()
         menu.addAction(QAction(self.tr('Add URL'), menu, )) # TODO Liens vers le panneau Options de QGIS
         return [menu]
-        
-    def createChildren(self):
-        children = []
+
+    def repopulate(self):
+        self.refresh()
         for pf in RemotePlatforms().plateforms :
             if pf.is_hidden() :
                 continue
             pf_collection = PlatformCollection(plateform=pf)
-            children.append(pf_collection)
-        return children
+            self.addChildItem(pf_collection, refresh=True)
+        self.setState(Qgis.BrowserItemState.Populated)
 
 
 class PlatformCollection(QgsDataCollectionItem):

plugin/idg/toolbelt/network_manager.py

@@ -0,0 +1,92 @@
+#! python3  # noqa: E265
+
+"""
+    Perform network request.
+    (from https://github.com/geotribu/qtribu/blob/main/qtribu/toolbelt/network_manager.py)
+"""
+
+# ############################################################################
+# ########## Imports ###############
+# ##################################
+
+# Standard library
+import logging
+from os import remove,  path
+from shutil import copy
+# PyQGIS
+from qgis.core import QgsBlockingNetworkRequest, QgsFileDownloader
+from qgis.PyQt.QtCore import QByteArray, QCoreApplication, QEventLoop, QUrl
+
+# project
+from idg.toolbelt.log_handler import PlgLogger
+
+
+# ############################################################################
+# ########## Globals ###############
+# ##################################
+
+logger = logging.getLogger(__name__)
+
+# ############################################################################
+# ########## Classes ###############
+# ##################################
+
+
+class NetworkRequestsManager:
+    """Helper on network operations.
+
+    :param tr: method to translate
+    :type tr: func
+    """
+
+    def __init__(self):
+        """Initialization."""
+        self.log = PlgLogger().log
+        self.ntwk_requester = QgsBlockingNetworkRequest()
+
+    def tr(self, message: str) -> str:
+        """Get the translation for a string using Qt translation API.
+
+        :param message: string to be translated.
+        :type message: str
+
+        :returns: Translated version of message.
+        :rtype: str
+        """
+        return QCoreApplication.translate(self.__class__.__name__, message)
+
+
+    def download_file(self, remote_url: str, local_path: str) -> str:
+        """Download a file from a remote web server accessible through HTTP.
+
+        :param remote_url: remote URL
+        :type remote_url: str
+        :param local_path: path to the local file
+        :type local_path: str
+        :return: output path
+        :rtype: str
+        """
+
+        def dlCompleted():
+            self.log(message=f"Download of {remote_url} to {local_path} succeedeed", log_level=3)
+            copy(local_path+'_tmp', local_path)
+            remove(local_path+'_tmp')
+
+        self.log(
+            message=f"Downloading file from {remote_url} to {local_path}", log_level=4
+        )
+        # download it
+        loop = QEventLoop()
+        file_downloader = QgsFileDownloader(
+            url = QUrl(remote_url ), outputFileName=local_path+'_tmp', delayStart=True # Le téléchargement se fait dans un fichier temporaire, pour garder l'ancien fichier en cas d'échec
+        )
+        file_downloader.downloadCompleted.connect(dlCompleted)
+        file_downloader.downloadError.connect(
+            lambda e : self.log(message=f"Download of {remote_url} to {local_path} error {e}", log_level=1)
+        )
+        file_downloader.downloadExited.connect(loop.quit)
+        file_downloader.startDownload()
+        loop.exec_()
+
+
+        return local_path

plugin/idg/toolbelt/tree_node_factory.py

@@ -11,11 +11,13 @@
     QgsProject,
     QgsNetworkAccessManager,
     QgsNetworkReplyContent,
+    QgsFileDownloader
 )
 from qgis.PyQt.QtNetwork import QNetworkRequest, QNetworkReply
-from qgis.PyQt.QtCore import QUrl
+from qgis.PyQt.QtCore import QUrl, QThread, pyqtSignal
 
 from idg.toolbelt import PluginGlobals
+from .network_manager import NetworkRequestsManager
 from .nodes import WmsLayerTreeNode, WmsStyleLayerTreeNode, WmtsLayerTreeNode, WfsFeatureTypeTreeNode
 from .nodes import WfsFeatureTypeFilterTreeNode, GdalWmsConfigFileTreeNode, FolderTreeNode
 
@@ -28,14 +30,16 @@ def download_default_idg_list(url='https://raw.githubusercontent.com/geo2france/
     response: QgsNetworkReplyContent = manager.blockingGet(
         request, forceRefresh=True
     )
-    if response.error() == QNetworkReply.NoError:
-        try:
-            os.remove(local_file)
-        except OSError:
-            pass
-        with open(local_file, "wb") as local_config_file:
-            local_config_file.write(response.content())
-        return json.loads(bytes(response.content()).decode())
+    qntwk = NetworkRequestsManager()
+    local_file_name = qntwk.download_file(url, os.path.join(PluginGlobals.instance().config_dir_path, 'default_idg.json'))
+    if local_file_name is not None:
+        #try:
+        #    os.remove(local_file)
+        #except OSError:
+        #    pass
+        with open(local_file, "r") as local_config_file:
+            out = json.load(local_config_file)
+        return out
     #TOD gérer les erreur (garder le fichier précédent + avertissement)
 
 def download_all_config_files(idgs): #remplacer la list par un dict ({idg_id:url})
@@ -47,52 +51,28 @@ def download_all_config_files(idgs): #remplacer la list par un dict ({idg_id:url
         key = IDG_id, value = url
         rename local file
     """
+    #TODO a passer dans RemotePlatforms
+    qntwk = NetworkRequestsManager()
     for idg_id, url in idgs.items():
+        #continue si l'IDG est masquée
         idg_id = str(idg_id)
         request = QNetworkRequest(QUrl(url))
         manager = QgsNetworkAccessManager.instance()
         response: QgsNetworkReplyContent = manager.blockingGet(
             request, forceRefresh=True
         )
         suffix = os.path.splitext(os.path.basename(url))[-1]
-        local_file_name = os.path.join(PluginGlobals.instance().config_dir_path, idg_id + suffix)
-        if response.error() == QNetworkReply.NoError:
-            # Creer le dossier si non existant
-            try:
-                os.makedirs(os.path.join(PluginGlobals.instance().config_dir_path))
-            except OSError:
-                if not os.path.isdir(os.path.join(PluginGlobals.instance().config_dir_path)):
-                    raise
-            # Supprimer le fichier si existant
-            try :
-                os.remove(os.path.join(PluginGlobals.instance().config_dir_path, idg_id + '.qgz') )
-            except OSError:
-                pass
-            try :
-                os.remove(os.path.join(PluginGlobals.instance().config_dir_path, idg_id + '.qgs') )
-            except OSError:
-                pass
+        local_file_name = qntwk.download_file(url, os.path.join(PluginGlobals.instance().config_dir_path, idg_id + suffix))
+        if local_file_name :
             with open(local_file_name, "wb") as local_config_file:
                 local_config_file.write(response.content())
             # Download icon if custom TODO a factoriser
             project = QgsProject()
             project.read(local_file_name, QgsProject.ReadFlags()|QgsProject.FlagDontResolveLayers|QgsProject.FlagDontLoadLayouts)
             for l in project.metadata().links():
                 if l.name.lower().strip() == 'icon':
-                    request = QNetworkRequest(QUrl(l.url))
-                    manager = QgsNetworkAccessManager.instance()
                     suffix = os.path.splitext(os.path.basename(l.url))[-1]
-                    response: QgsNetworkReplyContent = manager.blockingGet(
-                        request, forceRefresh=True
-                    )
-                    if response.error() == QNetworkReply.NoError:
-                        local_icon_file_name = os.path.join(PluginGlobals.instance().config_dir_path, idg_id + suffix) #TODO : vérifier qu'il s'agit d'un type image
-                        try:
-                            os.remove(local_icon_file_name)
-                        except OSError:
-                            pass
-                        with open(local_icon_file_name, "wb") as icon_file:
-                            icon_file.write(response.content())
+                    qntwk.download_file(l.url, os.path.join(PluginGlobals.instance().config_dir_path, idg_id + suffix) )
                     break
 
         else :
@@ -101,6 +81,33 @@ def download_all_config_files(idgs): #remplacer la list par un dict ({idg_id:url
                 "Erreur", short_message, level=Qgis.Warning
             )
 
+class DownloadAllConfigFilesAsync(QThread):
+    finished = pyqtSignal()
+    def __init__(self, idgs):
+        super(QThread, self).__init__()
+        self.idgs=idgs
+    def run(self):
+        qntwk = NetworkRequestsManager()
+
+        for idg_id, url in self.idgs.items():
+            # continue si l'IDG est masquée
+            idg_id = str(idg_id)
+            suffix = os.path.splitext(os.path.basename(url))[-1]
+            local_file_name = qntwk.download_file(url, os.path.join(PluginGlobals.instance().config_dir_path,
+                                                                    idg_id + suffix))
+            if local_file_name:
+                project = QgsProject()
+                project.read(local_file_name,
+                             QgsProject.ReadFlags() | QgsProject.FlagDontResolveLayers | QgsProject.FlagDontLoadLayouts)
+                for l in project.metadata().links():
+                    if l.name.lower().strip() == 'icon':
+                        suffix = os.path.splitext(os.path.basename(l.url))[-1]
+                        qntwk.download_file(l.url,
+                                            os.path.join(PluginGlobals.instance().config_dir_path, idg_id + suffix))
+                        break
+        self.finished.emit()
+
+
 def download_tree_config_file(file_url):
     """
     Download the resources tree file