[Processing] Add proxy support for Get scripts and models (fixes #13412)

python/plugins/processing/gui/GetScriptsAndModels.py

@@ -20,7 +20,7 @@
 
 __author__ = 'Victor Olaya'
 __date__ = 'June 2014'
-__copyright__ = '(C) 201, Victor Olaya'
+__copyright__ = '(C) 2014, Victor Olaya'
 
 # This will get replaced with a git SHA1 when you do a git archive
 
@@ -28,14 +28,16 @@ __revision__ = '$Format:%H$'
 
 import os
 import json
-import urllib2
+from functools import partial
-from urllib2 import HTTPError
 
 from PyQt4 import uic
-from PyQt4.QtCore import Qt, QCoreApplication
+from PyQt4.QtCore import Qt, QCoreApplication, QUrl
-from PyQt4.QtGui import QIcon, QMessageBox, QCursor, QApplication, QTreeWidgetItem
+from PyQt4.QtGui import QIcon, QCursor, QApplication, QTreeWidgetItem, QPushButton
+from PyQt4.QtNetwork import QNetworkReply, QNetworkRequest
 
-from qgis.utils import iface
+from qgis.utils import iface, show_message_log
+from qgis.core import QgsNetworkAccessManager, QgsMessageLog
+from qgis.gui import QgsMessageBar
 
 from processing.gui.ToolboxAction import ToolboxAction
 from processing.script.ScriptUtils import ScriptUtils
@@ -48,7 +50,6 @@ pluginPath = os.path.split(os.path.dirname(__file__))[0]
 WIDGET, BASE = uic.loadUiType(
     os.path.join(pluginPath, 'ui', 'DlgGetScriptsAndModels.ui'))
 
-
 class GetScriptsAction(ToolboxAction):
 
     def __init__(self):
@@ -59,15 +60,10 @@ class GetScriptsAction(ToolboxAction):
         return QIcon(os.path.join(pluginPath, 'images', 'script.png'))
 
     def execute(self):
-        try:
+        dlg = GetScriptsAndModelsDialog(GetScriptsAndModelsDialog.SCRIPTS)
-            dlg = GetScriptsAndModelsDialog(GetScriptsAndModelsDialog.SCRIPTS)
+        dlg.exec_()
-            dlg.exec_()
+        if dlg.updateToolbox:
-            if dlg.updateToolbox:
+            self.toolbox.updateProvider('script')
-                self.toolbox.updateProvider('script')
-        except HTTPError:
-            QMessageBox.critical(iface.mainWindow(),
-                                 self.tr('Connection problem', 'GetScriptsAction'),
-                                 self.tr('Could not connect to scripts/models repository', 'GetScriptsAction'))
 
 
 class GetRScriptsAction(ToolboxAction):
@@ -80,15 +76,10 @@ class GetRScriptsAction(ToolboxAction):
         return QIcon(os.path.join(pluginPath, 'images', 'r.png'))
 
     def execute(self):
-        try:
+        dlg = GetScriptsAndModelsDialog(GetScriptsAndModelsDialog.RSCRIPTS)
-            dlg = GetScriptsAndModelsDialog(GetScriptsAndModelsDialog.RSCRIPTS)
+        dlg.exec_()
-            dlg.exec_()
+        if dlg.updateToolbox:
-            if dlg.updateToolbox:
+            self.toolbox.updateProvider('r')
-                self.toolbox.updateProvider('r')
-        except HTTPError:
-            QMessageBox.critical(iface.mainWindow(),
-                                 self.tr('Connection problem', 'GetRScriptsAction'),
-                                 self.tr('Could not connect to scripts/models repository', 'GetRScriptsAction'))
 
 
 class GetModelsAction(ToolboxAction):
@@ -101,23 +92,10 @@ class GetModelsAction(ToolboxAction):
         return QIcon(os.path.join(pluginPath, 'images', 'model.png'))
 
     def execute(self):
-        try:
+        dlg = GetScriptsAndModelsDialog(GetScriptsAndModelsDialog.MODELS)
-            dlg = GetScriptsAndModelsDialog(GetScriptsAndModelsDialog.MODELS)
+        dlg.exec_()
-            dlg.exec_()
+        if dlg.updateToolbox:
-            if dlg.updateToolbox:
+            self.toolbox.updateProvider('model')
-                self.toolbox.updateProvider('model')
-        except (HTTPError, URLError):
-            QMessageBox.critical(iface.mainWindow(),
-                                 self.tr('Connection problem', 'GetModelsAction'),
-                                 self.tr('Could not connect to scripts/models repository', 'GetModelsAction'))
-
-
-def readUrl(url):
-    try:
-        QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
-        return urllib2.urlopen(url).read()
-    finally:
-        QApplication.restoreOverrideCursor()
 
 
 class GetScriptsAndModelsDialog(BASE, WIDGET):
@@ -137,9 +115,14 @@ class GetScriptsAndModelsDialog(BASE, WIDGET):
     SCRIPTS = 1
     RSCRIPTS = 2
 
+    tr_disambiguation = { 0: 'GetModelsAction',
+                          1: 'GetScriptsAction',
+                          2: 'GetRScriptsAction' }
+
     def __init__(self, resourceType):
         super(GetScriptsAndModelsDialog, self).__init__(iface.mainWindow())
         self.setupUi(self)
+        self.manager = QgsNetworkAccessManager.instance()
 
         self.resourceType = resourceType
         if self.resourceType == self.MODELS:
@@ -160,8 +143,30 @@ class GetScriptsAndModelsDialog(BASE, WIDGET):
         self.populateTree()
         self.buttonBox.accepted.connect(self.okPressed)
         self.buttonBox.rejected.connect(self.cancelPressed)
-        self.tree.currentItemChanged .connect(self.currentItemChanged)
+        self.tree.currentItemChanged.connect(self.currentItemChanged)
+    
+    def popupError(self, error=None, url=None):
+        """Popups an Error message bar for network errors."""
+        disambiguation = self.tr_disambiguation[self.resourceType]
+        widget = iface.messageBar().createMessage(self.tr('Connection problem', disambiguation),
+                                                  self.tr('Could not connect to scripts/models repository', disambiguation))
+        if error and url:
+            QgsMessageLog.logMessage(self.tr(u"Network error code: {} on URL: {}").format(error, url), u"Processing", QgsMessageLog.CRITICAL)
+            button = QPushButton(QCoreApplication.translate("Python", "View message log"), pressed=show_message_log)
+            widget.layout().addWidget(button)
 
+        iface.messageBar().pushWidget(widget, level=QgsMessageBar.CRITICAL, duration=5)
+        
+    def grabHTTP(self, url, loadFunction, arguments=None):
+        """Grab distant content via QGIS internal classes and QtNetwork."""
+        QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
+        request = QUrl(url)
+        reply = self.manager.get(QNetworkRequest(request))
+        if arguments:
+            reply.finished.connect(partial(loadFunction, reply, arguments))
+        else:
+            reply.finished.connect(partial(loadFunction, reply))
+        
     def populateTree(self):
         self.uptodateItem = QTreeWidgetItem()
         self.uptodateItem.setText(0, self.tr('Installed'))
@@ -172,35 +177,53 @@ class GetScriptsAndModelsDialog(BASE, WIDGET):
         self.toupdateItem.setIcon(0, self.icon)
         self.uptodateItem.setIcon(0, self.icon)
         self.notinstalledItem.setIcon(0, self.icon)
-        resources = readUrl(self.urlBase + 'list.txt').splitlines()
+        self.grabHTTP(self.urlBase + 'list.txt', self.treeLoaded)
-        resources = [r.split(',') for r in resources]
-        self.resources = {f: (v, n) for f, v, n in resources}
-        for filename, version, name in sorted(resources, key=lambda kv: kv[2].lower()):
-            treeBranch = self.getTreeBranchForState(filename, float(version))
-            item = TreeItem(filename, name, self.icon)
-            treeBranch.addChild(item)
-            if treeBranch != self.notinstalledItem:
-                item.setCheckState(0, Qt.Checked)
 
+    def treeLoaded(self, reply):
+        """
+        update the tree of scripts/models whenever 
+        HTTP request is finished
+        """
+        QApplication.restoreOverrideCursor()
+        if reply.error() != QNetworkReply.NoError:
+            self.popupError(reply.error(), reply.request().url().toString())
+        else:
+            resources = unicode(reply.readAll()).splitlines()
+            resources = [r.split(',') for r in resources]
+            self.resources = {f: (v, n) for f, v, n in resources}
+            for filename, version, name in sorted(resources, key=lambda kv: kv[2].lower()):
+                treeBranch = self.getTreeBranchForState(filename, float(version))
+                item = TreeItem(filename, name, self.icon)
+                treeBranch.addChild(item)
+                if treeBranch != self.notinstalledItem:
+                    item.setCheckState(0, Qt.Checked)
+
+        reply.deleteLater()
         self.tree.addTopLevelItem(self.toupdateItem)
         self.tree.addTopLevelItem(self.notinstalledItem)
         self.tree.addTopLevelItem(self.uptodateItem)
 
         self.webView.setHtml(self.HELP_TEXT)
 
+    def setHelp(self, reply, item):
+        """Change the webview HTML content"""
+        QApplication.restoreOverrideCursor()
+        if reply.error() != QNetworkReply.NoError:
+            html = self.tr('<h2>No detailed description available for this script</h2>')
+        else:
+            content = unicode(reply.readAll())
+            descriptions = json.loads(content)
+            html = '<h2>%s</h2>' % item.name
+            html += self.tr('<p><b>Description:</b> %s</p>') % getDescription(ALG_DESC, descriptions)
+            html += self.tr('<p><b>Created by:</b> %s') % getDescription(ALG_CREATOR, descriptions)
+            html += self.tr('<p><b>Version:</b> %s') % getDescription(ALG_VERSION, descriptions)
+        reply.deleteLater()
+        self.webView.setHtml(html)
+        
     def currentItemChanged(self, item, prev):
         if isinstance(item, TreeItem):
-            try:
+            url = self.urlBase + item.filename.replace(' ', '%20') + '.help'
-                url = self.urlBase + item.filename.replace(' ', '%20') + '.help'
+            self.grabHTTP(url, self.setHelp, item)
-                helpContent = readUrl(url)
-                descriptions = json.loads(helpContent)
-                html = '<h2>%s</h2>' % item.name
-                html += self.tr('<p><b>Description:</b> %s</p>') % getDescription(ALG_DESC, descriptions)
-                html += self.tr('<p><b>Created by:</b> %s') % getDescription(ALG_CREATOR, descriptions)
-                html += self.tr('<p><b>Version:</b> %s') % getDescription(ALG_VERSION, descriptions)
-            except HTTPError as e:
-                html = self.tr('<h2>No detailed description available for this script</h2>')
-            self.webView.setHtml(html)
         else:
             self.webView.setHtml(self.HELP_TEXT)
 
@@ -223,6 +246,26 @@ class GetScriptsAndModelsDialog(BASE, WIDGET):
     def cancelPressed(self):
         self.close()
 
+    def storeFile(self, reply, filename):
+        """store a script/model that has been downloaded"""
+        QApplication.restoreOverrideCursor()
+        if reply.error() != QNetworkReply.NoError:
+            if os.path.splitext(filename)[1].lower() == '.help':
+                content = '{"ALG_VERSION" : %s}' % self.resources[filename[:-5]][0]
+            else:
+                self.popupError(reply.error(), reply.request().url().toString())
+                content = None
+        else:
+            content = reply.readAll()
+
+        reply.deleteLater()
+        if content:
+            path = os.path.join(self.folder, filename)
+            with open(path, 'w') as f:
+                f.write(content)
+
+        self.progressBar.setValue(self.progressBar.value() + 1)
+
     def okPressed(self):
         toDownload = []
         for i in xrange(self.toupdateItem.childCount()):
@@ -235,39 +278,27 @@ class GetScriptsAndModelsDialog(BASE, WIDGET):
                 toDownload.append(item.filename)
 
         if toDownload:
-            self.progressBar.setMaximum(len(toDownload))
+            self.progressBar.setMaximum(len(toDownload) * 2)
             for i, filename in enumerate(toDownload):
                 QCoreApplication.processEvents()
                 url = self.urlBase + filename.replace(' ', '%20')
-                try:
+                self.grabHTTP(url, self.storeFile, filename)
-                    code = readUrl(url)
+                
-                    path = os.path.join(self.folder, filename)
-                    with open(path, 'w') as f:
-                        f.write(code)
-                except HTTPError:
-                    QMessageBox.critical(iface.mainWindow(),
-                                         self.tr('Connection problem'),
-                                         self.tr('Could not download file: %s') % filename)
-                    return
                 url += '.help'
-                try:
+                self.grabHTTP(url, self.storeFile, filename + '.help')
-                    html = readUrl(url)
-                except HTTPError:
-                    html = '{"ALG_VERSION" : %s}' % self.resources[filename][0]
-
-                path = os.path.join(self.folder, filename + '.help')
-                with open(path, 'w') as f:
-                    f.write(html)
-                self.progressBar.setValue(i + 1)
 
         toDelete = []
         for i in xrange(self.uptodateItem.childCount()):
             item = self.uptodateItem.child(i)
             if item.checkState(0) == Qt.Unchecked:
                 toDelete.append(item.filename)
+
+        # Remove py and help files if they exist
         for filename in toDelete:
-            path = os.path.join(self.folder, filename)
+            for pathname in (filename, filename + u".help"):
-            os.remove(path)
+                path = os.path.join(self.folder, pathname)
+                if os.path.exists(path):
+                    os.remove(path)
 
         self.updateToolbox = len(toDownload) + len(toDelete) > 0
         self.close()