[processing] refactor OGR algorithms to use commandline tools, not

bindings

python/plugins/processing/algs/gdal/GdalUtils.py

@@ -138,10 +138,10 @@ class GdalUtils:
     def escapeAndJoin(strList):
         joined = ''
         for s in strList:
-            if s[0]!='-' and ' ' in s:
+            if s[0] != '-' and ' ' in s:
                 escaped = '"' + s.replace('\\', '\\\\').replace('"', '\\"') \
                     + '"'
             else:
                 escaped = s
             joined += escaped + ' '
-        return joined.strip()
+        return joined.strip()

python/plugins/processing/algs/gdal/OgrAlgorithm.py

@@ -44,8 +44,6 @@ from processing.tools import dataobjects
 
 class OgrAlgorithm(GdalAlgorithm):
 
-    DB = 'DB'
-
     def ogrConnectionString(self, uri):
         ogrstr = None
 

python/plugins/processing/algs/gdal/information.py

@@ -64,6 +64,8 @@ class information(GdalAlgorithm):
                           progress)
         output = self.getOutputValue(information.OUTPUT)
         f = open(output, 'w')
+        f.write('<pre>')
         for s in GdalUtils.getConsoleOutput()[1:]:
-            f.write('<p>' + str(s) + '</p>')
+            f.write(unicode(s))
+        f.write('</pre>')
         f.close()

python/plugins/processing/algs/gdal/ogr2ogr.py

@@ -25,27 +25,22 @@ __copyright__ = '(C) 2012, Victor Olaya'
 
 __revision__ = '$Format:%H$'
 
-
+import os
-try:
-    from osgeo import gdal, ogr, osr
-    gdalAvailable = True
-except:
-    gdalAvailable = False
 
 from PyQt4.QtCore import *
 from PyQt4.QtGui import *
+
 from qgis.core import *
-from processing.core.GeoAlgorithmExecutionException import \
+
-        GeoAlgorithmExecutionException
 from processing.parameters.ParameterVector import ParameterVector
 from processing.parameters.ParameterString import ParameterString
 from processing.parameters.ParameterSelection import ParameterSelection
 from processing.outputs.OutputVector import OutputVector
 
-from OgrAlgorithm import OgrAlgorithm
+from processing.tools.system import *
-from pyogr.ogr2ogr import *
 
-GeomOperation = Enum(['NONE', 'SEGMENTIZE', 'SIMPLIFY_PRESERVE_TOPOLOGY'])
+from processing.algs.gdal.OgrAlgorithm import OgrAlgorithm
+from processing.algs.gdal.GdalUtils import GdalUtils
 
 FORMATS = [
     'ESRI Shapefile',
@@ -101,9 +96,8 @@ class Ogr2Ogr(OgrAlgorithm):
 
     OUTPUT_LAYER = 'OUTPUT_LAYER'
     INPUT_LAYER = 'INPUT_LAYER'
-    DEST_DS = 'DEST_DS'
+    FORMAT = 'FORMAT'
-    DEST_FORMAT = 'DEST_FORMAT'
+    OPTIONS = 'OPTIONS'
-    DEST_DSCO = 'DEST_DSCO'
 
     def defineCharacteristics(self):
         self.name = 'Convert format'
@@ -111,137 +105,51 @@ class Ogr2Ogr(OgrAlgorithm):
 
         self.addParameter(ParameterVector(self.INPUT_LAYER, 'Input layer',
                           [ParameterVector.VECTOR_TYPE_ANY], False))
-        self.addParameter(ParameterSelection(self.DEST_FORMAT,
+        self.addParameter(ParameterSelection(self.FORMAT,
                           'Destination Format', FORMATS))
-        self.addParameter(ParameterString(self.DEST_DSCO, 'Creation Options',
+        self.addParameter(ParameterString(self.OPTIONS, 'Creation Options',
-                          ''))
+                          '', optional=True))
 
         self.addOutput(OutputVector(self.OUTPUT_LAYER, 'Output layer'))
 
     def commandLineName(self):
         return "gdalogr:ogr2ogr"
 
-
     def processAlgorithm(self, progress):
-        if not gdalAvailable:
+        inLayer = self.getParameterValue(self.INPUT_LAYER)
-            raise GeoAlgorithmExecutionException(
+        ogrLayer = self.ogrConnectionString(inLayer)
-                    'GDAL bindings not installed.')
-
-        input = self.getParameterValue(self.INPUT_LAYER)
-        ogrLayer = self.ogrConnectionString(input)
 
         output = self.getOutputFromName(self.OUTPUT_LAYER)
-        outfile = output.value
+        outFile = output.value
-
-        formatIdx = self.getParameterValue(self.DEST_FORMAT)
 
+        formatIdx = self.getParameterValue(self.FORMAT)
+        outFormat = FORMATS[formatIdx]
         ext = EXTS[formatIdx]
-        if not outfile.endswith(ext):
+        if not outFile.endswith(ext):
-            outfile = outfile + ext
+            outFile += ext
-            output.value = outfile
+            output.value = outFile
 
-        dst_ds = self.ogrConnectionString(outfile)
+        output = self.ogrConnectionString(outFile)
-        dst_format = FORMATS[formatIdx]
+        options = unicode(self.getParameterValue(self.OPTIONS))
-        ogr_dsco = [self.getParameterValue(self.DEST_DSCO)]
 
-        poDS = ogr.Open(ogrLayer, False)
+        if outFormat == 'SQLite' and os.path.isfile(output):
-        if poDS is None:
+            os.remove(output)
-            raise GeoAlgorithmExecutionException(self.failure(ogrLayer))
 
-        if dst_format == 'SQLite' and os.path.isfile(dst_ds):
+        arguments = []
-            os.remove(dst_ds)
+        arguments.append('-f')
-        driver = ogr.GetDriverByName(str(dst_format))
+        arguments.append(outFormat)
-        poDstDS = driver.CreateDataSource(dst_ds, options=ogr_dsco)
+        if len(options) > 0:
-        if poDstDS is None:
+            arguments.append(options)
-            raise GeoAlgorithmExecutionException('Error creating %s' % dst_ds)
-            return
-        self.ogrtransform(poDS, poDstDS, bOverwrite=True)
 
-    def ogrtransform(
+        arguments.append(output)
-        self,
+        arguments.append(ogrLayer)
-        poSrcDS,
-        poDstDS,
-        papszLayers=[],
-        papszLCO=[],
-        bTransform=False,
-        bAppend=False,
-        bUpdate=False,
-        bOverwrite=False,
-        poOutputSRS=None,
-        poSourceSRS=None,
-        pszNewLayerName=None,
-        pszWHERE=None,
-        papszSelFields=None,
-        eGType=-2,
-        eGeomOp=GeomOperation.NONE,
-        dfGeomOpParam=0,
-        papszFieldTypesToString=[],
-        pfnProgress=None,
-        pProgressData=None,
-        nCountLayerFeatures=0,
-        poClipSrc=None,
-        poClipDst=None,
-        bExplodeCollections=False,
-        pszZField=None,
-        ):
 
-        # Process each data source layer
+        commands = []
-        if len(papszLayers) == 0:
+        if isWindows():
-            nLayerCount = poSrcDS.GetLayerCount()
+            commands = ['cmd.exe', '/C ', 'ogr2ogr.exe',
-            papoLayers = [None for i in range(nLayerCount)]
+                        GdalUtils.escapeAndJoin(arguments)]
-            iLayer = 0
-
-            for iLayer in range(nLayerCount):
-                poLayer = poSrcDS.GetLayer(iLayer)
-
-                if poLayer is None:
-                    raise GeoAlgorithmExecutionException(
-                            "FAILURE: Couldn't fetch advertised layer %d!"
-                            % iLayer)
-
-                papoLayers[iLayer] = poLayer
-                iLayer = iLayer + 1
         else:
-            # Process specified data source layers
+            commands = ['ogr2ogr', GdalUtils.escapeAndJoin(arguments)]
-            nLayerCount = len(papszLayers)
-            papoLayers = [None for i in range(nLayerCount)]
-            iLayer = 0
 
-            for layername in papszLayers:
+        GdalUtils.runGdal(commands, progress)
-                poLayer = poSrcDS.GetLayerByName(layername)
 
-                if poLayer is None:
-                    raise GeoAlgorithmExecutionException(
-                            "FAILURE: Couldn't fetch advertised layer %s!"
-                            % layername)
-
-                papoLayers[iLayer] = poLayer
-                iLayer = iLayer + 1
-
-        for poSrcLayer in papoLayers:
-            ok = TranslateLayer(
-                poSrcDS,
-                poSrcLayer,
-                poDstDS,
-                papszLCO,
-                pszNewLayerName,
-                bTransform,
-                poOutputSRS,
-                poSourceSRS,
-                papszSelFields,
-                bAppend,
-                eGType,
-                bOverwrite,
-                eGeomOp,
-                dfGeomOpParam,
-                papszFieldTypesToString,
-                nCountLayerFeatures,
-                poClipSrc,
-                poClipDst,
-                bExplodeCollections,
-                pszZField,
-                pszWHERE,
-                pfnProgress,
-                pProgressData,
-                )
-        return True

python/plugins/processing/algs/gdal/ogrinfo.py

@@ -25,14 +25,6 @@ __copyright__ = '(C) 2012, Victor Olaya'
 
 __revision__ = '$Format:%H$'
 
-import string
-import re
-
-try:
-    from osgeo import ogr
-    ogrAvailable = True
-except:
-    ogrAvailable = False
 
 from PyQt4.QtCore import *
 from PyQt4.QtGui import *
@@ -40,144 +32,38 @@ from qgis.core import *
 
 from processing.parameters.ParameterVector import ParameterVector
 from processing.outputs.OutputHTML import OutputHTML
-from OgrAlgorithm import OgrAlgorithm
+
+from processing.algs.gdal.GdalUtils import GdalUtils
+from processing.algs.gdal.OgrAlgorithm import OgrAlgorithm
 
 
 class OgrInfo(OgrAlgorithm):
 
+    INPUT = 'INPUT'
     OUTPUT = 'OUTPUT'
-    INPUT_LAYER = 'INPUT_LAYER'
 
     def defineCharacteristics(self):
         self.name = 'Information'
         self.group = '[OGR] Miscellaneous'
 
-        self.addParameter(ParameterVector(self.INPUT_LAYER, 'Input layer',
+        self.addParameter(ParameterVector(self.INPUT, 'Input layer',
                           [ParameterVector.VECTOR_TYPE_ANY], False))
 
         self.addOutput(OutputHTML(self.OUTPUT, 'Layer information'))
 
-    def commandLineName(self):
-        return "gdalogr:vectorinfo"
-
     def processAlgorithm(self, progress):
-
+        arguments = []
-        input = self.getParameterValue(self.INPUT_LAYER)
+        arguments.append('-al')
-        ogrLayer = self.ogrConnectionString(input)
+        arguments.append('-so')
-
+        layer = self.getParameterValue(self.INPUT)
+        conn = self.ogrConnectionString(layer)
+        arguments.append(conn)
+        GdalUtils.runGdal(['ogrinfo', GdalUtils.escapeAndJoin(arguments)],
+                          progress)
         output = self.getOutputValue(self.OUTPUT)
-
-        self.ogrinfo(ogrLayer)
-
         f = open(output, 'w')
-        f.write('<pre>' + self.info + '</pre>')
+        f.write('<pre>')
+        for s in GdalUtils.getConsoleOutput()[1:]:
+            f.write(unicode(s))
+        f.write('</pre>')
         f.close()
-
-    def out(self, text):
-        self.info = self.info + text + '\n'
-
-    def ogrinfo(self, pszDataSource):
-        bVerbose = True
-        bSummaryOnly = True
-        self.info = ''
-
-        if not ogrAvailable:
-            self.info = 'OGR bindings not installed'
-            return
-
-        qDebug("Opening data source '%s'" % pszDataSource)
-        poDS = ogr.Open(pszDataSource, False)
-
-        if poDS is None:
-            self.info = self.failure(pszDataSource)
-            return
-
-        poDriver = poDS.GetDriver()
-
-        if bVerbose:
-            self.out("INFO: Open of `%s'\n      using driver `%s' successful."
-                     % (pszDataSource, poDriver.GetName()))
-
-        poDS_Name = poDS.GetName()
-        if str(type(pszDataSource)) == "<type 'unicode'>" \
-            and str(type(poDS_Name)) == "<type 'str'>":
-            poDS_Name = unicode(poDS_Name, 'utf8')
-        if bVerbose and pszDataSource != poDS_Name:
-            self.out("INFO: Internal data source name '%s'\n \
-                      different from user name '%s'." \
-                      % (poDS_Name, pszDataSource))
-
-        # --------------------------------------------------------------------
-        #      Process each data source layer.
-        # --------------------------------------------------------------------
-
-        for iLayer in range(poDS.GetLayerCount()):
-            poLayer = poDS.GetLayer(iLayer)
-
-            if poLayer is None:
-                self.out("FAILURE: Couldn't fetch advertised layer %d!"
-                         % iLayer)
-                return 1
-
-            self.ReportOnLayer(poLayer)
-
-    def ReportOnLayer(
-        self,
-        poLayer,
-        pszWHERE=None,
-        poSpatialFilter=None,
-        ):
-        bVerbose = True
-
-        poDefn = poLayer.GetLayerDefn()
-
-    # --------------------------------------------------------------------
-    #      Set filters if provided.
-    # --------------------------------------------------------------------
-
-        if pszWHERE is not None:
-            if poLayer.SetAttributeFilter(pszWHERE) != 0:
-                self.out('FAILURE: SetAttributeFilter(%s) failed.' % pszWHERE)
-                return
-
-        if poSpatialFilter is not None:
-            poLayer.SetSpatialFilter(poSpatialFilter)
-
-    # --------------------------------------------------------------------
-    #      Report various overall information.
-    # --------------------------------------------------------------------
-
-        self.out('')
-
-        self.out('Layer name: %s' % poDefn.GetName())
-
-        if bVerbose:
-            self.out('Geometry: %s'
-                     % ogr.GeometryTypeToName(poDefn.GetGeomType()))
-
-            self.out('Feature Count: %d' % poLayer.GetFeatureCount())
-
-            oExt = poLayer.GetExtent(True, can_return_null=True)
-            if oExt is not None:
-                self.out('Extent: (%f, %f) - (%f, %f)' % (oExt[0], oExt[1],
-                         oExt[2], oExt[3]))
-
-            if poLayer.GetSpatialRef() is None:
-                pszWKT = '(unknown)'
-            else:
-                pszWKT = poLayer.GetSpatialRef().ExportToPrettyWkt()
-
-            self.out('Layer SRS WKT:\n%s' % pszWKT)
-
-            if len(poLayer.GetFIDColumn()) > 0:
-                self.out('FID Column = %s' % poLayer.GetFIDColumn())
-
-            if len(poLayer.GetGeometryColumn()) > 0:
-                self.out('Geometry Column = %s' % poLayer.GetGeometryColumn())
-
-            for iAttr in range(poDefn.GetFieldCount()):
-                poField = poDefn.GetFieldDefn(iAttr)
-
-                self.out('%s: %s (%d.%d)' % (poField.GetNameRef(),
-                         poField.GetFieldTypeName(poField.GetType()),
-                         poField.GetWidth(), poField.GetPrecision()))

python/plugins/processing/algs/gdal/ogrsql.py

@@ -25,106 +25,53 @@ __copyright__ = '(C) 2012, Victor Olaya'
 
 __revision__ = '$Format:%H$'
 
-import string
-import re
-
-try:
-    from osgeo import ogr
-    ogrAvailable = True
-except:
-    ogrAvailable = False
-
 from PyQt4.QtCore import *
 from PyQt4.QtGui import *
 from qgis.core import *
 
-from processing.core.ProcessingLog import ProcessingLog
+from processing.core.GeoAlgorithmExecutionException import \
+        GeoAlgorithmExecutionException
 from processing.parameters.ParameterVector import ParameterVector
 from processing.parameters.ParameterString import ParameterString
-from processing.outputs.OutputHTML import OutputHTML
+from processing.outputs.OutputVector import OutputVector
 
-from OgrAlgorithm import OgrAlgorithm
+from processing.algs.gdal.GdalUtils import GdalUtils
+from processing.algs.gdal.OgrAlgorithm import OgrAlgorithm
 
 
 class OgrSql(OgrAlgorithm):
 
+    INPUT = 'INPUT'
     OUTPUT = 'OUTPUT'
-    INPUT_LAYER = 'INPUT_LAYER'
     SQL = 'SQL'
 
     def defineCharacteristics(self):
         self.name = 'Execute SQL'
         self.group = '[OGR] Miscellaneous'
 
-        self.addParameter(ParameterVector(self.INPUT_LAYER, 'Input layer',
+        self.addParameter(ParameterVector(self.INPUT, 'Input layer',
                           [ParameterVector.VECTOR_TYPE_ANY], False))
         self.addParameter(ParameterString(self.SQL, 'SQL', ''))
 
-        self.addOutput(OutputHTML(self.OUTPUT, 'SQL result'))
+        self.addOutput(OutputVector(self.OUTPUT, 'SQL result'))
 
     def processAlgorithm(self, progress):
-        if not ogrAvailable:
-            ProcessingLog.addToLog(ProcessingLog.LOG_ERROR,
-                                   'OGR bindings not installed')
-            return
-
-        input = self.getParameterValue(self.INPUT_LAYER)
         sql = self.getParameterValue(self.SQL)
-        ogrLayer = self.ogrConnectionString(input)
+        if sql == '':
+            raise GeoAlgorithmExecutionException(
+                'Empty SQL. Please enter valid SQL expression and try again.')
 
-        output = self.getOutputValue(self.OUTPUT)
+        arguments = []
+        arguments.append('-sql')
+        arguments.append(sql)
 
-        qDebug("Opening data source '%s'" % ogrLayer)
+        output = self.getOutputFromName(self.OUTPUT)
-        poDS = ogr.Open(ogrLayer, False)
+        outFile = output.value
-        if poDS is None:
+        arguments.append(outFile)
-            ProcessingLog.addToLog(ProcessingLog.LOG_ERROR,
-                                   self.failure(ogrLayer))
-            return
 
-        result = self.select_values(poDS, sql)
+        layer = self.getParameterValue(self.INPUT)
+        conn = self.ogrConnectionString(layer)
+        arguments.append(conn)
 
-        f = open(output, 'w')
+        GdalUtils.runGdal(['ogr2ogr', GdalUtils.escapeAndJoin(arguments)],
-        f.write('<table>')
+                          progress)
-        for row in result:
-            f.write('<tr>')
-            for col in row:
-                f.write('<td>' + col + '</td>')
-            f.write('</tr>')
-        f.write('</table>')
-        f.close()
-
-    def execute_sql(self, ds, sql_statement):
-        poResultSet = ds.ExecuteSQL(sql_statement, None, None)
-        if poResultSet is not None:
-            ds.ReleaseResultSet(poResultSet)
-
-    def select_values(self, ds, sql_statement):
-        """Returns an array of the columns and values of SELECT
-        statement:
-
-        select_values(ds, "SELECT id FROM companies") => [['id'],[1],[2],[3]]
-        """
-
-        poResultSet = ds.ExecuteSQL(sql_statement, None, None)
-
-        # TODO: Redirect error messages
-        fields = []
-        rows = []
-        if poResultSet is not None:
-            poDefn = poResultSet.GetLayerDefn()
-            for iField in range(poDefn.GetFieldCount()):
-                poFDefn = poDefn.GetFieldDefn(iField)
-                fields.append(poFDefn.GetNameRef())
-
-            poFeature = poResultSet.GetNextFeature()
-            while poFeature is not None:
-                values = []
-                for iField in range(poDefn.GetFieldCount()):
-                    if poFeature.IsFieldSet(iField):
-                        values.append(poFeature.GetFieldAsString(iField))
-                    else:
-                        values.append('(null)')
-                rows.append(values)
-                poFeature = poResultSet.GetNextFeature()
-            ds.ReleaseResultSet(poResultSet)
-        return [fields] + rows