[processing] refactor Virtual vector layer algorithm

Add support for unioned layers (fix #12991)

python/plugins/processing/algs/qgis/BarPlot.py

@@ -50,9 +50,13 @@ class BarPlot(GeoAlgorithm):
 
         self.addParameter(ParameterTable(self.INPUT, self.tr('Input table')))
         self.addParameter(ParameterTableField(self.NAME_FIELD,
-                                              self.tr('Category name field'), self.INPUT))
+                                              self.tr('Category name field'),
+                                              self.INPUT,
+                                              ParameterTableField.DATA_TYPE_NUMBER))
         self.addParameter(ParameterTableField(self.VALUE_FIELD,
-                                              self.tr('Value field'), self.INPUT))
+                                              self.tr('Value field'),
+                                              self.INPUT,
+                                              ParameterTableField.DATA_TYPE_NUMBER))
 
         self.addOutput(OutputHTML(self.OUTPUT, self.tr('Bar plot')))
 

python/plugins/processing/algs/qgis/Datasources2Vrt.py

@@ -26,81 +26,61 @@ __copyright__ = '(C) 2015, Luigi Pirelli'
 __revision__ = '$Format:%H$'
 
 import os
+import codecs
+import xml.sax.saxutils
+
+from osgeo import ogr
 
 from processing.core.GeoAlgorithm import GeoAlgorithm
 from processing.core.GeoAlgorithmExecutionException import GeoAlgorithmExecutionException
-from processing.core.parameters import ParameterMultipleInput, ParameterBoolean
+from processing.core.parameters import ParameterMultipleInput
-from processing.core.outputs import OutputFile, OutputString
+from processing.core.parameters import ParameterBoolean
-
+from processing.core.outputs import OutputFile
-# algorithm specific imports
+from processing.core.outputs import OutputString
-from osgeo import ogr
-import xml.sax.saxutils
 
 
 class Datasources2Vrt(GeoAlgorithm):
+    DATASOURCES = 'DATASOURCES'
+    UNIONED = 'UNIONED'
 
-    """ This algorithm merge the layers of different data sources in
+    VRT_FILE = 'VRT_FILE'
-    a single vrt file
+    VRT_STRING = 'VRT_STRING'
-
-    This algo is especially useful in case an algo need multiple layers
-    but accept only one vrt in which the layers are specified
-    """
-
-    OUTPUT_VRT_FILE = 'OUTPUT_VRT_FILE'
-    OUTPUT_VRT_STRING = 'OUTPUT_VRT_STRING'
-    INPUT_DATASOURCES = 'INPUT_DATASOURCES'
-    INPUT_OVERWRITE_FLAG = 'INPUT_OVERWRITE_FLAG'
 
     def defineCharacteristics(self):
         self.name, self.i18n_name = self.trAlgorithm('Build virtual vector')
         self.group, self.i18n_group = self.trAlgorithm('Vector general tools')
 
-        self.addParameter(ParameterMultipleInput(name=self.INPUT_DATASOURCES,
+        self.addParameter(ParameterMultipleInput(self.DATASOURCES,
-                                                 description=self.tr('Input datasources'),
+                                                 self.tr('Input datasources'),
-                                                 datatype=ParameterMultipleInput.TYPE_VECTOR_ANY,
+                                                 ParameterMultipleInput.TYPE_VECTOR_ANY))
-                                                 optional=False))
+        self.addParameter(ParameterBoolean(self.UNIONED,
-        self.addParameter(ParameterBoolean(name=self.INPUT_OVERWRITE_FLAG,
+                                           self.tr('Create "unioned" VRT'),
-                                           description=self.tr('Overwrite output vrt'),
                                            default=False))
 
-        self.addOutput(OutputFile(self.OUTPUT_VRT_FILE,
+        self.addOutput(OutputFile(self.VRT_FILE,
                                   self.tr('Virtual vector'), ext='vrt'))
-        self.addOutput(OutputString(self.OUTPUT_VRT_STRING,
+        self.addOutput(OutputString(self.VRT_STRING,
                                     self.tr('Virtual string')))
 
     def processAlgorithm(self, progress):
-        input_layers = self.getParameterValue(self.INPUT_DATASOURCES)
+        input_layers = self.getParameterValue(self.DATASOURCES)
-        overwrite = self.getParameterValue(self.INPUT_OVERWRITE_FLAG)
+        unioned = self.getParameterValue(self.UNIONED)
-        outVrtPath = self.getOutputValue(self.OUTPUT_VRT_FILE)
+        vrtPath = self.getOutputValue(self.VRT_FILE)
-        outVrtString = self.getOutputValue(self.OUTPUT_VRT_STRING)
+        vrtString = self.getOutputValue(self.VRT_STRING)
 
-        ds = input_layers.split(";")
+        layers = input_layers.split(';')
-        if not isinstance(ds, list):
-            msg = "Input datasources would be a ';' separated list of path strings"
-            raise GeoAlgorithmExecutionException(msg)
 
-        if not ds:
+        vrtString = self.mergeDataSources2Vrt(layers,
-            msg = "Input data sources is empty"
+                                              vrtPath,
-            raise GeoAlgorithmExecutionException(msg)
+                                              union=unioned,
+                                              relative=False,
+                                              schema=False,
+                                              progress=progress)
 
-        if not overwrite and os.path.exists(outVrtPath):
+        self.setOutputValue(self.VRT_STRING, vrtString)
-            msg = "Output vrt: %s already exist and choosed to avoid overwrite it" % outVrtPath
-            raise GeoAlgorithmExecutionException(msg)
 
-        outVrtString = self.mergeDataSources2Vrt(data_sources=ds,
+    def mergeDataSources2Vrt(self, dataSources, outFile, union=False, relative=False,
-                                                 outfile=outVrtPath,
+                             schema=False, progress=None):
-                                                 relative=False,
-                                                 schema=False,
-                                                 progress=progress)
-
-        self.setOutputValue(self.OUTPUT_VRT_STRING, outVrtString)
-
-    def mergeDataSources2Vrt(self,
-                             data_sources=[],
-                             outfile=None,
-                             relative=False,
-                             schema=False,
-                             progress=None):
         '''Function to do the work of merging datasources in a single vrt format
 
         @param data_sources: Array of path strings
@@ -109,121 +89,115 @@ class Datasources2Vrt(GeoAlgorithm):
         @param schema: Schema flag
         @return: vrt in string format
         '''
-        if not data_sources or len(data_sources) == 0:
+        vrt = '<OGRVRTDataSource>'
-            return None
+        if union:
+            vrt += '<OGRVRTUnionLayer name="UnionedLayer">'
 
-        # Start the VRT file.
+        total = 100.0 / len(dataSources)
-        vrt = '<OGRVRTDataSource>\n'
+        for current, inFile in enumerate(dataSources):
+            progress.setPercentage(int(current * total))
 
-        # For each file open the datasource to read.
+            srcDS = ogr.Open(inFile, 0)
-        for i, infile in enumerate(data_sources):
+            if srcDS is None:
-            progress.setPercentage(int(100 * i / len(data_sources)))
+                raise GeoAlgorithmExecutionException(
-
+                    self.tr('Invalid datasource: {}'.format(infile)))
-            src_ds = ogr.Open(infile, update=0)
-
-            if src_ds is None:
-                msg = "Invalid datasource: %s" % infile
-                raise GeoAlgorithmExecutionException(msg)
 
             if schema:
-                infile = '@dummy@'
+                inFile = '@dummy@'
 
-            layer_list = []
+            for layer in srcDS:
-            for layer in src_ds:
+                layerDef = layer.GetLayerDefn()
-                layer_list.append(layer.GetLayerDefn().GetName())
+                layerName = layerDef.GetName()
 
-            # Process each source layer.
+                vrt += '<OGRVRTLayer name="{}">'.format(self.XmlEsc(layerName))
-            for name in layer_list:
+                vrt += '<SrcDataSource relativeToVRT="{}" shared="{}">{}</SrcDataSource>'.format(1 if relative else 0, not schema, self.XmlEsc(inFile))
-                layer = src_ds.GetLayerByName(name)
-                layerdef = layer.GetLayerDefn()
-
-                vrt += '  <OGRVRTLayer name="%s">\n' % self.XmlEsc(name)
-                vrt += '    <SrcDataSource relativeToVRT="%s" shared="%d">%s</SrcDataSource>\n' \
-                       % ('1' if relative else '0', not schema, self.XmlEsc(infile))
                 if schema:
-                    vrt += '    <SrcLayer>@dummy@</SrcLayer>\n'
+                    vrt += '<SrcLayer>@dummy@</SrcLayer>'
                 else:
-                    vrt += '    <SrcLayer>%s</SrcLayer>\n' % self.XmlEsc(name)
+                    vrt += '<SrcLayer>{}</SrcLayer>'.format(self.XmlEsc(layerName))
-                vrt += '    <GeometryType>%s</GeometryType>\n' \
+
-                       % self.GeomType2Name(layerdef.GetGeomType())
+                vrt += '<GeometryType>{}</GeometryType>'.format(self.GeomType2Name(layerDef.GetGeomType()))
-                srs = layer.GetSpatialRef()
+
-                if srs is not None:
+                crs = layer.GetSpatialRef()
-                    vrt += '    <LayerSRS>%s</LayerSRS>\n' \
+                if crs is not None:
-                           % (self.XmlEsc(srs.ExportToWkt()))
+                    vrt += '<LayerSRS>{}</LayerSRS>'.format(self.XmlEsc(crs.ExportToWkt()))
 
                 # Process all the fields.
-                for fld_index in range(layerdef.GetFieldCount()):
+                for fieldIdx in xrange(layerDef.GetFieldCount()):
-                    src_fd = layerdef.GetFieldDefn(fld_index)
+                    fieldDef = layerDef.GetFieldDefn(fieldIdx)
-                    if src_fd.GetType() == ogr.OFTInteger:
+                    vrt += '<Field name="{}" type="{}"'.format(self.XmlEsc(fieldDef.GetName()), self.fieldType2Name(fieldDef.GetType()))
-                        type = 'Integer'
-                    elif src_fd.GetType() == ogr.OFTString:
-                        type = 'String'
-                    elif src_fd.GetType() == ogr.OFTReal:
-                        type = 'Real'
-                    elif src_fd.GetType() == ogr.OFTStringList:
-                        type = 'StringList'
-                    elif src_fd.GetType() == ogr.OFTIntegerList:
-                        type = 'IntegerList'
-                    elif src_fd.GetType() == ogr.OFTRealList:
-                        type = 'RealList'
-                    elif src_fd.GetType() == ogr.OFTBinary:
-                        type = 'Binary'
-                    elif src_fd.GetType() == ogr.OFTDate:
-                        type = 'Date'
-                    elif src_fd.GetType() == ogr.OFTTime:
-                        type = 'Time'
-                    elif src_fd.GetType() == ogr.OFTDateTime:
-                        type = 'DateTime'
-                    else:
-                        type = 'String'
-
-                    vrt += '    <Field name="%s" type="%s"' \
-                           % (self.XmlEsc(src_fd.GetName()), type)
                     if not schema:
-                        vrt += ' src="%s"' % self.XmlEsc(src_fd.GetName())
+                        vrt += ' src="{}"'.format(self.XmlEsc(fieldDef.GetName()))
-                    if src_fd.GetWidth() > 0:
+                    if fieldDef.GetWidth() > 0:
-                        vrt += ' width="%d"' % src_fd.GetWidth()
+                        vrt += ' width="{}"'.format(fieldDef.GetWidth())
-                    if src_fd.GetPrecision() > 0:
+                    if fieldDef.GetPrecision() > 0:
-                        vrt += ' precision="%d"' % src_fd.GetPrecision()
+                        vrt += ' precision="{}"'.format(fieldDef.GetPrecision())
-                    vrt += '/>\n'
+                    vrt += '/>'
 
-                vrt += '  </OGRVRTLayer>\n'
+                vrt += '</OGRVRTLayer>'
 
-            # Сlose data source
+            srcDS.Destroy()
-            src_ds.Destroy()
 
-        vrt += '</OGRVRTDataSource>\n'
+        if union:
+            vrt += '</OGRVRTUnionLayer>'
 
-        if outfile is not None:
+        vrt += '</OGRVRTDataSource>'
-            f = open(outfile, "w")
+
-            f.write(vrt)
+        #TODO: pretty-print XML
-            f.close()
+
+        if outFile is not None:
+            with codecs.open(outFile, 'w') as f:
+                f.write(vrt)
 
         return vrt
 
-    def GeomType2Name(self, type):
+    def GeomType2Name(self, geomType):
-        if type == ogr.wkbUnknown:
+        if geomType == ogr.wkbUnknown:
             return 'wkbUnknown'
-        elif type == ogr.wkbPoint:
+        elif geomType == ogr.wkbPoint:
             return 'wkbPoint'
-        elif type == ogr.wkbLineString:
+        elif geomType == ogr.wkbLineString:
             return 'wkbLineString'
-        elif type == ogr.wkbPolygon:
+        elif geomType == ogr.wkbPolygon:
             return 'wkbPolygon'
-        elif type == ogr.wkbMultiPoint:
+        elif geomType == ogr.wkbMultiPoint:
             return 'wkbMultiPoint'
-        elif type == ogr.wkbMultiLineString:
+        elif geomType == ogr.wkbMultiLineString:
             return 'wkbMultiLineString'
-        elif type == ogr.wkbMultiPolygon:
+        elif geomType == ogr.wkbMultiPolygon:
             return 'wkbMultiPolygon'
-        elif type == ogr.wkbGeometryCollection:
+        elif geomType == ogr.wkbGeometryCollection:
             return 'wkbGeometryCollection'
-        elif type == ogr.wkbNone:
+        elif geomType == ogr.wkbNone:
             return 'wkbNone'
-        elif type == ogr.wkbLinearRing:
+        elif geomType == ogr.wkbLinearRing:
             return 'wkbLinearRing'
         else:
             return 'wkbUnknown'
 
-    def XmlEsc(self, x):
+    def fieldType2Name(self, fieldType):
-        return xml.sax.saxutils.escape(x)
+        if fieldType == ogr.OFTInteger:
+            return 'Integer'
+        elif fieldType == ogr.OFTString:
+            return 'String'
+        elif fieldType == ogr.OFTReal:
+            return 'Real'
+        elif fieldType == ogr.OFTStringList:
+            return 'StringList'
+        elif fieldType == ogr.OFTIntegerList:
+            return 'IntegerList'
+        elif fieldType == ogr.OFTRealList:
+            return 'RealList'
+        elif fieldType == ogr.OFTBinary:
+            return 'Binary'
+        elif fieldType == ogr.OFTDate:
+            return 'Date'
+        elif fieldType == ogr.OFTTime:
+            return 'Time'
+        elif fieldType == ogr.OFTDateTime:
+            return 'DateTime'
+        else:
+            return 'String'
+
+    def XmlEsc(self, text):
+        return xml.sax.saxutils.escape(text)