mirror of
https://github.com/qgis/QGIS.git
synced 2025-12-07 00:03:52 -05:00
[processing] refactor OGR algorithms to use commandline tools, not
bindings
This commit is contained in:
Alexander Bruy
parent
0df2b011fc
commit
a9216c7189
@ -138,10 +138,10 @@ class GdalUtils:
|
||||
def escapeAndJoin(strList):
|
||||
joined = ''
|
||||
for s in strList:
|
||||
if s[0]!='-' and ' ' in s:
|
||||
if s[0] != '-' and ' ' in s:
|
||||
escaped = '"' + s.replace('\\', '\\\\').replace('"', '\\"') \
|
||||
+ '"'
|
||||
else:
|
||||
escaped = s
|
||||
joined += escaped + ' '
|
||||
return joined.strip()
|
||||
return joined.strip()
|
||||
|
||||
@ -44,8 +44,6 @@ from processing.tools import dataobjects
|
||||
|
||||
class OgrAlgorithm(GdalAlgorithm):
|
||||
|
||||
DB = 'DB'
|
||||
|
||||
def ogrConnectionString(self, uri):
|
||||
ogrstr = None
|
||||
|
||||
|
||||
@ -64,6 +64,8 @@ class information(GdalAlgorithm):
|
||||
progress)
|
||||
output = self.getOutputValue(information.OUTPUT)
|
||||
f = open(output, 'w')
|
||||
f.write('<pre>')
|
||||
for s in GdalUtils.getConsoleOutput()[1:]:
|
||||
f.write('<p>' + str(s) + '</p>')
|
||||
f.write(unicode(s))
|
||||
f.write('</pre>')
|
||||
f.close()
|
||||
|
||||
@ -25,27 +25,22 @@ __copyright__ = '(C) 2012, Victor Olaya'
|
||||
|
||||
__revision__ = '$Format:%H$'
|
||||
|
||||
|
||||
try:
|
||||
from osgeo import gdal, ogr, osr
|
||||
gdalAvailable = True
|
||||
except:
|
||||
gdalAvailable = False
|
||||
import os
|
||||
|
||||
from PyQt4.QtCore import *
|
||||
from PyQt4.QtGui import *
|
||||
|
||||
from qgis.core import *
|
||||
from processing.core.GeoAlgorithmExecutionException import \
|
||||
GeoAlgorithmExecutionException
|
||||
|
||||
from processing.parameters.ParameterVector import ParameterVector
|
||||
from processing.parameters.ParameterString import ParameterString
|
||||
from processing.parameters.ParameterSelection import ParameterSelection
|
||||
from processing.outputs.OutputVector import OutputVector
|
||||
|
||||
from OgrAlgorithm import OgrAlgorithm
|
||||
from pyogr.ogr2ogr import *
|
||||
from processing.tools.system import *
|
||||
|
||||
GeomOperation = Enum(['NONE', 'SEGMENTIZE', 'SIMPLIFY_PRESERVE_TOPOLOGY'])
|
||||
from processing.algs.gdal.OgrAlgorithm import OgrAlgorithm
|
||||
from processing.algs.gdal.GdalUtils import GdalUtils
|
||||
|
||||
FORMATS = [
|
||||
'ESRI Shapefile',
|
||||
@ -101,9 +96,8 @@ class Ogr2Ogr(OgrAlgorithm):
|
||||
|
||||
OUTPUT_LAYER = 'OUTPUT_LAYER'
|
||||
INPUT_LAYER = 'INPUT_LAYER'
|
||||
DEST_DS = 'DEST_DS'
|
||||
DEST_FORMAT = 'DEST_FORMAT'
|
||||
DEST_DSCO = 'DEST_DSCO'
|
||||
FORMAT = 'FORMAT'
|
||||
OPTIONS = 'OPTIONS'
|
||||
|
||||
def defineCharacteristics(self):
|
||||
self.name = 'Convert format'
|
||||
@ -111,137 +105,51 @@ class Ogr2Ogr(OgrAlgorithm):
|
||||
|
||||
self.addParameter(ParameterVector(self.INPUT_LAYER, 'Input layer',
|
||||
[ParameterVector.VECTOR_TYPE_ANY], False))
|
||||
self.addParameter(ParameterSelection(self.DEST_FORMAT,
|
||||
self.addParameter(ParameterSelection(self.FORMAT,
|
||||
'Destination Format', FORMATS))
|
||||
self.addParameter(ParameterString(self.DEST_DSCO, 'Creation Options',
|
||||
''))
|
||||
self.addParameter(ParameterString(self.OPTIONS, 'Creation Options',
|
||||
'', optional=True))
|
||||
|
||||
self.addOutput(OutputVector(self.OUTPUT_LAYER, 'Output layer'))
|
||||
|
||||
def commandLineName(self):
|
||||
return "gdalogr:ogr2ogr"
|
||||
|
||||
|
||||
def processAlgorithm(self, progress):
|
||||
if not gdalAvailable:
|
||||
raise GeoAlgorithmExecutionException(
|
||||
'GDAL bindings not installed.')
|
||||
|
||||
input = self.getParameterValue(self.INPUT_LAYER)
|
||||
ogrLayer = self.ogrConnectionString(input)
|
||||
inLayer = self.getParameterValue(self.INPUT_LAYER)
|
||||
ogrLayer = self.ogrConnectionString(inLayer)
|
||||
|
||||
output = self.getOutputFromName(self.OUTPUT_LAYER)
|
||||
outfile = output.value
|
||||
|
||||
formatIdx = self.getParameterValue(self.DEST_FORMAT)
|
||||
outFile = output.value
|
||||
|
||||
formatIdx = self.getParameterValue(self.FORMAT)
|
||||
outFormat = FORMATS[formatIdx]
|
||||
ext = EXTS[formatIdx]
|
||||
if not outfile.endswith(ext):
|
||||
outfile = outfile + ext
|
||||
output.value = outfile
|
||||
if not outFile.endswith(ext):
|
||||
outFile += ext
|
||||
output.value = outFile
|
||||
|
||||
dst_ds = self.ogrConnectionString(outfile)
|
||||
dst_format = FORMATS[formatIdx]
|
||||
ogr_dsco = [self.getParameterValue(self.DEST_DSCO)]
|
||||
output = self.ogrConnectionString(outFile)
|
||||
options = unicode(self.getParameterValue(self.OPTIONS))
|
||||
|
||||
poDS = ogr.Open(ogrLayer, False)
|
||||
if poDS is None:
|
||||
raise GeoAlgorithmExecutionException(self.failure(ogrLayer))
|
||||
if outFormat == 'SQLite' and os.path.isfile(output):
|
||||
os.remove(output)
|
||||
|
||||
if dst_format == 'SQLite' and os.path.isfile(dst_ds):
|
||||
os.remove(dst_ds)
|
||||
driver = ogr.GetDriverByName(str(dst_format))
|
||||
poDstDS = driver.CreateDataSource(dst_ds, options=ogr_dsco)
|
||||
if poDstDS is None:
|
||||
raise GeoAlgorithmExecutionException('Error creating %s' % dst_ds)
|
||||
return
|
||||
self.ogrtransform(poDS, poDstDS, bOverwrite=True)
|
||||
arguments = []
|
||||
arguments.append('-f')
|
||||
arguments.append(outFormat)
|
||||
if len(options) > 0:
|
||||
arguments.append(options)
|
||||
|
||||
def ogrtransform(
|
||||
self,
|
||||
poSrcDS,
|
||||
poDstDS,
|
||||
papszLayers=[],
|
||||
papszLCO=[],
|
||||
bTransform=False,
|
||||
bAppend=False,
|
||||
bUpdate=False,
|
||||
bOverwrite=False,
|
||||
poOutputSRS=None,
|
||||
poSourceSRS=None,
|
||||
pszNewLayerName=None,
|
||||
pszWHERE=None,
|
||||
papszSelFields=None,
|
||||
eGType=-2,
|
||||
eGeomOp=GeomOperation.NONE,
|
||||
dfGeomOpParam=0,
|
||||
papszFieldTypesToString=[],
|
||||
pfnProgress=None,
|
||||
pProgressData=None,
|
||||
nCountLayerFeatures=0,
|
||||
poClipSrc=None,
|
||||
poClipDst=None,
|
||||
bExplodeCollections=False,
|
||||
pszZField=None,
|
||||
):
|
||||
arguments.append(output)
|
||||
arguments.append(ogrLayer)
|
||||
|
||||
# Process each data source layer
|
||||
if len(papszLayers) == 0:
|
||||
nLayerCount = poSrcDS.GetLayerCount()
|
||||
papoLayers = [None for i in range(nLayerCount)]
|
||||
iLayer = 0
|
||||
|
||||
for iLayer in range(nLayerCount):
|
||||
poLayer = poSrcDS.GetLayer(iLayer)
|
||||
|
||||
if poLayer is None:
|
||||
raise GeoAlgorithmExecutionException(
|
||||
"FAILURE: Couldn't fetch advertised layer %d!"
|
||||
% iLayer)
|
||||
|
||||
papoLayers[iLayer] = poLayer
|
||||
iLayer = iLayer + 1
|
||||
commands = []
|
||||
if isWindows():
|
||||
commands = ['cmd.exe', '/C ', 'ogr2ogr.exe',
|
||||
GdalUtils.escapeAndJoin(arguments)]
|
||||
else:
|
||||
# Process specified data source layers
|
||||
nLayerCount = len(papszLayers)
|
||||
papoLayers = [None for i in range(nLayerCount)]
|
||||
iLayer = 0
|
||||
commands = ['ogr2ogr', GdalUtils.escapeAndJoin(arguments)]
|
||||
|
||||
for layername in papszLayers:
|
||||
poLayer = poSrcDS.GetLayerByName(layername)
|
||||
GdalUtils.runGdal(commands, progress)
|
||||
|
||||
if poLayer is None:
|
||||
raise GeoAlgorithmExecutionException(
|
||||
"FAILURE: Couldn't fetch advertised layer %s!"
|
||||
% layername)
|
||||
|
||||
papoLayers[iLayer] = poLayer
|
||||
iLayer = iLayer + 1
|
||||
|
||||
for poSrcLayer in papoLayers:
|
||||
ok = TranslateLayer(
|
||||
poSrcDS,
|
||||
poSrcLayer,
|
||||
poDstDS,
|
||||
papszLCO,
|
||||
pszNewLayerName,
|
||||
bTransform,
|
||||
poOutputSRS,
|
||||
poSourceSRS,
|
||||
papszSelFields,
|
||||
bAppend,
|
||||
eGType,
|
||||
bOverwrite,
|
||||
eGeomOp,
|
||||
dfGeomOpParam,
|
||||
papszFieldTypesToString,
|
||||
nCountLayerFeatures,
|
||||
poClipSrc,
|
||||
poClipDst,
|
||||
bExplodeCollections,
|
||||
pszZField,
|
||||
pszWHERE,
|
||||
pfnProgress,
|
||||
pProgressData,
|
||||
)
|
||||
return True
|
||||
|
||||
@ -25,14 +25,6 @@ __copyright__ = '(C) 2012, Victor Olaya'
|
||||
|
||||
__revision__ = '$Format:%H$'
|
||||
|
||||
import string
|
||||
import re
|
||||
|
||||
try:
|
||||
from osgeo import ogr
|
||||
ogrAvailable = True
|
||||
except:
|
||||
ogrAvailable = False
|
||||
|
||||
from PyQt4.QtCore import *
|
||||
from PyQt4.QtGui import *
|
||||
@ -40,144 +32,38 @@ from qgis.core import *
|
||||
|
||||
from processing.parameters.ParameterVector import ParameterVector
|
||||
from processing.outputs.OutputHTML import OutputHTML
|
||||
from OgrAlgorithm import OgrAlgorithm
|
||||
|
||||
from processing.algs.gdal.GdalUtils import GdalUtils
|
||||
from processing.algs.gdal.OgrAlgorithm import OgrAlgorithm
|
||||
|
||||
|
||||
class OgrInfo(OgrAlgorithm):
|
||||
|
||||
INPUT = 'INPUT'
|
||||
OUTPUT = 'OUTPUT'
|
||||
INPUT_LAYER = 'INPUT_LAYER'
|
||||
|
||||
def defineCharacteristics(self):
|
||||
self.name = 'Information'
|
||||
self.group = '[OGR] Miscellaneous'
|
||||
|
||||
self.addParameter(ParameterVector(self.INPUT_LAYER, 'Input layer',
|
||||
self.addParameter(ParameterVector(self.INPUT, 'Input layer',
|
||||
[ParameterVector.VECTOR_TYPE_ANY], False))
|
||||
|
||||
self.addOutput(OutputHTML(self.OUTPUT, 'Layer information'))
|
||||
|
||||
def commandLineName(self):
|
||||
return "gdalogr:vectorinfo"
|
||||
|
||||
def processAlgorithm(self, progress):
|
||||
|
||||
input = self.getParameterValue(self.INPUT_LAYER)
|
||||
ogrLayer = self.ogrConnectionString(input)
|
||||
|
||||
arguments = []
|
||||
arguments.append('-al')
|
||||
arguments.append('-so')
|
||||
layer = self.getParameterValue(self.INPUT)
|
||||
conn = self.ogrConnectionString(layer)
|
||||
arguments.append(conn)
|
||||
GdalUtils.runGdal(['ogrinfo', GdalUtils.escapeAndJoin(arguments)],
|
||||
progress)
|
||||
output = self.getOutputValue(self.OUTPUT)
|
||||
|
||||
self.ogrinfo(ogrLayer)
|
||||
|
||||
f = open(output, 'w')
|
||||
f.write('<pre>' + self.info + '</pre>')
|
||||
f.write('<pre>')
|
||||
for s in GdalUtils.getConsoleOutput()[1:]:
|
||||
f.write(unicode(s))
|
||||
f.write('</pre>')
|
||||
f.close()
|
||||
|
||||
def out(self, text):
|
||||
self.info = self.info + text + '\n'
|
||||
|
||||
def ogrinfo(self, pszDataSource):
|
||||
bVerbose = True
|
||||
bSummaryOnly = True
|
||||
self.info = ''
|
||||
|
||||
if not ogrAvailable:
|
||||
self.info = 'OGR bindings not installed'
|
||||
return
|
||||
|
||||
qDebug("Opening data source '%s'" % pszDataSource)
|
||||
poDS = ogr.Open(pszDataSource, False)
|
||||
|
||||
if poDS is None:
|
||||
self.info = self.failure(pszDataSource)
|
||||
return
|
||||
|
||||
poDriver = poDS.GetDriver()
|
||||
|
||||
if bVerbose:
|
||||
self.out("INFO: Open of `%s'\n using driver `%s' successful."
|
||||
% (pszDataSource, poDriver.GetName()))
|
||||
|
||||
poDS_Name = poDS.GetName()
|
||||
if str(type(pszDataSource)) == "<type 'unicode'>" \
|
||||
and str(type(poDS_Name)) == "<type 'str'>":
|
||||
poDS_Name = unicode(poDS_Name, 'utf8')
|
||||
if bVerbose and pszDataSource != poDS_Name:
|
||||
self.out("INFO: Internal data source name '%s'\n \
|
||||
different from user name '%s'." \
|
||||
% (poDS_Name, pszDataSource))
|
||||
|
||||
# --------------------------------------------------------------------
|
||||
# Process each data source layer.
|
||||
# --------------------------------------------------------------------
|
||||
|
||||
for iLayer in range(poDS.GetLayerCount()):
|
||||
poLayer = poDS.GetLayer(iLayer)
|
||||
|
||||
if poLayer is None:
|
||||
self.out("FAILURE: Couldn't fetch advertised layer %d!"
|
||||
% iLayer)
|
||||
return 1
|
||||
|
||||
self.ReportOnLayer(poLayer)
|
||||
|
||||
def ReportOnLayer(
|
||||
self,
|
||||
poLayer,
|
||||
pszWHERE=None,
|
||||
poSpatialFilter=None,
|
||||
):
|
||||
bVerbose = True
|
||||
|
||||
poDefn = poLayer.GetLayerDefn()
|
||||
|
||||
# --------------------------------------------------------------------
|
||||
# Set filters if provided.
|
||||
# --------------------------------------------------------------------
|
||||
|
||||
if pszWHERE is not None:
|
||||
if poLayer.SetAttributeFilter(pszWHERE) != 0:
|
||||
self.out('FAILURE: SetAttributeFilter(%s) failed.' % pszWHERE)
|
||||
return
|
||||
|
||||
if poSpatialFilter is not None:
|
||||
poLayer.SetSpatialFilter(poSpatialFilter)
|
||||
|
||||
# --------------------------------------------------------------------
|
||||
# Report various overall information.
|
||||
# --------------------------------------------------------------------
|
||||
|
||||
self.out('')
|
||||
|
||||
self.out('Layer name: %s' % poDefn.GetName())
|
||||
|
||||
if bVerbose:
|
||||
self.out('Geometry: %s'
|
||||
% ogr.GeometryTypeToName(poDefn.GetGeomType()))
|
||||
|
||||
self.out('Feature Count: %d' % poLayer.GetFeatureCount())
|
||||
|
||||
oExt = poLayer.GetExtent(True, can_return_null=True)
|
||||
if oExt is not None:
|
||||
self.out('Extent: (%f, %f) - (%f, %f)' % (oExt[0], oExt[1],
|
||||
oExt[2], oExt[3]))
|
||||
|
||||
if poLayer.GetSpatialRef() is None:
|
||||
pszWKT = '(unknown)'
|
||||
else:
|
||||
pszWKT = poLayer.GetSpatialRef().ExportToPrettyWkt()
|
||||
|
||||
self.out('Layer SRS WKT:\n%s' % pszWKT)
|
||||
|
||||
if len(poLayer.GetFIDColumn()) > 0:
|
||||
self.out('FID Column = %s' % poLayer.GetFIDColumn())
|
||||
|
||||
if len(poLayer.GetGeometryColumn()) > 0:
|
||||
self.out('Geometry Column = %s' % poLayer.GetGeometryColumn())
|
||||
|
||||
for iAttr in range(poDefn.GetFieldCount()):
|
||||
poField = poDefn.GetFieldDefn(iAttr)
|
||||
|
||||
self.out('%s: %s (%d.%d)' % (poField.GetNameRef(),
|
||||
poField.GetFieldTypeName(poField.GetType()),
|
||||
poField.GetWidth(), poField.GetPrecision()))
|
||||
|
||||
@ -25,106 +25,53 @@ __copyright__ = '(C) 2012, Victor Olaya'
|
||||
|
||||
__revision__ = '$Format:%H$'
|
||||
|
||||
import string
|
||||
import re
|
||||
|
||||
try:
|
||||
from osgeo import ogr
|
||||
ogrAvailable = True
|
||||
except:
|
||||
ogrAvailable = False
|
||||
|
||||
from PyQt4.QtCore import *
|
||||
from PyQt4.QtGui import *
|
||||
from qgis.core import *
|
||||
|
||||
from processing.core.ProcessingLog import ProcessingLog
|
||||
from processing.core.GeoAlgorithmExecutionException import \
|
||||
GeoAlgorithmExecutionException
|
||||
from processing.parameters.ParameterVector import ParameterVector
|
||||
from processing.parameters.ParameterString import ParameterString
|
||||
from processing.outputs.OutputHTML import OutputHTML
|
||||
from processing.outputs.OutputVector import OutputVector
|
||||
|
||||
from OgrAlgorithm import OgrAlgorithm
|
||||
from processing.algs.gdal.GdalUtils import GdalUtils
|
||||
from processing.algs.gdal.OgrAlgorithm import OgrAlgorithm
|
||||
|
||||
|
||||
class OgrSql(OgrAlgorithm):
|
||||
|
||||
INPUT = 'INPUT'
|
||||
OUTPUT = 'OUTPUT'
|
||||
INPUT_LAYER = 'INPUT_LAYER'
|
||||
SQL = 'SQL'
|
||||
|
||||
def defineCharacteristics(self):
|
||||
self.name = 'Execute SQL'
|
||||
self.group = '[OGR] Miscellaneous'
|
||||
|
||||
self.addParameter(ParameterVector(self.INPUT_LAYER, 'Input layer',
|
||||
self.addParameter(ParameterVector(self.INPUT, 'Input layer',
|
||||
[ParameterVector.VECTOR_TYPE_ANY], False))
|
||||
self.addParameter(ParameterString(self.SQL, 'SQL', ''))
|
||||
|
||||
self.addOutput(OutputHTML(self.OUTPUT, 'SQL result'))
|
||||
self.addOutput(OutputVector(self.OUTPUT, 'SQL result'))
|
||||
|
||||
def processAlgorithm(self, progress):
|
||||
if not ogrAvailable:
|
||||
ProcessingLog.addToLog(ProcessingLog.LOG_ERROR,
|
||||
'OGR bindings not installed')
|
||||
return
|
||||
|
||||
input = self.getParameterValue(self.INPUT_LAYER)
|
||||
sql = self.getParameterValue(self.SQL)
|
||||
ogrLayer = self.ogrConnectionString(input)
|
||||
if sql == '':
|
||||
raise GeoAlgorithmExecutionException(
|
||||
'Empty SQL. Please enter valid SQL expression and try again.')
|
||||
|
||||
output = self.getOutputValue(self.OUTPUT)
|
||||
arguments = []
|
||||
arguments.append('-sql')
|
||||
arguments.append(sql)
|
||||
|
||||
qDebug("Opening data source '%s'" % ogrLayer)
|
||||
poDS = ogr.Open(ogrLayer, False)
|
||||
if poDS is None:
|
||||
ProcessingLog.addToLog(ProcessingLog.LOG_ERROR,
|
||||
self.failure(ogrLayer))
|
||||
return
|
||||
output = self.getOutputFromName(self.OUTPUT)
|
||||
outFile = output.value
|
||||
arguments.append(outFile)
|
||||
|
||||
result = self.select_values(poDS, sql)
|
||||
layer = self.getParameterValue(self.INPUT)
|
||||
conn = self.ogrConnectionString(layer)
|
||||
arguments.append(conn)
|
||||
|
||||
f = open(output, 'w')
|
||||
f.write('<table>')
|
||||
for row in result:
|
||||
f.write('<tr>')
|
||||
for col in row:
|
||||
f.write('<td>' + col + '</td>')
|
||||
f.write('</tr>')
|
||||
f.write('</table>')
|
||||
f.close()
|
||||
|
||||
def execute_sql(self, ds, sql_statement):
|
||||
poResultSet = ds.ExecuteSQL(sql_statement, None, None)
|
||||
if poResultSet is not None:
|
||||
ds.ReleaseResultSet(poResultSet)
|
||||
|
||||
def select_values(self, ds, sql_statement):
|
||||
"""Returns an array of the columns and values of SELECT
|
||||
statement:
|
||||
|
||||
select_values(ds, "SELECT id FROM companies") => [['id'],[1],[2],[3]]
|
||||
"""
|
||||
|
||||
poResultSet = ds.ExecuteSQL(sql_statement, None, None)
|
||||
|
||||
# TODO: Redirect error messages
|
||||
fields = []
|
||||
rows = []
|
||||
if poResultSet is not None:
|
||||
poDefn = poResultSet.GetLayerDefn()
|
||||
for iField in range(poDefn.GetFieldCount()):
|
||||
poFDefn = poDefn.GetFieldDefn(iField)
|
||||
fields.append(poFDefn.GetNameRef())
|
||||
|
||||
poFeature = poResultSet.GetNextFeature()
|
||||
while poFeature is not None:
|
||||
values = []
|
||||
for iField in range(poDefn.GetFieldCount()):
|
||||
if poFeature.IsFieldSet(iField):
|
||||
values.append(poFeature.GetFieldAsString(iField))
|
||||
else:
|
||||
values.append('(null)')
|
||||
rows.append(values)
|
||||
poFeature = poResultSet.GetNextFeature()
|
||||
ds.ReleaseResultSet(poResultSet)
|
||||
return [fields] + rows
|
||||
GdalUtils.runGdal(['ogr2ogr', GdalUtils.escapeAndJoin(arguments)],
|
||||
progress)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user