diff --git a/.gitignore b/.gitignore index d3a4446ffb9..3293e566de6 100644 --- a/.gitignore +++ b/.gitignore @@ -45,4 +45,6 @@ qgis-test.ctest i18n/*.qm .project .pydevproject -.idea \ No newline at end of file +.idea +/python/plugins/sextante/resources_rc.py +/python/plugins/sextante/about/ui_aboutdialogbase.py diff --git a/python/plugins/sextante/core/Sextante.py b/python/plugins/sextante/core/Sextante.py index e19a71e7bbd..8b8fa467da9 100644 --- a/python/plugins/sextante/core/Sextante.py +++ b/python/plugins/sextante/core/Sextante.py @@ -16,6 +16,7 @@ * * *************************************************************************** """ +from sextante.servertools.GeoServerToolsAlgorithmProvider import GeoServerToolsAlgorithmProvider __author__ = 'Victor Olaya' @@ -130,6 +131,7 @@ class Sextante: Sextante.addProvider(GrassAlgorithmProvider()) Sextante.addProvider(ScriptAlgorithmProvider()) Sextante.addProvider(TauDEMAlgorithmProvider()) + Sextante.addProvider(GeoServerToolsAlgorithmProvider()) Sextante.modeler.initializeSettings(); #and initialize SextanteLog.startLogging() diff --git a/python/plugins/sextante/database/DatabaseToolsAlgorithmProvider.py b/python/plugins/sextante/database/DatabaseToolsAlgorithmProvider.py new file mode 100644 index 00000000000..3257edca20e --- /dev/null +++ b/python/plugins/sextante/database/DatabaseToolsAlgorithmProvider.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- + +""" +*************************************************************************** + DatabaseToolProvider.py + --------------------- + Date : October 2012 + Copyright : (C) 2012 by Victor Olaya + Email : volayaf at gmail dot com +*************************************************************************** +* * +* This program is free software; you can redistribute it and/or modify * +* it under the terms of the GNU General Public License as published by * +* the Free Software Foundation; either version 2 of the License, or * +* (at your option) any later version. * +* * +*************************************************************************** +""" + +__author__ = 'Victor Olaya' +__date__ = 'October 2012' +__copyright__ = '(C) 2012, Victor Olaya' +# This will get replaced with a git SHA1 when you do a git archive +__revision__ = '$Format:%H$' + +from sextante.core.AlgorithmProvider import AlgorithmProvider +from PyQt4 import QtGui +import os + +class DatabaseToolsAlgorithmProvider(AlgorithmProvider): + + def __init__(self): + AlgorithmProvider.__init__(self) + self.alglist = []#PostGISSQL(), ImportIntoPostGIS(), CreateTable()] + + def initializeSettings(self): + AlgorithmProvider.initializeSettings(self) + + + def unload(self): + AlgorithmProvider.unload(self) + + + def getName(self): + return "database" + + def getDescription(self): + return "Database tools" + + def getIcon(self): + return QtGui.QIcon(os.path.dirname(__file__) + "/../images/postgis.png") + + def _loadAlgorithms(self): + self.algs = self.alglist + + def supportsNonFileBasedOutput(self): + return True \ No newline at end of file diff --git a/python/plugins/sextante/database/__init__.py b/python/plugins/sextante/database/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/plugins/sextante/gdal/translate.py b/python/plugins/sextante/gdal/translate.py index e3e287555ba..0d220686510 100644 --- a/python/plugins/sextante/gdal/translate.py +++ b/python/plugins/sextante/gdal/translate.py @@ -16,6 +16,7 @@ * * *************************************************************************** """ +from sextante.parameters.ParameterString import ParameterString __author__ = 'Victor Olaya' __date__ = 'August 2012' @@ -34,6 +35,7 @@ class translate(GeoAlgorithm): INPUT = "INPUT" OUTPUT = "OUTPUT" + EXTRA = "EXTRA" def getIcon(self): filepath = os.path.dirname(__file__) + "/icons/translate.png" @@ -43,14 +45,18 @@ class translate(GeoAlgorithm): self.name = "translate" self.group = "Conversion" self.addParameter(ParameterRaster(translate.INPUT, "Input layer", False)) + self.addParameter(ParameterString(translate.EXTRA, "Additional creation parameters")) self.addOutput(OutputRaster(translate.OUTPUT, "Output layer")) def processAlgorithm(self, progress): commands = ["gdal_translate"] commands.append("-of") out = self.getOutputValue(translate.OUTPUT) + extra = self.getOutputValue(translate.EXTRA) commands.append(GdalUtils.getFormatShortNameFromFilename(out)) + commands.append(extra) commands.append(self.getParameterValue(translate.INPUT)) commands.append(out) + GdalUtils.runGdal(commands, progress) diff --git a/python/plugins/sextante/grass/GrassAlgorithmProvider.py b/python/plugins/sextante/grass/GrassAlgorithmProvider.py index eebfec38549..1acf9d7174a 100644 --- a/python/plugins/sextante/grass/GrassAlgorithmProvider.py +++ b/python/plugins/sextante/grass/GrassAlgorithmProvider.py @@ -16,6 +16,7 @@ * * *************************************************************************** """ +from sextante.grass.nviz import nviz __author__ = 'Victor Olaya' __date__ = 'August 2012' @@ -37,10 +38,6 @@ class GrassAlgorithmProvider(AlgorithmProvider): def __init__(self): AlgorithmProvider.__init__(self) - #======================================================================= - # self.actions.append(DefineGrassRegionAction()) - # self.actions.append(DefineGrassRegionFromLayerAction()) - #======================================================================= self.createAlgsList() #preloading algorithms to speed up def initializeSettings(self): @@ -76,7 +73,7 @@ class GrassAlgorithmProvider(AlgorithmProvider): SextanteLog.addToLog(SextanteLog.LOG_ERROR, "Could not open GRASS algorithm: " + descriptionFile) except Exception,e: SextanteLog.addToLog(SextanteLog.LOG_ERROR, "Could not open GRASS algorithm: " + descriptionFile) - #self.preloadedAlgs.append(nviz()) + self.preloadedAlgs.append(nviz()) def _loadAlgorithms(self): self.algs = self.preloadedAlgs diff --git a/python/plugins/sextante/gui/ParametersPanel.py b/python/plugins/sextante/gui/ParametersPanel.py index 3359d5bd108..ea869cd746e 100644 --- a/python/plugins/sextante/gui/ParametersPanel.py +++ b/python/plugins/sextante/gui/ParametersPanel.py @@ -54,7 +54,6 @@ from sextante.outputs.OutputHTML import OutputHTML from sextante.outputs.OutputRaster import OutputRaster from sextante.outputs.OutputTable import OutputTable from sextante.outputs.OutputVector import OutputVector -from sextante.outputs.OutputNumber import OutputNumber from sextante.parameters.ParameterString import ParameterString class ParametersPanel(QtGui.QWidget): diff --git a/python/plugins/sextante/images/geoserver.png b/python/plugins/sextante/images/geoserver.png new file mode 100644 index 00000000000..78bb8909abd Binary files /dev/null and b/python/plugins/sextante/images/geoserver.png differ diff --git a/python/plugins/sextante/modeler/ModelerAlgorithm.py b/python/plugins/sextante/modeler/ModelerAlgorithm.py index b8d36d11c97..e1d283f2a3b 100644 --- a/python/plugins/sextante/modeler/ModelerAlgorithm.py +++ b/python/plugins/sextante/modeler/ModelerAlgorithm.py @@ -16,6 +16,7 @@ * * *************************************************************************** """ +from sextante.outputs.OutputString import OutputString __author__ = 'Victor Olaya' __date__ = 'August 2012' @@ -436,6 +437,8 @@ class ModelerAlgorithm(GeoAlgorithm): return "output html" elif isinstance(out, OutputNumber): return "output number" + elif isinstance(out, OutputString): + return "output string" def getAsPythonCode(self): diff --git a/python/plugins/sextante/modeler/ModelerArrowItem.py b/python/plugins/sextante/modeler/ModelerArrowItem.py index ca537c2b83e..12296025a6d 100644 --- a/python/plugins/sextante/modeler/ModelerArrowItem.py +++ b/python/plugins/sextante/modeler/ModelerArrowItem.py @@ -64,10 +64,14 @@ class ModelerArrowItem(QtGui.QGraphicsLineItem): return self.myEndItem def boundingRect(self): - extra = (self.pen().width() + 20) / 2.0 - p1 = self.line().p1() - p2 = self.line().p2() - return QtCore.QRectF(p1, QtCore.QSizeF(p2.x() - p1.x(), p2.y() - p1.y())).normalized().adjusted(-extra, -extra, extra, extra) + #this is a quick fix to avoid arrows not being drawn + return QtCore.QRectF(0, 0, 4000,4000) + #======================================================================= + # extra = (self.pen().width() + 20) / 2.0 + # p1 = self.line().p1() + # p2 = self.line().p2() + # return QtCore.QRectF(p1, QtCore.QSizeF(p2.x() - p1.x(), p2.y() - p1.y())).normalized().adjusted(-extra, -extra, extra, extra) + #======================================================================= def shape(self): path = super(ModelerArrowItem, self).shape() @@ -123,3 +127,4 @@ class ModelerArrowItem(QtGui.QGraphicsLineItem): painter.drawLine(line) painter.drawPolygon(self.arrowHead) + diff --git a/python/plugins/sextante/modeler/ModelerParametersDialog.py b/python/plugins/sextante/modeler/ModelerParametersDialog.py index 679e362b32f..7e779ff1202 100644 --- a/python/plugins/sextante/modeler/ModelerParametersDialog.py +++ b/python/plugins/sextante/modeler/ModelerParametersDialog.py @@ -17,6 +17,7 @@ *************************************************************************** """ from sextante.parameters.ParameterCrs import ParameterCrs +from sextante.outputs.OutputString import OutputString __author__ = 'Victor Olaya' __date__ = 'August 2012' @@ -317,6 +318,20 @@ class ModelerParametersDialog(QtGui.QDialog): for param in params: if isinstance(param, ParameterString): strings.append(AlgorithmAndParameter(AlgorithmAndParameter.PARENT_MODEL_ALGORITHM, param.name, "", param.description)) + + if self.algIndex is None: + dependent = [] + else: + dependent = self.model.getDependentAlgorithms(self.algIndex) + dependent.append(self.algIndex) + + i=0 + for alg in self.model.algs: + if i not in dependent: + for out in alg.outputs: + if isinstance(out, OutputString): + strings.append(AlgorithmAndParameter(i, out.name, alg.name, out.description)) + i+=1 return strings def getTableFields(self): diff --git a/python/plugins/sextante/modeler/ModelerScene.py b/python/plugins/sextante/modeler/ModelerScene.py index d83b22a8b6c..13bb71295bc 100644 --- a/python/plugins/sextante/modeler/ModelerScene.py +++ b/python/plugins/sextante/modeler/ModelerScene.py @@ -37,6 +37,7 @@ class ModelerScene(QtGui.QGraphicsScene): super(ModelerScene, self).__init__(parent) self.paramItems = [] self.algItems = [] + self.setItemIndexMethod(QtGui.QGraphicsScene.NoIndex); def getParameterPositions(self): pos = [] @@ -116,8 +117,7 @@ class ModelerScene(QtGui.QGraphicsScene): for sourceItem in sourceItems: arrow = ModelerArrowItem(sourceItem, self.algItems[iAlg]) self.addItem(arrow) - iAlg+=1 - + iAlg+=1 def mousePressEvent(self, mouseEvent): if (mouseEvent.button() != QtCore.Qt.LeftButton): diff --git a/python/plugins/sextante/modeler/ModelerUtils.py b/python/plugins/sextante/modeler/ModelerUtils.py index ababbc5e803..42c062105f3 100644 --- a/python/plugins/sextante/modeler/ModelerUtils.py +++ b/python/plugins/sextante/modeler/ModelerUtils.py @@ -24,7 +24,6 @@ __copyright__ = '(C) 2012, Victor Olaya' __revision__ = '$Format:%H$' import os -from sextante.core.SextanteUtils import SextanteUtils from sextante.core.SextanteUtils import mkdir from sextante.core.SextanteConfig import SextanteConfig @@ -37,8 +36,7 @@ class ModelerUtils: def modelsFolder(): folder = SextanteConfig.getSetting(ModelerUtils.MODELS_FOLDER) if folder == None: - #folder = os.path.join(os.path.dirname(__file__), "models") - folder = SextanteUtils.userFolder() + os.sep + "models" + folder = os.path.join(os.path.dirname(__file__), "models") mkdir(folder) return folder diff --git a/python/plugins/sextante/modeler/models/watersheds.model b/python/plugins/sextante/modeler/models/watersheds.model new file mode 100644 index 00000000000..0390107047b --- /dev/null +++ b/python/plugins/sextante/modeler/models/watersheds.model @@ -0,0 +1,75 @@ +NAME:Watersheds from DEM +GROUP:[Sample models] +PARAMETER:ParameterRaster|RASTERLAYER_DEM|DEM|False +458.0,50.0 +PARAMETER:ParameterNumber|NUMBER_INITIATIONTHRESHOLD|Initiation Threshold|None|None|10000000.0 +257.0,403.0 +VALUE:HARDCODEDPARAMVALUE_INIT_VALUE_1===10000000 +VALUE:HARDCODEDPARAMVALUE_SPLIT_3===0 +VALUE:HARDCODEDPARAMVALUE_Method_0===0 +VALUE:HARDCODEDPARAMVALUE_INIT_METHOD_1===2 +VALUE:HARDCODEDPARAMVALUE_CALC_METHOD_4===0 +VALUE:HARDCODEDPARAMVALUE_CLASS_ID_3===0 +VALUE:HARDCODEDPARAMVALUE_STEP_0===1 +VALUE:HARDCODEDPARAMVALUE_MINLEN_1===10 +VALUE:HARDCODEDPARAMVALUE_DOLINEAR _0===True +VALUE:HARDCODEDPARAMVALUE_MINSIZE_2===0 +VALUE:HARDCODEDPARAMVALUE_CLASS_ALL_3===1 +VALUE:HARDCODEDPARAMVALUE_LINEARTHRS_0===500.0 +VALUE:HARDCODEDPARAMVALUE_CONVERGENCE_0===1.0 +VALUE:HARDCODEDPARAMVALUE_DIV_CELLS_1===10 +ALGORITHM:saga:catchmentarea(parallel) +260.0,172.0 +-1|RASTERLAYER_DEM +None +None +None +None +-1|HARDCODEDPARAMVALUE_STEP_0 +-1|HARDCODEDPARAMVALUE_Method_0 +-1|HARDCODEDPARAMVALUE_DOLINEAR _0 +-1|HARDCODEDPARAMVALUE_LINEARTHRS_0 +None +None +-1|HARDCODEDPARAMVALUE_CONVERGENCE_0 +None +None +None +None +None +None +None +None +ALGORITHM:saga:channelnetwork +447.0,291.0 +-1|RASTERLAYER_DEM +None +0|CAREA +-1|HARDCODEDPARAMVALUE_INIT_METHOD_1 +-1|NUMBER_INITIATIONTHRESHOLD +None +-1|HARDCODEDPARAMVALUE_DIV_CELLS_1 +None +-1|HARDCODEDPARAMVALUE_MINLEN_1 +None +None +None +ALGORITHM:saga:watershedbasins +730.0,182.0 +-1|RASTERLAYER_DEM +1|CHNLNTWRK +None +-1|HARDCODEDPARAMVALUE_MINSIZE_2 +None +ALGORITHM:saga:vectorisinggridclasses +864.0,330.0 +2|BASINS +-1|HARDCODEDPARAMVALUE_CLASS_ALL_3 +-1|HARDCODEDPARAMVALUE_CLASS_ID_3 +-1|HARDCODEDPARAMVALUE_SPLIT_3 +None +ALGORITHM:ftools:export/addgeometrycolumns +655.0,442.0 +3|POLYGONS +-1|HARDCODEDPARAMVALUE_CALC_METHOD_4 +Watersheds diff --git a/python/plugins/sextante/outputs/OutputFactory.py b/python/plugins/sextante/outputs/OutputFactory.py index d1d9431fe2d..5971a393199 100644 --- a/python/plugins/sextante/outputs/OutputFactory.py +++ b/python/plugins/sextante/outputs/OutputFactory.py @@ -16,7 +16,6 @@ * * *************************************************************************** """ - __author__ = 'Victor Olaya' __date__ = 'August 2012' __copyright__ = '(C) 2012, Victor Olaya' @@ -29,12 +28,13 @@ from sextante.outputs.OutputTable import OutputTable from sextante.outputs.OutputVector import OutputVector from sextante.outputs.OutputNumber import OutputNumber from sextante.outputs.OutputFile import OutputFile +from sextante.outputs.OutputString import OutputString class OutputFactory(): @staticmethod def getFromString(s): - classes = [OutputRaster, OutputVector, OutputTable, OutputHTML, OutputNumber, OutputFile] + classes = [OutputRaster, OutputVector, OutputTable, OutputHTML, OutputNumber, OutputFile, OutputString] for clazz in classes: if s.startswith(clazz().outputTypeName()): tokens = s[len(clazz().outputTypeName())+1:].split("|") diff --git a/python/plugins/sextanteexampleprovider/__init__.py b/python/plugins/sextante/outputs/OutputString.py similarity index 53% rename from python/plugins/sextanteexampleprovider/__init__.py rename to python/plugins/sextante/outputs/OutputString.py index 749a01b0610..99f10835a2f 100644 --- a/python/plugins/sextanteexampleprovider/__init__.py +++ b/python/plugins/sextante/outputs/OutputString.py @@ -2,11 +2,11 @@ """ *************************************************************************** - __init__.py + OutputString.py --------------------- - Date : August 2012 - Copyright : (C) 2012 by Tim Sutton - Email : tim at linfiniti dot com + Date : October 2012 + Copyright : (C) 2012 by Victor Olaya + Email : volayaf at gmail dot com *************************************************************************** * * * This program is free software; you can redistribute it and/or modify * @@ -17,22 +17,18 @@ *************************************************************************** """ -__author__ = 'Tim Sutton' +__author__ = 'Victor Olaya' __date__ = 'August 2012' -__copyright__ = '(C) 2012, Tim Sutton' +__copyright__ = '(C) 2012, Victor Olaya' # This will get replaced with a git SHA1 when you do a git archive __revision__ = '$Format:%H$' -def name(): - return "SEXTANTE example provider" -def description(): - return "An example plugin that adds algorithms to SEXTANTE. Mainly created to guide developers in the process of creating plugins that add new capabilities to SEXTANTE" -def version(): - return "Version 1.0" -def icon(): - return "icon.png" -def qgisMinimumVersion(): - return "1.0" -def classFactory(iface): - from sextanteexampleprovider.SextanteExampleProviderPlugin import SextanteExampleProviderPlugin - return SextanteExampleProviderPlugin() +from sextante.outputs.Output import Output + +class OutputString(Output): + + def __init__(self, name="", description=""): + self.name = name + self.description = description + self.value = None + self.hidden = True diff --git a/python/plugins/sextante/pymorph/PymorphAlgorithmProvider.py b/python/plugins/sextante/pymorph/PymorphAlgorithmProvider.py index 34998b7db3a..da9d008b3a3 100644 --- a/python/plugins/sextante/pymorph/PymorphAlgorithmProvider.py +++ b/python/plugins/sextante/pymorph/PymorphAlgorithmProvider.py @@ -36,7 +36,7 @@ class PymorphAlgorithmProvider(AlgorithmProvider): def __init__(self): AlgorithmProvider.__init__(self) - #self.readAlgNames() + self.activate = False self.createAlgsList() def scriptsFolder(self): diff --git a/python/plugins/sextante/r/EditRScriptDialog.py b/python/plugins/sextante/r/EditRScriptDialog.py index e82afbbfe4f..6f128edcf7f 100644 --- a/python/plugins/sextante/r/EditRScriptDialog.py +++ b/python/plugins/sextante/r/EditRScriptDialog.py @@ -88,9 +88,11 @@ class EditRScriptDialog(QtGui.QDialog): def saveAlgorithm(self): if self.filename is None: - self.filename = QtGui.QFileDialog.getSaveFileName(self, "Save Script", RUtils.RScriptsFolder(), "SEXTANTE R script (*.rsx)") + self.filename = QtGui.QFileDialog.getSaveFileName(self, "Save Script", RUtils.RScriptsFolder(), "SEXTANTE R script (*.rsx)") if self.filename: + if not self.filename.endswith(".rsx"): + self.filename += ".rsx" text = str(self.text.toPlainText()) if self.alg is not None: self.alg.script = text diff --git a/python/plugins/sextante/script/EditScriptDialog.py b/python/plugins/sextante/script/EditScriptDialog.py index a87e595712d..4fe44658db5 100644 --- a/python/plugins/sextante/script/EditScriptDialog.py +++ b/python/plugins/sextante/script/EditScriptDialog.py @@ -91,6 +91,8 @@ class EditScriptDialog(QtGui.QDialog): self.filename = QtGui.QFileDialog.getSaveFileName(self, "Save Script", ScriptUtils.scriptsFolder(), "Python scripts (*.py)") if self.filename: + if not self.filename.endswith(".py"): + self.filename += ".py" text = str(self.text.toPlainText()) if self.alg is not None: self.alg.script = text diff --git a/python/plugins/sextante/servertools/CreateMosaicDatastore.py b/python/plugins/sextante/servertools/CreateMosaicDatastore.py new file mode 100644 index 00000000000..10d1f0fbb25 --- /dev/null +++ b/python/plugins/sextante/servertools/CreateMosaicDatastore.py @@ -0,0 +1,61 @@ +#=============================================================================== +# # -*- coding: utf-8 -*- +# +# """ +# *************************************************************************** +# CreateMosaicDatastore.py +# --------------------- +# Date : October 2012 +# Copyright : (C) 2012 by Victor Olaya +# Email : volayaf at gmail dot com +# *************************************************************************** +# * * +# * This program is free software; you can redistribute it and/or modify * +# * it under the terms of the GNU General Public License as published by * +# * the Free Software Foundation; either version 2 of the License, or * +# * (at your option) any later version. * +# * * +# *************************************************************************** +# """ +# from sextante.core.LayerExporter import LayerExporter +# from sextante.parameters.ParameterString import ParameterString +# from sextante.servertools.GeoServerToolsAlgorithm import GeoServerToolsAlgorithm +# +# __author__ = 'Victor Olaya' +# __date__ = 'October 2012' +# __copyright__ = '(C) 2012, Victor Olaya' +# # This will get replaced with a git SHA1 when you do a git archive +# __revision__ = '$Format:%H$' +# +# from qgis.core import * +# from sextante.parameters.ParameterVector import ParameterVector +# from sextante.core.QGisLayers import QGisLayers +# import os +# +# class CreateMosaicDatastore(GeoServerToolsAlgorithm): +# +# INPUT = "INPUT" +# WORKSPACE = "WORKSPACE" +# +# def processAlgorithm(self, progress): +# self.createCatalog() +# input = self.getParameterValue(self.INPUT) +# workspaceName = self.getParameterValue(self.WORKSPACE) +# connection = { +# 'shp': basepathname + '.shp', +# 'shx': basepathname + '.shx', +# 'dbf': basepathname + '.dbf', +# 'prj': basepathname + '.prj' +# } +# +# workspace = self.catalog.get_workspace(workspaceName) +# self.catalog.create_featurestore(basefilename, connection, workspace) +# +# +# def defineCharacteristics(self): +# self.addcaddBaseParameters() +# self.name = "Import into GeoServer" +# self.group = "GeoServer management tools" +# self.addParameter(ParameterVector(self.INPUT, "Layer to import", ParameterVector.VECTOR_TYPE_ANY)) +# self.addParameter(ParameterString(self.WORKSPACE, "Workspace")) +#=============================================================================== diff --git a/python/plugins/sextante/servertools/CreateWorkspace.py b/python/plugins/sextante/servertools/CreateWorkspace.py new file mode 100644 index 00000000000..5080203796a --- /dev/null +++ b/python/plugins/sextante/servertools/CreateWorkspace.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- + +""" +*************************************************************************** + CreateWorkspace.py + --------------------- + Date : October 2012 + Copyright : (C) 2012 by Victor Olaya + Email : volayaf at gmail dot com +*************************************************************************** +* * +* This program is free software; you can redistribute it and/or modify * +* it under the terms of the GNU General Public License as published by * +* the Free Software Foundation; either version 2 of the License, or * +* (at your option) any later version. * +* * +*************************************************************************** +""" +from sextante.servertools.GeoServerToolsAlgorithm import GeoServerToolsAlgorithm +from sextante.outputs.OutputString import OutputString + +__author__ = 'Victor Olaya' +__date__ = 'October 2012' +__copyright__ = '(C) 2012, Victor Olaya' +# This will get replaced with a git SHA1 when you do a git archive +__revision__ = '$Format:%H$' + +import os +from qgis.core import * +from PyQt4 import QtGui +from sextante.parameters.ParameterString import ParameterString + +class CreateWorkspace(GeoServerToolsAlgorithm): + + WORKSPACE = "WORKSPACE" + WORKSPACEURI = "WORKSPACEURI" + + def getIcon(self): + return QtGui.QIcon(os.path.dirname(__file__) + "/../images/geoserver.png") + + def processAlgorithm(self, progress): + self.createCatalog() + workspaceName = self.getParameterValue(self.WORKSPACE) + workspaceUri = self.getParameterValue(self.WORKSPACEURI) + self.catalog.create_workspace(workspaceName, workspaceUri) + + + def defineCharacteristics(self): + self.addBaseParameters() + self.name = "Create workspace" + self.group = "GeoServer management tools" + self.addParameter(ParameterString(self.WORKSPACE, "Workspace")) + self.addParameter(ParameterString(self.WORKSPACEURI, "Workspace URI")) + self.addOutput(OutputString(self.WORKSPACE, "Workspace")) + + diff --git a/python/plugins/sextante/servertools/DeleteDatastore.py b/python/plugins/sextante/servertools/DeleteDatastore.py new file mode 100644 index 00000000000..f2aabe829b4 --- /dev/null +++ b/python/plugins/sextante/servertools/DeleteDatastore.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- + +""" +*************************************************************************** + DeleteDatastore.py + --------------------- + Date : October 2012 + Copyright : (C) 2012 by Victor Olaya + Email : volayaf at gmail dot com +*************************************************************************** +* * +* This program is free software; you can redistribute it and/or modify * +* it under the terms of the GNU General Public License as published by * +* the Free Software Foundation; either version 2 of the License, or * +* (at your option) any later version. * +* * +*************************************************************************** +""" +from sextante.servertools.GeoServerToolsAlgorithm import GeoServerToolsAlgorithm + +__author__ = 'Victor Olaya' +__date__ = 'October 2012' +__copyright__ = '(C) 2012, Victor Olaya' +# This will get replaced with a git SHA1 when you do a git archive +__revision__ = '$Format:%H$' + +import os +from qgis.core import * +from PyQt4 import QtGui +from sextante.parameters.ParameterString import ParameterString + +class DeleteDatastore(GeoServerToolsAlgorithm): + + DATASTORE = "DATASTORE" + WORKSPACE = "WORKSPACE" + + def getIcon(self): + return QtGui.QIcon(os.path.dirname(__file__) + "/../images/geoserver.png") + + def processAlgorithm(self, progress): + self.createCatalog() + datastoreName = self.getParameterValue(self.DATASTORE) + workspaceName = self.getParameterValue(self.WORKSPACE) + ds = self.catalog.get_store(datastoreName, workspaceName) + self.catalog.delete(ds, recurse=True) + + + def defineCharacteristics(self): + self.addBaseParameters() + self.name = "Delete datastore" + self.group = "GeoServer management tools" + self.addParameter(ParameterString(self.DATASTORE, "Datastore name")) + self.addParameter(ParameterString(self.WORKSPACE, "Workspace")) + + + diff --git a/python/plugins/sextante/servertools/DeleteWorkspace.py b/python/plugins/sextante/servertools/DeleteWorkspace.py new file mode 100644 index 00000000000..0928fbb0636 --- /dev/null +++ b/python/plugins/sextante/servertools/DeleteWorkspace.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- + +""" +*************************************************************************** + DeleteWorkspace.py + --------------------- + Date : October 2012 + Copyright : (C) 2012 by Victor Olaya + Email : volayaf at gmail dot com +*************************************************************************** +* * +* This program is free software; you can redistribute it and/or modify * +* it under the terms of the GNU General Public License as published by * +* the Free Software Foundation; either version 2 of the License, or * +* (at your option) any later version. * +* * +*************************************************************************** +""" +from sextante.servertools.GeoServerToolsAlgorithm import GeoServerToolsAlgorithm + +__author__ = 'Victor Olaya' +__date__ = 'October 2012' +__copyright__ = '(C) 2012, Victor Olaya' +# This will get replaced with a git SHA1 when you do a git archive +__revision__ = '$Format:%H$' + +import os +from qgis.core import * +from PyQt4 import QtGui +from sextante.parameters.ParameterString import ParameterString + +class DeleteWorkspace(GeoServerToolsAlgorithm): + + WORKSPACE = "WORKSPACE" + + def getIcon(self): + return QtGui.QIcon(os.path.dirname(__file__) + "/../images/geoserver.png") + + def processAlgorithm(self, progress): + self.createCatalog() + workspaceName = self.getParameterValue(self.WORKSPACE) + ws = self.catalog.get_workspace(workspaceName) + self.catalog.delete(ws) + + + def defineCharacteristics(self): + self.addBaseParameters() + self.name = "Delete workspace" + self.group = "GeoServer management tools" + self.addParameter(ParameterString(self.WORKSPACE, "Workspace")) + + + diff --git a/python/plugins/sextante/servertools/GeoServerToolsAlgorithm.py b/python/plugins/sextante/servertools/GeoServerToolsAlgorithm.py new file mode 100644 index 00000000000..fb377a60a0b --- /dev/null +++ b/python/plugins/sextante/servertools/GeoServerToolsAlgorithm.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- + +""" +*************************************************************************** + GeoserverToolsAlgorithm.py + --------------------- + Date : October 2012 + Copyright : (C) 2012 by Victor Olaya + Email : volayaf at gmail dot com +*************************************************************************** +* * +* This program is free software; you can redistribute it and/or modify * +* it under the terms of the GNU General Public License as published by * +* the Free Software Foundation; either version 2 of the License, or * +* (at your option) any later version. * +* * +*************************************************************************** +""" +from sextante.parameters.ParameterString import ParameterString +from sextante.servertools.geoserver.catalog import Catalog + +__author__ = 'Victor Olaya' +__date__ = 'October 2012' +__copyright__ = '(C) 2012, Victor Olaya' +# This will get replaced with a git SHA1 when you do a git archive +__revision__ = '$Format:%H$' + +import os +from PyQt4 import QtGui +from sextante.core.GeoAlgorithm import GeoAlgorithm + +class GeoServerToolsAlgorithm(GeoAlgorithm): + + URL = "URL" + USER = "USER" + PASSWORD = "PASSWORD" + + def getIcon(self): + return QtGui.QIcon(os.path.dirname(__file__) + "/../images/geoserver.png") + + def addBaseParameters(self): + self.addParameter(ParameterString(self.URL, "URL", "http://localhost:8080/geoserver/rest")) + self.addParameter(ParameterString(self.USER, "User", "admin")) + self.addParameter(ParameterString(self.PASSWORD, "Password", "geoserver")) + + def createCatalog(self): + url = self.getParameterValue(self.URL) + user = self.getParameterValue(self.USER) + password = self.getParameterValue(self.PASSWORD) + self.catalog = Catalog(url, user, password) + + diff --git a/python/plugins/sextante/servertools/GeoServerToolsAlgorithmProvider.py b/python/plugins/sextante/servertools/GeoServerToolsAlgorithmProvider.py new file mode 100644 index 00000000000..f9ea1a22b7e --- /dev/null +++ b/python/plugins/sextante/servertools/GeoServerToolsAlgorithmProvider.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- + +""" +*************************************************************************** + GeoServerToolsAlgorithmProvider.py + --------------------- + Date : October 2012 + Copyright : (C) 2012 by Victor Olaya + Email : volayaf at gmail dot com +*************************************************************************** +* * +* This program is free software; you can redistribute it and/or modify * +* it under the terms of the GNU General Public License as published by * +* the Free Software Foundation; either version 2 of the License, or * +* (at your option) any later version. * +* * +*************************************************************************** +""" +from sextante.servertools.CreateWorkspace import CreateWorkspace +from sextante.servertools.ImportVectorIntoGeoServer import ImportVectorIntoGeoServer +from sextante.servertools.ImportRasterIntoGeoServer import ImportRasterIntoGeoServer +from sextante.servertools.DeleteWorkspace import DeleteWorkspace +from sextante.servertools.DeleteDatastore import DeleteDatastore + +__author__ = 'Victor Olaya' +__date__ = 'October 2012' +__copyright__ = '(C) 2012, Victor Olaya' +# This will get replaced with a git SHA1 when you do a git archive +__revision__ = '$Format:%H$' + +from sextante.core.AlgorithmProvider import AlgorithmProvider +from PyQt4 import QtGui +import os + +class GeoServerToolsAlgorithmProvider(AlgorithmProvider): + + def __init__(self): + AlgorithmProvider.__init__(self) + self.alglist = [ImportVectorIntoGeoServer(), ImportRasterIntoGeoServer(), + CreateWorkspace(), DeleteWorkspace(), DeleteDatastore()]#, CreateMosaicGeoserver(), StyleGeoserverLayer(), TruncateSeedGWC()] + + def initializeSettings(self): + AlgorithmProvider.initializeSettings(self) + + + def unload(self): + AlgorithmProvider.unload(self) + + + def getName(self): + return "geoserver" + + def getDescription(self): + return "Geoserver management tools" + + def getIcon(self): + return QtGui.QIcon(os.path.dirname(__file__) + "/../images/geoserver.png") + + def _loadAlgorithms(self): + self.algs = self.alglist + + def supportsNonFileBasedOutput(self): + return True \ No newline at end of file diff --git a/python/plugins/sextante/servertools/ImportRasterIntoGeoServer.py b/python/plugins/sextante/servertools/ImportRasterIntoGeoServer.py new file mode 100644 index 00000000000..d970a22f9d7 --- /dev/null +++ b/python/plugins/sextante/servertools/ImportRasterIntoGeoServer.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- + +""" +*************************************************************************** + ImportVectorIntoGeoServer.py + --------------------- + Date : October 2012 + Copyright : (C) 2012 by Victor Olaya + Email : volayaf at gmail dot com +*************************************************************************** +* * +* This program is free software; you can redistribute it and/or modify * +* it under the terms of the GNU General Public License as published by * +* the Free Software Foundation; either version 2 of the License, or * +* (at your option) any later version. * +* * +*************************************************************************** +""" +from sextante.core.LayerExporter import LayerExporter +from sextante.parameters.ParameterString import ParameterString +from sextante.servertools.GeoServerToolsAlgorithm import GeoServerToolsAlgorithm +from sextante.parameters.ParameterRaster import ParameterRaster + +__author__ = 'Victor Olaya' +__date__ = 'October 2012' +__copyright__ = '(C) 2012, Victor Olaya' +# This will get replaced with a git SHA1 when you do a git archive +__revision__ = '$Format:%H$' + +from qgis.core import * + +class ImportRasterIntoGeoServer(GeoServerToolsAlgorithm): + + INPUT = "INPUT" + WORKSPACE = "WORKSPACE" + NAME = "NAME" + + + def exportRasterLayer(self, inputFilename): + return inputFilename + + + def processAlgorithm(self, progress): + self.createCatalog() + inputFilename = self.getParameterValue(self.INPUT) + name = self.getParameterValue(self.NAME) + workspaceName = self.getParameterValue(self.WORKSPACE) + filename = self.exportRasterLayer(inputFilename) + workspace = self.catalog.get_workspace(workspaceName) + ds = self.catalog.create_coveragestore2(name, workspace) + ds.data_url = "file:" + filename; + self.catalog.save(ds) + + + def defineCharacteristics(self): + self.addBaseParameters() + self.name = "Import raster into GeoServer" + self.group = "GeoServer management tools" + self.addParameter(ParameterRaster(self.INPUT, "Layer to import")) + self.addParameter(ParameterString(self.WORKSPACE, "Workspace")) + self.addParameter(ParameterString(self.NAME, "Store name")) + diff --git a/python/plugins/sextante/servertools/ImportVectorIntoGeoServer.py b/python/plugins/sextante/servertools/ImportVectorIntoGeoServer.py new file mode 100644 index 00000000000..8c0c04b48ac --- /dev/null +++ b/python/plugins/sextante/servertools/ImportVectorIntoGeoServer.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- + +""" +*************************************************************************** + ImportVectorIntoGeoServer.py + --------------------- + Date : October 2012 + Copyright : (C) 2012 by Victor Olaya + Email : volayaf at gmail dot com +*************************************************************************** +* * +* This program is free software; you can redistribute it and/or modify * +* it under the terms of the GNU General Public License as published by * +* the Free Software Foundation; either version 2 of the License, or * +* (at your option) any later version. * +* * +*************************************************************************** +""" +from sextante.core.LayerExporter import LayerExporter +from sextante.parameters.ParameterString import ParameterString +from sextante.servertools.GeoServerToolsAlgorithm import GeoServerToolsAlgorithm + +__author__ = 'Victor Olaya' +__date__ = 'October 2012' +__copyright__ = '(C) 2012, Victor Olaya' +# This will get replaced with a git SHA1 when you do a git archive +__revision__ = '$Format:%H$' + +from qgis.core import * +from sextante.parameters.ParameterVector import ParameterVector +from sextante.core.QGisLayers import QGisLayers +import os + +class ImportVectorIntoGeoServer(GeoServerToolsAlgorithm): + + INPUT = "INPUT" + WORKSPACE = "WORKSPACE" + + def processAlgorithm(self, progress): + self.createCatalog() + inputFilename = self.getParameterValue(self.INPUT) + layer = QGisLayers.getObjectFromUri(inputFilename) + workspaceName = self.getParameterValue(self.WORKSPACE) + filename = LayerExporter.exportVectorLayer(layer) + basefilename = os.path.basename(filename) + basepathname = os.path.dirname(filename) + os.sep + basefilename[:basefilename.find('.')] + connection = { + 'shp': basepathname + '.shp', + 'shx': basepathname + '.shx', + 'dbf': basepathname + '.dbf', + 'prj': basepathname + '.prj' + } + + workspace = self.catalog.get_workspace(workspaceName) + self.catalog.create_featurestore(basefilename, connection, workspace) + + + def defineCharacteristics(self): + self.addBaseParameters() + self.name = "Import vector into GeoServer" + self.group = "GeoServer management tools" + self.addParameter(ParameterVector(self.INPUT, "Layer to import", ParameterVector.VECTOR_TYPE_ANY)) + self.addParameter(ParameterString(self.WORKSPACE, "Workspace")) + diff --git a/python/plugins/sextante/servertools/__init__.py b/python/plugins/sextante/servertools/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/plugins/sextante/servertools/geoserver/__init__.py b/python/plugins/sextante/servertools/geoserver/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/plugins/sextante/servertools/geoserver/catalog.py b/python/plugins/sextante/servertools/geoserver/catalog.py new file mode 100644 index 00000000000..d09631f38e7 --- /dev/null +++ b/python/plugins/sextante/servertools/geoserver/catalog.py @@ -0,0 +1,504 @@ +from datetime import datetime, timedelta +import logging +from sextante.servertools.geoserver.layer import Layer +from sextante.servertools.geoserver.store import coveragestore_from_index, datastore_from_index, \ + UnsavedDataStore, UnsavedCoverageStore +from sextante.servertools.geoserver.style import Style +from sextante.servertools.geoserver.support import prepare_upload_bundle, url +from sextante.servertools.geoserver.layergroup import LayerGroup, UnsavedLayerGroup +from sextante.servertools.geoserver.workspace import workspace_from_index, Workspace +from os import unlink +from xml.etree.ElementTree import XML +from xml.parsers.expat import ExpatError +from urlparse import urlparse +from sextante.servertools import httplib2 + +logger = logging.getLogger("gsconfig.catalog") + +class UploadError(Exception): + pass + +class ConflictingDataError(Exception): + pass + +class AmbiguousRequestError(Exception): + pass + +class FailedRequestError(Exception): + pass + +def _name(named): + """Get the name out of an object. This varies based on the type of the input: + * the "name" of a string is itself + * the "name" of None is itself + * the "name" of an object with a property named name is that property - + as long as it's a string + * otherwise, we raise a ValueError + """ + if isinstance(named, basestring) or named is None: + return named + elif hasattr(named, 'name') and isinstance(named.name, basestring): + return named.name + else: + raise ValueError("Can't interpret %s as a name or a configuration object" % named) + +class Catalog(object): + """ + The GeoServer catalog represents all of the information in the GeoServer + configuration. This includes: + - Stores of geospatial data + - Resources, or individual coherent datasets within stores + - Styles for resources + - Layers, which combine styles with resources to create a visible map layer + - LayerGroups, which alias one or more layers for convenience + - Workspaces, which provide logical grouping of Stores + - Maps, which provide a set of OWS services with a subset of the server's + Layers + - Namespaces, which provide unique identifiers for resources + """ + + def __init__(self, service_url, username="admin", password="geoserver", disable_ssl_certificate_validation=False): + self.service_url = service_url + if self.service_url.endswith("/"): + self.service_url = self.service_url.strip("/") + self.http = httplib2.Http( + disable_ssl_certificate_validation=disable_ssl_certificate_validation) + self.username = username + self.password = password + self.http.add_credentials(self.username, self.password) + netloc = urlparse(service_url).netloc + self.http.authorizations.append( + httplib2.BasicAuthentication( + (username, password), + netloc, + service_url, + {}, + None, + None, + self.http + )) + self._cache = dict() + + def delete(self, config_object, purge=False, recurse=False): + """ + send a delete request + XXX [more here] + """ + rest_url = config_object.href + + #params aren't supported fully in httplib2 yet, so: + params = [] + + # purge deletes the SLD from disk when a style is deleted + if purge: + params.append("purge=true") + + # recurse deletes the resource when a layer is deleted. + if recurse: + params.append("recurse=true") + + if params: + rest_url = rest_url + "?" + "&".join(params) + + headers = { + "Content-type": "application/xml", + "Accept": "application/xml" + } + response, content = self.http.request(rest_url, "DELETE", headers=headers) + self._cache.clear() + + if response.status == 200: + return (response, content) + else: + raise FailedRequestError("Tried to make a DELETE request to %s but got a %d status code: \n%s" % (rest_url, response.status, content)) + + def get_xml(self, rest_url): + logger.debug("GET %s", rest_url) + + cached_response = self._cache.get(rest_url) + + def is_valid(cached_response): + return cached_response is not None and datetime.now() - cached_response[0] < timedelta(seconds=5) + + def parse_or_raise(xml): + try: + return XML(xml) + except (ExpatError, SyntaxError), e: + msg = "GeoServer gave non-XML response for [GET %s]: %s" + msg = msg % (rest_url, xml) + raise Exception(msg, e) + + if is_valid(cached_response): + raw_text = cached_response[1] + return parse_or_raise(raw_text) + else: + response, content = self.http.request(rest_url) + if response.status == 200: + self._cache[rest_url] = (datetime.now(), content) + return parse_or_raise(content) + else: + raise FailedRequestError("Tried to make a GET request to %s but got a %d status code: \n%s" % (url, response.status, content)) + + def reload(self): + reload_url = url(self.service_url, ['reload']) + response = self.http.request(reload_url, "POST") + self._cache.clear() + return response + + def save(self, obj): + """ + saves an object to the REST service + + gets the object's REST location and the XML from the object, + then POSTS the request. + """ + rest_url = obj.href + message = obj.message() + + headers = { + "Content-type": "application/xml", + "Accept": "application/xml" + } + logger.debug("%s %s", obj.save_method, obj.href) + response = self.http.request(rest_url, obj.save_method, message, headers) + headers, body = response + self._cache.clear() + if 400 <= int(headers['status']) < 600: + raise FailedRequestError("Error code (%s) from GeoServer: %s" % + (headers['status'], body)) + return response + + def get_store(self, name, workspace=None): + #stores = [s for s in self.get_stores(workspace) if s.name == name] + if workspace is None: + store = None + for ws in self.get_workspaces(): + found = None + try: + found = self.get_store(name, ws) + except: + # don't expect every workspace to contain the named store + pass + if found: + if store: + raise AmbiguousRequestError("Multiple stores found named: " + name) + else: + store = found + if not store: + raise FailedRequestError("No store found named: " + name) + return store + else: # workspace is not None + if isinstance(workspace, basestring): + workspace = self.get_workspace(workspace) + if workspace is None: + return None + logger.debug("datastore url is [%s]", workspace.datastore_url ) + ds_list = self.get_xml(workspace.datastore_url) + cs_list = self.get_xml(workspace.coveragestore_url) + datastores = [n for n in ds_list.findall("dataStore") if n.find("name").text == name] + coveragestores = [n for n in cs_list.findall("coverageStore") if n.find("name").text == name] + ds_len, cs_len = len(datastores), len(coveragestores) + + if ds_len == 1 and cs_len == 0: + return datastore_from_index(self, workspace, datastores[0]) + elif ds_len == 0 and cs_len == 1: + return coveragestore_from_index(self, workspace, coveragestores[0]) + elif ds_len == 0 and cs_len == 0: + raise FailedRequestError("No store found in " + str(workspace) + " named: " + name) + else: + raise AmbiguousRequestError(str(workspace) + " and name: " + name + " do not uniquely identify a layer") + + def get_stores(self, workspace=None): + if workspace is not None: + if isinstance(workspace, basestring): + workspace = self.get_workspace(workspace) + ds_list = self.get_xml(workspace.datastore_url) + cs_list = self.get_xml(workspace.coveragestore_url) + datastores = [datastore_from_index(self, workspace, n) for n in ds_list.findall("dataStore")] + coveragestores = [coveragestore_from_index(self, workspace, n) for n in cs_list.findall("coverageStore")] + return datastores + coveragestores + else: + stores = [] + for ws in self.get_workspaces(): + a = self.get_stores(ws) + stores.extend(a) + return stores + + def create_datastore(self, name, workspace=None): + if isinstance(workspace, basestring): + workspace = self.get_workspace(workspace) + elif workspace is None: + workspace = self.get_default_workspace() + return UnsavedDataStore(self, name, workspace) + + def create_coveragestore2(self, name, workspace = None): + """ + Hm we already named the method that creates a coverage *resource* + create_coveragestore... time for an API break? + """ + if isinstance(workspace, basestring): + workspace = self.get_workspace(workspace) + elif workspace is None: + workspace = self.get_default_workspace() + return UnsavedCoverageStore(self, name, workspace) + + def add_data_to_store(self, store, name, data, workspace=None, overwrite = False, charset = None): + if isinstance(store, basestring): + store = self.get_store(store, workspace=workspace) + if workspace is not None: + workspace = _name(workspace) + assert store.workspace.name == workspace, "Specified store (%s) is not in specified workspace (%s)!" % (store, workspace) + else: + workspace = store.workspace.name + store = store.name + + if isinstance(data, dict): + bundle = prepare_upload_bundle(name, data) + else: + bundle = data + + params = dict() + if overwrite: + params["update"] = "overwrite" + if charset is not None: + params["charset"] = charset + + message = open(bundle) + headers = { 'Content-Type': 'application/zip', 'Accept': 'application/xml' } + upload_url = url(self.service_url, + ["workspaces", workspace, "datastores", store, "file.shp"], params) + + try: + headers, response = self.http.request(upload_url, "PUT", message, headers) + self._cache.clear() + if headers.status != 201: + raise UploadError(response) + finally: + unlink(bundle) + + def create_featurestore(self, name, data, workspace=None, overwrite=False, charset=None): + if not overwrite: + try: + store = self.get_store(name, workspace) + msg = "There is already a store named " + name + if workspace: + msg += " in " + str(workspace) + raise ConflictingDataError(msg) + except FailedRequestError: + # we don't really expect that every layer name will be taken + pass + + if workspace is None: + workspace = self.get_default_workspace() + workspace = _name(workspace) + params = dict() + if charset is not None: + params['charset'] = charset + ds_url = url(self.service_url, + ["workspaces", workspace, "datastores", name, "file.shp"], params) + + # PUT /workspaces//datastores//file.shp + headers = { + "Content-type": "application/zip", + "Accept": "application/xml" + } + if isinstance(data,dict): + logger.debug('Data is NOT a zipfile') + archive = prepare_upload_bundle(name, data) + else: + logger.debug('Data is a zipfile') + archive = data + message = open(archive) + try: + headers, response = self.http.request(ds_url, "PUT", message, headers) + self._cache.clear() + if headers.status != 201: + raise UploadError(response) + finally: + unlink(archive) + + def create_coveragestore(self, name, data, workspace=None, overwrite=False): + if not overwrite: + try: + store = self.get_store(name, workspace) + msg = "There is already a store named " + name + if workspace: + msg += " in " + str(workspace) + raise ConflictingDataError(msg) + except FailedRequestError: + # we don't really expect that every layer name will be taken + pass + + if workspace is None: + workspace = self.get_default_workspace() + headers = { + "Content-type": "image/tiff", + "Accept": "application/xml" + } + + archive = None + ext = "geotiff" + + if isinstance(data, dict): + archive = prepare_upload_bundle(name, data) + message = open(archive) + if "tfw" in data: + headers['Content-type'] = 'application/archive' + ext = "worldimage" + elif isinstance(data, basestring): + message = open(data) + else: + message = data + + cs_url = url(self.service_url, + ["workspaces", workspace.name, "coveragestores", name, "file." + ext]) + + try: + headers, response = self.http.request(cs_url, "PUT", message, headers) + self._cache.clear() + if headers.status != 201: + raise UploadError(response) + finally: + if archive is not None: + unlink(archive) + + def get_resource(self, name, store=None, workspace=None): + if store is not None: + candidates = [s for s in self.get_resources(store) if s.name == name] + if len(candidates) == 0: + return None + elif len(candidates) > 1: + raise AmbiguousRequestError + else: + return candidates[0] + + if workspace is not None: + for store in self.get_stores(workspace): + resource = self.get_resource(name, store) + if resource is not None: + return resource + return None + + for ws in self.get_workspaces(): + resource = self.get_resource(name, workspace=ws) + if resource is not None: + return resource + return None + + def get_resources(self, store=None, workspace=None): + if isinstance(workspace, basestring): + workspace = self.get_workspace(workspace) + if isinstance(store, basestring): + store = self.get_store(store, workspace) + if store is not None: + return store.get_resources() + if workspace is not None: + resources = [] + for store in self.get_stores(workspace): + resources.extend(self.get_resources(store)) + return resources + resources = [] + for ws in self.get_workspaces(): + resources.extend(self.get_resources(workspace=ws)) + return resources + + def get_layer(self, name): + try: + lyr = Layer(self, name) + lyr.fetch() + return lyr + except FailedRequestError: + return None + + def get_layers(self, resource=None): + if isinstance(resource, basestring): + resource = self.get_resource(resource) + layers_url = url(self.service_url, ["layers.xml"]) + description = self.get_xml(layers_url) + lyrs = [Layer(self, l.find("name").text) for l in description.findall("layer")] + if resource is not None: + lyrs = [l for l in lyrs if l.resource.href == resource.href] + # TODO: Filter by style + return lyrs + + def get_layergroup(self, name=None): + try: + group_url = url(self.service_url, ["layergroups", name + ".xml"]) + group = self.get_xml(group_url) + return LayerGroup(self, group.find("name").text) + except FailedRequestError: + return None + + def get_layergroups(self): + groups = self.get_xml("%s/layergroups.xml" % self.service_url) + return [LayerGroup(self, g.find("name").text) for g in groups.findall("layerGroup")] + + def create_layergroup(self, name, layers = (), styles = (), bounds = None): + if any(g.name == name for g in self.get_layergroups()): + raise ConflictingDataError("LayerGroup named %s already exists!" % name) + else: + return UnsavedLayerGroup(self, name, layers, styles, bounds) + + def get_style(self, name): + try: + style_url = url(self.service_url, ["styles", name + ".xml"]) + dom = self.get_xml(style_url) + return Style(self, dom.find("name").text) + except FailedRequestError: + return None + + def get_styles(self): + styles_url = url(self.service_url, ["styles.xml"]) + description = self.get_xml(styles_url) + return [Style(self, s.find('name').text) for s in description.findall("style")] + + def create_style(self, name, data, overwrite = False): + if overwrite == False and self.get_style(name) is not None: + raise ConflictingDataError("There is already a style named %s" % name) + + headers = { + "Content-type": "application/vnd.ogc.sld+xml", + "Accept": "application/xml" + } + + if overwrite: + style_url = url(self.service_url, ["styles", name + ".sld"]) + headers, response = self.http.request(style_url, "PUT", data, headers) + else: + style_url = url(self.service_url, ["styles"], dict(name=name)) + headers, response = self.http.request(style_url, "POST", data, headers) + + self._cache.clear() + if headers.status < 200 or headers.status > 299: raise UploadError(response) + + def create_workspace(self, name, uri): + xml = ("" + "{name}" + "{uri}" + "").format(name=name, uri=uri) + headers = { "Content-Type": "application/xml" } + workspace_url = self.service_url + "/namespaces/" + + headers, response = self.http.request(workspace_url, "POST", xml, headers) + assert 200 <= headers.status < 300, "Tried to create workspace but got " + str(headers.status) + ": " + response + self._cache.clear() + return self.get_workspace(name) + + def get_workspaces(self): + description = self.get_xml("%s/workspaces.xml" % self.service_url) + return [workspace_from_index(self, node) for node in description.findall("workspace")] + + def get_workspace(self, name): + candidates = [w for w in self.get_workspaces() if w.name == name] + if len(candidates) == 0: + return None + elif len(candidates) > 1: + raise AmbiguousRequestError() + else: + return candidates[0] + + def get_default_workspace(self): + return Workspace(self, "default") + + def set_default_workspace(self): + raise NotImplementedError() diff --git a/python/plugins/sextante/servertools/geoserver/layer.py b/python/plugins/sextante/servertools/geoserver/layer.py new file mode 100644 index 00000000000..f7271137033 --- /dev/null +++ b/python/plugins/sextante/servertools/geoserver/layer.py @@ -0,0 +1,132 @@ +from sextante.servertools.geoserver.support import ResourceInfo, xml_property, write_bool, url +from sextante.servertools.geoserver.style import Style + +class _attribution(object): + def __init__(self, title, width, height): + self.title = title + self.width = width + self.height = height + +def _read_attribution(node): + title = node.find("title") + width = node.find("logoWidth") + height = node.find("logoHeight") + + if title is not None: + title = title.text + if width is not None: + width = width.text + if height is not None: + height = height.text + + return _attribution(title, width, height) + +def _write_attribution(builder, attr): + builder.start("attribution", dict()) + if attr.title is not None: + builder.start("title", dict()) + builder.data(attr.title) + builder.end("title") + if attr.width is not None: + builder.start("logoWidth", dict()) + builder.data(attr.width) + builder.end("logoWidth") + if attr.height is not None: + builder.start("logoHeight", dict()) + builder.data(attr.height) + builder.end("logoHeight") + builder.end("attribution") + +def _write_default_style(builder, name): + builder.start("defaultStyle", dict()) + if name is not None: + builder.start("name", dict()) + builder.data(name) + builder.end("name") + builder.end("defaultStyle") + + +def _write_alternate_styles(builder, styles): + builder.start("styles", dict()) + for s in styles: + builder.start("style", dict()) + builder.start("name", dict()) + builder.data(s.name) + builder.end("name") + builder.end("style") + builder.end("styles") + + +class Layer(ResourceInfo): + def __init__(self, catalog, name): + super(Layer, self).__init__() + self.catalog = catalog + self.name = name + + resource_type = "layer" + save_method = "PUT" + + @property + def href(self): + return url(self.catalog.service_url, ["layers", self.name + ".xml"]) + + @property + def resource(self): + if self.dom is None: + self.fetch() + name = self.dom.find("resource/name").text + return self.catalog.get_resource(name) + + def _get_default_style(self): + if 'default_style' in self.dirty: + return self.dirty['default_style'] + if self.dom is None: + self.fetch() + name = self.dom.find("defaultStyle/name") + # aborted data uploads can result in no default style + if name is not None: + return self.catalog.get_style(name.text) + else: + return None + + def _set_default_style(self, style): + if isinstance(style, Style): + style = style.name + self.dirty["default_style"] = style + + def _get_alternate_styles(self): + if "alternate_styles" in self.dirty: + return self.dirty["alternate_styles"] + if self.dom is None: + self.fetch() + styles = self.dom.findall("styles/style/name") + return [Style(self.catalog, s.text) for s in styles] + + def _set_alternate_styles(self, styles): + self.dirty["alternate_styles"] = styles + + default_style = property(_get_default_style, _set_default_style) + styles = property(_get_alternate_styles, _set_alternate_styles) + + attribution_object = xml_property("attribution", _read_attribution) + enabled = xml_property("enabled", lambda x: x.text == "true") + + def _get_attr_text(self): + return self.attribution_object.title + + def _set_attr_text(self, text): + self.dirty["attribution"] = _attribution( + text, + self.attribution_object.width, + self.attribution_object.height + ) + assert self.attribution_object.title == text + + attribution = property(_get_attr_text, _set_attr_text) + + writers = dict( + attribution = _write_attribution, + enabled = write_bool("enabled"), + default_style = _write_default_style, + alternate_styles = _write_alternate_styles + ) diff --git a/python/plugins/sextante/servertools/geoserver/layergroup.py b/python/plugins/sextante/servertools/geoserver/layergroup.py new file mode 100644 index 00000000000..9030de3f038 --- /dev/null +++ b/python/plugins/sextante/servertools/geoserver/layergroup.py @@ -0,0 +1,84 @@ +from sextante.servertools.geoserver.support import ResourceInfo, bbox, write_bbox, \ + write_string, xml_property, url + +def _maybe_text(n): + if n is None: + return None + else: + return n.text + +def _layer_list(node): + if node is not None: + return [_maybe_text(n.find("name")) for n in node.findall("layer")] + +def _style_list(node): + if node is not None: + return [_maybe_text(n.find("name")) for n in node.findall("style")] + +def _write_layers(builder, layers): + builder.start("layers", dict()) + for l in layers: + builder.start("layer", dict()) + if l is not None: + builder.start("name", dict()) + builder.data(l) + builder.end("name") + builder.end("layer") + builder.end("layers") + +def _write_styles(builder, styles): + builder.start("styles", dict()) + for s in styles: + builder.start("style", dict()) + if s is not None: + builder.start("name", dict()) + builder.data(s) + builder.end("name") + builder.end("style") + builder.end("styles") + +class LayerGroup(ResourceInfo): + """ + Represents a layer group in geoserver + """ + + resource_type = "layerGroup" + save_method = "PUT" + + def __init__(self, catalog, name): + super(LayerGroup, self).__init__() + + assert isinstance(name, basestring) + + self.catalog = catalog + self.name = name + + @property + def href(self): + return url(self.catalog.service_url, ["layergroups", self.name + ".xml"]) + + styles = xml_property("styles", _style_list) + layers = xml_property("layers", _layer_list) + bounds = xml_property("bounds", bbox) + + writers = dict( + name = write_string("name"), + styles = _write_styles, + layers = _write_layers, + bounds = write_bbox("bounds") + ) + + def __str__(self): + return "" % self.name + + __repr__ = __str__ + +class UnsavedLayerGroup(LayerGroup): + save_method = "POST" + def __init__(self, catalog, name, layers, styles, bounds): + super(UnsavedLayerGroup, self).__init__(catalog, name) + self.dirty.update(name = name, layers = layers, styles = styles, bounds = bounds) + + @property + def href(self): + return "%s/layergroups?name=%s" % (self.catalog.service_url, self.name) diff --git a/python/plugins/sextante/servertools/geoserver/namespace.py b/python/plugins/sextante/servertools/geoserver/namespace.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/plugins/sextante/servertools/geoserver/resource.py b/python/plugins/sextante/servertools/geoserver/resource.py new file mode 100644 index 00000000000..633b616b71d --- /dev/null +++ b/python/plugins/sextante/servertools/geoserver/resource.py @@ -0,0 +1,176 @@ +from sextante.servertools.geoserver.support import ResourceInfo, xml_property, write_string, bbox, \ + write_bbox, string_list, write_string_list, attribute_list, write_bool, url + +def md_link(node): + """Extract a metadata link tuple from an xml node""" + mimetype = node.find("type") + mdtype = node.find("metadataType") + content = node.find("content") + if None in [mimetype, mdtype, content]: + return None + else: + return (mimetype.text, mdtype.text, content.text) + +def metadata_link_list(node): + if node is not None: + return [md_link(n) for n in node.findall("metadataLink")] + +def write_metadata_link_list(name): + def write(builder, md_links): + builder.start(name, dict()) + for (mime, md_type, content_url) in md_links: + builder.start("metadataLink", dict()) + builder.start("type", dict()) + builder.data(mime) + builder.end("type") + builder.start("metadataType", dict()) + builder.data(md_type) + builder.end("metadataType") + builder.start("content", dict()) + builder.data(content_url) + builder.end("content") + builder.end("metadataLink") + builder.end("metadataLinks") + return write + +def featuretype_from_index(catalog, workspace, store, node): + name = node.find("name") + return FeatureType(catalog, workspace, store, name.text) + +def coverage_from_index(catalog, workspace, store, node): + name = node.find("name") + return Coverage(catalog, workspace, store, name.text) + +class FeatureType(ResourceInfo): + resource_type = "featureType" + save_method = "PUT" + + def __init__(self, catalog, workspace, store, name): + super(FeatureType, self).__init__() + + assert isinstance(store, ResourceInfo) + assert isinstance(name, basestring) + + self.catalog = catalog + self.workspace = workspace + self.store = store + self.name = name + + @property + def href(self): + return url(self.catalog.service_url, + ["workspaces", self.workspace.name, + "datastores", self.store.name, + "featuretypes", self.name + ".xml"]) + + title = xml_property("title") + abstract = xml_property("abstract") + enabled = xml_property("enabled") + native_bbox = xml_property("nativeBoundingBox", bbox) + latlon_bbox = xml_property("latLonBoundingBox", bbox) + projection = xml_property("srs") + projection_policy = xml_property("projectionPolicy") + keywords = xml_property("keywords", string_list) + attributes = xml_property("attributes", attribute_list) + metadata_links = xml_property("metadataLinks", metadata_link_list) + + writers = dict( + title = write_string("title"), + abstract = write_string("abstract"), + enabled = write_bool("enabled"), + nativeBoundingBox = write_bbox("nativeBoundingBox"), + latLonBoundingBox = write_bbox("latLonBoundingBox"), + srs = write_string("srs"), + projectionPolicy = write_string("projectionPolicy"), + keywords = write_string_list("keywords"), + metadataLinks = write_metadata_link_list("metadataLinks") + ) + +class CoverageDimension(object): + def __init__(self, name, description, dimension_range): + self.name = name + self.description = description + self.dimension_range = dimension_range + +def coverage_dimension(node): + name = node.find("name") + name = name.text if name is not None else None + description = node.find("description") + description = description.text if description is not None else None + range_min = node.find("range/min") + range_max = node.find("range/max") + dimension_range = None + if None not in [min, max]: + dimension_range = float(range_min.text), float(range_max.text) + if None not in [name, description]: + return CoverageDimension(name, description, dimension_range) + else: + return None # should we bomb out more spectacularly here? + +def coverage_dimension_xml(builder, dimension): + builder.start("coverageDimension", dict()) + builder.start("name", dict()) + builder.data(dimension.name) + builder.end("name") + + builder.start("description", dict()) + builder.data(dimension.description) + builder.end("description") + + if dimension.range is not None: + builder.start("range", dict()) + builder.start("min", dict()) + builder.data(str(dimension.range[0])) + builder.end("min") + builder.start("max", dict()) + builder.data(str(dimension.range[1])) + builder.end("max") + builder.end("range") + + builder.end("coverageDimension") + +class Coverage(ResourceInfo): + def __init__(self, catalog, workspace, store, name): + super(Coverage, self).__init__() + self.catalog = catalog + self.workspace = workspace + self.store = store + self.name = name + + @property + def href(self): + return url(self.catalog.service_url, + ["workspaces", self.workspace.name, + "coveragestores", self.store.name, + "coverages", self.name + ".xml"]) + + resource_type = "coverage" + save_method = "PUT" + + title = xml_property("title") + abstract = xml_property("abstract") + enabled = xml_property("enabled") + native_bbox = xml_property("nativeBoundingBox", bbox) + latlon_bbox = xml_property("latLonBoundingBox", bbox) + projection = xml_property("srs") + projection_policy = xml_property("projectionPolicy") + keywords = xml_property("keywords", string_list) + request_srs_list = xml_property("requestSRS", string_list) + response_srs_list = xml_property("responseSRS", string_list) + supported_formats = xml_property("supportedFormats", string_list) + metadata_links = xml_property("metadataLinks", metadata_link_list) + + writers = dict( + title = write_string("title"), + abstract = write_string("abstract"), + enabled = write_bool("enabled"), + nativeBoundingBox = write_bbox("nativeBoundingBox"), + latLonBoundingBox = write_bbox("latLonBoundingBox"), + srs = write_string("srs"), + projection_policy = write_string("projectionPolicy"), + keywords = write_string_list("keywords"), + metadataLinks = write_metadata_link_list("metadataLinks"), + requestSRS = write_string_list("requestSRS"), + responseSRS = write_string_list("responseSRS"), + supportedFormats = write_string_list("supportedFormats") + ) diff --git a/python/plugins/sextante/servertools/geoserver/store.py b/python/plugins/sextante/servertools/geoserver/store.py new file mode 100644 index 00000000000..f5e446da5e6 --- /dev/null +++ b/python/plugins/sextante/servertools/geoserver/store.py @@ -0,0 +1,117 @@ +import sextante.servertools.geoserver.workspace as ws +from sextante.servertools.geoserver.resource import featuretype_from_index, coverage_from_index +from sextante.servertools.geoserver.support import ResourceInfo, xml_property, key_value_pairs, \ + write_bool, write_dict, write_string, url + +def datastore_from_index(catalog, workspace, node): + name = node.find("name") + return DataStore(catalog, workspace, name.text) + +def coveragestore_from_index(catalog, workspace, node): + name = node.find("name") + return CoverageStore(catalog, workspace, name.text) + +class DataStore(ResourceInfo): + resource_type = "dataStore" + save_method = "PUT" + + def __init__(self, catalog, workspace, name): + super(DataStore, self).__init__() + + assert isinstance(workspace, ws.Workspace) + assert isinstance(name, basestring) + self.catalog = catalog + self.workspace = workspace + self.name = name + + @property + def href(self): + return url(self.catalog.service_url, + ["workspaces", self.workspace.name, "datastores", self.name + ".xml"]) + + enabled = xml_property("enabled", lambda x: x.text == "true") + name = xml_property("name") + connection_parameters = xml_property("connectionParameters", key_value_pairs) + + writers = dict(enabled = write_bool("enabled"), + name = write_string("name"), + connectionParameters = write_dict("connectionParameters")) + + + def get_resources(self): + res_url = url(self.catalog.service_url, + ["workspaces", self.workspace.name, "datastores", self.name, "featuretypes.xml"]) + xml = self.catalog.get_xml(res_url) + def ft_from_node(node): + return featuretype_from_index(self.catalog, self.workspace, self, node) + + return [ft_from_node(node) for node in xml.findall("featureType")] + +class UnsavedDataStore(DataStore): + save_method = "POST" + + def __init__(self, catalog, name, workspace): + super(UnsavedDataStore, self).__init__(catalog, workspace, name) + self.dirty.update(dict( + name=name, enabled=True, connectionParameters=dict())) + + @property + def href(self): + path = [ "workspaces", + self.workspace.name, "datastores"] + query = dict(name=self.name) + return url(self.catalog.service_url, path, query) + +class CoverageStore(ResourceInfo): + resource_type = 'coverageStore' + save_method = "PUT" + + def __init__(self, catalog, workspace, name): + super(CoverageStore, self).__init__() + + assert isinstance(workspace, ws.Workspace) + assert isinstance(name, basestring) + + self.catalog = catalog + self.workspace = workspace + self.name = name + + @property + def href(self): + return url(self.catalog.service_url, + ["workspaces", self.workspace.name, "coveragestores", self.name + ".xml"]) + + enabled = xml_property("enabled", lambda x: x.text == "true") + name = xml_property("name") + url = xml_property("url") + type = xml_property("type") + + writers = dict(enabled = write_bool("enabled"), + name = write_string("name"), + url = write_string("url"), + type = write_string("type")) + + + def get_resources(self): + res_url = url(self.catalog.service_url, + ["workspaces", self.workspace.name, "coveragestores", self.name, "coverages.xml"]) + + xml = self.catalog.get_xml(res_url) + + def cov_from_node(node): + return coverage_from_index(self.catalog, self.workspace, self, node) + + return [cov_from_node(node) for node in xml.findall("coverage")] + +class UnsavedCoverageStore(CoverageStore): + save_method = "POST" + + def __init__(self, catalog, name, workspace): + super(UnsavedCoverageStore, self).__init__(catalog, workspace, name) + self.dirty.update(name=name, enabled = True, type="GeoTIFF", + url = "file:data/") + + @property + def href(self): + return url(self.catalog.service_url, + ["workspaces", self.workspace.name, "coveragestores"], dict(name=self.name)) diff --git a/python/plugins/sextante/servertools/geoserver/style.py b/python/plugins/sextante/servertools/geoserver/style.py new file mode 100644 index 00000000000..7cfb7971038 --- /dev/null +++ b/python/plugins/sextante/servertools/geoserver/style.py @@ -0,0 +1,46 @@ +from sextante.servertools.geoserver.support import ResourceInfo, url, xml_property + +class Style(ResourceInfo): + def __init__(self, catalog, name): + super(Style, self).__init__() + assert isinstance(name, basestring) + + self.catalog = catalog + self.name = name + self._sld_dom = None + + @property + def href(self): + return url(self.catalog.service_url, ["styles", self.name + ".xml"]) + + def body_href(self): + return url(self.catalog.service_url, ["styles", self.name + ".sld"]) + + filename = xml_property("filename") + + def _get_sld_dom(self): + if self._sld_dom is None: + self._sld_dom = self.catalog.get_xml(self.body_href()) + return self._sld_dom + + @property + def sld_title(self): + user_style = self._get_sld_dom().find("{http://www.opengis.net/sld}NamedLayer/{http://www.opengis.net/sld}UserStyle") + title_node = user_style.find("{http://www.opengis.net/sld}Title") + return title_node.text if title_node is not None else None + + @property + def sld_name(self): + user_style = self._get_sld_dom().find("{http://www.opengis.net/sld}NamedLayer/{http://www.opengis.net/sld}UserStyle") + name_node = user_style.find("{http://www.opengis.net/sld}Name") + return name_node.text if name_node is not None else None + + @property + def sld_body(self): + content = self.catalog.http.request(self.body_href())[1] + return content + + def update_body(self, body): + headers = { "Content-Type": "application/vnd.ogc.sld+xml" } + self.catalog.http.request( + self.body_href(), "PUT", body, headers) diff --git a/python/plugins/sextante/servertools/geoserver/support.py b/python/plugins/sextante/servertools/geoserver/support.py new file mode 100644 index 00000000000..ce1611d9309 --- /dev/null +++ b/python/plugins/sextante/servertools/geoserver/support.py @@ -0,0 +1,218 @@ +import logging +from xml.etree.ElementTree import TreeBuilder, tostring +import urllib +import urlparse +from zipfile import ZipFile +from sextante.core.SextanteUtils import SextanteUtils + + +logger = logging.getLogger("gsconfig.support") + +FORCE_DECLARED = "FORCE_DECLARED" +## The projection handling policy for layers that should use coordinates +## directly while reporting the configured projection to clients. This should be +## used when projection information is missing from the underlying datastore. + + +FORCE_NATIVE = "FORCE_NATIVE" +## The projection handling policy for layers that should use the projection +## information from the underlying storage mechanism directly, and ignore the +## projection setting. + +REPROJECT = "REPROJECT" +## The projection handling policy for layers that should use the projection +## information from the underlying storage mechanism to reproject to the +## configured projection. + +def url(base, seg, query=None): + """ + Create a URL from a list of path segments and an optional dict of query + parameters. + """ + + seg = (urllib.quote(s.strip('/')) for s in seg) + if query is None or len(query) == 0: + query_string = '' + else: + query_string = "?" + urllib.urlencode(query) + path = '/'.join(seg) + query_string + adjusted_base = base.rstrip('/') + '/' + return urlparse.urljoin(adjusted_base, path) + +def xml_property(path, converter = lambda x: x.text): + def getter(self): + if path in self.dirty: + return self.dirty[path] + else: + if self.dom is None: + self.fetch() + node = self.dom.find(path) + return converter(self.dom.find(path)) if node is not None else None + + def setter(self, value): + self.dirty[path] = value + + def delete(self): + self.dirty[path] = None + + return property(getter, setter, delete) + +def bbox(node): + if node is not None: + minx = node.find("minx") + maxx = node.find("maxx") + miny = node.find("miny") + maxy = node.find("maxy") + crs = node.find("crs") + crs = crs.text if crs is not None else None + + if (None not in [minx, maxx, miny, maxy]): + return (minx.text, maxx.text, miny.text, maxy.text, crs) + else: + return None + else: + return None + +def string_list(node): + if node is not None: + return [n.text for n in node.findall("string")] + +def attribute_list(node): + if node is not None: + return [n.text for n in node.findall("attribute/name")] + +def key_value_pairs(node): + if node is not None: + return dict((entry.attrib['key'], entry.text) for entry in node.findall("entry")) + +def write_string(name): + def write(builder, value): + builder.start(name, dict()) + if (value is not None): + builder.data(value) + builder.end(name) + return write + +def write_bool(name): + def write(builder, b): + builder.start(name, dict()) + builder.data("true" if b else "false") + builder.end(name) + return write + +def write_bbox(name): + def write(builder, b): + builder.start(name, dict()) + bbox_xml(builder, b) + builder.end(name) + return write + +def write_string_list(name): + def write(builder, words): + builder.start(name, dict()) + for w in words: + builder.start("string", dict()) + builder.data(w) + builder.end("string") + builder.end(name) + return write + +def write_dict(name): + def write(builder, pairs): + builder.start(name, dict()) + for k, v in pairs.iteritems(): + builder.start("entry", dict(key=k)) + builder.data(v) + builder.end("entry") + builder.end(name) + return write + +class ResourceInfo(object): + def __init__(self): + self.dom = None + self.dirty = dict() + + def fetch(self): + self.dom = self.catalog.get_xml(self.href) + + def clear(self): + self.dirty = dict() + + def refresh(self): + self.clear() + self.fetch() + + def serialize(self, builder): + # GeoServer will disable the resource if we omit the tag, + # so force it into the dirty dict before writing + if hasattr(self, "enabled"): + self.dirty['enabled'] = self.enabled + + for k, writer in self.writers.items(): + if k in self.dirty: + writer(builder, self.dirty[k]) + + def message(self): + builder = TreeBuilder() + builder.start(self.resource_type, dict()) + self.serialize(builder) + builder.end(self.resource_type) + msg = tostring(builder.close()) + return msg + +def prepare_upload_bundle(name, data): + """GeoServer's REST API uses ZIP archives as containers for file formats such + as Shapefile and WorldImage which include several 'boxcar' files alongside + the main data. In such archives, GeoServer assumes that all of the relevant + files will have the same base name and appropriate extensions, and live in + the root of the ZIP archive. This method produces a zip file that matches + these expectations, based on a basename, and a dict of extensions to paths or + file-like objects. The client code is responsible for deleting the zip + archive when it's done.""" + #we ut the zip file in the sextante temp dir, so it is deleted at the end. + f = SextanteUtils.getTempFilename('zip') + zip_file = ZipFile(f, 'w') + for ext, stream in data.iteritems(): + fname = "%s.%s" % (name, ext) + if (isinstance(stream, basestring)): + zip_file.write(stream, fname) + else: + zip_file.writestr(fname, stream.read()) + zip_file.close() + return f + +def atom_link(node): + if 'href' in node.attrib: + return node.attrib['href'] + else: + l = node.find("{http://www.w3.org/2005/Atom}link") + return l.get('href') + +def atom_link_xml(builder, href): + builder.start("atom:link", { + 'rel': 'alternate', + 'href': href, + 'type': 'application/xml', + 'xmlns:atom': 'http://www.w3.org/2005/Atom' + }) + builder.end("atom:link") + +def bbox_xml(builder, box): + minx, maxx, miny, maxy, crs = box + builder.start("minx", dict()) + builder.data(minx) + builder.end("minx") + builder.start("maxx", dict()) + builder.data(maxx) + builder.end("maxx") + builder.start("miny", dict()) + builder.data(miny) + builder.end("miny") + builder.start("maxy", dict()) + builder.data(maxy) + builder.end("maxy") + if crs is not None: + builder.start("crs", {"class": "projected"}) + builder.data(crs) + builder.end("crs") + diff --git a/python/plugins/sextante/servertools/geoserver/util.py b/python/plugins/sextante/servertools/geoserver/util.py new file mode 100644 index 00000000000..9788076ac00 --- /dev/null +++ b/python/plugins/sextante/servertools/geoserver/util.py @@ -0,0 +1,5 @@ +# shapefile_and_friends = None +# shapefile_plus_sidecars = shapefile_and_friends("test/data/states") + +def shapefile_and_friends(path): + return dict((ext, path + "." + ext) for ext in ['shx', 'shp', 'dbf', 'prj']) diff --git a/python/plugins/sextante/servertools/geoserver/workspace.py b/python/plugins/sextante/servertools/geoserver/workspace.py new file mode 100644 index 00000000000..0f2f23761ae --- /dev/null +++ b/python/plugins/sextante/servertools/geoserver/workspace.py @@ -0,0 +1,33 @@ +from sextante.servertools.geoserver.support import xml_property, write_bool, ResourceInfo, url + +def workspace_from_index(catalog, node): + name = node.find("name") + return Workspace(catalog, name.text) + +class Workspace(ResourceInfo): + resource_type = "workspace" + + def __init__(self, catalog, name): + super(Workspace, self).__init__() + self.catalog = catalog + self.name = name + + @property + def href(self): + return url(self.catalog.service_url, ["workspaces", self.name + ".xml"]) + + @property + def coveragestore_url(self): + return url(self.catalog.service_url, ["workspaces", self.name, "coveragestores.xml"]) + + @property + def datastore_url(self): + return url(self.catalog.service_url, ["workspaces", self.name, "datastores.xml"]) + + enabled = xml_property("enabled", lambda x: x.lower() == 'true') + writers = dict( + enabled = write_bool("enabled") + ) + + def __repr__(self): + return "%s @ %s" % (self.name, self.href) diff --git a/python/plugins/sextante/servertools/httplib2/__init__.py b/python/plugins/sextante/servertools/httplib2/__init__.py new file mode 100644 index 00000000000..fb74de2a248 --- /dev/null +++ b/python/plugins/sextante/servertools/httplib2/__init__.py @@ -0,0 +1,1675 @@ +from __future__ import generators +""" +httplib2 + +A caching http interface that supports ETags and gzip +to conserve bandwidth. + +Requires Python 2.3 or later + +Changelog: +2007-08-18, Rick: Modified so it's able to use a socks proxy if needed. + +""" + +__author__ = "Joe Gregorio (joe@bitworking.org)" +__copyright__ = "Copyright 2006, Joe Gregorio" +__contributors__ = ["Thomas Broyer (t.broyer@ltgt.net)", + "James Antill", + "Xavier Verges Farrero", + "Jonathan Feinberg", + "Blair Zajac", + "Sam Ruby", + "Louis Nyffenegger"] +__license__ = "MIT" +__version__ = "0.7.6" + +import re +import sys +import email +import email.Utils +import email.Message +import email.FeedParser +import StringIO +import gzip +import zlib +import httplib +import urlparse +import urllib +import base64 +import os +import copy +import calendar +import time +import random +import errno +try: + from hashlib import sha1 as _sha, md5 as _md5 +except ImportError: + # prior to Python 2.5, these were separate modules + import sha + import md5 + _sha = sha.new + _md5 = md5.new +import hmac +from gettext import gettext as _ +import socket + +try: + from httplib2 import socks +except ImportError: + try: + import socks + except ImportError: + socks = None + +# Build the appropriate socket wrapper for ssl +try: + import ssl # python 2.6 + ssl_SSLError = ssl.SSLError + def _ssl_wrap_socket(sock, key_file, cert_file, + disable_validation, ca_certs): + if disable_validation: + cert_reqs = ssl.CERT_NONE + else: + cert_reqs = ssl.CERT_REQUIRED + # We should be specifying SSL version 3 or TLS v1, but the ssl module + # doesn't expose the necessary knobs. So we need to go with the default + # of SSLv23. + return ssl.wrap_socket(sock, keyfile=key_file, certfile=cert_file, + cert_reqs=cert_reqs, ca_certs=ca_certs) +except (AttributeError, ImportError): + ssl_SSLError = None + def _ssl_wrap_socket(sock, key_file, cert_file, + disable_validation, ca_certs): + if not disable_validation: + raise CertificateValidationUnsupported( + "SSL certificate validation is not supported without " + "the ssl module installed. To avoid this error, install " + "the ssl module, or explicity disable validation.") + ssl_sock = socket.ssl(sock, key_file, cert_file) + return httplib.FakeSocket(sock, ssl_sock) + + +if sys.version_info >= (2,3): + from iri2uri import iri2uri +else: + def iri2uri(uri): + return uri + +def has_timeout(timeout): # python 2.6 + if hasattr(socket, '_GLOBAL_DEFAULT_TIMEOUT'): + return (timeout is not None and timeout is not socket._GLOBAL_DEFAULT_TIMEOUT) + return (timeout is not None) + +__all__ = ['Http', 'Response', 'ProxyInfo', 'HttpLib2Error', + 'RedirectMissingLocation', 'RedirectLimit', 'FailedToDecompressContent', + 'UnimplementedDigestAuthOptionError', 'UnimplementedHmacDigestAuthOptionError', + 'debuglevel', 'ProxiesUnavailableError'] + + +# The httplib debug level, set to a non-zero value to get debug output +debuglevel = 0 + +# A request will be tried 'RETRIES' times if it fails at the socket/connection level. +RETRIES = 2 + +# Python 2.3 support +if sys.version_info < (2,4): + def sorted(seq): + seq.sort() + return seq + +# Python 2.3 support +def HTTPResponse__getheaders(self): + """Return list of (header, value) tuples.""" + if self.msg is None: + raise httplib.ResponseNotReady() + return self.msg.items() + +if not hasattr(httplib.HTTPResponse, 'getheaders'): + httplib.HTTPResponse.getheaders = HTTPResponse__getheaders + +# All exceptions raised here derive from HttpLib2Error +class HttpLib2Error(Exception): pass + +# Some exceptions can be caught and optionally +# be turned back into responses. +class HttpLib2ErrorWithResponse(HttpLib2Error): + def __init__(self, desc, response, content): + self.response = response + self.content = content + HttpLib2Error.__init__(self, desc) + +class RedirectMissingLocation(HttpLib2ErrorWithResponse): pass +class RedirectLimit(HttpLib2ErrorWithResponse): pass +class FailedToDecompressContent(HttpLib2ErrorWithResponse): pass +class UnimplementedDigestAuthOptionError(HttpLib2ErrorWithResponse): pass +class UnimplementedHmacDigestAuthOptionError(HttpLib2ErrorWithResponse): pass + +class MalformedHeader(HttpLib2Error): pass +class RelativeURIError(HttpLib2Error): pass +class ServerNotFoundError(HttpLib2Error): pass +class ProxiesUnavailableError(HttpLib2Error): pass +class CertificateValidationUnsupported(HttpLib2Error): pass +class SSLHandshakeError(HttpLib2Error): pass +class NotSupportedOnThisPlatform(HttpLib2Error): pass +class CertificateHostnameMismatch(SSLHandshakeError): + def __init__(self, desc, host, cert): + HttpLib2Error.__init__(self, desc) + self.host = host + self.cert = cert + +# Open Items: +# ----------- +# Proxy support + +# Are we removing the cached content too soon on PUT (only delete on 200 Maybe?) + +# Pluggable cache storage (supports storing the cache in +# flat files by default. We need a plug-in architecture +# that can support Berkeley DB and Squid) + +# == Known Issues == +# Does not handle a resource that uses conneg and Last-Modified but no ETag as a cache validator. +# Does not handle Cache-Control: max-stale +# Does not use Age: headers when calculating cache freshness. + + +# The number of redirections to follow before giving up. +# Note that only GET redirects are automatically followed. +# Will also honor 301 requests by saving that info and never +# requesting that URI again. +DEFAULT_MAX_REDIRECTS = 5 + +# Default CA certificates file bundled with httplib2. +CA_CERTS = os.path.join( + os.path.dirname(os.path.abspath(__file__ )), "cacerts.txt") + +# Which headers are hop-by-hop headers by default +HOP_BY_HOP = ['connection', 'keep-alive', 'proxy-authenticate', 'proxy-authorization', 'te', 'trailers', 'transfer-encoding', 'upgrade'] + +def _get_end2end_headers(response): + hopbyhop = list(HOP_BY_HOP) + hopbyhop.extend([x.strip() for x in response.get('connection', '').split(',')]) + return [header for header in response.keys() if header not in hopbyhop] + +URI = re.compile(r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?") + +def parse_uri(uri): + """Parses a URI using the regex given in Appendix B of RFC 3986. + + (scheme, authority, path, query, fragment) = parse_uri(uri) + """ + groups = URI.match(uri).groups() + return (groups[1], groups[3], groups[4], groups[6], groups[8]) + +def urlnorm(uri): + (scheme, authority, path, query, fragment) = parse_uri(uri) + if not scheme or not authority: + raise RelativeURIError("Only absolute URIs are allowed. uri = %s" % uri) + authority = authority.lower() + scheme = scheme.lower() + if not path: + path = "/" + # Could do syntax based normalization of the URI before + # computing the digest. See Section 6.2.2 of Std 66. + request_uri = query and "?".join([path, query]) or path + scheme = scheme.lower() + defrag_uri = scheme + "://" + authority + request_uri + return scheme, authority, request_uri, defrag_uri + + +# Cache filename construction (original borrowed from Venus http://intertwingly.net/code/venus/) +re_url_scheme = re.compile(r'^\w+://') +re_slash = re.compile(r'[?/:|]+') + +def safename(filename): + """Return a filename suitable for the cache. + + Strips dangerous and common characters to create a filename we + can use to store the cache in. + """ + + try: + if re_url_scheme.match(filename): + if isinstance(filename,str): + filename = filename.decode('utf-8') + filename = filename.encode('idna') + else: + filename = filename.encode('idna') + except UnicodeError: + pass + if isinstance(filename,unicode): + filename=filename.encode('utf-8') + filemd5 = _md5(filename).hexdigest() + filename = re_url_scheme.sub("", filename) + filename = re_slash.sub(",", filename) + + # limit length of filename + if len(filename)>200: + filename=filename[:200] + return ",".join((filename, filemd5)) + +NORMALIZE_SPACE = re.compile(r'(?:\r\n)?[ \t]+') +def _normalize_headers(headers): + return dict([ (key.lower(), NORMALIZE_SPACE.sub(value, ' ').strip()) for (key, value) in headers.iteritems()]) + +def _parse_cache_control(headers): + retval = {} + if headers.has_key('cache-control'): + parts = headers['cache-control'].split(',') + parts_with_args = [tuple([x.strip().lower() for x in part.split("=", 1)]) for part in parts if -1 != part.find("=")] + parts_wo_args = [(name.strip().lower(), 1) for name in parts if -1 == name.find("=")] + retval = dict(parts_with_args + parts_wo_args) + return retval + +# Whether to use a strict mode to parse WWW-Authenticate headers +# Might lead to bad results in case of ill-formed header value, +# so disabled by default, falling back to relaxed parsing. +# Set to true to turn on, usefull for testing servers. +USE_WWW_AUTH_STRICT_PARSING = 0 + +# In regex below: +# [^\0-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+ matches a "token" as defined by HTTP +# "(?:[^\0-\x08\x0A-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?" matches a "quoted-string" as defined by HTTP, when LWS have already been replaced by a single space +# Actually, as an auth-param value can be either a token or a quoted-string, they are combined in a single pattern which matches both: +# \"?((?<=\")(?:[^\0-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?(?=\")|(?@,;:\\\"/[\]?={} \t]+(?!\"))\"? +WWW_AUTH_STRICT = re.compile(r"^(?:\s*(?:,\s*)?([^\0-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+)\s*=\s*\"?((?<=\")(?:[^\0-\x08\x0A-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?(?=\")|(?@,;:\\\"/[\]?={} \t]+(?!\"))\"?)(.*)$") +WWW_AUTH_RELAXED = re.compile(r"^(?:\s*(?:,\s*)?([^ \t\r\n=]+)\s*=\s*\"?((?<=\")(?:[^\\\"]|\\.)*?(?=\")|(? current_age: + retval = "FRESH" + return retval + +def _decompressContent(response, new_content): + content = new_content + try: + encoding = response.get('content-encoding', None) + if encoding in ['gzip', 'deflate']: + if encoding == 'gzip': + content = gzip.GzipFile(fileobj=StringIO.StringIO(new_content)).read() + if encoding == 'deflate': + content = zlib.decompress(content) + response['content-length'] = str(len(content)) + # Record the historical presence of the encoding in a way the won't interfere. + response['-content-encoding'] = response['content-encoding'] + del response['content-encoding'] + except IOError: + content = "" + raise FailedToDecompressContent(_("Content purported to be compressed with %s but failed to decompress.") % response.get('content-encoding'), response, content) + return content + +def _updateCache(request_headers, response_headers, content, cache, cachekey): + if cachekey: + cc = _parse_cache_control(request_headers) + cc_response = _parse_cache_control(response_headers) + if cc.has_key('no-store') or cc_response.has_key('no-store'): + cache.delete(cachekey) + else: + info = email.Message.Message() + for key, value in response_headers.iteritems(): + if key not in ['status','content-encoding','transfer-encoding']: + info[key] = value + + # Add annotations to the cache to indicate what headers + # are variant for this request. + vary = response_headers.get('vary', None) + if vary: + vary_headers = vary.lower().replace(' ', '').split(',') + for header in vary_headers: + key = '-varied-%s' % header + try: + info[key] = request_headers[header] + except KeyError: + pass + + status = response_headers.status + if status == 304: + status = 200 + + status_header = 'status: %d\r\n' % status + + header_str = info.as_string() + + header_str = re.sub("\r(?!\n)|(? 0: + service = "cl" + # No point in guessing Base or Spreadsheet + #elif request_uri.find("spreadsheets") > 0: + # service = "wise" + + auth = dict(Email=credentials[0], Passwd=credentials[1], service=service, source=headers['user-agent']) + resp, content = self.http.request("https://www.google.com/accounts/ClientLogin", method="POST", body=urlencode(auth), headers={'Content-Type': 'application/x-www-form-urlencoded'}) + lines = content.split('\n') + d = dict([tuple(line.split("=", 1)) for line in lines if line]) + if resp.status == 403: + self.Auth = "" + else: + self.Auth = d['Auth'] + + def request(self, method, request_uri, headers, content): + """Modify the request headers to add the appropriate + Authorization header.""" + headers['authorization'] = 'GoogleLogin Auth=' + self.Auth + + +AUTH_SCHEME_CLASSES = { + "basic": BasicAuthentication, + "wsse": WsseAuthentication, + "digest": DigestAuthentication, + "hmacdigest": HmacDigestAuthentication, + "googlelogin": GoogleLoginAuthentication +} + +AUTH_SCHEME_ORDER = ["hmacdigest", "googlelogin", "digest", "wsse", "basic"] + +class FileCache(object): + """Uses a local directory as a store for cached files. + Not really safe to use if multiple threads or processes are going to + be running on the same cache. + """ + def __init__(self, cache, safe=safename): # use safe=lambda x: md5.new(x).hexdigest() for the old behavior + self.cache = cache + self.safe = safe + if not os.path.exists(cache): + os.makedirs(self.cache) + + def get(self, key): + retval = None + cacheFullPath = os.path.join(self.cache, self.safe(key)) + try: + f = file(cacheFullPath, "rb") + retval = f.read() + f.close() + except IOError: + pass + return retval + + def set(self, key, value): + cacheFullPath = os.path.join(self.cache, self.safe(key)) + f = file(cacheFullPath, "wb") + f.write(value) + f.close() + + def delete(self, key): + cacheFullPath = os.path.join(self.cache, self.safe(key)) + if os.path.exists(cacheFullPath): + os.remove(cacheFullPath) + +class Credentials(object): + def __init__(self): + self.credentials = [] + + def add(self, name, password, domain=""): + self.credentials.append((domain.lower(), name, password)) + + def clear(self): + self.credentials = [] + + def iter(self, domain): + for (cdomain, name, password) in self.credentials: + if cdomain == "" or domain == cdomain: + yield (name, password) + +class KeyCerts(Credentials): + """Identical to Credentials except that + name/password are mapped to key/cert.""" + pass + +class AllHosts(object): + pass + +class ProxyInfo(object): + """Collect information required to use a proxy.""" + bypass_hosts = () + + def __init__(self, proxy_type, proxy_host, proxy_port, + proxy_rdns=None, proxy_user=None, proxy_pass=None): + """The parameter proxy_type must be set to one of socks.PROXY_TYPE_XXX + constants. For example: + + p = ProxyInfo(proxy_type=socks.PROXY_TYPE_HTTP, + proxy_host='localhost', proxy_port=8000) + """ + self.proxy_type = proxy_type + self.proxy_host = proxy_host + self.proxy_port = proxy_port + self.proxy_rdns = proxy_rdns + self.proxy_user = proxy_user + self.proxy_pass = proxy_pass + + def astuple(self): + return (self.proxy_type, self.proxy_host, self.proxy_port, + self.proxy_rdns, self.proxy_user, self.proxy_pass) + + def isgood(self): + return (self.proxy_host != None) and (self.proxy_port != None) + + @classmethod + def from_environment(cls, method='http'): + """ + Read proxy info from the environment variables. + """ + if method not in ['http', 'https']: + return + + env_var = method + '_proxy' + url = os.environ.get(env_var, os.environ.get(env_var.upper())) + if not url: + return + pi = cls.from_url(url, method) + + no_proxy = os.environ.get('no_proxy', os.environ.get('NO_PROXY', '')) + bypass_hosts = [] + if no_proxy: + bypass_hosts = no_proxy.split(',') + # special case, no_proxy=* means all hosts bypassed + if no_proxy == '*': + bypass_hosts = AllHosts + + pi.bypass_hosts = bypass_hosts + return pi + + @classmethod + def from_url(cls, url, method='http'): + """ + Construct a ProxyInfo from a URL (such as http_proxy env var) + """ + url = urlparse.urlparse(url) + username = None + password = None + port = None + if '@' in url[1]: + ident, host_port = url[1].split('@', 1) + if ':' in ident: + username, password = ident.split(':', 1) + else: + password = ident + else: + host_port = url[1] + if ':' in host_port: + host, port = host_port.split(':', 1) + else: + host = host_port + + if port: + port = int(port) + else: + port = dict(https=443, http=80)[method] + + proxy_type = 3 # socks.PROXY_TYPE_HTTP + return cls( + proxy_type = proxy_type, + proxy_host = host, + proxy_port = port, + proxy_user = username or None, + proxy_pass = password or None, + ) + + def applies_to(self, hostname): + return not self.bypass_host(hostname) + + def bypass_host(self, hostname): + """Has this host been excluded from the proxy config""" + if self.bypass_hosts is AllHosts: + return True + + bypass = False + for domain in self.bypass_hosts: + if hostname.endswith(domain): + bypass = True + + return bypass + + +class HTTPConnectionWithTimeout(httplib.HTTPConnection): + """ + HTTPConnection subclass that supports timeouts + + All timeouts are in seconds. If None is passed for timeout then + Python's default timeout for sockets will be used. See for example + the docs of socket.setdefaulttimeout(): + http://docs.python.org/library/socket.html#socket.setdefaulttimeout + """ + + def __init__(self, host, port=None, strict=None, timeout=None, proxy_info=None): + httplib.HTTPConnection.__init__(self, host, port, strict) + self.timeout = timeout + self.proxy_info = proxy_info + + def connect(self): + """Connect to the host and port specified in __init__.""" + # Mostly verbatim from httplib.py. + if self.proxy_info and socks is None: + raise ProxiesUnavailableError( + 'Proxy support missing but proxy use was requested!') + msg = "getaddrinfo returns an empty list" + if self.proxy_info and self.proxy_info.isgood(): + use_proxy = True + proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass = self.proxy_info.astuple() + else: + use_proxy = False + if use_proxy and proxy_rdns: + host = proxy_host + port = proxy_port + else: + host = self.host + port = self.port + + for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM): + af, socktype, proto, canonname, sa = res + try: + if use_proxy: + self.sock = socks.socksocket(af, socktype, proto) + self.sock.setproxy(proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass) + else: + self.sock = socket.socket(af, socktype, proto) + self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + # Different from httplib: support timeouts. + if has_timeout(self.timeout): + self.sock.settimeout(self.timeout) + # End of difference from httplib. + if self.debuglevel > 0: + print "connect: (%s, %s) ************" % (self.host, self.port) + if use_proxy: + print "proxy: %s ************" % str((proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass)) + + self.sock.connect((self.host, self.port) + sa[2:]) + except socket.error, msg: + if self.debuglevel > 0: + print "connect fail: (%s, %s)" % (self.host, self.port) + if use_proxy: + print "proxy: %s" % str((proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass)) + if self.sock: + self.sock.close() + self.sock = None + continue + break + if not self.sock: + raise socket.error, msg + +class HTTPSConnectionWithTimeout(httplib.HTTPSConnection): + """ + This class allows communication via SSL. + + All timeouts are in seconds. If None is passed for timeout then + Python's default timeout for sockets will be used. See for example + the docs of socket.setdefaulttimeout(): + http://docs.python.org/library/socket.html#socket.setdefaulttimeout + """ + def __init__(self, host, port=None, key_file=None, cert_file=None, + strict=None, timeout=None, proxy_info=None, + ca_certs=None, disable_ssl_certificate_validation=False): + httplib.HTTPSConnection.__init__(self, host, port=port, key_file=key_file, + cert_file=cert_file, strict=strict) + self.timeout = timeout + self.proxy_info = proxy_info + if ca_certs is None: + ca_certs = CA_CERTS + self.ca_certs = ca_certs + self.disable_ssl_certificate_validation = \ + disable_ssl_certificate_validation + + # The following two methods were adapted from https_wrapper.py, released + # with the Google Appengine SDK at + # http://googleappengine.googlecode.com/svn-history/r136/trunk/python/google/appengine/tools/https_wrapper.py + # under the following license: + # + # Copyright 2007 Google Inc. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + # + + def _GetValidHostsForCert(self, cert): + """Returns a list of valid host globs for an SSL certificate. + + Args: + cert: A dictionary representing an SSL certificate. + Returns: + list: A list of valid host globs. + """ + if 'subjectAltName' in cert: + return [x[1] for x in cert['subjectAltName'] + if x[0].lower() == 'dns'] + else: + return [x[0][1] for x in cert['subject'] + if x[0][0].lower() == 'commonname'] + + def _ValidateCertificateHostname(self, cert, hostname): + """Validates that a given hostname is valid for an SSL certificate. + + Args: + cert: A dictionary representing an SSL certificate. + hostname: The hostname to test. + Returns: + bool: Whether or not the hostname is valid for this certificate. + """ + hosts = self._GetValidHostsForCert(cert) + for host in hosts: + host_re = host.replace('.', '\.').replace('*', '[^.]*') + if re.search('^%s$' % (host_re,), hostname, re.I): + return True + return False + + def connect(self): + "Connect to a host on a given (SSL) port." + + msg = "getaddrinfo returns an empty list" + if self.proxy_info and self.proxy_info.isgood(): + use_proxy = True + proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass = self.proxy_info.astuple() + else: + use_proxy = False + if use_proxy and proxy_rdns: + host = proxy_host + port = proxy_port + else: + host = self.host + port = self.port + + for family, socktype, proto, canonname, sockaddr in socket.getaddrinfo( + host, port, 0, socket.SOCK_STREAM): + try: + if use_proxy: + sock = socks.socksocket(family, socktype, proto) + + sock.setproxy(proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass) + else: + sock = socket.socket(family, socktype, proto) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + + if has_timeout(self.timeout): + sock.settimeout(self.timeout) + sock.connect((self.host, self.port)) + self.sock =_ssl_wrap_socket( + sock, self.key_file, self.cert_file, + self.disable_ssl_certificate_validation, self.ca_certs) + if self.debuglevel > 0: + print "connect: (%s, %s)" % (self.host, self.port) + if use_proxy: + print "proxy: %s" % str((proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass)) + if not self.disable_ssl_certificate_validation: + cert = self.sock.getpeercert() + hostname = self.host.split(':', 0)[0] + if not self._ValidateCertificateHostname(cert, hostname): + raise CertificateHostnameMismatch( + 'Server presented certificate that does not match ' + 'host %s: %s' % (hostname, cert), hostname, cert) + except ssl_SSLError, e: + if sock: + sock.close() + if self.sock: + self.sock.close() + self.sock = None + # Unfortunately the ssl module doesn't seem to provide any way + # to get at more detailed error information, in particular + # whether the error is due to certificate validation or + # something else (such as SSL protocol mismatch). + if e.errno == ssl.SSL_ERROR_SSL: + raise SSLHandshakeError(e) + else: + raise + except (socket.timeout, socket.gaierror): + raise + except socket.error, msg: + if self.debuglevel > 0: + print "connect fail: (%s, %s)" % (self.host, self.port) + if use_proxy: + print "proxy: %s" % str((proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass)) + if self.sock: + self.sock.close() + self.sock = None + continue + break + if not self.sock: + raise socket.error, msg + +SCHEME_TO_CONNECTION = { + 'http': HTTPConnectionWithTimeout, + 'https': HTTPSConnectionWithTimeout + } + +# Use a different connection object for Google App Engine +try: + from google.appengine.api import apiproxy_stub_map + if apiproxy_stub_map.apiproxy.GetStub('urlfetch') is None: + raise ImportError # Bail out; we're not actually running on App Engine. + from google.appengine.api.urlfetch import fetch + from google.appengine.api.urlfetch import InvalidURLError + from google.appengine.api.urlfetch import DownloadError + from google.appengine.api.urlfetch import ResponseTooLargeError + from google.appengine.api.urlfetch import SSLCertificateError + + + class ResponseDict(dict): + """Is a dictionary that also has a read() method, so + that it can pass itself off as an httlib.HTTPResponse().""" + def read(self): + pass + + + class AppEngineHttpConnection(object): + """Emulates an httplib.HTTPConnection object, but actually uses the Google + App Engine urlfetch library. This allows the timeout to be properly used on + Google App Engine, and avoids using httplib, which on Google App Engine is + just another wrapper around urlfetch. + """ + def __init__(self, host, port=None, key_file=None, cert_file=None, + strict=None, timeout=None, proxy_info=None, ca_certs=None, + disable_ssl_certificate_validation=False): + self.host = host + self.port = port + self.timeout = timeout + if key_file or cert_file or proxy_info or ca_certs: + raise NotSupportedOnThisPlatform() + self.response = None + self.scheme = 'http' + self.validate_certificate = not disable_ssl_certificate_validation + self.sock = True + + def request(self, method, url, body, headers): + # Calculate the absolute URI, which fetch requires + netloc = self.host + if self.port: + netloc = '%s:%s' % (self.host, self.port) + absolute_uri = '%s://%s%s' % (self.scheme, netloc, url) + try: + try: # 'body' can be a stream. + body = body.read() + except AttributeError: + pass + response = fetch(absolute_uri, payload=body, method=method, + headers=headers, allow_truncated=False, follow_redirects=False, + deadline=self.timeout, + validate_certificate=self.validate_certificate) + self.response = ResponseDict(response.headers) + self.response['status'] = str(response.status_code) + self.response['reason'] = httplib.responses.get(response.status_code, 'Ok') + self.response.status = response.status_code + setattr(self.response, 'read', lambda : response.content) + + # Make sure the exceptions raised match the exceptions expected. + except InvalidURLError: + raise socket.gaierror('') + except (DownloadError, ResponseTooLargeError, SSLCertificateError): + raise httplib.HTTPException() + + def getresponse(self): + if self.response: + return self.response + else: + raise httplib.HTTPException() + + def set_debuglevel(self, level): + pass + + def connect(self): + pass + + def close(self): + pass + + + class AppEngineHttpsConnection(AppEngineHttpConnection): + """Same as AppEngineHttpConnection, but for HTTPS URIs.""" + def __init__(self, host, port=None, key_file=None, cert_file=None, + strict=None, timeout=None, proxy_info=None, ca_certs=None, + disable_ssl_certificate_validation=False): + AppEngineHttpConnection.__init__(self, host, port, key_file, cert_file, + strict, timeout, proxy_info, ca_certs, disable_ssl_certificate_validation) + self.scheme = 'https' + + # Update the connection classes to use the Googel App Engine specific ones. + SCHEME_TO_CONNECTION = { + 'http': AppEngineHttpConnection, + 'https': AppEngineHttpsConnection + } + +except ImportError: + pass + + +class Http(object): + """An HTTP client that handles: +- all methods +- caching +- ETags +- compression, +- HTTPS +- Basic +- Digest +- WSSE + +and more. + """ + def __init__(self, cache=None, timeout=None, + proxy_info=ProxyInfo.from_environment, + ca_certs=None, disable_ssl_certificate_validation=False): + """If 'cache' is a string then it is used as a directory name for + a disk cache. Otherwise it must be an object that supports the + same interface as FileCache. + + All timeouts are in seconds. If None is passed for timeout + then Python's default timeout for sockets will be used. See + for example the docs of socket.setdefaulttimeout(): + http://docs.python.org/library/socket.html#socket.setdefaulttimeout + + `proxy_info` may be: + - a callable that takes the http scheme ('http' or 'https') and + returns a ProxyInfo instance per request. By default, uses + ProxyInfo.from_environment. + - a ProxyInfo instance (static proxy config). + - None (proxy disabled). + + ca_certs is the path of a file containing root CA certificates for SSL + server certificate validation. By default, a CA cert file bundled with + httplib2 is used. + + If disable_ssl_certificate_validation is true, SSL cert validation will + not be performed. + """ + self.proxy_info = proxy_info + self.ca_certs = ca_certs + self.disable_ssl_certificate_validation = \ + disable_ssl_certificate_validation + + # Map domain name to an httplib connection + self.connections = {} + # The location of the cache, for now a directory + # where cached responses are held. + if cache and isinstance(cache, basestring): + self.cache = FileCache(cache) + else: + self.cache = cache + + # Name/password + self.credentials = Credentials() + + # Key/cert + self.certificates = KeyCerts() + + # authorization objects + self.authorizations = [] + + # If set to False then no redirects are followed, even safe ones. + self.follow_redirects = True + + # Which HTTP methods do we apply optimistic concurrency to, i.e. + # which methods get an "if-match:" etag header added to them. + self.optimistic_concurrency_methods = ["PUT", "PATCH"] + + # If 'follow_redirects' is True, and this is set to True then + # all redirecs are followed, including unsafe ones. + self.follow_all_redirects = False + + self.ignore_etag = False + + self.force_exception_to_status_code = False + + self.timeout = timeout + + # Keep Authorization: headers on a redirect. + self.forward_authorization_headers = False + + def _auth_from_challenge(self, host, request_uri, headers, response, content): + """A generator that creates Authorization objects + that can be applied to requests. + """ + challenges = _parse_www_authenticate(response, 'www-authenticate') + for cred in self.credentials.iter(host): + for scheme in AUTH_SCHEME_ORDER: + if challenges.has_key(scheme): + yield AUTH_SCHEME_CLASSES[scheme](cred, host, request_uri, headers, response, content, self) + + def add_credentials(self, name, password, domain=""): + """Add a name and password that will be used + any time a request requires authentication.""" + self.credentials.add(name, password, domain) + + def add_certificate(self, key, cert, domain): + """Add a key and cert that will be used + any time a request requires authentication.""" + self.certificates.add(key, cert, domain) + + def clear_credentials(self): + """Remove all the names and passwords + that are used for authentication""" + self.credentials.clear() + self.authorizations = [] + + def _conn_request(self, conn, request_uri, method, body, headers): + for i in range(RETRIES): + try: + if conn.sock is None: + conn.connect() + conn.request(method, request_uri, body, headers) + except socket.timeout: + raise + except socket.gaierror: + conn.close() + raise ServerNotFoundError("Unable to find the server at %s" % conn.host) + except ssl_SSLError: + conn.close() + raise + except socket.error, e: + err = 0 + if hasattr(e, 'args'): + err = getattr(e, 'args')[0] + else: + err = e.errno + if err == errno.ECONNREFUSED: # Connection refused + raise + except httplib.HTTPException: + # Just because the server closed the connection doesn't apparently mean + # that the server didn't send a response. + if conn.sock is None: + if i < RETRIES-1: + conn.close() + conn.connect() + continue + else: + conn.close() + raise + if i < RETRIES-1: + conn.close() + conn.connect() + continue + try: + response = conn.getresponse() + except (socket.error, httplib.HTTPException): + if i < RETRIES-1: + conn.close() + conn.connect() + continue + else: + raise + else: + content = "" + if method == "HEAD": + conn.close() + else: + content = response.read() + response = Response(response) + if method != "HEAD": + content = _decompressContent(response, content) + break + return (response, content) + + + def _request(self, conn, host, absolute_uri, request_uri, method, body, headers, redirections, cachekey): + """Do the actual request using the connection object + and also follow one level of redirects if necessary""" + + auths = [(auth.depth(request_uri), auth) for auth in self.authorizations if auth.inscope(host, request_uri)] + auth = auths and sorted(auths)[0][1] or None + if auth: + auth.request(method, request_uri, headers, body) + + (response, content) = self._conn_request(conn, request_uri, method, body, headers) + + if auth: + if auth.response(response, body): + auth.request(method, request_uri, headers, body) + (response, content) = self._conn_request(conn, request_uri, method, body, headers ) + response._stale_digest = 1 + + if response.status == 401: + for authorization in self._auth_from_challenge(host, request_uri, headers, response, content): + authorization.request(method, request_uri, headers, body) + (response, content) = self._conn_request(conn, request_uri, method, body, headers, ) + if response.status != 401: + self.authorizations.append(authorization) + authorization.response(response, body) + break + + if (self.follow_all_redirects or (method in ["GET", "HEAD"]) or response.status == 303): + if self.follow_redirects and response.status in [300, 301, 302, 303, 307]: + # Pick out the location header and basically start from the beginning + # remembering first to strip the ETag header and decrement our 'depth' + if redirections: + if not response.has_key('location') and response.status != 300: + raise RedirectMissingLocation( _("Redirected but the response is missing a Location: header."), response, content) + # Fix-up relative redirects (which violate an RFC 2616 MUST) + if response.has_key('location'): + location = response['location'] + (scheme, authority, path, query, fragment) = parse_uri(location) + if authority == None: + response['location'] = urlparse.urljoin(absolute_uri, location) + if response.status == 301 and method in ["GET", "HEAD"]: + response['-x-permanent-redirect-url'] = response['location'] + if not response.has_key('content-location'): + response['content-location'] = absolute_uri + _updateCache(headers, response, content, self.cache, cachekey) + if headers.has_key('if-none-match'): + del headers['if-none-match'] + if headers.has_key('if-modified-since'): + del headers['if-modified-since'] + if 'authorization' in headers and not self.forward_authorization_headers: + del headers['authorization'] + if response.has_key('location'): + location = response['location'] + old_response = copy.deepcopy(response) + if not old_response.has_key('content-location'): + old_response['content-location'] = absolute_uri + redirect_method = method + if response.status in [302, 303]: + redirect_method = "GET" + body = None + (response, content) = self.request(location, redirect_method, body=body, headers = headers, redirections = redirections - 1) + response.previous = old_response + else: + raise RedirectLimit("Redirected more times than rediection_limit allows.", response, content) + elif response.status in [200, 203] and method in ["GET", "HEAD"]: + # Don't cache 206's since we aren't going to handle byte range requests + if not response.has_key('content-location'): + response['content-location'] = absolute_uri + _updateCache(headers, response, content, self.cache, cachekey) + + return (response, content) + + def _normalize_headers(self, headers): + return _normalize_headers(headers) + +# Need to catch and rebrand some exceptions +# Then need to optionally turn all exceptions into status codes +# including all socket.* and httplib.* exceptions. + + + def request(self, uri, method="GET", body=None, headers=None, redirections=DEFAULT_MAX_REDIRECTS, connection_type=None): + """ Performs a single HTTP request. +The 'uri' is the URI of the HTTP resource and can begin +with either 'http' or 'https'. The value of 'uri' must be an absolute URI. + +The 'method' is the HTTP method to perform, such as GET, POST, DELETE, etc. +There is no restriction on the methods allowed. + +The 'body' is the entity body to be sent with the request. It is a string +object. + +Any extra headers that are to be sent with the request should be provided in the +'headers' dictionary. + +The maximum number of redirect to follow before raising an +exception is 'redirections. The default is 5. + +The return value is a tuple of (response, content), the first +being and instance of the 'Response' class, the second being +a string that contains the response entity body. + """ + try: + if headers is None: + headers = {} + else: + headers = self._normalize_headers(headers) + + if not headers.has_key('user-agent'): + headers['user-agent'] = "Python-httplib2/%s (gzip)" % __version__ + + uri = iri2uri(uri) + + (scheme, authority, request_uri, defrag_uri) = urlnorm(uri) + domain_port = authority.split(":")[0:2] + if len(domain_port) == 2 and domain_port[1] == '443' and scheme == 'http': + scheme = 'https' + authority = domain_port[0] + + proxy_info = self._get_proxy_info(scheme, authority) + + conn_key = scheme+":"+authority + if conn_key in self.connections: + conn = self.connections[conn_key] + else: + if not connection_type: + connection_type = SCHEME_TO_CONNECTION[scheme] + certs = list(self.certificates.iter(authority)) + if scheme == 'https': + if certs: + conn = self.connections[conn_key] = connection_type( + authority, key_file=certs[0][0], + cert_file=certs[0][1], timeout=self.timeout, + proxy_info=proxy_info, + ca_certs=self.ca_certs, + disable_ssl_certificate_validation= + self.disable_ssl_certificate_validation) + else: + conn = self.connections[conn_key] = connection_type( + authority, timeout=self.timeout, + proxy_info=proxy_info, + ca_certs=self.ca_certs, + disable_ssl_certificate_validation= + self.disable_ssl_certificate_validation) + else: + conn = self.connections[conn_key] = connection_type( + authority, timeout=self.timeout, + proxy_info=proxy_info) + conn.set_debuglevel(debuglevel) + + if 'range' not in headers and 'accept-encoding' not in headers: + headers['accept-encoding'] = 'gzip, deflate' + + info = email.Message.Message() + cached_value = None + if self.cache: + cachekey = defrag_uri + cached_value = self.cache.get(cachekey) + if cached_value: + # info = email.message_from_string(cached_value) + # + # Need to replace the line above with the kludge below + # to fix the non-existent bug not fixed in this + # bug report: http://mail.python.org/pipermail/python-bugs-list/2005-September/030289.html + try: + info, content = cached_value.split('\r\n\r\n', 1) + feedparser = email.FeedParser.FeedParser() + feedparser.feed(info) + info = feedparser.close() + feedparser._parse = None + except (IndexError, ValueError): + self.cache.delete(cachekey) + cachekey = None + cached_value = None + else: + cachekey = None + + if method in self.optimistic_concurrency_methods and self.cache and info.has_key('etag') and not self.ignore_etag and 'if-match' not in headers: + # http://www.w3.org/1999/04/Editing/ + headers['if-match'] = info['etag'] + + if method not in ["GET", "HEAD"] and self.cache and cachekey: + # RFC 2616 Section 13.10 + self.cache.delete(cachekey) + + # Check the vary header in the cache to see if this request + # matches what varies in the cache. + if method in ['GET', 'HEAD'] and 'vary' in info: + vary = info['vary'] + vary_headers = vary.lower().replace(' ', '').split(',') + for header in vary_headers: + key = '-varied-%s' % header + value = info[key] + if headers.get(header, None) != value: + cached_value = None + break + + if cached_value and method in ["GET", "HEAD"] and self.cache and 'range' not in headers: + if info.has_key('-x-permanent-redirect-url'): + # Should cached permanent redirects be counted in our redirection count? For now, yes. + if redirections <= 0: + raise RedirectLimit("Redirected more times than rediection_limit allows.", {}, "") + (response, new_content) = self.request(info['-x-permanent-redirect-url'], "GET", headers = headers, redirections = redirections - 1) + response.previous = Response(info) + response.previous.fromcache = True + else: + # Determine our course of action: + # Is the cached entry fresh or stale? + # Has the client requested a non-cached response? + # + # There seems to be three possible answers: + # 1. [FRESH] Return the cache entry w/o doing a GET + # 2. [STALE] Do the GET (but add in cache validators if available) + # 3. [TRANSPARENT] Do a GET w/o any cache validators (Cache-Control: no-cache) on the request + entry_disposition = _entry_disposition(info, headers) + + if entry_disposition == "FRESH": + if not cached_value: + info['status'] = '504' + content = "" + response = Response(info) + if cached_value: + response.fromcache = True + return (response, content) + + if entry_disposition == "STALE": + if info.has_key('etag') and not self.ignore_etag and not 'if-none-match' in headers: + headers['if-none-match'] = info['etag'] + if info.has_key('last-modified') and not 'last-modified' in headers: + headers['if-modified-since'] = info['last-modified'] + elif entry_disposition == "TRANSPARENT": + pass + + (response, new_content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey) + + if response.status == 304 and method == "GET": + # Rewrite the cache entry with the new end-to-end headers + # Take all headers that are in response + # and overwrite their values in info. + # unless they are hop-by-hop, or are listed in the connection header. + + for key in _get_end2end_headers(response): + info[key] = response[key] + merged_response = Response(info) + if hasattr(response, "_stale_digest"): + merged_response._stale_digest = response._stale_digest + _updateCache(headers, merged_response, content, self.cache, cachekey) + response = merged_response + response.status = 200 + response.fromcache = True + + elif response.status == 200: + content = new_content + else: + self.cache.delete(cachekey) + content = new_content + else: + cc = _parse_cache_control(headers) + if cc.has_key('only-if-cached'): + info['status'] = '504' + response = Response(info) + content = "" + else: + (response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey) + except Exception, e: + if self.force_exception_to_status_code: + if isinstance(e, HttpLib2ErrorWithResponse): + response = e.response + content = e.content + response.status = 500 + response.reason = str(e) + elif isinstance(e, socket.timeout): + content = "Request Timeout" + response = Response( { + "content-type": "text/plain", + "status": "408", + "content-length": len(content) + }) + response.reason = "Request Timeout" + else: + content = str(e) + response = Response( { + "content-type": "text/plain", + "status": "400", + "content-length": len(content) + }) + response.reason = "Bad Request" + else: + raise + + + return (response, content) + + def _get_proxy_info(self, scheme, authority): + """Return a ProxyInfo instance (or None) based on the scheme + and authority. + """ + hostname, port = urllib.splitport(authority) + proxy_info = self.proxy_info + if callable(proxy_info): + proxy_info = proxy_info(scheme) + + if (hasattr(proxy_info, 'applies_to') + and not proxy_info.applies_to(hostname)): + proxy_info = None + return proxy_info + + +class Response(dict): + """An object more like email.Message than httplib.HTTPResponse.""" + + """Is this response from our local cache""" + fromcache = False + + """HTTP protocol version used by server. 10 for HTTP/1.0, 11 for HTTP/1.1. """ + version = 11 + + "Status code returned by server. " + status = 200 + + """Reason phrase returned by server.""" + reason = "Ok" + + previous = None + + def __init__(self, info): + # info is either an email.Message or + # an httplib.HTTPResponse object. + if isinstance(info, httplib.HTTPResponse): + for key, value in info.getheaders(): + self[key.lower()] = value + self.status = info.status + self['status'] = str(self.status) + self.reason = info.reason + self.version = info.version + elif isinstance(info, email.Message.Message): + for key, value in info.items(): + self[key.lower()] = value + self.status = int(self['status']) + else: + for key, value in info.iteritems(): + self[key.lower()] = value + self.status = int(self.get('status', self.status)) + self.reason = self.get('reason', self.reason) + + + def __getattr__(self, name): + if name == 'dict': + return self + else: + raise AttributeError, name diff --git a/python/plugins/sextante/servertools/httplib2/iri2uri.py b/python/plugins/sextante/servertools/httplib2/iri2uri.py new file mode 100644 index 00000000000..70667edf858 --- /dev/null +++ b/python/plugins/sextante/servertools/httplib2/iri2uri.py @@ -0,0 +1,110 @@ +""" +iri2uri + +Converts an IRI to a URI. + +""" +__author__ = "Joe Gregorio (joe@bitworking.org)" +__copyright__ = "Copyright 2006, Joe Gregorio" +__contributors__ = [] +__version__ = "1.0.0" +__license__ = "MIT" +__history__ = """ +""" + +import urlparse + + +# Convert an IRI to a URI following the rules in RFC 3987 +# +# The characters we need to enocde and escape are defined in the spec: +# +# iprivate = %xE000-F8FF / %xF0000-FFFFD / %x100000-10FFFD +# ucschar = %xA0-D7FF / %xF900-FDCF / %xFDF0-FFEF +# / %x10000-1FFFD / %x20000-2FFFD / %x30000-3FFFD +# / %x40000-4FFFD / %x50000-5FFFD / %x60000-6FFFD +# / %x70000-7FFFD / %x80000-8FFFD / %x90000-9FFFD +# / %xA0000-AFFFD / %xB0000-BFFFD / %xC0000-CFFFD +# / %xD0000-DFFFD / %xE1000-EFFFD + +escape_range = [ + (0xA0, 0xD7FF ), + (0xE000, 0xF8FF ), + (0xF900, 0xFDCF ), + (0xFDF0, 0xFFEF), + (0x10000, 0x1FFFD ), + (0x20000, 0x2FFFD ), + (0x30000, 0x3FFFD), + (0x40000, 0x4FFFD ), + (0x50000, 0x5FFFD ), + (0x60000, 0x6FFFD), + (0x70000, 0x7FFFD ), + (0x80000, 0x8FFFD ), + (0x90000, 0x9FFFD), + (0xA0000, 0xAFFFD ), + (0xB0000, 0xBFFFD ), + (0xC0000, 0xCFFFD), + (0xD0000, 0xDFFFD ), + (0xE1000, 0xEFFFD), + (0xF0000, 0xFFFFD ), + (0x100000, 0x10FFFD) +] + +def encode(c): + retval = c + i = ord(c) + for low, high in escape_range: + if i < low: + break + if i >= low and i <= high: + retval = "".join(["%%%2X" % ord(o) for o in c.encode('utf-8')]) + break + return retval + + +def iri2uri(uri): + """Convert an IRI to a URI. Note that IRIs must be + passed in a unicode strings. That is, do not utf-8 encode + the IRI before passing it into the function.""" + if isinstance(uri ,unicode): + (scheme, authority, path, query, fragment) = urlparse.urlsplit(uri) + authority = authority.encode('idna') + # For each character in 'ucschar' or 'iprivate' + # 1. encode as utf-8 + # 2. then %-encode each octet of that utf-8 + uri = urlparse.urlunsplit((scheme, authority, path, query, fragment)) + uri = "".join([encode(c) for c in uri]) + return uri + +if __name__ == "__main__": + import unittest + + class Test(unittest.TestCase): + + def test_uris(self): + """Test that URIs are invariant under the transformation.""" + invariant = [ + u"ftp://ftp.is.co.za/rfc/rfc1808.txt", + u"http://www.ietf.org/rfc/rfc2396.txt", + u"ldap://[2001:db8::7]/c=GB?objectClass?one", + u"mailto:John.Doe@example.com", + u"news:comp.infosystems.www.servers.unix", + u"tel:+1-816-555-1212", + u"telnet://192.0.2.16:80/", + u"urn:oasis:names:specification:docbook:dtd:xml:4.1.2" ] + for uri in invariant: + self.assertEqual(uri, iri2uri(uri)) + + def test_iri(self): + """ Test that the right type of escaping is done for each part of the URI.""" + self.assertEqual("http://xn--o3h.com/%E2%98%84", iri2uri(u"http://\N{COMET}.com/\N{COMET}")) + self.assertEqual("http://bitworking.org/?fred=%E2%98%84", iri2uri(u"http://bitworking.org/?fred=\N{COMET}")) + self.assertEqual("http://bitworking.org/#%E2%98%84", iri2uri(u"http://bitworking.org/#\N{COMET}")) + self.assertEqual("#%E2%98%84", iri2uri(u"#\N{COMET}")) + self.assertEqual("/fred?bar=%E2%98%9A#%E2%98%84", iri2uri(u"/fred?bar=\N{BLACK LEFT POINTING INDEX}#\N{COMET}")) + self.assertEqual("/fred?bar=%E2%98%9A#%E2%98%84", iri2uri(iri2uri(u"/fred?bar=\N{BLACK LEFT POINTING INDEX}#\N{COMET}"))) + self.assertNotEqual("/fred?bar=%E2%98%9A#%E2%98%84", iri2uri(u"/fred?bar=\N{BLACK LEFT POINTING INDEX}#\N{COMET}".encode('utf-8'))) + + unittest.main() + + diff --git a/python/plugins/sextante/servertools/httplib2/socks.py b/python/plugins/sextante/servertools/httplib2/socks.py new file mode 100644 index 00000000000..0991f4cf6e0 --- /dev/null +++ b/python/plugins/sextante/servertools/httplib2/socks.py @@ -0,0 +1,438 @@ +"""SocksiPy - Python SOCKS module. +Version 1.00 + +Copyright 2006 Dan-Haim. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. +3. Neither the name of Dan Haim nor the names of his contributors may be used + to endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY DAN HAIM "AS IS" AND ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL DAN HAIM OR HIS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA +OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMANGE. + + +This module provides a standard socket-like interface for Python +for tunneling connections through SOCKS proxies. + +""" + +""" + +Minor modifications made by Christopher Gilbert (http://motomastyle.com/) +for use in PyLoris (http://pyloris.sourceforge.net/) + +Minor modifications made by Mario Vilas (http://breakingcode.wordpress.com/) +mainly to merge bug fixes found in Sourceforge + +""" + +import base64 +import socket +import struct +import sys + +if getattr(socket, 'socket', None) is None: + raise ImportError('socket.socket missing, proxy support unusable') + +PROXY_TYPE_SOCKS4 = 1 +PROXY_TYPE_SOCKS5 = 2 +PROXY_TYPE_HTTP = 3 +PROXY_TYPE_HTTP_NO_TUNNEL = 4 + +_defaultproxy = None +_orgsocket = socket.socket + +class ProxyError(Exception): pass +class GeneralProxyError(ProxyError): pass +class Socks5AuthError(ProxyError): pass +class Socks5Error(ProxyError): pass +class Socks4Error(ProxyError): pass +class HTTPError(ProxyError): pass + +_generalerrors = ("success", + "invalid data", + "not connected", + "not available", + "bad proxy type", + "bad input") + +_socks5errors = ("succeeded", + "general SOCKS server failure", + "connection not allowed by ruleset", + "Network unreachable", + "Host unreachable", + "Connection refused", + "TTL expired", + "Command not supported", + "Address type not supported", + "Unknown error") + +_socks5autherrors = ("succeeded", + "authentication is required", + "all offered authentication methods were rejected", + "unknown username or invalid password", + "unknown error") + +_socks4errors = ("request granted", + "request rejected or failed", + "request rejected because SOCKS server cannot connect to identd on the client", + "request rejected because the client program and identd report different user-ids", + "unknown error") + +def setdefaultproxy(proxytype=None, addr=None, port=None, rdns=True, username=None, password=None): + """setdefaultproxy(proxytype, addr[, port[, rdns[, username[, password]]]]) + Sets a default proxy which all further socksocket objects will use, + unless explicitly changed. + """ + global _defaultproxy + _defaultproxy = (proxytype, addr, port, rdns, username, password) + +def wrapmodule(module): + """wrapmodule(module) + Attempts to replace a module's socket library with a SOCKS socket. Must set + a default proxy using setdefaultproxy(...) first. + This will only work on modules that import socket directly into the namespace; + most of the Python Standard Library falls into this category. + """ + if _defaultproxy != None: + module.socket.socket = socksocket + else: + raise GeneralProxyError((4, "no proxy specified")) + +class socksocket(socket.socket): + """socksocket([family[, type[, proto]]]) -> socket object + Open a SOCKS enabled socket. The parameters are the same as + those of the standard socket init. In order for SOCKS to work, + you must specify family=AF_INET, type=SOCK_STREAM and proto=0. + """ + + def __init__(self, family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0, _sock=None): + _orgsocket.__init__(self, family, type, proto, _sock) + if _defaultproxy != None: + self.__proxy = _defaultproxy + else: + self.__proxy = (None, None, None, None, None, None) + self.__proxysockname = None + self.__proxypeername = None + self.__httptunnel = True + + def __recvall(self, count): + """__recvall(count) -> data + Receive EXACTLY the number of bytes requested from the socket. + Blocks until the required number of bytes have been received. + """ + data = self.recv(count) + while len(data) < count: + d = self.recv(count-len(data)) + if not d: raise GeneralProxyError((0, "connection closed unexpectedly")) + data = data + d + return data + + def sendall(self, content, *args): + """ override socket.socket.sendall method to rewrite the header + for non-tunneling proxies if needed + """ + if not self.__httptunnel: + content = self.__rewriteproxy(content) + return super(socksocket, self).sendall(content, *args) + + def __rewriteproxy(self, header): + """ rewrite HTTP request headers to support non-tunneling proxies + (i.e. those which do not support the CONNECT method). + This only works for HTTP (not HTTPS) since HTTPS requires tunneling. + """ + host, endpt = None, None + hdrs = header.split("\r\n") + for hdr in hdrs: + if hdr.lower().startswith("host:"): + host = hdr + elif hdr.lower().startswith("get") or hdr.lower().startswith("post"): + endpt = hdr + if host and endpt: + hdrs.remove(host) + hdrs.remove(endpt) + host = host.split(" ")[1] + endpt = endpt.split(" ") + if (self.__proxy[4] != None and self.__proxy[5] != None): + hdrs.insert(0, self.__getauthheader()) + hdrs.insert(0, "Host: %s" % host) + hdrs.insert(0, "%s http://%s%s %s" % (endpt[0], host, endpt[1], endpt[2])) + return "\r\n".join(hdrs) + + def __getauthheader(self): + auth = self.__proxy[4] + ":" + self.__proxy[5] + return "Proxy-Authorization: Basic " + base64.b64encode(auth) + + def setproxy(self, proxytype=None, addr=None, port=None, rdns=True, username=None, password=None): + """setproxy(proxytype, addr[, port[, rdns[, username[, password]]]]) + Sets the proxy to be used. + proxytype - The type of the proxy to be used. Three types + are supported: PROXY_TYPE_SOCKS4 (including socks4a), + PROXY_TYPE_SOCKS5 and PROXY_TYPE_HTTP + addr - The address of the server (IP or DNS). + port - The port of the server. Defaults to 1080 for SOCKS + servers and 8080 for HTTP proxy servers. + rdns - Should DNS queries be preformed on the remote side + (rather than the local side). The default is True. + Note: This has no effect with SOCKS4 servers. + username - Username to authenticate with to the server. + The default is no authentication. + password - Password to authenticate with to the server. + Only relevant when username is also provided. + """ + self.__proxy = (proxytype, addr, port, rdns, username, password) + + def __negotiatesocks5(self, destaddr, destport): + """__negotiatesocks5(self,destaddr,destport) + Negotiates a connection through a SOCKS5 server. + """ + # First we'll send the authentication packages we support. + if (self.__proxy[4]!=None) and (self.__proxy[5]!=None): + # The username/password details were supplied to the + # setproxy method so we support the USERNAME/PASSWORD + # authentication (in addition to the standard none). + self.sendall(struct.pack('BBBB', 0x05, 0x02, 0x00, 0x02)) + else: + # No username/password were entered, therefore we + # only support connections with no authentication. + self.sendall(struct.pack('BBB', 0x05, 0x01, 0x00)) + # We'll receive the server's response to determine which + # method was selected + chosenauth = self.__recvall(2) + if chosenauth[0:1] != chr(0x05).encode(): + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + # Check the chosen authentication method + if chosenauth[1:2] == chr(0x00).encode(): + # No authentication is required + pass + elif chosenauth[1:2] == chr(0x02).encode(): + # Okay, we need to perform a basic username/password + # authentication. + self.sendall(chr(0x01).encode() + chr(len(self.__proxy[4])) + self.__proxy[4] + chr(len(self.__proxy[5])) + self.__proxy[5]) + authstat = self.__recvall(2) + if authstat[0:1] != chr(0x01).encode(): + # Bad response + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + if authstat[1:2] != chr(0x00).encode(): + # Authentication failed + self.close() + raise Socks5AuthError((3, _socks5autherrors[3])) + # Authentication succeeded + else: + # Reaching here is always bad + self.close() + if chosenauth[1] == chr(0xFF).encode(): + raise Socks5AuthError((2, _socks5autherrors[2])) + else: + raise GeneralProxyError((1, _generalerrors[1])) + # Now we can request the actual connection + req = struct.pack('BBB', 0x05, 0x01, 0x00) + # If the given destination address is an IP address, we'll + # use the IPv4 address request even if remote resolving was specified. + try: + ipaddr = socket.inet_aton(destaddr) + req = req + chr(0x01).encode() + ipaddr + except socket.error: + # Well it's not an IP number, so it's probably a DNS name. + if self.__proxy[3]: + # Resolve remotely + ipaddr = None + req = req + chr(0x03).encode() + chr(len(destaddr)).encode() + destaddr + else: + # Resolve locally + ipaddr = socket.inet_aton(socket.gethostbyname(destaddr)) + req = req + chr(0x01).encode() + ipaddr + req = req + struct.pack(">H", destport) + self.sendall(req) + # Get the response + resp = self.__recvall(4) + if resp[0:1] != chr(0x05).encode(): + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + elif resp[1:2] != chr(0x00).encode(): + # Connection failed + self.close() + if ord(resp[1:2])<=8: + raise Socks5Error((ord(resp[1:2]), _socks5errors[ord(resp[1:2])])) + else: + raise Socks5Error((9, _socks5errors[9])) + # Get the bound address/port + elif resp[3:4] == chr(0x01).encode(): + boundaddr = self.__recvall(4) + elif resp[3:4] == chr(0x03).encode(): + resp = resp + self.recv(1) + boundaddr = self.__recvall(ord(resp[4:5])) + else: + self.close() + raise GeneralProxyError((1,_generalerrors[1])) + boundport = struct.unpack(">H", self.__recvall(2))[0] + self.__proxysockname = (boundaddr, boundport) + if ipaddr != None: + self.__proxypeername = (socket.inet_ntoa(ipaddr), destport) + else: + self.__proxypeername = (destaddr, destport) + + def getproxysockname(self): + """getsockname() -> address info + Returns the bound IP address and port number at the proxy. + """ + return self.__proxysockname + + def getproxypeername(self): + """getproxypeername() -> address info + Returns the IP and port number of the proxy. + """ + return _orgsocket.getpeername(self) + + def getpeername(self): + """getpeername() -> address info + Returns the IP address and port number of the destination + machine (note: getproxypeername returns the proxy) + """ + return self.__proxypeername + + def __negotiatesocks4(self,destaddr,destport): + """__negotiatesocks4(self,destaddr,destport) + Negotiates a connection through a SOCKS4 server. + """ + # Check if the destination address provided is an IP address + rmtrslv = False + try: + ipaddr = socket.inet_aton(destaddr) + except socket.error: + # It's a DNS name. Check where it should be resolved. + if self.__proxy[3]: + ipaddr = struct.pack("BBBB", 0x00, 0x00, 0x00, 0x01) + rmtrslv = True + else: + ipaddr = socket.inet_aton(socket.gethostbyname(destaddr)) + # Construct the request packet + req = struct.pack(">BBH", 0x04, 0x01, destport) + ipaddr + # The username parameter is considered userid for SOCKS4 + if self.__proxy[4] != None: + req = req + self.__proxy[4] + req = req + chr(0x00).encode() + # DNS name if remote resolving is required + # NOTE: This is actually an extension to the SOCKS4 protocol + # called SOCKS4A and may not be supported in all cases. + if rmtrslv: + req = req + destaddr + chr(0x00).encode() + self.sendall(req) + # Get the response from the server + resp = self.__recvall(8) + if resp[0:1] != chr(0x00).encode(): + # Bad data + self.close() + raise GeneralProxyError((1,_generalerrors[1])) + if resp[1:2] != chr(0x5A).encode(): + # Server returned an error + self.close() + if ord(resp[1:2]) in (91, 92, 93): + self.close() + raise Socks4Error((ord(resp[1:2]), _socks4errors[ord(resp[1:2]) - 90])) + else: + raise Socks4Error((94, _socks4errors[4])) + # Get the bound address/port + self.__proxysockname = (socket.inet_ntoa(resp[4:]), struct.unpack(">H", resp[2:4])[0]) + if rmtrslv != None: + self.__proxypeername = (socket.inet_ntoa(ipaddr), destport) + else: + self.__proxypeername = (destaddr, destport) + + def __negotiatehttp(self, destaddr, destport): + """__negotiatehttp(self,destaddr,destport) + Negotiates a connection through an HTTP server. + """ + # If we need to resolve locally, we do this now + if not self.__proxy[3]: + addr = socket.gethostbyname(destaddr) + else: + addr = destaddr + headers = ["CONNECT ", addr, ":", str(destport), " HTTP/1.1\r\n"] + headers += ["Host: ", destaddr, "\r\n"] + if (self.__proxy[4] != None and self.__proxy[5] != None): + headers += [self.__getauthheader(), "\r\n"] + headers.append("\r\n") + self.sendall("".join(headers).encode()) + # We read the response until we get the string "\r\n\r\n" + resp = self.recv(1) + while resp.find("\r\n\r\n".encode()) == -1: + resp = resp + self.recv(1) + # We just need the first line to check if the connection + # was successful + statusline = resp.splitlines()[0].split(" ".encode(), 2) + if statusline[0] not in ("HTTP/1.0".encode(), "HTTP/1.1".encode()): + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + try: + statuscode = int(statusline[1]) + except ValueError: + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + if statuscode != 200: + self.close() + raise HTTPError((statuscode, statusline[2])) + self.__proxysockname = ("0.0.0.0", 0) + self.__proxypeername = (addr, destport) + + def connect(self, destpair): + """connect(self, despair) + Connects to the specified destination through a proxy. + destpar - A tuple of the IP/DNS address and the port number. + (identical to socket's connect). + To select the proxy server use setproxy(). + """ + # Do a minimal input check first + if (not type(destpair) in (list,tuple)) or (len(destpair) < 2) or (not isinstance(destpair[0], basestring)) or (type(destpair[1]) != int): + raise GeneralProxyError((5, _generalerrors[5])) + if self.__proxy[0] == PROXY_TYPE_SOCKS5: + if self.__proxy[2] != None: + portnum = self.__proxy[2] + else: + portnum = 1080 + _orgsocket.connect(self, (self.__proxy[1], portnum)) + self.__negotiatesocks5(destpair[0], destpair[1]) + elif self.__proxy[0] == PROXY_TYPE_SOCKS4: + if self.__proxy[2] != None: + portnum = self.__proxy[2] + else: + portnum = 1080 + _orgsocket.connect(self,(self.__proxy[1], portnum)) + self.__negotiatesocks4(destpair[0], destpair[1]) + elif self.__proxy[0] == PROXY_TYPE_HTTP: + if self.__proxy[2] != None: + portnum = self.__proxy[2] + else: + portnum = 8080 + _orgsocket.connect(self,(self.__proxy[1], portnum)) + self.__negotiatehttp(destpair[0], destpair[1]) + elif self.__proxy[0] == PROXY_TYPE_HTTP_NO_TUNNEL: + if self.__proxy[2] != None: + portnum = self.__proxy[2] + else: + portnum = 8080 + _orgsocket.connect(self,(self.__proxy[1],portnum)) + if destpair[1] == 443: + self.__negotiatehttp(destpair[0],destpair[1]) + else: + self.__httptunnel = False + elif self.__proxy[0] == None: + _orgsocket.connect(self, (destpair[0], destpair[1])) + else: + raise GeneralProxyError((4, _generalerrors[4])) diff --git a/python/plugins/sextanteexampleprovider/ExampleAlgorithm.py b/python/plugins/sextanteexampleprovider/ExampleAlgorithm.py deleted file mode 100644 index b645870cccb..00000000000 --- a/python/plugins/sextanteexampleprovider/ExampleAlgorithm.py +++ /dev/null @@ -1,104 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -*************************************************************************** - ExampleAlgorithm.py - --------------------- - Date : August 2012 - Copyright : (C) 2012 by Tim Sutton - Email : tim at linfiniti dot com -*************************************************************************** -* * -* This program is free software; you can redistribute it and/or modify * -* it under the terms of the GNU General Public License as published by * -* the Free Software Foundation; either version 2 of the License, or * -* (at your option) any later version. * -* * -*************************************************************************** -""" - -__author__ = 'Tim Sutton' -__date__ = 'August 2012' -__copyright__ = '(C) 2012, Tim Sutton' -# This will get replaced with a git SHA1 when you do a git archive -__revision__ = '$Format:%H$' - -from sextante.core.GeoAlgorithm import GeoAlgorithm -from sextante.outputs.OutputVector import OutputVector -from sextante.parameters.ParameterVector import ParameterVector -from sextante.core.Sextante import Sextante -from qgis.core import * -from PyQt4.QtCore import * -from PyQt4.QtGui import * -from sextante.parameters.ParameterExtent import ParameterExtent -from sextante.parameters.ParameterCrs import ParameterCrs - - -class ExampleAlgorithm(GeoAlgorithm): - '''This is an example algorithm that takes a vector layer and creates - a new one just with just those features of the input layer that are - selected. - It is meant to be used as an example of how to create your own SEXTANTE - algorithms and explain methods and variables used to do it. - An algorithm like this will be available in all SEXTANTE elements, and - there is not need for additional work. - - All SEXTANTE algorithms should extend the GeoAlgorithm class''' - - #constants used to refer to parameters and outputs. - #They will be used when calling the algorithm from another algorithm, - #or when calling SEXTANTE from the QGIS console. - OUTPUT_LAYER = "OUTPUT_LAYER" - INPUT_LAYER = "INPUT_LAYER" - - def defineCharacteristics(self): - '''Here we define the inputs and output of the algorithm, along - with some other properties''' - - #the name that the user will see in the toolbox - self.name = "Create new layer with selected features" - - #the branch of the toolbox under which the algorithm will appear - self.group = "Algorithms for vector layers" - - #we add the input vector layer. It can have any kind of geometry - #It is a mandatory (not optional) one, hence the False argument - self.addParameter(ParameterVector(self.INPUT_LAYER, "Input layer", ParameterVector.VECTOR_TYPE_ANY, False)) - self.addParameter(ParameterExtent("EXTENT","EXTENT")) - self.addParameter(ParameterCrs("CRS", "CRS")) - # we add a vector layer as output - self.addOutput(OutputVector(self.OUTPUT_LAYER, "Output layer with selected features")) - - - def processAlgorithm(self, progress): - '''Here is where the processing itself takes place''' - - #the first thing to do is retrieve the values of the parameters - #entered by the user - inputFilename = self.getParameterValue(self.INPUT_LAYER) - output = self.getOutputValue(self.OUTPUT_LAYER) - - #input layers values are always a string with its location. - #That string can be converted into a QGIS object (a QgsVectorLayer in this case)) - #using the Sextante.getObject() method - vectorLayer = Sextante.getObject(inputFilename) - - #And now we can process - - #First we create the output layer. - #The output value entered by the user is a string containing a filename, - #so we can use it directly - settings = QSettings() - systemEncoding = settings.value( "/UI/encoding", "System" ).toString() - provider = vectorLayer.dataProvider() - writer = QgsVectorFileWriter( output, systemEncoding, provider.fields(), provider.geometryType(), provider.crs() ) - - #Now we take the selected features and add them to the output layer - selection = vectorLayer.selectedFeatures() - for feat in selection: - writer.addFeature(feat) - del writer - - #There is nothing more to do here. We do not have to open the layer that we have created. - #SEXTANTE will take care of that, or will handle it if this algorithm is executed within - #a complex model diff --git a/python/plugins/sextanteexampleprovider/ExampleAlgorithmProvider.py b/python/plugins/sextanteexampleprovider/ExampleAlgorithmProvider.py deleted file mode 100644 index bd03d6490ee..00000000000 --- a/python/plugins/sextanteexampleprovider/ExampleAlgorithmProvider.py +++ /dev/null @@ -1,78 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -*************************************************************************** - ExampleAlgorithmProvider.py - --------------------- - Date : August 2012 - Copyright : (C) 2012 by Tim Sutton - Email : tim at linfiniti dot com -*************************************************************************** -* * -* This program is free software; you can redistribute it and/or modify * -* it under the terms of the GNU General Public License as published by * -* the Free Software Foundation; either version 2 of the License, or * -* (at your option) any later version. * -* * -*************************************************************************** -""" - -__author__ = 'Tim Sutton' -__date__ = 'August 2012' -__copyright__ = '(C) 2012, Tim Sutton' -# This will get replaced with a git SHA1 when you do a git archive -__revision__ = '$Format:%H$' - -from sextante.core.AlgorithmProvider import AlgorithmProvider -from sextanteexampleprovider.ExampleAlgorithm import ExampleAlgorithm -from sextante.core.SextanteConfig import Setting, SextanteConfig - -class ExampleAlgorithmProvider(AlgorithmProvider): - - MY_DUMMY_SETTING = "MY_DUMMY_SETTING" - - def __init__(self): - AlgorithmProvider.__init__(self) - self.alglist = [ExampleAlgorithm()] - - def initializeSettings(self): - '''In this method we add settings needed to configure our provider. - Do not forget to call the parent method, since it takes care or - automatically adding a setting for activating or deactivating the - algorithms in the provider''' - AlgorithmProvider.initializeSettings(self) - SextanteConfig.addSetting(Setting("Example algorithms", ExampleAlgorithmProvider.MY_DUMMY_SETTING, "Example setting", "Default value")) - '''To get the parameter of a setting parameter, use SextanteConfig.getSetting(name_of_parameter)''' - - def unload(self): - '''Setting should be removed here, so they do not appear anymore - when the plugin is unloaded''' - AlgorithmProvider.unload(self) - SextanteConfig.removeSetting( ExampleAlgorithmProvider.MY_DUMMY_SETTING) - - - def getName(self): - '''This name is used to create the command line name of all the algorithms - from this provider''' - return "exampleprovider" - - def getDescription(self): - '''This is the name that will appear on the toolbox group.''' - return "Example algorithms" - - def getIcon(self): - '''We return the default icon''' - return AlgorithmProvider.getIcon(self) - - - def _loadAlgorithms(self): - '''Here we fill the list of algorithms in self.algs. - This method is called whenever the list of algorithms should be updated. - If the list of algorithms can change while executing SEXTANTE for QGIS - (for instance, if it contains algorithms from user-defined scripts and - a new script might have been added), you should create the list again - here. - In this case, since the list is always the same, we assign from the pre-made list. - This assignment has to be done in this method even if the list does not change, - since the self.algs list is cleared before calling this method''' - self.algs = self.alglist diff --git a/python/plugins/sextanteexampleprovider/SextanteExampleProviderPlugin.py b/python/plugins/sextanteexampleprovider/SextanteExampleProviderPlugin.py deleted file mode 100644 index 1e0cfd25a3d..00000000000 --- a/python/plugins/sextanteexampleprovider/SextanteExampleProviderPlugin.py +++ /dev/null @@ -1,46 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -*************************************************************************** - SextanteExampleProviderPlugin.py - --------------------- - Date : August 2012 - Copyright : (C) 2012 by Tim Sutton - Email : tim at linfiniti dot com -*************************************************************************** -* * -* This program is free software; you can redistribute it and/or modify * -* it under the terms of the GNU General Public License as published by * -* the Free Software Foundation; either version 2 of the License, or * -* (at your option) any later version. * -* * -*************************************************************************** -""" - -__author__ = 'Tim Sutton' -__date__ = 'August 2012' -__copyright__ = '(C) 2012, Tim Sutton' -# This will get replaced with a git SHA1 when you do a git archive -__revision__ = '$Format:%H$' - -from qgis.core import * -import os, sys -import inspect -from sextante.core.Sextante import Sextante -from sextanteexampleprovider.ExampleAlgorithmProvider import ExampleAlgorithmProvider - - -cmd_folder = os.path.split(inspect.getfile( inspect.currentframe() ))[0] -if cmd_folder not in sys.path: - sys.path.insert(0, cmd_folder) - -class SextanteExampleProviderPlugin: - - def __init__(self): - self.provider = ExampleAlgorithmProvider() - def initGui(self): - Sextante.addProvider(self.provider) - - def unload(self): - Sextante.removeProvider(self.provider) - diff --git a/python/plugins/sextanteexampleprovider/metadata.txt b/python/plugins/sextanteexampleprovider/metadata.txt deleted file mode 100644 index 7cc0cc5b9e9..00000000000 --- a/python/plugins/sextanteexampleprovider/metadata.txt +++ /dev/null @@ -1,11 +0,0 @@ -[general] -name=SEXTANTE Example Provider -description=An example plugin that adds algorithms to SEXTANTE. Mainly created to guide developers in the process of creating plugins that add new capabilities to SEXTANTE -version=1.0 -qgisMinimumVersion=1.0 - -name=Victor Olaya -email=volayaf@gmail.com -website=www.sextantegis.com - -class_name=SextanteExampleProviderPlugin