sharpetronics/QGIS
1
0
Fork 0
mirror of https://github.com/qgis/QGIS.git synced 2025-02-25 00:58:06 -05:00
Code Issues Packages Projects Releases Wiki Activity

[feature] Allow input parameter values for qgis_process to be

Browse Source 
specified as a JSON object passed via stdin to qgis_process

This provides a mechanism to support complex input parameters
for algorithms, and a way for qgis_process to gain support
for parameter types which are themselves specified as a dictionary
type object.

To indicate that parameters will be specified via stdin then
the qgis_process command must follow the format

    qgis_process run algid -

(with a trailing - in place of the usual arguments list).

The JSON object must contain an "inputs" key, which is a map
of the input parameter values.

E.g.

    echo "{"inputs": {\"INPUT\": \"my_shape.shp\", DISTANCE: 5}}" | qgis_process run native:buffer -

Specifying input parameters via stdin implies automatically
the --json output format for results.

One big motivation behind this enhancement is to provide a way for
the qgisprocess R libraries to support parameter types such as
aggregates.

Refs https://github.com/paleolimbot/qgisprocess/issues/56
Refs https://github.com/paleolimbot/qgisprocess/issues/44

Sponsored by the Research Institute for Nature and Forest, Flemish Govt
This commit is contained in:
Nyall Dawson 2021-12-15 11:34:56 +10:00
parent 2406286a0b
commit 5fd9b209ac
3 changed files with 216 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
python/core/auto_generated/qgsjsonutils.sip.in
View File

@ -345,6 +345,7 @@ Parse a simple array (depth=1)
};
/************************************************************************

158
src/process/qgsprocess.cpp
View File

@ -342,42 +342,112 @@ int QgsProcessingExec::run( const QStringList &constArgs )
QgsUnitTypes::AreaUnit areaUnit = QgsUnitTypes::AreaUnknownUnit;
QString projectPath;
QVariantMap params;
int i = 3;
for ( ; i < args.count(); i++ )
{
QString arg = args.at( i );
if ( arg == QLatin1String( "--" ) )
if ( args.size() == 4 && args.at( 3 ) == '-' )
{
// read arguments as JSON value from stdin
std::string stdin;
for ( std::string line; std::getline( std::cin, line ); )
{
break;
stdin.append( line + '\n' );
}
if ( arg.startsWith( QLatin1String( "--" ) ) )
arg = arg.mid( 2 );
const QStringList parts = arg.split( '=' );
if ( parts.count() >= 2 )
QString error;
const QVariantMap json = QgsJsonUtils::parseJson( stdin, error ).toMap();
if ( !error.isEmpty() )
{
const QString name = parts.at( 0 );
std::cerr << QStringLiteral( "Could not parse JSON parameters: %1" ).arg( error ).toLocal8Bit().constData() << std::endl;
return 1;
}
if ( !json.contains( QStringLiteral( "inputs" ) ) )
{
std::cerr << QStringLiteral( "JSON parameters object must contain an \"inputs\" key." ).toLocal8Bit().constData() << std::endl;
return 1;
}
if ( name.compare( QLatin1String( "ellipsoid" ), Qt::CaseInsensitive ) == 0 )
params = json.value( QStringLiteral( "inputs" ) ).toMap();
// JSON format for input parameters implies JSON output format
useJson = true;
}
else
{
int i = 3;
for ( ; i < args.count(); i++ )
{
QString arg = args.at( i );
if ( arg == QLatin1String( "--" ) )
{
ellipsoid = parts.mid( 1 ).join( '=' );
break;
}
else if ( name.compare( QLatin1String( "distance_units" ), Qt::CaseInsensitive ) == 0 )
if ( arg.startsWith( QLatin1String( "--" ) ) )
arg = arg.mid( 2 );
const QStringList parts = arg.split( '=' );
if ( parts.count() >= 2 )
{
distanceUnit = QgsUnitTypes::decodeDistanceUnit( parts.mid( 1 ).join( '=' ) );
}
else if ( name.compare( QLatin1String( "area_units" ), Qt::CaseInsensitive ) == 0 )
{
areaUnit = QgsUnitTypes::decodeAreaUnit( parts.mid( 1 ).join( '=' ) );
}
else if ( name.compare( QLatin1String( "project_path" ), Qt::CaseInsensitive ) == 0 )
{
projectPath = parts.mid( 1 ).join( '=' );
const QString name = parts.at( 0 );
if ( name.compare( QLatin1String( "ellipsoid" ), Qt::CaseInsensitive ) == 0 )
{
ellipsoid = parts.mid( 1 ).join( '=' );
}
else if ( name.compare( QLatin1String( "distance_units" ), Qt::CaseInsensitive ) == 0 )
{
distanceUnit = QgsUnitTypes::decodeDistanceUnit( parts.mid( 1 ).join( '=' ) );
}
else if ( name.compare( QLatin1String( "area_units" ), Qt::CaseInsensitive ) == 0 )
{
areaUnit = QgsUnitTypes::decodeAreaUnit( parts.mid( 1 ).join( '=' ) );
}
else if ( name.compare( QLatin1String( "project_path" ), Qt::CaseInsensitive ) == 0 )
{
projectPath = parts.mid( 1 ).join( '=' );
}
else
{
const QString value = parts.mid( 1 ).join( '=' );
if ( params.contains( name ) )
{
// parameter specified multiple times, store all of them in a list...
if ( params.value( name ).type() == QVariant::StringList )
{
// append to existing list
QStringList listValue = params.value( name ).toStringList();
listValue << value;
params.insert( name, listValue );
}
else
{
// upgrade previous value to list
QStringList listValue = QStringList() << params.value( name ).toString()
<< value;
params.insert( name, listValue );
}
}
else
{
params.insert( name, value );
}
}
}
else
{
std::cerr << QStringLiteral( "Invalid parameter value %1. Parameter values must be entered after \"--\" e.g.\n Example:\n qgis_process run algorithm_name -- PARAM1=VALUE PARAM2=42\"\n" ).arg( arg ).toLocal8Bit().constData();
return 1;
}
}
// After '--' we only have params
for ( ; i < args.count(); i++ )
{
const QString arg = args.at( i );
const QStringList parts = arg.split( '=' );
if ( parts.count() >= 2 )
{
const QString name = parts.first();
const QString value = parts.mid( 1 ).join( '=' );
if ( params.contains( name ) )
{
@ -403,45 +473,6 @@ int QgsProcessingExec::run( const QStringList &constArgs )
}
}
}
else
{
std::cerr << QStringLiteral( "Invalid parameter value %1. Parameter values must be entered after \"--\" e.g.\n Example:\n qgis_process run algorithm_name -- PARAM1=VALUE PARAM2=42\"\n" ).arg( arg ).toLocal8Bit().constData();
return 1;
}
}
// After '--' we only have params
for ( ; i < args.count(); i++ )
{
const QString arg = args.at( i );
const QStringList parts = arg.split( '=' );
if ( parts.count() >= 2 )
{
const QString name = parts.first();
const QString value = parts.mid( 1 ).join( '=' );
if ( params.contains( name ) )
{
// parameter specified multiple times, store all of them in a list...
if ( params.value( name ).type() == QVariant::StringList )
{
// append to existing list
QStringList listValue = params.value( name ).toStringList();
listValue << value;
params.insert( name, listValue );
}
else
{
// upgrade previous value to list
QStringList listValue = QStringList() << params.value( name ).toString()
<< value;
params.insert( name, listValue );
}
}
else
{
params.insert( name, value );
}
}
}
return execute( algId, params, ellipsoid, distanceUnit, areaUnit, logLevel, useJson, projectPath );
@ -473,6 +504,7 @@ void QgsProcessingExec::showUsage( const QString &appName )
<< "\tlist\t\tlist all available processing algorithms\n"
<< "\thelp\t\tshow help for an algorithm. The algorithm id or a path to a model file must be specified.\n"
<< "\trun\t\truns an algorithm. The algorithm id or a path to a model file and parameter values must be specified. Parameter values are specified after -- with PARAMETER=VALUE syntax. Ordered list values for a parameter can be created by specifying the parameter multiple times, e.g. --LAYERS=layer1.shp --LAYERS=layer2.shp\n"
<< "\t\t\tAlternatively, a '-' character in place of the parameters argument indicates that the parameters should be read from STDIN as a JSON object. The JSON should be structured as a map containing at least the \"inputs\" key specifying a map of input parameter values. This implies the --json option for output as a JSON object.\n"
<< "\t\t\tIf required, the ellipsoid to use for distance and area calculations can be specified via the \"--ELLIPSOID=name\" argument.\n"
<< "\t\t\tIf required, an existing QGIS project to use during the algorithm execution can be specified via the \"--PROJECT_PATH=path\" argument.\n";

120
tests/src/python/test_qgsprocessexecutable.py
View File

@ -56,6 +56,19 @@ class TestQgsProcessExecutable(unittest.TestCase):
return rc, output.decode(), err.decode()
def run_process_stdin(self, arguments, stdin_string: str):
call = [QGIS_PROCESS_BIN] + arguments
print(' '.join(call))
myenv = os.environ.copy()
myenv["QGIS_DEBUG"] = '0'
p = subprocess.Popen(call, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, env=myenv)
output, err = p.communicate(input=stdin_string.encode())
rc = p.returncode
return rc, output.decode(), err.decode()
def testNoArgs(self):
rc, output, err = self.run_process([])
self.assertIn('Available commands', output)
@ -187,6 +200,60 @@ class TestQgsProcessExecutable(unittest.TestCase):
self.assertTrue(os.path.exists(output_file))
self.assertEqual(rc, 0)
def testAlgorithmRunStdIn(self):
output_file = self.TMP_DIR + '/polygon_centroid_json.shp'
params = {
'inputs': {
'INPUT': TEST_DATA_DIR + '/polys.shp',
'OUTPUT': output_file
}
}
rc, output, err = self.run_process_stdin(['run', 'native:centroids', '-'], json.dumps(params))
if os.environ.get('TRAVIS', '') != 'true':
# Travis DOES have errors, due to QStandardPaths: XDG_RUNTIME_DIR not set warnings raised by Qt
self.assertFalse(err)
res = json.loads(output)
self.assertIn('gdal_version', res)
self.assertIn('geos_version', res)
self.assertIn('proj_version', res)
self.assertIn('python_version', res)
self.assertIn('qt_version', res)
self.assertIn('qgis_version', res)
self.assertEqual(res['algorithm_details']['name'], 'Centroids')
self.assertEqual(res['inputs']['INPUT'], TEST_DATA_DIR + '/polys.shp')
self.assertEqual(res['inputs']['OUTPUT'], output_file)
self.assertEqual(res['results']['OUTPUT'], output_file)
self.assertTrue(os.path.exists(output_file))
self.assertEqual(rc, 0)
def testAlgorithmRunStdInMissingInputKey(self):
output_file = self.TMP_DIR + '/polygon_centroid_json.shp'
params = {
'INPUT': TEST_DATA_DIR + '/polys.shp',
'OUTPUT': output_file
}
rc, output, err = self.run_process_stdin(['run', 'native:centroids', '-'], json.dumps(params))
self.assertEqual(rc, 1)
self.assertIn('JSON parameters object must contain an "inputs" key.', err)
def testAlgorithmRunStdInNoInput(self):
rc, output, err = self.run_process_stdin(['run', 'native:centroids', '-'], '')
self.assertEqual(rc, 1)
self.assertIn('Could not parse JSON parameters', err)
def testAlgorithmRunStdInBadInput(self):
rc, output, err = self.run_process_stdin(['run', 'native:centroids', '-'], '{"not valid json"}')
self.assertEqual(rc, 1)
self.assertIn('Could not parse JSON parameters', err)
def testAlgorithmRunJson(self):
output_file = self.TMP_DIR + '/polygon_centroid2.shp'
rc, output, err = self.run_process(['run', '--json', 'native:centroids', '--', 'INPUT={}'.format(TEST_DATA_DIR + '/polys.shp'), 'OUTPUT={}'.format(output_file)])
@ -254,6 +321,32 @@ class TestQgsProcessExecutable(unittest.TestCase):
self.assertIn('results', output.lower())
self.assertTrue(os.path.exists(output_file))
def testModelRunStdIn(self):
output_file = self.TMP_DIR + '/model_output_stdin.shp'
params = {
'inputs': {
'FEATS': TEST_DATA_DIR + '/polys.shp',
'native:centroids_1:CENTROIDS': output_file
}
}
rc, output, err = self.run_process_stdin(['run', TEST_DATA_DIR + '/test_model.model3', '-'], json.dumps(params))
if os.environ.get('TRAVIS', '') != 'true':
# Travis DOES have errors, due to QStandardPaths: XDG_RUNTIME_DIR not set warnings raised by Qt
self.assertFalse(err)
self.assertEqual(rc, 0)
res = json.loads(output)
self.assertIn('gdal_version', res)
self.assertIn('geos_version', res)
self.assertIn('proj_version', res)
self.assertIn('python_version', res)
self.assertIn('qt_version', res)
self.assertIn('qgis_version', res)
self.assertEqual(res['algorithm_details']['id'], 'Test model')
self.assertTrue(os.path.exists(output_file))
def testModelRunJson(self):
output_file = self.TMP_DIR + '/model_output2.shp'
rc, output, err = self.run_process(['run', TEST_DATA_DIR + '/test_model.model3', '--json', '--', 'FEATS={}'.format(TEST_DATA_DIR + '/polys.shp'), 'native:centroids_1:CENTROIDS={}'.format(output_file)])
@ -320,6 +413,33 @@ class TestQgsProcessExecutable(unittest.TestCase):
self.assertEqual(res['algorithm_details']['id'], 'script:converttouppercase')
self.assertEqual(res['results']['OUTPUT'], 'ABC')
def testScriptRunStdIn(self):
output_file = self.TMP_DIR + '/model_output_stdin.shp'
params = {
'inputs':
{
'INPUT': 'abc def'
}
}
rc, output, err = self.run_process_stdin(['run', TEST_DATA_DIR + '/convert_to_upper.py', '-'], json.dumps(params))
if os.environ.get('TRAVIS', '') != 'true':
# Travis DOES have errors, due to QStandardPaths: XDG_RUNTIME_DIR not set warnings raised by Qt
self.assertFalse(err)
self.assertEqual(rc, 0)
res = json.loads(output)
self.assertIn('gdal_version', res)
self.assertIn('geos_version', res)
self.assertIn('proj_version', res)
self.assertIn('python_version', res)
self.assertIn('qt_version', res)
self.assertIn('qgis_version', res)
self.assertEqual(res['algorithm_details']['id'], 'script:converttouppercase')
self.assertEqual(res['results']['OUTPUT'], 'ABC DEF')
def testPythonScriptRunNotAlgorithm(self):
rc, output, err = self.run_process(['run', TEST_DATA_DIR + '/not_a_processing_script.py'])
self.assertEqual(rc, 1)