mirror of
https://github.com/qgis/QGIS.git
synced 2025-10-31 00:06:02 -04:00
433 lines
16 KiB
Python
433 lines
16 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
"""
|
|
***************************************************************************
|
|
OTBTester.py
|
|
---------------------
|
|
Copyright : (C) 2013 by CS Systemes d'information (CS SI)
|
|
Email : otb at c-s dot fr (CS SI)
|
|
Contributors : Julien Malik (CS SI)
|
|
Oscar Picas (CS SI)
|
|
***************************************************************************
|
|
* *
|
|
* This program is free software; you can redistribute it and/or modify *
|
|
* it under the terms of the GNU General Public License as published by *
|
|
* the Free Software Foundation; either version 2 of the License, or *
|
|
* (at your option) any later version. *
|
|
* *
|
|
***************************************************************************
|
|
"""
|
|
__author__ = 'Julien Malik, Oscar Picas'
|
|
__copyright__ = '(C) 2013, CS Systemes d\'information (CS SI)'
|
|
# This will get replaced with a git SHA1 when you do a git archive
|
|
__revision__ = '$Format:%H$'
|
|
|
|
from parsing import parse
|
|
|
|
from string import Template
|
|
import os
|
|
import traceback
|
|
|
|
from ConfigParser import SafeConfigParser
|
|
|
|
from processing.otb.OTBHelper import get_OTB_log
|
|
|
|
|
|
class LowerTemplate(Template):
|
|
|
|
def safe_substitute(self, param):
|
|
ret = super(LowerTemplate, self).safe_substitute(param).lower()
|
|
return ret
|
|
|
|
|
|
class MakefileParser(object):
|
|
|
|
def __init__(self):
|
|
self.maxDiff = None
|
|
self.parser = SafeConfigParser()
|
|
self.parser.read('otbcfg.ini')
|
|
if not os.path.exists('otbcfg.ini'):
|
|
raise Exception("OTB_SOURCE_DIR and OTB_BINARY_DIR must be specified in the file otbcfg.ini")
|
|
|
|
self.root_dir = self.parser.get('otb', 'checkout_dir')
|
|
if not os.path.exists(self.root_dir):
|
|
raise Exception("Check otbcfg.ini : OTB_SOURCE_DIR and OTB_BINARY_DIR must be specified there")
|
|
self.build_dir = self.parser.get('otb', 'build_dir')
|
|
if not os.path.exists(self.build_dir):
|
|
raise Exception("Check otbcfg.ini : OTB_SOURCE_DIR and OTB_BINARY_DIR must be specified there")
|
|
self.logger = get_OTB_log()
|
|
|
|
def test_CMakelists(self):
|
|
provided = {}
|
|
provided["OTB_SOURCE_DIR"] = self.root_dir
|
|
provided["OTB_BINARY_DIR"] = self.build_dir
|
|
provided["OTB_DATA_LARGEINPUT_ROOT"] = os.path.normpath(os.path.join(self.root_dir, "../OTB-Data/Input"))
|
|
|
|
try:
|
|
with open(os.path.join(self.root_dir, "CMakeLists.txt")) as file_input:
|
|
content = file_input.read()
|
|
output = parse(content)
|
|
|
|
defined_paths = [each for each in output if 'Command' in unicode(type(each)) and "FIND_PATH" in each.name]
|
|
the_paths = {key.body[0].contents: [thing.contents for thing in key.body[1:]] for key in defined_paths}
|
|
|
|
the_sets = [each for each in output if 'Command' in unicode(type(each)) and "SET" in each.name.upper()]
|
|
the_sets = {key.body[0].contents: [thing.contents for thing in key.body[1:]] for key in the_sets}
|
|
the_sets = {key: " ".join(the_sets[key]) for key in the_sets}
|
|
|
|
the_strings = set([each.body[-1].contents for each in output if 'Command' in unicode(type(each)) and "STRING" in each.name.upper()])
|
|
|
|
def mini_clean(item):
|
|
if item.startswith('"') and item.endswith('"') and " " not in item:
|
|
return item[1:-1]
|
|
return item
|
|
|
|
the_sets = {key: mini_clean(the_sets[key]) for key in the_sets}
|
|
|
|
def templatize(item):
|
|
if "$" in item:
|
|
return Template(item)
|
|
return item
|
|
|
|
for key in the_sets:
|
|
if key in the_strings:
|
|
the_sets[key] = the_sets[key].lower()
|
|
|
|
the_sets = {key: templatize(the_sets[key]) for key in the_sets}
|
|
|
|
for path in the_paths:
|
|
target_file = the_paths[path][1]
|
|
suggested_paths = []
|
|
if len(the_paths[path]) > 2:
|
|
suggested_paths = the_paths[path][2:]
|
|
|
|
try:
|
|
provided[path] = find_file(target_file)
|
|
except Exception as e:
|
|
for each in suggested_paths:
|
|
st = Template(each)
|
|
pac = os.path.abspath(st.safe_substitute(provided))
|
|
if os.path.exists(pac):
|
|
provided[path] = pac
|
|
break
|
|
|
|
resolve_dict(provided, the_sets)
|
|
provided.update(the_sets)
|
|
|
|
return provided
|
|
except Exception as e:
|
|
traceback.print_exc()
|
|
self.fail(e.message)
|
|
|
|
def add_make(self, previous_context, new_file):
|
|
input = open(new_file).read()
|
|
output = parse(input)
|
|
apps = [each for each in output if 'Command' in unicode(type(each))]
|
|
setcommands = [each for each in apps if 'SET' in each.name.upper()]
|
|
stringcommands = [each for each in apps if 'STRING' in each.name.upper()]
|
|
|
|
environment = previous_context
|
|
|
|
def mini_clean(item):
|
|
if item.startswith('"') and item.endswith('"') and " " not in item:
|
|
return item[1:-1]
|
|
return item
|
|
|
|
new_env = {}
|
|
for command in setcommands:
|
|
key = command.body[0].contents
|
|
ct = " ".join([item.contents for item in command.body[1:]])
|
|
ct = mini_clean(ct)
|
|
|
|
if "$" in ct:
|
|
values = Template(ct)
|
|
else:
|
|
values = ct
|
|
|
|
new_env[key] = values
|
|
|
|
for stringcommand in stringcommands:
|
|
key = stringcommand.body[-1].contents
|
|
ct = stringcommand.body[-2].contents
|
|
ct = mini_clean(ct.lower())
|
|
|
|
if "$" in ct:
|
|
values = LowerTemplate(ct)
|
|
else:
|
|
values = ct
|
|
new_env[key] = values
|
|
|
|
resolve_dict(environment, new_env)
|
|
environment.update(new_env)
|
|
|
|
return environment
|
|
|
|
def get_apps(self, the_makefile, the_dict):
|
|
input = open(the_makefile).read()
|
|
output = parse(input)
|
|
apps = [each for each in output if 'Command' in unicode(type(each))]
|
|
otb_apps = [each for each in apps if 'OTB_TEST_APPLICATION' in each.name.upper()]
|
|
return otb_apps
|
|
|
|
def get_tests(self, the_makefile, the_dict):
|
|
input = open(the_makefile).read()
|
|
output = parse(input)
|
|
apps = [each for each in output if 'Command' in unicode(type(each))]
|
|
otb_tests = [each for each in apps if 'ADD_TEST' in each.name.upper()]
|
|
return otb_tests
|
|
|
|
def get_apps_with_context(self, the_makefile, the_dict):
|
|
input = open(the_makefile).read()
|
|
output = parse(input)
|
|
|
|
def is_a_command(item):
|
|
return 'Command' in unicode(type(item))
|
|
|
|
appz = []
|
|
context = []
|
|
for each in output:
|
|
if is_a_command(each):
|
|
if 'FOREACH' in each.name and 'ENDFOREACH' not in each.name:
|
|
args = [item.contents for item in each.body]
|
|
context.append(args)
|
|
elif 'ENDFOREACH' in each.name:
|
|
context.pop()
|
|
elif 'OTB_TEST_APPLICATION' in each.name.upper():
|
|
appz.append((each, context[:]))
|
|
return appz
|
|
|
|
def get_name_line(self, the_list, the_dict):
|
|
items = ('NAME', 'APP', 'OPTIONS', 'TESTENVOPTIONS', 'VALID')
|
|
itemz = [[], [], [], [], []]
|
|
last_index = 0
|
|
for each in the_list:
|
|
if each.contents in items:
|
|
last_index = items.index(each.contents)
|
|
else:
|
|
itemz[last_index].append(each.contents)
|
|
result = itemz[0][0]
|
|
the_string = Template(result).safe_substitute(the_dict)
|
|
|
|
if '$' in the_string:
|
|
neo_dict = the_dict
|
|
the_string = Template(the_string).safe_substitute(neo_dict)
|
|
while '$' in the_string:
|
|
try:
|
|
the_string = Template(the_string).substitute(neo_dict)
|
|
except KeyError as e:
|
|
self.logger.warning("Key %s is not found in makefiles" % e.message)
|
|
neo_dict[e.message] = ""
|
|
|
|
if 'string.Template' in the_string:
|
|
raise Exception("Unexpected toString call in %s" % the_string)
|
|
|
|
return the_string
|
|
|
|
def get_command_line(self, the_list, the_dict):
|
|
items = ('NAME', 'APP', 'OPTIONS', 'TESTENVOPTIONS', 'VALID')
|
|
itemz = [[], [], [], [], []]
|
|
last_index = 0
|
|
for each in the_list:
|
|
if each.contents in items:
|
|
last_index = items.index(each.contents)
|
|
else:
|
|
itemz[last_index].append(each.contents)
|
|
result = []
|
|
result.extend(["otbcli_%s" % each for each in itemz[1]])
|
|
|
|
if len(result[0]) == 7:
|
|
raise Exception("App name is empty !")
|
|
|
|
result.extend(itemz[2])
|
|
result.append("-testenv")
|
|
result.extend(itemz[3])
|
|
the_string = Template(" ".join(result)).safe_substitute(the_dict)
|
|
|
|
if '$' in the_string:
|
|
neo_dict = the_dict
|
|
the_string = Template(" ".join(result)).safe_substitute(neo_dict)
|
|
while '$' in the_string:
|
|
try:
|
|
the_string = Template(the_string).substitute(neo_dict)
|
|
except KeyError as e:
|
|
self.logger.warning("Key %s is not found in makefiles" % e.message)
|
|
neo_dict[e.message] = ""
|
|
|
|
if 'string.Template' in the_string:
|
|
raise Exception("Unexpected toString call in %s" % the_string)
|
|
|
|
return the_string
|
|
|
|
def get_test(self, the_list, the_dict):
|
|
items = ('NAME', 'APP', 'OPTIONS', 'TESTENVOPTIONS', 'VALID')
|
|
itemz = [[], [], [], [], []]
|
|
last_index = 0
|
|
for each in the_list:
|
|
if each.contents in items:
|
|
last_index = items.index(each.contents)
|
|
else:
|
|
itemz[last_index].append(each.contents)
|
|
result = ["otbTestDriver"]
|
|
result.extend(itemz[4])
|
|
|
|
if len(result) == 1:
|
|
return ""
|
|
|
|
the_string = Template(" ".join(result)).safe_substitute(the_dict)
|
|
|
|
if '$' in the_string:
|
|
neo_dict = the_dict
|
|
the_string = Template(" ".join(result)).safe_substitute(neo_dict)
|
|
while '$' in the_string:
|
|
try:
|
|
the_string = Template(the_string).substitute(neo_dict)
|
|
except KeyError as e:
|
|
self.logger.warning("Key %s is not found in makefiles" % e.message)
|
|
neo_dict[e.message] = ""
|
|
|
|
if 'string.Template' in the_string:
|
|
raise Exception("Unexpected toString call in %s" % the_string)
|
|
|
|
return the_string
|
|
|
|
def test_algos(self):
|
|
tests = {}
|
|
|
|
algos_dir = os.path.join(self.root_dir, "Testing/Applications")
|
|
makefiles = find_files("CMakeLists.txt", algos_dir)
|
|
to_be_excluded = os.path.join(self.root_dir, "Testing/Applications/CMakeLists.txt")
|
|
if to_be_excluded in makefiles:
|
|
makefiles.remove(to_be_excluded)
|
|
|
|
resolve_algos = {}
|
|
for makefile in makefiles:
|
|
intermediate_makefiles = []
|
|
path = makefile.split(os.sep)[len(self.root_dir.split(os.sep)):-1]
|
|
for ind in range(len(path)):
|
|
tmp_path = path[:ind + 1]
|
|
tmp_path.append("CMakeLists.txt")
|
|
tmp_path = os.sep.join(tmp_path)
|
|
candidate_makefile = os.path.join(self.root_dir, tmp_path)
|
|
if os.path.exists(candidate_makefile):
|
|
intermediate_makefiles.append(candidate_makefile)
|
|
resolve_algos[makefile] = intermediate_makefiles
|
|
|
|
dict_for_algo = {}
|
|
for makefile in makefiles:
|
|
basic = self.test_CMakelists()
|
|
last_context = self.add_make(basic, os.path.join(self.root_dir, "Testing/Utilities/CMakeLists.txt"))
|
|
for intermediate_makefile in resolve_algos[makefile]:
|
|
last_context = self.add_make(last_context, intermediate_makefile)
|
|
dict_for_algo[makefile] = last_context
|
|
|
|
for makefile in makefiles:
|
|
appz = self.get_apps_with_context(makefile, dict_for_algo[makefile])
|
|
|
|
for app, context in appz:
|
|
if len(context) == 0:
|
|
import copy
|
|
ddi = copy.deepcopy(dict_for_algo[makefile])
|
|
tk_dict = autoresolve(ddi)
|
|
tk_dict = autoresolve(tk_dict)
|
|
|
|
name_line = self.get_name_line(app.body, tk_dict)
|
|
command_line = self.get_command_line(app.body, tk_dict)
|
|
test_line = self.get_test(app.body, tk_dict)
|
|
|
|
if '$' in test_line or '$' in command_line:
|
|
if '$' in command_line:
|
|
self.logger.error(command_line)
|
|
if '$' in test_line:
|
|
self.logger.warning(test_line)
|
|
else:
|
|
tests[name_line] = (command_line, test_line)
|
|
else:
|
|
contexts = {}
|
|
for iteration in context:
|
|
key = iteration[0]
|
|
values = [each[1:-1].lower() for each in iteration[1:]]
|
|
contexts[key] = values
|
|
|
|
keyorder = contexts.keys()
|
|
import itertools
|
|
pool = [each for each in itertools.product(*contexts.values())]
|
|
|
|
import copy
|
|
for poolinstance in pool:
|
|
neo_dict = copy.deepcopy(dict_for_algo[makefile])
|
|
zipped = zip(keyorder, poolinstance)
|
|
for each in zipped:
|
|
neo_dict[each[0]] = each[1]
|
|
|
|
ak_dict = autoresolve(neo_dict)
|
|
ak_dict = autoresolve(ak_dict)
|
|
ak_dict = autoresolve(ak_dict)
|
|
|
|
ddi = ak_dict
|
|
|
|
name_line = self.get_name_line(app.body, ddi)
|
|
command_line = self.get_command_line(app.body, ddi)
|
|
test_line = self.get_test(app.body, ddi)
|
|
|
|
if '$' in command_line or '$' not in test_line:
|
|
if '$' in command_line:
|
|
self.logger.error(command_line)
|
|
if '$' in test_line:
|
|
self.logger.warning(test_line)
|
|
else:
|
|
tests[name_line] = (command_line, test_line)
|
|
|
|
return tests
|
|
|
|
|
|
def autoresolve(a_dict):
|
|
def as_template(item, b_dict):
|
|
if hasattr(item, 'safe_substitute'):
|
|
return item.safe_substitute(b_dict)
|
|
ate = Template(item)
|
|
return ate.safe_substitute(b_dict)
|
|
templatized = {key: as_template(a_dict[key], a_dict) for key in a_dict.keys()}
|
|
return templatized
|
|
|
|
|
|
def find_file(file_name, base_dir=os.curdir):
|
|
import os
|
|
for root, dirs, files in os.walk(base_dir, topdown=False):
|
|
for name in files:
|
|
if name == file_name:
|
|
return os.path.join(root, name)
|
|
raise Exception("File not found %s" % file_name)
|
|
|
|
|
|
def find_files(file_name, base_dir=os.curdir):
|
|
import os
|
|
result = []
|
|
for root, dirs, files in os.walk(base_dir, topdown=False):
|
|
for name in files:
|
|
if name == file_name:
|
|
result.append(os.path.join(root, name))
|
|
return result
|
|
|
|
|
|
def resolve_dict(adia, adib):
|
|
init = len(adia)
|
|
fin = len(adia) + 1
|
|
|
|
def _resolve_dict(dia, dib):
|
|
for key in dib:
|
|
cand_value = dib[key]
|
|
if hasattr(cand_value, 'safe_substitute'):
|
|
value = cand_value.safe_substitute(dia)
|
|
if isinstance(value, str) and "$" not in value:
|
|
dia[key] = value
|
|
else:
|
|
dia[key] = cand_value
|
|
for key in dia:
|
|
if key in dib:
|
|
del dib[key]
|
|
|
|
while(init != fin):
|
|
init = len(adia)
|
|
_resolve_dict(adia, adib)
|
|
fin = len(adia)
|