diff --git a/.ci/azure-pipelines/azure-pipelines.yml b/.ci/azure-pipelines/azure-pipelines.yml new file mode 100644 index 00000000000..0c8131ca188 --- /dev/null +++ b/.ci/azure-pipelines/azure-pipelines.yml @@ -0,0 +1,237 @@ +variables: + LR: release-3_10 + LTR: release-3_4 + CTEST_CUSTOM_TESTS_IGNORE: "ProcessingGdalAlgorithmsRasterTest;ProcessingGdalAlgorithmsVectorTest;ProcessingGrass7AlgorithmsImageryTest;ProcessingGrass7AlgorithmsRasterTest;ProcessingGrass7AlgorithmsVectorTest;ProcessingGuiTest;ProcessingOtbAlgorithmsTest;ProcessingQgisAlgorithmsTest;ProcessingQgisAlgorithmsTestPt2;ProcessingQgisAlgorithmsTestPt3;ProcessingQgisAlgorithmsTestPt4;ProcessingScriptUtilsTest;PyQgsAnnotation;PyQgsAppStartup;PyQgsAuthManagerOAuth2OWSTest;PyQgsAuthManagerPKIOWSTest;PyQgsAuthManagerPasswordOWSTest;PyQgsAuthManagerProxy;PyQgsAuthSettingsWidget;PyQgsAuxiliaryStorage;PyQgsBlockingNetworkRequest;PyQgsExifTools;PyQgsFileDownloader;PyQgsFileUtils;PyQgsGeometryTest;PyQgsImageCache;PyQgsImportIntoPostGIS;PyQgsLayoutAtlas;PyQgsLayoutLegend;PyQgsLayoutMap;PyQgsLayoutMapGrid;PyQgsMapLayer;PyQgsOGRProvider;PyQgsOGRProviderGpkg;PyQgsOGRProviderSqlite;PyQgsOfflineEditingWFS;PyQgsPalLabelingCanvas;PyQgsPalLabelingLayout;PyQgsPalLabelingPlacement;PyQgsPointDisplacementRenderer;PyQgsProject;PyQgsProviderConnectionGpkg;PyQgsProviderConnectionPostgres;PyQgsPythonProvider;PyQgsRasterFileWriter;PyQgsRasterLayer;PyQgsSelectiveMasking;PyQgsServerAccessControlWMSGetlegendgraphic;PyQgsServerApi;PyQgsServerCacheManager;PyQgsServerLocaleOverride;PyQgsServerSecurity;PyQgsServerSettings;PyQgsServerWMS;PyQgsServerWMSDimension;PyQgsServerWMSGetFeatureInfo;PyQgsServerWMSGetLegendGraphic;PyQgsServerWMSGetMap;PyQgsServerWMSGetPrint;PyQgsServerWMTS;PyQgsSettings;PyQgsShapefileProvider;PyQgsSpatialiteProvider;PyQgsSvgCache;PyQgsSymbolLayer;PyQgsTaskManager;PyQgsTextRenderer;PyQgsVectorFileWriter;PyQgsVectorLayer;PyQgsVectorLayerUtils;PyQgsVirtualLayerProvider;PyQgsWFSProviderGUI;PyQgsZipUtils;qgis_3drenderingtest;qgis_alignrastertest;qgis_banned_keywords;qgis_browsermodeltest;qgis_callouttest;qgis_compositionconvertertest;qgis_datadefinedsizelegendtest;qgis_diagramtest;qgis_doxygen_order;qgis_dxfexporttest;qgis_expressiontest;qgis_filedownloader;qgis_geometrycheckstest;qgis_geometrytest;qgis_geonodeconnectiontest;qgis_grassprovidertest7;qgis_labelingenginetest;qgis_layout3dmaptest;qgis_layouthtmltest;qgis_layoutlabeltest;qgis_layoutmapgridtest;qgis_layoutmaptest;qgis_layoutpicturetest;qgis_layoutscalebartest;qgis_layouttabletest;qgis_legendrenderertest;qgis_licenses;qgis_maprendererjobtest;qgis_maprotationtest;qgis_mapsettingsutilstest;qgis_mimedatautilstest;qgis_networkaccessmanagertest;qgis_openclutilstest;qgis_painteffecttest;qgis_pallabelingtest;qgis_processingtest;qgis_projecttest;qgis_qgisappclipboard;qgis_rasterlayersaveasdialog;qgis_shellcheck;qgis_sip_include;qgis_sip_uptodate;qgis_sipify;qgis_spelling;qgis_styletest;qgis_svgcachetest;qgis_taskmanagertest;qgis_valuerelationwidgetwrapper;qgis_vectorfilewritertest;qgis_wcsprovidertest;qgis_ziplayertest;qgis_arcgisrestutilstest;qgis_coordinatereferencesystemtest;qgis_imagecachetest;qgis_invertedpolygonrenderertest" + Agent.Source.Git.ShallowFetchDepth: 1 + +trigger: + branches: + include: + - master + - $(LR) + - $(LTR) + - azure-pipelines + +pr: +- master +- $(LR) +- $(LTR) + +jobs: +- job: OSGeo4W + pool: + vmImage: vs2015-win2012r2 + timeoutInMinutes: 360 + continueOnError: true + strategy: + maxParallel: 4 + matrix: + x86: + OSGEO4W_ROOT: C:\OSGeo4W + OSGEO4W_ARCH: x86 + CLCACHE_DIR: c:\clcache-x86 + PLATFORM: x86 + CC: C:\OSGeo4W\bin\clcache.bat + CXX: C:\OSGeo4W\bin\clcache.bat + + x86_64: + OSGEO4W_ROOT: C:\OSGeo4W64 + OSGEO4W_ARCH: x86_64 + CLCACHE_DIR: c:\clcache-x86_64 + PLATFORM: x64 + CC: C:\OSGeo4W64\bin\clcache.bat + CXX: C:\OSGeo4W64\bin\clcache.bat + + steps: + - bash: | + echo "BUILD_SOURCEBRANCHNAME: ${BUILD_SOURCEBRANCHNAME}" + echo "LR: ${LR}" + echo "LTR: ${LTR}" + case "${BUILD_SOURCEBRANCHNAME}" in + "${LR}") + echo "##vso[task.setvariable variable=OSGEO4W_PKG]qgis-ltr-dev" + echo "##vso[task.setvariable variable=OSGEO4W_DEPS]qgis-ltr-dev-deps" + ;; + "${LTR}") + echo "##vso[task.setvariable variable=OSGEO4W_PKG]qgis-rel-dev" + echo "##vso[task.setvariable variable=OSGEO4W_DEPS]qgis-rel-dev-deps" + ;; + *) + echo "##vso[task.setvariable variable=OSGEO4W_PKG]qgis-dev" + echo "##vso[task.setvariable variable=OSGEO4W_DEPS]qgis-dev-deps" + ;; + esac + displayName: 'Get package name from branch' + + - script: curl --output c:\setup-x86.exe https://cygwin.com/setup-x86.exe + env: + OSGEO4W_ARCH: $(OSGEO4W_ARCH) + displayName: 'Download cygwin Installer' + + - script: curl --output c:\osgeo4w-setup.exe https://download.osgeo.org/osgeo4w/osgeo4w-setup-%OSGEO4W_ARCH%.exe + env: + OSGEO4W_ARCH: $(OSGEO4W_ARCH) + displayName: 'Download OSGeo4W Installer' + + - script: curl --location-trusted --output c:\cmake.msi https://github.com/Kitware/CMake/releases/download/v3.15.5/cmake-3.15.5-win64-x64.msi + displayName: 'Download CMake Installer' + + - script: curl --location-trusted --output c:\ninja.zip https://github.com/ninja-build/ninja/releases/download/v1.9.0/ninja-win.zip + displayName: 'Download Ninja' + +# - script: curl --location-trusted --output c:\depends.zip http://www.dependencywalker.com/depends22_%PLATFORM%.zip +# displayName: 'Download Dependency walker' +# env: +# PLATFORM: $(PLATFORM) + +# Too large… +# - task: Cache@2 +# inputs: +# key: 'cygwin | $(Date:yyyyMMdd)' +# path: 'c:\cygwin' +# restoreKeys: | +# cygwin | $(Date:yyyyMM) +# cygwin | $(Date:yyyy) +# cygwin +# displayName: Cache cygwin + + - powershell: ms-windows/osgeo4w/runasadmin.ps1 c:\setup-x86.exe -qnNdO -R C:/cygwin -s http://cygwin.mirror.constant.com -l C:/temp/cygwin -P "bison,flex,poppler,doxygen,git,unzip" + displayName: 'Installing cygwin' + +# Too large… +# - task: Cache@2 +# inputs: +# key: 'osgeo4w | $(OSGEO4W_ARCH) | $(Date:yyyyMMdd)' +# path: '$(OSGEO4W_ROOT)' +# restoreKeys: | +# osgeo4w | $(OSGEO4W_ARCH) | $(Date:yyyyMMdd) +# osgeo4w | $(OSGEO4W_ARCH) | $(Date:yyyyMM) +# osgeo4w | $(OSGEO4W_ARCH) | $(Date:yyyy) +# osgeo4w | $(OSGEO4W_ARCH) +# displayName: Cache OSGeo4W + + - powershell: ms-windows/osgeo4w/runasadmin.ps1 c:\osgeo4w-setup.exe --autoaccept --advanced --arch $env:OSGEO4W_ARCH --quiet-mode --upgrade-also --root $env:OSGEO4W_ROOT --only-site -s http://download.osgeo.org/osgeo4w -l c:\temp\osgeo4w -P $env:OSGEO4W_DEPS -P python3-clcache + env: + OSGEO4W_ARCH: $(OSGEO4W_ARCH) + OSGEO4W_ROOT: $(OSGEO4W_ROOT) + OSGEO4W_DEPS: $(OSGEO4W_DEPS) + displayName: 'Installing OSGeo4W' + + - script: | + rmdir /s /q c:\temp\cygwin + rmdir /s /q c:\temp\osgeo4w + env: + OSGEO4W_ROOT: $(OSGEO4W_ROOT) + displayName: 'Clear package caches' + + - powershell: ms-windows/osgeo4w/runasadmin.ps1 msiexec.exe /X C:\Windows\Installer\146218.msi /qn /norestart /l*v c:\cmake-uninstall.log + displayName: 'Uninstalling old CMake' + + - powershell: ms-windows/osgeo4w/runasadmin.ps1 msiexec.exe /I c:\cmake.msi /qn /norestart /l*v c:\cmake-install.log + displayName: 'Installing CMake' + + - script: c:\cygwin\bin\unzip -o c:\ninja.zip -d %OSGEO4W_ROOT%\bin + displayName: 'Extracting Ninja' + env: + OSGEO4W_ROOT: $(OSGEO4W_ROOT) + +# - script: c:\cygwin\bin\unzip -o c:\depends.zip -d %OSGEO4W_ROOT%\bin +# displayName: 'Extracting Dependency Walker' +# env: +# OSGEO4W_ROOT: $(OSGEO4W_ROOT) + + - script: | + PATH %OSGEO4W_ROOT%\bin;%ProgramFiles%\CMake\bin;%PATH% + cmake --version + ctest --version + ninja --version + displayName: 'Display tool versions' + env: + OSGEO4W_ROOT: $(OSGEO4W_ROOT) + + - script: curl --location-trusted --output c:\LATEST.sha https://download.osgeo.org/osgeo4w/$(OSGEO4W_ARCH)/release/qgis/$(OSGEO4W_PKG)/LATEST.sha + displayName: 'Download LATEST.sha' + env: + OSGEO4W_ARCH: $(OSGEO4W_ARCH) + OSGEO4W_PKG: $(OSGEO4W_PKG) + +# Too large… +# - task: Cache@2 +# inputs: +# key: 'clcache | $(OSGEO4W_ARCH) | $(OSGEO4W_PKG) | $(Date:yyyyMMdd) | $(Hours)' +# path: '$(CLCACHE_DIR)' +# restoreKeys: | +# clcache | $(OSGEO4W_ARCH) | $(OSGEO4W_PKG) | $(Date:yyyyMMdd) | $(Hours) +# clcache | $(OSGEO4W_ARCH) | $(OSGEO4W_PKG) | $(Date:yyyyMMdd) +# clcache | $(OSGEO4W_ARCH) | $(OSGEO4W_PKG) | $(Date:yyyyMM) +# clcache | $(OSGEO4W_ARCH) | $(OSGEO4W_PKG) | $(Date:yyyy) +# clcache | $(OSGEO4W_ARCH) | $(OSGEO4W_PKG) +# displayName: Cache clcache + + - script: | + echo on + PATH c:\cygwin\bin;%OSGEO4W_ROOT%\bin;%PATH% + sed -ne 's/^SET(CPACK_PACKAGE_VERSION_MAJOR "\([0-9]*\)")\s*$/set major=\1/p' CMakeLists.txt >version.cmd + sed -ne 's/^SET(CPACK_PACKAGE_VERSION_MINOR "\([0-9]*\)")\s*$/set minor=\1/p' CMakeLists.txt >>version.cmd + sed -ne 's/^SET(CPACK_PACKAGE_VERSION_PATCH "\([0-9]*\)")\s*$/set patch=\1/p' CMakeLists.txt >>version.cmd + sed -ne 's/^\([0-9]*\):.*$/set binary=\1/p' c:\latest.sha >>version.cmd + call version.cmd + del version.cmd + set /A binary=%binary%+1 + cd ms-windows\osgeo4w + touch skippackage + set OSGEO4W_CXXFLAGS=/MD /MP /Od /D NDEBUG + package-nightly.cmd %major%.%minor%.%patch% %binary% %OSGEO4W_PKG% %OSGEO4W_ARCH% %REPO_COMMIT:~0,10% azure-pipelines + env: + OSGEO4W_ROOT: $(OSGEO4W_ROOT) + OSGEO4W_ARCH: $(OSGEO4W_ARCH) + OSGEO4W_PKG: $(OSGEO4W_PKG) + BUILD_NUMBER: $(Build.BuildId) + CTEST_CUSTOM_TESTS_IGNORE: $(CTEST_CUSTOM_TESTS_IGNORE) + REPO_COMMIT: $(Build.SourceVersion) + CLCACHE_DIR: $(CLCACHE_DIR) + TARGET: Experimental + CC: $(CC) + CXX: $(CXX) + displayName: 'Building QGIS' + +# - script: | +# echo on +# PATH %OSGEO4W_ROOT%\bin;%PATH% +# cd ms-windows\osgeo4w\build-%OSGEO4W_PKG%-%OSGEO4W_ARCH% +# set /P tag=" %BUILDDIR%\Testing\%tag%\Build.xml >nul +if not errorlevel 1 ( + cmake --build %BUILDDIR% --target %TARGET%Submit --config %BUILDCONF% if errorlevel 1 echo SUBMITTING BUILD ERRORS WAS NOT SUCCESSFUL. - echo build failed twice + echo build failed goto error ) @@ -219,18 +200,19 @@ for %%g IN (%GRASS_VERSIONS%) do ( PATH %path%;%BUILDDIR%\output\plugins set QT_PLUGIN_PATH=%BUILDDIR%\output\plugins;%OSGEO4W_ROOT%\apps\qt5\plugins -cmake --build %BUILDDIR% --target NightlyTest --config %BUILDCONF% +cmake --build %BUILDDIR% --target %TARGET%Test --config %BUILDCONF% if errorlevel 1 echo TESTS WERE NOT SUCCESSFUL. -:skiptests - set TEMP=%oldtemp% set TMP=%oldtmp% PATH %oldpath% -cmake --build %BUILDDIR% --target NightlySubmit --config %BUILDCONF% +cmake --build %BUILDDIR% --target %TARGET%Submit --config %BUILDCONF% if errorlevel 1 echo TEST SUBMISSION WAS NOT SUCCESSFUL. +:skiptests +if exist ..\skippackage goto end + if exist "%PKGDIR%" ( echo REMOVE: %DATE% %TIME% rmdir /s /q "%PKGDIR%" diff --git a/ms-windows/osgeo4w/package.cmd b/ms-windows/osgeo4w/package.cmd index 0c25a3bca1a..798c5eb01ca 100644 --- a/ms-windows/osgeo4w/package.cmd +++ b/ms-windows/osgeo4w/package.cmd @@ -38,33 +38,15 @@ call msvc-env.bat %ARCH% set O4W_ROOT=%OSGEO4W_ROOT:\=/% set LIB_DIR=%O4W_ROOT% -if "%ARCH%"=="x86" goto cmake_x86 -goto cmake_x86_64 +if "%ARCH%"=="x86" ( + set CMAKE_OPT=^ + -D SPATIALINDEX_LIBRARY=%O4W_ROOT%/lib/spatialindex-32.lib +) else ( + set CMAKE_OPT=^ + -D SPATIALINDEX_LIBRARY=%O4W_ROOT%/lib/spatialindex-64.lib ^ + -D CMAKE_INSTALL_SYSTEM_RUNTIME_LIBS_NO_WARNINGS=TRUE +) -:cmake_x86 -set CMAKE_COMPILER_PATH=%PF86%\Microsoft Visual Studio 14.0\VC\bin -set DBGHLP_PATH=%PF86%\Microsoft Visual Studio 14.0\Common7\IDE\Remote Debugger\x86 -set SETUPAPI_LIBRARY=%PF86%\Windows Kits\10\Lib\10.0.14393.0\um\x86\SetupAPI.Lib -if not exist "%SETUPAPI_LIBRARY%" set SETUPAPI_LIBRARY=%PF86%\Windows Kits\8.0\Lib\win8\um\x86\SetupAPI.Lib -if not exist "%SETUPAPI_LIBRARY%" (echo SETUPAPI_LIBRARY not found & goto error) - -set CMAKE_OPT=^ - -D SPATIALINDEX_LIBRARY=%O4W_ROOT%/lib/spatialindex-32.lib -goto cmake - -:cmake_x86_64 -set CMAKE_COMPILER_PATH=%PF86%\Microsoft Visual Studio 14.0\VC\bin\amd64 -set DBGHLP_PATH=%PF86%\Microsoft Visual Studio 14.0\Common7\IDE\Remote Debugger\x64 -set SETUPAPI_LIBRARY=%PF86%\Windows Kits\10\Lib\10.0.14393.0\um\x64\SetupAPI.Lib -if not exist "%SETUPAPI_LIBRARY%" set SETUPAPI_LIBRARY=%PF86%\Windows Kits\8.0\Lib\win8\um\x64\SetupAPI.Lib -if not exist "%SETUPAPI_LIBRARY%" (echo SETUPAPI_LIBRARY not found & goto error) - -set CMAKE_OPT=^ - -D SPATIALINDEX_LIBRARY=%O4W_ROOT%/lib/spatialindex-64.lib ^ - -D SETUPAPI_LIBRARY="%SETUPAPI_LIBRARY%" ^ - -D CMAKE_INSTALL_SYSTEM_RUNTIME_LIBS_NO_WARNINGS=TRUE - -:cmake for %%i in ("%GRASS_PREFIX%") do set GRASS7_VERSION=%%~nxi set GRASS_VERSIONS=%GRASS7_VERSION% @@ -72,8 +54,6 @@ set TAR=tar.exe if exist "c:\cygwin\bin\tar.exe" set TAR=c:\cygwin\bin\tar.exe if exist "c:\cygwin64\bin\tar.exe" set TAR=c:\cygwin64\bin\tar.exe -PROMPT qgis%VERSION%$g - set BUILDCONF=Release cd ..\.. @@ -125,13 +105,20 @@ echo CMAKE: %DATE% %TIME% if errorlevel 1 goto error if "%CMAKEGEN%"=="" set CMAKEGEN=Ninja +if "%CC%"=="" set CC="%CMAKE_COMPILER_PATH:\=/%/cl.exe" +if "%CXX%"=="" set CXX="%CMAKE_COMPILER_PATH:\=/%/cl.exe" +if "%OSGEO4W_CXXFLAGS%"=="" set OSGEO4W_CXXFLAGS=/MD /Z7 /MP /O2 /Ob2 /D NDEBUG for %%i in (%PYTHONHOME%) do set PYVER=%%~ni cmake -G "%CMAKEGEN%" ^ - -D CMAKE_CXX_COMPILER="%CMAKE_COMPILER_PATH:\=/%/cl.exe" ^ - -D CMAKE_C_COMPILER="%CMAKE_COMPILER_PATH:\=/%/cl.exe" ^ + -D CMAKE_CXX_COMPILER="%CXX:\=/%" ^ + -D CMAKE_C_COMPILER="%CC:\=/%" ^ -D CMAKE_LINKER="%CMAKE_COMPILER_PATH:\=/%/link.exe" ^ + -D CMAKE_CXX_FLAGS_RELEASE="%OSGEO4W_CXXFLAGS%" ^ + -D CMAKE_PDB_OUTPUT_DIRECTORY_RELEASE=%BUILDDIR%\apps\%PACKAGENAME%\pdb ^ + -D CMAKE_SHARED_LINKER_FLAGS_RELEASE="/INCREMENTAL:NO /DEBUG /OPT:REF /OPT:ICF" ^ + -D CMAKE_MODULE_LINKER_FLAGS_RELEASE="/INCREMENTAL:NO /DEBUG /OPT:REF /OPT:ICF" ^ -D BUILDNAME="%PACKAGENAME%-%VERSION%%SHA%-Release-VC14-%ARCH%" ^ -D SITE="%SITE%" ^ -D PEDANTIC=TRUE ^ @@ -145,12 +132,9 @@ cmake -G "%CMAKEGEN%" ^ -D WITH_GLOBE=FALSE ^ -D WITH_ORACLE=TRUE ^ -D WITH_CUSTOM_WIDGETS=TRUE ^ - -D CMAKE_CXX_FLAGS_RELEASE="/MD /Zi /MP /O2 /Ob2 /D NDEBUG" ^ - -D CMAKE_SHARED_LINKER_FLAGS_RELEASE="/INCREMENTAL:NO /DEBUG /OPT:REF /OPT:ICF" ^ - -D CMAKE_MODULE_LINKER_FLAGS_RELEASE="/INCREMENTAL:NO /DEBUG /OPT:REF /OPT:ICF" ^ - -D CMAKE_PDB_OUTPUT_DIRECTORY_RELEASE=%BUILDDIR%\apps\%PACKAGENAME%\pdb ^ -D CMAKE_BUILD_TYPE=%BUILDCONF% ^ -D CMAKE_CONFIGURATION_TYPES=%BUILDCONF% ^ + -D SETUPAPI_LIBRARY="%SETUPAPI_LIBRARY%" ^ -D GEOS_LIBRARY=%O4W_ROOT%/lib/geos_c.lib ^ -D SQLITE3_LIBRARY=%O4W_ROOT%/lib/sqlite3_i.lib ^ -D SPATIALITE_LIBRARY=%O4W_ROOT%/lib/spatialite_i.lib ^ @@ -183,8 +167,7 @@ if errorlevel 1 (echo clean failed & goto error) if exist ..\skipbuild (echo skip build & goto skipbuild) echo ALL_BUILD: %DATE% %TIME% cmake --build %BUILDDIR% --config %BUILDCONF% -if errorlevel 1 cmake --build %BUILDDIR% --config %BUILDCONF% -if errorlevel 1 (echo build failed twice & goto error) +if errorlevel 1 (echo build failed & goto error) :skipbuild if exist ..\skiptests goto skiptests @@ -217,6 +200,7 @@ set TMP=%oldtmp% PATH %oldpath% :skiptests +if exist ..\skippackage goto end if exist "%PKGDIR%" ( echo REMOVE: %DATE% %TIME% diff --git a/ms-windows/osgeo4w/runasadmin.ps1 b/ms-windows/osgeo4w/runasadmin.ps1 new file mode 100644 index 00000000000..ab2b4fca35a --- /dev/null +++ b/ms-windows/osgeo4w/runasadmin.ps1 @@ -0,0 +1,4 @@ +#-RunAsAdministrator +Write-Output ($args -join ' ') +$cmd, $args = $args +Start-Process $cmd -Wait -ArgumentList $args -NoNewWindow