From 40ada01de5db93186781e435732bbd716e8aa14f Mon Sep 17 00:00:00 2001 From: Michael Tryby Date: Mon, 25 Feb 2019 17:03:11 -0500 Subject: [PATCH] Improving reg testing helper scripts --- appveyor.yml | 22 +++++++++------ tools/before-test.cmd | 66 +++++++++++++++++++++++++++---------------- tools/gen-config.cmd | 28 ++++++++++++------ tools/run-nrtest.cmd | 35 ++++++++++++----------- 4 files changed, 94 insertions(+), 57 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index c958610..655ab56 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -21,14 +21,19 @@ environment: GENERATOR: "Visual Studio 15 2017" GROUP: "SUPPORTED" BOOST_ROOT: "C:/Libraries/boost_1_67_0" + PLATFORM: "win32" + REF_BUILD_ID: "220dev5" # New build on Visual Studio 15 2017 - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 GENERATOR: "Visual Studio 15 2017 Win64" GROUP: "EXPERIMENTAL" BOOST_ROOT: "C:/Libraries/boost_1_67_0" + PLATFORM: "win64" + REF_BUILD_ID: # called before repo cloning init: + - set SUT_BUILD_ID=%APPVEYOR_BUILD_NUMBER%_%APPVEYOR_JOB_NUMBER% - set EPANET_HOME=%APPVEYOR_BUILD_FOLDER% - set BUILD_HOME=buildprod - set TEST_HOME=nrtestsuite @@ -58,7 +63,7 @@ build_script: before_test: - cd %EPANET_HOME% - - tools\before-test.cmd %APPVEYOR_REPO_COMMIT% + - tools\before-test.cmd %PLATFORM% %REF_BUILD_ID% %SUT_BUILD_ID% %APPVEYOR_REPO_COMMIT% # run custom test script test_script: @@ -67,13 +72,14 @@ test_script: - ctest -C Release # run regression tests - cd %EPANET_HOME% - - tools\run-nrtest.cmd %APPVEYOR_REPO_COMMIT% + - tools\run-nrtest.cmd %REF_BUILD_ID% %SUT_BUILD_ID% -after_test: +on_success: + - cd %TEST_HOME%\benchmark + - appveyor PushArtifact receipt.json + +on_failure: - cd %TEST_HOME%\benchmark # zip up the SUT benchmarks - - 7z a epanet-benchmark.zip \epanet-%APPVEYOR_REPO_COMMIT%\ - -artifacts: - - path: epanet-benchmark.zip - name: epanet-benchmark.zip + - 7z a benchmark-%PLATFORM%-%SUT_BUILD_ID%.zip .\epanet-%SUT_BUILD_ID% + - appveyor PushArtifact benchmark-%PLATFORM%-%SUT_BUILD_ID%.zip diff --git a/tools/before-test.cmd b/tools/before-test.cmd index 30cdc68..4926189 100644 --- a/tools/before-test.cmd +++ b/tools/before-test.cmd @@ -7,8 +7,11 @@ :: US EPA - ORD/NRMRL :: :: Arguments: -:: 1 - build identifier for software under test -:: 2 - (relative path regression test file staging location) +:: 1 - (platform) +:: 2 - (build identifier for reference) +:: 3 - (build identifier for software under test) +:: 4 - (version identifier for software under test) +:: 5 - (relative path regression test file staging location) :: :: Note: :: Tests and benchmark files are stored in the epanet-example-networks repo. @@ -18,36 +21,50 @@ :: @echo off -setlocal +setlocal EnableExtensions -:: CHANGE THESE VARIABLES TO UPDATE BENCHMARK -set EXAMPLES_VER=1.0.2-dev.5 -set BENCHMARK_VER=220dev5 +IF [%1]==[] ( set PLATFORM= +) ELSE ( set "PLATFORM=%~1" ) +IF [%2]==[] ( echo "ERROR: REF_BUILD_ID must be defined" & exit /B 1 +) ELSE (set "REF_BUILD_ID=%~2" ) -set "SCRIPT_HOME=%~dp0" -set "EXE_HOME=Release" +IF [%3]==[] ( set "SUT_BUILD_ID=local" +) ELSE ( set "SUT_BUILD_ID=%~3" ) -::: Determine SUT executable path -:: TODO: This may fail when there is more than one cmake buildprod folder -for /d /r "%SCRIPT_HOME%..\" %%a in (*) do if /i "%%~nxa"=="bin" set "BUILD_HOME=%%a" -set SUT_PATH=%BUILD_HOME%\%EXE_HOME% +IF [%4]==[] (set SUT_VERSION= +) ELSE ( set "SUT_VERSION=%~4" ) -:: Check existence and apply default arguments -IF NOT [%1]==[] ( set "SUT_VER=%~1" -) ELSE ( set "SUT_VER=vXXX" ) - -IF NOT [%2]==[] ( set "TEST_HOME=%~2" -) ELSE ( set "TEST_HOME=nrtestsuite" ) - - -set TESTFILES_URL=https://github.com/OpenWaterAnalytics/epanet-example-networks/archive/v%EXAMPLES_VER%.zip -set BENCHFILES_URL=https://github.com/OpenWaterAnalytics/epanet-example-networks/releases/download/v%EXAMPLES_VER%/epanet-benchmark-%BENCHMARK_VER%.zip +IF [%5]==[] ( set "TEST_HOME=nrtestsuite" +) ELSE ( set "TEST_HOME=%~5" ) echo INFO: Staging files for regression testing + +:: determine SUT executable path +set "SCRIPT_HOME=%~dp0" +:: TODO: This may fail when there is more than one cmake buildprod folder +FOR /D /R "%SCRIPT_HOME%..\" %%a IN (*) DO IF /i "%%~nxa"=="bin" set "BUILD_HOME=%%a" +set "SUT_PATH=%BUILD_HOME%\Release" + + +:: determine platform from CMakeCache.txt +IF NOT DEFINED PLATFORM ( + FOR /F "tokens=*" %%p IN ( 'findstr CMAKE_SHARED_LINKER_FLAGS:STRING %BUILD_HOME%\..\CmakeCache.txt' ) DO ( set "FLAG=%%p" ) + FOR /F "delims=: tokens=3" %%m IN ( 'echo %FLAG%' ) DO IF "%%m"=="x64" ( set "PLATFORM=win64" ) ELSE ( set "PLATFORM=win32" ) +) + +:: hack to determine latest tag in epanet-example-networks repo +:: TODO: use GitHub api instead +set "LATEST_URL=https://github.com/OpenWaterAnalytics/epanet-example-networks/releases/latest" +FOR /F delims^=^"^ tokens^=2 %%g IN ('curl --silent %LATEST_URL%') DO ( set "LATEST_TAG=%%~nxg" ) + +set "TESTFILES_URL=https://github.com/OpenWaterAnalytics/epanet-example-networks/archive/%LATEST_TAG%.zip" +set "BENCHFILES_URL=https://github.com/OpenWaterAnalytics/epanet-example-networks/releases/download/%LATEST_TAG%/benchmark-%PLATFORM%-%REF_BUILD_ID%.zip" + + :: create a clean directory for staging regression tests if exist %TEST_HOME% ( rmdir /s /q %TEST_HOME% @@ -55,6 +72,7 @@ if exist %TEST_HOME% ( mkdir %TEST_HOME% cd %TEST_HOME% + :: retrieve epanet-examples for regression testing curl -fsSL -o examples.zip %TESTFILES_URL% @@ -67,9 +85,9 @@ curl -fsSL -o benchmark.zip %BENCHFILES_URL% 7z x benchmark.zip -obenchmark\ > nul :: set up symlink for tests directory -mklink /D .\tests .\epanet-example-networks-%EXAMPLES_VER%\epanet-tests +mklink /D .\tests .\epanet-example-networks-%LATEST_TAG:~1%\epanet-tests > nul :: generate json configuration file for software under test mkdir apps -%SCRIPT_HOME%\gen-config.cmd %SUT_PATH% > apps\epanet-%SUT_VER%.json +%SCRIPT_HOME%\gen-config.cmd %SUT_PATH% %PLATFORM% %SUT_BUILD_ID% %SUT_VERSION% > apps\epanet-%SUT_BUILD_ID%.json diff --git a/tools/gen-config.cmd b/tools/gen-config.cmd index c59902e..760ac60 100644 --- a/tools/gen-config.cmd +++ b/tools/gen-config.cmd @@ -1,29 +1,41 @@ :: :: gen-config.cmd - Generated nrtest app configuration file for test executable -:: +:: :: Date Created: 1/8/2018 :: :: Author: Michael E. Tryby :: US EPA - ORD/NRMRL :: -:: Arguments: +:: Arguments: :: 1 - absolute path to test executable (valid path seperator for nrtest is "/") -:: +:: 2 - (platform) +:: 3 - (build identifier for SUT) +:: 4 - (commit hash string) @echo off setlocal -:: process path to remove quotes and convert backward to forward slash +:: swmm target created by the cmake build script +set TEST_CMD=runepanet.exe + +:: remove quotes from path and convert backward to forward slash set ABS_BUILD_PATH=%~1 set ABS_BUILD_PATH=%ABS_BUILD_PATH:\=/% -:: this is the target created by the cmake build script -set TEST_CMD=runepanet.exe +IF [%2]==[] ( set "PLATFORM=unknown" +) ELSE ( set "PLATFORM=%~2" ) + +IF [%3]==[] ( set "BUILD_ID=unknown" +) ELSE ( set "BUILD_ID=%~3" ) + +IF [%4]==[] ( set "VERSION=unknown" +) ELSE ( set "VERSION=%~4" ) + echo { echo "name" : "epanet", -echo "version" : "", -echo "description" : "", +echo "version" : "%VERSION%", +echo "description" : "%PLATFORM% %BUILD_ID%", echo "setup_script" : "", echo "exe" : "%ABS_BUILD_PATH%/%TEST_CMD%" echo } diff --git a/tools/run-nrtest.cmd b/tools/run-nrtest.cmd index c39becb..30c7867 100644 --- a/tools/run-nrtest.cmd +++ b/tools/run-nrtest.cmd @@ -7,39 +7,40 @@ :: US EPA - ORD/NRMRL :: :: Arguments: -:: 1 - version/build identifier -:: 2 - (test suite path) +:: 1 - (REF build identifier) +:: 2 - (SUT build identifier) +:: 3 - (test suite path) :: @echo off setlocal -:: CHANGE THIS VARIABLES TO UPDATE BENCHMARK -set BENCHMARK_VER=220dev5 +:: Check existence and apply default arguments +IF [%1]==[] ( echo "ERROR: REF_BUILD_ID must be defined" & exit /B 1 +) ELSE ( set "REF_BUILD_ID=%~1" ) + +IF [%2]==[] ( set "SUT_BUILD_ID=local" +) ELSE ( set "SUT_BUILD_ID=%~2" ) + +IF [%3]==[] ( set "TEST_SUITE_PATH=nrtestsuite" +) ELSE ( set "TEST_SUITE_PATH=%~3" ) -:: Determine location of python Scripts folder +:: determine location of python Scripts folder FOR /F "tokens=*" %%G IN ('where python') DO ( set PYTHON_DIR=%%~dpG ) set "NRTEST_SCRIPT_PATH=%PYTHON_DIR%Scripts" -:: Check existence and apply default arguments -IF NOT [%1]==[] ( set "SUT_VER=%~1" -) ELSE ( set "SUT_VER=vXXX" ) - -IF NOT [%2]==[] ( set "TEST_SUITE_PATH=%~2" -) ELSE ( set "TEST_SUITE_PATH=nrtestsuite" ) - set NRTEST_EXECUTE_CMD=python %NRTEST_SCRIPT_PATH%\nrtest execute -set TEST_APP_PATH=apps\epanet-%SUT_VER%.json +set TEST_APP_PATH=apps\epanet-%SUT_BUILD_ID%.json set TESTS=tests\examples tests\exeter tests\large tests\network_one tests\press_depend tests\small tests\tanks tests\valves -set TEST_OUTPUT_PATH=benchmark\epanet-%SUT_VER% +set TEST_OUTPUT_PATH=benchmark\epanet-%SUT_BUILD_ID% set NRTEST_COMPARE_CMD=python %NRTEST_SCRIPT_PATH%\nrtest compare -set REF_OUTPUT_PATH=benchmark\epanet-%BENCHMARK_VER% +set REF_OUTPUT_PATH=benchmark\epanet-%REF_BUILD_ID% set RTOL_VALUE=0.01 set ATOL_VALUE=0.0 @@ -51,13 +52,13 @@ if exist %TEST_OUTPUT_PATH% ( rmdir /s /q %TEST_OUTPUT_PATH% ) -echo INFO: Creating test benchmark +echo INFO: Creating SUT %SUT_BUILD_ID% artifacts set NRTEST_COMMAND=%NRTEST_EXECUTE_CMD% %TEST_APP_PATH% %TESTS% -o %TEST_OUTPUT_PATH% :: if there is an error exit the script with error value 1 %NRTEST_COMMAND% || exit /B 1 echo. -echo INFO: Comparing test and ref benchmark +echo INFO: Comparing SUT artifacts to REF %REF_BUILD_ID% set NRTEST_COMMAND=%NRTEST_COMPARE_CMD% %TEST_OUTPUT_PATH% %REF_OUTPUT_PATH% --rtol %RTOL_VALUE% --atol %ATOL_VALUE% -o receipt.json %NRTEST_COMMAND%