Improving reg testing helper scripts
This commit is contained in:
Michael Tryby
22
appveyor.yml
22
appveyor.yml
@@ -21,14 +21,19 @@ environment:
|
||||
GENERATOR: "Visual Studio 15 2017"
|
||||
GROUP: "SUPPORTED"
|
||||
BOOST_ROOT: "C:/Libraries/boost_1_67_0"
|
||||
PLATFORM: "win32"
|
||||
REF_BUILD_ID: "220dev5"
|
||||
# New build on Visual Studio 15 2017
|
||||
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
|
||||
GENERATOR: "Visual Studio 15 2017 Win64"
|
||||
GROUP: "EXPERIMENTAL"
|
||||
BOOST_ROOT: "C:/Libraries/boost_1_67_0"
|
||||
PLATFORM: "win64"
|
||||
REF_BUILD_ID:
|
||||
|
||||
# called before repo cloning
|
||||
init:
|
||||
- set SUT_BUILD_ID=%APPVEYOR_BUILD_NUMBER%_%APPVEYOR_JOB_NUMBER%
|
||||
- set EPANET_HOME=%APPVEYOR_BUILD_FOLDER%
|
||||
- set BUILD_HOME=buildprod
|
||||
- set TEST_HOME=nrtestsuite
|
||||
@@ -58,7 +63,7 @@ build_script:
|
||||
|
||||
before_test:
|
||||
- cd %EPANET_HOME%
|
||||
- tools\before-test.cmd %APPVEYOR_REPO_COMMIT%
|
||||
- tools\before-test.cmd %PLATFORM% %REF_BUILD_ID% %SUT_BUILD_ID% %APPVEYOR_REPO_COMMIT%
|
||||
|
||||
# run custom test script
|
||||
test_script:
|
||||
@@ -67,13 +72,14 @@ test_script:
|
||||
- ctest -C Release
|
||||
# run regression tests
|
||||
- cd %EPANET_HOME%
|
||||
- tools\run-nrtest.cmd %APPVEYOR_REPO_COMMIT%
|
||||
- tools\run-nrtest.cmd %REF_BUILD_ID% %SUT_BUILD_ID%
|
||||
|
||||
after_test:
|
||||
on_success:
|
||||
- cd %TEST_HOME%\benchmark
|
||||
- appveyor PushArtifact receipt.json
|
||||
|
||||
on_failure:
|
||||
- cd %TEST_HOME%\benchmark
|
||||
# zip up the SUT benchmarks
|
||||
- 7z a epanet-benchmark.zip \epanet-%APPVEYOR_REPO_COMMIT%\
|
||||
|
||||
artifacts:
|
||||
- path: epanet-benchmark.zip
|
||||
name: epanet-benchmark.zip
|
||||
- 7z a benchmark-%PLATFORM%-%SUT_BUILD_ID%.zip .\epanet-%SUT_BUILD_ID%
|
||||
- appveyor PushArtifact benchmark-%PLATFORM%-%SUT_BUILD_ID%.zip
|
||||
|
||||
@@ -7,8 +7,11 @@
|
||||
:: US EPA - ORD/NRMRL
|
||||
::
|
||||
:: Arguments:
|
||||
:: 1 - build identifier for software under test
|
||||
:: 2 - (relative path regression test file staging location)
|
||||
:: 1 - (platform)
|
||||
:: 2 - (build identifier for reference)
|
||||
:: 3 - (build identifier for software under test)
|
||||
:: 4 - (version identifier for software under test)
|
||||
:: 5 - (relative path regression test file staging location)
|
||||
::
|
||||
:: Note:
|
||||
:: Tests and benchmark files are stored in the epanet-example-networks repo.
|
||||
@@ -18,36 +21,50 @@
|
||||
::
|
||||
|
||||
@echo off
|
||||
setlocal
|
||||
setlocal EnableExtensions
|
||||
|
||||
|
||||
:: CHANGE THESE VARIABLES TO UPDATE BENCHMARK
|
||||
set EXAMPLES_VER=1.0.2-dev.5
|
||||
set BENCHMARK_VER=220dev5
|
||||
IF [%1]==[] ( set PLATFORM=
|
||||
) ELSE ( set "PLATFORM=%~1" )
|
||||
|
||||
IF [%2]==[] ( echo "ERROR: REF_BUILD_ID must be defined" & exit /B 1
|
||||
) ELSE (set "REF_BUILD_ID=%~2" )
|
||||
|
||||
set "SCRIPT_HOME=%~dp0"
|
||||
set "EXE_HOME=Release"
|
||||
IF [%3]==[] ( set "SUT_BUILD_ID=local"
|
||||
) ELSE ( set "SUT_BUILD_ID=%~3" )
|
||||
|
||||
::: Determine SUT executable path
|
||||
:: TODO: This may fail when there is more than one cmake buildprod folder
|
||||
for /d /r "%SCRIPT_HOME%..\" %%a in (*) do if /i "%%~nxa"=="bin" set "BUILD_HOME=%%a"
|
||||
set SUT_PATH=%BUILD_HOME%\%EXE_HOME%
|
||||
IF [%4]==[] (set SUT_VERSION=
|
||||
) ELSE ( set "SUT_VERSION=%~4" )
|
||||
|
||||
:: Check existence and apply default arguments
|
||||
IF NOT [%1]==[] ( set "SUT_VER=%~1"
|
||||
) ELSE ( set "SUT_VER=vXXX" )
|
||||
|
||||
IF NOT [%2]==[] ( set "TEST_HOME=%~2"
|
||||
) ELSE ( set "TEST_HOME=nrtestsuite" )
|
||||
|
||||
|
||||
set TESTFILES_URL=https://github.com/OpenWaterAnalytics/epanet-example-networks/archive/v%EXAMPLES_VER%.zip
|
||||
set BENCHFILES_URL=https://github.com/OpenWaterAnalytics/epanet-example-networks/releases/download/v%EXAMPLES_VER%/epanet-benchmark-%BENCHMARK_VER%.zip
|
||||
IF [%5]==[] ( set "TEST_HOME=nrtestsuite"
|
||||
) ELSE ( set "TEST_HOME=%~5" )
|
||||
|
||||
|
||||
echo INFO: Staging files for regression testing
|
||||
|
||||
|
||||
:: determine SUT executable path
|
||||
set "SCRIPT_HOME=%~dp0"
|
||||
:: TODO: This may fail when there is more than one cmake buildprod folder
|
||||
FOR /D /R "%SCRIPT_HOME%..\" %%a IN (*) DO IF /i "%%~nxa"=="bin" set "BUILD_HOME=%%a"
|
||||
set "SUT_PATH=%BUILD_HOME%\Release"
|
||||
|
||||
|
||||
:: determine platform from CMakeCache.txt
|
||||
IF NOT DEFINED PLATFORM (
|
||||
FOR /F "tokens=*" %%p IN ( 'findstr CMAKE_SHARED_LINKER_FLAGS:STRING %BUILD_HOME%\..\CmakeCache.txt' ) DO ( set "FLAG=%%p" )
|
||||
FOR /F "delims=: tokens=3" %%m IN ( 'echo %FLAG%' ) DO IF "%%m"=="x64" ( set "PLATFORM=win64" ) ELSE ( set "PLATFORM=win32" )
|
||||
)
|
||||
|
||||
:: hack to determine latest tag in epanet-example-networks repo
|
||||
:: TODO: use GitHub api instead
|
||||
set "LATEST_URL=https://github.com/OpenWaterAnalytics/epanet-example-networks/releases/latest"
|
||||
FOR /F delims^=^"^ tokens^=2 %%g IN ('curl --silent %LATEST_URL%') DO ( set "LATEST_TAG=%%~nxg" )
|
||||
|
||||
set "TESTFILES_URL=https://github.com/OpenWaterAnalytics/epanet-example-networks/archive/%LATEST_TAG%.zip"
|
||||
set "BENCHFILES_URL=https://github.com/OpenWaterAnalytics/epanet-example-networks/releases/download/%LATEST_TAG%/benchmark-%PLATFORM%-%REF_BUILD_ID%.zip"
|
||||
|
||||
|
||||
:: create a clean directory for staging regression tests
|
||||
if exist %TEST_HOME% (
|
||||
rmdir /s /q %TEST_HOME%
|
||||
@@ -55,6 +72,7 @@ if exist %TEST_HOME% (
|
||||
mkdir %TEST_HOME%
|
||||
cd %TEST_HOME%
|
||||
|
||||
|
||||
:: retrieve epanet-examples for regression testing
|
||||
curl -fsSL -o examples.zip %TESTFILES_URL%
|
||||
|
||||
@@ -67,9 +85,9 @@ curl -fsSL -o benchmark.zip %BENCHFILES_URL%
|
||||
7z x benchmark.zip -obenchmark\ > nul
|
||||
|
||||
:: set up symlink for tests directory
|
||||
mklink /D .\tests .\epanet-example-networks-%EXAMPLES_VER%\epanet-tests
|
||||
mklink /D .\tests .\epanet-example-networks-%LATEST_TAG:~1%\epanet-tests > nul
|
||||
|
||||
|
||||
:: generate json configuration file for software under test
|
||||
mkdir apps
|
||||
%SCRIPT_HOME%\gen-config.cmd %SUT_PATH% > apps\epanet-%SUT_VER%.json
|
||||
%SCRIPT_HOME%\gen-config.cmd %SUT_PATH% %PLATFORM% %SUT_BUILD_ID% %SUT_VERSION% > apps\epanet-%SUT_BUILD_ID%.json
|
||||
|
||||
@@ -1,29 +1,41 @@
|
||||
::
|
||||
:: gen-config.cmd - Generated nrtest app configuration file for test executable
|
||||
::
|
||||
::
|
||||
:: Date Created: 1/8/2018
|
||||
::
|
||||
:: Author: Michael E. Tryby
|
||||
:: US EPA - ORD/NRMRL
|
||||
::
|
||||
:: Arguments:
|
||||
:: Arguments:
|
||||
:: 1 - absolute path to test executable (valid path seperator for nrtest is "/")
|
||||
::
|
||||
:: 2 - (platform)
|
||||
:: 3 - (build identifier for SUT)
|
||||
:: 4 - (commit hash string)
|
||||
|
||||
@echo off
|
||||
setlocal
|
||||
|
||||
:: process path to remove quotes and convert backward to forward slash
|
||||
:: swmm target created by the cmake build script
|
||||
set TEST_CMD=runepanet.exe
|
||||
|
||||
:: remove quotes from path and convert backward to forward slash
|
||||
set ABS_BUILD_PATH=%~1
|
||||
set ABS_BUILD_PATH=%ABS_BUILD_PATH:\=/%
|
||||
|
||||
:: this is the target created by the cmake build script
|
||||
set TEST_CMD=runepanet.exe
|
||||
IF [%2]==[] ( set "PLATFORM=unknown"
|
||||
) ELSE ( set "PLATFORM=%~2" )
|
||||
|
||||
IF [%3]==[] ( set "BUILD_ID=unknown"
|
||||
) ELSE ( set "BUILD_ID=%~3" )
|
||||
|
||||
IF [%4]==[] ( set "VERSION=unknown"
|
||||
) ELSE ( set "VERSION=%~4" )
|
||||
|
||||
|
||||
echo {
|
||||
echo "name" : "epanet",
|
||||
echo "version" : "",
|
||||
echo "description" : "",
|
||||
echo "version" : "%VERSION%",
|
||||
echo "description" : "%PLATFORM% %BUILD_ID%",
|
||||
echo "setup_script" : "",
|
||||
echo "exe" : "%ABS_BUILD_PATH%/%TEST_CMD%"
|
||||
echo }
|
||||
|
||||
@@ -7,39 +7,40 @@
|
||||
:: US EPA - ORD/NRMRL
|
||||
::
|
||||
:: Arguments:
|
||||
:: 1 - version/build identifier
|
||||
:: 2 - (test suite path)
|
||||
:: 1 - (REF build identifier)
|
||||
:: 2 - (SUT build identifier)
|
||||
:: 3 - (test suite path)
|
||||
::
|
||||
|
||||
@echo off
|
||||
setlocal
|
||||
|
||||
|
||||
:: CHANGE THIS VARIABLES TO UPDATE BENCHMARK
|
||||
set BENCHMARK_VER=220dev5
|
||||
:: Check existence and apply default arguments
|
||||
IF [%1]==[] ( echo "ERROR: REF_BUILD_ID must be defined" & exit /B 1
|
||||
) ELSE ( set "REF_BUILD_ID=%~1" )
|
||||
|
||||
IF [%2]==[] ( set "SUT_BUILD_ID=local"
|
||||
) ELSE ( set "SUT_BUILD_ID=%~2" )
|
||||
|
||||
IF [%3]==[] ( set "TEST_SUITE_PATH=nrtestsuite"
|
||||
) ELSE ( set "TEST_SUITE_PATH=%~3" )
|
||||
|
||||
|
||||
:: Determine location of python Scripts folder
|
||||
:: determine location of python Scripts folder
|
||||
FOR /F "tokens=*" %%G IN ('where python') DO (
|
||||
set PYTHON_DIR=%%~dpG
|
||||
)
|
||||
set "NRTEST_SCRIPT_PATH=%PYTHON_DIR%Scripts"
|
||||
|
||||
:: Check existence and apply default arguments
|
||||
IF NOT [%1]==[] ( set "SUT_VER=%~1"
|
||||
) ELSE ( set "SUT_VER=vXXX" )
|
||||
|
||||
IF NOT [%2]==[] ( set "TEST_SUITE_PATH=%~2"
|
||||
) ELSE ( set "TEST_SUITE_PATH=nrtestsuite" )
|
||||
|
||||
|
||||
set NRTEST_EXECUTE_CMD=python %NRTEST_SCRIPT_PATH%\nrtest execute
|
||||
set TEST_APP_PATH=apps\epanet-%SUT_VER%.json
|
||||
set TEST_APP_PATH=apps\epanet-%SUT_BUILD_ID%.json
|
||||
set TESTS=tests\examples tests\exeter tests\large tests\network_one tests\press_depend tests\small tests\tanks tests\valves
|
||||
set TEST_OUTPUT_PATH=benchmark\epanet-%SUT_VER%
|
||||
set TEST_OUTPUT_PATH=benchmark\epanet-%SUT_BUILD_ID%
|
||||
|
||||
set NRTEST_COMPARE_CMD=python %NRTEST_SCRIPT_PATH%\nrtest compare
|
||||
set REF_OUTPUT_PATH=benchmark\epanet-%BENCHMARK_VER%
|
||||
set REF_OUTPUT_PATH=benchmark\epanet-%REF_BUILD_ID%
|
||||
set RTOL_VALUE=0.01
|
||||
set ATOL_VALUE=0.0
|
||||
|
||||
@@ -51,13 +52,13 @@ if exist %TEST_OUTPUT_PATH% (
|
||||
rmdir /s /q %TEST_OUTPUT_PATH%
|
||||
)
|
||||
|
||||
echo INFO: Creating test benchmark
|
||||
echo INFO: Creating SUT %SUT_BUILD_ID% artifacts
|
||||
set NRTEST_COMMAND=%NRTEST_EXECUTE_CMD% %TEST_APP_PATH% %TESTS% -o %TEST_OUTPUT_PATH%
|
||||
:: if there is an error exit the script with error value 1
|
||||
%NRTEST_COMMAND% || exit /B 1
|
||||
|
||||
echo.
|
||||
|
||||
echo INFO: Comparing test and ref benchmark
|
||||
echo INFO: Comparing SUT artifacts to REF %REF_BUILD_ID%
|
||||
set NRTEST_COMMAND=%NRTEST_COMPARE_CMD% %TEST_OUTPUT_PATH% %REF_OUTPUT_PATH% --rtol %RTOL_VALUE% --atol %ATOL_VALUE% -o receipt.json
|
||||
%NRTEST_COMMAND%
|
||||
|
||||
Reference in New Issue
Block a user