Improving reg testing helper scripts

tools/before-test.cmd

@@ -7,8 +7,11 @@
 ::          US EPA - ORD/NRMRL
 ::
 ::  Arguments:
-::    1 - build identifier for software under test
-::    2 - (relative path regression test file staging location)
+::    1 - (platform)
+::    2 - (build identifier for reference)
+::    3 - (build identifier for software under test)
+::    4 - (version identifier for software under test)
+::    5 - (relative path regression test file staging location)
 ::
 ::  Note:
 ::    Tests and benchmark files are stored in the epanet-example-networks repo.
@@ -18,36 +21,50 @@
 ::
 
 @echo off
-setlocal
+setlocal EnableExtensions
 
 
-:: CHANGE THESE VARIABLES TO UPDATE BENCHMARK
-set EXAMPLES_VER=1.0.2-dev.5
-set BENCHMARK_VER=220dev5
+IF [%1]==[] ( set PLATFORM=
+) ELSE ( set "PLATFORM=%~1" )
 
+IF [%2]==[] ( echo "ERROR: REF_BUILD_ID must be defined" & exit /B 1
+) ELSE (set "REF_BUILD_ID=%~2" )
 
-set "SCRIPT_HOME=%~dp0"
-set "EXE_HOME=Release"
+IF [%3]==[] ( set "SUT_BUILD_ID=local"
+) ELSE ( set "SUT_BUILD_ID=%~3" )
 
-::: Determine SUT executable path
-:: TODO: This may fail when there is more than one cmake buildprod folder
-for /d /r "%SCRIPT_HOME%..\" %%a in (*) do if /i "%%~nxa"=="bin" set "BUILD_HOME=%%a"
-set SUT_PATH=%BUILD_HOME%\%EXE_HOME%
+IF [%4]==[] (set SUT_VERSION=
+) ELSE ( set "SUT_VERSION=%~4" )
 
-:: Check existence and apply default arguments
-IF NOT [%1]==[] ( set "SUT_VER=%~1"
-) ELSE ( set "SUT_VER=vXXX" )
-
-IF NOT [%2]==[] ( set "TEST_HOME=%~2"
-) ELSE ( set "TEST_HOME=nrtestsuite" )
-
-
-set TESTFILES_URL=https://github.com/OpenWaterAnalytics/epanet-example-networks/archive/v%EXAMPLES_VER%.zip
-set BENCHFILES_URL=https://github.com/OpenWaterAnalytics/epanet-example-networks/releases/download/v%EXAMPLES_VER%/epanet-benchmark-%BENCHMARK_VER%.zip
+IF [%5]==[] ( set "TEST_HOME=nrtestsuite"
+) ELSE ( set "TEST_HOME=%~5" )
 
 
 echo INFO: Staging files for regression testing
 
+
+:: determine SUT executable path
+set "SCRIPT_HOME=%~dp0"
+:: TODO: This may fail when there is more than one cmake buildprod folder
+FOR /D /R "%SCRIPT_HOME%..\" %%a IN (*) DO IF /i "%%~nxa"=="bin" set "BUILD_HOME=%%a"
+set "SUT_PATH=%BUILD_HOME%\Release"
+
+
+:: determine platform from CMakeCache.txt
+IF NOT DEFINED PLATFORM (
+  FOR /F "tokens=*" %%p IN ( 'findstr CMAKE_SHARED_LINKER_FLAGS:STRING %BUILD_HOME%\..\CmakeCache.txt' ) DO ( set "FLAG=%%p" )
+  FOR /F "delims=: tokens=3" %%m IN ( 'echo %FLAG%' ) DO IF "%%m"=="x64" ( set "PLATFORM=win64" ) ELSE ( set "PLATFORM=win32" )
+)
+
+:: hack to determine latest tag in epanet-example-networks repo
+:: TODO: use GitHub api instead
+set "LATEST_URL=https://github.com/OpenWaterAnalytics/epanet-example-networks/releases/latest"
+FOR /F delims^=^"^ tokens^=2 %%g IN ('curl --silent %LATEST_URL%') DO ( set "LATEST_TAG=%%~nxg" )
+
+set "TESTFILES_URL=https://github.com/OpenWaterAnalytics/epanet-example-networks/archive/%LATEST_TAG%.zip"
+set "BENCHFILES_URL=https://github.com/OpenWaterAnalytics/epanet-example-networks/releases/download/%LATEST_TAG%/benchmark-%PLATFORM%-%REF_BUILD_ID%.zip"
+
+
 :: create a clean directory for staging regression tests
 if exist %TEST_HOME% (
   rmdir /s /q %TEST_HOME%
@@ -55,6 +72,7 @@ if exist %TEST_HOME% (
 mkdir %TEST_HOME%
 cd %TEST_HOME%
 
+
 :: retrieve epanet-examples for regression testing
 curl -fsSL -o examples.zip %TESTFILES_URL%
 
@@ -67,9 +85,9 @@ curl -fsSL -o benchmark.zip %BENCHFILES_URL%
 7z x benchmark.zip -obenchmark\ > nul
 
 :: set up symlink for tests directory
-mklink /D .\tests .\epanet-example-networks-%EXAMPLES_VER%\epanet-tests
+mklink /D .\tests .\epanet-example-networks-%LATEST_TAG:~1%\epanet-tests > nul
 
 
 :: generate json configuration file for software under test
 mkdir apps
-%SCRIPT_HOME%\gen-config.cmd %SUT_PATH% > apps\epanet-%SUT_VER%.json
+%SCRIPT_HOME%\gen-config.cmd %SUT_PATH% %PLATFORM% %SUT_BUILD_ID% %SUT_VERSION% > apps\epanet-%SUT_BUILD_ID%.json

tools/gen-config.cmd

@@ -1,29 +1,41 @@
 ::
 ::  gen-config.cmd - Generated nrtest app configuration file for test executable
-::  
+::
 ::  Date Created: 1/8/2018
 ::
 ::  Author: Michael E. Tryby
 ::          US EPA - ORD/NRMRL
 ::
-::  Arguments: 
+::  Arguments:
 ::    1 - absolute path to test executable (valid path seperator for nrtest is "/")
-::
+::    2 - (platform)
+::    3 - (build identifier for SUT)
+::    4 - (commit hash string)
 
 @echo off
 setlocal
 
-:: process path to remove quotes and convert backward to forward slash
+:: swmm target created by the cmake build script
+set TEST_CMD=runepanet.exe
+
+:: remove quotes from path and convert backward to forward slash
 set ABS_BUILD_PATH=%~1
 set ABS_BUILD_PATH=%ABS_BUILD_PATH:\=/%
 
-:: this is the target created by the cmake build script
-set TEST_CMD=runepanet.exe
+IF [%2]==[] ( set "PLATFORM=unknown"
+) ELSE ( set "PLATFORM=%~2" )
+
+IF [%3]==[] ( set "BUILD_ID=unknown"
+) ELSE ( set "BUILD_ID=%~3" )
+
+IF [%4]==[] ( set "VERSION=unknown"
+) ELSE ( set "VERSION=%~4" )
+
 
 echo {
 echo     "name" : "epanet",
-echo     "version" : "",
-echo     "description" : "",
+echo     "version" : "%VERSION%",
+echo     "description" : "%PLATFORM% %BUILD_ID%",
 echo     "setup_script" : "",
 echo     "exe" : "%ABS_BUILD_PATH%/%TEST_CMD%"
 echo }

tools/run-nrtest.cmd

@@ -7,39 +7,40 @@
 ::          US EPA - ORD/NRMRL
 ::
 ::  Arguments:
-::    1 - version/build identifier
-::    2 - (test suite path)
+::    1 - (REF build identifier)
+::    2 - (SUT build identifier)
+::    3 - (test suite path)
 ::
 
 @echo off
 setlocal
 
 
-:: CHANGE THIS VARIABLES TO UPDATE BENCHMARK
-set BENCHMARK_VER=220dev5
+:: Check existence and apply default arguments
+IF [%1]==[] ( echo "ERROR: REF_BUILD_ID must be defined" & exit /B 1
+) ELSE ( set "REF_BUILD_ID=%~1" )
+
+IF [%2]==[] ( set "SUT_BUILD_ID=local"
+) ELSE ( set "SUT_BUILD_ID=%~2" )
+
+IF [%3]==[] ( set "TEST_SUITE_PATH=nrtestsuite"
+) ELSE ( set "TEST_SUITE_PATH=%~3" )
 
 
-:: Determine location of python Scripts folder
+:: determine location of python Scripts folder
 FOR /F "tokens=*" %%G IN ('where python') DO (
   set PYTHON_DIR=%%~dpG
 )
 set "NRTEST_SCRIPT_PATH=%PYTHON_DIR%Scripts"
 
-:: Check existence and apply default arguments
-IF NOT [%1]==[] ( set "SUT_VER=%~1"
-) ELSE ( set "SUT_VER=vXXX" )
-
-IF NOT [%2]==[] ( set "TEST_SUITE_PATH=%~2"
-) ELSE ( set "TEST_SUITE_PATH=nrtestsuite" )
-
 
 set NRTEST_EXECUTE_CMD=python %NRTEST_SCRIPT_PATH%\nrtest execute
-set TEST_APP_PATH=apps\epanet-%SUT_VER%.json
+set TEST_APP_PATH=apps\epanet-%SUT_BUILD_ID%.json
 set TESTS=tests\examples tests\exeter tests\large tests\network_one tests\press_depend tests\small tests\tanks tests\valves
-set TEST_OUTPUT_PATH=benchmark\epanet-%SUT_VER%
+set TEST_OUTPUT_PATH=benchmark\epanet-%SUT_BUILD_ID%
 
 set NRTEST_COMPARE_CMD=python %NRTEST_SCRIPT_PATH%\nrtest compare
-set REF_OUTPUT_PATH=benchmark\epanet-%BENCHMARK_VER%
+set REF_OUTPUT_PATH=benchmark\epanet-%REF_BUILD_ID%
 set RTOL_VALUE=0.01
 set ATOL_VALUE=0.0
 
@@ -51,13 +52,13 @@ if exist %TEST_OUTPUT_PATH% (
   rmdir /s /q %TEST_OUTPUT_PATH%
 )
 
-echo INFO: Creating test benchmark
+echo INFO: Creating SUT %SUT_BUILD_ID% artifacts
 set NRTEST_COMMAND=%NRTEST_EXECUTE_CMD% %TEST_APP_PATH% %TESTS% -o %TEST_OUTPUT_PATH%
 :: if there is an error exit the script with error value 1
 %NRTEST_COMMAND% || exit /B 1
 
 echo.
 
-echo INFO: Comparing test and ref benchmark
+echo INFO: Comparing SUT artifacts to REF %REF_BUILD_ID%
 set NRTEST_COMMAND=%NRTEST_COMPARE_CMD% %TEST_OUTPUT_PATH% %REF_OUTPUT_PATH% --rtol %RTOL_VALUE% --atol %ATOL_VALUE% -o receipt.json
 %NRTEST_COMMAND%