Improving bash scripts for running reg tests

.gitignore

@@ -225,6 +225,7 @@ nrtestsuite/
 tests/data/
 
 #Cmake stuff
+__cmake_systeminformation/
 buildprod*/
 *_export.h
 

tools/before-test.sh

@@ -9,9 +9,11 @@
 #                US EPA - ORD/NRMRL
 #
 #  Arguments:
-#    1 - relative path regression test file staging location 
+#    1 - (platform)
-#    2 - absolute path to location of software under test
+#    2 - (build id for reference)
-#    3 - build identifier for software under test
+#    3 - (build id for software under test)
+#    4 - (version id for software under test)
+#    5 - (relative path regression test file staging location)
 #
 #  Note:
 #    Tests and benchmark files are stored in the epanet-example-networks repo.
@@ -19,14 +21,49 @@
 #    GitHub and stages the files for nrtest to run. The script assumes that
 #    before-test.sh and gen-config.sh are located together in the same folder.
 
+if [ -z $1 ]; then
+  unset PLATFORM;
+else
+  PLATFORM=$1;
+fi
+
+if [ -z $2 ]; then
+  echo "ERROR: REF_BUILD_ID must be defined"; exit 1;
+else
+  REF_BUILD_ID=$2;
+fi
+
+if [ -z $3 ]; then
+  SUT_BUILD_ID="local";
+else
+  SUT_BUILD_ID=$3;
+fi
+
+if [ -z $4 ]; then
+  SUT_VERSION="unknown";
+else
+  SUT_VERSION=$4; fi
+
+if [ -z $5 ]; then
+  TEST_HOME="nrtestsuite";
+else
+  TEST_HOME=$5; fi
+
+
 SCRIPT_HOME="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-TEST_HOME=$1
+BUILD_HOME="$(dirname "$SCRIPT_HOME")"
 
-EXAMPLES_VER="1.0.2-dev.1"
-BENCHMARK_VER="220dev1"
 
-TEST_URL="https://github.com/OpenWaterAnalytics/epanet-example-networks/archive/v${EXAMPLES_VER}.tar.gz"
+SUT_PATH=(`find $BUILD_HOME -name "bin" -type d`)
-BENCH_URL="https://github.com/OpenWaterAnalytics/epanet-example-networks/releases/download/v${EXAMPLES_VER}/epanet-benchmark-${BENCHMARK_VER}.tar.gz"
+
+
+# TODO: determine platform
+
+# determine latest tag from GitHub API
+LATEST_TAG=$(curl --silent "https://api.github.com/repos/openwateranalytics/epanet-example-networks/releases/latest" | grep -Po '"tag_name": "\K.*?(?=")')
+
+TEST_URL="https://github.com/OpenWaterAnalytics/epanet-example-networks/archive/${LATEST_TAG}.tar.gz"
+BENCH_URL="https://github.com/OpenWaterAnalytics/epanet-example-networks/releases/download/${LATEST_TAG}/benchmark-${PLATFORM}-${REF_BUILD_ID}.tar.gz"
 
 
 echo INFO: Staging files for regression testing
@@ -47,7 +84,7 @@ curl -fsSL -o benchmark.tar.gz ${BENCH_URL}
 
 # extract tests and benchmarks
 tar xzf examples.tar.gz
-ln -s epanet-example-networks-${EXAMPLES_VER}/epanet-tests tests
+ln -s epanet-example-networks-${LATEST_TAG:1}/epanet-tests tests
 
 mkdir benchmark
 tar xzf benchmark.tar.gz -C benchmark
@@ -55,4 +92,4 @@ tar xzf benchmark.tar.gz -C benchmark
 
 # generate json configuration file for software under test
 mkdir apps
-${SCRIPT_HOME}/gen-config.sh $2 > apps/epanet-$3.json
+${SCRIPT_HOME}/gen-config.sh ${SUT_PATH} ${PLATFORM} ${SUT_BUILD_ID} ${SUT_VERSION} > apps/epanet-${SUT_BUILD_ID}.json

tools/gen-config.sh

@@ -10,10 +10,9 @@
 #
 #  Arguments:
 #    1 - absolute path to test executable
-# 
+#    2 - platform
-#  NOT IMPLEMENTED YET
+#    3 - SUT build id
-#    2 - test executable version number
+#    4 - SUT version id
-#    3 - build description
 #
 
 unameOut="$(uname -s)"
@@ -32,14 +31,11 @@ case "${unameOut}" in
     *)          # Machine unknown
 esac
 
-version=""
-build_description=""
-
 cat<<EOF
 {
     "name" : "epanet",
-    "version" : "${version}",
+    "version" : "$4",
-    "description" : "${build_description}", 
+    "description" : "$2 $3", 
     "setup_script" : "",
     "exe" : "${abs_build_path}/${test_cmd}"
 }

tools/run-nrtest.sh

@@ -9,8 +9,9 @@
 #                US EPA - ORD/NRMRL
 #
 #  Arguments:
-#   1 - relative path to location there test suite is staged
+#   1 - REF build identifier
-#   2 - version/build identifier
+#   2 - SUT build identifier
+#   3 - relative path to location there test suite is staged
 #
 
 run-nrtest()
@@ -18,17 +19,16 @@ run-nrtest()
 
 return_value=0
 
-test_suite_path=$2
+test_suite_path=$4
-benchmark_ver="220dev1"
 
 
 nrtest_execute_cmd="nrtest execute"
-test_app_path="apps/epanet-$3.json"
+sut_app_path="apps/epanet-$3.json"
 tests="tests/examples tests/exeter tests/large tests/network_one tests/small tests/tanks tests/valves"
-test_output_path="benchmark/epanet-$3"
+sut_output_path="benchmark/epanet-$3"
 
 nrtest_compare_cmd="nrtest compare"
-ref_output_path="benchmark/epanet-${benchmark_ver}"
+ref_output_path="benchmark/epanet-$2"
 rtol_value=0.1
 atol_value=0.0
 
@@ -40,14 +40,14 @@ cd ${test_suite_path}
 rm -rf ${test_output_path}
 
 echo INFO: Creating test benchmark
-nrtest_command="${nrtest_execute_cmd} ${test_app_path} ${tests} -o ${test_output_path}"
+nrtest_command="${nrtest_execute_cmd} ${sut_app_path} ${tests} -o ${sut_output_path}"
 echo INFO: "$nrtest_command"
 return_value=$( $nrtest_command )
 
 if [ $1 = 'true' ]; then
   echo
   echo INFO: Comparing test and ref benchmarks
-  nrtest_command="${nrtest_compare_cmd} ${test_output_path} ${ref_output_path} --rtol ${rtol_value} --atol ${atol_value}"
+  nrtest_command="${nrtest_compare_cmd} ${sut_output_path} ${ref_output_path} --rtol ${rtol_value} --atol ${atol_value} --output benchmark\receipt.json"
   echo INFO: "$nrtest_command"
   return_value=$( $nrtest_command )
 fi
@@ -60,28 +60,39 @@ print_usage() {
    echo "run-nrtest.sh -  generates artifacts for SUT and performes benchmark comparison "
    echo " "
    echo "options:"
-   echo "-c,             don't compare SUT and benchmark artifacts"
+   echo "-c               don't compare SUT and REF artifacts"
+   echo "-r ref_build id  REF build identifier"
+   echo "-s sut build id  SUT build identifier"
    echo "-t test_path     relative path to location where test suite is staged"
-   echo "-v version      version/build identifier"
+   echo " "
 }
 
 # Default option values
 compare='true'
+ref_build_id=''
+sut_build_id='local'
 test_path='nrtestsuite'
 version='vXXX'
 
-while getopts ":ct:v:" flag; do
+while getopts "cr:s:t:" flag; do
   case "${flag}" in
     c  ) compare='false' ;;
+    r  ) ref_build_id=${OPTARG} ;;
+    s  ) sut_build_id=${OPTARG} ;;
     t  ) test_path="${OPTARG}" ;;
-    v  ) version=${OPTARG} ;;
     \? ) print_usage
          exit 1 ;;
   esac
 done
 shift $(($OPTIND - 1))
 
+
+if [ -z $ref_build_id ]; then
+  echo "ERROR: REF_BUILD_ID must be defined"
+  exit 1;
+fi
+
 # Invoke command
-run_command="run-nrtest ${compare} ${test_path} ${version}"
+run_command="run-nrtest ${compare} ${ref_build_id} ${sut_build_id} ${test_path}"
 echo INFO: "$run_command"
 $run_command