Improving bash scripts for running reg tests
This commit is contained in:
Michael Tryby
1
.gitignore
vendored
1
.gitignore
vendored
@@ -225,6 +225,7 @@ nrtestsuite/
|
||||
tests/data/
|
||||
|
||||
#Cmake stuff
|
||||
__cmake_systeminformation/
|
||||
buildprod*/
|
||||
*_export.h
|
||||
|
||||
|
||||
@@ -9,9 +9,11 @@
|
||||
# US EPA - ORD/NRMRL
|
||||
#
|
||||
# Arguments:
|
||||
# 1 - relative path regression test file staging location
|
||||
# 2 - absolute path to location of software under test
|
||||
# 3 - build identifier for software under test
|
||||
# 1 - (platform)
|
||||
# 2 - (build id for reference)
|
||||
# 3 - (build id for software under test)
|
||||
# 4 - (version id for software under test)
|
||||
# 5 - (relative path regression test file staging location)
|
||||
#
|
||||
# Note:
|
||||
# Tests and benchmark files are stored in the epanet-example-networks repo.
|
||||
@@ -19,14 +21,49 @@
|
||||
# GitHub and stages the files for nrtest to run. The script assumes that
|
||||
# before-test.sh and gen-config.sh are located together in the same folder.
|
||||
|
||||
if [ -z $1 ]; then
|
||||
unset PLATFORM;
|
||||
else
|
||||
PLATFORM=$1;
|
||||
fi
|
||||
|
||||
if [ -z $2 ]; then
|
||||
echo "ERROR: REF_BUILD_ID must be defined"; exit 1;
|
||||
else
|
||||
REF_BUILD_ID=$2;
|
||||
fi
|
||||
|
||||
if [ -z $3 ]; then
|
||||
SUT_BUILD_ID="local";
|
||||
else
|
||||
SUT_BUILD_ID=$3;
|
||||
fi
|
||||
|
||||
if [ -z $4 ]; then
|
||||
SUT_VERSION="unknown";
|
||||
else
|
||||
SUT_VERSION=$4; fi
|
||||
|
||||
if [ -z $5 ]; then
|
||||
TEST_HOME="nrtestsuite";
|
||||
else
|
||||
TEST_HOME=$5; fi
|
||||
|
||||
|
||||
SCRIPT_HOME="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
TEST_HOME=$1
|
||||
BUILD_HOME="$(dirname "$SCRIPT_HOME")"
|
||||
|
||||
EXAMPLES_VER="1.0.2-dev.1"
|
||||
BENCHMARK_VER="220dev1"
|
||||
|
||||
TEST_URL="https://github.com/OpenWaterAnalytics/epanet-example-networks/archive/v${EXAMPLES_VER}.tar.gz"
|
||||
BENCH_URL="https://github.com/OpenWaterAnalytics/epanet-example-networks/releases/download/v${EXAMPLES_VER}/epanet-benchmark-${BENCHMARK_VER}.tar.gz"
|
||||
SUT_PATH=(`find $BUILD_HOME -name "bin" -type d`)
|
||||
|
||||
|
||||
# TODO: determine platform
|
||||
|
||||
# determine latest tag from GitHub API
|
||||
LATEST_TAG=$(curl --silent "https://api.github.com/repos/openwateranalytics/epanet-example-networks/releases/latest" | grep -Po '"tag_name": "\K.*?(?=")')
|
||||
|
||||
TEST_URL="https://github.com/OpenWaterAnalytics/epanet-example-networks/archive/${LATEST_TAG}.tar.gz"
|
||||
BENCH_URL="https://github.com/OpenWaterAnalytics/epanet-example-networks/releases/download/${LATEST_TAG}/benchmark-${PLATFORM}-${REF_BUILD_ID}.tar.gz"
|
||||
|
||||
|
||||
echo INFO: Staging files for regression testing
|
||||
@@ -47,7 +84,7 @@ curl -fsSL -o benchmark.tar.gz ${BENCH_URL}
|
||||
|
||||
# extract tests and benchmarks
|
||||
tar xzf examples.tar.gz
|
||||
ln -s epanet-example-networks-${EXAMPLES_VER}/epanet-tests tests
|
||||
ln -s epanet-example-networks-${LATEST_TAG:1}/epanet-tests tests
|
||||
|
||||
mkdir benchmark
|
||||
tar xzf benchmark.tar.gz -C benchmark
|
||||
@@ -55,4 +92,4 @@ tar xzf benchmark.tar.gz -C benchmark
|
||||
|
||||
# generate json configuration file for software under test
|
||||
mkdir apps
|
||||
${SCRIPT_HOME}/gen-config.sh $2 > apps/epanet-$3.json
|
||||
${SCRIPT_HOME}/gen-config.sh ${SUT_PATH} ${PLATFORM} ${SUT_BUILD_ID} ${SUT_VERSION} > apps/epanet-${SUT_BUILD_ID}.json
|
||||
|
||||
@@ -10,10 +10,9 @@
|
||||
#
|
||||
# Arguments:
|
||||
# 1 - absolute path to test executable
|
||||
#
|
||||
# NOT IMPLEMENTED YET
|
||||
# 2 - test executable version number
|
||||
# 3 - build description
|
||||
# 2 - platform
|
||||
# 3 - SUT build id
|
||||
# 4 - SUT version id
|
||||
#
|
||||
|
||||
unameOut="$(uname -s)"
|
||||
@@ -32,14 +31,11 @@ case "${unameOut}" in
|
||||
*) # Machine unknown
|
||||
esac
|
||||
|
||||
version=""
|
||||
build_description=""
|
||||
|
||||
cat<<EOF
|
||||
{
|
||||
"name" : "epanet",
|
||||
"version" : "${version}",
|
||||
"description" : "${build_description}",
|
||||
"version" : "$4",
|
||||
"description" : "$2 $3",
|
||||
"setup_script" : "",
|
||||
"exe" : "${abs_build_path}/${test_cmd}"
|
||||
}
|
||||
|
||||
@@ -9,8 +9,9 @@
|
||||
# US EPA - ORD/NRMRL
|
||||
#
|
||||
# Arguments:
|
||||
# 1 - relative path to location there test suite is staged
|
||||
# 2 - version/build identifier
|
||||
# 1 - REF build identifier
|
||||
# 2 - SUT build identifier
|
||||
# 3 - relative path to location there test suite is staged
|
||||
#
|
||||
|
||||
run-nrtest()
|
||||
@@ -18,17 +19,16 @@ run-nrtest()
|
||||
|
||||
return_value=0
|
||||
|
||||
test_suite_path=$2
|
||||
benchmark_ver="220dev1"
|
||||
test_suite_path=$4
|
||||
|
||||
|
||||
nrtest_execute_cmd="nrtest execute"
|
||||
test_app_path="apps/epanet-$3.json"
|
||||
sut_app_path="apps/epanet-$3.json"
|
||||
tests="tests/examples tests/exeter tests/large tests/network_one tests/small tests/tanks tests/valves"
|
||||
test_output_path="benchmark/epanet-$3"
|
||||
sut_output_path="benchmark/epanet-$3"
|
||||
|
||||
nrtest_compare_cmd="nrtest compare"
|
||||
ref_output_path="benchmark/epanet-${benchmark_ver}"
|
||||
ref_output_path="benchmark/epanet-$2"
|
||||
rtol_value=0.1
|
||||
atol_value=0.0
|
||||
|
||||
@@ -40,14 +40,14 @@ cd ${test_suite_path}
|
||||
rm -rf ${test_output_path}
|
||||
|
||||
echo INFO: Creating test benchmark
|
||||
nrtest_command="${nrtest_execute_cmd} ${test_app_path} ${tests} -o ${test_output_path}"
|
||||
nrtest_command="${nrtest_execute_cmd} ${sut_app_path} ${tests} -o ${sut_output_path}"
|
||||
echo INFO: "$nrtest_command"
|
||||
return_value=$( $nrtest_command )
|
||||
|
||||
if [ $1 = 'true' ]; then
|
||||
echo
|
||||
echo INFO: Comparing test and ref benchmarks
|
||||
nrtest_command="${nrtest_compare_cmd} ${test_output_path} ${ref_output_path} --rtol ${rtol_value} --atol ${atol_value}"
|
||||
nrtest_command="${nrtest_compare_cmd} ${sut_output_path} ${ref_output_path} --rtol ${rtol_value} --atol ${atol_value} --output benchmark\receipt.json"
|
||||
echo INFO: "$nrtest_command"
|
||||
return_value=$( $nrtest_command )
|
||||
fi
|
||||
@@ -60,28 +60,39 @@ print_usage() {
|
||||
echo "run-nrtest.sh - generates artifacts for SUT and performes benchmark comparison "
|
||||
echo " "
|
||||
echo "options:"
|
||||
echo "-c, don't compare SUT and benchmark artifacts"
|
||||
echo "-c don't compare SUT and REF artifacts"
|
||||
echo "-r ref_build id REF build identifier"
|
||||
echo "-s sut build id SUT build identifier"
|
||||
echo "-t test_path relative path to location where test suite is staged"
|
||||
echo "-v version version/build identifier"
|
||||
echo " "
|
||||
}
|
||||
|
||||
# Default option values
|
||||
compare='true'
|
||||
ref_build_id=''
|
||||
sut_build_id='local'
|
||||
test_path='nrtestsuite'
|
||||
version='vXXX'
|
||||
|
||||
while getopts ":ct:v:" flag; do
|
||||
while getopts "cr:s:t:" flag; do
|
||||
case "${flag}" in
|
||||
c ) compare='false' ;;
|
||||
r ) ref_build_id=${OPTARG} ;;
|
||||
s ) sut_build_id=${OPTARG} ;;
|
||||
t ) test_path="${OPTARG}" ;;
|
||||
v ) version=${OPTARG} ;;
|
||||
\? ) print_usage
|
||||
exit 1 ;;
|
||||
esac
|
||||
done
|
||||
shift $(($OPTIND - 1))
|
||||
|
||||
|
||||
if [ -z $ref_build_id ]; then
|
||||
echo "ERROR: REF_BUILD_ID must be defined"
|
||||
exit 1;
|
||||
fi
|
||||
|
||||
# Invoke command
|
||||
run_command="run-nrtest ${compare} ${test_path} ${version}"
|
||||
run_command="run-nrtest ${compare} ${ref_build_id} ${sut_build_id} ${test_path}"
|
||||
echo INFO: "$run_command"
|
||||
$run_command
|
||||
|
||||
Reference in New Issue
Block a user