Merge pull request #401 from michaeltryby/dev

Reorganizing unit tests and improving CI helper scripts on Travis

.gitignore

@@ -225,6 +225,7 @@ nrtestsuite/
 tests/data/
 
 #Cmake stuff
+__cmake_systeminformation/
 buildprod*/
 *_export.h
 

.travis.yml

@@ -1,25 +1,48 @@
 language: python
 
+matrix:
+  include:
+    # works on Precise and Trusty
+    - os: linux
+      addons:
+        apt:
+          sources:
+            - sourceline: 'ppa:mhier/libboost-latest'
+            - ubuntu-toolchain-r-test
+          packages:
+            - g++-7
+            - boost1.67
+      env:
+         - MATRIX_EVAL="CC=gcc-7 && CXX=g++-7"
+         - PLATFORM="linux"
+         - REF_BUILD_ID="220dev1"
+
 python:
     - "3.6"
 
 env:
   global:
+    - SUT_BUILD_ID=$TRAVIS_JOB_NUMBER
     - EPANET_HOME=`pwd`
     - BUILD_HOME=buildprod
     - TEST_HOME=nrtestsuite
 
 before_install:
   - sudo apt-get -qq update
+  - eval "${MATRIX_EVAL}"
 
 install:
-  - sudo apt-get install -y libboost-test-dev
+  - sudo apt-get install jq     
-  - sudo apt-get install -y libboost-thread-dev
+#  - sudo apt-get install -y libboost-test-dev
+#  - sudo apt-get install -y libboost-thread-dev
 
 before_script:
   - mkdir -p $BUILD_HOME
   - cd $BUILD_HOME
-  - cmake -DBUILD_TESTS=ON -DBUILD_COVERAGE=ON ..
+  - cmake -DCMAKE_C_COMPILER=${CC}
+          -DCMAKE_CXX_COMPILER=${CXX}
+          -DBUILD_TESTS=ON
+          -DBUILD_COVERAGE=ON ..
 
 script:
   - cmake --build .
@@ -29,8 +52,8 @@ script:
   # run regression tests
   - cd $EPANET_HOME
   - pip install -r tools/requirements.txt
-  - tools/before-test.sh $TEST_HOME $EPANET_HOME/$BUILD_HOME/bin $TRAVIS_COMMIT
+  - tools/before-test.sh $PLATFORM $REF_BUILD_ID $SUT_BUILD_ID $TRAVIS_COMMIT
-  - tools/run-nrtest.sh -c -t $TEST_HOME -v $TRAVIS_COMMIT
+  - tools/run-nrtest.sh -c -s $SUT_BUILD_ID
 
 after_success:
   - bash <(curl -s https://codecov.io/bash)

include/epanet2_2.h

@@ -220,7 +220,7 @@ typedef struct Project *EN_Project;
 
   Do not call this function while the hydraulics solver is open.
   */
-  int DLLEXPORT EN_usehydfile(EN_Project ph, char *filename);
+  int DLLEXPORT EN_usehydfile(EN_Project ph, const char *filename);
 
   /**
   @brief Opens a project's hydraulic solver.
@@ -343,7 +343,7 @@ typedef struct Project *EN_Project;
   called ::EN_solveH or the ::EN_initH - ::EN_runH - ::EN_nextH sequence with the initflag
   argument of ::EN_initH set to `EN_SAVE` or `EN_SAVE_AND_INIT`.
   */
-  int DLLEXPORT EN_savehydfile(EN_Project ph, char *filename);
+  int DLLEXPORT EN_savehydfile(EN_Project ph, const char *filename);
 
   /**
   @brief Closes the hydraulic solver freeing all of its allocated memory.
@@ -617,7 +617,7 @@ typedef struct Project *EN_Project;
   @return an error code
   */
   int  DLLEXPORT EN_getstatistic(EN_Project ph, int type, double* value);
-  
+
   /********************************************************************
 
   Analysis Options Functions

include/epanet_py.h

@@ -36,6 +36,7 @@ int EXPORT_PY_API proj_init(Handle ph, const char *rptFile, const char *outFile,
 int EXPORT_PY_API proj_open(Handle ph, const char *inpFile, const char *rptFile, const char *binOutFile);
 int EXPORT_PY_API proj_gettitle(Handle ph, char *line1, char *line2, char *line3);
 int EXPORT_PY_API proj_settitle(Handle ph, const char *line1, const char *line2, const char *line3);
+int EXPORT_PY_API proj_getcount(Handle ph, EN_CountType code, int *count);
 int EXPORT_PY_API proj_savefile(Handle ph, const char *inpfilename);
 int EXPORT_PY_API proj_close(Handle ph);
 
@@ -66,7 +67,6 @@ int EXPORT_PY_API rprt_clear(Handle ph);
 int EXPORT_PY_API rprt_reset(Handle ph);
 int EXPORT_PY_API rprt_set(Handle ph, char *reportCommand);
 int EXPORT_PY_API rprt_setlevel(Handle ph, EN_StatusReport code);
-int EXPORT_PY_API rprt_getcount(Handle ph, EN_CountType code, int *count);
 int EXPORT_PY_API rprt_anlysstats(Handle ph, EN_AnalysisStatistic code, double* value);
 
 

src/epanet.c

@@ -247,9 +247,9 @@ int DLLEXPORT EN_gettitle(EN_Project p, char *line1, char *line2, char *line3)
 */
 {
     if (!p->Openflag) return 102;
-    strcpy(line1, p->Title[0]);
+    strncpy(line1, p->Title[0], TITLELEN);
-    strcpy(line2, p->Title[1]);
+    strncpy(line2, p->Title[1], TITLELEN);
-    strcpy(line3, p->Title[2]);
+    strncpy(line3, p->Title[2], TITLELEN);
     return 0;
 }
 
@@ -265,7 +265,7 @@ int DLLEXPORT EN_settitle(EN_Project p, char *line1, char *line2, char *line3)
     strncpy(p->Title[0], line1, TITLELEN);
     strncpy(p->Title[1], line2, TITLELEN);
     strncpy(p->Title[2], line3, TITLELEN);
-    return 123;
+    return 0;
 }
 
 int DLLEXPORT EN_getcount(EN_Project p, int object, int *count)
@@ -571,7 +571,7 @@ int DLLEXPORT EN_closeH(EN_Project p)
   return 0;
 }
 
-int DLLEXPORT EN_savehydfile(EN_Project p, char *filename)
+int DLLEXPORT EN_savehydfile(EN_Project p, const char *filename)
 /*----------------------------------------------------------------
 **  Input:   filename = name of file to which hydraulic results are saved
 **  Output:  none
@@ -599,7 +599,7 @@ int DLLEXPORT EN_savehydfile(EN_Project p, char *filename)
     return 0;
 }
 
-int DLLEXPORT EN_usehydfile(EN_Project p, char *filename)
+int DLLEXPORT EN_usehydfile(EN_Project p, const char *filename)
 /*----------------------------------------------------------------
 **  Input:   filename = name of previously saved hydraulics file
 **  Output:  none
@@ -1031,7 +1031,7 @@ int DLLEXPORT EN_getstatistic(EN_Project p, int type, double *value)
         break;
     default:
         *value = 0.0;
-        break;
+        return 251;
     }
     return 0;
 }
@@ -1366,12 +1366,12 @@ int DLLEXPORT EN_gettimeparam(EN_Project p, int param, long *value)
     case EN_REPORTSTART:
         *value = time->Rstart;
         break;
-    case EN_STATISTIC:
-        *value = rpt->Tstatflag;
-        break;
     case EN_RULESTEP:
         *value = time->Rulestep;
         break;
+    case EN_STATISTIC:
+        *value = rpt->Tstatflag;
+        break;
     case EN_PERIODS:
         *value = rpt->Nperiods;
         break;
@@ -1381,6 +1381,10 @@ int DLLEXPORT EN_gettimeparam(EN_Project p, int param, long *value)
     case EN_HTIME:
         *value = time->Htime;
         break;
+    case EN_QTIME:
+        *value = time->Qtime;
+    case EN_HALTFLAG:
+        break;
     case EN_NEXTEVENT:
         *value = time->Hstep; // find the lesser of the hydraulic time step length,
                               // or the time to next full/empty tank
@@ -1391,6 +1395,8 @@ int DLLEXPORT EN_gettimeparam(EN_Project p, int param, long *value)
         i = tanktimestep(p, value);
         *value = i;
         break;
+    default:
+        return 251;
     }
     return 0;
 }
@@ -2474,7 +2480,7 @@ int DLLEXPORT EN_settankdata(EN_Project p, int index, double elev,
     if (initlvl < 0.0 || minlvl < 0.0 || maxlvl < 0.0) return 209;
     if (minlvl > initlvl || minlvl > maxlvl || initlvl > maxlvl) return 225;
     if (diam < 0.0 || minvol < 0.0) return 209;
-                            
+
     // volume curve supplied
     if (strlen(volcurve) > 0)
     {
@@ -3449,13 +3455,13 @@ int DLLEXPORT EN_getlinkvalue(EN_Project p, int index, int property, double *val
             v = (double)Pump[findpump(&p->network, index)].Ecurve;
         }
         break;
-        
+
     case EN_PUMP_ECOST:
         if  (Link[index].Type == PUMP)
         {
             v = (double)Pump[findpump(&p->network, index)].Ecost;
         }
-        break;    
+        break;
 
     case EN_PUMP_EPAT:
         if (Link[index].Type == PUMP)
@@ -4382,7 +4388,7 @@ int DLLEXPORT EN_setcurve(EN_Project p, int index, double *xValues,
     if (!p->Openflag) return 102;
     if (index <= 0 || index > net->Ncurves) return 206;
     if (nPoints <= 0) return 202;
-    
+
     // Check that x values are increasing
     for (j = 1; j < nPoints; j++) if (xValues[j-1] >= xValues[j]) return 230;
 

src/epanet_py.c

@@ -96,6 +96,12 @@ int EXPORT_PY_API proj_settitle(Handle ph, const char *line1, const char *line2,
     return error_set(pr->error, EN_settitle(pr->project, line1, line2, line3));
 }
 
+int EXPORT_PY_API rprt_getcount(Handle ph, EN_CountType code, int *count)
+{
+    handle_t *pr = (handle_t *)ph;
+    return error_set(pr->error, EN_getcount(pr->project, code, count));
+}
+
 int EXPORT_PY_API proj_savefile(Handle ph, const char *filename)
 {
     handle_t *pr = (handle_t *)ph;
@@ -249,12 +255,6 @@ int EXPORT_PY_API rprt_setlevel(Handle ph, EN_StatusReport code)
     return error_set(pr->error, EN_setstatusreport(pr->project, code));
 }
 
-int EXPORT_PY_API rprt_getcount(Handle ph, EN_CountType code, int *count)
-{
-    handle_t *pr = (handle_t *)ph;
-    return error_set(pr->error, EN_getcount(pr->project, code, count));
-}
-
 int EXPORT_PY_API rprt_anlysstats(Handle ph, EN_AnalysisStatistic code, double* value)
 {
     handle_t *pr = (handle_t *)ph;

src/input2.c

@@ -272,7 +272,7 @@ int newline(Project *pr, int sect, char *line)
           {
               n = (int)strlen(line);
               if (line[n - 1] == 10)
-              line[n - 1] = ' ';
+              line[n - 1] = '\0';
               strncpy(pr->Title[parser->Ntitle], line, TITLELEN);
               parser->Ntitle++;
           }

tests/CMakeLists.txt

@@ -27,7 +27,7 @@ if(MSVC)
     set(Boost_USE_STATIC_LIBS       ON)
 endif(MSVC)
 set(Boost_THREAD_FOUND              OFF)
-find_package(Boost COMPONENTS thread)
+find_package(Boost COMPONENTS system thread filesystem)
 include_directories (${Boost_INCLUDE_DIRS})
 
 
@@ -44,7 +44,11 @@ foreach(testSrc ${TEST_SRCS})
         add_executable(${testName} ${testSrc})
 
         #link to Boost libraries AND your targets and dependencies
-        target_link_libraries(${testName} ${Boost_LIBRARIES} epanet2 epanet-output)
+        IF(MSVC)
+            target_link_libraries(${testName} ${Boost_LIBRARIES} epanet2 epanet-output)
+        ELSE(TRUE)
+            target_link_libraries(${testName} ${Boost_LIBRARIES} pthread epanet2 epanet-output)
+        ENDIF(MSVC)
 
         #Finally add it to test execution
         #Notice the WORKING_DIRECTORY and COMMAND

tests/test_hydrqual.cpp

@@ -0,0 +1,173 @@
+//
+// test_hydrqual.cpp
+//
+// Date Created: February 28, 2019
+//
+// Author: Michael E. Tryby
+//         US EPA - ORD/NRMRL
+//
+
+//#define BOOST_TEST_DYN_LINK
+
+//#ifdef _WIN32
+//#define _CRTDBG_MAP_ALLOC
+//#include <stdlib.h>
+//#include <crtdbg.h>
+//#else
+#include <stdlib.h>
+//#endif
+
+
+#define BOOST_TEST_MODULE hydqual
+
+#include "test_shared.hpp"
+
+
+using namespace std;
+using namespace boost;
+
+
+BOOST_AUTO_TEST_SUITE (test_hydrqual)
+
+BOOST_FIXTURE_TEST_CASE(test_solveH_solveQ, FixtureOpenClose)
+{
+    error = EN_solveH(ph);
+    BOOST_REQUIRE(error == 0);
+
+    error = EN_solveQ(ph);
+    BOOST_REQUIRE(error == 0);
+
+    error = EN_report(ph);
+    BOOST_REQUIRE(error == 0);
+}
+
+BOOST_FIXTURE_TEST_CASE(test_hyd_step, FixtureOpenClose)
+{
+    int flag = 00;
+    long t, tstep;
+
+    error = EN_openH(ph);
+    BOOST_REQUIRE(error == 0);
+
+    error = EN_initH(ph, flag);
+    BOOST_REQUIRE(error == 0);
+
+    do {
+        error = EN_runH(ph, &t);
+        BOOST_REQUIRE(error == 0);
+
+        error = EN_nextH(ph, &tstep);
+        BOOST_REQUIRE(error == 0);
+
+    } while (tstep > 0);
+
+    error = EN_closeH(ph);
+    BOOST_REQUIRE(error == 0);
+}
+
+BOOST_FIXTURE_TEST_CASE(test_qual_step, FixtureOpenClose)
+{
+    int flag = 0;
+    long t, tstep;
+
+    error = EN_solveH(ph);
+    BOOST_REQUIRE(error == 0);
+
+    error = EN_openQ(ph);
+    BOOST_REQUIRE(error == 0);
+
+    error = EN_initQ(ph, flag);
+    BOOST_REQUIRE(error == 0);
+
+    do {
+        error = EN_runQ(ph, &t);
+        BOOST_REQUIRE(error == 0);
+
+        error = EN_stepQ(ph, &tstep);
+        BOOST_REQUIRE(error == 0);
+
+    } while (tstep > 0);
+
+    error = EN_closeQ(ph);
+    BOOST_REQUIRE(error == 0);
+}
+
+BOOST_FIXTURE_TEST_CASE(test_progressive_step, FixtureOpenClose)
+{
+    int flag = EN_NOSAVE;
+    long t, tstep_h, tstep_q;
+
+    error = EN_openH(ph);
+    BOOST_REQUIRE(error == 0);
+
+    error = EN_initH(ph, flag);
+    BOOST_REQUIRE(error == 0);
+
+    error = EN_openQ(ph);
+    BOOST_REQUIRE(error == 0);
+
+    error = EN_initQ(ph, flag);
+    BOOST_REQUIRE(error == 0);
+
+    do {
+        error = EN_runH(ph, &t);
+        BOOST_REQUIRE(error == 0);
+
+        error = EN_runQ(ph, &t);
+        BOOST_REQUIRE(error == 0);
+
+        error = EN_nextH(ph, &tstep_h);
+        BOOST_REQUIRE(error == 0);
+
+        error = EN_nextQ(ph, &tstep_q);
+        BOOST_REQUIRE(error == 0);
+
+    } while (tstep_h > 0);
+
+    error = EN_closeH(ph);
+    BOOST_REQUIRE(error == 0);
+
+    error = EN_closeQ(ph);
+    BOOST_REQUIRE(error == 0);
+
+}
+
+BOOST_FIXTURE_TEST_CASE(test_hydr_save, FixtureOpenClose)
+{
+    error = EN_solveH(ph);
+    BOOST_REQUIRE(error == 0);
+
+    error = EN_saveH(ph);
+    BOOST_REQUIRE(error == 0);
+
+    error = EN_report(ph);
+    BOOST_REQUIRE(error == 0);
+}
+
+BOOST_FIXTURE_TEST_CASE(test_hydr_savefile, FixtureOpenClose)
+{
+    string hyd_file("test_savefile.hyd");
+
+    error = EN_solveH(ph);
+    BOOST_REQUIRE(error == 0);
+
+    error = EN_savehydfile(ph, hyd_file.c_str());
+    BOOST_REQUIRE(error == 0);
+
+    BOOST_CHECK(filesystem::exists(hyd_file) == true);
+}
+
+BOOST_FIXTURE_TEST_CASE(test_hydr_usefile, FixtureOpenClose, * unit_test::depends_on("test_hydrqual/test_hydr_savefile"))
+{
+    string hyd_file("test_savefile.hyd");
+
+    error = EN_usehydfile(ph, hyd_file.c_str());
+    BOOST_REQUIRE(error == 0);
+
+    error = EN_solveQ(ph);
+    BOOST_REQUIRE(error == 0);
+}
+
+
+
+BOOST_AUTO_TEST_SUITE_END()

tests/test_output.cpp

@@ -12,23 +12,20 @@
 // NOTE: Can not dyn link boost using Visual Studio 10 2010
 //#define BOOST_TEST_DYN_LINK
 
-#define BOOST_TEST_MODULE "output"
-#include <boost/test/included/unit_test.hpp>
 
 #include <stdlib.h>
 #include <stdio.h>
 #include <string>
 #include <math.h>
 
+#include <boost/filesystem.hpp>
+#define BOOST_TEST_MODULE "output"
+#include <boost/test/included/unit_test.hpp>
+
 #include "epanet_output.h"
 
-#define DATA_PATH "./example1.out"
 
-using namespace std;
+boost::test_tools::predicate_result check_cdd_float(std::vector<float>& test,
-
-// Custom test to check the minimum number of correct decimal digits between
-// the test and the ref vectors.
-boost::test_tools::predicate_result check_cdd(std::vector<float>& test,
     std::vector<float>& ref, long cdd_tol){
     float tmp, min_cdd = 10.0;
 
@@ -62,6 +59,7 @@ boost::test_tools::predicate_result check_cdd(std::vector<float>& test,
     return floor(min_cdd) >= cdd_tol;
 }
 
+
 boost::test_tools::predicate_result check_string(std::string test, std::string ref)
 {
     if (ref.compare(test) == 0)
@@ -70,10 +68,13 @@ boost::test_tools::predicate_result check_string(std::string test, std::string r
         return false;
 }
 
+
+#define DATA_PATH_OUTPUT "./example1.out"
+
 BOOST_AUTO_TEST_SUITE (test_output_auto)
 
 BOOST_AUTO_TEST_CASE(OpenCloseTest) {
-    std::string path = std::string(DATA_PATH);
+    std::string path = std::string(DATA_PATH_OUTPUT);
 
 	ENR_Handle p_handle = NULL;
     ENR_init(&p_handle);
@@ -89,9 +90,9 @@ BOOST_AUTO_TEST_CASE(OpenCloseTest) {
 BOOST_AUTO_TEST_SUITE_END()
 
 
-struct Fixture{
+struct FixtureOutput{
-    Fixture() {
+    FixtureOutput() {
-        path = std::string(DATA_PATH);
+        path = std::string(DATA_PATH_OUTPUT);
 
         error = ENR_init(&p_handle);
         ENR_clearError(p_handle);
@@ -100,7 +101,7 @@ struct Fixture{
         array = NULL;
         array_dim = 0;
     }
-    ~Fixture() {
+    ~FixtureOutput() {
         free((void*)array);
         error = ENR_close(&p_handle);
     }
@@ -113,9 +114,10 @@ struct Fixture{
     int array_dim;
 };
 
+
 BOOST_AUTO_TEST_SUITE(test_output_fixture)
 
-BOOST_FIXTURE_TEST_CASE(test_getNetSize, Fixture)
+BOOST_FIXTURE_TEST_CASE(test_getNetSize, FixtureOutput)
 {
     int *i_array = NULL;
 
@@ -133,7 +135,7 @@ BOOST_FIXTURE_TEST_CASE(test_getNetSize, Fixture)
     free((void*)i_array);
 }
 
-BOOST_FIXTURE_TEST_CASE(test_getUnits, Fixture) {
+BOOST_FIXTURE_TEST_CASE(test_getUnits, FixtureOutput) {
     int flag;
 
     error = ENR_getUnits(p_handle, ENR_qualUnits, &flag);
@@ -142,7 +144,7 @@ BOOST_FIXTURE_TEST_CASE(test_getUnits, Fixture) {
 	BOOST_CHECK_EQUAL(flag, ENR_MGL);
 }
 
-BOOST_FIXTURE_TEST_CASE(test_getElementName, Fixture) {
+BOOST_FIXTURE_TEST_CASE(test_getElementName, FixtureOutput) {
     char* name;
     int length, index = 1;
 
@@ -156,7 +158,7 @@ BOOST_FIXTURE_TEST_CASE(test_getElementName, Fixture) {
     free((void *)name);
 }
 
-BOOST_FIXTURE_TEST_CASE(test_getNodeAttribute, Fixture) {
+BOOST_FIXTURE_TEST_CASE(test_getNodeAttribute, FixtureOutput) {
 
     error = ENR_getNodeAttribute(p_handle, 1, ENR_quality, &array, &array_dim);
     BOOST_REQUIRE(error == 0);
@@ -176,10 +178,10 @@ BOOST_FIXTURE_TEST_CASE(test_getNodeAttribute, Fixture) {
     std::vector<float> test_vec;
     test_vec.assign(array, array + array_dim);
 
-    BOOST_CHECK(check_cdd(test_vec, ref_vec, 3));
+    BOOST_CHECK(check_cdd_float(test_vec, ref_vec, 3));
 }
 
-BOOST_FIXTURE_TEST_CASE(test_getLinkAttribute, Fixture) {
+BOOST_FIXTURE_TEST_CASE(test_getLinkAttribute, FixtureOutput) {
 
     error = ENR_getLinkAttribute(p_handle, 1, ENR_flow, &array ,&array_dim);
     BOOST_REQUIRE(error == 0);
@@ -201,10 +203,10 @@ BOOST_FIXTURE_TEST_CASE(test_getLinkAttribute, Fixture) {
     std::vector<float> test_vec;
     test_vec.assign(array, array + array_dim);
 
-    BOOST_CHECK(check_cdd(test_vec, ref_vec, 3));
+    BOOST_CHECK(check_cdd_float(test_vec, ref_vec, 3));
 }
 
-BOOST_FIXTURE_TEST_CASE(test_getNodeResult, Fixture) {
+BOOST_FIXTURE_TEST_CASE(test_getNodeResult, FixtureOutput) {
 
     error = ENR_getNodeResult(p_handle, 1, 2, &array, &array_dim);
     BOOST_REQUIRE(error == 0);
@@ -217,10 +219,10 @@ BOOST_FIXTURE_TEST_CASE(test_getNodeResult, Fixture) {
     std::vector<float> test_vec;
     test_vec.assign(array, array + array_dim);
 
-    BOOST_CHECK(check_cdd(test_vec, ref_vec, 3));
+    BOOST_CHECK(check_cdd_float(test_vec, ref_vec, 3));
 }
 
-BOOST_FIXTURE_TEST_CASE(test_getLinkResult, Fixture) {
+BOOST_FIXTURE_TEST_CASE(test_getLinkResult, FixtureOutput) {
 
     error = ENR_getLinkResult(p_handle, 24, 13, &array, &array_dim);
     BOOST_REQUIRE(error == 0);
@@ -237,10 +239,10 @@ BOOST_FIXTURE_TEST_CASE(test_getLinkResult, Fixture) {
     std::vector<float> test_vec;
     test_vec.assign(array, array + array_dim);
 
-    BOOST_CHECK(check_cdd(test_vec, ref_vec, 3));
+    BOOST_CHECK(check_cdd_float(test_vec, ref_vec, 3));
 }
 
-BOOST_FIXTURE_TEST_CASE(test_getNodeSeries, Fixture){
+BOOST_FIXTURE_TEST_CASE(test_getNodeSeries, FixtureOutput){
 
     error = ENR_getNodeSeries(p_handle, 2, ENR_pressure, 0, 10, &array, &array_dim);
     BOOST_REQUIRE(error == 0);
@@ -259,10 +261,10 @@ BOOST_FIXTURE_TEST_CASE(test_getNodeSeries, Fixture){
     std::vector<float> test_vec;
     test_vec.assign(array, array + array_dim);
 
-    BOOST_CHECK(check_cdd(test_vec, ref_vec, 3));
+    BOOST_CHECK(check_cdd_float(test_vec, ref_vec, 3));
 }
 
-BOOST_FIXTURE_TEST_CASE(test_getLinkSeries, Fixture) {
+BOOST_FIXTURE_TEST_CASE(test_getLinkSeries, FixtureOutput) {
 
     error = ENR_getLinkSeries(p_handle, 2, ENR_flow, 0, 10, &array, &array_dim);
     BOOST_REQUIRE(error == 0);
@@ -281,10 +283,10 @@ BOOST_FIXTURE_TEST_CASE(test_getLinkSeries, Fixture) {
     std::vector<float> test_vec;
     test_vec.assign(array, array + array_dim);
 
-    BOOST_CHECK(check_cdd(test_vec, ref_vec, 3));
+    BOOST_CHECK(check_cdd_float(test_vec, ref_vec, 3));
 }
 
-BOOST_FIXTURE_TEST_CASE(test_getNetReacts, Fixture) {
+BOOST_FIXTURE_TEST_CASE(test_getNetReacts, FixtureOutput) {
 
     error = ENR_getNetReacts(p_handle, &array, &array_dim);
     BOOST_REQUIRE(error == 0);
@@ -297,10 +299,10 @@ BOOST_FIXTURE_TEST_CASE(test_getNetReacts, Fixture) {
     std::vector<float> test_vec;
     test_vec.assign(array, array + array_dim);
 
-    BOOST_CHECK(check_cdd(test_vec, ref_vec, 2));
+    BOOST_CHECK(check_cdd_float(test_vec, ref_vec, 2));
 }
 
-BOOST_FIXTURE_TEST_CASE(test_getEnergyUsage, Fixture) {
+BOOST_FIXTURE_TEST_CASE(test_getEnergyUsage, FixtureOutput) {
 
     int linkIdx;
 
@@ -317,7 +319,7 @@ BOOST_FIXTURE_TEST_CASE(test_getEnergyUsage, Fixture) {
     std::vector<float> test_vec;
     test_vec.assign(array, array + array_dim);
 
-    BOOST_CHECK(check_cdd(test_vec, ref_vec, 3));
+    BOOST_CHECK(check_cdd_float(test_vec, ref_vec, 3));
 }
 
 BOOST_AUTO_TEST_SUITE_END()

tests/test_proj.cpp

@@ -1,5 +1,5 @@
 //
-// test_epanet_toolkit.cpp
+// test_project.cpp
 //
 // Date Created: January 24, 2018
 //
@@ -9,31 +9,27 @@
 
 //#define BOOST_TEST_DYN_LINK
 
-#ifdef _WIN32
+//#ifdef _WIN32
-#define _CRTDBG_MAP_ALLOC
+//#define _CRTDBG_MAP_ALLOC
+//#include <stdlib.h>
+//#include <crtdbg.h>
+//#else
 #include <stdlib.h>
-#include <crtdbg.h>
+//#endif
-#else
-#include <stdlib.h>
-#endif
 
-#define BOOST_TEST_MODULE "toolkit"
-#include <boost/test/included/unit_test.hpp>
 
-#include <string>
+#define BOOST_TEST_MODULE "project"
-#include "epanet2_2.h"
+
+#include "test_shared.hpp"
 
-// NOTE: Project Home needs to be updated to run unit test
-#define DATA_PATH_INP "./net1.inp"
-#define DATA_PATH_RPT "./test.rpt"
-#define DATA_PATH_OUT "./test.out"
 
 using namespace std;
+using namespace boost;
 
 
-BOOST_AUTO_TEST_SUITE (test_toolkit)
+BOOST_AUTO_TEST_SUITE (test_proj)
 
-BOOST_AUTO_TEST_CASE (test_alloc_free)
+BOOST_AUTO_TEST_CASE (test_proj_create_delete)
 {
     int error = 0;
     EN_Project ph = NULL;
@@ -49,7 +45,7 @@ BOOST_AUTO_TEST_CASE (test_alloc_free)
     BOOST_CHECK(ph == NULL);
 }
 
-BOOST_AUTO_TEST_CASE (test_open_close)
+BOOST_AUTO_TEST_CASE (test_proj_open_close)
 {
 	string path_inp(DATA_PATH_INP);
 	string path_rpt(DATA_PATH_RPT);
@@ -67,34 +63,53 @@ BOOST_AUTO_TEST_CASE (test_open_close)
     EN_deleteproject(&ph);
 }
 
-BOOST_AUTO_TEST_CASE(test_save_reopen)
+BOOST_AUTO_TEST_CASE(test_proj_savefile)
 {
-	string path_inp(DATA_PATH_INP);
+    int error;
-	string inp_save("test_reopen.inp");
+
+    string path_inp(DATA_PATH_INP);
+    string inp_save("test_reopen.inp");
 	string path_rpt(DATA_PATH_RPT);
 	string path_out(DATA_PATH_OUT);
 
 	EN_Project ph_save;
-    EN_Project ph_reopen;
+
 
 	EN_createproject(&ph_save);
+    error = EN_open(ph_save, path_inp.c_str(), path_rpt.c_str(), path_out.c_str());
+    BOOST_REQUIRE(error == 0);
 
-	EN_open(ph_save, path_inp.c_str(), path_rpt.c_str(), path_out.c_str());
+    error = EN_saveinpfile(ph_save, inp_save.c_str());
-	EN_saveinpfile(ph_save, inp_save.c_str());
+    BOOST_REQUIRE(error == 0);
-	EN_close(ph_save);
 
-	EN_deleteproject(&ph_save);
+	BOOST_CHECK(filesystem::exists(inp_save) == true);
-	BOOST_TEST_CHECKPOINT("Saved input file");
 
-	EN_createproject(&ph_reopen);
+	error = EN_close(ph_save);
+	BOOST_REQUIRE(error == 0);
+    EN_deleteproject(&ph_save);
+}
 
-	EN_open(ph_reopen, inp_save.c_str(), path_rpt.c_str(), path_out.c_str());
+BOOST_AUTO_TEST_CASE(test_proj_reopen, * unit_test::depends_on("test_proj/test_proj_savefile"))
-	EN_close(ph_reopen);
+{
+    int error;
 
+    string inp_save("test_reopen.inp");
+    string path_rpt(DATA_PATH_RPT);
+	string path_out(DATA_PATH_OUT);
+
+    EN_Project ph_reopen;
+
+
+    EN_createproject(&ph_reopen);
+	error = EN_open(ph_reopen, inp_save.c_str(), path_rpt.c_str(), path_out.c_str());
+    BOOST_REQUIRE(error == 0);
+
+    error = EN_close(ph_reopen);
+    BOOST_REQUIRE(error == 0);
 	EN_deleteproject(&ph_reopen);
 }
 
-BOOST_AUTO_TEST_CASE(test_epanet)
+BOOST_AUTO_TEST_CASE(test_proj_run)
 {
     string path_inp(DATA_PATH_INP);
     string path_rpt(DATA_PATH_RPT);
@@ -113,135 +128,50 @@ BOOST_AUTO_TEST_CASE(test_epanet)
 BOOST_AUTO_TEST_SUITE_END()
 
 
-struct Fixture{
-    Fixture() {
-        path_inp = string(DATA_PATH_INP);
-        path_rpt = string(DATA_PATH_RPT);
-        path_out = string(DATA_PATH_OUT);
 
-        EN_createproject(&ph);
+BOOST_AUTO_TEST_SUITE(test_proj_fixture)
-        error = EN_open(ph, path_inp.c_str(), path_rpt.c_str(), path_out.c_str());
+
+BOOST_FIXTURE_TEST_CASE(test_proj_title, FixtureOpenClose)
+{
+    // How is the API user supposed to know array size?
+    char c_test[3][80], c_ref[3][80];
+
+    strncpy(c_ref[0], " EPANET Example Network 1", 26);
+    strncpy(c_ref[1], "A simple example of modeling chlorine decay. Both bulk and", 59);
+    strncpy(c_ref[2], "wall reactions are included. ", 30);
+
+    error = EN_gettitle(ph, c_test[0], c_test[1], c_test[2]);
+    BOOST_REQUIRE(error == 0);
+
+   for (int i = 0; i < 3; i++) {
+       string test (c_test[i]);
+       string ref (c_ref[i]);
+       BOOST_CHECK(check_string(test, ref));
+   }
+
+   // Need a test for EN_settitle
+}
+
+BOOST_FIXTURE_TEST_CASE(test_proj_getcount, FixtureOpenClose)
+{
+    int i, array[7];
+
+	std::vector<int> test;
+	vector<int> ref = { 11, 2, 13, 1, 1, 2, 0 };
+
+    for (i=EN_NODECOUNT; i<=EN_RULECOUNT; i++) {
+        error = EN_getcount(ph, i, &array[i]);
+        BOOST_REQUIRE(error == 0);
     }
 
-    ~Fixture() {
+    test.assign(array, array + 7);
-      error = EN_close(ph);
+    BOOST_CHECK_EQUAL_COLLECTIONS(ref.begin(), ref.end(), test.begin(), test.end());
-      EN_deleteproject(&ph);
-  }
 
-  string path_inp;
+	error = EN_getcount(ph, 7, &i);
-  string path_rpt;
+	BOOST_CHECK(error == 251);
-  string path_out;
-
-  int error;
-  EN_Project ph;
-};
-
-BOOST_AUTO_TEST_SUITE(test_epanet_fixture)
-
-BOOST_FIXTURE_TEST_CASE(test_epanet, Fixture)
-{
-    error = EN_solveH(ph);
-    BOOST_REQUIRE(error == 0);
-
-    error = EN_solveQ(ph);
-    BOOST_REQUIRE(error == 0);
-
-    error = EN_report(ph);
-    BOOST_REQUIRE(error == 0);
 }
 
-BOOST_FIXTURE_TEST_CASE(test_hyd_step, Fixture)
+BOOST_FIXTURE_TEST_CASE(test_setdemandpattern, FixtureOpenClose)
-{
-    int flag = 00;
-    long t, tstep;
-
-    error = EN_openH(ph);
-    BOOST_REQUIRE(error == 0);
-
-    error = EN_initH(ph, flag);
-    BOOST_REQUIRE(error == 0);
-
-    do {
-        error = EN_runH(ph, &t);
-        BOOST_REQUIRE(error == 0);
-
-        error = EN_nextH(ph, &tstep);
-        BOOST_REQUIRE(error == 0);
-
-    } while (tstep > 0);
-
-    error = EN_closeH(ph);
-    BOOST_REQUIRE(error == 0);
-}
-
-BOOST_FIXTURE_TEST_CASE(test_qual_step, Fixture)
-{
-    int flag = 0;
-    long t, tstep;
-
-    error = EN_solveH(ph);
-    BOOST_REQUIRE(error == 0);
-
-    error = EN_openQ(ph);
-    BOOST_REQUIRE(error == 0);
-
-    error = EN_initQ(ph, flag);
-    BOOST_REQUIRE(error == 0);
-
-    do {
-        error = EN_runQ(ph, &t);
-        BOOST_REQUIRE(error == 0);
-
-        error = EN_nextQ(ph, &tstep);
-        BOOST_REQUIRE(error == 0);
-
-    } while (tstep > 0);
-
-    error = EN_closeQ(ph);
-    BOOST_REQUIRE(error == 0);
-}
-
-BOOST_FIXTURE_TEST_CASE(test_progressive_stepping, Fixture)
-{
-    int flag = EN_NOSAVE;
-    long t, tstep_h, tstep_q;
-
-    error = EN_openH(ph);
-    BOOST_REQUIRE(error == 0);
-
-    error = EN_initH(ph, flag);
-    BOOST_REQUIRE(error == 0);
-
-    error = EN_openQ(ph);
-    BOOST_REQUIRE(error == 0);
-
-    error = EN_initQ(ph, flag);
-    BOOST_REQUIRE(error == 0);
-
-    do {
-        error = EN_runH(ph, &t);
-        BOOST_REQUIRE(error == 0);
-
-        error = EN_runQ(ph, &t);
-        BOOST_REQUIRE(error == 0);
-
-        error = EN_nextH(ph, &tstep_h);
-        BOOST_REQUIRE(error == 0);
-
-        error = EN_nextQ(ph, &tstep_q);
-        BOOST_REQUIRE(error == 0);
-
-    } while (tstep_h > 0);
-
-    error = EN_closeH(ph);
-    BOOST_REQUIRE(error == 0);
-
-    error = EN_closeQ(ph);
-    BOOST_REQUIRE(error == 0);
-
-}
-
-BOOST_FIXTURE_TEST_CASE(test_setdemandpattern, Fixture)
 {
     int i, j, pat_index, pat_index_2, numDemands, nnodes;
 	char newpat[] = "new_pattern";
@@ -274,7 +204,8 @@ BOOST_FIXTURE_TEST_CASE(test_setdemandpattern, Fixture)
 		}
 	}
 }
-BOOST_FIXTURE_TEST_CASE(test_addpattern, Fixture)
+
+BOOST_FIXTURE_TEST_CASE(test_addpattern, FixtureOpenClose)
 {
     int pat_index, n_patterns_1, n_patterns_2;
     char newpat[] = "new_pattern";
@@ -297,7 +228,7 @@ BOOST_FIXTURE_TEST_CASE(test_addpattern, Fixture)
     BOOST_CHECK(pat_index == n_patterns_2);
 }
 
-BOOST_FIXTURE_TEST_CASE(test_add_control, Fixture)
+BOOST_FIXTURE_TEST_CASE(test_add_control, FixtureOpenClose)
 {
     int flag = 00;
     long t, tstep;

tests/test_rprtanlys.cpp

@@ -0,0 +1,114 @@
+//
+// test_rprtanlys.cpp
+//
+// Date Created: February 28, 2019
+//
+// Author: Michael E. Tryby
+//         US EPA - ORD/NRMRL
+//
+
+//#define BOOST_TEST_DYN_LINK
+
+//#ifdef _WIN32
+//#define _CRTDBG_MAP_ALLOC
+//#include <stdlib.h>
+//#include <crtdbg.h>
+//#else
+#include <stdlib.h>
+//#endif
+
+
+#define BOOST_TEST_MODULE hydqual
+
+#include "test_shared.hpp"
+
+
+using namespace std;
+using namespace boost;
+
+
+BOOST_AUTO_TEST_SUITE (test_rprtanlys)
+
+BOOST_FIXTURE_TEST_CASE(test_rprt_anlysstats, FixtureOpenClose)
+{
+    int i;
+    double array[5];
+
+    std::vector<double> test;
+	vector<double> ref = {3.0, 7.0799498320679432e-06, 1.6680242187483429e-08,
+        0.0089173150106518495, 0.99999998187144024};
+
+    error = EN_solveH(ph);
+    BOOST_REQUIRE(error == 0);
+
+    error = EN_solveQ(ph);
+    BOOST_REQUIRE(error == 0);
+
+
+    for (i=EN_ITERATIONS; i<=EN_MASSBALANCE; i++) {
+        error = EN_getstatistic(ph, i, &array[i]);
+        BOOST_REQUIRE(error == 0);
+    }
+
+    test.assign(array, array + 5);
+//    BOOST_CHECK_EQUAL_COLLECTIONS(ref.begin(), ref.end(), test.begin(), test.end());
+    BOOST_CHECK(check_cdd_double(test, ref, 3));
+
+    error = EN_getstatistic(ph, 8, &array[0]);
+    BOOST_CHECK(error == 251);
+}
+
+BOOST_FIXTURE_TEST_CASE(test_anlys_getoption, FixtureOpenClose)
+{
+    int i;
+    double array[13];
+
+    std::vector<double> test;
+	vector<double> ref = {40.0, 0.001, 0.01, 0.5, 1.0, 0.0, 0.0, 1.0, 0.0, 75.0, 0.0, 0.0, 0.0};
+
+    error = EN_solveH(ph);
+    BOOST_REQUIRE(error == 0);
+
+    error = EN_solveQ(ph);
+    BOOST_REQUIRE(error == 0);
+
+
+    for (i=EN_TRIALS; i<=EN_DEMANDCHARGE; i++) {
+        error = EN_getoption(ph, i, &array[i]);
+        BOOST_REQUIRE(error == 0);
+    }
+
+    test.assign(array, array + 13);
+    BOOST_CHECK_EQUAL_COLLECTIONS(ref.begin(), ref.end(), test.begin(), test.end());
+
+    error = EN_getoption(ph, 18, &array[0]);
+    BOOST_CHECK(error == 251);
+}
+
+BOOST_FIXTURE_TEST_CASE(test_anlys_gettimeparam, FixtureOpenClose)
+{
+    int i;
+    long array[16];
+
+    std::vector<long> test;
+	vector<long> ref = {86400, 3600, 300, 7200, 0, 3600, 0, 360, 0, 25, 0, 86400, 86400, 0, 3600, 0};
+
+    error = EN_solveH(ph);
+    BOOST_REQUIRE(error == 0);
+
+    error = EN_solveQ(ph);
+    BOOST_REQUIRE(error == 0);
+
+
+    for (i=EN_DURATION; i<=EN_NEXTEVENTTANK; i++) {
+        error = EN_gettimeparam(ph, i, &array[i]);
+        BOOST_REQUIRE(error == 0);
+    }
+
+    test.assign(array, array + 16);
+    BOOST_CHECK_EQUAL_COLLECTIONS(ref.begin(), ref.end(), test.begin(), test.end());
+
+    error = EN_gettimeparam(ph, 18, &array[0]);
+    BOOST_CHECK(error == 251);
+}
+BOOST_AUTO_TEST_SUITE_END()

tests/test_shared.hpp

@@ -0,0 +1,84 @@
+
+
+#include <string>
+#include <math.h>
+
+#include <boost/filesystem.hpp>
+#include <boost/test/included/unit_test.hpp>
+
+#include "epanet2_2.h"
+
+
+
+// Custom test to check the minimum number of correct decimal digits between
+// the test and the ref vectors.
+boost::test_tools::predicate_result check_cdd_double(std::vector<double>& test,
+    std::vector<double>& ref, long cdd_tol){
+    double tmp, min_cdd = 10.0;
+
+    // TODO: What if the vectors aren't the same length?
+
+    std::vector<double>::iterator test_it;
+    std::vector<double>::iterator ref_it;
+
+    for (test_it = test.begin(), ref_it = ref.begin();
+        (test_it < test.end()) && (ref_it < ref.end());
+        ++test_it, ++ref_it)
+    {
+        if (*test_it != *ref_it) {
+            // Compute log absolute error
+            tmp = abs(*test_it - *ref_it);
+            if (tmp < 1.0e-7)
+                tmp = 1.0e-7;
+
+            else if (tmp > 2.0)
+                tmp = 1.0;
+
+            tmp = -log10(tmp);
+            if (tmp < 0.0)
+                tmp = 0.0;
+
+            if (tmp < min_cdd)
+                min_cdd = tmp;
+        }
+    }
+
+    return floor(min_cdd) >= cdd_tol;
+}
+
+boost::test_tools::predicate_result check_string(std::string test, std::string ref)
+{
+    if (ref.compare(test) == 0)
+        return true;
+    else
+        return false;
+}
+
+
+// NOTE: Project Home needs to be updated to run unit test
+#define DATA_PATH_INP "./net1.inp"
+#define DATA_PATH_RPT "./test.rpt"
+#define DATA_PATH_OUT "./test.out"
+
+struct FixtureOpenClose{
+    FixtureOpenClose() {
+        path_inp = std::string(DATA_PATH_INP);
+        path_rpt = std::string(DATA_PATH_RPT);
+        path_out = std::string(DATA_PATH_OUT);
+
+        EN_createproject(&ph);
+        error = EN_open(ph, path_inp.c_str(), path_rpt.c_str(), path_out.c_str());
+    }
+
+    ~FixtureOpenClose() {
+      error = EN_close(ph);
+      EN_deleteproject(&ph);
+  }
+
+  std::string path_inp;
+  std::string path_rpt;
+  std::string path_out;
+
+  int error;
+  EN_Project ph;
+};

tools/before-test.cmd

@@ -80,9 +80,11 @@ curl -fsSL -o examples.zip %TESTFILES_URL%
 curl -fsSL -o benchmark.zip %BENCHFILES_URL%
 
 
-:: extract tests and benchmarks
+:: extract tests, benchmarks, and manifest
 7z x examples.zip *\epanet-tests\* > nul
 7z x benchmark.zip -obenchmark\ > nul
+7z e benchmark.zip -o. manifest.json -r > nul
+
 
 :: set up symlink for tests directory
 mklink /D .\tests .\epanet-example-networks-%LATEST_TAG:~1%\epanet-tests > nul

tools/before-test.sh

@@ -1,6 +1,6 @@
 #! /bin/bash
 
-# 
+#
 #  before-test.sh - Prepares Travis CI worker to run epanet regression tests
 #
 #  Date Created: 04/04/2018
@@ -8,25 +8,67 @@
 #  Author:       Michael E. Tryby
 #                US EPA - ORD/NRMRL
 #
-#  Arguments: 
+#  Arguments:
-#    1 - relative path regression test file staging location 
+#    1 - (platform)
-#    2 - absolute path to location of software under test
+#    2 - (build id for reference)
-#    3 - build identifier for software under test
+#    3 - (build id for software under test)
-#  
+#    4 - (version id for software under test)
-#  Note: 
+#    5 - (relative path regression test file staging location)
+#
+#  Note:
 #    Tests and benchmark files are stored in the epanet-example-networks repo.
-#    This script retreives them using a stable URL associated with a release on 
+#    This script retreives them using a stable URL associated with a release on
-#    GitHub and stages the files for nrtest to run. The script assumes that 
+#    GitHub and stages the files for nrtest to run. The script assumes that
-#    before-test.sh and gen-config.sh are located together in the same folder. 
+#    before-test.sh and gen-config.sh are located together in the same folder.
+
+if [ -z $1 ]; then
+  unset PLATFORM;
+else
+  PLATFORM=$1;
+fi
+
+if [ -z $2 ]; then
+  echo "ERROR: REF_BUILD_ID must be defined"; exit 1;
+else
+  REF_BUILD_ID=$2;
+fi
+
+if [ -z $3 ]; then
+  SUT_BUILD_ID="local";
+else
+  SUT_BUILD_ID=$3;
+fi
+
+if [ -z $4 ]; then
+  SUT_VERSION="unknown";
+else
+  SUT_VERSION=$4; fi
+
+if [ -z $5 ]; then
+  TEST_HOME="nrtestsuite";
+else
+  TEST_HOME=$5; fi
+
 
 SCRIPT_HOME="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-TEST_HOME=$1
+BUILD_HOME="$(dirname "$SCRIPT_HOME")"
 
-EXAMPLES_VER="1.0.2-dev.1"
-BENCHMARK_VER="220dev1"
 
-TEST_URL="https://github.com/OpenWaterAnalytics/epanet-example-networks/archive/v${EXAMPLES_VER}.tar.gz"
+SUT_PATH=(`find $BUILD_HOME -name "bin" -type d`)
-BENCH_URL="https://github.com/OpenWaterAnalytics/epanet-example-networks/releases/download/v${EXAMPLES_VER}/epanet-benchmark-${BENCHMARK_VER}.tar.gz"
+
+
+# TODO: determine platform
+
+# determine latest tag from GitHub API
+LATEST_URL="https://api.github.com/repos/openwateranalytics/epanet-example-networks/releases/latest"
+LATEST_TAG=(`curl --silent ${LATEST_URL} | jq -r .tag_name`)
+if [ -z $LATEST_TAG ]; then
+    echo "ERROR: curl | jq - ${LATEST_URL}"
+    exit 1
+fi
+
+TEST_URL="https://github.com/OpenWaterAnalytics/epanet-example-networks/archive/${LATEST_TAG}.tar.gz"
+BENCH_URL="https://github.com/OpenWaterAnalytics/epanet-example-networks/releases/download/${LATEST_TAG}/benchmark-${PLATFORM}-${REF_BUILD_ID}.tar.gz"
 
 
 echo INFO: Staging files for regression testing
@@ -38,21 +80,26 @@ fi
 mkdir ${TEST_HOME}
 cd ${TEST_HOME}
 
+
 # retrieve epanet-examples for regression testing
-curl -fsSL -o examples.tar.gz ${TEST_URL}
+if ! curl -fsSL -o examples.tar.gz ${TEST_URL}; then
+    echo "ERROR: curl - ${TEST_URL}"
+fi
 
 # retrieve epanet benchmark results
-curl -fsSL -o benchmark.tar.gz ${BENCH_URL}
+if ! curl -fsSL -o benchmark.tar.gz ${BENCH_URL}; then
+    echo "ERROR: curl - ${BENCH_URL}"
+fi
 
-
+# extract tests, benchmarks, and manifest
-# extract tests and benchmarks
 tar xzf examples.tar.gz
-ln -s epanet-example-networks-${EXAMPLES_VER}/epanet-tests tests
+ln -s epanet-example-networks-${LATEST_TAG:1}/epanet-tests tests
 
 mkdir benchmark
 tar xzf benchmark.tar.gz -C benchmark
+tar xzf benchmark.tar.gz --wildcards --no-anchored --strip-components=1 '*/manifest.json' -C .
 
 
 # generate json configuration file for software under test
 mkdir apps
-${SCRIPT_HOME}/gen-config.sh $2 > apps/epanet-$3.json
+${SCRIPT_HOME}/gen-config.sh ${SUT_PATH} ${PLATFORM} ${SUT_BUILD_ID} ${SUT_VERSION} > apps/epanet-${SUT_BUILD_ID}.json

tools/gen-config.sh

@@ -1,6 +1,6 @@
 #! /bin/bash
 
-# 
+#
 #  gen-config.sh - Generates nrtest app configuration file for test executable
 #
 #  Date Created: 10/16/2017
@@ -10,10 +10,9 @@
 #
 #  Arguments:
 #    1 - absolute path to test executable
-# 
+#    2 - platform
-#  NOT IMPLEMENTED YET
+#    3 - SUT build id
-#    2 - test executable version number
+#    4 - SUT version id
-#    3 - build description
 #
 
 unameOut="$(uname -s)"
@@ -28,18 +27,15 @@ case "${unameOut}" in
                 abs_build_path="$( echo "$1" | sed -e 's#/c##' )"
                 test_cmd="runepanet.exe"
                 ;;
-				
+
     *)          # Machine unknown
 esac
 
-version=""
-build_description=""
-
 cat<<EOF
 {
     "name" : "epanet",
-    "version" : "${version}",
+    "version" : "$4",
-    "description" : "${build_description}", 
+    "description" : "$2 $3", 
     "setup_script" : "",
     "exe" : "${abs_build_path}/${test_cmd}"
 }

tools/run-nrtest.sh

@@ -9,8 +9,9 @@
 #                US EPA - ORD/NRMRL
 #
 #  Arguments:
-#   1 - relative path to location there test suite is staged
+#   1 - REF build identifier
-#   2 - version/build identifier
+#   2 - SUT build identifier
+#   3 - relative path to location there test suite is staged
 #
 
 run-nrtest()
@@ -18,17 +19,16 @@ run-nrtest()
 
 return_value=0
 
-test_suite_path=$2
+test_suite_path=$4
-benchmark_ver="220dev1"
 
 
 nrtest_execute_cmd="nrtest execute"
-test_app_path="apps/epanet-$3.json"
+sut_app_path="apps/epanet-$3.json"
 tests="tests/examples tests/exeter tests/large tests/network_one tests/small tests/tanks tests/valves"
-test_output_path="benchmark/epanet-$3"
+sut_output_path="benchmark/epanet-$3"
 
 nrtest_compare_cmd="nrtest compare"
-ref_output_path="benchmark/epanet-${benchmark_ver}"
+ref_output_path="benchmark/epanet-$2"
 rtol_value=0.1
 atol_value=0.0
 
@@ -40,14 +40,14 @@ cd ${test_suite_path}
 rm -rf ${test_output_path}
 
 echo INFO: Creating test benchmark
-nrtest_command="${nrtest_execute_cmd} ${test_app_path} ${tests} -o ${test_output_path}"
+nrtest_command="${nrtest_execute_cmd} ${sut_app_path} ${tests} -o ${sut_output_path}"
 echo INFO: "$nrtest_command"
 return_value=$( $nrtest_command )
 
 if [ $1 = 'true' ]; then
   echo
   echo INFO: Comparing test and ref benchmarks
-  nrtest_command="${nrtest_compare_cmd} ${test_output_path} ${ref_output_path} --rtol ${rtol_value} --atol ${atol_value}"
+  nrtest_command="${nrtest_compare_cmd} ${sut_output_path} ${ref_output_path} --rtol ${rtol_value} --atol ${atol_value} --output benchmark\receipt.json"
   echo INFO: "$nrtest_command"
   return_value=$( $nrtest_command )
 fi
@@ -57,31 +57,42 @@ return $return_value
 
 print_usage() {
    echo " "
-   echo "run-nrtest.sh - generates artifacts for SUT and performes benchmark comparison "
+   echo "run-nrtest.sh -  generates artifacts for SUT and performes benchmark comparison "
    echo " "
    echo "options:"
-   echo "-c,             don't compare SUT and benchmark artifacts"
+   echo "-c               don't compare SUT and REF artifacts"
-   echo "-t test_path    relative path to location where test suite is staged"
+   echo "-r ref_build id  REF build identifier"
-   echo "-v version      version/build identifier"
+   echo "-s sut build id  SUT build identifier"
+   echo "-t test_path     relative path to location where test suite is staged"
+   echo " "
 }
 
 # Default option values
 compare='true'
+ref_build_id='unknown'
+sut_build_id='local'
 test_path='nrtestsuite'
-version='vXXX'
 
-while getopts ":ct:v:" flag; do
+while getopts "cr:s:t:" flag; do
   case "${flag}" in
     c  ) compare='false' ;;
+    r  ) ref_build_id=${OPTARG} ;;
+    s  ) sut_build_id=${OPTARG} ;;
     t  ) test_path="${OPTARG}" ;;
-    v  ) version=${OPTARG} ;;
     \? ) print_usage
          exit 1 ;;
   esac
 done
 shift $(($OPTIND - 1))
 
+
+# determine ref_build_id from manifest file
+if [[ $ref_build_id == 'unknown' ]] && [[ $compare == 'true' ]]; then
+  description=(`cat ${test_path}/manifest.json | jq '.Application.description | splits(" ")'`)
+  ref_build_id=${description[1]//\"/}
+fi
+
 # Invoke command
-run_command="run-nrtest ${compare} ${test_path} ${version}"
+run_command="run-nrtest ${compare} ${ref_build_id} ${sut_build_id} ${test_path}"
 echo INFO: "$run_command"
 $run_command