diff --git a/tools/nrtest-epanet/main.py b/tools/nrtest-epanet/main.py index d51c44e..c962dab 100644 --- a/tools/nrtest-epanet/main.py +++ b/tools/nrtest-epanet/main.py @@ -130,6 +130,8 @@ def nrtest_execute(app_path, test_path, output_path): exit(not success) +import report_diff as rd + if __name__ == "__main__": # app_path = "apps\\swmm-5.1.11.json" # test_path = "tests\\examples\\example1.json" @@ -141,7 +143,9 @@ if __name__ == "__main__": # print(nrtest_compare(test_path, ref_path, (0.001, 0.0))) benchmark_path = "C:\\Users\\mtryby\\Workspace\\GitRepo\\michaeltryby\\epanet-lr\\nrtestsuite\\benchmarks\\" - path_test = benchmark_path + "epanet-220dev\\example1\\example1.out" - path_ref = benchmark_path + "epanet-2012\\example1\\example1.out" + path_test = benchmark_path + "epanet-220dev\\example2\\example2.out" + path_ref = benchmark_path + "epanet-2012\\example2\\example2.out" - result_compare(path_test, path_ref, (0.001, 0.0)) + #result_compare(path_test, path_ref, (0.001, 0.0)) + rd.report_diff(path_test, path_ref, 2) + \ No newline at end of file diff --git a/tools/nrtest-epanet/nrtest_epanet/__init__.py b/tools/nrtest-epanet/nrtest_epanet/__init__.py index d6ff9bd..a118695 100644 --- a/tools/nrtest-epanet/nrtest_epanet/__init__.py +++ b/tools/nrtest-epanet/nrtest_epanet/__init__.py @@ -118,7 +118,7 @@ def epanet_mincdd_compare(path_test, path_ref, rtol, atol): idx = np.unravel_index(np.argmax(diff), diff.shape) if diff[idx] != 0.0: - tmp = - np.log10(diff[idx]) + tmp = - np.log10(diff[idx]) if tmp < min_cdd: min_cdd = tmp; diff --git a/tools/nrtest-epanet/report_diff.py b/tools/nrtest-epanet/report_diff.py new file mode 100644 index 0000000..689d7aa --- /dev/null +++ b/tools/nrtest-epanet/report_diff.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- + +# +# report_diff.py +# +# Date Created: July 11, 2018 +# +# Author: Michael E. Tryby +# US EPA - ORD/NRMRL +# + +# system imports +import itertools as it + +# third party imports +import numpy as np + +# project imports +import nrtest_epanet.output_reader as ordr + + +def report_diff(path_test, path_ref, min_cdd): + for (test, ref) in it.izip(ordr.output_generator(path_test), + ordr.output_generator(path_ref)): + + if len(test[0]) != len(ref[0]): + raise ValueError('Inconsistent lengths') + + # Skip over arrays that are equal + if np.array_equal(test[0], ref[0]): + continue + else: + lre = log_relative_error(test[0], ref[0]) + idx = np.unravel_index(np.argmin(lre), lre.shape) + + if lre[idx] < min_cdd: + print_diff(idx, lre, test, ref) + + return + + +def log_relative_error(q, c): + ''' + Computes log relative error, a measure of numerical accuracy. + + Single precision machine epsilon is between 2^-24 and 2^-23. + + Reference: + McCullough, B. D. "Assessing the Reliability of Statistical Software: Part I." + The American Statistician, vol. 52, no. 4, 1998, pp. 358�366. + JSTOR, JSTOR, www.jstor.org/stable/2685442. + ''' + diff = np.subtract(q, c) + tmp_c = np.copy(c) + # If ref value is small compute absolute error + tmp_c[np.fabs(tmp_c) <= 1.0e-6] = 1.0 + + re = np.fabs(diff)/np.fabs(tmp_c) + # If re is tiny set lre to number of digits + re[re < 1.0e-7] = 1.0e-7 + # If re is very large set lre to zero + re[re > 2.0] = 1.0 + + return np.negative(np.log10(re)) + + +def print_diff(idx, lre, test, ref): + print("LRE: %f\nIdx: %s\nSut: %f\nRef: %f\n" + % ((lre[idx]),(idx[0], ref[1]),(test[0][idx[0]]),(ref[0][idx[0]]))) \ No newline at end of file