From f38a637aaf1dc6af1724969b5ded392940f3924b Mon Sep 17 00:00:00 2001 From: Michael Tryby Date: Tue, 10 Jul 2018 18:50:33 -0400 Subject: [PATCH] Adding mincdd comparison and better debug information --- tools/nrtest-epanet/main.py | 39 +++++++----- tools/nrtest-epanet/nrtest_epanet/__init__.py | 63 ++++++++++++++++--- .../nrtest_epanet/output_reader.py | 6 +- tools/nrtest-epanet/setup.py | 1 + 4 files changed, 84 insertions(+), 25 deletions(-) diff --git a/tools/nrtest-epanet/main.py b/tools/nrtest-epanet/main.py index 1c0f41d..d51c44e 100644 --- a/tools/nrtest-epanet/main.py +++ b/tools/nrtest-epanet/main.py @@ -4,7 +4,9 @@ import time import cStringIO import itertools as it -import epanet_reader as er +# project import +import nrtest_epanet.output_reader as er + def result_compare(path_test, path_ref, comp_args): @@ -15,29 +17,36 @@ def result_compare(path_test, path_ref, comp_args): total = 0 output = cStringIO.StringIO() eps = np.finfo(float).eps + min_cdd = 100.0 start = time.time() - test_reader = er.reader(path_test) - ref_reader = er.reader(path_ref) + test_reader = er.output_generator(path_test) + ref_reader = er.output_generator(path_ref) for test, ref in it.izip(test_reader, ref_reader): total += 1 if total%100000 == 0: print(total) - if test.size != ref.size: + if len(test[0]) != len(ref[0]): raise ValueError('Inconsistent lengths') # Skip results if they are zero or equal - if np.array_equal(test, ref): - equal += 1 - continue + #if np.array_equal(test, ref): + # equal += 1 + # continue else: try: - np.testing.assert_allclose(test, ref, 1.0e-06, 2*eps) - close += 1 + diff = np.fabs(np.subtract(test[0], ref[0])) + idx = np.unravel_index(np.argmax(diff), diff.shape) + if diff[idx] != 0.0: + tmp = - np.log10(diff[idx]) + + if tmp < min_cdd: + min_cdd = tmp; + except AssertionError as ae: notclose += 1 output.write(str(ae)) @@ -49,8 +58,9 @@ def result_compare(path_test, path_ref, comp_args): print(output.getvalue()) output.close() - print('equal: %d close: %d notclose: %d total: %d in %f (sec)\n' % - (equal, close, notclose, total, (stop - start))) + print('mincdd: %d in %f (sec)' % (np.floor(min_cdd), (stop - start))) + #print('equal: %d close: %d notclose: %d total: %d in %f (sec)\n' % + # (equal, close, notclose, total, (stop - start))) if notclose > 0: print('%d differences found\n' % notclose) @@ -130,7 +140,8 @@ if __name__ == "__main__": # ref_path = "C:\\Users\\mtryby\\Workspace\\GitRepo\\Local\\epanet-testsuite\\benchmarks\\v2012" # print(nrtest_compare(test_path, ref_path, (0.001, 0.0))) - - path_test = "C:\\Users\\mtryby\\Workspace\\GitRepo\\Local\\epanet-testsuite\\benchmarks\\v2011a\\Example_3\\example3.out" - path_ref = "C:\\Users\\mtryby\\Workspace\\GitRepo\\Local\\epanet-testsuite\\benchmarks\\v2012\\Example_3\\example3.out" + benchmark_path = "C:\\Users\\mtryby\\Workspace\\GitRepo\\michaeltryby\\epanet-lr\\nrtestsuite\\benchmarks\\" + path_test = benchmark_path + "epanet-220dev\\example1\\example1.out" + path_ref = benchmark_path + "epanet-2012\\example1\\example1.out" + result_compare(path_test, path_ref, (0.001, 0.0)) diff --git a/tools/nrtest-epanet/nrtest_epanet/__init__.py b/tools/nrtest-epanet/nrtest_epanet/__init__.py index 7bdf41b..6135873 100644 --- a/tools/nrtest-epanet/nrtest_epanet/__init__.py +++ b/tools/nrtest-epanet/nrtest_epanet/__init__.py @@ -38,7 +38,7 @@ __status = "Development" def epanet_allclose_compare(path_test, path_ref, rtol, atol): ''' - Compares results in two EPANET binary files. Using the comparison criteria + Compares results in two EPANET binary files using the comparison criteria described in the numpy assert_allclose documentation. (test_value - ref_value) <= atol + rtol * abs(ref_value) @@ -67,22 +67,67 @@ def epanet_allclose_compare(path_test, path_ref, rtol, atol): for (test, ref) in it.izip(ordr.output_generator(path_test), ordr.output_generator(path_ref)): - if len(test) != len(ref): + if len(test[0]) != len(ref[0]): raise ValueError('Inconsistent lengths') # Skip over arrays that are equal - if np.array_equal(test, ref): + if np.array_equal(test[0], ref[0]): continue else: - np.testing.assert_allclose(test, ref, rtol, atol) + np.testing.assert_allclose(test[0], ref[0], rtol, atol) return True -# def epanet_better_compare(path_test, path_ref, rtol, atol): -# ''' -# If you don't like assert_allclose you can add another function here. -# ''' -# pass + +def epanet_mincdd_compare(path_test, path_ref, rtol, atol): + ''' + Compares the results of two EPANET binary files using a correct decimal + digits (cdd) comparison criteria: + + min cdd(test, ref) >= atol + + Returns true if min cdd in the file is greater than or equal to atol, + otherwise an AssertionError is thrown. + + Arguments: + path_test - path to result file being testedgit + path_ref - path to reference result file + rtol - ignored + atol - minimum allowable cdd value (i.e. 3) + + Returns: + True + + Raises: + ValueError() + AssertionError() + ''' + min_cdd = 100.0 + + for (test, ref) in it.izip(ordr.output_generator(path_test), + ordr.output_generator(path_ref)): + + if len(test[0]) != len(ref[0]): + raise ValueError('Inconsistent lengths') + + # Skip over arrays that are equal + if np.array_equal(test[0], ref[0]): + continue + else: + diff = np.fabs(np.subtract(test[0], ref[0])) + idx = np.unravel_index(np.argmax(diff), diff.shape) + + if diff[idx] != 0.0: + tmp = - np.log10(diff[idx]) + + if tmp < min_cdd: + min_cdd = tmp; + + if np.floor(min_cdd) >= atol: + return True + else: + raise AssertionError('min_cdd=%d less than atol=%g' % (min_cdd, atol)) + def epanet_report_compare(path_test, path_ref, rtol, atol): ''' diff --git a/tools/nrtest-epanet/nrtest_epanet/output_reader.py b/tools/nrtest-epanet/nrtest_epanet/output_reader.py index 0bf640f..6877d91 100644 --- a/tools/nrtest-epanet/nrtest_epanet/output_reader.py +++ b/tools/nrtest-epanet/nrtest_epanet/output_reader.py @@ -23,7 +23,8 @@ def output_generator(path_ref): yield element attributes. It is useful for comparing contents of binary files for numerical regression testing. - The generator yields a Python list containing element attributes. + The generator yields a Python tuple containing an array of element + attributes and a tuple containing the element type, period, and attribute. Arguments: path_ref - path to result file @@ -38,7 +39,8 @@ def output_generator(path_ref): for element_type in oapi.ElementType: for attribute in br.elementAttributes[element_type]: - yield br.element_attribute(element_type, period_index, attribute) + yield (br.element_attribute(element_type, period_index, attribute), + (element_type, period_index, attribute)) class OutputReader(): diff --git a/tools/nrtest-epanet/setup.py b/tools/nrtest-epanet/setup.py index 897e621..1a18763 100644 --- a/tools/nrtest-epanet/setup.py +++ b/tools/nrtest-epanet/setup.py @@ -17,6 +17,7 @@ except ImportError: entry_points = { 'nrtest.compare': [ 'epanet allclose = nrtest_epanet:epanet_allclose_compare', + 'epanet mincdd = nrtest_epanet:epanet_mincdd_compare', 'epanet report = nrtest_epanet:epanet_report_compare', # Add entry point for new comparison functions here ]