JIANG/EPANET
1
0
Fork 0
Code Issues Pull Requests Actions 4 Packages Projects Releases Wiki Activity

Merge branch 'contributor-lr' of https://github.com/LRossman/EPANET into contributor-lr

Browse Source
This commit is contained in:
Michael Tryby
2018-08-24 16:14:54 -04:00
parent 66bb150f99 f495d2c707
commit f847a3ed6e
11 changed files with 467 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
tools/nrtest-epanet/main.py
View File

@@ -4,7 +4,9 @@ import time
import cStringIO
import itertools as it
import epanet_reader as er
# project import
import nrtest_epanet.output_reader as er
def result_compare(path_test, path_ref, comp_args):
@@ -15,29 +17,36 @@ def result_compare(path_test, path_ref, comp_args):
total = 0
output = cStringIO.StringIO()
eps = np.finfo(float).eps
min_cdd = 100.0
start = time.time()
test_reader = er.reader(path_test)
ref_reader = er.reader(path_ref)
test_reader = er.output_generator(path_test)
ref_reader = er.output_generator(path_ref)
for test, ref in it.izip(test_reader, ref_reader):
total += 1
if total%100000 == 0:
print(total)
if test.size != ref.size:
if len(test[0]) != len(ref[0]):
raise ValueError('Inconsistent lengths')
# Skip results if they are zero or equal
if np.array_equal(test, ref):
equal += 1
continue
#if np.array_equal(test, ref):
# equal += 1
# continue
else:
try:
np.testing.assert_allclose(test, ref, 1.0e-06, 2*eps)
close += 1
diff = np.fabs(np.subtract(test[0], ref[0]))
idx = np.unravel_index(np.argmax(diff), diff.shape)
if diff[idx] != 0.0:
tmp = - np.log10(diff[idx])
if tmp < min_cdd:
min_cdd = tmp;
except AssertionError as ae:
notclose += 1
output.write(str(ae))
@@ -49,8 +58,9 @@ def result_compare(path_test, path_ref, comp_args):
print(output.getvalue())
output.close()
print('equal: %d close: %d notclose: %d total: %d in %f (sec)\n' %
(equal, close, notclose, total, (stop - start)))
print('mincdd: %d in %f (sec)' % (np.floor(min_cdd), (stop - start)))
#print('equal: %d close: %d notclose: %d total: %d in %f (sec)\n' %
# (equal, close, notclose, total, (stop - start)))
if notclose > 0:
print('%d differences found\n' % notclose)
@@ -120,6 +130,8 @@ def nrtest_execute(app_path, test_path, output_path):
exit(not success)
import report_diff as rd
if __name__ == "__main__":
# app_path = "apps\\swmm-5.1.11.json"
# test_path = "tests\\examples\\example1.json"
@@ -130,7 +142,10 @@ if __name__ == "__main__":
# ref_path = "C:\\Users\\mtryby\\Workspace\\GitRepo\\Local\\epanet-testsuite\\benchmarks\\v2012"
# print(nrtest_compare(test_path, ref_path, (0.001, 0.0)))
path_test = "C:\\Users\\mtryby\\Workspace\\GitRepo\\Local\\epanet-testsuite\\benchmarks\\v2011a\\Example_3\\example3.out"
path_ref = "C:\\Users\\mtryby\\Workspace\\GitRepo\\Local\\epanet-testsuite\\benchmarks\\v2012\\Example_3\\example3.out"
result_compare(path_test, path_ref, (0.001, 0.0))
benchmark_path = "C:\\Users\\mtryby\\Workspace\\GitRepo\\michaeltryby\\epanet-lr\\nrtestsuite\\benchmarks\\"
path_test = benchmark_path + "epanet-220dev\\example2\\example2.out"
path_ref = benchmark_path + "epanet-2012\\example2\\example2.out"
#result_compare(path_test, path_ref, (0.001, 0.0))
rd.report_diff(path_test, path_ref, 2)

65
tools/nrtest-epanet/nrtest_epanet/__init__.py
View File

@@ -28,7 +28,7 @@ __copyright__ = "None"
__credits__ = "Colleen Barr, Maurizio Cingi, Mark Gray, David Hall, Bryant McDonnell"
__license__ = "CC0 1.0 Universal"
__version__ = "0.4.0"
__version__ = "0.5.0"
__date__ = "September 6, 2017"
__maintainer__ = "Michael Tryby"
@@ -38,7 +38,7 @@ __status = "Development"
def epanet_allclose_compare(path_test, path_ref, rtol, atol):
'''
Compares results in two EPANET binary files. Using the comparison criteria
Compares results in two EPANET binary files using the comparison criteria
described in the numpy assert_allclose documentation.
(test_value - ref_value) <= atol + rtol * abs(ref_value)
@@ -67,22 +67,67 @@ def epanet_allclose_compare(path_test, path_ref, rtol, atol):
for (test, ref) in it.izip(ordr.output_generator(path_test),
ordr.output_generator(path_ref)):
if len(test) != len(ref):
if len(test[0]) != len(ref[0]):
raise ValueError('Inconsistent lengths')
# Skip over arrays that are equal
if np.array_equal(test, ref):
if np.array_equal(test[0], ref[0]):
continue
else:
np.testing.assert_allclose(test, ref, rtol, atol)
np.testing.assert_allclose(test[0], ref[0], rtol, atol)
return True
# def epanet_better_compare(path_test, path_ref, rtol, atol):
# '''
# If you don't like assert_allclose you can add another function here.
# '''
# pass
def epanet_mincdd_compare(path_test, path_ref, rtol, atol):
'''
Compares the results of two EPANET binary files using a correct decimal
digits (cdd) comparison criteria:
min cdd(test, ref) >= atol
Returns true if min cdd in the file is greater than or equal to atol,
otherwise an AssertionError is thrown.
Arguments:
path_test - path to result file being testedgit
path_ref - path to reference result file
rtol - ignored
atol - minimum allowable cdd value (i.e. 3)
Returns:
True
Raises:
ValueError()
AssertionError()
'''
min_cdd = 100.0
for (test, ref) in it.izip(ordr.output_generator(path_test),
ordr.output_generator(path_ref)):
if len(test[0]) != len(ref[0]):
raise ValueError('Inconsistent lengths')
# Skip over arrays that are equal
if np.array_equal(test[0], ref[0]):
continue
else:
diff = np.fabs(np.subtract(test[0], ref[0]))
idx = np.unravel_index(np.argmax(diff), diff.shape)
if diff[idx] != 0.0:
tmp = - np.log10(diff[idx])
if tmp < min_cdd:
min_cdd = tmp;
if np.floor(min_cdd) >= atol:
return True
else:
raise AssertionError('min_cdd=%d less than atol=%g' % (min_cdd, atol))
def epanet_report_compare(path_test, path_ref, rtol, atol):
'''

6
tools/nrtest-epanet/nrtest_epanet/output_reader.py
View File

@@ -23,7 +23,8 @@ def output_generator(path_ref):
yield element attributes. It is useful for comparing contents of binary
files for numerical regression testing.
The generator yields a Python list containing element attributes.
The generator yields a Python tuple containing an array of element
attributes and a tuple containing the element type, period, and attribute.
Arguments:
path_ref - path to result file
@@ -38,7 +39,8 @@ def output_generator(path_ref):
for element_type in oapi.ElementType:
for attribute in br.elementAttributes[element_type]:
yield br.element_attribute(element_type, period_index, attribute)
yield (br.element_attribute(element_type, period_index, attribute),
(element_type, period_index, attribute))
class OutputReader():

96
tools/nrtest-epanet/report_diff.py Normal file
View File

@@ -0,0 +1,96 @@
# -*- coding: utf-8 -*-
#
# report_diff.py
#
# Date Created: July 11, 2018
#
# Author: Michael E. Tryby
# US EPA - ORD/NRMRL
#
# system imports
import itertools as it
# third party imports
import numpy as np
# project imports
import nrtest_epanet.output_reader as ordr
def _binary_diff(path_test, path_ref, min_cdd):
for (test, ref) in it.izip(ordr.output_generator(path_test),
ordr.output_generator(path_ref)):
if len(test[0]) != len(ref[0]):
raise ValueError('Inconsistent lengths')
# Skip over arrays that are equal
if np.array_equal(test[0], ref[0]):
continue
else:
lre = _log_relative_error(test[0], ref[0])
idx = np.unravel_index(np.argmin(lre), lre.shape)
if lre[idx] < min_cdd:
_print_diff(idx, lre, test, ref)
return
def _log_relative_error(q, c):
'''
Computes log relative error, a measure of numerical accuracy.
Single precision machine epsilon is between 2^-24 and 2^-23.
Reference:
McCullough, B. D. "Assessing the Reliability of Statistical Software: Part I."
The American Statistician, vol. 52, no. 4, 1998, pp. 358-366.
'''
diff = np.subtract(q, c)
tmp_c = np.copy(c)
# If ref value is small compute absolute error
tmp_c[np.fabs(tmp_c) < 1.0e-6] = 1.0
re = np.fabs(diff)/np.fabs(tmp_c)
# If re is tiny set lre to number of digits
re[re < 1.0e-7] = 1.0e-7
# If re is very large set lre to zero
re[re > 2.0] = 1.0
return np.negative(np.log10(re))
def _print_diff(idx, lre, test, ref):
idx_val = (idx[0], ref[1])
test_val = (test[0][idx[0]])
ref_val = (ref[0][idx[0]])
diff_val = (test_val - ref_val)
lre_val = (lre[idx[0]])
print("Idx: %s\nSut: %f Ref: %f Diff: %f LRE: %f\n"
% (idx_val, test_val, ref_val, diff_val, lre_val))
def report(args):
_binary_diff(args.test, args.ref, args.mincdd)
if __name__ == '__main__':
from argparse import ArgumentParser
parser = ArgumentParser(description='EPANET benchmark difference reporting')
parser.set_defaults(func=report)
parser.add_argument('-t', '--test', default=None,
help='Path to test benchmark')
parser.add_argument('-r', '--ref', default=None,
help='Path to reference benchmark')
parser.add_argument('-mc', '--mincdd', type=int, default=3,
help='Minimum correct decimal digits')
args = parser.parse_args()
args.func(args)

3
tools/nrtest-epanet/setup.py
View File

@@ -17,6 +17,7 @@ except ImportError:
entry_points = {
'nrtest.compare': [
'epanet allclose = nrtest_epanet:epanet_allclose_compare',
'epanet mincdd = nrtest_epanet:epanet_mincdd_compare',
'epanet report = nrtest_epanet:epanet_report_compare',
# Add entry point for new comparison functions here
]
@@ -24,7 +25,7 @@ entry_points = {
setup(
name='nrtest-epanet',
version='0.4.0',
version='0.5.0',
description="EPANET extension for nrtest",
author="Michael E. Tryby",