JIANG/EPANET
1
0
Fork 0
Code Issues Pull Requests Actions 4 Packages Projects Releases Wiki Activity

Merge branch 'dev' into contributor-lr

Browse Source
This commit is contained in:
Michael Tryby
2018-08-24 16:09:21 -04:00
committed by GitHub
parent 364e4987d8 761199b08e
commit f495d2c707
11 changed files with 467 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
tools/nrtest-epanet/main.py
View File

@@ -4,7 +4,9 @@ import time
import cStringIO
import itertools as it
import epanet_reader as er
# project import
import nrtest_epanet.output_reader as er
def result_compare(path_test, path_ref, comp_args):
@@ -15,29 +17,36 @@ def result_compare(path_test, path_ref, comp_args):
total = 0
output = cStringIO.StringIO()
eps = np.finfo(float).eps
min_cdd = 100.0
start = time.time()
test_reader = er.reader(path_test)
ref_reader = er.reader(path_ref)
test_reader = er.output_generator(path_test)
ref_reader = er.output_generator(path_ref)
for test, ref in it.izip(test_reader, ref_reader):
total += 1
if total%100000 == 0:
print(total)
if test.size != ref.size:
if len(test[0]) != len(ref[0]):
raise ValueError('Inconsistent lengths')
# Skip results if they are zero or equal
if np.array_equal(test, ref):
equal += 1
continue
#if np.array_equal(test, ref):
# equal += 1
# continue
else:
try:
np.testing.assert_allclose(test, ref, 1.0e-06, 2*eps)
close += 1
diff = np.fabs(np.subtract(test[0], ref[0]))
idx = np.unravel_index(np.argmax(diff), diff.shape)
if diff[idx] != 0.0:
tmp = - np.log10(diff[idx])
if tmp < min_cdd:
min_cdd = tmp;
except AssertionError as ae:
notclose += 1
output.write(str(ae))
@@ -49,8 +58,9 @@ def result_compare(path_test, path_ref, comp_args):
print(output.getvalue())
output.close()
print('equal: %d close: %d notclose: %d total: %d in %f (sec)\n' %
(equal, close, notclose, total, (stop - start)))
print('mincdd: %d in %f (sec)' % (np.floor(min_cdd), (stop - start)))
#print('equal: %d close: %d notclose: %d total: %d in %f (sec)\n' %
# (equal, close, notclose, total, (stop - start)))
if notclose > 0:
print('%d differences found\n' % notclose)
@@ -120,6 +130,8 @@ def nrtest_execute(app_path, test_path, output_path):
exit(not success)
import report_diff as rd
if __name__ == "__main__":
# app_path = "apps\\swmm-5.1.11.json"
# test_path = "tests\\examples\\example1.json"
@@ -130,7 +142,10 @@ if __name__ == "__main__":
# ref_path = "C:\\Users\\mtryby\\Workspace\\GitRepo\\Local\\epanet-testsuite\\benchmarks\\v2012"
# print(nrtest_compare(test_path, ref_path, (0.001, 0.0)))
path_test = "C:\\Users\\mtryby\\Workspace\\GitRepo\\Local\\epanet-testsuite\\benchmarks\\v2011a\\Example_3\\example3.out"
path_ref = "C:\\Users\\mtryby\\Workspace\\GitRepo\\Local\\epanet-testsuite\\benchmarks\\v2012\\Example_3\\example3.out"
result_compare(path_test, path_ref, (0.001, 0.0))
benchmark_path = "C:\\Users\\mtryby\\Workspace\\GitRepo\\michaeltryby\\epanet-lr\\nrtestsuite\\benchmarks\\"
path_test = benchmark_path + "epanet-220dev\\example2\\example2.out"
path_ref = benchmark_path + "epanet-2012\\example2\\example2.out"
#result_compare(path_test, path_ref, (0.001, 0.0))
rd.report_diff(path_test, path_ref, 2)

65
tools/nrtest-epanet/nrtest_epanet/__init__.py
View File

@@ -28,7 +28,7 @@ __copyright__ = "None"
__credits__ = "Colleen Barr, Maurizio Cingi, Mark Gray, David Hall, Bryant McDonnell"
__license__ = "CC0 1.0 Universal"
__version__ = "0.4.0"
__version__ = "0.5.0"
__date__ = "September 6, 2017"
__maintainer__ = "Michael Tryby"
@@ -38,7 +38,7 @@ __status = "Development"
def epanet_allclose_compare(path_test, path_ref, rtol, atol):
'''
Compares results in two EPANET binary files. Using the comparison criteria
Compares results in two EPANET binary files using the comparison criteria
described in the numpy assert_allclose documentation.
(test_value - ref_value) <= atol + rtol * abs(ref_value)
@@ -67,22 +67,67 @@ def epanet_allclose_compare(path_test, path_ref, rtol, atol):
for (test, ref) in it.izip(ordr.output_generator(path_test),
ordr.output_generator(path_ref)):
if len(test) != len(ref):
if len(test[0]) != len(ref[0]):
raise ValueError('Inconsistent lengths')
# Skip over arrays that are equal
if np.array_equal(test, ref):
if np.array_equal(test[0], ref[0]):
continue
else:
np.testing.assert_allclose(test, ref, rtol, atol)
np.testing.assert_allclose(test[0], ref[0], rtol, atol)
return True
# def epanet_better_compare(path_test, path_ref, rtol, atol):
# '''
# If you don't like assert_allclose you can add another function here.
# '''
# pass
def epanet_mincdd_compare(path_test, path_ref, rtol, atol):
'''
Compares the results of two EPANET binary files using a correct decimal
digits (cdd) comparison criteria:
min cdd(test, ref) >= atol
Returns true if min cdd in the file is greater than or equal to atol,
otherwise an AssertionError is thrown.
Arguments:
path_test - path to result file being testedgit
path_ref - path to reference result file
rtol - ignored
atol - minimum allowable cdd value (i.e. 3)
Returns:
True
Raises:
ValueError()
AssertionError()
'''
min_cdd = 100.0
for (test, ref) in it.izip(ordr.output_generator(path_test),
ordr.output_generator(path_ref)):
if len(test[0]) != len(ref[0]):
raise ValueError('Inconsistent lengths')
# Skip over arrays that are equal
if np.array_equal(test[0], ref[0]):
continue
else:
diff = np.fabs(np.subtract(test[0], ref[0]))
idx = np.unravel_index(np.argmax(diff), diff.shape)
if diff[idx] != 0.0:
tmp = - np.log10(diff[idx])
if tmp < min_cdd:
min_cdd = tmp;
if np.floor(min_cdd) >= atol:
return True
else:
raise AssertionError('min_cdd=%d less than atol=%g' % (min_cdd, atol))
def epanet_report_compare(path_test, path_ref, rtol, atol):
'''

6
tools/nrtest-epanet/nrtest_epanet/output_reader.py
View File

@@ -23,7 +23,8 @@ def output_generator(path_ref):
yield element attributes. It is useful for comparing contents of binary
files for numerical regression testing.
The generator yields a Python list containing element attributes.
The generator yields a Python tuple containing an array of element
attributes and a tuple containing the element type, period, and attribute.
Arguments:
path_ref - path to result file
@@ -38,7 +39,8 @@ def output_generator(path_ref):
for element_type in oapi.ElementType:
for attribute in br.elementAttributes[element_type]:
yield br.element_attribute(element_type, period_index, attribute)
yield (br.element_attribute(element_type, period_index, attribute),
(element_type, period_index, attribute))
class OutputReader():

96
tools/nrtest-epanet/report_diff.py Normal file
View File

@@ -0,0 +1,96 @@
# -*- coding: utf-8 -*-
#
# report_diff.py
#
# Date Created: July 11, 2018
#
# Author: Michael E. Tryby
# US EPA - ORD/NRMRL
#
# system imports
import itertools as it
# third party imports
import numpy as np
# project imports
import nrtest_epanet.output_reader as ordr
def _binary_diff(path_test, path_ref, min_cdd):
for (test, ref) in it.izip(ordr.output_generator(path_test),
ordr.output_generator(path_ref)):
if len(test[0]) != len(ref[0]):
raise ValueError('Inconsistent lengths')
# Skip over arrays that are equal
if np.array_equal(test[0], ref[0]):
continue
else:
lre = _log_relative_error(test[0], ref[0])
idx = np.unravel_index(np.argmin(lre), lre.shape)
if lre[idx] < min_cdd:
_print_diff(idx, lre, test, ref)
return
def _log_relative_error(q, c):
'''
Computes log relative error, a measure of numerical accuracy.
Single precision machine epsilon is between 2^-24 and 2^-23.
Reference:
McCullough, B. D. "Assessing the Reliability of Statistical Software: Part I."
The American Statistician, vol. 52, no. 4, 1998, pp. 358-366.
'''
diff = np.subtract(q, c)
tmp_c = np.copy(c)
# If ref value is small compute absolute error
tmp_c[np.fabs(tmp_c) < 1.0e-6] = 1.0
re = np.fabs(diff)/np.fabs(tmp_c)
# If re is tiny set lre to number of digits
re[re < 1.0e-7] = 1.0e-7
# If re is very large set lre to zero
re[re > 2.0] = 1.0
return np.negative(np.log10(re))
def _print_diff(idx, lre, test, ref):
idx_val = (idx[0], ref[1])
test_val = (test[0][idx[0]])
ref_val = (ref[0][idx[0]])
diff_val = (test_val - ref_val)
lre_val = (lre[idx[0]])
print("Idx: %s\nSut: %f Ref: %f Diff: %f LRE: %f\n"
% (idx_val, test_val, ref_val, diff_val, lre_val))
def report(args):
_binary_diff(args.test, args.ref, args.mincdd)
if __name__ == '__main__':
from argparse import ArgumentParser
parser = ArgumentParser(description='EPANET benchmark difference reporting')
parser.set_defaults(func=report)
parser.add_argument('-t', '--test', default=None,
help='Path to test benchmark')
parser.add_argument('-r', '--ref', default=None,
help='Path to reference benchmark')
parser.add_argument('-mc', '--mincdd', type=int, default=3,
help='Minimum correct decimal digits')
args = parser.parse_args()
args.func(args)

3
tools/nrtest-epanet/setup.py
View File

@@ -17,6 +17,7 @@ except ImportError:
entry_points = {
'nrtest.compare': [
'epanet allclose = nrtest_epanet:epanet_allclose_compare',
'epanet mincdd = nrtest_epanet:epanet_mincdd_compare',
'epanet report = nrtest_epanet:epanet_report_compare',
# Add entry point for new comparison functions here
]
@@ -24,7 +25,7 @@ entry_points = {
setup(
name='nrtest-epanet',
version='0.4.0',
version='0.5.0',
description="EPANET extension for nrtest",
author="Michael E. Tryby",