Adding mincdd comparison and better debug information

tools/nrtest-epanet/main.py

@@ -4,7 +4,9 @@ import time
 import cStringIO
 import itertools as it
 
-import epanet_reader as er
+# project import
+import nrtest_epanet.output_reader as er
+
 
 def result_compare(path_test, path_ref, comp_args):
 
@@ -15,28 +17,35 @@ def result_compare(path_test, path_ref, comp_args):
     total = 0
     output = cStringIO.StringIO()
     eps =  np.finfo(float).eps
+    min_cdd = 100.0
     
     start = time.time()
     
-    test_reader = er.reader(path_test)
+    test_reader = er.output_generator(path_test)
-    ref_reader = er.reader(path_ref)
+    ref_reader = er.output_generator(path_ref)
     
     for test, ref in it.izip(test_reader, ref_reader):
         total += 1
         if total%100000 == 0:
             print(total)
         
-        if test.size != ref.size:
+        if len(test[0]) != len(ref[0]):
             raise ValueError('Inconsistent lengths')
         
         # Skip results if they are zero or equal
-        if np.array_equal(test, ref):
+        #if np.array_equal(test, ref):
-            equal += 1
+        #    equal += 1
-            continue
+        #    continue
         else:
             try:
-                np.testing.assert_allclose(test, ref, 1.0e-06, 2*eps)
+                diff = np.fabs(np.subtract(test[0], ref[0]))
-                close += 1
+                idx = np.unravel_index(np.argmax(diff), diff.shape)
+                
+                if diff[idx] != 0.0:
+                    tmp = - np.log10(diff[idx])
+                    
+                    if tmp < min_cdd:
+                        min_cdd = tmp;
                         
             except AssertionError as ae:
                 notclose += 1
@@ -49,8 +58,9 @@ def result_compare(path_test, path_ref, comp_args):
     print(output.getvalue())
     output.close()
     
-    print('equal: %d  close: %d  notclose: %d  total: %d  in %f (sec)\n' % 
+    print('mincdd: %d in %f (sec)' % (np.floor(min_cdd), (stop - start)))
-          (equal, close, notclose, total, (stop - start)))
+    #print('equal: %d  close: %d  notclose: %d  total: %d  in %f (sec)\n' % 
+    #      (equal, close, notclose, total, (stop - start)))
 
     if notclose > 0:
         print('%d differences found\n' % notclose)
@@ -130,7 +140,8 @@ if __name__ == "__main__":
 #    ref_path  = "C:\\Users\\mtryby\\Workspace\\GitRepo\\Local\\epanet-testsuite\\benchmarks\\v2012"
 #    print(nrtest_compare(test_path, ref_path, (0.001, 0.0)))
 
+    benchmark_path = "C:\\Users\\mtryby\\Workspace\\GitRepo\\michaeltryby\\epanet-lr\\nrtestsuite\\benchmarks\\"
+    path_test = benchmark_path + "epanet-220dev\\example1\\example1.out"
+    path_ref  = benchmark_path + "epanet-2012\\example1\\example1.out"
 
-    path_test = "C:\\Users\\mtryby\\Workspace\\GitRepo\\Local\\epanet-testsuite\\benchmarks\\v2011a\\Example_3\\example3.out"
-    path_ref  = "C:\\Users\\mtryby\\Workspace\\GitRepo\\Local\\epanet-testsuite\\benchmarks\\v2012\\Example_3\\example3.out"    
     result_compare(path_test, path_ref, (0.001, 0.0))

tools/nrtest-epanet/nrtest_epanet/__init__.py

@@ -38,7 +38,7 @@ __status  = "Development"
 
 def epanet_allclose_compare(path_test, path_ref, rtol, atol):
     ''' 
-    Compares results in two EPANET binary files. Using the comparison criteria 
+    Compares results in two EPANET binary files using the comparison criteria 
     described in the numpy assert_allclose documentation. 
             
         (test_value - ref_value) <= atol + rtol * abs(ref_value) 
@@ -67,22 +67,67 @@ def epanet_allclose_compare(path_test, path_ref, rtol, atol):
     for (test, ref) in it.izip(ordr.output_generator(path_test), 
                                ordr.output_generator(path_ref)):
         
-        if len(test) != len(ref):
+        if len(test[0]) != len(ref[0]):
             raise ValueError('Inconsistent lengths')
         
         # Skip over arrays that are equal
-        if np.array_equal(test, ref):
+        if np.array_equal(test[0], ref[0]):
             continue
         else:
-            np.testing.assert_allclose(test, ref, rtol, atol)
+            np.testing.assert_allclose(test[0], ref[0], rtol, atol)
 
     return True
 
-# def epanet_better_compare(path_test, path_ref, rtol, atol):
+
-#     '''
+def epanet_mincdd_compare(path_test, path_ref, rtol, atol):
-#     If you don't like assert_allclose you can add another function here. 
+    '''
-#     '''
+    Compares the results of two EPANET binary files using a correct decimal
-#     pass
+    digits (cdd) comparison criteria:
+
+           min cdd(test, ref) >= atol
+
+    Returns true if min cdd in the file is greater than or equal to atol,
+    otherwise an AssertionError is thrown.
+
+    Arguments:
+        path_test - path to result file being testedgit
+        path_ref  - path to reference result file
+        rtol - ignored
+        atol - minimum allowable cdd value (i.e. 3)
+
+    Returns:
+        True
+
+    Raises:
+        ValueError()
+        AssertionError()
+    '''
+    min_cdd = 100.0
+
+    for (test, ref) in it.izip(ordr.output_generator(path_test), 
+                               ordr.output_generator(path_ref)):
+
+        if len(test[0]) != len(ref[0]):
+            raise ValueError('Inconsistent lengths')
+
+        # Skip over arrays that are equal
+        if np.array_equal(test[0], ref[0]):
+            continue
+        else:
+            diff = np.fabs(np.subtract(test[0], ref[0]))
+            idx = np.unravel_index(np.argmax(diff), diff.shape)
+
+            if diff[idx] != 0.0:
+                tmp =  - np.log10(diff[idx])
+
+                if tmp < min_cdd:
+                    min_cdd = tmp;
+
+    if np.floor(min_cdd) >= atol:
+        return True
+    else:
+        raise AssertionError('min_cdd=%d less than atol=%g' % (min_cdd, atol))
+
 
 def epanet_report_compare(path_test, path_ref, rtol, atol):
     '''

tools/nrtest-epanet/nrtest_epanet/output_reader.py

@@ -23,7 +23,8 @@ def output_generator(path_ref):
     yield element attributes. It is useful for comparing contents of binary 
     files for numerical regression testing. 
     
-    The generator yields a Python list containing element attributes. 
+    The generator yields a Python tuple containing an array of element 
+    attributes and a tuple containing the element type, period, and attribute. 
     
     Arguments: 
         path_ref - path to result file
@@ -38,7 +39,8 @@ def output_generator(path_ref):
             for element_type in oapi.ElementType:
                 for attribute in br.elementAttributes[element_type]:
         
-                    yield br.element_attribute(element_type, period_index, attribute)
+                    yield (br.element_attribute(element_type, period_index, attribute), 
+                           (element_type, period_index, attribute))
 
 
 class OutputReader():    

tools/nrtest-epanet/setup.py

@@ -17,6 +17,7 @@ except ImportError:
 entry_points = {
     'nrtest.compare': [
         'epanet allclose = nrtest_epanet:epanet_allclose_compare',
+        'epanet mincdd = nrtest_epanet:epanet_mincdd_compare',
         'epanet report = nrtest_epanet:epanet_report_compare',
         # Add entry point for new comparison functions here
     ]