Jakub/iid node algorithm dependencies

.gitignore

@@ -11,4 +11,5 @@
 /CMakeSettings.json
 /cmake-build-*
 **/__pycache__
-/benchmarks/pending/__debug__*
+/benchmarks/pending/__debug__*
+/benchmarks/fast/_.c

CMakeLists.txt

@@ -1,7 +1,7 @@
-project(fizz)
-
 cmake_minimum_required(VERSION 3.20 FATAL_ERROR)
 
+project(fizzer)
+
 macro(append_compiler_flags FLAGS)
     set(CMAKE_CXX_FLAGS  "${CMAKE_CXX_FLAGS} ${FLAGS}")
 endmacro()
@@ -70,12 +70,13 @@ find_package(Threads REQUIRED)
 
 # find and add Boost
 message("Searching for Boost library ...")
-find_package(Boost 1.69 REQUIRED COMPONENTS filesystem)
+set(Boost_USE_STATIC_LIBS   ON)
+find_package(Boost REQUIRED COMPONENTS filesystem)
 message("Boost STATUS:
     Includes ${Boost_INCLUDE_DIRS}
-    Libs ${Boost_LIBRARIES}
-")
+    Libs ${Boost_LIBRARIES}")
 include_directories(${Boost_INCLUDE_DIRS})
+set(BOOST_LIST_OF_LIBRARIES_TO_LINK_WITH "${Boost_LIBRARIES}")
 
 set(LLVM_EXPORT_SYMBOLS_FOR_PLUGINS "yes")
 find_package(LLVM REQUIRED CONFIG)
@@ -92,8 +93,7 @@ message("LLVM STATUS:
     Definitions ${LLVM_DEFINITIONS}
     Includes    ${LLVM_INCLUDE_DIRS}
     Libraries   ${LLVM_LIBRARY_DIRS}
-    Targets     ${LLVM_TARGETS_TO_BUILD}"
-)
+    Targets     ${LLVM_TARGETS_TO_BUILD}")
 
 
 if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
@@ -113,7 +113,9 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/src")
 # Add project specific sources
 add_subdirectory(./src)
 
-install(DIRECTORY ./benchmarks DESTINATION .)
-install(FILES ./LICENSE.txt DESTINATION .)
+if(FIZZ_BUILD_LIBS_32_BIT STREQUAL "No")
+    install(DIRECTORY ./benchmarks DESTINATION .)
+    install(FILES ./LICENSE.txt DESTINATION .)
+endif()
 
 message("Generating build files ...")

README.md

@@ -77,6 +77,11 @@ start with the **age** project:
         tasks, e.g., building benchmarks and killing non-terminating clients.
         You only need to copy the file from the `setup` folder to the folder
         `.vscode` folder.
+- (Optional) If you also want to analyze 32-bit programs, then you must also
+  build 32-bit version of Fizzer's libraries. That is done automatically via
+  Fizzer's `build.sh` script. However, 32-bit version of C++ standard library
+  must be available in the C++ compiler. On Linux (Ubuntu) you can install
+  this library using: `sudo apt install g++-multilib`
 - (optional) **SmartGit** Git GUI client: https://www.syntevo.com/smartgit/
 
 ## Downloading **SBT-Fizzer**

benchmarks/benman.py

@@ -106,99 +106,69 @@ def _execute_and_check_output(self, cmdline : str, desired_output : str, work_di
         self._execute(cmdline, os.path.dirname(desired_output) if work_dir is None else work_dir)
         ASSUMPTION(os.path.isfile(desired_output), "_execute_and_check_output(): the output is missing: " + desired_output)
 
-    def _check_outcomes(self, config : dict, outcomes : dict):
-        checked_properties_and_comparators = {
-            "termination_type": "EQ",
-            "termination_reason": "EQ",
-            "num_executions": "LE",
-            "num_covered_branchings": "GE",
-            "covered_branchings": None,
-            "num_generated_tests": "GE",
-            "num_crashes": "GE",
-            "num_boundary_violations": "LE"
-        }
-
-        def is_valid(obtained, expected, op : str) -> bool:
-            if op == "EQ": return obtained == expected
-            if op == "NE": return obtained != expected
-            if op == "LT": return obtained < expected
-            if op == "LE": return obtained <= expected
-            if op == "GT": return obtained > expected
-            if op == "GE": return obtained >= expected
-            raise Exception("Invalid comparison operator '" + op + "'.")
-
-        for property, expected_value in config["results"].items():
-            ASSUMPTION(
-                property in checked_properties_and_comparators,
-                "Unsupported key '" + property + "' in the 'results' section of benchmark's config JSON file."
-                )
-            ASSUMPTION(
-                property in outcomes,
-                "The valid key '" + property + "' was not found in the 'outcomes' JSON file."
-                )
-            if type(expected_value) in [int, float, str]:
-                if not is_valid(outcomes[property], expected_value, checked_properties_and_comparators[property]):
+    @staticmethod
+    def _add_error_message(text: str, errors: list, properties: list):
+        errors.append(("In " + "/".join(properties) + ": " if len(properties) > 0 else "") + text)
+
+    @staticmethod
+    def _epsilon_for_property(properties):
+        if len(properties) == 0: return None
+        if properties[-1] == "num_executions": return 5.0
+        return None
+
+    @staticmethod
+    def _check_outcomes(obtained, expected, errors: list, properties = []) -> bool:
+        if type(expected) is dict:
+            if type(obtained) is not dict:
+                Benchmark._add_error_message("Mismatch in JSON structure. Expected dictionary.", errors, properties)
+                return False
+            result = True
+            for key in expected:
+                if key not in obtained:
+                    Benchmark._add_error_message("Missing property: " + key, errors, properties)
+                    return False
+                r = Benchmark._check_outcomes(obtained[key], expected[key], errors, properties + [key])
+                result = result and r
+            return result
+        elif type(expected) is list:
+            if type(obtained) is not list:
+                Benchmark._add_error_message("Mismatch in JSON structure. Expected list.", errors, properties)
+                return False
+            if len(obtained) != len(expected):
+                Benchmark._add_error_message("Different list size.", errors, properties)
+                return False
+            result = True
+            for i in range(min(len(obtained), len(expected))):
+                r = Benchmark._check_outcomes(obtained[i], expected[i], errors, properties)
+                result = result and r
+            return result
+        elif type(expected) in [int, float]:
+            if type(obtained) not in [int, float]:
+                Benchmark._add_error_message("Mismatch in JSON structure. Expected int or float.", errors, properties)
+                return False
+            epsilon = Benchmark._epsilon_for_property(properties)
+            if epsilon is None:
+                if obtained != expected:
+                    Benchmark._add_error_message("Expected " + str(expected) + ", obtained " + str(obtained), errors, properties)
                     return False
             else:
-                ASSUMPTION(property == "covered_branchings", "Only 'covered_branchings' can be a 'list' property to check.")
-                ASSUMPTION(len(expected_value) % 2 == 0, "Expected covered branchings list must have even number of elements.")
-                ASSUMPTION(len(outcomes[property]) % 2 == 0, "Obtained covered branchings list must have even number of elements.")
-                def get_branchings(seq : list) -> set:
-                    result = set()
-                    if len(seq) > 0:
-                        for i in range(0, len(seq)-1, 2):
-                            result.add((seq[i], seq[i+1]))
-                    return result
-                expected_branchings = get_branchings(expected_value)
-                obtained_branchings = get_branchings(outcomes[property])
-                for x in expected_branchings:
-                    if x not in obtained_branchings:
-                        return False
-        return True
-
-    def _ok_stats_message(self, config : dict, outcomes : dict) -> str:
-        max_num_execution = config["results"]["num_executions"]
-        try:
-            num_execution = outcomes["num_executions"]
-        except Exception as e:
-            return "Unknown executions count"
-        percentage = 100.0 * num_execution / max_num_execution
-        return "#" + ("%.2f" % (percentage - 100)) + "%" if max_num_execution >= 100 and percentage < 90 else ""
-
-    def _fail_stats_message(self, config : dict, outcomes : dict) -> str:
-        expected_termination_type = config["results"]["termination_type"]
-        try:
-            termination_type = outcomes["termination_type"]
-        except Exception as e:
-            return "Unknown termination type"
-        if termination_type != expected_termination_type:
-            return termination_type
-
-        result = ""
-
-        expected_termination_reason = config["results"]["termination_reason"]
-        try:
-            termination_reason = outcomes["termination_reason"]
-        except Exception as e:
-            return "Unknown termination reason"
-        if termination_reason != expected_termination_reason:
-            result += termination_reason
-
-        max_num_execution = config["results"]["num_executions"]
-        try:
-            num_execution = outcomes["num_executions"]
-        except Exception as e:
-            return result + ("" if len(result) == 0 else ", ") + "unknown executions count"
-        if len(result) == 0 and num_execution > max_num_execution:
-            percentage = 100.0 * num_execution / max_num_execution
-            result += ("" if len(result) == 0 else ", ") + "#" + ("+%.2f" % (percentage - 100)) + "%"
-
-        return result
-
-    def _embrace_stats_message(self, msg : str) -> str:
-        if len(msg) == 0:
-            return msg
-        return "[" + msg + "]"
+                percentage = (100.0 * obtained) / expected if expected > 0 else 100.0 * obtained + 100.0
+                error = percentage - 100.0
+                if abs(error) > epsilon:
+                    Benchmark._add_error_message("Expected " + str(expected) + ", obtained " + str(obtained) + " [error: " + ("%.2f" % error) + "%]", errors, properties)
+                    return False
+            return True
+        elif type(expected) is str:
+            if type(obtained) is not str:
+                Benchmark._add_error_message("Mismatch in JSON structure. Expected string.", errors, properties)
+                return False
+            if obtained != expected:
+                Benchmark._add_error_message("Expected " + expected + ", obtained " + obtained, errors, properties)
+                return False
+            return True
+        else:
+            Benchmark._add_error_message("Unexpected JSON content [type: " + str(type(expected)) + "].", errors, properties)
+            return False
 
     def build(self, benchmarks_root_dir : str, output_root_dir : str) -> None:
         self.log("===")
@@ -216,7 +186,7 @@ def build(self, benchmarks_root_dir : str, output_root_dir : str) -> None:
                 "--skip_fuzzing",
                 "--input_file", self.src_file,
                 "--output_dir",  self.work_dir,
-                "--silent_build",
+                "--silent_mode",
                 "--save_mapping"
             ] + (["--m32"] if "m32" in self.config["args"] and self.config["args"]["m32"] is True else []),
             self.fuzz_target_file,
@@ -263,19 +233,20 @@ def fuzz(self, benchmarks_root_dir : str, output_root_dir : str) -> bool:
             output_dir
             )
 
+        errors = []
         try:
             outcomes_pathname = os.path.join(output_dir, self.name + "_outcomes.json")
             with open(outcomes_pathname, "rb") as fp:
                 outcomes = json.load(fp)
-            if self._check_outcomes(self.config, outcomes) is True:
-                stats_msg = self._embrace_stats_message(self._ok_stats_message(self.config, outcomes))
-                self.log("The outcomes are as expected => the test has PASSED. [Details: " + stats_msg + "]", "ok " + stats_msg + "\n")
+            if self._check_outcomes(outcomes, self.config["results"], errors) is True:
+                ASSUMPTION(len(errors) == 0)
+                self.log("The outcomes are as expected => the test has PASSED.", "ok\n")
                 return True
         except Exception as e:
             self.log("FAILURE due to an EXCEPTION: " + str(e), "EXCEPTION[" + str(e) + "]\n")
             return False
-        stats_msg = self._embrace_stats_message(self._fail_stats_message(self.config, outcomes))
-        self.log("The outcomes are NOT as expected => the test has FAILED. Details: " + stats_msg, "FAILED " + stats_msg + "\n")
+        error_messages = "\n    " + "\n    ".join(errors)
+        self.log("The outcomes are NOT as expected => the test has FAILED. Details:" + error_messages, "FAILED " + error_messages + "\n")
         return False
 
     def clear(self, benchmarks_root_dir : str, output_root_dir : str) -> None:
@@ -328,7 +299,7 @@ def search_for_benchmarks(folder : str) -> list:
                             pass
             return benchmarks
 
-        kinds = ["fast", "medium", "slow", "pending"]
+        kinds = ["fast", "iid_testing", "testcomp-selection-selection", "medium", "slow", "pending"]
         benchmarks = []
         if name == "all":
             for kind in kinds:

benchmarks/fast/array-1.json

@@ -7,11 +7,12 @@
         "max_stdin_bytes": 6400,
         "max_exec_milliseconds": 250,
         "max_exec_megabytes": 1024,
-        "stdin_model": "stdin_replay_bytes_then_repeat_85",
+        "stdin_model": "stdin_replay_bytes_then_repeat_zero",
         "stdout_model": "stdout_void",
         "optimizer_max_seconds": 10,
         "optimizer_max_trace_length": 1000000,
-        "optimizer_max_stdin_bytes": 1000000
+        "optimizer_max_stdin_bytes": 1000000,
+        "m32": false
     },
     "results": {
         "termination_type": "NORMAL",
@@ -21,8 +22,17 @@
         "covered_branchings": [
             2,0,    3,0
         ],
-        "num_generated_tests": 2,
-        "num_crashes": 0,
-        "num_boundary_violations": 0
+        "output_statistics": {
+            "sensitivity_analysis": {
+                "num_generated_tests": 1,
+                "num_crashes": 0,
+                "num_boundary_violations": 0
+            },
+            "STARTUP": {
+                "num_generated_tests": 1,
+                "num_crashes": 0,
+                "num_boundary_violations": 0
+            }
+        }
     }
 }

benchmarks/fast/array1_pattern.json

@@ -7,11 +7,12 @@
         "max_stdin_bytes": 6400,
         "max_exec_milliseconds": 250,
         "max_exec_megabytes": 1024,
-        "stdin_model": "stdin_replay_bytes_then_repeat_85",
+        "stdin_model": "stdin_replay_bytes_then_repeat_zero",
         "stdout_model": "stdout_void",
         "optimizer_max_seconds": 10,
         "optimizer_max_trace_length": 1000000,
-        "optimizer_max_stdin_bytes": 1000000
+        "optimizer_max_stdin_bytes": 1000000,
+        "m32": false
     },
     "results": {
         "termination_type": "NORMAL",
@@ -22,8 +23,17 @@
             1,2654435899,    1,2654435962,    3,0,    4,0,    
             5,0,    6,0,    7,0,    8,0
         ],
-        "num_generated_tests": 7,
-        "num_crashes": 0,
-        "num_boundary_violations": 2
+        "output_statistics": {
+            "sensitivity_analysis": {
+                "num_generated_tests": 4,
+                "num_crashes": 1,
+                "num_boundary_violations": 2
+            },
+            "STARTUP": {
+                "num_generated_tests": 1,
+                "num_crashes": 1,
+                "num_boundary_violations": 0
+            }
+        }
     }
 }

benchmarks/fast/big_issue.json

@@ -11,18 +11,28 @@
         "stdout_model": "stdout_void",
         "optimizer_max_seconds": 10,
         "optimizer_max_trace_length": 1000000,
-        "optimizer_max_stdin_bytes": 1000000
+        "optimizer_max_stdin_bytes": 1000000,
+        "m32": false
     },
     "results": {
         "termination_type": "NORMAL",
         "termination_reason": "FUZZING_STRATEGY_DEPLETED",
-        "num_executions": 105,
+        "num_executions": 103,
         "num_covered_branchings": 2,
         "covered_branchings": [
             2,0,    4,0
         ],
-        "num_generated_tests": 3,
-        "num_crashes": 0,
-        "num_boundary_violations": 0
+        "output_statistics": {
+            "sensitivity_analysis": {
+                "num_generated_tests": 2,
+                "num_crashes": 0,
+                "num_boundary_violations": 0
+            },
+            "STARTUP": {
+                "num_generated_tests": 1,
+                "num_crashes": 0,
+                "num_boundary_violations": 0
+            }
+        }
     }
 }