Introduce record/replay testing mechanism

nob.c

@@ -30,7 +30,7 @@ const char *test_names[] = {
 };
 #define test_names_count ARRAY_LEN(test_names)
 
-bool build_and_run_test(Cmd *cmd, const char *test_name)
+bool build_and_run_test(Cmd *cmd, const char *test_name, bool record)
 {
     size_t mark = temp_save();
 
@@ -46,15 +46,80 @@ bool build_and_run_test(Cmd *cmd, const char *test_name)
     if (!mkdir_if_not_exists(test_cwd_path)) return false;
     if (!set_current_dir(test_cwd_path)) return false;
     cmd_append(cmd, temp_sprintf("../%s", test_name));
-    if (!cmd_run(cmd))    return false;
+    const char *test_stdout_path = temp_sprintf("../%s.stdout.txt", test_name);
+    const char *test_stderr_path = temp_sprintf("../%s.stderr.txt", test_name);
+    if (!cmd_run(cmd, .stdout_path = test_stdout_path, .stderr_path = test_stderr_path)) return false;
     if (!set_current_dir("../../../")) return false;
 
+    // TODO: implement record/replay testing for windows
+#ifndef _WIN32
+    const char *src_stdout_path = temp_sprintf("%s%s.stdout.txt", BUILD_FOLDER TESTS_FOLDER, test_name);
+    const char *src_stderr_path = temp_sprintf("%s%s.stderr.txt", BUILD_FOLDER TESTS_FOLDER, test_name);
+    const char *dst_stdout_path = temp_sprintf("%s%s.stdout.txt", TESTS_FOLDER, test_name);
+    const char *dst_stderr_path = temp_sprintf("%s%s.stderr.txt", TESTS_FOLDER, test_name);
+    if (record) {
+        if (!copy_file(src_stdout_path, dst_stdout_path)) return 1;
+        if (!copy_file(src_stderr_path, dst_stderr_path)) return 1;
+    } else {
+        cmd_append(cmd, "diff");
+        cmd_append(cmd, "-u");
+        cmd_append(cmd, src_stdout_path);
+        cmd_append(cmd, dst_stdout_path);
+        if (!cmd_run(cmd)) return false;
+
+        cmd_append(cmd, "diff");
+        cmd_append(cmd, "-u");
+        cmd_append(cmd, src_stderr_path);
+        cmd_append(cmd, dst_stderr_path);
+        if (!cmd_run(cmd)) return false;
+    }
+#endif // _WIN32
+
     nob_log(INFO, "--- %s finished ---", bin_path);
 
     temp_rewind(mark);
     return true;
 }
 
+typedef struct {
+    const char *name;
+    const char *signature;
+    const char *description;
+} Command;
+
+typedef struct {
+    Command *items;
+    size_t count;
+    size_t capacity;
+} Commands;
+
+bool command(const char *arg, Commands *commands, const char *name, const char *signature, const char *description)
+{
+    Command command = {
+        .name = name,
+        .signature = signature,
+        .description = description,
+    };
+    da_append(commands, command);
+    return strcmp(arg, name) == 0;
+}
+
+void print_available_commands(Commands commands)
+{
+    size_t max_name_width = 0;
+    size_t max_sign_width = 0;
+    da_foreach(Command, command, &commands) {
+        size_t name_width = strlen(command->name);
+        size_t sign_width = strlen(command->signature);
+        if (name_width > max_name_width) max_name_width = name_width;
+        if (sign_width > max_sign_width) max_sign_width = sign_width;
+    }
+    nob_log(INFO, "Available commands:");
+    da_foreach(Command, command, &commands) {
+        nob_log(INFO, "    %-*s %-*s - %s", (int)max_name_width, command->name, (int)max_sign_width, command->signature, command->description);
+    }
+}
+
 int main(int argc, char **argv)
 {
     set_log_handler(cancer_log_handler);
@@ -67,25 +132,45 @@ int main(int argc, char **argv)
     const char *command_name = "test";
     if (argc > 0) command_name = shift(argv, argc);
 
-    if (!mkdir_if_not_exists(BUILD_FOLDER)) return 1;
-    if (!mkdir_if_not_exists(BUILD_FOLDER TESTS_FOLDER)) return 1;
+    Commands commands = {0};
+
+    if (command(command_name, &commands, "test", "[test_names...]", "Run the tests checking their expected output")) {
+        if (!mkdir_if_not_exists(BUILD_FOLDER)) return 1;
+        if (!mkdir_if_not_exists(BUILD_FOLDER TESTS_FOLDER)) return 1;
+
+        if (argc <= 0) {
+            for (size_t i = 0; i < test_names_count; ++i) {
+                if (!build_and_run_test(&cmd, test_names[i], false)) return 1;
+            }
+            return 0;
+        }
+
+        while (argc > 0) {
+            const char *test_name = shift(argv, argc);
+            if (!build_and_run_test(&cmd, test_name, false)) return 1;
+        }
+        return 0;
+    }
+
+    if (command(command_name, &commands, "record", "[test_names...]", "Record expected output of the tests")) {
+        if (!mkdir_if_not_exists(BUILD_FOLDER)) return 1;
+        if (!mkdir_if_not_exists(BUILD_FOLDER TESTS_FOLDER)) return 1;
 
-    if (strcmp(command_name, "test") == 0) {
         if (argc <= 0) {
             for (size_t i = 0; i < test_names_count; ++i) {
-                if (!build_and_run_test(&cmd, test_names[i])) return 1;
+                if (!build_and_run_test(&cmd, test_names[i], true)) return 1;
             }
             return 0;
         }
 
         while (argc > 0) {
             const char *test_name = shift(argv, argc);
-            if (!build_and_run_test(&cmd, test_name)) return 1;
+            if (!build_and_run_test(&cmd, test_name, true)) return 1;
         }
         return 0;
     }
 
-    if (strcmp(command_name, "list") == 0) {
+    if (command(command_name, &commands, "list", "", "List available tests")) {
         nob_log(INFO, "Tests:");
         for (size_t i = 0; i < test_names_count; ++i) {
             nob_log(INFO, "    %s", test_names[i]);
@@ -94,6 +179,12 @@ int main(int argc, char **argv)
         return 0;
     }
 
+    if (command(command_name, &commands, "help", "", "Print this help message")) {
+        print_available_commands(commands);
+        return 0;
+    }
+
+    print_available_commands(commands);
     nob_log(ERROR, "Unknown command %s", command_name);
     return 1;
 }

tests/cmd_args_passing.stderr.txt

@@ -0,0 +1,2 @@
+[INFO] CMD: cc -Wall -Wextra -o print_args print_args.c
+[INFO] CMD: ./print_args foo bar 'Hello, world' '"Hello, world"' '"\` %$*@'

tests/cmd_args_passing.stdout.txt

@@ -0,0 +1,5 @@
+1: foo
+2: bar
+3: Hello, world
+4: "Hello, world"
+5: "\` %$*@

tests/cmd_redirect.stderr.txt

@@ -0,0 +1,3 @@
+[INFO] CMD: cc -Wall -Wextra -o ./echo ./echo.c
+[INFO] CMD: ./echo 'Hello, World'
+[INFO] OK

tests/cmd_run_dont_reset.stderr.txt

@@ -0,0 +1,10 @@
+[INFO] CMD: cc -Wall -Wextra -o empty empty.c
+[INFO] Reset:
+[INFO] CMD: cc -Wall -Wextra -o empty empty.c
+[INFO] Don't Reset:
+[INFO]   cc
+[INFO]   -Wall
+[INFO]   -Wextra
+[INFO]   -o
+[INFO]   empty
+[INFO]   empty.c

tests/da_append.stderr.txt

@@ -0,0 +1,10 @@
+[INFO] count = 1, capacity = 4
+[INFO] count = 2, capacity = 4
+[INFO] count = 3, capacity = 4
+[INFO] count = 4, capacity = 4
+[INFO] count = 5, capacity = 8
+[INFO] count = 6, capacity = 8
+[INFO] count = 7, capacity = 8
+[INFO] count = 8, capacity = 8
+[INFO] count = 9, capacity = 16
+[INFO] count = 10, capacity = 16

tests/da_foreach.stderr.txt

@@ -0,0 +1,3 @@
+[INFO] 0: 69
+[INFO] 1: 420
+[INFO] 2: 1337

tests/da_last.stderr.txt

@@ -0,0 +1,6 @@
+[INFO] da_last:
+[INFO] 17
+[INFO] 16
+[INFO] 15
+[INFO] 14
+[INFO] 13

tests/da_remove_unordered.stderr.txt

@@ -0,0 +1,4 @@
+[INFO] 12, 13, 14, 15
+[INFO] 15, 13, 14
+[INFO] 14, 13
+[INFO] 13

tests/da_resize.stderr.txt

@@ -0,0 +1,3 @@
+[INFO] count = 3, capacity = 4
+[INFO] count = 1, capacity = 4
+[INFO] count = 10, capacity = 16

tests/minimal_log_level.stderr.txt

@@ -0,0 +1,6 @@
+[INFO] Info Test Message
+[WARNING] Warning Test Message
+[ERROR] Error Test Message
+[WARNING] Warning Test Message
+[ERROR] Error Test Message
+[ERROR] Error Test Message

tests/nob_sv_end_with.stderr.txt

@@ -0,0 +1,8 @@
+[INFO] [SUCCESS] nob_sv_end_with(sv1, "./example.exe")
+[INFO] [SUCCESS] nob_sv_end_with(sv1, ".exe")
+[INFO] [SUCCESS] nob_sv_end_with(sv1, "e")
+[INFO] [SUCCESS] nob_sv_end_with(sv1, "")
+[INFO] [SUCCESS] nob_sv_end_with(sv2, "")
+[INFO] [SUCCESS] nob_sv_end_with(sv1, ".png")
+[INFO] [SUCCESS] nob_sv_end_with(sv1, "/path/to/example.exe")
+[INFO] [SUCCESS] nob_sv_end_with(sv2, ".obj")

tests/read_entire_dir.c

@@ -2,6 +2,13 @@
 #define NOB_IMPLEMENTATION
 #include "nob.h"
 
+int compar_paths(const void *a, const void *b)
+{
+    const char* const* ap = (const char* const*)a;
+    const char* const* bp = (const char* const*)b;
+    return strcmp(*ap, *bp);
+}
+
 int main(void)
 {
     if (!write_entire_file("foo.txt", NULL, 0)) return 1;
@@ -10,6 +17,7 @@ int main(void)
 
     Nob_File_Paths children = {0};
     if (!nob_read_entire_dir(".", &children)) return 1;
+    qsort(children.items, children.count, sizeof(*children.items), compar_paths);
     nob_log(INFO, "Tests:");
     for (size_t i = 0; i < children.count; ++i) {
         if (*children.items[i] == '.') continue;

tests/read_entire_dir.stderr.txt

@@ -0,0 +1,4 @@
+[INFO] Tests:
+[INFO]     bar.txt
+[INFO]     baz.txt
+[INFO]     foo.txt

tests/sb_appendf.stderr.txt

@@ -0,0 +1,2 @@
+[INFO] count = 0, capacity = 0, items = ""
+[INFO] count = 10, capacity = 16, items = "Hello, 69."

tests/set_get_current_dir.c

@@ -5,15 +5,11 @@ int main(void)
 {
     const char *test_current_dir = "/";
 
-    const char *current_dir = get_current_dir_temp();
-    if (current_dir == NULL) return 1;
-    nob_log(INFO, "Current Dir: %s", current_dir);
-
     nob_log(INFO, "Setting Current Dir to %s", test_current_dir);
     if (!set_current_dir(test_current_dir)) return 1;
 
-    current_dir = get_current_dir_temp();
+    const char *current_dir = get_current_dir_temp();
     if (current_dir == NULL) return 1;
-    nob_log(INFO, "Current Dir: %s", get_current_dir_temp());
+    nob_log(INFO, "Current Dir: %s", current_dir);
     return 0;
 }

tests/set_get_current_dir.stderr.txt

@@ -0,0 +1,2 @@
+[INFO] Setting Current Dir to /
+[INFO] Current Dir: /

tests/temp_aligned_alloc.stdout.txt

@@ -0,0 +1,12 @@
+temp_alloc(  0) → temp_size ==   0
+temp_alloc(  2) → temp_size ==   8
+temp_alloc(  3) → temp_size ==  16
+temp_alloc(  5) → temp_size ==  24
+temp_alloc(  7) → temp_size ==  32
+temp_alloc( 11) → temp_size ==  48
+temp_alloc( 13) → temp_size ==  64
+temp_alloc( 11) → temp_size ==  80
+temp_alloc(  7) → temp_size ==  88
+temp_alloc(  5) → temp_size ==  96
+temp_alloc(  3) → temp_size == 104
+temp_alloc(  2) → temp_size == 112

tests/temp_path_comps.stderr.txt

@@ -0,0 +1,27 @@
+[INFO] temp_dir_name:
+[INFO]     / => /
+[INFO]     /home/rexim => /home
+[INFO]     /home/rexim/ => /home
+[INFO]     /home/rexim//// => /home
+[INFO]     /home/rexim/file.txt => /home/rexim
+[INFO]      => .
+[INFO]     . => .
+[INFO]     (null) => .
+[INFO] temp_file_name:
+[INFO]     / => /
+[INFO]     /home/rexim => rexim
+[INFO]     /home/rexim/ => rexim
+[INFO]     /home/rexim//// => rexim
+[INFO]     /home/rexim/file.txt => file.txt
+[INFO]      => .
+[INFO]     . => .
+[INFO]     (null) => .
+[INFO] temp_file_ext:
+[INFO]     / => (null)
+[INFO]     /home/rexim => (null)
+[INFO]     /home/rexim/ => (null)
+[INFO]     /home/rexim//// => (null)
+[INFO]     /home/rexim/file.txt => .txt
+[INFO]      => .
+[INFO]     . => .
+[INFO]     (null) => .

tests/temp_running_executable_path.c

@@ -3,6 +3,6 @@
 
 int main(void)
 {
-    nob_log(INFO, "%s", temp_running_executable_path());
+    nob_log(INFO, "%s", temp_file_name(temp_running_executable_path()));
     return 0;
 }

tests/temp_running_executable_path.stderr.txt

@@ -0,0 +1 @@
+[INFO] temp_running_executable_path