Added generic replace build step

JSON.md

@@ -264,6 +264,15 @@ In `cfbs.json`'s `"steps"`, the build step name must be separated from the rest
   - Converts the input data for a module into the augments format and merges it with the target augments file.
   - Source is relative to module directory and target is relative to `out/masterfiles`.
     - In most cases, the build step should be: `input ./input.json def.json`
+- `replace <n> <a> <b> <filename>`
+
+  - Replace string `<a>` with string `<b>`, exactly `<n>` times, in file `filename`.
+  - string `<b>` must not contain string `<a>`, as that could lead to confusing / recursive replacement situations.
+  - The number of occurences is strict: It will error if the string cannot be found, cannot be replaced exactly `<n>` times, or can still be found after replacements are done.
+    (This is to try to catch mistakes).
+  - `n` must be an integer, from 1 to 1000, and may optionally have a trailing `+` to signify "or more".
+    At most 1000 replacements will be performed, regardless of whether you specify `+` or not.
+
 - `replace_version <to_replace> <filename>`
   - Replace the string inside the file with the version number of that module.
   - The module must have a version and the string must occur exactly once in the file.

cfbs/build.py

@@ -1,7 +1,18 @@
+"""
+Functions for performing the core part of 'cfbs build'
+
+This module contains the code for performing the actual build,
+converting a project into a ready to deploy policy set.
+To achieve this, we iterate over all the build steps in all
+the modules running the appropriate file and shell operations.
+
+There are some preliminary parts of 'cfbs build' implemented
+elsewhere, like validation and downloading modules.
+"""
+
 import os
 import logging as log
 import shutil
-from typing import List, Tuple
 from cfbs.utils import (
     canonify,
     cp,
@@ -19,19 +30,12 @@
     write_json,
 )
 from cfbs.pretty import pretty, pretty_file
-
-AVAILABLE_BUILD_STEPS = {
-    "copy": 2,
-    "run": "1+",
-    "delete": "1+",
-    "json": 2,
-    "append": 2,
-    "directory": 2,
-    "input": 2,
-    "policy_files": "1+",
-    "bundles": "1+",
-    "replace_version": 2,  # string to replace and filename
-}
+from cfbs.validate import (
+    AVAILABLE_BUILD_STEPS,
+    MAX_REPLACEMENTS,
+    step_has_valid_arg_count,
+    split_build_step,
+)
 
 
 def init_out_folder():
@@ -74,27 +78,53 @@ def _generate_augment(module_name, input_data):
     return augment
 
 
-def split_build_step(command) -> Tuple[str, List[str]]:
-    terms = command.split(" ")
-    operation, args = terms[0], terms[1:]
-    return operation, args
-
-
-def step_has_valid_arg_count(args, expected):
-    actual = len(args)
-
-    if type(expected) is int:
-        if actual != expected:
-            return False
-
-    else:
-        # Only other option is a string of 1+, 2+ or similar:
-        assert type(expected) is str and expected.endswith("+")
-        expected = int(expected[0:-1])
-        if actual < expected:
-            return False
+def _perform_replace_step(n, a, b, filename):
+    or_more = False
+    if n.endswith("+"):
+        n = n[0:-1]
+        or_more = True
+    n = int(n)
+    if n <= 0:
+        user_error("replace build step cannot replace something %s times" % (n))
+    if n > MAX_REPLACEMENTS or n == MAX_REPLACEMENTS and or_more:
+        user_error(
+            "replace build step cannot replace something more than %s times"
+            % (MAX_REPLACEMENTS)
+        )
+    if a in b and (n >= 2 or or_more):
+        user_error(
+            "'%s' must not contain '%s' (could lead to recursive replacing)" % (a, b)
+        )
+    if not os.path.isfile(filename):
+        user_error("No such file '%s' in replace build step" % (filename,))
+    try:
+        with open(filename, "r") as f:
+            content = f.read()
+    except:
+        user_error("Could not open/read '%s' in replace build step" % (filename,))
+    new_content = previous_content = content
+    for i in range(0, n):
+        previous_content = new_content
+        new_content = previous_content.replace(a, b, 1)
+        if new_content == previous_content:
+            user_error(
+                "replace build step could only replace '%s' in '%s' %s times, not %s times (required)"
+                % (a, filename, i, n)
+            )
 
-    return True
+    if or_more:
+        for i in range(n, MAX_REPLACEMENTS):
+            previous_content = new_content
+            new_content = previous_content.replace(a, b, 1)
+            if new_content == previous_content:
+                break
+    if a in new_content:
+        user_error("too many occurences of '%s' in '%s'" % (a, filename))
+    try:
+        with open(filename, "w") as f:
+            f.write(new_content)
+    except:
+        user_error("Failed to write to '%s'" % (filename,))
 
 
 def _perform_build_step(module, step, max_length):
@@ -260,6 +290,12 @@ def _perform_build_step(module, step, max_length):
             merged = augment
         log.debug("Merged def.json: %s", pretty(merged))
         write_json(path, merged)
+    elif operation == "replace":
+        assert len(args) == 4
+        print("%s replace '%s'" % (prefix, "' '".join(args)))
+        n, a, b, file = args
+        file = os.path.join(destination, file)
+        _perform_replace_step(n, a, b, file)
     elif operation == "replace_version":
         assert len(args) == 2
         print("%s replace_version '%s'" % (prefix, "' '".join(args)))

cfbs/validate.py

@@ -1,15 +1,75 @@
+"""
+Functions for performing the core part of 'cfbs validate'
+
+Iterate over the JSON structure from cfbs.json, and check
+the contents against validation rules.
+
+Currently, we are not very strict with validation in other
+commands, when you run something like 'cfbs build',
+many things only produce warnings. This is for backwards
+compatibility and we might choose to turn those warnings
+into errors in the future.
+
+Be careful about introducing dependencies to other parts
+of the codebase, such as build.py - We want validate.py
+to be relatively easy to reuse in various places without
+accidentally introducing circular dependencies.
+Thus, for example, the common parts needed by both build.py
+and validate.py, should be in utils.py or validate.py,
+not in build.py.
+"""
+
 import argparse
 import sys
 import re
 from collections import OrderedDict
+from typing import List, Tuple
 
 from cfbs.utils import (
     is_a_commit_hash,
     user_error,
 )
 from cfbs.pretty import TOP_LEVEL_KEYS, MODULE_KEYS
 from cfbs.cfbs_config import CFBSConfig
-from cfbs.build import AVAILABLE_BUILD_STEPS, step_has_valid_arg_count, split_build_step
+
+AVAILABLE_BUILD_STEPS = {
+    "copy": 2,
+    "run": "1+",
+    "delete": "1+",
+    "json": 2,
+    "append": 2,
+    "directory": 2,
+    "input": 2,
+    "policy_files": "1+",
+    "bundles": "1+",
+    "replace": 4,  # n, a, b, filename
+    "replace_version": 2,  # string to replace and filename
+}
+
+MAX_REPLACEMENTS = 1000
+
+
+def split_build_step(command) -> Tuple[str, List[str]]:
+    terms = command.split(" ")
+    operation, args = terms[0], terms[1:]
+    return operation, args
+
+
+def step_has_valid_arg_count(args, expected):
+    actual = len(args)
+
+    if type(expected) is int:
+        if actual != expected:
+            return False
+
+    else:
+        # Only other option is a string of 1+, 2+ or similar:
+        assert type(expected) is str and expected.endswith("+")
+        expected = int(expected[0:-1])
+        if actual < expected:
+            return False
+
+    return True
 
 
 class CFBSValidationError(Exception):
@@ -130,6 +190,31 @@ def validate_config(config, empty_build_list_ok=False):
         return 1
 
 
+def validate_build_step(name, i, operation, args):
+    if not operation in AVAILABLE_BUILD_STEPS:
+        raise CFBSValidationError(
+            name,
+            'Unknown operation "%s" in "steps", must be one of: %s (build step %s in module "%s")'
+            % (operation, ", ".join(AVAILABLE_BUILD_STEPS), i, name),
+        )
+    expected = AVAILABLE_BUILD_STEPS[operation]
+    actual = len(args)
+    if not step_has_valid_arg_count(args, expected):
+        if type(expected) is int:
+            raise CFBSValidationError(
+                name,
+                "The %s build step expects %d arguments, %d were given (build step "
+                % (operation, expected, actual),
+            )
+        else:
+            expected = int(expected[0:-1])
+            raise CFBSValidationError(
+                name,
+                "The %s build step expects %d or more arguments, %d were given"
+                % (operation, expected, actual),
+            )
+
+
 def _validate_module_object(context, name, module, config):
     def validate_alias(name, module, context):
         if context == "index":
@@ -261,37 +346,15 @@ def validate_steps(name, module):
             raise CFBSValidationError(name, '"steps" must be of type list')
         if not module["steps"]:
             raise CFBSValidationError(name, '"steps" must be non-empty')
-        for step in module["steps"]:
+        for i, step in enumerate(module["steps"]):
             if type(step) != str:
                 raise CFBSValidationError(name, '"steps" must be a list of strings')
             if not step or step.strip() == "":
                 raise CFBSValidationError(
                     name, '"steps" must be a list of non-empty / non-whitespace strings'
                 )
             operation, args = split_build_step(step)
-            if not operation in AVAILABLE_BUILD_STEPS:
-                x = ", ".join(AVAILABLE_BUILD_STEPS)
-                raise CFBSValidationError(
-                    name,
-                    'Unknown operation "%s" in "steps", must be one of: (%s)'
-                    % (operation, x),
-                )
-            expected = AVAILABLE_BUILD_STEPS[operation]
-            actual = len(args)
-            if not step_has_valid_arg_count(args, expected):
-                if type(expected) is int:
-                    raise CFBSValidationError(
-                        name,
-                        "The %s build step expects %d arguments, %d were given"
-                        % (operation, expected, actual),
-                    )
-                else:
-                    expected = int(expected[0:-1])
-                    raise CFBSValidationError(
-                        name,
-                        "The %s build step expects %d or more arguments, %d were given"
-                        % (operation, expected, actual),
-                    )
+            validate_build_step(name, i, operation, args)
 
     def validate_url_field(name, module, field):
         assert field in module

tests/shell/044_replace.sh

@@ -0,0 +1,21 @@
+set -e
+set -x
+cd tests/
+mkdir -p ./tmp/
+cd ./tmp/
+
+# Set up the project we will build:
+cp ../shell/044_replace/example-cfbs.json ./cfbs.json
+mkdir -p subdir
+cp ../shell/044_replace/subdir/example.py ./subdir/example.py
+cp ../shell/044_replace/subdir/example.expected.py ./subdir/example.expected.py
+
+cfbs build
+
+ls out/masterfiles/services/cfbs/subdir/example.py
+
+# Replace should have changed it:
+! diff ./subdir/example.py out/masterfiles/services/cfbs/subdir/example.py > /dev/null
+
+# This is the expected content:
+diff ./subdir/example.expected.py out/masterfiles/services/cfbs/subdir/example.py

tests/shell/044_replace/example-cfbs.json

@@ -0,0 +1,19 @@
+{
+  "name": "Example project",
+  "description": "Example description",
+  "type": "policy-set",
+  "git": true,
+  "build": [
+    {
+      "name": "./subdir/",
+      "description": "Local subdirectory added using cfbs command line",
+      "added_by": "cfbs add",
+      "steps": [
+        "copy example.py services/cfbs/subdir/example.py",
+        "replace 1 foo bar services/cfbs/subdir/example.py",
+        "replace 2 alice bob services/cfbs/subdir/example.py",
+        "replace 1+ lorem ipsum services/cfbs/subdir/example.py"
+      ]
+    }
+  ]
+}

tests/shell/044_replace/subdir/example.expected.py

@@ -0,0 +1,4 @@
+if __name__ == "__main__":
+    print("bar")
+    print("bob,bob")
+    print("ipsum,ipsum,ipsum")

tests/shell/044_replace/subdir/example.py

@@ -0,0 +1,4 @@
+if __name__ == "__main__":
+    print("foo")
+    print("alice,alice")
+    print("lorem,lorem,lorem")

tests/shell/all.sh

@@ -47,5 +47,6 @@ bash tests/shell/040_add_added_by_field_update_2.sh
 bash tests/shell/041_add_multidep.sh
 bash tests/shell/042_update_from_url.sh
 bash tests/shell/043_replace_version.sh
+bash tests/shell/044_replace.sh
 
 echo "All cfbs shell tests completed successfully!"