Generic Source Provenance

src/buildstream/_project.py

@@ -125,6 +125,8 @@ def __init__(
         self.sandbox: Optional[MappingNode] = None
         self.splits: Optional[MappingNode] = None
 
+        self.source_provenance_fields: Optional[MappingNode] = None  # Source provenance fields and their description
+
         #
         # Private members
         #
@@ -726,6 +728,7 @@ def _validate_toplevel_node(self, node, *, first_pass=False):
                 "sources",
                 "source-caches",
                 "junctions",
+                "source-provenance-fields",
                 "(@)",
                 "(?)",
             ]
@@ -1005,6 +1008,7 @@ def _load_second_pass(self):
                 mount = _HostMount(path, host_path, optional)
 
             self._shell_host_files.append(mount)
+        self.source_provenance_fields = config.get_mapping("source-provenance-fields")
 
     # _load_pass():
     #

src/buildstream/data/projectconfig.yaml

@@ -175,6 +175,12 @@ shell:
   #
   command: [ 'sh', '-i' ]
 
+# Define the set of fields accepted in `provenance` dictionaries of sources.
+#
+source-provenance-fields:
+  homepage: "The project homepage URL"
+  issue-tracker: "The project's issue tracking URL"
+
 # Defaults for bst commands
 #
 defaults:

src/buildstream/downloadablefilesource.py

@@ -269,6 +269,7 @@ class DownloadableFileSource(Source):
     def configure(self, node):
         self.original_url = node.get_str("url")
         self.ref = node.get_str("ref", None)
+        self.source_provenance = node.get_mapping("provenance", None)
 
         extra_data = {}
         self.url = self.translate_url(self.original_url, extra_data=extra_data)
@@ -310,6 +311,15 @@ def get_ref(self):
     def set_ref(self, ref, node):
         node["ref"] = self.ref = ref
 
+    def load_source_provenance(self, node):
+        self.source_provenance = node.get_str("provenance", None)
+
+    def get_source_provenance(self):
+        return self.source_provenance
+
+    def set_source_provenance(self, source_provenance, node):
+        node["provenance"] = self.source_provenance = source_provenance
+
     def track(self):  # pylint: disable=arguments-differ
         # there is no 'track' field in the source to determine what/whether
         # or not to update refs, because tracking a ref is always a conscious
@@ -325,7 +335,7 @@ def track(self):  # pylint: disable=arguments-differ
             )
             self.warn("Potential man-in-the-middle attack!", detail=detail)
 
-        return new_ref
+        return new_ref, None
 
     def fetch(self):  # pylint: disable=arguments-differ
 

src/buildstream/element.py

@@ -90,7 +90,7 @@
 from .sandbox import _SandboxFlags, SandboxCommandError
 from .sandbox._config import SandboxConfig
 from .sandbox._sandboxremote import SandboxRemote
-from .types import _Scope, _CacheBuildTrees, _KeyStrength, OverlapAction, _DisplayKey, _SourceProvenance
+from .types import _Scope, _CacheBuildTrees, _KeyStrength, OverlapAction, _DisplayKey
 from ._artifact import Artifact
 from ._elementproxy import ElementProxy
 from ._elementsources import ElementSources
@@ -2635,19 +2635,18 @@ def __load_sources(self, load_element):
                     del source[Symbol.DIRECTORY]
 
                 # Provenance is optional
-                provenance_node = source.get_mapping(Symbol.PROVENANCE, default=None)
-                provenance = None
+                provenance_node: MappingNode = source.get_mapping(Symbol.PROVENANCE, default=None)
                 if provenance_node:
                     del source[Symbol.PROVENANCE]
-                    provenance = _SourceProvenance.new_from_node(provenance_node)
+                    provenance_node.validate_keys(project.source_provenance_fields.keys())
 
                 meta_source = MetaSource(
                     self.name,
                     index,
                     self.get_kind(),
                     kind.as_str(),
                     directory,
-                    provenance,
+                    provenance_node,
                     source,
                     load_element.first_pass,
                 )

src/buildstream/exceptions.py

@@ -155,3 +155,8 @@ class LoadErrorReason(Enum):
     This warning will be produced when a filename for a target contains invalid
     characters in its name.
     """
+
+    UNDEFINED_SOURCE_PROVENANCE_ATTRIBUTE = 29
+    """
+    Thee source provenance attribute specified was not defined in the project config
+    """

src/buildstream/plugin.py

@@ -129,7 +129,19 @@
 import sys
 import traceback
 from contextlib import contextmanager, suppress
-from typing import IO, Any, Callable, Generator, Optional, Sequence, Dict, Tuple, TypeVar, Union, TYPE_CHECKING
+from typing import (
+    IO,
+    Any,
+    Callable,
+    Generator,
+    Optional,
+    Sequence,
+    Dict,
+    Tuple,
+    TypeVar,
+    Union,
+    TYPE_CHECKING,
+)
 from weakref import WeakValueDictionary
 
 from . import utils, _signals
@@ -160,6 +172,7 @@
 _STR_BYTES_PATH = Union[str, bytes, "os.PathLike[str]", "os.PathLike[bytes]"]
 _CMD = Union[_STR_BYTES_PATH, Sequence[_STR_BYTES_PATH]]
 
+
 # _background_job_wrapper()
 #
 # Wrapper for running jobs in the background, transparently for users
@@ -172,7 +185,9 @@
 #   target: function to execute in the background
 #   args: positional arguments to give to the target function
 #
-def _background_job_wrapper(result_queue: multiprocessing.Queue, target: Callable[..., T1], args: Any) -> None:
+def _background_job_wrapper(
+    result_queue: multiprocessing.Queue, target: Callable[..., T1], args: Any
+) -> None:
     result = None
 
     try:
@@ -265,7 +280,8 @@ class Foo(Source):
 
     try:
         __multiprocessing_context: Union[
-            multiprocessing.context.ForkServerContext, multiprocessing.context.SpawnContext
+            multiprocessing.context.ForkServerContext,
+            multiprocessing.context.SpawnContext,
         ] = multiprocessing.get_context("forkserver")
     except ValueError:
         # We are on a system without `forkserver` support. Let's default to
@@ -286,7 +302,6 @@ def __init__(
         type_tag: str,
         unique_id: Optional[int] = None,
     ):
-
         self.name = name
         """The plugin name
 
@@ -371,7 +386,9 @@ def configure(self, node: MappingNode) -> None:
 
         """
         raise ImplError(
-            "{tag} plugin '{kind}' does not implement configure()".format(tag=self.__type_tag, kind=self.get_kind())
+            "{tag} plugin '{kind}' does not implement configure()".format(
+                tag=self.__type_tag, kind=self.get_kind()
+            )
         )
 
     def preflight(self) -> None:
@@ -393,7 +410,9 @@ def preflight(self) -> None:
         will raise an error automatically informing the user that a host tool is needed.
         """
         raise ImplError(
-            "{tag} plugin '{kind}' does not implement preflight()".format(tag=self.__type_tag, kind=self.get_kind())
+            "{tag} plugin '{kind}' does not implement preflight()".format(
+                tag=self.__type_tag, kind=self.get_kind()
+            )
         )
 
     def get_unique_key(self) -> SourceRef:
@@ -440,7 +459,9 @@ def get_kind(self) -> str:
         """
         return self.__kind
 
-    def node_get_project_path(self, node, *, check_is_file=False, check_is_dir=False) -> str:
+    def node_get_project_path(
+        self, node, *, check_is_file=False, check_is_dir=False
+    ) -> str:
         """Fetches a project path from a dictionary node and validates it
 
         Paths are asserted to never lead to a directory outside of the
@@ -476,7 +497,9 @@ def node_get_project_path(self, node, *, check_is_file=False, check_is_dir=False
 
         """
 
-        return self.__project.get_path_from_node(node, check_is_file=check_is_file, check_is_dir=check_is_dir)
+        return self.__project.get_path_from_node(
+            node, check_is_file=check_is_file, check_is_dir=check_is_dir
+        )
 
     def debug(self, brief: str, *, detail: Optional[str] = None) -> None:
         """Print a debugging message
@@ -514,7 +537,13 @@ def info(self, brief: str, *, detail: Optional[str] = None) -> None:
         """
         self.__message(MessageType.INFO, brief, detail=detail)
 
-    def warn(self, brief: str, *, detail: Optional[str] = None, warning_token: Optional[str] = None) -> None:
+    def warn(
+        self,
+        brief: str,
+        *,
+        detail: Optional[str] = None,
+        warning_token: Optional[str] = None,
+    ) -> None:
         """Print a warning message, checks warning_token against project configuration
 
         Args:
@@ -533,7 +562,9 @@ def warn(self, brief: str, *, detail: Optional[str] = None, warning_token: Optio
 
             if project._warning_is_fatal(warning_token):
                 detail = detail if detail else ""
-                raise PluginError(message="{}\n{}".format(brief, detail), reason=warning_token)
+                raise PluginError(
+                    message="{}\n{}".format(brief, detail), reason=warning_token
+                )
 
         self.__message(MessageType.WARN, brief=brief, detail=detail)
 
@@ -551,7 +582,11 @@ def log(self, brief: str, *, detail: Optional[str] = None) -> None:
 
     @contextmanager
     def timed_activity(
-        self, activity_name: str, *, detail: Optional[str] = None, silent_nested: bool = False
+        self,
+        activity_name: str,
+        *,
+        detail: Optional[str] = None,
+        silent_nested: bool = False,
     ) -> Generator[None, None, None]:
         """Context manager for performing timed activities in plugins
 
@@ -589,7 +624,7 @@ def blocking_activity(
         activity_name: str,
         *,
         detail: Optional[str] = None,
-        silent_nested: bool = False
+        silent_nested: bool = False,
     ) -> T1:
         """Execute a blocking activity in the background.
 
@@ -616,7 +651,10 @@ def blocking_activity(
             the return value from `target`.
         """
         with self.__context.messenger.timed_activity(
-            activity_name, element_name=self._get_full_name(), detail=detail, silent_nested=silent_nested
+            activity_name,
+            element_name=self._get_full_name(),
+            detail=detail,
+            silent_nested=silent_nested,
         ):
             result_queue = self.__multiprocessing_context.Queue()
             proc = None
@@ -636,7 +674,10 @@ def resume_proc():
                     with suppress(ProcessLookupError):
                         os.kill(proc.pid, signal.SIGCONT)
 
-            with _signals.suspendable(suspend_proc, resume_proc), _signals.terminator(kill_proc):
+            with (
+                _signals.suspendable(suspend_proc, resume_proc),
+                _signals.terminator(kill_proc),
+            ):
                 proc = self.__multiprocessing_context.Process(
                     target=_background_job_wrapper, args=(result_queue, target, args)
                 )
@@ -659,16 +700,24 @@ def resume_proc():
                             should_continue = False
                         continue
                 else:
-                    raise PluginError("Background process died with error code {}".format(proc.exitcode))
+                    raise PluginError(
+                        "Background process died with error code {}".format(
+                            proc.exitcode
+                        )
+                    )
 
                 try:
                     proc.join(timeout=15)
                     proc.terminate()
                 except TimeoutError:
-                    raise PluginError("Background process didn't exit after 15 seconds and got killed.")
+                    raise PluginError(
+                        "Background process didn't exit after 15 seconds and got killed."
+                    )
 
             if err is not None:
-                raise PluginError("An error happened while running a blocking activity", detail=err)
+                raise PluginError(
+                    "An error happened while running a blocking activity", detail=err
+                )
 
             return result
 
@@ -944,10 +993,15 @@ def __call(
 
             self.__note_command(output_file, args, cwd)
 
-            exit_code, output = utils._call(args, cwd=cwd, env=env, stdin=stdin, stdout=stdout, stderr=stderr)
+            exit_code, output = utils._call(
+                args, cwd=cwd, env=env, stdin=stdin, stdout=stdout, stderr=stderr
+            )
 
             if fail and exit_code:
-                raise PluginError("{plugin}: {message}".format(plugin=self, message=fail), temporary=fail_temporarily)
+                raise PluginError(
+                    "{plugin}: {message}".format(plugin=self, message=fail),
+                    temporary=fail_temporarily,
+                )
 
         return (exit_code, output)
 
@@ -999,7 +1053,9 @@ def __get_full_name(self):
 
 # A local table for _prefix_warning()
 #
-__CORE_WARNINGS = [value for name, value in CoreWarnings.__dict__.items() if not name.startswith("__")]
+__CORE_WARNINGS = [
+    value for name, value in CoreWarnings.__dict__.items() if not name.startswith("__")
+]
 
 
 # _prefix_warning():

src/buildstream/plugins/sources/local.py

@@ -93,6 +93,15 @@ def get_ref(self):
     def set_ref(self, ref, node):
         pass  # pragma: nocover
 
+    def load_source_provenance(self, node):
+        pass
+
+    def get_source_provenance(self):
+        return None
+
+    def set_source_provenance(self, source_provenance, node):
+        pass
+
     def fetch(self):  # pylint: disable=arguments-differ
         # Nothing to do here for a local source
         pass  # pragma: nocover