From 040415b14a23e6749f985f6f3599941e3ddb3165 Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Mon, 4 May 2026 16:19:06 +0200 Subject: [PATCH 01/20] first part of the provenance collection draft --- src/hermes/commands/harvest/base.py | 67 ++++++++++- src/hermes/model/provenance/ld_prov.py | 158 +++++++++++++++++++++++++ src/hermes/model/types/ld_list.py | 2 +- 3 files changed, 225 insertions(+), 2 deletions(-) create mode 100644 src/hermes/model/provenance/ld_prov.py diff --git a/src/hermes/commands/harvest/base.py b/src/hermes/commands/harvest/base.py index 0d3d9e5f..93b23601 100644 --- a/src/hermes/commands/harvest/base.py +++ b/src/hermes/commands/harvest/base.py @@ -12,6 +12,8 @@ from hermes.error import HermesPluginRunError, MisconfigurationError from hermes.model.context_manager import HermesContext from hermes.model import SoftwareMetadata +from hermes.model.provenance.ld_prov import ld_prov_list, ld_prov_node +from hermes.model.types.ld_context import ALL_CONTEXTS class HermesHarvestPlugin(HermesPlugin): @@ -19,10 +21,19 @@ class HermesHarvestPlugin(HermesPlugin): TODO: describe the harvesting process and how this is mapped to this plugin. """ + def __init__(self): + self.io_operations: list[tuple[dict, dict, dict]] = [] + super().__init__() def __call__(self, command: HermesCommand) -> SoftwareMetadata: pass + def load(): + pass + + def write(): + pass + class HarvestSettings(BaseModel): """Generic harvesting settings.""" @@ -37,9 +48,11 @@ class HermesHarvestCommand(HermesCommand): settings_class = HarvestSettings def __call__(self, args: argparse.Namespace) -> None: - self.log.info("# Metadata harvesting") self.args = args + self.log.info("# Load provenance from old harvest or create new document.") + prov_doc, base_plugin = self.init_provenance_document() + self.log.info("# Metadata harvesting") if len(self.settings.sources) == 0: self.log.critical("# No harvest plugin was configured to be run and loaded.") raise MisconfigurationError("No harvest plugin was configured to be run and loaded.") @@ -66,6 +79,24 @@ def __call__(self, args: argparse.Namespace) -> None: except Exception: self.log.exception(f"### Unknown error while executing the {plugin_name} plugin, skipping it now.") continue + self.remove_provenance_info_for_plugin(prov_doc, plugin_name) + + plugin = prov_doc.add_hermes_plugin("harvest", plugin_name) + plugin_io_operations = plugin_func.io_operations # liste von drei Tupeln (input_file, load_function, output) + for plugin_io_operation in plugin_io_operations: + loaded_source = prov_doc.add_entity() + loaded_source.update(plugin_io_operation[0]) + io_op = prov_doc.add_activity() + plugin_io_operation[1]["prov:wasAssociatedWith"] = [base_plugin.ref, plugin.ref] + plugin_io_operation[1]["prov:used"] = loaded_source.ref + io_op.update(plugin_io_operation[1]) + loaded_data = prov_doc.add_entity() + plugin_io_operation[2].update({ + "prov:wasAttributedTo": plugin.ref, + "prov:wasDerivedFrom": loaded_source.ref, + "prov:wasGeneratedBy": io_op.ref + }) + loaded_data.update(plugin_io_operations[2]) self.log.info(f"### Store metadata harvested by {plugin_name} plugin") # store harvested data @@ -76,3 +107,37 @@ def __call__(self, args: argparse.Namespace) -> None: if not harvested_any: self.log.critical("No harvest plugin ran successfully.") raise HermesPluginRunError("No harvest plugin ran successfully.") + + def init_provenance_document(self) -> tuple[ld_prov_list, ld_prov_node]: + ctx = HermesContext() + ctx.prepare_step("harvest") + with ctx["provenance"] as cache: + try: + ld_prov_doc = ld_prov_list.from_list(cache["codemeta"], container_type="@graph", context=ALL_CONTEXTS) + return ld_prov_doc, ld_prov_doc.shallow_search({"schema:name": lambda doc, node: node["schema:name"][0].find("harvest base plugin") != -1})[0] + except KeyError: + pass + prov_doc = ld_prov_list() + prov_doc.init_hermes_agents() + return prov_doc, prov_doc.add_hermes_base_plugin("harvest") + + def remove_provenance_info_for_plugin(self, prov_doc, plugin) -> None: + plugin = prov_doc.shallow_search({ + "schema:name": (lambda doc, node: f"harvest plugin {plugin}" in node["schema:name"]), + }) + if len(plugin) == 0: + return + # two passes are needed because the nodes are nested exactly two levels + related = prov_doc.shallow_search({ + "prov:wasAssociatedWith": (lambda doc, node: plugin.ref in node["prov:wasAssociatedWith"]), + "prov:wasAttributedTo": (lambda doc, node: plugin.ref in node["prov:wasAttributedTo"]) + }) + ids = [plugin.ref, *(rel.ref for rel in related)] + related = prov_doc.shallow_search({ + f"prov:{key}": (lambda doc, node: any(id in node[f"prov:{key}"] for id in ids)) for key in [ + "wasAssociatedWith", "wasAttributedTo", "wasGeneratedBy", "used", "wasDerivedFrom", "wasInformedBy" + ] + }) + for item in related: + items = prov_doc.shallow_search({"@id": (lambda doc, node: node["@id"] == item["@id"])}) + del prov_doc[items[0].index] diff --git a/src/hermes/model/provenance/ld_prov.py b/src/hermes/model/provenance/ld_prov.py new file mode 100644 index 00000000..2c2a10f7 --- /dev/null +++ b/src/hermes/model/provenance/ld_prov.py @@ -0,0 +1,158 @@ +# SPDX-FileCopyrightText: 2026 German Aerospace Center (DLR) +# +# SPDX-License-Identifier: Apache-2.0 + +# SPDX-FileContributor: Michael Fritzsche + +from typing import Optional, Union +from typing_extensions import Self +import uuid + +from hermes import utils +from hermes.model.types import ld_dict, ld_list +from hermes.model.types.ld_container import BASIC_TYPE, EXPANDED_JSON_LD_VALUE, JSON_LD_CONTEXT_DICT, TIME_TYPE +from hermes.model.types.ld_context import ALL_CONTEXTS, iri_map + + +class ld_prov_container: + def _to_python( + self: Self, + full_iri: str, + ld_value: Union[EXPANDED_JSON_LD_VALUE, dict[str, EXPANDED_JSON_LD_VALUE], list[str], str] + ) -> Union["ld_prov_node", "ld_prov_list", BASIC_TYPE, TIME_TYPE]: + item = super()._to_python(full_iri, ld_value) + if isinstance(item, ld_list): + return ld_prov_list( + data=item._data, parent=item.parent, key=item.key, index=item.index, context=item.context + ) + elif isinstance(item, ld_dict): + return ld_prov_node( + data=item._data, parent=item.parent, key=item.key, index=item.index, context=item.context + ) + return item + + +class ld_prov_list(ld_list): + NODE_IRI_FORMAT = "graph://{uuid}/{index}" + PROV_DOC_IRI = iri_map['hermes-rt', "graph"] + + def __init__( + self: Self, + *, + data: EXPANDED_JSON_LD_VALUE = [{"@graph": []}], + parent: Optional[Union[ld_dict, ld_list]] = None, + key: Optional[str] = PROV_DOC_IRI, + index: Optional[int] = None, + context: Optional[list[Union[str, JSON_LD_CONTEXT_DICT]]] = ALL_CONTEXTS + ) -> None: + self.id = uuid.uuid1() + self.node_index = 0 + super().__init__([{"@graph": []}], parent=parent, key=key, index=index, context=context) + + def __getitem__( + self: Self, index: Union[int, slice] + ) -> Union[ + BASIC_TYPE, + TIME_TYPE, + "ld_prov_node", + "ld_prov_list", + list[Union[BASIC_TYPE, TIME_TYPE, "ld_prov_node", "ld_prov_list"]] + ]: + item = super().__getitem__(index) + if isinstance(item, ld_list): + return ld_prov_list( + data=item._data, parent=item.parent, key=item.key, index=item.index, context=item.context + ) + elif isinstance(item, ld_dict): + return ld_prov_node( + data=item._data, parent=item.parent, key=item.key, index=item.index, context=item.context + ) + return item + + def next_node_iri(self) -> str: + self.node_index += 1 + return self.NODE_IRI_FORMAT.format(uuid=self.id, index=self.node_index) + + def add_activity(self) -> "ld_prov_node": + self.append({"@id": self.next_node_iri(), "@type": "prov:Activity"}) + return self[-1] + + def add_agent(self) -> "ld_prov_node": + self.append({"@id": self.next_node_iri(), "@type": "prov:Agent"}) + return self[-1] + + def add_entity(self) -> "ld_prov_node": + self.append({"@id": self.next_node_iri(), "@type": "prov:Entity"}) + return self[-1] + + def init_hermes_agents(self) -> "ld_prov_node": + hermes = self.add_agent() + hermes.update({ + "schema:name": utils.hermes_name, + "schema:version": utils.hermes_version, + "schema:url": utils.hermes_urls, + }) + hermes["@type"].append("schema:SoftwareApplication") + node = self.add_agent() + node.update({ + "schema:name": utils.hermes_name + " cache", + "schema:version": utils.hermes_version, + "prov:actedOnBehalfOf": hermes.ref + }) + node["@type"].append("schema:SoftwareApplication") + return node + + def add_hermes_command(self, step) -> "ld_prov_node": + node = self.add_agent() + node.update({ + "schema:name": f"{utils.hermes_name} {step} command", + "schema:version": utils.hermes_version, + "prov:actedOnBehalfOf": self.shallow_search( + {"schema:name": (lambda doc, node: node["schema:name"] == utils.hermes_name)} + ) + }) + node["@type"].append("schema:SoftwareApplication") + return node + + def add_hermes_base_plugin(self, step) -> "ld_prov_node": + node = self.add_agent() + node.update({ + "schema:name": f"{utils.hermes_name} {step} base plugin", + "schema:version": utils.hermes_version, + "prov:actedOnBehalfOf": self.shallow_search( + {"schema:name": (lambda doc, node: node["schema:name"] == f"{utils.hermes_name} {step} command")} + ) + }) + node["@type"].append("schema:SoftwareApplication") + return node + + def add_hermes_plugin(self, step, name) -> "ld_prov_node": + node = self.add_agent() + # TODO: add version + node.update({ + "schema:name": f"{utils.hermes_name} {step} plugin '{name}'", + "prov:actedOnBehalfOf": self.shallow_search( + {"schema:name": (lambda doc, node: node["schema:name"] == f"{utils.hermes_name} {step} base plugin")} + ) + }) + node["@type"].append("schema:SoftwareApplication") + return node + + def shallow_search(self, query: dict) -> list["ld_prov_node"]: + return [ + item for item in self for key, test in query.items() if key in item and test(self, item) + ] + + +class ld_prov_node(ld_dict): + def __init__( + self: Self, + data: list[dict[str, EXPANDED_JSON_LD_VALUE]], + *, + parent: Optional[Union[ld_dict, ld_list]] = None, + key: Optional[str] = None, + index: Optional[int] = None, + context: Optional[list[Union[str, JSON_LD_CONTEXT_DICT]]] = ALL_CONTEXTS + ) -> None: + self.id = uuid.uuid1() + super().__init__(data, parent=parent, key=key, index=index, context=context) diff --git a/src/hermes/model/types/ld_list.py b/src/hermes/model/types/ld_list.py index 14331472..003cda82 100644 --- a/src/hermes/model/types/ld_list.py +++ b/src/hermes/model/types/ld_list.py @@ -589,7 +589,7 @@ def from_list( key: Optional[str] = None, context: Optional[Union[str, JSON_LD_CONTEXT_DICT, list[Union[str, JSON_LD_CONTEXT_DICT]]]] = None, container_type: str = "@set" - ) -> ld_list: + ) -> Self: """ Creates a ld_list from the given list with the given parent, key, context and container_type.\n Note that only container_type '@set' is valid for key '@type'.\n From a96821eb7964dfa9c1802c3049cae1e48a3d70b1 Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Mon, 4 May 2026 17:21:28 +0200 Subject: [PATCH 02/20] first draft for provenance recording of harvest command --- src/hermes/commands/harvest/base.py | 64 +++++++++++++++++++++++--- src/hermes/model/provenance/ld_prov.py | 12 ++--- 2 files changed, 64 insertions(+), 12 deletions(-) diff --git a/src/hermes/commands/harvest/base.py b/src/hermes/commands/harvest/base.py index 93b23601..61e2b3e7 100644 --- a/src/hermes/commands/harvest/base.py +++ b/src/hermes/commands/harvest/base.py @@ -79,10 +79,18 @@ def __call__(self, args: argparse.Namespace) -> None: except Exception: self.log.exception(f"### Unknown error while executing the {plugin_name} plugin, skipping it now.") continue + + self.log.info(f"### Store metadata harvested by {plugin_name} plugin") + # store harvested data + harvested_data.write_to_cache(ctx, plugin_name) + harvested_any = True + self.remove_provenance_info_for_plugin(prov_doc, plugin_name) plugin = prov_doc.add_hermes_plugin("harvest", plugin_name) - plugin_io_operations = plugin_func.io_operations # liste von drei Tupeln (input_file, load_function, output) + plugin_io_operations = plugin_func.io_operations + outputs = [] + io_ops = [] for plugin_io_operation in plugin_io_operations: loaded_source = prov_doc.add_entity() loaded_source.update(plugin_io_operation[0]) @@ -97,11 +105,52 @@ def __call__(self, args: argparse.Namespace) -> None: "prov:wasGeneratedBy": io_op.ref }) loaded_data.update(plugin_io_operations[2]) + outputs.append(loaded_data.ref) + io_ops.append(io_op.ref) + + map_activity = prov_doc.add_activity() + map_activity.update({ + "prov:wasInformedBy": io_ops, + "prov:used": outputs, + "prov:wasAssociatedWith": plugin.ref + }) + data_output = prov_doc.add_entity() + data_output.update({ + "prov:wasAttributedTo": plugin.ref, + "prov:wasGeneratedBy": map_activity.ref, + "prov:wasDerivedFrom": outputs + }) + + write = prov_doc.add_activity() + write.update({ + "prov:wasAssociatedWith": [ + prov_doc.shallow_search({ + "schema:name": lambda doc, node: node["schema:name"][0].find("harvest command") != -1 + })[0].ref, + prov_doc.shallow_search({ + "schema:name": lambda doc, node: node["schema:name"][0].find(" cache") != -1 + })[0].ref, + plugin.ref + ], + "prov:used": data_output.ref, + "prov:wasInformedBy": map_activity.ref + }) + # TODO: add more info + write_output = prov_doc.add_entity() + write_output.update({ + "prov:wasGeneratedBy": write.ref, "prov:wasDerivedFrom": data_output.ref + }) + write_output = prov_doc.add_entity() + write_output.update({ + "prov:wasGeneratedBy": write.ref, "prov:wasDerivedFrom": data_output.ref + }) + write_output = prov_doc.add_entity() + write_output.update({ + "prov:wasGeneratedBy": write.ref, "prov:wasDerivedFrom": data_output.ref + }) - self.log.info(f"### Store metadata harvested by {plugin_name} plugin") - # store harvested data - harvested_data.write_to_cache(ctx, plugin_name) - harvested_any = True + with ctx["provenance"] as cache: + cache["codemeta"] = prov_doc.ld_value ctx.finalize_step('harvest') if not harvested_any: @@ -114,11 +163,14 @@ def init_provenance_document(self) -> tuple[ld_prov_list, ld_prov_node]: with ctx["provenance"] as cache: try: ld_prov_doc = ld_prov_list.from_list(cache["codemeta"], container_type="@graph", context=ALL_CONTEXTS) - return ld_prov_doc, ld_prov_doc.shallow_search({"schema:name": lambda doc, node: node["schema:name"][0].find("harvest base plugin") != -1})[0] + return ld_prov_doc, ld_prov_doc.shallow_search({ + "schema:name": lambda doc, node: node["schema:name"][0].find("harvest base plugin") != -1 + })[0] except KeyError: pass prov_doc = ld_prov_list() prov_doc.init_hermes_agents() + prov_doc.add_hermes_command("harvest") return prov_doc, prov_doc.add_hermes_base_plugin("harvest") def remove_provenance_info_for_plugin(self, prov_doc, plugin) -> None: diff --git a/src/hermes/model/provenance/ld_prov.py b/src/hermes/model/provenance/ld_prov.py index 2c2a10f7..a8419da9 100644 --- a/src/hermes/model/provenance/ld_prov.py +++ b/src/hermes/model/provenance/ld_prov.py @@ -108,8 +108,8 @@ def add_hermes_command(self, step) -> "ld_prov_node": "schema:name": f"{utils.hermes_name} {step} command", "schema:version": utils.hermes_version, "prov:actedOnBehalfOf": self.shallow_search( - {"schema:name": (lambda doc, node: node["schema:name"] == utils.hermes_name)} - ) + {"schema:name": (lambda doc, node: node["schema:name"][0] == utils.hermes_name)} + )[0].ref }) node["@type"].append("schema:SoftwareApplication") return node @@ -120,8 +120,8 @@ def add_hermes_base_plugin(self, step) -> "ld_prov_node": "schema:name": f"{utils.hermes_name} {step} base plugin", "schema:version": utils.hermes_version, "prov:actedOnBehalfOf": self.shallow_search( - {"schema:name": (lambda doc, node: node["schema:name"] == f"{utils.hermes_name} {step} command")} - ) + {"schema:name": (lambda doc, node: node["schema:name"][0] == f"{utils.hermes_name} {step} command")} + )[0].ref }) node["@type"].append("schema:SoftwareApplication") return node @@ -132,8 +132,8 @@ def add_hermes_plugin(self, step, name) -> "ld_prov_node": node.update({ "schema:name": f"{utils.hermes_name} {step} plugin '{name}'", "prov:actedOnBehalfOf": self.shallow_search( - {"schema:name": (lambda doc, node: node["schema:name"] == f"{utils.hermes_name} {step} base plugin")} - ) + {"schema:name": (lambda doc, node: node["schema:name"][0] == f"{utils.hermes_name} {step} base plugin")} + )[0].ref }) node["@type"].append("schema:SoftwareApplication") return node From 4f85996d53dda0997706283c3a6aa6682b889999 Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Thu, 7 May 2026 16:33:13 +0200 Subject: [PATCH 03/20] improved ld_prov_list and adjusted the collection of provenance accordingly --- src/hermes/commands/harvest/base.py | 83 ++++------ src/hermes/model/provenance/ld_prov.py | 216 ++++++++++++------------- 2 files changed, 135 insertions(+), 164 deletions(-) diff --git a/src/hermes/commands/harvest/base.py b/src/hermes/commands/harvest/base.py index 61e2b3e7..e6fbfc4c 100644 --- a/src/hermes/commands/harvest/base.py +++ b/src/hermes/commands/harvest/base.py @@ -12,7 +12,7 @@ from hermes.error import HermesPluginRunError, MisconfigurationError from hermes.model.context_manager import HermesContext from hermes.model import SoftwareMetadata -from hermes.model.provenance.ld_prov import ld_prov_list, ld_prov_node +from hermes.model.provenance.ld_prov import ld_prov_list from hermes.model.types.ld_context import ALL_CONTEXTS @@ -50,7 +50,8 @@ class HermesHarvestCommand(HermesCommand): def __call__(self, args: argparse.Namespace) -> None: self.args = args self.log.info("# Load provenance from old harvest or create new document.") - prov_doc, base_plugin = self.init_provenance_document() + prov_doc = self.init_provenance_document() + base_plugin = prov_doc.get_hermes_base_plugin("harvest") self.log.info("# Metadata harvesting") if len(self.settings.sources) == 0: @@ -92,60 +93,48 @@ def __call__(self, args: argparse.Namespace) -> None: outputs = [] io_ops = [] for plugin_io_operation in plugin_io_operations: - loaded_source = prov_doc.add_entity() - loaded_source.update(plugin_io_operation[0]) - io_op = prov_doc.add_activity() - plugin_io_operation[1]["prov:wasAssociatedWith"] = [base_plugin.ref, plugin.ref] - plugin_io_operation[1]["prov:used"] = loaded_source.ref - io_op.update(plugin_io_operation[1]) - loaded_data = prov_doc.add_entity() + loaded_source = prov_doc.add_entity(data=plugin_io_operation[0]) + plugin_io_operation[1].update( + {"prov:wasAssociatedWith": [base_plugin.ref, plugin.ref], "prov:used": loaded_source.ref} + ) + io_op = prov_doc.add_activity(data=plugin_io_operation[1]) plugin_io_operation[2].update({ "prov:wasAttributedTo": plugin.ref, "prov:wasDerivedFrom": loaded_source.ref, "prov:wasGeneratedBy": io_op.ref }) - loaded_data.update(plugin_io_operations[2]) + loaded_data = prov_doc.add_entity(data=plugin_io_operations[2]) outputs.append(loaded_data.ref) io_ops.append(io_op.ref) - map_activity = prov_doc.add_activity() - map_activity.update({ + map_activity = prov_doc.add_activity(data={ "prov:wasInformedBy": io_ops, "prov:used": outputs, "prov:wasAssociatedWith": plugin.ref }) - data_output = prov_doc.add_entity() - data_output.update({ + data_output = prov_doc.add_entity(data={ "prov:wasAttributedTo": plugin.ref, "prov:wasGeneratedBy": map_activity.ref, "prov:wasDerivedFrom": outputs }) - write = prov_doc.add_activity() - write.update({ + write = prov_doc.add_activity(data={ "prov:wasAssociatedWith": [ - prov_doc.shallow_search({ - "schema:name": lambda doc, node: node["schema:name"][0].find("harvest command") != -1 - })[0].ref, - prov_doc.shallow_search({ - "schema:name": lambda doc, node: node["schema:name"][0].find(" cache") != -1 - })[0].ref, + prov_doc.get_hermes_command("harvest").ref, + prov_doc.get_hermes_cache().ref, plugin.ref ], "prov:used": data_output.ref, "prov:wasInformedBy": map_activity.ref }) # TODO: add more info - write_output = prov_doc.add_entity() - write_output.update({ + prov_doc.add_entity(data={ "prov:wasGeneratedBy": write.ref, "prov:wasDerivedFrom": data_output.ref }) - write_output = prov_doc.add_entity() - write_output.update({ + prov_doc.add_entity(data={ "prov:wasGeneratedBy": write.ref, "prov:wasDerivedFrom": data_output.ref }) - write_output = prov_doc.add_entity() - write_output.update({ + prov_doc.add_entity(data={ "prov:wasGeneratedBy": write.ref, "prov:wasDerivedFrom": data_output.ref }) @@ -157,39 +146,37 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.critical("No harvest plugin ran successfully.") raise HermesPluginRunError("No harvest plugin ran successfully.") - def init_provenance_document(self) -> tuple[ld_prov_list, ld_prov_node]: + def init_provenance_document(self) -> ld_prov_list: ctx = HermesContext() ctx.prepare_step("harvest") with ctx["provenance"] as cache: try: ld_prov_doc = ld_prov_list.from_list(cache["codemeta"], container_type="@graph", context=ALL_CONTEXTS) - return ld_prov_doc, ld_prov_doc.shallow_search({ - "schema:name": lambda doc, node: node["schema:name"][0].find("harvest base plugin") != -1 - })[0] + return ld_prov_doc except KeyError: pass prov_doc = ld_prov_list() prov_doc.init_hermes_agents() - prov_doc.add_hermes_command("harvest") - return prov_doc, prov_doc.add_hermes_base_plugin("harvest") - - def remove_provenance_info_for_plugin(self, prov_doc, plugin) -> None: - plugin = prov_doc.shallow_search({ - "schema:name": (lambda doc, node: f"harvest plugin {plugin}" in node["schema:name"]), - }) - if len(plugin) == 0: + return prov_doc + + def remove_provenance_info_for_plugin(self, prov_doc: ld_prov_list, plugin) -> None: + plugin = prov_doc.get_hermes_plugin("harvest", plugin) + if plugin is None: return # two passes are needed because the nodes are nested exactly two levels - related = prov_doc.shallow_search({ - "prov:wasAssociatedWith": (lambda doc, node: plugin.ref in node["prov:wasAssociatedWith"]), - "prov:wasAttributedTo": (lambda doc, node: plugin.ref in node["prov:wasAttributedTo"]) - }) + related = prov_doc.shallow_search(lambda doc, node: ( + ("prov:wasAssociatedWith" in node and plugin.ref in node["prov:wasAssociatedWith"]) or + ("prov:wasAttributedTo" in node and plugin.ref in node["prov:wasAttributedTo"]) + )) + if len(related) == 0: + del prov_doc[plugin.index] + return ids = [plugin.ref, *(rel.ref for rel in related)] - related = prov_doc.shallow_search({ - f"prov:{key}": (lambda doc, node: any(id in node[f"prov:{key}"] for id in ids)) for key in [ + related = prov_doc.shallow_search(lambda doc, node: any( + (f"prov:{key}" in node and id in node[f"prov:{key}"]) for id in ids for key in [ "wasAssociatedWith", "wasAttributedTo", "wasGeneratedBy", "used", "wasDerivedFrom", "wasInformedBy" ] - }) + )) for item in related: - items = prov_doc.shallow_search({"@id": (lambda doc, node: node["@id"] == item["@id"])}) + items = prov_doc.shallow_search(lambda doc, node: ("@id" in node and node["@id"] == item["@id"])) del prov_doc[items[0].index] diff --git a/src/hermes/model/provenance/ld_prov.py b/src/hermes/model/provenance/ld_prov.py index a8419da9..df4b8788 100644 --- a/src/hermes/model/provenance/ld_prov.py +++ b/src/hermes/model/provenance/ld_prov.py @@ -6,35 +6,22 @@ from typing import Optional, Union from typing_extensions import Self -import uuid from hermes import utils from hermes.model.types import ld_dict, ld_list -from hermes.model.types.ld_container import BASIC_TYPE, EXPANDED_JSON_LD_VALUE, JSON_LD_CONTEXT_DICT, TIME_TYPE +from hermes.model.types.ld_container import EXPANDED_JSON_LD_VALUE, JSON_LD_CONTEXT_DICT from hermes.model.types.ld_context import ALL_CONTEXTS, iri_map -class ld_prov_container: - def _to_python( - self: Self, - full_iri: str, - ld_value: Union[EXPANDED_JSON_LD_VALUE, dict[str, EXPANDED_JSON_LD_VALUE], list[str], str] - ) -> Union["ld_prov_node", "ld_prov_list", BASIC_TYPE, TIME_TYPE]: - item = super()._to_python(full_iri, ld_value) - if isinstance(item, ld_list): - return ld_prov_list( - data=item._data, parent=item.parent, key=item.key, index=item.index, context=item.context - ) - elif isinstance(item, ld_dict): - return ld_prov_node( - data=item._data, parent=item.parent, key=item.key, index=item.index, context=item.context - ) - return item - - class ld_prov_list(ld_list): - NODE_IRI_FORMAT = "graph://{uuid}/{index}" + NODE_IRI_FORMAT = "_:{type}/{index}" + HERMES_ID = f"https://doi.org/{utils.hermes_doi}" + HERMES_CACHE_ID = "_:hermes/cache" + HERMES_COMMAND_ID_FORMAT = "_:hermes/command/{step}" + HERMES_PLUGIN_ID_FORMAT = "_:hermes/plugin/{step}/{name}" + HERMES_BASE_PLUGIN_ID_FORMAT = "_:hermes/base_plugin/{step}" PROV_DOC_IRI = iri_map['hermes-rt', "graph"] + INDICES = {} def __init__( self: Self, @@ -45,114 +32,111 @@ def __init__( index: Optional[int] = None, context: Optional[list[Union[str, JSON_LD_CONTEXT_DICT]]] = ALL_CONTEXTS ) -> None: - self.id = uuid.uuid1() - self.node_index = 0 super().__init__([{"@graph": []}], parent=parent, key=key, index=index, context=context) - def __getitem__( - self: Self, index: Union[int, slice] - ) -> Union[ - BASIC_TYPE, - TIME_TYPE, - "ld_prov_node", - "ld_prov_list", - list[Union[BASIC_TYPE, TIME_TYPE, "ld_prov_node", "ld_prov_list"]] - ]: - item = super().__getitem__(index) - if isinstance(item, ld_list): - return ld_prov_list( - data=item._data, parent=item.parent, key=item.key, index=item.index, context=item.context - ) - elif isinstance(item, ld_dict): - return ld_prov_node( - data=item._data, parent=item.parent, key=item.key, index=item.index, context=item.context - ) - return item - - def next_node_iri(self) -> str: - self.node_index += 1 - return self.NODE_IRI_FORMAT.format(uuid=self.id, index=self.node_index) - - def add_activity(self) -> "ld_prov_node": - self.append({"@id": self.next_node_iri(), "@type": "prov:Activity"}) - return self[-1] - - def add_agent(self) -> "ld_prov_node": - self.append({"@id": self.next_node_iri(), "@type": "prov:Agent"}) - return self[-1] - - def add_entity(self) -> "ld_prov_node": - self.append({"@id": self.next_node_iri(), "@type": "prov:Entity"}) - return self[-1] - - def init_hermes_agents(self) -> "ld_prov_node": - hermes = self.add_agent() - hermes.update({ + def next_node_iri(self, type) -> str: + if type not in ld_prov_list.INDICES: + ld_prov_list.INDICES[type] = 0 + ld_prov_list.INDICES[type] += 1 + return self.NODE_IRI_FORMAT.format(type=type, index=ld_prov_list.INDICES[type]) + + def add_activity(self, *, data={}) -> ld_dict: + self.append(data) + activity = self[-1] + if "@type" not in data: + activity["@type"] = "prov:Activity" + else: + activity["@type"].append("prov:Activity") + if "@id" not in data: + activity["@id"] = self.next_node_iri("Activity") + return activity + + def add_agent(self, *, data={}) -> ld_dict: + self.append(data) + agent = self[-1] + if "@type" not in data: + agent["@type"] = "prov:Agent" + else: + agent["@type"].append("prov:Agent") + if "@id" not in data: + agent["@id"] = self.next_node_iri("Agent") + return agent + + def add_entity(self, *, data={}) -> ld_dict: + self.append(data) + entity = self[-1] + if "@type" not in data: + entity["@type"] = "prov:Entity" + else: + entity["@type"].append("prov:Entity") + if "@id" not in data: + entity["@id"] = self.next_node_iri("Entity") + return entity + + def init_hermes_agents(self) -> None: + hermes = self.add_agent(data={ + "@id": ld_prov_list.HERMES_ID, + "@type": "schema:SoftwareApplication", "schema:name": utils.hermes_name, "schema:version": utils.hermes_version, - "schema:url": utils.hermes_urls, + "schema:url": [*set(utils.hermes_urls.values())] }) - hermes["@type"].append("schema:SoftwareApplication") - node = self.add_agent() - node.update({ + self.add_agent(data={ + "@id": ld_prov_list.HERMES_CACHE_ID, + "@type": "schema:SoftwareApplication", "schema:name": utils.hermes_name + " cache", "schema:version": utils.hermes_version, "prov:actedOnBehalfOf": hermes.ref }) - node["@type"].append("schema:SoftwareApplication") - return node - - def add_hermes_command(self, step) -> "ld_prov_node": - node = self.add_agent() - node.update({ - "schema:name": f"{utils.hermes_name} {step} command", - "schema:version": utils.hermes_version, - "prov:actedOnBehalfOf": self.shallow_search( - {"schema:name": (lambda doc, node: node["schema:name"][0] == utils.hermes_name)} - )[0].ref - }) - node["@type"].append("schema:SoftwareApplication") - return node - - def add_hermes_base_plugin(self, step) -> "ld_prov_node": - node = self.add_agent() - node.update({ - "schema:name": f"{utils.hermes_name} {step} base plugin", - "schema:version": utils.hermes_version, - "prov:actedOnBehalfOf": self.shallow_search( - {"schema:name": (lambda doc, node: node["schema:name"][0] == f"{utils.hermes_name} {step} command")} - )[0].ref - }) - node["@type"].append("schema:SoftwareApplication") - return node - - def add_hermes_plugin(self, step, name) -> "ld_prov_node": - node = self.add_agent() + for step in ["harvest", "process", "curate", "deposit", "postprocess"]: + command = self.add_agent(data={ + "@id": ld_prov_list.HERMES_COMMAND_ID_FORMAT.format(step=step), + "@type": "schema:SoftwareApplication", + "schema:name": f"{utils.hermes_name} {step} command", + "schema:version": utils.hermes_version, + "prov:actedOnBehalfOf": hermes.ref + }) + self.add_agent(data={ + "@id": ld_prov_list.HERMES_BASE_PLUGIN_ID_FORMAT.format(step=step), + "@type": "schema:SoftwareApplication", + "schema:name": f"{utils.hermes_name} {step} base plugin", + "schema:version": utils.hermes_version, + "prov:actedOnBehalfOf": command.ref + }) + + def add_hermes_plugin(self, step, name) -> ld_dict: # TODO: add version - node.update({ + node = self.add_agent(data={ + "@id": ld_prov_list.HERMES_PLUGIN_ID_FORMAT.format(step=step, name=name), + "@type": "schema:SoftwareApplication", "schema:name": f"{utils.hermes_name} {step} plugin '{name}'", - "prov:actedOnBehalfOf": self.shallow_search( - {"schema:name": (lambda doc, node: node["schema:name"][0] == f"{utils.hermes_name} {step} base plugin")} - )[0].ref + "prov:actedOnBehalfOf": self.get_hermes_base_plugin(step) }) - node["@type"].append("schema:SoftwareApplication") return node - def shallow_search(self, query: dict) -> list["ld_prov_node"]: - return [ - item for item in self for key, test in query.items() if key in item and test(self, item) - ] + def shallow_search(self, query) -> list[ld_dict]: + return [item for item in self if query(self, item)] + def get_hermes(self) -> ld_dict: + return self.shallow_search(lambda doc, node: ("@id" in node and node["@id"] == ld_prov_list.HERMES_ID))[0] -class ld_prov_node(ld_dict): - def __init__( - self: Self, - data: list[dict[str, EXPANDED_JSON_LD_VALUE]], - *, - parent: Optional[Union[ld_dict, ld_list]] = None, - key: Optional[str] = None, - index: Optional[int] = None, - context: Optional[list[Union[str, JSON_LD_CONTEXT_DICT]]] = ALL_CONTEXTS - ) -> None: - self.id = uuid.uuid1() - super().__init__(data, parent=parent, key=key, index=index, context=context) + def get_hermes_cache(self) -> ld_dict: + return self.shallow_search(lambda doc, node: ("@id" in node and node["@id"] == ld_prov_list.HERMES_CACHE_ID))[0] + + def get_hermes_base_plugin(self, step) -> ld_dict: + return self.shallow_search(lambda doc, node: ( + "@id" in node and node["@id"] == ld_prov_list.HERMES_BASE_PLUGIN_ID_FORMAT.format(step=step) + ))[0] + + def get_hermes_plugin(self, step, name) -> Union[ld_dict, None]: + search_result = self.shallow_search(lambda doc, node: ( + "@id" in node and node["@id"] == ld_prov_list.HERMES_PLUGIN_ID_FORMAT.format(step=step, name=name) + )) + if search_result: + return search_result[0] + return None + + def get_hermes_command(self, step) -> ld_dict: + return self.shallow_search(lambda doc, node: ( + "@id" in node and node["@id"] == ld_prov_list.HERMES_COMMAND_ID_FORMAT.format(step=step) + ))[0] From 893096c9bbb21f765636ed2f9d09c434fd0d2842 Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Thu, 21 May 2026 11:53:36 +0200 Subject: [PATCH 04/20] updated compaction of ld_lists --- src/hermes/model/types/ld_container.py | 2 +- src/hermes/model/types/ld_list.py | 41 ++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/hermes/model/types/ld_container.py b/src/hermes/model/types/ld_container.py index abc37fa9..10d2a82c 100644 --- a/src/hermes/model/types/ld_container.py +++ b/src/hermes/model/types/ld_container.py @@ -316,7 +316,7 @@ def compact( COMPACTED_JSON_LD_VALUE: The compacted version of selfs JSON-LD representation. """ return self.ld_proc.compact( - self.ld_value, context or self.context, {"documentLoader": bundled_loader, "skipExpand": True} + self.ld_value, context or self.full_context, {"documentLoader": bundled_loader, "skipExpand": True} ) def to_python(self): diff --git a/src/hermes/model/types/ld_list.py b/src/hermes/model/types/ld_list.py index 003cda82..01a1c265 100644 --- a/src/hermes/model/types/ld_list.py +++ b/src/hermes/model/types/ld_list.py @@ -15,6 +15,7 @@ from typing_extensions import Self from .ld_container import ( + COMPACTED_JSON_LD_VALUE, ld_container, JSON_LD_CONTEXT_DICT, EXPANDED_JSON_LD_VALUE, @@ -23,6 +24,7 @@ TIME_TYPE, BASIC_TYPE, ) +from .pyld_util import bundled_loader if TYPE_CHECKING: from .ld_dict import ld_dict @@ -548,6 +550,45 @@ def to_python(self: Self) -> list[Union[BASIC_TYPE, TIME_TYPE, PYTHONIZED_LD_CON for item in self ] + def compact( + self: Self, context: Optional[Union[list[Union[JSON_LD_CONTEXT_DICT, str]], JSON_LD_CONTEXT_DICT, str]] = None + ) -> COMPACTED_JSON_LD_VALUE: + """ + Returns the compacted version of the given ld_list using its context only if none was supplied. + The returned object is of the form `{"@context": the_context, container_type: compacted_content}`. + + Args: + context (list[JSON_LD_CONTEXT_DICT | str] | JSON_LD_CONTEXT_DICT | str | None): + The context to use for the compaction. If None the context of self is used. + + Returns: + COMPACTED_JSON_LD_VALUE: The compacted version of selfs JSON-LD representation. + """ + # compact the ld_list standalone if necessary + if self.key is None: + return self.ld_proc.compact( + self.ld_value, context or self.full_context, {"documentLoader": bundled_loader, "skipExpand": True} + ) + # compact the ld_list within a temporary dictionary + temp_dict = self.ld_proc.compact( + [{self.ld_proc.expand_iri(self.active_ctx, self.key): self.ld_value}], + context or self.full_context, + {"documentLoader": bundled_loader, "skipExpand": True} + ) + context = temp_dict["@context"] + temp_container = temp_dict[ + self.ld_proc.compact_iri(self.active_ctx, self.ld_proc.expand_iri(self.active_ctx, self.key)) + ] + if self.container_type != "@set": + return { + "@context": context, + **temp_container + } + return { + "@context": context, + "@set": temp_container if isinstance(temp_container, list) else [temp_container] + } + @classmethod def is_ld_list(cls: type[Self], ld_value: Any) -> bool: """ From 4686353f5fda49ca3c5434b2e904b51f8690d6de Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Thu, 21 May 2026 13:25:18 +0200 Subject: [PATCH 05/20] fixed provenance document loading in harvest --- src/hermes/commands/harvest/base.py | 5 ++--- src/hermes/model/provenance/ld_prov.py | 19 +++++++++++++++++-- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/hermes/commands/harvest/base.py b/src/hermes/commands/harvest/base.py index e6fbfc4c..30c00f23 100644 --- a/src/hermes/commands/harvest/base.py +++ b/src/hermes/commands/harvest/base.py @@ -139,7 +139,7 @@ def __call__(self, args: argparse.Namespace) -> None: }) with ctx["provenance"] as cache: - cache["codemeta"] = prov_doc.ld_value + cache["result"] = prov_doc.ld_value ctx.finalize_step('harvest') if not harvested_any: @@ -151,8 +151,7 @@ def init_provenance_document(self) -> ld_prov_list: ctx.prepare_step("harvest") with ctx["provenance"] as cache: try: - ld_prov_doc = ld_prov_list.from_list(cache["codemeta"], container_type="@graph", context=ALL_CONTEXTS) - return ld_prov_doc + return ld_prov_list.load_ld_prov_list(cache["result"]) except KeyError: pass prov_doc = ld_prov_list() diff --git a/src/hermes/model/provenance/ld_prov.py b/src/hermes/model/provenance/ld_prov.py index df4b8788..4ce93a4b 100644 --- a/src/hermes/model/provenance/ld_prov.py +++ b/src/hermes/model/provenance/ld_prov.py @@ -25,14 +25,29 @@ class ld_prov_list(ld_list): def __init__( self: Self, - *, data: EXPANDED_JSON_LD_VALUE = [{"@graph": []}], + *, parent: Optional[Union[ld_dict, ld_list]] = None, key: Optional[str] = PROV_DOC_IRI, index: Optional[int] = None, context: Optional[list[Union[str, JSON_LD_CONTEXT_DICT]]] = ALL_CONTEXTS ) -> None: - super().__init__([{"@graph": []}], parent=parent, key=key, index=index, context=context) + super().__init__(data, parent=parent, key=key, index=index, context=context) + + @classmethod + def load_ld_prov_list(cls, data) -> "ld_prov_list": + if cls.INDICES != {}: + raise RuntimeError("Only zero or one objects of class 'ld_prov_list' may exist at every point in time.") + prov_list = cls.from_list(data[0]["@graph"], container_type="@graph", context=ALL_CONTEXTS, key=cls.PROV_DOC_IRI) + for item in prov_list: + if not ("@id" in item and item["@id"].startswith("_:")): + continue + item_id = item["@id"][2:].split("/") + if not (len(item_id) == 2 and item_id[1].isnumeric()): + continue + if cls.INDICES.get(item_id[0], 0) < int(item_id[1]): + cls.INDICES[item_id[0]] = int(item_id[1]) + return prov_list def next_node_iri(self, type) -> str: if type not in ld_prov_list.INDICES: From c05601b7753a7cb6619983475c87ca602879d723 Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Thu, 21 May 2026 13:29:33 +0200 Subject: [PATCH 06/20] flake8 --- src/hermes/commands/harvest/base.py | 1 - src/hermes/model/provenance/ld_prov.py | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/hermes/commands/harvest/base.py b/src/hermes/commands/harvest/base.py index 30c00f23..f1ae63b7 100644 --- a/src/hermes/commands/harvest/base.py +++ b/src/hermes/commands/harvest/base.py @@ -13,7 +13,6 @@ from hermes.model.context_manager import HermesContext from hermes.model import SoftwareMetadata from hermes.model.provenance.ld_prov import ld_prov_list -from hermes.model.types.ld_context import ALL_CONTEXTS class HermesHarvestPlugin(HermesPlugin): diff --git a/src/hermes/model/provenance/ld_prov.py b/src/hermes/model/provenance/ld_prov.py index 4ce93a4b..3c6ea7c3 100644 --- a/src/hermes/model/provenance/ld_prov.py +++ b/src/hermes/model/provenance/ld_prov.py @@ -38,7 +38,9 @@ def __init__( def load_ld_prov_list(cls, data) -> "ld_prov_list": if cls.INDICES != {}: raise RuntimeError("Only zero or one objects of class 'ld_prov_list' may exist at every point in time.") - prov_list = cls.from_list(data[0]["@graph"], container_type="@graph", context=ALL_CONTEXTS, key=cls.PROV_DOC_IRI) + prov_list = cls.from_list( + data[0]["@graph"], key=cls.PROV_DOC_IRI, context=ALL_CONTEXTS, container_type="@graph" + ) for item in prov_list: if not ("@id" in item and item["@id"].startswith("_:")): continue From 379fe0fbd90731f0c288f3ab85ac08627c13d0b4 Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Thu, 21 May 2026 16:31:45 +0200 Subject: [PATCH 07/20] first draft of provenance recording for process step --- src/hermes/commands/harvest/base.py | 18 ++-- src/hermes/commands/process/base.py | 112 ++++++++++++++++++++++++- src/hermes/model/provenance/ld_prov.py | 12 +-- 3 files changed, 128 insertions(+), 14 deletions(-) diff --git a/src/hermes/commands/harvest/base.py b/src/hermes/commands/harvest/base.py index f1ae63b7..51744a50 100644 --- a/src/hermes/commands/harvest/base.py +++ b/src/hermes/commands/harvest/base.py @@ -128,13 +128,19 @@ def __call__(self, args: argparse.Namespace) -> None: }) # TODO: add more info prov_doc.add_entity(data={ - "prov:wasGeneratedBy": write.ref, "prov:wasDerivedFrom": data_output.ref + "prov:wasGeneratedBy": write.ref, + "prov:wasDerivedFrom": data_output.ref, + "prov:wasAttributedTo": prov_doc.get_hermes_cache().ref }) prov_doc.add_entity(data={ - "prov:wasGeneratedBy": write.ref, "prov:wasDerivedFrom": data_output.ref + "prov:wasGeneratedBy": write.ref, + "prov:wasDerivedFrom": data_output.ref, + "prov:wasAttributedTo": prov_doc.get_hermes_cache().ref }) prov_doc.add_entity(data={ - "prov:wasGeneratedBy": write.ref, "prov:wasDerivedFrom": data_output.ref + "prov:wasGeneratedBy": write.ref, + "prov:wasDerivedFrom": data_output.ref, + "prov:wasAttributedTo": prov_doc.get_hermes_cache().ref }) with ctx["provenance"] as cache: @@ -162,7 +168,7 @@ def remove_provenance_info_for_plugin(self, prov_doc: ld_prov_list, plugin) -> N if plugin is None: return # two passes are needed because the nodes are nested exactly two levels - related = prov_doc.shallow_search(lambda doc, node: ( + related = prov_doc.shallow_search(lambda node: ( ("prov:wasAssociatedWith" in node and plugin.ref in node["prov:wasAssociatedWith"]) or ("prov:wasAttributedTo" in node and plugin.ref in node["prov:wasAttributedTo"]) )) @@ -170,11 +176,11 @@ def remove_provenance_info_for_plugin(self, prov_doc: ld_prov_list, plugin) -> N del prov_doc[plugin.index] return ids = [plugin.ref, *(rel.ref for rel in related)] - related = prov_doc.shallow_search(lambda doc, node: any( + related = prov_doc.shallow_search(lambda node: any( (f"prov:{key}" in node and id in node[f"prov:{key}"]) for id in ids for key in [ "wasAssociatedWith", "wasAttributedTo", "wasGeneratedBy", "used", "wasDerivedFrom", "wasInformedBy" ] )) for item in related: - items = prov_doc.shallow_search(lambda doc, node: ("@id" in node and node["@id"] == item["@id"])) + items = prov_doc.shallow_search(lambda node: ("@id" in node and node["@id"] == item["@id"])) del prov_doc[items[0].index] diff --git a/src/hermes/commands/process/base.py b/src/hermes/commands/process/base.py index 725f6487..a603c650 100644 --- a/src/hermes/commands/process/base.py +++ b/src/hermes/commands/process/base.py @@ -5,7 +5,7 @@ # SPDX-FileContributor: Michael Meinel import argparse -from typing import Union +from typing import Optional from pydantic import BaseModel @@ -15,12 +15,13 @@ from hermes.model.context_manager import HermesContext from hermes.model.merge.action import MergeAction from hermes.model.merge.container import ld_merge_dict +from hermes.model.provenance.ld_prov import ld_prov_list class HermesProcessPlugin(HermesPlugin): """ Base plugin that defines additional merge strategies.""" - def __call__(self, command: HermesCommand) -> dict[Union[str, None], dict[Union[str, None], MergeAction]]: + def __call__(self, command: HermesCommand) -> dict[Optional[str], dict[Optional[str], MergeAction]]: pass @@ -38,6 +39,12 @@ class HermesProcessCommand(HermesCommand): settings_class = ProcessSettings def __call__(self, args: argparse.Namespace) -> None: + self.log.info("# Load provenance data from harvest step") + prov_doc = self.load_prov_doc() + if prov_doc is not None: + process_command = prov_doc.get_hermes_command("process") + hermes_cache = prov_doc.get_hermes_cache() + self.log.info("# Metadata processing") merged_doc = ld_merge_dict([{}]) @@ -56,6 +63,7 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.info("## Load and run the plugins") any_strategies_loaded = False + strategy_action, merged_strategies = None, None # add the strategies from the plugins for plugin_name in reversed(self.settings.plugins): self.log.info(f"### Load {plugin_name} plugin") @@ -79,6 +87,28 @@ def __call__(self, args: argparse.Namespace) -> None: merged_doc.add_strategy(additional_strategies) any_strategies_loaded = True + if prov_doc is None: + continue + plugin = prov_doc.add_hermes_plugin("process", plugin_name) + new_strategy_generation = prov_doc.add_activity(data={"prov:wasAssociatedWith": plugin.ref}) + new_strategies = prov_doc.add_entity( + data={"prov:wasAttributedTo": plugin.ref, "prov:wasGeneratedBy": new_strategy_generation.ref} + ) + if merged_strategies is None: + merged_strategies = new_strategies + strategy_action = new_strategy_generation + continue + strategy_action = prov_doc.add_activity(data={ + "prov:used": [merged_strategies.ref, new_strategies.ref], + "prov:wasInformedBy": [strategy_action.ref, new_strategy_generation.ref], + "prov:wasAssociatedWith": process_command.ref + }) + merged_strategies = prov_doc.add_entity(data={ + "prov:wasDerivedFrom": [merged_strategies.ref, new_strategies.ref], + "prov:wasGeneratedBy": strategy_action.ref, + "prov:wasAttributedTo": process_command.ref + }) + if not any_strategies_loaded: self.log.critical("## No process plugin was ran successfully.") raise HermesPluginRunError("No process plugin was ran successfully.") @@ -89,6 +119,7 @@ def __call__(self, args: argparse.Namespace) -> None: # merge data from harvesters self.log.info("## Merge the metadata of the harvesters") merged_any = False + merge_action, merged_data = None, None for harvester in harvester_names: self.log.info(f"### Load data from {harvester} plugin") # load data from harvester @@ -110,6 +141,43 @@ def __call__(self, args: argparse.Namespace) -> None: raise RuntimeError(f"Merging the data from {harvester} plugin failed.") from e merged_any = True + if prov_doc is None: + continue + harvest_plugin = prov_doc.get_hermes_plugin("harvest", harvester) + harvest_command = prov_doc.get_hermes_command("harvest") + store_action = prov_doc.shallow_search(lambda node: ( + "prov:wasAssociatedWith" in node and + node["prov:wasAssociatedWith"] == [harvest_plugin.ref, hermes_cache.ref, harvest_command.ref] + ))[0] + stored_results = [ + result.ref for result in prov_doc.shallow_search( + lambda node: ("prov:wasGeneratedBy" in node and node["prov:wasGeneratedBy"] == store_action.ref) + ) + ] + new_data_load = prov_doc.add_activity(data={ + "prov:wasAssociatedWith": [process_command.ref, hermes_cache.ref], + "prov:used": stored_results + }) + new_data = prov_doc.add_entity(data={ + "prov:wasAttributedTo": plugin.ref, + "prov:wasGeneratedBy": new_data_load.ref, + "prov:wasDerivedFrom": stored_results + }) + if merged_data is None: + merged_data = new_data + merge_action = new_data_load + continue + merge_action = prov_doc.add_activity(data={ + "prov:used": [merged_data.ref, new_data.ref, merged_strategies.ref], + "prov:wasInformedBy": [merge_action.ref, new_data_load.ref], + "prov:wasAssociatedWith": process_command.ref + }) + merged_data = prov_doc.add_entity(data={ + "prov:wasDerivedFrom": [merged_data.ref, new_data.ref], + "prov:wasGeneratedBy": merge_action.ref, + "prov:wasAttributedTo": process_command.ref + }) + # error if nothing was merged if not merged_any: self.log.critical("No metadata has been merged, the loading of the data failed for all harvesters.") @@ -122,6 +190,46 @@ def __call__(self, args: argparse.Namespace) -> None: result_ctx["codemeta"] = merged_doc.compact() result_ctx["context"] = {"@context": merged_doc.full_context} result_ctx["expanded"] = merged_doc.ld_value + + if prov_doc is not None: + write = prov_doc.add_activity(data={ + "prov:wasAssociatedWith": [process_command.ref, hermes_cache.ref, plugin.ref], + "prov:used": merged_data.ref, + "prov:wasInformedBy": merge_action.ref + }) + # TODO: add more info + prov_doc.add_entity(data={ + "prov:wasGeneratedBy": write.ref, + "prov:wasDerivedFrom": merged_data.ref, + "prov:wasAttributedTo": hermes_cache.ref + }) + prov_doc.add_entity(data={ + "prov:wasGeneratedBy": write.ref, + "prov:wasDerivedFrom": merged_data.ref, + "prov:wasAttributedTo": hermes_cache.ref + }) + prov_doc.add_entity(data={ + "prov:wasGeneratedBy": write.ref, + "prov:wasDerivedFrom": merged_data.ref, + "prov:wasAttributedTo": hermes_cache.ref + }) + + with ctx["provenance"] as cache: + cache["result"] = prov_doc.ld_value + ctx.finalize_step("process") ctx.finalize_step("harvest") + + def load_prov_doc(self) -> Optional[ld_prov_list]: + ctx = HermesContext() + ctx.prepare_step("harvest") + with ctx["provenance"] as cache: + try: + return ld_prov_list.load_ld_prov_list(cache["result"]) + except Exception: + self.log.warning( + "The provenance data from the harvest step could not be loaded." + "Processing will proceed without collecting provenance data.", + exc_info=1 + ) diff --git a/src/hermes/model/provenance/ld_prov.py b/src/hermes/model/provenance/ld_prov.py index 3c6ea7c3..9edc4019 100644 --- a/src/hermes/model/provenance/ld_prov.py +++ b/src/hermes/model/provenance/ld_prov.py @@ -132,21 +132,21 @@ def add_hermes_plugin(self, step, name) -> ld_dict: return node def shallow_search(self, query) -> list[ld_dict]: - return [item for item in self if query(self, item)] + return [item for item in self if query(item)] def get_hermes(self) -> ld_dict: - return self.shallow_search(lambda doc, node: ("@id" in node and node["@id"] == ld_prov_list.HERMES_ID))[0] + return self.shallow_search(lambda node: ("@id" in node and node["@id"] == ld_prov_list.HERMES_ID))[0] def get_hermes_cache(self) -> ld_dict: - return self.shallow_search(lambda doc, node: ("@id" in node and node["@id"] == ld_prov_list.HERMES_CACHE_ID))[0] + return self.shallow_search(lambda node: ("@id" in node and node["@id"] == ld_prov_list.HERMES_CACHE_ID))[0] def get_hermes_base_plugin(self, step) -> ld_dict: - return self.shallow_search(lambda doc, node: ( + return self.shallow_search(lambda node: ( "@id" in node and node["@id"] == ld_prov_list.HERMES_BASE_PLUGIN_ID_FORMAT.format(step=step) ))[0] def get_hermes_plugin(self, step, name) -> Union[ld_dict, None]: - search_result = self.shallow_search(lambda doc, node: ( + search_result = self.shallow_search(lambda node: ( "@id" in node and node["@id"] == ld_prov_list.HERMES_PLUGIN_ID_FORMAT.format(step=step, name=name) )) if search_result: @@ -154,6 +154,6 @@ def get_hermes_plugin(self, step, name) -> Union[ld_dict, None]: return None def get_hermes_command(self, step) -> ld_dict: - return self.shallow_search(lambda doc, node: ( + return self.shallow_search(lambda node: ( "@id" in node and node["@id"] == ld_prov_list.HERMES_COMMAND_ID_FORMAT.format(step=step) ))[0] From 40a848ab1abccd0ca1e6a5c1fce0d8c6b7a3a68e Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Thu, 28 May 2026 14:13:28 +0200 Subject: [PATCH 08/20] improved provenance collection for harvest --- src/hermes/commands/harvest/base.py | 47 +++++++++++++++++++++----- src/hermes/commands/process/base.py | 2 +- src/hermes/model/provenance/ld_prov.py | 12 +++---- 3 files changed, 46 insertions(+), 15 deletions(-) diff --git a/src/hermes/commands/harvest/base.py b/src/hermes/commands/harvest/base.py index 51744a50..f720f8b7 100644 --- a/src/hermes/commands/harvest/base.py +++ b/src/hermes/commands/harvest/base.py @@ -5,6 +5,7 @@ # SPDX-FileContributor: Michael Meinel import argparse +import datetime from pydantic import BaseModel @@ -28,9 +29,11 @@ def __call__(self, command: HermesCommand) -> SoftwareMetadata: pass def load(): + # TODO: Implement pass def write(): + # TODO: Implement pass @@ -75,14 +78,17 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.info(f"### Run {plugin_name} plugin") # run plugin try: - harvested_data = plugin_func(self) + harvested_data: SoftwareMetadata = plugin_func(self) except Exception: self.log.exception(f"### Unknown error while executing the {plugin_name} plugin, skipping it now.") continue + returned_at_time = datetime.datetime.now().isoformat() self.log.info(f"### Store metadata harvested by {plugin_name} plugin") # store harvested data + begin_store_at_time = datetime.datetime.now().isoformat() harvested_data.write_to_cache(ctx, plugin_name) + stored_at_time = datetime.datetime.now().isoformat() harvested_any = True self.remove_provenance_info_for_plugin(prov_doc, plugin_name) @@ -107,40 +113,64 @@ def __call__(self, args: argparse.Namespace) -> None: io_ops.append(io_op.ref) map_activity = prov_doc.add_activity(data={ + "schema:description": "Maps the loaded data to the JSON-LD contexts vocabulary.", "prov:wasInformedBy": io_ops, "prov:used": outputs, - "prov:wasAssociatedWith": plugin.ref + "prov:wasAssociatedWith": plugin.ref, + "prov:startedAtTime": returned_at_time }) data_output = prov_doc.add_entity(data={ + "schema:description": "the harvested metadata", "prov:wasAttributedTo": plugin.ref, "prov:wasGeneratedBy": map_activity.ref, - "prov:wasDerivedFrom": outputs + "prov:wasDerivedFrom": outputs, + "prov:generatedAtTime": returned_at_time }) write = prov_doc.add_activity(data={ + "schema:description": "Writes the harvested metadata into the HERMES cache.", "prov:wasAssociatedWith": [ prov_doc.get_hermes_command("harvest").ref, prov_doc.get_hermes_cache().ref, plugin.ref ], "prov:used": data_output.ref, - "prov:wasInformedBy": map_activity.ref + "prov:wasInformedBy": map_activity.ref, + "prov:startedAtTime": begin_store_at_time, + "prov:endedAtTime": stored_at_time }) - # TODO: add more info prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "The compacted version of the harvested metadata.", + "schema:text": str(harvested_data.compact()), + "schema:encodingFormat": "application/json", + "schema:url": (ctx.cache_dir / "harvest" / plugin_name / "codemeta.json").as_uri(), "prov:wasGeneratedBy": write.ref, "prov:wasDerivedFrom": data_output.ref, - "prov:wasAttributedTo": prov_doc.get_hermes_cache().ref + "prov:wasAttributedTo": prov_doc.get_hermes_cache().ref, + "prov:generatedAtTime": stored_at_time }) prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "The context of the harvested metadata.", + "schema:text": str({"@context": harvested_data.full_context}), + "schema:encodingFormat": "application/json", + "schema:url": (ctx.cache_dir / "harvest" / plugin_name / "context.json").as_uri(), "prov:wasGeneratedBy": write.ref, "prov:wasDerivedFrom": data_output.ref, - "prov:wasAttributedTo": prov_doc.get_hermes_cache().ref + "prov:wasAttributedTo": prov_doc.get_hermes_cache().ref, + "prov:generatedAtTime": stored_at_time }) prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "The expanded version of the harvested metadata.", + "schema:text": str(harvested_data.ld_value), + "schema:encodingFormat": "application/json", + "schema:url": (ctx.cache_dir / "harvest" / plugin_name / "expanded.json").as_uri(), "prov:wasGeneratedBy": write.ref, "prov:wasDerivedFrom": data_output.ref, - "prov:wasAttributedTo": prov_doc.get_hermes_cache().ref + "prov:wasAttributedTo": prov_doc.get_hermes_cache().ref, + "prov:generatedAtTime": stored_at_time }) with ctx["provenance"] as cache: @@ -181,6 +211,7 @@ def remove_provenance_info_for_plugin(self, prov_doc: ld_prov_list, plugin) -> N "wasAssociatedWith", "wasAttributedTo", "wasGeneratedBy", "used", "wasDerivedFrom", "wasInformedBy" ] )) + del prov_doc[plugin.index] for item in related: items = prov_doc.shallow_search(lambda node: ("@id" in node and node["@id"] == item["@id"])) del prov_doc[items[0].index] diff --git a/src/hermes/commands/process/base.py b/src/hermes/commands/process/base.py index a603c650..51bac589 100644 --- a/src/hermes/commands/process/base.py +++ b/src/hermes/commands/process/base.py @@ -229,7 +229,7 @@ def load_prov_doc(self) -> Optional[ld_prov_list]: return ld_prov_list.load_ld_prov_list(cache["result"]) except Exception: self.log.warning( - "The provenance data from the harvest step could not be loaded." + "The provenance data from the harvest step could not be loaded. " "Processing will proceed without collecting provenance data.", exc_info=1 ) diff --git a/src/hermes/model/provenance/ld_prov.py b/src/hermes/model/provenance/ld_prov.py index 9edc4019..6604df1c 100644 --- a/src/hermes/model/provenance/ld_prov.py +++ b/src/hermes/model/provenance/ld_prov.py @@ -61,9 +61,9 @@ def add_activity(self, *, data={}) -> ld_dict: self.append(data) activity = self[-1] if "@type" not in data: - activity["@type"] = "prov:Activity" + activity["@type"] = ["prov:Activity", "schema:Action"] else: - activity["@type"].append("prov:Activity") + activity["@type"].extend(["prov:Activity", "schema:Action"]) if "@id" not in data: activity["@id"] = self.next_node_iri("Activity") return activity @@ -72,9 +72,9 @@ def add_agent(self, *, data={}) -> ld_dict: self.append(data) agent = self[-1] if "@type" not in data: - agent["@type"] = "prov:Agent" + agent["@type"] = ["prov:Agent", "schema:SoftwareApplication"] else: - agent["@type"].append("prov:Agent") + agent["@type"].extend(["prov:Agent", "schema:SoftwareApplication"]) if "@id" not in data: agent["@id"] = self.next_node_iri("Agent") return agent @@ -83,9 +83,9 @@ def add_entity(self, *, data={}) -> ld_dict: self.append(data) entity = self[-1] if "@type" not in data: - entity["@type"] = "prov:Entity" + entity["@type"] = ["prov:Entity", "schema:Thing"] else: - entity["@type"].append("prov:Entity") + entity["@type"].extend(["prov:Entity", "schema:Thing"]) if "@id" not in data: entity["@id"] = self.next_node_iri("Entity") return entity From d637f84832f6bf2944ce2a731ca8a1d34ba75c2b Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Mon, 1 Jun 2026 16:26:14 +0200 Subject: [PATCH 09/20] implement load wrapper for harvest plugins and fixed minor bug --- src/hermes/commands/harvest/base.py | 39 ++++++++++++++++++------- src/hermes/commands/harvest/cff.py | 2 +- src/hermes/commands/harvest/codemeta.py | 2 +- 3 files changed, 31 insertions(+), 12 deletions(-) diff --git a/src/hermes/commands/harvest/base.py b/src/hermes/commands/harvest/base.py index f720f8b7..7545c52f 100644 --- a/src/hermes/commands/harvest/base.py +++ b/src/hermes/commands/harvest/base.py @@ -6,6 +6,8 @@ import argparse import datetime +from io import IOBase +from pathlib import Path from pydantic import BaseModel @@ -28,9 +30,24 @@ def __init__(self): def __call__(self, command: HermesCommand) -> SoftwareMetadata: pass - def load(): - # TODO: Implement - pass + def load(self, func, source, *args, **kwargs): + source_metadata = {"schema:description": "metadata source"} + if isinstance(source, IOBase): + source_metadata["schema:url"] = Path(source.name).absolute().as_uri() + elif isinstance(source, Path): + source_metadata["schema:url"] = source.absolute().as_uri() + elif isinstance(source, str): + source_metadata["schema:url"] = Path(source).absolute().as_uri() + io_operation = { + "schema:description": "Load operation called with (" + f"{source_metadata['schema:url'] if 'schema:url' in source_metadata else str(source)}" + f"{', ' + str(args) if args else ''}{', ' + str(kwargs) if kwargs else ''}).", + "schema:name": f"{func.__module__}.{func.__qualname__}" + } + result = func(source, *args, **kwargs) + loaded_metadata = {"schema:description": "the loaded data", "schema:text": str(result)} + self.io_operations.append((source_metadata, io_operation, loaded_metadata)) + return result def write(): # TODO: Implement @@ -108,7 +125,7 @@ def __call__(self, args: argparse.Namespace) -> None: "prov:wasDerivedFrom": loaded_source.ref, "prov:wasGeneratedBy": io_op.ref }) - loaded_data = prov_doc.add_entity(data=plugin_io_operations[2]) + loaded_data = prov_doc.add_entity(data=plugin_io_operation[2]) outputs.append(loaded_data.ref) io_ops.append(io_op.ref) @@ -144,7 +161,7 @@ def __call__(self, args: argparse.Namespace) -> None: "schema:description": "The compacted version of the harvested metadata.", "schema:text": str(harvested_data.compact()), "schema:encodingFormat": "application/json", - "schema:url": (ctx.cache_dir / "harvest" / plugin_name / "codemeta.json").as_uri(), + "schema:url": (ctx.cache_dir / "harvest" / plugin_name / "codemeta.json").absolute().as_uri(), "prov:wasGeneratedBy": write.ref, "prov:wasDerivedFrom": data_output.ref, "prov:wasAttributedTo": prov_doc.get_hermes_cache().ref, @@ -155,7 +172,7 @@ def __call__(self, args: argparse.Namespace) -> None: "schema:description": "The context of the harvested metadata.", "schema:text": str({"@context": harvested_data.full_context}), "schema:encodingFormat": "application/json", - "schema:url": (ctx.cache_dir / "harvest" / plugin_name / "context.json").as_uri(), + "schema:url": (ctx.cache_dir / "harvest" / plugin_name / "context.json").absolute().as_uri(), "prov:wasGeneratedBy": write.ref, "prov:wasDerivedFrom": data_output.ref, "prov:wasAttributedTo": prov_doc.get_hermes_cache().ref, @@ -166,7 +183,7 @@ def __call__(self, args: argparse.Namespace) -> None: "schema:description": "The expanded version of the harvested metadata.", "schema:text": str(harvested_data.ld_value), "schema:encodingFormat": "application/json", - "schema:url": (ctx.cache_dir / "harvest" / plugin_name / "expanded.json").as_uri(), + "schema:url": (ctx.cache_dir / "harvest" / plugin_name / "expanded.json").absolute().as_uri(), "prov:wasGeneratedBy": write.ref, "prov:wasDerivedFrom": data_output.ref, "prov:wasAttributedTo": prov_doc.get_hermes_cache().ref, @@ -197,7 +214,6 @@ def remove_provenance_info_for_plugin(self, prov_doc: ld_prov_list, plugin) -> N plugin = prov_doc.get_hermes_plugin("harvest", plugin) if plugin is None: return - # two passes are needed because the nodes are nested exactly two levels related = prov_doc.shallow_search(lambda node: ( ("prov:wasAssociatedWith" in node and plugin.ref in node["prov:wasAssociatedWith"]) or ("prov:wasAttributedTo" in node and plugin.ref in node["prov:wasAttributedTo"]) @@ -206,7 +222,9 @@ def remove_provenance_info_for_plugin(self, prov_doc: ld_prov_list, plugin) -> N del prov_doc[plugin.index] return ids = [plugin.ref, *(rel.ref for rel in related)] - related = prov_doc.shallow_search(lambda node: any( + used_entities = [rel["prov:used"][0]["@id"] for rel in related if "prov:used" in rel] + related = prov_doc.shallow_search(lambda node: node["@id"] in used_entities) + related += prov_doc.shallow_search(lambda node: any( (f"prov:{key}" in node and id in node[f"prov:{key}"]) for id in ids for key in [ "wasAssociatedWith", "wasAttributedTo", "wasGeneratedBy", "used", "wasDerivedFrom", "wasInformedBy" ] @@ -214,4 +232,5 @@ def remove_provenance_info_for_plugin(self, prov_doc: ld_prov_list, plugin) -> N del prov_doc[plugin.index] for item in related: items = prov_doc.shallow_search(lambda node: ("@id" in node and node["@id"] == item["@id"])) - del prov_doc[items[0].index] + if len(items) == 1: + del prov_doc[items[0].index] diff --git a/src/hermes/commands/harvest/cff.py b/src/hermes/commands/harvest/cff.py index 5a2d16c1..f2b648e6 100644 --- a/src/hermes/commands/harvest/cff.py +++ b/src/hermes/commands/harvest/cff.py @@ -43,7 +43,7 @@ def __call__(self, command: HermesHarvestCommand) -> tuple[SoftwareMetadata, dic 'Aborting harvesting for this metadata source.') # Read the content - cff_data = cff_file.read_text() + cff_data = self.load(pathlib.Path.read_text, cff_file) cff_dict = self._load_cff_from_file(cff_data) if command.settings.cff.enable_validation: diff --git a/src/hermes/commands/harvest/codemeta.py b/src/hermes/commands/harvest/codemeta.py index 3dc84296..07645647 100644 --- a/src/hermes/commands/harvest/codemeta.py +++ b/src/hermes/commands/harvest/codemeta.py @@ -34,7 +34,7 @@ def __call__(self, command: HermesHarvestCommand) -> tuple[SoftwareMetadata, dic ) # Read the content - codemeta_str = codemeta_file.read_text() + codemeta_str = self.load(pathlib.Path.read_text, codemeta_file) if not self._validate(codemeta_file): raise HermesValidationError(codemeta_file) From 4d8e62da8d8ba024e7dc024318c972169ec2e9fc Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Mon, 8 Jun 2026 17:34:38 +0200 Subject: [PATCH 10/20] implemented draft of provenance collection druing merge --- src/hermes/commands/harvest/base.py | 65 +++++++++---------- src/hermes/commands/process/base.py | 88 +++++++++++++------------- src/hermes/model/merge/container.py | 51 ++++++++++++++- src/hermes/model/provenance/ld_prov.py | 2 +- 4 files changed, 127 insertions(+), 79 deletions(-) diff --git a/src/hermes/commands/harvest/base.py b/src/hermes/commands/harvest/base.py index 7545c52f..56799e9c 100644 --- a/src/hermes/commands/harvest/base.py +++ b/src/hermes/commands/harvest/base.py @@ -38,10 +38,10 @@ def load(self, func, source, *args, **kwargs): source_metadata["schema:url"] = source.absolute().as_uri() elif isinstance(source, str): source_metadata["schema:url"] = Path(source).absolute().as_uri() - io_operation = { + io_operation = { "schema:description": "Load operation called with (" - f"{source_metadata['schema:url'] if 'schema:url' in source_metadata else str(source)}" - f"{', ' + str(args) if args else ''}{', ' + str(kwargs) if kwargs else ''}).", + f"{source_metadata['schema:url'] if 'schema:url' in source_metadata else str(source)}" + f"{', ' + str(args) if args else ''}{', ' + str(kwargs) if kwargs else ''}).", "schema:name": f"{func.__module__}.{func.__qualname__}" } result = func(source, *args, **kwargs) @@ -60,6 +60,32 @@ class HarvestSettings(BaseModel): sources: list[str] = [] +def remove_harvest_plugin_from_prov_doc(prov_doc: ld_prov_list, plugin: str) -> None: + plugin = prov_doc.get_hermes_plugin("harvest", plugin) + if plugin is None: + return + related = prov_doc.shallow_search(lambda node: ( + ("prov:wasAssociatedWith" in node and plugin.ref in node["prov:wasAssociatedWith"]) or + ("prov:wasAttributedTo" in node and plugin.ref in node["prov:wasAttributedTo"]) + )) + if len(related) == 0: + del prov_doc[plugin.index] + return + ids = [plugin.ref, *(rel.ref for rel in related)] + used_entities = [rel["prov:used"][0]["@id"] for rel in related if "prov:used" in rel] + related = prov_doc.shallow_search(lambda node: node["@id"] in used_entities) + related += prov_doc.shallow_search(lambda node: any( + (f"prov:{key}" in node and id in node[f"prov:{key}"]) for id in ids for key in [ + "wasAssociatedWith", "wasAttributedTo", "wasGeneratedBy", "used", "wasDerivedFrom", "wasInformedBy" + ] + )) + del prov_doc[plugin.index] + for item in related: + items = prov_doc.shallow_search(lambda node: ("@id" in node and node["@id"] == item["@id"])) + if len(items) == 1: + del prov_doc[items[0].index] + + class HermesHarvestCommand(HermesCommand): """ Harvest metadata from configured sources. """ @@ -108,7 +134,7 @@ def __call__(self, args: argparse.Namespace) -> None: stored_at_time = datetime.datetime.now().isoformat() harvested_any = True - self.remove_provenance_info_for_plugin(prov_doc, plugin_name) + remove_harvest_plugin_from_prov_doc(prov_doc, plugin_name) plugin = prov_doc.add_hermes_plugin("harvest", plugin_name) plugin_io_operations = plugin_func.io_operations @@ -159,7 +185,7 @@ def __call__(self, args: argparse.Namespace) -> None: prov_doc.add_entity(data={ "@type": "schema:CreativeWork", "schema:description": "The compacted version of the harvested metadata.", - "schema:text": str(harvested_data.compact()), + "schema:text": str(harvested_data.compact()), # TODO: maybe "prov:value" instead? "schema:encodingFormat": "application/json", "schema:url": (ctx.cache_dir / "harvest" / plugin_name / "codemeta.json").absolute().as_uri(), "prov:wasGeneratedBy": write.ref, @@ -170,7 +196,7 @@ def __call__(self, args: argparse.Namespace) -> None: prov_doc.add_entity(data={ "@type": "schema:CreativeWork", "schema:description": "The context of the harvested metadata.", - "schema:text": str({"@context": harvested_data.full_context}), + "schema:text": str({"@context": harvested_data.full_context}), # TODO: maybe "prov:value" instead? "schema:encodingFormat": "application/json", "schema:url": (ctx.cache_dir / "harvest" / plugin_name / "context.json").absolute().as_uri(), "prov:wasGeneratedBy": write.ref, @@ -181,7 +207,7 @@ def __call__(self, args: argparse.Namespace) -> None: prov_doc.add_entity(data={ "@type": "schema:CreativeWork", "schema:description": "The expanded version of the harvested metadata.", - "schema:text": str(harvested_data.ld_value), + "schema:text": str(harvested_data.ld_value), # TODO: maybe "prov:value" instead? "schema:encodingFormat": "application/json", "schema:url": (ctx.cache_dir / "harvest" / plugin_name / "expanded.json").absolute().as_uri(), "prov:wasGeneratedBy": write.ref, @@ -209,28 +235,3 @@ def init_provenance_document(self) -> ld_prov_list: prov_doc = ld_prov_list() prov_doc.init_hermes_agents() return prov_doc - - def remove_provenance_info_for_plugin(self, prov_doc: ld_prov_list, plugin) -> None: - plugin = prov_doc.get_hermes_plugin("harvest", plugin) - if plugin is None: - return - related = prov_doc.shallow_search(lambda node: ( - ("prov:wasAssociatedWith" in node and plugin.ref in node["prov:wasAssociatedWith"]) or - ("prov:wasAttributedTo" in node and plugin.ref in node["prov:wasAttributedTo"]) - )) - if len(related) == 0: - del prov_doc[plugin.index] - return - ids = [plugin.ref, *(rel.ref for rel in related)] - used_entities = [rel["prov:used"][0]["@id"] for rel in related if "prov:used" in rel] - related = prov_doc.shallow_search(lambda node: node["@id"] in used_entities) - related += prov_doc.shallow_search(lambda node: any( - (f"prov:{key}" in node and id in node[f"prov:{key}"]) for id in ids for key in [ - "wasAssociatedWith", "wasAttributedTo", "wasGeneratedBy", "used", "wasDerivedFrom", "wasInformedBy" - ] - )) - del prov_doc[plugin.index] - for item in related: - items = prov_doc.shallow_search(lambda node: ("@id" in node and node["@id"] == item["@id"])) - if len(items) == 1: - del prov_doc[items[0].index] diff --git a/src/hermes/commands/process/base.py b/src/hermes/commands/process/base.py index 51bac589..5711b197 100644 --- a/src/hermes/commands/process/base.py +++ b/src/hermes/commands/process/base.py @@ -10,6 +10,7 @@ from pydantic import BaseModel from hermes.commands.base import HermesCommand, HermesPlugin +from hermes.commands.harvest.base import remove_harvest_plugin_from_prov_doc from hermes.error import HermesPluginRunError, MisconfigurationError from hermes.model.api import SoftwareMetadata from hermes.model.context_manager import HermesContext @@ -46,7 +47,7 @@ def __call__(self, args: argparse.Namespace) -> None: hermes_cache = prov_doc.get_hermes_cache() self.log.info("# Metadata processing") - merged_doc = ld_merge_dict([{}]) + merged_doc = ld_merge_dict([{}], prov_doc) if not self.settings.plugins: self.log.critical( @@ -119,7 +120,6 @@ def __call__(self, args: argparse.Namespace) -> None: # merge data from harvesters self.log.info("## Merge the metadata of the harvesters") merged_any = False - merge_action, merged_data = None, None for harvester in harvester_names: self.log.info(f"### Load data from {harvester} plugin") # load data from harvester @@ -127,56 +127,56 @@ def __call__(self, args: argparse.Namespace) -> None: metadata = SoftwareMetadata.load_from_cache(ctx, harvester) except Exception: # skip this harvester when the data is invalid + if prov_doc is not None: + remove_harvest_plugin_from_prov_doc(prov_doc, harvester) self.log.exception( f"### The data from the harvester {harvester} could not be loaded or is invalid, skipping it now." ) continue + if prov_doc is not None: + harvest_plugin = prov_doc.get_hermes_plugin("harvest", harvester) + harvest_command = prov_doc.get_hermes_command("harvest") + store_action = prov_doc.shallow_search(lambda node: ( + "prov:wasAssociatedWith" in node and + node["prov:wasAssociatedWith"] == [harvest_plugin.ref, hermes_cache.ref, harvest_command.ref] + ))[0] + stored_results = [ + result.ref for result in prov_doc.shallow_search(lambda node: ( + "prov:wasGeneratedBy" in node and node["prov:wasGeneratedBy"] == [store_action.ref] + )) + ] + new_action = prov_doc.add_activity(data={ # load of new data + "prov:wasAssociatedWith": [process_command.ref, hermes_cache.ref], + "prov:used": stored_results + }) + new_data = prov_doc.add_entity(data={ # new data to be merged + "prov:wasAttributedTo": plugin.ref, + "prov:wasGeneratedBy": new_action.ref, + "prov:wasDerivedFrom": stored_results + }) + if merged_any: + # One pass must have been completed already. + new_action = prov_doc.add_activity(data={ + "prov:used": [last_data.ref, new_data.ref], + "prov:wasInformedBy": [last_action.ref, new_action.ref], + "prov:wasAssociatedWith": process_command.ref + }) # initial merge action of the merge + merged_doc.prov_objects = [new_action, new_data, last_data] + self.log.info(f"### Merge data from {harvester} plugin") # merge data into the merge dict try: merged_doc.update(metadata) except Exception as e: + # TODO: Maybe this state is recoverable by starting over again and skipping this plugin. self.log.critical(f"### Merging the data from {harvester} plugin resulted in an error.", exc_info=True) raise RuntimeError(f"Merging the data from {harvester} plugin failed.") from e - merged_any = True - if prov_doc is None: - continue - harvest_plugin = prov_doc.get_hermes_plugin("harvest", harvester) - harvest_command = prov_doc.get_hermes_command("harvest") - store_action = prov_doc.shallow_search(lambda node: ( - "prov:wasAssociatedWith" in node and - node["prov:wasAssociatedWith"] == [harvest_plugin.ref, hermes_cache.ref, harvest_command.ref] - ))[0] - stored_results = [ - result.ref for result in prov_doc.shallow_search( - lambda node: ("prov:wasGeneratedBy" in node and node["prov:wasGeneratedBy"] == store_action.ref) - ) - ] - new_data_load = prov_doc.add_activity(data={ - "prov:wasAssociatedWith": [process_command.ref, hermes_cache.ref], - "prov:used": stored_results - }) - new_data = prov_doc.add_entity(data={ - "prov:wasAttributedTo": plugin.ref, - "prov:wasGeneratedBy": new_data_load.ref, - "prov:wasDerivedFrom": stored_results - }) - if merged_data is None: - merged_data = new_data - merge_action = new_data_load - continue - merge_action = prov_doc.add_activity(data={ - "prov:used": [merged_data.ref, new_data.ref, merged_strategies.ref], - "prov:wasInformedBy": [merge_action.ref, new_data_load.ref], - "prov:wasAssociatedWith": process_command.ref - }) - merged_data = prov_doc.add_entity(data={ - "prov:wasDerivedFrom": [merged_data.ref, new_data.ref], - "prov:wasGeneratedBy": merge_action.ref, - "prov:wasAttributedTo": process_command.ref - }) + if prov_doc is not None: + last_action = merged_doc.prov_objects[0] if merged_any else new_action + last_data = merged_doc.prov_objects[2] if merged_any else new_data + merged_any = True # error if nothing was merged if not merged_any: @@ -194,23 +194,23 @@ def __call__(self, args: argparse.Namespace) -> None: if prov_doc is not None: write = prov_doc.add_activity(data={ "prov:wasAssociatedWith": [process_command.ref, hermes_cache.ref, plugin.ref], - "prov:used": merged_data.ref, - "prov:wasInformedBy": merge_action.ref + "prov:used": last_data.ref, + "prov:wasInformedBy": last_action.ref }) # TODO: add more info prov_doc.add_entity(data={ "prov:wasGeneratedBy": write.ref, - "prov:wasDerivedFrom": merged_data.ref, + "prov:wasDerivedFrom": last_data.ref, "prov:wasAttributedTo": hermes_cache.ref }) prov_doc.add_entity(data={ "prov:wasGeneratedBy": write.ref, - "prov:wasDerivedFrom": merged_data.ref, + "prov:wasDerivedFrom": last_data.ref, "prov:wasAttributedTo": hermes_cache.ref }) prov_doc.add_entity(data={ "prov:wasGeneratedBy": write.ref, - "prov:wasDerivedFrom": merged_data.ref, + "prov:wasDerivedFrom": last_data.ref, "prov:wasAttributedTo": hermes_cache.ref }) diff --git a/src/hermes/model/merge/container.py b/src/hermes/model/merge/container.py index e9cc03a2..a286c7e8 100644 --- a/src/hermes/model/merge/container.py +++ b/src/hermes/model/merge/container.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, Callable, Optional, Union from typing_extensions import Self +from hermes.model.provenance.ld_prov import ld_prov_list from hermes.model.types import ld_container, ld_context, ld_dict, ld_list from hermes.model.types.ld_container import ( BASIC_TYPE, EXPANDED_JSON_LD_VALUE, JSON_LD_CONTEXT_DICT, JSON_LD_VALUE, TIME_TYPE @@ -50,6 +51,8 @@ def _to_python( if isinstance(value, ld_dict) and not isinstance(value, ld_merge_dict): value = ld_merge_dict( value.ld_value, + self.prov_doc, + self.prov_objects, parent=value.parent, key=value.key, index=value.index, @@ -60,6 +63,8 @@ def _to_python( if isinstance(value, ld_list) and not isinstance(value, ld_merge_list): value = ld_merge_list( value.ld_value, + self.prov_doc, + self.prov_objects, parent=value.parent, key=value.key, index=value.index, @@ -82,6 +87,8 @@ class ld_merge_list(_ld_merge_container, ld_list): def __init__( self: "ld_merge_list", data: Union[list[str], list[dict[str, EXPANDED_JSON_LD_VALUE]]], + prov_doc: ld_prov_list = None, + prov_objects: list[ld_dict] = 3*[None], *, parent: Optional[ld_container] = None, key: Optional[str] = None, @@ -108,6 +115,8 @@ def __init__( super().__init__(data, parent=parent, key=key, index=index, context=context) self.strategies = strategies + self.prov_doc = prov_doc + self.prov_objects = prov_objects class ld_merge_dict(_ld_merge_container, ld_dict): @@ -123,6 +132,8 @@ class ld_merge_dict(_ld_merge_container, ld_dict): def __init__( self: Self, data: list[dict[str, EXPANDED_JSON_LD_VALUE]], + prov_doc: ld_prov_list = None, + prov_objects: list[ld_dict] = 3*[None], *, parent: Optional[Union[ld_dict, ld_list]] = None, key: Optional[str] = None, @@ -154,6 +165,8 @@ def __init__( # add strategies self.strategies = strategies + self.prov_doc = prov_doc + self.prov_objects = prov_objects def update_context( self: Self, other_context: Union[list[Union[str, JSON_LD_CONTEXT_DICT]], None] @@ -225,9 +238,32 @@ def __setitem__(self: Self, key: str, value: Union[JSON_LD_VALUE, BASIC_TYPE, TI """ # create the new item if self[key] and value have to be merged. if key in self: - value = self._merge_item(key, value) + if self.prov_objects[0] is not None: + last_merged_data = self.prov_objects[2] + merge_activity, value = self._merge_item(key, value) + if self.prov_objects[0] is not None: + create_new_merged_data = last_merged_data is self.prov_objects[2] + elif self.prov_objects[0] is not None: + merge_activity = self.prov_doc.add_activity(data={ + "schema:name": "merge", + "schema:description": "foo", + "prov:used": {"@list": [self.prov_objects[2].ref, str(self.path+[key])]}, + "prov:wasInformedBy": self.prov_objects[0].ref + }) + create_new_merged_data = True # update the entry of self[key] super().__setitem__(key, value) + if self.prov_objects[0] is None: + return + self.prov_objects[0] = merge_activity + if create_new_merged_data: + self.prov_objects[2] = self.prov_doc.add_entity(data={ + "prov:wasAttributedTo": self.prov_doc.get_hermes_command("process").ref, + "prov:wasGeneratedBy": merge_activity.ref, + "prov:wasDerivedFrom": {"@list": [self.prov_objects[1].ref, self.prov_objects[2].ref]} + }) + else: + self.prov_objects[2]["prov:wasGeneratedBy"].append(merge_activity.ref) def match( self: Self, @@ -281,7 +317,18 @@ def _merge_item( merger = strategy.get(key, strategy.get(None, None)) if merger is None: raise MergeError(f"Can't merge, no strategy found for key '{key}'.") - return merger.merge(self, [*self.path, key], self[key], value) + if self.prov_objects[0] is not None: + merge_activity = self.prov_doc.add_activity(data={ + "schema:name": "merge", + "schema:description": "foo", + "prov:wasAssociatedWith": self.prov_doc.get_hermes_command("process").ref, + "prov:used": {"@list": [self.prov_objects[1].ref, self.prov_objects[2].ref, str(self.path+[key])]}, + "prov:wasInformedBy": self.prov_objects[0].ref + }) + self.prov_objects[0] = merge_activity + else: + merge_activity = None + return merge_activity, merger.merge(self, [*self.path, key], self[key], value) def _add_related( self: Self, rel: str, key: str, value: Union[BASIC_TYPE, TIME_TYPE, ld_dict, ld_list] diff --git a/src/hermes/model/provenance/ld_prov.py b/src/hermes/model/provenance/ld_prov.py index 6604df1c..767d58ad 100644 --- a/src/hermes/model/provenance/ld_prov.py +++ b/src/hermes/model/provenance/ld_prov.py @@ -127,7 +127,7 @@ def add_hermes_plugin(self, step, name) -> ld_dict: "@id": ld_prov_list.HERMES_PLUGIN_ID_FORMAT.format(step=step, name=name), "@type": "schema:SoftwareApplication", "schema:name": f"{utils.hermes_name} {step} plugin '{name}'", - "prov:actedOnBehalfOf": self.get_hermes_base_plugin(step) + "prov:actedOnBehalfOf": self.get_hermes_base_plugin(step).ref }) return node From 038b72a50c884944faeaa9019b094302936df301 Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Thu, 2 Jul 2026 12:58:45 +0200 Subject: [PATCH 11/20] completed provenance recording in the process step at least for now --- src/hermes/commands/process/base.py | 56 +++++++++++++++++++++++------ src/hermes/model/merge/container.py | 24 +++++++++---- 2 files changed, 64 insertions(+), 16 deletions(-) diff --git a/src/hermes/commands/process/base.py b/src/hermes/commands/process/base.py index 5711b197..b7d83a79 100644 --- a/src/hermes/commands/process/base.py +++ b/src/hermes/commands/process/base.py @@ -5,6 +5,7 @@ # SPDX-FileContributor: Michael Meinel import argparse +import datetime from typing import Optional from pydantic import BaseModel @@ -91,20 +92,27 @@ def __call__(self, args: argparse.Namespace) -> None: if prov_doc is None: continue plugin = prov_doc.add_hermes_plugin("process", plugin_name) - new_strategy_generation = prov_doc.add_activity(data={"prov:wasAssociatedWith": plugin.ref}) - new_strategies = prov_doc.add_entity( - data={"prov:wasAttributedTo": plugin.ref, "prov:wasGeneratedBy": new_strategy_generation.ref} - ) + new_strategy_generation = prov_doc.add_activity(data={ + "schema:description": "generate new merge strategies", + "prov:wasAssociatedWith": plugin.ref + }) + new_strategies = prov_doc.add_entity(data={ # TODO: record strategies + "schema:description": f"new merge strategies of plugin {plugin_name}", + "prov:wasAttributedTo": plugin.ref, + "prov:wasGeneratedBy": new_strategy_generation.ref + }) if merged_strategies is None: merged_strategies = new_strategies strategy_action = new_strategy_generation continue strategy_action = prov_doc.add_activity(data={ + "schema:description": "merging the new strategies into the others", "prov:used": [merged_strategies.ref, new_strategies.ref], "prov:wasInformedBy": [strategy_action.ref, new_strategy_generation.ref], "prov:wasAssociatedWith": process_command.ref }) - merged_strategies = prov_doc.add_entity(data={ + merged_strategies = prov_doc.add_entity(data={ # TODO: record strategies + "schema:description": "the merge strategies of multiple plugins merged together", "prov:wasDerivedFrom": [merged_strategies.ref, new_strategies.ref], "prov:wasGeneratedBy": strategy_action.ref, "prov:wasAttributedTo": process_command.ref @@ -147,10 +155,14 @@ def __call__(self, args: argparse.Namespace) -> None: )) ] new_action = prov_doc.add_activity(data={ # load of new data + "schema:description": f"loads the data from {harvester} plugin", "prov:wasAssociatedWith": [process_command.ref, hermes_cache.ref], "prov:used": stored_results }) new_data = prov_doc.add_entity(data={ # new data to be merged + "@type": "schema:CreativeWork", + "schema:description": f"data loaded from {harvester} plugin", + "schema:text": str(metadata.compact()), # TODO: maybe "prov:value" instead? "prov:wasAttributedTo": plugin.ref, "prov:wasGeneratedBy": new_action.ref, "prov:wasDerivedFrom": stored_results @@ -158,11 +170,12 @@ def __call__(self, args: argparse.Namespace) -> None: if merged_any: # One pass must have been completed already. new_action = prov_doc.add_activity(data={ + "schema:description": "merges the old data object with the new data", "prov:used": [last_data.ref, new_data.ref], "prov:wasInformedBy": [last_action.ref, new_action.ref], "prov:wasAssociatedWith": process_command.ref }) # initial merge action of the merge - merged_doc.prov_objects = [new_action, new_data, last_data] + merged_doc.prov_objects = [new_action, new_data, last_data] # set the starting objects of the merge self.log.info(f"### Merge data from {harvester} plugin") # merge data into the merge dict @@ -186,32 +199,55 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.info("## Store processed metadata") # store processed data ctx.prepare_step("process") + begin_store_at_time = datetime.datetime.now().isoformat() with ctx["result"] as result_ctx: result_ctx["codemeta"] = merged_doc.compact() result_ctx["context"] = {"@context": merged_doc.full_context} result_ctx["expanded"] = merged_doc.ld_value + stored_at_time = datetime.datetime.now().isoformat() if prov_doc is not None: write = prov_doc.add_activity(data={ + "schema:description": "Writes the processed metadata into the HERMES cache.", "prov:wasAssociatedWith": [process_command.ref, hermes_cache.ref, plugin.ref], "prov:used": last_data.ref, - "prov:wasInformedBy": last_action.ref + "prov:wasInformedBy": last_action.ref, + "prov:startedAtTime": begin_store_at_time, + "prov:endedAtTime": stored_at_time }) # TODO: add more info prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "The compacted version of the processed metadata.", + "schema:text": str(merged_doc.compact()), # TODO: maybe "prov:value" instead? + "schema:encodingFormat": "application/json", + "schema:url": (ctx.cache_dir / "process" / "result" / "codemeta.json").absolute().as_uri(), "prov:wasGeneratedBy": write.ref, "prov:wasDerivedFrom": last_data.ref, - "prov:wasAttributedTo": hermes_cache.ref + "prov:wasAttributedTo": hermes_cache.ref, + "prov:generatedAtTime": stored_at_time }) prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "The context of the processed metadata.", + "schema:text": str({"@context": merged_doc.full_context}), # TODO: maybe "prov:value" instead? + "schema:encodingFormat": "application/json", + "schema:url": (ctx.cache_dir / "process" / "result" / "context.json").absolute().as_uri(), "prov:wasGeneratedBy": write.ref, "prov:wasDerivedFrom": last_data.ref, - "prov:wasAttributedTo": hermes_cache.ref + "prov:wasAttributedTo": hermes_cache.ref, + "prov:generatedAtTime": stored_at_time }) prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "The expanded version of the processed metadata.", + "schema:text": str(merged_doc.ld_value), # TODO: maybe "prov:value" instead? + "schema:encodingFormat": "application/json", + "schema:url": (ctx.cache_dir / "process" / "result" / "expanded.json").absolute().as_uri(), "prov:wasGeneratedBy": write.ref, "prov:wasDerivedFrom": last_data.ref, - "prov:wasAttributedTo": hermes_cache.ref + "prov:wasAttributedTo": hermes_cache.ref, + "prov:generatedAtTime": stored_at_time }) with ctx["provenance"] as cache: diff --git a/src/hermes/model/merge/container.py b/src/hermes/model/merge/container.py index a286c7e8..470d3161 100644 --- a/src/hermes/model/merge/container.py +++ b/src/hermes/model/merge/container.py @@ -245,9 +245,10 @@ def __setitem__(self: Self, key: str, value: Union[JSON_LD_VALUE, BASIC_TYPE, TI create_new_merged_data = last_merged_data is self.prov_objects[2] elif self.prov_objects[0] is not None: merge_activity = self.prov_doc.add_activity(data={ - "schema:name": "merge", - "schema:description": "foo", - "prov:used": {"@list": [self.prov_objects[2].ref, str(self.path+[key])]}, + "schema:name": f"merge values at {str(self.path+[key])}", + "schema:description": f"inserting value in the second 'used' value at {str(self.path+[key])} into the " + "first 'used' value at the same point", + "prov:used": {"@list": [self.prov_objects[2].ref, self.prov_objects[1].ref]}, "prov:wasInformedBy": self.prov_objects[0].ref }) create_new_merged_data = True @@ -257,7 +258,13 @@ def __setitem__(self: Self, key: str, value: Union[JSON_LD_VALUE, BASIC_TYPE, TI return self.prov_objects[0] = merge_activity if create_new_merged_data: + outer_most_parent = self + while outer_most_parent.parent != None: + outer_most_parent = outer_most_parent.parent self.prov_objects[2] = self.prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": f"software metadata after merge of values at {str(self.path+[key])}", + "schema:text": str(outer_most_parent.compact()), # TODO: maybe "prov:value" instead? "prov:wasAttributedTo": self.prov_doc.get_hermes_command("process").ref, "prov:wasGeneratedBy": merge_activity.ref, "prov:wasDerivedFrom": {"@list": [self.prov_objects[1].ref, self.prov_objects[2].ref]} @@ -319,10 +326,15 @@ def _merge_item( raise MergeError(f"Can't merge, no strategy found for key '{key}'.") if self.prov_objects[0] is not None: merge_activity = self.prov_doc.add_activity(data={ - "schema:name": "merge", - "schema:description": "foo", + "schema:name": f"merge values at {str(self.path+[key])}", + "schema:description": f"merge value in the second 'used' value at {str(self.path+[key])} into the " + "first 'used' value at the same point using the third 'used' value", "prov:wasAssociatedWith": self.prov_doc.get_hermes_command("process").ref, - "prov:used": {"@list": [self.prov_objects[1].ref, self.prov_objects[2].ref, str(self.path+[key])]}, + "prov:used": {"@list": [ + self.prov_objects[2].ref, + self.prov_objects[1].ref, + f"{merger.merge.__module__}.{merger.merge.__qualname__}" + ]}, "prov:wasInformedBy": self.prov_objects[0].ref }) self.prov_objects[0] = merge_activity From 2813e02701647818acafa6b99ec3f8d0141b65ff Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Fri, 24 Jul 2026 12:32:26 +0200 Subject: [PATCH 12/20] add curate provenance recording --- src/hermes/commands/curate/base.py | 108 ++++++++++++++++++++++++++++ src/hermes/commands/process/base.py | 6 +- 2 files changed, 112 insertions(+), 2 deletions(-) diff --git a/src/hermes/commands/curate/base.py b/src/hermes/commands/curate/base.py index 51f2da08..c250fb9d 100644 --- a/src/hermes/commands/curate/base.py +++ b/src/hermes/commands/curate/base.py @@ -5,6 +5,8 @@ # SPDX-FileContributor: Michael Meinel import argparse +import datetime +from typing import Optional from pydantic import BaseModel @@ -13,6 +15,7 @@ from hermes.model import SoftwareMetadata from hermes.model.context_manager import HermesContext from hermes.model.error import HermesValidationError +from hermes.model.provenance.ld_prov import ld_prov_list class HermesCuratePlugin(HermesPlugin): @@ -35,6 +38,14 @@ class HermesCurateCommand(HermesCommand): settings_class = CurateSettings def __call__(self, args: argparse.Namespace) -> None: + self.log.info("# Load provenance data from process step") + prov_doc = self.load_prov_doc() + if prov_doc is not None: + curate_command = prov_doc.get_hermes_command("curate") + curate_base_plugin = prov_doc.get_hermes_base_plugin("curate") + process_command = prov_doc.get_hermes_command("process") + hermes_cache = prov_doc.get_hermes_cache() + self.log.info("# Metadata curation") plugin_name = self.settings.plugin @@ -54,6 +65,9 @@ def __call__(self, args: argparse.Namespace) -> None: raise HermesValidationError("The results of the process step are invalid.") from e ctx.finalize_step("process") + # save loaded metadata now, because it could be altered in curation + loaded_metadata_str = str(metadata.compact()) + self.log.info(f"## Load curation plugin {plugin_name}") # load plugin try: @@ -72,6 +86,100 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.info("## Store curated data") # store metadata + begin_store_at_time = datetime.datetime.now().isoformat() curated_metadata.write_to_cache(ctx, "result") + stored_at_time = datetime.datetime.now().isoformat() + + if prov_doc is not None: + curate_plugin = prov_doc.add_hermes_plugin("curate", plugin_name) + store_action_of_process = prov_doc.shallow_search(lambda node: ( + "prov:wasAssociatedWith" in node and + node["prov:wasAssociatedWith"] == [process_command.ref, hermes_cache.ref] and + "prov:wasInformedBy" in node + ))[0] + stored_results_of_process = [res.ref for res in prov_doc.shallow_search(lambda node: ( + "prov:wasGeneratedBy" in node and node["prov:wasGeneratedBy"] == [store_action_of_process.ref] + ))] + load_action = prov_doc.add_activity(data={ + "schema:description": "loads the data from process step", + "prov:wasAssociatedWith": [process_command.ref, hermes_cache.ref], + "prov:used": stored_results_of_process + }) + loaded_data = prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "data loaded from process step", + "schema:text": loaded_metadata_str, # TODO: maybe "prov:value" instead? + "prov:wasAttributedTo": [curate_command.ref, curate_plugin.ref, hermes_cache.ref], + "prov:wasGeneratedBy": load_action.ref, + "prov:wasDerivedFrom": stored_results_of_process + }) + curated_data = prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "curated metadata", + "schema:text": str(curated_metadata.compact()), # TODO: maybe "prov:value" instead? + "prov:wasAttributedTo": [curate_plugin.ref, curate_base_plugin.ref, curate_command.ref], + "prov:wasInfluencedBy": curate_plugin.ref, + "prov:wasGeneratedBy": load_action.ref, + "prov:wasDerivedFrom": loaded_data.ref + }) + write = prov_doc.add_activity(data={ + "schema:description": "Writes the processed metadata into the HERMES cache.", + "prov:wasAssociatedWith": [process_command.ref, hermes_cache.ref, curate_plugin.ref], + "prov:used": curated_data.ref, + "prov:startedAtTime": begin_store_at_time, + "prov:endedAtTime": stored_at_time + }) + # TODO: add more info + prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "The compacted version of the processed metadata.", + "schema:text": str(curated_metadata.compact()), # TODO: maybe "prov:value" instead? + "schema:encodingFormat": "application/json", + "schema:url": (ctx.cache_dir / "curate" / "result" / "codemeta.json").absolute().as_uri(), + "prov:wasGeneratedBy": write.ref, + "prov:wasDerivedFrom": curated_data.ref, + "prov:wasAttributedTo": hermes_cache.ref, + "prov:generatedAtTime": stored_at_time + }) + prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "The context of the processed metadata.", + "schema:text": str({"@context": curated_metadata.full_context}), # TODO: maybe "prov:value" instead? + "schema:encodingFormat": "application/json", + "schema:url": (ctx.cache_dir / "curate" / "result" / "context.json").absolute().as_uri(), + "prov:wasGeneratedBy": write.ref, + "prov:wasDerivedFrom": curated_data.ref, + "prov:wasAttributedTo": hermes_cache.ref, + "prov:generatedAtTime": stored_at_time + }) + prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "The expanded version of the processed metadata.", + "schema:text": str(curated_metadata.ld_value), # TODO: maybe "prov:value" instead? + "schema:encodingFormat": "application/json", + "schema:url": (ctx.cache_dir / "curate" / "result" / "expanded.json").absolute().as_uri(), + "prov:wasGeneratedBy": write.ref, + "prov:wasDerivedFrom": curated_data.ref, + "prov:wasAttributedTo": hermes_cache.ref, + "prov:generatedAtTime": stored_at_time + }) + + with ctx["provenance"] as cache: + cache["result"] = prov_doc.ld_value ctx.finalize_step("curate") + + def load_prov_doc(self) -> Optional[ld_prov_list]: + ctx = HermesContext() + ctx.prepare_step("process") + with ctx["provenance"] as cache: + try: + return ld_prov_list.load_ld_prov_list(cache["result"]) + except Exception: + self.log.warning( + "The provenance data from the harvest step could not be loaded. " + "Processing will proceed without collecting provenance data.", + exc_info=1 + ) + finally: + ctx.finalize_step("process") diff --git a/src/hermes/commands/process/base.py b/src/hermes/commands/process/base.py index b7d83a79..95e23d34 100644 --- a/src/hermes/commands/process/base.py +++ b/src/hermes/commands/process/base.py @@ -163,7 +163,7 @@ def __call__(self, args: argparse.Namespace) -> None: "@type": "schema:CreativeWork", "schema:description": f"data loaded from {harvester} plugin", "schema:text": str(metadata.compact()), # TODO: maybe "prov:value" instead? - "prov:wasAttributedTo": plugin.ref, + "prov:wasAttributedTo": [process_command.ref, hermes_cache.ref], "prov:wasGeneratedBy": new_action.ref, "prov:wasDerivedFrom": stored_results }) @@ -209,7 +209,7 @@ def __call__(self, args: argparse.Namespace) -> None: if prov_doc is not None: write = prov_doc.add_activity(data={ "schema:description": "Writes the processed metadata into the HERMES cache.", - "prov:wasAssociatedWith": [process_command.ref, hermes_cache.ref, plugin.ref], + "prov:wasAssociatedWith": [process_command.ref, hermes_cache.ref], "prov:used": last_data.ref, "prov:wasInformedBy": last_action.ref, "prov:startedAtTime": begin_store_at_time, @@ -269,3 +269,5 @@ def load_prov_doc(self) -> Optional[ld_prov_list]: "Processing will proceed without collecting provenance data.", exc_info=1 ) + finally: + ctx.finalize_step("harvest") From 7eb72cde11efcdfb992b5240c33abf7a4124d8e4 Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Fri, 24 Jul 2026 15:12:15 +0200 Subject: [PATCH 13/20] add recording of version of plugins --- src/hermes/commands/curate/base.py | 2 +- src/hermes/commands/harvest/base.py | 11 ++++++++--- src/hermes/commands/process/base.py | 2 +- src/hermes/model/provenance/ld_prov.py | 16 +++++++++++----- 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/src/hermes/commands/curate/base.py b/src/hermes/commands/curate/base.py index c250fb9d..e5479fdd 100644 --- a/src/hermes/commands/curate/base.py +++ b/src/hermes/commands/curate/base.py @@ -91,7 +91,7 @@ def __call__(self, args: argparse.Namespace) -> None: stored_at_time = datetime.datetime.now().isoformat() if prov_doc is not None: - curate_plugin = prov_doc.add_hermes_plugin("curate", plugin_name) + curate_plugin = prov_doc.add_hermes_plugin("curate", plugin_name, plugin_func) store_action_of_process = prov_doc.shallow_search(lambda node: ( "prov:wasAssociatedWith" in node and node["prov:wasAssociatedWith"] == [process_command.ref, hermes_cache.ref] and diff --git a/src/hermes/commands/harvest/base.py b/src/hermes/commands/harvest/base.py index 56799e9c..9d902b78 100644 --- a/src/hermes/commands/harvest/base.py +++ b/src/hermes/commands/harvest/base.py @@ -37,20 +37,25 @@ def load(self, func, source, *args, **kwargs): elif isinstance(source, Path): source_metadata["schema:url"] = source.absolute().as_uri() elif isinstance(source, str): - source_metadata["schema:url"] = Path(source).absolute().as_uri() + try: + source_metadata["schema:url"] = Path(source).absolute().as_uri() + except Exception: + source_metadata["schema:url"] = source io_operation = { "schema:description": "Load operation called with (" f"{source_metadata['schema:url'] if 'schema:url' in source_metadata else str(source)}" f"{', ' + str(args) if args else ''}{', ' + str(kwargs) if kwargs else ''}).", "schema:name": f"{func.__module__}.{func.__qualname__}" } + io_operation["prov:startedAtTime"] = datetime.datetime.now().isoformat() result = func(source, *args, **kwargs) + io_operation["prov:endedAtTime"] = datetime.datetime.now().isoformat() loaded_metadata = {"schema:description": "the loaded data", "schema:text": str(result)} self.io_operations.append((source_metadata, io_operation, loaded_metadata)) return result def write(): - # TODO: Implement + # TODO: Is this needed? If yes, it needs to be implemented pass @@ -136,7 +141,7 @@ def __call__(self, args: argparse.Namespace) -> None: remove_harvest_plugin_from_prov_doc(prov_doc, plugin_name) - plugin = prov_doc.add_hermes_plugin("harvest", plugin_name) + plugin = prov_doc.add_hermes_plugin("harvest", plugin_name, plugin_func) plugin_io_operations = plugin_func.io_operations outputs = [] io_ops = [] diff --git a/src/hermes/commands/process/base.py b/src/hermes/commands/process/base.py index 95e23d34..d3a4d2ee 100644 --- a/src/hermes/commands/process/base.py +++ b/src/hermes/commands/process/base.py @@ -91,7 +91,7 @@ def __call__(self, args: argparse.Namespace) -> None: if prov_doc is None: continue - plugin = prov_doc.add_hermes_plugin("process", plugin_name) + plugin = prov_doc.add_hermes_plugin("process", plugin_name, plugin_func) new_strategy_generation = prov_doc.add_activity(data={ "schema:description": "generate new merge strategies", "prov:wasAssociatedWith": plugin.ref diff --git a/src/hermes/model/provenance/ld_prov.py b/src/hermes/model/provenance/ld_prov.py index 767d58ad..45f12078 100644 --- a/src/hermes/model/provenance/ld_prov.py +++ b/src/hermes/model/provenance/ld_prov.py @@ -4,6 +4,7 @@ # SPDX-FileContributor: Michael Fritzsche +from importlib.metadata import metadata from typing import Optional, Union from typing_extensions import Self @@ -121,14 +122,19 @@ def init_hermes_agents(self) -> None: "prov:actedOnBehalfOf": command.ref }) - def add_hermes_plugin(self, step, name) -> ld_dict: - # TODO: add version - node = self.add_agent(data={ + def add_hermes_plugin(self, step, name, plugin) -> ld_dict: + data = { "@id": ld_prov_list.HERMES_PLUGIN_ID_FORMAT.format(step=step, name=name), "@type": "schema:SoftwareApplication", - "schema:name": f"{utils.hermes_name} {step} plugin '{name}'", + "schema:name": f"{plugin.__module__}.{plugin.__class__.__qualname__}", + "schema:description": f"{utils.hermes_name} {step} plugin '{name}'", "prov:actedOnBehalfOf": self.get_hermes_base_plugin(step).ref - }) + } + try: + data["version"] = metadata(plugin.__module__)["version"] + except Exception: + pass + node = self.add_agent(data=data) return node def shallow_search(self, query) -> list[ld_dict]: From 483ea1ae4d124a76504a66515b3a3982dc4d4c59 Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Fri, 31 Jul 2026 14:08:37 +0200 Subject: [PATCH 14/20] add recording of settings and args of commands --- src/hermes/commands/curate/base.py | 5 +- src/hermes/commands/harvest/base.py | 4 +- src/hermes/commands/process/base.py | 5 +- src/hermes/model/provenance/ld_prov.py | 90 +++++++++++++++++++++++++- 4 files changed, 99 insertions(+), 5 deletions(-) diff --git a/src/hermes/commands/curate/base.py b/src/hermes/commands/curate/base.py index e5479fdd..eb556e17 100644 --- a/src/hermes/commands/curate/base.py +++ b/src/hermes/commands/curate/base.py @@ -38,9 +38,12 @@ class HermesCurateCommand(HermesCommand): settings_class = CurateSettings def __call__(self, args: argparse.Namespace) -> None: + self.args = args self.log.info("# Load provenance data from process step") prov_doc = self.load_prov_doc() if prov_doc is not None: + prov_doc.add_hermes_settings(self) + prov_doc.add_settings_to_command("curate", self) curate_command = prov_doc.get_hermes_command("curate") curate_base_plugin = prov_doc.get_hermes_base_plugin("curate") process_command = prov_doc.get_hermes_command("process") @@ -91,7 +94,7 @@ def __call__(self, args: argparse.Namespace) -> None: stored_at_time = datetime.datetime.now().isoformat() if prov_doc is not None: - curate_plugin = prov_doc.add_hermes_plugin("curate", plugin_name, plugin_func) + curate_plugin = prov_doc.add_hermes_plugin("curate", plugin_name, plugin_func, self) store_action_of_process = prov_doc.shallow_search(lambda node: ( "prov:wasAssociatedWith" in node and node["prov:wasAssociatedWith"] == [process_command.ref, hermes_cache.ref] and diff --git a/src/hermes/commands/harvest/base.py b/src/hermes/commands/harvest/base.py index 9d902b78..680f24b3 100644 --- a/src/hermes/commands/harvest/base.py +++ b/src/hermes/commands/harvest/base.py @@ -102,6 +102,8 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.info("# Load provenance from old harvest or create new document.") prov_doc = self.init_provenance_document() base_plugin = prov_doc.get_hermes_base_plugin("harvest") + prov_doc.add_hermes_settings(self) + prov_doc.add_settings_to_command("harvest", self) self.log.info("# Metadata harvesting") if len(self.settings.sources) == 0: @@ -141,7 +143,7 @@ def __call__(self, args: argparse.Namespace) -> None: remove_harvest_plugin_from_prov_doc(prov_doc, plugin_name) - plugin = prov_doc.add_hermes_plugin("harvest", plugin_name, plugin_func) + plugin = prov_doc.add_hermes_plugin("harvest", plugin_name, plugin_func, self) plugin_io_operations = plugin_func.io_operations outputs = [] io_ops = [] diff --git a/src/hermes/commands/process/base.py b/src/hermes/commands/process/base.py index d3a4d2ee..62741e28 100644 --- a/src/hermes/commands/process/base.py +++ b/src/hermes/commands/process/base.py @@ -41,9 +41,12 @@ class HermesProcessCommand(HermesCommand): settings_class = ProcessSettings def __call__(self, args: argparse.Namespace) -> None: + self.args = args self.log.info("# Load provenance data from harvest step") prov_doc = self.load_prov_doc() if prov_doc is not None: + prov_doc.add_hermes_settings(self) + prov_doc.add_settings_to_command("process", self) process_command = prov_doc.get_hermes_command("process") hermes_cache = prov_doc.get_hermes_cache() @@ -91,7 +94,7 @@ def __call__(self, args: argparse.Namespace) -> None: if prov_doc is None: continue - plugin = prov_doc.add_hermes_plugin("process", plugin_name, plugin_func) + plugin = prov_doc.add_hermes_plugin("process", plugin_name, plugin_func, self) new_strategy_generation = prov_doc.add_activity(data={ "schema:description": "generate new merge strategies", "prov:wasAssociatedWith": plugin.ref diff --git a/src/hermes/model/provenance/ld_prov.py b/src/hermes/model/provenance/ld_prov.py index 45f12078..3ed95a24 100644 --- a/src/hermes/model/provenance/ld_prov.py +++ b/src/hermes/model/provenance/ld_prov.py @@ -9,8 +9,9 @@ from typing_extensions import Self from hermes import utils +from hermes.commands.base import HermesCommand, HermesPlugin from hermes.model.types import ld_dict, ld_list -from hermes.model.types.ld_container import EXPANDED_JSON_LD_VALUE, JSON_LD_CONTEXT_DICT +from hermes.model.types.ld_container import BASIC_TYPE, EXPANDED_JSON_LD_VALUE, JSON_LD_CONTEXT_DICT from hermes.model.types.ld_context import ALL_CONTEXTS, iri_map @@ -122,14 +123,99 @@ def init_hermes_agents(self) -> None: "prov:actedOnBehalfOf": command.ref }) - def add_hermes_plugin(self, step, name, plugin) -> ld_dict: + def add_hermes_settings(self, command: HermesCommand) -> None: + hermes = self.get_hermes() + hermes.emplace("schema:supportingData") + hermes["schema:supportingData"].append({ + "@type": "schema:DataFeed", + "schema:dataFeedElement": [ + { + "@type": "schema:DataFeedItem", + "schema:name": name, + "schema:item": [ + { + "@type": "schema:Item", + "schema:description": value + } + ], + "schema:description": "setting provided by command line (or its default value)" + } + for name, value in [ + ("path", command.args.path.absolute().as_uri()), + ("config", command.args.config.absolute().as_uri()), + ("options", str(command.args.options)) + ] + ], + "schema:description": f"options for run {len(hermes['schema:supportingData']) + 1} of some hermes step" + }) + for name, values in command.root_settings.model_dump(mode="json").items(): + if not isinstance(values, list): + values = [values] + hermes["schema:supportingData"][-1]["schema:dataFeedElement"].append({ + "@type": "schema:DataFeedItem", + "schema:name": name, + "schema:item": [ + { + "@type": "schema:Item", + "schema:description": value + } + for value in values + ], + "schema:description": "setting loaded from the config file" + }) + + def add_settings_to_command(self, step: str, command: HermesCommand) -> None: + command_prov = self.get_hermes_command(step) + command_prov.emplace("schema:supportingData") + command_prov["schema:supportingData"].append({ + "@type": "schema:DataFeed", + "schema:dataFeedElement": [], + "schema:description": f"options for run {len(command_prov['schema:supportingData']) + 1} of step {step}" + }) + for name, values in command.settings.model_dump(mode="json").items(): + if not isinstance(values, list): + values = [values] + command_prov["schema:supportingData"][-1]["schema:dataFeedElement"].append({ + "@type": "schema:DataFeedItem", + "schema:name": name, + "schema:item": [ + { + "@type": "schema:Item", + "schema:description": value + } + for value in values + ] + }) + + def add_hermes_plugin(self, step: str, name: str, plugin: HermesPlugin, command: HermesCommand) -> ld_dict: data = { "@id": ld_prov_list.HERMES_PLUGIN_ID_FORMAT.format(step=step, name=name), "@type": "schema:SoftwareApplication", "schema:name": f"{plugin.__module__}.{plugin.__class__.__qualname__}", "schema:description": f"{utils.hermes_name} {step} plugin '{name}'", + "schema:supportingData": { + "@type": "schema:DataFeed", + "schema:dataFeedElement": [] + }, "prov:actedOnBehalfOf": self.get_hermes_base_plugin(step).ref } + try: + for name, values in getattr(command.settings, name).model_dump(mode="json").items(): + if not isinstance(values, list): + values = [values] + data["schema:supportingData"]["schema:dataFeedElement"].append({ + "@type": "schema:DataFeedItem", + "schema:name": name, + "schema:item": [ + { + "@type": "schema:Item", + "schema:description": value + } + for value in values + ] + }) + except Exception: + del data["schema:supportingData"] try: data["version"] = metadata(plugin.__module__)["version"] except Exception: From 22a9d7b105dfcd90183b515aaba007be4ec1401f Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Wed, 5 Aug 2026 13:03:06 +0200 Subject: [PATCH 15/20] add provenance recording for deposit --- src/hermes/commands/curate/base.py | 6 +- src/hermes/commands/deposit/base.py | 150 ++++++++++++++++++++++++- src/hermes/model/provenance/ld_prov.py | 5 +- 3 files changed, 150 insertions(+), 11 deletions(-) diff --git a/src/hermes/commands/curate/base.py b/src/hermes/commands/curate/base.py index eb556e17..4e056079 100644 --- a/src/hermes/commands/curate/base.py +++ b/src/hermes/commands/curate/base.py @@ -112,7 +112,7 @@ def __call__(self, args: argparse.Namespace) -> None: "@type": "schema:CreativeWork", "schema:description": "data loaded from process step", "schema:text": loaded_metadata_str, # TODO: maybe "prov:value" instead? - "prov:wasAttributedTo": [curate_command.ref, curate_plugin.ref, hermes_cache.ref], + "prov:wasAttributedTo": hermes_cache.ref, "prov:wasGeneratedBy": load_action.ref, "prov:wasDerivedFrom": stored_results_of_process }) @@ -127,7 +127,7 @@ def __call__(self, args: argparse.Namespace) -> None: }) write = prov_doc.add_activity(data={ "schema:description": "Writes the processed metadata into the HERMES cache.", - "prov:wasAssociatedWith": [process_command.ref, hermes_cache.ref, curate_plugin.ref], + "prov:wasAssociatedWith": [curate_command.ref, hermes_cache.ref], "prov:used": curated_data.ref, "prov:startedAtTime": begin_store_at_time, "prov:endedAtTime": stored_at_time @@ -180,7 +180,7 @@ def load_prov_doc(self) -> Optional[ld_prov_list]: return ld_prov_list.load_ld_prov_list(cache["result"]) except Exception: self.log.warning( - "The provenance data from the harvest step could not be loaded. " + "The provenance data from the process step could not be loaded. " "Processing will proceed without collecting provenance data.", exc_info=1 ) diff --git a/src/hermes/commands/deposit/base.py b/src/hermes/commands/deposit/base.py index 57bed627..ace1e614 100644 --- a/src/hermes/commands/deposit/base.py +++ b/src/hermes/commands/deposit/base.py @@ -7,6 +7,8 @@ import abc import argparse +import datetime +from typing import Optional from pydantic import BaseModel @@ -15,6 +17,7 @@ from hermes.model.context_manager import HermesContext from hermes.model import SoftwareMetadata from hermes.model.error import HermesValidationError +from hermes.model.provenance.ld_prov import ld_prov_list class BaseDepositPlugin(HermesPlugin): @@ -23,36 +26,143 @@ class BaseDepositPlugin(HermesPlugin): TODO: describe workflow... needs refactoring to be less stateful! """ - def __call__(self, command: HermesCommand) -> None: + def __call__(self, command: HermesCommand, prov_doc: ld_prov_list) -> None: """Initiate the deposition process. This calls a list of additional methods on the class, none of which need to be implemented. """ self.command = command + target = command.settings.target self.ctx = HermesContext() self.ctx.prepare_step("deposit") self.ctx.prepare_step("curate") try: + start_of_load = datetime.datetime.now().isoformat() self.metadata = SoftwareMetadata.load_from_cache(self.ctx, "result") + end_of_load = datetime.datetime.now().isoformat() except Exception as e: raise HermesValidationError("The results of the curate step are invalid.") from e self.ctx.finalize_step("curate") + if prov_doc is not None: + plugin = prov_doc.add_hermes_plugin("deposit", target, self, command) + deposit_command = prov_doc.get_hermes_command("deposit") + curate_command = prov_doc.get_hermes_command("curate") + deposit_base_plugin = prov_doc.get_hermes_base_plugin("deposit") + hermes_cache = prov_doc.get_hermes_cache() + store_action_curate = prov_doc.shallow_search(lambda node: ( + "prov:wasAssociatedWith" in node and + node["prov:wasAssociatedWith"] == [curate_command.ref, hermes_cache.ref] and + "prov:used" in node and + len(node["prov:used"]) == 1 + ))[0] + results_curate = [item.ref for item in prov_doc.shallow_search(lambda node: ( + "prov:wasGeneratedBy" in node and node["prov:wasGeneratedBy"] == [store_action_curate.ref] + ))] + load_action = prov_doc.add_activity(data={ + "schema:description": "Loads the results of the curate step.", + "prov:used": results_curate, + "prov:wasAssociatedWith": [hermes_cache.ref, deposit_command.ref, deposit_base_plugin.ref], + "prov:startedAtTime": start_of_load, + "prov:endedAtTime": end_of_load + }) + loaded_data = prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "data loaded from curate step", + "schema:text": str(self.metadata.compact()), # TODO: maybe "prov:value" instead? + "prov:wasAttributedTo": hermes_cache.ref, + "prov:wasGeneratedBy": load_action.ref, + "prov:wasDerivedFrom": results_curate, + "prov:generatedAtTime": end_of_load + }) + self.prepare() + start_of_map = datetime.datetime.now().isoformat() deposit = self.map_metadata() - with self.ctx[command.settings.target] as cache: + end_of_map = datetime.datetime.now().isoformat() + with self.ctx[target] as cache: + start_of_store = datetime.datetime.now().isoformat() cache["deposit"] = deposit + end_of_store = datetime.datetime.now().isoformat() + + if prov_doc is not None: + map_action = prov_doc.add_activity(data={ + "schema:description": "Maps the metadata to the format required by the deposition target.", + "prov:used": loaded_data.ref, + "prov:wasAssociatedWith": plugin.ref, + "prov:startedAtTime": start_of_map, + "prov:endedAtTime": end_of_map + }) + mapped_data = prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "The metadata mapped to the format required by the deposition target.", + "schema:text": str(deposit), # TODO: maybe "prov:value" instead? + "prov:wasAttributedTo": plugin.ref, + "prov:wasGeneratedBy": map_action.ref, + "prov:wasDerivedFrom": loaded_data.ref, + "prov:generatedAtTime": end_of_load + }) + store_mapped_data = prov_doc.add_activity(data={ + "schema:description": "Stores the mapped metadata.", + "prov:used": mapped_data.ref, + "prov:wasAssociatedWith": [hermes_cache.ref, deposit_command.ref, deposit_base_plugin.ref], + "prov:startedAtTime": start_of_store, + "prov:endedAtTime": end_of_store + }) + prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "The stored version of the mapped metadata.", + "schema:text": str(deposit), # TODO: maybe "prov:value" instead? + "schema:encodingFormat": "application/json", + "schema:url": (self.ctx.cache_dir / "deposit" / target / "deposit.json").absolute().as_uri(), + "prov:wasGeneratedBy": store_mapped_data.ref, + "prov:wasDerivedFrom": mapped_data.ref, + "prov:wasAttributedTo": hermes_cache.ref, + "prov:generatedAtTime": end_of_store + }) if self.is_initial_publication(): self.create_initial_version() else: self.create_new_version() - deposit = self.update_metadata() - with self.ctx[command.settings.target] as cache: - cache["result"] = deposit + updated_deposit = self.update_metadata() + end_of_update_map = datetime.datetime.now().isoformat() + with self.ctx[target] as cache: + start_of_second_store = datetime.datetime.now().isoformat() + cache["result"] = updated_deposit + end_of_second_store = datetime.datetime.now().isoformat() self.ctx.finalize_step("deposit") + + if prov_doc is not None: + updated_mapped_data = prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "The updated mapped metadata.", + "schema:text": str(updated_deposit), # TODO: maybe "prov:value" instead? + "prov:wasInfluencedBy": plugin.ref, + "prov:wasDerivedFrom": mapped_data.ref, + "prov:generatedAtTime": end_of_update_map + }) + store_updated_mapped_data = prov_doc.add_activity(data={ + "schema:description": "Stores the mapped metadata.", + "prov:used": updated_mapped_data.ref, + "prov:wasAssociatedWith": [hermes_cache.ref, deposit_command.ref, deposit_base_plugin.ref], + "prov:startedAtTime": start_of_second_store, + "prov:endedAtTime": end_of_second_store + }) + prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "The stored version of the updated mapped metadata.", + "schema:text": str(updated_deposit), # TODO: maybe "prov:value" instead? + "schema:encodingFormat": "application/json", + "schema:url": (self.ctx.cache_dir / "deposit" / target / "result.json").absolute().as_uri(), + "prov:wasGeneratedBy": store_updated_mapped_data.ref, + "prov:wasDerivedFrom": updated_mapped_data.ref, + "prov:wasAttributedTo": hermes_cache.ref, + "prov:generatedAtTime": end_of_second_store + }) + self.delete_artifacts() self.upload_artifacts() self.publish() @@ -138,6 +248,10 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.info("# Metadata deposition") self.args = args plugin_name = self.settings.target + prov_doc = self.load_prov_doc() + if prov_doc is not None: + prov_doc.add_hermes_settings(self) + prov_doc.add_settings_to_command("deposit", self) self.log.info(f"## Load deposit plugin {plugin_name}") # load plugin @@ -150,9 +264,33 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.info(f"## Run deposit plugin {plugin_name}") # run plugin try: - plugin_func(self) + plugin_func(self, prov_doc) except HermesValidationError as e: self.log.critical(f"## Error while executing {plugin_name} plugin.", exc_info=1) raise HermesPluginRunError( f"Something went wrong while running the deposit plugin {self.settings.plugin}" ) from e + + if prov_doc is None: + return + + ctx = HermesContext() + ctx.prepare_step("deposit") + with ctx["provenance"] as cache: + cache["result"] = prov_doc.ld_value + ctx.finalize_step("deposit") + + def load_prov_doc(self) -> Optional[ld_prov_list]: + ctx = HermesContext() + ctx.prepare_step("curate") + with ctx["provenance"] as cache: + try: + return ld_prov_list.load_ld_prov_list(cache["result"]) + except Exception: + self.log.warning( + "The provenance data from the curate step could not be loaded. " + "Deposition will proceed without collecting provenance data.", + exc_info=1 + ) + finally: + ctx.finalize_step("curate") diff --git a/src/hermes/model/provenance/ld_prov.py b/src/hermes/model/provenance/ld_prov.py index 3ed95a24..e9c1850a 100644 --- a/src/hermes/model/provenance/ld_prov.py +++ b/src/hermes/model/provenance/ld_prov.py @@ -170,8 +170,9 @@ def add_settings_to_command(self, step: str, command: HermesCommand) -> None: command_prov["schema:supportingData"].append({ "@type": "schema:DataFeed", "schema:dataFeedElement": [], - "schema:description": f"options for run {len(command_prov['schema:supportingData']) + 1} of step {step}" - }) + "schema:description": f"options for run {len(command_prov['schema:supportingData']) + 1} of step {step} out" + f" of {len(self.get_hermes()['schema:supportingData'])} runs of some hermes step" + }) # Needs add_hermes_settings to be called before add_settings_to_command is called! for name, values in command.settings.model_dump(mode="json").items(): if not isinstance(values, list): values = [values] From 7063755771842a126c0cc2a242c1b6de072add52 Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Wed, 5 Aug 2026 15:37:31 +0200 Subject: [PATCH 16/20] revised provenance recording --- src/hermes/commands/curate/base.py | 13 ++++++++--- src/hermes/commands/harvest/base.py | 4 +++- src/hermes/commands/process/base.py | 32 +++++++++++++++++++++----- src/hermes/model/merge/container.py | 9 +++++++- src/hermes/model/provenance/ld_prov.py | 2 +- 5 files changed, 48 insertions(+), 12 deletions(-) diff --git a/src/hermes/commands/curate/base.py b/src/hermes/commands/curate/base.py index 4e056079..b8fc536b 100644 --- a/src/hermes/commands/curate/base.py +++ b/src/hermes/commands/curate/base.py @@ -59,7 +59,9 @@ def __call__(self, args: argparse.Namespace) -> None: # load processed data ctx.prepare_step("process") try: + begin_load_at_time = datetime.datetime.now().isoformat() metadata = SoftwareMetadata.load_from_cache(ctx, "result") + end_load_at_time = datetime.datetime.now().isoformat() except Exception as e: self.log.critical( "## The data from the process step could not be loaded or is invalid for some reason.", @@ -83,6 +85,7 @@ def __call__(self, args: argparse.Namespace) -> None: # run plugin try: curated_metadata = plugin_func(self, metadata) + end_curation_time = datetime.datetime.now().isoformat() except Exception as e: self.log.critical(f"## Unknown error while executing the {plugin_name} plugin.", exc_info=1) raise HermesPluginRunError(f"Something went wrong while running the curate plugin {plugin_name}") from e @@ -106,7 +109,9 @@ def __call__(self, args: argparse.Namespace) -> None: load_action = prov_doc.add_activity(data={ "schema:description": "loads the data from process step", "prov:wasAssociatedWith": [process_command.ref, hermes_cache.ref], - "prov:used": stored_results_of_process + "prov:used": stored_results_of_process, + "prov:startedAtTime": begin_load_at_time, + "prov:endedAtTime": end_load_at_time }) loaded_data = prov_doc.add_entity(data={ "@type": "schema:CreativeWork", @@ -114,7 +119,8 @@ def __call__(self, args: argparse.Namespace) -> None: "schema:text": loaded_metadata_str, # TODO: maybe "prov:value" instead? "prov:wasAttributedTo": hermes_cache.ref, "prov:wasGeneratedBy": load_action.ref, - "prov:wasDerivedFrom": stored_results_of_process + "prov:wasDerivedFrom": stored_results_of_process, + "prov:generatedAtTime": end_load_at_time }) curated_data = prov_doc.add_entity(data={ "@type": "schema:CreativeWork", @@ -123,7 +129,8 @@ def __call__(self, args: argparse.Namespace) -> None: "prov:wasAttributedTo": [curate_plugin.ref, curate_base_plugin.ref, curate_command.ref], "prov:wasInfluencedBy": curate_plugin.ref, "prov:wasGeneratedBy": load_action.ref, - "prov:wasDerivedFrom": loaded_data.ref + "prov:wasDerivedFrom": loaded_data.ref, + "prov:generatedAtTime": end_curation_time }) write = prov_doc.add_activity(data={ "schema:description": "Writes the processed metadata into the HERMES cache.", diff --git a/src/hermes/commands/harvest/base.py b/src/hermes/commands/harvest/base.py index 680f24b3..32550181 100644 --- a/src/hermes/commands/harvest/base.py +++ b/src/hermes/commands/harvest/base.py @@ -167,10 +167,12 @@ def __call__(self, args: argparse.Namespace) -> None: "prov:wasInformedBy": io_ops, "prov:used": outputs, "prov:wasAssociatedWith": plugin.ref, - "prov:startedAtTime": returned_at_time + "prov:endedAtTime": returned_at_time }) data_output = prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", "schema:description": "the harvested metadata", + "schema:text": str(harvested_data.compact()), # TODO: maybe "prov:value" instead? "prov:wasAttributedTo": plugin.ref, "prov:wasGeneratedBy": map_activity.ref, "prov:wasDerivedFrom": outputs, diff --git a/src/hermes/commands/process/base.py b/src/hermes/commands/process/base.py index 62741e28..eda4962c 100644 --- a/src/hermes/commands/process/base.py +++ b/src/hermes/commands/process/base.py @@ -82,14 +82,18 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.info(f"### Run {plugin_name} plugin") # run plugin try: + generate_strategies_start = datetime.datetime.now().isoformat() additional_strategies = plugin_func(self) + generate_strategies_end = datetime.datetime.now().isoformat() except Exception: self.log.exception(f"### Unknown error while executing the {plugin_name} plugin, skipping it now.") continue self.log.info(f"### Add the strategies to the merge document {plugin_name} plugin") # add strategies to the merge document + merge_strategies_start = datetime.datetime.now().isoformat() merged_doc.add_strategy(additional_strategies) + merge_strategies_end = datetime.datetime.now().isoformat() any_strategies_loaded = True if prov_doc is None: @@ -97,12 +101,15 @@ def __call__(self, args: argparse.Namespace) -> None: plugin = prov_doc.add_hermes_plugin("process", plugin_name, plugin_func, self) new_strategy_generation = prov_doc.add_activity(data={ "schema:description": "generate new merge strategies", - "prov:wasAssociatedWith": plugin.ref + "prov:wasAssociatedWith": plugin.ref, + "prov:startedAtTime": generate_strategies_start, + "prov:endedAtTime": generate_strategies_end }) new_strategies = prov_doc.add_entity(data={ # TODO: record strategies "schema:description": f"new merge strategies of plugin {plugin_name}", "prov:wasAttributedTo": plugin.ref, - "prov:wasGeneratedBy": new_strategy_generation.ref + "prov:wasGeneratedBy": new_strategy_generation.ref, + "prov:generatedAtTime": generate_strategies_end }) if merged_strategies is None: merged_strategies = new_strategies @@ -112,13 +119,16 @@ def __call__(self, args: argparse.Namespace) -> None: "schema:description": "merging the new strategies into the others", "prov:used": [merged_strategies.ref, new_strategies.ref], "prov:wasInformedBy": [strategy_action.ref, new_strategy_generation.ref], - "prov:wasAssociatedWith": process_command.ref + "prov:wasAssociatedWith": process_command.ref, + "prov:startedAtTime": merge_strategies_start, + "prov:endedAtTime": merge_strategies_end }) merged_strategies = prov_doc.add_entity(data={ # TODO: record strategies "schema:description": "the merge strategies of multiple plugins merged together", "prov:wasDerivedFrom": [merged_strategies.ref, new_strategies.ref], "prov:wasGeneratedBy": strategy_action.ref, - "prov:wasAttributedTo": process_command.ref + "prov:wasAttributedTo": process_command.ref, + "prov:generatedAtTime": merge_strategies_end }) if not any_strategies_loaded: @@ -135,7 +145,9 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.info(f"### Load data from {harvester} plugin") # load data from harvester try: + load_start = datetime.datetime.now().isoformat() metadata = SoftwareMetadata.load_from_cache(ctx, harvester) + load_end = datetime.datetime.now().isoformat() except Exception: # skip this harvester when the data is invalid if prov_doc is not None: @@ -160,7 +172,9 @@ def __call__(self, args: argparse.Namespace) -> None: new_action = prov_doc.add_activity(data={ # load of new data "schema:description": f"loads the data from {harvester} plugin", "prov:wasAssociatedWith": [process_command.ref, hermes_cache.ref], - "prov:used": stored_results + "prov:used": stored_results, + "prov:startedAtTime": load_start, + "prov:endedAtTime": load_end }) new_data = prov_doc.add_entity(data={ # new data to be merged "@type": "schema:CreativeWork", @@ -168,7 +182,8 @@ def __call__(self, args: argparse.Namespace) -> None: "schema:text": str(metadata.compact()), # TODO: maybe "prov:value" instead? "prov:wasAttributedTo": [process_command.ref, hermes_cache.ref], "prov:wasGeneratedBy": new_action.ref, - "prov:wasDerivedFrom": stored_results + "prov:wasDerivedFrom": stored_results, + "prov:generatedAtTime": load_end }) if merged_any: # One pass must have been completed already. @@ -183,13 +198,18 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.info(f"### Merge data from {harvester} plugin") # merge data into the merge dict try: + merge_start = datetime.datetime.now().isoformat() merged_doc.update(metadata) + merge_end = datetime.datetime.now().isoformat() except Exception as e: # TODO: Maybe this state is recoverable by starting over again and skipping this plugin. self.log.critical(f"### Merging the data from {harvester} plugin resulted in an error.", exc_info=True) raise RuntimeError(f"Merging the data from {harvester} plugin failed.") from e if prov_doc is not None: + if merged_any: + new_action["prov:startedAtTime"] = merge_start + new_action["prov:endedAtTime"] = merge_end last_action = merged_doc.prov_objects[0] if merged_any else new_action last_data = merged_doc.prov_objects[2] if merged_any else new_data merged_any = True diff --git a/src/hermes/model/merge/container.py b/src/hermes/model/merge/container.py index 470d3161..c2330029 100644 --- a/src/hermes/model/merge/container.py +++ b/src/hermes/model/merge/container.py @@ -7,6 +7,7 @@ from __future__ import annotations +import datetime from typing import TYPE_CHECKING, Any, Callable, Optional, Union from typing_extensions import Self @@ -237,6 +238,7 @@ def __setitem__(self: Self, key: str, value: Union[JSON_LD_VALUE, BASIC_TYPE, TI ``self[key]``. """ # create the new item if self[key] and value have to be merged. + merge_start = datetime.datetime.now().isoformat() if key in self: if self.prov_objects[0] is not None: last_merged_data = self.prov_objects[2] @@ -254,8 +256,12 @@ def __setitem__(self: Self, key: str, value: Union[JSON_LD_VALUE, BASIC_TYPE, TI create_new_merged_data = True # update the entry of self[key] super().__setitem__(key, value) + merge_end = datetime.datetime.now().isoformat() if self.prov_objects[0] is None: return + if merge_activity is not None: + merge_activity["prov:startedAtTime"] = merge_start + merge_activity["prov:endedAtTime"] = merge_end self.prov_objects[0] = merge_activity if create_new_merged_data: outer_most_parent = self @@ -267,7 +273,8 @@ def __setitem__(self: Self, key: str, value: Union[JSON_LD_VALUE, BASIC_TYPE, TI "schema:text": str(outer_most_parent.compact()), # TODO: maybe "prov:value" instead? "prov:wasAttributedTo": self.prov_doc.get_hermes_command("process").ref, "prov:wasGeneratedBy": merge_activity.ref, - "prov:wasDerivedFrom": {"@list": [self.prov_objects[1].ref, self.prov_objects[2].ref]} + "prov:wasDerivedFrom": {"@list": [self.prov_objects[1].ref, self.prov_objects[2].ref]}, + "prov:generatedAtTime": merge_end }) else: self.prov_objects[2]["prov:wasGeneratedBy"].append(merge_activity.ref) diff --git a/src/hermes/model/provenance/ld_prov.py b/src/hermes/model/provenance/ld_prov.py index e9c1850a..ba86c274 100644 --- a/src/hermes/model/provenance/ld_prov.py +++ b/src/hermes/model/provenance/ld_prov.py @@ -218,7 +218,7 @@ def add_hermes_plugin(self, step: str, name: str, plugin: HermesPlugin, command: except Exception: del data["schema:supportingData"] try: - data["version"] = metadata(plugin.__module__)["version"] + data["schema:softwareVersion"] = metadata(plugin.__module__)["version"] except Exception: pass node = self.add_agent(data=data) From 1e83028a00a9b89390a42d34c13c316498710d92 Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Wed, 5 Aug 2026 15:38:22 +0200 Subject: [PATCH 17/20] add diagrams of provenance for future use in adr (exists only on develop) --- .../hermes-prov-diagram/hermes-prov.drawio | 4162 +++++++++++++++++ .../hermes-prov.drawio.license | 3 + docs/adr/hermes-prov-diagram/hermes-prov.svg | 4 + .../hermes-prov.svg.license | 3 + 4 files changed, 4172 insertions(+) create mode 100644 docs/adr/hermes-prov-diagram/hermes-prov.drawio create mode 100644 docs/adr/hermes-prov-diagram/hermes-prov.drawio.license create mode 100644 docs/adr/hermes-prov-diagram/hermes-prov.svg create mode 100644 docs/adr/hermes-prov-diagram/hermes-prov.svg.license diff --git a/docs/adr/hermes-prov-diagram/hermes-prov.drawio b/docs/adr/hermes-prov-diagram/hermes-prov.drawio new file mode 100644 index 00000000..70b06617 --- /dev/null +++ b/docs/adr/hermes-prov-diagram/hermes-prov.drawio @@ -0,0 +1,4162 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/adr/hermes-prov-diagram/hermes-prov.drawio.license b/docs/adr/hermes-prov-diagram/hermes-prov.drawio.license new file mode 100644 index 00000000..e4d2c6e9 --- /dev/null +++ b/docs/adr/hermes-prov-diagram/hermes-prov.drawio.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2026 German Aerospace Center (DLR) + +SPDX-License-Identifier: CC-BY-SA-4.0 \ No newline at end of file diff --git a/docs/adr/hermes-prov-diagram/hermes-prov.svg b/docs/adr/hermes-prov-diagram/hermes-prov.svg new file mode 100644 index 00000000..56e2a032 --- /dev/null +++ b/docs/adr/hermes-prov-diagram/hermes-prov.svg @@ -0,0 +1,4 @@ + + + +
wasGeneratedBy
used
used
wasDerivedFrom
wasDerivedFrom
wasGeneratedBy
wasAssociatedWith
wasAttributedTo
wasAssociatedWith
wasAssociatedWith
wasAssociatedWith
wasInfluencedBy
used
wasDerivedFrom
actedOnBehalfOf
used
used
used
wasDerivedFrom
wasDerivedFrom
wasGeneratedBy
actedOnBehalfOf
wasAttributedTo
wasAssociatedWith
actedOnBehalfOf
wasDerivedFrom
wasInfluencedBy
wasAttributedTo
wasAttributedTo
wasAttributedTo
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasGeneratedBy
wasGeneratedBy
wasGeneratedBy
wasAssociatedWith
wasAssociatedWith
wasAssociatedWith
wasAttributedTo
wasAttributedTo
wasAttributedTo
wasAssociatedWith
wasAttributedTo
wasAttributedTo
used
used
wasAssociatedWith
wasGeneratedBy
wasAssociatedWith
wasAttributedTo
wasAssociatedWith
wasInformedBy
wasAttributedTo
wasGeneratedBy
wasInformedBy
wasGeneratedBy
used
used
wasGeneratedBy
wasInformedBy
used
wasDerivedFrom
wasDerivedFrom
actedOnBehalfOf
wasAttributedTo
wasAssociatedWith
used
used
used
used
used
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
used
used
wasAssociatedWith
wasInformedBy
used
used
wasInformedBy
wasGeneratedBy
wasInformedBy
used
wasDerivedFrom
wasGeneratedBy
used
wasInformedBy
wasDerivedFrom
wasGeneratedBy
wasDerivedFrom
used
wasGeneratedBy
wasInformedBy
used
wasGeneratedBy
wasGeneratedBy
used
used
used
used
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasAssociatedWith
wasAssociatedWith
wasAttributedTo
wasAttributedTo
wasAssociatedWith
wasAssociatedWith
wasAttributedTo
wasAttributedTo
wasAssociatedWith
wasInformedBy
used
used
used
used
used
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
used
wasAssociatedWith
wasInformedBy
used
wasInformedBy
wasGeneratedBy
wasInformedBy
used
wasDerivedFrom
wasGeneratedBy
used
wasInformedBy
wasDerivedFrom
wasGeneratedBy
wasDerivedFrom
used
wasGeneratedBy
wasInformedBy
used
wasGeneratedBy
wasGeneratedBy
used
used
used
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasAssociatedWith
wasAssociatedWith
wasAttributedTo
wasAttributedTo
wasAssociatedWith
wasAssociatedWith
wasAttributedTo
wasAttributedTo
wasAssociatedWith
wasAttributedTo
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasDerivedFrom
wasAssociatedWith
wasInformedBy
wasAttributedTo
wasGeneratedBy
used
wasDerivedFrom
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasAttributedTo
wasAssociatedWith
wasAttributedTo
wasAttributedTo
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
used
wasGeneratedBy
wasGeneratedBy
wasGeneratedBy
wasAssociatedWith
wasAssociatedWith
actedOnBehalfOf
actedOnBehalfOf
wasGeneratedBy
wasGeneratedBy
actedOnBehalfOf
wasGeneratedBy
wasAttributedTo
wasAssociatedWith
actedOnBehalfOf
used
wasAssociatedWith
wasAttributedTo
wasAttributedTo
wasAttributedTo
wasAttributedTo
used
wasAssociatedWith
wasAttributedTo
wasAttributedTo
wasAttributedTo
wasAttributedTo
wasAttributedTo
used
used
used
wasAssociatedWith
used
used
wasAssociatedWith
wasInformedBy
wasInformedBy
wasAttributedTo
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasGeneratedBy
wasGeneratedBy
used
used
used
wasAttributedTo
wasAssociatedWith
wasDerivedFrom
wasGeneratedBy
wasDerivedFrom
wasGeneratedBy
wasGeneratedBy
wasAssociatedWith
wasAssociatedWith
actedOnBehalfOf
wasAssociatedWith
wasAssociatedWith
wasAssociatedWith
wasAssociatedWith
actedOnBehalfOf
wasAssociatedWith
wasInformedBy
wasInformedBy
wasAssociatedWith
wasAttributedTo
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasGeneratedBy
wasGeneratedBy
used
used
used
wasAttributedTo
wasAssociatedWith
wasDerivedFrom
wasGeneratedBy
wasDerivedFrom
wasGeneratedBy
wasGeneratedBy
wasAssociatedWith
wasInformedBy
wasInformedBy
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
actedOnBehalfOf
actedOnBehalfOf
wasAttributedTo
wasDerivedFrom
wasGeneratedBy
wasGeneratedBy
used
used
used
wasAttributedTo
wasAssociatedWith
wasGeneratedBy
wasGeneratedBy
wasGeneratedBy
wasAssociatedWith
wasAssociatedWith
wasAssociatedWith
wasAssociatedWith
actedOnBehalfOf
actedOnBehalfOf
harvest plugin

name, version, settings
harvest source

path uri
.hermes/harvest/
{plugin_name}/codemeta.json

text, path uri, date created
harvested metadata

data
software-metadata

date, time
map

end time
write

start time, end time
.hermes/harvest/
{plugin_name}/expanded.json

text, path uri, date created
.hermes/harvest/
{plugin_name}/context.json

text, path uri, date created
Legend
design
meaning
provenance: Agent
provenance: Entity
provenance: Activity
bold text
record those properties always
solid lining
record as detailed as possible
dashed lining
record without many details
grayed out
optional / not always existent
name
properties
name
properties
name
properties
harvest plugin

name, version, settings
harvest source

path uri
.hermes/harvest/
{plugin_name}/codemeta.json

text, path uri, date created
harvested metadata

data
software-metadata

data, time
map

end time
write

start time, end time
.hermes/harvest/
{plugin_name}/expanded.json

text, path uri, date created
HARVEST
hermes

version
HERMES cache
load

func, args, kwargs, source, time
harvest base plugin

settings
load

func, args, kwargs, source, time
harvest command

settings
.hermes/harvest/
{plugin_name}/context.json

text, path uri, date created
process plugin

name, version, settings
merge strategies

strategies, time
process plugin

name, version, settings
.hermes/process/result/
codemeta.json

text, path uri, date created
merge strategies

strategies, time
merge strategies

start time, end time
write

start time, end time
.hermes/process/result/
expanded.json

text, path uri, date created
PROCESS
generate merge strategies

start time, end time
process base plugin

settings
generate merge strategies

start time, end time
process command

settings
.hermes/process/result/
context.json

text, path uri, date created
process plugin

name, version, settings
generate merge strategies

start time, end time
merge strategies

start time, end time
merged strategies

strategies, time
merged strategies

strategies, time
harvest plugin

name, version, settings
harvest source

path uri
.hermes/harvest/
{plugin_name}/codemeta.json

text, path uri, date created
harvested metadata

data
software-metadata

data, time
map

end time
write

start time, end time
.hermes/harvest/
{plugin_name}/expanded.json

text, path uri, date created
.hermes/harvest/
{plugin_name}/context.json

text, path uri, date created
load

func, args, kwargs, source, time
software-metadata

data, time
load

start time, end time
software-metadata

data, time
load

start time, end time
software-metadata

data, time
load

start time, end time
used
used
wasAssociatedWith
reject/ replace/ ...
value with other value

start time, end time, strategy used
merge value at key

key, strategy used
merge

start time, end time
software-metadata

time, data
reject/ replace/ ...
value with other value

start time, end time, strategy used
software-metadata

time, data
reject/ replace/ ...
value with other value

start time, end time, strategy used
software-metadata

time, data
software-metadata

time, data
reject/ replace/ ...
value with other value

start time, end time, strategy used
reject/ replace/ ...
value with other value

start time, end time, strategy used
merge value at key

key, strategy used
merge

start time, end time
software-metadata

time, data
reject/ replace/ ...
value with other value

start time, end time, strategy used
software-metadata

time, data
reject/ replace/ ...
value with other value

start time, end time, strategy used
software-metadata

time, data
software-metadata

time, data
reject/ replace/ ...
value with other value

start time, end time, strategy used
merge strategies

strategies, time
curate command

settings
software-metadata

data, time
load

start time, end time
curate base plugin

settings
curate plugin

name, version, settings
software-metadata

data, time
.hermes/curate/result/
codemeta.json

text, path uri, time created
write

start time, end time
.hermes/curate/result/
expanded.json

text, path uri, time created
.hermes/curate/result/
context.json

text, path uri, time created
CURATE
used
wasDerivedFrom
actedOnBehalfOf
used
used
used
wasDerivedFrom
wasDerivedFrom
wasGeneratedBy
actedOnBehalfOf
wasAttributedTo
wasAssociatedWith
wasAssociatedWith
actedOnBehalfOf
wasDerivedFrom
wasAttributedTo
wasAttributedTo
wasDerivedFrom
wasGeneratedBy
wasAssociatedWith
wasAssociatedWith
wasAssociatedWith
wasAssociatedWith
deposit command

settings
software-metadata

data, time
load

start time, end time
deposit base plugin

settings
deposit plugin

name, version, settings
mapped data for deposit

data, time
.hermes/deposit/
{deposit_plugin}/deposit.json

text, path uri, time created
write

start time, end time
DEPOSIT
map

start time, end time
updated metadata

data, time
.hermes/deposit/
{deposit_plugin}/result.json

text, path uri, time created
write

start time, end time
 \ No newline at end of file diff --git a/docs/adr/hermes-prov-diagram/hermes-prov.svg.license b/docs/adr/hermes-prov-diagram/hermes-prov.svg.license new file mode 100644 index 00000000..e4d2c6e9 --- /dev/null +++ b/docs/adr/hermes-prov-diagram/hermes-prov.svg.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2026 German Aerospace Center (DLR) + +SPDX-License-Identifier: CC-BY-SA-4.0 \ No newline at end of file From 66c61b2728523ceacd36ad6c12bb51d4f7c802e3 Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Thu, 6 Aug 2026 11:55:48 +0200 Subject: [PATCH 18/20] add recording of merge strategies --- src/hermes/commands/process/base.py | 3 +++ src/hermes/model/merge/action.py | 18 ++++++++++++++++++ src/hermes/model/merge/container.py | 18 ++++++++++++------ 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/src/hermes/commands/process/base.py b/src/hermes/commands/process/base.py index eda4962c..1fcba96e 100644 --- a/src/hermes/commands/process/base.py +++ b/src/hermes/commands/process/base.py @@ -106,7 +106,9 @@ def __call__(self, args: argparse.Namespace) -> None: "prov:endedAtTime": generate_strategies_end }) new_strategies = prov_doc.add_entity(data={ # TODO: record strategies + "@type": "schema:CreativeWork", "schema:description": f"new merge strategies of plugin {plugin_name}", + "schema:text": str(additional_strategies), # TODO: maybe "prov:value" instead? "prov:wasAttributedTo": plugin.ref, "prov:wasGeneratedBy": new_strategy_generation.ref, "prov:generatedAtTime": generate_strategies_end @@ -125,6 +127,7 @@ def __call__(self, args: argparse.Namespace) -> None: }) merged_strategies = prov_doc.add_entity(data={ # TODO: record strategies "schema:description": "the merge strategies of multiple plugins merged together", + "schema:text": str(merged_doc.strategies), # TODO: maybe "prov:value" instead? "prov:wasDerivedFrom": [merged_strategies.ref, new_strategies.ref], "prov:wasGeneratedBy": strategy_action.ref, "prov:wasAttributedTo": process_command.ref, diff --git a/src/hermes/model/merge/action.py b/src/hermes/model/merge/action.py index f2cfc7b3..9c52115c 100644 --- a/src/hermes/model/merge/action.py +++ b/src/hermes/model/merge/action.py @@ -49,6 +49,16 @@ def merge( """ raise NotImplementedError() + def __repr__(self) -> str: + """ + A generic stringify method for MergeActions. + Please overwrite this method if your MergeAction should be represented differently in the provenance data. + (I.e. if not all important attributes are recorded or some attributes string representation is not adequat.) + """ + if self.__dict__: + return f"{self.__module__}.{self.__class__.__qualname__} with attributes {str(self.__dict__)}" + return f"{self.__module__}.{self.__class__.__qualname__}" + class Reject(MergeAction): """ :class:`MergeAction` providing a merge function for rejecting the incoming item. """ @@ -209,6 +219,10 @@ def merge( return value + def __repr__(self): + return f"{self.__module__}.{self.__class__.__qualname__} with attributes " \ + f"{{'match': {self.match.__module__}.{self.match.__qualname__}, 'reject_incoming': {self.reject_incoming}}}" + class MergeSet(MergeAction): """ @@ -275,6 +289,10 @@ def merge( # Return the merged values. return value + def __repr__(self): + return f"{self.__module__}.{self.__class__.__qualname__} with attributes " \ + f"{{'match': {self.match.__module__}.{self.match.__qualname__}}}" + class IdMerge(MergeAction): """ :class:`MergeAction` providing a merge function for merging ids, i.e. error if not equals else do nothing. """ diff --git a/src/hermes/model/merge/container.py b/src/hermes/model/merge/container.py index c2330029..86c5a531 100644 --- a/src/hermes/model/merge/container.py +++ b/src/hermes/model/merge/container.py @@ -248,8 +248,8 @@ def __setitem__(self: Self, key: str, value: Union[JSON_LD_VALUE, BASIC_TYPE, TI elif self.prov_objects[0] is not None: merge_activity = self.prov_doc.add_activity(data={ "schema:name": f"merge values at {str(self.path+[key])}", - "schema:description": f"inserting value in the second 'used' value at {str(self.path+[key])} into the " - "first 'used' value at the same point", + "schema:description": f"Inserting value in the second 'used' value at {str(self.path+[key])} into the " + "first 'used' value at the same point, no merger needed.", "prov:used": {"@list": [self.prov_objects[2].ref, self.prov_objects[1].ref]}, "prov:wasInformedBy": self.prov_objects[0].ref }) @@ -324,8 +324,14 @@ def _merge_item( # search for all applicable strategies strategy = {**self.strategies.get(None, {})} ld_types = self.data_dict.get('@type', []) + type_of_used_strategy = None + key_of_used_strategy = None for ld_type in ld_types: strategy.update(self.strategies.get(ld_type, {})) + if key in self.strategies.get(ld_type, {}): + type_of_used_strategy = ld_type + key_of_used_strategy = key + # choose one merge strategy and return the item returned by following the merge startegy merger = strategy.get(key, strategy.get(None, None)) @@ -334,13 +340,13 @@ def _merge_item( if self.prov_objects[0] is not None: merge_activity = self.prov_doc.add_activity(data={ "schema:name": f"merge values at {str(self.path+[key])}", - "schema:description": f"merge value in the second 'used' value at {str(self.path+[key])} into the " - "first 'used' value at the same point using the third 'used' value", + "schema:description": f"Merge value in the second 'used' value at {str(self.path+[key])} into the " + f"first 'used' value at the same point using the merger {merger} for type " + f"{type_of_used_strategy} and key {key_of_used_strategy}", "prov:wasAssociatedWith": self.prov_doc.get_hermes_command("process").ref, "prov:used": {"@list": [ self.prov_objects[2].ref, - self.prov_objects[1].ref, - f"{merger.merge.__module__}.{merger.merge.__qualname__}" + self.prov_objects[1].ref ]}, "prov:wasInformedBy": self.prov_objects[0].ref }) From b775072a040bf0be11c1a1502c1b459551e4deae Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Thu, 6 Aug 2026 17:03:07 +0200 Subject: [PATCH 19/20] add merge strategies for invenio publish --- hermes.toml | 3 + pyproject.toml | 1 + src/hermes/commands/process/invenio_merge.py | 93 ++++++++++++++++++++ 3 files changed, 97 insertions(+) create mode 100644 src/hermes/commands/process/invenio_merge.py diff --git a/hermes.toml b/hermes.toml index a42a9406..dab72523 100644 --- a/hermes.toml +++ b/hermes.toml @@ -5,6 +5,9 @@ [harvest] sources = [ "cff", "toml" ] # ordered priority (first one is most important) +[process] +plugins = [ "invenio", "codemeta" ] + [curate] plugin = "pass_curate" diff --git a/pyproject.toml b/pyproject.toml index 17ea5087..8bb98ac6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,6 +75,7 @@ codemeta_doi = "hermes.commands.postprocess.invenio:codemeta_doi" [project.entry-points."hermes.process"] codemeta = "hermes.commands.process.standard_merge:CodemetaProcessPlugin" +invenio = "hermes.commands.process.invenio_merge:InvenioProcessPlugin" [project.entry-points."hermes.curate"] pass_curate = "hermes.commands.curate.pass_curate:DoNothingCuratePlugin" diff --git a/src/hermes/commands/process/invenio_merge.py b/src/hermes/commands/process/invenio_merge.py new file mode 100644 index 00000000..39e5c6ec --- /dev/null +++ b/src/hermes/commands/process/invenio_merge.py @@ -0,0 +1,93 @@ +# SPDX-FileCopyrightText: 2026 German Aerospace Center (DLR) +# +# SPDX-License-Identifier: Apache-2.0 + +# SPDX-FileContributor: Michael Fritzsche + + +from typing import Union +from typing_extensions import Self + +from hermes.commands.base import HermesCommand +from hermes.model.merge.action import MergeAction +from hermes.model.merge.container import ld_merge_dict, ld_merge_list +from hermes.model.types import ld_dict, ld_list +from hermes.model.types.ld_container import BASIC_TYPE, TIME_TYPE +from hermes.model.types.ld_context import iri_map as iri +from .base import HermesProcessPlugin + + +class InvenioMerge(MergeAction): + """ :class:`MergeAction` providing a merge function that tries to conform with Invenios metadata restrictions. """ + def merge( + self: Self, + target: ld_merge_dict, + key: list[Union[str, int]], + value: Union[ld_merge_list, str], + update: Union[BASIC_TYPE, TIME_TYPE, ld_dict, ld_list] + ) -> ld_merge_list: + print(key, value, update) + types = target.get("@type", []) + print(types) + if key[-1] == iri["schema:license"] and (iri["schema:SoftwareSourceCode"] in types or iri["schema:SoftwareApplication"] in types): + if len(value) == 1: + if isinstance(value[0], str) or ( + isinstance(value[0], (dict, ld_merge_dict)) and [*value[0].keys()] == ["@id"] + ): + if value != update: + target.reject(key, update) + return value + if isinstance(update, ld_list) and len(update) == 1: + if isinstance(update[0], str) or ( + isinstance(update[0], (dict, ld_merge_dict)) and [*update[0].keys()] == ["@id"] + ): + target.replace(key, value) + return update + target.reject(key, update) + return value + if ((key[-1] == iri["schema:familyName"] and iri["schema:Person"] in types) or + (key[-1] == iri["schema:name"] and iri["schema:Person"] in types) or + (key[-1] == iri["schema:name"] and (iri["schema:SoftwareSourceCode"] in types or iri["schema:SoftwareApplication"] in types)) + ): + if len(value) == 1: + if value != update: + target.reject(key, update) + return value + if len(update) == 1: + target.replace(key, value) + return update + if len(value) == len(update) == 0: + return value + target.reject(key, update) + return value + if ((key[-1] == iri["schema:version"] or key[-1] == iri["schema:description"]) and + (iri["schema:SoftwareSourceCode"] in types or iri["schema:SoftwareApplication"] in types) + ): + if len(value) == 1: + if value != update: + target.reject(key, update) + return value + if len(update) == 1: + target.replace(key, value) + return update + if len(value) == 0 or len(update) == 0: + return [] + target.reject(key, update) + return value + print("fail") + + +class InvenioProcessPlugin(HermesProcessPlugin): + def __call__(self, command: HermesCommand) -> dict[Union[str, None], dict[Union[str, None], MergeAction]]: + merger = InvenioMerge() + return { + iri["schema:SoftwareSourceCode"]: { + iri["schema:"+term]: merger for term in ["version", "name", "description", "license"] + }, + iri["schema:SoftwareApplication"]: { + iri["schema:"+term]: merger for term in ["version", "name", "description", "license"] + }, + iri["schema:Person"]: { + iri["schema:"+term]: merger for term in ["familyName", "name"] + } + } From 0cb53579a315145fc0e08f160f38649fff84245a Mon Sep 17 00:00:00 2001 From: notactuallyfinn Date: Thu, 6 Aug 2026 17:11:06 +0200 Subject: [PATCH 20/20] fix issue 434 --- src/hermes/commands/curate/base.py | 10 +++++----- src/hermes/commands/deposit/base.py | 18 +++++++++--------- src/hermes/commands/harvest/base.py | 13 +++++++------ src/hermes/commands/process/base.py | 20 ++++++++++---------- src/hermes/model/merge/container.py | 4 ++-- src/hermes/model/types/ld_container.py | 8 ++++++++ 6 files changed, 41 insertions(+), 32 deletions(-) diff --git a/src/hermes/commands/curate/base.py b/src/hermes/commands/curate/base.py index b8fc536b..b151e9f0 100644 --- a/src/hermes/commands/curate/base.py +++ b/src/hermes/commands/curate/base.py @@ -59,9 +59,9 @@ def __call__(self, args: argparse.Namespace) -> None: # load processed data ctx.prepare_step("process") try: - begin_load_at_time = datetime.datetime.now().isoformat() + begin_load_at_time = datetime.datetime.now() metadata = SoftwareMetadata.load_from_cache(ctx, "result") - end_load_at_time = datetime.datetime.now().isoformat() + end_load_at_time = datetime.datetime.now() except Exception as e: self.log.critical( "## The data from the process step could not be loaded or is invalid for some reason.", @@ -85,16 +85,16 @@ def __call__(self, args: argparse.Namespace) -> None: # run plugin try: curated_metadata = plugin_func(self, metadata) - end_curation_time = datetime.datetime.now().isoformat() + end_curation_time = datetime.datetime.now() except Exception as e: self.log.critical(f"## Unknown error while executing the {plugin_name} plugin.", exc_info=1) raise HermesPluginRunError(f"Something went wrong while running the curate plugin {plugin_name}") from e self.log.info("## Store curated data") # store metadata - begin_store_at_time = datetime.datetime.now().isoformat() + begin_store_at_time = datetime.datetime.now() curated_metadata.write_to_cache(ctx, "result") - stored_at_time = datetime.datetime.now().isoformat() + stored_at_time = datetime.datetime.now() if prov_doc is not None: curate_plugin = prov_doc.add_hermes_plugin("curate", plugin_name, plugin_func, self) diff --git a/src/hermes/commands/deposit/base.py b/src/hermes/commands/deposit/base.py index ace1e614..b8beb1c5 100644 --- a/src/hermes/commands/deposit/base.py +++ b/src/hermes/commands/deposit/base.py @@ -38,9 +38,9 @@ def __call__(self, command: HermesCommand, prov_doc: ld_prov_list) -> None: self.ctx.prepare_step("curate") try: - start_of_load = datetime.datetime.now().isoformat() + start_of_load = datetime.datetime.now() self.metadata = SoftwareMetadata.load_from_cache(self.ctx, "result") - end_of_load = datetime.datetime.now().isoformat() + end_of_load = datetime.datetime.now() except Exception as e: raise HermesValidationError("The results of the curate step are invalid.") from e self.ctx.finalize_step("curate") @@ -78,13 +78,13 @@ def __call__(self, command: HermesCommand, prov_doc: ld_prov_list) -> None: }) self.prepare() - start_of_map = datetime.datetime.now().isoformat() + start_of_map = datetime.datetime.now() deposit = self.map_metadata() - end_of_map = datetime.datetime.now().isoformat() + end_of_map = datetime.datetime.now() with self.ctx[target] as cache: - start_of_store = datetime.datetime.now().isoformat() + start_of_store = datetime.datetime.now() cache["deposit"] = deposit - end_of_store = datetime.datetime.now().isoformat() + end_of_store = datetime.datetime.now() if prov_doc is not None: map_action = prov_doc.add_activity(data={ @@ -128,11 +128,11 @@ def __call__(self, command: HermesCommand, prov_doc: ld_prov_list) -> None: self.create_new_version() updated_deposit = self.update_metadata() - end_of_update_map = datetime.datetime.now().isoformat() + end_of_update_map = datetime.datetime.now() with self.ctx[target] as cache: - start_of_second_store = datetime.datetime.now().isoformat() + start_of_second_store = datetime.datetime.now() cache["result"] = updated_deposit - end_of_second_store = datetime.datetime.now().isoformat() + end_of_second_store = datetime.datetime.now() self.ctx.finalize_step("deposit") if prov_doc is not None: diff --git a/src/hermes/commands/harvest/base.py b/src/hermes/commands/harvest/base.py index 32550181..ec02f948 100644 --- a/src/hermes/commands/harvest/base.py +++ b/src/hermes/commands/harvest/base.py @@ -47,9 +47,9 @@ def load(self, func, source, *args, **kwargs): f"{', ' + str(args) if args else ''}{', ' + str(kwargs) if kwargs else ''}).", "schema:name": f"{func.__module__}.{func.__qualname__}" } - io_operation["prov:startedAtTime"] = datetime.datetime.now().isoformat() + io_operation["prov:startedAtTime"] = datetime.datetime.now() result = func(source, *args, **kwargs) - io_operation["prov:endedAtTime"] = datetime.datetime.now().isoformat() + io_operation["prov:endedAtTime"] = datetime.datetime.now() loaded_metadata = {"schema:description": "the loaded data", "schema:text": str(result)} self.io_operations.append((source_metadata, io_operation, loaded_metadata)) return result @@ -132,13 +132,13 @@ def __call__(self, args: argparse.Namespace) -> None: except Exception: self.log.exception(f"### Unknown error while executing the {plugin_name} plugin, skipping it now.") continue - returned_at_time = datetime.datetime.now().isoformat() + returned_at_time = datetime.datetime.now() self.log.info(f"### Store metadata harvested by {plugin_name} plugin") # store harvested data - begin_store_at_time = datetime.datetime.now().isoformat() + begin_store_at_time = datetime.datetime.now() harvested_data.write_to_cache(ctx, plugin_name) - stored_at_time = datetime.datetime.now().isoformat() + stored_at_time = datetime.datetime.now() harvested_any = True remove_harvest_plugin_from_prov_doc(prov_doc, plugin_name) @@ -233,7 +233,8 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.critical("No harvest plugin ran successfully.") raise HermesPluginRunError("No harvest plugin ran successfully.") - def init_provenance_document(self) -> ld_prov_list: + @classmethod + def init_provenance_document(cls) -> ld_prov_list: ctx = HermesContext() ctx.prepare_step("harvest") with ctx["provenance"] as cache: diff --git a/src/hermes/commands/process/base.py b/src/hermes/commands/process/base.py index 1fcba96e..8bea1b6b 100644 --- a/src/hermes/commands/process/base.py +++ b/src/hermes/commands/process/base.py @@ -82,18 +82,18 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.info(f"### Run {plugin_name} plugin") # run plugin try: - generate_strategies_start = datetime.datetime.now().isoformat() + generate_strategies_start = datetime.datetime.now() additional_strategies = plugin_func(self) - generate_strategies_end = datetime.datetime.now().isoformat() + generate_strategies_end = datetime.datetime.now() except Exception: self.log.exception(f"### Unknown error while executing the {plugin_name} plugin, skipping it now.") continue self.log.info(f"### Add the strategies to the merge document {plugin_name} plugin") # add strategies to the merge document - merge_strategies_start = datetime.datetime.now().isoformat() + merge_strategies_start = datetime.datetime.now() merged_doc.add_strategy(additional_strategies) - merge_strategies_end = datetime.datetime.now().isoformat() + merge_strategies_end = datetime.datetime.now() any_strategies_loaded = True if prov_doc is None: @@ -148,9 +148,9 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.info(f"### Load data from {harvester} plugin") # load data from harvester try: - load_start = datetime.datetime.now().isoformat() + load_start = datetime.datetime.now() metadata = SoftwareMetadata.load_from_cache(ctx, harvester) - load_end = datetime.datetime.now().isoformat() + load_end = datetime.datetime.now() except Exception: # skip this harvester when the data is invalid if prov_doc is not None: @@ -201,9 +201,9 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.info(f"### Merge data from {harvester} plugin") # merge data into the merge dict try: - merge_start = datetime.datetime.now().isoformat() + merge_start = datetime.datetime.now() merged_doc.update(metadata) - merge_end = datetime.datetime.now().isoformat() + merge_end = datetime.datetime.now() except Exception as e: # TODO: Maybe this state is recoverable by starting over again and skipping this plugin. self.log.critical(f"### Merging the data from {harvester} plugin resulted in an error.", exc_info=True) @@ -225,12 +225,12 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.info("## Store processed metadata") # store processed data ctx.prepare_step("process") - begin_store_at_time = datetime.datetime.now().isoformat() + begin_store_at_time = datetime.datetime.now() with ctx["result"] as result_ctx: result_ctx["codemeta"] = merged_doc.compact() result_ctx["context"] = {"@context": merged_doc.full_context} result_ctx["expanded"] = merged_doc.ld_value - stored_at_time = datetime.datetime.now().isoformat() + stored_at_time = datetime.datetime.now() if prov_doc is not None: write = prov_doc.add_activity(data={ diff --git a/src/hermes/model/merge/container.py b/src/hermes/model/merge/container.py index 86c5a531..fc77fdcc 100644 --- a/src/hermes/model/merge/container.py +++ b/src/hermes/model/merge/container.py @@ -238,7 +238,7 @@ def __setitem__(self: Self, key: str, value: Union[JSON_LD_VALUE, BASIC_TYPE, TI ``self[key]``. """ # create the new item if self[key] and value have to be merged. - merge_start = datetime.datetime.now().isoformat() + merge_start = datetime.datetime.now() if key in self: if self.prov_objects[0] is not None: last_merged_data = self.prov_objects[2] @@ -256,7 +256,7 @@ def __setitem__(self: Self, key: str, value: Union[JSON_LD_VALUE, BASIC_TYPE, TI create_new_merged_data = True # update the entry of self[key] super().__setitem__(key, value) - merge_end = datetime.datetime.now().isoformat() + merge_end = datetime.datetime.now() if self.prov_objects[0] is None: return if merge_activity is not None: diff --git a/src/hermes/model/types/ld_container.py b/src/hermes/model/types/ld_container.py index 10d2a82c..19fd70ae 100644 --- a/src/hermes/model/types/ld_container.py +++ b/src/hermes/model/types/ld_container.py @@ -13,6 +13,8 @@ from typing import Any, Optional, TypeAlias, TYPE_CHECKING, Union from typing_extensions import Self +from hermes.model.types.ld_context import iri_map + from .pyld_util import JsonLdProcessor, bundled_loader if TYPE_CHECKING: from .ld_dict import ld_dict @@ -462,6 +464,12 @@ def typed_ld_to_py(cls: type[Self], data: list[dict[str, BASIC_TYPE]], **kwargs) """ # FIXME: #434 dates are not returned as datetime/ date/ time but as string ld_value = data[0]['@value'] + if iri_map["schema:DateTime"] == data[0]['@type']: + ld_value = datetime.fromisoformat(ld_value) + elif iri_map["schema:Date"] == data[0]['@type']: + ld_value = date.fromisoformat(ld_value) + elif iri_map["schema:Time"] == data[0]['@type']: + ld_value = time.fromisoformat(ld_value) return ld_value