diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 9e7621e..811b499 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: actions/setup-python@v2 with: python-version: "3.10" diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 4006fab..7340aac 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -15,7 +15,7 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: conda-incubator/setup-miniconda@v3 with: mamba-version: "*" @@ -35,4 +35,4 @@ jobs: uses: codecov/codecov-action@v4.0.1 with: token: ${{ secrets.CODECOV_TOKEN }} - slug: pyscal/atomRDF + slug: OCDO/tools4RDF diff --git a/pyproject.toml b/pyproject.toml index 3ea0fb7..707cc56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ readme = "README.md" requires-python = ">=3.8" license = {text = "MIT"} authors = [ - {name = "Abril Azocar Guzman", email = "sarath.menon@pyscal.org"}, + {name = "Abril Azocar Guzman"}, {name = "Sarath Menon", email = "sarath.menon@pyscal.org"} ] maintainers = [ @@ -30,7 +30,6 @@ classifiers = [ "Topic :: Scientific/Engineering", ] dependencies = [ - "numpy", "rdflib", "pyyaml", "graphviz", diff --git a/requirements.txt b/requirements.txt index 962c3fa..9fba33f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ -numpy rdflib pyyaml graphviz diff --git a/tests/test_regressions.py b/tests/test_regressions.py new file mode 100644 index 0000000..e0fa9ce --- /dev/null +++ b/tests/test_regressions.py @@ -0,0 +1,475 @@ +""" +Regression tests for defects found in the 2026-09 audit. +""" + +import pytest +from rdflib import Graph, Namespace, RDF, RDFS, OWL, XSD, BNode, Literal +from rdflib.collection import Collection +from rdflib.plugins.sparql import prepareQuery + +from tools4rdf.network.parser import OntoParser +from tools4rdf.network.network import OntologyNetworkBase +from tools4rdf.network.ontology import read_ontology +from tools4rdf.network.term import OntoTerm + +EX = Namespace("http://example.org/o#") + + +def make_graph(classes=(), triples=()): + g = Graph() + g.bind("ex", EX) + for c in classes: + g.add((EX[c], RDF.type, OWL.Class)) + for t in triples: + g.add(t) + return g + + +def add_union(g, subclass, members): + bn = BNode() + lst = BNode() + Collection(g, lst, [EX[m] for m in members]) + g.add((bn, RDF.type, OWL.Class)) + g.add((bn, OWL.unionOf, lst)) + g.add((EX[subclass], RDFS.subClassOf, bn)) + + +def test_unions_do_not_leak_between_mappings(): + g = make_graph("ABCDEF") + add_union(g, "E", ["A", "B"]) + add_union(g, "F", ["C", "D"]) + parser = OntoParser(g) + items = sorted(sorted(v["items"]) for v in parser.mappings.values()) + assert items == [["ex:A", "ex:B"], ["ex:C", "ex:D"]] + assert parser.attributes["class"]["ex:A"].subclasses == ["ex:E"] + + +def test_unions_do_not_leak_between_parsers(): + g = make_graph("ABE") + add_union(g, "E", ["A", "B"]) + OntoParser(g).attributes + # a second, unrelated ontology must still parse + onto = read_ontology() + assert "cmso:AtomicScaleSample" in onto.attributes["class"] + + +def test_nested_class_expression_as_superclass(): + g = make_graph("ABC") + inner = BNode() + inner_list = BNode() + Collection(g, inner_list, [EX.A, EX.B]) + g.add((inner, RDF.type, OWL.Class)) + g.add((inner, OWL.intersectionOf, inner_list)) + outer = BNode() + outer_list = BNode() + Collection(g, outer_list, [inner, EX.B]) + g.add((outer, RDF.type, OWL.Class)) + g.add((outer, OWL.unionOf, outer_list)) + g.add((EX.C, RDFS.subClassOf, outer)) + parser = OntoParser(g) + assert "ex:C" in parser.attributes["class"]["ex:B"].subclasses + + +def test_superclass_from_unloaded_ontology(): + ext = Namespace("http://external.org/") + g = make_graph("AC") + bn = BNode() + lst = BNode() + Collection(g, lst, [EX.A, ext.Z]) + g.add((bn, RDF.type, OWL.Class)) + g.add((bn, OWL.unionOf, lst)) + g.add((EX.C, RDFS.subClassOf, bn)) + parser = OntoParser(g) + assert "ex:C" in parser.attributes["class"]["ex:A"].subclasses + + +def test_named_individual_of_unloaded_class(): + ext = Namespace("http://external.org/") + g = make_graph("A") + g.add((EX.ind, RDF.type, OWL.NamedIndividual)) + g.add((EX.ind, RDF.type, ext.Z)) + g.add((EX.ind2, RDF.type, OWL.NamedIndividual)) + g.add((EX.ind2, RDF.type, EX.A)) + parser = OntoParser(g) + assert "ex:ind" in parser.attributes["class"] + assert parser.attributes["class"]["ex:A"].named_individuals == ["ex:ind2"] + + +def object_property_network(): + g = make_graph( + "ABC", + [ + (EX.p, RDF.type, OWL.ObjectProperty), + (EX.p, RDFS.domain, EX.A), + (EX.p, RDFS.range, EX.B), + ], + ) + for d in ["x", "y"]: + g.add((EX[d], RDF.type, OWL.DatatypeProperty)) + g.add((EX[d], RDFS.domain, EX.A)) + g.add((EX[d], RDFS.range, XSD.float)) + return OntologyNetworkBase(OntoParser(g)) + + +def test_object_property_source_with_several_destinations(): + net = object_property_network() + t = net.terms.ex + query = net.create_query(t.p, [t.x, t.y]) + assert "?A ex:p ?p ." in query + assert "?A ex:x ?xvalue ." in query + assert "?A ex:y ?yvalue ." in query + Graph().query(query) + + +def test_object_property_already_in_destinations(): + net = object_property_network() + t = net.terms.ex + query = net.create_query(t.p, [t.p, t.x]) + assert query.count("ex:p") == 1 + + +def test_filter_declares_xsd_prefix(): + onto = read_ontology() + c = onto.terms.cmso + query = onto.create_query(c.AtomicScaleSample, [c.hasNumberOfAtoms == 4]) + assert "PREFIX xsd: " in query + # must parse without rdflib's default namespace bindings + prepareQuery(query, initNs={}) + + +def test_term_label_is_kept(): + term = OntoTerm(uri="http://example.org/o#A", label="my label") + assert term.label == "my label" + + +def test_parser_reads_labels(): + g = make_graph( + "AB", + [ + (EX.A, RDFS.label, Literal("Klasse A", lang="de")), + (EX.A, RDFS.label, Literal("class A", lang="en")), + ], + ) + parser = OntoParser(g) + assert parser.attributes["class"]["ex:A"].label == "class A" + assert parser.attributes["class"]["ex:B"].label is None + + +def test_and_or_methods_return_combined_term(): + onto = read_ontology() + c = onto.terms.cmso + combined = (c.hasNumberOfAtoms > 1).and_(c.hasNumberOfAtoms < 10) + assert "&&" in combined._condition + combined = (c.hasNumberOfAtoms > 1).or_(c.hasNumberOfAtoms < 10) + assert "||" in combined._condition + + +def test_terms_are_hashable_and_comparable(): + onto = read_ontology() + c = onto.terms.cmso + assert len({c.AtomicScaleSample, c.AtomicScaleSample, c.hasNumberOfAtoms}) == 2 + assert c.AtomicScaleSample != "cmso:AtomicScaleSample" + assert not (c.AtomicScaleSample == "anything") + # data properties compared to terms do not produce conditions + assert (c.hasNumberOfAtoms == c.hasNumberOfAtoms) is True + assert c.hasNumberOfAtoms not in [c.hasVolume] + + +def data_property_network(range_): + triples = [ + (EX.v, RDF.type, OWL.DatatypeProperty), + (EX.v, RDFS.domain, EX.A), + ] + if range_ is not None: + triples.append((EX.v, RDFS.range, range_)) + return OntologyNetworkBase(OntoParser(make_graph("A", triples))) + + +def run_filter(range_, stored, value): + net = data_property_network(range_) + kg = Graph() + kg.add((EX.i, RDF.type, EX.A)) + kg.add((EX.i, EX.v, stored)) + query = net.create_query(net.terms.ex.A, [net.terms.ex.v == value]) + return query, len(kg.query(query)) + + +def test_filter_string_with_quotes(): + query, rows = run_filter(XSD.string, Literal('P"1'), 'P"1') + assert rows == 1 + + +def test_filter_boolean_uses_canonical_lexical_form(): + query, rows = run_filter(XSD.boolean, Literal(True), True) + assert '"true"^^xsd:boolean' in query + assert rows == 1 + + +def test_filter_rdfs_literal_range(): + query, rows = run_filter(RDFS.Literal, Literal("x"), "x") + assert "xsd:Literal" not in query + assert rows == 1 + + +def test_filter_without_range(): + query, rows = run_filter(None, Literal(3), 3) + assert rows == 1 + + +def test_hyphenated_intermediate_class(): + g = make_graph( + ["A", "Twin-like"], + [ + (EX.p, RDF.type, OWL.ObjectProperty), + (EX.p, RDFS.domain, EX.A), + (EX.p, RDFS.range, EX["Twin-like"]), + (EX.x, RDF.type, OWL.DatatypeProperty), + (EX.x, RDFS.domain, EX["Twin-like"]), + (EX.x, RDFS.range, XSD.float), + ], + ) + net = OntologyNetworkBase(OntoParser(g)) + kg = Graph() + kg.add((EX.a, RDF.type, EX.A)) + kg.add((EX.a, EX.p, EX.t)) + kg.add((EX.t, EX.x, Literal(1.0))) + df = net.query(kg, net.terms.ex.A, [net.terms.ex.x]) + assert len(df) == 1 + + +def test_every_source_keeps_the_filter(): + g = make_graph( + "AB", + [ + (EX.x, RDF.type, OWL.DatatypeProperty), + (EX.x, RDFS.domain, EX.A), + (EX.x, RDFS.domain, EX.B), + (EX.x, RDFS.range, XSD.float), + ], + ) + net = OntologyNetworkBase(OntoParser(g)) + t = net.terms.ex + queries = net.create_query([t.A, t.B], [t.x > 3]) + assert len(queries) == 2 + assert all("FILTER" in q for q in queries) + + +def test_create_query_does_not_mutate_shared_terms(): + onto = read_ontology() + c = onto.terms.cmso + with pytest.raises(Exception): + # no path exists, so this raises half-way through + onto.create_query( + c.AtomicScaleSample, [[c.hasSpaceGroupSymbol, c.hasNumberOfAtoms]] + ) + assert onto.terms.cmso.hasNumberOfAtoms._parents == [] + onto.create_query(c.AtomicScaleSample, [[c.Material, c.CrystalStructure]]) + assert onto.terms.cmso.CrystalStructure._parents == [] + + +def test_add_path_creates_edges_for_class_and_subclasses(): + g = make_graph( + "ABC", + [ + (EX.C, RDFS.subClassOf, EX.A), + (EX.p, RDF.type, OWL.ObjectProperty), + ], + ) + net = OntologyNetworkBase(OntoParser(g)) + net.add_path(("ex:A", "ex:p", "ex:B")) + assert net.g.has_edge("ex:A", "ex:p") + assert net.g.has_edge("ex:C", "ex:p") + assert net.g.has_edge("ex:p", "ex:B") + assert "?A ex:p ?p ." in net.create_query(net.terms.ex.A, [net.terms.ex.p]) + + +def test_add_path_rdfs_label_like_atomrdf(): + net = OntologyNetworkBase(OntoParser(make_graph("A"))) + net.add_namespace("rdfs", "http://www.w3.org/2000/01/rdf-schema#") + net.add_term( + "http://www.w3.org/2000/01/rdf-schema#label", + "data_property", + rn=["str"], + ) + net.add_path(("ex:A", "rdfs:label", "string")) + kg = Graph() + kg.add((EX.a, RDF.type, EX.A)) + kg.add((EX.a, RDFS.label, Literal("name"))) + df = net.query(kg, net.terms.ex.A, [net.terms.rdfs.label == "name"]) + assert len(df) == 1 + + +def test_add_path_survives_combining_networks(): + net = OntologyNetworkBase(OntoParser(make_graph("AB", [(EX.p, RDF.type, OWL.ObjectProperty)]))) + net.add_path(("ex:A", "ex:p", "ex:B")) + other = OntologyNetworkBase(OntoParser(make_graph("D"))) + combined = net + other + assert combined.g.has_edge("ex:A", "ex:p") + + +def test_add_path_rejects_unknown_property(): + net = OntologyNetworkBase(OntoParser(make_graph("AB"))) + with pytest.raises(ValueError): + net.add_path(("ex:A", "ex:missing", "ex:B")) + + +def test_most_specific_namespace_wins(): + g = Graph() + g.bind("base", Namespace("http://example.org/")) + g.bind("ex", EX) + g.add((EX.A, RDF.type, OWL.Class)) + parser = OntoParser(g) + assert "ex:A" in parser.attributes["class"] + assert "base:A" not in parser.attributes["class"] + + +def test_same_local_name_in_two_namespaces(): + Y = Namespace("http://other.org/y#") + g = make_graph( + "AB", + [ + (EX.hasB, RDF.type, OWL.ObjectProperty), + (EX.hasB, RDFS.domain, EX.A), + (EX.hasB, RDFS.range, EX.B), + (EX.value, RDFS.domain, EX.A), + (Y.value, RDFS.domain, EX.B), + ], + ) + g.bind("y", Y) + for ns in (EX, Y): + g.add((ns.value, RDF.type, OWL.DatatypeProperty)) + g.add((ns.value, RDFS.range, XSD.float)) + net = OntologyNetworkBase(OntoParser(g)) + kg = Graph() + kg.add((EX.a, RDF.type, EX.A)) + kg.add((EX.a, EX.hasB, EX.b)) + kg.add((EX.a, EX.value, Literal(1.0))) + kg.add((EX.b, Y.value, Literal(2.0))) + df = net.query(kg, net.terms.ex.A, [net.terms.ex.value, net.terms.y.value > 1]) + assert list(df.columns) == ["A", "ex_valuevalue", "y_valuevalue"] + assert len(df) == 1 + df = net.query(kg, net.terms.ex.A, [net.terms.ex.value, net.terms.y.value > 5]) + assert len(df) == 0 + + +def test_object_property_sources_start_from_first_common_domain(): + import subprocess + import sys + + script = """ +from rdflib import Graph, Namespace, RDF, RDFS, OWL +from tools4rdf.network.parser import OntoParser +from tools4rdf.network.network import OntologyNetworkBase +EX = Namespace("http://example.org/o#") +g = Graph() +g.bind("ex", EX) +for c in ["Alpha", "Beta", "Gamma", "Delta"]: + g.add((EX[c], RDF.type, OWL.Class)) +for p in ["p", "q"]: + g.add((EX[p], RDF.type, OWL.ObjectProperty)) + g.add((EX[p], RDFS.range, EX.Delta)) + for c in ["Alpha", "Beta", "Gamma"]: + g.add((EX[p], RDFS.domain, EX[c])) +net = OntologyNetworkBase(OntoParser(g)) +print([l for l in net.create_query([net.terms.ex.p, net.terms.ex.q]).splitlines() if l.startswith("SELECT")]) +""" + outputs = set() + for seed in ["1", "2", "3", "4", "5", "6"]: + env = {"PYTHONHASHSEED": seed, "PATH": ""} + out = subprocess.run( + [sys.executable, "-c", script], capture_output=True, text=True, env=env + ) + assert out.returncode == 0, out.stderr + outputs.add(out.stdout.strip()) + assert len(outputs) == 1 + + +def test_prefixes_are_sorted(): + onto = read_ontology() + c = onto.terms.cmso + query = onto.create_query(c.AtomicScaleSample, [c.hasNumberOfAtoms == 4]) + prefixes = [line for line in query.splitlines() if line.startswith("PREFIX")] + assert prefixes == sorted(prefixes) + + +def test_remote_query_is_sent_to_the_endpoint(monkeypatch): + from rdflib.plugins.stores.sparqlstore import SPARQLStore + + sent = {} + + def fake_query(self, query, *args, **kwargs): + sent["endpoint"] = self.query_endpoint + sent["query"] = query + return [] + + monkeypatch.setattr(SPARQLStore, "query", fake_query) + onto = read_ontology() + c = onto.terms.cmso + onto.query( + "https://example.org/sparql", c.AtomicScaleSample, [c.hasNumberOfAtoms], limit=5 + ) + assert sent["endpoint"] == "https://example.org/sparql" + assert "SERVICE" not in sent["query"] + assert sent["query"].rstrip().endswith("LIMIT 5") + + +def test_query_rejects_non_url_string(): + onto = read_ontology() + c = onto.terms.cmso + with pytest.raises(ValueError): + onto.query("not a url", c.AtomicScaleSample, [c.hasNumberOfAtoms]) + + +def test_named_individual_as_source_raises(): + g = make_graph("A", [(EX.ind, RDF.type, OWL.NamedIndividual), (EX.ind, RDF.type, EX.A)]) + net = OntologyNetworkBase(OntoParser(g)) + assert net.terms.ex.ind.node_type == "named_individual" + with pytest.raises(ValueError): + net.create_query(net.terms.ex.ind) + + +def test_punned_class_keeps_its_subclasses(): + g = make_graph( + "AB", + [(EX.B, RDFS.subClassOf, EX.A), (EX.A, RDF.type, OWL.NamedIndividual)], + ) + parser = OntoParser(g) + assert parser.attributes["class"]["ex:A"].node_type == "class" + assert parser.attributes["class"]["ex:A"].subclasses == ["ex:B"] + + +def test_terms_can_be_deep_copied(): + import copy + + onto = read_ontology() + terms = copy.deepcopy(onto.terms) + assert terms.cmso.AtomicScaleSample.name == "cmso:AtomicScaleSample" + + +def test_add_term_expands_prefixed_names(): + net = OntologyNetworkBase(OntoParser(make_graph("AB"))) + net.add_term("http://example.org/o#q", "object_property", dm=["ex:A"], rn=["ex:B"]) + net.add_term("http://example.org/o#v", "data_property", dm=["ex:B"], rn=["float"]) + assert net.attributes["object_property"]["ex:q"].domain == ["ex:A"] + assert net.attributes["object_property"]["ex:q"].range == ["ex:B"] + assert net.attributes["data_property"]["ex:v"].range == ["float"] + query = net.create_query(net.terms.ex.A, [net.terms.ex.v > 1]) + assert "?A ex:q ?ex_B ." in query + + +def test_owl_thing_is_not_its_own_subclass(): + parser = OntoParser(make_graph("A")) + subclasses = parser.attributes["class"]["owl:Thing"].subclasses + assert "owl:Thing" not in subclasses + assert "ex:A" in subclasses + + +def test_visualize_graph_with_default_style(): + from tools4rdf.visualize import visualize_graph + + g = Graph() + g.add((EX.a, EX.p, Literal("fcc lattice"))) + g.add((EX.a, EX.q, EX.b)) + dot = visualize_graph(g) + assert "fcc lattice" in dot.source + assert "Fcc Lattice" not in dot.source diff --git a/tools4rdf/network/attrsetter.py b/tools4rdf/network/attrsetter.py index 6a28cd4..48ea97c 100644 --- a/tools4rdf/network/attrsetter.py +++ b/tools4rdf/network/attrsetter.py @@ -6,7 +6,6 @@ import yaml import os -import numpy as np doc_fields = ["doc", "url"] @@ -63,10 +62,13 @@ def __getattr__(self, key): key which add ons: for example `position_selected` or `position_not_selected` key does not exist: raise `AttributeError` """ - if key in self._map_dict.keys(): + # read _map_dict from __dict__: during copy/unpickling it is not set yet, + # and self._map_dict would recurse into __getattr__ + map_dict = self.__dict__.get("_map_dict", {}) + if key in map_dict.keys(): if self.head is None: self.head = self - return self._map_dict[key] + return map_dict[key] else: raise AttributeError(f"Attribute {key} not found") diff --git a/tools4rdf/network/network.py b/tools4rdf/network/network.py index adb9829..afeef6f 100644 --- a/tools4rdf/network/network.py +++ b/tools4rdf/network/network.py @@ -12,14 +12,22 @@ import networkx as nx import graphviz import pandas as pd +import copy import itertools +import re -from rdflib import URIRef, Literal, RDF, OWL, Graph +from rdflib import URIRef, Literal, RDF, RDFS, OWL, XSD, Graph +from rdflib.plugins.stores.sparqlstore import SPARQLStore from tools4rdf.network.attrsetter import AttrSetter from tools4rdf.network.parser import parse_ontology, OntoParser from tools4rdf.network.term import OntoTerm, is_url +def _to_variable(name): + """Turn a term name such as 'ex:Twin-like' into a valid SPARQL variable name.""" + return re.sub(r"\W", "_", name) + + def _replace_name(name): return ".".join(name.split(":")) @@ -284,7 +292,9 @@ def _insert_namespaces(self, namespaces): to resolve the URIs for the provided namespace prefixes. """ query = [] - ns = self.namespaces | self.extra_namespaces + ns = {**self.namespaces, **self.extra_namespaces} + # filters always use xsd datatypes, even if the ontology does not bind it + ns.setdefault("xsd", str(XSD)) for key in namespaces: query.append(f"PREFIX {key}: <{ns[key]}>") return query @@ -292,25 +302,29 @@ def _insert_namespaces(self, namespaces): @staticmethod def _modify_destinations(destinations): """look for lists within destinations to phase out the @ operator""" + # work on copies; the terms passed in are shared with `terms` modified_destinations = [] for count, destination in enumerate(destinations): if isinstance(destination, (list, tuple)): - last_destination = destination[-1] + last_destination = copy.deepcopy(destination[-1]) for d in destination[:-1]: last_destination._parents.append(d) modified_destinations.append(last_destination) else: - modified_destinations.append(destination) + modified_destinations.append(copy.deepcopy(destination)) # now update conditions for modified_destination in modified_destinations: modified_destination._update_condition_string() return modified_destinations def _is_already_in_destinations(self, object_property, destinations): - if object_property in destinations: - return True + # compare by name; OntoTerm.__eq__ builds conditions for data properties for d in destinations: - if object_property in d._parents: + if isinstance(d, (list, tuple)): + chain = list(d) + else: + chain = [*d._parents, d] + if any(x.name == object_property.name for x in chain): return True return False @@ -389,6 +403,11 @@ def create_query( for s in source: if s.node_type == "data_property": raise ValueError("Data properties are not allowed as source nodes.") + if s.node_type not in ("class", "object_property"): + raise ValueError( + f"{s.name} is a {s.node_type}; only classes and object " + "properties can be used as source nodes." + ) # separate sources into classes and object properties classes = [s for s in source if s.node_type == "class"] @@ -398,12 +417,18 @@ def create_query( # common classes in the domains # Do only if we have object properties if len(object_properties) > 0: + # keep the domain order so the chosen class is the same on every run domains = [a.domain for a in object_properties] - common_classes = set(domains[0]) - for d in domains[1:]: - common_classes = common_classes.intersection(set(d)) + common_names = [] + for x in domains[0]: + if ( + x not in common_names + and x in self.onto.attributes["class"] + and all(x in d for d in domains[1:]) + ): + common_names.append(x) # now if we do not have any common classes, raise an error - common_classes = [self.onto.attributes["class"][x] for x in common_classes] + common_classes = [self.onto.attributes["class"][x] for x in common_names] if len(common_classes) == 0: raise ValueError( "No common classes found in the domains of the object properties." @@ -427,13 +452,13 @@ def create_query( else: for object_property in object_properties: if not self._is_already_in_destinations( - object_property, *destinations + object_property, destinations ): destinations.append(object_property) destinations = self._modify_destinations(destinations) elif len(object_properties) > 0: - destinations = object_properties + destinations = [copy.deepcopy(op) for op in object_properties] if destinations is None: destinations = [] @@ -510,6 +535,21 @@ def _prepare_destinations(self, destinations=None, source=None): destinations.append(parent) return destinations + @staticmethod + def _disambiguate_variables(destinations): + """ + Qualify variable names with the namespace prefix where different terms + (e.g. ex:value and y:value) would otherwise share a SPARQL variable. + """ + by_variable = {} + for destination in destinations: + by_variable.setdefault(destination.variable_name, []).append(destination) + for terms in by_variable.values(): + if len({term.name for term in terms}) > 1: + for term in terms: + term._qualified = True + term._update_condition_string() + def _create_query_prefix(self, source, destinations, remote_source=None): """ Constructs the prefix of a SPARQL query with a SELECT DISTINCT clause. @@ -603,9 +643,9 @@ def _get_triples(self, source, destinations, num_paths=1): for triple in triples: namespace.extend([x.split(":")[0] for x in triple if ":" in x]) line_text = " ?%s %s ?%s ." % ( - triple[0].replace(":", "_"), + _to_variable(triple[0]), triple[1], - triple[2].replace(":", "_"), + _to_variable(triple[2]), ) if line_text not in query: query.append(line_text) @@ -766,8 +806,7 @@ def _add_filters(self, destinations, remote_source=None): This method processes a list of destination objects, extracts their filter conditions, and constructs a SPARQL query fragment with the appropriate FILTER clause. It also replaces query names with variable names in the - filter conditions and ensures that the destination objects are refreshed - after processing. + filter conditions. Parameters ---------- @@ -799,10 +838,6 @@ def _add_filters(self, destinations, remote_source=None): if remote_source is not None: query.append(" }") query.append("}") - - # finished, clean up the terms; - for destination in destinations: - destination.refresh() return query def _add_limit(self, limit=None): @@ -867,6 +902,7 @@ def _create_query( destinations = self._prepare_destinations( destinations=destinations, source=source ) + self._disambiguate_variables(destinations) query_header = self._create_query_prefix( source, destinations, remote_source=remote_source ) @@ -889,16 +925,20 @@ def _create_query( ) query_filter = self._add_filters(destinations, remote_source=remote_source) + if any("xsd:" in line for line in query_filter): + namespaces_used.append("xsd") created_queries = [] for query in queries: query_header_new = ( self._insert_namespaces( - set( - namespaces_used - + namespaces - + namespaces_source - + namespaces_dest + sorted( + set( + namespaces_used + + namespaces + + namespaces_source + + namespaces_dest + ) ) ) + query_header @@ -939,16 +979,18 @@ def query( Returns ------- pandas.DataFrame or list - If `return_df` is True, returns a pandas DataFrame containing the query results. + If `return_df` is True, returns a pandas DataFrame containing the query results + (empty if nothing matches). Values are rdflib terms (URIRef, Literal). If `return_df` is False, returns a list of query results. - Returns None if no results are found. + Returns None only if no query could be generated. """ - remote_source = None if isinstance(kg, str): - if is_url(kg): - remote_source = kg - kg = Graph() + if not is_url(kg): + raise ValueError(f"{kg} is not a valid SPARQL endpoint url") + # send the query to the endpoint itself, so that FILTER and LIMIT + # are evaluated remotely instead of through a local SERVICE call + kg = SPARQLStore(kg, method="POST_FORM", returnFormat="json") query_strings = self.create_query( source, @@ -956,7 +998,6 @@ def query( return_list=True, num_paths=num_paths, limit=limit, - remote_source=remote_source, ) res = [] for query_string in query_strings: @@ -1088,46 +1129,68 @@ def add_path(self, triple): """ Add a triple as path. - Note that all attributes of the triple should already exist in the graph. - The ontology itself is not modified. Only the graph representation of it is. - The expected use is to bridge between two (or more) different ontologies. + The path is recorded as an extra rdfs:domain (and rdfs:range) of the + property in the in-memory ontology graph, so it survives rebuilding the + network and combining ontologies, and it also applies to subclasses. + The ontology file itself is not modified. The expected use is to bridge + between two (or more) different ontologies. Parameters ---------- triple : tuple - A tuple representing the triple to be added. The tuple should contain three elements: - subject, predicate, and object. + A tuple (subject, predicate, object) of prefixed names or IRIs. + The subject must be a class and the predicate an object or data + property. For an object property the object must be a class; for a + data property it is a datatype name such as "string" or "xsd:float", + used only if the property has no range yet. Raises ------ ValueError - If the subject or object of the triple is not found in the attributes of the ontology. + If the subject, predicate or object is not found in the ontology. """ + graph = self.onto.graph - def to_uri(tag, namespaces): + def to_uri(tag): + if is_url(tag): + return URIRef(tag) if ":" in tag: - prefix, term = tag.split(":") - return URIRef(namespaces[prefix] + term) - else: - return Literal(tag) + prefix, term = tag.split(":", 1) + if prefix not in self.namespaces: + raise ValueError(f"prefix {prefix} of {tag} is not known") + return URIRef(self.namespaces[prefix] + term) + return None - sub, pred, obj = [to_uri(t, self.namespaces) for t in triple] + sub_tag, pred_tag, obj_tag = triple + sub = to_uri(sub_tag) + pred = to_uri(pred_tag) - if (sub, RDF.type, OWL.Class) not in self.onto.graph: + if sub is None or (sub, RDF.type, OWL.Class) not in graph: raise ValueError( - f"{sub} not found in {list(self.onto.graph.subjects(RDF.type, OWL.Class))}" + f"{sub_tag} not found in {list(graph.subjects(RDF.type, OWL.Class))}" ) - if ( - isinstance(obj, URIRef) - and (obj, RDF.type, OWL.Class) not in self.onto.graph - ): - raise ValueError( - f"{obj} not found in {list(self.onto.graph.subjects(RDF.type, OWL.Class))}" - ) + if pred is not None and (pred, RDF.type, OWL.ObjectProperty) in graph: + obj = to_uri(obj_tag) + if obj is None or (obj, RDF.type, OWL.Class) not in graph: + raise ValueError( + f"{obj_tag} not found in {list(graph.subjects(RDF.type, OWL.Class))}" + ) + graph.add((pred, RDFS.domain, sub)) + graph.add((pred, RDFS.range, obj)) + elif pred is not None and (pred, RDF.type, OWL.DatatypeProperty) in graph: + graph.add((pred, RDFS.domain, sub)) + if (pred, RDFS.range, None) not in graph: + datatype = to_uri(obj_tag) + if datatype is None: + name = "string" if obj_tag == "str" else obj_tag + datatype = XSD[name] + graph.add((pred, RDFS.range, datatype)) + else: + raise ValueError(f"{pred_tag} is not an object or data property") - self.onto.graph.add((sub, pred, obj)) + self.onto._data_dict = None self._terms = None self._g = None diff --git a/tools4rdf/network/ontology.py b/tools4rdf/network/ontology.py index 2d221d1..4c89dd3 100644 --- a/tools4rdf/network/ontology.py +++ b/tools4rdf/network/ontology.py @@ -1,19 +1,18 @@ -import os -from tools4rdf.network.network import OntologyNetwork - - -def read_ontology(): - """ - Read in ontologies and perform necessary operations. - - Returns - ------- - combo: OntologyNetwork, Combined ontology network. - """ - # read in ontologies - file_location = os.path.dirname(__file__).split("/") - file_location = "/".join(file_location[:-1]) - - cmso = OntologyNetwork(os.path.join(file_location, "data/cmso.owl")) - # return - return cmso +import os +from tools4rdf.network.network import OntologyNetwork + + +def read_ontology(): + """ + Read in ontologies and perform necessary operations. + + Returns + ------- + combo: OntologyNetwork, Combined ontology network. + """ + # read in ontologies + file_location = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + cmso = OntologyNetwork(os.path.join(file_location, "data/cmso.owl")) + # return + return cmso diff --git a/tools4rdf/network/parser.py b/tools4rdf/network/parser.py index 2f649bb..3441006 100644 --- a/tools4rdf/network/parser.py +++ b/tools4rdf/network/parser.py @@ -27,9 +27,9 @@ import networkx as nx import warnings -from tools4rdf.network.term import OntoTerm, strip_name +from tools4rdf.network.term import OntoTerm, strip_name, is_url, _XSD_DATATYPES from tools4rdf.network.patch import patch_terms -from rdflib import Graph, RDF, RDFS, OWL, BNode, URIRef +from rdflib import Graph, RDF, RDFS, OWL, XSD, BNode, URIRef def parse_ontology(infile, format="xml"): @@ -393,6 +393,30 @@ def get_description(self, cls): comment = "" return comment + def get_label(self, cls): + """ + Retrieve the rdfs:label of a given term from the RDF graph. + + Untagged or English labels are preferred over other languages. + + Parameters + ---------- + cls : rdflib.term.URIRef + The RDF term for which the label is to be retrieved. + + Returns + ------- + str or None + The label of the term, or None if it has no label. + """ + labels = list(self.graph.objects(cls, RDFS.label)) + if len(labels) == 0: + return None + for label in labels: + if getattr(label, "language", None) in (None, "en"): + return str(label) + return str(labels[0]) + def lookup_node(self, term): """ Maps a given term to its corresponding domain, range, and related hierarchies. @@ -599,6 +623,7 @@ def create_term(self, cls): uri=iri, namespace=self._lookup_namespace(iri), description=self.get_description(cls), + label=self.get_label(cls), target=cls, ) return term @@ -608,8 +633,8 @@ def _lookup_namespace(self, uri): Look up the namespace prefix for a given URI. This method iterates through the stored namespaces and checks if the - provided URI starts with any of the namespace values. If a match is found, - the corresponding namespace prefix is returned. + provided URI starts with any of the namespace values. If several match, + the prefix of the longest (most specific) namespace is returned. Parameters ---------- @@ -621,12 +646,15 @@ def _lookup_namespace(self, uri): str or None The namespace prefix if a match is found, otherwise None. """ + match = None + match_length = -1 for key, value in self.namespaces.items(): - if uri.startswith(value): - return key - return None + if uri.startswith(value) and len(value) > match_length: + match = key + match_length = len(value) + return match - def unravel_relation(self, term, unravel_list=[]): + def unravel_relation(self, term, unravel_list=None): """ Recursively unravels an RDF collection into a Python list. @@ -649,6 +677,8 @@ def unravel_relation(self, term, unravel_list=[]): A Python list containing the elements of the RDF collection in the order they appear. """ + if unravel_list is None: + unravel_list = [] if term == RDF.nil: return unravel_list first_term = self.graph.value(term, RDF.first) @@ -670,7 +700,12 @@ def parse_subclasses(self): for obj in self.graph.objects(cls.target, RDFS.subClassOf): superclasses = self.lookup_class(obj) for superclass in superclasses: - self.attributes["class"][superclass].subclasses.append(cls.name) + # superclasses can be nested class expressions or classes + # from ontologies that are not loaded; skip those + if superclass in self.attributes["class"]: + self.attributes["class"][superclass].subclasses.append( + cls.name + ) def recursively_add_subclasses(self, item_type="class"): """ @@ -726,6 +761,8 @@ def add_subclasses_to_owlThing(self): of "owl:Thing". """ for key, cls in self.attributes["class"].items(): + if key == "owl:Thing": + continue objects = list(self.graph.objects(cls.target, RDFS.subClassOf)) if len(objects) == 0: self.attributes["class"]["owl:Thing"].subclasses.append(cls.name) @@ -804,16 +841,21 @@ def parse_named_individuals(self): for cls in named_individuals: # find parent term = self.create_term(cls) - self.attributes["class"][term.name] = term + # a punned term that is also a class keeps its class entry + if term.name not in self.attributes["class"]: + term.node_type = "named_individual" + self.attributes["class"][term.name] = term parents = list(self.graph.objects(cls, RDF.type)) for parent in parents: if parent not in [OWL.NamedIndividual, OWL.Class]: - self.attributes["class"][ - strip_name( - parent.toPython(), - namespace=self._lookup_namespace(parent.toPython()), + parent_name = strip_name( + parent.toPython(), + namespace=self._lookup_namespace(parent.toPython()), + ) + if parent_name in self.attributes["class"]: + self.attributes["class"][parent_name].named_individuals.append( + term.name ) - ].named_individuals.append(term.name) def get_attributes(self): """ @@ -905,17 +947,18 @@ def add_term( node_type : str The type of the node. namespace : str, optional - The namespace of the node. + Not used; kept for backwards compatibility. dm : list, optional - The domain metadata of the node. + The domain of the node, as IRIs or prefixed names (e.g. "cmso:Material"). rn : list, optional - The range metadata of the node. + The range of the node, as IRIs or prefixed names. For data properties, + XSD datatype names such as "float" or "str" are also accepted. data_type : str, optional - The data type of the node. + Not used; kept for backwards compatibility. node_id : str, optional - The ID of the node. + Not used; kept for backwards compatibility. delimiter : str, optional - The delimiter used for parsing the URI. + Not used; kept for backwards compatibility. Raises ------ @@ -932,11 +975,28 @@ def add_term( else: raise ValueError("Node type not found") for r in rn: - self.graph.add((URIRef(uri), RDFS.range, URIRef(r))) + self.graph.add((URIRef(uri), RDFS.range, self._expand(r))) for d in dm: - self.graph.add((URIRef(uri), RDFS.domain, URIRef(d))) + self.graph.add((URIRef(uri), RDFS.domain, self._expand(d))) self._data_dict = None + def _expand(self, tag): + """ + Expand a prefixed name (e.g. "cmso:Material") or an XSD datatype name + (e.g. "float", "str") into a URIRef. IRIs are returned unchanged. + """ + if is_url(tag): + return URIRef(tag) + if ":" in tag: + prefix, name = tag.split(":", 1) + if prefix in self.namespaces: + return URIRef(self.namespaces[prefix] + name) + else: + name = "string" if tag == "str" else tag + if name in _XSD_DATATYPES: + return XSD[name] + return URIRef(tag) + def add_namespace(self, namespace_name, namespace_iri): """ Add a new namespace. @@ -948,10 +1008,9 @@ def add_namespace(self, namespace_name, namespace_iri): namespace_iri : str The IRI of the namespace. - Raises - ------ - KeyError - If the namespace already exists. + Notes + ----- + If a namespace with this name already exists, it is left unchanged. """ if namespace_name not in self.namespaces.keys(): diff --git a/tools4rdf/network/term.py b/tools4rdf/network/term.py index 9eab032..bd2df21 100644 --- a/tools4rdf/network/term.py +++ b/tools4rdf/network/term.py @@ -18,13 +18,17 @@ making it easier to work with semantic data and SPARQL queries. """ -from rdflib import URIRef +from rdflib import URIRef, Literal, XSD import numbers +import re import copy import warnings from urllib.parse import urlparse +_XSD_DATATYPES = set(XSD.__annotations__) + + def is_url(string): parsed = urlparse(string) return bool(parsed.netloc) and bool(parsed.scheme) @@ -165,8 +169,8 @@ def __init__( self.delimiter = delimiter self._description = None self.description = description - self.label = label self._label = None + self.label = label self.is_domain_of = [] self.is_range_of = [] self._condition = None @@ -186,6 +190,7 @@ def __init__( self._enforce_type = True self._add_subclass = True self._old_variable_name = None + self._qualified = False @property def URIRef(self): @@ -351,7 +356,12 @@ def variable_name(self): """ name_list = [x.name_without_prefix for x in self._parents] - name_list.append(self.name_without_prefix) + if self._qualified: + # used when two terms of different namespaces share a local name + prefix = re.sub(r"\W", "_", self.name.split(":")[0]) + name_list.append(f"{prefix}_{self.name_without_prefix}") + else: + name_list.append(self.name_without_prefix) name = "_".join(name_list) if self.node_type == "data_property": @@ -441,30 +451,46 @@ def _is_data_node(self): def _update_condition_string( self, ): - condition_string = self._condition - if self._old_variable_name is not None: - condition_string = condition_string.replace( - self._old_variable_name, self.variable_name + if self._condition is not None and self._old_variable_name is not None: + self._condition = self._condition.replace( + f"?{self._old_variable_name}", f"?{self.variable_name}" ) - self._condition = condition_string + self._old_variable_name = self.variable_name + + def _format_literal(self, val): + """ + Serialise a value as a SPARQL literal matching the range of the term. + + Values are escaped and use the canonical lexical form of the datatype. + If the range is missing or not an XSD datatype (e.g. rdfs:Literal), + the literal is left untyped (strings) or typed from the Python value. + """ + datatype = self._clean_datatype(self.range[0]) if len(self.range) > 0 else None + if datatype in _XSD_DATATYPES: + lexical = str(Literal(val, datatype=XSD[datatype])) + return f"{Literal(lexical).n3()}^^xsd:{datatype}" + return Literal(val).n3() def _create_condition_string(self, condition, val): self._old_variable_name = self.variable_name - return f'(?{self.variable_name}{condition}"{val}"^^xsd:{self._clean_datatype(self.range[0])})' + return f"(?{self.variable_name}{condition}{self._format_literal(val)})" # overloading operators def __eq__(self, val): """ = """ - # print("eq") - # print(f'lhs {self} rhs {val}') + # comparing two terms is an identity check, never a condition + if isinstance(val, OntoTerm): + return self.name == val.name if self.node_type == "data_property": item = copy.deepcopy(self) item._condition = item._create_condition_string("=", val) return item - else: - return self.name == val.name + return NotImplemented + + def __hash__(self): + return hash(self.name) def __lt__(self, val): self._is_number(val) @@ -481,8 +507,10 @@ def __le__(self, val): return item def __ne__(self, val): - # self._is_number(val) - self._is_data_node() + if isinstance(val, OntoTerm): + return self.name != val.name + if self.node_type != "data_property": + return NotImplemented item = copy.deepcopy(self) item._condition = item._create_condition_string("!=", val) return item @@ -520,7 +548,7 @@ def __and__(self, term): return item def and_(self, term): - self.__and__(term) + return self.__and__(term) def __or__(self, term): self._is_term(term) @@ -539,7 +567,7 @@ def __or__(self, term): return item def or_(self, term): - self.__or__(term) + return self.__or__(term) def __matmul__(self, term): # we will phase out this operator soon diff --git a/tools4rdf/visualize.py b/tools4rdf/visualize.py index 03a8ce1..a4e29d7 100644 --- a/tools4rdf/visualize.py +++ b/tools4rdf/visualize.py @@ -87,13 +87,32 @@ def parse_object(x): elif isinstance(x, URIRef): return get_string_from_URI(x) elif isinstance(x, Literal): - return str(x.title()), "Literal" + return str(x), "Literal" styledict = { - "BNode": {"color": "#ffe6ff", "shape": "box", "style": "filled"}, - "URIRef": {"color": "#ffffcc", "shape": "box", "style": "filled"}, - "Literal": {"color": "#e6ffcc", "shape": "ellipse", "style": "filled"}, + "BNode": { + "color": "#ffe6ff", + "shape": "box", + "style": "filled", + "fontsize": "8", + "fontname": "Helvetica", + }, + "URIRef": { + "color": "#ffffcc", + "shape": "box", + "style": "filled", + "fontsize": "8", + "fontname": "Helvetica", + }, + "Literal": { + "color": "#e6ffcc", + "shape": "ellipse", + "style": "filled", + "fontsize": "8", + "fontname": "Helvetica", + }, + "edgecolor": "#263238", }