diff --git a/examples/02_advanced_query_building.ipynb b/examples/02_advanced_query_building.ipynb index 53bb8aa..cf61d55 100644 --- a/examples/02_advanced_query_building.ipynb +++ b/examples/02_advanced_query_building.ipynb @@ -860,6 +860,352 @@ "These selectors become particularly useful when combined with comparison operators and filters." ] }, + { + "cell_type": "markdown", + "id": "8b8991f1", + "metadata": {}, + "source": [ + "## Several Independent Conditions\n", + "\n", + "Instead of combining conditions with `&`, you can pass them as separate destinations. Each condition becomes its own `FILTER`, and all of them must hold. This also makes range queries on a single property easy to write:" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "038a63b0", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
PizzahasPricevaluehasCalorificContentValuevalue
0American:245aa2f110.01292.0
\n", + "
" + ], + "text/plain": [ + " Pizza hasPricevalue hasCalorificContentValuevalue\n", + "0 American:245aa2f1 10.0 1292.0" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "q = onto.query(g, onto.terms.pizza.Pizza,\n", + " [onto.terms.pizza.hasPrice >= 10,\n", + " onto.terms.pizza.hasPrice <= 11,\n", + " onto.terms.pizza.hasCalorificContentValue > 1000])\n", + "q" + ] + }, + { + "cell_type": "markdown", + "id": "09f99440", + "metadata": {}, + "source": [ + "## Optional Properties\n", + "\n", + "By default, every destination is required: an item missing any of the requested properties does not appear in the results. Use `.optional` to keep such items; the missing value is then empty. Only some pizzas in the knowledge graph have a spiciness:" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "95c12455", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
PizzahasPricevaluehasSpiciness
0American:245aa2f110.0None
1AmericanHot:81efd53711.5Hot:3ba60921
2Pizza:bec7e5f211.5None
3Margherita:3cfeae458.99None
4Margherita:e424656a9.5None
5Mushroom:b6b736c911.0None
6Pizza:71307a5a11.0Hot:3ba60921
7Pizza:f9f81adc12.99Mild:0bf38aee
\n", + "
" + ], + "text/plain": [ + " Pizza hasPricevalue hasSpiciness\n", + "0 American:245aa2f1 10.0 None\n", + "1 AmericanHot:81efd537 11.5 Hot:3ba60921\n", + "2 Pizza:bec7e5f2 11.5 None\n", + "3 Margherita:3cfeae45 8.99 None\n", + "4 Margherita:e424656a 9.5 None\n", + "5 Mushroom:b6b736c9 11.0 None\n", + "6 Pizza:71307a5a 11.0 Hot:3ba60921\n", + "7 Pizza:f9f81adc 12.99 Mild:0bf38aee" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "q = onto.query(g, onto.terms.pizza.Pizza,\n", + " [onto.terms.pizza.hasPrice,\n", + " onto.terms.pizza.hasSpiciness.optional])\n", + "q" + ] + }, + { + "cell_type": "markdown", + "id": "b52ed58f", + "metadata": {}, + "source": [ + "A condition on an optional property only applies to that property: items that do not satisfy it are kept, with an empty value. In the generated query, the path, type and condition of the optional property are placed in their own `OPTIONAL` block:" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "3a3b4a1f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " OPTIONAL {\n", + " ?Pizza pizza:hasCalorificContentValue ?hasCalorificContentValuevalue .\n", + " FILTER (?hasCalorificContentValuevalue>\"1300\"^^xsd:float)\n", + " }\n", + "}\n" + ] + } + ], + "source": [ + "query = onto.create_query(onto.terms.pizza.Pizza,\n", + " [onto.terms.pizza.hasPrice,\n", + " (onto.terms.pizza.hasCalorificContentValue > 1300).optional])\n", + "print(query[query.index(\" OPTIONAL\"):])" + ] + }, + { + "cell_type": "markdown", + "id": "0374b411", + "metadata": {}, + "source": [ + "## Python Values\n", + "\n", + "Query results contain `rdflib` terms by default. With `python_values=True`, literals are converted to Python values and IRIs to strings, so numeric columns can be used directly for sorting, statistics or plotting:" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "240ec56f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
PizzahasPricevaluehasCalorificContentValuevalue
2Margherita:3cfeae458.99848.0
3Margherita:e424656a9.50898.0
0American:245aa2f110.001292.0
4Mushroom:b6b736c911.00820.0
5Pizza:71307a5a11.00838.0
\n", + "
" + ], + "text/plain": [ + " Pizza hasPricevalue hasCalorificContentValuevalue\n", + "2 Margherita:3cfeae45 8.99 848.0\n", + "3 Margherita:e424656a 9.50 898.0\n", + "0 American:245aa2f1 10.00 1292.0\n", + "4 Mushroom:b6b736c9 11.00 820.0\n", + "5 Pizza:71307a5a 11.00 838.0" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "q = onto.query(g, onto.terms.pizza.Pizza,\n", + " [onto.terms.pizza.hasPrice,\n", + " onto.terms.pizza.hasCalorificContentValue],\n", + " python_values=True)\n", + "q.sort_values(\"hasPricevalue\").head()" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ad4f7ce6", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Pizza object\n", + "hasPricevalue float64\n", + "hasCalorificContentValuevalue float64\n", + "dtype: object" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "q.dtypes" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -873,6 +1219,9 @@ "3. **Multiple Properties**: Query multiple properties simultaneously \n", "4. **Nested Conditions**: Build sophisticated queries by nesting logical operators for complex filtering logic\n", "5. **Class Selectors**: Control query behavior with `.all_subtypes`, `.any`, and `.only` to precisely target classes and hierarchies\n", + "6. **Independent Conditions**: Pass several conditions as separate destinations; all of them must hold\n", + "7. **Optional Properties**: Use `.optional` to keep items that lack a property\n", + "8. **Python Values**: Use `python_values=True` to get plain Python values and numeric columns\n", "\n", "These features allow you to construct complex SPARQL queries using intuitive Python syntax, making knowledge graph querying more accessible without requiring deep SPARQL expertise." ] diff --git a/tests/test_features.py b/tests/test_features.py new file mode 100644 index 0000000..a427d92 --- /dev/null +++ b/tests/test_features.py @@ -0,0 +1,168 @@ +""" +Tests for query features: several filters, optional destinations and +Python values in results. +""" + +import pytest +from rdflib import Graph, Namespace, RDF, RDFS, OWL, XSD, Literal + +from tools4rdf.network.parser import OntoParser +from tools4rdf.network.network import OntologyNetworkBase + +EX = Namespace("http://example.org/o#") + + +@pytest.fixture +def net(): + """ + Sample --hasCell--> Cell ; Sample has natoms (int), label (string); + Cell has volume (float). + """ + g = Graph() + g.bind("ex", EX) + for c in ["Sample", "Cell"]: + g.add((EX[c], RDF.type, OWL.Class)) + g.add((EX.hasCell, RDF.type, OWL.ObjectProperty)) + g.add((EX.hasCell, RDFS.domain, EX.Sample)) + g.add((EX.hasCell, RDFS.range, EX.Cell)) + for prop, domain, rng in [ + ("natoms", "Sample", XSD.integer), + ("name", "Sample", XSD.string), + ("volume", "Cell", XSD.float), + ]: + g.add((EX[prop], RDF.type, OWL.DatatypeProperty)) + g.add((EX[prop], RDFS.domain, EX[domain])) + g.add((EX[prop], RDFS.range, rng)) + return OntologyNetworkBase(OntoParser(g)) + + +@pytest.fixture +def kg(): + """s1: 4 atoms, volume 10, name 'fcc'; s2: 100 atoms, volume 50, no name; + s3: 200 atoms, no cell, name 'bcc'.""" + g = Graph() + for s, natoms in [("s1", 4), ("s2", 100), ("s3", 200)]: + g.add((EX[s], RDF.type, EX.Sample)) + g.add((EX[s], EX.natoms, Literal(natoms))) + for s, cell, volume in [("s1", "c1", 10.0), ("s2", "c2", 50.0)]: + g.add((EX[s], EX.hasCell, EX[cell])) + g.add((EX[cell], RDF.type, EX.Cell)) + g.add((EX[cell], EX.volume, Literal(volume, datatype=XSD.float))) + g.add((EX.s1, EX.name, Literal("fcc"))) + g.add((EX.s3, EX.name, Literal("bcc"))) + return g + + +def samples(df): + return sorted(str(x).split("#")[-1] for x in df["Sample"]) + + +def test_two_filters_on_different_properties(net, kg): + t = net.terms.ex + df = net.query(kg, t.Sample, [t.natoms > 50, t.volume < 60]) + assert samples(df) == ["s2"] + + +def test_range_filter_on_one_property(net, kg): + t = net.terms.ex + query = net.create_query(t.Sample, [t.natoms > 50, t.natoms < 150]) + select = [line for line in query.splitlines() if line.startswith("SELECT")][0] + assert select.count("?natomsvalue") == 1 + assert samples(net.query(kg, t.Sample, [t.natoms > 50, t.natoms < 150])) == ["s2"] + + +def test_or_condition_is_not_narrowed_by_its_operands(net, kg): + t = net.terms.ex + df = net.query(kg, t.Sample, [(t.natoms < 10) | (t.volume > 40)]) + assert samples(df) == ["s1", "s2"] + + +def test_and_condition_combined_with_a_separate_filter(net, kg): + t = net.terms.ex + df = net.query(kg, t.Sample, [(t.natoms > 1) & (t.volume > 1), t.natoms < 50]) + assert samples(df) == ["s1"] + + +def test_optional_keeps_rows_without_the_value(net, kg): + t = net.terms.ex + df = net.query(kg, t.Sample, [t.natoms, t.name.optional]) + assert samples(df) == ["s1", "s2", "s3"] + names = {str(s).split("#")[-1]: n for s, n in zip(df["Sample"], df["namevalue"])} + assert str(names["s1"]) == "fcc" + assert names["s2"] is None + + +def test_required_destination_still_filters_rows(net, kg): + t = net.terms.ex + # s3 has no cell, so it drops out; s2 has no name but is kept + df = net.query(kg, t.Sample, [t.volume, t.name.optional]) + assert samples(df) == ["s1", "s2"] + + +def test_optional_path_through_another_class(net, kg): + t = net.terms.ex + query = net.create_query(t.Sample, [t.natoms, t.volume.optional]) + optional_block = query.split("OPTIONAL {")[1].split("}")[0] + assert "ex:hasCell" in optional_block + assert "ex:volume" in optional_block + df = net.query(kg, t.Sample, [t.natoms, t.volume.optional]) + assert samples(df) == ["s1", "s2", "s3"] + + +def test_condition_on_optional_stays_inside_its_block(net, kg): + t = net.terms.ex + query = net.create_query(t.Sample, [t.natoms, (t.volume > 20).optional]) + assert "FILTER" in query.split("OPTIONAL {")[1].split("}")[0] + df = net.query(kg, t.Sample, [t.natoms, (t.volume > 20).optional]) + # every sample is kept; only s2 has a volume above 20 + assert samples(df) == ["s1", "s2", "s3"] + volumes = [v for v in df["volumevalue"] if v is not None] + assert [float(v) for v in volumes] == [50.0] + + +def test_optional_query_parses_without_default_prefixes(net): + from rdflib.plugins.sparql import prepareQuery + + t = net.terms.ex + query = net.create_query(t.Sample, [t.natoms > 1, (t.volume > 20).optional]) + prepareQuery(query, initNs={}) + + +def test_optional_does_not_mutate_shared_term(net): + t = net.terms.ex + t.name.optional + assert net.terms.ex.name._optional is False + + +def test_optional_last_step_of_stepped_query(net, kg): + t = net.terms.ex + df = net.query(kg, t.Sample, [t.natoms, [t.hasCell, t.volume.optional]]) + assert samples(df) == ["s1", "s2", "s3"] + + +def test_python_values_gives_numeric_columns(net, kg): + t = net.terms.ex + df = net.query(kg, t.Sample, [t.natoms, t.volume], python_values=True) + assert df["natomsvalue"].dtype.kind == "i" + assert df["volumevalue"].dtype.kind == "f" + assert sorted(df["natomsvalue"]) == [4, 100] + assert all(isinstance(s, str) for s in df["Sample"]) + assert list(df.index) == list(range(len(df))) + + +def test_python_values_with_missing_optional_values(net, kg): + t = net.terms.ex + df = net.query( + kg, t.Sample, [t.natoms, t.volume.optional, t.name.optional], python_values=True + ) + assert df["volumevalue"].dtype.kind == "f" + assert df["volumevalue"].isna().sum() == 1 + names = dict(zip(df["Sample"], df["namevalue"])) + assert names[str(EX.s1)] == "fcc" + assert names[str(EX.s2)] is None + + +def test_default_keeps_rdflib_terms(net, kg): + t = net.terms.ex + df = net.query(kg, t.Sample, [t.natoms]) + assert all(isinstance(v, Literal) for v in df["natomsvalue"]) diff --git a/tests/test_network.py b/tests/test_network.py index aed3244..5f18d68 100644 --- a/tests/test_network.py +++ b/tests/test_network.py @@ -229,16 +229,14 @@ def test_prepare_destinations_source_any_error(network_fixture): network_fixture._prepare_destinations(None, source) -def test_prepare_destinations_multiple_conditions_error(network_fixture): +def test_prepare_destinations_multiple_conditions(network_fixture): source = network_fixture.terms.ex.Pizza - dest1 = network_fixture.terms.ex.Pizza - dest2 = network_fixture.terms.ex.Pizza + dest1 = network_fixture.terms.ex.Pizza.any + dest2 = network_fixture.terms.ex.Food.any dest1._condition = "condition1" dest2._condition = "condition2" - with pytest.raises(ValueError): - network_fixture._prepare_destinations([dest1, dest2], source) - dest1._condition = None - dest2._condition = None + result = network_fixture._prepare_destinations([dest1, dest2], source) + assert len(result) == 2 def test_create_query_prefix(network_fixture): diff --git a/tests/test_regressions.py b/tests/test_regressions.py index e0fa9ce..1102eee 100644 --- a/tests/test_regressions.py +++ b/tests/test_regressions.py @@ -473,3 +473,12 @@ def test_visualize_graph_with_default_style(): dot = visualize_graph(g) assert "fcc lattice" in dot.source assert "Fcc Lattice" not in dot.source + + +def test_stepped_query_through_object_property(): + onto = read_ontology() + c = onto.terms.cmso + query = onto.create_query(c.AtomicScaleSample, [[c.hasMaterial, c.hasAltName]]) + assert "?AtomicScaleSample cmso:hasMaterial ?hasMaterial ." in query + assert "?hasMaterial cmso:hasAltName ?hasMaterial_hasAltNamevalue ." in query + prepareQuery(query, initNs={}) diff --git a/tools4rdf/network/network.py b/tools4rdf/network/network.py index afeef6f..374d15f 100644 --- a/tools4rdf/network/network.py +++ b/tools4rdf/network/network.py @@ -14,6 +14,7 @@ import pandas as pd import copy import itertools +import numbers import re from rdflib import URIRef, Literal, RDF, RDFS, OWL, XSD, Graph @@ -28,6 +29,17 @@ def _to_variable(name): return re.sub(r"\W", "_", name) +def _to_python(value): + """Convert an rdflib term from a result row into a plain Python value.""" + if value is None: + return None + if isinstance(value, Literal): + python_value = value.toPython() + # ill-typed and language-tagged literals come back as Literal + return str(python_value) if isinstance(python_value, Literal) else python_value + return str(value) + + def _replace_name(name): return ".".join(name.split(":")) @@ -254,7 +266,12 @@ def get_shortest_path(self, source, target, triples=False, num_paths=1): # this will also always be of shape 1, x so we can directly take first element temp_path = temp_path[0] # ok now we need to merge temp_path into path - if len(temp_path) == 2: + if temp_source.node_type == "object_property": + # temp_path starts [property, range class, ...]; the path so far + # already ends with the object of that property, which takes + # the place of the range class + path.extend(temp_path[2:]) + elif len(temp_path) == 2: path[-1] = temp_path[-1] else: path.extend(temp_path[1:]) @@ -511,8 +528,6 @@ def _prepare_destinations(self, destinations=None, source=None): ------ ValueError If `destinations` is None and `source.any` is used without enforcing type. - ValueError - If more than one destination has an associated condition. """ # if destinations is None, we need to check if source.any is used if destinations is None and not source._enforce_type: @@ -520,18 +535,14 @@ def _prepare_destinations(self, destinations=None, source=None): "If no destinations are provided, source.any cannot be used!." ) - # check if more than one of them have an associated condition -> if so throw error - no_of_conditions = 0 - for destination in destinations: - if destination._condition is not None: - no_of_conditions += 1 - if no_of_conditions > 1: - raise ValueError("Only one condition is allowed") - # iterate through the list, if they have condition parents, add them explicitely + # only for their paths: their conditions are already part of the combined + # condition, and filtering on them again would turn || into && for destination in destinations: for parent in destination._condition_parents: if parent.variable_name not in [d.variable_name for d in destinations]: + parent = copy.deepcopy(parent) + parent.refresh_condition() destinations.append(parent) return destinations @@ -584,10 +595,11 @@ def _create_query_prefix(self, source, destinations, remote_source=None): # construct the select distinct command: # add source `variable_name` # iterate over destinations, add their `variable_name` - select_destinations = [ - "?" + destination.variable_name for destination in destinations - ] - select_destinations = ["?" + source.variable_name] + select_destinations + select_destinations = [] + for term in [source, *destinations]: + # the same term may appear twice, e.g. [x > 1, x < 5] + if "?" + term.variable_name not in select_destinations: + select_destinations.append("?" + term.variable_name) query.append(f'SELECT DISTINCT {" ".join(select_destinations)}') query.append("WHERE {") # if remote_source is provided, use it as the source @@ -620,6 +632,32 @@ def _get_triples(self, source, destinations, num_paths=1): namespaces : list of str A list of namespaces extracted from the triples. """ + combinations, namespaces = self._get_triples_per_destination( + source, destinations, num_paths=num_paths + ) + queries = [] + for combination in combinations: + query = [] + for lines in combination: + for line_text in lines: + if line_text not in query: + query.append(line_text) + queries.append(query) + return queries, namespaces + + def _get_triples_per_destination(self, source, destinations, num_paths=1): + """ + Like `_get_triples`, but keeps the triple patterns of each destination + separate, so that they can be placed in OPTIONAL blocks. + + Returns + ------- + combinations : list of list of list of str + For each combination of paths, one list of triple patterns per + destination, in the order of `destinations`. + namespaces : list of str + A list of namespaces extracted from the triples. + """ # for each source and destinations, we need to find num_paths paths # then these have to be combined; and each set to be made into individual queries complete_triples = [] @@ -640,6 +678,7 @@ def _get_triples(self, source, destinations, num_paths=1): query = [] namespace = [] for triples in combination: + lines = [] for triple in triples: namespace.extend([x.split(":")[0] for x in triple if ":" in x]) line_text = " ?%s %s ?%s ." % ( @@ -647,8 +686,9 @@ def _get_triples(self, source, destinations, num_paths=1): triple[1], _to_variable(triple[2]), ) - if line_text not in query: - query.append(line_text) + if line_text not in lines: + lines.append(line_text) + query.append(lines) queries.append(query) namespaces.extend(namespace) @@ -804,9 +844,9 @@ def _add_filters(self, destinations, remote_source=None): Adds filter conditions to a SPARQL query based on the provided destinations. This method processes a list of destination objects, extracts their filter - conditions, and constructs a SPARQL query fragment with the appropriate - FILTER clause. It also replaces query names with variable names in the - filter conditions. + conditions, and constructs a SPARQL query fragment with one FILTER clause + per condition; SPARQL combines them with a logical AND. It also replaces + query names with variable names in the filter conditions. Parameters ---------- @@ -821,25 +861,32 @@ def _add_filters(self, destinations, remote_source=None): A list of strings representing the SPARQL query fragment, including the FILTER clause and a closing brace. """ - filter_text = "" - query = [] - for destination in destinations: - if destination._condition is not None: - filter_text = destination._condition - break - - # replace the query_name with variable_name - if filter_text != "": - for destination in destinations: - filter_text = filter_text.replace( - destination.query_name, destination.variable_name - ) - query.append(f"FILTER {filter_text}") + query = self._filter_lines(destinations) if remote_source is not None: query.append(" }") query.append("}") return query + @staticmethod + def _filter_lines(destinations, all_destinations=None): + """ + One FILTER line per condition of `destinations`. Query names are replaced + by the variable names of `all_destinations` (default: `destinations`). + """ + if all_destinations is None: + all_destinations = destinations + query = [] + for destination in destinations: + if destination._condition is None: + continue + # replace the query_name with variable_name + filter_text = destination._condition + for other in all_destinations: + filter_text = filter_text.replace(other.query_name, other.variable_name) + if f"FILTER {filter_text}" not in query: + query.append(f"FILTER {filter_text}") + return query + def _add_limit(self, limit=None): """ Adds a LIMIT clause to a SPARQL query if a limit is specified. @@ -912,50 +959,85 @@ def _create_query( namespaces_used.append(source.name.split(":")[0]) namespaces_used.append("rdf") - # get a list of queries - queries, namespaces = self._get_triples( + # get the triple patterns of each destination, for every path combination + combinations, namespaces = self._get_triples_per_destination( source, destinations, num_paths=num_paths ) + required = [d for d in destinations if not d._optional] query_footer_source_types, namespaces_source = self._add_types_for_source( source ) query_footer_dest_types, namespaces_dest = self._add_types_for_destination( - destinations + required ) + query_filter = self._filter_lines(required, destinations) - query_filter = self._add_filters(destinations, remote_source=remote_source) - if any("xsd:" in line for line in query_filter): + # optional destinations keep their types and conditions inside their block + optional_parts = {} + for destination in destinations: + if destination._optional: + types, namespaces_optional = self._add_types_for_destination( + [destination] + ) + namespaces_dest = namespaces_dest + namespaces_optional + optional_parts[id(destination)] = types + self._filter_lines( + [destination], destinations + ) + + all_filters = query_filter + [ + line for part in optional_parts.values() for line in part + ] + if any("xsd:" in line for line in all_filters): namespaces_used.append("xsd") - created_queries = [] - for query in queries: - query_header_new = ( - self._insert_namespaces( - sorted( - set( - namespaces_used - + namespaces - + namespaces_source - + namespaces_dest - ) - ) + query_footer = [] + if remote_source is not None: + query_footer.append(" }") + query_footer.append("}") + + query_header = ( + self._insert_namespaces( + sorted( + set(namespaces_used + namespaces + namespaces_source + namespaces_dest) ) - + query_header ) + + query_header + ) + + created_queries = [] + for combination in combinations: + body = [] + for destination, lines in zip(destinations, combination): + if not destination._optional: + body.extend(line for line in lines if line not in body) + body += query_footer_source_types + query_footer_dest_types + for destination, lines in zip(destinations, combination): + if destination._optional: + block = [line for line in lines if line not in body] + block += optional_parts[id(destination)] + body.append(" OPTIONAL {") + body.extend(" " + line.strip() for line in block) + body.append(" }") query = ( - query_header_new - + query - + query_footer_source_types - + query_footer_dest_types + query_header + + body + query_filter + + query_footer + self._add_limit(limit) ) created_queries.append("\n".join(query)) return created_queries def query( - self, kg, source, destinations=None, return_df=True, num_paths=1, limit=None + self, + kg, + source, + destinations=None, + return_df=True, + num_paths=1, + limit=None, + python_values=False, ): """ Executes queries on a knowledge graph (KG) to retrieve data from a SPARQL query. @@ -975,12 +1057,17 @@ def query( The number of paths to retrieve for each query. limit : int, optional The maximum number of results to return. If None, no limit is applied. + python_values : bool, default=False + If True and `return_df` is True, literals are converted to Python values + (int, float, str, datetime, ...) and IRIs to strings, so that numeric + columns get a numeric dtype. Missing optional values become NaN/None. Returns ------- pandas.DataFrame or list If `return_df` is True, returns a pandas DataFrame containing the query results - (empty if nothing matches). Values are rdflib terms (URIRef, Literal). + (empty if nothing matches). Values are rdflib terms (URIRef, Literal), + unless `python_values` is True. If `return_df` is False, returns a list of query results. Returns None only if no query could be generated. """ @@ -1001,16 +1088,18 @@ def query( ) res = [] for query_string in query_strings: - r = self._query(kg, query_string, return_df=return_df) + r = self._query( + kg, query_string, return_df=return_df, python_values=python_values + ) if r is not None: res.append(r) if len(res) == 0: return None if return_df: - res = pd.concat(res) + res = pd.concat(res, ignore_index=python_values) return res - def _query(self, kg, query_string, return_df=True): + def _query(self, kg, query_string, return_df=True, python_values=False): """ Executes a SPARQL query on the given knowledge graph (KG) and optionally returns the result as a pandas DataFrame. @@ -1024,6 +1113,9 @@ def _query(self, kg, query_string, return_df=True): return_df : bool, optional If True, the query result is returned as a pandas DataFrame. Defaults to True. + python_values : bool, optional + If True, DataFrame cells are converted to Python values; see `query`. + Defaults to False. Returns ------- @@ -1047,7 +1139,18 @@ def _query(self, kg, query_string, return_df=True): if "SELECT DISTINCT" in line: break labels = [x[1:] for x in line.split()[2:]] - return pd.DataFrame(res, columns=labels) + if not python_values: + return pd.DataFrame(res, columns=labels) + rows = [[_to_python(value) for value in row] for row in res] + df = pd.DataFrame(rows, columns=labels) + for column in df.columns: + values = df[column].dropna() + if len(values) > 0 and all( + isinstance(v, numbers.Number) and not isinstance(v, bool) + for v in values + ): + df[column] = pd.to_numeric(df[column]) + return df return res diff --git a/tools4rdf/network/term.py b/tools4rdf/network/term.py index bd2df21..96efddf 100644 --- a/tools4rdf/network/term.py +++ b/tools4rdf/network/term.py @@ -191,6 +191,7 @@ def __init__( self._add_subclass = True self._old_variable_name = None self._qualified = False + self._optional = False @property def URIRef(self): @@ -413,6 +414,14 @@ def only(self): item._enforce_type = True return item + @property + def optional(self): + # this indicates that results without this term should be kept; + # its path, type and condition go into an OPTIONAL block + item = copy.deepcopy(self) + item._optional = True + return item + def toPython(self): return self.uri