diff --git a/CLAUDE.md b/CLAUDE.md index 94694bde64..74a15d2d71 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -80,10 +80,11 @@ These are structural properties of the graph model, not validation rules layered - **Every node has at least one label.** An empty label set is not representable; there is no "unlabeled node". An import file with a node whose label list is empty/null is malformed input, not a node to be created label-less. - **Every edge has exactly one edge type.** A null/empty edge type is not representable; an edge cannot exist without a type. A null `__type` in an import file is malformed input. -- **A property value is a scalar, a list or a map.** The scalar types are `ValueType`'s Int64, UInt64, Double, Bool, String, Embedding, DateTime and Duration; `ValueType::List` is the list one, held by `TypedPropertyContainer` over a `ListContainer` that owns its elements (see `docs/lists.md`). A stored list may be heterogeneous and may hold nested lists, so a LIST-typed column in an import file is a property to ingest, not unsupported input. `ValueType::Map` is the map one, held by `TypedPropertyContainer` over a `MapContainer` (see `docs/maps.md`): string keys, stored sorted, over values that may be scalars, lists or nested maps. A stored list may hold maps too, owned by its `ListContainer`'s own `MapContainer`. -- **`ValueType` and `CellTag` are append-only.** Both are written to disk as their raw ordinals — the property tag in `PropertyTypeMapDumper`, and the cell tag of an element inside an `EncodedList` and of a value inside an `EncodedMap` — with no format version beside them. A new member goes immediately before the sentinel; inserting one anywhere else silently reinterprets every datapart already written. The same holds for `net::proto::ColumnInternalKind` on the wire. +- **A property value is a scalar, a list or a map.** The scalar types are `ValueType`'s Int64, UInt64, Double, Bool, String, Embedding, DateTime, Duration and Point; `ValueType::List` is the list one, held by `TypedPropertyContainer` over a `ListContainer` that owns its elements (see `docs/lists.md`). A stored list may be heterogeneous and may hold nested lists, so a LIST-typed column in an import file is a property to ingest, not unsupported input. `ValueType::Map` is the map one, held by `TypedPropertyContainer` over a `MapContainer` (see `docs/maps.md`): string keys, stored sorted, over values that may be scalars, lists or nested maps. A stored list may hold maps too, owned by its `ListContainer`'s own `MapContainer`. +- **`ValueType` and `CellTag` are append-only.** Both are written to disk as their raw ordinals — the property tag in `PropertyTypeMapDumper`, and the cell tag of an element inside an `EncodedList` and of a value inside an `EncodedMap` — with no format version beside them. A new member goes immediately before the sentinel; inserting one anywhere else silently reinterprets every datapart already written. The same holds for `net::proto::ColumnInternalKind` on the wire, and for `PointCRS`, whose ordinal every stored point carries. - **`DateTime` is microseconds since the Unix epoch, UTC** (`storage/metadata/DateTime.h`). `datetime("...")` parses ISO-8601 and spends any offset on the instant, so `...T16:05:00+02:00` and `...T14:05:00Z` are one value and both render as `Z`. One int64 carries no zone; a zoned datetime would be a second type, not a widening of this one. - **`Duration` is a signed count of microseconds** (`storage/metadata/Duration.h`). It has no month component, because a month has no fixed length; a calendar duration would be a second type. `duration()` reads the count as microseconds, and a duration renders as ISO-8601 hours, minutes and seconds (`PT25H1M1S`). +- **A `Point` is x, y and z in one of four CRSs** (`storage/metadata/Point.h`): cartesian, cartesian-3d, wgs-84 and wgs-84-3d, Neo4j's. It is 32 bytes, so two equal points are equal in every byte: a 2D point holds a z of 0, no coordinate is -0.0, and the padding is zero. No coordinate is non-finite, a geographic latitude lies in [-90, 90] and its longitude in [-180, 180]. A point renders as the text `point({x: 1.0, y: 2.0, crs: 'cartesian'})`. ### Server (`server/`, `net/`) - Single TCP listener speaking TuringDB's binary protocol framed as HTTP/1.1 chunked transfer. The in-tree `net/http_parser/HTTPParser` reads the request (endpoints routed in `server/DBURIParser.h` / `server/Endpoints.h`), `server/TuringProtoServerProcessor` handles dispatch, and `net/turing_proto_server/TuringProtoWriter` emits one proto packet per HTTP chunk. There is no JSON protocol and no third-party web framework. Client side: `net/turing_proto_client/TuringClient` (C++), `python/turingdb/binary_client.py` (`BinaryClient`), `wasm/npm` (JS). diff --git a/build_npm.sh b/build_npm.sh index 1bd79fb43e..f8a81518af 100755 --- a/build_npm.sh +++ b/build_npm.sh @@ -67,6 +67,7 @@ node decoder_smoke.mjs cd "$PACKAGE_DIR" node client_smoke.mjs +node point_column_values_smoke.mjs node parameter_errors_smoke.mjs node split_error_smoke.mjs node payload_limit_smoke.mjs diff --git a/common/StringUtils.cpp b/common/StringUtils.cpp index b7598aa479..f6e6416324 100644 --- a/common/StringUtils.cpp +++ b/common/StringUtils.cpp @@ -1,5 +1,9 @@ #include "StringUtils.h" +#include + +#include + void StringUtils::splitString(std::string_view str, char sep, std::vector& res) { @@ -17,3 +21,11 @@ void StringUtils::splitString(std::string_view str, } } } + +bool StringUtils::equalsIgnoringCase(std::string_view lhs, std::string_view rhs) { + const auto sameLetter = [](char lhsCharacter, char rhsCharacter) { + return tolower(static_cast(lhsCharacter)) == tolower(static_cast(rhsCharacter)); + }; + + return std::ranges::equal(lhs, rhs, sameLetter); +} diff --git a/common/StringUtils.h b/common/StringUtils.h index 6f2ea9a04f..4de9ccc9ff 100644 --- a/common/StringUtils.h +++ b/common/StringUtils.h @@ -10,4 +10,6 @@ class StringUtils { static void splitString(std::string_view str, char sep, std::vector& res); + + static bool equalsIgnoringCase(std::string_view lhs, std::string_view rhs); }; diff --git a/docs/maps.md b/docs/maps.md index a7a9400b89..13846c455b 100644 --- a/docs/maps.md +++ b/docs/maps.md @@ -128,11 +128,11 @@ as `[u64 length][EncodedMap bytes]`. # Map projection `v {.name, .*, rank: v.age + 1, x}` builds a map out of `v`, a node, a relationship, a map, a -datetime or a duration. `.key` holds `v.key`, null where `v` has no such property, and `.*` every -property of `v`. On a datetime or a duration `.key` holds the component `v.key` reads: -`WITH duration({hours: 1}) AS d RETURN d {.hours, x: 1}` is `{hours: 1, x: 1}`. A temporal has -no properties, so `.*` and a name that is no component are rejected by the analyzer where `v` is -typed a datetime or a duration, and raise on the row where a tagged cell holds one. +datetime, a duration or a point. `.key` holds `v.key`, null where `v` has no such property, and +`.*` every property of `v`. On a datetime, a duration or a point `.key` holds the component +`v.key` reads: `WITH duration({hours: 1}) AS d RETURN d {.hours, x: 1}` is `{hours: 1, x: 1}`. +These values have no properties, so `.*` and a name that is no component are rejected by the +analyzer where `v` is typed one of them, and raise on the row where a tagged cell holds one. `key: expr` holds the expression, and `x` the variable `x` under the key `x`. A later entry shadows an earlier one under the same key, as in a map literal: `v {x: 1 / 0, x: 2}` is `{x: 2}`, and `v {.name, name: 1 / 0}` raises. An entry wins over the property `.*` reads under diff --git a/docs/points.md b/docs/points.md new file mode 100644 index 0000000000..853074c6fe --- /dev/null +++ b/docs/points.md @@ -0,0 +1,124 @@ +We support the Cypher point type as a stored property value, following Neo4j's spatial values: +https://neo4j.com/docs/cypher-manual/current/values-and-types/spatial/. + +# The value + +`ValueType::Point` is a scalar property type, held by `TypedPropertyContainer`. +`db::Point` (`storage/metadata/Point.h`) is 32 trivially copyable bytes: `x`, `y` and `z` as +doubles, then a one-byte `PointCRS` and 7 bytes of zero padding. The CRS is one of Neo4j's four: + +``` +PointCRS name SRID dimension +Cartesian cartesian 7203 2 +Cartesian3D cartesian-3d 9157 3 +WGS84 wgs-84 4326 2 +WGS84_3D wgs-84-3d 4979 3 +``` + +`PointCRS` is written to disk as its ordinal, inside every stored point, so it is append-only. + +The constructor makes two equal points equal in every byte, so a dump, a list encoding and a +DISTINCT key can all compare raw bytes: a 2D point holds a `z` of 0, `-0.0` becomes `0.0`, and +the padding is zero. It also rejects what no point holds: a non-finite coordinate, and a +geographic latitude outside [-90, 90]. A geographic longitude outside [-180, 180] is wrapped into +it, so `point({longitude: 190, latitude: 0})` has an x of -170. + +A point is a `CellTag::Point` inside a list or a map, written as its 32 bytes. + +# Building one + +`point(map)` reads the map's keys, ignoring case and ignoring any key it does not know: + +``` +point({x, y}) cartesian +point({x, y, z}) cartesian-3d +point({longitude, latitude}) wgs-84 +point({longitude, latitude, height}) wgs-84-3d; z may stand for height +point({..., crs: 'wgs-84'}) the CRS by name, ignoring case +point({..., srid: 4326}) the CRS by SRID +``` + +`x` and `y` win over `longitude` and `latitude` where a map holds both, and are read as longitude +and latitude in a geographic CRS. A null map, or a null value under any key, gives null. + +It raises on a map naming both a `crs` and an `srid`, on an unknown CRS, on longitude and latitude +in a cartesian CRS, on a number of coordinates the CRS does not have, and on a coordinate that is +no number. Neo4j parses a numeric string coordinate; TuringDB does not. + +# Reading one + +`p.x`, `p.y` and `p.z` are its coordinates, and `p.longitude`, `p.latitude` and `p.height` the +same ones in a geographic CRS; all six are doubles. `p.crs` is the CRS name and `p.srid` its SRID. +`p.z` and `p.height` raise on a 2D point, and the three geographic names raise on a cartesian +point. The CRS is a property of the value, not of the column, so a column of points mixing CRSs +raises on the row that holds the wrong one. + +A point renders as the text Neo4j's `toString` writes, in `toString(p)` and in the shell: +`point({x: 2.3522, y: 48.8566, crs: 'wgs-84'})`. The binary protocol carries it as its 32 bytes, +under the `POINT` column kind. + +# Comparing + +Two points are equal when they share a CRS and every coordinate. Points have one total order, by +SRID, then by x, y and z: ORDER BY, DISTINCT, grouping, `min()`, `max()` and `<`, `<=`, `>` and +`>=` all follow it, so every wgs-84 point (4326) is less than every cartesian one (7203). Neo4j +answers null for `<` between two different points. A point against a value of another type +compares to null. Between types, ORDER BY sorts points after paths and before datetimes, as in +Neo4j. + +# Aggregating + +`count`, `collect`, `min` and `max` take points, over a property, over the points a list holds, +per group and with DISTINCT. `avg` of points is their centroid, coordinate by coordinate: a point +of the CRS they share. Points of two CRSs raise, as points mixed with numbers do, and no point +averages to null. A geographic centroid is the mean of the longitudes and of the latitudes, not a +spherical mean, so points on both sides of the 180th meridian average to a longitude near 0. +`sum` takes no point. + +# Distance and bounding boxes + +`point.distance(a, b)` is the Euclidean distance between two cartesian points. Between two +wgs-84 points it is the great-circle distance in metres, by the haversine formula on a sphere of +the IUGG mean Earth radius, 6,371,008.8 m. Neo4j uses 6,378,140 m, so its distances are 0.11% +longer. Between two wgs-84-3d points the great circle is taken at the mean of the two heights and +combined with the height difference, as Neo4j does. It is null for points in different CRSs. + +`point.withinBBox(p, lowerLeft, upperRight)` is whether `p` lies in the box, bounds included. In a +geographic CRS a `lowerLeft` east of `upperRight` makes a box that crosses the 180th meridian, and +a `lowerLeft` north of `upperRight` an empty one. It is null unless the three points share a CRS. + +# Importing + +`LOAD JSONL` reads a point out of a JSON object under a name a `WITH POINTS [...]` clause lists, +by the rules `point()` reads a map with: + +``` +LOAD JSONL '/data/cities.jsonl' AS cities WITH POINTS [at] + +{"type": "node", "id": "0", "labels": ["City"], + "properties": {"name": "Paris", "at": {"longitude": 2.3522, "latitude": 48.8566}}} +``` + +A JSON null leaves the property unset. An object naming no point, or holding a null, raises. +`WITH POINTS`, `WITH DATETIMES` and `WITH DURATIONS` fill one `TypedPropertySpec`, which maps +each name to the type it is read as, and a name given two types is rejected. + +`LOAD PARQUET` reads points out of a geometry column with no clause of its own: a column +Parquet's GEOMETRY or GEOGRAPHY logical type marks, or one GeoParquet's `geo` file metadata names +with the WKB encoding (`io/parquet/ParquetGeometry.h`). Each value is a WKB point, little- or +big-endian, ISO or EWKB: a 2D point or a 3D one, and an empty point reads as null. A list of +geometries reads as a list of points. + +``` +CRS of the column points +OGC:CRS84, EPSG:4326, EPSG:4979, the default wgs-84, or wgs-84-3d for a 3D point +GeoParquet "crs": null cartesian, or cartesian-3d +any other rejected +``` + +A CRS given as PROJJSON is read by its `id`: GeoParquet's `crs` object, a `projjson:` naming +a key of the file metadata, or the PROJJSON text Arrow writes in the logical type itself. + +A geometry that is no point, a point with a measure, and bytes that are no WKB are rejected, a +WKB type outside ISO's dimension codes 0 to 3 included. An EWKB SRID other than 0 has to name the +column's CRS: 4326 or 4979 in a wgs-84 column, none in a cartesian one. diff --git a/fuzz/cypher.dict b/fuzz/cypher.dict index 71844e2da7..b5ea46bb61 100644 --- a/fuzz/cypher.dict +++ b/fuzz/cypher.dict @@ -70,6 +70,8 @@ "EMBEDDINGS" "AVAILABLE" "DATETIMES" +"DURATIONS" +"POINTS" # --- Boolean and null --- "TRUE" diff --git a/import/common/CMakeLists.txt b/import/common/CMakeLists.txt index dec338d67d..c37714bcca 100644 --- a/import/common/CMakeLists.txt +++ b/import/common/CMakeLists.txt @@ -1,8 +1,8 @@ add_library(turing_db_import_common_s STATIC - DateTimeSpec.cpp - DurationSpec.cpp EmbeddingsSpec.cpp + TypedPropertySpec.cpp ) target_include_directories(turing_db_import_common_s PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) +target_link_libraries(turing_db_import_common_s PUBLIC turing_db_storage_s) diff --git a/import/common/DateTimeSpec.cpp b/import/common/DateTimeSpec.cpp deleted file mode 100644 index 2ab573e6fe..0000000000 --- a/import/common/DateTimeSpec.cpp +++ /dev/null @@ -1,9 +0,0 @@ -#include "DateTimeSpec.h" - -#include - -using namespace db; - -bool DateTimeSpec::contains(std::string_view propName) const { - return _names.contains(propName); -} diff --git a/import/common/DateTimeSpec.h b/import/common/DateTimeSpec.h deleted file mode 100644 index ca63733481..0000000000 --- a/import/common/DateTimeSpec.h +++ /dev/null @@ -1,29 +0,0 @@ -#pragma once - -#include -#include - -namespace db { - -// The properties a JSONL import reads as instants. A datetime arrives as a plain JSON -// string, which a string property is spelled as too, so the import is told which names -// carry one rather than guessing from what the text looks like. -class DateTimeSpec { -public: - using SetType = std::unordered_set; - using ConstIterator = SetType::const_iterator; - - bool contains(std::string_view propName) const; - - ConstIterator begin() const { return _names.begin(); }; - ConstIterator end() const { return _names.end(); }; - - bool empty() const { return _names.empty(); } - - void emplace(std::string_view name) { _names.emplace(name); } - -private: - SetType _names; -}; - -} diff --git a/import/common/DurationSpec.cpp b/import/common/DurationSpec.cpp deleted file mode 100644 index fab65fbfc2..0000000000 --- a/import/common/DurationSpec.cpp +++ /dev/null @@ -1,9 +0,0 @@ -#include "DurationSpec.h" - -#include - -using namespace db; - -bool DurationSpec::contains(std::string_view propertyName) const { - return _names.contains(propertyName); -} diff --git a/import/common/DurationSpec.h b/import/common/DurationSpec.h deleted file mode 100644 index e87a607087..0000000000 --- a/import/common/DurationSpec.h +++ /dev/null @@ -1,29 +0,0 @@ -#pragma once - -#include -#include - -namespace db { - -// The properties an import reads as durations. Neither JSONL nor Parquet has a duration -// type, so a duration arrives as a plain integer count of microseconds and the import is -// told which names carry one. -class DurationSpec { -public: - using SetType = std::unordered_set; - using ConstIterator = SetType::const_iterator; - - bool contains(std::string_view propertyName) const; - - ConstIterator begin() const { return _names.begin(); }; - ConstIterator end() const { return _names.end(); }; - - bool empty() const { return _names.empty(); } - - void emplace(std::string_view name) { _names.emplace(name); } - -private: - SetType _names; -}; - -} diff --git a/import/common/JsonlImportSpecs.h b/import/common/JsonlImportSpecs.h index 2abd779dd5..517395b147 100644 --- a/import/common/JsonlImportSpecs.h +++ b/import/common/JsonlImportSpecs.h @@ -2,9 +2,8 @@ #include -#include "DateTimeSpec.h" -#include "DurationSpec.h" #include "EmbeddingsSpec.h" +#include "TypedPropertySpec.h" namespace db { @@ -13,8 +12,7 @@ namespace db { // text or an instant - so the query names the properties that are neither by default. struct JsonlImportSpecs { EmbeddingsSpec _embeddings; - DateTimeSpec _dateTimes; - DurationSpec _durations; + TypedPropertySpec _types; // Takes the hints another clause of the same statement gave void absorb(JsonlImportSpecs&& other) { @@ -22,12 +20,8 @@ struct JsonlImportSpecs { _embeddings.emplace(name, dimension); } - for (const std::string_view name : other._dateTimes) { - _dateTimes.emplace(name); - } - - for (const std::string_view name : other._durations) { - _durations.emplace(name); + for (const auto& [name, type] : other._types) { + _types.emplace(name, type); } } }; diff --git a/import/common/TypedPropertySpec.cpp b/import/common/TypedPropertySpec.cpp new file mode 100644 index 0000000000..5f7e5fd903 --- /dev/null +++ b/import/common/TypedPropertySpec.cpp @@ -0,0 +1,20 @@ +#include "TypedPropertySpec.h" + +using namespace db; + +ValueType TypedPropertySpec::getType(std::string_view propertyName) const { + const auto it = _types.find(propertyName); + if (it == _types.end()) { + return ValueType::Invalid; + } + + return it->second; +} + +void TypedPropertySpec::emplace(std::string_view propertyName, ValueType type) { + const auto [it, inserted] = _types.emplace(propertyName, type); + + if (!inserted && it->second != type) { + _clashes.push_back(Clash {._name = propertyName, ._first = it->second, ._second = type}); + } +} diff --git a/import/common/TypedPropertySpec.h b/import/common/TypedPropertySpec.h new file mode 100644 index 0000000000..8f031c1302 --- /dev/null +++ b/import/common/TypedPropertySpec.h @@ -0,0 +1,43 @@ +#pragma once + +#include +#include +#include + +#include "metadata/PropertyType.h" + +namespace db { + +// The properties an import reads as a type its file format does not spell: a datetime is a +// JSON string, a duration an integer count of microseconds, a point a JSON object. The query +// names them, so the import does not guess from what a value looks like. +class TypedPropertySpec { +public: + // A name the query gave a second type. The first one stands. + struct Clash { + std::string_view _name; + ValueType _first {ValueType::Invalid}; + ValueType _second {ValueType::Invalid}; + }; + + using MapType = std::unordered_map; + using ConstIterator = MapType::const_iterator; + + // ValueType::Invalid where the query names no type for @param propertyName + ValueType getType(std::string_view propertyName) const; + + void emplace(std::string_view propertyName, ValueType type); + + const std::vector& getClashes() const { return _clashes; } + + ConstIterator begin() const { return _types.begin(); } + ConstIterator end() const { return _types.end(); } + + bool empty() const { return _types.empty(); } + +private: + MapType _types; + std::vector _clashes; +}; + +} diff --git a/import/jsonl/JsonlImportResult.h b/import/jsonl/JsonlImportResult.h index 42ddfa3031..8f3e0d4afd 100644 --- a/import/jsonl/JsonlImportResult.h +++ b/import/jsonl/JsonlImportResult.h @@ -19,6 +19,7 @@ enum class JsonlImportErrorType : uint8_t { NON_NUMERIC_EMB, NON_DATETIME_VALUE, NON_DURATION_VALUE, + NON_POINT_VALUE, UNKNOWN_EDGE_SRC_ID, UNKNOWN_EDGE_TGT_ID, @@ -39,6 +40,7 @@ using JsonlImportErrorTypeDescription = EnumToString::Crea EnumStringPair, EnumStringPair, EnumStringPair, + EnumStringPair, EnumStringPair, EnumStringPair >; diff --git a/import/jsonl/JsonlParser.cpp b/import/jsonl/JsonlParser.cpp index 1d3bf8065a..85460a014e 100644 --- a/import/jsonl/JsonlParser.cpp +++ b/import/jsonl/JsonlParser.cpp @@ -5,22 +5,24 @@ #include #include #include +#include #include "ID.h" #include "JsonlImportResult.h" #include "list/ListContainer.h" #include "list/ListView.h" +#include "map/MapContainer.h" #include "metadata/PropertyNull.h" #include "metadata/PropertyType.h" #include "versioning/ChangeAccessor.h" #include "versioning/CommitBuilder.h" #include "writers/DataPartBuilder.h" -#include "DateTimeSpec.h" -#include "DurationSpec.h" #include "EmbeddingsSpec.h" +#include "TypedPropertySpec.h" #include "Profiler.h" +#include "TuringException.h" #include "writers/MetadataBuilder.h" using json = nlohmann::json; @@ -147,22 +149,13 @@ ListView fillList(const json& array, ListContainer& lists) { return lists.insert(elements); } -// The instant a property the query named as a datetime holds. A name it did not is left -// alone, so a string that happens to read as a date stays the text it was written as. -JsonlImportResult> tryGetDateTime(std::string_view name, - const json& value, - const DateTimeSpec& dateTimeSpecs, - size_t lineNo) { - if (!dateTimeSpecs.contains(name)) { - return std::optional {}; - } - +JsonlImportResult tryGetDateTime(std::string_view name, const json& value, size_t lineNumber) { // The query said this property holds an instant, so a value that names none is the // malformed input it reports rather than a property silently written as something // else. A null is not such a value - the caller drops it before reaching here. if (!value.is_string()) { return JsonlImportError::result(JsonlImportErrorType::NON_DATETIME_VALUE, - lineNo, + lineNumber, fmt::format("property '{}' is not a string", name)); } @@ -170,21 +163,14 @@ JsonlImportResult> tryGetDateTime(std::string_view name, const std::optional instant = DateTime::parse(text); if (!instant) { return JsonlImportError::result(JsonlImportErrorType::NON_DATETIME_VALUE, - lineNo, + lineNumber, fmt::format("property '{}' reads '{}'", name, text)); } - return instant; + return *instant; } -JsonlImportResult> tryGetDuration(std::string_view name, - const json& value, - const DurationSpec& durationSpecs, - size_t lineNumber) { - if (!durationSpecs.contains(name)) { - return std::optional {}; - } - +JsonlImportResult tryGetDuration(std::string_view name, const json& value, size_t lineNumber) { const bool isUnsignedPastInt64 = value.is_number_unsigned() && value.get() > static_cast(INT64_MAX); if (!value.is_number_integer() || isUnsignedPastInt64) { @@ -193,7 +179,112 @@ JsonlImportResult> tryGetDuration(std::string_view name, fmt::format("property '{}' reads '{}'", name, value.dump())); } - return std::optional {Duration(value.get())}; + return Duration(value.get()); +} + +// The map point() reads a JSON object as. point() reads no deeper than the kind of a value, +// so a nested object or array goes in empty. +MapView fillPointMap(const json& object, MapContainer& maps, ListContainer& lists) { + std::vector entries; + entries.reserve(object.size()); + + for (const auto& [key, field] : object.items()) { + if (field.is_object()) { + entries.push_back({key, maps.insert({})}); + } else if (field.is_array()) { + entries.push_back({key, lists.insert({})}); + } else if (field.is_null()) { + entries.push_back({key, PropertyNull {}}); + } else if (field.is_boolean()) { + entries.push_back({key, types::Bool::Primitive(field.get())}); + } else if (field.is_number_unsigned()) { + entries.push_back({key, field.get()}); + } else if (field.is_number_integer()) { + entries.push_back({key, field.get()}); + } else if (field.is_number_float()) { + entries.push_back({key, field.get()}); + } else { + entries.push_back({key, field.get()}); + } + } + + return maps.insert(entries); +} + +JsonlImportResult tryGetPoint(std::string_view name, + const json& value, + MapContainer& maps, + ListContainer& lists, + size_t lineNumber) { + if (!value.is_object()) { + return JsonlImportError::result(JsonlImportErrorType::NON_POINT_VALUE, + lineNumber, + fmt::format("property '{}' is not an object", name)); + } + + Point point; + try { + if (!Point::fromMap(fillPointMap(value, maps, lists), point)) { + return JsonlImportError::result(JsonlImportErrorType::NON_POINT_VALUE, + lineNumber, + fmt::format("property '{}' holds a null", name)); + } + } catch (const TuringException& e) { + return JsonlImportError::result(JsonlImportErrorType::NON_POINT_VALUE, + lineNumber, + fmt::format("property '{}': {}", name, e.what())); + } + + return point; +} + +// A name the query did not type is left alone, so a string that happens to read as a date +// stays the text it was written as. +using TypedValue = std::variant; + +JsonlImportResult> tryGetTypedValue(std::string_view name, + const json& value, + ValueType type, + MapContainer& maps, + ListContainer& lists, + size_t lineNumber) { + if (type == ValueType::DateTime) { + const JsonlImportResult instant = tryGetDateTime(name, value, lineNumber); + if (!instant) { + return instant.get_unexpected(); + } + + return std::optional {instant.value()}; + } else if (type == ValueType::Duration) { + const JsonlImportResult duration = tryGetDuration(name, value, lineNumber); + if (!duration) { + return duration.get_unexpected(); + } + + return std::optional {duration.value()}; + } else if (type == ValueType::Point) { + const JsonlImportResult point = tryGetPoint(name, value, maps, lists, lineNumber); + if (!point) { + return point.get_unexpected(); + } + + return std::optional {point.value()}; + } + + return std::optional {}; +} + +template +void visitTypedValue(const TypedValue& typed, Add&& add) { + std::visit([&add](const Value& held) { + if constexpr (std::is_same_v) { + add.template operator()(held); + } else if constexpr (std::is_same_v) { + add.template operator()(held); + } else { + add.template operator()(held); + } + }, typed); } std::optional tryGetEmbDim(std::string_view name, @@ -215,8 +306,7 @@ std::optional tryGetEmbDim(std::string_view name, JsonlImportResult JsonlParser::parse(ChangeAccessor& change, std::istream& stream, const EmbeddingsSpec& embeddingSpecs, - const DateTimeSpec& dateTimeSpecs, - const DurationSpec& durationSpecs) { + const TypedPropertySpec& typedProperties) { Profile profile("JsonlParser::parse"); std::string line; @@ -231,15 +321,15 @@ JsonlImportResult JsonlParser::parse(ChangeAccessor& change, std::vector embBacking; ListContainer listBacking; + MapContainer mapBacking; while (std::getline(stream, line)) { ++lineNumber; // Each list the last line built has been copied into the datapart, so the scratch // is released here rather than growing into a second copy of the whole file. - if (listBacking.size() > 0) { - listBacking.clear(); - } + listBacking.clear(); + mapBacking.clear(); // If empty line, skip if (line.empty()) { @@ -302,25 +392,17 @@ JsonlImportResult JsonlParser::parse(ChangeAccessor& change, continue; } - const JsonlImportResult> instant = - tryGetDateTime(ptName, value, dateTimeSpecs, lineNumber); - if (!instant) { - return instant.get_unexpected(); - } - - const JsonlImportResult> duration = - tryGetDuration(ptName, value, durationSpecs, lineNumber); - if (!duration) { - return duration.get_unexpected(); + const ValueType typedType = typedProperties.getType(ptName); + const JsonlImportResult> typed = + tryGetTypedValue(ptName, value, typedType, mapBacking, listBacking, lineNumber); + if (!typed) { + return typed.get_unexpected(); } - const bool valueIsDateTime = instant.value().has_value(); - const bool valueIsDuration = duration.value().has_value(); + const std::optional& typedValue = typed.value(); - if (valueIsDateTime) { - vt = ValueType::DateTime; - } else if (valueIsDuration) { - vt = ValueType::Duration; + if (typedValue) { + vt = typedType; } else if (value.is_number_float()) { vt = ValueType::Double; } else if (value.is_boolean()) { @@ -341,10 +423,10 @@ JsonlImportResult JsonlParser::parse(ChangeAccessor& change, pt = metadataBuilder.getOrCreatePropertyType(ptName, vt); } - if (valueIsDateTime) { - builder.addNodeProperty(nodeID, pt._id, *instant.value()); - } else if (valueIsDuration) { - builder.addNodeProperty(nodeID, pt._id, *duration.value()); + if (typedValue) { + visitTypedValue(*typedValue, [&](const typename Type::Primitive& held) { + builder.addNodeProperty(nodeID, pt._id, held); + }); } else if (value.is_number_float()) { builder.addNodeProperty(nodeID, pt._id, value.get()); } else if (value.is_boolean()) { @@ -417,25 +499,17 @@ JsonlImportResult JsonlParser::parse(ChangeAccessor& change, continue; } - const JsonlImportResult> instant = - tryGetDateTime(ptName, value, dateTimeSpecs, lineNumber); - if (!instant) { - return instant.get_unexpected(); - } - - const JsonlImportResult> duration = - tryGetDuration(ptName, value, durationSpecs, lineNumber); - if (!duration) { - return duration.get_unexpected(); + const ValueType typedType = typedProperties.getType(ptName); + const JsonlImportResult> typed = + tryGetTypedValue(ptName, value, typedType, mapBacking, listBacking, lineNumber); + if (!typed) { + return typed.get_unexpected(); } - const bool valueIsDateTime = instant.value().has_value(); - const bool valueIsDuration = duration.value().has_value(); + const std::optional& typedValue = typed.value(); - if (valueIsDateTime) { - vt = ValueType::DateTime; - } else if (valueIsDuration) { - vt = ValueType::Duration; + if (typedValue) { + vt = typedType; } else if (value.is_number_float()) { vt = ValueType::Double; } else if (value.is_boolean()) { @@ -456,10 +530,10 @@ JsonlImportResult JsonlParser::parse(ChangeAccessor& change, pt = metadataBuilder.getOrCreatePropertyType(ptName, vt); } - if (valueIsDateTime) { - builder.addEdgeProperty(edge, pt._id, *instant.value()); - } else if (valueIsDuration) { - builder.addEdgeProperty(edge, pt._id, *duration.value()); + if (typedValue) { + visitTypedValue(*typedValue, [&](const typename Type::Primitive& held) { + builder.addEdgeProperty(edge, pt._id, held); + }); } else if (value.is_number_float()) { builder.addEdgeProperty(edge, pt._id, value.get()); } else if (value.is_boolean()) { diff --git a/import/jsonl/JsonlParser.h b/import/jsonl/JsonlParser.h index 36ecc61630..a722c62bb3 100644 --- a/import/jsonl/JsonlParser.h +++ b/import/jsonl/JsonlParser.h @@ -5,9 +5,8 @@ #include "JsonlImportResult.h" -#include "DateTimeSpec.h" -#include "DurationSpec.h" #include "EmbeddingsSpec.h" +#include "TypedPropertySpec.h" namespace db { @@ -26,8 +25,7 @@ class JsonlParser { [[nodiscard]] static JsonlImportResult parse(ChangeAccessor& change, std::istream& stream, const EmbeddingsSpec& embeddingSpecs = {}, - const DateTimeSpec& dateTimeSpecs = {}, - const DurationSpec& durationSpecs = {}); + const TypedPropertySpec& typedProperties = {}); }; } diff --git a/import/parquet/ParquetImporter.cpp b/import/parquet/ParquetImporter.cpp index 0a05d02a76..f22ca65525 100644 --- a/import/parquet/ParquetImporter.cpp +++ b/import/parquet/ParquetImporter.cpp @@ -12,14 +12,14 @@ void ParquetImporter::import() { bioassert(_builder, "Null CommitBuilder."); { - ParquetNodeVisitor nodeVisitor(_builder, _nodeIDs, _durationSpecs); + ParquetNodeVisitor nodeVisitor(_builder, _nodeIDs, _typedProperties); ParquetReader reader(_nodeFile, nodeVisitor); reader.ensureFileOpen(); while (reader.nextChunk()) { } } { - ParquetEdgeVisitor edgeVisitor(_builder, _nodeIDs, _durationSpecs); + ParquetEdgeVisitor edgeVisitor(_builder, _nodeIDs, _typedProperties); ParquetReader reader(_edgeFile, edgeVisitor); reader.ensureFileOpen(); while (reader.nextChunk()) { } diff --git a/import/parquet/ParquetImporter.h b/import/parquet/ParquetImporter.h index c5e4dcb26d..67db8c2acf 100644 --- a/import/parquet/ParquetImporter.h +++ b/import/parquet/ParquetImporter.h @@ -4,8 +4,8 @@ #include "ParquetImportVisitor.h" -#include "DurationSpec.h" #include "Path.h" +#include "TypedPropertySpec.h" namespace db { @@ -33,11 +33,11 @@ class ParquetImporter { ParquetImporter(fs::Path nodeFile, fs::Path edgeFile, CommitBuilder* builder, - const DurationSpec& durationSpecs) + const TypedPropertySpec& typedProperties) : _nodeFile(std::move(nodeFile)), _edgeFile(std::move(edgeFile)), _builder(builder), - _durationSpecs(durationSpecs) + _typedProperties(typedProperties) { } @@ -49,7 +49,7 @@ class ParquetImporter { CommitBuilder* _builder {nullptr}; - const DurationSpec& _durationSpecs; + const TypedPropertySpec& _typedProperties; // Source ID -> NodeID, filled while importing nodes and consumed by the edges. ParquetImportVisitor::IDMap _nodeIDs; diff --git a/io/parquet/CMakeLists.txt b/io/parquet/CMakeLists.txt index ecf4a16c14..f365579c73 100644 --- a/io/parquet/CMakeLists.txt +++ b/io/parquet/CMakeLists.txt @@ -4,6 +4,7 @@ add_library(turing_db_io_parquet_s STATIC ParquetImportVisitor.cpp ParquetNodeVisitor.cpp ParquetEdgeVisitor.cpp + ParquetGeometry.cpp ) target_include_directories(turing_db_io_parquet_s PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/io/parquet/ParquetEdgeVisitor.cpp b/io/parquet/ParquetEdgeVisitor.cpp index 6e44edd160..d921795985 100644 --- a/io/parquet/ParquetEdgeVisitor.cpp +++ b/io/parquet/ParquetEdgeVisitor.cpp @@ -30,6 +30,7 @@ namespace rv = ranges::views; bool ParquetEdgeVisitor::onFileStart(const parquet::FileMetaData& metadata) { const parquet::SchemaDescriptor* schema = metadata.schema(); const int numColumns = metadata.num_columns(); + const ParquetGeometry geometry(metadata); for (int columnIndex = 0; columnIndex < numColumns; columnIndex++) { const parquet::ColumnDescriptor* desc = schema->Column(columnIndex); @@ -59,7 +60,7 @@ bool ParquetEdgeVisitor::onFileStart(const parquet::FileMetaData& metadata) { _edgetypeColIdx = columnIndex; } else { - discoverPropertyColumn(columnIndex, path, *desc); + discoverPropertyColumn(columnIndex, path, *desc, geometry); } } @@ -278,6 +279,16 @@ void ParquetEdgeVisitor::addEdgeProperty(const EdgeRecord& edge, } break; + case ValueType::Point: { + const std::string& wkb = _propByteArrayVals.at(columnIndex).at(valueIndex); + + Point point; + if (ParquetGeometry::readPoint(prop.name, wkb, prop.geometryCRS, point)) { + builder.addEdgeProperty(edge, prop.propertyTypeID, point); + } + } + break; + // Handled by applyEdgeListProperty, which groups the repeated values into lists // before any row reaches here case ValueType::List: diff --git a/io/parquet/ParquetGeometry.cpp b/io/parquet/ParquetGeometry.cpp new file mode 100644 index 0000000000..cc04c74b3a --- /dev/null +++ b/io/parquet/ParquetGeometry.cpp @@ -0,0 +1,336 @@ +#include "ParquetGeometry.h" + +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "StringUtils.h" +#include "TuringException.h" + +using namespace db; + +using json = nlohmann::json; + +namespace { + +constexpr uint32_t WKB_POINT = 1; +constexpr uint32_t WKB_DIMENSION_STEP = 1000; +constexpr uint32_t WKB_DIMENSION_CODE_COUNT = 4; +constexpr uint32_t EWKB_Z_FLAG = 0x80000000; +constexpr uint32_t EWKB_M_FLAG = 0x40000000; +constexpr uint32_t EWKB_SRID_FLAG = 0x20000000; +constexpr uint32_t EWKB_FLAGS = EWKB_Z_FLAG | EWKB_M_FLAG | EWKB_SRID_FLAG; + +constexpr std::string_view GEOPARQUET_KEY = "geo"; +constexpr std::string_view PROJJSON_PREFIX = "projjson:"; + +class WKBReader { +public: + explicit WKBReader(std::string_view bytes) + : _bytes(bytes) + { + } + + template + bool read(Value& value) { + if (_offset + sizeof(Value) > _bytes.size()) { + return false; + } + + std::array raw; + memcpy(raw.data(), _bytes.data() + _offset, sizeof(Value)); + + const bool swapsBytes = _bigEndian != (std::endian::native == std::endian::big); + if (swapsBytes) { + std::reverse(raw.begin(), raw.end()); + } + + memcpy(&value, raw.data(), sizeof(Value)); + _offset += sizeof(Value); + + return true; + } + + void setBigEndian(bool bigEndian) { _bigEndian = bigEndian; } + + bool atEnd() const { return _offset == _bytes.size(); } + +private: + std::string_view _bytes; + size_t _offset {0}; + bool _bigEndian {false}; +}; + +std::string_view geometryName(uint32_t kind) { + switch (kind) { + case 1: + return "Point"; + break; + case 2: + return "LineString"; + break; + case 3: + return "Polygon"; + break; + case 4: + return "MultiPoint"; + break; + case 5: + return "MultiLineString"; + break; + case 6: + return "MultiPolygon"; + break; + case 7: + return "GeometryCollection"; + break; + default: + return "geometry of an unknown type"; + break; + } +} + +[[noreturn]] void throwNoWKB(std::string_view columnName) { + throw TuringException(fmt::format("Geometry column '{}' holds a value that is no WKB.", columnName)); +} + +bool namesAGeographicCRS(std::string_view crs) { + constexpr std::array GEOGRAPHIC_CRS_NAMES = { + "OGC:CRS84", "OGC:CRS84h", "EPSG:4326", "EPSG:4979", "srid:4326", "srid:4979", + }; + + return std::ranges::any_of(GEOGRAPHIC_CRS_NAMES, [crs](std::string_view name) { + return StringUtils::equalsIgnoringCase(name, crs); + }); +} + +// The text a JSON value spells: a string's own characters, any other value's JSON +void jsonText(const json& value, std::string& text) { + text = value.is_string() ? value.get() : value.dump(); +} + +ParquetGeometryCRS crsOfProjJson(std::string_view columnName, const json& projjson) { + const auto id = projjson.find("id"); + const bool hasAnID = id != projjson.end() && id->is_object(); + + if (hasAnID) { + const auto authority = id->find("authority"); + const auto code = id->find("code"); + const bool namesAnAuthority = authority != id->end() && authority->is_string(); + const bool namesACode = code != id->end(); + + if (namesAnAuthority && namesACode) { + std::string codeText; + jsonText(*code, codeText); + + if (namesAGeographicCRS(fmt::format("{}:{}", authority->get(), codeText))) { + return ParquetGeometryCRS::Geographic; + } + } + } + + const auto name = projjson.find("name"); + std::string crsName; + jsonText(name != projjson.end() ? *name : projjson, crsName); + + throw TuringException(fmt::format("Geometry column '{}' is in the CRS '{}', and a point is cartesian or wgs-84.", + columnName, + crsName)); +} + +} + +ParquetGeometry::ParquetGeometry(const parquet::FileMetaData& metadata) + : _keyValues(metadata.key_value_metadata().get()) +{ + readGeoParquetColumns(); +} + +ParquetGeometry::~ParquetGeometry() { +} + +bool ParquetGeometry::findColumn(const std::string& name, + const parquet::ColumnDescriptor& descriptor, + ParquetGeometryCRS& crs) const { + const parquet::LogicalType* logicalType = descriptor.logical_type().get(); + + if (logicalType && logicalType->is_geometry()) { + crs = crsOfString(name, static_cast(*logicalType).crs()); + return true; + } else if (logicalType && logicalType->is_geography()) { + crs = crsOfString(name, static_cast(*logicalType).crs()); + return true; + } + + const auto it = _geoParquetColumns.find(name); + if (it == _geoParquetColumns.end()) { + return false; + } + + crs = it->second; + + return true; +} + +bool ParquetGeometry::readPoint(std::string_view columnName, + std::string_view wkb, + ParquetGeometryCRS crs, + Point& point) { + WKBReader reader(wkb); + + uint8_t byteOrder = 0; + if (!reader.read(byteOrder) || byteOrder > 1) { + throwNoWKB(columnName); + } + + reader.setBigEndian(byteOrder == 0); + + uint32_t type = 0; + if (!reader.read(type)) { + throwNoWKB(columnName); + } + + // EWKB flags its dimensions in the top bits, ISO WKB in the thousands of the type + const uint32_t isoType = type & ~EWKB_FLAGS; + const uint32_t kind = isoType % WKB_DIMENSION_STEP; + const uint32_t isoDimensions = isoType / WKB_DIMENSION_STEP; + if (isoDimensions >= WKB_DIMENSION_CODE_COUNT) { + throwNoWKB(columnName); + } + + const bool hasZ = (type & EWKB_Z_FLAG) != 0 || isoDimensions == 1 || isoDimensions == 3; + const bool hasM = (type & EWKB_M_FLAG) != 0 || isoDimensions == 2 || isoDimensions == 3; + + uint32_t srid = 0; + const bool hasSRID = (type & EWKB_SRID_FLAG) != 0; + if (hasSRID && !reader.read(srid)) { + throwNoWKB(columnName); + } + + if (kind != WKB_POINT) { + throw TuringException(fmt::format("Geometry column '{}' holds a {}, not a point.", columnName, geometryName(kind))); + } else if (hasM) { + throw TuringException(fmt::format("Geometry column '{}' holds a point with a measure, which no point carries.", + columnName)); + } + + double x = 0.0; + double y = 0.0; + double z = 0.0; + const bool readsCoordinates = reader.read(x) && reader.read(y) && (!hasZ || reader.read(z)); + + if (!readsCoordinates || !reader.atEnd()) { + throwNoWKB(columnName); + } + + const bool isGeographic = crs == ParquetGeometryCRS::Geographic; + PointCRS sridCRS {PointCRS::Cartesian}; + const bool sridIsGeographic = Point::crsOfSRID(srid, sridCRS) && Point::isGeographic(sridCRS); + const bool sridNamesTheColumnCRS = srid == 0 || (isGeographic && sridIsGeographic); + + if (!sridNamesTheColumnCRS) { + throw TuringException(fmt::format("Geometry column '{}' holds a point with SRID {}, which is not the column's CRS.", + columnName, + srid)); + } + + // WKB spells an empty point with NaN coordinates + if (isnan(x) && isnan(y)) { + return false; + } + + const PointCRS pointCRS = isGeographic ? (hasZ ? PointCRS::WGS84_3D : PointCRS::WGS84) + : (hasZ ? PointCRS::Cartesian3D : PointCRS::Cartesian); + + point = Point(pointCRS, x, y, z); + + return true; +} + +// GeoParquet names its geometry columns in the JSON of the file's "geo" key. A column with no +// "crs" is in OGC:CRS84, and one whose "crs" is null in an undefined, cartesian CRS. +void ParquetGeometry::readGeoParquetColumns() { + const std::string* geo = findKeyValue(GEOPARQUET_KEY); + if (!geo) { + return; + } + + const json document = json::parse(*geo, nullptr, false); + const auto columns = document.find("columns"); + const bool namesColumns = columns != document.end() && columns->is_object(); + + if (!namesColumns) { + throw TuringException("The GeoParquet metadata of the file names no columns."); + } + + for (const auto& [name, column] : columns->items()) { + if (!column.is_object()) { + throw TuringException(fmt::format("The GeoParquet metadata of column '{}' is no object.", name)); + } + + const auto encoding = column.find("encoding"); + std::string encodingText; + if (encoding != column.end()) { + jsonText(*encoding, encodingText); + } + + if (!StringUtils::equalsIgnoringCase(encodingText, "WKB")) { + throw TuringException(fmt::format("Geometry column '{}' is encoded as '{}', not as WKB.", name, encodingText)); + } + + const auto crs = column.find("crs"); + if (crs == column.end()) { + _geoParquetColumns.emplace(name, ParquetGeometryCRS::Geographic); + } else if (crs->is_null()) { + _geoParquetColumns.emplace(name, ParquetGeometryCRS::Cartesian); + } else if (crs->is_string()) { + _geoParquetColumns.emplace(name, crsOfString(name, crs->get())); + } else { + _geoParquetColumns.emplace(name, crsOfProjJson(name, *crs)); + } + } +} + +const std::string* ParquetGeometry::findKeyValue(std::string_view key) const { + if (!_keyValues) { + return nullptr; + } + + const int index = _keyValues->FindKey(key); + if (index < 0) { + return nullptr; + } + + return &_keyValues->value(index); +} + +// Parquet's logical types name a CRS by a string, OGC:CRS84 where it is empty. A +// "projjson:" one names a key of the file metadata holding the PROJJSON, and Arrow writes +// any other CRS as the PROJJSON text itself. +ParquetGeometryCRS ParquetGeometry::crsOfString(std::string_view columnName, std::string_view crs) const { + if (crs.empty() || namesAGeographicCRS(crs)) { + return ParquetGeometryCRS::Geographic; + } + + const std::string* referenced = crs.starts_with(PROJJSON_PREFIX) ? findKeyValue(crs.substr(PROJJSON_PREFIX.size())) : nullptr; + const std::string_view projjsonText = referenced ? std::string_view(*referenced) : crs; + const json projjson = json::parse(projjsonText.begin(), projjsonText.end(), nullptr, false); + + if (projjson.is_object()) { + return crsOfProjJson(columnName, projjson); + } + + throw TuringException(fmt::format("Geometry column '{}' is in the CRS '{}', and a point is cartesian or wgs-84.", + columnName, + crs)); +} diff --git a/io/parquet/ParquetGeometry.h b/io/parquet/ParquetGeometry.h new file mode 100644 index 0000000000..4db7251471 --- /dev/null +++ b/io/parquet/ParquetGeometry.h @@ -0,0 +1,61 @@ +#pragma once + +#include + +#include +#include +#include + +#include "metadata/Point.h" + +namespace arrow { +class KeyValueMetadata; +} + +namespace parquet { +class ColumnDescriptor; +class FileMetaData; +} + +namespace db { + +// The dimension of each point comes from its own WKB, so a column names no 3D CRS +enum class ParquetGeometryCRS : uint8_t { + Geographic, + Cartesian, +}; + +// The geometry columns of a Parquet file, which hold one WKB value per row: the columns +// Parquet's GEOMETRY or GEOGRAPHY logical type marks, and the ones GeoParquet's "geo" file +// metadata names. It reads the file metadata it is built from, which outlives it. +class ParquetGeometry { +public: + explicit ParquetGeometry(const parquet::FileMetaData& metadata); + ~ParquetGeometry(); + + // Whether the column holds geometries, and the CRS of their points. Raises for a CRS no + // point is in. + bool findColumn(const std::string& name, + const parquet::ColumnDescriptor& descriptor, + ParquetGeometryCRS& crs) const; + + // The point a WKB value holds, false for an empty point. Raises on a value that holds + // another geometry, a measure, an SRID that is not the column's CRS, or bytes that are + // no WKB. + static bool readPoint(std::string_view columnName, + std::string_view wkb, + ParquetGeometryCRS crs, + Point& point); + +private: + const arrow::KeyValueMetadata* _keyValues {nullptr}; + std::unordered_map _geoParquetColumns; + + void readGeoParquetColumns(); + + const std::string* findKeyValue(std::string_view key) const; + + ParquetGeometryCRS crsOfString(std::string_view columnName, std::string_view crs) const; +}; + +} diff --git a/io/parquet/ParquetImportVisitor.cpp b/io/parquet/ParquetImportVisitor.cpp index 527c268f55..a70770bcf0 100644 --- a/io/parquet/ParquetImportVisitor.cpp +++ b/io/parquet/ParquetImportVisitor.cpp @@ -126,7 +126,8 @@ bool ParquetImportVisitor::onBoolValues(size_t columnIndex, std::spanmetadata(); const parquet::Type::type physicalType = descriptor.physical_type(); @@ -180,7 +181,7 @@ void ParquetImportVisitor::discoverPropertyColumn(size_t columnIndex, const bool isList = maxRepLevel > 0; const std::string name {isList ? listPropertyName(path) : std::string_view {path}}; - const bool holdsDurations = _durationSpecs.contains(name); + const bool holdsDurations = _typedProperties.getType(name) == ValueType::Duration; if (holdsDurations) { const bool hasNoLogicalType = !logicalType || logicalType->is_none(); const bool isSignedInt = logicalType && logicalType->is_int() @@ -194,6 +195,16 @@ void ParquetImportVisitor::discoverPropertyColumn(size_t columnIndex, valueType = ValueType::Duration; } + ParquetGeometryCRS geometryCRS {ParquetGeometryCRS::Geographic}; + const bool holdsGeometries = geometry.findColumn(name, descriptor, geometryCRS); + if (holdsGeometries) { + if (physicalType != parquet::Type::BYTE_ARRAY) { + throw TuringException(fmt::format("Geometry column '{}' must be a BYTE_ARRAY column of WKB.", name)); + } + + valueType = ValueType::Point; + } + if (isList) { valueType = ValueType::List; } @@ -218,7 +229,9 @@ void ParquetImportVisitor::discoverPropertyColumn(size_t columnIndex, .maxDefLevel = maxDefLevel, .maxRepLevel = maxRepLevel, .timeUnit = timeUnit, - .holdsDurations = holdsDurations}; + .holdsDurations = holdsDurations, + .holdsGeometries = holdsGeometries, + .geometryCRS = geometryCRS}; if (isList) { collectListDefLevels(descriptor, col.listDefLevels); @@ -459,8 +472,20 @@ ListContainer::ListItemVariant ParquetImportVisitor::listElement(const PropertyC case parquet::Type::BOOLEAN: return types::Bool::Primitive(_propListBoolVals.at(columnIndex).at(valueIndex) != 0); break; - case parquet::Type::BYTE_ARRAY: - return types::String::Primitive {_propByteArrayVals.at(columnIndex).at(valueIndex)}; + case parquet::Type::BYTE_ARRAY: { + const std::string& bytes = _propByteArrayVals.at(columnIndex).at(valueIndex); + + if (prop.holdsGeometries) { + Point point; + if (!ParquetGeometry::readPoint(prop.name, bytes, prop.geometryCRS, point)) { + return PropertyNull {}; + } + + return point; + } + + return types::String::Primitive {bytes}; + } break; case parquet::Type::FLOAT: diff --git a/io/parquet/ParquetImportVisitor.h b/io/parquet/ParquetImportVisitor.h index fb9442f8fd..b832f89129 100644 --- a/io/parquet/ParquetImportVisitor.h +++ b/io/parquet/ParquetImportVisitor.h @@ -11,10 +11,11 @@ #include +#include "ParquetGeometry.h" #include "ParquetReader.h" -#include "DurationSpec.h" #include "ID.h" +#include "TypedPropertySpec.h" #include "list/ListContainer.h" #include "list/ListView.h" #include "metadata/PropertyType.h" @@ -36,10 +37,10 @@ class ParquetImportVisitor : public ParquetSaxVisitor { using NodeIDs = std::span; using IDMap = std::unordered_map; - ParquetImportVisitor(CommitBuilder* builder, IDMap& nodeIDs, const DurationSpec& durationSpecs) + ParquetImportVisitor(CommitBuilder* builder, IDMap& nodeIDs, const TypedPropertySpec& typedProperties) : _builder(builder), _nodeIDs(nodeIDs), - _durationSpecs(durationSpecs) + _typedProperties(typedProperties) { } @@ -64,6 +65,9 @@ class ParquetImportVisitor : public ParquetSaxVisitor { bool holdsDurations {false}; + bool holdsGeometries {false}; + ParquetGeometryCRS geometryCRS {ParquetGeometryCRS::Geographic}; + // Indexed by repetition depth: the definition level a value must reach for the // list at that depth to hold an element. Index 0 is unused. std::vector listDefLevels; @@ -79,7 +83,7 @@ class ParquetImportVisitor : public ParquetSaxVisitor { // @ref DataPartBuilder used when importing IDMap& _nodeIDs; - const DurationSpec& _durationSpecs; + const TypedPropertySpec& _typedProperties; std::unordered_map _propertyColumns; @@ -125,7 +129,8 @@ class ParquetImportVisitor : public ParquetSaxVisitor { // Infers a property column value type and registers property type void discoverPropertyColumn(size_t columnIndex, const std::string& path, - const parquet::ColumnDescriptor& descriptor); + const parquet::ColumnDescriptor& descriptor, + const ParquetGeometry& geometry); // Capture helpers for property columns, called by the derived value callbacks // once they have handled their entity-specific columns. diff --git a/io/parquet/ParquetNodeVisitor.cpp b/io/parquet/ParquetNodeVisitor.cpp index ec2e81676e..a0ca67f40f 100644 --- a/io/parquet/ParquetNodeVisitor.cpp +++ b/io/parquet/ParquetNodeVisitor.cpp @@ -29,6 +29,7 @@ namespace rv = ranges::views; bool ParquetNodeVisitor::onFileStart(const parquet::FileMetaData& metadata) { const parquet::SchemaDescriptor* schema = metadata.schema(); const int numColumns = metadata.num_columns(); + const ParquetGeometry geometry(metadata); for (int columnIndex = 0; columnIndex < numColumns; columnIndex++) { const parquet::ColumnDescriptor* desc = schema->Column(columnIndex); @@ -53,7 +54,7 @@ bool ParquetNodeVisitor::onFileStart(const parquet::FileMetaData& metadata) { _lblColIdx = columnIndex; _lblMaxDefLevel = maxDefLevel; } else { - discoverPropertyColumn(columnIndex, path, *desc); + discoverPropertyColumn(columnIndex, path, *desc, geometry); } } @@ -286,6 +287,16 @@ void ParquetNodeVisitor::addNodeProperty(NodeID id, } break; + case ValueType::Point: { + const std::string& wkb = _propByteArrayVals.at(columnIndex).at(valueIndex); + + Point point; + if (ParquetGeometry::readPoint(prop.name, wkb, prop.geometryCRS, point)) { + builder.addNodeProperty(id, prop.propertyTypeID, point); + } + } + break; + // Handled by applyNodeListProperty, which groups the repeated values into lists // before any row reaches here case ValueType::List: diff --git a/memory/LocalMemory.h b/memory/LocalMemory.h index 8ddee05c5b..496b447452 100644 --- a/memory/LocalMemory.h +++ b/memory/LocalMemory.h @@ -80,6 +80,7 @@ class LocalMemory { MakeMemoryPool>::type, MakeMemoryPool>::type, MakeMemoryPool>::type, + MakeMemoryPool>::type, MakeMemoryPool>>::type, MakeMemoryPool>>::type, @@ -91,6 +92,7 @@ class LocalMemory { MakeMemoryPool>>::type, MakeMemoryPool>>::type, MakeMemoryPool>>::type, + MakeMemoryPool>>::type, MakeMemoryPool>::type, @@ -102,6 +104,7 @@ class LocalMemory { MakeMemoryPool>::type, MakeMemoryPool>::type, MakeMemoryPool>::type, + MakeMemoryPool>::type, MakeMemoryPool>::type, MakeMemoryPool>::type, @@ -114,6 +117,7 @@ class LocalMemory { MakeMemoryPool>::type, MakeMemoryPool>::type, MakeMemoryPool>::type, + MakeMemoryPool>::type, MakeMemoryPool>::type, MakeMemoryPool>::type, MakeMemoryPool>::type, diff --git a/net/decoders/ParameterMapSink.cpp b/net/decoders/ParameterMapSink.cpp index 0672405901..e295bf894e 100644 --- a/net/decoders/ParameterMapSink.cpp +++ b/net/decoders/ParameterMapSink.cpp @@ -71,6 +71,9 @@ void readScalar(const char* bytes, db::ParameterValue& slot) { case db::CellTag::Duration: slot.setDuration(readValue(value)); break; + case db::CellTag::Point: + throwUnsupportedType("Point"); + break; default: throw TuringException(fmt::format("Invalid value tag {} in a parameter", static_cast(tag))); break; diff --git a/net/decoders/TuringProtoDecoder.h b/net/decoders/TuringProtoDecoder.h index 75f5cb2a21..2ebb35afd8 100644 --- a/net/decoders/TuringProtoDecoder.h +++ b/net/decoders/TuringProtoDecoder.h @@ -125,6 +125,8 @@ decltype(auto) TuringProtoDecoder::dispatchColumnType(ColumnInternalKind t return fn.template operator()(encoding); case ColumnInternalKind::GRAPH_PATH: return fn.template operator()(encoding); + case ColumnInternalKind::POINT: + return fn.template operator()(encoding); } throw TuringException("Unsupported incoming column type"); diff --git a/net/decoders/TuringProtoDecoderConcepts.h b/net/decoders/TuringProtoDecoderConcepts.h index 724ef59507..7fd653129a 100644 --- a/net/decoders/TuringProtoDecoderConcepts.h +++ b/net/decoders/TuringProtoDecoderConcepts.h @@ -16,6 +16,7 @@ concept TrivialInternalTypes = std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v +|| std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v diff --git a/net/encoders/OutputValues.h b/net/encoders/OutputValues.h index 252d9d33d6..21e4c89976 100644 --- a/net/encoders/OutputValues.h +++ b/net/encoders/OutputValues.h @@ -85,6 +85,9 @@ concept IsDateTime = std::is_same_v, types::Date template concept IsDuration = std::is_same_v, types::Duration::Primitive>; +template +concept IsPoint = std::is_same_v, types::Point::Primitive>; + struct ColumnTypeGenerator { std::string& _name; @@ -96,6 +99,8 @@ struct ColumnTypeGenerator { _name = fmt::format("DateTime"); } else if constexpr (IsDuration) { _name = fmt::format("Duration"); + } else if constexpr (IsPoint) { + _name = fmt::format("Point"); } else if constexpr (IsUInt64) { _name = fmt::format("UInt64"); } else if constexpr (IsInt64) { diff --git a/net/encoders/TuringProtoEncoder.h b/net/encoders/TuringProtoEncoder.h index 2fd94cd099..a1fa505ee3 100644 --- a/net/encoders/TuringProtoEncoder.h +++ b/net/encoders/TuringProtoEncoder.h @@ -60,6 +60,8 @@ struct ColInternalKindToProtoEnum { return Enum::DATE_TIME; } else if constexpr (db::IsDuration) { return Enum::DURATION; + } else if constexpr (db::IsPoint) { + return Enum::POINT; } else if constexpr (db::IsListView) { return Enum::LIST_VIEW; } else if constexpr (db::IsMap) { diff --git a/net/turing_proto_common/TuringProtoHeaders.h b/net/turing_proto_common/TuringProtoHeaders.h index 26cdd5d9cd..5f79bf096a 100644 --- a/net/turing_proto_common/TuringProtoHeaders.h +++ b/net/turing_proto_common/TuringProtoHeaders.h @@ -80,6 +80,7 @@ enum class ColumnInternalKind : uint32_t { DATE_TIME, DURATION, GRAPH_PATH, + POINT, }; enum class ColumnKind : uint8_t { diff --git a/procedures/ProcUtils.cpp b/procedures/ProcUtils.cpp index a4c9869334..3ca5f8c45b 100644 --- a/procedures/ProcUtils.cpp +++ b/procedures/ProcUtils.cpp @@ -92,6 +92,13 @@ void appendCell(std::string& out, CellView element) { ProcUtils::appendJsonString(out, formatted); } break; + case CellTag::Point: { + std::string formatted; + Point::format(formatted, element.getAs()); + + ProcUtils::appendJsonString(out, formatted); + } + break; case CellTag::MapView: appendMapValue(out, element.getAs()); break; @@ -171,6 +178,11 @@ void appendPropertyValue(std::string& out, const PropertyVariant& value) { std::string formatted; Duration::format(formatted, *ptr); + ProcUtils::appendJsonString(out, formatted); + } else if constexpr (std::is_same_v) { + std::string formatted; + Point::format(formatted, *ptr); + ProcUtils::appendJsonString(out, formatted); } else if constexpr (std::is_same_v) { appendMapValue(out, *ptr); diff --git a/python/turingdb/__init__.py b/python/turingdb/__init__.py index f0db7a9fe6..55014785a2 100644 --- a/python/turingdb/__init__.py +++ b/python/turingdb/__init__.py @@ -6,13 +6,14 @@ from .binary_client import BinaryClient from .embedded_client import EmbeddedClient from .exceptions import TuringDBException +from .point import Point try: from ._version import version as __version__ except ImportError: __version__ = "unknown" -__all__ = ["TuringDB", "BinaryClient", "EmbeddedClient", "TuringDBException", "__version__"] +__all__ = ["TuringDB", "BinaryClient", "EmbeddedClient", "TuringDBException", "Point", "__version__"] def get_executable_path(executable_name="turingdb"): """Get the path to a compiled C++ executable shipped with the package.""" diff --git a/python/turingdb/_common/NanobindUtils.cpp b/python/turingdb/_common/NanobindUtils.cpp index f4728a08ba..4c18f2dffe 100644 --- a/python/turingdb/_common/NanobindUtils.cpp +++ b/python/turingdb/_common/NanobindUtils.cpp @@ -190,6 +190,7 @@ namespace { // them as members of one struct lets them call each other without a forward declaration. struct ValueToPyObject { mutable PyTimeTypes _times; + mutable nb::object _point; nb::object duration(db::Duration value) const { return _times.getTimedelta()(nb::arg("microseconds") = value.getMicroseconds()); @@ -199,6 +200,30 @@ struct ValueToPyObject { return _times.getEpoch() + duration(db::Duration {value.getMicroseconds()}); } + nb::object point(const db::Point& value) const { + if (!_point.is_valid()) { + _point = nb::module_::import_("turingdb.point").attr("Point"); + } + + const db::PointCRS crs = value.getCRS(); + const nb::object z = db::Point::getDimension(crs) == 3 ? nb::cast(value.getZ()) : nb::none(); + + return _point(db::Point::getSRID(crs), value.getX(), value.getY(), z); + } + + nb::object point(const std::optional& value) const { + return value.has_value() ? point(*value) : nb::none(); + } + + template + nb::object pointColumn(const std::vector& src) const { + nb::list out; + for (const T& value : src) { + out.append(point(value)); + } + return out; + } + nb::object view(const db::ListView& listView) const { nb::list out; for (const db::CellView element : listView.elements()) { @@ -241,6 +266,8 @@ struct ValueToPyObject { return duration(element.getAs()); } else if constexpr (std::is_same_v) { return dateTime(element.getAs()); + } else if constexpr (std::is_same_v) { + return point(element.getAs()); } else if constexpr (std::is_same_v) { nb::list ids; for (const uint64_t id : element.getAs().getIDs()) { @@ -367,6 +394,12 @@ nb::dict dataframeToNumpy(db::Dataframe* df) { dtypeName = "Duration"; break; } + case db::ColumnVector::staticKind(): { + const auto& src = static_cast*>(col)->getRaw(); + value = listVisitor.pointColumn(src); + dtypeName = "Point"; + break; + } case db::ColumnVector::staticKind(): { const auto& src = static_cast*>(col)->getRaw(); nb::list lst; @@ -533,6 +566,13 @@ nb::dict dataframeToNumpy(db::Dataframe* df) { break; } + case db::ColumnOptVector::staticKind(): { + const auto& src = static_cast*>(col)->getRaw(); + value = listVisitor.pointColumn(src); + dtypeName = "Point"; + break; + } + case db::ColumnConst::staticKind(): { const auto& v = static_cast*>(col)->getRaw(); value = repeatValueAsNdarray(v, rowCount); @@ -614,6 +654,28 @@ nb::dict dataframeToNumpy(db::Dataframe* df) { dtypeName = "Duration"; break; } + case db::ColumnConst::staticKind(): { + const auto& constant = static_cast*>(col)->getRaw(); + const nb::object point = listVisitor.point(constant); + nb::list points; + for (size_t row = 0; row < rowCount; ++row) { + points.append(point); + } + value = points; + dtypeName = "Point"; + break; + } + case db::ColumnConst>::staticKind(): { + const auto& constant = static_cast>*>(col)->getRaw(); + const nb::object point = listVisitor.point(constant); + nb::list points; + for (size_t row = 0; row < rowCount; ++row) { + points.append(point); + } + value = points; + dtypeName = "Point"; + break; + } case db::ColumnConst::staticKind(): { const auto& v = static_cast*>(col)->getRaw(); nb::object py = embeddingToNdarray(v); diff --git a/python/turingdb/_common/PyParameters.cpp b/python/turingdb/_common/PyParameters.cpp index c756710008..60031b1ac2 100644 --- a/python/turingdb/_common/PyParameters.cpp +++ b/python/turingdb/_common/PyParameters.cpp @@ -65,8 +65,29 @@ std::string getKey(nb::handle key) { return nb::cast(key); } -// A bool is checked before an int: in Python, bool is a subclass of int. -void readValue(nb::handle source, db::ParameterValue& target, PyTimeTypes& times, std::vector& worklist) { +bool isPoint(nb::handle value, nb::object& pointType) { + if (!pointType.is_valid()) { + pointType = nb::module_::import_("turingdb.point").attr("Point"); + } + + const int isInstance = PyObject_IsInstance(value.ptr(), pointType.ptr()); + if (isInstance < 0) { + throw nb::python_error(); + } + + return isInstance == 1; +} + +// A bool is checked before an int: in Python, bool is a subclass of int. A Point is a tuple +// too, but a value rather than a list, and no parameter holds a point. +void readValue(nb::handle source, + db::ParameterValue& target, + PyTimeTypes& times, + nb::object& pointType, + std::vector& worklist) { + const bool readsAsAList = nb::isinstance(source) + || (nb::isinstance(source) && !isPoint(source, pointType)); + if (source.is_none()) { target.setNull(); } else if (nb::isinstance(source)) { @@ -83,7 +104,7 @@ void readValue(nb::handle source, db::ParameterValue& target, PyTimeTypes& times target.setDouble(nb::cast(source)); } else if (nb::isinstance(source)) { target.setString(nb::cast(source)); - } else if (nb::isinstance(source) || nb::isinstance(source)) { + } else if (readsAsAList) { db::ParameterValue::List& list = target.setList(); list.resize(nb::len(source)); @@ -123,6 +144,7 @@ void readParameters(nb::handle source, db::ParameterMap& parameters) { } PyTimeTypes times; + nb::object pointType; std::vector worklist; for (const auto& [name, value] : nb::borrow(source)) { @@ -133,7 +155,7 @@ void readParameters(nb::handle source, db::ParameterMap& parameters) { const auto [value, target] = worklist.back(); worklist.pop_back(); - readValue(value, *target, times, worklist); + readValue(value, *target, times, pointType, worklist); } } diff --git a/python/turingdb/point.py b/python/turingdb/point.py new file mode 100644 index 0000000000..1f7e506afd --- /dev/null +++ b/python/turingdb/point.py @@ -0,0 +1,16 @@ +from typing import NamedTuple, Optional + +_CRS_NAMES = {7203: "cartesian", 9157: "cartesian-3d", 4326: "wgs-84", 4979: "wgs-84-3d"} + + +class Point(NamedTuple): + """A point a query returns. z is None for a 2D point.""" + + srid: int + x: float + y: float + z: Optional[float] = None + + @property + def crs(self) -> str: + return _CRS_NAMES[self.srid] diff --git a/query/AST/FunctionDecls.cpp b/query/AST/FunctionDecls.cpp index 8075eca6d1..ba7cbe4ef5 100644 --- a/query/AST/FunctionDecls.cpp +++ b/query/AST/FunctionDecls.cpp @@ -9,6 +9,7 @@ FunctionDecls::FunctionDecls() { initDefault(); initCollectionFunctions(); + initSpatialFunctions(); } FunctionDecls::~FunctionDecls() { @@ -157,6 +158,11 @@ void FunctionDecls::initDefault() { countDurations->setReturnTypes({{EvaluatedType::Integer}}); countDurations->setIsAggregate(true); + FunctionSignature* countPoints = createFunction("count"); + countPoints->setArguments({EvaluatedType::Point}); + countPoints->setReturnTypes({{EvaluatedType::Integer}}); + countPoints->setIsAggregate(true); + FunctionSignature* countAnyValues = createFunction("count"); countAnyValues->setArguments({EvaluatedType::Any}); countAnyValues->setReturnTypes({{EvaluatedType::Integer}}); @@ -200,6 +206,12 @@ void FunctionDecls::initDefault() { collectDurations->setIsAggregate(true); collectDurations->setCollectsItsArgument(true); + FunctionSignature* collectPoints = createFunction("collect"); + collectPoints->setArguments({EvaluatedType::Point}); + collectPoints->setReturnTypes({{EvaluatedType::List}}); + collectPoints->setIsAggregate(true); + collectPoints->setCollectsItsArgument(true); + FunctionSignature* collectNodes = createFunction("collect"); collectNodes->setArguments({EvaluatedType::NodePattern}); collectNodes->setReturnTypes({{EvaluatedType::List}}); @@ -364,6 +376,17 @@ void FunctionDecls::initDefault() { minDuration->setReturnTypes({{EvaluatedType::Duration}}); minDuration->setIsAggregate(true); + // Points order by SRID, then by coordinates, as ORDER BY sorts them + FunctionSignature* minPoint = createFunction("min"); + minPoint->setArguments({EvaluatedType::Point}); + minPoint->setReturnTypes({{EvaluatedType::Point}}); + minPoint->setIsAggregate(true); + + FunctionSignature* maxPoint = createFunction("max"); + maxPoint->setArguments({EvaluatedType::Point}); + maxPoint->setReturnTypes({{EvaluatedType::Point}}); + maxPoint->setIsAggregate(true); + FunctionSignature* maxDuration = createFunction("max"); maxDuration->setArguments({EvaluatedType::Duration}); maxDuration->setReturnTypes({{EvaluatedType::Duration}}); @@ -433,10 +456,16 @@ void FunctionDecls::initDefault() { avgDuration->setReturnTypes({{EvaluatedType::Duration}}); avgDuration->setIsAggregate(true); + // The centroid of points of one CRS, coordinate by coordinate + FunctionSignature* avgPoint = createFunction("avg"); + avgPoint->setArguments({EvaluatedType::Point}); + avgPoint->setReturnTypes({{EvaluatedType::Point}}); + avgPoint->setIsAggregate(true); + // avg over a type-erased column of tagged cells - what a list mixing numeric types, // holding a null or holding nothing unwinds into. Each cell is read through its tag: - // the mean of numbers is a double and the mean of durations a duration, so the - // average is a cell too. + // the mean of numbers is a double, the mean of durations a duration and the mean of + // points a point, so the average is a cell too. FunctionSignature* avgAnyValues = createFunction("avg"); avgAnyValues->setArguments({EvaluatedType::Any}); avgAnyValues->setReturnTypes({{EvaluatedType::Any}}); @@ -658,7 +687,8 @@ void FunctionDecls::initDefault() { EvaluatedType::Integer, {EvaluatedType::String, EvaluatedType::Integer, EvaluatedType::Double, EvaluatedType::Bool}, {EvaluatedType::List, EvaluatedType::Map, EvaluatedType::NodePattern, EvaluatedType::EdgePattern, - EvaluatedType::GraphPath, EvaluatedType::DateTime, EvaluatedType::Duration, EvaluatedType::Embedding}, + EvaluatedType::GraphPath, EvaluatedType::DateTime, EvaluatedType::Duration, EvaluatedType::Point, + EvaluatedType::Embedding}, }, { "toFloat", @@ -666,7 +696,8 @@ void FunctionDecls::initDefault() { EvaluatedType::Double, {EvaluatedType::String, EvaluatedType::Integer, EvaluatedType::Double}, {EvaluatedType::Bool, EvaluatedType::List, EvaluatedType::Map, EvaluatedType::NodePattern, EvaluatedType::EdgePattern, - EvaluatedType::GraphPath, EvaluatedType::DateTime, EvaluatedType::Duration, EvaluatedType::Embedding}, + EvaluatedType::GraphPath, EvaluatedType::DateTime, EvaluatedType::Duration, EvaluatedType::Point, + EvaluatedType::Embedding}, }, { "toBoolean", @@ -674,14 +705,15 @@ void FunctionDecls::initDefault() { EvaluatedType::Bool, {EvaluatedType::String, EvaluatedType::Integer, EvaluatedType::Bool}, {EvaluatedType::Double, EvaluatedType::List, EvaluatedType::Map, EvaluatedType::NodePattern, EvaluatedType::EdgePattern, - EvaluatedType::GraphPath, EvaluatedType::DateTime, EvaluatedType::Duration, EvaluatedType::Embedding}, + EvaluatedType::GraphPath, EvaluatedType::DateTime, EvaluatedType::Duration, EvaluatedType::Point, + EvaluatedType::Embedding}, }, { "toString", "toStringOrNull", EvaluatedType::String, {EvaluatedType::String, EvaluatedType::Integer, EvaluatedType::Double, EvaluatedType::Bool, - EvaluatedType::DateTime, EvaluatedType::Duration}, + EvaluatedType::DateTime, EvaluatedType::Duration, EvaluatedType::Point}, {EvaluatedType::List, EvaluatedType::Map, EvaluatedType::NodePattern, EvaluatedType::EdgePattern, EvaluatedType::GraphPath, EvaluatedType::Embedding}, }, @@ -743,6 +775,14 @@ void FunctionDecls::initDefault() { durationOfCell->setArguments({EvaluatedType::Any}); durationOfCell->setReturnTypes({{EvaluatedType::Duration}}); + FunctionSignature* pointOfMap = createFunction("point"); + pointOfMap->setArguments({EvaluatedType::Map}); + pointOfMap->setReturnTypes({{EvaluatedType::Point}}); + + FunctionSignature* pointOfCell = createFunction("point"); + pointOfCell->setArguments({EvaluatedType::Any}); + pointOfCell->setReturnTypes({{EvaluatedType::Point}}); + // Numeric functions for (const EvaluatedType numberType : {EvaluatedType::Integer, EvaluatedType::Double, EvaluatedType::Any}) { FunctionSignature* abs = createFunction("abs"); @@ -837,7 +877,7 @@ void FunctionDecls::initDefault() { const std::vector nonString = { EvaluatedType::Integer, EvaluatedType::Double, EvaluatedType::Bool, EvaluatedType::List, EvaluatedType::Map, EvaluatedType::NodePattern, EvaluatedType::EdgePattern, EvaluatedType::GraphPath, - EvaluatedType::DateTime, EvaluatedType::Duration, EvaluatedType::Embedding, + EvaluatedType::DateTime, EvaluatedType::Duration, EvaluatedType::Point, EvaluatedType::Embedding, }; const std::vector normalForm = {EvaluatedType::NormalForm}; @@ -901,6 +941,7 @@ void FunctionDecls::initCollectionFunctions() { EvaluatedType::Bool, EvaluatedType::DateTime, EvaluatedType::Duration, + EvaluatedType::Point, EvaluatedType::Embedding, EvaluatedType::NodePattern, EvaluatedType::EdgePattern, @@ -918,6 +959,26 @@ void FunctionDecls::initCollectionFunctions() { } } +void FunctionDecls::initSpatialFunctions() { + FunctionNamespace* point = _root.createNamespace("point"); + + const PossibleArgTypes pointOrCell {EvaluatedType::Point, EvaluatedType::Any}; + + std::vector distances; + point->createOverloads("distance", {pointOrCell, pointOrCell}, distances); + + for (FunctionSignature* distance : distances) { + distance->setReturnTypes({{EvaluatedType::Double}}); + } + + std::vector boxes; + point->createOverloads("withinBBox", {pointOrCell, pointOrCell, pointOrCell}, boxes); + + for (FunctionSignature* box : boxes) { + box->setReturnTypes({{EvaluatedType::Bool}}); + } +} + FunctionSignature* FunctionDecls::createFunction(std::string_view name) { return _root.createFunction(name); } diff --git a/query/AST/FunctionDecls.h b/query/AST/FunctionDecls.h index 4376b02f02..4a570a7f16 100644 --- a/query/AST/FunctionDecls.h +++ b/query/AST/FunctionDecls.h @@ -33,6 +33,8 @@ class FunctionDecls : public FunctionResolver { void initCollectionFunctions(); + void initSpatialFunctions(); + FunctionSignature* createFunction(std::string_view name); void addAlias(std::string_view alias, std::string_view name); diff --git a/query/AST/LoadJsonlQuery.h b/query/AST/LoadJsonlQuery.h index 56e4f43f87..714f85e969 100644 --- a/query/AST/LoadJsonlQuery.h +++ b/query/AST/LoadJsonlQuery.h @@ -5,9 +5,8 @@ #include "Path.h" #include "QueryCommand.h" -#include "DateTimeSpec.h" -#include "DurationSpec.h" #include "EmbeddingsSpec.h" +#include "TypedPropertySpec.h" namespace db { @@ -23,8 +22,7 @@ class LoadJsonlQuery : public QueryCommand { void setGraphName(std::string_view graphName) { _graphName = graphName; } void setEmbeddingSpecs(EmbeddingsSpec&& specs) { _embeddingSpecs = std::move(specs); } - void setDateTimeSpecs(DateTimeSpec&& specs) { _dateTimeSpecs = std::move(specs); } - void setDurationSpecs(DurationSpec&& specs) { _durationSpecs = std::move(specs); } + void setTypedPropertySpec(TypedPropertySpec&& specs) { _typedPropertySpec = std::move(specs); } const fs::Path& getFilePath() const { return _path; } std::string_view getGraphName() const { return _graphName; } @@ -32,18 +30,13 @@ class LoadJsonlQuery : public QueryCommand { return _embeddingSpecs; } - const DateTimeSpec& getDateTimeSpecs() const { - return _dateTimeSpecs; - } - - const DurationSpec& getDurationSpecs() const { return _durationSpecs; } + const TypedPropertySpec& getTypedPropertySpec() const { return _typedPropertySpec; } private: fs::Path _path; std::string_view _graphName; EmbeddingsSpec _embeddingSpecs; - DateTimeSpec _dateTimeSpecs; - DurationSpec _durationSpecs; + TypedPropertySpec _typedPropertySpec; LoadJsonlQuery(DeclContext* declContext, fs::Path&& path); ~LoadJsonlQuery() override; diff --git a/query/AST/LoadParquetQuery.h b/query/AST/LoadParquetQuery.h index de6b4558fa..a1ad48cd57 100644 --- a/query/AST/LoadParquetQuery.h +++ b/query/AST/LoadParquetQuery.h @@ -4,7 +4,7 @@ #include "QueryCommand.h" -#include "DurationSpec.h" +#include "TypedPropertySpec.h" #include "Path.h" @@ -23,15 +23,15 @@ class LoadParquetQuery : public QueryCommand { std::string_view getGraphName() const { return _graphName; } void setGraphName(std::string_view name) { _graphName = name; } - void setDurationSpecs(DurationSpec&& specs) { _durationSpecs = std::move(specs); } + void setTypedPropertySpec(TypedPropertySpec&& specs) { _typedPropertySpec = std::move(specs); } const fs::Path& getFilePath() const { return _filePath; } - const DurationSpec& getDurationSpecs() const { return _durationSpecs; } + const TypedPropertySpec& getTypedPropertySpec() const { return _typedPropertySpec; } private: fs::Path _filePath; std::string_view _graphName; - DurationSpec _durationSpecs; + TypedPropertySpec _typedPropertySpec; LoadParquetQuery(DeclContext* declContext, fs::Path&& filePath); diff --git a/query/AST/decl/EvaluatedType.h b/query/AST/decl/EvaluatedType.h index 90e622aefd..30cb34c4c1 100644 --- a/query/AST/decl/EvaluatedType.h +++ b/query/AST/decl/EvaluatedType.h @@ -39,6 +39,7 @@ enum class EvaluatedType : uint8_t { EdgeType, DateTime, Duration, + Point, NormalForm, _SIZE, @@ -93,11 +94,12 @@ using EvaluatedTypeName = EnumToString::Create< EnumStringPair, EnumStringPair, EnumStringPair, + EnumStringPair, EnumStringPair>; // Evaluated Type to ValueType conversion static constexpr size_t VALUE_TYPE_COUNT = std::to_underlying(ValueType::_SIZE); -static_assert(VALUE_TYPE_COUNT == 11, "ValueType added: please update below map."); +static_assert(VALUE_TYPE_COUNT == 12, "ValueType added: please update below map."); static constexpr std::pair EVConversions[VALUE_TYPE_COUNT + 1] = { {ValueType::Int64, EvaluatedType::Integer }, @@ -109,6 +111,7 @@ static constexpr std::pair EVConversions[VALUE_TYPE_CO {ValueType::List, EvaluatedType::List }, {ValueType::DateTime, EvaluatedType::DateTime }, {ValueType::Duration, EvaluatedType::Duration }, + {ValueType::Point, EvaluatedType::Point }, {ValueType::Map, EvaluatedType::Map }, {ValueType::Invalid, EvaluatedType::Invalid }, @@ -170,6 +173,7 @@ inline bool namesAComparableValue(EvaluatedType e) { || e == EvaluatedType::List || e == EvaluatedType::DateTime || e == EvaluatedType::Duration + || e == EvaluatedType::Point || e == EvaluatedType::Map; } @@ -277,7 +281,7 @@ inline EvaluatedType commonType(EvaluatedType carried, EvaluatedType value) { } // The types <, <=, > and >= take: every value, entities and paths included, though only -// numbers, strings, booleans, datetimes, durations and lists have an order +// numbers, strings, booleans, datetimes, durations, points and lists have an order inline bool namesAnOrderingOperand(EvaluatedType e) { return namesAnEqualityOperand(e); } @@ -297,6 +301,7 @@ inline std::optional orderClassOf(EvaluatedType e) { case EvaluatedType::Bool: case EvaluatedType::DateTime: case EvaluatedType::Duration: + case EvaluatedType::Point: case EvaluatedType::List: return e; break; diff --git a/query/AST/expr/PropertyExpr.cpp b/query/AST/expr/PropertyExpr.cpp index 44a4b4b257..fb4e69374d 100644 --- a/query/AST/expr/PropertyExpr.cpp +++ b/query/AST/expr/PropertyExpr.cpp @@ -13,16 +13,6 @@ PropertyExpr* PropertyExpr::create(CypherAST* ast, QualifiedName* name) { return expr; } -void PropertyExpr::setDateTimePart(DateTimePart part) { - _dateTimePart = part; - _readsADateTimeComponent = true; -} - -void PropertyExpr::setDurationPart(DurationPart part) { - _durationPart = part; - _readsADurationComponent = true; -} - void PropertyExpr::setMapKey(std::string_view key) { _mapKey = key; _readsAMapKey = true; diff --git a/query/AST/expr/PropertyExpr.h b/query/AST/expr/PropertyExpr.h index 8d7468349f..95bc71fb54 100644 --- a/query/AST/expr/PropertyExpr.h +++ b/query/AST/expr/PropertyExpr.h @@ -1,8 +1,11 @@ #pragma once +#include + #include "Expr.h" #include "metadata/PropertyType.h" +#include "metadata/ValueComponent.h" namespace db { @@ -35,15 +38,9 @@ class PropertyExpr : public Expr { _stringTableHeaderAccess = csvHeaderAccess; } - // A calendar field read off a variable bound to an instant, d.year. The part is - // meaningless unless this says so. - bool readsADateTimeComponent() const { return _readsADateTimeComponent; } - DateTimePart getDateTimePart() const { return _dateTimePart; } - void setDateTimePart(DateTimePart part); - - bool readsADurationComponent() const { return _readsADurationComponent; } - DurationPart getDurationPart() const { return _durationPart; } - void setDurationPart(DurationPart part); + // The component read off a variable bound to an instant, a duration or a point: d.year + const std::optional& getComponent() const { return _component; } + void setComponent(ValueComponent component) { _component = component; } // The key a map variable is read under, m.key. The key is meaningless unless this says so. bool readsAMapKey() const { return _readsAMapKey; } @@ -64,11 +61,8 @@ class PropertyExpr : public Expr { std::string_view _propName; std::string_view _mapKey; ValueType _valueType {ValueType::Invalid}; - DateTimePart _dateTimePart {DateTimePart::Year}; - DurationPart _durationPart {DurationPart::Years}; + std::optional _component; bool _stringTableHeaderAccess {false}; - bool _readsADateTimeComponent {false}; - bool _readsADurationComponent {false}; bool _readsAMapKey {false}; PropertyExpr(QualifiedName* name) diff --git a/query/AST/expr/PropertyLookupExpr.cpp b/query/AST/expr/PropertyLookupExpr.cpp index 2355c01006..8e6a916ee2 100644 --- a/query/AST/expr/PropertyLookupExpr.cpp +++ b/query/AST/expr/PropertyLookupExpr.cpp @@ -20,16 +20,6 @@ PropertyLookupExpr* PropertyLookupExpr::create(CypherAST* ast, Expr* base, std:: return expr; } -void PropertyLookupExpr::setDateTimePart(DateTimePart part) { - _dateTimePart = part; - _readsADateTimeComponent = true; -} - -void PropertyLookupExpr::setDurationPart(DurationPart part) { - _durationPart = part; - _readsADurationComponent = true; -} - void PropertyLookupExpr::setReadsAMapKey() { _readsAMapKey = true; } diff --git a/query/AST/expr/PropertyLookupExpr.h b/query/AST/expr/PropertyLookupExpr.h index 09df10d8e0..c671c56afa 100644 --- a/query/AST/expr/PropertyLookupExpr.h +++ b/query/AST/expr/PropertyLookupExpr.h @@ -1,10 +1,11 @@ #pragma once +#include + #include "Expr.h" -#include "metadata/DateTime.h" -#include "metadata/Duration.h" #include "metadata/PropertyType.h" +#include "metadata/ValueComponent.h" namespace db { @@ -22,13 +23,8 @@ class PropertyLookupExpr : public Expr { ValueType getValueType() const { return _valueType; } void setValueType(ValueType valueType) { _valueType = valueType; } - bool readsADateTimeComponent() const { return _readsADateTimeComponent; } - DateTimePart getDateTimePart() const { return _dateTimePart; } - void setDateTimePart(DateTimePart part); - - bool readsADurationComponent() const { return _readsADurationComponent; } - DurationPart getDurationPart() const { return _durationPart; } - void setDurationPart(DurationPart part); + const std::optional& getComponent() const { return _component; } + void setComponent(ValueComponent component) { _component = component; } bool readsAMapKey() const { return _readsAMapKey; } void setReadsAMapKey(); @@ -37,10 +33,7 @@ class PropertyLookupExpr : public Expr { Expr* _base {nullptr}; std::string_view _propName; ValueType _valueType {ValueType::Invalid}; - DateTimePart _dateTimePart {DateTimePart::Year}; - DurationPart _durationPart {DurationPart::Years}; - bool _readsADateTimeComponent {false}; - bool _readsADurationComponent {false}; + std::optional _component; bool _readsAMapKey {false}; PropertyLookupExpr(Expr* base, std::string_view propName); diff --git a/query/AST/expr/StructuralExpressionComparator.cpp b/query/AST/expr/StructuralExpressionComparator.cpp index d3f5fb2131..611658cf1b 100644 --- a/query/AST/expr/StructuralExpressionComparator.cpp +++ b/query/AST/expr/StructuralExpressionComparator.cpp @@ -118,25 +118,7 @@ bool StructuralExpressionComparator::equal(const Expr* lhs, const Expr* rhs) { // Two components of one value are two values, so the component is part of // what the access reads: d.year and d.day read the same variable - const bool lhsReadsADateTime = lhsProperty->readsADateTimeComponent(); - const bool rhsReadsADateTime = rhsProperty->readsADateTimeComponent(); - const bool lhsReadsADuration = lhsProperty->readsADurationComponent(); - const bool rhsReadsADuration = rhsProperty->readsADurationComponent(); - - const bool neitherReadsAComponent = !lhsReadsADateTime - && !rhsReadsADateTime - && !lhsReadsADuration - && !rhsReadsADuration; - - const bool sameDateTimeComponent = lhsReadsADateTime - && rhsReadsADateTime - && lhsProperty->getDateTimePart() == rhsProperty->getDateTimePart(); - - const bool sameDurationComponent = lhsReadsADuration - && rhsReadsADuration - && lhsProperty->getDurationPart() == rhsProperty->getDurationPart(); - - const bool sameComponent = neitherReadsAComponent || sameDateTimeComponent || sameDurationComponent; + const bool sameComponent = lhsProperty->getComponent() == rhsProperty->getComponent(); // And two keys of one map are two values for the same reason. The key is empty // on every read that is not one, so the comparison stands for all of them. diff --git a/query/analyzer/CypherAnalyzer.cpp b/query/analyzer/CypherAnalyzer.cpp index c82cc3cdad..c769c69713 100644 --- a/query/analyzer/CypherAnalyzer.cpp +++ b/query/analyzer/CypherAnalyzer.cpp @@ -84,6 +84,23 @@ using namespace db; namespace { +std::string_view importedTypeName(ValueType type) { + switch (type) { + case ValueType::DateTime: + return "a datetime"; + break; + case ValueType::Duration: + return "a duration"; + break; + case ValueType::Point: + return "a point"; + break; + default: + return ValueTypeName::value(type); + break; + } +} + const FunctionSignature* aggregateSignatureOf(const Expr* expr) { if (expr->getKind() != Expr::Kind::FUNCTION_INVOCATION) { return nullptr; @@ -1874,13 +1891,19 @@ void CypherAnalyzer::analyze(LoadJsonlQuery* loadJsonl) { throwError("All embedding properties must have dimension greater than 1."); } - const DateTimeSpec& dateTimeSpecs = loadJsonl->getDateTimeSpecs(); - for (const std::string_view name : loadJsonl->getDurationSpecs()) { - if (dateTimeSpecs.contains(name)) { - throwError(fmt::format("Property '{}' is named both a datetime and a duration.", name), - loadJsonl); - } else if (embDims.find(name) != embDims.end()) { - throwError(fmt::format("Property '{}' is named both an embedding and a duration.", name), + const TypedPropertySpec& typedProperties = loadJsonl->getTypedPropertySpec(); + + for (const TypedPropertySpec::Clash& clash : typedProperties.getClashes()) { + throwError(fmt::format("Property '{}' is named both {} and {}.", + clash._name, + importedTypeName(clash._first), + importedTypeName(clash._second)), + loadJsonl); + } + + for (const auto& [name, type] : typedProperties) { + if (embDims.find(name) != embDims.end()) { + throwError(fmt::format("Property '{}' is named both an embedding and {}.", name, importedTypeName(type)), loadJsonl); } } diff --git a/query/analyzer/ExprAnalyzer.cpp b/query/analyzer/ExprAnalyzer.cpp index 5c373695e9..6fcf9ec1d3 100644 --- a/query/analyzer/ExprAnalyzer.cpp +++ b/query/analyzer/ExprAnalyzer.cpp @@ -42,6 +42,8 @@ #include "embedding/EmbeddingBucket.h" #include "metadata/DateTime.h" #include "metadata/Duration.h" +#include "metadata/Point.h" +#include "metadata/ValueComponent.h" #include "StringBucket.h" #include "expr/All.h" @@ -64,6 +66,56 @@ constexpr std::string_view mapKeyNotAProperty = "A map key is a value inside a property, not a property of its own. " "Only a property can be set, removed or indexed."; +constexpr std::string_view POINT_COMPONENT_NOT_A_PROPERTY = + "A point component is computed from a property, not stored as one. " + "Only a property can be set, removed or indexed."; + +bool hasComponents(EvaluatedType type) { + return type == EvaluatedType::DateTime || type == EvaluatedType::Duration || type == EvaluatedType::Point; +} + +template +std::optional componentOf(std::string_view name) { + Part part {}; + if (!Owner::partNamed(name, part)) { + return std::nullopt; + } + + return ValueComponent {part}; +} + +std::optional componentOfAnyType(std::string_view name) { + if (const std::optional dateTimeComponent = componentOf(name)) { + return dateTimeComponent; + } else if (const std::optional durationComponent = componentOf(name)) { + return durationComponent; + } + + return componentOf(name); +} + +EvaluatedType componentType(const ValueComponent& component) { + const PointPart* pointPart = std::get_if(&component); + + if (!pointPart || *pointPart == PointPart::SRID) { + return EvaluatedType::Integer; + } else if (*pointPart == PointPart::CRS) { + return EvaluatedType::String; + } + + return EvaluatedType::Double; +} + +std::string_view componentNotAProperty(const ValueComponent& component) { + if (std::holds_alternative(component)) { + return datetimeComponentNotAProperty; + } else if (std::holds_alternative(component)) { + return "A duration component cannot name a property."; + } + + return POINT_COMPONENT_NOT_A_PROPERTY; +} + // The type a reduce's accumulator takes once it has held @param accumulator and then // @param value: their common type, an integer beside a double widening as the arithmetic // updating it does, and a tagged cell when the two share no column type @@ -579,36 +631,22 @@ ValueType ExprAnalyzer::analyzePropertyExpr(PropertyExpr* expr, bool allowCreate throwTargetInsideAProperty(expr, varDecl); } - // d.year or d.hours, where d was bound to an instant or a duration rather than to an entity - const bool bindsAValueWithComponents = varType == EvaluatedType::DateTime - || varType == EvaluatedType::Duration; - - if (bindsAValueWithComponents) { - const std::string_view name = propName->getName(); - - if (varType == EvaluatedType::DateTime) { - DateTimePart part {DateTimePart::Year}; - if (!DateTime::partNamed(name, part)) { - throwError(fmt::format("'{}' is not a component of a datetime", name), expr); - } - - expr->setDateTimePart(part); - } else { - DurationPart part {DurationPart::Years}; - if (!Duration::partNamed(name, part)) { - throwError(fmt::format("'{}' is not a component of a duration", name), expr); - } + // d.year, d.hours or p.x, where d or p was bound to an instant, a duration or a point + // rather than to an entity + if (hasComponents(varType)) { + ValueComponent component; + resolveComponent(expr, varType, propName->getName(), component); - expr->setDurationPart(part); - } + const EvaluatedType readType = componentType(component); + expr->setComponent(component); expr->setEntityVarDecl(varDecl); - expr->setType(EvaluatedType::Integer); + expr->setType(readType); expr->setDynamic(); - expr->setExprVarDecl(_ctxt->createUnnamedVariable(_ast, EvaluatedType::Integer)); + expr->setExprVarDecl(_ctxt->createUnnamedVariable(_ast, readType)); - return ValueType::Int64; + return *toValueType(readType); } if (varType == EvaluatedType::StringTable) { @@ -750,10 +788,7 @@ void ExprAnalyzer::throwTargetInsideAProperty(const PropertyExpr* expr, const Va const std::string_view propName = qualifiedName->get(1)->getName(); const std::string_view componentName = qualifiedName->get(2)->getName(); - DateTimePart part {DateTimePart::Year}; - DurationPart durationPart {DurationPart::Years}; - const bool namesADateTimePart = DateTime::partNamed(componentName, part); - const bool namesADurationPart = Duration::partNamed(componentName, durationPart); + const std::optional namedComponent = componentOfAnyType(componentName); const EvaluatedType varType = varDecl->getType(); const bool readsAMapVariable = varType == EvaluatedType::Map || varType == EvaluatedType::Any; @@ -761,27 +796,50 @@ void ExprAnalyzer::throwTargetInsideAProperty(const PropertyExpr* expr, const Va if (readsAMapVariable || storesAMap) { throwError(mapKeyNotAProperty, expr); - } else if (namesADateTimePart) { - throwError(datetimeComponentNotAProperty, expr); - } else if (namesADurationPart) { - throwError("A duration component cannot name a property.", expr); + } else if (namedComponent) { + throwError(componentNotAProperty(*namedComponent), expr); } else { - throwError(fmt::format("'{}' is not a component of a datetime or a duration", componentName), expr); + throwError(fmt::format("'{}' is not a component of a datetime, a duration or a point", componentName), expr); } } // A datetime or duration component and a map key are all read out of a property rather than // being one, so none of them can be the property a write or an index targets. void ExprAnalyzer::throwIfReadsPartOfAProperty(const PropertyExpr* expr) { - if (expr->readsADateTimeComponent()) { - throwError(datetimeComponentNotAProperty, expr); - } else if (expr->readsADurationComponent()) { - throwError("A duration component cannot name a property.", expr); + const std::optional& component = expr->getComponent(); + + if (component) { + throwError(componentNotAProperty(*component), expr); } else if (expr->readsAMapKey()) { throwError(mapKeyNotAProperty, expr); } } +void ExprAnalyzer::resolveComponent(const Expr* expr, + EvaluatedType ownerType, + std::string_view name, + ValueComponent& component) const { + std::optional namedComponent; + std::string_view ownerName; + + if (ownerType == EvaluatedType::DateTime) { + namedComponent = componentOf(name); + ownerName = "datetime"; + } else if (ownerType == EvaluatedType::Duration) { + namedComponent = componentOf(name); + ownerName = "duration"; + } else { + namedComponent = componentOf(name); + ownerName = "point"; + } + + if (!namedComponent) { + throwError(fmt::format("'{}' is not a component of a {}", name, ownerName), expr); + } + + component = *namedComponent; +} + void ExprAnalyzer::analyzePropertyLookupExpr(PropertyLookupExpr* expr) { Expr* base = expr->getBase(); analyzeExpr(base); @@ -798,22 +856,12 @@ void ExprAnalyzer::analyzePropertyLookupExpr(PropertyLookupExpr* expr) { EvaluatedType type = EvaluatedType::Null; - if (baseType == EvaluatedType::DateTime) { - DateTimePart part {DateTimePart::Year}; - if (!DateTime::partNamed(propName, part)) { - throwError(fmt::format("'{}' is not a component of a datetime", propName), expr); - } - - expr->setDateTimePart(part); - type = EvaluatedType::Integer; - } else if (baseType == EvaluatedType::Duration) { - DurationPart part {DurationPart::Years}; - if (!Duration::partNamed(propName, part)) { - throwError(fmt::format("'{}' is not a component of a duration", propName), expr); - } + if (hasComponents(baseType)) { + ValueComponent component; + resolveComponent(expr, baseType, propName, component); - expr->setDurationPart(part); - type = EvaluatedType::Integer; + expr->setComponent(component); + type = componentType(component); } else if (readsAnEntity) { const ValueType valueType = resolvePropertyType(propName); @@ -1491,6 +1539,8 @@ bool ExprAnalyzer::propTypeCompatible(ValueType vt, EvaluatedType exprType) { return vt == ValueType::DateTime; case EvaluatedType::Duration: return vt == ValueType::Duration; + case EvaluatedType::Point: + return vt == ValueType::Point; case EvaluatedType::Map: return vt == ValueType::Map; case EvaluatedType::Wildcard: @@ -2020,12 +2070,13 @@ void ExprAnalyzer::analyzeMapProjectionExpr(MapProjectionExpr* expr) { || baseType == EvaluatedType::EdgePattern; const bool projectsAMap = baseType == EvaluatedType::Map || baseType == EvaluatedType::Any; - const bool projectsATemporal = baseType == EvaluatedType::DateTime - || baseType == EvaluatedType::Duration; + const bool projectsComponents = baseType == EvaluatedType::DateTime + || baseType == EvaluatedType::Duration + || baseType == EvaluatedType::Point; const bool projectsNull = baseType == EvaluatedType::Null; - if (!projectsAnEntity && !projectsAMap && !projectsATemporal && !projectsNull) { - throwError(fmt::format("A map projection reads a node, a relationship, a map, a datetime or a duration, " + if (!projectsAnEntity && !projectsAMap && !projectsComponents && !projectsNull) { + throwError(fmt::format("A map projection reads a node, a relationship, a map, a datetime, a duration or a point, " "and '{}' is '{}'", baseSymbol->getName(), EvaluatedTypeName::value(baseType)), @@ -2038,7 +2089,7 @@ void ExprAnalyzer::analyzeMapProjectionExpr(MapProjectionExpr* expr) { const bool readsAProperty = item._kind == MapProjectionExpr::ItemKind::Property; const bool readsEveryProperty = item._kind == MapProjectionExpr::ItemKind::AllProperties; - if (readsEveryProperty && projectsATemporal) { + if (readsEveryProperty && projectsComponents) { throwError(fmt::format("A map projection reads .* off a node, a relationship or a map, and '{}' is '{}'", baseSymbol->getName(), EvaluatedTypeName::value(baseType)), diff --git a/query/analyzer/ExprAnalyzer.h b/query/analyzer/ExprAnalyzer.h index 928c6e3931..5f02ad4dc7 100644 --- a/query/analyzer/ExprAnalyzer.h +++ b/query/analyzer/ExprAnalyzer.h @@ -10,6 +10,7 @@ #include "decl/ExprType.h" #include "expr/CaseExpr.h" #include "metadata/PropertyType.h" +#include "metadata/ValueComponent.h" #include "views/GraphView.h" namespace db { @@ -204,6 +205,13 @@ class ExprAnalyzer { [[noreturn]] void throwTargetInsideAProperty(const PropertyExpr* expr, const VarDecl* varDecl); + // The component @param name names of a value of @param ownerType, an instant, a duration + // or a point. Raises where it names none. + void resolveComponent(const Expr* expr, + EvaluatedType ownerType, + std::string_view name, + ValueComponent& component) const; + LoadCSVStmt* findCSVSource(const VarDecl* alias) const; // The declaration the load publishes field @param slot under, created by the first diff --git a/query/analyzer/WriteStmtAnalyzer.cpp b/query/analyzer/WriteStmtAnalyzer.cpp index f4ed685e25..94ef4ed1f8 100644 --- a/query/analyzer/WriteStmtAnalyzer.cpp +++ b/query/analyzer/WriteStmtAnalyzer.cpp @@ -610,6 +610,8 @@ db::ValueType WriteStmtAnalyzer::evaluatedToValueType(EvaluatedType type) { return ValueType::DateTime; case EvaluatedType::Duration: return ValueType::Duration; + case EvaluatedType::Point: + return ValueType::Point; case EvaluatedType::Map: return ValueType::Map; case EvaluatedType::Null: diff --git a/query/common/ExplainRequest.cpp b/query/common/ExplainRequest.cpp index e964253186..189c7a6948 100644 --- a/query/common/ExplainRequest.cpp +++ b/query/common/ExplainRequest.cpp @@ -1,21 +1,14 @@ #include "ExplainRequest.h" -#include #include #include +#include "StringUtils.h" + using namespace db; namespace { -bool equalsInsensitive(std::string_view left, std::string_view right) { - const auto sameLetter = [](char leftChar, char rightChar) { - return tolower(static_cast(leftChar)) == tolower(static_cast(rightChar)); - }; - - return std::ranges::equal(left, right, sameLetter); -} - struct StageWord { std::string_view word; ExplainStage stage {ExplainStage::AST}; @@ -32,17 +25,17 @@ const std::array stageWords {{ bool isStageWord(std::string_view word) { return std::ranges::any_of(stageWords, [word](const StageWord& stageWord) { - return equalsInsensitive(stageWord.word, word); + return StringUtils::equalsIgnoringCase(stageWord.word, word); }); } bool isPassSelector(std::string_view word) { - return equalsInsensitive(word, "around") || equalsInsensitive(word, "before") || equalsInsensitive(word, "after"); + return StringUtils::equalsIgnoringCase(word, "around") || StringUtils::equalsIgnoringCase(word, "before") || StringUtils::equalsIgnoringCase(word, "after"); } bool contains(const std::vector& passNames, std::string_view passName) { return std::ranges::any_of(passNames, [passName](std::string_view name) { - return equalsInsensitive(name, passName); + return StringUtils::equalsIgnoringCase(name, passName); }); } @@ -67,7 +60,7 @@ void ExplainRequest::requestAll() { bool ExplainRequest::requestStage(std::string_view word) { for (const StageWord& stageWord : stageWords) { - if (equalsInsensitive(stageWord.word, word)) { + if (StringUtils::equalsIgnoringCase(stageWord.word, word)) { request(stageWord.stage); return true; } @@ -77,9 +70,9 @@ bool ExplainRequest::requestStage(std::string_view word) { } bool ExplainRequest::requestPass(std::string_view selector, std::string_view passName) { - const bool isAround = equalsInsensitive(selector, "around"); - const bool printsBefore = isAround || equalsInsensitive(selector, "before"); - const bool printsAfter = isAround || equalsInsensitive(selector, "after"); + const bool isAround = StringUtils::equalsIgnoringCase(selector, "around"); + const bool printsBefore = isAround || StringUtils::equalsIgnoringCase(selector, "before"); + const bool printsAfter = isAround || StringUtils::equalsIgnoringCase(selector, "after"); if (!printsBefore && !printsAfter) { return false; @@ -112,7 +105,7 @@ std::string_view ExplainRequest::findUnknownPass(std::span* named : {&_passesBefore, &_passesAfter}) { for (const std::string_view passName : *named) { const bool isInPipeline = std::ranges::any_of(pipelinePasses, [passName](std::string_view name) { - return equalsInsensitive(name, passName); + return StringUtils::equalsIgnoringCase(name, passName); }); if (!isInPipeline) { diff --git a/query/ir/codegen/DBProgramGenerator.cpp b/query/ir/codegen/DBProgramGenerator.cpp index 42e681332b..f5003c2ab5 100644 --- a/query/ir/codegen/DBProgramGenerator.cpp +++ b/query/ir/codegen/DBProgramGenerator.cpp @@ -513,6 +513,7 @@ const std::unordered_map unaryFunctionEm {"toBooleanOrNull", &emitUnaryFunction}, {"datetime", &emitUnaryFunction}, {"duration", &emitUnaryFunction}, + {"point", &emitUnaryFunction}, {"size", &emitUnaryFunction}, {"length", &emitUnaryFunction}, {"isEmpty", &emitUnaryFunction}, @@ -9850,13 +9851,12 @@ mlir::Type DBProgramGenerator::propertyValueType(ValueType valueType) { mlir::Value DBProgramGenerator::translatePropertyExpr(const PropertyExpr* propExpr) { const mlir::Value value = translatePropertyRead(propExpr); + const std::optional& component = propExpr->getComponent(); if (propExpr->readsAMapKey()) { return emitStaticMapKey(value, propExpr->getMapKey()); - } else if (propExpr->readsADateTimeComponent()) { - return emitDateTimeComponent(propExpr->getDateTimePart(), value); - } else if (propExpr->readsADurationComponent()) { - return emitDurationComponent(propExpr->getDurationPart(), value); + } else if (component) { + return emitComponent(*component, value); } return value; @@ -9874,15 +9874,14 @@ mlir::Value DBProgramGenerator::translatePropertyLookupExpr(const PropertyLookup bioassert(_part._exprMap.contains(base), "Property lookup with unknown base."); const mlir::Value baseColumn = _part._exprMap.at(base); + const std::optional& component = lookupExpr->getComponent(); if (lookupExpr->readsAMapKey()) { return emitStaticMapKey(baseColumn, lookupExpr->getPropName()); } - if (lookupExpr->readsADateTimeComponent()) { - return emitDateTimeComponent(lookupExpr->getDateTimePart(), baseColumn); - } else if (lookupExpr->readsADurationComponent()) { - return emitDurationComponent(lookupExpr->getDurationPart(), baseColumn); + if (component) { + return emitComponent(*component, baseColumn); } else if (readType == EvaluatedType::Any) { return emitStaticMapKey(baseColumn, lookupExpr->getPropName()); } @@ -9925,7 +9924,11 @@ mlir::Value DBProgramGenerator::emitDynamicMapKey(mlir::Value map, mlir::Value k return _opBuilder.create(loc, cellType, map, key).getResult(); } -mlir::Value DBProgramGenerator::emitDateTimeComponent(DateTimePart part, mlir::Value instant) { +mlir::Value DBProgramGenerator::emitComponent(const ValueComponent& component, mlir::Value value) { + return std::visit([this, value](auto part) { return emitComponent(part, value); }, component); +} + +mlir::Value DBProgramGenerator::emitComponent(DateTimePart part, mlir::Value instant) { const mlir::Location loc = _opBuilder.getUnknownLoc(); const mlir::db::ColumnType noneType = allocColumnType(mlir::NoneType::get(_mlirCtxt)); @@ -9966,7 +9969,48 @@ mlir::Value DBProgramGenerator::emitDateTimeComponent(DateTimePart part, mlir::V return mlir::Value(); } -mlir::Value DBProgramGenerator::emitDurationComponent(DurationPart part, mlir::Value duration) { +mlir::Value DBProgramGenerator::emitComponent(PointPart part, mlir::Value point) { + const mlir::Location loc = _opBuilder.getUnknownLoc(); + const mlir::db::ColumnType noneType = allocColumnType(mlir::NoneType::get(_mlirCtxt)); + + switch (part) { + case PointPart::X: + return _opBuilder.create(loc, noneType, point).getResult(); + break; + + case PointPart::Y: + return _opBuilder.create(loc, noneType, point).getResult(); + break; + + case PointPart::Z: + return _opBuilder.create(loc, noneType, point).getResult(); + break; + + case PointPart::Longitude: + return _opBuilder.create(loc, noneType, point).getResult(); + break; + + case PointPart::Latitude: + return _opBuilder.create(loc, noneType, point).getResult(); + break; + + case PointPart::Height: + return _opBuilder.create(loc, noneType, point).getResult(); + break; + + case PointPart::CRS: + return _opBuilder.create(loc, noneType, point).getResult(); + break; + + case PointPart::SRID: + return _opBuilder.create(loc, noneType, point).getResult(); + break; + } + + return mlir::Value(); +} + +mlir::Value DBProgramGenerator::emitComponent(DurationPart part, mlir::Value duration) { const mlir::Location loc = _opBuilder.getUnknownLoc(); const mlir::db::ColumnType noneType = allocColumnType(mlir::NoneType::get(_mlirCtxt)); @@ -10052,12 +10096,13 @@ mlir::Value DBProgramGenerator::translatePropertyRead(const PropertyExpr* propEx const std::string_view varName = entityDecl->getName(); const std::string_view propName = propExpr->getPropName(); - // d.year, d.hours and m.key name no property: the value the component or the key is - // read off is the column the variable itself was bound to + // d.year, d.hours, p.x and m.key name no property: the value the component or the key + // is read off is the column the variable itself was bound to const EvaluatedType entityType = entityDecl->getType(); const bool readsOffTheVariableItself = entityType == EvaluatedType::DateTime || entityType == EvaluatedType::Duration + || entityType == EvaluatedType::Point || entityType == EvaluatedType::Map || entityType == EvaluatedType::Any; @@ -10396,6 +10441,12 @@ void DBProgramGenerator::translateFunctionExpr(const Expr* expr, } else if (funcName == "replace") { translateReplace(expr, args); return; + } else if (funcName == "point.distance") { + translatePointDistance(expr, args); + return; + } else if (funcName == "point.withinBBox") { + translatePointWithinBBox(expr, args); + return; } else if (funcName == "normalize") { translateNormalFormFunction(expr, args, mlir::storage::StringType::get(_mlirCtxt)); return; @@ -10488,6 +10539,42 @@ void DBProgramGenerator::translateFunctionExpr(const Expr* expr, throwError(fmt::format("Unsupported function: {}", funcName), expr); } +void DBProgramGenerator::translatePointDistance(const Expr* expr, const ExprChain* args) { + bioassert(args && args->size() == 2, "point.distance() takes 2 arguments."); + + const ExprChain::ExprVector& argExprs = args->getExprs(); + + const mlir::Value lhs = translateArg(argExprs[0]); + const mlir::Value rhs = translateArg(argExprs[1]); + + mlir::db::PointDistance distance = _opBuilder.create(_opBuilder.getUnknownLoc(), + allocColumnType(_opBuilder.getF64Type()), + lhs, + rhs); + + _part._exprMap[expr] = distance.getResult(); +} + +void DBProgramGenerator::translatePointWithinBBox(const Expr* expr, const ExprChain* args) { + bioassert(args && args->size() == 3, "point.withinBBox() takes 3 arguments."); + + const ExprChain::ExprVector& argExprs = args->getExprs(); + + const mlir::Value point = translateArg(argExprs[0]); + const mlir::Value lowerLeft = translateArg(argExprs[1]); + const mlir::Value upperRight = translateArg(argExprs[2]); + + const mlir::Type boolType = mlir::storage::BoolType::get(_mlirCtxt); + + mlir::db::PointWithinBBox within = _opBuilder.create(_opBuilder.getUnknownLoc(), + allocColumnType(boolType), + point, + lowerLeft, + upperRight); + + _part._exprMap[expr] = within.getResult(); +} + void DBProgramGenerator::translateRange(const Expr* expr, const ExprChain* args) { const size_t argCount = args ? args->size() : 0; const bool boundsGiven = argCount == 2 || argCount == 3; diff --git a/query/ir/codegen/DBProgramGenerator.h b/query/ir/codegen/DBProgramGenerator.h index 5ccfffd6de..209100f1e1 100644 --- a/query/ir/codegen/DBProgramGenerator.h +++ b/query/ir/codegen/DBProgramGenerator.h @@ -23,6 +23,8 @@ #include "expr/CaseExpr.h" #include "stmt/SetItem.h" #include "metadata/DateTime.h" +#include "metadata/Point.h" +#include "metadata/ValueComponent.h" #include "ExplainRequest.h" #include "InsertionOrderedMap.h" @@ -1391,6 +1393,8 @@ class DBProgramGenerator { void translateSubstring(const Expr* expr, const ExprChain* args); void translateSplit(const Expr* expr, const ExprChain* args); void translateReplace(const Expr* expr, const ExprChain* args); + void translatePointDistance(const Expr* expr, const ExprChain* args); + void translatePointWithinBBox(const Expr* expr, const ExprChain* args); template void translateStringEnd(const Expr* expr, const ExprChain* args); @@ -1411,8 +1415,10 @@ class DBProgramGenerator { mlir::Value translatePropertyLookupExpr(const PropertyLookupExpr* lookupExpr); mlir::Value emitStaticMapKey(mlir::Value map, std::string_view key); mlir::Value emitDynamicMapKey(mlir::Value map, mlir::Value key); - mlir::Value emitDateTimeComponent(DateTimePart part, mlir::Value instant); - mlir::Value emitDurationComponent(DurationPart part, mlir::Value duration); + mlir::Value emitComponent(const ValueComponent& component, mlir::Value value); + mlir::Value emitComponent(DateTimePart part, mlir::Value instant); + mlir::Value emitComponent(DurationPart part, mlir::Value duration); + mlir::Value emitComponent(PointPart part, mlir::Value point); mlir::Value translateEntityTypeExpr(const EntityTypeExpr* typeExpr); // labels() / type() of an entity the CREATE wrote, as the constant string it reads: diff --git a/query/ir/codegen/DBSystemProgramGenerator.cpp b/query/ir/codegen/DBSystemProgramGenerator.cpp index 7eb6bfc55e..45f08de5a7 100644 --- a/query/ir/codegen/DBSystemProgramGenerator.cpp +++ b/query/ir/codegen/DBSystemProgramGenerator.cpp @@ -7,6 +7,7 @@ #include "mlir/IR/MLIRContext.h" #include "DBOps.h" +#include "IRValueTypes.h" #include "StorageEnums.h" #include "StorageTypes.h" @@ -28,6 +29,7 @@ #include "QueryCommand.h" #include "S3ConnectQuery.h" #include "S3TransferQuery.h" +#include "TypedPropertySpec.h" #include "expr/PropertyExpr.h" #include "BioAssert.h" @@ -40,18 +42,17 @@ llvm::StringRef toStringRef(std::string_view text) { return llvm::StringRef(text.data(), text.size()); } -// A named property carries no value of its own, so each name is a unit entry. A dictionary -// rather than an array because it sorts and uniques its own names, which the hash set the -// spec holds does neither. -template -mlir::DictionaryAttr propertyNamesAttribute(mlir::OpBuilder* builder, const PropertyNameSpec& names) { - if (names.empty()) { +// Each named property maps to the element type it is read as. A dictionary rather than an +// array because it sorts its names itself, which the hash map the spec holds does not. +mlir::DictionaryAttr typedPropertiesAttribute(mlir::OpBuilder* builder, const TypedPropertySpec& specs) { + if (specs.empty()) { return mlir::DictionaryAttr(); } llvm::SmallVector entries; - for (const std::string_view propertyName : names) { - entries.emplace_back(builder->getStringAttr(toStringRef(propertyName)), builder->getUnitAttr()); + for (const auto& [propertyName, type] : specs) { + const mlir::Type elementType = valueTypeToElementType(*builder, type); + entries.emplace_back(builder->getStringAttr(toStringRef(propertyName)), mlir::TypeAttr::get(elementType)); } return builder->getDictionaryAttr(entries); @@ -323,8 +324,7 @@ void DBSystemProgramGenerator::generateLoadJsonl(const LoadJsonlQuery* query) { query->getGraphName(), mlir::storage::GraphImportFormat::Jsonl, embeddings, - propertyNamesAttribute(_opBuilder, query->getDateTimeSpecs()), - propertyNamesAttribute(_opBuilder, query->getDurationSpecs())); + typedPropertiesAttribute(_opBuilder, query->getTypedPropertySpec())); } void DBSystemProgramGenerator::generateLoadGML(const LoadGMLQuery* query) { @@ -339,24 +339,21 @@ void DBSystemProgramGenerator::generateLoadParquet(const LoadParquetQuery* query query->getGraphName(), mlir::storage::GraphImportFormat::Parquet, mlir::DictionaryAttr(), - mlir::DictionaryAttr(), - propertyNamesAttribute(_opBuilder, query->getDurationSpecs())); + typedPropertiesAttribute(_opBuilder, query->getTypedPropertySpec())); } void DBSystemProgramGenerator::generateImportGraph(std::string_view path, std::string_view graphName, mlir::storage::GraphImportFormat format, mlir::DictionaryAttr embeddings, - mlir::DictionaryAttr dateTimes, - mlir::DictionaryAttr durations) { + mlir::DictionaryAttr types) { mlir::db::ImportGraph op = _opBuilder->create(_opBuilder->getUnknownLoc(), stringColumnType(), toStringRef(path), toStringRef(graphName), format, embeddings, - dateTimes, - durations); + types); emitOutput(op.getGraph(), {"graphName"}); } diff --git a/query/ir/codegen/DBSystemProgramGenerator.h b/query/ir/codegen/DBSystemProgramGenerator.h index 02d0657302..5c23e86863 100644 --- a/query/ir/codegen/DBSystemProgramGenerator.h +++ b/query/ir/codegen/DBSystemProgramGenerator.h @@ -95,8 +95,7 @@ class DBSystemProgramGenerator { std::string_view graphName, mlir::storage::GraphImportFormat format, mlir::DictionaryAttr embeddings, - mlir::DictionaryAttr dateTimes = {}, - mlir::DictionaryAttr durations = {}); + mlir::DictionaryAttr types = {}); void generatePropertyIndex(std::string_view indexName, const PropertyExpr* propertyExpr, diff --git a/query/ir/common/IRValueTypes.cpp b/query/ir/common/IRValueTypes.cpp index 35ed0508da..dccd691cc2 100644 --- a/query/ir/common/IRValueTypes.cpp +++ b/query/ir/common/IRValueTypes.cpp @@ -3,6 +3,7 @@ #include "mlir/IR/BuiltinAttributes.h" #include "mlir/IR/Builders.h" +#include "StorageEnums.h" #include "StorageTypes.h" #include "IRException.h" @@ -68,6 +69,10 @@ mlir::Type db::valueTypeToElementType(mlir::OpBuilder& builder, ValueType valueT return mlir::storage::DurationType::get(builder.getContext()); break; + case ValueType::Point: + return mlir::storage::PointType::get(builder.getContext()); + break; + case ValueType::Map: return mlir::storage::MapType::get(builder.getContext()); break; @@ -92,6 +97,8 @@ ValueType db::elementTypeToValueType(mlir::Type elementType) { return ValueType::DateTime; } else if (mlir::isa(elementType)) { return ValueType::Duration; + } else if (mlir::isa(elementType)) { + return ValueType::Point; } else if (mlir::isa(elementType)) { return ValueType::Map; } else if (mlir::isa(elementType)) { @@ -126,6 +133,30 @@ bool db::namesPropertyElementTypes(mlir::DictionaryAttr types) { return true; } +bool db::namesImportedPropertyTypes(mlir::storage::GraphImportFormat format, mlir::DictionaryAttr types) { + if (!types) { + return true; + } + + for (const mlir::NamedAttribute entry : types) { + const mlir::TypeAttr typeAttr = mlir::dyn_cast(entry.getValue()); + if (!typeAttr) { + return false; + } + + const ValueType type = elementTypeToValueType(typeAttr.getValue()); + const bool readsJsonlType = format == mlir::storage::GraphImportFormat::Jsonl + && (type == ValueType::DateTime || type == ValueType::Duration || type == ValueType::Point); + const bool readsParquetType = format == mlir::storage::GraphImportFormat::Parquet && type == ValueType::Duration; + + if (!readsJsonlType && !readsParquetType) { + return false; + } + } + + return true; +} + mlir::Type db::unwoundElementType(mlir::MLIRContext* context, mlir::Type sourceElement) { // A cell that may be absent - what an index into a list hands back - contributes no // row where it is absent, so what the drain hands on is the tagged scalar itself. diff --git a/query/ir/common/IRValueTypes.h b/query/ir/common/IRValueTypes.h index db0efa95dc..b87890018f 100644 --- a/query/ir/common/IRValueTypes.h +++ b/query/ir/common/IRValueTypes.h @@ -1,5 +1,7 @@ #pragma once +#include + #include "metadata/PropertyType.h" namespace mlir { @@ -9,6 +11,10 @@ class OpBuilder; class Type; } +namespace mlir::storage { +enum class GraphImportFormat : uint64_t; +} + namespace db { // Map a stored property value type to the MLIR element type baked into the nullable value @@ -24,6 +30,11 @@ ValueType elementTypeToValueType(mlir::Type elementType); // property value column is laid out for bool namesPropertyElementTypes(mlir::DictionaryAttr types); +// Whether each entry of @param types, absent or not, is a TypeAttr of a type a @param format +// import reads a property as where the file spells none: a datetime, duration or point from +// JSONL, a duration from Parquet +bool namesImportedPropertyTypes(mlir::storage::GraphImportFormat format, mlir::DictionaryAttr types); + // The element type an unwind of @param sourceElement produces: a drained list gives up the // type its own elements carry, falling back to the type-erased !storage.any when they share // none, and any other source keeps its own element, its cells being the elements themselves diff --git a/query/ir/dialect/db/DBColumnElement.cpp b/query/ir/dialect/db/DBColumnElement.cpp index 2a8bad1e1c..190fae3061 100644 --- a/query/ir/dialect/db/DBColumnElement.cpp +++ b/query/ir/dialect/db/DBColumnElement.cpp @@ -72,6 +72,10 @@ mlir::Type durationElement(mlir::MLIRContext* context, mlir::Type inputElement) return storage::DurationType::get(context); } +mlir::Type pointElement(mlir::MLIRContext* context, mlir::Type inputElement) { + return storage::PointType::get(context); +} + // A list function reads a list cell, or the type-erased cell an unwind of a list of lists // hands its nested lists on as bool readsAList(mlir::Type inputElement) { @@ -142,6 +146,7 @@ const std::unordered_map elementRules = { {"db.to_boolean", &booleanElement}, {"db.to_datetime", &dateTimeElement}, {"db.to_duration", &durationElement}, + {"db.to_point", &pointElement}, {"db.datetime_year", &integerElement}, {"db.datetime_month", &integerElement}, @@ -171,6 +176,15 @@ const std::unordered_map elementRules = { {"db.duration_milliseconds_of_second", &integerElement}, {"db.duration_microseconds_of_second", &integerElement}, + {"db.point_x", &floatElement}, + {"db.point_y", &floatElement}, + {"db.point_z", &floatElement}, + {"db.point_longitude", &floatElement}, + {"db.point_latitude", &floatElement}, + {"db.point_height", &floatElement}, + {"db.point_crs", &stringElement}, + {"db.point_srid", &integerElement}, + {"db.to_string", &stringElement}, {"db.to_integer_or_null", &integerElement}, {"db.to_float_or_null", &floatElement}, diff --git a/query/ir/dialect/db/DBHeldTags.cpp b/query/ir/dialect/db/DBHeldTags.cpp index e7ef8ad024..9eaa9462b5 100644 --- a/query/ir/dialect/db/DBHeldTags.cpp +++ b/query/ir/dialect/db/DBHeldTags.cpp @@ -61,6 +61,8 @@ CellTag elementTag(mlir::Type element) { return CellTag::DateTime; } else if (mlir::isa(element)) { return CellTag::Duration; + } else if (mlir::isa(element)) { + return CellTag::Point; } else if (mlir::isa(element)) { return CellTag::Embedding; } diff --git a/query/ir/dialect/db/DBOps.cpp b/query/ir/dialect/db/DBOps.cpp index 8be2005ed8..86efd03c13 100644 --- a/query/ir/dialect/db/DBOps.cpp +++ b/query/ir/dialect/db/DBOps.cpp @@ -74,6 +74,14 @@ Speculation::Speculatability speculatableOverNumbers(Operation* op) { return speculatableWhen(llvm::all_of(op->getOperands(), isNumberColumn)); } +Speculation::Speculatability speculatableWhenReadingPoints(Operation* op) { + const bool readsPoints = llvm::all_of(op->getOperands(), [](Value operand) { + return isa(columnElement(operand)); + }); + + return speculatableWhen(readsPoints); +} + // A tagged cell, or a column whose type lowering resolves, may hold a value of any type. The // null literal is a nullable none. bool mayHoldAnyType(Value operand) { @@ -265,7 +273,10 @@ bool isMaskColumn(Type type) { return isa(value) || isa(value) || value.isInteger(1); } -bool isStringColumn(Type type) { +// Whether @param type is a column of @param Element, nullable or not, or of a cell typed at +// run time +template +bool isColumnOf(Type type) { const auto column = dyn_cast(type); if (!column) { return false; @@ -275,7 +286,26 @@ bool isStringColumn(Type type) { const auto nullable = dyn_cast(element); const Type value = nullable ? nullable.getValueType() : element; - return isa(value); + return isa(value); +} + +// A point function reads columns of points into a column of @param Result +template +LogicalResult verifyPointFunction(Operation* op, Value result, StringRef resultName) { + const bool readsPoints = llvm::all_of(op->getOperands(), [](Value operand) { + return isColumnOf(operand.getType()); + }); + + if (!readsPoints) { + return op->emitOpError("operands must be point columns"); + } + + const ColumnType resultColumn = dyn_cast(result.getType()); + if (!resultColumn || !isa(resultColumn.getType())) { + return op->emitOpError("result must be a column of ") << resultName; + } + + return success(); } bool isIntegerColumn(Value column) { @@ -293,7 +323,7 @@ LogicalResult verifyStringResult(Operation* op, Value result) { } LogicalResult verifyStringEnd(Operation* op, Value string, Value length, Value result) { - const bool readsAStringAndALength = isStringColumn(string.getType()) && isIntegerColumn(length); + const bool readsAStringAndALength = isColumnOf(string.getType()) && isIntegerColumn(length); if (!readsAStringAndALength) { return op->emitOpError("operands must be a string column and an integer column"); } @@ -1608,6 +1638,14 @@ LogicalResult SetEdgeProperties::verify() { return success(); } +LogicalResult ImportGraph::verify() { + if (!::db::namesImportedPropertyTypes(getFormat(), getTypesAttr())) { + return emitOpError("types each property as a datetime, duration or point a jsonl import reads, or a duration a parquet import reads"); + } + + return success(); +} + // A label scan must name at least one label to filter by; a label-free scan of // every node is db.scan_nodes, so an empty label list is malformed IR here. LogicalResult ScanNodesByLabel::verify() { @@ -2462,7 +2500,7 @@ LogicalResult Range::verify() { LogicalResult Replace::verify() { const bool readsStrings = llvm::all_of(getOperation()->getOperands(), [](Value operand) { - return isStringColumn(operand.getType()); + return isColumnOf(operand.getType()); }); if (!readsStrings) { @@ -2472,11 +2510,19 @@ LogicalResult Replace::verify() { return verifyStringResult(getOperation(), getResult()); } +LogicalResult PointDistance::verify() { + return verifyPointFunction(getOperation(), getResult(), "f64"); +} + +LogicalResult PointWithinBBox::verify() { + return verifyPointFunction(getOperation(), getResult(), "booleans"); +} + LogicalResult Split::verify() { const Value delimiter = getDelimiter(); - const bool readsAString = isStringColumn(getString().getType()); - const bool readsDelimiters = isStringColumn(delimiter.getType()) || isa(columnElement(delimiter)); + const bool readsAString = isColumnOf(getString().getType()); + const bool readsDelimiters = isColumnOf(delimiter.getType()) || isa(columnElement(delimiter)); if (!readsAString || !readsDelimiters) { return emitOpError("operands must be a string column, then a string or list column"); } @@ -3188,6 +3234,14 @@ Speculation::Speculatability Replace::getSpeculatability() { return speculatableWhen(readsStrings); } +Speculation::Speculatability PointDistance::getSpeculatability() { + return speculatableWhenReadingPoints(getOperation()); +} + +Speculation::Speculatability PointWithinBBox::getSpeculatability() { + return speculatableWhenReadingPoints(getOperation()); +} + Speculation::Speculatability Split::getSpeculatability() { const bool readsAString = isa(columnElement(getString())); diff --git a/query/ir/dialect/db/DBOps.td b/query/ir/dialect/db/DBOps.td index 528eaacecf..d409526175 100644 --- a/query/ir/dialect/db/DBOps.td +++ b/query/ir/dialect/db/DBOps.td @@ -3005,6 +3005,10 @@ def ToFloat : ConversionFunctionOp<"to_float">; def ToBoolean : ConversionFunctionOp<"to_boolean">; def ToDateTime : TypedUnaryFunctionOp<"to_datetime", "::mlir::storage::StringType">; def ToDuration : TypedUnaryFunctionOp<"to_duration", "::mlir::IntegerType">; + +// Any map can name an invalid point - a latitude past 90, an unknown crs - so it is never +// speculatable +def ToPoint : UnaryFunctionOp<"to_point", [MayRaise]>; def ToString : ConversionFunctionOp<"to_string">; def ToIntegerOrNull : UnaryFunctionOp<"to_integer_or_null", [Pure]>; @@ -3044,6 +3048,18 @@ def DurationSecondsOfMinute : TypedUnaryFunctionOp<"duration_seconds_of_minute", def DurationMillisecondsOfSecond : TypedUnaryFunctionOp<"duration_milliseconds_of_second", "::mlir::storage::DurationType">; def DurationMicrosecondsOfSecond : TypedUnaryFunctionOp<"duration_microseconds_of_second", "::mlir::storage::DurationType">; +// The fields of a point: its coordinates as float columns, the name of its CRS as a string +// column and its SRID as an integer column. A 2D point has no z or height and a cartesian +// one no longitude, latitude or height, so those four raise over a column of points too. +def PointX : TypedUnaryFunctionOp<"point_x", "::mlir::storage::PointType">; +def PointY : TypedUnaryFunctionOp<"point_y", "::mlir::storage::PointType">; +def PointZ : UnaryFunctionOp<"point_z", [MayRaise]>; +def PointLongitude : UnaryFunctionOp<"point_longitude", [MayRaise]>; +def PointLatitude : UnaryFunctionOp<"point_latitude", [MayRaise]>; +def PointHeight : UnaryFunctionOp<"point_height", [MayRaise]>; +def PointCRSName : TypedUnaryFunctionOp<"point_crs", "::mlir::storage::PointType">; +def PointSRID : TypedUnaryFunctionOp<"point_srid", "::mlir::storage::PointType">; + def CurrentDateTime : TuringOp<"current_datetime", [Pure, ConstantThroughOperands]> { let summary = "The instant the query runs, as a constant column"; let description = [{ @@ -3204,6 +3220,47 @@ def Replace : TuringOp<"replace", [MayRaise, ConstantThroughOperands, DeclareOpI }]; } +def PointDistance : TuringOp<"point_distance", [MayRaise, ConstantThroughOperands, DeclareOpInterfaceMethods]> { + let summary = "The distance between the points of each row, null for two points in different CRSs"; + + let description = [{ + Euclidean between two cartesian points, and great-circle metres between two + geographic ones. It raises on a tagged cell holding no point. + + %d = db.point_distance(%a, %b) : (!db.column, !db.column) + -> !db.column + }]; + + let arguments = (ins Column:$lhs, Column:$rhs); + let results = (outs Column:$result); + let hasVerifier = 1; + + let assemblyFormat = [{ + `(` $lhs `,` $rhs `)` attr-dict `:` functional-type(operands, results) + }]; +} + +def PointWithinBBox : TuringOp<"point_within_bbox", [MayRaise, ConstantThroughOperands, DeclareOpInterfaceMethods]> { + let summary = "Whether each row's point lies in its box, null unless the three share a CRS"; + + let description = [{ + The box is bounded by its lower left and upper right corners, both included. It + raises on a tagged cell holding no point. + + %in = db.point_within_bbox(%p, %lowerLeft, %upperRight) + : (!db.column, !db.column, !db.column) + -> !db.column + }]; + + let arguments = (ins Column:$point, Column:$lowerLeft, Column:$upperRight); + let results = (outs Column:$result); + let hasVerifier = 1; + + let assemblyFormat = [{ + `(` $point `,` $lowerLeft `,` $upperRight `)` attr-dict `:` functional-type(operands, results) + }]; +} + class BinaryFunctionOp : TuringOp { let summary = "Apply a binary function row-wise to two value columns"; diff --git a/query/ir/dialect/db/DBSystemOps.td b/query/ir/dialect/db/DBSystemOps.td index aec9ff49e9..9a90299b39 100644 --- a/query/ir/dialect/db/DBSystemOps.td +++ b/query/ir/dialect/db/DBSystemOps.td @@ -74,32 +74,31 @@ def ImportGraph : SystemOp<"import_graph"> { `LOAD JSONL`: one entry per property that holds a vector, mapping the property name to its dimension. The other two formats never carry it. - `date_times` is its `WITH DATETIMES [prop, ...]` clause: the properties whose - strings name an instant rather than text. A datetime is spelled as a string - in JSON, so the query says which names carry one. The names carry no value of - their own, so each is a unit entry - a dictionary rather than an array because - that sorts and uniques its names itself. - %g = db.import_graph jsonl("/data/social.jsonl") as "social" embeddings {vec = 128 : ui64} : !db.column - `durations` is the `WITH DURATIONS [prop, ...]` clause of `LOAD JSONL` and - `LOAD PARQUET`: the properties whose integers count microseconds. Neither - format has a duration type. Each name is a unit entry, as in `date_times`. + `types` maps each property the query names in a `WITH DATETIMES`, `WITH + DURATIONS` or `WITH POINTS` clause to the element type it is read as: a type + the format does not spell. A datetime is a string in JSON, a duration an + integer count of microseconds in JSON and Parquet, a point an object in JSON. + + %g = db.import_graph jsonl("/data/cities.jsonl") as "cities" + types {at = !storage.point, founded = !storage.datetime} : !db.column }]; let arguments = (ins StrAttr:$path, StrAttr:$graph_name, GraphImportFormat:$format, OptionalAttr:$embeddings, - OptionalAttr:$date_times, - OptionalAttr:$durations); + OptionalAttr:$types); let results = (outs ColumnStrings:$graph); let assemblyFormat = [{ $format `(` $path `)` `as` $graph_name (`embeddings` $embeddings^)? - (`datetimes` $date_times^)? (`durations` $durations^)? attr-dict `:` qualified(type($graph)) + (`types` $types^)? attr-dict `:` qualified(type($graph)) }]; + + let hasVerifier = 1; } /** diff --git a/query/ir/dialect/nl/NLOps.cpp b/query/ir/dialect/nl/NLOps.cpp index 156b8871d2..77e43161c5 100644 --- a/query/ir/dialect/nl/NLOps.cpp +++ b/query/ir/dialect/nl/NLOps.cpp @@ -1403,7 +1403,7 @@ LogicalResult AggregateUpdate::verify() { const Type inputValueType = untypedNull ? IntegerType::get(getContext(), 64) : declaredValueType; if (!foldsIntoDouble && inputValueType != accumulatorType) { - return emitOpError("sum/min/max and avg of durations must fold into an accumulator of the input's value type"); + return emitOpError("sum/min/max and avg of durations or points must fold into an accumulator of the input's value type"); } } @@ -1547,6 +1547,14 @@ LogicalResult SetEdgeProperties::verify() { return success(); } +LogicalResult ImportGraph::verify() { + if (!::db::namesImportedPropertyTypes(getFormat(), getTypesAttr())) { + return emitOpError("types each property as a datetime, duration or point a jsonl import reads, or a duration a parquet import reads"); + } + + return success(); +} + // A grouped accumulator needs at least one grouping key and one aggregate (with // no key it is a whole-stream aggregate, with no aggregate a projection), and // every kind must be a valid GroupAggregateKind. The column count is not known diff --git a/query/ir/dialect/nl/NLOps.td b/query/ir/dialect/nl/NLOps.td index 3962524ec6..636f62d613 100644 --- a/query/ir/dialect/nl/NLOps.td +++ b/query/ir/dialect/nl/NLOps.td @@ -3123,6 +3123,7 @@ def ToFloat : NLUnaryFunctionOp<"to_float", [ConstantThroughOperands]>; def ToBoolean : NLUnaryFunctionOp<"to_boolean", [ConstantThroughOperands]>; def ToDateTime : NLUnaryFunctionOp<"to_datetime", [ConstantThroughOperands]>; def ToDuration : NLUnaryFunctionOp<"to_duration", [ConstantThroughOperands]>; +def ToPoint : NLUnaryFunctionOp<"to_point", [ConstantThroughOperands]>; def ToString : NLUnaryFunctionOp<"to_string", [ConstantThroughOperands]>; def ToIntegerOrNull : NLUnaryFunctionOp<"to_integer_or_null", [ConstantThroughOperands]>; @@ -3162,6 +3163,15 @@ def DurationSecondsOfMinute : NLUnaryFunctionOp<"duration_seconds_of_minute", [C def DurationMillisecondsOfSecond : NLUnaryFunctionOp<"duration_milliseconds_of_second", [ConstantThroughOperands]>; def DurationMicrosecondsOfSecond : NLUnaryFunctionOp<"duration_microseconds_of_second", [ConstantThroughOperands]>; +def PointX : NLUnaryFunctionOp<"point_x", [ConstantThroughOperands]>; +def PointY : NLUnaryFunctionOp<"point_y", [ConstantThroughOperands]>; +def PointZ : NLUnaryFunctionOp<"point_z", [ConstantThroughOperands]>; +def PointLongitude : NLUnaryFunctionOp<"point_longitude", [ConstantThroughOperands]>; +def PointLatitude : NLUnaryFunctionOp<"point_latitude", [ConstantThroughOperands]>; +def PointHeight : NLUnaryFunctionOp<"point_height", [ConstantThroughOperands]>; +def PointCRSName : NLUnaryFunctionOp<"point_crs", [ConstantThroughOperands]>; +def PointSRID : NLUnaryFunctionOp<"point_srid", [ConstantThroughOperands]>; + def CurrentDateTime : NLOp<"current_datetime", [Pure, ConstantThroughOperands, RowAlignedThroughOperands]> { let summary = "The instant the query runs, as a single-value chunk"; let description = [{ @@ -3268,6 +3278,24 @@ def Replace : NLOp<"replace", [Pure, RowAlignedThroughOperands]> { let assemblyFormat = "$string `,` $search `,` $replacement attr-dict `:` functional-type(operands, results)"; } +def PointDistance : NLOp<"point_distance", [Pure, RowAlignedThroughOperands]> { + let summary = "Row-wise distance between two points, null across CRSs"; + + let arguments = (ins NLChunk:$lhs, NLChunk:$rhs); + let results = (outs NLChunk:$result); + + let assemblyFormat = "$lhs `,` $rhs attr-dict `:` functional-type(operands, results)"; +} + +def PointWithinBBox : NLOp<"point_within_bbox", [Pure, RowAlignedThroughOperands]> { + let summary = "Row-wise test of a point against a box, null unless the three share a CRS"; + + let arguments = (ins NLChunk:$point, NLChunk:$lowerLeft, NLChunk:$upperRight); + let results = (outs NLChunk:$result); + + let assemblyFormat = "$point `,` $lowerLeft `,` $upperRight attr-dict `:` functional-type(operands, results)"; +} + class NLBinaryFunctionOp : NLOp { let summary = "Apply a binary scalar function row-wise to two value chunks"; diff --git a/query/ir/dialect/nl/NLSystemOps.td b/query/ir/dialect/nl/NLSystemOps.td index 47d5c94056..0921e059a5 100644 --- a/query/ir/dialect/nl/NLSystemOps.td +++ b/query/ir/dialect/nl/NLSystemOps.td @@ -70,14 +70,15 @@ def ImportGraph : NLSystemOp<"import_graph"> { StrAttr:$graph_name, GraphImportFormat:$format, OptionalAttr:$embeddings, - OptionalAttr:$date_times, - OptionalAttr:$durations); + OptionalAttr:$types); let results = (outs NLChunk:$graph); let assemblyFormat = [{ $format `(` $path `)` `as` $graph_name (`embeddings` $embeddings^)? - (`datetimes` $date_times^)? (`durations` $durations^)? attr-dict `:` qualified(type($graph)) + (`types` $types^)? attr-dict `:` qualified(type($graph)) }]; + + let hasVerifier = 1; } /** diff --git a/query/ir/dialect/storage/StorageTypes.cpp b/query/ir/dialect/storage/StorageTypes.cpp index 379e59c09b..ad0eda87ee 100644 --- a/query/ir/dialect/storage/StorageTypes.cpp +++ b/query/ir/dialect/storage/StorageTypes.cpp @@ -20,7 +20,7 @@ mlir::LogicalResult NullableType::verify(llvm::function_ref(values); + const bool averagesNumbers = kind == AggregateKind::Avg && !isa(values); return averagesNumbers || kind == AggregateKind::StDev diff --git a/query/ir/dialect/storage/StorageTypes.td b/query/ir/dialect/storage/StorageTypes.td index 61b1c06b3e..c5fcd16016 100644 --- a/query/ir/dialect/storage/StorageTypes.td +++ b/query/ir/dialect/storage/StorageTypes.td @@ -111,6 +111,15 @@ def Duration : StorageType<"Duration", "duration"> { }]; } +def Point : StorageType<"Point", "point"> { + let summary = "A point property value"; + let description = [{ + The element type of a chunk holding one location per row - storage's + ColumnVector. It holds x, y and z and the coordinate reference system + they are given in. + }]; +} + def Nullable : StorageType<"Nullable", "nullable"> { let summary = "A nullable property value"; let description = [{ diff --git a/query/ir/exec/NLEntityProperties.cpp b/query/ir/exec/NLEntityProperties.cpp index 5135e6a510..dd5c717a5b 100644 --- a/query/ir/exec/NLEntityProperties.cpp +++ b/query/ir/exec/NLEntityProperties.cpp @@ -57,6 +57,9 @@ MapBuffer<>::MapItemVariant storedValue(const PropertyContainer& container, Enti case ValueType::Duration: return container.cast().get(entity); break; + case ValueType::Point: + return container.cast().get(entity); + break; case ValueType::Invalid: case ValueType::_SIZE: break; @@ -113,6 +116,9 @@ MapBuffer<>::MapItemVariant writtenValue(NLWrittenValues& written, case ValueType::Duration: return readWritten(written, readValues, value); break; + case ValueType::Point: + return readWritten(written, readValues, value); + break; case ValueType::Invalid: case ValueType::_SIZE: break; diff --git a/query/ir/exec/NLExecutor.cpp b/query/ir/exec/NLExecutor.cpp index 1b55e2123c..f3bbe4a22c 100644 --- a/query/ir/exec/NLExecutor.cpp +++ b/query/ir/exec/NLExecutor.cpp @@ -2970,6 +2970,12 @@ void distinctAppendElementBytes(std::string& key, const CellView element) { return; break; + case CellTag::Point: + key.push_back(static_cast(tag)); + distinctAppendValueBytes(key, element.getAs()); + return; + break; + case CellTag::Embedding: throw IRException("cannot dedup by an embedding element"); break; @@ -3139,6 +3145,10 @@ uint64_t hashKeyValue(types::Duration::Primitive value) { return mixKeyBits(static_cast(value.getMicroseconds())); } +uint64_t hashKeyValue(types::Point::Primitive value) { + return mixKeyBits(std::hash {}(value)); +} + // The two keys of a join need not share a type, so two values `=` holds equal hash alike // whatever they are held as: a double holding a whole number hashes as that integer. -0.0 // hashes as +0.0, the one value it equals; a NaN never reaches a lookup. @@ -3240,6 +3250,10 @@ uint64_t hashKeyValue(CellView cell) { return hashKeyValue(cell.getAs()); break; + case CellTag::Point: + return hashKeyValue(cell.getAs()); + break; + case CellTag::Path: return hashKeyValue(cell.getAs()); break; @@ -3464,14 +3478,22 @@ Primitive numericAdd(Primitive accumulator, Primitive value) { } } +// A number sums as an f64 and a point keeps its running mean. A duration's total is the +// state's 128-bit one. template -using AverageTotal = std::conditional_t, - types::Duration::Primitive, +using AverageTotal = std::conditional_t + || std::is_same_v, + Primitive, types::Double::Primitive>; +// @param count is how many values @param total already holds, which a running mean needs template -types::Double::Primitive addToAverageTotal(types::Double::Primitive total, Primitive value) { - return total + static_cast(value); +AverageTotal addToAverageTotal(AverageTotal total, Primitive value, uint64_t count) { + if constexpr (std::is_same_v) { + return Point::addToMean(total, count, value); + } else { + return total + static_cast(value); + } } types::Double::Primitive averageOf(types::Double::Primitive total, uint64_t count) { @@ -3497,6 +3519,10 @@ types::Duration::Primitive durationOfTotal(__int128 total) { return types::Duration::Primitive(static_cast(total)); } +types::Point::Primitive averageOf(types::Point::Primitive mean, uint64_t count) { + return mean; +} + template void dispatchAverageTotal(ValueType totalType, Select&& select) { switch (totalType) { @@ -3508,8 +3534,12 @@ void dispatchAverageTotal(ValueType totalType, Select&& select) { select.template operator()(); break; + case ValueType::Point: + select.template operator()(); + break; + default: - throw IRException("avg answers a double or a duration"); + throw IRException("avg answers a double, a duration or a point"); break; } } @@ -3841,16 +3871,24 @@ constexpr std::byte DURATION_ZERO_CELL[sizeof(CellTag) + sizeof(types::Duration: }; // The running sum of tagged cells, as Cypher adds them: an integer while every number -// summed is one, a float from the first float on, and a duration over durations. A null -// sum has summed nothing yet. Integers wrap as a typed column's sum does. The sum is its -// cell plus a 128-bit total of microseconds: durations add into the total, and the cell -// of a duration sum is PT0S, so its tag still names what the sum holds. +// summed is one, a float from the first float on, a duration over durations, and the +// running mean of the points an average reads. A null sum has summed nothing yet. Integers +// wrap as a typed column's sum does. The sum is its cell plus a 128-bit total of +// microseconds: durations add into the total, and the cell of a duration sum is PT0S, so +// its tag still names what the sum holds. class NLTaggedSum { public: NLTaggedSum(const std::optional& sum, __int128 durationTotal) : _durationTotal(durationTotal) { - if (sum.has_value()) { + if (!sum.has_value()) { + return; + } + + if (sum->getTag() == CellTag::Point) { + _tag = CellTag::Point; + _point = sum->getAs(); + } else { add(*sum); } } @@ -3881,13 +3919,16 @@ class NLTaggedSum { } } - // An average reads every number as a float, as it does over a typed column - void addToAverage(CellView value) { + // An average reads every number as a float, as it does over a typed column. @param count + // is how many cells the average already holds, which the running mean of points needs. + void addToAverage(CellView value, uint64_t count) { const CellTag tag = value.getTag(); const bool isNumber = tag == CellTag::Int || tag == CellTag::UInt || tag == CellTag::Double; if (isNumber) { addReal(tag, taggedNumericValue(value)); + } else if (tag == CellTag::Point) { + addPoint(value.getAs(), count); } else { add(value); } @@ -3902,6 +3943,8 @@ class NLTaggedSum { return CellView {DURATION_ZERO_CELL}; } else if (_tag == CellTag::Double) { item = _real; + } else if (_tag == CellTag::Point) { + item = _point; } return values.ownCell(item); @@ -3926,6 +3969,8 @@ class NLTaggedSum { return std::nullopt; } else if (_tag == CellTag::Duration) { return stagedCell(&cells, averageOf(_durationTotal, count)); + } else if (_tag == CellTag::Point) { + return stagedCell(&cells, _point); } bioassert(_tag == CellTag::Double, "An average sums its numbers as floats"); @@ -3938,10 +3983,17 @@ class NLTaggedSum { types::Int64::Primitive _integer {0}; types::Double::Primitive _real {0.0}; __int128 _durationTotal {0}; + types::Point::Primitive _point {}; + + // Numbers sum together, and a duration or a point only with its own kind + static CellTag sumKind(CellTag tag) { + const bool sumsAlone = tag == CellTag::Duration || tag == CellTag::Point; + return sumsAlone ? tag : CellTag::Double; + } void throwIfMixing(CellTag tag) const { const bool summedAny = _tag != CellTag::Null; - const bool mixes = (_tag == CellTag::Duration) != (tag == CellTag::Duration); + const bool mixes = sumKind(_tag) != sumKind(tag); if (summedAny && mixes) { throw IRException(fmt::format("sum() and avg() cannot add '{}' to a sum of '{}'", @@ -3978,6 +4030,13 @@ class NLTaggedSum { _tag = CellTag::Duration; _durationTotal += value.getMicroseconds(); } + + void addPoint(types::Point::Primitive value, uint64_t count) { + throwIfMixing(CellTag::Point); + + _tag = CellTag::Point; + _point = Point::addToMean(_point, count, value); + } }; // Fold a chunk of tagged cells into the running sum of a sum or an average, which is @@ -3999,7 +4058,7 @@ void aggregateUpdateTaggedSum(NLAggregateState* state, const Column* input) { } if constexpr (Averages) { - sum.addToAverage(*value); + sum.addToAverage(*value, state->getCount() + added); } else { sum.add(*value); } @@ -4077,7 +4136,7 @@ void aggregateUpdateAvg(NLAggregateState* state, const Column* input) { size_t seen = 0; for (const std::optional& value : inputRaw) { if (value.has_value()) { - running = addToAverageTotal(running, *value); + running = addToAverageTotal(running, *value, state->getCount() + seen); seen++; } } @@ -4180,8 +4239,12 @@ NLAggregateUpdateFunction selectAvgUpdate(ValueType inputType) { return &aggregateUpdateDurationTotal; break; + case ValueType::Point: + return &aggregateUpdateAvg; + break; + default: - throw IRException("avg requires a numeric or a duration column"); + throw IRException("avg requires a numeric, a duration or a point column"); break; } @@ -4223,6 +4286,10 @@ NLAggregateUpdateFunction selectMinMaxUpdate(ValueType inputType) { return &aggregateUpdateMinMax; break; + case ValueType::Point: + return &aggregateUpdateMinMax; + break; + case ValueType::List: return &aggregateUpdateMinMax; break; @@ -4478,7 +4545,7 @@ void groupFoldTaggedSum(Column* accumulator, NLTaggedSum sum(running, durationTotal); if constexpr (Averages) { - sum.addToAverage(*value); + sum.addToAverage(*value, counts[group]); counts[group]++; } else { sum.add(*value); @@ -4615,7 +4682,7 @@ void groupFoldAvg(Column* accumulator, if (value.has_value()) { const size_t group = groups[row]; std::optional& running = raw[group]; - running = addToAverageTotal(running.value(), *value); + running = addToAverageTotal(running.value(), *value, counts[group]); counts[group]++; } } @@ -4653,7 +4720,7 @@ void groupFoldAvgDistinct(Column* accumulator, } std::optional& running = raw[group]; - running = addToAverageTotal(running.value(), *value); + running = addToAverageTotal(running.value(), *value, counts[group]); counts[group]++; } } @@ -5222,6 +5289,10 @@ Item taggedItem(const CellView element) { return Item {element.getAs()}; break; + case CellTag::Point: + return Item {element.getAs()}; + break; + case CellTag::MapView: return Item {element.getAs()}; break; @@ -5550,6 +5621,9 @@ NLKeyedValue readKeyedCell(const CellView cell) { case CellTag::Duration: return cell.getAs(); break; + case CellTag::Point: + return cell.getAs(); + break; case CellTag::Null: return std::monostate {}; break; @@ -5631,6 +5705,14 @@ bool findKeyedEntry(const NLKeyedValue& keyed, return true; } +MapBuffer<>::MapItemVariant mapItemOf(int64_t component) { + return component; +} + +MapBuffer<>::MapItemVariant mapItemOf(const Point::Component& component) { + return std::visit([](auto value) -> MapBuffer<>::MapItemVariant { return value; }, component); +} + // The component @param part names of @param value, written into @param mapBuffer as the // value of a map of one entry, as an entity's property is template @@ -5645,7 +5727,7 @@ CellView componentCell(Owner value, const MapBuffer<>::MapKeyValuePair component { .key = key, - .value = Owner::component(value, *part), + .value = mapItemOf(Owner::component(value, *part)), }; const MapView map = mapBuffer.insert(std::span::MapKeyValuePair> {&component, 1}); @@ -5654,7 +5736,8 @@ CellView componentCell(Owner value, } // The value a property read finds in @param keyed: a key of its map, a property of its node -// or edge, or a component of its instant or duration. Null where it holds null or no such key. +// or edge, or a component of its instant, duration or point. Null where it holds null or no +// such key. CellView readStaticKey(const NLKeyedValue& keyed, const NLStaticMapKeyData* mapKey, NLEntityProperties& properties, @@ -5662,9 +5745,11 @@ CellView readStaticKey(const NLKeyedValue& keyed, const std::string_view key = mapKey->getKey(); if (const DateTime* instant = std::get_if(&keyed)) { - return componentCell(*instant, mapKey->getDateTimePart(), key, "datetime", mapBuffer); + return componentCell(*instant, mapKey->getPart(), key, "datetime", mapBuffer); } else if (const Duration* duration = std::get_if(&keyed)) { - return componentCell(*duration, mapKey->getDurationPart(), key, "duration", mapBuffer); + return componentCell(*duration, mapKey->getPart(), key, "duration", mapBuffer); + } else if (const Point* point = std::get_if(&keyed)) { + return componentCell(*point, mapKey->getPart(), key, "point", mapBuffer); } else if (const NLKeylessValue* keyless = std::get_if(&keyed)) { throw IRException(fmt::format("Type mismatch: a value of type '{}' has no property '{}'", queryTypeName(keyless->_tag), @@ -5784,6 +5869,28 @@ std::optional unsignedValueArgument(const Column* input return unsignedCharacterArgument(*cell, functionName); } +template +std::optional pointValueArgument(const Column* input, size_t row, std::string_view) { + return (*static_cast(input))[row]; +} + +template +std::optional cellPointArgument(const Column* input, size_t row, std::string_view functionName) { + const std::optional cell = (*static_cast(input))[row]; + if (!cell.has_value()) { + return std::nullopt; + } + + const CellTag tag = cell->getTag(); + if (tag == CellTag::Point) { + return cell->getAs(); + } else if (tag == CellTag::Null) { + return std::nullopt; + } + + throwCellTypeError(functionName, "a point", tag); +} + template std::optional cellIntegerArgument(const Column* input, size_t row, std::string_view functionName) { const std::optional cell = (*static_cast(input))[row]; @@ -5890,6 +5997,7 @@ NLSubscript subscriptOf(const CellView cell) { case CellTag::MapView: case CellTag::Duration: case CellTag::Path: + case CellTag::Point: return NLNeitherPositionNorKey {tag}; break; case CellTag::INVALID: @@ -6444,8 +6552,12 @@ NLGroupAggregateFoldFunction selectGroupAvgFold(ValueType inputType) { return &groupFoldDurationTotal; break; + case ValueType::Point: + return &groupFoldAvg; + break; + default: - throw IRException("avg requires a numeric or a duration column"); + throw IRException("avg requires a numeric, a duration or a point column"); break; } @@ -6472,8 +6584,12 @@ NLGroupAggregateFoldFunction selectGroupAvgDistinctFold(ValueType inputType) { return &groupFoldDurationTotal; break; + case ValueType::Point: + return &groupFoldAvgDistinct; + break; + default: - throw IRException("avg requires a numeric or a duration column"); + throw IRException("avg requires a numeric, a duration or a point column"); break; } @@ -6514,6 +6630,10 @@ NLGroupAggregateFoldFunction selectGroupMinMaxFold(ValueType inputType) { return &groupFoldMinMax; break; + case ValueType::Point: + return &groupFoldMinMax; + break; + case ValueType::List: return &groupFoldMinMax; break; @@ -6562,6 +6682,10 @@ NLGroupAggregateFoldFunction selectGroupFirstFold(ValueType inputType) { return &groupFoldFirst; break; + case ValueType::Point: + return &groupFoldFirst; + break; + case ValueType::List: return &groupFoldFirst; break; @@ -9958,6 +10082,64 @@ void NLExecutor::runReplace(NLExecutionContext*, NLFunctionData* data) { template void NLExecutor::runReplace(NLExecutionContext* context, NLFunctionData* data); template void NLExecutor::runReplace>(NLExecutionContext* context, NLFunctionData* data); +void NLExecutor::runPointDistance(NLExecutionContext*, NLFunctionData* data) { + const NLPointFunctionData* function = static_cast(data); + + const std::vector& arguments = function->getArguments(); + const NLPointFunctionData::PointArgument& lhs = arguments[0]; + const NLPointFunctionData::PointArgument& rhs = arguments[1]; + + const size_t rowCount = lhs._column->size(); + bioassert(rhs._column->size() == rowCount, "Columns of a point.distance are not row-aligned."); + + std::vector>& outputRaw = + static_cast*>(function->getResult())->getRaw(); + outputRaw.resize(rowCount); + + for (size_t rowIndex = 0; rowIndex < rowCount; rowIndex++) { + const std::optional from = lhs._read(lhs._column, rowIndex, "point.distance"); + const std::optional to = rhs._read(rhs._column, rowIndex, "point.distance"); + + if (from.has_value() && to.has_value()) { + outputRaw[rowIndex] = Point::distance(*from, *to); + } else { + outputRaw[rowIndex] = std::nullopt; + } + } +} + +void NLExecutor::runPointWithinBBox(NLExecutionContext*, NLFunctionData* data) { + const NLPointFunctionData* function = static_cast(data); + + const std::vector& arguments = function->getArguments(); + const NLPointFunctionData::PointArgument& point = arguments[0]; + const NLPointFunctionData::PointArgument& lowerLeft = arguments[1]; + const NLPointFunctionData::PointArgument& upperRight = arguments[2]; + + const size_t rowCount = point._column->size(); + bioassert(lowerLeft._column->size() == rowCount, "Columns of a point.withinBBox are not row-aligned."); + bioassert(upperRight._column->size() == rowCount, "Columns of a point.withinBBox are not row-aligned."); + + std::vector>& outputRaw = + static_cast*>(function->getResult())->getRaw(); + outputRaw.resize(rowCount); + + for (size_t rowIndex = 0; rowIndex < rowCount; rowIndex++) { + const std::optional value = point._read(point._column, rowIndex, "point.withinBBox"); + const std::optional lower = lowerLeft._read(lowerLeft._column, rowIndex, "point.withinBBox"); + const std::optional upper = upperRight._read(upperRight._column, rowIndex, "point.withinBBox"); + + const bool readsANull = !value.has_value() || !lower.has_value() || !upper.has_value(); + const std::optional within = readsANull ? std::nullopt : Point::withinBBox(*value, *lower, *upper); + + if (within.has_value()) { + outputRaw[rowIndex] = types::Bool::Primitive {*within}; + } else { + outputRaw[rowIndex] = std::nullopt; + } + } +} + void NLExecutor::runRange(NLExecutionContext*, NLFunctionData* data) { const NLRangeData* range = static_cast(data); @@ -10380,6 +10562,30 @@ NLStringListArgumentRead NLExecutor::selectStringListArgumentRead(const Column* throw IRException("a string function reads a list of strings out of a nullable string column, a list column, or a column of tagged cells"); } +NLPointArgumentRead NLExecutor::selectPointArgumentRead(const Column* input) { + using PointValue = types::Point::Primitive; + + const ColumnKind::Code kind = input->getKind(); + + if (kind == ColumnOptVector::staticKind()) { + return &pointValueArgument>; + } else if (kind == ColumnConst::staticKind()) { + return &pointValueArgument>; + } else if (kind == ColumnConst>::staticKind()) { + return &pointValueArgument>>; + } else if (kind == ColumnVector::staticKind()) { + return &cellPointArgument>; + } else if (kind == ColumnOptVector::staticKind()) { + return &cellPointArgument>; + } else if (kind == ColumnConst::staticKind()) { + return &cellPointArgument>; + } else if (kind == ColumnConst>::staticKind()) { + return &cellPointArgument>>; + } + + throw IRException("a point function reads a point argument out of a nullable point column, or a column of tagged cells"); +} + NLIntegerArgumentRead NLExecutor::selectIntegerArgumentRead(const Column* input) { const ColumnKind::Code kind = input->getKind(); @@ -10993,6 +11199,10 @@ NLUnaryFn NLExecutor::selectToNullable(ValueType valueType, const Column* operan return selectToNullableOf(operand, memory, result); break; + case ValueType::Point: + return selectToNullableOf(operand, memory, result); + break; + // A cell is a view into the query's list buffer, so the nullable column holds the // views the plain one held and an empty optional where a row has no list at all case ValueType::List: @@ -11183,6 +11393,7 @@ NLUnaryFunctionKernel NLExecutor::selectConversion(const Column* input, bool inp selectForElement.template operator()(); selectForElement.template operator()(); selectForElement.template operator()(); + selectForElement.template operator()(); if (kernel) { return kernel; @@ -11389,6 +11600,14 @@ template NLUnaryFunctionKernel NLExecutor::selectFunction>(const Column* input, bool inputNullable, LocalMemory* memory, Column*& result); template NLUnaryFunctionKernel NLExecutor::selectFunction>(const Column* input, bool inputNullable, LocalMemory* memory, Column*& result); template NLUnaryFunctionKernel NLExecutor::selectFunction>(const Column* input, bool inputNullable, LocalMemory* memory, Column*& result); +template NLUnaryFunctionKernel NLExecutor::selectFunction>(const Column* input, bool inputNullable, LocalMemory* memory, Column*& result); +template NLUnaryFunctionKernel NLExecutor::selectFunction>(const Column* input, bool inputNullable, LocalMemory* memory, Column*& result); +template NLUnaryFunctionKernel NLExecutor::selectFunction>(const Column* input, bool inputNullable, LocalMemory* memory, Column*& result); +template NLUnaryFunctionKernel NLExecutor::selectFunction>(const Column* input, bool inputNullable, LocalMemory* memory, Column*& result); +template NLUnaryFunctionKernel NLExecutor::selectFunction>(const Column* input, bool inputNullable, LocalMemory* memory, Column*& result); +template NLUnaryFunctionKernel NLExecutor::selectFunction>(const Column* input, bool inputNullable, LocalMemory* memory, Column*& result); +template NLUnaryFunctionKernel NLExecutor::selectFunction>(const Column* input, bool inputNullable, LocalMemory* memory, Column*& result); +template NLUnaryFunctionKernel NLExecutor::selectFunction>(const Column* input, bool inputNullable, LocalMemory* memory, Column*& result); template NLUnaryFunctionKernel NLExecutor::selectFunction(const Column* input, bool inputNullable, LocalMemory* memory, Column*& result); template NLUnaryFunctionKernel NLExecutor::selectFunction(const Column* input, bool inputNullable, LocalMemory* memory, Column*& result); template NLUnaryFunctionKernel NLExecutor::selectFunction(const Column* input, bool inputNullable, LocalMemory* memory, Column*& result); @@ -11413,6 +11632,7 @@ template NLUnaryFunctionKernel NLExecutor::selectConversion(co template NLUnaryFunctionKernel NLExecutor::selectConversion(const Column* input, bool inputNullable, LocalMemory* memory, Column*& result); template NLUnaryFunctionKernel NLExecutor::selectConversion(const Column* input, bool inputNullable, LocalMemory* memory, Column*& result); template NLUnaryFunctionKernel NLExecutor::selectConversion(const Column* input, bool inputNullable, LocalMemory* memory, Column*& result); +template NLUnaryFunctionKernel NLExecutor::selectConversion(const Column* input, bool inputNullable, LocalMemory* memory, Column*& result); template NLUnaryFunctionKernel NLExecutor::selectOrNull(const Column* input, bool inputNullable, LocalMemory* memory, Column*& result) { @@ -12314,6 +12534,10 @@ NLCollectFoldFunction NLExecutor::selectCollectFold(ValueType valueType) { return &collectFold; break; + case ValueType::Point: + return &collectFold; + break; + default: throw IRException("collect does not support this value type"); break; @@ -12359,6 +12583,10 @@ NLCollectFoldFunction NLExecutor::selectCollectDistinctFold(ValueType valueType) return &collectFoldDistinct; break; + case ValueType::Point: + return &collectFoldDistinct; + break; + default: throw IRException("collect does not support this value type"); break; @@ -12469,6 +12697,10 @@ NLUnwindCollectValueEmitFunction NLExecutor::selectUnwindCollectValueEmit(ValueT return &unwindCollectValueEmit; break; + case ValueType::Point: + return &unwindCollectValueEmit; + break; + default: throw IRException("unwind does not support this value type"); break; @@ -12511,6 +12743,10 @@ NLCollectListEmitFunction NLExecutor::selectCollectListEmit(ValueType valueType) return &collectListEmit; break; + case ValueType::Point: + return &collectListEmit; + break; + default: throw IRException("collect does not support this value type"); break; @@ -13020,6 +13256,7 @@ NLUnaryFn NLExecutor::selectRowConstant(const Column* row, LocalMemory* memory, selectForElement.template operator()(); selectForElement.template operator()(); selectForElement.template operator()(); + selectForElement.template operator()(); selectForElement.template operator()>(); selectForElement.template operator()>(); selectForElement.template operator()>(); @@ -13028,6 +13265,7 @@ NLUnaryFn NLExecutor::selectRowConstant(const Column* row, LocalMemory* memory, selectForElement.template operator()>(); selectForElement.template operator()>(); selectForElement.template operator()>(); + selectForElement.template operator()>(); selectForElement.template operator()(); selectForElement.template operator()(); selectForElement.template operator()(); @@ -13441,6 +13679,10 @@ NLKeyAppendFunction NLExecutor::selectKeyAppendFunction(NLChunkKind kind) { return &distinctKeyAppendPlainColumn; break; + case NLChunkKind::Point: + return &distinctKeyAppendPlainColumn; + break; + case NLChunkKind::PathRef: throw IRException("A path column cannot be a DISTINCT or grouping key: expand it into its list first"); break; @@ -13493,6 +13735,10 @@ NLKeyAppendFunction NLExecutor::selectOptKeyAppendFunction(ValueType valueType) return &distinctKeyAppendOptColumn; break; + case ValueType::Point: + return &distinctKeyAppendOptColumn; + break; + case ValueType::Map: return &distinctKeyAppendOptMapColumn; break; @@ -13573,6 +13819,10 @@ NLJoinKeyFunctions NLExecutor::selectJoinKeyFunctions(NLChunkKind kind) { return joinKeyFunctions(); break; + case NLChunkKind::Point: + return joinKeyFunctions(); + break; + case NLChunkKind::PathRef: throw IRException("A path column cannot be a join key: expand it into its list first"); break; @@ -14051,6 +14301,10 @@ NLGroupAggregateFoldFunction NLExecutor::selectGroupCountDistinctFold(ValueType return &groupFoldCountDistinctPresent; break; + case ValueType::Point: + return &groupFoldCountDistinctPresent; + break; + case ValueType::Map: return &groupFoldCountDistinctPresentMap; break; @@ -14283,6 +14537,11 @@ NLKeyAppendFunction NLExecutor::selectPlainMergeKeyAppendFunction(NLChunkKind ki throwUnlessKeyedAsItsOwnType(ValueType::Duration, keyType); return &mergeKeyAppendPlainColumn; break; + + case NLChunkKind::Point: + throwUnlessKeyedAsItsOwnType(ValueType::Point, keyType); + return &mergeKeyAppendPlainColumn; + break; } bioassert(false, "Unknown NLChunkKind"); @@ -14331,6 +14590,11 @@ NLKeyAppendFunction NLExecutor::selectConstMergeKeyAppendFunction(ValueType valu return &mergeKeyAppendConstColumn; break; + case ValueType::Point: + throwUnlessKeyedAsItsOwnType(ValueType::Point, keyType); + return &mergeKeyAppendConstColumn; + break; + case ValueType::Map: throw IRException("a MERGE pattern cannot constrain a property to a map"); break; @@ -14516,6 +14780,10 @@ NLCompareFunction NLExecutor::selectCompareFunction(NLChunkKind kind) { return &compareColumn; break; + case NLChunkKind::Point: + return &compareColumn; + break; + case NLChunkKind::PathRef: throw IRException("A path column cannot be a sort key: expand it into its list first"); break; @@ -14567,6 +14835,10 @@ NLCompareFunction NLExecutor::selectOptCompareFunction(ValueType valueType) { return &compareOptColumn; break; + case ValueType::Point: + return &compareOptColumn; + break; + case ValueType::Map: return &compareOptContainerColumn; break; @@ -14658,6 +14930,7 @@ template void NLExecutor::runPropertyFetch(NLExecution template void NLExecutor::runPropertyFetch(NLExecutionContext*, NLFunctionData*); template void NLExecutor::runPropertyFetch(NLExecutionContext*, NLFunctionData*); template void NLExecutor::runPropertyFetch(NLExecutionContext*, NLFunctionData*); +template void NLExecutor::runPropertyFetch(NLExecutionContext*, NLFunctionData*); template void NLExecutor::runPropertyFetch(NLExecutionContext*, NLFunctionData*); template void NLExecutor::runPropertyFetch(NLExecutionContext*, NLFunctionData*); template void NLExecutor::runPropertyFetch(NLExecutionContext*, NLFunctionData*); @@ -14668,6 +14941,7 @@ template void NLExecutor::runPropertyFetch(NLExecution template void NLExecutor::runPropertyFetch(NLExecutionContext*, NLFunctionData*); template void NLExecutor::runPropertyFetch(NLExecutionContext*, NLFunctionData*); template void NLExecutor::runPropertyFetch(NLExecutionContext*, NLFunctionData*); +template void NLExecutor::runPropertyFetch(NLExecutionContext*, NLFunctionData*); template void NLExecutor::runPropertyFetch(NLExecutionContext*, NLFunctionData*); void NLExecutor::runGetNodeLabelSet(NLExecutionContext* context, NLFunctionData* data) { diff --git a/query/ir/exec/NLExecutor.h b/query/ir/exec/NLExecutor.h index 81cdced8bd..51ebc1eb9b 100644 --- a/query/ir/exec/NLExecutor.h +++ b/query/ir/exec/NLExecutor.h @@ -425,6 +425,8 @@ class NLExecutor { static void runStringEnd(NLExecutionContext* context, NLFunctionData* data); template static void runReplace(NLExecutionContext* context, NLFunctionData* data); + static void runPointDistance(NLExecutionContext* context, NLFunctionData* data); + static void runPointWithinBBox(NLExecutionContext* context, NLFunctionData* data); // Build one map per row (nl.make_map): row r of the result maps each key to what its // value column holds at r, as one contiguous run of the query's map buffer. @@ -473,6 +475,7 @@ class NLExecutor { static NLStringArgumentRead selectStringArgumentRead(const Column* input); static NLStringListArgumentRead selectStringListArgumentRead(const Column* input); static NLIntegerArgumentRead selectIntegerArgumentRead(const Column* input); + static NLPointArgumentRead selectPointArgumentRead(const Column* input); // Build one list per row (nl.list_comprehension): the body runs over the elements of // the step's cells, a chunkful at a time, and row r takes the ones it kept of its own diff --git a/query/ir/exec/NLProgram.h b/query/ir/exec/NLProgram.h index 3131caeb54..993e34af5c 100644 --- a/query/ir/exec/NLProgram.h +++ b/query/ir/exec/NLProgram.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -39,6 +40,7 @@ #include "list/PathTrie.h" #include "metadata/DateTime.h" #include "metadata/Duration.h" +#include "metadata/Point.h" #include "metadata/LabelSet.h" #include "metadata/LabelSetHandle.h" #include "metadata/PropertyType.h" @@ -84,6 +86,7 @@ enum class NLChunkKind { Path, DateTime, Duration, + Point, PathRef, }; @@ -161,6 +164,10 @@ void dispatchChunkKind(NLChunkKind kind, Handler&& handler) { return handler.template operator()(); break; + case NLChunkKind::Point: + return handler.template operator()(); + break; + case NLChunkKind::PathRef: return handler.template operator()(); break; @@ -4248,6 +4255,34 @@ using NLIntegerArgumentRead = std::optional (*)(const C size_t row, std::string_view functionName); +using NLPointArgumentRead = std::optional (*)(const Column* input, + size_t row, + std::string_view functionName); + +// The arguments of point.distance or point.withinBBox, each a column read one point per row +class NLPointFunctionData : public NLFunctionData { +public: + struct PointArgument { + const Column* _column {nullptr}; + NLPointArgumentRead _read {nullptr}; + }; + + explicit NLPointFunctionData(Column* result) + : _result(result) + { + } + + Column* getResult() const { return _result; } + + const std::vector& getArguments() const { return _arguments; } + + void addArgument(const PointArgument& argument) { _arguments.push_back(argument); } + +private: + Column* _result {nullptr}; + std::vector _arguments; +}; + class NLSubstringData : public NLFunctionData { public: struct StringArgument { @@ -4420,9 +4455,9 @@ struct NLKeylessValue { }; // What one row of an nl.static_map_key or an nl.dynamic_map_key reads its key in: a map, the -// node or the edge whose properties are its keys, or the instant or the duration whose -// components are. Nothing where the row holds null. -using NLKeyedValue = std::variant; +// node or the edge whose properties are its keys, or the instant, the duration or the point +// whose components are. Nothing where the row holds null. +using NLKeyedValue = std::variant; // The keyed value one column of a map key read holds at @param row. One per column shape, // selected during translation. @@ -4434,23 +4469,24 @@ using NLStringReadFunction = std::optional (*)(const C size_t row); // Row-wise map key read (nl.static_map_key): row r of the result views the value row r's map -// holds under the key, or a null cell where it holds none. The components of an instant and -// of a duration the key names are resolved during translation. +// holds under the key, or a null cell where it holds none. The components of an instant, a +// duration and a point the key names are resolved during translation. class NLStaticMapKeyData : public NLFunctionData { public: + // The part the key names of each type with components, as the row's value picks the type + using ComponentParts = std::tuple, std::optional, std::optional>; + NLStaticMapKeyData(const Column* input, Column* result, NLKeyedReadFunction keyedRead, std::string_view key, - std::optional dateTimePart, - std::optional durationPart, + const ComponentParts& parts, NLStepBuffers* stepBuffers) : _input(input), _result(result), _keyedRead(keyedRead), _key(key), - _dateTimePart(dateTimePart), - _durationPart(durationPart), + _parts(parts), _stepBuffers(stepBuffers) { } @@ -4459,17 +4495,17 @@ class NLStaticMapKeyData : public NLFunctionData { Column* getResult() const { return _result; } NLKeyedReadFunction getKeyedRead() const { return _keyedRead; } std::string_view getKey() const { return _key; } - std::optional getDateTimePart() const { return _dateTimePart; } - std::optional getDurationPart() const { return _durationPart; } NLStepBuffers* getStepBuffers() const { return _stepBuffers; } + template + std::optional getPart() const { return std::get>(_parts); } + private: const Column* _input {nullptr}; Column* _result {nullptr}; NLKeyedReadFunction _keyedRead {nullptr}; std::string_view _key {}; - std::optional _dateTimePart; - std::optional _durationPart; + ComponentParts _parts; NLStepBuffers* _stepBuffers {nullptr}; }; diff --git a/query/ir/exec/NLSystemData.h b/query/ir/exec/NLSystemData.h index fcd4f62717..924c34757b 100644 --- a/query/ir/exec/NLSystemData.h +++ b/query/ir/exec/NLSystemData.h @@ -5,10 +5,9 @@ #include #include -#include "DateTimeSpec.h" -#include "DurationSpec.h" #include "EmbeddingsSpec.h" #include "Path.h" +#include "TypedPropertySpec.h" #include "VecLibMetadata.h" #include "columns/ColumnVector.h" @@ -71,22 +70,21 @@ class NLGraphCommandData : public NLFunctionData { }; // LOAD JSONL / LOAD GML / LOAD PARQUET: the file to import, the graph to build -// from it, the per-property embedding dimensions (JSONL alone ever carries any) -// and the statement to blame when the import fails. +// from it, the per-property embedding dimensions (JSONL alone ever carries any), +// the properties it reads as a type the format does not spell, and the statement +// to blame when the import fails. class NLImportGraphData : public NLFunctionData { public: NLImportGraphData(const fs::Path& path, std::string_view graphName, const EmbeddingsSpec& embeddings, - const DateTimeSpec& dateTimes, - const DurationSpec& durations, + const TypedPropertySpec& typedProperties, std::string_view statement, NLViewColumn* graph) : _path(path), _graphName(graphName), _embeddings(embeddings), - _dateTimes(dateTimes), - _durations(durations), + _typedProperties(typedProperties), _statement(statement), _graph(graph) { @@ -95,8 +93,7 @@ class NLImportGraphData : public NLFunctionData { const fs::Path& getPath() const { return _path; } std::string_view getGraphName() const { return _graphName; } const EmbeddingsSpec& getEmbeddings() const { return _embeddings; } - const DateTimeSpec& getDateTimes() const { return _dateTimes; } - const DurationSpec& getDurations() const { return _durations; } + const TypedPropertySpec& getTypedProperties() const { return _typedProperties; } std::string_view getStatement() const { return _statement; } NLViewColumn* getGraph() const { return _graph; } @@ -104,8 +101,7 @@ class NLImportGraphData : public NLFunctionData { fs::Path _path; std::string_view _graphName; EmbeddingsSpec _embeddings; - DateTimeSpec _dateTimes; - DurationSpec _durations; + TypedPropertySpec _typedProperties; std::string_view _statement; NLViewColumn* _graph {nullptr}; }; diff --git a/query/ir/exec/NLSystemExecutor.cpp b/query/ir/exec/NLSystemExecutor.cpp index 25c6fa471e..c47c31499f 100644 --- a/query/ir/exec/NLSystemExecutor.cpp +++ b/query/ir/exec/NLSystemExecutor.cpp @@ -191,8 +191,7 @@ void NLSystemExecutor::runImportGraph(NLExecutionContext* context, NLFunctionDat const Graph* graph = accessor.importGraph(command->getPath(), graphName, command->getEmbeddings(), - command->getDateTimes(), - command->getDurations()); + command->getTypedProperties()); if (!graph) { throw IRException(fmt::format("{}: failed to import graph '{}' from '{}'", command->getStatement(), diff --git a/query/ir/exec/NLSystemTranslator.cpp b/query/ir/exec/NLSystemTranslator.cpp index f9b39f3bde..6d250469da 100644 --- a/query/ir/exec/NLSystemTranslator.cpp +++ b/query/ir/exec/NLSystemTranslator.cpp @@ -12,10 +12,10 @@ #include "NLOps.h" #include "StorageEnums.h" -#include "DateTimeSpec.h" -#include "DurationSpec.h" #include "EmbeddingsSpec.h" +#include "IRValueTypes.h" #include "Path.h" +#include "TypedPropertySpec.h" #include "LocalMemory.h" @@ -172,17 +172,17 @@ void fillEmbeddingsSpec(mlir::DictionaryAttr attribute, EmbeddingsSpec& specs) { } } -// The properties an import named as instants or durations, one unit entry each - the name -// is the whole of it. A null attribute - an import with no such clause - leaves the spec -// empty. -template -void fillPropertyNameSpec(mlir::DictionaryAttr attribute, PropertyNameSpec& specs) { +// The properties an import reads as a type its format does not spell, each name mapped to +// that type's element type. A null attribute - an import with no such clause - leaves the +// spec empty. +void fillTypedPropertySpec(mlir::DictionaryAttr attribute, TypedPropertySpec& specs) { if (!attribute) { return; } for (const mlir::NamedAttribute& entry : attribute) { - specs.emplace(toStringView(entry.getName().getValue())); + const mlir::TypeAttr type = mlir::cast(entry.getValue()); + specs.emplace(toStringView(entry.getName().getValue()), elementTypeToValueType(type.getValue())); } } @@ -221,18 +221,14 @@ bool NLSystemTranslator::translate(mlir::Operation& operation, NLStmtContainer* EmbeddingsSpec embeddings; fillEmbeddingsSpec(importGraph.getEmbeddingsAttr(), embeddings); - DateTimeSpec dateTimes; - fillPropertyNameSpec(importGraph.getDateTimesAttr(), dateTimes); - - DurationSpec durations; - fillPropertyNameSpec(importGraph.getDurationsAttr(), durations); + TypedPropertySpec typedProperties; + fillTypedPropertySpec(importGraph.getTypesAttr(), typedProperties); NLImportGraphData* data = _program->allocFunctionData(fs::Path(importGraph.getPath().str()), toStringView(importGraph.getGraphName()), embeddings, - dateTimes, - durations, + typedProperties, importStatement(importGraph.getFormat()), allocResult(importGraph.getGraph())); body->emplaceStmt(&NLSystemExecutor::runImportGraph, data); diff --git a/query/ir/exec/NLTranslator.cpp b/query/ir/exec/NLTranslator.cpp index 54d3d4d660..a1ca8c19d3 100644 --- a/query/ir/exec/NLTranslator.cpp +++ b/query/ir/exec/NLTranslator.cpp @@ -378,6 +378,7 @@ const std::unordered_map unaryFunctio {"nl.to_string_or_null", &NLExecutor::selectOrNull<&NLExecutor::selectConversion, TaggedToStringOrNullFunction>}, {"nl.to_datetime", &NLExecutor::selectConversion}, {"nl.to_duration", &NLExecutor::selectConversion}, + {"nl.to_point", &NLExecutor::selectConversion}, {"nl.datetime_year", &NLExecutor::selectFunction>}, {"nl.datetime_month", &NLExecutor::selectFunction>}, @@ -407,6 +408,15 @@ const std::unordered_map unaryFunctio {"nl.duration_milliseconds_of_second", &NLExecutor::selectFunction>}, {"nl.duration_microseconds_of_second", &NLExecutor::selectFunction>}, + {"nl.point_x", &NLExecutor::selectFunction>}, + {"nl.point_y", &NLExecutor::selectFunction>}, + {"nl.point_z", &NLExecutor::selectFunction>}, + {"nl.point_longitude", &NLExecutor::selectFunction>}, + {"nl.point_latitude", &NLExecutor::selectFunction>}, + {"nl.point_height", &NLExecutor::selectFunction>}, + {"nl.point_crs", &NLExecutor::selectFunction>}, + {"nl.point_srid", &NLExecutor::selectFunction>}, + {"nl.element_id", &NLExecutor::selectId}, {"nl.size", &NLExecutor::selectSize}, {"nl.is_empty", &NLExecutor::selectIsEmpty}, @@ -1319,6 +1329,10 @@ void NLTranslator::translateBlock(mlir::Block& block, NLStmtContainer* body) { translateStringEnd(right, body); } else if (nl::Replace replace = mlir::dyn_cast(operation)) { translateReplace(replace, body); + } else if (mlir::isa(operation)) { + translatePointFunction(&operation, &NLExecutor::runPointDistance, body); + } else if (mlir::isa(operation)) { + translatePointFunction(&operation, &NLExecutor::runPointWithinBBox, body); } else if (nl::ListComprehension listComprehension = mlir::dyn_cast(operation)) { translateListComprehension(listComprehension, body); } else if (nl::ListPredicate listPredicate = mlir::dyn_cast(operation)) { @@ -4089,13 +4103,15 @@ void NLTranslator::translateStaticMapKey(nl::StaticMapKey mapKey, NLStmtContaine _valueSlots[resultValue] = result; const std::string_view key = ownedCharacters(mapKey.getKey()); + const NLStaticMapKeyData::ComponentParts parts {componentNamed(key), + componentNamed(key), + componentNamed(key)}; NLStaticMapKeyData* data = _program->allocFunctionData(input, result, NLExecutor::selectKeyedRead(input), key, - componentNamed(key), - componentNamed(key), + parts, _program->allocStepBuffers()); body->emplaceStmt(&NLExecutor::runStaticMapKey, data); @@ -4371,6 +4387,22 @@ void NLTranslator::translateReplace(nl::Replace replace, NLStmtContainer* body) } } +void NLTranslator::translatePointFunction(mlir::Operation* operation, NLHandlerFunction run, NLStmtContainer* body) { + const mlir::Value resultValue = operation->getResult(0); + + Column* const result = allocColumnForChunkType(resultValue.getType()); + _valueSlots[resultValue] = result; + + NLPointFunctionData* data = _program->allocFunctionData(result); + + for (const mlir::Value operand : operation->getOperands()) { + const Column* const argument = getColumn(operand); + data->addArgument(NLPointFunctionData::PointArgument {argument, NLExecutor::selectPointArgumentRead(argument)}); + } + + body->emplaceStmt(run, data); +} + template Data* NLTranslator::allocElementData(Op op, Extra... extra) { const mlir::Value sourceValue = op.getSource(); @@ -6890,6 +6922,10 @@ Column* NLTranslator::allocValueColumnForValueType(ValueType valueType) { return _memory->alloc>(); break; + case ValueType::Point: + return _memory->alloc>(); + break; + default: throw IRException("collect does not support this value type"); break; @@ -8020,6 +8056,8 @@ NLChunkKind NLTranslator::chunkKindFromElementType(mlir::Type elementType) { return NLChunkKind::DateTime; } else if (mlir::isa(elementType)) { return NLChunkKind::Duration; + } else if (mlir::isa(elementType)) { + return NLChunkKind::Point; } else if (mlir::isa(elementType)) { return NLChunkKind::PathRef; } else if (mlir::isa(elementType)) { diff --git a/query/ir/exec/NLTranslator.h b/query/ir/exec/NLTranslator.h index 573d4f79a5..fda9eeb866 100644 --- a/query/ir/exec/NLTranslator.h +++ b/query/ir/exec/NLTranslator.h @@ -1107,6 +1107,7 @@ class NLTranslator { void translateSubstring(mlir::nl::Substring substring, NLStmtContainer* body); void translateSplit(mlir::nl::Split split, NLStmtContainer* body); void translateReplace(mlir::nl::Replace replace, NLStmtContainer* body); + void translatePointFunction(mlir::Operation* operation, NLHandlerFunction run, NLStmtContainer* body); template void translateStringEnd(StringEndOp stringEnd, NLStmtContainer* body); diff --git a/query/ir/exec/NLWriteProperties.cpp b/query/ir/exec/NLWriteProperties.cpp index 762dda7344..8433f81cbf 100644 --- a/query/ir/exec/NLWriteProperties.cpp +++ b/query/ir/exec/NLWriteProperties.cpp @@ -372,6 +372,9 @@ void stageCellAsItsType(const CellView cell, case ValueType::Duration: value = std::optional {cell.getAs()}; break; + case ValueType::Point: + value = std::optional {cell.getAs()}; + break; case ValueType::String: value = std::optional {std::in_place, cell.getAs()}; break; diff --git a/query/ir/exec/NLWrittenValues.cpp b/query/ir/exec/NLWrittenValues.cpp index 29429a02cd..710262085d 100644 --- a/query/ir/exec/NLWrittenValues.cpp +++ b/query/ir/exec/NLWrittenValues.cpp @@ -155,5 +155,6 @@ template std::optional NLWrittenValues::read NLWrittenValues::read(const Value& value, NLValueStore& values); template std::optional NLWrittenValues::read(const Value& value, NLValueStore& values); template std::optional NLWrittenValues::read(const Value& value, NLValueStore& values); +template std::optional NLWrittenValues::read(const Value& value, NLValueStore& values); } diff --git a/query/ir/lowering/DBLowering.cpp b/query/ir/lowering/DBLowering.cpp index b34a801cec..1cb8f35108 100644 --- a/query/ir/lowering/DBLowering.cpp +++ b/query/ir/lowering/DBLowering.cpp @@ -208,6 +208,7 @@ const std::unordered_map unaryFunctionL {"db.to_boolean", {&emitNLUnaryFunction, ResultNullability::AlwaysNullable}}, {"db.to_datetime", {&emitNLUnaryFunction, ResultNullability::AlwaysNullable}}, {"db.to_duration", {&emitNLUnaryFunction, ResultNullability::AlwaysNullable}}, + {"db.to_point", {&emitNLUnaryFunction, ResultNullability::AlwaysNullable}}, // A calendar field of an instant is always readable, so the field is null exactly // where the instant it was read off is @@ -238,6 +239,15 @@ const std::unordered_map unaryFunctionL {"db.duration_seconds_of_minute", {&emitNLUnaryFunction, ResultNullability::FollowsInput}}, {"db.duration_milliseconds_of_second", {&emitNLUnaryFunction, ResultNullability::FollowsInput}}, {"db.duration_microseconds_of_second", {&emitNLUnaryFunction, ResultNullability::FollowsInput}}, + + {"db.point_x", {&emitNLUnaryFunction, ResultNullability::FollowsInput}}, + {"db.point_y", {&emitNLUnaryFunction, ResultNullability::FollowsInput}}, + {"db.point_z", {&emitNLUnaryFunction, ResultNullability::FollowsInput}}, + {"db.point_longitude", {&emitNLUnaryFunction, ResultNullability::FollowsInput}}, + {"db.point_latitude", {&emitNLUnaryFunction, ResultNullability::FollowsInput}}, + {"db.point_height", {&emitNLUnaryFunction, ResultNullability::FollowsInput}}, + {"db.point_crs", {&emitNLUnaryFunction, ResultNullability::FollowsInput}}, + {"db.point_srid", {&emitNLUnaryFunction, ResultNullability::FollowsInput}}, {"db.to_string", {&emitNLUnaryFunction, ResultNullability::FollowsInput}}, {"db.to_integer_or_null", {&emitNLUnaryFunction, ResultNullability::AlwaysNullable}}, {"db.to_float_or_null", {&emitNLUnaryFunction, ResultNullability::AlwaysNullable}}, @@ -420,6 +430,7 @@ mlir::Type aggregateResultElementType(mlir::OpBuilder& builder, const bool isString = mlir::isa(inputElement); const bool isDateTime = mlir::isa(inputElement); const bool isDuration = mlir::isa(inputElement); + const bool isPoint = mlir::isa(inputElement); const bool isTaggedCell = mlir::isa(inputElement); // An untyped null holds no value to reduce - a name no property in the graph carries, @@ -445,8 +456,8 @@ mlir::Type aggregateResultElementType(mlir::OpBuilder& builder, break; case storage::AggregateKind::Avg: { - if (!isNumeric && !isDuration && !isTaggedCell) { - throw IRException("db.avg requires a numeric or a duration column"); + if (!isNumeric && !isDuration && !isPoint && !isTaggedCell) { + throw IRException("db.avg requires a numeric, a duration or a point column"); } return storage::reducesToADouble(kind, inputElement) ? builder.getF64Type() : inputElement; } @@ -464,12 +475,12 @@ mlir::Type aggregateResultElementType(mlir::OpBuilder& builder, case storage::AggregateKind::Min: case storage::AggregateKind::Max: { // min/max order the values, so anything Cypher orders is fine - numbers, - // strings, bools, instants, lists, maps, and cells of different types - but - // an embedding has no order. + // strings, bools, instants, points, lists, maps, and cells of different types - + // but an embedding has no order. const bool isList = mlir::isa(inputElement); const bool isMap = mlir::isa(inputElement); const bool isOrderable = isNumeric || isString || isBool || isDateTime || isDuration - || isList || isMap || isTaggedCell; + || isPoint || isList || isMap || isTaggedCell; if (!isOrderable) { throw IRException("db.min/db.max requires an orderable column"); } @@ -1498,6 +1509,10 @@ void DBLowering::lowerOperationByKind(mlir::Operation& operation) { lowerStringEnd(right); } else if (mlir::db::Replace replace = mlir::dyn_cast(operation)) { lowerReplace(replace); + } else if (mlir::isa(operation)) { + lowerPointFunction(&operation, _builder.getF64Type()); + } else if (mlir::isa(operation)) { + lowerPointFunction(&operation, _builder.getI1Type()); } else if (mlir::db::ListSlice listSlice = mlir::dyn_cast(operation)) { lowerListSlice(listSlice); } else if (mlir::isa(operation)) { @@ -2925,6 +2940,23 @@ void DBLowering::lowerReplace(mlir::db::Replace replace) { _valueMap[replace.getResult()] = replaced.getResult(); } +// Two points in different CRSs have no distance and no box, so the result is nullable +// whatever its arguments are +template +void DBLowering::lowerPointFunction(mlir::Operation* operation, mlir::Type valueType) { + llvm::SmallVector arguments; + containerCellChunks(operation->getOperands(), arguments); + + mlir::MLIRContext* const context = _builder.getContext(); + const nl::ChunkType resultType = nl::ChunkType::get(context, storage::NullableType::get(context, valueType)); + + setInsertionForNaryOp(arguments); + + NLOp lowered = _builder.create(_builder.getUnknownLoc(), mlir::TypeRange {resultType}, arguments); + + _valueMap[operation->getResult(0)] = lowered.getResult(); +} + void DBLowering::lowerScanEdges(mlir::db::ScanEdges scanEdges) { // The edge sibling of lowerScanNodes: a scan reads no column, so its loop // sits at the top of the current root block. The nl.scan_edges iterator diff --git a/query/ir/lowering/DBLowering.h b/query/ir/lowering/DBLowering.h index b1857409ca..3d9597986d 100644 --- a/query/ir/lowering/DBLowering.h +++ b/query/ir/lowering/DBLowering.h @@ -304,6 +304,9 @@ class DBLowering { void lowerSplit(mlir::db::Split split); void lowerReplace(mlir::db::Replace replace); + template + void lowerPointFunction(mlir::Operation* operation, mlir::Type valueType); + template void lowerStringEnd(StringEndOp stringEnd); diff --git a/query/parser/CypherGrammar.h b/query/parser/CypherGrammar.h index d9b08f573a..5c464fa2c7 100644 --- a/query/parser/CypherGrammar.h +++ b/query/parser/CypherGrammar.h @@ -364,8 +364,7 @@ class CypherGrammar { LoadJsonlQuery* parseLoadJsonl(); void parseJsonlOptions(JsonlImportSpecs& specs); void parseJsonlOption(JsonlImportSpecs& specs); - template - void parseSpecNames(Spec& specs); + void parseTypedNames(TypedPropertySpec& specs, ValueType type); void parseEmbeddingSpecs(EmbeddingsSpec& specs); std::pair parseEmbeddingSpec(); ListGraphQuery* parseListGraphQuery(); diff --git a/query/parser/CypherGrammarQueries.cpp b/query/parser/CypherGrammarQueries.cpp index 16a30b20b9..9cda383fc2 100644 --- a/query/parser/CypherGrammarQueries.cpp +++ b/query/parser/CypherGrammarQueries.cpp @@ -556,8 +556,7 @@ LoadJsonlQuery* CypherGrammar::parseLoadJsonl() { if (hasOptions) { query->setEmbeddingSpecs(std::move(specs._embeddings)); - query->setDateTimeSpecs(std::move(specs._dateTimes)); - query->setDurationSpecs(std::move(specs._durations)); + query->setTypedPropertySpec(std::move(specs._types)); } locate(query, start); @@ -583,27 +582,28 @@ void CypherGrammar::parseJsonlOption(JsonlImportSpecs& specs) { parseEmbeddingSpecs(specs._embeddings); expect(CypherTokenKind::CBRACK); } else if (accept(CypherTokenKind::DATETIMES)) { - expect(CypherTokenKind::OBRACK); - parseSpecNames(specs._dateTimes); - expect(CypherTokenKind::CBRACK); + parseTypedNames(specs._types, ValueType::DateTime); } else if (accept(CypherTokenKind::DURATIONS)) { - expect(CypherTokenKind::OBRACK); - parseSpecNames(specs._durations); - expect(CypherTokenKind::CBRACK); + parseTypedNames(specs._types, ValueType::Duration); + } else if (accept(CypherTokenKind::POINTS)) { + parseTypedNames(specs._types, ValueType::Point); } else { throwUnexpectedToken(); } } -template -void CypherGrammar::parseSpecNames(Spec& specs) { +void CypherGrammar::parseTypedNames(TypedPropertySpec& specs, ValueType type) { + expect(CypherTokenKind::OBRACK); + do { if (accept(CypherTokenKind::STRING_LITERAL)) { - specs.emplace(_previous._text); + specs.emplace(_previous._text, type); } else { - specs.emplace(parseName()->getName()); + specs.emplace(parseName()->getName(), type); } } while (accept(CypherTokenKind::COMMA)); + + expect(CypherTokenKind::CBRACK); } void CypherGrammar::parseEmbeddingSpecs(EmbeddingsSpec& specs) { @@ -927,13 +927,11 @@ LoadParquetQuery* CypherGrammar::parseLoadParquet() { graphName = parseName(); } - DurationSpec durations; + TypedPropertySpec types; const bool hasDurations = accept(CypherTokenKind::WITH); if (hasDurations) { expect(CypherTokenKind::DURATIONS); - expect(CypherTokenKind::OBRACK); - parseSpecNames(durations); - expect(CypherTokenKind::CBRACK); + parseTypedNames(types, ValueType::Duration); } LoadParquetQuery* query = LoadParquetQuery::create(_ast, fs::Path(std::string(path))); @@ -943,7 +941,7 @@ LoadParquetQuery* CypherGrammar::parseLoadParquet() { } if (hasDurations) { - query->setDurationSpecs(std::move(durations)); + query->setTypedPropertySpec(std::move(types)); } locate(query, start); diff --git a/query/parser/CypherLexer.cpp b/query/parser/CypherLexer.cpp index 0d62577c91..93647d1d6d 100644 --- a/query/parser/CypherLexer.cpp +++ b/query/parser/CypherLexer.cpp @@ -109,6 +109,7 @@ constexpr auto KEYWORDS = std::to_array({ {"OR", CypherTokenKind::OR}, {"ORDER", CypherTokenKind::ORDER}, {"PARQUET", CypherTokenKind::PARQUET}, + {"POINTS", CypherTokenKind::POINTS}, {"PROCEDURES", CypherTokenKind::PROCEDURES}, {"PULL", CypherTokenKind::PULL}, {"PUSH", CypherTokenKind::PUSH}, diff --git a/query/parser/CypherToken.cpp b/query/parser/CypherToken.cpp index 4eb9a994a6..72fdbeed16 100644 --- a/query/parser/CypherToken.cpp +++ b/query/parser/CypherToken.cpp @@ -97,6 +97,7 @@ constexpr auto TOKEN_NAMES = std::to_array({ {CypherTokenKind::HEADERS, "HEADERS"}, {CypherTokenKind::DATETIMES, "DATETIMES"}, {CypherTokenKind::DURATIONS, "DURATIONS"}, + {CypherTokenKind::POINTS, "POINTS"}, {CypherTokenKind::EMBEDDINGS, "EMBEDDINGS"}, {CypherTokenKind::JSONL, "JSONL"}, {CypherTokenKind::LIST, "LIST"}, diff --git a/query/parser/CypherToken.h b/query/parser/CypherToken.h index e034775f17..ebb9133af7 100644 --- a/query/parser/CypherToken.h +++ b/query/parser/CypherToken.h @@ -94,6 +94,7 @@ enum class CypherTokenKind : uint8_t { HEADERS, DATETIMES, DURATIONS, + POINTS, EMBEDDINGS, JSONL, LIST, diff --git a/regress/journey_planner_npm/cypher.ts b/regress/journey_planner_npm/cypher.ts index 1c7a07b90a..20fb3f8ffd 100644 --- a/regress/journey_planner_npm/cypher.ts +++ b/regress/journey_planner_npm/cypher.ts @@ -1,9 +1,9 @@ import assert from "node:assert/strict"; import { Column, ColumnEncoding, ColumnType, TuringClient, TuringQueryError, double } from "turingdb"; -import type { Parameters, QueryResult, Value } from "turingdb"; +import type { Parameters, Point, QueryResult, Value } from "turingdb"; -import { at, embedding, minutes } from "./harness.js"; +import { at, embedding, minutes, point } from "./harness.js"; import type { Case, Check } from "./harness.js"; import { TIMETABLE_GRAPH } from "./timetable.js"; @@ -11,7 +11,8 @@ const OXFORD_FACILITIES = { parking: true, cycleSpaces: 400n, retail: ["café", // Every value type the engine stores or computes, as the npm client decodes it: a 64-bit // integer or an ID as a BigInt, a double as a number, a DateTime as microseconds since the -// epoch and a Duration as microseconds, both BigInt, and an embedding as a Float32Array +// epoch and a Duration as microseconds, both BigInt, an embedding as a Float32Array and a +// point as a Point export const VALUE_CASES: Case[] = [ { name: "reads the scalar properties of the stations", @@ -39,6 +40,42 @@ export const VALUE_CASES: Case[] = [ ["PAD", at("1854-01-16T00:00:00Z"), minutes(10), 1854n], ], }, + { + name: "reads the point property of the stations", + query: "MATCH (s:STATION) RETURN s.crs, s.location ORDER BY s.crs", + types: ["STRING", "POINT"], + ordered: true, + rows: [ + ["DID", point(4326, -1.2429, 51.611)], + ["OXF", point(4326, -1.27, 51.7535)], + ["PAD", point(4326, -0.1755, 51.5154)], + ["RDG", point(4326, -0.9718, 51.4589)], + ["SWI", point(4326, -1.7856, 51.5655)], + ], + }, + { + name: "reads the components of a station's location", + query: "MATCH (s:STATION {crs: 'OXF'}) " + + "RETURN s.location.longitude, s.location.latitude, s.location.x, s.location.crs, s.location.srid", + types: ["DOUBLE", "DOUBLE", "DOUBLE", "STRING", "INT64"], + rows: [[-1.27, 51.7535, -1.27, "wgs-84", 4326n]], + }, + { + name: "reads the location of a node that has none as null", + query: "MATCH (s:STATION {crs: 'OXF'})-[:THU]->(m:MODE) RETURN s.location, m.location", + types: ["POINT", "POINT"], + rows: [[point(4326, -1.27, 51.7535), null]], + }, + { + name: "returns a point in each coordinate reference system", + query: "RETURN point({x: 1.5, y: -2}), point({x: 1, y: 2, z: 3}), " + + "point({longitude: 2.3522, latitude: 48.8566}), point({longitude: 6.8652, latitude: 45.8326, height: 4808})", + types: ["POINT", "POINT", "POINT", "POINT"], + rows: [[point(7203, 1.5, -2), point(9157, 1, 2, 3), point(4326, 2.3522, 48.8566), point(4979, 6.8652, 45.8326, 4808)]], + check: ({ rows }) => { + assert.deepEqual((rows[0] as Point[]).map((value) => value.crs), ["cartesian", "cartesian-3d", "wgs-84", "wgs-84-3d"]); + }, + }, { name: "reads the list and map properties of the stations", query: "MATCH (s:STATION) RETURN s.crs, s.platforms, s.facilities ORDER BY s.crs", @@ -94,6 +131,7 @@ export const VALUE_CASES: Case[] = [ maxInterchange: 60n, lat: 51.7535, lon: -1.27, + location: point(4326, -1.27, 51.7535), stepFree: true, opened: at("1852-10-01T00:00:00Z"), minConnection: minutes(5), @@ -108,7 +146,7 @@ export const VALUE_CASES: Case[] = [ query: "MATCH (s:STATION {crs: 'OXF'}) UNWIND keys(s) AS key RETURN key ORDER BY key", ordered: true, rows: [ - ["crs"], ["facilities"], ["lat"], ["lon"], ["maxInterchange"], ["minConnection"], ["minInterchange"], + ["crs"], ["facilities"], ["lat"], ["location"], ["lon"], ["maxInterchange"], ["minConnection"], ["minInterchange"], ["name"], ["nlc"], ["opened"], ["platforms"], ["profile"], ["sign"], ["stepFree"], ], }, @@ -132,8 +170,9 @@ export const VALUE_CASES: Case[] = [ }, { name: "returns a list holding every kind of value", - query: "RETURN [1, 2.5, 'three', true, null, [4, [5]], {six: 6}, datetime('2026-11-04T08:00:00Z'), duration({minutes: 5})]", - rows: [[[1n, 2.5, "three", true, null, [4n, [5n]], { six: 6n }, at("2026-11-04T08:00:00Z"), minutes(5)]]], + query: "RETURN [1, 2.5, 'three', true, null, [4, [5]], {six: 6}, datetime('2026-11-04T08:00:00Z'), duration({minutes: 5}), " + + "point({x: 1, y: 2})]", + rows: [[[1n, 2.5, "three", true, null, [4n, [5n]], { six: 6n }, at("2026-11-04T08:00:00Z"), minutes(5), point(7203, 1, 2)]]], }, { name: "returns maps nested in maps and lists", @@ -198,11 +237,11 @@ export const VALUE_CASES: Case[] = [ }, { name: "returns a cell of each kind, one kind per row", - query: "UNWIND [1, 'two', 2.5, true, null, [3], {four: 4}, datetime('2026-11-04T08:00:00Z'), duration({minutes: 5})] AS cell " + - "RETURN cell", + query: "UNWIND [1, 'two', 2.5, true, null, [3], {four: 4}, datetime('2026-11-04T08:00:00Z'), duration({minutes: 5}), " + + "point({x: 1, y: 2})] AS cell RETURN cell", types: ["CELL_VIEW"], ordered: true, - rows: [[1n], ["two"], [2.5], [true], [null], [[3n]], [{ four: 4n }], [at("2026-11-04T08:00:00Z")], [minutes(5)]], + rows: [[1n], ["two"], [2.5], [true], [null], [[3n]], [{ four: 4n }], [at("2026-11-04T08:00:00Z")], [minutes(5)], [point(7203, 1, 2)]], }, { name: "returns cells holding a node, an edge and a path", @@ -562,6 +601,15 @@ export function clientChecks(url: string): Check[] { await assert.rejects(client.query("RETURN 1", { parameters }), expectStatus("PARAMETER_ERROR", "$when")); }, }, + { + name: "rejects a point a query returned as a parameter before sending it", + run: async (client) => { + const location = firstValue(await client.query("MATCH (s:STATION {crs: 'OXF'}) RETURN s.location")); + const parameters = { at: location } as unknown as Parameters; + + await assert.rejects(client.query("RETURN 1", { parameters }), expectStatus("PARAMETER_ERROR", "$at is a Point")); + }, + }, { name: "returns plain numbers from queryData", run: async (client) => { @@ -761,7 +809,7 @@ export function authChecks(url: string, token: string): Check[] { } // db::ValueType's ordinals, which are append-only on disk -const VALUE_TYPE = { INT64: 1, DOUBLE: 3, STRING: 4, BOOL: 5, EMBEDDING: 6, LIST: 7, DATE_TIME: 8, MAP: 9, DURATION: 10 }; +const VALUE_TYPE = { INT64: 1, DOUBLE: 3, STRING: 4, BOOL: 5, EMBEDDING: 6, LIST: 7, DATE_TIME: 8, MAP: 9, DURATION: 10, POINT: 11 }; export const SYSTEM_CASES: Case[] = [ { @@ -787,7 +835,7 @@ export const SYSTEM_CASES: Case[] = [ types: ["PROPERTY_TYPE_ID", "STRING", "VALUE_TYPE"], check: ({ rows }) => { const ids = rows.map((row) => row[0] as number).sort((left, right) => left - right); - assert.deepEqual(ids, Array.from({ length: 22 }, (_, index) => index)); + assert.deepEqual(ids, Array.from({ length: 23 }, (_, index) => index)); }, }, { @@ -802,6 +850,7 @@ export const SYSTEM_CASES: Case[] = [ ["end", VALUE_TYPE.INT64], ["facilities", VALUE_TYPE.MAP], ["lat", VALUE_TYPE.DOUBLE], + ["location", VALUE_TYPE.POINT], ["lon", VALUE_TYPE.DOUBLE], ["maxInterchange", VALUE_TYPE.INT64], ["minConnection", VALUE_TYPE.DURATION], diff --git a/regress/journey_planner_npm/harness.ts b/regress/journey_planner_npm/harness.ts index 0e2c843dec..a5e5636264 100644 --- a/regress/journey_planner_npm/harness.ts +++ b/regress/journey_planner_npm/harness.ts @@ -1,6 +1,6 @@ import { isDeepStrictEqual } from "node:util"; -import { ColumnType, TuringClient, TuringQueryError } from "turingdb"; +import { ColumnType, Point, TuringClient, TuringQueryError } from "turingdb"; import type { ColumnTypeName, QueryOptions, Value } from "turingdb"; export type Row = Value[]; @@ -59,6 +59,13 @@ export function embedding(...values: number[]): Float32Array { return new Float32Array(values); } +// The typings keep Point's constructor private, since no parameter holds a point, so the +// expected value is built through the class the client decodes into +export function point(srid: Point["srid"], x: number, y: number, z?: number): Point { + const PointConstructor = Point as unknown as new (srid: number, x: number, y: number, z: number | undefined) => Point; + return new PointConstructor(srid, x, y, z); +} + // Renders a decoded value so an Int64 (480) reads apart from a Double (480.0) function describeValue(value: Value | undefined): string { if (value === undefined) { @@ -75,6 +82,9 @@ function describeValue(value: Value | undefined): string { return value.toString(); } else if (value instanceof Float32Array) { return `embedding(${Array.from(value).join(", ")})`; + } else if (value instanceof Point) { + const z = value.z === undefined ? "" : `, ${describeValue(value.z)}`; + return `point(${value.srid}, ${describeValue(value.x)}, ${describeValue(value.y)}${z})`; } else if (Array.isArray(value)) { return `[${value.map(describeValue).join(", ")}]`; } diff --git a/regress/journey_planner_npm/journey.ts b/regress/journey_planner_npm/journey.ts index 561ef8430e..e4ad5d466f 100644 --- a/regress/journey_planner_npm/journey.ts +++ b/regress/journey_planner_npm/journey.ts @@ -636,4 +636,19 @@ export const JOURNEY_PLUS_CASES: Case[] = [ ordered: true, rows: [["DID", 16.0], ["RDG", 39.0], ["SWI", 41.0], ["PAD", 80.0]], }, + { + name: "measures each station's distance from Oxford between their locations", + query: "MATCH (o:STATION {crs: 'OXF'}), (s:STATION) WHERE s <> o " + + "WITH s, point.distance(o.location, s.location) / 1000 AS kilometres " + + "RETURN s.crs, round(kilometres) ORDER BY kilometres", + ordered: true, + rows: [["DID", 16.0], ["RDG", 39.0], ["SWI", 41.0], ["PAD", 80.0]], + }, + { + name: "finds the stations inside a box around Didcot and Reading", + query: "MATCH (s:STATION) " + + "WHERE point.withinBBox(s.location, point({longitude: -1.3, latitude: 51.4}), point({longitude: -0.9, latitude: 51.7})) " + + "RETURN s.crs", + rows: [["DID"], ["RDG"]], + }, ]; diff --git a/regress/journey_planner_npm/timetable.ts b/regress/journey_planner_npm/timetable.ts index 8809f98b6e..37ca972126 100644 --- a/regress/journey_planner_npm/timetable.ts +++ b/regress/journey_planner_npm/timetable.ts @@ -29,19 +29,19 @@ type FixedLink = { }; const STATIONS = - "CREATE (:STATION {crs: 'OXF', nlc: '3115', name: 'Oxford', minInterchange: 5, maxInterchange: 60, lat: 51.7535, lon: -1.27, " + + "CREATE (:STATION {crs: 'OXF', nlc: '3115', name: 'Oxford', minInterchange: 5, maxInterchange: 60, lat: 51.7535, lon: -1.27, location: point({longitude: -1.27, latitude: 51.7535}), " + "stepFree: true, opened: datetime('1852-10-01T00:00:00Z'), minConnection: duration({minutes: 5}), platforms: [1, 2, 3, 4], " + "facilities: {parking: true, cycleSpaces: 400, retail: ['café', 'newsagent']}, profile: (0.9, 0.3, 0.5), sign: 'Oxford 🚆'}), " + - "(:STATION {crs: 'DID', nlc: '3128', name: 'Didcot Parkway', minInterchange: 4, maxInterchange: 45, lat: 51.611, lon: -1.2429, " + + "(:STATION {crs: 'DID', nlc: '3128', name: 'Didcot Parkway', minInterchange: 4, maxInterchange: 45, lat: 51.611, lon: -1.2429, location: point({longitude: -1.2429, latitude: 51.611}), " + "stepFree: true, opened: datetime('1844-06-12T00:00:00Z'), minConnection: duration({minutes: 4}), platforms: [1, 2, 3, 4, 5], " + "facilities: {parking: true, cycleSpaces: 120, retail: ['café']}, profile: (0.4, 0.8, 0.2)}), " + - "(:STATION {crs: 'RDG', nlc: '3149', name: 'Reading', minInterchange: 6, maxInterchange: 50, lat: 51.4589, lon: -0.9718, " + + "(:STATION {crs: 'RDG', nlc: '3149', name: 'Reading', minInterchange: 6, maxInterchange: 50, lat: 51.4589, lon: -0.9718, location: point({longitude: -0.9718, latitude: 51.4589}), " + "stepFree: true, opened: datetime('1840-03-30T00:00:00Z'), minConnection: duration({minutes: 6}), platforms: [7, 8, 9, 10, 11, 12, 13, 14, 15], " + "facilities: {parking: true, cycleSpaces: 800, retail: ['café', 'pharmacy', 'bookshop']}, profile: (0.8, 0.9, 0.4)}), " + - "(:STATION {crs: 'PAD', nlc: '3087', name: 'London Paddington', minInterchange: 10, maxInterchange: 90, lat: 51.5154, lon: -0.1755, " + + "(:STATION {crs: 'PAD', nlc: '3087', name: 'London Paddington', minInterchange: 10, maxInterchange: 90, lat: 51.5154, lon: -0.1755, location: point({longitude: -0.1755, latitude: 51.5154}), " + "stepFree: true, opened: datetime('1854-01-16T00:00:00Z'), minConnection: duration({minutes: 10}), platforms: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], " + "facilities: {parking: false, cycleSpaces: 250, retail: ['bookshop', 'café', 'florist']}, profile: (1.0, 0.7, 0.9)}), " + - "(:STATION {crs: 'SWI', nlc: '3138', name: 'Swindon', minInterchange: 5, maxInterchange: 40, lat: 51.5655, lon: -1.7856, " + + "(:STATION {crs: 'SWI', nlc: '3138', name: 'Swindon', minInterchange: 5, maxInterchange: 40, lat: 51.5655, lon: -1.7856, location: point({longitude: -1.7856, latitude: 51.5655}), " + "stepFree: false, opened: datetime('1842-07-17T00:00:00Z'), minConnection: duration({minutes: 5}), platforms: [1, 2, 3, 4], " + "facilities: {parking: true, cycleSpaces: 60, retail: []}, profile: (0.5, 0.6, 0.3)})"; diff --git a/samples/list-buffer/main.cpp b/samples/list-buffer/main.cpp index e9c94c5414..89294fe6f6 100644 --- a/samples/list-buffer/main.cpp +++ b/samples/list-buffer/main.cpp @@ -37,6 +37,11 @@ int main() { std::string formatted; Duration::format(formatted, value); + spdlog::info("element: {}", formatted); + } else if constexpr (std::is_same_v) { + std::string formatted; + Point::format(formatted, value); + spdlog::info("element: {}", formatted); } else { spdlog::info("element: {}", value); diff --git a/samples/mlir/main.cpp b/samples/mlir/main.cpp index 463c755600..de378adc1e 100644 --- a/samples/mlir/main.cpp +++ b/samples/mlir/main.cpp @@ -485,6 +485,14 @@ class PrintingSink : public NLOutputSink { } break; + case CellTag::Point: { + std::string formatted; + Point::format(formatted, element.getAs()); + + std::cout << formatted; + } + break; + case CellTag::MapView: case CellTag::Path: case CellTag::INVALID: diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt index 25d5a7e0da..1e5a68350e 100644 --- a/storage/CMakeLists.txt +++ b/storage/CMakeLists.txt @@ -45,6 +45,7 @@ set(storage_sources metadata/PropertyTypeMap.cpp metadata/LabelMap.cpp metadata/LabelSetMap.cpp + metadata/Point.cpp views/EntityPropertyView.cpp views/GraphView.cpp diff --git a/storage/CellOrder.cpp b/storage/CellOrder.cpp index 2a48c33022..45eedd1694 100644 --- a/storage/CellOrder.cpp +++ b/storage/CellOrder.cpp @@ -30,6 +30,7 @@ enum class CellOrderClass { Edge, List, Path, + Point, DateTime, Duration, String, @@ -90,6 +91,10 @@ CellOrderClass orderClassOf(CellTag tag) { return CellOrderClass::Path; break; + case CellTag::Point: + return CellOrderClass::Point; + break; + case CellTag::INVALID: throw FatalException("Cannot order an untagged list element."); break; @@ -211,6 +216,10 @@ std::strong_ordering compareValues(const Lhs lhs, const Rhs rhs) { } break; + case CellOrderClass::Point: + return valueOf(lhs) <=> valueOf(rhs); + break; + case CellOrderClass::String: return valueOf(lhs) <=> valueOf(rhs); break; @@ -348,6 +357,10 @@ std::optional taggedComparisonOrder(const Lhs lhs, const return sameTypeComparisonOrder(lhs, rhsTag, valueOf(rhs)); break; + case CellTag::Point: + return sameTypeComparisonOrder(lhs, rhsTag, valueOf(rhs)); + break; + case CellTag::ListView: return sameTypeComparisonOrder(lhs, rhsTag, valueOf(rhs)); break; @@ -597,6 +610,10 @@ std::optional db::comparisonOrder(const CellView element, return sameTypeComparisonOrder(element, CellTag::Duration, value); } +std::optional db::comparisonOrder(const CellView element, const types::Point::Primitive value) { + return sameTypeComparisonOrder(element, CellTag::Point, value); +} + std::optional db::comparisonOrder(const CellView element, const ListView value) { return sameTypeComparisonOrder(element, CellTag::ListView, value); } @@ -618,3 +635,12 @@ std::strong_ordering db::operator<=>(const CellView element, const types::Durati return element.getAs() <=> value; } + +std::strong_ordering db::operator<=>(const CellView element, const types::Point::Primitive value) { + const CellOrderClass elementClass = orderClassOf(element.getTag()); + if (elementClass != CellOrderClass::Point) { + return elementClass <=> CellOrderClass::Point; + } + + return element.getAs() <=> value; +} diff --git a/storage/CellOrder.h b/storage/CellOrder.h index 977d543979..bc1bd3f47a 100644 --- a/storage/CellOrder.h +++ b/storage/CellOrder.h @@ -116,6 +116,11 @@ inline bool operator==(const CellView element, const types::Duration::Primitive && element.getAs() == value; } +inline bool operator==(const CellView element, const types::Point::Primitive value) { + return element.getTag() == CellTag::Point + && element.getAs() == value; +} + inline bool operator==(const CellView element, const types::Embedding::Primitive value) { return element.getTag() == CellTag::Embedding && EmbeddingEqual {}(element.getAs(), value); @@ -164,11 +169,12 @@ std::strong_ordering operator<=>(CellView element, types::String::Primitive valu std::strong_ordering operator<=>(CellView element, types::Bool::Primitive value); std::strong_ordering operator<=>(CellView element, types::DateTime::Primitive value); std::strong_ordering operator<=>(CellView element, types::Duration::Primitive value); +std::strong_ordering operator<=>(CellView element, types::Point::Primitive value); /** * @brief Compares an element of a @ref ListByteBuffer as <, <=, > and >= do. Unlike the order - * above, only two numbers, strings, booleans, datetimes, durations or lists compare: any other - * pair, a null on either side included, is absent. A NaN is unordered against every number. + * above, only two numbers, strings, booleans, datetimes, durations, points or lists compare: any + * other pair, a null on either side included, is absent. A NaN is unordered against every number. */ std::optional comparisonOrder(CellView lhs, CellView rhs); std::optional comparisonOrder(ListView lhs, ListView rhs); @@ -180,5 +186,6 @@ std::optional comparisonOrder(CellView element, types::St std::optional comparisonOrder(CellView element, types::Bool::Primitive value); std::optional comparisonOrder(CellView element, types::DateTime::Primitive value); std::optional comparisonOrder(CellView element, types::Duration::Primitive value); +std::optional comparisonOrder(CellView element, types::Point::Primitive value); } diff --git a/storage/CellTag.h b/storage/CellTag.h index b5df797a7e..49b4f5bbcb 100644 --- a/storage/CellTag.h +++ b/storage/CellTag.h @@ -21,6 +21,7 @@ enum class CellTag : uint8_t { MapView, Duration, Path, + Point, INVALID, }; @@ -39,5 +40,6 @@ static_assert(static_cast(CellTag::DateTime) == 10); static_assert(static_cast(CellTag::MapView) == 11); static_assert(static_cast(CellTag::Duration) == 12); static_assert(static_cast(CellTag::Path) == 13); +static_assert(static_cast(CellTag::Point) == 14); } diff --git a/storage/CellUtils.h b/storage/CellUtils.h index 852b059953..036903b6d7 100644 --- a/storage/CellUtils.h +++ b/storage/CellUtils.h @@ -61,6 +61,9 @@ struct CellTagDispatcher { case CellTag::Path: return executor.template operator()(view); break; + case CellTag::Point: + return executor.template operator()(view); + break; case CellTag::INVALID: break; @@ -143,4 +146,9 @@ struct TypeToCellTag { static constexpr CellTag Tag = CellTag::Path; }; +template <> +struct TypeToCellTag { + static constexpr CellTag Tag = CellTag::Point; +}; + } diff --git a/storage/DebugDump.cpp b/storage/DebugDump.cpp index d35bd812d7..d1d705922d 100644 --- a/storage/DebugDump.cpp +++ b/storage/DebugDump.cpp @@ -62,6 +62,13 @@ void DebugDump::dump(std::ostream& out, CellView view, bool isInList) { dumper.execute(dumpTyped, view); } +void DebugDump::dump(std::ostream& out, Point value) { + std::string text; + Point::format(text, value); + + out << text; +} + void DebugDump::dump(std::ostream& out, ListView list) { if (list.empty()) { out << "[]\n"; diff --git a/storage/DebugDump.h b/storage/DebugDump.h index 97a5db16ff..c7d2b26689 100644 --- a/storage/DebugDump.h +++ b/storage/DebugDump.h @@ -78,6 +78,8 @@ class DebugDump { static void dump(std::ostream& out, PathView path); + static void dump(std::ostream& out, Point value); + private: static void dumpImpl(std::ostream& out, uint64_t data); }; diff --git a/storage/QueryTypeName.cpp b/storage/QueryTypeName.cpp index c154077034..501f617ee8 100644 --- a/storage/QueryTypeName.cpp +++ b/storage/QueryTypeName.cpp @@ -46,6 +46,9 @@ std::string_view db::queryTypeName(CellTag tag) { case CellTag::Path: return "GraphPath"; break; + case CellTag::Point: + return "Point"; + break; case CellTag::INVALID: return "Invalid"; break; @@ -84,6 +87,9 @@ std::string_view db::queryTypeName(ValueType type) { case ValueType::Duration: return "Duration"; break; + case ValueType::Point: + return "Point"; + break; case ValueType::Invalid: case ValueType::_SIZE: return "Invalid"; diff --git a/storage/columns/AllowedKinds.h b/storage/columns/AllowedKinds.h index 408bd5800c..70ec4bc18a 100644 --- a/storage/columns/AllowedKinds.h +++ b/storage/columns/AllowedKinds.h @@ -106,6 +106,7 @@ struct SearchedValueKindPairs { typename SearchPairs::Pairs, typename SearchPairs::Pairs, typename SearchPairs::Pairs, + typename SearchPairs::Pairs, typename SearchPairs::Pairs, typename SearchPairs::Pairs, typename SearchPairs::Pairs, @@ -189,6 +190,7 @@ struct PairRestrictions { OptionalKindPairs::Pairs, OptionalKindPairs::Pairs, OptionalKindPairs::Pairs, + OptionalKindPairs::Pairs, OptionalKindPairs::Pairs, OptionalKindPairs::Pairs, @@ -200,6 +202,7 @@ struct PairRestrictions { CellKindPairs::Pairs, CellKindPairs::Pairs, CellKindPairs::Pairs, + CellKindPairs::Pairs, CellKindPairs::Pairs, OptionalKindPairs::Pairs, CellKindPairs::Pairs, @@ -274,6 +277,9 @@ struct PairRestrictions { OptionalKindPairs::Pairs, + // A point orders against another point, across CRSs as ORDER BY sorts them + OptionalKindPairs::Pairs, + OptionalKindPairs::Pairs, // Lexicographic ordering of strings, whether each side borrows its characters @@ -293,6 +299,7 @@ struct PairRestrictions { CellKindPairs::Pairs, CellKindPairs::Pairs, CellKindPairs::Pairs, + CellKindPairs::Pairs, OptionalKindPairs::Pairs, // A map value is a type-erased cell too, and orders the same way @@ -730,6 +737,7 @@ struct OutputtedTypes { types::Embedding::Primitive, types::DateTime::Primitive, types::Duration::Primitive, + types::Point::Primitive, std::optional, std::optional, std::optional, @@ -739,6 +747,7 @@ struct OutputtedTypes { std::optional, std::optional, std::optional, + std::optional, PropertyNull, std::optional, @@ -823,6 +832,7 @@ struct WriteProcessorPropertyTypes { types::Embedding::Primitive, types::DateTime::Primitive, types::Duration::Primitive, + types::Point::Primitive, ListView, MapView, @@ -834,6 +844,7 @@ struct WriteProcessorPropertyTypes { std::optional, std::optional, std::optional, + std::optional, std::optional, std::optional, @@ -853,6 +864,7 @@ struct WriteProcessorPropertyTypes { types::Embedding::Primitive, types::DateTime::Primitive, types::Duration::Primitive, + types::Point::Primitive, ListView, MapView, @@ -864,6 +876,7 @@ struct WriteProcessorPropertyTypes { std::optional, std::optional, std::optional, + std::optional, std::optional, std::string // For LOAD CSV inputs @@ -1003,6 +1016,9 @@ struct CartesianProductKinds { types::Duration::Primitive, std::optional, + types::Point::Primitive, + std::optional, + // Entities and metadata NodeID, EdgeID, diff --git a/storage/columns/BinaryOperators.h b/storage/columns/BinaryOperators.h index 203e3ed405..25fd3c7d32 100644 --- a/storage/columns/BinaryOperators.h +++ b/storage/columns/BinaryOperators.h @@ -184,6 +184,7 @@ inline std::optional cellNumber(const CellView cell) { case CellTag::MapView: case CellTag::Duration: case CellTag::Path: + case CellTag::Point: case CellTag::INVALID: return std::nullopt; break; diff --git a/storage/columns/ColumnDispatcher.h b/storage/columns/ColumnDispatcher.h index 7760796c05..6b390496fb 100644 --- a/storage/columns/ColumnDispatcher.h +++ b/storage/columns/ColumnDispatcher.h @@ -39,6 +39,7 @@ using AllValueTypes = TypeList< types::Embedding::Primitive, types::DateTime::Primitive, types::Duration::Primitive, + types::Point::Primitive, std::optional, std::optional, std::optional, @@ -47,6 +48,7 @@ using AllValueTypes = TypeList< std::optional, std::optional, std::optional, + std::optional, std::optional, std::optional, std::string, diff --git a/storage/columns/Functions.h b/storage/columns/Functions.h index 666826a4e1..e489b9797c 100644 --- a/storage/columns/Functions.h +++ b/storage/columns/Functions.h @@ -419,6 +419,25 @@ class MapToDurationFunction { } }; +// point() over a map, the only value it reads. A map holding a null reads as null. +class ToPointFunction { +public: + using ArgType = MapView; + using ResultType = std::optional; + using TaggedCounterpart = TaggedConversionFunction; + + static constexpr std::string_view NAME = "point"; + + ResultType operator()(const MapView map) const { + types::Point::Primitive value; + if (!Point::fromMap(map, value)) { + return std::nullopt; + } + + return value; + } +}; + class MapToDateTimeFunction { public: using ArgType = MapView; @@ -529,6 +548,53 @@ class DurationComponentFunction { } }; +template +using PointComponent = std::conditional_t>; + +template +PointComponent readPointComponent(const types::Point::Primitive value) { + return std::get>(Point::component(value, part)); +} + +template +class TaggedPointComponentFunction { +public: + using ArgType = CellView; + using ResultType = std::optional>; + + ResultType operator()(const ArgType cell) const { + const CellTag tag = cell.getTag(); + + if (tag == CellTag::Point) { + return readPointComponent(cell.getAs()); + } else if (tag == CellTag::Null) { + return std::nullopt; + } + + throw TuringException(fmt::format("a point component reads a point, not '{}'", queryTypeName(tag))); + } + + ResultType operator()(const std::optional& cell) const { + return cell.has_value() ? (*this)(*cell) : std::nullopt; + } +}; + +template +class PointComponentFunction { +public: + using ArgType = types::Point::Primitive; + using ResultType = PointComponent; + using TaggedCounterpart = TaggedPointComponentFunction; + + ResultType operator()(const types::Point::Primitive value) { + return readPointComponent(value); + } +}; + // toInteger() and toFloat() over a number rather than a string. Every conversion keeps an // optional result so one column type carries it whatever the argument was, and so a double // that no integer can represent reads as null. @@ -600,20 +666,20 @@ class ToBoolFromValueFunction { } }; -template -class ToStringFromTemporalFunction { +template +class ToStringFromFormattedFunction { public: - using ArgType = Temporal; + using ArgType = Formatted; using ResultType = types::String::Primitive; - explicit ToStringFromTemporalFunction(StringBuffer* stringBuffer) + explicit ToStringFromFormattedFunction(StringBuffer* stringBuffer) : _stringBuffer(stringBuffer) { } - ResultType operator()(const Temporal value) { + ResultType operator()(const Formatted value) { _formatted.clear(); - Temporal::format(_formatted, value); + Formatted::format(_formatted, value); return _stringBuffer->insert(_formatted); } @@ -632,9 +698,11 @@ concept ConvertibleToBool = std::is_same_v || std::is_same_v || std::is_same_v; +// A value whose text is the one its type's format writes template -concept TemporalValue = std::is_same_v - || std::is_same_v; +concept FormattedValue = std::is_same_v + || std::is_same_v + || std::is_same_v; // The element a column of the given type converts, with any optional unwrapped: what a // conversion is applied to row by row. @@ -668,9 +736,9 @@ struct ConversionFunctorFor { using Type = ToStringFromValueFunction; }; -template +template struct ConversionFunctorFor { - using Type = ToStringFromTemporalFunction; + using Type = ToStringFromFormattedFunction; }; template <> @@ -769,7 +837,8 @@ class TaggedConversionFunction { _fromString(stringBuffer), _fromDateTime(stringBuffer), _fromDuration(stringBuffer), - _fromMap(stringBuffer) + _fromMap(stringBuffer), + _fromPoint(stringBuffer) { } @@ -801,6 +870,9 @@ class TaggedConversionFunction { case CellTag::MapView: return convert(cell, _fromMap); break; + case CellTag::Point: + return convert(cell, _fromPoint); + break; case CellTag::Null: return std::nullopt; break; @@ -832,6 +904,7 @@ class TaggedConversionFunction { ConversionRule> _fromDateTime; ConversionRule> _fromDuration; ConversionRule> _fromMap; + ConversionRule> _fromPoint; template ResultType convert(const ArgType cell, ConversionRule>& rule) { diff --git a/storage/columns/InternalKind.h b/storage/columns/InternalKind.h index b46d9705d6..fa2d2a9aab 100644 --- a/storage/columns/InternalKind.h +++ b/storage/columns/InternalKind.h @@ -75,6 +75,7 @@ class InternalKind { types::Embedding::Primitive, types::DateTime::Primitive, types::Duration::Primitive, + types::Point::Primitive, std::optional, std::optional, std::optional, @@ -91,6 +92,7 @@ class InternalKind { std::optional, std::optional, std::optional, + std::optional, std::optional, std::optional, std::optional, diff --git a/storage/comparators/PropertyContainerComparator.h b/storage/comparators/PropertyContainerComparator.h index f2d71f99e1..fff345cece 100644 --- a/storage/comparators/PropertyContainerComparator.h +++ b/storage/comparators/PropertyContainerComparator.h @@ -184,6 +184,15 @@ class PropertyContainerComparator { break; } + case ValueType::Point: { + if (!TypedPropertyContainerComparator::same( + a->cast(), + b->cast())) { + return false; + } + break; + } + case ValueType::Map: { if (!TypedPropertyContainerComparator::same(a->cast(), b->cast())) { return false; diff --git a/storage/dump/DataPartDumper.cpp b/storage/dump/DataPartDumper.cpp index c8e3ac9433..dba2f69f5e 100644 --- a/storage/dump/DataPartDumper.cpp +++ b/storage/dump/DataPartDumper.cpp @@ -89,6 +89,13 @@ DumpResult dumpProperties(fs::FilePageWriter& writer, PropertyContainer* c } break; } + case ValueType::Point: { + TrivialPropertyContainerDumper dumper(writer); + if (auto res = dumper.dump(container->cast()); !res) { + return res.get_unexpected(); + } + break; + } case ValueType::Map: { MapPropertyContainerDumper dumper(writer); if (auto res = dumper.dump(container->cast()); !res) { diff --git a/storage/dump/DataPartLoader.cpp b/storage/dump/DataPartLoader.cpp index bbcd4dc532..ff29b8d4b9 100644 --- a/storage/dump/DataPartLoader.cpp +++ b/storage/dump/DataPartLoader.cpp @@ -206,6 +206,8 @@ DumpResult> DataPartLoader::load(DataPartID partID, manager._dateTimes.emplace(pt->_id, static_cast(ptr)); } else if constexpr (std::is_same_v) { manager._durations.emplace(pt->_id, static_cast(ptr)); + } else if constexpr (std::is_same_v) { + manager._points.emplace(pt->_id, static_cast(ptr)); } else { bioassert(false, "Missing trivial property type"); } @@ -301,6 +303,12 @@ DumpResult> DataPartLoader::load(DataPartID partID, } break; } + case ValueType::Point: { + if (auto res = storeTrivialContainer.operator()(manager); !res) { + return res.get_unexpected(); + } + break; + } case ValueType::Map: { MapPropertyContainerLoader loader(reader.value()); diff --git a/storage/dump/PropertyContainerDumpConstants.h b/storage/dump/PropertyContainerDumpConstants.h index e517af3dc0..d1511e8340 100644 --- a/storage/dump/PropertyContainerDumpConstants.h +++ b/storage/dump/PropertyContainerDumpConstants.h @@ -5,6 +5,7 @@ #include #include "ID.h" +#include "metadata/Point.h" #include "metadata/SupportedType.h" #include "DumpConfig.h" #include "StringBucket.h" @@ -19,6 +20,9 @@ static_assert(std::is_trivially_copyable_v, "Dump format writes Entity static_assert(sizeof(StringBucket::StringLimits) == 2 * sizeof(uint32_t), "Dump format writes StringLimits object bytes as packed uint32 pairs"); static_assert(offsetof(StringBucket::StringLimits, _offset) == 0, "Loaders read StringLimits as offset then count"); static_assert(offsetof(StringBucket::StringLimits, _count) == sizeof(uint32_t), "Loaders read StringLimits as offset then count"); +static_assert(sizeof(Point) == 4 * sizeof(double), "Dump format writes a Point as x, y, z, then its CRS ordinal and padding"); +static_assert(std::is_trivially_copyable_v, "Dump format writes Point object bytes"); +static_assert(std::is_trivially_default_constructible_v, "Loaders read Point values straight out of the mapped page"); template class TrivialPropertyContainerDumpConstants { diff --git a/storage/indexes/IndexManager.cpp b/storage/indexes/IndexManager.cpp index 7f8115244d..2faf42474a 100644 --- a/storage/indexes/IndexManager.cpp +++ b/storage/indexes/IndexManager.cpp @@ -56,6 +56,7 @@ template WeakArc IndexManager::createNodeIndex(std::str template WeakArc IndexManager::createNodeIndex(std::string_view indexName, PropertyTypeID ptID); template WeakArc IndexManager::createNodeIndex(std::string_view indexName, PropertyTypeID ptID); template WeakArc IndexManager::createNodeIndex(std::string_view indexName, PropertyTypeID ptID); +template WeakArc IndexManager::createNodeIndex(std::string_view indexName, PropertyTypeID ptID); template WeakArc IndexManager::createNodeIndex(std::string_view indexName, PropertyTypeID ptID); template WeakArc IndexManager::createEdgeIndex(std::string_view indexName, PropertyTypeID ptID); @@ -67,5 +68,6 @@ template WeakArc IndexManager::createEdgeIndex(std::str template WeakArc IndexManager::createEdgeIndex(std::string_view indexName, PropertyTypeID ptID); template WeakArc IndexManager::createEdgeIndex(std::string_view indexName, PropertyTypeID ptID); template WeakArc IndexManager::createEdgeIndex(std::string_view indexName, PropertyTypeID ptID); +template WeakArc IndexManager::createEdgeIndex(std::string_view indexName, PropertyTypeID ptID); template WeakArc IndexManager::createEdgeIndex(std::string_view indexName, PropertyTypeID ptID); } diff --git a/storage/indexes/PropertyHashIndex.cpp b/storage/indexes/PropertyHashIndex.cpp index 9df3a6b99e..4080d26ffa 100644 --- a/storage/indexes/PropertyHashIndex.cpp +++ b/storage/indexes/PropertyHashIndex.cpp @@ -220,6 +220,7 @@ template class PropertyHashIndex; template class PropertyHashIndex; template class PropertyHashIndex; template class PropertyHashIndex; +template class PropertyHashIndex; template class PropertyHashIndex; template class PropertyHashIndex; @@ -231,5 +232,6 @@ template class PropertyHashIndex; template class PropertyHashIndex; template class PropertyHashIndex; template class PropertyHashIndex; +template class PropertyHashIndex; template class PropertyHashIndex; } diff --git a/storage/iterators/GetPropertiesWithNullIterator.cpp b/storage/iterators/GetPropertiesWithNullIterator.cpp index 973d2cfa57..b7956cae7a 100644 --- a/storage/iterators/GetPropertiesWithNullIterator.cpp +++ b/storage/iterators/GetPropertiesWithNullIterator.cpp @@ -118,6 +118,7 @@ template class GetPropertiesIteratorWithNull; template class GetPropertiesIteratorWithNull; template class GetPropertiesIteratorWithNull; template class GetPropertiesIteratorWithNull; +template class GetPropertiesIteratorWithNull; template class GetPropertiesIteratorWithNull; template class GetPropertiesIteratorWithNull; template class GetPropertiesIteratorWithNull; @@ -128,6 +129,7 @@ template class GetPropertiesIteratorWithNull; template class GetPropertiesIteratorWithNull; template class GetPropertiesIteratorWithNull; template class GetPropertiesIteratorWithNull; +template class GetPropertiesIteratorWithNull; template class GetPropertiesIteratorWithNull; } @@ -165,6 +167,7 @@ template class GetPropertiesWithNullChunkWriter; template class GetPropertiesWithNullChunkWriter; template class GetPropertiesWithNullChunkWriter; template class GetPropertiesWithNullChunkWriter; +template class GetPropertiesWithNullChunkWriter; template class GetPropertiesWithNullChunkWriter; template class GetPropertiesWithNullChunkWriter; template class GetPropertiesWithNullChunkWriter; @@ -175,6 +178,7 @@ template class GetPropertiesWithNullChunkWriter; template class GetPropertiesWithNullChunkWriter; template class GetPropertiesWithNullChunkWriter; template class GetPropertiesWithNullChunkWriter; +template class GetPropertiesWithNullChunkWriter; template class GetPropertiesWithNullChunkWriter; } diff --git a/storage/iterators/ScanEdgePropertiesIterator.cpp b/storage/iterators/ScanEdgePropertiesIterator.cpp index 4287655392..de5762b521 100644 --- a/storage/iterators/ScanEdgePropertiesIterator.cpp +++ b/storage/iterators/ScanEdgePropertiesIterator.cpp @@ -230,6 +230,7 @@ template class ScanEdgePropertiesIterator; template class ScanEdgePropertiesIterator; template class ScanEdgePropertiesIterator; template class ScanEdgePropertiesIterator; +template class ScanEdgePropertiesIterator; template class ScanEdgePropertiesIterator; template class ScanEdgePropertiesChunkWriter; @@ -241,6 +242,7 @@ template class ScanEdgePropertiesChunkWriter; template class ScanEdgePropertiesChunkWriter; template class ScanEdgePropertiesChunkWriter; template class ScanEdgePropertiesChunkWriter; +template class ScanEdgePropertiesChunkWriter; template class ScanEdgePropertiesChunkWriter; } diff --git a/storage/iterators/ScanNodePropertiesByLabelIterator.cpp b/storage/iterators/ScanNodePropertiesByLabelIterator.cpp index 68d2665bb4..41dff3cd7f 100644 --- a/storage/iterators/ScanNodePropertiesByLabelIterator.cpp +++ b/storage/iterators/ScanNodePropertiesByLabelIterator.cpp @@ -269,6 +269,7 @@ template class ScanNodePropertiesByLabelIterator; template class ScanNodePropertiesByLabelIterator; template class ScanNodePropertiesByLabelIterator; template class ScanNodePropertiesByLabelIterator; +template class ScanNodePropertiesByLabelIterator; template class ScanNodePropertiesByLabelIterator; template class ScanNodePropertiesByLabelChunkWriter; @@ -280,6 +281,7 @@ template class ScanNodePropertiesByLabelChunkWriter; template class ScanNodePropertiesByLabelChunkWriter; template class ScanNodePropertiesByLabelChunkWriter; template class ScanNodePropertiesByLabelChunkWriter; +template class ScanNodePropertiesByLabelChunkWriter; template class ScanNodePropertiesByLabelChunkWriter; } diff --git a/storage/iterators/ScanNodePropertiesIterator.cpp b/storage/iterators/ScanNodePropertiesIterator.cpp index 6e1e9f6cf6..9b5d536a5b 100644 --- a/storage/iterators/ScanNodePropertiesIterator.cpp +++ b/storage/iterators/ScanNodePropertiesIterator.cpp @@ -228,6 +228,7 @@ template class ScanNodePropertiesIterator; template class ScanNodePropertiesIterator; template class ScanNodePropertiesIterator; template class ScanNodePropertiesIterator; +template class ScanNodePropertiesIterator; template class ScanNodePropertiesIterator; template class ScanNodePropertiesChunkWriter; @@ -239,6 +240,7 @@ template class ScanNodePropertiesChunkWriter; template class ScanNodePropertiesChunkWriter; template class ScanNodePropertiesChunkWriter; template class ScanNodePropertiesChunkWriter; +template class ScanNodePropertiesChunkWriter; template class ScanNodePropertiesChunkWriter; } diff --git a/storage/list/EncodedList.cpp b/storage/list/EncodedList.cpp index dc4337b953..672db9723a 100644 --- a/storage/list/EncodedList.cpp +++ b/storage/list/EncodedList.cpp @@ -61,6 +61,9 @@ void encodeElement(CellView element, std::vector& out) { case CellTag::Duration: appendValue(out, element.getAs()); break; + case CellTag::Point: + appendValue(out, element.getAs()); + break; case CellTag::Null: break; case CellTag::String: { @@ -190,6 +193,9 @@ class ListDecoder { case CellTag::Duration: return read(); break; + case CellTag::Point: + return read(); + break; case CellTag::Null: return PropertyNull {}; break; diff --git a/storage/list/ListBuffer.h b/storage/list/ListBuffer.h index 5d63936ec9..d9ffade789 100644 --- a/storage/list/ListBuffer.h +++ b/storage/list/ListBuffer.h @@ -28,7 +28,7 @@ using ListableTypesImpl = db::types::Double::Primitive, db::types::String::Primitive, db::types::Bool::Primitive, db::types::Embedding::Primitive, db::ListView, db::PropertyNull, db::NodeID, db::EdgeID, db::types::DateTime::Primitive, db::MapView, - db::types::Duration::Primitive, db::PathView>; + db::types::Duration::Primitive, db::PathView, db::types::Point::Primitive>; } namespace db { diff --git a/storage/list/ListByteBuffer.cpp b/storage/list/ListByteBuffer.cpp index e78a832434..d4fafe1e19 100644 --- a/storage/list/ListByteBuffer.cpp +++ b/storage/list/ListByteBuffer.cpp @@ -127,4 +127,5 @@ template CellView ListByteBuffer<>::write(CellTag, const types::DateTime::Primit template CellView ListByteBuffer<>::write(CellTag, const types::Duration::Primitive&); template CellView ListByteBuffer<>::write(CellTag, const MapView&); template CellView ListByteBuffer<>::write(CellTag, const PathView&); +template CellView ListByteBuffer<>::write(CellTag, const types::Point::Primitive&); } diff --git a/storage/list/ListHash.cpp b/storage/list/ListHash.cpp index 22a01bc8fe..1cfc64006d 100644 --- a/storage/list/ListHash.cpp +++ b/storage/list/ListHash.cpp @@ -68,6 +68,9 @@ size_t db::hashCell(CellView element) { case CellTag::Duration: return combine(seed, hashValue(element.getAs())); break; + case CellTag::Point: + return combine(seed, hashValue(element.getAs())); + break; case CellTag::Embedding: { size_t hash = seed; for (const float value : element.getAs()) { diff --git a/storage/map/EncodedMap.cpp b/storage/map/EncodedMap.cpp index 7eca97a84a..6a35e8af71 100644 --- a/storage/map/EncodedMap.cpp +++ b/storage/map/EncodedMap.cpp @@ -61,6 +61,9 @@ void encodeValue(MapEntryView entry, std::vector& out) { case CellTag::Duration: appendValue(out, entry.getValue().getAs()); break; + case CellTag::Point: + appendValue(out, entry.getValue().getAs()); + break; case CellTag::Null: break; case CellTag::String: { @@ -209,6 +212,9 @@ class MapDecoder { case CellTag::Duration: return read(); break; + case CellTag::Point: + return read(); + break; case CellTag::Null: return PropertyNull {}; break; diff --git a/storage/map/MapBuffer.h b/storage/map/MapBuffer.h index ebe61de36f..75c800a2c4 100644 --- a/storage/map/MapBuffer.h +++ b/storage/map/MapBuffer.h @@ -23,7 +23,8 @@ using MappableTypesImpl = db::types::Double::Primitive, db::types::Bool::Primitive, db::types::String::Primitive, db::types::Embedding::Primitive, db::ListView, db::MapView, db::PropertyNull, db::NodeID, db::EdgeID, - db::types::DateTime::Primitive, db::types::Duration::Primitive, db::PathView>; + db::types::DateTime::Primitive, db::types::Duration::Primitive, db::PathView, + db::types::Point::Primitive>; } namespace db { diff --git a/storage/map/MapByteBuffer.cpp b/storage/map/MapByteBuffer.cpp index ff25e43e38..823ec23e77 100644 --- a/storage/map/MapByteBuffer.cpp +++ b/storage/map/MapByteBuffer.cpp @@ -134,4 +134,5 @@ template MapEntryView MapByteBuffer<>::write(std::string_view, CellTag, const Ed template MapEntryView MapByteBuffer<>::write(std::string_view, CellTag, const types::DateTime::Primitive&); template MapEntryView MapByteBuffer<>::write(std::string_view, CellTag, const types::Duration::Primitive&); template MapEntryView MapByteBuffer<>::write(std::string_view, CellTag, const PathView&); +template MapEntryView MapByteBuffer<>::write(std::string_view, CellTag, const types::Point::Primitive&); } diff --git a/storage/map/MapHash.cpp b/storage/map/MapHash.cpp index 23dc058176..f906967929 100644 --- a/storage/map/MapHash.cpp +++ b/storage/map/MapHash.cpp @@ -63,6 +63,9 @@ size_t hashMapValue(MapEntryView entry) { case CellTag::Duration: return combine(seed, hashValue(entry.getValue().getAs())); break; + case CellTag::Point: + return combine(seed, hashValue(entry.getValue().getAs())); + break; case CellTag::Embedding: { size_t hash = seed; for (const float value : entry.getValue().getAs()) { @@ -129,6 +132,9 @@ bool sameMapValue(MapEntryView lhs, MapEntryView rhs) { case CellTag::Duration: return lhs.getValue().getAs() == rhs.getValue().getAs(); break; + case CellTag::Point: + return lhs.getValue().getAs() == rhs.getValue().getAs(); + break; case CellTag::Embedding: { const types::Embedding::Primitive lhsValues = lhs.getValue().getAs(); const types::Embedding::Primitive rhsValues = rhs.getValue().getAs(); diff --git a/storage/map/MapWriteCursor.cpp b/storage/map/MapWriteCursor.cpp index 8a967391ab..fb5bf41fab 100644 --- a/storage/map/MapWriteCursor.cpp +++ b/storage/map/MapWriteCursor.cpp @@ -72,4 +72,5 @@ template MapEntryView MapWriteCursor::writeValue(CellTag, const NodeID&); template MapEntryView MapWriteCursor::writeValue(CellTag, const EdgeID&); template MapEntryView MapWriteCursor::writeValue(CellTag, const types::DateTime::Primitive&); template MapEntryView MapWriteCursor::writeValue(CellTag, const types::Duration::Primitive&); +template MapEntryView MapWriteCursor::writeValue(CellTag, const types::Point::Primitive&); } diff --git a/storage/metadata/Point.cpp b/storage/metadata/Point.cpp new file mode 100644 index 0000000000..4feebe241f --- /dev/null +++ b/storage/metadata/Point.cpp @@ -0,0 +1,583 @@ +#include "Point.h" + +#include +#include + +#include +#include +#include +#include + +#include + +#include "QueryTypeName.h" +#include "StringUtils.h" +#include "TuringException.h" +#include "columns/ValueText.h" +#include "map/MapView.h" + +using namespace db; + +namespace { + +constexpr double EARTH_MEAN_RADIUS_METRES = 6371008.8; + +constexpr std::array CRS_SYSTEMS = { + PointCRS::Cartesian, + PointCRS::Cartesian3D, + PointCRS::WGS84, + PointCRS::WGS84_3D, +}; + +// The name of each PointPart, at its ordinal +constexpr std::array PART_NAMES = { + "x", "y", "z", "longitude", "latitude", "height", "crs", "srid", +}; + +static_assert(PART_NAMES.size() == static_cast(PointPart::SRID) + 1); + +struct PointKeys { + std::array _coordinates {}; + std::array _present {}; + std::string_view _crsName; + int64_t _srid {0}; + + bool has(PointPart part) const { return _present[static_cast(part)]; } + double get(PointPart part) const { return _coordinates[static_cast(part)]; } +}; + +// point() reads its keys whatever their case, as Neo4j does +bool keyNamed(std::string_view name, PointPart& part) { + const auto it = std::ranges::find_if(PART_NAMES, [name](std::string_view partName) { + return StringUtils::equalsIgnoringCase(partName, name); + }); + + if (it == PART_NAMES.end()) { + return false; + } + + part = static_cast(std::distance(PART_NAMES.begin(), it)); + + return true; +} + +double coordinateOf(std::string_view key, CellView value) { + const CellTag tag = value.getTag(); + + if (tag == CellTag::Int) { + return static_cast(value.getAs()); + } else if (tag == CellTag::UInt) { + return static_cast(value.getAs()); + } else if (tag == CellTag::Double) { + return value.getAs(); + } + + throw TuringException(fmt::format("point() reads a number for '{}', not '{}'", key, queryTypeName(tag))); +} + +// Reads the keys point() knows; it ignores any other, as Neo4j does. False where a value is null. +bool readKeys(const MapView& map, PointKeys& keys) { + bool holdsANull = false; + + for (const MapEntryView entry : map) { + const std::string_view name = entry.getKey(); + const CellView value = entry.getValue(); + const CellTag tag = value.getTag(); + + if (tag == CellTag::Null) { + holdsANull = true; + continue; + } + + PointPart part {PointPart::X}; + if (!keyNamed(name, part)) { + continue; + } + + const size_t index = static_cast(part); + if (keys._present[index]) { + throw TuringException(fmt::format("point() reads the key '{}' twice", name)); + } + + keys._present[index] = true; + + if (part == PointPart::CRS) { + if (tag != CellTag::String) { + throw TuringException(fmt::format("point() reads a string for 'crs', not '{}'", queryTypeName(tag))); + } + + keys._crsName = value.getAs(); + } else if (part == PointPart::SRID) { + if (tag == CellTag::Int) { + keys._srid = value.getAs(); + } else if (tag == CellTag::UInt) { + const uint64_t srid = value.getAs(); + keys._srid = srid > static_cast(INT64_MAX) ? -1 : static_cast(srid); + } else { + throw TuringException(fmt::format("point() reads an integer for 'srid', not '{}'", queryTypeName(tag))); + } + } else { + keys._coordinates[index] = coordinateOf(name, value); + } + } + + return !holdsANull; +} + +// The CRS the map names with 'crs' or 'srid', false where it names none +bool namedCRS(const PointKeys& keys, PointCRS& crs) { + const bool namesACRS = keys.has(PointPart::CRS); + const bool namesAnSRID = keys.has(PointPart::SRID); + + if (namesACRS && namesAnSRID) { + throw TuringException("point() takes a 'crs' or an 'srid', not both"); + } else if (namesACRS) { + if (!Point::crsNamed(keys._crsName, crs)) { + throw TuringException(fmt::format("Unknown coordinate reference system: '{}'", keys._crsName)); + } + + return true; + } else if (namesAnSRID) { + if (!Point::crsOfSRID(keys._srid, crs)) { + throw TuringException(fmt::format("Unknown coordinate reference system SRID: {}", keys._srid)); + } + + return true; + } + + return false; +} + +double finiteCoordinate(double coordinate) { + if (!isfinite(coordinate)) { + throw TuringException("A point cannot have a non-finite coordinate"); + } + + return coordinate == 0.0 ? 0.0 : coordinate; +} + +bool withinRange(double value, double lower, double upper) { + return value >= lower && value <= upper; +} + +double wrappedLongitude(double longitude) { + if (withinRange(longitude, -180.0, 180.0)) { + return longitude; + } + + double wrapped = fmod(longitude, 360.0); + if (wrapped > 180.0) { + wrapped -= 360.0; + } else if (wrapped < -180.0) { + wrapped += 360.0; + } + + return wrapped == 0.0 ? 0.0 : wrapped; +} + +std::strong_ordering compareCoordinates(double lhs, double rhs) { + if (lhs < rhs) { + return std::strong_ordering::less; + } else if (lhs > rhs) { + return std::strong_ordering::greater; + } + + return std::strong_ordering::equal; +} + +// A longitude range whose lower end lies east of its upper end crosses the 180th meridian +bool withinLongitudes(double longitude, double west, double east) { + if (west <= east) { + return withinRange(longitude, west, east); + } + + return longitude >= west || longitude <= east; +} + +} + +Point::Point(PointCRS crs, double x, double y, double z) + : _x(finiteCoordinate(x)), + _y(finiteCoordinate(y)), + _z(getDimension(crs) == 3 ? finiteCoordinate(z) : 0.0), + _crs(crs), + _padding {} +{ + if (!isGeographic(crs)) { + return; + } + + if (!withinRange(_y, -90.0, 90.0)) { + throw TuringException(fmt::format("The latitude of a {} point lies in [-90, 90], not {}", getCRSName(crs), _y)); + } + + _x = wrappedLongitude(_x); +} + +bool Point::operator==(const Point& other) const { + return _crs == other._crs && _x == other._x && _y == other._y && _z == other._z; +} + +std::strong_ordering Point::operator<=>(const Point& other) const { + const std::strong_ordering bySRID = getSRID(_crs) <=> getSRID(other._crs); + if (bySRID != 0) { + return bySRID; + } + + const std::strong_ordering byX = compareCoordinates(_x, other._x); + if (byX != 0) { + return byX; + } + + const std::strong_ordering byY = compareCoordinates(_y, other._y); + if (byY != 0) { + return byY; + } + + return compareCoordinates(_z, other._z); +} + +size_t Point::getDimension(PointCRS crs) { + switch (crs) { + case PointCRS::Cartesian: + case PointCRS::WGS84: + return 2; + break; + case PointCRS::Cartesian3D: + case PointCRS::WGS84_3D: + return 3; + break; + } + + return 2; +} + +bool Point::isGeographic(PointCRS crs) { + return crs == PointCRS::WGS84 || crs == PointCRS::WGS84_3D; +} + +int64_t Point::getSRID(PointCRS crs) { + switch (crs) { + case PointCRS::Cartesian: + return 7203; + break; + case PointCRS::Cartesian3D: + return 9157; + break; + case PointCRS::WGS84: + return 4326; + break; + case PointCRS::WGS84_3D: + return 4979; + break; + } + + return 0; +} + +std::string_view Point::getCRSName(PointCRS crs) { + switch (crs) { + case PointCRS::Cartesian: + return "cartesian"; + break; + case PointCRS::Cartesian3D: + return "cartesian-3d"; + break; + case PointCRS::WGS84: + return "wgs-84"; + break; + case PointCRS::WGS84_3D: + return "wgs-84-3d"; + break; + } + + return ""; +} + +bool Point::partNamed(std::string_view name, PointPart& part) { + const auto it = std::ranges::find(PART_NAMES, name); + if (it == PART_NAMES.end()) { + return false; + } + + part = static_cast(std::distance(PART_NAMES.begin(), it)); + + return true; +} + +std::string_view Point::getPartName(PointPart part) { + return PART_NAMES[static_cast(part)]; +} + +double Point::coordinate(Point value, PointPart part) { + const PointCRS crs = value.getCRS(); + const bool isThreeDimensional = getDimension(crs) == 3; + const bool namesAGeographicPart = part == PointPart::Longitude + || part == PointPart::Latitude + || part == PointPart::Height; + const bool namesAThirdCoordinate = part == PointPart::Z || part == PointPart::Height; + + const bool missesThePart = (namesAGeographicPart && !isGeographic(crs)) + || (namesAThirdCoordinate && !isThreeDimensional); + + if (missesThePart) { + std::string text; + format(text, value); + throw TuringException(fmt::format("The point {} has no {}", text, getPartName(part))); + } + + switch (part) { + case PointPart::X: + case PointPart::Longitude: + return value.getX(); + break; + case PointPart::Y: + case PointPart::Latitude: + return value.getY(); + break; + case PointPart::Z: + case PointPart::Height: + return value.getZ(); + break; + case PointPart::CRS: + case PointPart::SRID: + throw TuringException("The crs and the srid of a point are not coordinates"); + break; + } + + return 0.0; +} + +Point::Component Point::component(Point value, PointPart part) { + if (part == PointPart::CRS) { + return getCRSName(value.getCRS()); + } else if (part == PointPart::SRID) { + return getSRID(value.getCRS()); + } + + return coordinate(value, part); +} + +bool Point::fromMap(const MapView& map, Point& value) { + PointKeys keys; + if (!readKeys(map, keys)) { + return false; + } + + PointCRS crs {PointCRS::Cartesian}; + const bool namesACRS = namedCRS(keys, crs); + + const bool hasXY = keys.has(PointPart::X) && keys.has(PointPart::Y); + const bool hasLongitudeLatitude = keys.has(PointPart::Longitude) && keys.has(PointPart::Latitude); + + double x = 0.0; + double y = 0.0; + double z = 0.0; + size_t dimension = 2; + + if (hasXY) { + x = keys.get(PointPart::X); + y = keys.get(PointPart::Y); + + if (keys.has(PointPart::Z)) { + z = keys.get(PointPart::Z); + dimension = 3; + } + + if (!namesACRS) { + crs = dimension == 3 ? PointCRS::Cartesian3D : PointCRS::Cartesian; + } + } else if (hasLongitudeLatitude) { + x = keys.get(PointPart::Longitude); + y = keys.get(PointPart::Latitude); + + if (keys.has(PointPart::Z)) { + z = keys.get(PointPart::Z); + dimension = 3; + } else if (keys.has(PointPart::Height)) { + z = keys.get(PointPart::Height); + dimension = 3; + } + + if (!namesACRS) { + crs = dimension == 3 ? PointCRS::WGS84_3D : PointCRS::WGS84; + } else if (!isGeographic(crs)) { + throw TuringException(fmt::format("A point given by longitude and latitude takes a geographic CRS, not '{}'", + getCRSName(crs))); + } + } else if (namesACRS) { + throw TuringException(fmt::format("A {} point needs {}", getCRSName(crs), requiredKeys(crs))); + } else { + throw TuringException("point() needs 'x' and 'y', or 'longitude' and 'latitude'"); + } + + const size_t crsDimension = getDimension(crs); + if (crsDimension != dimension) { + throw TuringException(fmt::format("A {} point has {} coordinates, not {}", + getCRSName(crs), + crsDimension, + dimension)); + } + + value = Point(crs, x, y, z); + + return true; +} + +bool Point::crsNamed(std::string_view name, PointCRS& crs) { + for (const PointCRS system : CRS_SYSTEMS) { + if (StringUtils::equalsIgnoringCase(getCRSName(system), name)) { + crs = system; + return true; + } + } + + return false; +} + +bool Point::crsOfSRID(int64_t srid, PointCRS& crs) { + for (const PointCRS system : CRS_SYSTEMS) { + if (getSRID(system) == srid) { + crs = system; + return true; + } + } + + return false; +} + +std::string_view Point::requiredKeys(PointCRS crs) { + switch (crs) { + case PointCRS::Cartesian: + return "'x' and 'y'"; + break; + case PointCRS::Cartesian3D: + return "'x', 'y' and 'z'"; + break; + case PointCRS::WGS84: + return "'longitude' and 'latitude'"; + break; + case PointCRS::WGS84_3D: + return "'longitude', 'latitude' and 'height'"; + break; + } + + return ""; +} + +double Point::haversineDistance(Point lhs, Point rhs, double radius) { + constexpr double RADIANS_PER_DEGREE = std::numbers::pi / 180.0; + + const double lhsLatitude = lhs.getY(); + const double rhsLatitude = rhs.getY(); + + // Subtracting in degrees keeps a point at 0 from itself and the distance the same either + // way, which a product contracted into an FMA breaks + const double halfLatitudeSine = sin((rhsLatitude - lhsLatitude) * RADIANS_PER_DEGREE / 2.0); + const double halfLongitudeSine = sin((rhs.getX() - lhs.getX()) * RADIANS_PER_DEGREE / 2.0); + const double latitudeCosines = cos(lhsLatitude * RADIANS_PER_DEGREE) * cos(rhsLatitude * RADIANS_PER_DEGREE); + + // The square of a half chord, which rounding can push past 1 between antipodes + const double alpha = std::clamp(halfLatitudeSine * halfLatitudeSine + latitudeCosines * halfLongitudeSine * halfLongitudeSine, + 0.0, + 1.0); + + return radius * 2.0 * atan2(sqrt(alpha), sqrt(1.0 - alpha)); +} + +std::optional Point::distance(Point lhs, Point rhs) { + const PointCRS crs = lhs.getCRS(); + if (crs != rhs.getCRS()) { + return std::nullopt; + } + + const double lhsHeight = lhs.getZ(); + const double rhsHeight = rhs.getZ(); + const double heightDifference = lhsHeight - rhsHeight; + + if (!isGeographic(crs)) { + const double xDifference = lhs.getX() - rhs.getX(); + const double yDifference = lhs.getY() - rhs.getY(); + + return sqrt(xDifference * xDifference + yDifference * yDifference + heightDifference * heightDifference); + } else if (getDimension(crs) == 2) { + return haversineDistance(lhs, rhs, EARTH_MEAN_RADIUS_METRES); + } + + const double averageHeight = (lhsHeight + rhsHeight) / 2.0; + const double surfaceDistance = haversineDistance(lhs, rhs, EARTH_MEAN_RADIUS_METRES + averageHeight); + + return sqrt(surfaceDistance * surfaceDistance + heightDifference * heightDifference); +} + +std::optional Point::withinBBox(Point value, Point lowerLeft, Point upperRight) { + const PointCRS crs = value.getCRS(); + const bool sharesTheCRS = lowerLeft.getCRS() == crs && upperRight.getCRS() == crs; + if (!sharesTheCRS) { + return std::nullopt; + } + + const bool withinHeights = withinRange(value.getZ(), lowerLeft.getZ(), upperRight.getZ()); + + if (!isGeographic(crs)) { + return withinRange(value.getX(), lowerLeft.getX(), upperRight.getX()) + && withinRange(value.getY(), lowerLeft.getY(), upperRight.getY()) + && withinHeights; + } + + return withinLongitudes(value.getX(), lowerLeft.getX(), upperRight.getX()) + && withinRange(value.getY(), lowerLeft.getY(), upperRight.getY()) + && withinHeights; +} + +Point Point::addToMean(Point mean, uint64_t count, Point value) { + if (count == 0) { + return value; + } + + const PointCRS crs = mean.getCRS(); + const PointCRS valueCRS = value.getCRS(); + if (valueCRS != crs) { + throw TuringException(fmt::format("avg() reads points of one CRS, not {} and {}", + getCRSName(crs), + getCRSName(valueCRS))); + } + + const double meanX = mean.getX(); + const double meanY = mean.getY(); + const double meanZ = mean.getZ(); + + const double total = static_cast(count + 1); + const double x = meanX + (value.getX() - meanX) / total; + const double y = meanY + (value.getY() - meanY) / total; + const double z = meanZ + (value.getZ() - meanZ) / total; + + return Point(crs, x, y, z); +} + +void Point::format(std::string& out, Point value) { + ValueTextScratch scratch; + const auto output = std::back_inserter(out); + + const PointCRS crs = value.getCRS(); + + fmt::format_to(output, "point({{x: {}", valueTextInto(value.getX(), scratch)); + fmt::format_to(output, ", y: {}", valueTextInto(value.getY(), scratch)); + + if (getDimension(crs) == 3) { + fmt::format_to(output, ", z: {}", valueTextInto(value.getZ(), scratch)); + } + + fmt::format_to(output, ", crs: '{}'}})", getCRSName(crs)); +} + +size_t std::hash::operator()(const Point& value) const noexcept { + const std::hash hasher; + + size_t hash = static_cast(value.getCRS()); + hash = hash * 31 + hasher(value.getX()); + hash = hash * 31 + hasher(value.getY()); + hash = hash * 31 + hasher(value.getZ()); + + return hash; +} diff --git a/storage/metadata/Point.h b/storage/metadata/Point.h new file mode 100644 index 0000000000..79413b765b --- /dev/null +++ b/storage/metadata/Point.h @@ -0,0 +1,117 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include + +namespace db { + +class MapView; + +// A coordinate reference system, written to disk as its ordinal inside every point +enum class PointCRS : uint8_t { + Cartesian = 0, + Cartesian3D, + WGS84, + WGS84_3D, +}; + +// A field of a point. Longitude, latitude and height name x, y and z in a geographic CRS. +enum class PointPart : uint8_t { + X, + Y, + Z, + Longitude, + Latitude, + Height, + CRS, + SRID, +}; + +// A location in one of the coordinate reference systems Cypher names. A 2D point holds a z +// of 0, and no coordinate is -0.0, so two equal points are equal in every byte. +class Point { +public: + // The value of a component: a coordinate, the name of the CRS or its SRID + using Component = std::variant; + + Point() = default; + + // Raises on a non-finite coordinate and on a latitude outside [-90, 90]. A longitude + // outside [-180, 180] is wrapped into it. + Point(PointCRS crs, double x, double y, double z); + + PointCRS getCRS() const { return _crs; } + double getX() const { return _x; } + double getY() const { return _y; } + double getZ() const { return _z; } + + bool operator==(const Point& other) const; + + // The order ORDER BY sorts points in: by SRID, then by coordinates + std::strong_ordering operator<=>(const Point& other) const; + + static size_t getDimension(PointCRS crs); + + static bool isGeographic(PointCRS crs); + + static int64_t getSRID(PointCRS crs); + + static std::string_view getCRSName(PointCRS crs); + + // The CRS @param name names, whatever its case + static bool crsNamed(std::string_view name, PointCRS& crs); + + static bool crsOfSRID(int64_t srid, PointCRS& crs); + + static bool partNamed(std::string_view name, PointPart& part); + + static std::string_view getPartName(PointPart part); + + // A coordinate the point has. Raises for z of a 2D point, and for longitude, latitude + // and height of a cartesian one. + static double coordinate(Point value, PointPart part); + + static Component component(Point value, PointPart part); + + // point() of a map. False where the map holds a null. + static bool fromMap(const MapView& map, Point& value); + + // Nullopt for two points in different CRSs + static std::optional distance(Point lhs, Point rhs); + + // Nullopt unless the three points share a CRS + static std::optional withinBBox(Point value, Point lowerLeft, Point upperRight); + + // The mean of @param count points whose mean is @param mean, and @param value. A mean + // of valid points of one CRS is a valid point of it, so a running mean is a Point. + // Raises for a value in another CRS. + static Point addToMean(Point mean, uint64_t count, Point value); + + static void format(std::string& out, Point value); + +private: + // No initializers: the datapart loader reads a trivial property's values straight out of + // the mapped page, which needs a trivial default constructor + double _x; + double _y; + double _z; + PointCRS _crs; + uint8_t _padding[7]; + + static std::string_view requiredKeys(PointCRS crs); + + static double haversineDistance(Point lhs, Point rhs, double radius); +}; + +} + +template <> +struct std::hash { + size_t operator()(const db::Point& value) const noexcept; +}; diff --git a/storage/metadata/PropertyType.h b/storage/metadata/PropertyType.h index 59d8dfde4a..51029bf467 100644 --- a/storage/metadata/PropertyType.h +++ b/storage/metadata/PropertyType.h @@ -10,6 +10,7 @@ #include "metadata/DateTime.h" #include "metadata/Duration.h" +#include "metadata/Point.h" #include "list/EncodedList.h" #include "list/ListHash.h" @@ -34,6 +35,7 @@ enum class ValueType : uint8_t { DateTime, Map, Duration, + Point, _SIZE, }; @@ -49,7 +51,8 @@ using ValueTypeName = EnumToString::Create< EnumStringPair, EnumStringPair, EnumStringPair, - EnumStringPair>; + EnumStringPair, + EnumStringPair>; struct CustomBool { CustomBool() = default; @@ -159,6 +162,13 @@ struct Duration : public PropertyType { static constexpr auto _valueType = ValueType::Duration; }; +struct Point : public PropertyType { + using Primitive = db::Point; + using MandatorySpan = std::span; + using OptionalSpan = std::span>; + static constexpr auto _valueType = ValueType::Point; +}; + struct Map : public PropertyType { using Primitive = MapView; using OwningPrimitive = EncodedMap; @@ -227,6 +237,9 @@ struct ValueTypeDispatcher { case ValueType::Duration: executor.template operator()(); break; + case ValueType::Point: + executor.template operator()(); + break; case ValueType::Map: executor.template operator()(); break; diff --git a/storage/metadata/PropertyTypeDispatcher.h b/storage/metadata/PropertyTypeDispatcher.h index f1058837a5..06fe8f7a55 100644 --- a/storage/metadata/PropertyTypeDispatcher.h +++ b/storage/metadata/PropertyTypeDispatcher.h @@ -33,6 +33,8 @@ struct PropertyTypeDispatcher { return executor.template operator()(); case ValueType::Duration: return executor.template operator()(); + case ValueType::Point: + return executor.template operator()(); case ValueType::Map: return executor.template operator()(); case ValueType::_SIZE: diff --git a/storage/metadata/ValueComponent.h b/storage/metadata/ValueComponent.h new file mode 100644 index 0000000000..d47b2db992 --- /dev/null +++ b/storage/metadata/ValueComponent.h @@ -0,0 +1,14 @@ +#pragma once + +#include + +#include "metadata/DateTime.h" +#include "metadata/Duration.h" +#include "metadata/Point.h" + +namespace db { + +// The component a read takes of a value that has components: d.year, d.hours, p.x +using ValueComponent = std::variant; + +} diff --git a/storage/properties/PropertyManager.cpp b/storage/properties/PropertyManager.cpp index 4d0f892f59..1aa76b55e4 100644 --- a/storage/properties/PropertyManager.cpp +++ b/storage/properties/PropertyManager.cpp @@ -84,6 +84,11 @@ void PropertyManager::fillEntityPropertyView(EntityID entityID, fill(container, ptID); } + for (const auto& [ptID, rawContainer] : _points) { + const auto& container = rawContainer->cast(); + fill(container, ptID); + } + for (const auto& [ptID, rawContainer] : _maps) { const auto& container = rawContainer->cast(); fill(container, ptID); diff --git a/storage/properties/PropertyManager.h b/storage/properties/PropertyManager.h index a3b5d79ff0..6f7517e01c 100644 --- a/storage/properties/PropertyManager.h +++ b/storage/properties/PropertyManager.h @@ -57,6 +57,8 @@ class PropertyManager { _dateTimes.emplace(ptID, static_cast(ptr)); } else if constexpr (std::is_same_v) { _durations.emplace(ptID, static_cast(ptr)); + } else if constexpr (std::is_same_v) { + _points.emplace(ptID, static_cast(ptr)); } else if constexpr (std::is_same_v) { _maps.emplace(ptID, static_cast(ptr)); } @@ -241,6 +243,7 @@ class PropertyManager { PropertyContainerReferences _lists; PropertyContainerReferences _dateTimes; PropertyContainerReferences _durations; + PropertyContainerReferences _points; PropertyContainerReferences _maps; PropertyIndexer _indexers; diff --git a/storage/reader/GraphReader.cpp b/storage/reader/GraphReader.cpp index a7708113c9..b0225ffa7d 100644 --- a/storage/reader/GraphReader.cpp +++ b/storage/reader/GraphReader.cpp @@ -580,6 +580,7 @@ template std::optional GraphReader::tryGetNo template std::optional GraphReader::tryGetNodeProperty(PropertyTypeID ptID, NodeID nodeID) const; template std::optional GraphReader::tryGetNodeProperty(PropertyTypeID ptID, NodeID nodeID) const; template std::optional GraphReader::tryGetNodeProperty(PropertyTypeID ptID, NodeID nodeID) const; +template std::optional GraphReader::tryGetNodeProperty(PropertyTypeID ptID, NodeID nodeID) const; template std::optional GraphReader::tryGetNodeProperty(PropertyTypeID ptID, NodeID nodeID) const; template std::optional GraphReader::tryGetEdgeProperty(PropertyTypeID ptID, EdgeID edgeID) const; @@ -591,4 +592,5 @@ template std::optional GraphReader::tryGetEd template std::optional GraphReader::tryGetEdgeProperty(PropertyTypeID ptID, EdgeID edgeID) const; template std::optional GraphReader::tryGetEdgeProperty(PropertyTypeID ptID, EdgeID edgeID) const; template std::optional GraphReader::tryGetEdgeProperty(PropertyTypeID ptID, EdgeID edgeID) const; +template std::optional GraphReader::tryGetEdgeProperty(PropertyTypeID ptID, EdgeID edgeID) const; template std::optional GraphReader::tryGetEdgeProperty(PropertyTypeID ptID, EdgeID edgeID) const; diff --git a/storage/versioning/CommitBuilder.cpp b/storage/versioning/CommitBuilder.cpp index 6935142969..f0f7027b14 100644 --- a/storage/versioning/CommitBuilder.cpp +++ b/storage/versioning/CommitBuilder.cpp @@ -283,6 +283,7 @@ template WeakArc CommitBuilder::newNodePropertyIndex(st template WeakArc CommitBuilder::newNodePropertyIndex(std::string_view indexName, PropertyTypeID ptID); template WeakArc CommitBuilder::newNodePropertyIndex(std::string_view indexName, PropertyTypeID ptID); template WeakArc CommitBuilder::newNodePropertyIndex(std::string_view indexName, PropertyTypeID ptID); +template WeakArc CommitBuilder::newNodePropertyIndex(std::string_view indexName, PropertyTypeID ptID); template WeakArc CommitBuilder::newNodePropertyIndex(std::string_view indexName, PropertyTypeID ptID); template WeakArc CommitBuilder::newEdgePropertyIndex(std::string_view indexName, PropertyTypeID ptID); @@ -294,5 +295,6 @@ template WeakArc CommitBuilder::newEdgePropertyIndex(st template WeakArc CommitBuilder::newEdgePropertyIndex(std::string_view indexName, PropertyTypeID ptID); template WeakArc CommitBuilder::newEdgePropertyIndex(std::string_view indexName, PropertyTypeID ptID); template WeakArc CommitBuilder::newEdgePropertyIndex(std::string_view indexName, PropertyTypeID ptID); +template WeakArc CommitBuilder::newEdgePropertyIndex(std::string_view indexName, PropertyTypeID ptID); template WeakArc CommitBuilder::newEdgePropertyIndex(std::string_view indexName, PropertyTypeID ptID); } diff --git a/storage/versioning/CommitWriteBuffer.cpp b/storage/versioning/CommitWriteBuffer.cpp index 966bd7f46e..c17585f837 100644 --- a/storage/versioning/CommitWriteBuffer.cpp +++ b/storage/versioning/CommitWriteBuffer.cpp @@ -60,6 +60,10 @@ struct PrimitiveToTag { using Type = types::Duration; }; template <> +struct PrimitiveToTag { + using Type = types::Point; +}; +template <> struct PrimitiveToTag { using Type = types::Embedding; }; diff --git a/storage/versioning/CommitWriteBuffer.h b/storage/versioning/CommitWriteBuffer.h index 12be520c6b..5651ec4ae3 100644 --- a/storage/versioning/CommitWriteBuffer.h +++ b/storage/versioning/CommitWriteBuffer.h @@ -46,6 +46,7 @@ class CommitWriteBuffer { std::optional, std::optional, std::optional, + std::optional, std::optional >; using UntypedProperties = std::vector; diff --git a/storage/versioning/DataPartRebaser.cpp b/storage/versioning/DataPartRebaser.cpp index 09b29a308b..ba6eeee958 100644 --- a/storage/versioning/DataPartRebaser.cpp +++ b/storage/versioning/DataPartRebaser.cpp @@ -149,6 +149,7 @@ bool DataPartRebaser::rebase(const MetadataRebaser& metadata, std::unordered_map lists; std::unordered_map dateTimes; std::unordered_map durations; + std::unordered_map points; std::unordered_map maps; for (auto& [ptID, container] : nodeProperties->_containers) { @@ -202,6 +203,11 @@ bool DataPartRebaser::rebase(const MetadataRebaser& metadata, durations[newPT._id] = container; } + for (const auto& [ptID, container] : nodeProperties->_points) { + const auto newPT = metadata.getPropertyTypeMapping(ptID); + points[newPT._id] = container; + } + for (const auto& [ptID, container] : nodeProperties->_maps) { const auto newPT = metadata.getPropertyTypeMapping(ptID); maps[newPT._id] = container; @@ -217,8 +223,9 @@ bool DataPartRebaser::rebase(const MetadataRebaser& metadata, nodeProperties->_lists = std::move(lists); nodeProperties->_dateTimes = std::move(dateTimes); nodeProperties->_durations = std::move(durations); + nodeProperties->_points = std::move(points); nodeProperties->_maps = std::move(maps); - static_assert((size_t)ValueType::_SIZE == 11 && "A value type was added"); + static_assert((size_t)ValueType::_SIZE == 12 && "A value type was added"); { PropertyIndexer newIndexers; @@ -258,6 +265,7 @@ bool DataPartRebaser::rebase(const MetadataRebaser& metadata, std::unordered_map lists; std::unordered_map dateTimes; std::unordered_map durations; + std::unordered_map points; std::unordered_map maps; for (auto& [ptID, container] : edgeProperties->_containers) { @@ -311,6 +319,11 @@ bool DataPartRebaser::rebase(const MetadataRebaser& metadata, durations[newPT._id] = container; } + for (const auto& [ptID, container] : edgeProperties->_points) { + const auto newPT = metadata.getPropertyTypeMapping(ptID); + points[newPT._id] = container; + } + for (const auto& [ptID, container] : edgeProperties->_maps) { const auto newPT = metadata.getPropertyTypeMapping(ptID); maps[newPT._id] = container; @@ -326,8 +339,9 @@ bool DataPartRebaser::rebase(const MetadataRebaser& metadata, edgeProperties->_lists = std::move(lists); edgeProperties->_dateTimes = std::move(dateTimes); edgeProperties->_durations = std::move(durations); + edgeProperties->_points = std::move(points); edgeProperties->_maps = std::move(maps); - static_assert((size_t)ValueType::_SIZE == 11 && "A value type was added"); + static_assert((size_t)ValueType::_SIZE == 12 && "A value type was added"); { // Update property indexers based on our new LabelSetMapping PropertyIndexer newIndexers; diff --git a/storage/versioning/VersionController.cpp b/storage/versioning/VersionController.cpp index 36e1b0416c..2e64d650f9 100644 --- a/storage/versioning/VersionController.cpp +++ b/storage/versioning/VersionController.cpp @@ -255,6 +255,7 @@ template WeakArc VersionController::createNodePropertyIndex VersionController::createNodePropertyIndex(std::string_view indexName, PropertyTypeID ptID); template WeakArc VersionController::createNodePropertyIndex(std::string_view indexName, PropertyTypeID ptID); template WeakArc VersionController::createNodePropertyIndex(std::string_view indexName, PropertyTypeID ptID); +template WeakArc VersionController::createNodePropertyIndex(std::string_view indexName, PropertyTypeID ptID); template WeakArc VersionController::createNodePropertyIndex(std::string_view indexName, PropertyTypeID ptID); template WeakArc VersionController::createEdgePropertyIndex(std::string_view indexName, PropertyTypeID ptID); @@ -266,5 +267,6 @@ template WeakArc VersionController::createEdgePropertyIndex VersionController::createEdgePropertyIndex(std::string_view indexName, PropertyTypeID ptID); template WeakArc VersionController::createEdgePropertyIndex(std::string_view indexName, PropertyTypeID ptID); template WeakArc VersionController::createEdgePropertyIndex(std::string_view indexName, PropertyTypeID ptID); +template WeakArc VersionController::createEdgePropertyIndex(std::string_view indexName, PropertyTypeID ptID); template WeakArc VersionController::createEdgePropertyIndex(std::string_view indexName, PropertyTypeID ptID); } diff --git a/storage/views/EntityPropertyView.h b/storage/views/EntityPropertyView.h index 63c1a0a710..1e5e7fbf46 100644 --- a/storage/views/EntityPropertyView.h +++ b/storage/views/EntityPropertyView.h @@ -97,6 +97,6 @@ class EntityPropertyView { void removeProperty(PropertyTypeID propTypeID); }; -static_assert((size_t)ValueType::_SIZE == 11); +static_assert((size_t)ValueType::_SIZE == 12); } diff --git a/storage/views/PropertyView.h b/storage/views/PropertyView.h index 29d399bbdc..2015984163 100644 --- a/storage/views/PropertyView.h +++ b/storage/views/PropertyView.h @@ -16,6 +16,7 @@ using PropertyVariant = std::variant; struct PropertyView { diff --git a/storage/writers/DataPartBuilder.cpp b/storage/writers/DataPartBuilder.cpp index c4f6fddbf2..71abf8409a 100644 --- a/storage/writers/DataPartBuilder.cpp +++ b/storage/writers/DataPartBuilder.cpp @@ -306,4 +306,5 @@ INSTANTIATE(types::Bool); INSTANTIATE(types::List); INSTANTIATE(types::DateTime); INSTANTIATE(types::Duration); +INSTANTIATE(types::Point); INSTANTIATE(types::Map); diff --git a/storage/writers/GraphWriter.cpp b/storage/writers/GraphWriter.cpp index e89e2b40c2..d5c2eee1a5 100644 --- a/storage/writers/GraphWriter.cpp +++ b/storage/writers/GraphWriter.cpp @@ -219,6 +219,7 @@ template void GraphWriter::addNodeProperty(NodeID, std::string template void GraphWriter::addNodeProperty(NodeID, std::string_view, types::List::Primitive&&); template void GraphWriter::addNodeProperty(NodeID, std::string_view, types::DateTime::Primitive&&); template void GraphWriter::addNodeProperty(NodeID, std::string_view, types::Duration::Primitive&&); +template void GraphWriter::addNodeProperty(NodeID, std::string_view, types::Point::Primitive&&); template void GraphWriter::addNodeProperty(NodeID, std::string_view, types::Map::Primitive&&); template void GraphWriter::addNodeProperty(NodeID, PropertyType, types::Int64::Primitive&&); @@ -230,6 +231,7 @@ template void GraphWriter::addNodeProperty(NodeID, PropertyTyp template void GraphWriter::addNodeProperty(NodeID, PropertyType, types::List::Primitive&&); template void GraphWriter::addNodeProperty(NodeID, PropertyType, types::DateTime::Primitive&&); template void GraphWriter::addNodeProperty(NodeID, PropertyType, types::Duration::Primitive&&); +template void GraphWriter::addNodeProperty(NodeID, PropertyType, types::Point::Primitive&&); template void GraphWriter::addNodeProperty(NodeID, PropertyType, types::Map::Primitive&&); template void GraphWriter::addEdgeProperty(const EdgeRecord&, std::string_view, types::Int64::Primitive&&); @@ -240,6 +242,7 @@ template void GraphWriter::addEdgeProperty(const EdgeRecord&, std:: template void GraphWriter::addEdgeProperty(const EdgeRecord&, std::string_view, types::Embedding::Primitive&&); template void GraphWriter::addEdgeProperty(const EdgeRecord&, std::string_view, types::DateTime::Primitive&&); template void GraphWriter::addEdgeProperty(const EdgeRecord&, std::string_view, types::Duration::Primitive&&); +template void GraphWriter::addEdgeProperty(const EdgeRecord&, std::string_view, types::Point::Primitive&&); template void GraphWriter::addEdgeProperty(const EdgeRecord&, std::string_view, types::List::Primitive&&); template void GraphWriter::addEdgeProperty(const EdgeRecord&, std::string_view, types::Map::Primitive&&); @@ -252,4 +255,5 @@ template void GraphWriter::addEdgeProperty(const EdgeRecord&, template void GraphWriter::addEdgeProperty(const EdgeRecord&, PropertyType, types::List::Primitive&&); template void GraphWriter::addEdgeProperty(const EdgeRecord&, PropertyType, types::DateTime::Primitive&&); template void GraphWriter::addEdgeProperty(const EdgeRecord&, PropertyType, types::Duration::Primitive&&); +template void GraphWriter::addEdgeProperty(const EdgeRecord&, PropertyType, types::Point::Primitive&&); template void GraphWriter::addEdgeProperty(const EdgeRecord&, PropertyType, types::Map::Primitive&&); diff --git a/system/GraphManager.cpp b/system/GraphManager.cpp index 21b5fb9392..8a54e5098b 100644 --- a/system/GraphManager.cpp +++ b/system/GraphManager.cpp @@ -177,8 +177,7 @@ Graph* GraphManager::importGraph(std::string_view graphName, const fs::Path& filePath, JobSystem* jobSystem, const EmbeddingsSpec& embeddingSpecs, - const DateTimeSpec& dateTimeSpecs, - const DurationSpec& durationSpecs) { + const TypedPropertySpec& typedProperties) { const fs::Path graphPath = _config->getGraphsDir() / filePath; // Step 1. Check if graph was already loaded || is already loading @@ -207,18 +206,18 @@ Graph* GraphManager::importGraph(std::string_view graphName, return loadGmlDB(graphName, absolute, jobSystem); break; case GraphFileType::JSONL: - return loadJsonlDB(graphName, absolute, jobSystem, embeddingSpecs, dateTimeSpecs, durationSpecs); + return loadJsonlDB(graphName, absolute, jobSystem, embeddingSpecs, typedProperties); break; case GraphFileType::BINARY: return loadBinaryDB(graphName, absolute, jobSystem); break; case GraphFileType::PARQUET: - return loadParquetDB(graphName, absolute, jobSystem, durationSpecs); + return loadParquetDB(graphName, absolute, jobSystem, typedProperties); break; case GraphFileType::UNKNOWN: // If we can not determine the file type, assume it is a JSONL graph // FIXME To be changed - return loadJsonlDB(graphName, absolute, jobSystem, embeddingSpecs, dateTimeSpecs, durationSpecs); + return loadJsonlDB(graphName, absolute, jobSystem, embeddingSpecs, typedProperties); break; case GraphFileType::_SIZE: throw TuringException("Unsupported graph type"); @@ -303,8 +302,7 @@ Graph* GraphManager::loadJsonlDB(std::string_view graphName, const fs::Path& dbPath, JobSystem* jobSystem, const EmbeddingsSpec& embeddingSpecs, - const DateTimeSpec& dateTimeSpecs, - const DurationSpec& durationSpecs) { + const TypedPropertySpec& typedProperties) { fs::Path graphPath; getGraphPath(graphName, graphPath); if (graphPath == dbPath) { @@ -333,7 +331,7 @@ Graph* GraphManager::loadJsonlDB(std::string_view graphName, Change* change = _changes.createChange(graph.get(), CommitHash::head()); ChangeAccessor changeAccessor = change->access(); - const auto importRes = JsonlParser::parse(changeAccessor, file, embeddingSpecs, dateTimeSpecs, durationSpecs); + const auto importRes = JsonlParser::parse(changeAccessor, file, embeddingSpecs, typedProperties); if (!importRes) { _changes.deleteChange(changeAccessor, change->id()); @@ -415,7 +413,7 @@ Graph* GraphManager::loadGmlDB(std::string_view graphName, Graph* GraphManager::loadParquetDB(std::string_view graphName, const fs::Path& dbPath, JobSystem* jobSystem, - const DurationSpec& durationSpecs) { + const TypedPropertySpec& typedProperties) { fs::Path graphPath; getGraphPath(graphName, graphPath); if (graphPath == dbPath) { @@ -451,7 +449,7 @@ Graph* GraphManager::loadParquetDB(std::string_view graphName, // Unfortunately I think we need to have this try-catch due to the dependency on the // Parquet lib which throws. Removing would require a rewrite of the reader. // TODO: Attempt to remove this try-catch and make the importer/reader exception free - ParquetImporter importer(nodeFile, edgeFile, commitBuilder, durationSpecs); + ParquetImporter importer(nodeFile, edgeFile, commitBuilder, typedProperties); try { importer.import(); } catch (...) { diff --git a/system/GraphManager.h b/system/GraphManager.h index 5935104698..4eb8e50e16 100644 --- a/system/GraphManager.h +++ b/system/GraphManager.h @@ -7,13 +7,12 @@ #include #include "ChangeManager.h" -#include "DateTimeSpec.h" -#include "DurationSpec.h" #include "EmbeddingsSpec.h" #include "GraphFileType.h" #include "GraphLoadStatus.h" #include "ObjectMap.h" #include "Path.h" +#include "TypedPropertySpec.h" #include "dump/LoadGraphResult.h" namespace db { @@ -52,8 +51,7 @@ class GraphManager { const fs::Path& filePath, JobSystem* jobSystem, const EmbeddingsSpec& embeddingSpecs = {}, - const DateTimeSpec& dateTimeSpecs = {}, - const DurationSpec& durationSpecs = {}); + const TypedPropertySpec& typedProperties = {}); GraphFileType getGraphFileType(const fs::Path& graphPath) const; @@ -86,8 +84,7 @@ class GraphManager { const fs::Path& dbPath, JobSystem* jobSystem, const EmbeddingsSpec& embeddingSpecs, - const DateTimeSpec& dateTimeSpecs, - const DurationSpec& durationSpecs); + const TypedPropertySpec& typedProperties); Graph* loadGmlDB(std::string_view graphName, const fs::Path& dbPath, @@ -100,6 +97,6 @@ class GraphManager { Graph* loadParquetDB(std::string_view graphName, const fs::Path& dbPath, JobSystem* jobSystem, - const DurationSpec& durationSpecs); + const TypedPropertySpec& typedProperties); }; } diff --git a/system/SystemAccessor.cpp b/system/SystemAccessor.cpp index eee7c6ab06..d1e1416321 100644 --- a/system/SystemAccessor.cpp +++ b/system/SystemAccessor.cpp @@ -67,9 +67,8 @@ DumpResult SystemAccessor::loadCommit(std::string_view name, CommitHash ha Graph* SystemAccessor::importGraph(const fs::Path& path, std::string_view name, const EmbeddingsSpec& embeddingSpecs, - const DateTimeSpec& dateTimeSpecs, - const DurationSpec& durationSpecs) { - return _sysMan->importGraph(path, name, embeddingSpecs, dateTimeSpecs, durationSpecs); + const TypedPropertySpec& typedProperties) { + return _sysMan->importGraph(path, name, embeddingSpecs, typedProperties); } GraphFileType SystemAccessor::getGraphFileType(const fs::Path& path) const { diff --git a/system/SystemAccessor.h b/system/SystemAccessor.h index 1801694955..e24032f63f 100644 --- a/system/SystemAccessor.h +++ b/system/SystemAccessor.h @@ -13,9 +13,8 @@ #include "dump/LoadGraphResult.h" #include "mergers/DataPartMergeResult.h" -#include "DateTimeSpec.h" -#include "DurationSpec.h" #include "EmbeddingsSpec.h" +#include "TypedPropertySpec.h" #include "GraphFileType.h" #include "Path.h" @@ -73,8 +72,7 @@ class SystemAccessor : public Accessor { Graph* importGraph(const fs::Path& path, std::string_view name, const EmbeddingsSpec& embeddingSpecs = {}, - const DateTimeSpec& dateTimeSpecs = {}, - const DurationSpec& durationSpecs = {}); + const TypedPropertySpec& typedProperties = {}); GraphFileType getGraphFileType(const fs::Path& path) const; diff --git a/system/SystemManager.cpp b/system/SystemManager.cpp index f563c28585..31aae93e97 100644 --- a/system/SystemManager.cpp +++ b/system/SystemManager.cpp @@ -187,9 +187,8 @@ void SystemManager::listGraphs(std::vector& names) const { Graph* SystemManager::importGraph(const fs::Path& path, std::string_view graphName, const EmbeddingsSpec& embeddingSpecs, - const DateTimeSpec& dateTimeSpecs, - const DurationSpec& durationSpecs) { - return _graphManager.importGraph(graphName, path, &_jobSystem, embeddingSpecs, dateTimeSpecs, durationSpecs); + const TypedPropertySpec& typedProperties) { + return _graphManager.importGraph(graphName, path, &_jobSystem, embeddingSpecs, typedProperties); } DumpResult SystemManager::dumpGraph(std::string_view graphName) { diff --git a/system/SystemManager.h b/system/SystemManager.h index d62d23e827..50a0992957 100644 --- a/system/SystemManager.h +++ b/system/SystemManager.h @@ -6,10 +6,9 @@ #include #include -#include "DateTimeSpec.h" -#include "DurationSpec.h" #include "EmbeddingsSpec.h" #include "GraphManager.h" +#include "TypedPropertySpec.h" #include "SystemAccessor.h" @@ -106,8 +105,7 @@ class SystemManager { Graph* importGraph(const fs::Path& path, std::string_view graphName, const EmbeddingsSpec& embeddingSpecs, - const DateTimeSpec& dateTimeSpecs, - const DurationSpec& durationSpecs); + const TypedPropertySpec& typedProperties); GraphFileType getGraphFileType(const fs::Path& graphPath) const; diff --git a/test/net/TuringProtoRoundTripTest.cpp b/test/net/TuringProtoRoundTripTest.cpp index 393293af8f..6ed9de42a9 100644 --- a/test/net/TuringProtoRoundTripTest.cpp +++ b/test/net/TuringProtoRoundTripTest.cpp @@ -45,6 +45,7 @@ using Bool = db::types::Bool::Primitive; using Embedding = db::types::Embedding::Primitive; using DateTime = db::types::DateTime::Primitive; using Duration = db::types::Duration::Primitive; +using Point = db::types::Point::Primitive; struct FramedPacket { net::proto::MessageTypes _type; @@ -493,6 +494,77 @@ TEST(TuringProtoRoundTripTest, RoundTripsDurationColumnsAcrossChunkSizes) { } } +TEST(TuringProtoRoundTripTest, RoundTripsPointColumnsAcrossChunkSizes) { + using OptionalPoint = std::optional; + + const Point cartesian(db::PointCRS::Cartesian, 1.5, -2.0, 0.0); + const Point cartesian3D(db::PointCRS::Cartesian3D, 1.5, -2.0, 3.25); + const Point wgs84(db::PointCRS::WGS84, 2.3522, 48.8566, 0.0); + const Point wgs84With3D(db::PointCRS::WGS84_3D, 2.3522, 48.8566, 35.0); + + for (const size_t chunkSize : std::array {48, 64, 97, 256}) { + SCOPED_TRACE(::testing::Message() << "chunkSize=" << chunkSize); + + db::LocalMemory localMem; + db::DataframeManager dfMan; + db::Dataframe source; + + auto* at = localMem.alloc>(); + at->push_back(cartesian); + at->push_back(cartesian3D); + at->push_back(wgs84); + at->push_back(wgs84With3D); + addColumn(&dfMan, &source, "at", at); + + auto* home = localMem.alloc>(); + home->push_back(wgs84); + home->push_back(std::nullopt); + home->push_back(cartesian3D); + home->push_back(std::nullopt); + addColumn(&dfMan, &source, "home", home); + + std::vector::ListItemVariant> items; + items.emplace_back(cartesian); + items.emplace_back(wgs84With3D); + + auto* route = localMem.alloc>(); + route->set(localMem.listBuffer().insert(items)); + addColumn(&dfMan, &source, "route", route); + + const auto packets = encodeDataframeWithChunkSize(source, chunkSize); + expectPacketSequence(packets, true); + + net::proto::ChunkedBuffer embeddingBuffer; + net::proto::ChunkedBuffer pathBuffer; + net::proto::ChunkedBuffer stringBuffer; + db::ListBuffer<> listBuffer; + db::MapBuffer<> mapBuffer; + db::Dataframe decoded; + std::vector schemas; + decodeChunkPackets(packets, &localMem, &embeddingBuffer, &pathBuffer, &stringBuffer, &listBuffer, &mapBuffer, &dfMan, &decoded, &schemas); + + ASSERT_EQ(decoded.cols().size(), 3u); + EXPECT_EQ(decoded.getLogicalRowCount(), 4u); + + const auto* decodedAt = decoded.cols().at(0)->as>(); + const auto* decodedHome = decoded.cols().at(1)->as>(); + const auto* decodedRoute = decoded.cols().at(2)->as>(); + ASSERT_NE(decodedAt, nullptr); + ASSERT_NE(decodedHome, nullptr); + ASSERT_NE(decodedRoute, nullptr); + + EXPECT_EQ(decodedAt->getRaw(), (std::vector {cartesian, cartesian3D, wgs84, wgs84With3D})); + EXPECT_EQ(decodedHome->getRaw(), (std::vector {wgs84, std::nullopt, cartesian3D, std::nullopt})); + + const std::span elements = decodedRoute->at(0).elements(); + ASSERT_EQ(elements.size(), 2u); + ASSERT_EQ(elements[0].getTag(), db::CellTag::Point); + EXPECT_EQ(elements[0].getAs(), cartesian); + ASSERT_EQ(elements[1].getTag(), db::CellTag::Point); + EXPECT_EQ(elements[1].getAs(), wgs84With3D); + } +} + // A label predicate answers with a mask, a boolean column that carries no nulls of its // own, so it goes on the wire as a plain BOOL vector. TEST(TuringProtoRoundTripTest, RoundTripsMaskColumns) { diff --git a/test/query-test-suite/QueryResultFormatter.cpp b/test/query-test-suite/QueryResultFormatter.cpp index 639c6e0e31..cfc70aa2c9 100644 --- a/test/query-test-suite/QueryResultFormatter.cpp +++ b/test/query-test-suite/QueryResultFormatter.cpp @@ -106,6 +106,13 @@ template return result; } +[[maybe_unused]] std::string valueToString(const db::Point& value) { + std::string result; + db::Point::format(result, value); + + return result; +} + [[maybe_unused]] std::string valueToString(const db::CommitBuilder* value) { return value ? "commit_builder_ptr" : "null"; } diff --git a/test/query/ir/CMakeLists.txt b/test/query/ir/CMakeLists.txt index 21d1aadb51..d29fdaf79d 100644 --- a/test/query/ir/CMakeLists.txt +++ b/test/query/ir/CMakeLists.txt @@ -81,6 +81,7 @@ add_subdirectory(optional_match) add_subdirectory(parameter) add_subdirectory(passes) add_subdirectory(path) +add_subdirectory(point) add_subdirectory(projection) add_subdirectory(set) add_subdirectory(subquery) diff --git a/test/query/ir/IRTestRows.cpp b/test/query/ir/IRTestRows.cpp index 587247ead5..bb7848c04f 100644 --- a/test/query/ir/IRTestRows.cpp +++ b/test/query/ir/IRTestRows.cpp @@ -140,6 +140,11 @@ void renderCellView(const CellView& element, std::string& out) { return; break; + case CellTag::Point: + Point::format(out, element.getAs()); + return; + break; + case CellTag::MapView: renderMap(element.getAs(), out); return; @@ -204,6 +209,9 @@ void renderValue(const T& value, std::string& out) { } else if constexpr (std::is_same_v) { out.clear(); Duration::format(out, value); + } else if constexpr (std::is_same_v) { + out.clear(); + Point::format(out, value); } else { out = std::to_string(value); } @@ -339,7 +347,8 @@ void turing::test::renderCell(const Column* column, size_t row, std::string& out || renderValueCell(column, row, out) || renderValueCell(column, row, out) || renderValueCell(column, row, out) - || renderValueCell(column, row, out)) { + || renderValueCell(column, row, out) + || renderValueCell(column, row, out)) { // Rendered by the helper for whichever value type matched } else { throw std::runtime_error("IRTestRows: unsupported output column type"); diff --git a/test/query/ir/StringRowSink.cpp b/test/query/ir/StringRowSink.cpp index de8411aefb..4331563dac 100644 --- a/test/query/ir/StringRowSink.cpp +++ b/test/query/ir/StringRowSink.cpp @@ -24,6 +24,7 @@ #include "map/MapView.h" #include "metadata/DateTime.h" #include "metadata/Duration.h" +#include "metadata/Point.h" #include "metadata/PropertyNull.h" #include "metadata/PropertyType.h" #include "versioning/CommitHash.h" @@ -92,6 +93,10 @@ void appendValueText(Duration value, std::string& text) { Duration::format(text, value); } +void appendValueText(Point value, std::string& text) { + Point::format(text, value); +} + void appendValueText(PropertyNull value, std::string& text) { text += "null"; } diff --git a/test/query/ir/engine/EquivalenceTest.cpp b/test/query/ir/engine/EquivalenceTest.cpp index 49946c4f6c..31570f58e4 100644 --- a/test/query/ir/engine/EquivalenceTest.cpp +++ b/test/query/ir/engine/EquivalenceTest.cpp @@ -122,6 +122,13 @@ std::string valueToString(const DateTime value) { return result; } +std::string valueToString(const Point value) { + std::string result; + Point::format(result, value); + + return result; +} + std::string valueToString(const Duration value) { std::string result; Duration::format(result, value); diff --git a/test/query/ir/load/CMakeLists.txt b/test/query/ir/load/CMakeLists.txt index 80fdc158ec..3460d747f1 100644 --- a/test/query/ir/load/CMakeLists.txt +++ b/test/query/ir/load/CMakeLists.txt @@ -46,3 +46,7 @@ target_include_directories(test_query_ir_load_jsonl_edge_endpoint PRIVATE ${IR_T add_turing_test(test_query_ir_load_csv_with_where LoadCSVWithWhereTest.cpp) target_sources(test_query_ir_load_csv_with_where PRIVATE ${IR_TEST_DIR}/StringRowSink.cpp) target_include_directories(test_query_ir_load_csv_with_where PRIVATE ${IR_TEST_DIR}) + +add_turing_test(test_query_ir_load_jsonl_list LoadJsonlListTest.cpp) +target_sources(test_query_ir_load_jsonl_list PRIVATE ${IR_TEST_DIR}/StringRowSink.cpp) +target_include_directories(test_query_ir_load_jsonl_list PRIVATE ${IR_TEST_DIR}) diff --git a/test/query/ir/load/LoadJsonlListTest.cpp b/test/query/ir/load/LoadJsonlListTest.cpp new file mode 100644 index 0000000000..cdce239c52 --- /dev/null +++ b/test/query/ir/load/LoadJsonlListTest.cpp @@ -0,0 +1,123 @@ +#include + +#include + +#include +#include +#include +#include +#include + +#include + +#include "QueryInterpreterV3.h" +#include "QueryStatus.h" + +#include "FileUtils.h" +#include "Graph.h" +#include "SimpleGraph.h" +#include "SystemAccessor.h" +#include "SystemManager.h" +#include "versioning/ChangeID.h" +#include "versioning/CommitHash.h" + +#include "StringRowSink.h" +#include "TuringTest.h" +#include "TuringTestEnv.h" + +using namespace db; +using namespace turing::test; + +// LOAD JSONL builds each line's lists in a scratch it clears before the next line, so every +// list has to read back whole once the lines after it have been imported. +class LoadJsonlListTest : public TuringTest { +public: + void initialize() override { + _env = TuringTestEnv::create(fs::Path {_outDir} / "turing"); + + SystemAccessor system = _env->getSystemManager().accessUnique(); + Graph* graph = system.createGraph(_sessionGraph); + SimpleGraph::createSimpleGraph(graph); + + _interpreter = std::make_unique(&_env->getSystemManager(), &_env->getMem(), &_env->getCompilerContext()); + } + +protected: + void writeFixture(std::string_view fileName, std::string_view contents) { + const FileUtils::Path dataDir {_env->getConfig().getDataDir().get()}; + + std::ofstream file(dataDir / std::string {fileName}); + ASSERT_TRUE(file.is_open()); + + file << contents; + } + + void runQuery(std::string_view query, std::string_view graphName, NLOutputSink& sink) { + QueryStatus status; + _interpreter->execute(status, + query, + graphName, + CommitHash::head(), + ChangeID::head(), + &sink); + + ASSERT_TRUE(status.isOk()) << query << ": " << status.getError(); + } + + void load(std::string_view query) { + StringRowSink sink; + runQuery(query, _sessionGraph, sink); + } + + void expectSortedRows(std::string_view query, + std::string_view graphName, + const std::vector& expected) { + StringRowSink sink; + runQuery(query, graphName, sink); + + std::vector rows; + sink.sortedRows(rows); + EXPECT_EQ(rows, expected) << query; + } + + const std::string _sessionGraph = "simpledb"; + std::unique_ptr _env; + std::unique_ptr _interpreter; +}; + +TEST_F(LoadJsonlListTest, ReadsTheListsOfEveryLine) { + writeFixture("items.jsonl", + R"({"type":"node","id":"0","labels":["Item"],"properties":{"name":"a","tags":["red","green"]}} +{"type":"node","id":"1","labels":["Item"],"properties":{"name":"b","tags":["blue"],"grid":[[1,2],[3]]}} +{"type":"node","id":"2","labels":["Item"],"properties":{"name":"c","tags":[]}} +{"type":"relationship","id":"3","label":"NEXT","start":{"id":"0"},"end":{"id":"1"},"properties":{"stops":["x","y"]}} +)"); + + load(R"(LOAD JSONL "items.jsonl" AS items)"); + + expectSortedRows("MATCH (n:Item) RETURN n.name, n.tags, n.grid", + "items", + {{"a", "red, green", "null"}, {"b", "blue", "[1, 2], [3]"}, {"c", "", "null"}}); + + expectSortedRows("MATCH ()-[r:NEXT]->() RETURN r.stops", "items", {{"x, y"}}); +} + +TEST_F(LoadJsonlListTest, ReadsTheListsOfAFileLongerThanOneScratchChunk) { + constexpr size_t LINE_COUNT = 2000; + + std::string contents; + for (size_t line = 0; line < LINE_COUNT; line++) { + contents += fmt::format(R"({{"type":"node","id":"{0}","labels":["Item"],"properties":{{"name":"n{0}","tags":["first-{0}","second-{0}"]}}}})", + line); + contents += '\n'; + } + + writeFixture("long.jsonl", contents); + + load(R"(LOAD JSONL "long.jsonl" AS long)"); + + expectSortedRows("MATCH (n:Item) RETURN count(n)", "long", {{"2000"}}); + expectSortedRows("MATCH (n:Item) WHERE n.name IN ['n0', 'n1000', 'n1999'] RETURN n.name, n.tags", + "long", + {{"n0", "first-0, second-0"}, {"n1000", "first-1000, second-1000"}, {"n1999", "first-1999, second-1999"}}); +} diff --git a/test/query/ir/map/MapProjectionTest.cpp b/test/query/ir/map/MapProjectionTest.cpp index 4d2dba4a0a..b0ca364781 100644 --- a/test/query/ir/map/MapProjectionTest.cpp +++ b/test/query/ir/map/MapProjectionTest.cpp @@ -184,5 +184,5 @@ TEST_F(MapProjectionTest, readsANodePatternWithPropertiesAsAPattern) { } TEST_F(MapProjectionTest, rejectsABaseThatIsNoEntityOrMap) { - expectError("WITH 1 AS x RETURN x {.a} AS m", "A map projection reads a node, a relationship, a map, a datetime or a duration"); + expectError("WITH 1 AS x RETURN x {.a} AS m", "A map projection reads a node, a relationship, a map, a datetime, a duration or a point"); } diff --git a/test/query/ir/map/TemporalMapProjectionTest.cpp b/test/query/ir/map/TemporalMapProjectionTest.cpp index 3bdd885395..90bebfedc8 100644 --- a/test/query/ir/map/TemporalMapProjectionTest.cpp +++ b/test/query/ir/map/TemporalMapProjectionTest.cpp @@ -184,7 +184,7 @@ TEST_F(TemporalMapProjectionTest, rejectsANameNoComponentOfATypedTemporalHas) { TEST_F(TemporalMapProjectionTest, rejectsAValueWithNoComponents) { expectRejected("WITH 1 AS d RETURN d {.hours}", - "A map projection reads a node, a relationship, a map, a datetime or a duration, and 'd' is 'Integer'"); + "A map projection reads a node, a relationship, a map, a datetime, a duration or a point, and 'd' is 'Integer'"); } TEST_F(TemporalMapProjectionTest, raisesOnEveryPropertyOfATemporalCell) { diff --git a/test/query/ir/point/CMakeLists.txt b/test/query/ir/point/CMakeLists.txt new file mode 100644 index 0000000000..4d42934d8b --- /dev/null +++ b/test/query/ir/point/CMakeLists.txt @@ -0,0 +1,35 @@ +add_turing_test(test_query_ir_point_property PointPropertyTest.cpp) +target_include_directories(test_query_ir_point_property PRIVATE ${IR_TEST_DIR}) + +add_turing_test(test_query_ir_point_value_flow PointValueFlowTest.cpp) +target_include_directories(test_query_ir_point_value_flow PRIVATE ${IR_TEST_DIR}) + +add_turing_test(test_query_ir_point_component PointComponentTest.cpp) +target_include_directories(test_query_ir_point_component PRIVATE ${IR_TEST_DIR}) + +add_turing_test(test_query_ir_point_spatial_functions PointSpatialFunctionsTest.cpp) +target_include_directories(test_query_ir_point_spatial_functions PRIVATE ${IR_TEST_DIR}) + +function(add_point_import_test exe_name cpp_file) + add_turing_test(${exe_name} ${cpp_file}) + target_sources(${exe_name} PRIVATE ${IR_TEST_DIR}/StringRowSink.cpp ${CMAKE_CURRENT_SOURCE_DIR}/PointImportTest.cpp) + target_include_directories(${exe_name} PRIVATE ${IR_TEST_DIR}) +endfunction() + +function(add_parquet_point_import_test exe_name cpp_file) + add_point_import_test(${exe_name} ${cpp_file}) + target_sources(${exe_name} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/ParquetPointImportTest.cpp) + target_compile_definitions(${exe_name} PRIVATE PARQUET_TEST_DATA_DIR="${CMAKE_SOURCE_DIR}/test/import/parquet") +endfunction() + +add_point_import_test(test_query_ir_load_jsonl_point LoadJsonlPointTest.cpp) +add_point_import_test(test_query_ir_jsonl_point_map JsonlPointMapTest.cpp) + +add_parquet_point_import_test(test_query_ir_load_parquet_point LoadParquetPointTest.cpp) +add_parquet_point_import_test(test_query_ir_geoparquet_metadata GeoParquetMetadataTest.cpp) +add_parquet_point_import_test(test_query_ir_parquet_point_srid ParquetPointSRIDTest.cpp) +add_parquet_point_import_test(test_query_ir_parquet_wkb_type ParquetWKBTypeTest.cpp) +add_parquet_point_import_test(test_query_ir_parquet_inline_crs ParquetInlineCRSTest.cpp) + +add_turing_test(test_query_ir_point_order_and_aggregate PointOrderAndAggregateTest.cpp) +target_include_directories(test_query_ir_point_order_and_aggregate PRIVATE ${IR_TEST_DIR}) diff --git a/test/query/ir/point/GeoParquetMetadataTest.cpp b/test/query/ir/point/GeoParquetMetadataTest.cpp new file mode 100644 index 0000000000..9dac7bd4c9 --- /dev/null +++ b/test/query/ir/point/GeoParquetMetadataTest.cpp @@ -0,0 +1,59 @@ +#include + +#include + +#include +#include +#include +#include + +#include "ParquetPointImportTest.h" + +using namespace turing::test; + +// GeoParquet's "geo" file metadata is JSON the file's writer chose. A value of the wrong +// kind anywhere in it is malformed input, rejected with a message that names the column. +class GeoParquetMetadataTest : public ParquetPointImportTest { +}; + +TEST_F(GeoParquetMetadataTest, RejectsAValueOfTheWrongKindByNamingTheColumn) { + const std::vector> cases = { + {R"({"columns": {"at": 5}})", + "The GeoParquet metadata of column 'at' is no object."}, + {R"({"columns": {"at": {"encoding": 1}}})", + "Geometry column 'at' is encoded as '1', not as WKB."}, + {R"({"columns": {"at": {"encoding": "WKB", "crs": 5}}})", + "Geometry column 'at' is in the CRS '5', and a point is cartesian or wgs-84."}, + {R"({"columns": {"at": {"encoding": "WKB", "crs": {"name": 7, "id": {"authority": "EPSG", "code": 3857}}}}})", + "Geometry column 'at' is in the CRS '7', and a point is cartesian or wgs-84."}, + {R"({"columns": {"at": {"encoding": "WKB", "crs": {"name": "WGS 84", "id": {"authority": 4, "code": 4326}}}}})", + "Geometry column 'at' is in the CRS 'WGS 84', and a point is cartesian or wgs-84."}, + }; + + size_t caseIndex = 0; + for (const auto& [geo, reason] : cases) { + const std::string directory = "malformed" + std::to_string(caseIndex++); + writeImport(directory, GeometryMarking::GeoParquetMetadata, geo, {wkbPoint(1, 2), wkbPoint(3, 4)}); + + runQueryExpectingError("LOAD PARQUET '" + directory + "' AS " + directory, reason); + } +} + +TEST_F(GeoParquetMetadataTest, RejectsMetadataNamingNoColumns) { + writeImport("bare", GeometryMarking::GeoParquetMetadata, "5", {wkbPoint(1, 2), wkbPoint(3, 4)}); + + runQueryExpectingError("LOAD PARQUET 'bare' AS bare", "The GeoParquet metadata of the file names no columns."); +} + +TEST_F(GeoParquetMetadataTest, ReadsTheEPSGCodeOfAPROJJSONCRSWrittenAsAString) { + writeImport("cities", + GeometryMarking::GeoParquetMetadata, + R"({"columns": {"at": {"encoding": "WKB", "crs": {"name": "WGS 84", "id": {"authority": "EPSG", "code": "4326"}}}}})", + {wkbPoint(2.3522, 48.8566), wkbPoint(1, 2)}); + + load("LOAD PARQUET 'cities' AS cities"); + + expectSortedRows("MATCH (n:City) RETURN n.at", + "cities", + {{"point({x: 1.0, y: 2.0, crs: 'wgs-84'})"}, {"point({x: 2.3522, y: 48.8566, crs: 'wgs-84'})"}}); +} diff --git a/test/query/ir/point/JsonlPointMapTest.cpp b/test/query/ir/point/JsonlPointMapTest.cpp new file mode 100644 index 0000000000..b556608810 --- /dev/null +++ b/test/query/ir/point/JsonlPointMapTest.cpp @@ -0,0 +1,62 @@ +#include + +#include +#include +#include + +#include "PointImportTest.h" + +using namespace turing::test; + +// LOAD JSONL reads the object of a WITH POINTS property by the rules point() reads a map +// with: it ignores a key it does not know unless that key holds null, which makes point() +// null, and raises point()'s own error for a known key holding a value of the wrong kind. +class JsonlPointMapTest : public PointImportTest { +}; + +TEST_F(JsonlPointMapTest, IgnoresAKeyPointDoesNotKnowHoldingAnyValueButNull) { + writeFixture("tagged.jsonl", + R"({"type":"node","id":"0","labels":["City"],"properties":{"at":{"longitude":2.35,"latitude":48.85,"verified":true,"tags":["a",1],"source":{"name":"gps"}}}} +)"); + + load(R"(LOAD JSONL "tagged.jsonl" AS tagged WITH POINTS [at])"); + + expectSortedRows("MATCH (n:City) RETURN n.at", "tagged", {{"point({x: 2.35, y: 48.85, crs: 'wgs-84'})"}}); + + expectSortedRows("RETURN point({longitude: 2.35, latitude: 48.85, verified: true, tags: ['a', 1], source: {name: 'gps'}})", + _sessionGraph, + {{"point({x: 2.35, y: 48.85, crs: 'wgs-84'})"}}); +} + +TEST_F(JsonlPointMapTest, RejectsAKnownKeyHoldingAValueOfTheWrongKindAsPointDoes) { + const std::vector> cases = { + {R"({"type":"node","id":"0","labels":["City"],"properties":{"at":{"x":true,"y":2}}})", + "property 'at': point() reads a number for 'x', not 'Bool'"}, + {R"({"type":"node","id":"0","labels":["City"],"properties":{"at":{"x":[1],"y":2}}})", + "property 'at': point() reads a number for 'x', not 'List'"}, + {R"({"type":"node","id":"0","labels":["City"],"properties":{"at":{"x":{"value":1},"y":2}}})", + "property 'at': point() reads a number for 'x', not 'Map'"}, + {R"({"type":"node","id":"0","labels":["City"],"properties":{"at":{"x":1,"y":2,"crs":false}}})", + "property 'at': point() reads a string for 'crs', not 'Bool'"}, + }; + + for (const auto& [line, reason] : cases) { + writeFixture("bad.jsonl", line); + runQueryExpectingError(R"(LOAD JSONL "bad.jsonl" AS bad WITH POINTS [at])", reason); + } + + runQueryExpectingError("RETURN point({x: true, y: 2})", "point() reads a number for 'x', not 'Bool'"); + runQueryExpectingError("RETURN point({x: [1], y: 2})", "point() reads a number for 'x', not 'List'"); + runQueryExpectingError("RETURN point({x: {value: 1}, y: 2})", "point() reads a number for 'x', not 'Map'"); + runQueryExpectingError("RETURN point({x: 1, y: 2, crs: false})", "point() reads a string for 'crs', not 'Bool'"); +} + +TEST_F(JsonlPointMapTest, RejectsANullUnderAKeyPointDoesNotKnowAsPointIsNull) { + writeFixture("unset.jsonl", + R"({"type":"node","id":"0","labels":["City"],"properties":{"at":{"x":1,"y":2,"accuracy":null}}} +)"); + + runQueryExpectingError(R"(LOAD JSONL "unset.jsonl" AS unset WITH POINTS [at])", "property 'at' holds a null"); + + expectSortedRows("RETURN point({x: 1, y: 2, accuracy: null})", _sessionGraph, {{"null"}}); +} diff --git a/test/query/ir/point/LoadJsonlPointTest.cpp b/test/query/ir/point/LoadJsonlPointTest.cpp new file mode 100644 index 0000000000..62d5d81114 --- /dev/null +++ b/test/query/ir/point/LoadJsonlPointTest.cpp @@ -0,0 +1,109 @@ +#include + +#include +#include +#include + +#include "PointImportTest.h" + +using namespace turing::test; + +// JSON has no point, so LOAD JSONL is told which object properties hold one with +// WITH POINTS [...]. An object is read as point() reads a map. +class LoadJsonlPointTest : public PointImportTest { +}; + +TEST_F(LoadJsonlPointTest, ReadsANamedObjectAsAPoint) { + writeFixture("cities.jsonl", + R"({"type":"node","id":"0","labels":["City"],"properties":{"name":"Paris","at":{"longitude":2.3522,"latitude":48.8566},"meta":{"x":1}}} +{"type":"node","id":"1","labels":["City"],"properties":{"name":"Tower","at":{"x":1,"y":2,"z":3}}} +{"type":"node","id":"2","labels":["City"],"properties":{"name":"Grid","at":{"x":1,"y":2,"crs":"cartesian"}}} +{"type":"node","id":"3","labels":["City"],"properties":{"name":"Peak","at":{"longitude":6.8652,"latitude":45.8326,"height":4808,"srid":4979}}} +)"); + + load(R"(LOAD JSONL "cities.jsonl" AS cities WITH POINTS [at])"); + + expectSortedRows("MATCH (n:City) RETURN n.name, n.at", + "cities", + {{"Grid", "point({x: 1.0, y: 2.0, crs: 'cartesian'})"}, + {"Paris", "point({x: 2.3522, y: 48.8566, crs: 'wgs-84'})"}, + {"Peak", "point({x: 6.8652, y: 45.8326, z: 4808.0, crs: 'wgs-84-3d'})"}, + {"Tower", "point({x: 1.0, y: 2.0, z: 3.0, crs: 'cartesian-3d'})"}}); + + expectSortedRows("CALL db.propertyTypes() YIELD propertyType, valueType RETURN propertyType, valueType", + "cities", + {{"at", "Point"}, {"meta", "String"}, {"name", "String"}}); +} + +TEST_F(LoadJsonlPointTest, ReadsAPointOnAnEdge) { + writeFixture("roads.jsonl", + R"({"type":"node","id":"0","labels":["City"],"properties":{"name":"a"}} +{"type":"node","id":"1","labels":["City"],"properties":{"name":"b"}} +{"type":"relationship","id":"2","label":"ROAD","start":{"id":"0"},"end":{"id":"1"},"properties":{"midpoint":{"x":0.5,"y":0.5}}} +)"); + + load(R"(LOAD JSONL "roads.jsonl" AS roads WITH POINTS [midpoint])"); + + expectSortedRows("MATCH ()-[r:ROAD]->() RETURN r.midpoint", "roads", {{"point({x: 0.5, y: 0.5, crs: 'cartesian'})"}}); +} + +TEST_F(LoadJsonlPointTest, SkipsANullPoint) { + writeFixture("sparse.jsonl", + R"({"type":"node","id":"0","labels":["City"],"properties":{"name":"a","at":null}} +{"type":"node","id":"1","labels":["City"],"properties":{"name":"b","at":{"x":1,"y":2}}} +)"); + + load(R"(LOAD JSONL "sparse.jsonl" AS sparse WITH POINTS [at])"); + + expectSortedRows("MATCH (n:City) RETURN n.name, n.at", "sparse", {{"a", "null"}, {"b", "point({x: 1.0, y: 2.0, crs: 'cartesian'})"}}); +} + +TEST_F(LoadJsonlPointTest, MeasuresBetweenImportedPoints) { + writeFixture("pair.jsonl", + R"({"type":"node","id":"0","labels":["City"],"properties":{"name":"a","at":{"x":0,"y":0}}} +{"type":"node","id":"1","labels":["City"],"properties":{"name":"b","at":{"x":3,"y":4}}} +)"); + + load(R"(LOAD JSONL "pair.jsonl" AS pair WITH POINTS [at])"); + + expectSortedRows("MATCH (a:City {name: 'a'}), (b:City {name: 'b'}) RETURN point.distance(a.at, b.at)", + "pair", + {{"5"}}); +} + +TEST_F(LoadJsonlPointTest, RejectsAValueThatIsNoObject) { + writeFixture("text.jsonl", + R"({"type":"node","id":"0","labels":["City"],"properties":{"at":"1,2"}} +)"); + + runQueryExpectingError(R"(LOAD JSONL "text.jsonl" AS text WITH POINTS [at])", "property 'at' is not an object"); +} + +TEST_F(LoadJsonlPointTest, RejectsAnObjectNamingNoPoint) { + const std::vector> cases = { + {R"({"type":"node","id":"0","labels":["City"],"properties":{"at":{"x":1}}})", + "property 'at': point() needs 'x' and 'y', or 'longitude' and 'latitude'"}, + {R"({"type":"node","id":"0","labels":["City"],"properties":{"at":{"longitude":0,"latitude":91}}})", + "property 'at': The latitude of a wgs-84 point lies in [-90, 90], not 91"}, + {R"({"type":"node","id":"0","labels":["City"],"properties":{"at":{"x":1,"y":null}}})", + "property 'at' holds a null"}, + {R"({"type":"node","id":"0","labels":["City"],"properties":{"at":{"x":[1],"y":2}}})", + "property 'at': point() reads a number for 'x', not 'List'"}, + }; + + for (const auto& [line, reason] : cases) { + writeFixture("bad.jsonl", line); + runQueryExpectingError(R"(LOAD JSONL "bad.jsonl" AS bad WITH POINTS [at])", reason); + } +} + +TEST_F(LoadJsonlPointTest, RejectsANameGivenTwoTypes) { + runQueryExpectingError(R"(LOAD JSONL "clash.jsonl" AS clash WITH DATETIMES ["at"] WITH POINTS ["at"])", + "Property 'at' is named both a datetime and a point."); + runQueryExpectingError(R"(LOAD JSONL "clash.jsonl" AS clash WITH POINTS ["v"] WITH EMBEDDINGS [{"v", 3}])", + "Property 'v' is named both an embedding and a point."); +} + +TEST_F(LoadJsonlPointTest, StillReadsPointsAsAnIdentifier) { + expectSortedRows("WITH 1 AS points RETURN points", _sessionGraph, {{"1"}}); +} diff --git a/test/query/ir/point/LoadParquetPointTest.cpp b/test/query/ir/point/LoadParquetPointTest.cpp new file mode 100644 index 0000000000..b69c098965 --- /dev/null +++ b/test/query/ir/point/LoadParquetPointTest.cpp @@ -0,0 +1,173 @@ +#include + +#include +#include + +#include "ParquetPointImportTest.h" + +using namespace turing::test; + +// A Parquet column holds points where Parquet's GEOMETRY or GEOGRAPHY logical type marks it, +// or GeoParquet's "geo" file metadata names it: one WKB point per row. +class LoadParquetPointTest : public ParquetPointImportTest { +}; + +TEST_F(LoadParquetPointTest, ReadsAGeometryColumn) { + writeImport("cities", + GeometryMarking::GeometryLogicalType, + "OGC:CRS84", + {wkbPoint(2.3522, 48.8566), wkbPoint(6.8652, 45.8326, 4808.0), std::nullopt}); + + load("LOAD PARQUET 'cities' AS cities"); + + expectSortedRows("MATCH (n:City) RETURN n.at", + "cities", + {{"null"}, + {"point({x: 2.3522, y: 48.8566, crs: 'wgs-84'})"}, + {"point({x: 6.8652, y: 45.8326, z: 4808.0, crs: 'wgs-84-3d'})"}}); + + expectSortedRows("CALL db.propertyTypes() YIELD propertyType, valueType RETURN propertyType, valueType", + "cities", + {{"at", "Point"}}); +} + +TEST_F(LoadParquetPointTest, ReadsAGeographyColumn) { + writeImport("cities", GeometryMarking::GeographyLogicalType, "EPSG:4326", {wkbPoint(2.3522, 48.8566), wkbPoint(0, 0)}); + + load("LOAD PARQUET 'cities' AS cities"); + + expectSortedRows("MATCH (n:City) RETURN n.at", + "cities", + {{"point({x: 0.0, y: 0.0, crs: 'wgs-84'})"}, {"point({x: 2.3522, y: 48.8566, crs: 'wgs-84'})"}}); +} + +TEST_F(LoadParquetPointTest, ReadsTheDefaultCRSAsWGS84) { + writeImport("cities", GeometryMarking::GeometryLogicalType, "", {wkbPoint(1, 2), wkbPoint(3, 4)}); + + load("LOAD PARQUET 'cities' AS cities"); + + expectSortedRows("MATCH (n:City) RETURN n.at.crs", "cities", {{"wgs-84"}, {"wgs-84"}}); +} + +TEST_F(LoadParquetPointTest, ReadsAColumnGeoParquetNames) { + writeImport("cities", + GeometryMarking::GeoParquetMetadata, + R"({"version": "1.1.0", "primary_column": "at", "columns": {"at": {"encoding": "WKB", "geometry_types": ["Point"]}}})", + {wkbPoint(2.3522, 48.8566), wkbPoint(1, 2)}); + + load("LOAD PARQUET 'cities' AS cities"); + + expectSortedRows("MATCH (n:City) RETURN n.at", + "cities", + {{"point({x: 1.0, y: 2.0, crs: 'wgs-84'})"}, {"point({x: 2.3522, y: 48.8566, crs: 'wgs-84'})"}}); +} + +TEST_F(LoadParquetPointTest, ReadsAnUndefinedCRSAsCartesian) { + writeImport("grid", + GeometryMarking::GeoParquetMetadata, + R"({"version": "1.1.0", "primary_column": "at", "columns": {"at": {"encoding": "WKB", "crs": null}}})", + {wkbPoint(1, 2), wkbPoint(1, 2, 3)}); + + load("LOAD PARQUET 'grid' AS grid"); + + expectSortedRows("MATCH (n:City) RETURN n.at", + "grid", + {{"point({x: 1.0, y: 2.0, crs: 'cartesian'})"}, {"point({x: 1.0, y: 2.0, z: 3.0, crs: 'cartesian-3d'})"}}); +} + +TEST_F(LoadParquetPointTest, ReadsAPROJJSONCRS) { + writeImport("cities", + GeometryMarking::GeoParquetMetadata, + R"({"version": "1.1.0", "primary_column": "at", "columns": {"at": {"encoding": "WKB", + "crs": {"type": "GeographicCRS", "name": "WGS 84", "id": {"authority": "EPSG", "code": 4326}}}}})", + {wkbPoint(1, 2), wkbPoint(3, 4)}); + + load("LOAD PARQUET 'cities' AS cities"); + + expectSortedRows("MATCH (n:City) RETURN n.at.srid", "cities", {{"4326"}, {"4326"}}); +} + +TEST_F(LoadParquetPointTest, ReadsBigEndianEWKB) { + writeImport("cities", GeometryMarking::GeometryLogicalType, "OGC:CRS84", {bigEndianEWKBPoint(2.3522, 48.8566), wkbPoint(1, 2)}); + + load("LOAD PARQUET 'cities' AS cities"); + + expectSortedRows("MATCH (n:City) RETURN n.at", + "cities", + {{"point({x: 1.0, y: 2.0, crs: 'wgs-84'})"}, {"point({x: 2.3522, y: 48.8566, crs: 'wgs-84'})"}}); +} + +TEST_F(LoadParquetPointTest, ReadsAnEmptyPointAsNull) { + const double nan = std::numeric_limits::quiet_NaN(); + writeImport("cities", GeometryMarking::GeometryLogicalType, "OGC:CRS84", {wkbPoint(nan, nan), wkbPoint(1, 2)}); + + load("LOAD PARQUET 'cities' AS cities"); + + expectSortedRows("MATCH (n:City) RETURN n.at", "cities", {{"null"}, {"point({x: 1.0, y: 2.0, crs: 'wgs-84'})"}}); +} + +TEST_F(LoadParquetPointTest, MeasuresBetweenImportedPoints) { + writeImport("cities", GeometryMarking::GeometryLogicalType, "OGC:CRS84", {wkbPoint(0, 0), wkbPoint(1, 0)}); + + load("LOAD PARQUET 'cities' AS cities"); + + expectSortedRows("MATCH (a:City), (b:City) WHERE id(a) < id(b) RETURN point.distance(a.at, b.at)", + "cities", + {{"111195.0802335329"}}); +} + +TEST_F(LoadParquetPointTest, RejectsAGeometryThatIsNoPoint) { + writeImport("lines", GeometryMarking::GeometryLogicalType, "OGC:CRS84", {wkbOfType(2, {}), wkbPoint(1, 2)}); + + runQueryExpectingError("LOAD PARQUET 'lines' AS lines", "Geometry column 'at' holds a LineString, not a point."); +} + +TEST_F(LoadParquetPointTest, RejectsAPointWithAMeasure) { + writeImport("measured", GeometryMarking::GeometryLogicalType, "OGC:CRS84", {wkbOfType(2001, {1, 2, 3}), wkbPoint(1, 2)}); + + runQueryExpectingError("LOAD PARQUET 'measured' AS measured", "Geometry column 'at' holds a point with a measure"); +} + +TEST_F(LoadParquetPointTest, RejectsBytesThatAreNoWKB) { + writeImport("truncated", GeometryMarking::GeometryLogicalType, "OGC:CRS84", {wkbPoint(1, 2).substr(0, 9), wkbPoint(1, 2)}); + + runQueryExpectingError("LOAD PARQUET 'truncated' AS truncated", "Geometry column 'at' holds a value that is no WKB."); +} + +TEST_F(LoadParquetPointTest, RejectsACRSNoPointIsIn) { + writeImport("mercator", GeometryMarking::GeometryLogicalType, "EPSG:3857", {wkbPoint(1, 2), wkbPoint(3, 4)}); + + runQueryExpectingError("LOAD PARQUET 'mercator' AS mercator", + "Geometry column 'at' is in the CRS 'EPSG:3857', and a point is cartesian or wgs-84."); +} + +TEST_F(LoadParquetPointTest, RejectsAnEncodingThatIsNoWKB) { + writeImport("native", + GeometryMarking::GeoParquetMetadata, + R"({"version": "1.1.0", "primary_column": "at", "columns": {"at": {"encoding": "point"}}})", + {wkbPoint(1, 2), wkbPoint(3, 4)}); + + runQueryExpectingError("LOAD PARQUET 'native' AS native", "Geometry column 'at' is encoded as 'point', not as WKB."); +} + +TEST_F(LoadParquetPointTest, RejectsALatitudePastAPole) { + writeImport("pole", GeometryMarking::GeometryLogicalType, "OGC:CRS84", {wkbPoint(0, 91), wkbPoint(1, 2)}); + + runQueryExpectingError("LOAD PARQUET 'pole' AS pole", "The latitude of a wgs-84 point lies in [-90, 90], not 91"); +} + +TEST_F(LoadParquetPointTest, ReadsAListOfGeometries) { + writeImport("routes", + GeometryMarking::GeometryLogicalType, + "OGC:CRS84", + {wkbPoint(0, 0), wkbPoint(1, 1)}, + {{wkbPoint(0, 0), wkbPoint(0, 1, 5.0)}, {}}); + + load("LOAD PARQUET 'routes' AS routes"); + + expectSortedRows("MATCH (n:City) RETURN n.at, n.route", + "routes", + {{"point({x: 0.0, y: 0.0, crs: 'wgs-84'})", + "point({x: 0.0, y: 0.0, crs: 'wgs-84'}), point({x: 0.0, y: 1.0, z: 5.0, crs: 'wgs-84-3d'})"}, + {"point({x: 1.0, y: 1.0, crs: 'wgs-84'})", ""}}); +} diff --git a/test/query/ir/point/ParquetInlineCRSTest.cpp b/test/query/ir/point/ParquetInlineCRSTest.cpp new file mode 100644 index 0000000000..d80937b76d --- /dev/null +++ b/test/query/ir/point/ParquetInlineCRSTest.cpp @@ -0,0 +1,34 @@ +#include + +#include "ParquetPointImportTest.h" + +using namespace turing::test; + +// Arrow writes the CRS of a GEOMETRY or GEOGRAPHY logical type other than OGC:CRS84 as the +// PROJJSON text itself, not as a "projjson:" reference, and that text names its CRS as +// the referenced PROJJSON does. +class ParquetInlineCRSTest : public ParquetPointImportTest { +}; + +TEST_F(ParquetInlineCRSTest, ReadsAGeographicCRSWrittenAsInlinePROJJSON) { + writeImport("peaks", + GeometryMarking::GeometryLogicalType, + R"json({"type": "GeographicCRS", "name": "WGS 84 (3D)", "id": {"authority": "EPSG", "code": 4979}})json", + {wkbPoint(6.8652, 45.8326, 4808.0), wkbPoint(1, 2)}); + + load("LOAD PARQUET 'peaks' AS peaks"); + + expectSortedRows("MATCH (n:City) RETURN n.at", + "peaks", + {{"point({x: 1.0, y: 2.0, crs: 'wgs-84'})"}, {"point({x: 6.8652, y: 45.8326, z: 4808.0, crs: 'wgs-84-3d'})"}}); +} + +TEST_F(ParquetInlineCRSTest, RejectsAProjectedCRSWrittenAsInlinePROJJSON) { + writeImport("mercator", + GeometryMarking::GeographyLogicalType, + R"json({"type": "ProjectedCRS", "name": "WGS 84 / Pseudo-Mercator", "id": {"authority": "EPSG", "code": 3857}})json", + {wkbPoint(1, 2), wkbPoint(3, 4)}); + + runQueryExpectingError("LOAD PARQUET 'mercator' AS mercator", + "Geometry column 'at' is in the CRS 'WGS 84 / Pseudo-Mercator', and a point is cartesian or wgs-84."); +} diff --git a/test/query/ir/point/ParquetPointImportTest.cpp b/test/query/ir/point/ParquetPointImportTest.cpp new file mode 100644 index 0000000000..53cd9ec0e6 --- /dev/null +++ b/test/query/ir/point/ParquetPointImportTest.cpp @@ -0,0 +1,211 @@ +#include "ParquetPointImportTest.h" + +#include + +#include + +#include +#include +#include +#include +#include + +#include "FileUtils.h" + +using namespace db; +using namespace turing::test; + +namespace { + +constexpr uint32_t EWKB_POINT_WITH_SRID = 0x20000001; +constexpr uint32_t EWKB_Z_FLAG = 0x80000000; + +template +void appendValue(std::string& bytes, Value value) { + char raw[sizeof(Value)]; + memcpy(raw, &value, sizeof(Value)); + bytes.append(raw, sizeof(Value)); +} + +} + +std::string turing::test::wkbPoint(double x, double y, std::optional z) { + std::string bytes; + appendValue(bytes, 1); + appendValue(bytes, z ? 1001 : 1); + appendValue(bytes, x); + appendValue(bytes, y); + + if (z) { + appendValue(bytes, *z); + } + + return bytes; +} + +std::string turing::test::wkbOfType(uint32_t type, std::initializer_list values) { + std::string bytes; + appendValue(bytes, 1); + appendValue(bytes, type); + + for (const double value : values) { + appendValue(bytes, value); + } + + return bytes; +} + +std::string turing::test::bigEndianEWKBPoint(double x, double y) { + std::string bytes; + bytes.push_back(0); + + const auto appendBigEndian = [&bytes](Value value) { + char raw[sizeof(Value)]; + memcpy(raw, &value, sizeof(Value)); + for (size_t index = sizeof(Value); index > 0; index--) { + bytes.push_back(raw[index - 1]); + } + }; + + appendBigEndian(EWKB_POINT_WITH_SRID); + appendBigEndian(uint32_t {4326}); + appendBigEndian(x); + appendBigEndian(y); + + return bytes; +} + +std::string turing::test::ewkbPoint(double x, double y, uint32_t srid, std::optional z) { + std::string bytes; + appendValue(bytes, 1); + appendValue(bytes, z ? EWKB_POINT_WITH_SRID | EWKB_Z_FLAG : EWKB_POINT_WITH_SRID); + appendValue(bytes, srid); + appendValue(bytes, x); + appendValue(bytes, y); + + if (z) { + appendValue(bytes, *z); + } + + return bytes; +} + +void ParquetPointImportTest::writeImport(std::string_view directory, + GeometryMarking marking, + std::string_view crs, + const std::vector>& points, + const std::vector>& routes) { + FileUtils::Path dataDir; + getDataDir(dataDir); + + const FileUtils::Path importDir = dataDir / std::string {directory}; + FileUtils::createDirectory(importDir); + + const FileUtils::Path fixtureDir {PARQUET_TEST_DATA_DIR}; + FileUtils::copy(fixtureDir / "minimal_edges.parquet", importDir / "edges.parquet"); + + namespace schema = parquet::schema; + + const std::string crsText {crs}; + std::shared_ptr atType = parquet::LogicalType::None(); + std::shared_ptr keyValues; + + if (marking == GeometryMarking::GeometryLogicalType) { + atType = parquet::LogicalType::Geometry(crsText); + } else if (marking == GeometryMarking::GeographyLogicalType) { + atType = parquet::LogicalType::Geography(crsText); + } else { + keyValues = arrow::key_value_metadata({"geo"}, {crsText}); + } + + const schema::NodePtr element = schema::PrimitiveNode::Make("element", + parquet::Repetition::OPTIONAL, + parquet::LogicalType::String(), + parquet::Type::BYTE_ARRAY); + const schema::NodePtr list = schema::GroupNode::Make("list", parquet::Repetition::REPEATED, {element}); + + schema::NodeVector fields = { + schema::PrimitiveNode::Make("__id", parquet::Repetition::REQUIRED, parquet::Type::INT64), + schema::GroupNode::Make("__labels", parquet::Repetition::OPTIONAL, {list}, parquet::LogicalType::List()), + schema::PrimitiveNode::Make("at", parquet::Repetition::OPTIONAL, atType, parquet::Type::BYTE_ARRAY), + }; + + if (!routes.empty()) { + const schema::NodePtr stop = schema::PrimitiveNode::Make("element", + parquet::Repetition::OPTIONAL, + atType, + parquet::Type::BYTE_ARRAY); + const schema::NodePtr stops = schema::GroupNode::Make("list", parquet::Repetition::REPEATED, {stop}); + fields.push_back(schema::GroupNode::Make("route", parquet::Repetition::OPTIONAL, {stops}, parquet::LogicalType::List())); + } + + const auto root = std::static_pointer_cast( + schema::GroupNode::Make("schema", parquet::Repetition::REQUIRED, fields)); + + const std::string nodePath = (importDir / "nodes.parquet").string(); + auto file = arrow::io::FileOutputStream::Open(nodePath).ValueOrDie(); + std::unique_ptr writer = + parquet::ParquetFileWriter::Open(file, root, parquet::default_writer_properties(), keyValues); + + parquet::RowGroupWriter* rowGroup = writer->AppendRowGroup(); + const int64_t rowCount = static_cast(points.size()); + + std::vector ids(points.size()); + for (size_t index = 0; index < points.size(); index++) { + ids[index] = static_cast(index); + } + + static_cast(rowGroup->NextColumn())->WriteBatch(rowCount, nullptr, nullptr, ids.data()); + + parquet::ByteArrayWriter* labelWriter = static_cast(rowGroup->NextColumn()); + const int16_t labelDefLevel = labelWriter->descr()->max_definition_level(); + const std::vector labelDefLevels(points.size(), labelDefLevel); + const std::vector labelRepLevels(points.size(), 0); + const std::vector labels(points.size(), parquet::ByteArray("City")); + labelWriter->WriteBatch(rowCount, labelDefLevels.data(), labelRepLevels.data(), labels.data()); + + std::vector atDefLevels; + std::vector atValues; + for (const std::optional& point : points) { + atDefLevels.push_back(point ? 1 : 0); + if (point) { + atValues.emplace_back(static_cast(point->size()), + reinterpret_cast(point->data())); + } + } + + static_cast(rowGroup->NextColumn())->WriteBatch(rowCount, + atDefLevels.data(), + nullptr, + atValues.data()); + + if (!routes.empty()) { + parquet::ByteArrayWriter* routeWriter = static_cast(rowGroup->NextColumn()); + const int16_t stopDefLevel = routeWriter->descr()->max_definition_level(); + + std::vector routeDefLevels; + std::vector routeRepLevels; + std::vector routeValues; + + for (const std::vector& route : routes) { + if (route.empty()) { + routeDefLevels.push_back(stopDefLevel - 2); + routeRepLevels.push_back(0); + } + + for (size_t index = 0; index < route.size(); index++) { + routeDefLevels.push_back(stopDefLevel); + routeRepLevels.push_back(index == 0 ? 0 : 1); + routeValues.emplace_back(static_cast(route[index].size()), + reinterpret_cast(route[index].data())); + } + } + + routeWriter->WriteBatch(static_cast(routeDefLevels.size()), + routeDefLevels.data(), + routeRepLevels.data(), + routeValues.data()); + } + + writer->Close(); +} diff --git a/test/query/ir/point/ParquetPointImportTest.h b/test/query/ir/point/ParquetPointImportTest.h new file mode 100644 index 0000000000..ef55de62c7 --- /dev/null +++ b/test/query/ir/point/ParquetPointImportTest.h @@ -0,0 +1,46 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include + +#include "PointImportTest.h" + +namespace turing::test { + +// How a node file marks its "at" column as holding geometries +enum class GeometryMarking { + GeometryLogicalType, + GeographyLogicalType, + GeoParquetMetadata, +}; + +// A little-endian ISO WKB point, 3D where @param z is given +std::string wkbPoint(double x, double y, std::optional z = std::nullopt); + +std::string wkbOfType(uint32_t type, std::initializer_list values); + +// A big-endian EWKB point carrying the SRID 4326 +std::string bigEndianEWKBPoint(double x, double y); + +// A little-endian EWKB point carrying @param srid, 3D where @param z is given +std::string ewkbPoint(double x, double y, uint32_t srid, std::optional z = std::nullopt); + +// A PointImportTest that writes its imports as Parquet node and edge files +class ParquetPointImportTest : public PointImportTest { +protected: + // Writes nodes 0, 1, ... labelled City, node i holding @param points[i] under "at", and + // the minimal edge file, into the import directory @param directory. @param crs is the + // logical type's CRS, or the whole "geo" metadata for GeoParquetMetadata. + void writeImport(std::string_view directory, + GeometryMarking marking, + std::string_view crs, + const std::vector>& points, + const std::vector>& routes = {}); +}; + +} diff --git a/test/query/ir/point/ParquetPointSRIDTest.cpp b/test/query/ir/point/ParquetPointSRIDTest.cpp new file mode 100644 index 0000000000..6c5d26e8f3 --- /dev/null +++ b/test/query/ir/point/ParquetPointSRIDTest.cpp @@ -0,0 +1,63 @@ +#include + +#include + +#include "ParquetPointImportTest.h" + +using namespace turing::test; + +namespace { + +constexpr std::string_view UNDEFINED_CRS_GEO = + R"({"version": "1.1.0", "primary_column": "at", "columns": {"at": {"encoding": "WKB", "crs": null}}})"; + +} + +// An EWKB value carries the SRID of its point. A non-zero one names a CRS, which has to be +// the CRS of the point's column: a column's points are all in one CRS. +class ParquetPointSRIDTest : public ParquetPointImportTest { +}; + +TEST_F(ParquetPointSRIDTest, ReadsAGeographicSRIDInAGeographicColumn) { + writeImport("cities", + GeometryMarking::GeometryLogicalType, + "OGC:CRS84", + {ewkbPoint(2.3522, 48.8566, 4326), ewkbPoint(6.8652, 45.8326, 4979, 4808.0), ewkbPoint(1, 2, 4326, 3.0)}); + + load("LOAD PARQUET 'cities' AS cities"); + + expectSortedRows("MATCH (n:City) RETURN n.at", + "cities", + {{"point({x: 1.0, y: 2.0, z: 3.0, crs: 'wgs-84-3d'})"}, + {"point({x: 2.3522, y: 48.8566, crs: 'wgs-84'})"}, + {"point({x: 6.8652, y: 45.8326, z: 4808.0, crs: 'wgs-84-3d'})"}}); +} + +TEST_F(ParquetPointSRIDTest, ReadsAnSRIDOfZeroAsNamingNoCRS) { + writeImport("grid", GeometryMarking::GeoParquetMetadata, UNDEFINED_CRS_GEO, {ewkbPoint(1, 2, 0), wkbPoint(3, 4)}); + writeImport("cities", GeometryMarking::GeometryLogicalType, "OGC:CRS84", {ewkbPoint(2.3522, 48.8566, 0), wkbPoint(1, 2)}); + + load("LOAD PARQUET 'grid' AS grid"); + load("LOAD PARQUET 'cities' AS cities"); + + expectSortedRows("MATCH (n:City) RETURN n.at", + "grid", + {{"point({x: 1.0, y: 2.0, crs: 'cartesian'})"}, {"point({x: 3.0, y: 4.0, crs: 'cartesian'})"}}); + expectSortedRows("MATCH (n:City) RETURN n.at", + "cities", + {{"point({x: 1.0, y: 2.0, crs: 'wgs-84'})"}, {"point({x: 2.3522, y: 48.8566, crs: 'wgs-84'})"}}); +} + +TEST_F(ParquetPointSRIDTest, RejectsAGeographicSRIDInAColumnOfUndefinedCRS) { + writeImport("grid", GeometryMarking::GeoParquetMetadata, UNDEFINED_CRS_GEO, {wkbPoint(1, 2), ewkbPoint(2.3522, 48.8566, 4326)}); + + runQueryExpectingError("LOAD PARQUET 'grid' AS grid", + "Geometry column 'at' holds a point with SRID 4326, which is not the column's CRS."); +} + +TEST_F(ParquetPointSRIDTest, RejectsAProjectedSRIDInAGeographicColumn) { + writeImport("mercator", GeometryMarking::GeometryLogicalType, "OGC:CRS84", {wkbPoint(1, 2), ewkbPoint(1000.0, 50.0, 3857)}); + + runQueryExpectingError("LOAD PARQUET 'mercator' AS mercator", + "Geometry column 'at' holds a point with SRID 3857, which is not the column's CRS."); +} diff --git a/test/query/ir/point/ParquetWKBTypeTest.cpp b/test/query/ir/point/ParquetWKBTypeTest.cpp new file mode 100644 index 0000000000..bfde03bb5f --- /dev/null +++ b/test/query/ir/point/ParquetWKBTypeTest.cpp @@ -0,0 +1,34 @@ +#include + +#include + +#include + +#include "ParquetPointImportTest.h" + +using namespace turing::test; + +// ISO WKB spells the dimensions of a geometry in the thousands of its type: 0 for XY, 1 for +// XYZ, 2 for XYM and 3 for XYZM. A type with any other thousands is no WKB. +class ParquetWKBTypeTest : public ParquetPointImportTest { +}; + +TEST_F(ParquetWKBTypeTest, ReadsEachISODimensionCodeOfAPoint) { + writeImport("cities", GeometryMarking::GeometryLogicalType, "OGC:CRS84", {wkbOfType(1, {1, 2}), wkbOfType(1001, {3, 4, 5})}); + + load("LOAD PARQUET 'cities' AS cities"); + + expectSortedRows("MATCH (n:City) RETURN n.at", + "cities", + {{"point({x: 1.0, y: 2.0, crs: 'wgs-84'})"}, {"point({x: 3.0, y: 4.0, z: 5.0, crs: 'wgs-84-3d'})"}}); +} + +TEST_F(ParquetWKBTypeTest, RejectsATypeWithNoISODimensionCode) { + for (const uint32_t type : {4001u, 1000001u, 9001u}) { + const std::string directory = "dimensions" + std::to_string(type); + writeImport(directory, GeometryMarking::GeometryLogicalType, "OGC:CRS84", {wkbOfType(type, {1, 2}), wkbPoint(1, 2)}); + + runQueryExpectingError("LOAD PARQUET '" + directory + "' AS " + directory, + "Geometry column 'at' holds a value that is no WKB."); + } +} diff --git a/test/query/ir/point/PointComponentTest.cpp b/test/query/ir/point/PointComponentTest.cpp new file mode 100644 index 0000000000..8173957ec0 --- /dev/null +++ b/test/query/ir/point/PointComponentTest.cpp @@ -0,0 +1,218 @@ +#include + +#include +#include +#include +#include + +#include "QueryInterpreterV3.h" +#include "QueryStatus.h" + +#include "Graph.h" +#include "QueryConfig.h" +#include "SimpleGraph.h" +#include "SystemAccessor.h" +#include "SystemManager.h" +#include "TuringDB.h" +#include "versioning/ChangeID.h" +#include "versioning/CommitHash.h" + +#include "IRTestRows.h" +#include "TuringTest.h" +#include "TuringTestEnv.h" + +using namespace db; +using namespace turing::test; + +class PointComponentTest : public TuringTest { +protected: + void initialize() override { + _env = TuringTestEnv::create(fs::Path {_outDir} / "turing"); + _interpreter = std::make_unique(&_env->getSystemManager(), &_env->getMem(), &_env->getCompilerContext()); + + SystemAccessor system = _env->getSystemManager().accessUnique(); + Graph* graph = system.createGraph(_graphName); + SimpleGraph::createSimpleGraph(graph); + } + + void openChange(ChangeID& changeID) { + SystemAccessor system = _env->getSystemManager().accessUnique(); + const auto res = system.newChange(_graphName); + ASSERT_TRUE(res); + + changeID = res.value()->id(); + } + + void submit(const ChangeID& changeID) { + const QueryState submitState(_graphName, + &_env->getMem(), + &_env->getCompilerContext(), + &_queryConfig, + nullptr, + CommitHash::head(), + changeID); + const QueryStatus status = _env->getDB().query("CHANGE SUBMIT", submitState); + ASSERT_TRUE(status.isOk()) << "CHANGE SUBMIT failed"; + } + + void write(std::string_view query) { + ChangeID changeID; + openChange(changeID); + + RowSink sink; + QueryStatus status; + _interpreter->execute(status, + query, + _graphName, + CommitHash::head(), + changeID, + &sink); + ASSERT_TRUE(status.isOk()) << "query: " << query << "\nerror: " << status.getError(); + + submit(changeID); + } + + void run(std::string_view query, RowSink& sink, QueryStatus& status) { + _interpreter->execute(status, + query, + _graphName, + CommitHash::head(), + ChangeID::head(), + &sink); + } + + void expectRows(std::string_view query, const Rows& expected) { + RowSink sink; + QueryStatus status; + run(query, sink, status); + ASSERT_TRUE(status.isOk()) << "query: " << query << "\nerror: " << status.getError(); + + Rows actual; + sink.sortedRows(actual); + + Rows sortedExpected = expected; + std::sort(sortedExpected.begin(), sortedExpected.end()); + + std::string actualText; + describeRows(actual, actualText); + + EXPECT_EQ(actual, sortedExpected) << "query: " << query << "\ngot:\n" << actualText; + } + + void expectOrderedRows(std::string_view query, const Rows& expected) { + RowSink sink; + QueryStatus status; + run(query, sink, status); + ASSERT_TRUE(status.isOk()) << "query: " << query << "\nerror: " << status.getError(); + + std::string actualText; + describeRows(sink.rows(), actualText); + + EXPECT_EQ(sink.rows(), expected) << "query: " << query << "\ngot:\n" << actualText; + } + + void expectError(std::string_view query, std::string_view expectedError) { + RowSink sink; + QueryStatus status; + run(query, sink, status); + + ASSERT_FALSE(status.isOk()) << "query: " << query << "\nexpected it to fail"; + EXPECT_NE(status.getError().find(expectedError), std::string::npos) + << "query: " << query << "\nerror: " << status.getError(); + } + + void createFourCities() { + write("CREATE (n:City {name: 'Paris', at: point({longitude: 2.3522, latitude: 48.8566})})"); + write("CREATE (n:City {name: 'London', at: point({longitude: -0.1276, latitude: 51.5072})})"); + write("CREATE (n:City {name: 'Grid', at: point({x: 3, y: 4})})"); + write("CREATE (n:City {name: 'Tower', at: point({x: 3, y: 4, z: 5})})"); + } + + const std::string _graphName = "simpledb"; + std::unique_ptr _env; + std::unique_ptr _interpreter; + QueryConfig _queryConfig; +}; + +TEST_F(PointComponentTest, readsTheCoordinatesOfAPoint) { + expectRows("RETURN point({x: 1, y: 2}).x, point({x: 1, y: 2}).y", {{"1.000000", "2.000000"}}); + expectRows("WITH point({x: 1, y: 2, z: 3}) AS p RETURN p.x, p.y, p.z", {{"1.000000", "2.000000", "3.000000"}}); +} + +TEST_F(PointComponentTest, readsTheGeographicNamesOfTheCoordinates) { + expectRows("WITH point({longitude: 2.5, latitude: 48.5, height: 35}) AS p " + "RETURN p.longitude, p.latitude, p.height, p.x, p.y, p.z", + {{"2.500000", "48.500000", "35.000000", "2.500000", "48.500000", "35.000000"}}); +} + +TEST_F(PointComponentTest, readsTheCRSAndTheSRID) { + expectRows("UNWIND [{x: 1, y: 2}, {x: 1, y: 2, z: 3}, {longitude: 1, latitude: 2}, " + "{longitude: 1, latitude: 2, height: 3}] AS m " + "WITH point(m) AS p RETURN p.crs, p.srid", + {{"cartesian", "7203"}, {"cartesian-3d", "9157"}, {"wgs-84", "4326"}, {"wgs-84-3d", "4979"}}); +} + +TEST_F(PointComponentTest, readsTheComponentsOfAStoredPoint) { + write("CREATE (n:City {name: 'Paris', at: point({longitude: 2.5, latitude: 48.5})})"); + expectRows("MATCH (n:City) RETURN n.at.longitude, n.at.latitude, n.at.crs", + {{"2.500000", "48.500000", "wgs-84"}}); +} + +TEST_F(PointComponentTest, computesWithAComponent) { + expectRows("WITH point({x: 3, y: 4}) AS p RETURN sqrt(p.x * p.x + p.y * p.y)", {{"5.000000"}}); +} + +TEST_F(PointComponentTest, filtersAndOrdersOnAComponent) { + write("CREATE (n:City {name: 'a', at: point({x: 3, y: 1})})"); + write("CREATE (n:City {name: 'b', at: point({x: 1, y: 2})})"); + write("CREATE (n:City {name: 'c', at: point({x: 2, y: 3})})"); + + expectRows("MATCH (n:City) WHERE n.at.x >= 2 RETURN n.name", {{"a"}, {"c"}}); + expectOrderedRows("MATCH (n:City) RETURN n.name ORDER BY n.at.y DESC", {{"c"}, {"b"}, {"a"}}); +} + +TEST_F(PointComponentTest, readsNullOffANullPoint) { + write("CREATE (n:City {name: 'a'})"); + expectRows("MATCH (n:City) RETURN n.at.x, n.at.crs, n.at.srid", {{"null", "null", "null"}}); +} + +TEST_F(PointComponentTest, rejectsAThirdCoordinateOfA2DPoint) { + expectError("WITH point({x: 1, y: 2}) AS p RETURN p.z", + "The point point({x: 1.0, y: 2.0, crs: 'cartesian'}) has no z"); + expectError("WITH point({longitude: 1, latitude: 2}) AS p RETURN p.height", + "The point point({x: 1.0, y: 2.0, crs: 'wgs-84'}) has no height"); +} + +TEST_F(PointComponentTest, rejectsAGeographicNameOfACartesianPoint) { + expectError("WITH point({x: 1, y: 2}) AS p RETURN p.latitude", "has no latitude"); + expectError("WITH point({x: 1, y: 2, z: 3}) AS p RETURN p.height", "has no height"); +} + +TEST_F(PointComponentTest, rejectsANameThatIsNoComponent) { + expectError("WITH point({x: 1, y: 2}) AS p RETURN p.w", "'w' is not a component of a point"); + expectError("RETURN point({x: 1, y: 2}).w", "'w' is not a component of a point"); +} + +TEST_F(PointComponentTest, readsTheComponentOfAPointAListHolds) { + expectRows("UNWIND [point({x: 1, y: 2}), point({longitude: 3, latitude: 4})] AS p RETURN p.x, p.crs, p.srid", + {{"1.000000", "cartesian", "7203"}, {"3.000000", "wgs-84", "4326"}}); +} + +TEST_F(PointComponentTest, rejectsANameThatIsNoComponentOfAPointAListHolds) { + expectError("UNWIND [point({x: 1, y: 2})] AS p RETURN p.w", "'w' is not a component of a point"); + expectError("UNWIND [point({x: 1, y: 2})] AS p RETURN p.z", "has no z"); +} + +TEST_F(PointComponentTest, projectsTheComponentsOfAPoint) { + expectRows("WITH point({longitude: 1, latitude: 2}) AS p RETURN p {.longitude, .crs, n: 1}", + {{"{crs: wgs-84, longitude: 1.000000, n: 1}"}}); +} + +TEST_F(PointComponentTest, rejectsEveryPropertyOfAPoint) { + expectError("WITH point({x: 1, y: 2}) AS p RETURN p {.*}", "A map projection reads .* off a node, a relationship or a map"); +} + +TEST_F(PointComponentTest, rejectsAComponentAsTheTargetOfAWrite) { + write("CREATE (n:City {name: 'a', at: point({x: 1, y: 2})})"); + expectError("MATCH (n:City) SET n.at.x = 3", "component"); +} diff --git a/test/query/ir/point/PointImportTest.cpp b/test/query/ir/point/PointImportTest.cpp new file mode 100644 index 0000000000..97d16af870 --- /dev/null +++ b/test/query/ir/point/PointImportTest.cpp @@ -0,0 +1,94 @@ +#include "PointImportTest.h" + +#include + +#include + +#include "QueryInterpreterV3.h" +#include "QueryStatus.h" + +#include "Graph.h" +#include "SimpleGraph.h" +#include "SystemAccessor.h" +#include "SystemManager.h" +#include "versioning/ChangeID.h" +#include "versioning/CommitHash.h" + +#include "TuringTestEnv.h" + +using namespace db; +using namespace turing::test; + +PointImportTest::PointImportTest() { +} + +PointImportTest::~PointImportTest() { +} + +void PointImportTest::initialize() { + _env = TuringTestEnv::create(fs::Path {_outDir} / "turing"); + + SystemAccessor system = _env->getSystemManager().accessUnique(); + Graph* graph = system.createGraph(_sessionGraph); + SimpleGraph::createSimpleGraph(graph); + + _interpreter = std::make_unique(&_env->getSystemManager(), &_env->getMem(), &_env->getCompilerContext()); +} + +void PointImportTest::getDataDir(FileUtils::Path& dataDir) const { + dataDir = FileUtils::Path {_env->getConfig().getDataDir().get()}; +} + +void PointImportTest::writeFixture(std::string_view fileName, std::string_view contents) { + FileUtils::Path dataDir; + getDataDir(dataDir); + + std::ofstream file(dataDir / std::string {fileName}); + ASSERT_TRUE(file.is_open()); + + file << contents; +} + +void PointImportTest::runQuery(std::string_view query, std::string_view graphName, NLOutputSink& sink) { + QueryStatus status; + _interpreter->execute(status, + query, + graphName, + CommitHash::head(), + ChangeID::head(), + &sink); + + ASSERT_TRUE(status.isOk()) << query << ": " << status.getError(); +} + +void PointImportTest::load(std::string_view query) { + StringRowSink sink; + runQuery(query, _sessionGraph, sink); +} + +void PointImportTest::runQueryExpectingError(std::string_view query, std::string_view reason) { + StringRowSink sink; + QueryStatus status; + _interpreter->execute(status, + query, + _sessionGraph, + CommitHash::head(), + ChangeID::head(), + &sink); + + ASSERT_FALSE(status.isOk()) << "accepted: " << query; + + const std::string error = status.getError(); + EXPECT_NE(error.find(reason), std::string::npos) << query << ": " << error; +} + +void PointImportTest::expectSortedRows(std::string_view query, + std::string_view graphName, + const std::vector& expected) { + StringRowSink sink; + runQuery(query, graphName, sink); + + std::vector rows; + sink.sortedRows(rows); + EXPECT_EQ(rows, expected) << query; +} diff --git a/test/query/ir/point/PointImportTest.h b/test/query/ir/point/PointImportTest.h new file mode 100644 index 0000000000..72c0222f6a --- /dev/null +++ b/test/query/ir/point/PointImportTest.h @@ -0,0 +1,48 @@ +#pragma once + +#include +#include +#include +#include + +#include "FileUtils.h" +#include "StringRowSink.h" +#include "TuringTest.h" + +namespace db { +class NLOutputSink; +class QueryInterpreterV3; +} + +namespace turing::test { + +class TuringTestEnv; + +// Drives QueryInterpreterV3 over simpledb, for tests that import a file of points into a +// graph of its own and read that graph back +class PointImportTest : public TuringTest { +public: + PointImportTest(); + ~PointImportTest() override; + + void initialize() override; + +protected: + const std::string _sessionGraph {"simpledb"}; + + // The directory LOAD JSONL and LOAD PARQUET read a relative path from + void getDataDir(FileUtils::Path& dataDir) const; + void writeFixture(std::string_view fileName, std::string_view contents); + void runQuery(std::string_view query, std::string_view graphName, db::NLOutputSink& sink); + void load(std::string_view query); + void runQueryExpectingError(std::string_view query, std::string_view reason); + void expectSortedRows(std::string_view query, + std::string_view graphName, + const std::vector& expected); + +private: + std::unique_ptr _env; + std::unique_ptr _interpreter; +}; + +} diff --git a/test/query/ir/point/PointOrderAndAggregateTest.cpp b/test/query/ir/point/PointOrderAndAggregateTest.cpp new file mode 100644 index 0000000000..7e1ad718f9 --- /dev/null +++ b/test/query/ir/point/PointOrderAndAggregateTest.cpp @@ -0,0 +1,256 @@ +#include + +#include +#include +#include +#include + +#include "QueryInterpreterV3.h" +#include "QueryStatus.h" + +#include "Graph.h" +#include "QueryConfig.h" +#include "SimpleGraph.h" +#include "SystemAccessor.h" +#include "SystemManager.h" +#include "TuringDB.h" +#include "versioning/ChangeID.h" +#include "versioning/CommitHash.h" + +#include "IRTestRows.h" +#include "TuringTest.h" +#include "TuringTestEnv.h" + +using namespace db; +using namespace turing::test; + +class PointOrderAndAggregateTest : public TuringTest { +protected: + void initialize() override { + _env = TuringTestEnv::create(fs::Path {_outDir} / "turing"); + _interpreter = std::make_unique(&_env->getSystemManager(), &_env->getMem(), &_env->getCompilerContext()); + + SystemAccessor system = _env->getSystemManager().accessUnique(); + Graph* graph = system.createGraph(_graphName); + SimpleGraph::createSimpleGraph(graph); + } + + void openChange(ChangeID& changeID) { + SystemAccessor system = _env->getSystemManager().accessUnique(); + const auto res = system.newChange(_graphName); + ASSERT_TRUE(res); + + changeID = res.value()->id(); + } + + void submit(const ChangeID& changeID) { + const QueryState submitState(_graphName, + &_env->getMem(), + &_env->getCompilerContext(), + &_queryConfig, + nullptr, + CommitHash::head(), + changeID); + const QueryStatus status = _env->getDB().query("CHANGE SUBMIT", submitState); + ASSERT_TRUE(status.isOk()) << "CHANGE SUBMIT failed"; + } + + void write(std::string_view query) { + ChangeID changeID; + openChange(changeID); + + RowSink sink; + QueryStatus status; + _interpreter->execute(status, + query, + _graphName, + CommitHash::head(), + changeID, + &sink); + ASSERT_TRUE(status.isOk()) << "query: " << query << "\nerror: " << status.getError(); + + submit(changeID); + } + + void run(std::string_view query, RowSink& sink, QueryStatus& status) { + _interpreter->execute(status, + query, + _graphName, + CommitHash::head(), + ChangeID::head(), + &sink); + } + + void expectRows(std::string_view query, const Rows& expected) { + RowSink sink; + QueryStatus status; + run(query, sink, status); + ASSERT_TRUE(status.isOk()) << "query: " << query << "\nerror: " << status.getError(); + + Rows actual; + sink.sortedRows(actual); + + Rows sortedExpected = expected; + std::sort(sortedExpected.begin(), sortedExpected.end()); + + std::string actualText; + describeRows(actual, actualText); + + EXPECT_EQ(actual, sortedExpected) << "query: " << query << "\ngot:\n" << actualText; + } + + void expectOrderedRows(std::string_view query, const Rows& expected) { + RowSink sink; + QueryStatus status; + run(query, sink, status); + ASSERT_TRUE(status.isOk()) << "query: " << query << "\nerror: " << status.getError(); + + std::string actualText; + describeRows(sink.rows(), actualText); + + EXPECT_EQ(sink.rows(), expected) << "query: " << query << "\ngot:\n" << actualText; + } + + void expectError(std::string_view query, std::string_view expectedError) { + RowSink sink; + QueryStatus status; + run(query, sink, status); + + ASSERT_FALSE(status.isOk()) << "query: " << query << "\nexpected it to fail"; + EXPECT_NE(status.getError().find(expectedError), std::string::npos) + << "query: " << query << "\nerror: " << status.getError(); + } + + void createFourCities() { + write("CREATE (n:City {name: 'Paris', at: point({longitude: 2.3522, latitude: 48.8566})})"); + write("CREATE (n:City {name: 'London', at: point({longitude: -0.1276, latitude: 51.5072})})"); + write("CREATE (n:City {name: 'Grid', at: point({x: 3, y: 4})})"); + write("CREATE (n:City {name: 'Tower', at: point({x: 3, y: 4, z: 5})})"); + } + + const std::string _graphName = "simpledb"; + std::unique_ptr _env; + std::unique_ptr _interpreter; + QueryConfig _queryConfig; +}; + +TEST_F(PointOrderAndAggregateTest, comparesPointsOfOneCRSByCoordinates) { + expectRows("RETURN point({x: 1, y: 9}) < point({x: 2, y: 0}), point({x: 1, y: 1}) < point({x: 1, y: 2})", + {{"true", "true"}}); + expectRows("RETURN point({x: 1, y: 2, z: 3}) > point({x: 1, y: 2, z: 2})", {{"true"}}); + expectRows("RETURN point({x: 1, y: 2}) <= point({x: 1, y: 2}), point({x: 1, y: 2}) >= point({x: 1, y: 2})", + {{"true", "true"}}); + expectRows("RETURN point({x: 1, y: 2}) < point({x: 1, y: 2}), point({x: 1, y: 2}) > point({x: 1, y: 2})", + {{"false", "false"}}); +} + +TEST_F(PointOrderAndAggregateTest, comparesPointsAcrossCRSsBySRID) { + expectRows("RETURN point({x: 9, y: 9, crs: 'wgs-84'}) < point({x: 0, y: 0})", {{"true"}}); + expectRows("RETURN point({x: 9, y: 9}) < point({x: 0, y: 0, z: 0})", {{"true"}}); + expectRows("RETURN point({x: 0, y: 0, z: 0, crs: 'wgs-84-3d'}) > point({x: 9, y: 9, crs: 'wgs-84'})", {{"true"}}); +} + +TEST_F(PointOrderAndAggregateTest, comparesNothingButAPointWithAPoint) { + expectRows("RETURN point({x: 1, y: 2}) < 3, point({x: 1, y: 2}) < 'a'", {{"null", "null"}}); + expectRows("RETURN point({x: 1, y: 2}) < null", {{"null"}}); +} + +TEST_F(PointOrderAndAggregateTest, comparesPointsAListHolds) { + expectRows("UNWIND [point({x: 1, y: 2}), point({x: 3, y: 4}), 5] AS p RETURN p, p < point({x: 2, y: 0})", + {{"5", "null"}, + {"point({x: 1.0, y: 2.0, crs: 'cartesian'})", "true"}, + {"point({x: 3.0, y: 4.0, crs: 'cartesian'})", "false"}}); +} + +TEST_F(PointOrderAndAggregateTest, filtersOnAPointOrder) { + createFourCities(); + expectRows("MATCH (n:City) WHERE n.at < point({x: 0, y: 0}) RETURN n.name", {{"Paris"}, {"London"}}); + expectRows("MATCH (n:City) WHERE n.at >= point({x: 3, y: 4}) RETURN n.name", {{"Grid"}, {"Tower"}}); +} + +TEST_F(PointOrderAndAggregateTest, takesTheLeastAndGreatestPoint) { + createFourCities(); + expectRows("MATCH (n:City) RETURN min(n.at), max(n.at)", + {{"point({x: -0.1276, y: 51.5072, crs: 'wgs-84'})", "point({x: 3.0, y: 4.0, z: 5.0, crs: 'cartesian-3d'})"}}); +} + +TEST_F(PointOrderAndAggregateTest, takesTheExtremesOfEachGroup) { + write("CREATE (n:City {name: 'a', region: 'north', at: point({x: 1, y: 5})})"); + write("CREATE (n:City {name: 'b', region: 'north', at: point({x: 2, y: 0})})"); + write("CREATE (n:City {name: 'c', region: 'south', at: point({x: 7, y: 7})})"); + write("CREATE (n:City {name: 'd', region: 'south'})"); + + expectRows("MATCH (n:City) RETURN n.region, min(n.at), max(n.at)", + {{"north", "point({x: 1.0, y: 5.0, crs: 'cartesian'})", "point({x: 2.0, y: 0.0, crs: 'cartesian'})"}, + {"south", "point({x: 7.0, y: 7.0, crs: 'cartesian'})", "point({x: 7.0, y: 7.0, crs: 'cartesian'})"}}); + expectRows("MATCH (n:City) RETURN n.region, min(DISTINCT n.at)", + {{"north", "point({x: 1.0, y: 5.0, crs: 'cartesian'})"}, {"south", "point({x: 7.0, y: 7.0, crs: 'cartesian'})"}}); +} + +TEST_F(PointOrderAndAggregateTest, takesTheExtremesOfPointsAListHolds) { + expectRows("UNWIND [point({x: 3, y: 0}), point({x: 1, y: 9}), null] AS p RETURN min(p), max(p)", + {{"point({x: 1.0, y: 9.0, crs: 'cartesian'})", "point({x: 3.0, y: 0.0, crs: 'cartesian'})"}}); +} + +TEST_F(PointOrderAndAggregateTest, takesNoExtremeOfNoPoint) { + write("CREATE (n:City {name: 'a', at: point({x: 1, y: 2})})"); + expectRows("MATCH (n:City) WHERE n.name = 'b' RETURN min(n.at), max(n.at)", {{"null", "null"}}); +} + +TEST_F(PointOrderAndAggregateTest, averagesPointsIntoTheirCentroid) { + write("CREATE (n:City {name: 'a', at: point({x: 0, y: 0})})"); + write("CREATE (n:City {name: 'b', at: point({x: 4, y: 0})})"); + write("CREATE (n:City {name: 'c', at: point({x: 2, y: 6})})"); + write("CREATE (n:City {name: 'd'})"); + + expectRows("MATCH (n:City) RETURN avg(n.at)", {{"point({x: 2.0, y: 2.0, crs: 'cartesian'})"}}); +} + +TEST_F(PointOrderAndAggregateTest, averagesPointsOfEveryCRS) { + expectRows("UNWIND [point({x: 0, y: 0, z: 0}), point({x: 2, y: 4, z: 6})] AS p RETURN avg(p)", + {{"point({x: 1.0, y: 2.0, z: 3.0, crs: 'cartesian-3d'})"}}); + expectRows("UNWIND [point({longitude: 2, latitude: 48}), point({longitude: 4, latitude: 50})] AS p RETURN avg(p)", + {{"point({x: 3.0, y: 49.0, crs: 'wgs-84'})"}}); +} + +TEST_F(PointOrderAndAggregateTest, averagesThePointsOfEachGroup) { + write("CREATE (n:City {name: 'a', region: 'north', at: point({x: 0, y: 0})})"); + write("CREATE (n:City {name: 'b', region: 'north', at: point({x: 2, y: 2})})"); + write("CREATE (n:City {name: 'c', region: 'north', at: point({x: 2, y: 2})})"); + write("CREATE (n:City {name: 'd', region: 'south', at: point({x: 5, y: 5})})"); + + expectRows("MATCH (n:City) RETURN n.region, avg(n.at)", + {{"north", "point({x: 1.3333333333333333, y: 1.3333333333333333, crs: 'cartesian'})"}, + {"south", "point({x: 5.0, y: 5.0, crs: 'cartesian'})"}}); + expectRows("MATCH (n:City) RETURN n.region, avg(DISTINCT n.at)", + {{"north", "point({x: 1.0, y: 1.0, crs: 'cartesian'})"}, {"south", "point({x: 5.0, y: 5.0, crs: 'cartesian'})"}}); +} + +TEST_F(PointOrderAndAggregateTest, averagesThePointsOfEachGroupAListHolds) { + expectRows("UNWIND [['a', point({x: 0, y: 0})], ['a', point({x: 4, y: 2})], ['b', point({x: 1, y: 1})]] AS row " + "RETURN row[0], avg(row[1])", + {{"a", "point({x: 2.0, y: 1.0, crs: 'cartesian'})"}, {"b", "point({x: 1.0, y: 1.0, crs: 'cartesian'})"}}); +} + +TEST_F(PointOrderAndAggregateTest, averagesNoPointToNull) { + write("CREATE (n:City {name: 'a', at: point({x: 1, y: 2})})"); + expectRows("MATCH (n:City) WHERE n.name = 'b' RETURN avg(n.at)", {{"null"}}); +} + +TEST_F(PointOrderAndAggregateTest, rejectsAnAverageAcrossCRSs) { + createFourCities(); + expectError("MATCH (n:City) RETURN avg(n.at)", "avg() reads points of one CRS, not wgs-84 and cartesian"); + expectError("UNWIND [point({x: 1, y: 2}), point({x: 1, y: 2, z: 3})] AS p RETURN avg(p)", + "avg() reads points of one CRS, not cartesian and cartesian-3d"); +} + +TEST_F(PointOrderAndAggregateTest, rejectsAnAverageOfPointsAndNumbers) { + expectError("UNWIND [point({x: 1, y: 2}), 3] AS p RETURN avg(p)", "sum() and avg() cannot add 'Integer' to a sum of 'Point'"); + expectError("UNWIND [3, point({x: 1, y: 2})] AS p RETURN avg(p)", "sum() and avg() cannot add 'Point' to a sum of 'Double'"); +} + +TEST_F(PointOrderAndAggregateTest, rejectsASumOfPoints) { + expectError("RETURN sum(point({x: 1, y: 2}))", "Invalid arguments for function 'sum'"); + expectError("UNWIND [point({x: 1, y: 2})] AS p RETURN sum(p)", "Invalid arguments for function 'sum'"); + expectError("UNWIND [point({x: 1, y: 2}), 1] AS p RETURN sum(p)", "sum() and avg() read numbers or durations, not 'Point'"); +} diff --git a/test/query/ir/point/PointPropertyTest.cpp b/test/query/ir/point/PointPropertyTest.cpp new file mode 100644 index 0000000000..30b56ab963 --- /dev/null +++ b/test/query/ir/point/PointPropertyTest.cpp @@ -0,0 +1,347 @@ +#include + +#include +#include +#include +#include + +#include "QueryInterpreterV3.h" +#include "QueryStatus.h" + +#include "Graph.h" +#include "QueryConfig.h" +#include "SimpleGraph.h" +#include "SystemAccessor.h" +#include "SystemManager.h" +#include "TuringDB.h" +#include "versioning/ChangeID.h" +#include "versioning/CommitHash.h" + +#include "IRTestRows.h" +#include "TuringTest.h" +#include "TuringTestEnv.h" + +using namespace db; +using namespace turing::test; + +class PointPropertyTest : public TuringTest { +protected: + void initialize() override { + _env = TuringTestEnv::create(fs::Path {_outDir} / "turing"); + _interpreter = std::make_unique(&_env->getSystemManager(), &_env->getMem(), &_env->getCompilerContext()); + + SystemAccessor system = _env->getSystemManager().accessUnique(); + Graph* graph = system.createGraph(_graphName); + SimpleGraph::createSimpleGraph(graph); + } + + void openChange(ChangeID& changeID) { + SystemAccessor system = _env->getSystemManager().accessUnique(); + const auto res = system.newChange(_graphName); + ASSERT_TRUE(res); + + changeID = res.value()->id(); + } + + void submit(const ChangeID& changeID) { + const QueryState submitState(_graphName, + &_env->getMem(), + &_env->getCompilerContext(), + &_queryConfig, + nullptr, + CommitHash::head(), + changeID); + const QueryStatus status = _env->getDB().query("CHANGE SUBMIT", submitState); + ASSERT_TRUE(status.isOk()) << "CHANGE SUBMIT failed"; + } + + void write(std::string_view query) { + ChangeID changeID; + openChange(changeID); + + RowSink sink; + QueryStatus status; + _interpreter->execute(status, + query, + _graphName, + CommitHash::head(), + changeID, + &sink); + ASSERT_TRUE(status.isOk()) << "query: " << query << "\nerror: " << status.getError(); + + submit(changeID); + } + + void run(std::string_view query, RowSink& sink, QueryStatus& status) { + _interpreter->execute(status, + query, + _graphName, + CommitHash::head(), + ChangeID::head(), + &sink); + } + + void expectRows(std::string_view query, const Rows& expected) { + RowSink sink; + QueryStatus status; + run(query, sink, status); + ASSERT_TRUE(status.isOk()) << "query: " << query << "\nerror: " << status.getError(); + + Rows actual; + sink.sortedRows(actual); + + Rows sortedExpected = expected; + std::sort(sortedExpected.begin(), sortedExpected.end()); + + std::string actualText; + describeRows(actual, actualText); + + EXPECT_EQ(actual, sortedExpected) << "query: " << query << "\ngot:\n" << actualText; + } + + void expectOrderedRows(std::string_view query, const Rows& expected) { + RowSink sink; + QueryStatus status; + run(query, sink, status); + ASSERT_TRUE(status.isOk()) << "query: " << query << "\nerror: " << status.getError(); + + std::string actualText; + describeRows(sink.rows(), actualText); + + EXPECT_EQ(sink.rows(), expected) << "query: " << query << "\ngot:\n" << actualText; + } + + void expectError(std::string_view query, std::string_view expectedError) { + RowSink sink; + QueryStatus status; + run(query, sink, status); + + ASSERT_FALSE(status.isOk()) << "query: " << query << "\nexpected it to fail"; + EXPECT_NE(status.getError().find(expectedError), std::string::npos) + << "query: " << query << "\nerror: " << status.getError(); + } + + void createFourCities() { + write("CREATE (n:City {name: 'Paris', at: point({longitude: 2.3522, latitude: 48.8566})})"); + write("CREATE (n:City {name: 'London', at: point({longitude: -0.1276, latitude: 51.5072})})"); + write("CREATE (n:City {name: 'Grid', at: point({x: 3, y: 4})})"); + write("CREATE (n:City {name: 'Tower', at: point({x: 3, y: 4, z: 5})})"); + } + + const std::string _graphName = "simpledb"; + std::unique_ptr _env; + std::unique_ptr _interpreter; + QueryConfig _queryConfig; +}; + +TEST_F(PointPropertyTest, returnsACartesianPoint) { + expectRows("RETURN point({x: 1, y: 2})", {{"point({x: 1.0, y: 2.0, crs: 'cartesian'})"}}); + expectRows("RETURN point({x: 1.5, y: -2, z: 3})", {{"point({x: 1.5, y: -2.0, z: 3.0, crs: 'cartesian-3d'})"}}); +} + +TEST_F(PointPropertyTest, returnsAGeographicPoint) { + expectRows("RETURN point({longitude: 2.3522, latitude: 48.8566})", + {{"point({x: 2.3522, y: 48.8566, crs: 'wgs-84'})"}}); + expectRows("RETURN point({longitude: 2.3522, latitude: 48.8566, height: 35})", + {{"point({x: 2.3522, y: 48.8566, z: 35.0, crs: 'wgs-84-3d'})"}}); + expectRows("RETURN point({longitude: 2.3522, latitude: 48.8566, z: 35})", + {{"point({x: 2.3522, y: 48.8566, z: 35.0, crs: 'wgs-84-3d'})"}}); +} + +TEST_F(PointPropertyTest, readsTheCRSANameOrAnSRIDGives) { + expectRows("RETURN point({x: 2.3522, y: 48.8566, crs: 'wgs-84'})", + {{"point({x: 2.3522, y: 48.8566, crs: 'wgs-84'})"}}); + expectRows("RETURN point({x: 1, y: 2, crs: 'WGS-84'})", {{"point({x: 1.0, y: 2.0, crs: 'wgs-84'})"}}); + expectRows("RETURN point({x: 1, y: 2, srid: 4326})", {{"point({x: 1.0, y: 2.0, crs: 'wgs-84'})"}}); + expectRows("RETURN point({x: 1, y: 2, z: 3, srid: 9157})", + {{"point({x: 1.0, y: 2.0, z: 3.0, crs: 'cartesian-3d'})"}}); + expectRows("RETURN point({longitude: 1, latitude: 2, height: 3, srid: 4979})", + {{"point({x: 1.0, y: 2.0, z: 3.0, crs: 'wgs-84-3d'})"}}); +} + +TEST_F(PointPropertyTest, readsItsKeysWhateverTheirCase) { + expectRows("RETURN point({X: 1, Y: 2})", {{"point({x: 1.0, y: 2.0, crs: 'cartesian'})"}}); + expectRows("RETURN point({Longitude: 1, LATITUDE: 2})", {{"point({x: 1.0, y: 2.0, crs: 'wgs-84'})"}}); +} + +TEST_F(PointPropertyTest, ignoresTheKeysItDoesNotKnow) { + expectRows("RETURN point({x: 1, y: 2, name: 'a'})", {{"point({x: 1.0, y: 2.0, crs: 'cartesian'})"}}); +} + +TEST_F(PointPropertyTest, wrapsALongitudeIntoTheRange) { + expectRows("RETURN point({longitude: 190, latitude: 0})", {{"point({x: -170.0, y: 0.0, crs: 'wgs-84'})"}}); + expectRows("RETURN point({longitude: -540, latitude: 0})", {{"point({x: -180.0, y: 0.0, crs: 'wgs-84'})"}}); + expectRows("RETURN point({longitude: 360, latitude: 0})", {{"point({x: 0.0, y: 0.0, crs: 'wgs-84'})"}}); +} + +TEST_F(PointPropertyTest, returnsNullForANullMapOrANullValue) { + expectRows("RETURN point(null)", {{"null"}}); + expectRows("RETURN point({x: null, y: 2})", {{"null"}}); + expectRows("RETURN point({x: 1, y: 2, crs: null})", {{"null"}}); +} + +TEST_F(PointPropertyTest, rejectsALatitudePastAPole) { + expectError("RETURN point({longitude: 0, latitude: 90.5})", "The latitude of a wgs-84 point lies in [-90, 90], not 90.5"); + expectError("RETURN point({x: 0, y: -91, crs: 'wgs-84'})", "lies in [-90, 90], not -91"); +} + +TEST_F(PointPropertyTest, rejectsANonFiniteCoordinate) { + expectError("RETURN point({x: 1e308 * 10, y: 0})", "A point cannot have a non-finite coordinate"); +} + +TEST_F(PointPropertyTest, rejectsAMapNamingBothACRSAndAnSRID) { + expectError("RETURN point({x: 1, y: 2, crs: 'cartesian', srid: 7203})", "point() takes a 'crs' or an 'srid', not both"); +} + +TEST_F(PointPropertyTest, rejectsAnUnknownCRS) { + expectError("RETURN point({x: 1, y: 2, crs: 'mercator'})", "Unknown coordinate reference system: 'mercator'"); + expectError("RETURN point({x: 1, y: 2, srid: 3857})", "Unknown coordinate reference system SRID: 3857"); +} + +TEST_F(PointPropertyTest, rejectsAGeographicPointInACartesianCRS) { + expectError("RETURN point({longitude: 1, latitude: 2, crs: 'cartesian'})", + "A point given by longitude and latitude takes a geographic CRS, not 'cartesian'"); +} + +TEST_F(PointPropertyTest, rejectsCoordinatesTheCRSDoesNotHave) { + expectError("RETURN point({x: 1, y: 2, crs: 'cartesian-3d'})", "A cartesian-3d point has 3 coordinates, not 2"); + expectError("RETURN point({x: 1, y: 2, z: 3, crs: 'wgs-84'})", "A wgs-84 point has 2 coordinates, not 3"); +} + +TEST_F(PointPropertyTest, rejectsAMapMissingACoordinate) { + expectError("RETURN point({x: 1})", "point() needs 'x' and 'y', or 'longitude' and 'latitude'"); + expectError("RETURN point({latitude: 1, crs: 'wgs-84'})", "A wgs-84 point needs 'longitude' and 'latitude'"); +} + +TEST_F(PointPropertyTest, rejectsACoordinateThatIsNoNumber) { + expectError("RETURN point({x: '1', y: 2})", "point() reads a number for 'x', not 'String'"); +} + +TEST_F(PointPropertyTest, rejectsAKeyGivenTwice) { + expectError("RETURN point({x: 1, X: 2, y: 3})", "point() reads the key 'x' twice"); +} + +TEST_F(PointPropertyTest, buildsAPointFromAMapItReadsPerRow) { + expectRows("UNWIND [{x: 1, y: 2}, {longitude: 3, latitude: 4}, null] AS m RETURN point(m)", + {{"point({x: 1.0, y: 2.0, crs: 'cartesian'})"}, + {"point({x: 3.0, y: 4.0, crs: 'wgs-84'})"}, + {"null"}}); +} + +TEST_F(PointPropertyTest, rejectsAValueThatIsNoMap) { + expectError("UNWIND [1] AS m RETURN point(m)", "point"); +} + +TEST_F(PointPropertyTest, writesAPointAsText) { + expectRows("RETURN toString(point({x: 1, y: 2}))", {{"point({x: 1.0, y: 2.0, crs: 'cartesian'})"}}); + expectRows("UNWIND [point({x: 1, y: 2})] AS p RETURN toString(p)", {{"point({x: 1.0, y: 2.0, crs: 'cartesian'})"}}); +} + +TEST_F(PointPropertyTest, storesAndReadsBackAPoint) { + write("CREATE (n:City {name: 'Paris', at: point({longitude: 2.3522, latitude: 48.8566})})"); + expectRows("MATCH (n:City) RETURN n.at", {{"point({x: 2.3522, y: 48.8566, crs: 'wgs-84'})"}}); +} + +TEST_F(PointPropertyTest, setsThePointOfAnExistingNode) { + write("MATCH (n:Person {name: 'Remy'}) SET n.home = point({x: 1, y: 2, z: 3})"); + expectRows("MATCH (n:Person {name: 'Remy'}) RETURN n.home", {{"point({x: 1.0, y: 2.0, z: 3.0, crs: 'cartesian-3d'})"}}); +} + +TEST_F(PointPropertyTest, storesAPointOnAnEdge) { + write("MATCH (a:Person {name: 'Remy'}), (b:Person {name: 'Adam'}) " + "CREATE (a)-[:MET {at: point({x: 1, y: 2})}]->(b)"); + + expectRows("MATCH (:Person)-[e:MET]->(:Person) RETURN e.at", {{"point({x: 1.0, y: 2.0, crs: 'cartesian'})"}}); +} + +TEST_F(PointPropertyTest, readsNullFromANodeCarryingNoPoint) { + write("CREATE (a:City {name: 'a', at: point({x: 1, y: 2})})"); + write("CREATE (b:City {name: 'b'})"); + + expectRows("MATCH (n:City) RETURN n.name, n.at", {{"a", "point({x: 1.0, y: 2.0, crs: 'cartesian'})"}, {"b", "null"}}); +} + +TEST_F(PointPropertyTest, holdsPointsOfDifferentCRSsInOneProperty) { + createFourCities(); + expectRows("MATCH (n:City) RETURN n.name, n.at", + {{"Paris", "point({x: 2.3522, y: 48.8566, crs: 'wgs-84'})"}, + {"London", "point({x: -0.1276, y: 51.5072, crs: 'wgs-84'})"}, + {"Grid", "point({x: 3.0, y: 4.0, crs: 'cartesian'})"}, + {"Tower", "point({x: 3.0, y: 4.0, z: 5.0, crs: 'cartesian-3d'})"}}); +} + +TEST_F(PointPropertyTest, filtersOnEquality) { + createFourCities(); + expectRows("MATCH (n:City) WHERE n.at = point({x: 3, y: 4}) RETURN n.name", {{"Grid"}}); + expectRows("MATCH (n:City) WHERE n.at <> point({x: 3, y: 4}) RETURN n.name", {{"Paris"}, {"London"}, {"Tower"}}); +} + +TEST_F(PointPropertyTest, tellsPointsOfDifferentCRSsApart) { + expectRows("RETURN point({x: 1, y: 2}) = point({x: 1, y: 2, crs: 'wgs-84'})", {{"false"}}); + expectRows("RETURN point({x: 1, y: 2}) = point({x: 1, y: 2, z: 0})", {{"false"}}); + expectRows("RETURN point({x: 1, y: 2}) = point({x: 1.0, y: 2.0})", {{"true"}}); +} + +TEST_F(PointPropertyTest, equatesAZeroToANegativeZero) { + expectRows("RETURN point({x: -0.0, y: 0}) = point({x: 0.0, y: 0})", {{"true"}}); + expectRows("UNWIND [point({x: -0.0, y: 0}), point({x: 0.0, y: 0})] AS p RETURN DISTINCT p", + {{"point({x: 0.0, y: 0.0, crs: 'cartesian'})"}}); +} + +TEST_F(PointPropertyTest, ordersAPointBeforeAnother) { + expectRows("RETURN point({x: 1, y: 2}) < point({x: 3, y: 4})", {{"true"}}); +} + +TEST_F(PointPropertyTest, ordersBySRIDThenByCoordinates) { + createFourCities(); + write("CREATE (n:City {name: 'Origin', at: point({x: 0, y: 9})})"); + + expectOrderedRows("MATCH (n:City) RETURN n.name ORDER BY n.at", + {{"London"}, {"Paris"}, {"Origin"}, {"Grid"}, {"Tower"}}); + expectOrderedRows("MATCH (n:City) RETURN n.name ORDER BY n.at DESC", + {{"Tower"}, {"Grid"}, {"Origin"}, {"Paris"}, {"London"}}); +} + +TEST_F(PointPropertyTest, sortsPointsAfterPathsAndBeforeDateTimes) { + expectOrderedRows("UNWIND [datetime('2026-01-01'), point({x: 1, y: 2}), [1]] AS v RETURN v ORDER BY v", + {{"[1]"}, {"point({x: 1.0, y: 2.0, crs: 'cartesian'})"}, {"2026-01-01T00:00:00Z"}}); +} + +TEST_F(PointPropertyTest, countsAndCollectsPoints) { + write("CREATE (a:City {name: 'a', at: point({x: 1, y: 2})})"); + write("CREATE (b:City {name: 'b', at: point({x: 1, y: 2})})"); + + expectRows("MATCH (n:City) RETURN count(n.at), count(DISTINCT n.at)", {{"2", "1"}}); + expectRows("MATCH (n:City) RETURN collect(n.at)", + {{"[point({x: 1.0, y: 2.0, crs: 'cartesian'}), point({x: 1.0, y: 2.0, crs: 'cartesian'})]"}}); +} + +TEST_F(PointPropertyTest, groupsByPoint) { + write("CREATE (a:City {name: 'a', at: point({x: 1, y: 2})})"); + write("CREATE (b:City {name: 'b', at: point({x: 1, y: 2})})"); + write("CREATE (c:City {name: 'c', at: point({x: 1, y: 2, crs: 'wgs-84'})})"); + + expectRows("MATCH (n:City) RETURN n.at, count(*)", + {{"point({x: 1.0, y: 2.0, crs: 'cartesian'})", "2"}, {"point({x: 1.0, y: 2.0, crs: 'wgs-84'})", "1"}}); +} + +TEST_F(PointPropertyTest, storesPointsInAList) { + write("CREATE (n:City {name: 'a', route: [point({x: 1, y: 2}), point({longitude: 3, latitude: 4})]})"); + expectRows("MATCH (n:City) RETURN n.route", + {{"[point({x: 1.0, y: 2.0, crs: 'cartesian'}), point({x: 3.0, y: 4.0, crs: 'wgs-84'})]"}}); +} + +TEST_F(PointPropertyTest, storesAPointInAMap) { + write("CREATE (n:City {name: 'a', attrs: {at: point({x: 1, y: 2}), n: 1}})"); + expectRows("MATCH (n:City) RETURN n.attrs", {{"{at: point({x: 1.0, y: 2.0, crs: 'cartesian'}), n: 1}"}}); +} + +TEST_F(PointPropertyTest, findsAPointInAList) { + expectRows("RETURN point({x: 1, y: 2}) IN [point({x: 3, y: 4}), point({x: 1, y: 2})]", {{"true"}}); + expectRows("RETURN point({x: 1, y: 2}) IN [point({x: 1, y: 2, crs: 'wgs-84'})]", {{"false"}}); +} + +TEST_F(PointPropertyTest, unwindsAListOfPoints) { + expectRows("UNWIND [point({x: 3, y: 4}), point({x: 1, y: 2})] AS p RETURN p", + {{"point({x: 1.0, y: 2.0, crs: 'cartesian'})"}, {"point({x: 3.0, y: 4.0, crs: 'cartesian'})"}}); +} diff --git a/test/query/ir/point/PointSpatialFunctionsTest.cpp b/test/query/ir/point/PointSpatialFunctionsTest.cpp new file mode 100644 index 0000000000..6bfdd0543a --- /dev/null +++ b/test/query/ir/point/PointSpatialFunctionsTest.cpp @@ -0,0 +1,264 @@ +#include + +#include +#include +#include +#include + +#include "QueryInterpreterV3.h" +#include "QueryStatus.h" + +#include "Graph.h" +#include "QueryConfig.h" +#include "SimpleGraph.h" +#include "SystemAccessor.h" +#include "SystemManager.h" +#include "TuringDB.h" +#include "versioning/ChangeID.h" +#include "versioning/CommitHash.h" + +#include "IRTestRows.h" +#include "TuringTest.h" +#include "TuringTestEnv.h" + +using namespace db; +using namespace turing::test; + +class PointSpatialFunctionsTest : public TuringTest { +protected: + void initialize() override { + _env = TuringTestEnv::create(fs::Path {_outDir} / "turing"); + _interpreter = std::make_unique(&_env->getSystemManager(), &_env->getMem(), &_env->getCompilerContext()); + + SystemAccessor system = _env->getSystemManager().accessUnique(); + Graph* graph = system.createGraph(_graphName); + SimpleGraph::createSimpleGraph(graph); + } + + void openChange(ChangeID& changeID) { + SystemAccessor system = _env->getSystemManager().accessUnique(); + const auto res = system.newChange(_graphName); + ASSERT_TRUE(res); + + changeID = res.value()->id(); + } + + void submit(const ChangeID& changeID) { + const QueryState submitState(_graphName, + &_env->getMem(), + &_env->getCompilerContext(), + &_queryConfig, + nullptr, + CommitHash::head(), + changeID); + const QueryStatus status = _env->getDB().query("CHANGE SUBMIT", submitState); + ASSERT_TRUE(status.isOk()) << "CHANGE SUBMIT failed"; + } + + void write(std::string_view query) { + ChangeID changeID; + openChange(changeID); + + RowSink sink; + QueryStatus status; + _interpreter->execute(status, + query, + _graphName, + CommitHash::head(), + changeID, + &sink); + ASSERT_TRUE(status.isOk()) << "query: " << query << "\nerror: " << status.getError(); + + submit(changeID); + } + + void run(std::string_view query, RowSink& sink, QueryStatus& status) { + _interpreter->execute(status, + query, + _graphName, + CommitHash::head(), + ChangeID::head(), + &sink); + } + + void expectRows(std::string_view query, const Rows& expected) { + RowSink sink; + QueryStatus status; + run(query, sink, status); + ASSERT_TRUE(status.isOk()) << "query: " << query << "\nerror: " << status.getError(); + + Rows actual; + sink.sortedRows(actual); + + Rows sortedExpected = expected; + std::sort(sortedExpected.begin(), sortedExpected.end()); + + std::string actualText; + describeRows(actual, actualText); + + EXPECT_EQ(actual, sortedExpected) << "query: " << query << "\ngot:\n" << actualText; + } + + void expectOrderedRows(std::string_view query, const Rows& expected) { + RowSink sink; + QueryStatus status; + run(query, sink, status); + ASSERT_TRUE(status.isOk()) << "query: " << query << "\nerror: " << status.getError(); + + std::string actualText; + describeRows(sink.rows(), actualText); + + EXPECT_EQ(sink.rows(), expected) << "query: " << query << "\ngot:\n" << actualText; + } + + void expectError(std::string_view query, std::string_view expectedError) { + RowSink sink; + QueryStatus status; + run(query, sink, status); + + ASSERT_FALSE(status.isOk()) << "query: " << query << "\nexpected it to fail"; + EXPECT_NE(status.getError().find(expectedError), std::string::npos) + << "query: " << query << "\nerror: " << status.getError(); + } + + void createFourCities() { + write("CREATE (n:City {name: 'Paris', at: point({longitude: 2.3522, latitude: 48.8566})})"); + write("CREATE (n:City {name: 'London', at: point({longitude: -0.1276, latitude: 51.5072})})"); + write("CREATE (n:City {name: 'Grid', at: point({x: 3, y: 4})})"); + write("CREATE (n:City {name: 'Tower', at: point({x: 3, y: 4, z: 5})})"); + } + + const std::string _graphName = "simpledb"; + std::unique_ptr _env; + std::unique_ptr _interpreter; + QueryConfig _queryConfig; +}; + +TEST_F(PointSpatialFunctionsTest, measuresACartesianDistance) { + expectRows("RETURN point.distance(point({x: 1, y: 2}), point({x: 4, y: 6}))", {{"5.000000"}}); + expectRows("RETURN point.distance(point({x: 1, y: 2, z: 3}), point({x: 4, y: 6, z: 15}))", {{"13.000000"}}); +} + +TEST_F(PointSpatialFunctionsTest, measuresAGreatCircleDistanceInMetres) { + expectRows("RETURN point.distance(point({longitude: 2.3522, latitude: 48.8566}), " + "point({longitude: -0.1276, latitude: 51.5072}))", + {{"343530.339999"}}); + expectRows("RETURN point.distance(point({longitude: 0, latitude: 0}), point({longitude: 1, latitude: 0}))", + {{"111195.080234"}}); +} + +TEST_F(PointSpatialFunctionsTest, measuresAcrossTheAntimeridian) { + expectRows("RETURN point.distance(point({longitude: 179.5, latitude: 0}), point({longitude: -179.5, latitude: 0}))", + {{"111195.080234"}}); +} + +TEST_F(PointSpatialFunctionsTest, measuresBetweenTwoHeights) { + expectRows("RETURN point.distance(point({longitude: 2.3522, latitude: 48.8566, height: 35}), " + "point({longitude: -0.1276, latitude: 51.5072, height: 100}))", + {{"343533.985807"}}); +} + +TEST_F(PointSpatialFunctionsTest, measuresNothingAcrossCRSs) { + expectRows("RETURN point.distance(point({x: 1, y: 2}), point({x: 1, y: 2, crs: 'wgs-84'}))", {{"null"}}); + expectRows("RETURN point.distance(point({x: 1, y: 2}), point({x: 1, y: 2, z: 0}))", {{"null"}}); +} + +TEST_F(PointSpatialFunctionsTest, measuresNothingFromANull) { + expectRows("RETURN point.distance(null, point({x: 1, y: 2}))", {{"null"}}); + expectRows("RETURN point.distance(point({x: 1, y: 2}), null)", {{"null"}}); +} + +TEST_F(PointSpatialFunctionsTest, filtersOnADistance) { + write("CREATE (n:City {name: 'Paris', at: point({longitude: 2.3522, latitude: 48.8566})})"); + write("CREATE (n:City {name: 'London', at: point({longitude: -0.1276, latitude: 51.5072})})"); + write("CREATE (n:City {name: 'Versailles', at: point({longitude: 2.1301, latitude: 48.8049})})"); + + expectRows("MATCH (a:City {name: 'Paris'}), (b:City) " + "WHERE point.distance(a.at, b.at) < 50000 " + "RETURN b.name", + {{"Paris"}, {"Versailles"}}); +} + +TEST_F(PointSpatialFunctionsTest, measuresBetweenPointsAListHolds) { + expectRows("UNWIND [[point({x: 0, y: 0}), point({x: 3, y: 4})], [point({x: 0, y: 0}), point({longitude: 0, latitude: 0})]] AS pair " + "RETURN point.distance(pair[0], pair[1])", + {{"5.000000"}, {"null"}}); +} + +TEST_F(PointSpatialFunctionsTest, rejectsADistanceToAValueThatIsNoPoint) { + expectError("RETURN point.distance(point({x: 1, y: 2}), 1)", "point.distance"); + expectError("UNWIND [1] AS v RETURN point.distance(point({x: 1, y: 2}), v)", "point.distance"); +} + +TEST_F(PointSpatialFunctionsTest, findsACartesianPointInABox) { + expectRows("RETURN point.withinBBox(point({x: 1, y: 2}), point({x: 0, y: 0}), point({x: 2, y: 2}))", {{"true"}}); + expectRows("RETURN point.withinBBox(point({x: 3, y: 2}), point({x: 0, y: 0}), point({x: 2, y: 2}))", {{"false"}}); + expectRows("RETURN point.withinBBox(point({x: 2, y: 0}), point({x: 0, y: 0}), point({x: 2, y: 2}))", {{"true"}}); + expectRows("RETURN point.withinBBox(point({x: 1, y: 1, z: 5}), point({x: 0, y: 0, z: 0}), point({x: 2, y: 2, z: 2}))", + {{"false"}}); +} + +TEST_F(PointSpatialFunctionsTest, findsNoPointInABoxWhoseCornersAreSwapped) { + expectRows("RETURN point.withinBBox(point({x: 1, y: 1}), point({x: 2, y: 2}), point({x: 0, y: 0}))", {{"false"}}); + expectRows("RETURN point.withinBBox(point({longitude: 0, latitude: 1}), " + "point({longitude: -1, latitude: 2}), point({longitude: 1, latitude: 0}))", + {{"false"}}); +} + +TEST_F(PointSpatialFunctionsTest, findsAGeographicPointInABoxAcrossTheAntimeridian) { + expectRows("RETURN point.withinBBox(point({longitude: 179.5, latitude: 0}), " + "point({longitude: 179, latitude: -1}), point({longitude: -179, latitude: 1}))", + {{"true"}}); + expectRows("RETURN point.withinBBox(point({longitude: -179.5, latitude: 0}), " + "point({longitude: 179, latitude: -1}), point({longitude: -179, latitude: 1}))", + {{"true"}}); + expectRows("RETURN point.withinBBox(point({longitude: 0, latitude: 0}), " + "point({longitude: 179, latitude: -1}), point({longitude: -179, latitude: 1}))", + {{"false"}}); +} + +TEST_F(PointSpatialFunctionsTest, findsNothingAcrossCRSsOrFromANull) { + expectRows("RETURN point.withinBBox(point({x: 1, y: 1}), point({x: 0, y: 0, crs: 'wgs-84'}), point({x: 2, y: 2}))", + {{"null"}}); + expectRows("RETURN point.withinBBox(null, point({x: 0, y: 0}), point({x: 2, y: 2}))", {{"null"}}); + expectRows("RETURN point.withinBBox(point({x: 1, y: 1}), null, point({x: 2, y: 2}))", {{"null"}}); +} + +TEST_F(PointSpatialFunctionsTest, filtersOnABox) { + write("CREATE (n:City {name: 'Paris', at: point({longitude: 2.3522, latitude: 48.8566})})"); + write("CREATE (n:City {name: 'London', at: point({longitude: -0.1276, latitude: 51.5072})})"); + + expectRows("MATCH (n:City) " + "WHERE point.withinBBox(n.at, point({longitude: 0, latitude: 45}), point({longitude: 5, latitude: 50})) " + "RETURN n.name", + {{"Paris"}}); +} + +TEST_F(PointSpatialFunctionsTest, ordersByDistance) { + write("CREATE (n:City {name: 'Paris', at: point({longitude: 2.3522, latitude: 48.8566})})"); + write("CREATE (n:City {name: 'London', at: point({longitude: -0.1276, latitude: 51.5072})})"); + write("CREATE (n:City {name: 'Versailles', at: point({longitude: 2.1301, latitude: 48.8049})})"); + + expectOrderedRows("MATCH (n:City) RETURN n.name " + "ORDER BY point.distance(n.at, point({longitude: 2.3522, latitude: 48.8566}))", + {{"Paris"}, {"Versailles"}, {"London"}}); +} + +TEST_F(PointSpatialFunctionsTest, measuresNothingFromAMissingProperty) { + write("CREATE (n:City {name: 'Paris', at: point({longitude: 2.3522, latitude: 48.8566})})"); + write("CREATE (n:City {name: 'Nowhere'})"); + + expectRows("MATCH (n:City) RETURN n.name, point.distance(n.at, point({longitude: 2.3522, latitude: 48.8566}))", + {{"Paris", "0.000000"}, {"Nowhere", "null"}}); +} + +TEST_F(PointSpatialFunctionsTest, answersTheTicketQuery) { + write("CREATE (:City {name: 'London', location: point({latitude: 51.5072, longitude: -0.1276})})"); + write("CREATE (:City {name: 'Paris', location: point({latitude: 48.8566, longitude: 2.3522})})"); + write("CREATE (:City {name: 'Croydon', location: point({latitude: 51.3762, longitude: -0.0982})})"); + + expectRows("MATCH (a:City {name: 'London'}), (b:City) " + "WHERE point.distance(a.location, b.location) < 50000 " + "RETURN b.name, point.distance(a.location, b.location) AS metres", + {{"London", "0.000000"}, {"Croydon", "14708.388652"}}); +} diff --git a/test/query/ir/point/PointValueFlowTest.cpp b/test/query/ir/point/PointValueFlowTest.cpp new file mode 100644 index 0000000000..dc4dc0c3e5 --- /dev/null +++ b/test/query/ir/point/PointValueFlowTest.cpp @@ -0,0 +1,238 @@ +#include + +#include +#include +#include +#include + +#include "QueryInterpreterV3.h" +#include "QueryStatus.h" + +#include "Graph.h" +#include "QueryConfig.h" +#include "SimpleGraph.h" +#include "SystemAccessor.h" +#include "SystemManager.h" +#include "TuringDB.h" +#include "versioning/ChangeID.h" +#include "versioning/CommitHash.h" + +#include "IRTestRows.h" +#include "TuringTest.h" +#include "TuringTestEnv.h" + +using namespace db; +using namespace turing::test; + +class PointValueFlowTest : public TuringTest { +protected: + void initialize() override { + _env = TuringTestEnv::create(fs::Path {_outDir} / "turing"); + _interpreter = std::make_unique(&_env->getSystemManager(), &_env->getMem(), &_env->getCompilerContext()); + + SystemAccessor system = _env->getSystemManager().accessUnique(); + Graph* graph = system.createGraph(_graphName); + SimpleGraph::createSimpleGraph(graph); + } + + void openChange(ChangeID& changeID) { + SystemAccessor system = _env->getSystemManager().accessUnique(); + const auto res = system.newChange(_graphName); + ASSERT_TRUE(res); + + changeID = res.value()->id(); + } + + void submit(const ChangeID& changeID) { + const QueryState submitState(_graphName, + &_env->getMem(), + &_env->getCompilerContext(), + &_queryConfig, + nullptr, + CommitHash::head(), + changeID); + const QueryStatus status = _env->getDB().query("CHANGE SUBMIT", submitState); + ASSERT_TRUE(status.isOk()) << "CHANGE SUBMIT failed"; + } + + void write(std::string_view query) { + ChangeID changeID; + openChange(changeID); + + RowSink sink; + QueryStatus status; + _interpreter->execute(status, + query, + _graphName, + CommitHash::head(), + changeID, + &sink); + ASSERT_TRUE(status.isOk()) << "query: " << query << "\nerror: " << status.getError(); + + submit(changeID); + } + + void run(std::string_view query, RowSink& sink, QueryStatus& status) { + _interpreter->execute(status, + query, + _graphName, + CommitHash::head(), + ChangeID::head(), + &sink); + } + + void expectRows(std::string_view query, const Rows& expected) { + RowSink sink; + QueryStatus status; + run(query, sink, status); + ASSERT_TRUE(status.isOk()) << "query: " << query << "\nerror: " << status.getError(); + + Rows actual; + sink.sortedRows(actual); + + Rows sortedExpected = expected; + std::sort(sortedExpected.begin(), sortedExpected.end()); + + std::string actualText; + describeRows(actual, actualText); + + EXPECT_EQ(actual, sortedExpected) << "query: " << query << "\ngot:\n" << actualText; + } + + void expectOrderedRows(std::string_view query, const Rows& expected) { + RowSink sink; + QueryStatus status; + run(query, sink, status); + ASSERT_TRUE(status.isOk()) << "query: " << query << "\nerror: " << status.getError(); + + std::string actualText; + describeRows(sink.rows(), actualText); + + EXPECT_EQ(sink.rows(), expected) << "query: " << query << "\ngot:\n" << actualText; + } + + void expectError(std::string_view query, std::string_view expectedError) { + RowSink sink; + QueryStatus status; + run(query, sink, status); + + ASSERT_FALSE(status.isOk()) << "query: " << query << "\nexpected it to fail"; + EXPECT_NE(status.getError().find(expectedError), std::string::npos) + << "query: " << query << "\nerror: " << status.getError(); + } + + void createFourCities() { + write("CREATE (n:City {name: 'Paris', at: point({longitude: 2.3522, latitude: 48.8566})})"); + write("CREATE (n:City {name: 'London', at: point({longitude: -0.1276, latitude: 51.5072})})"); + write("CREATE (n:City {name: 'Grid', at: point({x: 3, y: 4})})"); + write("CREATE (n:City {name: 'Tower', at: point({x: 3, y: 4, z: 5})})"); + } + + const std::string _graphName = "simpledb"; + std::unique_ptr _env; + std::unique_ptr _interpreter; + QueryConfig _queryConfig; +}; + +TEST_F(PointValueFlowTest, coalescesAPoint) { + expectRows("RETURN coalesce(null, point({x: 1, y: 2}))", {{"point({x: 1.0, y: 2.0, crs: 'cartesian'})"}}); +} + +TEST_F(PointValueFlowTest, selectsAPointInACase) { + expectRows("UNWIND [true, false] AS b RETURN CASE WHEN b THEN point({x: 1, y: 2}) ELSE null END", + {{"point({x: 1.0, y: 2.0, crs: 'cartesian'})"}, {"null"}}); + expectRows("UNWIND [true, false] AS b RETURN CASE WHEN b THEN point({x: 1, y: 2}) ELSE 1 END", + {{"point({x: 1.0, y: 2.0, crs: 'cartesian'})"}, {"1"}}); +} + +TEST_F(PointValueFlowTest, unitesPoints) { + expectRows("RETURN point({x: 1, y: 2}) AS p UNION RETURN point({x: 1, y: 2}) AS p", + {{"point({x: 1.0, y: 2.0, crs: 'cartesian'})"}}); + expectRows("RETURN point({x: 1, y: 2}) AS p UNION ALL RETURN point({x: 1, y: 2, crs: 'wgs-84'}) AS p", + {{"point({x: 1.0, y: 2.0, crs: 'cartesian'})"}, {"point({x: 1.0, y: 2.0, crs: 'wgs-84'})"}}); +} + +TEST_F(PointValueFlowTest, comparesListsAndMapsHoldingPoints) { + expectRows("RETURN [point({x: 1, y: 2})] = [point({x: 1, y: 2})]", {{"true"}}); + expectRows("RETURN [point({x: 1, y: 2})] = [point({x: 1, y: 3})]", {{"false"}}); + expectRows("RETURN {a: point({x: 1, y: 2})} = {a: point({x: 1, y: 2})}", {{"true"}}); +} + +TEST_F(PointValueFlowTest, comparesAPointCarriedByAVariable) { + expectRows("WITH point({x: 1, y: 2}) AS p RETURN p = p, p <> p", {{"true", "false"}}); +} + +TEST_F(PointValueFlowTest, findsAPointAmongValuesOfOtherTypes) { + expectRows("RETURN point({x: 1, y: 2}) IN [1, 'a', point({x: 1, y: 2})]", {{"true"}}); + expectRows("RETURN coll.indexOf([1, point({x: 1, y: 2})], point({x: 1, y: 2}))", {{"1"}}); +} + +TEST_F(PointValueFlowTest, ordersPointsAmongValuesOfOtherTypes) { + expectOrderedRows("UNWIND [1, 'a', point({x: 1, y: 2}), true] AS v RETURN v ORDER BY v", + {{"point({x: 1.0, y: 2.0, crs: 'cartesian'})"}, {"a"}, {"true"}, {"1"}}); +} + +TEST_F(PointValueFlowTest, countsDistinctPointsOfAList) { + expectRows("UNWIND [point({x: 1, y: 2}), point({x: 1, y: 2}), point({x: 1, y: 2, crs: 'wgs-84'})] AS p " + "RETURN count(DISTINCT p)", + {{"2"}}); +} + +TEST_F(PointValueFlowTest, collectsDistinctPointProperties) { + createFourCities(); + write("CREATE (n:City {name: 'Twin', at: point({x: 3, y: 4})})"); + + expectRows("MATCH (n:City) WHERE n.name IN ['Grid', 'Twin'] RETURN collect(DISTINCT n.at)", + {{"[point({x: 3.0, y: 4.0, crs: 'cartesian'})]"}}); +} + +TEST_F(PointValueFlowTest, filtersOnAListOfPoints) { + createFourCities(); + expectRows("MATCH (n:City) WHERE n.at IN [point({x: 3, y: 4}), point({x: 3, y: 4, z: 5})] RETURN n.name", + {{"Grid"}, {"Tower"}}); +} + +TEST_F(PointValueFlowTest, joinsOnAPoint) { + createFourCities(); + write("CREATE (n:City {name: 'Twin', at: point({x: 3, y: 4})})"); + + expectRows("MATCH (a:City), (b:City) WHERE a.at = b.at AND a.name < b.name RETURN a.name, b.name", + {{"Grid", "Twin"}}); +} + +TEST_F(PointValueFlowTest, mergesOnAPoint) { + write("MERGE (n:City {at: point({x: 1, y: 2})})"); + write("MERGE (n:City {at: point({x: 1, y: 2})})"); + write("MERGE (n:City {at: point({x: 1, y: 2, crs: 'wgs-84'})})"); + + expectRows("MATCH (n:City) RETURN n.at", + {{"point({x: 1.0, y: 2.0, crs: 'cartesian'})"}, {"point({x: 1.0, y: 2.0, crs: 'wgs-84'})"}}); +} + +TEST_F(PointValueFlowTest, matchesAPointGivenInline) { + createFourCities(); + expectRows("MATCH (n:City {at: point({x: 3, y: 4})}) RETURN n.name", {{"Grid"}}); +} + +TEST_F(PointValueFlowTest, projectsAPointPropertyIntoAMap) { + write("CREATE (n:City {name: 'Grid', at: point({x: 3, y: 4})})"); + + expectRows("MATCH (n:City) RETURN n {.at}", {{"{at: point({x: 3.0, y: 4.0, crs: 'cartesian'})}"}}); + expectRows("MATCH (n:City) RETURN properties(n)", {{"{at: point({x: 3.0, y: 4.0, crs: 'cartesian'}), name: Grid}"}}); +} + +TEST_F(PointValueFlowTest, buildsAPointFromAMapParameterOfARow) { + expectRows("WITH {longitude: 2.3522, latitude: 48.8566} AS m RETURN point(m)", + {{"point({x: 2.3522, y: 48.8566, crs: 'wgs-84'})"}}); +} + +TEST_F(PointValueFlowTest, writesAPointReadOffAListElement) { + write("UNWIND [point({x: 1, y: 2})] AS p CREATE (n:City {at: p})"); + expectRows("MATCH (n:City) RETURN n.at", {{"point({x: 1.0, y: 2.0, crs: 'cartesian'})"}}); +} + +TEST_F(PointValueFlowTest, convertsAPointToNothingButAString) { + expectError("RETURN toInteger(point({x: 1, y: 2}))", "toInteger"); + expectRows("RETURN toIntegerOrNull(point({x: 1, y: 2}))", {{"null"}}); + expectRows("RETURN toStringOrNull(point({x: 1, y: 2}))", {{"point({x: 1.0, y: 2.0, crs: 'cartesian'})"}}); +} diff --git a/test/query/ir/system/CMakeLists.txt b/test/query/ir/system/CMakeLists.txt index 19aa217333..20d00621d6 100644 --- a/test/query/ir/system/CMakeLists.txt +++ b/test/query/ir/system/CMakeLists.txt @@ -7,6 +7,8 @@ add_ir_tests(test_query_ir_system_ops_dialect SystemOpsDialectTest.cpp) target_compile_definitions(test_query_ir_system_ops_dialect PRIVATE MLIR_SAMPLES_DIR="${CMAKE_SOURCE_DIR}/samples/mlir") +add_ir_tests(test_query_ir_import_graph_types ImportGraphTypesTest.cpp) + add_ir_tests(test_query_ir_system_command_lowering SystemCommandLoweringTest.cpp) target_compile_definitions(test_query_ir_system_command_lowering PRIVATE MLIR_SAMPLES_DIR="${CMAKE_SOURCE_DIR}/samples/mlir") diff --git a/test/query/ir/system/ImportGraphTypesTest.cpp b/test/query/ir/system/ImportGraphTypesTest.cpp new file mode 100644 index 0000000000..f442e53946 --- /dev/null +++ b/test/query/ir/system/ImportGraphTypesTest.cpp @@ -0,0 +1,73 @@ +#include + +#include +#include + +#include "mlir/Dialect/Func/IR/FuncOps.h" +#include "mlir/IR/BuiltinOps.h" +#include "mlir/IR/Diagnostics.h" +#include "mlir/IR/MLIRContext.h" +#include "mlir/Parser/Parser.h" + +#include "DBDialect.h" +#include "NLDialect.h" +#include "StorageDialect.h" + +namespace { + +// The `types` of db.import_graph and nl.import_graph map a property to the type an import +// reads it as where its format spells none: a datetime, duration or point from JSONL, a +// duration from Parquet. Any other entry is malformed IR. +class ImportGraphTypesTest : public ::testing::Test { +protected: + ImportGraphTypesTest() { + _context.getOrLoadDialect(); + _context.getOrLoadDialect(); + _context.getOrLoadDialect(); + _context.getOrLoadDialect(); + } + + bool verifies(std::string_view import) { + const mlir::ScopedDiagnosticHandler handler(&_context, [](mlir::Diagnostic&) { + return mlir::success(); + }); + + const std::string text = "module { func.func @main() { " + std::string(import) + " return } }"; + + return static_cast(mlir::parseSourceString(text, mlir::ParserConfig(&_context))); + } + + mlir::MLIRContext _context; +}; + +} + +TEST_F(ImportGraphTypesTest, acceptsTheTypesEachFormatReads) { + EXPECT_TRUE(verifies(R"(%g = db.import_graph jsonl("/data/c.jsonl") as "c" + types {at = !storage.point, founded = !storage.datetime, span = !storage.duration} : !db.column)")); + EXPECT_TRUE(verifies(R"(%g = db.import_graph parquet("/data/c") as "c" + types {span = !storage.duration} : !db.column)")); + EXPECT_TRUE(verifies(R"(%g = nl.import_graph jsonl("/data/c.jsonl") as "c" + types {at = !storage.point, founded = !storage.datetime, span = !storage.duration} : !nl.chunk)")); + EXPECT_TRUE(verifies(R"(%g = nl.import_graph parquet("/data/c") as "c" + types {span = !storage.duration} : !nl.chunk)")); +} + +TEST_F(ImportGraphTypesTest, rejectsAnEntryThatIsNoType) { + EXPECT_FALSE(verifies(R"(%g = db.import_graph jsonl("/data/c.jsonl") as "c" types {at} : !db.column)")); + EXPECT_FALSE(verifies(R"(%g = db.import_graph jsonl("/data/c.jsonl") as "c" types {at = 1 : i64} : !db.column)")); + EXPECT_FALSE(verifies(R"(%g = nl.import_graph jsonl("/data/c.jsonl") as "c" types {at} : !nl.chunk)")); +} + +TEST_F(ImportGraphTypesTest, rejectsATypeTheFormatSpellsItself) { + EXPECT_FALSE(verifies(R"(%g = db.import_graph jsonl("/data/c.jsonl") as "c" types {at = f64} : !db.column)")); + EXPECT_FALSE(verifies(R"(%g = nl.import_graph jsonl("/data/c.jsonl") as "c" types {at = f64} : !nl.chunk)")); +} + +TEST_F(ImportGraphTypesTest, rejectsATypeTheFormatDoesNotRead) { + EXPECT_FALSE(verifies(R"(%g = db.import_graph parquet("/data/c") as "c" types {g = !storage.point} : !db.column)")); + EXPECT_FALSE(verifies(R"(%g = db.import_graph parquet("/data/c") as "c" types {g = !storage.datetime} : !db.column)")); + EXPECT_FALSE(verifies(R"(%g = db.import_graph gml("/data/c.gml") as "c" types {g = !storage.duration} : !db.column)")); + EXPECT_FALSE(verifies(R"(%g = nl.import_graph parquet("/data/c") as "c" types {g = !storage.point} : !nl.chunk)")); + EXPECT_FALSE(verifies(R"(%g = nl.import_graph gml("/data/c.gml") as "c" types {g = !storage.duration} : !nl.chunk)")); +} diff --git a/test/query/parser/CypherASTComparator.cpp b/test/query/parser/CypherASTComparator.cpp index 2e8d364d3b..ca2c056b91 100644 --- a/test/query/parser/CypherASTComparator.cpp +++ b/test/query/parser/CypherASTComparator.cpp @@ -15,12 +15,11 @@ #include "CreateNodePropertyIndexQuery.h" #include "CreateVectorIndexQuery.h" #include "CypherAST.h" -#include "DateTimeSpec.h" #include "DeleteVectorIndexQuery.h" #include "DropIndexQuery.h" -#include "DurationSpec.h" #include "EdgePattern.h" #include "EmbeddingsSpec.h" +#include "TypedPropertySpec.h" #include "ExplainRequest.h" #include "FunctionInvocation.h" #include "InstallExtensionQuery.h" @@ -648,7 +647,7 @@ void renderBound(int64_t bound, std::string& rendered) { fmt::format_to(std::back_inserter(rendered), "{}", bound); } -void renderNames(const std::vector& names, std::string& rendered) { +void renderTypedNames(const std::vector>& names, std::string& rendered) { rendered += '{'; for (size_t index = 0; index < names.size(); index++) { @@ -656,7 +655,9 @@ void renderNames(const std::vector& names, std::string& render rendered += ", "; } - renderQuoted(names[index], rendered); + renderQuoted(names[index].first, rendered); + rendered += ": "; + rendered += ValueTypeName::value(names[index].second); } rendered += '}'; @@ -809,10 +810,11 @@ bool CypherASTComparator::compareEnum(std::string_view field, Enum expected, Enu return false; } -template -bool CypherASTComparator::compareNameSpec(std::string_view field, const Spec& expected, const Spec& actual) { - std::vector expectedNames(expected.begin(), expected.end()); - std::vector actualNames(actual.begin(), actual.end()); +bool CypherASTComparator::compareTypedPropertySpec(std::string_view field, + const TypedPropertySpec& expected, + const TypedPropertySpec& actual) { + std::vector> expectedNames(expected.begin(), expected.end()); + std::vector> actualNames(actual.begin(), actual.end()); std::ranges::sort(expectedNames); std::ranges::sort(actualNames); @@ -822,10 +824,10 @@ bool CypherASTComparator::compareNameSpec(std::string_view field, const Spec& ex } std::string expectedText; - renderNames(expectedNames, expectedText); + renderTypedNames(expectedNames, expectedText); std::string actualText; - renderNames(actualNames, actualText); + renderTypedNames(actualNames, actualText); const PathScope scope(_path, field); reportDifference(expectedText, actualText); @@ -1183,8 +1185,7 @@ bool CypherASTComparator::compareLoadJsonlQuery(const LoadJsonlQuery* expected, return compareString("filePath", expected->getFilePath().get(), actual->getFilePath().get()) && compareString("graphName", expected->getGraphName(), actual->getGraphName()) && compareEmbeddingSpec("embeddingSpecs", expected->getEmbeddingSpecs(), actual->getEmbeddingSpecs()) - && compareNameSpec("dateTimeSpecs", expected->getDateTimeSpecs(), actual->getDateTimeSpecs()) - && compareNameSpec("durationSpecs", expected->getDurationSpecs(), actual->getDurationSpecs()); + && compareTypedPropertySpec("typedPropertySpec", expected->getTypedPropertySpec(), actual->getTypedPropertySpec()); } bool CypherASTComparator::compareChangeQuery(const ChangeQuery* expected, const ChangeQuery* actual) { @@ -1203,7 +1204,7 @@ bool CypherASTComparator::compareLoadGMLQuery(const LoadGMLQuery* expected, cons bool CypherASTComparator::compareLoadParquetQuery(const LoadParquetQuery* expected, const LoadParquetQuery* actual) { return compareString("filePath", expected->getFilePath().get(), actual->getFilePath().get()) && compareString("graphName", expected->getGraphName(), actual->getGraphName()) - && compareNameSpec("durationSpecs", expected->getDurationSpecs(), actual->getDurationSpecs()); + && compareTypedPropertySpec("typedPropertySpec", expected->getTypedPropertySpec(), actual->getTypedPropertySpec()); } bool CypherASTComparator::compareS3ConnectQuery(const S3ConnectQuery* expected, const S3ConnectQuery* actual) { diff --git a/test/query/parser/CypherASTComparator.h b/test/query/parser/CypherASTComparator.h index eeeb0b7503..2669951da5 100644 --- a/test/query/parser/CypherASTComparator.h +++ b/test/query/parser/CypherASTComparator.h @@ -91,6 +91,7 @@ class StringExpr; class Symbol; class SymbolChain; class SymbolExpr; +class TypedPropertySpec; class UnaryExpr; class UnwindStmt; class VectorSearchStmt; @@ -143,8 +144,9 @@ class CypherASTComparator { template bool compareEnum(std::string_view field, Enum expected, Enum actual); - template - bool compareNameSpec(std::string_view field, const Spec& expected, const Spec& actual); + bool compareTypedPropertySpec(std::string_view field, + const db::TypedPropertySpec& expected, + const db::TypedPropertySpec& actual); bool compareSize(std::string_view field, size_t expected, size_t actual); bool comparePresence(std::string_view field, const void* expected, const void* actual, std::string_view className); diff --git a/test/query/parser/CypherASTComparatorTest.cpp b/test/query/parser/CypherASTComparatorTest.cpp index cd69b91122..f7563b131a 100644 --- a/test/query/parser/CypherASTComparatorTest.cpp +++ b/test/query/parser/CypherASTComparatorTest.cpp @@ -157,7 +157,7 @@ const DifferenceCase DIFFERENCES[] = { "queries[0] CreateGraphQuery > graphName: expected \"alpha\", actual \"beta\""}, {"LOAD JSONL 'g.jsonl' WITH DURATIONS [a, b]", "LOAD JSONL 'g.jsonl' WITH DURATIONS [a]", - "queries[0] LoadJsonlQuery > durationSpecs: expected {\"a\", \"b\"}, actual {\"a\"}"}, + "queries[0] LoadJsonlQuery > typedPropertySpec: expected {\"a\": Duration, \"b\": Duration}, actual {\"a\": Duration}"}, {"RETURN 1; RETURN 2", "RETURN 1", "queries: expected size 2, actual size 1"}, diff --git a/test/query/parser/KeywordObjectNameTest.cpp b/test/query/parser/KeywordObjectNameTest.cpp index 2cff4f0381..9a710720c3 100644 --- a/test/query/parser/KeywordObjectNameTest.cpp +++ b/test/query/parser/KeywordObjectNameTest.cpp @@ -139,23 +139,23 @@ TEST_F(KeywordObjectNameTest, importsParquetAsAGraphMatch) { const LoadParquetQuery* load = static_cast(command); EXPECT_EQ(load->getGraphName(), "match"); - EXPECT_TRUE(load->getDurationSpecs().contains("commit")); + EXPECT_EQ(load->getTypedPropertySpec().getType("commit"), ValueType::Duration); } TEST_F(KeywordObjectNameTest, readsDatetimesCommit) { const QueryCommand* command = parseCommand("LOAD JSONL 'f.jsonl' WITH DATETIMES [commit, `created at`]", QueryCommand::Kind::LOAD_JSONL_QUERY); const LoadJsonlQuery* load = static_cast(command); - EXPECT_TRUE(load->getDateTimeSpecs().contains("commit")); - EXPECT_TRUE(load->getDateTimeSpecs().contains("created at")); + EXPECT_EQ(load->getTypedPropertySpec().getType("commit"), ValueType::DateTime); + EXPECT_EQ(load->getTypedPropertySpec().getType("created at"), ValueType::DateTime); } TEST_F(KeywordObjectNameTest, readsDurationsMatch) { const QueryCommand* command = parseCommand("LOAD JSONL 'f.jsonl' WITH DURATIONS [match, `time spent`]", QueryCommand::Kind::LOAD_JSONL_QUERY); const LoadJsonlQuery* load = static_cast(command); - EXPECT_TRUE(load->getDurationSpecs().contains("match")); - EXPECT_TRUE(load->getDurationSpecs().contains("time spent")); + EXPECT_EQ(load->getTypedPropertySpec().getType("match"), ValueType::Duration); + EXPECT_EQ(load->getTypedPropertySpec().getType("time spent"), ValueType::Duration); } TEST_F(KeywordObjectNameTest, readsEmbeddingsEmbedding) { diff --git a/test/query/parser/ParseEdgeCaseTest.cpp b/test/query/parser/ParseEdgeCaseTest.cpp index 79fbef6598..efb09d2f1f 100644 --- a/test/query/parser/ParseEdgeCaseTest.cpp +++ b/test/query/parser/ParseEdgeCaseTest.cpp @@ -374,22 +374,20 @@ TEST_F(ParseEdgeCaseTest, parsesAdministrativeQueries) { embeddings.emplace("e", 4); load->setEmbeddingSpecs(std::move(embeddings)); - DateTimeSpec dateTimes; - dateTimes.emplace("d"); - dateTimes.emplace("t"); - load->setDateTimeSpecs(std::move(dateTimes)); - - DurationSpec durations; - durations.emplace("u"); - load->setDurationSpecs(std::move(durations)); - - expectParse("LOAD JSONL 'f' WITH EMBEDDINGS [{e, 4}] WITH DATETIMES ['d', t] WITH DURATIONS [u]", {load}); + TypedPropertySpec types; + types.emplace("d", ValueType::DateTime); + types.emplace("t", ValueType::DateTime); + types.emplace("u", ValueType::Duration); + types.emplace("p", ValueType::Point); + load->setTypedPropertySpec(std::move(types)); + + expectParse("LOAD JSONL 'f' WITH EMBEDDINGS [{e, 4}] WITH DATETIMES ['d', t] WITH DURATIONS [u] WITH POINTS [p]", {load}); } expectError("LOAD JSONL 'f' WITH 5", " 1 | LOAD JSONL 'f' WITH 5\n" " | ^\n" - "-------* syntax error, unexpected DIGIT, expecting DATETIMES or DURATIONS or EMBEDDINGS"); + "-------* syntax error, unexpected DIGIT, expecting DATETIMES or DURATIONS or POINTS or EMBEDDINGS"); expectParse("LOAD GML 'f'", {LoadGMLQuery::create(getExpectedAST(), fs::Path(std::string("f")))}); @@ -404,10 +402,10 @@ TEST_F(ParseEdgeCaseTest, parsesAdministrativeQueries) { LoadParquetQuery* load = LoadParquetQuery::create(getExpectedAST(), fs::Path(std::string("f"))); load->setGraphName("g"); - DurationSpec durations; - durations.emplace("d"); - durations.emplace("e"); - load->setDurationSpecs(std::move(durations)); + TypedPropertySpec types; + types.emplace("d", ValueType::Duration); + types.emplace("e", ValueType::Duration); + load->setTypedPropertySpec(std::move(types)); expectParse("LOAD PARQUET 'f' AS g WITH DURATIONS [d, 'e']", {load}); } diff --git a/test/storage/CMakeLists.txt b/test/storage/CMakeLists.txt index 25fdab173c..a677ba3609 100644 --- a/test/storage/CMakeLists.txt +++ b/test/storage/CMakeLists.txt @@ -9,6 +9,9 @@ add_turing_test(test_storage_datetime_parse DateTimeParseTest.cpp) add_turing_test(test_storage_datetime_component DateTimeComponentTest.cpp) add_turing_test(test_storage_duration_format DurationFormatTest.cpp) add_turing_test(test_storage_duration_component DurationComponentTest.cpp) +add_turing_test(test_storage_point PointTest.cpp) +add_turing_test(test_storage_point_distance PointDistanceTest.cpp) +add_turing_test(test_storage_point_distance_symmetry PointDistanceSymmetryTest.cpp) add_turing_test(test_storage_mappropertycontainer MapPropertyContainerTest.cpp) add_turing_test(test_storage_property_hash_index_numeric_key PropertyHashIndexNumericKeyTest.cpp) @@ -126,6 +129,7 @@ add_turing_test(test_storage_dump_embedding_graph dump/EmbeddingGraphDumpLoadTes add_turing_test(test_storage_dump_list_graph dump/ListGraphDumpLoadTest.cpp) add_turing_test(test_storage_dump_datetime_graph dump/DateTimeGraphDumpLoadTest.cpp) add_turing_test(test_storage_dump_duration_graph dump/DurationGraphDumpLoadTest.cpp) +add_turing_test(test_storage_dump_point_graph dump/PointGraphDumpLoadTest.cpp) add_turing_test(test_storage_dump_map_graph dump/MapGraphDumpLoadTest.cpp) add_turing_test(test_storage_dump_map_datetime_graph dump/MapDateTimeGraphDumpLoadTest.cpp) add_turing_test(test_storage_dump_celltagdumpformat dump/CellTagDumpFormatTest.cpp) diff --git a/test/storage/PointDistanceSymmetryTest.cpp b/test/storage/PointDistanceSymmetryTest.cpp new file mode 100644 index 0000000000..286ca39fe1 --- /dev/null +++ b/test/storage/PointDistanceSymmetryTest.cpp @@ -0,0 +1,51 @@ +#include + +#include +#include + +#include "metadata/Point.h" + +using namespace db; + +namespace { + +void geographicPoints(PointCRS crs, std::vector& points) { + for (int latitude = -85; latitude <= 85; latitude += 17) { + for (int longitude = -175; longitude <= 175; longitude += 23) { + points.emplace_back(crs, longitude + 0.35, latitude + 0.85, 120.5); + } + } +} + +} + +TEST(PointDistanceSymmetryTest, measuresZeroFromAGeographicPointToItself) { + for (const PointCRS crs : {PointCRS::WGS84, PointCRS::WGS84_3D}) { + std::vector points; + geographicPoints(crs, points); + + for (const Point& point : points) { + const std::optional distance = Point::distance(point, point); + + ASSERT_TRUE(distance.has_value()); + ASSERT_EQ(*distance, 0.0) << point.getX() << ", " << point.getY(); + } + } +} + +TEST(PointDistanceSymmetryTest, measuresTheSameDistanceEitherWay) { + for (const PointCRS crs : {PointCRS::WGS84, PointCRS::WGS84_3D}) { + std::vector points; + geographicPoints(crs, points); + + for (size_t lhs = 0; lhs < points.size(); lhs++) { + for (size_t rhs = lhs + 1; rhs < points.size(); rhs++) { + const std::optional there = Point::distance(points[lhs], points[rhs]); + const std::optional back = Point::distance(points[rhs], points[lhs]); + + ASSERT_TRUE(there.has_value() && back.has_value()); + ASSERT_EQ(*there, *back) << lhs << " and " << rhs; + } + } + } +} diff --git a/test/storage/PointDistanceTest.cpp b/test/storage/PointDistanceTest.cpp new file mode 100644 index 0000000000..f64c1e62bc --- /dev/null +++ b/test/storage/PointDistanceTest.cpp @@ -0,0 +1,47 @@ +#include + +#include + +#include +#include + +#include "metadata/Point.h" + +using namespace db; + +namespace { + +constexpr double EARTH_MEAN_RADIUS_METRES = 6371008.8; +constexpr double HALF_THE_EARTH_METRES = std::numbers::pi * EARTH_MEAN_RADIUS_METRES; + +} + +TEST(PointDistanceTest, measuresHalfTheEarthBetweenAntipodes) { + const std::optional distance = Point::distance(Point(PointCRS::WGS84, -178.0, 8.0, 0.0), + Point(PointCRS::WGS84, 2.0, -8.0, 0.0)); + + ASSERT_TRUE(distance.has_value()); + EXPECT_NEAR(*distance, HALF_THE_EARTH_METRES, 1.0); +} + +TEST(PointDistanceTest, measuresHalfTheEarthBetweenEveryPairOfIntegerAntipodes) { + for (int latitude = -89; latitude <= 89; latitude++) { + for (int longitude = -179; longitude <= 0; longitude++) { + const Point point(PointCRS::WGS84, longitude, latitude, 0.0); + const Point antipode(PointCRS::WGS84, longitude + 180, -latitude, 0.0); + + const std::optional distance = Point::distance(point, antipode); + + ASSERT_TRUE(distance.has_value()); + ASSERT_NEAR(*distance, HALF_THE_EARTH_METRES, 1.0) << "longitude " << longitude << ", latitude " << latitude; + } + } +} + +TEST(PointDistanceTest, measuresHalfTheEarthBetween3DAntipodesAtSeaLevel) { + const std::optional distance = Point::distance(Point(PointCRS::WGS84_3D, -178.0, 8.0, 0.0), + Point(PointCRS::WGS84_3D, 2.0, -8.0, 0.0)); + + ASSERT_TRUE(distance.has_value()); + EXPECT_NEAR(*distance, HALF_THE_EARTH_METRES, 1.0); +} diff --git a/test/storage/PointTest.cpp b/test/storage/PointTest.cpp new file mode 100644 index 0000000000..96410b81d0 --- /dev/null +++ b/test/storage/PointTest.cpp @@ -0,0 +1,100 @@ +#include + +#include +#include + +#include +#include +#include + +#include "metadata/Point.h" +#include "TuringException.h" + +using namespace db; + +namespace { + +std::string formatted(const Point& value) { + std::string out; + Point::format(out, value); + + return out; +} + +} + +TEST(PointTest, writesTheCoordinatesItsCRSHas) { + EXPECT_EQ(formatted(Point(PointCRS::Cartesian, 1.0, 2.5, 9.0)), "point({x: 1.0, y: 2.5, crs: 'cartesian'})"); + EXPECT_EQ(formatted(Point(PointCRS::Cartesian3D, 1.0, 2.5, 9.0)), "point({x: 1.0, y: 2.5, z: 9.0, crs: 'cartesian-3d'})"); + EXPECT_EQ(formatted(Point(PointCRS::WGS84, 2.3522, 48.8566, 0.0)), "point({x: 2.3522, y: 48.8566, crs: 'wgs-84'})"); + EXPECT_EQ(formatted(Point(PointCRS::WGS84_3D, 2.3522, 48.8566, 35.0)), "point({x: 2.3522, y: 48.8566, z: 35.0, crs: 'wgs-84-3d'})"); +} + +TEST(PointTest, holdsAZOfZeroIn2D) { + const Point point(PointCRS::Cartesian, 1.0, 2.0, 9.0); + + EXPECT_EQ(point.getZ(), 0.0); + EXPECT_EQ(point, Point(PointCRS::Cartesian, 1.0, 2.0, 0.0)); +} + +TEST(PointTest, holdsNoNegativeZero) { + const Point negative(PointCRS::Cartesian3D, -0.0, -0.0, -0.0); + const Point positive(PointCRS::Cartesian3D, 0.0, 0.0, 0.0); + + EXPECT_EQ(memcmp(&negative, &positive, sizeof(Point)), 0); + EXPECT_EQ(std::hash {}(negative), std::hash {}(positive)); +} + +TEST(PointTest, wrapsALongitudeIntoTheRange) { + EXPECT_EQ(Point(PointCRS::WGS84, 190.0, 0.0, 0.0).getX(), -170.0); + EXPECT_EQ(Point(PointCRS::WGS84, -190.0, 0.0, 0.0).getX(), 170.0); + EXPECT_EQ(Point(PointCRS::WGS84, 180.0, 0.0, 0.0).getX(), 180.0); + EXPECT_EQ(Point(PointCRS::WGS84, -180.0, 0.0, 0.0).getX(), -180.0); + EXPECT_EQ(Point(PointCRS::WGS84, 720.0, 0.0, 0.0).getX(), 0.0); + EXPECT_EQ(Point(PointCRS::Cartesian, 720.0, 0.0, 0.0).getX(), 720.0); +} + +TEST(PointTest, rejectsALatitudePastAPole) { + EXPECT_THROW(Point(PointCRS::WGS84, 0.0, 90.5, 0.0), TuringException); + EXPECT_THROW(Point(PointCRS::WGS84_3D, 0.0, -90.5, 0.0), TuringException); + EXPECT_NO_THROW(Point(PointCRS::WGS84, 0.0, 90.0, 0.0)); + EXPECT_NO_THROW(Point(PointCRS::Cartesian, 0.0, 90.5, 0.0)); +} + +TEST(PointTest, rejectsANonFiniteCoordinate) { + EXPECT_THROW(Point(PointCRS::Cartesian, std::numeric_limits::infinity(), 0.0, 0.0), TuringException); + EXPECT_THROW(Point(PointCRS::Cartesian3D, 0.0, 0.0, std::numeric_limits::quiet_NaN()), TuringException); +} + +TEST(PointTest, ordersBySRIDThenByCoordinates) { + const Point wgs84(PointCRS::WGS84, 100.0, 0.0, 0.0); + const Point wgs84With3D(PointCRS::WGS84_3D, -100.0, 0.0, 0.0); + const Point cartesian(PointCRS::Cartesian, -100.0, 0.0, 0.0); + const Point cartesian3D(PointCRS::Cartesian3D, -100.0, 0.0, 0.0); + + EXPECT_LT(wgs84, wgs84With3D); + EXPECT_LT(wgs84With3D, cartesian); + EXPECT_LT(cartesian, cartesian3D); + + EXPECT_LT(Point(PointCRS::Cartesian, 1.0, 9.0, 0.0), Point(PointCRS::Cartesian, 2.0, 0.0, 0.0)); + EXPECT_LT(Point(PointCRS::Cartesian, 1.0, 1.0, 0.0), Point(PointCRS::Cartesian, 1.0, 2.0, 0.0)); + EXPECT_LT(Point(PointCRS::Cartesian3D, 1.0, 1.0, 1.0), Point(PointCRS::Cartesian3D, 1.0, 1.0, 2.0)); +} + +TEST(PointTest, namesTheCRSAndItsSRID) { + EXPECT_EQ(Point::getCRSName(PointCRS::Cartesian), "cartesian"); + EXPECT_EQ(Point::getSRID(PointCRS::Cartesian), 7203); + EXPECT_EQ(Point::getCRSName(PointCRS::Cartesian3D), "cartesian-3d"); + EXPECT_EQ(Point::getSRID(PointCRS::Cartesian3D), 9157); + EXPECT_EQ(Point::getCRSName(PointCRS::WGS84), "wgs-84"); + EXPECT_EQ(Point::getSRID(PointCRS::WGS84), 4326); + EXPECT_EQ(Point::getCRSName(PointCRS::WGS84_3D), "wgs-84-3d"); + EXPECT_EQ(Point::getSRID(PointCRS::WGS84_3D), 4979); +} + +TEST(PointTest, keepsTheOrdinalsOfItsCRSs) { + EXPECT_EQ(static_cast(PointCRS::Cartesian), 0); + EXPECT_EQ(static_cast(PointCRS::Cartesian3D), 1); + EXPECT_EQ(static_cast(PointCRS::WGS84), 2); + EXPECT_EQ(static_cast(PointCRS::WGS84_3D), 3); +} diff --git a/test/storage/QueryTypeNameTest.cpp b/test/storage/QueryTypeNameTest.cpp index 9fbb4ff7a5..95539d9cb0 100644 --- a/test/storage/QueryTypeNameTest.cpp +++ b/test/storage/QueryTypeNameTest.cpp @@ -33,6 +33,7 @@ constexpr std::array CELL_TAG_TYPES { CellTagType {CellTag::MapView, EvaluatedType::Map}, CellTagType {CellTag::Duration, EvaluatedType::Duration}, CellTagType {CellTag::Path, EvaluatedType::GraphPath}, + CellTagType {CellTag::Point, EvaluatedType::Point}, CellTagType {CellTag::INVALID, EvaluatedType::Invalid}, }; diff --git a/test/storage/dump/CellTagDumpFormatTest.cpp b/test/storage/dump/CellTagDumpFormatTest.cpp index ae2f15ad3b..2f9e1b1920 100644 --- a/test/storage/dump/CellTagDumpFormatTest.cpp +++ b/test/storage/dump/CellTagDumpFormatTest.cpp @@ -10,6 +10,7 @@ #include "map/MapContainer.h" #include "metadata/DateTime.h" #include "metadata/Duration.h" +#include "metadata/Point.h" #include "metadata/PropertyNull.h" #include "FileHash.h" #include "FilePageReader.h" @@ -132,6 +133,74 @@ TEST_F(CellTagDumpFormatTest, mapHoldingEveryCellTag) { } } +TEST_F(CellTagDumpFormatTest, listAndMapHoldingAPointOfEveryCRS) { + const fs::Path outDir(_outDir.c_str()); + const fs::Path listPath = outDir / "point_lists"; + const fs::Path mapPath = outDir / "point_maps"; + + ListContainer lists; + const std::vector items = {Point(PointCRS::Cartesian, 1.5, -2.0, 0.0), + Point(PointCRS::Cartesian3D, 1.5, -2.0, 3.25), + Point(PointCRS::WGS84, 2.3522, 48.8566, 0.0), + Point(PointCRS::WGS84_3D, 2.3522, 48.8566, 35.0)}; + + TypedPropertyContainer originalLists; + originalLists.add(EntityID(0), lists.insert(items)); + + MapContainer maps; + const std::vector entries = {{"a", Point(PointCRS::Cartesian, 1.5, -2.0, 0.0)}, + {"b", Point(PointCRS::Cartesian3D, 1.5, -2.0, 3.25)}, + {"c", Point(PointCRS::WGS84, 2.3522, 48.8566, 0.0)}, + {"d", Point(PointCRS::WGS84_3D, 2.3522, 48.8566, 35.0)}}; + + TypedPropertyContainer originalMaps; + originalMaps.add(EntityID(0), maps.insert(entries)); + + { + auto writer = fs::FilePageWriter::open(listPath); + ASSERT_TRUE(writer); + ListPropertyContainerDumper dumper(writer.value()); + ASSERT_TRUE(dumper.dump(originalLists)); + } + + { + auto writer = fs::FilePageWriter::open(mapPath); + ASSERT_TRUE(writer); + MapPropertyContainerDumper dumper(writer.value()); + ASSERT_TRUE(dumper.dump(originalMaps)); + } + + uint64_t listHash = 0; + hashDumpFileContent(listPath, listHash); + EXPECT_EQ(listHash, 1837138699016962021ull); + + uint64_t mapHash = 0; + hashDumpFileContent(mapPath, mapHash); + EXPECT_EQ(mapHash, 8917834194145396159ull); + + { + auto reader = fs::FilePageReader::open(listPath, DumpConfig::PAGE_SIZE); + ASSERT_TRUE(reader); + ListPropertyContainerLoader loader(reader.value()); + auto result = loader.load(); + ASSERT_TRUE(result); + + const auto& loaded = result.value()->cast(); + ASSERT_TRUE(PropertyContainerComparator::same(&originalLists, &loaded)); + } + + { + auto reader = fs::FilePageReader::open(mapPath, DumpConfig::PAGE_SIZE); + ASSERT_TRUE(reader); + MapPropertyContainerLoader loader(reader.value()); + auto result = loader.load(); + ASSERT_TRUE(result); + + const auto& loaded = result.value()->cast(); + ASSERT_TRUE(PropertyContainerComparator::same(&originalMaps, &loaded)); + } +} + int main(int argc, char** argv) { return turing::test::turingTestMain(argc, argv); } diff --git a/test/storage/dump/PointGraphDumpLoadTest.cpp b/test/storage/dump/PointGraphDumpLoadTest.cpp new file mode 100644 index 0000000000..0a82b9ca81 --- /dev/null +++ b/test/storage/dump/PointGraphDumpLoadTest.cpp @@ -0,0 +1,135 @@ +#include "TuringTest.h" + +#include + +#include + +#include "dump/GraphDumper.h" +#include "dump/GraphLoader.h" +#include "Path.h" +#include "TuringException.h" + +#include "Graph.h" +#include "comparators/GraphComparator.h" +#include "list/ListContainer.h" +#include "map/MapContainer.h" +#include "metadata/Point.h" +#include "writers/GraphWriter.h" + +#include "JobSystem.h" + +using namespace db; +using namespace turing::test; + +namespace { + +using ListItem = ListContainer::ListItemVariant; +using MapEntry = MapContainer::MapKeyValuePair; + +constexpr PointCRS CRS_CYCLE[] = {PointCRS::Cartesian, PointCRS::Cartesian3D, PointCRS::WGS84, PointCRS::WGS84_3D}; + +Point pointAt(size_t index) { + const double coordinate = static_cast(index) - 40.0; + const PointCRS crs = CRS_CYCLE[index % 4]; + + return Point(crs, coordinate, coordinate / 2.0, coordinate * 3.0); +} + +} + +class PointGraphDumpLoadTest : public TuringTest { +protected: + void initialize() override { + } + + void dumpAndCompare(Graph* graph) { + const fs::Path dumpPath = fs::Path {_outDir} / "dump"; + + const auto res = GraphDumper::dump(graph, dumpPath); + if (!res) { + throw TuringException("Failed to dump graph:\n" + res.error().fmtMessage()); + } + + auto loadedGraph = Graph::create(); + const auto loadRes = GraphLoader::load(loadedGraph.get(), dumpPath); + if (!loadRes) { + throw TuringException("Failed to load graph:\n" + loadRes.error().fmtMessage()); + } + + ASSERT_TRUE(GraphComparator::same(*graph, *loadedGraph)); + } +}; + +TEST_F(PointGraphDumpLoadTest, NodePoints) { + constexpr size_t nodeCount = 80; + + auto graph = Graph::create("pointgraph", fs::Path {_outDir} / "original"); + + { + JobSystem jobSystem; + jobSystem.init(); + GraphWriter writer(graph.get(), &jobSystem); + + for (size_t i = 0; i < nodeCount; i++) { + const NodeID node = writer.addNode({"City"}); + writer.addNodeProperty(node, "at", pointAt(i)); + } + + writer.submit(); + } + + dumpAndCompare(graph.get()); +} + +TEST_F(PointGraphDumpLoadTest, EdgePoints) { + constexpr size_t edgeCount = 40; + + auto graph = Graph::create("pointgraph", fs::Path {_outDir} / "original"); + + { + JobSystem jobSystem; + jobSystem.init(); + GraphWriter writer(graph.get(), &jobSystem); + + const NodeID source = writer.addNode({"City"}); + + for (size_t i = 0; i < edgeCount; i++) { + const NodeID target = writer.addNode({"City"}); + const EdgeRecord edge = writer.addEdge("ROAD", source, target); + + writer.addEdgeProperty(edge, "midpoint", pointAt(i)); + } + + writer.submit(); + } + + dumpAndCompare(graph.get()); +} + +TEST_F(PointGraphDumpLoadTest, ListsAndMapsHoldingPoints) { + constexpr size_t nodeCount = 20; + + auto graph = Graph::create("pointgraph", fs::Path {_outDir} / "original"); + ListContainer lists; + MapContainer maps; + + { + JobSystem jobSystem; + jobSystem.init(); + GraphWriter writer(graph.get(), &jobSystem); + + for (size_t i = 0; i < nodeCount; i++) { + const NodeID node = writer.addNode({"City"}); + + const std::vector route = {pointAt(i), pointAt(i + 1), static_cast(i)}; + writer.addNodeProperty(node, "route", lists.insert(route)); + + const std::vector entries = {{"at", pointAt(i)}, {"n", static_cast(i)}}; + writer.addNodeProperty(node, "attrs", maps.insert(entries)); + } + + writer.submit(); + } + + dumpAndCompare(graph.get()); +} diff --git a/test/storage/dump/PropertyDumpFormatTest.cpp b/test/storage/dump/PropertyDumpFormatTest.cpp index 743eac4eb3..168ab15509 100644 --- a/test/storage/dump/PropertyDumpFormatTest.cpp +++ b/test/storage/dump/PropertyDumpFormatTest.cpp @@ -5,6 +5,7 @@ #include "dump/PropertyContainerLoader.h" #include "comparators/PropertyContainerComparator.h" #include "embedding/EmbeddingBucket.h" +#include "metadata/Point.h" #include "FileHash.h" #include "FilePageReader.h" #include "FilePageWriter.h" @@ -144,6 +145,43 @@ TEST_F(PropertyDumpFormatTest, embeddingDumpBytes) { } } +TEST_F(PropertyDumpFormatTest, pointDumpBytes) { + const fs::Path outDir(_outDir.c_str()); + const fs::Path path = outDir / "points"; + + // 40'000 points of 32 bytes span several value pages, and cycle through every CRS so + // the ordinal each is written with shows in the bytes + constexpr PointCRS systems[] = {PointCRS::Cartesian, PointCRS::Cartesian3D, PointCRS::WGS84, PointCRS::WGS84_3D}; + + TypedPropertyContainer original; + for (size_t i = 0; i < 40'000; i++) { + const double coordinate = static_cast(i % 180) - 89.5; + original.add(EntityID(i), Point(systems[i % 4], coordinate, coordinate, static_cast(i))); + } + + { + auto writer = fs::FilePageWriter::open(path); + ASSERT_TRUE(writer); + TrivialPropertyContainerDumper dumper(writer.value()); + ASSERT_TRUE(dumper.dump(original)); + } + + uint64_t hash = 0; + hashDumpFileContent(path, hash); + EXPECT_EQ(hash, 934115674187105631ull); + + { + auto reader = fs::FilePageReader::open(path, DumpConfig::PAGE_SIZE); + ASSERT_TRUE(reader); + TrivialPropertyContainerLoader loader(reader.value()); + auto result = loader.load(); + ASSERT_TRUE(result); + + const auto& loaded = result.value()->cast(); + ASSERT_TRUE(PropertyContainerComparator::same(&original, &loaded)); + } +} + int main(int argc, char** argv) { return turing::test::turingTestMain(argc, argv); } diff --git a/test/system/FailedImportClearsLoadingTest.cpp b/test/system/FailedImportClearsLoadingTest.cpp index c25349f9c2..43594b6bb0 100644 --- a/test/system/FailedImportClearsLoadingTest.cpp +++ b/test/system/FailedImportClearsLoadingTest.cpp @@ -37,7 +37,7 @@ class FailedImportClearsLoadingTest : public TuringTest { Graph* graph = nullptr; try { - graph = system.importGraph(fs::Path {std::string {fileName}}, "g", {}, {}, {}); + graph = system.importGraph(fs::Path {std::string {fileName}}, "g", {}, {}); } catch (const TuringException&) { } @@ -66,7 +66,7 @@ TEST_F(FailedImportClearsLoadingTest, aMalformedFileLeavesTheNameFree) { } SystemAccessor system = _env->getSystemManager().accessShared(); - EXPECT_NE(system.importGraph(fs::Path {"good.jsonl"}, "g", {}, {}, {}), nullptr); + EXPECT_NE(system.importGraph(fs::Path {"good.jsonl"}, "g", {}, {}), nullptr); } int main(int argc, char** argv) { diff --git a/test/system/ListAvailableGraphsDuringImportTest.cpp b/test/system/ListAvailableGraphsDuringImportTest.cpp index e26ac83f99..44c31ed34b 100644 --- a/test/system/ListAvailableGraphsDuringImportTest.cpp +++ b/test/system/ListAvailableGraphsDuringImportTest.cpp @@ -41,7 +41,7 @@ TEST_F(ListAvailableGraphsDuringImportTest, aListedGraphStaysListedWhileItFinish std::thread importer([&systemManager, &importing, &importedCount] { for (size_t graph = 0; graph < GRAPH_COUNT; graph++) { SystemAccessor system = systemManager.accessShared(); - if (system.importGraph(fs::Path {"tiny.jsonl"}, "g" + std::to_string(graph), {}, {}, {})) { + if (system.importGraph(fs::Path {"tiny.jsonl"}, "g" + std::to_string(graph), {}, {})) { importedCount++; } } diff --git a/tools/turingdb/TuringShell.cpp b/tools/turingdb/TuringShell.cpp index 0d52e0bee3..c27103538b 100644 --- a/tools/turingdb/TuringShell.cpp +++ b/tools/turingdb/TuringShell.cpp @@ -569,6 +569,10 @@ void asString(std::string& out, const db::Duration v) { db::Duration::format(out, v); } +void asString(std::string& out, const db::Point v) { + db::Point::format(out, v); +} + template void asString(std::string& out, const TemplateCommitHash& hash) { hash.appendString(out); diff --git a/wasm/WasmSink.h b/wasm/WasmSink.h index ebc657fd63..a60c1a3162 100644 --- a/wasm/WasmSink.h +++ b/wasm/WasmSink.h @@ -56,6 +56,8 @@ consteval ColumnType wireCodeOfElement() { return ColumnType::DATE_TIME; } else if constexpr (std::is_same_v) { return ColumnType::DURATION; + } else if constexpr (std::is_same_v) { + return ColumnType::POINT; } else if constexpr (std::is_same_v) { return ColumnType::GRAPH_PATH; } else if constexpr (std::is_same_v) { diff --git a/wasm/npm/TuringClient.d.ts b/wasm/npm/TuringClient.d.ts index 35485c3c94..033c64b4ab 100644 --- a/wasm/npm/TuringClient.d.ts +++ b/wasm/npm/TuringClient.d.ts @@ -24,6 +24,7 @@ export declare const ColumnType: Readonly<{ DATE_TIME: 19; DURATION: 20; GRAPH_PATH: 21; + POINT: 22; }>; export type ColumnTypeName = keyof typeof ColumnType; @@ -38,6 +39,16 @@ export declare const ColumnEncoding: Readonly<{ export type ColumnEncodingCode = (typeof ColumnEncoding)[keyof typeof ColumnEncoding]; +/** A point a query returns. z is undefined for a 2D point. */ +export declare class Point { + private constructor(); + readonly srid: 7203 | 9157 | 4326 | 4979; + readonly x: number; + readonly y: number; + readonly z: number | undefined; + readonly crs: "cartesian" | "cartesian-3d" | "wgs-84" | "wgs-84-3d"; +} + /** A row value as Column.get and Column.toArray return it. A path is its node IDs. */ export type Value = | bigint @@ -46,6 +57,7 @@ export type Value = | string | null | Float32Array + | Point | Value[] | { [key: string]: Value }; @@ -56,6 +68,7 @@ export type NumberValue = | string | null | Float32Array + | Point | NumberValue[] | { [key: string]: NumberValue }; @@ -115,7 +128,7 @@ export declare class Column { readonly isOptional: boolean; /** * Fixed-width vector columns as their typed array, one element per row. Null for - * constants and variable-width kinds. A missing optional row holds 0: read validity. + * constants, variable-width kinds and points. A missing optional row holds 0: read validity. */ readonly values: | BigUint64Array diff --git a/wasm/npm/TuringClient.mjs b/wasm/npm/TuringClient.mjs index 3fcdb758fe..11dbd7303f 100644 --- a/wasm/npm/TuringClient.mjs +++ b/wasm/npm/TuringClient.mjs @@ -57,6 +57,7 @@ export const ColumnType = Object.freeze({ DATE_TIME: 19, DURATION: 20, GRAPH_PATH: 21, + POINT: 22, }); // The column types whose values the nested reader walks out of the flat nested bytes @@ -84,6 +85,12 @@ const CELL_TAG_DATETIME = 10; const CELL_TAG_MAP_VIEW = 11; const CELL_TAG_DURATION = 12; const CELL_TAG_PATH = 13; +const CELL_TAG_POINT = 14; + +// Mirrors db::PointCRS (storage/metadata/Point.h): the SRID of each CRS ordinal. +const POINT_SRIDS = [7203, 9157, 4326, 4979]; +const POINT_CRS_NAMES = { 7203: "cartesian", 9157: "cartesian-3d", 4326: "wgs-84", 4979: "wgs-84-3d" }; +const POINT_SIZE = 32; // Mirrors db::QueryStatus::Status (base/QueryStatus.h); the ERROR packet's first // payload byte indexes into this. @@ -269,6 +276,30 @@ export function double(value) { return new DoubleValue(value); } +// A point a query returns. z is undefined for a 2D point. +export class Point { + constructor(srid, x, y, z) { + this.srid = srid; + this.x = x; + this.y = y; + this.z = z; + Object.freeze(this); + } + + get crs() { + return POINT_CRS_NAMES[this.srid]; + } +} + +// A point's 32 bytes: x, y and z as doubles, then the ordinal of its CRS +function readPoint(view, offset) { + const srid = POINT_SRIDS[view.getUint8(offset + 24)]; + const isThreeDimensional = srid === 9157 || srid === 4979; + const z = isThreeDimensional ? view.getFloat64(offset + 16, true) : undefined; + + return new Point(srid, view.getFloat64(offset, true), view.getFloat64(offset + 8, true), z); +} + function throwParameterError(path, message) { throw new TuringQueryError("PARAMETER_ERROR", `Parameter ${path} ${message}`); } @@ -415,6 +446,7 @@ const FIXED_WIDTH_KINDS = { // count is handed over as it arrived so no precision is lost on the way [ColumnType.DATE_TIME]: { size: 8, wrap: (buffer) => new BigInt64Array(buffer) }, [ColumnType.DURATION]: { size: 8, wrap: (buffer) => new BigInt64Array(buffer) }, + [ColumnType.POINT]: { size: POINT_SIZE, wrap: (buffer) => new DataView(buffer), read: (view, index) => readPoint(view, index * POINT_SIZE) }, }; // Strings are decoded with one TextDecoder call over the whole column and sliced by @@ -509,6 +541,9 @@ export class NestedReader { case CELL_TAG_DOUBLE: this._cursor = payload + 8; return this._view.getFloat64(payload, true); + case CELL_TAG_POINT: + this._cursor = payload + POINT_SIZE; + return readPoint(this._view, payload); case CELL_TAG_BOOL: this._cursor = payload + 1; return this._bytes[payload] !== 0; @@ -556,7 +591,7 @@ function makeItemReader(buffers, nestedReader) { } const values = fixed.wrap(buffers.values.buffer); const read = fixed.read ?? ((array, index) => array[index]); - return { read: (index) => read(values, index), values, direct: fixed.read === undefined }; + return { read: (index) => read(values, index), values: values instanceof DataView ? null : values, direct: fixed.read === undefined }; } const offsets = buffers.offsets !== undefined ? new Uint32Array(buffers.offsets.buffer) : null; @@ -620,8 +655,9 @@ export class Column { } // Fixed-width vector columns as their typed array (Float64Array, BigUint64Array, ...), - // one element per row with no per-row work; null for constants and variable-width - // kinds. A missing optional row holds 0 here: consult validity. + // one element per row with no per-row work; null for constants, variable-width kinds + // and points, which no typed array holds. A missing optional row holds 0 here: consult + // validity. get values() { return this.isConstant ? null : this._typedValues; } diff --git a/wasm/npm/client_smoke.mjs b/wasm/npm/client_smoke.mjs index 52e44c1f16..4d68220f74 100644 --- a/wasm/npm/client_smoke.mjs +++ b/wasm/npm/client_smoke.mjs @@ -8,7 +8,7 @@ import assert from "assert"; -import { Column, TuringClient, TuringQueryError, double } from "./TuringClient.mjs"; +import { Column, Point, TuringClient, TuringQueryError, double } from "./TuringClient.mjs"; const MESSAGE_CHUNK_HEADER = 0; const MESSAGE_CHUNK = 1; @@ -26,6 +26,7 @@ const TYPE_NODE_ID = 9; const TYPE_PROPERTY_NULL = 17; const TYPE_MAP_VIEW = 18; const TYPE_GRAPH_PATH = 21; +const TYPE_POINT = 22; const ENCODING_VECTOR = 0; const ENCODING_OPTIONAL_VECTOR = 1; @@ -40,6 +41,10 @@ const TAG_LIST_VIEW = 6; const TAG_NULL = 7; const TAG_NODE_ID = 8; const TAG_MAP_VIEW = 11; +const TAG_POINT = 14; + +const CRS_CARTESIAN = 0; +const CRS_WGS84_3D = 3; const encoder = new TextEncoder(); @@ -96,6 +101,14 @@ class Writer { raw(byteArray) { this.bytes.push(...byteArray); } + // A point as the server stores it: x, y and z, then its CRS ordinal and 7 bytes of padding + point(crs, x, y, z) { + this.f64(x); + this.f64(y); + this.f64(z); + this.u8(crs); + this.raw(new Uint8Array(7)); + } toUint8Array() { return Uint8Array.from(this.bytes); } @@ -751,6 +764,52 @@ async function testNullGraphPath() { assert.deepStrictEqual(data, [[[[7], null]]]); } +// A point reads as a Point, in a column of its own and inside a list. A 2D point has no z. +async function testPoints() { + const header = makeHeader([ + { name: "at", typeCode: TYPE_POINT, encoding: ENCODING_VECTOR }, + { name: "route", typeCode: TYPE_LIST_VIEW, encoding: ENCODING_VECTOR }, + ]); + + const chunk = new Writer(); + chunk.u32(2); + chunk.point(CRS_CARTESIAN, 1.5, -2, 0); + chunk.point(CRS_WGS84_3D, 2.3522, 48.8566, 35); + + chunk.u32(2); + chunk.u32(1); + chunk.u32(32); + chunk.u8(TAG_POINT); + chunk.point(CRS_WGS84_3D, 2.3522, 48.8566, 35); + chunk.u32(0); + chunk.u32(0); + + const bytes = concatPackets([frame(MESSAGE_CHUNK_HEADER, header), frame(MESSAGE_CHUNK, chunk), makeEndChunk(2), makeEnd(1.0)]); + const { chunks } = await new TuringClient({ fetch: makeFakeFetch(bytes, {}) }).query("RETURN 1"); + const [at, route] = chunks[0].columns; + + const cartesian = at.get(0); + assert.ok(cartesian instanceof Point); + assert.strictEqual(cartesian.srid, 7203); + assert.strictEqual(cartesian.crs, "cartesian"); + assert.strictEqual(cartesian.x, 1.5); + assert.strictEqual(cartesian.y, -2); + assert.strictEqual(cartesian.z, undefined); + + const paris = at.get(1); + assert.strictEqual(paris.srid, 4979); + assert.strictEqual(paris.crs, "wgs-84-3d"); + assert.deepStrictEqual([paris.x, paris.y, paris.z], [2.3522, 48.8566, 35]); + + const [inList] = route.get(0); + assert.ok(inList instanceof Point); + assert.deepStrictEqual([inList.srid, inList.x, inList.y, inList.z], [4979, 2.3522, 48.8566, 35]); + assert.deepStrictEqual(route.get(1), []); + + const data = await new TuringClient({ fetch: makeFakeFetch(bytes, {}) }).queryData("RETURN 1"); + assert.ok(data[0][0][0] instanceof Point); +} + await testQuery(); await testParameters(); await testRejectsParametersAQueryCannotHold(); @@ -767,5 +826,6 @@ await testMaps(true); await testRaggedQueryData(); await testGraphPathIdsStayExact(); await testNullGraphPath(); +await testPoints(); -console.log("wasm client smoke test passed: query, parameters, buffer growth, constant nulls, optional constant lists, queryData, multi-byte body, error, rejected request, error without packets, maps, ragged, paths, null paths"); +console.log("wasm client smoke test passed: query, parameters, buffer growth, constant nulls, optional constant lists, queryData, multi-byte body, error, rejected request, error without packets, maps, ragged, paths, null paths, points"); diff --git a/wasm/npm/point_column_values_smoke.mjs b/wasm/npm/point_column_values_smoke.mjs new file mode 100644 index 0000000000..1673954cc1 --- /dev/null +++ b/wasm/npm/point_column_values_smoke.mjs @@ -0,0 +1,31 @@ +// Node test for the values of a point column: a point has no typed array, so Column.values is +// null, as for a variable-width kind, and each row reads as a Point through get(). +// Run from wasm/npm/: node point_column_values_smoke.mjs + +import assert from "assert"; + +import { Column, Point } from "./TuringClient.mjs"; + +const TYPE_POINT = 22; +const ENCODING_VECTOR = 0; +const POINT_SIZE = 32; +const CRS_WGS84 = 2; + +const bytes = new Uint8Array(2 * POINT_SIZE); +const view = new DataView(bytes.buffer); +for (let row = 0; row < 2; row++) { + view.setFloat64(row * POINT_SIZE, row + 1, true); + view.setFloat64(row * POINT_SIZE + 8, row + 2, true); + view.setUint8(row * POINT_SIZE + 24, CRS_WGS84); +} + +const column = new Column("at", { typeCode: TYPE_POINT, encoding: ENCODING_VECTOR, count: 2, elementSize: POINT_SIZE, values: bytes }, null, 2); + +assert.strictEqual(column.values, null); + +const second = column.get(1); +assert.ok(second instanceof Point); +assert.deepStrictEqual([second.srid, second.x, second.y, second.z], [4326, 2, 3, undefined]); +assert.deepStrictEqual(column.toArray().map((point) => point.x), [1, 2]); + +console.log("npm client point column values test passed");