From 9a1d0ab5f89fc29fca8d999f8b02abaaa75d135a Mon Sep 17 00:00:00 2001 From: jampukka Date: Sun, 18 Oct 2026 21:06:29 +0300 Subject: [PATCH] Add hakunapi-tiles-generator: seed a GeoPackage tile pyramid and convert it to PMTiles --- CHANGELOG.md | 1 + docs/overview/modules.md | 2 + pom.xml | 5 + .../hakunapi-tiles-generator/LICENSE | 21 + .../hakunapi-tiles-generator/README.md | 130 ++++++ .../hakunapi-tiles-generator/SUBMITTERS | 3 + .../hakunapi-tiles-generator/pom.xml | 81 ++++ .../src/assembly/jar-with-dependencies.xml | 20 + .../tiles/generator/GeneratorBootstrap.java | 152 +++++++ .../tiles/generator/GeneratorControl.java | 180 ++++++++ .../generator/GeneratorServiceConfig.java | 39 ++ .../nls/hakunapi/tiles/generator/MvtTile.java | 52 +++ .../tiles/generator/PMTilesConvert.java | 312 ++++++++++++++ .../tiles/generator/PyramidGenerator.java | 258 ++++++++++++ .../tiles/generator/PyramidTables.java | 114 +++++ .../hakunapi/tiles/generator/TileAddress.java | 10 + .../hakunapi/tiles/generator/TileGzip.java | 31 ++ .../tiles/generator/TilePyramidWriter.java | 127 ++++++ .../hakunapi/tiles/generator/TileRange.java | 40 ++ .../hakunapi/tiles/generator/TileSources.java | 85 ++++ .../tiles/generator/TilesGenerator.java | 244 +++++++++++ .../hakunapi/tiles/generator/ZoomRange.java | 25 ++ .../tiles/generator/GeneratorControlTest.java | 139 ++++++ .../hakunapi/tiles/generator/MvtTileTest.java | 58 +++ .../tiles/generator/PyramidGeneratorTest.java | 309 ++++++++++++++ .../tiles/generator/StubCollection.java | 395 ++++++++++++++++++ .../tiles/generator/TileGzipTest.java | 45 ++ .../tiles/generator/TileRangeTest.java | 45 ++ .../tiles/generator/TileSourcesTest.java | 70 ++++ .../tiles/source/pmtiles/DirectoryWriter.java | 76 ++++ .../tiles/source/pmtiles/HeaderWriter.java | 63 +++ .../tiles/source/pmtiles/PMTilesWriter.java | 278 ++++++++++++ .../source/pmtiles/PMTilesWriterTest.java | 155 +++++++ src-community/pom.xml | 1 + .../main/java/fi/nls/hakunapi/gpkg/GPKG.java | 103 +++++ .../fi/nls/hakunapi/gpkg/GPKGTileMatrix.java | 61 +++ .../fi/nls/hakunapi/gpkg/GPKGTilesTable.java | 61 +++ 37 files changed, 3791 insertions(+) create mode 100644 src-community/hakunapi-tiles-generator/LICENSE create mode 100644 src-community/hakunapi-tiles-generator/README.md create mode 100644 src-community/hakunapi-tiles-generator/SUBMITTERS create mode 100644 src-community/hakunapi-tiles-generator/pom.xml create mode 100644 src-community/hakunapi-tiles-generator/src/assembly/jar-with-dependencies.xml create mode 100644 src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/GeneratorBootstrap.java create mode 100644 src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/GeneratorControl.java create mode 100644 src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/GeneratorServiceConfig.java create mode 100644 src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/MvtTile.java create mode 100644 src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/PMTilesConvert.java create mode 100644 src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/PyramidGenerator.java create mode 100644 src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/PyramidTables.java create mode 100644 src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/TileAddress.java create mode 100644 src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/TileGzip.java create mode 100644 src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/TilePyramidWriter.java create mode 100644 src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/TileRange.java create mode 100644 src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/TileSources.java create mode 100644 src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/TilesGenerator.java create mode 100644 src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/ZoomRange.java create mode 100644 src-community/hakunapi-tiles-generator/src/test/java/fi/nls/hakunapi/tiles/generator/GeneratorControlTest.java create mode 100644 src-community/hakunapi-tiles-generator/src/test/java/fi/nls/hakunapi/tiles/generator/MvtTileTest.java create mode 100644 src-community/hakunapi-tiles-generator/src/test/java/fi/nls/hakunapi/tiles/generator/PyramidGeneratorTest.java create mode 100644 src-community/hakunapi-tiles-generator/src/test/java/fi/nls/hakunapi/tiles/generator/StubCollection.java create mode 100644 src-community/hakunapi-tiles-generator/src/test/java/fi/nls/hakunapi/tiles/generator/TileGzipTest.java create mode 100644 src-community/hakunapi-tiles-generator/src/test/java/fi/nls/hakunapi/tiles/generator/TileRangeTest.java create mode 100644 src-community/hakunapi-tiles-generator/src/test/java/fi/nls/hakunapi/tiles/generator/TileSourcesTest.java create mode 100644 src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/DirectoryWriter.java create mode 100644 src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/HeaderWriter.java create mode 100644 src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/PMTilesWriter.java create mode 100644 src-community/hakunapi-tiles-source-pmtiles/src/test/java/fi/nls/hakunapi/tiles/source/pmtiles/PMTilesWriterTest.java create mode 100644 src/hakunapi-gpkg/src/main/java/fi/nls/hakunapi/gpkg/GPKGTileMatrix.java create mode 100644 src/hakunapi-gpkg/src/main/java/fi/nls/hakunapi/gpkg/GPKGTilesTable.java diff --git a/CHANGELOG.md b/CHANGELOG.md index c64e3386..63d2feed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ - New community module `hakunapi-tiles-source-vectortile`: a `VectorTileSource` that renders Mapbox Vector Tiles in-JVM from the configured feature collections, one layer per collection, with no new third-party dependencies. Not part of the reference webapp. - New community module `hakunapi-tiles-source-gpkg`: a `MapTileSource` and a `VectorTileSource` serving the tiles stored in a GeoPackage tile pyramid, read through `sqlite-jdbc`. Not part of the reference webapp. - New community module `hakunapi-tiles-source-pmtiles`: a `MapTileSource` and a `VectorTileSource` serving the tiles of a PMTiles v3 archive, read in place from a local file or over HTTP range requests. Not part of the reference webapp. +- New community module `hakunapi-tiles-generator`: the seed tool `TilesGenerator` writes a vector tile pyramid into a GeoPackage (`hakunapi-tiles-source-gpkg` can serve it), and `PMTilesConvert` converts such a pyramid into a PMTiles v3 archive. Not part of the reference webapp. ## 2.0.0 diff --git a/docs/overview/modules.md b/docs/overview/modules.md index 411ca1f7..10cb7c36 100644 --- a/docs/overview/modules.md +++ b/docs/overview/modules.md @@ -295,6 +295,8 @@ Key differences between PostGIS and GeoPackage support in Hakunapi: [`hakunapi-tiles-source-pmtiles`](../../src-community/hakunapi-tiles-source-pmtiles/README.md) *(community module)* is a `MapTileSource` and a `VectorTileSource` serving the tiles of a PMTiles v3 archive, read in place from a local file or over HTTP range requests. It is found with `ServiceLoader`, so it is enabled by being on the classpath, and is not bundled in the reference webapp. It depends on `hakunapi-tiles-core` and `slf4j-api`; it adds no new third-party dependency. +[`hakunapi-tiles-generator`](../../src-community/hakunapi-tiles-generator/README.md) *(community module)* is a command-line seed tool, not a server module: `TilesGenerator` writes a vector tile pyramid into a GeoPackage from the feature sources of a service configuration, and `PMTilesConvert` turns such a pyramid into a PMTiles v3 archive. Run `TilesGenerator` with its jar (`java -cp fi.nls.hakunapi.tiles.generator.TilesGenerator `), and `PMTilesConvert` the same way with its main class and ` `. It depends at compile time on `hakunapi-tiles-core`, `hakunapi-tiles-source-vectortile`, `hakunapi-gpkg`, `hakunapi-cql2` and `hakunapi-tiles-source-pmtiles`, and at runtime on `hakunapi-crs-properties`, which is loaded with `ServiceLoader` and not imported. Its only test dependency is `junit`. It is not bundled in the reference webapp. + ## Output data formats (stable) ### GeoJSON diff --git a/pom.xml b/pom.xml index faf439d0..a2bab26d 100644 --- a/pom.xml +++ b/pom.xml @@ -94,6 +94,11 @@ hakunapi-tiles-source-pmtiles ${project.version} + + fi.nls.hakunapi + hakunapi-tiles-generator + ${project.version} + fi.nls.hakunapi hakunapi-config-test diff --git a/src-community/hakunapi-tiles-generator/LICENSE b/src-community/hakunapi-tiles-generator/LICENSE new file mode 100644 index 00000000..0eb4c375 --- /dev/null +++ b/src-community/hakunapi-tiles-generator/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Janne Heikkilä + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/src-community/hakunapi-tiles-generator/README.md b/src-community/hakunapi-tiles-generator/README.md new file mode 100644 index 00000000..2208db06 --- /dev/null +++ b/src-community/hakunapi-tiles-generator/README.md @@ -0,0 +1,130 @@ +# hakunapi-tiles-generator + +Community module: a command-line tool that renders a vector tile pyramid offline +into a [GeoPackage](https://www.geopackage.org/), from the feature sources a +normal hakunapi service configuration describes. + +Reading such a pyramid back is already implemented — `hakunapi-tiles-source-gpkg` +serves a precomputed tile table — so a generated file can be served with no +feature database in the request path. + +Tiles are encoded by the same `VectorTileGenerator` the server renders on-the-fly +tiles with, so this tool is a driver rather than a second rendering +implementation. + +## Usage + +``` +hakunapi-tiles-generator [options] + + --min-zoom coarsest zoom to write (default: the control file's minimum) + --max-zoom finest zoom to write (default: the control file's maximum) + --bbox + limit to this extent, in the tiling scheme's CRS + --threads parallel render workers (default: available processors) + --generator vector tile generator type (default: hakunapi) +``` + +The system property `-Dhakunapi.tiles.block=` sets the side, in tiles, of the square blocks rendered together (default 4; 1 disables blocks). + +The module is not bundled in the reference webapp. Build it and run its +`jar-with-dependencies`, adding the feature source and projection factory your +configuration names: + +``` +mvn -pl src-community/hakunapi-tiles-generator -am install +java -cp target/hakunapi-tiles-generator.jar:.jar:.jar \ + fi.nls.hakunapi.tiles.generator.TilesGenerator service.properties control.properties tiles.gpkg +``` + +## The two configuration files + +The **service properties file** is an ordinary hakunapi configuration and says +what the data *is*: `db.classes`, `collections.*`, and the `tileMatrixSets.*` +block defining the tiling scheme. `tiles.layers.*` is **not** read — a run's +content comes from the control file. + +The **control file** defines the run: + +```properties +tileMatrixSet = ETRS-TM35FIN +ranges = overview, mid, detail + +overview.zoom = 0-7 +overview.collections = maastoalue, tieviiva +overview.maastoalue.filter = kohdeluokka IN (36200, 36313) +overview.tieviiva.filter = kohdeluokka <= 12132 + +mid.zoom = 8-11 +mid.collections = maastoalue, tieviiva, rakennus +mid.tieviiva.filter = kohdeluokka <= 12141 + +detail.zoom = 12-14 +detail.collections = maastoalue, tieviiva, rakennus, korkeuskayra +``` + +| key | meaning | +| --- | --- | +| `tileMatrixSet` | tiling scheme to render into, by id; its definition is read from the service file's `tileMatrixSets.*` block | +| `ranges` | the zoom ranges, by name | +| `.zoom` | inclusive zoom range, `n` or `n-m` | +| `.collections` | collection ids rendered at these zooms, one MVT layer each | +| `..filter` | CQL2 text, parsed with the API's own parser | +| `sources` | output tables, by source id, comma-separated; default: one table per rendered collection | +| `.collections` | collection ids in this source, comma-separated; each rendered collection belongs to exactly one source | + +Ranges must cover the rendered zooms exactly: a gap or an overlap is a +configuration error. The whole file — collection ids and filters included — is +validated before any rendering starts, so a typo cannot fail a long run an hour +in. + +**Sources** group collections into output tables. Without `sources`, each +rendered collection is its own table, named by its id. With `sources`, each +source id is one table, named by that id, and the layers of its collections +share each tile. The control file is rejected if a source is listed twice, has +no collections, lists a collection that another source also lists, names a +collection that no zoom range renders, or leaves a rendered collection in no +source. Source ids are table names as given, so the [Output](#output) rules +apply to them. + +**Why a separate file.** Rendering the same source data across a dozen zoom +levels is not what anyone wants: at small scales you select different +collections, filter by attribute, and read a pre-generalised geometry. The server +has no notion of this and deliberately does not gain one — an offline pyramid is +where "what is drawn at what scale" is decided. Pre-generalised geometry needs no +syntax of its own: configure it as its own collection in the service file, and +name that collection for the coarse zooms. + +## How the pyramid is built + +**Every zoom level is rendered from features.** Deriving a coarse tile by scaling +its four children's MVT geometry would be cheaper but is wrong twice over: +halving the coordinates keeps the finer level's point density instead of +generalising at the coarser resolution, and the children are clipped at seams +that are interior to the parent, so a feature crossing one arrives as two +fragments whose seam stays visible. + +An empty tile writes no row — the GeoPackage tiles model has no empty-tile +marker, and a reader signals "no tile" by a missing row. An empty coarse tile +does **not** prune its subtree: a coarse zoom may render a filtered subset of a +different collection than its children, so "empty here" says nothing about +"empty below". + +Rendering runs on a worker pool; writing is single-threaded, because SQLite +takes one writer at a time. Tiles are written in whichever order they finish +rendering — waiting for the next address in traversal order instead left a slow +tile holding every finished tile behind it, and the workers idle. + +## Output + +A GeoPackage may hold several pyramids, so the rules are per *table*: + +| output file | table | behaviour | +| --- | --- | --- | +| absent | — | create file and table | +| present | absent | create the table in it | +| present | present | error: refuses to overwrite | + +A run that is interrupted leaves whatever it had committed. That is a partial +pyramid with holes in it, not a prefix of the traversal, so it cannot be +continued — rerun into a fresh output. diff --git a/src-community/hakunapi-tiles-generator/SUBMITTERS b/src-community/hakunapi-tiles-generator/SUBMITTERS new file mode 100644 index 00000000..46d30fd3 --- /dev/null +++ b/src-community/hakunapi-tiles-generator/SUBMITTERS @@ -0,0 +1,3 @@ +Submitted and maintained by: + +jampukka diff --git a/src-community/hakunapi-tiles-generator/pom.xml b/src-community/hakunapi-tiles-generator/pom.xml new file mode 100644 index 00000000..a27d7082 --- /dev/null +++ b/src-community/hakunapi-tiles-generator/pom.xml @@ -0,0 +1,81 @@ + + 4.0.0 + + fi.nls.hakunapi + src-community + 2.1.0-SNAPSHOT + + hakunapi-tiles-generator + + + fi.nls.hakunapi + hakunapi-core + + + fi.nls.hakunapi + hakunapi-tiles-core + + + fi.nls.hakunapi + hakunapi-tiles-source-vectortile + + + fi.nls.hakunapi + hakunapi-gpkg + + + + fi.nls.hakunapi + hakunapi-tiles-source-pmtiles + + + fi.nls.hakunapi + hakunapi-cql2 + + + + fi.nls.hakunapi + hakunapi-crs-properties + runtime + + + org.slf4j + slf4j-api + + + junit + junit + test + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 3.3.0 + + + package + + single + + + hakunapi-tiles-generator + + + fi.nls.hakunapi.tiles.generator.TilesGenerator + + + + src/assembly/jar-with-dependencies.xml + + false + + + + + + + diff --git a/src-community/hakunapi-tiles-generator/src/assembly/jar-with-dependencies.xml b/src-community/hakunapi-tiles-generator/src/assembly/jar-with-dependencies.xml new file mode 100644 index 00000000..7ee0a5e3 --- /dev/null +++ b/src-community/hakunapi-tiles-generator/src/assembly/jar-with-dependencies.xml @@ -0,0 +1,20 @@ + + jar-with-dependencies + + jar + + false + + + metaInf-services + + + + + / + true + true + runtime + + + diff --git a/src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/GeneratorBootstrap.java b/src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/GeneratorBootstrap.java new file mode 100644 index 00000000..0709426c --- /dev/null +++ b/src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/GeneratorBootstrap.java @@ -0,0 +1,152 @@ +package fi.nls.hakunapi.tiles.generator; + +import java.io.BufferedReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import fi.nls.hakunapi.core.FeatureType; +import fi.nls.hakunapi.core.SRIDCode; +import fi.nls.hakunapi.core.SimpleFeatureType; +import fi.nls.hakunapi.core.SimpleSource; +import fi.nls.hakunapi.core.config.HakunaConfigParser; +import fi.nls.hakunapi.core.CRSRegistryProvider; +import fi.nls.hakunapi.tiles.TileMatrixSetRegistry; +import fi.nls.hakunapi.tiles.config.TilesConfigParser; + +public class GeneratorBootstrap implements AutoCloseable { + + private static final Logger LOG = LoggerFactory.getLogger(GeneratorBootstrap.class); + + private static final String[] DEFAULT_SOURCE_CLASSES = { + "fi.nls.hakunapi.simple.postgis.PostGISSimpleSource" + }; + + private final GeneratorServiceConfig service; + private final TileMatrixSetRegistry tileMatrixSets; + private final List sources; + private final HakunaConfigParser parser; + private final Path configPath; + private final Map sourcesByType; + + private GeneratorBootstrap(GeneratorServiceConfig service, TileMatrixSetRegistry tileMatrixSets, + List sources, + HakunaConfigParser parser, Path configPath, Map sourcesByType) { + this.service = service; + this.tileMatrixSets = tileMatrixSets; + this.sources = sources; + this.parser = parser; + this.configPath = configPath; + this.sourcesByType = sourcesByType; + } + + public static GeneratorBootstrap read(Path configPath) throws Exception { + Properties properties = new Properties(); + try (BufferedReader r = Files.newBufferedReader(configPath, StandardCharsets.UTF_8)) { + properties.load(r); + } + HakunaConfigParser parser = new HakunaConfigParser(properties); + + List sources = createSources(parser); + Map sourcesByType = new HashMap<>(); + for (int i = 0; i < sources.size(); i++) { + SimpleSource source = sources.get(i); + sourcesByType.put(source.getType(), source); + } + + List knownSrids = CRSRegistryProvider.getCRSRegistry() + .map(reg -> reg.detect(properties, parser.getKnownSrids())) + .orElseThrow(() -> new IllegalStateException("Could not find any CRS registry!")); + + Map collections = new LinkedHashMap<>(); + try { + for (String collectionId : parser.readCollectionIds()) { + collections.put(collectionId, parser.readCollection(configPath, sourcesByType, collectionId)); + } + } catch (Exception e) { + closeSources(sources); + throw e; + } + if (collections.isEmpty()) { + closeSources(sources); + throw new IllegalArgumentException("Missing required config property 'collections'"); + } + LOG.info("Read {} collections from {}", collections.size(), configPath); + + GeneratorServiceConfig service = new GeneratorServiceConfig(collections); + service.setKnownSrids(knownSrids); + + TileMatrixSetRegistry tileMatrixSets = new TilesConfigParser(parser).readTileMatrixSets(); + + return new GeneratorBootstrap(service, tileMatrixSets, sources, + parser, configPath, sourcesByType); + } + + private static List createSources(HakunaConfigParser cfg) { + String[] classNames = cfg.getMultiple("db.classes", DEFAULT_SOURCE_CLASSES); + List sources = new ArrayList<>(classNames.length); + for (String className : classNames) { + try { + sources.add((SimpleSource) Class.forName(className).getDeclaredConstructor().newInstance()); + } catch (ReflectiveOperationException e) { + throw new IllegalArgumentException("db.classes names source '" + className + + "', which could not be instantiated: " + e); + } + } + if (sources.isEmpty()) { + throw new IllegalArgumentException("db.classes names no feature sources"); + } + LOG.info("Sources: {}", classNames.length == 0 ? "(none)" : String.join(", ", classNames)); + return sources; + } + + /** Re-reads the config: FeatureType has no copy operation and sources cast it to their concrete type. */ + public SimpleFeatureType readCollection(String collectionId) throws Exception { + FeatureType ft = parser.readCollection(configPath, sourcesByType, collectionId); + if (!(ft instanceof SimpleFeatureType sft)) { + throw new IllegalStateException("Collection " + collectionId + " is a " + + ft.getClass().getName() + ", which cannot carry a zoom range filter"); + } + return sft; + } + + public GeneratorServiceConfig getService() { + return service; + } + + public TileMatrixSetRegistry getTileMatrixSets() { + return tileMatrixSets; + } + + public SRIDCode getSridCode(int srid) { + return service.getSridCode(srid) + .orElseThrow(() -> new IllegalArgumentException("Unknown SRID " + srid + + "; add it to the service configuration's srid list")); + } + + @Override + public void close() { + closeSources(sources); + } + + private static void closeSources(List sources) { + for (int i = 0; i < sources.size(); i++) { + SimpleSource source = sources.get(i); + try { + source.close(); + } catch (Exception e) { + LOG.warn("Failed to close source {}", source.getType(), e); + } + } + } + +} diff --git a/src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/GeneratorControl.java b/src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/GeneratorControl.java new file mode 100644 index 00000000..9b98b5ed --- /dev/null +++ b/src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/GeneratorControl.java @@ -0,0 +1,180 @@ +package fi.nls.hakunapi.tiles.generator; + +import java.io.BufferedReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import fi.nls.hakunapi.core.FeatureServiceConfig; +import fi.nls.hakunapi.core.FeatureType; +import fi.nls.hakunapi.core.FilterParser; +import fi.nls.hakunapi.core.SRIDCode; +import fi.nls.hakunapi.core.config.HakunaConfigParser; +import fi.nls.hakunapi.core.filter.Filter; + +public class GeneratorControl { + + public static final String PROP_TILE_MATRIX_SET = "tileMatrixSet"; + public static final String PROP_RANGES = "ranges"; + + private final String tileMatrixSetId; + private final List ranges; + + public GeneratorControl(String tileMatrixSetId, List ranges) { + this.tileMatrixSetId = tileMatrixSetId; + this.ranges = List.copyOf(ranges); + } + + public static HakunaConfigParser load(Path path) throws Exception { + Properties properties = new Properties(); + try (BufferedReader r = Files.newBufferedReader(path, StandardCharsets.UTF_8)) { + properties.load(r); + } + return new HakunaConfigParser(properties); + } + + public static String readTileMatrixSetId(HakunaConfigParser cfg) { + return cfg.getRequired(PROP_TILE_MATRIX_SET); + } + + public static GeneratorControl readRanges(HakunaConfigParser cfg, FeatureServiceConfig service, + List filters, SRIDCode srid) { + String tileMatrixSetId = readTileMatrixSetId(cfg); + + String[] rangeNames = cfg.getMultiple(PROP_RANGES); + if (rangeNames.length == 0) { + throw new IllegalArgumentException("control file: " + PROP_RANGES + " must name at least one zoom range"); + } + + List ranges = new ArrayList<>(rangeNames.length); + for (String name : rangeNames) { + ranges.add(readRange(cfg, name, service, filters, srid)); + } + ranges.sort((a, b) -> Integer.compare(a.minZoom(), b.minZoom())); + validateContiguous(ranges); + + return new GeneratorControl(tileMatrixSetId, ranges); + } + + private static ZoomRange readRange(HakunaConfigParser cfg, String name, FeatureServiceConfig service, + List filters, SRIDCode srid) { + String p = name + "."; + int[] zoom = parseZoomRange(name, cfg.getRequired(p + "zoom")); + + String[] collectionIds = cfg.getMultiple(p + "collections"); + if (collectionIds.length == 0) { + throw new IllegalArgumentException("control file: range " + name + + " must name at least one collection in " + p + "collections"); + } + + Map rangeFilters = new LinkedHashMap<>(); + for (String collectionId : collectionIds) { + FeatureType ft = service.getCollection(collectionId); + if (ft == null) { + throw new IllegalArgumentException("control file: range " + name + + " names unknown collection '" + collectionId + "'"); + } + String cql2 = cfg.get(p + collectionId + ".filter"); + if (cql2 != null && !cql2.isBlank()) { + rangeFilters.put(collectionId, parseFilter(name, collectionId, cql2, ft, filters, srid)); + } + } + + return new ZoomRange(name, zoom[0], zoom[1], List.of(collectionIds), rangeFilters); + } + + private static Filter parseFilter(String rangeName, String collectionId, String cql2, FeatureType ft, + List filters, SRIDCode srid) { + if (filters.isEmpty()) { + throw new IllegalStateException("No filter parser available to parse range filters"); + } + FilterParser parser = filters.get(0); + try { + return parser.parse(ft, cql2, srid); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("control file: range " + rangeName + " filter for collection '" + + collectionId + "' is not valid " + parser.getCode() + ": " + e.getMessage()); + } + } + + private static int[] parseZoomRange(String name, String value) { + String s = value.trim(); + int dash = s.indexOf('-'); + try { + if (dash < 0) { + int z = Integer.parseInt(s); + return new int[] { z, z }; + } + int min = Integer.parseInt(s.substring(0, dash).trim()); + int max = Integer.parseInt(s.substring(dash + 1).trim()); + if (min > max) { + throw new IllegalArgumentException("control file: range " + name + " zoom '" + value + + "' is inverted"); + } + return new int[] { min, max }; + } catch (NumberFormatException e) { + throw new IllegalArgumentException("control file: range " + name + " zoom '" + value + + "' is not a zoom level or an n-m range"); + } + } + + private static void validateContiguous(List ranges) { + for (int i = 1; i < ranges.size(); i++) { + ZoomRange prev = ranges.get(i - 1); + ZoomRange next = ranges.get(i); + if (next.minZoom() <= prev.maxZoom()) { + throw new IllegalArgumentException("control file: ranges " + prev.name() + " (" + prev.minZoom() + + "-" + prev.maxZoom() + ") and " + next.name() + " (" + next.minZoom() + "-" + + next.maxZoom() + ") overlap"); + } + if (next.minZoom() != prev.maxZoom() + 1) { + throw new IllegalArgumentException("control file: no range covers zoom " + + (prev.maxZoom() + 1) + " between ranges " + prev.name() + " and " + next.name()); + } + } + } + + public String getTileMatrixSetId() { + return tileMatrixSetId; + } + + public List getRanges() { + return ranges; + } + + public List getAllCollectionIds() { + List all = new ArrayList<>(); + for (int i = 0; i < ranges.size(); i++) { + for (String id : ranges.get(i).collectionIds()) { + if (!all.contains(id)) { + all.add(id); + } + } + } + return all; + } + + public int getMinZoom() { + return ranges.get(0).minZoom(); + } + + public int getMaxZoom() { + return ranges.get(ranges.size() - 1).maxZoom(); + } + + public ZoomRange getRange(int zoom) { + for (int i = 0; i < ranges.size(); i++) { + ZoomRange range = ranges.get(i); + if (range.covers(zoom)) { + return range; + } + } + throw new IllegalArgumentException("No zoom range covers zoom level " + zoom); + } + +} diff --git a/src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/GeneratorServiceConfig.java b/src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/GeneratorServiceConfig.java new file mode 100644 index 00000000..0997fc7d --- /dev/null +++ b/src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/GeneratorServiceConfig.java @@ -0,0 +1,39 @@ +package fi.nls.hakunapi.tiles.generator; + +import java.util.Collection; +import java.util.Collections; +import java.util.Map; + +import fi.nls.hakunapi.core.FeatureServiceConfig; +import fi.nls.hakunapi.core.FeatureType; +import fi.nls.hakunapi.core.OutputFormat; + +public class GeneratorServiceConfig extends FeatureServiceConfig { + + private final Map collections; + + public GeneratorServiceConfig(Map collections) { + this.collections = collections; + } + + @Override + public Collection getCollections() { + return collections.values(); + } + + @Override + public FeatureType getCollection(String name) { + return collections.get(name); + } + + @Override + public Collection getOutputFormats() { + return Collections.emptyList(); + } + + @Override + public OutputFormat getOutputFormat(String f) { + return null; + } + +} diff --git a/src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/MvtTile.java b/src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/MvtTile.java new file mode 100644 index 00000000..79becd84 --- /dev/null +++ b/src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/MvtTile.java @@ -0,0 +1,52 @@ +package fi.nls.hakunapi.tiles.generator; + +import fi.nls.hakunapi.tiles.mvt.Protobuf; + +public final class MvtTile { + + private static final int TILE_LAYERS = 3; + private static final int LAYER_FEATURES = 2; + + private MvtTile() {} + + public static boolean hasFeatures(byte[] tile) { + if (tile == null) { + return false; + } + int pos = 0; + int end = tile.length; + while (pos < end) { + long t = Protobuf.varint(tile, pos); + int tag = Protobuf.value(t); + pos = Protobuf.end(t); + int wire = Protobuf.wire(tag); + if (Protobuf.field(tag) == TILE_LAYERS && wire == Protobuf.WIRE_LEN) { + long l = Protobuf.varint(tile, pos); + int len = Protobuf.value(l); + pos = Protobuf.end(l); + if (layerHasFeatures(tile, pos, pos + len)) { + return true; + } + pos += len; + } else { + pos = Protobuf.skip(tile, pos, wire); + } + } + return false; + } + + private static boolean layerHasFeatures(byte[] tile, int pos, int end) { + while (pos < end) { + long t = Protobuf.varint(tile, pos); + int tag = Protobuf.value(t); + pos = Protobuf.end(t); + int wire = Protobuf.wire(tag); + if (Protobuf.field(tag) == LAYER_FEATURES && wire == Protobuf.WIRE_LEN) { + return true; + } + pos = Protobuf.skip(tile, pos, wire); + } + return false; + } + +} diff --git a/src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/PMTilesConvert.java b/src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/PMTilesConvert.java new file mode 100644 index 00000000..32881050 --- /dev/null +++ b/src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/PMTilesConvert.java @@ -0,0 +1,312 @@ +package fi.nls.hakunapi.tiles.generator; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import java.util.zip.GZIPInputStream; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import fi.nls.hakunapi.gpkg.GPKG; +import fi.nls.hakunapi.tiles.mvt.Protobuf; +import fi.nls.hakunapi.tiles.source.pmtiles.Compression; +import fi.nls.hakunapi.tiles.source.pmtiles.PMTilesWriter; +import fi.nls.hakunapi.tiles.source.pmtiles.TileId; + +public class PMTilesConvert { + + private static final Logger LOG = LoggerFactory.getLogger(PMTilesConvert.class); + + private static final byte TILE_TYPE_MVT = 1; + private static final int LAYER_SAMPLE_PER_ZOOM = 8; + private static final int TILE_LAYERS = 3; + private static final int LAYER_NAME = 1; + + public static void main(String[] args) { + try { + System.exit(run(args)); + } catch (Exception e) { + System.err.println("ERROR: " + e.getMessage()); + LOG.debug("Conversion failed", e); + System.exit(1); + } + } + + private static int run(String[] args) throws Exception { + Args parsed = Args.parse(args); + if (parsed == null) { + usage(); + return 2; + } + if (!Files.exists(parsed.pyramidPath)) { + throw new IllegalArgumentException("No such pyramid: " + parsed.pyramidPath); + } + + long started = System.currentTimeMillis(); + long written; + try (Connection c = open(parsed.pyramidPath)) { + checkSquareMatrices(c, parsed.table); + List addresses = readAddresses(c, parsed.table, parsed.flip); + if (addresses.isEmpty()) { + throw new IllegalArgumentException( + "Table " + parsed.table + " holds no tiles in " + parsed.pyramidPath); + } + LOG.info("Converting {} tiles from {} table {}", addresses.size(), + parsed.pyramidPath, parsed.table); + if (parsed.centerZoom < 0) { + parsed.centerZoom = (int) addresses.get(0)[1]; + } + written = convert(c, parsed, addresses); + } + long seconds = (System.currentTimeMillis() - started) / 1000; + System.out.printf("Wrote %s (%,d bytes) in %d s%n", + parsed.outputPath, written, seconds); + return 0; + } + + /** The PMTiles tile id is a Hilbert index over a 2^z square grid. */ + private static void checkSquareMatrices(Connection c, String table) throws SQLException { + String sql = "SELECT m.zoom_level, m.matrix_width, m.matrix_height FROM gpkg_tile_matrix m" + + " WHERE m.table_name = ? AND EXISTS (SELECT 1 FROM " + GPKG.quote(table) + + " t WHERE t.zoom_level = m.zoom_level)"; + try (PreparedStatement ps = c.prepareStatement(sql)) { + ps.setString(1, table); + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + int zoom = rs.getInt(1); + long width = rs.getLong(2); + long height = rs.getLong(3); + long dim = 1L << zoom; + if (width != dim || height != dim) { + throw new IllegalArgumentException("Zoom " + zoom + " of table " + table + + " has a " + width + "x" + height + " tile matrix, but PMTiles needs " + + dim + "x" + dim + " (2^zoom)"); + } + } + } + } + } + + /** Sorted by Hilbert tile id, which the PMTiles writer requires and SQLite cannot order by. */ + private static List readAddresses(Connection c, String table, boolean flip) + throws SQLException { + List addresses = new ArrayList<>(); + String sql = "SELECT zoom_level, tile_column, tile_row FROM " + GPKG.quote(table); + try (Statement st = c.createStatement(); ResultSet rs = st.executeQuery(sql)) { + while (rs.next()) { + int zoom = rs.getInt(1); + long col = rs.getLong(2); + long row = rs.getLong(3); + long y = flip ? (1L << zoom) - 1 - row : row; + addresses.add(new long[] { TileId.fromZXY(zoom, col, y), zoom, col, row }); + } + } + addresses.sort((a, b) -> Long.compare(a[0], b[0])); + return addresses; + } + + private static long convert(Connection c, Args parsed, List addresses) throws Exception { + String sql = "SELECT tile_data FROM " + GPKG.quote(parsed.table) + + " WHERE zoom_level = ? AND tile_column = ? AND tile_row = ?"; + try (PMTilesWriter writer = new PMTilesWriter(parsed.outputPath, + parsed.tileCompression, TILE_TYPE_MVT); + PreparedStatement ps = c.prepareStatement(sql)) { + Set layers = new LinkedHashSet<>(); + int[] sampled = new int[32]; + for (int i = 0; i < addresses.size(); i++) { + long[] a = addresses.get(i); + int zoom = (int) a[1]; + long col = a[2]; + long row = a[3]; + ps.setInt(1, zoom); + ps.setLong(2, col); + ps.setLong(3, row); + byte[] tile; + try (ResultSet rs = ps.executeQuery()) { + if (!rs.next()) { + throw new IllegalStateException("Tile vanished: " + zoom + "/" + row + "/" + col); + } + tile = rs.getBytes(1); + } + long y = parsed.flip ? (1L << zoom) - 1 - row : row; + if (tile != null && sampled[zoom]++ < LAYER_SAMPLE_PER_ZOOM) { + collectLayerNames(parsed.tileCompression == Compression.GZIP ? gunzip(tile) : tile, layers); + } + writer.add(zoom, col, y, tile); + if ((i + 1) % 100_000 == 0) { + LOG.info("{} / {} tiles", i + 1, addresses.size()); + } + } + return writer.finish(metadata(parsed.table, layers), parsed.bboxE7, parsed.centerE7, parsed.centerZoom); + } + } + + private static byte[] gunzip(byte[] bytes) throws IOException { + try (GZIPInputStream in = new GZIPInputStream(new ByteArrayInputStream(bytes))) { + return in.readAllBytes(); + } + } + + private static void collectLayerNames(byte[] tile, Set names) { + int pos = 0; + while (pos < tile.length) { + long t = Protobuf.varint(tile, pos); + int tag = Protobuf.value(t); + int afterTag = Protobuf.end(t); + int wire = Protobuf.wire(tag); + if (Protobuf.field(tag) == TILE_LAYERS && wire == Protobuf.WIRE_LEN) { + long l = Protobuf.varint(tile, afterTag); + int body = Protobuf.end(l); + int end = body + Protobuf.value(l); + for (int p = body; p < end;) { + long lt = Protobuf.varint(tile, p); + int ltag = Protobuf.value(lt); + int after = Protobuf.end(lt); + int lwire = Protobuf.wire(ltag); + if (Protobuf.field(ltag) == LAYER_NAME && lwire == Protobuf.WIRE_LEN) { + long nl = Protobuf.varint(tile, after); + names.add(new String(tile, Protobuf.end(nl), Protobuf.value(nl), StandardCharsets.UTF_8)); + break; + } + p = Protobuf.skip(tile, after, lwire); + } + pos = end; + } else { + pos = Protobuf.skip(tile, afterTag, wire); + } + } + } + + private static String jsonString(String s) { + StringBuilder sb = new StringBuilder("\""); + for (int i = 0; i < s.length(); i++) { + char ch = s.charAt(i); + if (ch == '"' || ch == '\\') { + sb.append('\\').append(ch); + } else if (ch < 0x20) { + sb.append(String.format("\\u%04x", (int) ch)); + } else { + sb.append(ch); + } + } + return sb.append('"').toString(); + } + + /** Spec: MVT archives MUST carry vector_layers. Third-party viewers assume WebMercatorQuad; the note names the actual tiling scheme. */ + private static String metadata(String table, Set layers) { + if (layers.isEmpty()) { + layers.add(table); + } + StringBuilder vectorLayers = new StringBuilder("["); + for (String layer : layers) { + vectorLayers.append(vectorLayers.length() > 1 ? "," : "") + .append("{\"id\":").append(jsonString(layer)).append('}'); + } + vectorLayers.append(']'); + return "{\"name\":" + jsonString(table) + "," + + "\"format\":\"pbf\"," + + "\"vector_layers\":" + vectorLayers + "," + + "\"generator\":\"hakunapi-tiles-generator\"," + + "\"hakunapi:note\":\"Tiles are addressed on the hakunapi tile matrix set they " + + "were generated for, which need not be WebMercatorQuad. The PMTiles tile id is a " + + "Hilbert index over a square 2^z grid and carries no projection; a viewer that " + + "assumes WebMercatorQuad will place these tiles incorrectly.\"}"; + } + + private static Connection open(Path path) throws Exception { + return DriverManager.getConnection("jdbc:sqlite:" + path.toAbsolutePath() + "?open_mode=1"); + } + + private static void usage() { + System.err.println("Usage: PMTilesConvert [options]
"); + System.err.println(); + System.err.println(" --raw the pyramid's tiles are uncompressed"); + System.err.println(" (default: gzip, which is what the generator stores)"); + System.err.println(" --flip the pyramid counts tile_row from the bottom (TMS)"); + System.err.println(" --bbox "); + System.err.println(" WGS84 extent recorded in the header (default: Finland, 19,59.4,31.8,70.1)"); + System.err.println(" --center-zoom zoom a viewer should open at (default: the lowest zoom in the pyramid)"); + } + + private static class Args { + + Path pyramidPath; + String table; + Path outputPath; + byte tileCompression = Compression.GZIP; + boolean flip; + int[] bboxE7 = { 190000000, 594000000, 318000000, 701000000 }; + int[] centerE7 = { 250000000, 645000000 }; + int centerZoom = -1; + + static Args parse(String[] args) { + Args a = new Args(); + List positional = new ArrayList<>(3); + for (int i = 0; i < args.length; i++) { + String arg = args[i]; + if (("--bbox".equals(arg) || "--center-zoom".equals(arg)) && i + 1 >= args.length) { + System.err.println("Option " + arg + " needs a value"); + return null; + } + switch (arg) { + case "--raw": + a.tileCompression = Compression.NONE; + break; + case "--flip": + a.flip = true; + break; + case "--bbox": + a.bboxE7 = parseBboxE7(args[++i]); + a.centerE7 = new int[] { + (int) (((long) a.bboxE7[0] + a.bboxE7[2]) / 2), + (int) (((long) a.bboxE7[1] + a.bboxE7[3]) / 2) }; + break; + case "--center-zoom": + a.centerZoom = Integer.parseInt(args[++i]); + break; + default: + if (arg.startsWith("--")) { + System.err.println("Unknown option: " + arg); + return null; + } + positional.add(arg); + } + } + if (positional.size() != 3) { + return null; + } + a.pyramidPath = Paths.get(positional.get(0)).toAbsolutePath(); + a.table = positional.get(1); + a.outputPath = Paths.get(positional.get(2)).toAbsolutePath(); + return a; + } + + private static int[] parseBboxE7(String csv) { + String[] parts = csv.split(","); + if (parts.length != 4) { + throw new IllegalArgumentException("--bbox must be minLon,minLat,maxLon,maxLat"); + } + int[] bbox = new int[4]; + for (int i = 0; i < 4; i++) { + bbox[i] = (int) Math.round(Double.parseDouble(parts[i].trim()) * 1e7); + } + return bbox; + } + + } + +} diff --git a/src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/PyramidGenerator.java b/src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/PyramidGenerator.java new file mode 100644 index 00000000..504b78b0 --- /dev/null +++ b/src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/PyramidGenerator.java @@ -0,0 +1,258 @@ +package fi.nls.hakunapi.tiles.generator; + +import java.io.ByteArrayOutputStream; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.CompletionService; +import java.util.concurrent.ExecutorCompletionService; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import fi.nls.hakunapi.core.FeatureType; +import fi.nls.hakunapi.core.SimpleFeatureType; +import fi.nls.hakunapi.core.filter.Filter; +import fi.nls.hakunapi.tiles.TileBody; +import fi.nls.hakunapi.tiles.TileMatrix; +import fi.nls.hakunapi.tiles.TileMatrixSet; +import fi.nls.hakunapi.tiles.source.vectortile.JavaVectorTileGenerator; +import fi.nls.hakunapi.tiles.source.vectortile.TileContext; +import fi.nls.hakunapi.tiles.source.vectortile.VectorTileGenerator; +import fi.nls.hakunapi.tiles.source.vectortile.VectorTileGeneratorSource; + +public class PyramidGenerator { + + private static final Logger LOG = LoggerFactory.getLogger(PyramidGenerator.class); + + private static final int QUEUE_DEPTH_PER_WORKER = 4; + + private static final int BLOCK = Integer.getInteger("hakunapi.tiles.block", 4); + + private boolean useBlocks() { + return BLOCK > 1 && generator.supportsBlocks(); + } + + private final GeneratorControl control; + private final TileSources sources; + private final TileMatrixSet tms; + private final VectorTileGenerator generator; + private final String mediaType; + + private final Map> collectionsByRange; + + public PyramidGenerator(GeneratorControl control, TileSources sources, TileMatrixSet tms, + VectorTileGenerator generator, String mediaType, + Map> collectionsByRange) { + this.control = control; + this.sources = sources; + this.tms = tms; + this.generator = generator; + this.mediaType = mediaType; + this.collectionsByRange = collectionsByRange; + } + + public static Map> resolveCollections(GeneratorBootstrap bootstrap, + GeneratorControl control) throws Exception { + Map> byRange = new LinkedHashMap<>(); + for (ZoomRange range : control.getRanges()) { + List ids = range.collectionIds(); + List collections = new ArrayList<>(ids.size()); + for (int i = 0; i < ids.size(); i++) { + String id = ids.get(i); + Filter rangeFilter = range.filters().get(id); + if (rangeFilter == null) { + collections.add(bootstrap.getService().getCollection(id)); + } else { + SimpleFeatureType ft = bootstrap.readCollection(id); + List filters = new ArrayList<>(ft.getStaticFilters()); + filters.add(rangeFilter); + ft.setStaticFilters(filters); + collections.add(ft); + } + } + byRange.put(range.name(), collections); + } + return byRange; + } + + public long generate(PyramidTables tables, int minZoom, int maxZoom, + double minX, double minY, double maxX, double maxY, + int threads) throws Exception { + List ranges = new ArrayList<>(maxZoom - minZoom + 1); + long total = 0; + for (int zoom = minZoom; zoom <= maxZoom; zoom++) { + TileRange range = TileRange.of(tms.getTileMatrices().get(zoom), zoom, minX, minY, maxX, maxY); + ranges.add(range); + total += range.size(); + } + LOG.info("Generating {} tiles across zoom levels {}-{} with {} threads", total, minZoom, maxZoom, threads); + + ExecutorService pool = Executors.newFixedThreadPool(threads); + try { + return run(tables, ranges, pool, threads); + } finally { + pool.shutdownNow(); + pool.awaitTermination(30, TimeUnit.SECONDS); + } + } + + /** Writes in completion order: a GeoPackage tile table is addressed by key, not insertion order. */ + private long run(PyramidTables tables, List ranges, + ExecutorService pool, int threads) throws Exception { + int depth = Math.max(1, threads * QUEUE_DEPTH_PER_WORKER); + CompletionService rendered = new ExecutorCompletionService<>(pool); + int inFlight = 0; + long empty = 0; + + try { + for (int r = 0; r < ranges.size(); r++) { + TileRange range = ranges.get(r); + if (range.isEmpty()) { + continue; + } + int side = useBlocks() ? BLOCK : 1; + for (long row = range.minRow(); row <= range.maxRow(); row += side) { + long rowEnd = Math.min(row + side - 1, range.maxRow()); + for (long col = range.minCol(); col <= range.maxCol(); col += side) { + long colEnd = Math.min(col + side - 1, range.maxCol()); + rendered.submit(renderBlockTask(range.zoom(), row, rowEnd, col, colEnd)); + inFlight++; + if (inFlight >= depth) { + empty += write(tables, rendered.take().get()); + inFlight--; + } + } + } + } + while (inFlight > 0) { + empty += write(tables, rendered.take().get()); + inFlight--; + } + } catch (Exception e) { + LOG.warn("Generation failed: the tile tables {} are incomplete and were kept; they must be removed before a rerun", + tables.getWrittenBySource().keySet()); + salvage(tables, pool, e); + throw e; + } + tables.finish(); + + LOG.info("Wrote {} tiles across {} table(s), {} addresses were empty everywhere", + tables.getWritten(), tables.getWrittenBySource().size(), empty); + return tables.getWritten(); + } + + /** A flush failure must not mask the render failure. */ + private static void salvage(PyramidTables tables, ExecutorService pool, Exception cause) { + try { + tables.flush(); + } catch (Exception e) { + cause.addSuppressed(e); + } + pool.shutdownNow(); + } + + private record Rendered(List addresses, List> tiles) { + } + + /** Returns how many of the block's addresses wrote no row. */ + private long write(PyramidTables tables, Rendered r) throws SQLException { + long empty = 0; + for (int i = 0; i < r.addresses().size(); i++) { + Map tiles = r.tiles().get(i); + if (tiles.isEmpty()) { + empty++; + continue; + } + for (Map.Entry e : tiles.entrySet()) { + tables.write(e.getKey(), r.addresses().get(i), e.getValue()); + } + } + return empty; + } + + private List renderPerTile(List ctxs, FeatureType ft) throws Exception { + List layers = new ArrayList<>(ctxs.size()); + for (int t = 0; t < ctxs.size(); t++) { + TileBody body = generator.generate(ctxs.get(t), List.of(ft), + mediaType, Map.of()).orElse(null); + if (body == null) { + layers.add(new byte[0]); + continue; + } + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + body.writeTo(baos); + layers.add(baos.toByteArray()); + } + return layers; + } + + /** Empty tiles are absent from the result: GeoPackage has no empty-tile marker. */ + private Callable renderBlockTask(int zoom, long minRow, long maxRow, + long minCol, long maxCol) { + return () -> { + TileMatrix tm = tms.getTileMatrices().get(zoom); + ZoomRange range = control.getRange(zoom); + List collections = collectionsByRange.get(range.name()); + List ids = range.collectionIds(); + + List addresses = new ArrayList<>(); + List ctxs = new ArrayList<>(); + for (long row = minRow; row <= maxRow; row++) { + for (long col = minCol; col <= maxCol; col++) { + addresses.add(new TileAddress(zoom, row, col)); + ctxs.add(VectorTileGeneratorSource.toTileContext(tms, tm, row, col)); + } + } + + // Union of the tiles' clip rects, buffer included; partial edge blocks have no corner order. + double[] queryBox = { Double.MAX_VALUE, Double.MAX_VALUE, + -Double.MAX_VALUE, -Double.MAX_VALUE }; + for (int t = 0; t < ctxs.size(); t++) { + TileContext ctx = ctxs.get(t); + double margin = JavaVectorTileGenerator.clipMargin(ctx); + queryBox[0] = Math.min(queryBox[0], ctx.minX() - margin); + queryBox[1] = Math.min(queryBox[1], ctx.minY() - margin); + queryBox[2] = Math.max(queryBox[2], ctx.maxX() + margin); + queryBox[3] = Math.max(queryBox[3], ctx.maxY() + margin); + } + + List> bySource = new ArrayList<>(ctxs.size()); + for (int t = 0; t < ctxs.size(); t++) { + bySource.add(new LinkedHashMap<>()); + } + for (int i = 0; i < collections.size(); i++) { + List layers = useBlocks() + ? generator.generateBlock(ctxs, List.of(collections.get(i)), queryBox) + : renderPerTile(ctxs, collections.get(i)); + String source = sources.getSource(ids.get(i)); + for (int t = 0; t < layers.size(); t++) { + byte[] layer = layers.get(t); + if (layer.length == 0 || !MvtTile.hasFeatures(layer)) { + continue; + } + bySource.get(t).computeIfAbsent(source, __ -> new ByteArrayOutputStream()) + .write(layer); + } + } + + List> tiles = new ArrayList<>(ctxs.size()); + for (int t = 0; t < ctxs.size(); t++) { + Map src = bySource.get(t); + Map out = new LinkedHashMap<>(src.size()); + for (Map.Entry e : src.entrySet()) { + out.put(e.getKey(), TileGzip.compress(e.getValue().toByteArray())); + } + tiles.add(out); + } + return new Rendered(addresses, tiles); + }; + } + +} diff --git a/src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/PyramidTables.java b/src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/PyramidTables.java new file mode 100644 index 00000000..988eeca3 --- /dev/null +++ b/src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/PyramidTables.java @@ -0,0 +1,114 @@ +package fi.nls.hakunapi.tiles.generator; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import fi.nls.hakunapi.tiles.TileMatrixSet; + +public class PyramidTables implements AutoCloseable { + + private static final Logger LOG = LoggerFactory.getLogger(PyramidTables.class); + + private final Connection c; + + private final Map writers = new LinkedHashMap<>(); + + public PyramidTables(Connection c, List sourceIds) throws SQLException { + this.c = c; + try { + for (String sourceId : sourceIds) { + writers.put(sourceId, new TilePyramidWriter(c, sourceId)); + } + } catch (SQLException e) { + close(); + throw e; + } + c.setAutoCommit(false); + } + + public static void createTables(Connection c, List sourceIds, TileMatrixSet tms, + int minZoom, int maxZoom) throws SQLException { + c.setAutoCommit(false); + try { + for (String sourceId : sourceIds) { + TilePyramidWriter.createTable(c, sourceId, tms, minZoom, maxZoom); + } + c.commit(); + } catch (SQLException | RuntimeException e) { + c.rollback(); + throw e; + } finally { + c.setAutoCommit(true); + } + LOG.info("Created {} pyramid table(s) for zooms {}-{}", sourceIds.size(), minZoom, maxZoom); + } + + public static List existingTables(Connection c, List sourceIds) throws SQLException { + List existing = new ArrayList<>(); + for (String sourceId : sourceIds) { + if (TilePyramidWriter.tableExists(c, sourceId)) { + existing.add(sourceId); + } + } + return existing; + } + + public void write(String sourceId, TileAddress address, byte[] tileData) throws SQLException { + TilePyramidWriter writer = writers.get(sourceId); + if (writer == null) { + throw new IllegalArgumentException("No pyramid table open for source " + sourceId); + } + if (writer.write(address, tileData)) { + flush(); + } + } + + public void flush() throws SQLException { + boolean any = false; + for (TilePyramidWriter writer : writers.values()) { + any |= writer.flush(); + } + if (any) { + c.commit(); + } + } + + public long getWritten() { + long total = 0; + for (TilePyramidWriter writer : writers.values()) { + total += writer.getWritten(); + } + return total; + } + + public Map getWrittenBySource() { + Map byCollection = new LinkedHashMap<>(); + for (Map.Entry e : writers.entrySet()) { + byCollection.put(e.getKey(), e.getValue().getWritten()); + } + return byCollection; + } + + public void finish() throws SQLException { + for (TilePyramidWriter writer : writers.values()) { + writer.finish(); + } + c.commit(); + } + + @Override + public void close() { + for (TilePyramidWriter writer : writers.values()) { + writer.close(); + } + writers.clear(); + } + +} diff --git a/src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/TileAddress.java b/src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/TileAddress.java new file mode 100644 index 00000000..118e62e2 --- /dev/null +++ b/src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/TileAddress.java @@ -0,0 +1,10 @@ +package fi.nls.hakunapi.tiles.generator; + +public record TileAddress(int zoom, long row, long col) { + + @Override + public String toString() { + return zoom + "/" + row + "/" + col; + } + +} diff --git a/src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/TileGzip.java b/src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/TileGzip.java new file mode 100644 index 00000000..5ec1e207 --- /dev/null +++ b/src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/TileGzip.java @@ -0,0 +1,31 @@ +package fi.nls.hakunapi.tiles.generator; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.zip.GZIPOutputStream; + +public final class TileGzip { + + private TileGzip() {} + + public static byte[] compress(byte[] tile) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(Math.max(64, tile.length)); + try (GZIPOutputStream gz = new FastCompression(out)) { + gz.write(tile); + } + return out.toByteArray(); + } + + /** GZIPOutputStream has no level constructor; the level is set on its inherited Deflater. */ + private static class FastCompression extends GZIPOutputStream { + + private static final int LEVEL = 4; + + FastCompression(ByteArrayOutputStream out) throws IOException { + super(out, 8192); + def.setLevel(LEVEL); + } + + } + +} diff --git a/src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/TilePyramidWriter.java b/src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/TilePyramidWriter.java new file mode 100644 index 00000000..ecb9f554 --- /dev/null +++ b/src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/TilePyramidWriter.java @@ -0,0 +1,127 @@ +package fi.nls.hakunapi.tiles.generator; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import fi.nls.hakunapi.gpkg.GPKG; +import fi.nls.hakunapi.gpkg.GPKGTileMatrix; +import fi.nls.hakunapi.gpkg.GPKGTilesTable; +import fi.nls.hakunapi.tiles.TileMatrix; +import fi.nls.hakunapi.tiles.TileMatrixSet; + +/** Batches but never commits: PyramidTables owns the shared connection's transaction. */ +public class TilePyramidWriter implements AutoCloseable { + + private static final Logger LOG = LoggerFactory.getLogger(TilePyramidWriter.class); + + public static final int BATCH_LIMIT = 1024; + + private final Connection c; + private final String tableName; + private final PreparedStatement insert; + + private int pending; + private long written; + private boolean closed; + + public TilePyramidWriter(Connection c, String tableName) throws SQLException { + this.c = c; + this.tableName = tableName; + this.insert = c.prepareStatement("INSERT INTO " + GPKG.quote(tableName) + + " (zoom_level, tile_column, tile_row, tile_data) VALUES (?, ?, ?, ?)"); + } + + public static void createTable(Connection c, String tableName, TileMatrixSet tms, + int minZoom, int maxZoom) throws SQLException { + List matrices = tms.getTileMatrices(); + TileMatrix first = matrices.get(0); + + double minX = first.getPointOfOriginX(); + double maxY = first.getPointOfOriginY(); + double maxX = minX + first.getMatrixWidth() * first.getTileSpanX(); + double minY = maxY - first.getMatrixHeight() * first.getTileSpanY(); + + GPKGTilesTable table = new GPKGTilesTable(tableName, tableName, + "Vector tiles generated from " + tms.getId(), minX, minY, maxX, maxY, tms.getSrid()); + + GPKG.createTilePyramidTable(c, tableName); + GPKG.insertTilesTableEntry(c, table); + GPKG.insertTileMatrixSet(c, table); + + for (int zoom = minZoom; zoom <= maxZoom; zoom++) { + TileMatrix tm = matrices.get(zoom); + GPKG.insertTileMatrix(c, new GPKGTileMatrix(tableName, zoom, + tm.getMatrixWidth(), tm.getMatrixHeight(), + tm.getTileWidth(), tm.getTileHeight(), + tm.getCellSize(), tm.getCellSize())); + } + } + + public static boolean tableExists(Connection c, String tableName) throws SQLException { + String sql = "SELECT 1 FROM gpkg_contents WHERE table_name = ? AND data_type = 'tiles'"; + try (PreparedStatement ps = c.prepareStatement(sql)) { + ps.setString(1, tableName); + try (ResultSet rs = ps.executeQuery()) { + return rs.next(); + } + } + } + + public boolean write(TileAddress address, byte[] tileData) throws SQLException { + insert.setInt(1, address.zoom()); + insert.setLong(2, address.col()); + insert.setLong(3, address.row()); + insert.setBytes(4, tileData); + insert.addBatch(); + written++; + if (++pending >= BATCH_LIMIT) { + flush(); + return true; + } + return false; + } + + public boolean flush() throws SQLException { + if (pending == 0) { + return false; + } + insert.executeBatch(); + pending = 0; + return true; + } + + public long getWritten() { + return written; + } + + public void finish() throws SQLException { + flush(); + String sql = "UPDATE gpkg_contents SET last_change = strftime('%Y-%m-%dT%H:%M:%fZ','now')" + + " WHERE table_name = ?"; + try (PreparedStatement ps = c.prepareStatement(sql)) { + ps.setString(1, tableName); + ps.executeUpdate(); + } + } + + @Override + public void close() { + if (closed) { + return; + } + closed = true; + try { + insert.close(); + } catch (SQLException e) { + LOG.warn("Failed to close tile insert statement", e); + } + } + +} diff --git a/src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/TileRange.java b/src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/TileRange.java new file mode 100644 index 00000000..cca8c3f6 --- /dev/null +++ b/src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/TileRange.java @@ -0,0 +1,40 @@ +package fi.nls.hakunapi.tiles.generator; + +import fi.nls.hakunapi.tiles.TileMatrix; + +/** Row and column bounds are inclusive. */ +public record TileRange(int zoom, long minRow, long maxRow, long minCol, long maxCol) { + + /** Far edges are half-open: a boundary on a tile edge does not pull in the next tile. */ + public static TileRange of(TileMatrix tm, int zoom, double minX, double minY, double maxX, double maxY) { + double spanX = tm.getTileSpanX(); + double spanY = tm.getTileSpanY(); + double originX = tm.getPointOfOriginX(); + double originY = tm.getPointOfOriginY(); + + long minCol = (long) Math.floor((minX - originX) / spanX); + long maxCol = (long) Math.ceil((maxX - originX) / spanX) - 1; + // Rows grow downward from the origin, so the extent's maxY gives the first row. + long minRow = (long) Math.floor((originY - maxY) / spanY); + long maxRow = (long) Math.ceil((originY - minY) / spanY) - 1; + + minCol = Math.max(0, minCol); + minRow = Math.max(0, minRow); + maxCol = Math.min(tm.getMatrixWidth() - 1L, maxCol); + maxRow = Math.min(tm.getMatrixHeight() - 1L, maxRow); + + return new TileRange(zoom, minRow, maxRow, minCol, maxCol); + } + + public boolean isEmpty() { + return minRow > maxRow || minCol > maxCol; + } + + public long size() { + if (isEmpty()) { + return 0; + } + return (maxRow - minRow + 1) * (maxCol - minCol + 1); + } + +} diff --git a/src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/TileSources.java b/src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/TileSources.java new file mode 100644 index 00000000..942d7817 --- /dev/null +++ b/src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/TileSources.java @@ -0,0 +1,85 @@ +package fi.nls.hakunapi.tiles.generator; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import fi.nls.hakunapi.core.config.HakunaConfigParser; + +public class TileSources { + + public static final String PROP_SOURCES = "sources"; + + private final Map sourceByCollection; + + private final List sourceIds; + + private TileSources(Map sourceByCollection, List sourceIds) { + this.sourceByCollection = Map.copyOf(sourceByCollection); + this.sourceIds = List.copyOf(sourceIds); + } + + public static TileSources read(HakunaConfigParser cfg, GeneratorControl control) { + List rendered = control.getAllCollectionIds(); + + String[] ids = cfg.getMultiple(PROP_SOURCES); + if (ids.length == 0) { + Map identity = new LinkedHashMap<>(); + for (String id : rendered) { + identity.put(id, id); + } + return new TileSources(identity, rendered); + } + + Map byCollection = new LinkedHashMap<>(); + List sourceIds = new ArrayList<>(ids.length); + for (String sourceId : ids) { + if (sourceIds.contains(sourceId)) { + throw new IllegalArgumentException("control file: source '" + sourceId + "' is declared twice"); + } + sourceIds.add(sourceId); + String[] collectionIds = cfg.getMultiple(sourceId + ".collections"); + if (collectionIds.length == 0) { + throw new IllegalArgumentException("control file: source " + sourceId + + " must name at least one collection in " + sourceId + ".collections"); + } + for (String collectionId : collectionIds) { + String previous = byCollection.putIfAbsent(collectionId, sourceId); + if (previous != null) { + throw new IllegalArgumentException("control file: collection '" + collectionId + + "' is in both source " + previous + " and source " + sourceId + + "; a collection belongs to exactly one source"); + } + if (!rendered.contains(collectionId)) { + throw new IllegalArgumentException("control file: source " + sourceId + + " names collection '" + collectionId + "', which no zoom range renders"); + } + } + } + + Set unplaced = new LinkedHashSet<>(rendered); + unplaced.removeAll(byCollection.keySet()); + if (!unplaced.isEmpty()) { + throw new IllegalArgumentException("control file: collection(s) " + unplaced + + " are rendered by a zoom range but are in no source"); + } + + return new TileSources(byCollection, sourceIds); + } + + public List getSourceIds() { + return sourceIds; + } + + public String getSource(String collectionId) { + String sourceId = sourceByCollection.get(collectionId); + if (sourceId == null) { + throw new IllegalArgumentException("No source configured for collection '" + collectionId + "'"); + } + return sourceId; + } + +} diff --git a/src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/TilesGenerator.java b/src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/TilesGenerator.java new file mode 100644 index 00000000..161a4f7d --- /dev/null +++ b/src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/TilesGenerator.java @@ -0,0 +1,244 @@ +package fi.nls.hakunapi.tiles.generator; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.DriverManager; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import fi.nls.hakunapi.core.FeatureType; +import fi.nls.hakunapi.core.FilterParser; +import fi.nls.hakunapi.core.config.HakunaConfigParser; +import fi.nls.hakunapi.cql2.text.CQL2Text; +import fi.nls.hakunapi.gpkg.GPKG; +import fi.nls.hakunapi.gpkg.GPKGSpatialRefSystems; +import fi.nls.hakunapi.tiles.TileMatrix; +import fi.nls.hakunapi.tiles.TileMatrixSet; +import fi.nls.hakunapi.tiles.source.vectortile.JavaVectorTileGenerator; +import fi.nls.hakunapi.tiles.source.vectortile.VectorTileGenerator; +import fi.nls.hakunapi.tiles.source.vectortile.VectorTileGeneratorProvider; +import fi.nls.hakunapi.tiles.source.vectortile.VectorTileGeneratorSource; + +public class TilesGenerator { + + private static final Logger LOG = LoggerFactory.getLogger(TilesGenerator.class); + + public static void main(String[] args) { + try { + int status = run(args); + System.exit(status); + } catch (Exception e) { + Throwable root = e; + while (root instanceof ExecutionException && root.getCause() != null) { + root = root.getCause(); + } + System.err.println("ERROR: " + root.getMessage()); + LOG.debug("Generation failed", e); + System.exit(1); + } + } + + private static int run(String[] args) throws Exception { + Args parsed = Args.parse(args); + if (parsed == null) { + usage(); + return 2; + } + + try (GeneratorBootstrap bootstrap = GeneratorBootstrap.read(parsed.configPath)) { + List filters = List.of(CQL2Text.INSTANCE); + + HakunaConfigParser controlCfg = GeneratorControl.load(parsed.controlPath); + String tmsId = GeneratorControl.readTileMatrixSetId(controlCfg); + TileMatrixSet tms = bootstrap.getTileMatrixSets().get(tmsId) + .orElseThrow(() -> new IllegalArgumentException("Unknown tileMatrixSet '" + tmsId + + "'; define it as tileMatrixSets." + tmsId + ".* in the service configuration")); + + GeneratorControl control = GeneratorControl.readRanges(controlCfg, bootstrap.getService(), + filters, bootstrap.getSridCode(tms.getSrid())); + TileSources sources = TileSources.read(controlCfg, control); + + int minZoom = parsed.minZoom != null ? parsed.minZoom : control.getMinZoom(); + int maxZoom = parsed.maxZoom != null ? parsed.maxZoom : control.getMaxZoom(); + validateZooms(control, tms, minZoom, maxZoom); + + double[] bbox = parsed.bbox != null ? parsed.bbox : fullExtent(tms); + + VectorTileGenerator generator = resolveGenerator(parsed.generatorType); + LOG.info("Generating into {} using tileMatrixSet {} and generator {}", + parsed.outputPath, tms.getId(), generator.getType()); + + Map> collections = + PyramidGenerator.resolveCollections(bootstrap, control); + + return generate(parsed, control, sources, tms, generator, collections, minZoom, maxZoom, bbox); + } + } + + private static int generate(Args parsed, GeneratorControl control, TileSources sources, + TileMatrixSet tms, VectorTileGenerator generator, + Map> collections, + int minZoom, int maxZoom, double[] bbox) throws Exception { + boolean fileExists = Files.exists(parsed.outputPath); + List tables = sources.getSourceIds(); + + try (Connection c = open(parsed.outputPath)) { + if (!fileExists) { + GPKG.init(c); + } + GPKG.initTiles(c); + GPKG.insertSpatialRefSys(c, GPKGSpatialRefSystems.get(tms.getSrid())); + + List existing = PyramidTables.existingTables(c, tables); + if (!existing.isEmpty()) { + throw new IllegalArgumentException("Refusing to overwrite tiles table(s) " + existing + + " in " + parsed.outputPath); + } + PyramidTables.createTables(c, tables, tms, minZoom, maxZoom); + + PyramidGenerator pyramid = new PyramidGenerator(control, sources, tms, generator, + VectorTileGeneratorSource.DEFAULT_MEDIA_TYPE, collections); + try (PyramidTables pyramidTables = new PyramidTables(c, tables)) { + long written = pyramid.generate(pyramidTables, minZoom, maxZoom, + bbox[0], bbox[1], bbox[2], bbox[3], parsed.threads); + System.out.println("Wrote " + written + " tiles into " + parsed.outputPath); + for (Map.Entry e : pyramidTables.getWrittenBySource().entrySet()) { + if (e.getValue() > 0) { + System.out.printf(" %-28s %,8d%n", e.getKey(), e.getValue()); + } + } + } + } + return 0; + } + + /** Autocommit: GPKG.init's PRAGMA synchronous fails inside a transaction. */ + private static Connection open(Path path) throws Exception { + return DriverManager.getConnection("jdbc:sqlite:" + path.toAbsolutePath()); + } + + private static VectorTileGenerator resolveGenerator(String type) { + String wanted = type != null ? type : JavaVectorTileGenerator.TYPE; + VectorTileGenerator generator = VectorTileGeneratorProvider.get(wanted); + if (generator == null) { + throw new IllegalArgumentException("Unknown vector tile generator type '" + wanted + "'"); + } + return generator; + } + + private static void validateZooms(GeneratorControl control, TileMatrixSet tms, int minZoom, int maxZoom) { + if (minZoom > maxZoom) { + throw new IllegalArgumentException("--min-zoom " + minZoom + " is above --max-zoom " + maxZoom); + } + int levels = tms.getTileMatrices().size(); + if (minZoom < 0 || maxZoom >= levels) { + throw new IllegalArgumentException("Zoom levels " + minZoom + "-" + maxZoom + + " are outside tileMatrixSet " + tms.getId() + ", which has levels 0-" + (levels - 1)); + } + for (int zoom = minZoom; zoom <= maxZoom; zoom++) { + control.getRange(zoom); + } + } + + private static double[] fullExtent(TileMatrixSet tms) { + TileMatrix tm = tms.getTileMatrices().get(0); + double minX = tm.getPointOfOriginX(); + double maxY = tm.getPointOfOriginY(); + return new double[] { + minX, maxY - tm.getMatrixHeight() * tm.getTileSpanY(), + minX + tm.getMatrixWidth() * tm.getTileSpanX(), maxY }; + } + + private static void usage() { + System.err.println("Usage: hakunapi-tiles-generator [options] "); + System.err.println(); + System.err.println(" --min-zoom coarsest zoom to write (default: the control file's minimum)"); + System.err.println(" --max-zoom finest zoom to write (default: the control file's maximum)"); + System.err.println(" --bbox "); + System.err.println(" limit to this extent, in the tiling scheme's CRS"); + System.err.println(" --threads parallel render workers (default: available processors)"); + System.err.println(" --generator vector tile generator type (default: hakunapi)"); + } + + private static class Args { + + Path configPath; + Path controlPath; + Path outputPath; + Integer minZoom; + Integer maxZoom; + double[] bbox; + int threads = Runtime.getRuntime().availableProcessors(); + String generatorType; + + static Args parse(String[] args) { + Args a = new Args(); + List positional = new ArrayList<>(3); + for (int i = 0; i < args.length; i++) { + String arg = args[i]; + if (arg.startsWith("--") && i + 1 >= args.length) { + System.err.println("Option " + arg + " needs a value"); + return null; + } + switch (arg) { + case "--min-zoom": + a.minZoom = Integer.parseInt(args[++i]); + break; + case "--max-zoom": + a.maxZoom = Integer.parseInt(args[++i]); + break; + case "--bbox": + a.bbox = parseBbox(args[++i]); + break; + case "--threads": + a.threads = Integer.parseInt(args[++i]); + break; + case "--generator": + a.generatorType = args[++i]; + break; + default: + if (arg.startsWith("--")) { + System.err.println("Unknown option: " + arg); + return null; + } + positional.add(arg); + } + } + if (positional.size() != 3) { + return null; + } + if (a.threads < 1) { + System.err.println("--threads must be at least 1"); + return null; + } + a.configPath = Paths.get(positional.get(0)).toAbsolutePath(); + a.controlPath = Paths.get(positional.get(1)).toAbsolutePath(); + a.outputPath = Paths.get(positional.get(2)).toAbsolutePath(); + return a; + } + + private static double[] parseBbox(String csv) { + String[] parts = csv.split(","); + if (parts.length != 4) { + throw new IllegalArgumentException("--bbox must be minX,minY,maxX,maxY"); + } + double[] bbox = new double[4]; + for (int i = 0; i < 4; i++) { + bbox[i] = Double.parseDouble(parts[i].trim()); + } + if (bbox[0] >= bbox[2] || bbox[1] >= bbox[3]) { + throw new IllegalArgumentException("--bbox is empty or inverted"); + } + return bbox; + } + + } + +} diff --git a/src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/ZoomRange.java b/src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/ZoomRange.java new file mode 100644 index 00000000..40621aa3 --- /dev/null +++ b/src-community/hakunapi-tiles-generator/src/main/java/fi/nls/hakunapi/tiles/generator/ZoomRange.java @@ -0,0 +1,25 @@ +package fi.nls.hakunapi.tiles.generator; + +import java.util.List; +import java.util.Map; + +import fi.nls.hakunapi.core.filter.Filter; + +/** Zoom bounds are inclusive; filters has no entry for a collection without a filter. */ +public record ZoomRange( + String name, + int minZoom, + int maxZoom, + List collectionIds, + Map filters) { + + public ZoomRange { + collectionIds = List.copyOf(collectionIds); + filters = Map.copyOf(filters); + } + + public boolean covers(int zoom) { + return zoom >= minZoom && zoom <= maxZoom; + } + +} diff --git a/src-community/hakunapi-tiles-generator/src/test/java/fi/nls/hakunapi/tiles/generator/GeneratorControlTest.java b/src-community/hakunapi-tiles-generator/src/test/java/fi/nls/hakunapi/tiles/generator/GeneratorControlTest.java new file mode 100644 index 00000000..baeca9bf --- /dev/null +++ b/src-community/hakunapi-tiles-generator/src/test/java/fi/nls/hakunapi/tiles/generator/GeneratorControlTest.java @@ -0,0 +1,139 @@ +package fi.nls.hakunapi.tiles.generator; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import fi.nls.hakunapi.core.FeatureType; +import fi.nls.hakunapi.core.FilterParser; +import fi.nls.hakunapi.core.SRIDCode; +import fi.nls.hakunapi.core.config.HakunaConfigParser; +import fi.nls.hakunapi.cql2.text.CQL2Text; + +public class GeneratorControlTest { + + private static final int SRID = 3067; + private static final List FILTERS = List.of(CQL2Text.INSTANCE); + + @Rule + public TemporaryFolder tmp = new TemporaryFolder(); + + private static GeneratorServiceConfig service(String... collectionIds) { + Map byId = new LinkedHashMap<>(); + for (String id : collectionIds) { + byId.put(id, new StubCollection(id, SRID)); + } + GeneratorServiceConfig service = new GeneratorServiceConfig(byId); + service.setKnownSrids(List.of(new SRIDCode(SRID, false, false, + fi.nls.hakunapi.core.geom.HakunaGeometryDimension.XY))); + return service; + } + + private GeneratorControl read(String content, String... collectionIds) throws Exception { + Path path = tmp.newFolder().toPath().resolve("control.properties"); + Files.write(path, content.getBytes(StandardCharsets.UTF_8)); + HakunaConfigParser cfg = GeneratorControl.load(path); + return GeneratorControl.readRanges(cfg, service(collectionIds), FILTERS, + new SRIDCode(SRID, false, false, fi.nls.hakunapi.core.geom.HakunaGeometryDimension.XY)); + } + + private void assertRejected(String message, String content) { + IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> read(content, "maastoalue")); + assertTrue(e.getMessage(), e.getMessage().toLowerCase().contains(message)); + } + + @Test + public void rangesAreSortedAndCarryTheirCollectionsAndFilters() throws Exception { + GeneratorControl control = read(""" + tileMatrixSet = ETRS-TM35FIN + ranges = detail, overview + detail.zoom = 8-14 + detail.collections = maastoalue, tieviiva + detail.tieviiva.filter = kohdeluokka <= 12132 + overview.zoom = 0-7 + overview.collections = maastoalue + """, "maastoalue", "tieviiva"); + + assertEquals("ETRS-TM35FIN", control.getTileMatrixSetId()); + assertEquals("overview", control.getRanges().get(0).name()); + assertEquals(0, control.getMinZoom()); + assertEquals(14, control.getMaxZoom()); + assertEquals(List.of("maastoalue", "tieviiva"), control.getRange(10).collectionIds()); + assertNotNull(control.getRange(10).filters().get("tieviiva")); + assertNull(control.getRange(10).filters().get("maastoalue")); + assertThrows(IllegalArgumentException.class, () -> control.getRange(15)); + } + + @Test + public void invalidControlFilesAreRejectedWhenRead() { + assertRejected("overlap", """ + tileMatrixSet = T + ranges = a, b + a.zoom = 0-8 + a.collections = maastoalue + b.zoom = 7-14 + b.collections = maastoalue + """); + assertRejected("zoom 6", """ + tileMatrixSet = T + ranges = a, b + a.zoom = 0-5 + a.collections = maastoalue + b.zoom = 8-14 + b.collections = maastoalue + """); + assertRejected("eiolemassa", """ + tileMatrixSet = T + ranges = a + a.zoom = 0-2 + a.collections = eiolemassa + """); + assertRejected("maastoalue", """ + tileMatrixSet = T + ranges = a + a.zoom = 0-2 + a.collections = maastoalue + a.maastoalue.filter = this is not CQL2 ((( + """); + assertRejected("inverted", """ + tileMatrixSet = T + ranges = a + a.zoom = 9-2 + a.collections = maastoalue + """); + assertRejected("zoom", """ + tileMatrixSet = T + ranges = a + a.zoom = kaikki + a.collections = maastoalue + """); + assertRejected("collections", """ + tileMatrixSet = T + ranges = a + a.zoom = 0-2 + """); + assertRejected("ranges", """ + tileMatrixSet = T + """); + assertRejected("tilematrixset", """ + ranges = a + a.zoom = 0-2 + a.collections = maastoalue + """); + } + +} diff --git a/src-community/hakunapi-tiles-generator/src/test/java/fi/nls/hakunapi/tiles/generator/MvtTileTest.java b/src-community/hakunapi-tiles-generator/src/test/java/fi/nls/hakunapi/tiles/generator/MvtTileTest.java new file mode 100644 index 00000000..f595a19a --- /dev/null +++ b/src-community/hakunapi-tiles-generator/src/test/java/fi/nls/hakunapi/tiles/generator/MvtTileTest.java @@ -0,0 +1,58 @@ +package fi.nls.hakunapi.tiles.generator; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayOutputStream; + +import org.junit.Test; + +import fi.nls.hakunapi.tiles.source.vectortile.mvt.MvtLayerEncoder; +import fi.nls.hakunapi.tiles.source.vectortile.mvt.MvtTileWriter; + +public class MvtTileTest { + + private static byte[] tile(MvtLayerEncoder... layers) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + MvtTileWriter.write(out, layers); + return out.toByteArray(); + } + + private static MvtLayerEncoder emptyLayer(String name) { + MvtLayerEncoder layer = new MvtLayerEncoder(); + layer.reset(name); + layer.setExtent(4096); + layer.finish(); + return layer; + } + + private static MvtLayerEncoder pointLayer(String name) { + MvtLayerEncoder layer = new MvtLayerEncoder(); + layer.reset(name); + layer.setExtent(4096); + int key = layer.keyIndex("kohdeluokka"); + layer.beginFeature(); + layer.tagLong(key, 12132); + int[] geometry = new int[] { (1 << 3) | 1, 200, 400 }; + layer.endFeature(1, MvtLayerEncoder.GEOM_POINT, geometry, geometry.length); + layer.finish(); + return layer; + } + + @Test + public void aFeatureInAnyLayerCounts() throws Exception { + assertTrue(MvtTile.hasFeatures(tile(pointLayer("a")))); + assertTrue(MvtTile.hasFeatures(tile(emptyLayer("a"), pointLayer("b")))); + assertTrue(MvtTile.hasFeatures(tile(pointLayer("a"), emptyLayer("b")))); + } + + @Test + public void emptyLayersHaveNoFeaturesEvenThoughTheTileIsNotZeroBytes() throws Exception { + assertTrue(tile(emptyLayer("tieviiva")).length > 0); + assertFalse(MvtTile.hasFeatures(tile(emptyLayer("a"), emptyLayer("b")))); + assertFalse(MvtTile.hasFeatures(tile())); + assertFalse(MvtTile.hasFeatures(new byte[0])); + assertFalse(MvtTile.hasFeatures(null)); + } + +} diff --git a/src-community/hakunapi-tiles-generator/src/test/java/fi/nls/hakunapi/tiles/generator/PyramidGeneratorTest.java b/src-community/hakunapi-tiles-generator/src/test/java/fi/nls/hakunapi/tiles/generator/PyramidGeneratorTest.java new file mode 100644 index 00000000..2c120226 --- /dev/null +++ b/src-community/hakunapi-tiles-generator/src/test/java/fi/nls/hakunapi/tiles/generator/PyramidGeneratorTest.java @@ -0,0 +1,309 @@ +package fi.nls.hakunapi.tiles.generator; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayInputStream; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; +import java.util.concurrent.ExecutionException; +import java.util.zip.GZIPInputStream; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import fi.nls.hakunapi.core.FeatureType; +import fi.nls.hakunapi.core.config.HakunaConfigParser; +import fi.nls.hakunapi.gpkg.GPKG; +import fi.nls.hakunapi.gpkg.GPKGSpatialRefSystems; +import fi.nls.hakunapi.tiles.TileBody; +import fi.nls.hakunapi.tiles.TileMatrix; +import fi.nls.hakunapi.tiles.TileMatrixSet; +import fi.nls.hakunapi.tiles.source.vectortile.JavaVectorTileGenerator; +import fi.nls.hakunapi.tiles.source.vectortile.TileContext; +import fi.nls.hakunapi.tiles.source.vectortile.VectorTileGenerator; +import fi.nls.hakunapi.tiles.source.vectortile.VectorTileGeneratorSource; + +public class PyramidGeneratorTest { + + private static final int SRID = 3067; + + @Rule + public TemporaryFolder tmp = new TemporaryFolder(); + + private static TileMatrixSet scheme() { + List matrices = new ArrayList<>(4); + matrices.add(new TileMatrix("0", 1, 4.0, 0.0, 1024.0, 256, 256, 1, 1)); + matrices.add(new TileMatrix("1", 1, 2.0, 0.0, 1024.0, 256, 256, 2, 2)); + matrices.add(new TileMatrix("2", 1, 1.0, 0.0, 1024.0, 256, 256, 4, 4)); + matrices.add(new TileMatrix("3", 1, 0.5, 0.0, 1024.0, 256, 256, 8, 8)); + return new TileMatrixSet("TEST", "Test scheme", "http://example.org/crs/3067", SRID, matrices); + } + + private static GeneratorControl control(int minZoom, int maxZoom, String... collectionIds) { + ZoomRange range = new ZoomRange("all", minZoom, maxZoom, List.of(collectionIds), Map.of()); + return new GeneratorControl("TEST", List.of(range)); + } + + private static Map> collectionsOf(GeneratorControl control, + StubCollection... collections) { + Map byId = new LinkedHashMap<>(); + for (StubCollection c : collections) { + byId.put(c.getName(), c); + } + Map> byRange = new LinkedHashMap<>(); + for (ZoomRange range : control.getRanges()) { + List resolved = new ArrayList<>(range.collectionIds().size()); + for (String id : range.collectionIds()) { + resolved.add(byId.get(id)); + } + byRange.put(range.name(), resolved); + } + return byRange; + } + + private static TileSources sourcesOf(GeneratorControl control, String... keyValues) { + Properties p = new Properties(); + for (int i = 0; i < keyValues.length; i += 2) { + p.setProperty(keyValues[i], keyValues[i + 1]); + } + return TileSources.read(new HakunaConfigParser(p), control); + } + + private Path generate(StubCollection[] stubs, GeneratorControl control, TileSources sources, + VectorTileGenerator generator, int minZoom, int maxZoom) throws Exception { + Path out = tmp.newFolder().toPath().resolve("tiles.gpkg"); + TileMatrixSet tms = scheme(); + List tables = sources.getSourceIds(); + try (Connection c = open(out)) { + GPKG.init(c); + GPKG.initTiles(c); + GPKG.insertSpatialRefSys(c, GPKGSpatialRefSystems.get(SRID)); + PyramidTables.createTables(c, tables, tms, minZoom, maxZoom); + + PyramidGenerator pyramid = new PyramidGenerator(control, sources, tms, generator, + VectorTileGeneratorSource.DEFAULT_MEDIA_TYPE, collectionsOf(control, stubs)); + try (PyramidTables pyramidTables = new PyramidTables(c, tables)) { + pyramid.generate(pyramidTables, minZoom, maxZoom, 0, 0, 1024, 1024, 1); + } + } + return out; + } + + private Path generate(StubCollection[] stubs, GeneratorControl control, TileSources sources, + int minZoom, int maxZoom) throws Exception { + return generate(stubs, control, sources, new JavaVectorTileGenerator(), minZoom, maxZoom); + } + + private static VectorTileGenerator perTileOnly(VectorTileGenerator delegate) { + return new VectorTileGenerator() { + @Override + public String getType() { + return delegate.getType(); + } + + @Override + public Optional generate(TileContext ctx, List collections, + String mediaType, Map requestParams) throws Exception { + return delegate.generate(ctx, collections, mediaType, requestParams); + } + }; + } + + private static StubCollection scattered() { + return new StubCollection("tieviiva", SRID) + .add(1, 100, 900, 12132).add(2, 500, 500, 12141).add(3, 900, 100, 12132) + .add(4, 256, 256, 12132).add(5, 512, 512, 12141).add(6, 300, 700, 12132); + } + + private static Connection open(Path path) throws Exception { + return DriverManager.getConnection("jdbc:sqlite:" + path.toAbsolutePath()); + } + + @Test + public void blockOutputEqualsPerTileOutput() throws Exception { + GeneratorControl control = control(0, 3, "tieviiva"); + StubCollection blockStub = scattered(); + StubCollection tileStub = scattered(); + JavaVectorTileGenerator java = new JavaVectorTileGenerator(); + assertTrue(java.supportsBlocks()); + + Path blocks = generate(new StubCollection[] { blockStub }, control, + sourcesOf(control), java, 0, 3); + Path perTile = generate(new StubCollection[] { tileStub }, control, + sourcesOf(control), perTileOnly(java), 0, 3); + + Map expected = readAll(perTile, "tieviiva"); + assertFalse(expected.isEmpty()); + assertEquals(expected, readAll(blocks, "tieviiva")); + assertTrue(blockStub.queries < tileStub.queries); + } + + @Test + public void emptyTilesWriteNoRowAndEachCollectionKeepsItsOwnTable() throws Exception { + StubCollection tieviiva = new StubCollection("tieviiva", SRID).add(1, 100, 900, 12132); + StubCollection rakennus = new StubCollection("rakennus", SRID); + GeneratorControl control = control(2, 2, "tieviiva", "rakennus"); + + Path out = generate(new StubCollection[] { tieviiva, rakennus }, control, + sourcesOf(control), 2, 2); + + assertEquals(1, readAll(out, "tieviiva").size()); + assertTrue(readAll(out, "rakennus").isEmpty()); + byte[] tile = readTile(out, "tieviiva", 2, 0, 0); + assertNotNull(tile); + assertTrue(MvtTile.hasFeatures(tile)); + } + + @Test + public void aGroupedTileIsTheGzippedConcatenationOfItsCollectionsAndEmptySourcesWriteNoRow() + throws Exception { + StubCollection jarvi = new StubCollection("jarvi", SRID).add(1, 100, 900, 36200); + StubCollection meri = new StubCollection("meri", SRID).add(2, 120, 880, 36211); + StubCollection suo = new StubCollection("suo", SRID); + GeneratorControl control = control(2, 2, "jarvi", "meri", "suo"); + StubCollection[] stubs = { jarvi, meri, suo }; + + Path perCollection = generate(stubs, control, sourcesOf(control), 2, 2); + Path perSource = generate(stubs, control, sourcesOf(control, "sources", "vesi, maasto", + "vesi.collections", "jarvi, meri", "maasto.collections", "suo"), 2, 2); + + byte[] one = readTile(perCollection, "jarvi", 2, 0, 0); + byte[] two = readTile(perCollection, "meri", 2, 0, 0); + byte[] concatenated = new byte[one.length + two.length]; + System.arraycopy(one, 0, concatenated, 0, one.length); + System.arraycopy(two, 0, concatenated, one.length, two.length); + + assertArrayEquals(concatenated, readTile(perSource, "vesi", 2, 0, 0)); + byte[] stored = readStoredTile(perSource, "vesi", 2, 0, 0); + assertEquals((byte) 0x1f, stored[0]); + assertEquals((byte) 0x8b, stored[1]); + assertTrue(readAll(perSource, "maasto").isEmpty()); + } + + @Test + public void aFailedRenderKeepsTheWrittenTilesAndSkipsFinish() throws Exception { + GeneratorControl control = control(0, 3, "tieviiva"); + VectorTileGenerator java = new JavaVectorTileGenerator(); + VectorTileGenerator failing = new VectorTileGenerator() { + @Override + public String getType() { + return java.getType(); + } + + @Override + public Optional generate(TileContext ctx, List collections, + String mediaType, Map requestParams) throws Exception { + if (ctx.resolution() <= 0.5) { + throw new IllegalStateException("boom"); + } + return java.generate(ctx, collections, mediaType, requestParams); + } + }; + Path out = tmp.newFolder().toPath().resolve("tiles.gpkg"); + TileSources sources = sourcesOf(control); + List tables = sources.getSourceIds(); + TileMatrixSet tms = scheme(); + try (Connection c = open(out)) { + GPKG.init(c); + GPKG.initTiles(c); + GPKG.insertSpatialRefSys(c, GPKGSpatialRefSystems.get(SRID)); + PyramidTables.createTables(c, tables, tms, 0, 3); + PyramidGenerator pyramid = new PyramidGenerator(control, sources, tms, failing, + VectorTileGeneratorSource.DEFAULT_MEDIA_TYPE, + collectionsOf(control, scattered())); + try (PyramidTables pyramidTables = new PyramidTables(c, tables)) { + ExecutionException e = assertThrows(ExecutionException.class, + () -> pyramid.generate(pyramidTables, 0, 3, 0, 0, 1024, 1024, 1)); + assertEquals("boom", e.getCause().getMessage()); + } + } + Map kept = readAll(out, "tieviiva"); + assertFalse(kept.isEmpty()); + assertFalse(kept.keySet().stream().anyMatch(k -> k.startsWith("3/"))); + } + + @Test + public void tileMetadataIsCompleteAndInitTilesIsIdempotent() throws Exception { + Path out = tmp.newFolder().toPath().resolve("tiles.gpkg"); + try (Connection c = open(out)) { + GPKG.init(c); + GPKG.initTiles(c); + GPKG.initTiles(c); + GPKG.insertSpatialRefSys(c, GPKGSpatialRefSystems.get(SRID)); + PyramidTables.createTables(c, List.of("tieviiva"), scheme(), 1, 2); + assertTrue(PyramidTables.existingTables(c, List.of("tieviiva", "x")).equals(List.of("tieviiva"))); + try (Statement st = c.createStatement()) { + try (ResultSet rs = st.executeQuery( + "SELECT zoom_level, matrix_width FROM gpkg_tile_matrix ORDER BY zoom_level")) { + assertTrue(rs.next()); + assertEquals(1, rs.getInt(1)); + assertEquals(2, rs.getInt(2)); + assertTrue(rs.next()); + assertEquals(4, rs.getInt(2)); + assertFalse(rs.next()); + } + try (ResultSet rs = st.executeQuery( + "SELECT min_x, max_x, max_y FROM gpkg_contents WHERE table_name = 'tieviiva'")) { + assertTrue(rs.next()); + assertEquals(0.0, rs.getDouble(1), 0); + assertEquals(1024.0, rs.getDouble(2), 0); + assertEquals(1024.0, rs.getDouble(3), 0); + } + } + } + } + + private static Map readAll(Path gpkg, String table) throws Exception { + Map out = new LinkedHashMap<>(); + try (Connection c = open(gpkg); + PreparedStatement ps = c.prepareStatement( + "SELECT zoom_level, tile_row, tile_column, hex(tile_data) FROM " + + GPKG.quote(table) + " ORDER BY zoom_level, tile_row, tile_column"); + ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + out.put(rs.getInt(1) + "/" + rs.getInt(2) + "/" + rs.getInt(3), rs.getString(4)); + } + } + return out; + } + + private static byte[] readTile(Path gpkg, String table, int zoom, int row, int col) throws Exception { + byte[] stored = readStoredTile(gpkg, table, zoom, row, col); + if (stored == null) { + return null; + } + try (GZIPInputStream in = new GZIPInputStream(new ByteArrayInputStream(stored))) { + return in.readAllBytes(); + } + } + + private static byte[] readStoredTile(Path gpkg, String table, int zoom, int row, int col) throws Exception { + try (Connection c = open(gpkg); + PreparedStatement ps = c.prepareStatement("SELECT tile_data FROM " + GPKG.quote(table) + + " WHERE zoom_level = ? AND tile_row = ? AND tile_column = ?")) { + ps.setInt(1, zoom); + ps.setInt(2, row); + ps.setInt(3, col); + try (ResultSet rs = ps.executeQuery()) { + return rs.next() ? rs.getBytes(1) : null; + } + } + } + +} diff --git a/src-community/hakunapi-tiles-generator/src/test/java/fi/nls/hakunapi/tiles/generator/StubCollection.java b/src-community/hakunapi-tiles-generator/src/test/java/fi/nls/hakunapi/tiles/generator/StubCollection.java new file mode 100644 index 00000000..b3577d0c --- /dev/null +++ b/src-community/hakunapi-tiles-generator/src/test/java/fi/nls/hakunapi/tiles/generator/StubCollection.java @@ -0,0 +1,395 @@ +package fi.nls.hakunapi.tiles.generator; + +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.Envelope; +import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.geom.GeometryFactory; + +import fi.nls.hakunapi.core.DatetimeProperty; +import fi.nls.hakunapi.core.FeatureProducer; +import fi.nls.hakunapi.core.FeatureStream; +import fi.nls.hakunapi.core.FeatureType; +import fi.nls.hakunapi.core.OrderBy; +import fi.nls.hakunapi.core.PaginationStrategy; +import fi.nls.hakunapi.core.QueryContext; +import fi.nls.hakunapi.core.ValueMapper; +import fi.nls.hakunapi.core.ValueProvider; +import fi.nls.hakunapi.core.filter.Filter; +import fi.nls.hakunapi.core.filter.FilterOp; +import fi.nls.hakunapi.core.geom.HakunaGeometry; +import fi.nls.hakunapi.core.geom.HakunaGeometryJTS; +import fi.nls.hakunapi.core.geom.HakunaGeometryType; +import fi.nls.hakunapi.core.param.GetFeatureParam; +import fi.nls.hakunapi.core.projection.ProjectionTransformerFactory; +import fi.nls.hakunapi.core.property.HakunaProperty; +import fi.nls.hakunapi.core.property.HakunaPropertyType; +import fi.nls.hakunapi.core.property.HakunaPropertyWriter; +import fi.nls.hakunapi.core.property.simple.HakunaPropertyGeometry; +import fi.nls.hakunapi.core.request.GetFeatureCollection; +import fi.nls.hakunapi.core.request.GetFeatureRequest; +import fi.nls.hakunapi.core.util.StringPair; + +public class StubCollection implements FeatureType { + + private static final GeometryFactory GF = new GeometryFactory(); + + private static final HakunaPropertyWriter NOOP_WRITER = new HakunaPropertyWriter() { + @Override + public void write(ValueProvider p, int i, fi.nls.hakunapi.core.FeatureWriter w) {} + }; + + private final String name; + private final int srid; + private final HakunaProperty id; + private final HakunaPropertyGeometry geom; + private final HakunaProperty kohdeluokka; + private final List props; + private final List points = new ArrayList<>(); + + public volatile int queries; + + public StubCollection(String name, int srid) { + this.name = name; + this.srid = srid; + this.id = new StubProperty("id", HakunaPropertyType.LONG); + this.kohdeluokka = new StubProperty("kohdeluokka", HakunaPropertyType.INT); + this.geom = new HakunaPropertyGeometry("geom", "t", "geom", true, + HakunaGeometryType.POINT, new int[] { srid }, srid, 2, NOOP_WRITER); + this.props = new ArrayList<>(); + this.props.add(geom); + this.props.add(kohdeluokka); + } + + public StubCollection add(long fid, double x, double y, int kohdeluokka) { + points.add(new Point(fid, x, y, kohdeluokka)); + return this; + } + + private record Point(long fid, double x, double y, int kohdeluokka) {} + + @Override + public FeatureProducer getFeatureProducer() { + return new FeatureProducer() { + @Override + public int getNumberMatched(GetFeatureRequest r, GetFeatureCollection c) { + return points.size(); + } + + @Override + public FeatureStream getFeatures(GetFeatureRequest r, GetFeatureCollection c) { + queries++; + double[] bbox = bboxOf(c); + List rows = new ArrayList<>(); + for (int i = 0; i < points.size(); i++) { + Point p = points.get(i); + if (bbox == null || (p.x() >= bbox[0] && p.x() <= bbox[2] + && p.y() >= bbox[1] && p.y() <= bbox[3])) { + rows.add(row(p)); + } + } + return new StubStream(rows); + } + }; + } + + private static double[] bboxOf(GetFeatureCollection c) { + List filters = c.getFilters(); + for (int i = 0; i < filters.size(); i++) { + Filter f = filters.get(i); + if ((f.getOp() == FilterOp.INTERSECTS_INDEX || f.getOp() == FilterOp.INTERSECTS) + && f.getValue() instanceof Geometry g) { + Envelope e = g.getEnvelopeInternal(); + return new double[] { e.getMinX(), e.getMinY(), e.getMaxX(), e.getMaxY() }; + } + } + return null; + } + + private Row row(Point p) { + Geometry g = GF.createPoint(new Coordinate(p.x(), p.y())); + g.setSRID(srid); + HakunaGeometry hg = new HakunaGeometryJTS(g); + return new Row(new Object[] { p.fid(), hg, hg, p.kohdeluokka() }); + } + + @Override + public String getName() { + return name; + } + + @Override + public String getNS() { + return null; + } + + @Override + public String getSchemaLocation() { + return null; + } + + @Override + public String getTitle() { + return name; + } + + @Override + public String getDescription() { + return null; + } + + @Override + public Map getMetadata() { + return Collections.emptyMap(); + } + + @Override + public HakunaProperty getId() { + return id; + } + + @Override + public HakunaPropertyGeometry getGeom() { + return geom; + } + + @Override + public List getProperties() { + return props; + } + + @Override + public List getQueryableProperties() { + return props; + } + + @Override + public List getDatetimeProperties() { + return Collections.emptyList(); + } + + @Override + public double[] getSpatialExtent() { + return null; + } + + @Override + public Instant[] getTemporalExtent() { + return null; + } + + @Override + public List getParameters() { + return Collections.emptyList(); + } + + @Override + public List getStaticFilters() { + return Collections.emptyList(); + } + + @Override + public ProjectionTransformerFactory getProjectionTransformerFactory() { + return null; + } + + @Override + public PaginationStrategy getPaginationStrategy() { + return null; + } + + @Override + public List getDefaultOrderBy() { + return Collections.emptyList(); + } + + private static final class Row implements ValueProvider { + private final Object[] vals; + + Row(Object[] vals) { + this.vals = vals; + } + + @Override + public int size() { + return vals.length; + } + + @Override + public boolean isNull(int i) { + return vals[i] == null; + } + + @Override + public Boolean getBoolean(int i) { + return (Boolean) vals[i]; + } + + @Override + public Integer getInt(int i) { + return vals[i] instanceof Integer v ? v : null; + } + + @Override + public Long getLong(int i) { + if (vals[i] instanceof Long v) { + return v; + } + return vals[i] instanceof Integer v ? v.longValue() : null; + } + + @Override + public Float getFloat(int i) { + return (Float) vals[i]; + } + + @Override + public Double getDouble(int i) { + return (Double) vals[i]; + } + + @Override + public String getString(int i) { + return vals[i] == null ? null : vals[i].toString(); + } + + @Override + public Instant getInstant(int i) { + return (Instant) vals[i]; + } + + @Override + public LocalDateTime getLocalDateTime(int i) { + return vals[i] instanceof LocalDateTime v ? v : null; + } + + @Override + public LocalDate getLocalDate(int i) { + return vals[i] instanceof LocalDate v ? v : null; + } + + @Override + public HakunaGeometry getHakunaGeometry(int i) { + return (HakunaGeometry) vals[i]; + } + + @Override + public Object[] getArray(int i) { + return null; + } + + @Override + public UUID getUUID(int i) { + return null; + } + + @Override + public Object getObject(int i) { + return vals[i]; + } + } + + private static final class StubStream implements FeatureStream { + private final Iterator it; + + StubStream(List rows) { + this.it = rows.iterator(); + } + + @Override + public boolean hasNext() { + return it.hasNext(); + } + + @Override + public ValueProvider next() { + return it.next(); + } + + @Override + public void close() {} + } + + private static final class StubProperty implements HakunaProperty { + private final String name; + private final HakunaPropertyType type; + + StubProperty(String name, HakunaPropertyType type) { + this.name = name; + this.type = type; + } + + @Override + public String getName() { + return name; + } + + @Override + public String getTable() { + return null; + } + + @Override + public String getColumn() { + return name; + } + + @Override + public HakunaPropertyType getType() { + return type; + } + + @Override + public boolean nullable() { + return true; + } + + @Override + public boolean unique() { + return false; + } + + @Override + public io.swagger.v3.oas.models.media.Schema getSchema() { + return null; + } + + @Override + public boolean isStatic() { + return false; + } + + @Override + public FeatureType getFeatureType() { + return null; + } + + @Override + public void setFeatureType(FeatureType ft) {} + + @Override + public HakunaPropertyWriter getPropertyWriter() { + return null; + } + + @Override + public ValueMapper getMapper(Map columnToIndex, int out, QueryContext ctx) { + return null; + } + + @Override + public Object toInner(String value) { + return value; + } + } + +} diff --git a/src-community/hakunapi-tiles-generator/src/test/java/fi/nls/hakunapi/tiles/generator/TileGzipTest.java b/src-community/hakunapi-tiles-generator/src/test/java/fi/nls/hakunapi/tiles/generator/TileGzipTest.java new file mode 100644 index 00000000..ee3214d1 --- /dev/null +++ b/src-community/hakunapi-tiles-generator/src/test/java/fi/nls/hakunapi/tiles/generator/TileGzipTest.java @@ -0,0 +1,45 @@ +package fi.nls.hakunapi.tiles.generator; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.util.zip.GZIPInputStream; + +import org.junit.Test; + +public class TileGzipTest { + + private static byte[] gunzip(byte[] compressed) throws Exception { + try (GZIPInputStream in = new GZIPInputStream(new ByteArrayInputStream(compressed))) { + return in.readAllBytes(); + } + } + + @Test + public void roundTripsTilesIncludingAnEmptyOne() throws Exception { + byte[] tile = new byte[4096]; + for (int i = 0; i < tile.length; i++) { + tile[i] = (byte) (i * 31); + } + + assertArrayEquals(tile, gunzip(TileGzip.compress(tile))); + assertArrayEquals(new byte[0], gunzip(TileGzip.compress(new byte[0]))); + } + + @Test + public void outputIsAReproducibleGzipMember() throws Exception { + byte[] tile = "the same tile bytes, twice".getBytes(StandardCharsets.UTF_8); + byte[] out = TileGzip.compress(tile); + + assertArrayEquals(out, TileGzip.compress(tile)); + assertEquals((byte) 0x1f, out[0]); + assertEquals((byte) 0x8b, out[1]); + for (int i = 4; i < 8; i++) { + assertEquals(0, out[i]); + } + assertEquals((byte) 255, out[9]); + } + +} diff --git a/src-community/hakunapi-tiles-generator/src/test/java/fi/nls/hakunapi/tiles/generator/TileRangeTest.java b/src-community/hakunapi-tiles-generator/src/test/java/fi/nls/hakunapi/tiles/generator/TileRangeTest.java new file mode 100644 index 00000000..0b961006 --- /dev/null +++ b/src-community/hakunapi-tiles-generator/src/test/java/fi/nls/hakunapi/tiles/generator/TileRangeTest.java @@ -0,0 +1,45 @@ +package fi.nls.hakunapi.tiles.generator; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +import fi.nls.hakunapi.tiles.TileMatrix; + +public class TileRangeTest { + + private static TileMatrix matrix() { + return new TileMatrix("2", 1000, 1.0, 0.0, 1024.0, 256, 256, 4, 4); + } + + @Test + public void rowsGrowDownwardAndATileAlignedEdgeDoesNotPullInTheNextTile() { + TileRange bottomLeft = TileRange.of(matrix(), 2, 0, 0, 256, 256); + assertEquals(3, bottomLeft.minRow()); + assertEquals(3, bottomLeft.maxRow()); + assertEquals(0, bottomLeft.minCol()); + assertEquals(0, bottomLeft.maxCol()); + + TileRange topTwoColumns = TileRange.of(matrix(), 2, 0, 768, 512, 1024); + assertEquals(0, topTwoColumns.minRow()); + assertEquals(0, topTwoColumns.maxRow()); + assertEquals(0, topTwoColumns.minCol()); + assertEquals(1, topTwoColumns.maxCol()); + } + + @Test + public void extentsAreClampedToTheMatrixAndOutsideOnesAreEmpty() { + TileRange all = TileRange.of(matrix(), 2, -5000, -5000, 5000, 5000); + assertEquals(0, all.minRow()); + assertEquals(3, all.maxRow()); + assertEquals(0, all.minCol()); + assertEquals(3, all.maxCol()); + assertEquals(16, all.size()); + + TileRange outside = TileRange.of(matrix(), 2, 5000, 5000, 6000, 6000); + assertTrue(outside.isEmpty()); + assertEquals(0, outside.size()); + } + +} diff --git a/src-community/hakunapi-tiles-generator/src/test/java/fi/nls/hakunapi/tiles/generator/TileSourcesTest.java b/src-community/hakunapi-tiles-generator/src/test/java/fi/nls/hakunapi/tiles/generator/TileSourcesTest.java new file mode 100644 index 00000000..75de8a9f --- /dev/null +++ b/src-community/hakunapi-tiles-generator/src/test/java/fi/nls/hakunapi/tiles/generator/TileSourcesTest.java @@ -0,0 +1,70 @@ +package fi.nls.hakunapi.tiles.generator; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.junit.Test; + +import fi.nls.hakunapi.core.config.HakunaConfigParser; + +public class TileSourcesTest { + + private static GeneratorControl control(String... collectionIds) { + return new GeneratorControl("TEST", + List.of(new ZoomRange("all", 0, 2, List.of(collectionIds), Map.of()))); + } + + private static HakunaConfigParser cfg(String... keyValues) { + Properties p = new Properties(); + for (int i = 0; i < keyValues.length; i += 2) { + p.setProperty(keyValues[i], keyValues[i + 1]); + } + return new HakunaConfigParser(p); + } + + private static void assertRejected(HakunaConfigParser cfg, GeneratorControl control, String... messages) { + IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> TileSources.read(cfg, control)); + for (String message : messages) { + assertTrue(e.getMessage(), e.getMessage().contains(message)); + } + } + + @Test + public void collectionsAreGroupedIntoSourcesInConfigurationOrderOrEachIsItsOwnSource() { + TileSources grouped = TileSources.read(cfg( + "sources", "vesi, maasto", + "vesi.collections", "jarvi, meri", + "maasto.collections", "suo"), + control("jarvi", "meri", "suo")); + + assertEquals(List.of("vesi", "maasto"), grouped.getSourceIds()); + assertEquals("vesi", grouped.getSource("jarvi")); + assertEquals("vesi", grouped.getSource("meri")); + assertEquals("maasto", grouped.getSource("suo")); + + TileSources ungrouped = TileSources.read(cfg(), control("jarvi", "suo")); + + assertEquals(List.of("jarvi", "suo"), ungrouped.getSourceIds()); + assertEquals("suo", ungrouped.getSource("suo")); + } + + @Test + public void inconsistentGroupingsAreRejected() { + assertRejected(cfg("sources", "vesi, maasto", "vesi.collections", "jarvi", + "maasto.collections", "jarvi"), control("jarvi"), "jarvi", "exactly one source"); + assertRejected(cfg("sources", "vesi", "vesi.collections", "jarvi"), + control("jarvi", "suo"), "suo"); + assertRejected(cfg("sources", "vesi", "vesi.collections", "jarvi, meri"), + control("jarvi"), "meri", "no zoom range renders"); + assertRejected(cfg("sources", "vesi"), control("jarvi"), "vesi.collections"); + assertRejected(cfg("sources", "vesi, vesi", "vesi.collections", "jarvi"), + control("jarvi"), "declared twice"); + } + +} diff --git a/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/DirectoryWriter.java b/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/DirectoryWriter.java new file mode 100644 index 00000000..6a61f4c2 --- /dev/null +++ b/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/DirectoryWriter.java @@ -0,0 +1,76 @@ +package fi.nls.hakunapi.tiles.source.pmtiles; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.List; +import java.util.zip.GZIPOutputStream; + +public final class DirectoryWriter { + + private DirectoryWriter() { + } + + public static byte[] encode(List entries, byte compression) throws IOException { + byte[] plain = encode(entries); + if (compression == Compression.NONE || compression == Compression.UNKNOWN) { + return plain; + } + if (compression != Compression.GZIP) { + throw new IOException("Unsupported PMTiles internal compression: " + compression); + } + ByteArrayOutputStream out = new ByteArrayOutputStream(plain.length / 2 + 64); + try (GZIPOutputStream gz = new GZIPOutputStream(out)) { + gz.write(plain); + } + return out.toByteArray(); + } + + public static byte[] encode(List entries) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(entries.size() * 5 + 16); + writeVarint(out, entries.size()); + + long lastId = 0; + for (int i = 0; i < entries.size(); i++) { + long id = entries.get(i).tileId(); + if (id < lastId) { + throw new IOException("PMTiles directory entries must ascend by tile id, but " + + id + " follows " + lastId); + } + writeVarint(out, id - lastId); + lastId = id; + } + for (int i = 0; i < entries.size(); i++) { + writeVarint(out, entries.get(i).runLength()); + } + for (int i = 0; i < entries.size(); i++) { + writeVarint(out, entries.get(i).length()); + } + for (int i = 0; i < entries.size(); i++) { + Directory.Entry e = entries.get(i); + if (i > 0) { + Directory.Entry prev = entries.get(i - 1); + if (e.offset() == prev.offset() + prev.length()) { + // Offset 0 means contiguous with the previous entry (spec). + writeVarint(out, 0); + continue; + } + } + writeVarint(out, e.offset() + 1); + } + return out.toByteArray(); + } + + private static void writeVarint(OutputStream out, long value) throws IOException { + if (value < 0) { + throw new IOException("Cannot encode a negative varint: " + value); + } + long v = value; + while (v >= 0x80) { + out.write((int) ((v & 0x7f) | 0x80)); + v >>>= 7; + } + out.write((int) v); + } + +} diff --git a/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/HeaderWriter.java b/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/HeaderWriter.java new file mode 100644 index 00000000..2f0382d2 --- /dev/null +++ b/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/HeaderWriter.java @@ -0,0 +1,63 @@ +package fi.nls.hakunapi.tiles.source.pmtiles; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; + +public final class HeaderWriter { + + public static final byte CLUSTERED = 1; + public static final byte NOT_CLUSTERED = 0; + + private HeaderWriter() { + } + + public static byte[] build( + long rootDirOffset, long rootDirLength, + long metadataOffset, long metadataLength, + long leafDirsOffset, long leafDirsLength, + long tileDataOffset, long tileDataLength, + long addressedTiles, long tileEntries, long tileContents, + boolean clustered, + byte internalCompression, byte tileCompression, byte tileType, + int minZoom, int maxZoom, + int[] bboxE7, int[] centerE7, int centerZoom) { + + byte[] header = new byte[PMTilesHeader.LENGTH]; + ByteBuffer b = ByteBuffer.wrap(header).order(ByteOrder.LITTLE_ENDIAN); + + byte[] magic = "PMTiles".getBytes(StandardCharsets.US_ASCII); + System.arraycopy(magic, 0, header, 0, magic.length); + header[7] = 3; + + b.putLong(8, rootDirOffset); + b.putLong(16, rootDirLength); + b.putLong(24, metadataOffset); + b.putLong(32, metadataLength); + b.putLong(40, leafDirsOffset); + b.putLong(48, leafDirsLength); + b.putLong(56, tileDataOffset); + b.putLong(64, tileDataLength); + b.putLong(72, addressedTiles); + b.putLong(80, tileEntries); + b.putLong(88, tileContents); + + header[96] = clustered ? CLUSTERED : NOT_CLUSTERED; + header[97] = internalCompression; + header[98] = tileCompression; + header[99] = tileType; + header[100] = (byte) minZoom; + header[101] = (byte) maxZoom; + + b.putInt(102, bboxE7[0]); + b.putInt(106, bboxE7[1]); + b.putInt(110, bboxE7[2]); + b.putInt(114, bboxE7[3]); + header[118] = (byte) centerZoom; + b.putInt(119, centerE7[0]); + b.putInt(123, centerE7[1]); + + return header; + } + +} diff --git a/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/PMTilesWriter.java b/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/PMTilesWriter.java new file mode 100644 index 00000000..7da0b61f --- /dev/null +++ b/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/PMTilesWriter.java @@ -0,0 +1,278 @@ +package fi.nls.hakunapi.tiles.source.pmtiles; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.zip.GZIPOutputStream; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class PMTilesWriter implements AutoCloseable { + + private static final Logger LOG = LoggerFactory.getLogger(PMTilesWriter.class); + + /** Spec: header plus compressed root directory must fit the first 16384 bytes. */ + static final int ROOT_DIR_MAX = 16_384 - PMTilesHeader.LENGTH; + + private static final int FIRST_LEAF_SIZE = 4096; + + private final Path target; + private final Path tempData; + private final FileChannel data; + private final byte tileCompression; + private final byte tileType; + + private final List entries = new ArrayList<>(); + + private final Map> byHash = new HashMap<>(); + + private long dataLength; + private long addressedTiles; + private long lastTileId = -1; + private boolean closed; + private int minZoom = Integer.MAX_VALUE; + private int maxZoom = Integer.MIN_VALUE; + private boolean finished; + + public PMTilesWriter(Path target, byte tileCompression, byte tileType) throws IOException { + this.target = target; + this.tileCompression = tileCompression; + this.tileType = tileType; + this.tempData = Files.createTempFile( + target.toAbsolutePath().getParent(), ".pmtiles-data-", ".tmp"); + this.data = FileChannel.open(tempData, StandardOpenOption.WRITE, StandardOpenOption.READ); + } + + /** Tile ids must ascend: runs only collapse consecutive ids. */ + public void add(int z, long x, long y, byte[] tile) throws IOException { + if (finished) { + throw new IllegalStateException("Archive already finished"); + } + long tileId = TileId.fromZXY(z, x, y); + if (tileId <= lastTileId) { + throw new IllegalArgumentException("Tiles must be added in ascending tile id order, but " + + z + "/" + x + "/" + y + " is id " + tileId + " after " + lastTileId); + } + lastTileId = tileId; + addressedTiles++; + minZoom = Math.min(minZoom, z); + maxZoom = Math.max(maxZoom, z); + + ByteBuffer body = ByteBuffer.wrap(tile); + long hash = hash(tile); + long offset = findDuplicate(body, hash); + if (offset < 0) { + offset = dataLength; + writeFully(data, body); + dataLength += tile.length; + byHash.computeIfAbsent(hash, __ -> new ArrayList<>()) + .add(new long[] { offset, tile.length }); + } + + if (!entries.isEmpty()) { + Directory.Entry last = entries.get(entries.size() - 1); + if (last.tileId() + last.runLength() == tileId + && last.offset() == offset && last.length() == tile.length) { + entries.set(entries.size() - 1, new Directory.Entry( + last.tileId(), last.runLength() + 1, last.offset(), last.length())); + return; + } + } + entries.add(new Directory.Entry(tileId, 1, offset, tile.length)); + } + + private long findDuplicate(ByteBuffer tile, long hash) throws IOException { + List candidates = byHash.get(hash); + if (candidates == null) { + return -1; + } + ByteBuffer buf = null; + for (int i = 0; i < candidates.size(); i++) { + long[] candidate = candidates.get(i); + if (candidate[1] != tile.remaining()) { + continue; + } + if (buf == null) { + buf = ByteBuffer.allocate(tile.remaining()); + } + buf.clear(); + readFully(buf, candidate[0]); + if (buf.flip().equals(tile)) { + return candidate[0]; + } + } + return -1; + } + + private static void writeFully(FileChannel ch, ByteBuffer buf) throws IOException { + while (buf.hasRemaining()) { + ch.write(buf); + } + } + + private void readFully(ByteBuffer buf, long position) throws IOException { + long at = position; + while (buf.hasRemaining()) { + int n = data.read(buf, at); + if (n < 0) { + throw new IOException("Truncated tile data at " + at); + } + at += n; + } + } + + private static long hash(byte[] bytes) { + long h = 0xcbf29ce484222325L; + for (int i = 0; i < bytes.length; i++) { + h ^= bytes[i] & 0xff; + h *= 0x100000001b3L; + } + return h; + } + + public long finish(String metadataJson, int[] bboxE7, int[] centerE7, int centerZoom) + throws IOException { + if (finished) { + throw new IllegalStateException("Archive already finished"); + } + finished = true; + if (entries.isEmpty()) { + throw new IllegalStateException("Refusing to write an archive with no tiles"); + } + + byte internal = Compression.GZIP; + byte[] metadata = gzip(metadataJson.getBytes(StandardCharsets.UTF_8)); + Directories dirs = buildDirectories(internal); + + long rootDirOffset = PMTilesHeader.LENGTH; + long metadataOffset = rootDirOffset + dirs.root.length; + long leafDirsOffset = metadataOffset + metadata.length; + long tileDataOffset = leafDirsOffset + dirs.leaves.length; + + byte[] header = HeaderWriter.build( + rootDirOffset, dirs.root.length, + metadataOffset, metadata.length, + leafDirsOffset, dirs.leaves.length, + tileDataOffset, dataLength, + addressedTiles, entries.size(), countDistinct(), + true, internal, tileCompression, tileType, + minZoom, maxZoom, bboxE7, centerE7, centerZoom); + + Path tmp = target.resolveSibling(target.getFileName() + ".tmp"); + long total; + try { + total = writeArchive(tmp, header, dirs.root, metadata, dirs.leaves); + Files.move(tmp, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException | RuntimeException e) { + try { + Files.deleteIfExists(tmp); + } catch (IOException suppressed) { + e.addSuppressed(suppressed); + } + throw e; + } + + LOG.info("Wrote {} ({} addressed tiles, {} entries, {} distinct bodies, {} leaf dirs, {} bytes)", + target, addressedTiles, entries.size(), countDistinct(), dirs.leafCount, total); + return total; + } + + private long writeArchive(Path out, byte[] header, byte[] root, byte[] metadata, byte[] leaves) + throws IOException { + try (FileChannel ch = FileChannel.open(out, StandardOpenOption.CREATE, + StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING)) { + writeFully(ch, ByteBuffer.wrap(header)); + writeFully(ch, ByteBuffer.wrap(root)); + writeFully(ch, ByteBuffer.wrap(metadata)); + writeFully(ch, ByteBuffer.wrap(leaves)); + long moved = 0; + while (moved < dataLength) { + long n = data.transferTo(moved, dataLength - moved, ch); + if (n <= 0) { + throw new IOException("Failed to copy tile data at " + moved); + } + moved += n; + } + return ch.position(); + } + } + + private long countDistinct() { + long n = 0; + for (List bucket : byHash.values()) { + n += bucket.size(); + } + return n; + } + + private Directories buildDirectories(byte internal) throws IOException { + byte[] flat = DirectoryWriter.encode(entries, internal); + if (flat.length <= ROOT_DIR_MAX) { + return new Directories(flat, new byte[0], 0); + } + for (int leafSize = FIRST_LEAF_SIZE;; leafSize *= 2) { + List root = new ArrayList<>(); + ByteArrayOutputStream leaves = new ByteArrayOutputStream(); + int leafCount = 0; + for (int i = 0; i < entries.size(); i += leafSize) { + List chunk = entries.subList(i, Math.min(i + leafSize, entries.size())); + byte[] encoded = DirectoryWriter.encode(chunk, internal); + // runLength 0 marks a leaf pointer; offset/length address the leaf section. + root.add(new Directory.Entry(chunk.get(0).tileId(), 0, leaves.size(), encoded.length)); + leaves.write(encoded, 0, encoded.length); + leafCount++; + } + byte[] rootBytes = DirectoryWriter.encode(root, internal); + if (rootBytes.length <= ROOT_DIR_MAX) { + return new Directories(rootBytes, leaves.toByteArray(), leafCount); + } + } + } + + private record Directories(byte[] root, byte[] leaves, int leafCount) { + } + + private static byte[] gzip(byte[] bytes) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(bytes.length / 2 + 64); + try (OutputStream gz = new GZIPOutputStream(out)) { + gz.write(bytes); + } + return out.toByteArray(); + } + + public long getAddressedTiles() { + return addressedTiles; + } + + /** Idempotent and never throws, so cleanup cannot fail a finished archive. */ + @Override + public void close() { + if (closed) { + return; + } + closed = true; + try { + data.close(); + } catch (IOException e) { + LOG.warn("Failed to close the temporary tile data file {}", tempData, e); + } + try { + Files.deleteIfExists(tempData); + } catch (IOException e) { + LOG.warn("Failed to delete the temporary tile data file {}", tempData, e); + } + } + +} diff --git a/src-community/hakunapi-tiles-source-pmtiles/src/test/java/fi/nls/hakunapi/tiles/source/pmtiles/PMTilesWriterTest.java b/src-community/hakunapi-tiles-source-pmtiles/src/test/java/fi/nls/hakunapi/tiles/source/pmtiles/PMTilesWriterTest.java new file mode 100644 index 00000000..a74f738a --- /dev/null +++ b/src-community/hakunapi-tiles-source-pmtiles/src/test/java/fi/nls/hakunapi/tiles/source/pmtiles/PMTilesWriterTest.java @@ -0,0 +1,155 @@ +package fi.nls.hakunapi.tiles.source.pmtiles; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class PMTilesWriterTest { + + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + private static final int[] BBOX = { -1800000000, -850000000, 1800000000, 850000000 }; + private static final int[] CENTER = { 0, 0 }; + + private static byte[] body(String s) { + return s.getBytes(StandardCharsets.UTF_8); + } + + private Path write(Writing writing) throws Exception { + Path archive = folder.newFile("out.pmtiles").toPath(); + try (PMTilesWriter writer = new PMTilesWriter(archive, Compression.GZIP, (byte) 1)) { + writing.write(writer); + writer.finish("{\"name\":\"test\"}", BBOX, CENTER, 0); + } + return archive; + } + + private interface Writing { + void write(PMTilesWriter writer) throws Exception; + } + + private static byte[] read(Path archive, int z, long x, long y) throws Exception { + try (PMTilesArchive a = PMTilesArchive.open(new FileRangeReader(archive))) { + return a.getTile(z, x, y).orElse(null); + } + } + + private static int indexOf(byte[] haystack, byte[] needle, int from) { + outer: + for (int i = from; i <= haystack.length - needle.length; i++) { + for (int j = 0; j < needle.length; j++) { + if (haystack[i + j] != needle[j]) { + continue outer; + } + } + return i; + } + return -1; + } + + @Test + public void tilesReadBackAtTheirAddressAndIdenticalBodiesAreStoredOnce() throws Exception { + byte[] repeated = body("the very same bytes"); + Path archive = write(w -> { + w.add(0, 0, 0, body("zero")); + w.add(1, 0, 0, repeated); + w.add(1, 0, 1, repeated); + w.add(1, 1, 1, repeated); + }); + + assertArrayEquals(body("zero"), read(archive, 0, 0, 0)); + assertArrayEquals(repeated, read(archive, 1, 0, 0)); + assertArrayEquals(repeated, read(archive, 1, 0, 1)); + assertArrayEquals(repeated, read(archive, 1, 1, 1)); + assertNull(read(archive, 1, 1, 0)); + + byte[] file = Files.readAllBytes(archive); + int first = indexOf(file, repeated, 0); + assertTrue(first >= 0); + assertEquals(-1, indexOf(file, repeated, first + 1)); + ByteBuffer header = ByteBuffer.wrap(file).order(ByteOrder.LITTLE_ENDIAN); + assertEquals(4, header.getLong(72)); + assertEquals(2, header.getLong(88)); + } + + @Test + public void aLargeArchiveSplitsIntoLeafDirectoriesAndReadsBack() throws Exception { + int n = 100_000; + int z = 9; + long dim = 1L << z; + Random random = new Random(42); + List addresses = new ArrayList<>(n); + for (long i = 0; addresses.size() < n && i < dim * dim; i++) { + addresses.add(new long[] { i % dim, i / dim }); + } + addresses.sort((a, b) -> Long.compare( + TileId.fromZXY(z, a[0], a[1]), TileId.fromZXY(z, b[0], b[1]))); + String[] bodies = new String[n]; + for (int i = 0; i < n; i++) { + bodies[i] = "tile-" + i + "x".repeat(random.nextInt(200)); + } + + Path archive = write(w -> { + for (int i = 0; i < addresses.size(); i++) { + long[] a = addresses.get(i); + w.add(z, a[0], a[1], body(bodies[i])); + } + }); + + ByteBuffer header = ByteBuffer.wrap(Files.readAllBytes(archive)).order(ByteOrder.LITTLE_ENDIAN); + assertNotEquals(0, header.getLong(48)); + assertTrue(header.getLong(16) <= 16_384 - 127); + for (int i : new int[] { 0, addresses.size() / 2, addresses.size() - 1 }) { + long[] a = addresses.get(i); + assertArrayEquals(body(bodies[i]), read(archive, z, a[0], a[1])); + } + } + + @Test + public void theHeaderMatchesTheSpecIndependentlyOfTheReader() throws Exception { + Path archive = write(w -> w.add(0, 0, 0, body("zero"))); + + byte[] file = Files.readAllBytes(archive); + assertEquals("PMTiles", new String(file, 0, 7, StandardCharsets.US_ASCII)); + assertEquals(3, file[7]); + assertEquals(1, file[99]); + assertEquals(127, ByteBuffer.wrap(file).order(ByteOrder.LITTLE_ENDIAN).getLong(8)); + assertEquals(19078479L, TileId.fromZXY(12, 3423, 1763)); + } + + @Test + public void outOfOrderTilesAndEmptyArchivesAreRefused() throws Exception { + Path ordered = folder.newFile("order.pmtiles").toPath(); + try (PMTilesWriter writer = new PMTilesWriter(ordered, Compression.GZIP, (byte) 1)) { + writer.add(1, 1, 1, body("later")); + IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> writer.add(0, 0, 0, body("earlier"))); + assertTrue(e.getMessage(), e.getMessage().contains("ascending")); + } + + Path empty = folder.newFile("empty.pmtiles").toPath(); + try (PMTilesWriter writer = new PMTilesWriter(empty, Compression.GZIP, (byte) 1)) { + IllegalStateException e = assertThrows(IllegalStateException.class, + () -> writer.finish("{}", BBOX, CENTER, 0)); + assertTrue(e.getMessage(), e.getMessage().contains("no tiles")); + } + } + +} diff --git a/src-community/pom.xml b/src-community/pom.xml index 915ebfa1..910385ba 100644 --- a/src-community/pom.xml +++ b/src-community/pom.xml @@ -19,6 +19,7 @@ hakunapi-tiles-source-vectortile hakunapi-tiles-source-gpkg hakunapi-tiles-source-pmtiles + hakunapi-tiles-generator diff --git a/src/hakunapi-gpkg/src/main/java/fi/nls/hakunapi/gpkg/GPKG.java b/src/hakunapi-gpkg/src/main/java/fi/nls/hakunapi/gpkg/GPKG.java index 01cce125..d37ba9df 100644 --- a/src/hakunapi-gpkg/src/main/java/fi/nls/hakunapi/gpkg/GPKG.java +++ b/src/hakunapi-gpkg/src/main/java/fi/nls/hakunapi/gpkg/GPKG.java @@ -132,6 +132,109 @@ public static int insertFeaturesTableEntry(Connection c, GPKGFeaturesTable featu } } + /** Separate from init: a feature-only GeoPackage must not carry empty tile tables. */ + public static void initTiles(Connection c) throws SQLException { + createTileMatrixSetTable(c); + createTileMatrixTable(c); + } + + private static void createTileMatrixSetTable(Connection c) throws SQLException { + String sql = "" + + "CREATE TABLE IF NOT EXISTS gpkg_tile_matrix_set (" + + "table_name TEXT NOT NULL PRIMARY KEY," + + "srs_id INTEGER NOT NULL," + + "min_x DOUBLE NOT NULL," + + "min_y DOUBLE NOT NULL," + + "max_x DOUBLE NOT NULL," + + "max_y DOUBLE NOT NULL," + + "CONSTRAINT fk_gtms_table_name FOREIGN KEY (table_name) REFERENCES gpkg_contents(table_name)," + + "CONSTRAINT fk_gtms_srs FOREIGN KEY (srs_id) REFERENCES gpkg_spatial_ref_sys(srs_id)" + + ")"; + execute(c, sql); + } + + private static void createTileMatrixTable(Connection c) throws SQLException { + String sql = "" + + "CREATE TABLE IF NOT EXISTS gpkg_tile_matrix (" + + "table_name TEXT NOT NULL," + + "zoom_level INTEGER NOT NULL," + + "matrix_width INTEGER NOT NULL," + + "matrix_height INTEGER NOT NULL," + + "tile_width INTEGER NOT NULL," + + "tile_height INTEGER NOT NULL," + + "pixel_x_size DOUBLE NOT NULL," + + "pixel_y_size DOUBLE NOT NULL," + + "CONSTRAINT pk_ttm PRIMARY KEY (table_name, zoom_level)," + + "CONSTRAINT fk_tmm_table_name FOREIGN KEY (table_name) REFERENCES gpkg_contents(table_name)" + + ")"; + execute(c, sql); + } + + public static void createTilePyramidTable(Connection c, String tableName) throws SQLException { + String sql = "" + + "CREATE TABLE " + quote(tableName) + " (" + + "id INTEGER PRIMARY KEY AUTOINCREMENT," + + "zoom_level INTEGER NOT NULL," + + "tile_column INTEGER NOT NULL," + + "tile_row INTEGER NOT NULL," + + "tile_data BLOB NOT NULL," + + "UNIQUE (zoom_level, tile_column, tile_row)" + + ")"; + execute(c, sql); + } + + public static int insertTilesTableEntry(Connection c, GPKGTilesTable tilesTable) throws SQLException { + String sql = "INSERT INTO gpkg_contents (table_name, data_type, identifier, description, min_x, min_y, max_x, max_y, srs_id)" + + " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"; + try (PreparedStatement ps = c.prepareStatement(sql)) { + ps.setString(1, tilesTable.getTableName()); + ps.setString(2, GPKGTilesTable.DATA_TYPE); + ps.setString(3, tilesTable.getIdentifier()); + ps.setString(4, tilesTable.getDescription()); + ps.setDouble(5, tilesTable.getMinX()); + ps.setDouble(6, tilesTable.getMinY()); + ps.setDouble(7, tilesTable.getMaxX()); + ps.setDouble(8, tilesTable.getMaxY()); + ps.setInt(9, tilesTable.getSrid()); + return ps.executeUpdate(); + } + } + + public static int insertTileMatrixSet(Connection c, GPKGTilesTable tilesTable) throws SQLException { + String sql = "INSERT INTO gpkg_tile_matrix_set (table_name, srs_id, min_x, min_y, max_x, max_y)" + + " VALUES (?, ?, ?, ?, ?, ?)"; + try (PreparedStatement ps = c.prepareStatement(sql)) { + ps.setString(1, tilesTable.getTableName()); + ps.setInt(2, tilesTable.getSrid()); + ps.setDouble(3, tilesTable.getMinX()); + ps.setDouble(4, tilesTable.getMinY()); + ps.setDouble(5, tilesTable.getMaxX()); + ps.setDouble(6, tilesTable.getMaxY()); + return ps.executeUpdate(); + } + } + + public static int insertTileMatrix(Connection c, GPKGTileMatrix tileMatrix) throws SQLException { + String sql = "INSERT INTO gpkg_tile_matrix" + + " (table_name, zoom_level, matrix_width, matrix_height, tile_width, tile_height, pixel_x_size, pixel_y_size)" + + " VALUES (?, ?, ?, ?, ?, ?, ?, ?)"; + try (PreparedStatement ps = c.prepareStatement(sql)) { + ps.setString(1, tileMatrix.getTableName()); + ps.setInt(2, tileMatrix.getZoomLevel()); + ps.setInt(3, tileMatrix.getMatrixWidth()); + ps.setInt(4, tileMatrix.getMatrixHeight()); + ps.setInt(5, tileMatrix.getTileWidth()); + ps.setInt(6, tileMatrix.getTileHeight()); + ps.setDouble(7, tileMatrix.getPixelXSize()); + ps.setDouble(8, tileMatrix.getPixelYSize()); + return ps.executeUpdate(); + } + } + + public static String quote(String identifier) { + return '"' + identifier.replace("\"", "\"\"") + '"'; + } + public static int insertGeometryColumn(Connection c, GPKGGeometryColumn geometryColumn) throws SQLException { String sql = "INSERT INTO gpkg_geometry_columns (table_name, column_name, geometry_type_name, srs_id, z, m)" + " VALUES (?, ?, ?, ?, ?, ?)"; diff --git a/src/hakunapi-gpkg/src/main/java/fi/nls/hakunapi/gpkg/GPKGTileMatrix.java b/src/hakunapi-gpkg/src/main/java/fi/nls/hakunapi/gpkg/GPKGTileMatrix.java new file mode 100644 index 00000000..4765efab --- /dev/null +++ b/src/hakunapi-gpkg/src/main/java/fi/nls/hakunapi/gpkg/GPKGTileMatrix.java @@ -0,0 +1,61 @@ +package fi.nls.hakunapi.gpkg; + +/** pixelXSize and pixelYSize are ground units per pixel. */ +public class GPKGTileMatrix { + + public static final String GPKG_TABLE_TILE_MATRIX = "gpkg_tile_matrix"; + + private final String tableName; + private final int zoomLevel; + private final int matrixWidth; + private final int matrixHeight; + private final int tileWidth; + private final int tileHeight; + private final double pixelXSize; + private final double pixelYSize; + + public GPKGTileMatrix(String tableName, int zoomLevel, int matrixWidth, int matrixHeight, + int tileWidth, int tileHeight, double pixelXSize, double pixelYSize) { + this.tableName = tableName; + this.zoomLevel = zoomLevel; + this.matrixWidth = matrixWidth; + this.matrixHeight = matrixHeight; + this.tileWidth = tileWidth; + this.tileHeight = tileHeight; + this.pixelXSize = pixelXSize; + this.pixelYSize = pixelYSize; + } + + public String getTableName() { + return tableName; + } + + public int getZoomLevel() { + return zoomLevel; + } + + public int getMatrixWidth() { + return matrixWidth; + } + + public int getMatrixHeight() { + return matrixHeight; + } + + public int getTileWidth() { + return tileWidth; + } + + public int getTileHeight() { + return tileHeight; + } + + public double getPixelXSize() { + return pixelXSize; + } + + public double getPixelYSize() { + return pixelYSize; + } + +} diff --git a/src/hakunapi-gpkg/src/main/java/fi/nls/hakunapi/gpkg/GPKGTilesTable.java b/src/hakunapi-gpkg/src/main/java/fi/nls/hakunapi/gpkg/GPKGTilesTable.java new file mode 100644 index 00000000..f64a1771 --- /dev/null +++ b/src/hakunapi-gpkg/src/main/java/fi/nls/hakunapi/gpkg/GPKGTilesTable.java @@ -0,0 +1,61 @@ +package fi.nls.hakunapi.gpkg; + +/** The extent is the tiling scheme's, not that of the tiles written. */ +public class GPKGTilesTable { + + public static final String DATA_TYPE = "tiles"; + + private final String tableName; + private final String identifier; + private final String description; + private final double minX; + private final double minY; + private final double maxX; + private final double maxY; + private final int srid; + + public GPKGTilesTable(String tableName, String identifier, String description, + double minX, double minY, double maxX, double maxY, int srid) { + this.tableName = tableName; + this.identifier = identifier; + this.description = description; + this.minX = minX; + this.minY = minY; + this.maxX = maxX; + this.maxY = maxY; + this.srid = srid; + } + + public String getTableName() { + return tableName; + } + + public String getIdentifier() { + return identifier; + } + + public String getDescription() { + return description; + } + + public double getMinX() { + return minX; + } + + public double getMinY() { + return minY; + } + + public double getMaxX() { + return maxX; + } + + public double getMaxY() { + return maxY; + } + + public int getSrid() { + return srid; + } + +}