From eca308742a41c974bc11b8cf4af9ce18a3aa40eb Mon Sep 17 00:00:00 2001 From: jampukka Date: Sat, 17 Oct 2026 00:33:08 +0300 Subject: [PATCH] Add hakunapi-tiles-source-pmtiles: map and vector tile sources reading PMTiles v3 archives --- CHANGELOG.md | 1 + docs/overview/modules.md | 2 + pom.xml | 5 + .../hakunapi-tiles-source-pmtiles/LICENSE | 21 +++ .../hakunapi-tiles-source-pmtiles/README.md | 79 ++++++++ .../hakunapi-tiles-source-pmtiles/SUBMITTERS | 3 + .../hakunapi-tiles-source-pmtiles/pom.xml | 24 +++ .../tiles/source/pmtiles/BoxedTileIndex.java | 150 +++++++++++++++ .../tiles/source/pmtiles/Compression.java | 60 ++++++ .../tiles/source/pmtiles/Directory.java | 116 ++++++++++++ .../tiles/source/pmtiles/FileRangeReader.java | 32 ++++ .../tiles/source/pmtiles/HttpRangeReader.java | 50 +++++ .../tiles/source/pmtiles/PMTilesArchive.java | 115 ++++++++++++ .../tiles/source/pmtiles/PMTilesConfig.java | 18 ++ .../tiles/source/pmtiles/PMTilesHeader.java | 56 ++++++ .../source/pmtiles/PMTilesTileLayer.java | 38 ++++ .../source/pmtiles/PMTilesTileSource.java | 175 ++++++++++++++++++ .../pmtiles/PMTilesVectorTileLayer.java | 44 +++++ .../pmtiles/PMTilesVectorTileSource.java | 55 ++++++ .../tiles/source/pmtiles/RangeReader.java | 10 + .../hakunapi/tiles/source/pmtiles/TileId.java | 88 +++++++++ .../tiles/source/pmtiles/TileIndex.java | 120 ++++++++++++ .../fi.nls.hakunapi.tiles.MapTileSource | 1 + .../fi.nls.hakunapi.tiles.VectorTileSource | 1 + .../source/pmtiles/BoxedTileIndexTest.java | 56 ++++++ .../tiles/source/pmtiles/DirectoryTest.java | 23 +++ .../pmtiles/PMTilesVectorTileSourceTest.java | 90 +++++++++ .../tiles/source/pmtiles/TileIdTest.java | 43 +++++ .../src/test/resources/test_fixture_1.pmtiles | Bin 0 -> 468 bytes src-community/pom.xml | 1 + 30 files changed, 1477 insertions(+) create mode 100644 src-community/hakunapi-tiles-source-pmtiles/LICENSE create mode 100644 src-community/hakunapi-tiles-source-pmtiles/README.md create mode 100644 src-community/hakunapi-tiles-source-pmtiles/SUBMITTERS create mode 100644 src-community/hakunapi-tiles-source-pmtiles/pom.xml create mode 100644 src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/BoxedTileIndex.java create mode 100644 src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/Compression.java create mode 100644 src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/Directory.java create mode 100644 src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/FileRangeReader.java create mode 100644 src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/HttpRangeReader.java create mode 100644 src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/PMTilesArchive.java create mode 100644 src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/PMTilesConfig.java create mode 100644 src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/PMTilesHeader.java create mode 100644 src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/PMTilesTileLayer.java create mode 100644 src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/PMTilesTileSource.java create mode 100644 src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/PMTilesVectorTileLayer.java create mode 100644 src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/PMTilesVectorTileSource.java create mode 100644 src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/RangeReader.java create mode 100644 src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/TileId.java create mode 100644 src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/TileIndex.java create mode 100644 src-community/hakunapi-tiles-source-pmtiles/src/main/resources/META-INF/services/fi.nls.hakunapi.tiles.MapTileSource create mode 100644 src-community/hakunapi-tiles-source-pmtiles/src/main/resources/META-INF/services/fi.nls.hakunapi.tiles.VectorTileSource create mode 100644 src-community/hakunapi-tiles-source-pmtiles/src/test/java/fi/nls/hakunapi/tiles/source/pmtiles/BoxedTileIndexTest.java create mode 100644 src-community/hakunapi-tiles-source-pmtiles/src/test/java/fi/nls/hakunapi/tiles/source/pmtiles/DirectoryTest.java create mode 100644 src-community/hakunapi-tiles-source-pmtiles/src/test/java/fi/nls/hakunapi/tiles/source/pmtiles/PMTilesVectorTileSourceTest.java create mode 100644 src-community/hakunapi-tiles-source-pmtiles/src/test/java/fi/nls/hakunapi/tiles/source/pmtiles/TileIdTest.java create mode 100644 src-community/hakunapi-tiles-source-pmtiles/src/test/resources/test_fixture_1.pmtiles diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e2f735f..c64e3386 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ - A service no longer needs a `collections` property: with none, it starts as a collectionless service, serving the common conformance classes and whatever API extensions publish. - New community module `hakunapi-tiles-source-vectortile`: a `VectorTileSource` that renders Mapbox Vector Tiles in-JVM from the configured feature collections, one layer per collection, with no new third-party dependencies. Not part of the reference webapp. - New community module `hakunapi-tiles-source-gpkg`: a `MapTileSource` and a `VectorTileSource` serving the tiles stored in a GeoPackage tile pyramid, read through `sqlite-jdbc`. Not part of the reference webapp. +- New community module `hakunapi-tiles-source-pmtiles`: a `MapTileSource` and a `VectorTileSource` serving the tiles of a PMTiles v3 archive, read in place from a local file or over HTTP range requests. Not part of the reference webapp. ## 2.0.0 diff --git a/docs/overview/modules.md b/docs/overview/modules.md index e00bd27c..411ca1f7 100644 --- a/docs/overview/modules.md +++ b/docs/overview/modules.md @@ -293,6 +293,8 @@ Key differences between PostGIS and GeoPackage support in Hakunapi: [`hakunapi-tiles-source-gpkg`](../../src-community/hakunapi-tiles-source-gpkg/README.md) *(community module)* is a `MapTileSource` and a `VectorTileSource` serving the tiles stored in a GeoPackage tile pyramid, read through `sqlite-jdbc`. It is found with `ServiceLoader`, so it is enabled by being on the classpath, and is not bundled in the reference webapp. It depends on `hakunapi-tiles-core`, `slf4j-api` and `sqlite-jdbc` (already used by `hakunapi-gpkg`). +[`hakunapi-tiles-source-pmtiles`](../../src-community/hakunapi-tiles-source-pmtiles/README.md) *(community module)* is a `MapTileSource` and a `VectorTileSource` serving the tiles of a PMTiles v3 archive, read in place from a local file or over HTTP range requests. It is found with `ServiceLoader`, so it is enabled by being on the classpath, and is not bundled in the reference webapp. It depends on `hakunapi-tiles-core` and `slf4j-api`; it adds no new third-party dependency. + ## Output data formats (stable) ### GeoJSON diff --git a/pom.xml b/pom.xml index 8b2a75fe..faf439d0 100644 --- a/pom.xml +++ b/pom.xml @@ -89,6 +89,11 @@ hakunapi-tiles-source-gpkg ${project.version} + + fi.nls.hakunapi + hakunapi-tiles-source-pmtiles + ${project.version} + fi.nls.hakunapi hakunapi-config-test diff --git a/src-community/hakunapi-tiles-source-pmtiles/LICENSE b/src-community/hakunapi-tiles-source-pmtiles/LICENSE new file mode 100644 index 00000000..0eb4c375 --- /dev/null +++ b/src-community/hakunapi-tiles-source-pmtiles/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Janne Heikkilä + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/src-community/hakunapi-tiles-source-pmtiles/README.md b/src-community/hakunapi-tiles-source-pmtiles/README.md new file mode 100644 index 00000000..ab7b7f03 --- /dev/null +++ b/src-community/hakunapi-tiles-source-pmtiles/README.md @@ -0,0 +1,79 @@ +# hakunapi-tiles-source-pmtiles + +Community module serving tiles from a +[PMTiles v3](https://github.com/protomaps/PMTiles) archive as OGC API - Tiles +map tiles, under `/collections/{collectionId}/map/tiles`. + +PMTiles is a single-file tile archive addressed by byte range, so an archive is +read in place — from a local file, or from a remote HTTPS URL using HTTP range +requests, with no download or extraction step. The root directory is parsed at +startup and leaf directories are cached (LRU, 64 entries); a tile read is then a +range read plus a relay of the stored bytes. + +## Configuration + +The source registers as type `pmtiles` for raster archives (PNG, JPEG, WebP, +AVIF) and as type `pmtiles-vector` for Mapbox Vector Tile archives. An archive +of the other kind is rejected at startup. Give either a local `path` or a remote +`url`. + +``` +tiles.layers=basemap +tiles.layers.basemap.type=pmtiles +tiles.layers.basemap.title=Basemap +tiles.layers.basemap.tileMatrixSets=WebMercatorQuad + +tiles.layers.basemap.pmtiles.path=/data/basemap.pmtiles +``` + +Remote archive: + +``` +tiles.layers.basemap.pmtiles.url=https://example.org/basemap.pmtiles +``` + +| key | meaning | +| --- | --- | +| `pmtiles.path` | local archive file | +| `pmtiles.url` | remote archive over HTTPS, read with range requests | +| `pmtiles.format` | media type advertised; taken from the archive header when unset | +| `pmtiles.sendfile` | `pmtiles-vector` only; `true` has the container send tile ranges from the archive file; needs `pmtiles.path` | + +`pmtiles.sendfile` builds an in-memory index of tile locations at startup. + +PMTiles addresses tiles by XYZ zoom level, so every tile matrix set the layer is +served in must use numeric tile matrix ids. `WebMercatorQuad` does. + +### Compression + +Tile bytes are relayed exactly as stored. When the archive's `tile_compression` +is gzip, brotli or zstd, the tiles are served with `Content-Encoding: gzip`, +`br` or `zstd` rather than being decompressed and re-compressed. An archive +without compression is served without the header. + +The archive's own directory and metadata must be stored uncompressed or gzipped +— brotli and zstd `internal_compression` are rejected at startup, since +decompressing them would mean a dependency. + +## Usage + +```xml + + fi.nls.hakunapi + hakunapi-tiles-source-pmtiles + +``` + +The source is discovered at runtime via `ServiceLoader`, so it is enough for the +module to be on the classpath. Serving the tiles over HTTP additionally needs +`hakunapi-tiles-servlet-jakarta`. + +## Status and support + +This is a community module under `src-community/`. It is built and +(automatically) tested together with the Hakunapi core modules, but it is not +covered by the support the National Land Survey of Finland provides for core +modules. + +See [SUBMITTERS](SUBMITTERS) for authors and +[CONTRIBUTING.md](../../CONTRIBUTING.md) for how to contribute. diff --git a/src-community/hakunapi-tiles-source-pmtiles/SUBMITTERS b/src-community/hakunapi-tiles-source-pmtiles/SUBMITTERS new file mode 100644 index 00000000..46d30fd3 --- /dev/null +++ b/src-community/hakunapi-tiles-source-pmtiles/SUBMITTERS @@ -0,0 +1,3 @@ +Submitted and maintained by: + +jampukka diff --git a/src-community/hakunapi-tiles-source-pmtiles/pom.xml b/src-community/hakunapi-tiles-source-pmtiles/pom.xml new file mode 100644 index 00000000..0a8a49cb --- /dev/null +++ b/src-community/hakunapi-tiles-source-pmtiles/pom.xml @@ -0,0 +1,24 @@ + + 4.0.0 + + fi.nls.hakunapi + src-community + 2.1.0-SNAPSHOT + + hakunapi-tiles-source-pmtiles + + + fi.nls.hakunapi + hakunapi-tiles-core + + + org.slf4j + slf4j-api + + + junit + junit + test + + + diff --git a/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/BoxedTileIndex.java b/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/BoxedTileIndex.java new file mode 100644 index 00000000..b30a28ae --- /dev/null +++ b/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/BoxedTileIndex.java @@ -0,0 +1,150 @@ +package fi.nls.hakunapi.tiles.source.pmtiles; + +import fi.nls.hakunapi.tiles.TileLocator; + +public class BoxedTileIndex implements TileLocator { + private static final long EMPTY = 0L; + + public static final double MIN_DENSITY = 0.5; + + private final Zoom[] zooms; + + private final long tileDataOffset; + private final byte tileCompression; + private final String mediaType; + + private static final class Zoom { + final long xMin; + final long yMin; + final long width; + final long height; + final long[] locations; + + Zoom(long xMin, long yMin, long width, long height) { + this.xMin = xMin; + this.yMin = yMin; + this.width = width; + this.height = height; + this.locations = new long[Math.toIntExact(width * height)]; + } + } + + private BoxedTileIndex(Zoom[] zooms, long tileDataOffset, byte tileCompression, + String mediaType) { + this.zooms = zooms; + this.tileDataOffset = tileDataOffset; + this.tileCompression = tileCompression; + this.mediaType = mediaType; + } + + public static BoxedTileIndex of(TileIndex sorted) { + long[] xMin = new long[32]; + long[] xMax = new long[32]; + long[] yMin = new long[32]; + long[] yMax = new long[32]; + long[] counts = new long[32]; + java.util.Arrays.fill(xMin, Long.MAX_VALUE); + java.util.Arrays.fill(yMin, Long.MAX_VALUE); + java.util.Arrays.fill(xMax, Long.MIN_VALUE); + java.util.Arrays.fill(yMax, Long.MIN_VALUE); + + sorted.forEachEntry((tileId, packed) -> { + long[] zxy = TileId.toZXY(tileId); + int z = (int) zxy[0]; + long x = zxy[1]; + long y = zxy[2]; + counts[z]++; + if (x < xMin[z]) xMin[z] = x; + if (x > xMax[z]) xMax[z] = x; + if (y < yMin[z]) yMin[z] = y; + if (y > yMax[z]) yMax[z] = y; + }); + + Zoom[] zooms = new Zoom[32]; + for (int z = 0; z < 32; z++) { + if (counts[z] == 0) { + continue; + } + long w = xMax[z] - xMin[z] + 1; + long h = yMax[z] - yMin[z] + 1; + long slots = w * h; + + if (slots > Integer.MAX_VALUE || (double) counts[z] / slots < MIN_DENSITY) { + return null; + } + zooms[z] = new Zoom(xMin[z], yMin[z], w, h); + } + + sorted.forEachEntry((tileId, packed) -> { + long[] zxy = TileId.toZXY(tileId); + Zoom zoom = zooms[(int) zxy[0]]; + zoom.locations[slot(zoom, zxy[1], zxy[2])] = packed; + }); + + return new BoxedTileIndex(zooms, sorted.tileDataOffset(), sorted.tileCompression(), + sorted.getMediaType()); + } + + private static int slot(Zoom zoom, long x, long y) { + return Math.toIntExact((y - zoom.yMin) * zoom.width + (x - zoom.xMin)); + } + + @Override + public long find(int z, long x, long y) { + if (z < 0 || z >= zooms.length) { + return NOT_FOUND; + } + Zoom zoom = zooms[z]; + if (zoom == null) { + return NOT_FOUND; + } + long dx = x - zoom.xMin; + long dy = y - zoom.yMin; + + if (Long.compareUnsigned(dx, zoom.width) >= 0 + || Long.compareUnsigned(dy, zoom.height) >= 0) { + return NOT_FOUND; + } + long packed = zoom.locations[(int) (dy * zoom.width + dx)]; + if (packed == EMPTY) { + return NOT_FOUND; + } + return TileLocator.pack(tileDataOffset + TileLocator.offset(packed), + TileLocator.length(packed)); + } + + @Override + public String getContentEncoding() { + return Compression.contentEncoding(tileCompression); + } + + @Override + public String getMediaType() { + return mediaType; + } + + public long heapBytes() { + long bytes = 0; + for (Zoom zoom : zooms) { + if (zoom != null) { + bytes += (long) zoom.locations.length * Long.BYTES; + } + } + return bytes; + } + + public String describe() { + StringBuilder sb = new StringBuilder(); + for (int z = 0; z < zooms.length; z++) { + if (zooms[z] == null) { + continue; + } + if (sb.length() > 0) { + sb.append(", "); + } + sb.append('z').append(z).append(' ') + .append(zooms[z].width).append('x').append(zooms[z].height); + } + return sb.toString(); + } +} diff --git a/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/Compression.java b/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/Compression.java new file mode 100644 index 00000000..627c6ccf --- /dev/null +++ b/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/Compression.java @@ -0,0 +1,60 @@ +package fi.nls.hakunapi.tiles.source.pmtiles; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.zip.GZIPInputStream; + +public final class Compression { + public static final byte UNKNOWN = 0; + public static final byte NONE = 1; + public static final byte GZIP = 2; + public static final byte BROTLI = 3; + public static final byte ZSTD = 4; + + private static final int MAX_DECOMPRESSED = 64 << 20; + + private Compression() { + } + + public static byte[] decode(byte type, byte[] bytes) throws IOException { + switch (type) { + case NONE: + case UNKNOWN: + return bytes; + case GZIP: + return gunzip(bytes); + default: + throw new IOException("Unsupported PMTiles internal compression: " + type + + " (only none/gzip supported for directories)"); + } + } + + public static String contentEncoding(byte type) { + switch (type) { + case GZIP: + return "gzip"; + case BROTLI: + return "br"; + case ZSTD: + return "zstd"; + default: + return null; + } + } + + private static byte[] gunzip(byte[] bytes) throws IOException { + try (GZIPInputStream in = new GZIPInputStream(new ByteArrayInputStream(bytes))) { + ByteArrayOutputStream out = new ByteArrayOutputStream((int) Math.min(bytes.length * 2L, 1 << 20)); + byte[] buf = new byte[8192]; + int n; + while ((n = in.read(buf)) != -1) { + if (out.size() + n > MAX_DECOMPRESSED) { + throw new IOException("PMTiles gzip data inflates past " + MAX_DECOMPRESSED + " bytes"); + } + out.write(buf, 0, n); + } + return out.toByteArray(); + } + } +} diff --git a/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/Directory.java b/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/Directory.java new file mode 100644 index 00000000..9ee12b30 --- /dev/null +++ b/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/Directory.java @@ -0,0 +1,116 @@ +package fi.nls.hakunapi.tiles.source.pmtiles; + +import java.io.IOException; + +public final class Directory { + private final long[] tileIds; + private final long[] runLengths; + private final long[] offsets; + private final long[] lengths; + + private Directory(long[] tileIds, long[] runLengths, long[] offsets, long[] lengths) { + this.tileIds = tileIds; + this.runLengths = runLengths; + this.offsets = offsets; + this.lengths = lengths; + } + + public static Directory parse(byte[] bytes) throws IOException { + Varint v = new Varint(bytes); + long count = v.next(); + if (count < 0 || count > bytes.length / 4) { + throw new IOException("PMTiles directory entry count out of range: " + count); + } + int n = (int) count; + long[] tileIds = new long[n]; + long[] runLengths = new long[n]; + long[] offsets = new long[n]; + long[] lengths = new long[n]; + + long lastId = 0; + for (int i = 0; i < n; i++) { + lastId += v.next(); + tileIds[i] = lastId; + } + for (int i = 0; i < n; i++) { + runLengths[i] = v.next(); + } + for (int i = 0; i < n; i++) { + lengths[i] = v.next(); + } + for (int i = 0; i < n; i++) { + long o = v.next(); + if (o == 0) { + offsets[i] = i > 0 ? offsets[i - 1] + lengths[i - 1] : 0; + } else { + offsets[i] = o - 1; + } + } + return new Directory(tileIds, runLengths, offsets, lengths); + } + + public Entry find(long tileId) { + int lo = 0; + int hi = tileIds.length - 1; + int found = -1; + while (lo <= hi) { + int mid = (lo + hi) >>> 1; + if (tileIds[mid] <= tileId) { + found = mid; + lo = mid + 1; + } else { + hi = mid - 1; + } + } + if (found < 0) { + return null; + } + long rl = runLengths[found]; + if (rl == 0) { + return new Entry(tileIds[found], 0, offsets[found], lengths[found]); + } + if (tileId < tileIds[found] + rl) { + return new Entry(tileIds[found], rl, offsets[found], lengths[found]); + } + return null; + } + + public int size() { + return tileIds.length; + } + + public Entry get(int i) { + return new Entry(tileIds[i], runLengths[i], offsets[i], lengths[i]); + } + + public record Entry(long tileId, long runLength, long offset, long length) { + public boolean isLeaf() { + return runLength == 0; + } + } + + private static final class Varint { + private final byte[] buf; + private int pos; + + Varint(byte[] buf) { + this.buf = buf; + } + + long next() throws IOException { + long result = 0; + int shift = 0; + while (true) { + if (pos >= buf.length) { + throw new IOException("Truncated PMTiles directory varint"); + } + int b = buf[pos++] & 0xff; + result |= (long) (b & 0x7f) << shift; + if ((b & 0x80) == 0) { + return result; + } + shift += 7; + } + } + } +} diff --git a/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/FileRangeReader.java b/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/FileRangeReader.java new file mode 100644 index 00000000..03352d23 --- /dev/null +++ b/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/FileRangeReader.java @@ -0,0 +1,32 @@ +package fi.nls.hakunapi.tiles.source.pmtiles; + +import java.io.EOFException; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; + +public class FileRangeReader implements RangeReader { + private final FileChannel channel; + + public FileRangeReader(Path path) throws IOException { + this.channel = FileChannel.open(path, StandardOpenOption.READ); + } + + @Override + public byte[] readRange(long offset, int length) throws IOException { + ByteBuffer buf = ByteBuffer.allocate(length); + while (buf.hasRemaining()) { + if (channel.read(buf, offset + buf.position()) < 0) { + throw new EOFException("PMTiles range past end of file: offset=" + offset + " length=" + length); + } + } + return buf.array(); + } + + @Override + public void close() throws IOException { + channel.close(); + } +} diff --git a/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/HttpRangeReader.java b/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/HttpRangeReader.java new file mode 100644 index 00000000..e2f58aec --- /dev/null +++ b/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/HttpRangeReader.java @@ -0,0 +1,50 @@ +package fi.nls.hakunapi.tiles.source.pmtiles; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; + +public class HttpRangeReader implements RangeReader { + private final HttpClient http = HttpClient.newBuilder() + .connectTimeout(Duration.ofSeconds(10)) + .followRedirects(HttpClient.Redirect.NORMAL) + .build(); + + private final URI uri; + + public HttpRangeReader(URI uri) { + this.uri = uri; + } + + @Override + public byte[] readRange(long offset, int length) throws IOException, InterruptedException { + long end = offset + length - 1; + HttpRequest req = HttpRequest.newBuilder(uri) + .timeout(Duration.ofSeconds(30)) + .header("Range", "bytes=" + offset + "-" + end) + .GET() + .build(); + HttpResponse resp = http.send(req, HttpResponse.BodyHandlers.ofByteArray()); + int status = resp.statusCode(); + + if (status == 200) { + throw new IOException("PMTiles server does not support range requests: " + uri); + } + if (status != 206) { + throw new IOException("PMTiles range request failed: HTTP " + status + " for " + uri); + } + byte[] body = resp.body(); + if (body.length != length) { + throw new IOException("PMTiles short range read: wanted " + length + " got " + body.length); + } + return body; + } + + @Override + public void close() { + http.close(); + } +} diff --git a/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/PMTilesArchive.java b/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/PMTilesArchive.java new file mode 100644 index 00000000..76989d23 --- /dev/null +++ b/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/PMTilesArchive.java @@ -0,0 +1,115 @@ +package fi.nls.hakunapi.tiles.source.pmtiles; + +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; + +public class PMTilesArchive implements AutoCloseable { + private static final int MAX_LEAF_DEPTH = 4; + private static final long MAX_RANGE_LENGTH = 64L << 20; + private static final int LEAF_CACHE_SIZE = 64; + + private final RangeReader reader; + private final PMTilesHeader header; + private final Directory rootDir; + + private final Map leafCache = new LinkedHashMap<>(16, 0.75f, true) { + @Override + protected boolean removeEldestEntry(Map.Entry eldest) { + return size() > LEAF_CACHE_SIZE; + } + }; + + private PMTilesArchive(RangeReader reader, PMTilesHeader header, Directory rootDir) { + this.reader = reader; + this.header = header; + this.rootDir = rootDir; + } + + public static PMTilesArchive open(RangeReader reader) throws Exception { + byte[] headerBytes = reader.readRange(0, PMTilesHeader.LENGTH); + PMTilesHeader header = PMTilesHeader.parse(headerBytes); + byte[] rootRaw = reader.readRange(header.rootDirOffset(), toLength(header.rootDirLength())); + Directory rootDir = Directory.parse(Compression.decode(header.internalCompression(), rootRaw)); + return new PMTilesArchive(reader, header, rootDir); + } + + public PMTilesHeader getHeader() { + return header; + } + + public Optional getTile(int z, long x, long y) throws Exception { + TileMeta meta = getTileMeta(z, x, y); + if (meta == null) { + return Optional.empty(); + } + return Optional.of(reader.readRange(meta.offset(), meta.length())); + } + + public TileMeta getTileMeta(int z, long x, long y) throws Exception { + long tileId = TileId.fromZXY(z, x, y); + Directory dir = rootDir; + for (int depth = 0; depth <= MAX_LEAF_DEPTH; depth++) { + Directory.Entry e = dir.find(tileId); + if (e == null) { + return null; + } + if (!e.isLeaf()) { + return new TileMeta(header.tileDataOffset() + e.offset(), toLength(e.length()), + header.tileCompression(), header.tileMediaType()); + } + dir = leafDirectory(e.offset(), toLength(e.length())); + } + throw new IllegalStateException("PMTiles leaf directory chain exceeded depth " + MAX_LEAF_DEPTH); + } + + private Directory leafDirectory(long offset, int length) throws Exception { + synchronized (leafCache) { + Directory cached = leafCache.get(offset); + if (cached != null) { + return cached; + } + } + byte[] leafRaw = reader.readRange(header.leafDirsOffset() + offset, length); + Directory dir = Directory.parse(Compression.decode(header.internalCompression(), leafRaw)); + synchronized (leafCache) { + leafCache.put(offset, dir); + } + return dir; + } + + static int toLength(long v) throws IOException { + if (v < 0 || v > MAX_RANGE_LENGTH) { + throw new IOException("PMTiles range length out of range: " + v); + } + return (int) v; + } + + public record TileMeta(long offset, int length, byte compression, String mediaType) {} + + public void forEachEntry(TileIndex.EntryVisitor visitor) throws Exception { + visit(rootDir, visitor, 0); + } + + private void visit(Directory dir, TileIndex.EntryVisitor visitor, int depth) throws Exception { + if (depth > MAX_LEAF_DEPTH) { + throw new IllegalStateException("PMTiles leaf directory chain exceeded depth " + MAX_LEAF_DEPTH); + } + for (int i = 0; i < dir.size(); i++) { + Directory.Entry e = dir.get(i); + if (e.isLeaf()) { + byte[] leafRaw = reader.readRange(header.leafDirsOffset() + e.offset(), toLength(e.length())); + visit(Directory.parse(Compression.decode(header.internalCompression(), leafRaw)), + visitor, depth + 1); + } else { + visitor.accept(e.tileId(), e.runLength(), e.offset(), e.length()); + } + } + } + + @Override + public void close() throws Exception { + reader.close(); + } +} diff --git a/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/PMTilesConfig.java b/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/PMTilesConfig.java new file mode 100644 index 00000000..09d7743a --- /dev/null +++ b/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/PMTilesConfig.java @@ -0,0 +1,18 @@ +package fi.nls.hakunapi.tiles.source.pmtiles; + +import java.nio.file.Path; +import java.util.Set; + +import fi.nls.hakunapi.tiles.TileLocator; + +public record PMTilesConfig( + PMTilesArchive archive, + String mediaType, + Set tmsIds, + boolean sendfile, + TileLocator index, + Path archiveFile) { + public Path sendfileArchive() { + return sendfile ? archiveFile : null; + } +} diff --git a/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/PMTilesHeader.java b/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/PMTilesHeader.java new file mode 100644 index 00000000..18041ff4 --- /dev/null +++ b/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/PMTilesHeader.java @@ -0,0 +1,56 @@ +package fi.nls.hakunapi.tiles.source.pmtiles; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; + +public record PMTilesHeader( + long rootDirOffset, + long rootDirLength, + long leafDirsOffset, + long tileDataOffset, + byte internalCompression, + byte tileCompression, + byte tileType) { + /** Header size in bytes. */ + public static final int LENGTH = 127; + + private static final byte[] MAGIC = "PMTiles".getBytes(StandardCharsets.US_ASCII); + + public static PMTilesHeader parse(byte[] bytes) throws IOException { + if (bytes.length < LENGTH) { + throw new IOException("PMTiles header too short: " + bytes.length); + } + for (int i = 0; i < MAGIC.length; i++) { + if (bytes[i] != MAGIC[i]) { + throw new IOException("Not a PMTiles archive: bad magic"); + } + } + int version = bytes[7] & 0xff; + if (version != 3) { + throw new IOException("Unsupported PMTiles version: " + version + " (only v3)"); + } + ByteBuffer b = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN); + long rootDirOffset = b.getLong(8); + long rootDirLength = b.getLong(16); + long leafDirsOffset = b.getLong(40); + long tileDataOffset = b.getLong(56); + byte internalCompression = bytes[97]; + byte tileCompression = bytes[98]; + byte tileType = bytes[99]; + return new PMTilesHeader(rootDirOffset, rootDirLength, leafDirsOffset, + tileDataOffset, internalCompression, tileCompression, tileType); + } + + public String tileMediaType() { + switch (tileType) { + case 1: return "application/vnd.mapbox-vector-tile"; + case 2: return "image/png"; + case 3: return "image/jpeg"; + case 4: return "image/webp"; + case 5: return "image/avif"; + default: return "application/octet-stream"; + } + } +} diff --git a/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/PMTilesTileLayer.java b/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/PMTilesTileLayer.java new file mode 100644 index 00000000..aa54dc38 --- /dev/null +++ b/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/PMTilesTileLayer.java @@ -0,0 +1,38 @@ +package fi.nls.hakunapi.tiles.source.pmtiles; + +import java.nio.file.Path; +import java.util.List; + +import fi.nls.hakunapi.tiles.AcceleratedTileLayer; +import fi.nls.hakunapi.tiles.MapTileLayer; +import fi.nls.hakunapi.tiles.MapTileSource; +import fi.nls.hakunapi.tiles.TileLocator; + +public class PMTilesTileLayer extends MapTileLayer implements AcceleratedTileLayer { + private final PMTilesConfig pmtilesConfig; + + public PMTilesTileLayer(String id, String title, String description, MapTileSource source, + List tileMatrixSetIds, List mediaTypes, double[] bbox, PMTilesConfig pmtilesConfig) { + super(id, title, description, source, tileMatrixSetIds, mediaTypes, bbox); + this.pmtilesConfig = pmtilesConfig; + } + + public PMTilesConfig getPMTilesConfig() { + return pmtilesConfig; + } + + @Override + public TileLocator getTileLocator() { + return pmtilesConfig.index(); + } + + @Override + public String getInternalLocation() { + return null; + } + + @Override + public Path getTileFile() { + return pmtilesConfig.sendfileArchive(); + } +} diff --git a/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/PMTilesTileSource.java b/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/PMTilesTileSource.java new file mode 100644 index 00000000..4a7090a9 --- /dev/null +++ b/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/PMTilesTileSource.java @@ -0,0 +1,175 @@ +package fi.nls.hakunapi.tiles.source.pmtiles; + +import java.net.URI; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import fi.nls.hakunapi.core.config.HakunaConfigParser; +import fi.nls.hakunapi.tiles.BytesTile; +import fi.nls.hakunapi.tiles.Tile; +import fi.nls.hakunapi.tiles.MapTileLayer; +import fi.nls.hakunapi.tiles.TileMatrixSetRegistry; +import fi.nls.hakunapi.tiles.MapTileSource; +import fi.nls.hakunapi.tiles.config.TilesConfigParser; + +public class PMTilesTileSource implements MapTileSource { + private static final Logger LOG = LoggerFactory.getLogger(PMTilesTileSource.class); + + public static final String TYPE = "pmtiles"; + + static final String MVT = "application/vnd.mapbox-vector-tile"; + + private final List archives = new ArrayList<>(); + + @Override + public String getType() { + return TYPE; + } + + @Override + public MapTileLayer parse(HakunaConfigParser cfg, Path path, TileMatrixSetRegistry tms, String layerId) + throws Exception { + Parsed parsed = parse(cfg, path, layerId, archives, false); + return new PMTilesTileLayer(layerId, parsed.title(), parsed.description(), this, + parsed.tmsIds(), List.of(parsed.config().mediaType()), null, parsed.config()); + } + + record Parsed(String title, String description, List tmsIds, PMTilesConfig config) {} + + static Parsed parse(HakunaConfigParser cfg, Path path, String layerId, + List archives, boolean vector) throws Exception { + TilesConfigParser tiles = new TilesConfigParser(cfg); + String p = "tiles.layers." + layerId + "."; + String pm = p + "pmtiles."; + + String title = cfg.get(p + "title", layerId); + String description = cfg.get(p + "description", title); + List tmsIds = tiles.readTileMatrixSetIds(layerId); + + String filePath = cfg.get(pm + "path"); + String url = cfg.get(pm + "url"); + if ((filePath == null) == (url == null)) { + throw new IllegalArgumentException( + "tile layer " + layerId + ": exactly one of pmtiles.path or pmtiles.url is required"); + } + + RangeReader reader; + String location; + if (filePath != null) { + Path archivePath = path != null ? path.resolveSibling(filePath) : Path.of(filePath); + reader = new FileRangeReader(archivePath); + location = archivePath.toString(); + } else { + reader = new HttpRangeReader(URI.create(url)); + location = url; + } + + PMTilesArchive archive; + try { + archive = PMTilesArchive.open(reader); + } catch (Exception e) { + reader.close(); + throw e; + } + String mediaType = cfg.get(pm + "format", archive.getHeader().tileMediaType()); + + boolean mvt = MVT.equals(archive.getHeader().tileMediaType()); + if (mvt != vector) { + archive.close(); + throw new IllegalArgumentException("tile layer " + layerId + ": " + location + + " holds " + archive.getHeader().tileMediaType() + ", so its type is " + + (mvt ? PMTilesVectorTileSource.TYPE : TYPE) + ", not " + + (vector ? PMTilesVectorTileSource.TYPE : TYPE)); + } + archives.add(archive); + + boolean sendfile = Boolean.parseBoolean(cfg.get(pm + "sendfile", "false")); + if (sendfile && filePath == null) { + throw new IllegalArgumentException("tile layer " + layerId + + ": pmtiles.sendfile needs pmtiles.path; the container sends a range of a" + + " file on disk, and a remote archive is not one"); + } + if (!vector && sendfile) { + throw new IllegalArgumentException("tile layer " + layerId + + ": pmtiles.sendfile needs type " + PMTilesVectorTileSource.TYPE); + } + fi.nls.hakunapi.tiles.TileLocator index = null; + if (sendfile) { + long started = System.currentTimeMillis(); + TileIndex sorted = TileIndex.build(archive); + BoxedTileIndex boxed = BoxedTileIndex.of(sorted); + index = boxed != null ? boxed : sorted; + LOG.info("pmtiles tile layer {} indexed ({}) for sendfile -> {} tiles, {} kB{}, in {} ms", + layerId, boxed != null ? "per-zoom arrays" : "sorted", + sorted.size(), + (boxed != null ? boxed.heapBytes() : sorted.heapBytes()) / 1024, + boxed != null ? " [" + boxed.describe() + "]" : "", + System.currentTimeMillis() - started); + } + + PMTilesConfig config = new PMTilesConfig(archive, mediaType, java.util.Set.copyOf(tmsIds), + sendfile, index, filePath != null ? Path.of(location) : null); + LOG.info("pmtiles tile layer {} -> {} type={}", layerId, location, mediaType); + return new Parsed(title, description, tmsIds, config); + } + + @Override + public Optional getTile(MapTileLayer layer, String tileMatrixSetId, String tileMatrix, + long tileRow, long tileCol, String mediaType, Map requestParams) throws Exception { + return readTile(((PMTilesTileLayer) layer).getPMTilesConfig(), tileMatrixSetId, tileMatrix, + tileRow, tileCol); + } + + static Optional readTile(PMTilesConfig cfg, String tileMatrixSetId, String tileMatrix, + long tileRow, long tileCol) throws Exception { + if (!cfg.tmsIds().contains(tileMatrixSetId)) { + return Optional.empty(); + } + + int z; + try { + z = Integer.parseInt(tileMatrix); + } catch (NumberFormatException e) { + return Optional.empty(); + } + if (!TileId.inRange(z, tileCol, tileRow)) { + return Optional.empty(); + } + + Optional bytes = cfg.archive().getTile(z, tileCol, tileRow); + if (bytes.isEmpty()) { + return Optional.empty(); + } + String contentEncoding = Compression.contentEncoding(cfg.archive().getHeader().tileCompression()); + return Optional.of(new BytesTile(tileMatrixSetId, tileMatrix, tileRow, tileCol, + cfg.mediaType(), contentEncoding, bytes.get())); + } + + @Override + public void close() throws Exception { + closeAll(archives); + } + + static void closeAll(List archives) throws Exception { + Exception first = null; + for (PMTilesArchive a : archives) { + try { + a.close(); + } catch (Exception e) { + if (first == null) { + first = e; + } + } + } + archives.clear(); + if (first != null) { + throw first; + } + } +} diff --git a/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/PMTilesVectorTileLayer.java b/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/PMTilesVectorTileLayer.java new file mode 100644 index 00000000..d049e504 --- /dev/null +++ b/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/PMTilesVectorTileLayer.java @@ -0,0 +1,44 @@ +package fi.nls.hakunapi.tiles.source.pmtiles; + +import java.nio.file.Path; +import java.util.List; + +import fi.nls.hakunapi.tiles.AcceleratedTileLayer; +import fi.nls.hakunapi.tiles.TileLocator; +import fi.nls.hakunapi.tiles.VectorTileLayer; +import fi.nls.hakunapi.tiles.VectorTileSource; + +public class PMTilesVectorTileLayer extends VectorTileLayer implements AcceleratedTileLayer { + private final PMTilesConfig pmtilesConfig; + + public PMTilesVectorTileLayer(String id, String title, String description, VectorTileSource source, + List collectionIds, List tileMatrixSetIds, List mediaTypes, + double[] bbox, PMTilesConfig pmtilesConfig) { + super(id, title, description, source, collectionIds, tileMatrixSetIds, mediaTypes, bbox); + this.pmtilesConfig = pmtilesConfig; + } + + public PMTilesConfig getPMTilesConfig() { + return pmtilesConfig; + } + + @Override + public TileLocator getTileLocator() { + return pmtilesConfig.index(); + } + + @Override + public String getInternalLocation() { + return null; + } + + @Override + public Path getTileFile() { + return pmtilesConfig.sendfileArchive(); + } + + @Override + public boolean isStoredTileset() { + return true; + } +} diff --git a/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/PMTilesVectorTileSource.java b/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/PMTilesVectorTileSource.java new file mode 100644 index 00000000..ea3bd922 --- /dev/null +++ b/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/PMTilesVectorTileSource.java @@ -0,0 +1,55 @@ +package fi.nls.hakunapi.tiles.source.pmtiles; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import fi.nls.hakunapi.core.FeatureType; +import fi.nls.hakunapi.core.config.HakunaConfigParser; +import fi.nls.hakunapi.tiles.BytesTile; +import fi.nls.hakunapi.tiles.Tile; +import fi.nls.hakunapi.tiles.TileMatrixSetRegistry; +import fi.nls.hakunapi.tiles.VectorTileLayer; +import fi.nls.hakunapi.tiles.VectorTileSource; + +public class PMTilesVectorTileSource implements VectorTileSource { + public static final String TYPE = "pmtiles-vector"; + + private final List archives = new ArrayList<>(); + + @Override + public String getType() { + return TYPE; + } + + @Override + public VectorTileLayer parse(HakunaConfigParser cfg, Path path, TileMatrixSetRegistry tms, String layerId) + throws Exception { + PMTilesTileSource.Parsed parsed = PMTilesTileSource.parse(cfg, path, layerId, archives, true); + List collectionIds = List.of( + cfg.getMultiple("tiles.layers." + layerId + ".collections", new String[] { layerId })); + return new PMTilesVectorTileLayer(layerId, parsed.title(), parsed.description(), this, + collectionIds, parsed.tmsIds(), List.of(parsed.config().mediaType()), null, + parsed.config()); + } + + @Override + public boolean needsCollections(VectorTileLayer layer) { + return false; + } + + @Override + public Optional getTile(VectorTileLayer layer, String tileMatrixSetId, String tileMatrix, + long tileRow, long tileCol, List collections, String mediaType, + Map requestParams) throws Exception { + PMTilesConfig cfg = ((PMTilesVectorTileLayer) layer).getPMTilesConfig(); + return PMTilesTileSource.readTile(cfg, tileMatrixSetId, tileMatrix, tileRow, tileCol); + } + + @Override + public void close() throws Exception { + PMTilesTileSource.closeAll(archives); + } +} diff --git a/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/RangeReader.java b/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/RangeReader.java new file mode 100644 index 00000000..0378fffb --- /dev/null +++ b/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/RangeReader.java @@ -0,0 +1,10 @@ +package fi.nls.hakunapi.tiles.source.pmtiles; + +public interface RangeReader extends AutoCloseable { + /** Offset and length in bytes; the returned array is owned by the caller. */ + byte[] readRange(long offset, int length) throws Exception; + + @Override + default void close() throws Exception { + } +} diff --git a/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/TileId.java b/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/TileId.java new file mode 100644 index 00000000..38c7aee2 --- /dev/null +++ b/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/TileId.java @@ -0,0 +1,88 @@ +package fi.nls.hakunapi.tiles.source.pmtiles; + +public final class TileId { + private TileId() { + } + + public static boolean inRange(int z, long x, long y) { + if (z < 0 || z > 31) { + return false; + } + long dim = 1L << z; + return x >= 0 && y >= 0 && x < dim && y < dim; + } + + public static long fromZXY(int z, long x, long y) { + if (z > 31) { + throw new IllegalArgumentException("PMTiles zoom out of range: " + z); + } + long dim = 1L << z; + if (x < 0 || y < 0 || x >= dim || y >= dim) { + throw new IllegalArgumentException("Tile out of range for zoom " + z + ": x=" + x + " y=" + y); + } + + long acc = (dim * dim - 1) / 3; + long tx = x; + long ty = y; + long d = 0; + for (long s = dim / 2; s > 0; s /= 2) { + long rx = (tx & s) > 0 ? 1 : 0; + long ry = (ty & s) > 0 ? 1 : 0; + d += s * s * ((3 * rx) ^ ry); + + if (ry == 0) { + if (rx == 1) { + tx = s - 1 - tx; + ty = s - 1 - ty; + } + long t = tx; + tx = ty; + ty = t; + } + } + return acc + d; + } + + public static long[] toZXY(long tileId) { + if (tileId < 0) { + throw new IllegalArgumentException("Negative PMTiles tile id: " + tileId); + } + int z = 0; + long acc = 0; + while (z <= 31) { + long dim = 1L << z; + long levelTiles = dim * dim; + if (tileId < acc + levelTiles) { + break; + } + acc += levelTiles; + z++; + } + if (z > 31) { + throw new IllegalArgumentException("PMTiles tile id out of range: " + tileId); + } + + long d = tileId - acc; + long dim = 1L << z; + long tx = 0; + long ty = 0; + for (long s = 1; s < dim; s *= 2) { + long rx = 1 & (d / 2); + long ry = 1 & (d ^ rx); + + if (ry == 0) { + if (rx == 1) { + tx = s - 1 - tx; + ty = s - 1 - ty; + } + long t = tx; + tx = ty; + ty = t; + } + tx += s * rx; + ty += s * ry; + d /= 4; + } + return new long[] { z, tx, ty }; + } +} diff --git a/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/TileIndex.java b/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/TileIndex.java new file mode 100644 index 00000000..50ccf08c --- /dev/null +++ b/src-community/hakunapi-tiles-source-pmtiles/src/main/java/fi/nls/hakunapi/tiles/source/pmtiles/TileIndex.java @@ -0,0 +1,120 @@ +package fi.nls.hakunapi.tiles.source.pmtiles; + +import java.io.IOException; +import java.util.Arrays; + +import fi.nls.hakunapi.tiles.TileLocator; + +public class TileIndex implements TileLocator { + private final long[] ids; + private final long[] locations; + + private final long tileDataOffset; + + private final byte tileCompression; + private final String mediaType; + + private TileIndex(long[] ids, long[] locations, long tileDataOffset, + byte tileCompression, String mediaType) { + this.ids = ids; + this.locations = locations; + this.tileDataOffset = tileDataOffset; + this.tileCompression = tileCompression; + this.mediaType = mediaType; + } + + public static TileIndex build(PMTilesArchive archive) throws Exception { + Builder b = new Builder(); + archive.forEachEntry((tileId, runLength, offset, length) -> { + long packed = TileLocator.pack(offset, length); + + for (long i = 0; i < runLength; i++) { + b.add(tileId + i, packed); + } + }); + + PMTilesHeader header = archive.getHeader(); + return new TileIndex(b.ids(), b.locations(), header.tileDataOffset(), + header.tileCompression(), header.tileMediaType()); + } + + private static final class Builder { + private long[] ids = new long[1024]; + private long[] locations = new long[1024]; + private int n; + + void add(long id, long packed) { + if (n > 0 && id <= ids[n - 1]) { + throw new IllegalStateException("PMTiles entries are not in ascending tile id" + + " order: " + id + " after " + ids[n - 1]); + } + if (n == ids.length) { + ids = Arrays.copyOf(ids, n * 2); + locations = Arrays.copyOf(locations, n * 2); + } + ids[n] = id; + locations[n] = packed; + n++; + } + + long[] ids() { + return Arrays.copyOf(ids, n); + } + + long[] locations() { + return Arrays.copyOf(locations, n); + } + } + + @Override + public long find(int z, long x, long y) { + int i = Arrays.binarySearch(ids, TileId.fromZXY(z, x, y)); + if (i < 0) { + return NOT_FOUND; + } + long packed = locations[i]; + + return TileLocator.pack(tileDataOffset + TileLocator.offset(packed), + TileLocator.length(packed)); + } + + @Override + public String getContentEncoding() { + return Compression.contentEncoding(tileCompression); + } + + @Override + public String getMediaType() { + return mediaType; + } + + public int size() { + return ids.length; + } + + public long heapBytes() { + return 2L * ids.length * Long.BYTES; + } + + void forEachEntry(PackedVisitor visitor) { + for (int i = 0; i < ids.length; i++) { + visitor.accept(ids[i], locations[i]); + } + } + + long tileDataOffset() { + return tileDataOffset; + } + + byte tileCompression() { + return tileCompression; + } + + public interface EntryVisitor { + void accept(long tileId, long runLength, long offset, long length) throws IOException; + } + + public interface PackedVisitor { + void accept(long tileId, long packed); + } +} diff --git a/src-community/hakunapi-tiles-source-pmtiles/src/main/resources/META-INF/services/fi.nls.hakunapi.tiles.MapTileSource b/src-community/hakunapi-tiles-source-pmtiles/src/main/resources/META-INF/services/fi.nls.hakunapi.tiles.MapTileSource new file mode 100644 index 00000000..534c3539 --- /dev/null +++ b/src-community/hakunapi-tiles-source-pmtiles/src/main/resources/META-INF/services/fi.nls.hakunapi.tiles.MapTileSource @@ -0,0 +1 @@ +fi.nls.hakunapi.tiles.source.pmtiles.PMTilesTileSource diff --git a/src-community/hakunapi-tiles-source-pmtiles/src/main/resources/META-INF/services/fi.nls.hakunapi.tiles.VectorTileSource b/src-community/hakunapi-tiles-source-pmtiles/src/main/resources/META-INF/services/fi.nls.hakunapi.tiles.VectorTileSource new file mode 100644 index 00000000..644e939e --- /dev/null +++ b/src-community/hakunapi-tiles-source-pmtiles/src/main/resources/META-INF/services/fi.nls.hakunapi.tiles.VectorTileSource @@ -0,0 +1 @@ +fi.nls.hakunapi.tiles.source.pmtiles.PMTilesVectorTileSource diff --git a/src-community/hakunapi-tiles-source-pmtiles/src/test/java/fi/nls/hakunapi/tiles/source/pmtiles/BoxedTileIndexTest.java b/src-community/hakunapi-tiles-source-pmtiles/src/test/java/fi/nls/hakunapi/tiles/source/pmtiles/BoxedTileIndexTest.java new file mode 100644 index 00000000..5c8f3973 --- /dev/null +++ b/src-community/hakunapi-tiles-source-pmtiles/src/test/java/fi/nls/hakunapi/tiles/source/pmtiles/BoxedTileIndexTest.java @@ -0,0 +1,56 @@ +package fi.nls.hakunapi.tiles.source.pmtiles; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.junit.Test; + +import fi.nls.hakunapi.tiles.TileLocator; + +public class BoxedTileIndexTest { + private static PMTilesArchive openFixture() throws Exception { + Path path = Paths.get("src/test/resources/test_fixture_1.pmtiles"); + return PMTilesArchive.open(new FileRangeReader(path)); + } + + @Test + public void answersExactlyAsTheSortedIndexDoes() throws Exception { + try (PMTilesArchive archive = openFixture()) { + TileIndex sorted = TileIndex.build(archive); + BoxedTileIndex boxed = BoxedTileIndex.of(sorted); + assertNotNull("the fixture should be dense enough to box", boxed); + + int hits = 0; + for (int z = 0; z <= 4; z++) { + long dim = 1L << z; + + for (long y = 0; y <= dim; y++) { + for (long x = 0; x <= dim; x++) { + long a = x < dim && y < dim ? sorted.find(z, x, y) : TileLocator.NOT_FOUND; + long b = x < dim && y < dim ? boxed.find(z, x, y) : TileLocator.NOT_FOUND; + assertEquals("z" + z + "/" + y + "/" + x, a, b); + if (a != TileLocator.NOT_FOUND) { + hits++; + } + } + } + } + assertTrue("the fixture should hold some tiles in z0..4", hits > 0); + } + } + + @Test + public void describesTheArchiveTheSameWay() throws Exception { + try (PMTilesArchive archive = openFixture()) { + TileIndex sorted = TileIndex.build(archive); + BoxedTileIndex boxed = BoxedTileIndex.of(sorted); + assertNotNull(boxed); + assertEquals(sorted.getMediaType(), boxed.getMediaType()); + assertEquals(sorted.getContentEncoding(), boxed.getContentEncoding()); + } + } +} diff --git a/src-community/hakunapi-tiles-source-pmtiles/src/test/java/fi/nls/hakunapi/tiles/source/pmtiles/DirectoryTest.java b/src-community/hakunapi-tiles-source-pmtiles/src/test/java/fi/nls/hakunapi/tiles/source/pmtiles/DirectoryTest.java new file mode 100644 index 00000000..ebdca920 --- /dev/null +++ b/src-community/hakunapi-tiles-source-pmtiles/src/test/java/fi/nls/hakunapi/tiles/source/pmtiles/DirectoryTest.java @@ -0,0 +1,23 @@ +package fi.nls.hakunapi.tiles.source.pmtiles; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; + +import java.io.IOException; + +import org.junit.Test; + +public class DirectoryTest { + @Test + public void rejectsEntryCountThatCannotFitTheBytes() { + byte[] bytes = { (byte) 0x80, (byte) 0x80, (byte) 0x80, (byte) 0x80, (byte) 0x80, 0x20 }; + assertThrows(IOException.class, () -> Directory.parse(bytes)); + } + + @Test + public void parsesOneEntryWhenTheBytesHoldIt() throws IOException { + Directory dir = Directory.parse(new byte[] { 1, 0, 1, 1, 1 }); + assertEquals(1, dir.size()); + assertEquals(1, dir.get(0).runLength()); + } +} diff --git a/src-community/hakunapi-tiles-source-pmtiles/src/test/java/fi/nls/hakunapi/tiles/source/pmtiles/PMTilesVectorTileSourceTest.java b/src-community/hakunapi-tiles-source-pmtiles/src/test/java/fi/nls/hakunapi/tiles/source/pmtiles/PMTilesVectorTileSourceTest.java new file mode 100644 index 00000000..54211387 --- /dev/null +++ b/src-community/hakunapi-tiles-source-pmtiles/src/test/java/fi/nls/hakunapi/tiles/source/pmtiles/PMTilesVectorTileSourceTest.java @@ -0,0 +1,90 @@ +package fi.nls.hakunapi.tiles.source.pmtiles; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; + +import org.junit.Test; + +import fi.nls.hakunapi.core.config.HakunaConfigParser; +import fi.nls.hakunapi.tiles.DefaultTileMatrixSetRegistry; +import fi.nls.hakunapi.tiles.Tile; +import fi.nls.hakunapi.tiles.VectorTileLayer; + +public class PMTilesVectorTileSourceTest { + private static final String TMS = "WebMercatorQuad"; + + private Path fixture() throws Exception { + return Paths.get(getClass().getResource("/test_fixture_1.pmtiles").toURI()); + } + + private Properties baseProps(Path archive) { + Properties p = new Properties(); + p.setProperty("tiles.layers", "pm_layer"); + p.setProperty("tiles.layers.pm_layer.type", "pmtiles-vector"); + p.setProperty("tiles.layers.pm_layer.tileMatrixSets", TMS); + + p.setProperty("tiles.layers.pm_layer.pmtiles.path", archive.getFileName().toString()); + return p; + } + + private VectorTileLayer parse(PMTilesVectorTileSource source, Properties p, Path archive) throws Exception { + HakunaConfigParser cfg = new HakunaConfigParser(p); + Path configPath = archive.resolveSibling("service.properties"); + return source.parse(cfg, configPath, new DefaultTileMatrixSetRegistry(), "pm_layer"); + } + + @Test + public void servesTileBytesVerbatimWithGzipEncoding() throws Exception { + Path archive = fixture(); + try (PMTilesVectorTileSource source = new PMTilesVectorTileSource()) { + VectorTileLayer layer = parse(source, baseProps(archive), archive); + + assertEquals("application/vnd.mapbox-vector-tile", layer.getMediaTypes().get(0)); + + Optional tile = source.getTile(layer, TMS, "0", 0, 0, List.of(), + "application/vnd.mapbox-vector-tile", Map.of()); + + assertTrue(tile.isPresent()); + byte[] bytes = tile.get().getBytes(); + + assertEquals(69, bytes.length); + assertEquals((byte) 0x1f, bytes[0]); + assertEquals((byte) 0x8b, bytes[1]); + assertEquals("application/vnd.mapbox-vector-tile", tile.get().getMediaType()); + assertEquals("gzip", tile.get().getContentEncoding()); + } + } + + @Test + public void missingTileReturnsEmpty() throws Exception { + Path archive = fixture(); + try (PMTilesVectorTileSource source = new PMTilesVectorTileSource()) { + VectorTileLayer layer = parse(source, baseProps(archive), archive); + + Optional tile = source.getTile(layer, TMS, "1", 0, 0, List.of(), + "application/vnd.mapbox-vector-tile", Map.of()); + assertFalse(tile.isPresent()); + } + } + + @Test + public void outOfRangeTileReturnsEmpty() throws Exception { + Path archive = fixture(); + try (PMTilesVectorTileSource source = new PMTilesVectorTileSource()) { + VectorTileLayer layer = parse(source, baseProps(archive), archive); + + Optional tile = source.getTile(layer, TMS, "40", 0, 0, List.of(), + "application/vnd.mapbox-vector-tile", Map.of()); + assertFalse(tile.isPresent()); + } + } + +} diff --git a/src-community/hakunapi-tiles-source-pmtiles/src/test/java/fi/nls/hakunapi/tiles/source/pmtiles/TileIdTest.java b/src-community/hakunapi-tiles-source-pmtiles/src/test/java/fi/nls/hakunapi/tiles/source/pmtiles/TileIdTest.java new file mode 100644 index 00000000..5db236d5 --- /dev/null +++ b/src-community/hakunapi-tiles-source-pmtiles/src/test/java/fi/nls/hakunapi/tiles/source/pmtiles/TileIdTest.java @@ -0,0 +1,43 @@ +package fi.nls.hakunapi.tiles.source.pmtiles; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +public class TileIdTest { + @Test + public void roundTripsEveryAddressOfTheLowerZooms() { + for (int z = 0; z <= 7; z++) { + long dim = 1L << z; + for (long y = 0; y < dim; y++) { + for (long x = 0; x < dim; x++) { + long id = TileId.fromZXY(z, x, y); + assertArrayEquals("z" + z + "/" + y + "/" + x, + new long[] { z, x, y }, TileId.toZXY(id)); + } + } + } + } + + @Test + public void zoomsOccupyConsecutiveIdRanges() { + long expected = 0; + for (int z = 0; z <= 12; z++) { + assertEquals("start of z" + z, expected, TileId.toZXY(expected)[0] == z ? expected : -1); + long dim = 1L << z; + expected += dim * dim; + } + } + + @Test + public void inRangeRejectsAddressesOutsideTheZoom() { + assertTrue(TileId.inRange(3, 7, 0)); + assertFalse(TileId.inRange(3, 8, 0)); + assertFalse(TileId.inRange(3, -1, 0)); + assertFalse(TileId.inRange(-1, 0, 0)); + assertFalse(TileId.inRange(32, 0, 0)); + } +} diff --git a/src-community/hakunapi-tiles-source-pmtiles/src/test/resources/test_fixture_1.pmtiles b/src-community/hakunapi-tiles-source-pmtiles/src/test/resources/test_fixture_1.pmtiles new file mode 100644 index 0000000000000000000000000000000000000000..c86db1f27bba6d3818bb5da5ac6b78e8f6e3303a GIT binary patch literal 468 zcmWIW4av+&EoQD~fB;D-Jp)RAhtlX&KO;oM6)FMcGN988OiVB#pvwAbGZ-3x7!KsS zIlyct;pCKrl2jH3ANx)QR-h0r{uR(?Y;E0N${r@wV$Rqz?{?h$%CZ~pR-fD;chPxcGvD5#swCfUJ3q9X zyYJfGQaN*z>CwX)(_MHoE_$j>pZmpR;iPE+bNH@ihyD-PGDW#IC`?~Ew^Gu@^U;5~ zMLN|*T=}&jpE@sYjM(y1WHrOxzp@|qI{uNHt+DuR>j{?#b(QndSF&YxKiSJB=o0K; zmUMDvv&>(=%}+VLv)Zh>lkdN)q3RE>2ir|u&wq}OjgFMDdZaDyH+~%MwD)0=m+&ls zb+dE0Za)PE6vEq+FRf2_B&6%9tJmp&dX0vru2-+Q&I#Rhakunapi-tiles-servlet-jakarta hakunapi-tiles-source-vectortile hakunapi-tiles-source-gpkg + hakunapi-tiles-source-pmtiles