From a75d0e1adc7637fc358dcdffed3b6c8e5a4887fe Mon Sep 17 00:00:00 2001 From: AI Assistant Date: Fri, 27 Mar 2026 10:28:20 +0200 Subject: [PATCH] fix(geoportal): mount Martin config + upgrade v1.4 + enable building labels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: martin.yaml was never mounted in docker-compose.yml — Martin ran in auto-discovery mode which dropped cadastral_ref from gis_cladiri tiles. Changes: - docker-compose: mount martin.yaml, upgrade Martin v0.15→v1.4.0, use --config - map-viewer: add cladiriLabel layer (cadastral_ref at z16+), wire into visibility - martin.yaml: update version comment - geoportal/: tile server evaluation doc + 3 skill files (vector tiles, PMTiles, MapLibre perf) Co-Authored-By: Claude Opus 4.6 (1M context) --- docker-compose.yml | 6 +- geoportal/TILE-SERVER-EVALUATION.md | 343 ++++++++++++++++++ geoportal/skill-maplibre-performance.md | 292 +++++++++++++++ geoportal/skill-pmtiles-pipeline.md | 272 ++++++++++++++ geoportal/skill-vector-tile-serving.md | 181 +++++++++ martin.yaml | 2 +- .../geoportal/components/map-viewer.tsx | 15 +- 7 files changed, 1104 insertions(+), 7 deletions(-) create mode 100644 geoportal/TILE-SERVER-EVALUATION.md create mode 100644 geoportal/skill-maplibre-performance.md create mode 100644 geoportal/skill-pmtiles-pipeline.md create mode 100644 geoportal/skill-vector-tile-serving.md diff --git a/docker-compose.yml b/docker-compose.yml index 8e4a4f1..350805b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -102,11 +102,13 @@ services: start_period: 10s martin: - image: ghcr.io/maplibre/martin:v0.15.0 + image: ghcr.io/maplibre/martin:v1.4.0 container_name: martin restart: unless-stopped ports: - "3010:3000" - command: ["--default-srid", "3844"] + command: ["--config", "/config/martin.yaml"] environment: - DATABASE_URL=postgresql://architools_user:stictMyFon34!_gonY@10.10.10.166:5432/architools_db + volumes: + - ./martin.yaml:/config/martin.yaml:ro diff --git a/geoportal/TILE-SERVER-EVALUATION.md b/geoportal/TILE-SERVER-EVALUATION.md new file mode 100644 index 0000000..37693b3 --- /dev/null +++ b/geoportal/TILE-SERVER-EVALUATION.md @@ -0,0 +1,343 @@ +# Tile Server Evaluation — ArchiTools Geoportal (March 2026) + +## Context + +ArchiTools Geoportal serves vector tiles (MVT) from PostgreSQL 16 + PostGIS 3 via Martin. +Data: ~330K GIS features (parcels, buildings, admin boundaries) in EPSG:3844 (Stereo70), growing to 1M+. +Frontend: MapLibre GL JS 5.21, Next.js 16, Docker self-hosted. + +--- + +## Problem Statement + +1. Martin v0.15.0 was running in **auto-discovery mode** — the existing `martin.yaml` config was never mounted +2. Building labels (`cadastral_ref`) missing from MVT tiles despite the view exposing them +3. Performance concerns at scale (330K → 1M+ features) + +--- + +## Solutions Evaluated (7 options + emerging tech) + +### 1. Martin (Fix + Upgrade) — WINNER + +| Aspect | Detail | +|---|---| +| Root cause | `martin.yaml` not mounted in docker-compose — Martin ran in auto-discovery mode | +| Fix | Mount config + upgrade v0.15 → v1.4.0 | +| Performance | Fastest tile server benchmarked (2-3x faster than #2 Tegola) | +| EPSG:3844 | Native support via `default_srid: 3844` | +| New in v1.4 | ZSTD compression, MLT format, materialized views, better logging | + +**Status: IMPLEMENTED** — docker-compose.yml updated, building labels activated. + +### 2. pg_tileserv (CrunchyData) + +| Aspect | Detail | +|---|---| +| Architecture | Go binary, zero-config, delegates ST_AsMVT to PostGIS | +| Property control | Auto from schema + URL `?properties=` parameter | +| Performance | 2-3x slower than Martin (Rechsteiner benchmark) | +| EPSG:3844 | Supported (auto-reprojects via ST_Transform) | +| Killer feature | Function-based sources (full SQL tile functions) | +| Dealbreaker | View extent estimation bug (#156) affects all our views, development stagnant | + +**Verdict: NO** — slower, buggy with views, stagnant development. + +### 3. Tegola (Go-based) + +| Aspect | Detail | +|---|---| +| Architecture | Go, TOML config, explicit per-layer SQL | +| Performance | 2nd in benchmarks, but 2-3x slower than Martin | +| Built-in cache | File, S3/MinIO, Redis — with seed/purge CLI | +| EPSG:3844 | **NOT SUPPORTED** (only 3857/4326) — requires ST_Transform in every query | +| Killer feature | Built-in tile seeding and cache purging | + +**Verdict: NO** — EPSG:3844 not supported, dealbreaker for our data. + +### 4. t-rex (Rust-based) + +| Aspect | Detail | +|---|---| +| Status | **Abandoned/unmaintained** — no releases since 2023 | + +**Verdict: NO** — dead project. + +### 5. GeoJSON Direct from Next.js API + +| Aspect | Detail | +|---|---| +| 330K features | ~270 MB uncompressed, 800 MB–1.4 GB browser memory | +| Browser impact | 10-30s main thread freeze, mobile crash | +| Pan/zoom | Full re-fetch on every viewport change, flickering | +| Viable range | Only at zoom 16+ with <500 features in viewport | + +**Verdict: NO** — does not scale beyond ~20K features. + +### 6. PMTiles (Pre-generated) + +| Aspect | Detail | +|---|---| +| Architecture | Single-file tile archive, HTTP Range Requests, no server needed | +| Performance | ~5ms per tile (vs 200-2000ms for Martin on low-zoom) | +| Property control | tippecanoe gives explicit include/exclude per property | +| Update strategy | Full rebuild required (~3-7 min for 330K features) | +| EPSG:3844 | Requires reprojection to 4326 via ogr2ogr before tippecanoe | +| MinIO serving | Yes — direct HTTP Range Requests with CORS | + +**Verdict: YES as hybrid complement** — excellent for static UAT overview layers (z0-z12), Martin for live detail. + +### 7. Emerging Solutions + +| Solution | Status | Relevance | +|---|---|---| +| **mvt-rs** (Rust) | v0.16.2, active | Admin UI, auth per layer, cache — good for multi-tenant | +| **MLT format** | Stable Jan 2026 | 6x compression, 4x faster decode — Martin v1.3+ supports it | +| **BBOX** | Maturing | Similar to Tegola performance, unified raster+vector | +| **DuckDB tiles** | Early | Not PostGIS replacement, interesting for GeoParquet | +| **FlatGeobuf** | Stable | Good for <100K features, not a tile replacement | + +--- + +## Benchmark Reference (Rechsteiner, April 2025) + +| Rank | Server | Language | Relative Speed | +|---|---|---|---| +| 1 | **Martin** | Rust | 1x (fastest) | +| 2 | Tegola | Go | 2-3x slower | +| 3 | BBOX | Rust | ~same as Tegola | +| 4 | pg_tileserv | Go | ~4x slower | +| 5 | TiPg | Python | Slower | +| 6 | ldproxy | Java | 4-70x slower | + +Source: [github.com/FabianRechsteiner/vector-tiles-benchmark](https://github.com/FabianRechsteiner/vector-tiles-benchmark) + +--- + +## Implementation Roadmap + +### Phase 1: Martin Fix (DONE) + +Changes applied: +- `docker-compose.yml`: Martin v0.15 → v1.4.0, config mounted, command changed to `--config` +- `martin.yaml`: Comment updated to reflect v1.4 +- `map-viewer.tsx`: Building labels layer activated (`cladiriLabel` at minzoom 16) + +### Phase 2A: nginx Tile Cache + +**Impact**: 10-100x faster on repeat requests, zero PostGIS load for cached tiles. +**Effort**: ~2 hours. + +#### Implementation Prompt + +``` +Add an nginx reverse proxy cache in front of Martin for tile serving in ArchiTools. + +Context: +- Martin serves tiles at http://martin:3000 (container name: martin, port 3010 on host) +- Traefik proxies external traffic to ArchiTools at tools.beletage.ro +- Current tile URL pattern: https://tools.beletage.ro/tiles/{source}/{z}/{x}/{y} +- NEXT_PUBLIC_MARTIN_URL=https://tools.beletage.ro/tiles + +Requirements: +1. Create an nginx container `tile-cache` in docker-compose.yml +2. nginx config: proxy_cache for /tiles/* with: + - Cache zone: 2GB max, keys in shared memory + - Cache valid: 200 responses for 1 hour + - Stale serving on error/timeout + - Cache-Control headers passed through + - CORS headers for tiles (Access-Control-Allow-Origin: *) + - Gzip/brotli passthrough (Martin already compresses) +3. Route Martin traffic through tile-cache: + - tile-cache listens on port 3010 (replace Martin's host port) + - tile-cache proxies to http://martin:3000 +4. Add cache purge endpoint or script for post-sync invalidation +5. Volume for persistent cache across container restarts + +Files to modify: +- docker-compose.yml (add tile-cache service, adjust martin ports) +- Create nginx/tile-cache.conf + +Do NOT change the frontend NEXT_PUBLIC_MARTIN_URL — keep the same external URL. +Build with `npx next build` to verify zero errors. +``` + +### Phase 2B: PMTiles for UAT Overview Layers + +**Impact**: Sub-10ms overview tiles, zero PostGIS load for z0-z12. +**Effort**: ~4-6 hours. + +#### Implementation Prompt + +``` +Implement PMTiles pre-generation for UAT overview layers in ArchiTools Geoportal. + +Context: +- PostGIS at 10.10.10.166:5432, database architools_db +- Views: gis_uats_z0, gis_uats_z5, gis_uats_z8, gis_uats_z12, gis_administrativ +- All geometries EPSG:3844 (Stereo70), need reprojection to 4326 for tippecanoe +- MinIO at 10.10.10.166:9002, bucket for tiles +- Frontend: MapLibre GL JS 5.21 in src/modules/geoportal/components/map-viewer.tsx + +Requirements: + +1. Create `scripts/rebuild-overview-tiles.sh`: + - Export each view with ogr2ogr: -f FlatGeobuf -s_srs EPSG:3844 -t_srs EPSG:4326 + - Generate combined PMTiles with tippecanoe: + - --layer per view, --minimum-zoom=0, --maximum-zoom=14 + - --detect-shared-borders (critical for adjacent UAT polygons) + - --hilbert for compression + - Atomic upload to MinIO: upload as overview_new.pmtiles, then rename to overview.pmtiles + - Cleanup temp files + +2. Create Dockerfile for tippecanoe build container (or use ghcr.io/felt/tippecanoe) + +3. Add `tippecanoe` service to docker-compose.yml (one-shot, for manual/cron runs) + +4. Configure MinIO: + - Create bucket `tiles` with public read + - CORS: Allow GET/HEAD from tools.beletage.ro, expose Range/Content-Range headers + +5. Update map-viewer.tsx: + - npm install pmtiles + - Register pmtiles:// protocol on MapLibre + - Add PMTiles source for overview layers (z0-z14) + - Keep existing Martin sources for detail layers (z14+) + - Set zoom breakpoints: PMTiles below z14, Martin above z14 + +6. Add N8N webhook trigger or cron for nightly rebuild after weekend deep sync + +The UAT overview layers change rarely (only when new UATs are synced). +Parcel/building layers stay on Martin for live data freshness. + +Build with `npx next build` to verify zero errors. +Read CLAUDE.md for project conventions before starting. +``` + +### Phase 2C: MLT Format Testing + +**Impact**: 6x smaller tiles, 4x faster client decode. +**Effort**: ~1 hour to test. + +#### Implementation Prompt + +``` +Test MLT (MapLibre Tiles) format on one layer in ArchiTools Geoportal. + +Context: +- Martin v1.4.0 running with config at martin.yaml +- MapLibre GL JS 5.21 in src/modules/geoportal/components/map-viewer.tsx +- Test layer: gis_terenuri (largest layer, ~250K features) + +Requirements: + +1. Research how Martin v1.4 serves MLT format: + - Check if it's automatic via Accept header or needs config + - Check Martin docs for MLT serving configuration + +2. Update map-viewer.tsx to request MLT for one source (gis_terenuri): + - Add `encoding: "mlt"` to the vector source definition if MapLibre supports it + - Or configure via source URL parameter if Martin expects it + +3. Test and measure: + - Compare tile sizes: MVT vs MLT for same tile coordinates + - Compare decode time in browser DevTools Network tab + - Check that all properties (cadastral_ref, area_value, etc.) survive MLT encoding + - Check label rendering still works + +4. If MLT works correctly, apply to all Martin sources + If issues found, document them and revert to MVT + +This is experimental — keep MVT as fallback. Do not break existing functionality. +Build with `npx next build` to verify zero errors. +``` + +### Phase 2D: mvt-rs Evaluation (Future — Multi-Tenant) + +**Impact**: Built-in auth, admin UI, per-layer access control. +**Effort**: 1-2 days for evaluation + migration. + +#### Implementation Prompt + +``` +Evaluate mvt-rs as a replacement for Martin in ArchiTools Geoportal for multi-tenant deployment. + +Context: +- Current: Martin v1.4.0 serving 9 PostGIS sources (views in EPSG:3844) +- Goal: Expose geoportal to external clients with per-layer access control +- mvt-rs repo: https://github.com/mvt-proj/mvt-rs (v0.16.2+, Rust, Salvo framework) + +Requirements: + +1. Deploy mvt-rs as a Docker container alongside Martin (don't replace yet) + - Use same DATABASE_URL + - Map to different port (e.g., 3011) + +2. Configure mvt-rs with equivalent layers to martin.yaml: + - gis_uats_z0/z5/z8/z12, gis_administrativ, gis_terenuri, gis_cladiri + - gis_terenuri_status, gis_cladiri_status + - All with EPSG:3844, explicit properties + +3. Test: + - Do all properties appear in MVT output? (especially cadastral_ref on gis_cladiri) + - Performance comparison: curl timing for 10 representative tiles vs Martin + - Admin UI: create test user, assign layer permissions + - Cache: configure disk cache, measure cold vs warm tile times + +4. Document findings: + - Property inclusion: pass/fail per layer + - Performance delta vs Martin + - Admin UI capabilities and limitations + - Missing features vs Martin (PMTiles, MLT, etc.) + +5. Decision matrix: when to switch from Martin to mvt-rs + +Do NOT modify the production Martin setup. This is a parallel evaluation only. +``` + +--- + +## Key Technical Details + +### Martin v1.4.0 Config (validated compatible) + +The `martin.yaml` at project root defines 9 sources with explicit properties. +Config format unchanged from v0.15 to v1.4 — no migration needed. + +Key config features used: +- `auto_publish: false` — only explicitly listed sources are served +- `default_srid: 3844` — all sources use Stereo70 +- `properties:` map per source — explicit column name + PostgreSQL type +- `minzoom/maxzoom` per source — controls tile generation range + +### PostGIS View Chain + +``` +GisFeature table (Prisma) → gis_features view → gis_terenuri / gis_cladiri / gis_administrativ + → gis_terenuri_status / gis_cladiri_status (with JOINs) +GisUat table → gis_uats_z0/z5/z8/z12 (with ST_SimplifyPreserveTopology) +``` + +### MapLibre Layer Architecture + +``` +Sources (Martin): gis_uats_z0, z5, z8, z12, administrativ, terenuri, cladiri +Layers per source: fill + line + label (where applicable) +Selection: Separate highlight layers on terenuri source +Drawing: GeoJSON source for freehand/rect polygon +``` + +--- + +## References + +- [Martin Documentation](https://maplibre.org/martin/) +- [Martin Releases](https://github.com/maplibre/martin/releases) +- [Vector Tiles Benchmark (Rechsteiner 2025)](https://github.com/FabianRechsteiner/vector-tiles-benchmark) +- [PMTiles Specification](https://github.com/protomaps/PMTiles) +- [tippecanoe (Felt)](https://github.com/felt/tippecanoe) +- [MLT Format Announcement](https://maplibre.org/news/2026-01-23-mlt-release/) +- [mvt-rs](https://github.com/mvt-proj/mvt-rs) +- [pg_tileserv](https://github.com/CrunchyData/pg_tileserv) +- [Tegola](https://github.com/go-spatial/tegola) +- [Serving Vector Tiles Fast (Spatialists)](https://spatialists.ch/posts/2025/04/05-serving-vector-tiles-fast/) diff --git a/geoportal/skill-maplibre-performance.md b/geoportal/skill-maplibre-performance.md new file mode 100644 index 0000000..add3861 --- /dev/null +++ b/geoportal/skill-maplibre-performance.md @@ -0,0 +1,292 @@ +# Skill: MapLibre GL JS Performance for Large GIS Datasets + +## When to Use + +When building web maps with MapLibre GL JS that display large spatial datasets (>10K features). Covers source type selection, layer optimization, label rendering, and client-side performance tuning. + +--- + +## Source Type Decision Matrix + +| Dataset Size | Recommended Source | Reason | +|---|---|---| +| <2K features | GeoJSON | Simple, full property access, smooth | +| 2K-20K features | GeoJSON (careful) | Works but `setData()` updates lag 200-400ms | +| 20K-100K features | Vector tiles (MVT) | GeoJSON causes multi-second freezes | +| 100K+ features | Vector tiles (MVT) | GeoJSON crashes mobile, 1GB+ memory on desktop | +| Static/archival | PMTiles | Pre-generated, ~5ms per tile, zero server load | + +### GeoJSON Memory Profile + +| Features (polygons, ~20 coords each) | JSON Size | Browser Memory | Load Time | +|---|---|---|---| +| 1K | 0.8 MB | ~50 MB | <1s | +| 10K | 8 MB | ~200 MB | 1-3s | +| 50K | 41 MB | ~600 MB | 5-15s freeze | +| 100K | 82 MB | ~1.2 GB | 15-30s freeze | +| 330K | 270 MB | ~1.5 GB+ | Crash | + +The bottleneck is `JSON.stringify` on the main thread when data is transferred to the Web Worker for `geojson-vt` tiling. + +--- + +## Vector Tile Source Configuration + +### Zoom-Dependent Source Loading + +Don't load data you don't need. Set `minzoom`/`maxzoom` on sources and layers: + +```typescript +// Source: only request tiles in useful zoom range +map.addSource('parcels', { + type: 'vector', + tiles: ['https://tiles.example.com/parcels/{z}/{x}/{y}'], + minzoom: 10, // don't request below z10 + maxzoom: 18, // server maxzoom (tiles overzoom beyond this) +}); + +// Layer: only render when meaningful +map.addLayer({ + id: 'parcels-fill', + type: 'fill', + source: 'parcels', + 'source-layer': 'parcels', + minzoom: 13, // visible from z13 (even if source loads from z10) + maxzoom: 20, // render up to z20 (overzooming tiles from z18) + paint: { ... }, +}); +``` + +### Multiple Sources at Different Detail Levels + +For large datasets, serve simplified versions at low zoom: + +```typescript +// Simplified overview (server: ST_Simplify, fewer properties) +map.addSource('parcels-overview', { + type: 'vector', + tiles: ['https://tiles.example.com/parcels_simplified/{z}/{x}/{y}'], + minzoom: 6, maxzoom: 14, +}); + +// Full detail +map.addSource('parcels-detail', { + type: 'vector', + tiles: ['https://tiles.example.com/parcels/{z}/{x}/{y}'], + minzoom: 14, maxzoom: 18, +}); + +// Layers with zoom handoff +map.addLayer({ + id: 'parcels-overview-fill', source: 'parcels-overview', + minzoom: 10, maxzoom: 14, // disappears at z14 + ... +}); +map.addLayer({ + id: 'parcels-detail-fill', source: 'parcels-detail', + minzoom: 14, // appears at z14 + ... +}); +``` + +--- + +## Label Rendering Best Practices + +### Text Labels on Polygons + +```typescript +map.addLayer({ + id: 'parcel-labels', + type: 'symbol', + source: 'parcels', + 'source-layer': 'parcels', + minzoom: 16, // only show labels at high zoom + layout: { + 'text-field': ['coalesce', ['get', 'cadastral_ref'], ''], + 'text-font': ['Noto Sans Regular'], + 'text-size': 10, + 'text-anchor': 'center', + 'text-allow-overlap': false, // prevent label collisions + 'text-max-width': 8, // wrap long labels (in ems) + 'text-optional': true, // label is optional — feature renders without it + 'symbol-placement': 'point', // placed at polygon centroid + }, + paint: { + 'text-color': '#1e3a5f', + 'text-halo-color': '#ffffff', + 'text-halo-width': 1, // readability on any background + }, +}); +``` + +### Performance Tips for Labels + +- **`text-allow-overlap: false`** — essential for dense datasets, MapLibre auto-removes colliding labels +- **`text-optional: true`** — allow symbol layer to show icon without text if text collides +- **High `minzoom`** (16+) — labels are expensive to render, only show when meaningful +- **`text-font`** — use fonts available in the basemap style. Custom fonts require glyph server. +- **`symbol-sort-key`** — prioritize which labels show first (e.g., larger parcels) + +```typescript +layout: { + 'symbol-sort-key': ['*', -1, ['get', 'area_value']], // larger areas get priority +} +``` + +--- + +## Selection and Interaction Patterns + +### Click Selection (single feature) + +```typescript +map.on('click', 'parcels-fill', (e) => { + const feature = e.features?.[0]; + if (!feature) return; + const props = feature.properties; + + // Highlight via filter + map.setFilter('selection-highlight', ['==', 'object_id', props.object_id]); +}); +``` + +### queryRenderedFeatures for Box/Polygon Selection + +```typescript +// Rectangle selection +const features = map.queryRenderedFeatures( + [[x1, y1], [x2, y2]], // pixel bbox + { layers: ['parcels-fill'] } +); + +// Features are from rendered tiles — properties may be limited +// For full properties, fetch from API by ID +``` + +**Important:** `queryRenderedFeatures` only returns features currently rendered in the viewport tiles. Properties in MVT tiles may be a subset of the full database record. For detailed properties, use a separate API endpoint. + +### Highlight Layer Pattern + +Dedicated layer with dynamic filter for selection highlighting: + +```typescript +// Add once during map setup +map.addLayer({ + id: 'selection-fill', + type: 'fill', + source: 'parcels', + 'source-layer': 'parcels', + filter: ['==', 'object_id', '__NONE__'], // show nothing initially + paint: { 'fill-color': '#f59e0b', 'fill-opacity': 0.5 }, +}); + +// Update filter on selection +const ids = Array.from(selectedIds); +map.setFilter('selection-fill', + ids.length > 0 + ? ['in', ['to-string', ['get', 'object_id']], ['literal', ids]] + : ['==', 'object_id', '__NONE__'] +); +``` + +--- + +## Basemap Management + +### Multiple Basemap Support + +Switching basemaps requires recreating the map (MapLibre limitation). Preserve view state: + +```typescript +const viewStateRef = useRef({ center: DEFAULT_CENTER, zoom: DEFAULT_ZOOM }); + +// Save on every move +map.on('moveend', () => { + viewStateRef.current = { + center: map.getCenter().toArray(), + zoom: map.getZoom(), + }; +}); + +// On basemap switch: destroy map, recreate with saved view state +// All sources + layers must be re-added after style load +``` + +### Raster Basemaps + +```typescript +const style: StyleSpecification = { + version: 8, + sources: { + basemap: { + type: 'raster', + tiles: ['https://mt0.google.com/vt/lyrs=s&x={x}&y={y}&z={z}'], + tileSize: 256, + attribution: '© Google', + }, + }, + layers: [{ + id: 'basemap', type: 'raster', source: 'basemap', + minzoom: 0, maxzoom: 20, + }], +}; +``` + +### Vector Basemaps (OpenFreeMap, MapTiler) + +```typescript +// Style URL — includes all sources + layers +const map = new maplibregl.Map({ + style: 'https://tiles.openfreemap.org/styles/liberty', +}); + +// Hide unwanted built-in layers (e.g., admin boundaries you'll replace) +for (const layer of map.getStyle().layers) { + if (/boundar|admin/i.test(layer.id)) { + map.setLayoutProperty(layer.id, 'visibility', 'none'); + } +} +``` + +--- + +## Performance Checklist + +### Server Side +- [ ] Spatial index (GiST) on geometry column +- [ ] Zoom-dependent simplified views for overview levels +- [ ] `minzoom`/`maxzoom` per tile source to prevent pathological tiles +- [ ] HTTP cache (nginx proxy_cache / Varnish) in front of tile server +- [ ] PMTiles for static layers (no DB hit) +- [ ] Exclude large geometry columns from list queries + +### Client Side +- [ ] Set `minzoom` on layers to avoid rendering at useless zoom levels +- [ ] `text-allow-overlap: false` on all symbol layers +- [ ] Use `text-optional: true` for labels +- [ ] Don't add GeoJSON sources for >20K features +- [ ] Use `queryRenderedFeatures` (not `querySourceFeatures`) for interaction +- [ ] Preserve view state across basemap switches (ref, not state) +- [ ] Debounce viewport-dependent API calls (search, feature loading) + +### Memory Management +- [ ] Remove unused sources/layers when switching views +- [ ] Clear GeoJSON sources with `setData(emptyFeatureCollection)` before removing +- [ ] Use `map.remove()` in cleanup (useEffect return) +- [ ] Don't store large GeoJSON in React state (use refs) + +--- + +## Common Pitfalls + +1. **GeoJSON `setData()` freezes main thread** — `JSON.stringify` runs synchronously for every update +2. **`queryRenderedFeatures` returns simplified geometry** — don't use for area/distance calculations +3. **Vector tile properties may be truncated** — tile servers can drop properties to fit tile size limits +4. **Basemap switch requires full map recreation** — save/restore view state and re-add all overlay layers +5. **`text-font` must match basemap fonts** — if using vector basemap, use its font stack; if raster, you need a glyph server +6. **Popup/tooltip on dense data causes flicker** — debounce mousemove handlers +7. **Large fill layers without `minzoom` tank performance** — 100K polygons at z0 is pathological +8. **`map.setFilter` with huge ID lists is slow** — for >1000 selected features, consider a separate GeoJSON source +9. **MapLibre CSS must be loaded manually in SSR frameworks** — inject `` in `useEffect` or import statically +10. **React strict mode double-mounts effects** — guard map initialization with ref check diff --git a/geoportal/skill-pmtiles-pipeline.md b/geoportal/skill-pmtiles-pipeline.md new file mode 100644 index 0000000..4db78c1 --- /dev/null +++ b/geoportal/skill-pmtiles-pipeline.md @@ -0,0 +1,272 @@ +# Skill: PMTiles Generation Pipeline from PostGIS + +## When to Use + +When you need to pre-generate vector tiles from PostGIS data for fast static serving. Ideal for overview/boundary layers that change infrequently, serving from S3/MinIO/CDN without a tile server, or eliminating database load for tile serving. + +--- + +## Complete Pipeline + +### Prerequisites + +| Tool | Purpose | Install | +|---|---|---| +| ogr2ogr (GDAL) | PostGIS export + reprojection | `apt install gdal-bin` or Docker | +| tippecanoe | MVT tile generation → PMTiles | `ghcr.io/felt/tippecanoe` Docker image | +| mc (MinIO client) | Upload to MinIO/S3 | `brew install minio/stable/mc` | + +### Step 1: Export from PostGIS + +```bash +# Single layer — FlatGeobuf is fastest for tippecanoe input +ogr2ogr -f FlatGeobuf \ + -s_srs EPSG:3844 \ # source SRID (your data) + -t_srs EPSG:4326 \ # tippecanoe REQUIRES WGS84 + parcels.fgb \ + "PG:host=10.10.10.166 dbname=mydb user=myuser password=mypass" \ + -sql "SELECT id, name, area, geom FROM my_view WHERE geom IS NOT NULL" + +# Multiple layers in parallel +ogr2ogr -f FlatGeobuf -s_srs EPSG:3844 -t_srs EPSG:4326 \ + parcels.fgb "PG:..." -sql "SELECT ... FROM gis_terenuri" & +ogr2ogr -f FlatGeobuf -s_srs EPSG:3844 -t_srs EPSG:4326 \ + buildings.fgb "PG:..." -sql "SELECT ... FROM gis_cladiri" & +ogr2ogr -f FlatGeobuf -s_srs EPSG:3844 -t_srs EPSG:4326 \ + uats.fgb "PG:..." -sql "SELECT ... FROM gis_uats_z12" & +wait +``` + +**Why FlatGeobuf over GeoJSON:** +- Binary columnar format — tippecanoe reads it 3-5x faster +- No JSON parsing overhead +- Streaming read (no need to load entire file in memory) +- tippecanoe native support since v2.17+ + +### Step 2: Generate PMTiles with tippecanoe + +```bash +# Single layer +tippecanoe \ + -o parcels.pmtiles \ + --name="Parcels" \ + --layer="parcels" \ + --minimum-zoom=6 \ + --maximum-zoom=15 \ + --base-zoom=15 \ + --drop-densest-as-needed \ + --extend-zooms-if-still-dropping \ + --detect-shared-borders \ + --simplification=10 \ + --hilbert \ + --force \ + parcels.fgb + +# Multi-layer (combined file) +tippecanoe \ + -o combined.pmtiles \ + --named-layer=parcels:parcels.fgb \ + --named-layer=buildings:buildings.fgb \ + --named-layer=uats:uats.fgb \ + --minimum-zoom=0 \ + --maximum-zoom=15 \ + --drop-densest-as-needed \ + --detect-shared-borders \ + --hilbert \ + --force +``` + +#### Key tippecanoe Flags + +| Flag | Purpose | When to Use | +|---|---|---| +| `--minimum-zoom=N` | Lowest zoom level | Always set | +| `--maximum-zoom=N` | Highest zoom level (full detail) | Always set | +| `--base-zoom=N` | Zoom where ALL features kept (no dropping) | Set to max-zoom | +| `--drop-densest-as-needed` | Drop features in dense areas at low zoom | Large polygon datasets | +| `--extend-zooms-if-still-dropping` | Auto-increase max zoom if needed | Safety net | +| `--detect-shared-borders` | Prevent gaps between adjacent polygons | **Critical for parcels/admin boundaries** | +| `--coalesce-densest-as-needed` | Merge small features at low zoom | Building footprints | +| `--simplification=N` | Pixel tolerance for geometry simplification | Reduce tile size at low zoom | +| `--hilbert` | Hilbert curve ordering | Better compression, always use | +| `-y col1 -y col2` | Include ONLY these properties | Reduce tile size | +| `-x col1 -x col2` | Exclude these properties | Remove large/unnecessary fields | +| `--force` | Overwrite existing output | Scripts | +| `--no-feature-limit` | No limit per tile | When density matters | +| `--no-tile-size-limit` | No tile byte limit | When completeness matters | + +#### Property Control + +```bash +# Include only specific properties (whitelist) +tippecanoe -o out.pmtiles -y name -y area -y type parcels.fgb + +# Exclude specific properties (blacklist) +tippecanoe -o out.pmtiles -x raw_json -x internal_id parcels.fgb + +# Zoom-dependent properties (different attributes per zoom) +# Use tippecanoe-json format with per-feature "tippecanoe" key +``` + +### Step 3: Upload to MinIO (Atomic Swap) + +```bash +# Upload to temp name first +mc cp combined.pmtiles myminio/tiles/combined_new.pmtiles + +# Atomic rename (zero-downtime swap) +mc mv myminio/tiles/combined_new.pmtiles myminio/tiles/combined.pmtiles +``` + +### Step 4: MinIO CORS Configuration + +```bash +# Required for browser-direct Range Requests +mc admin config set myminio api cors_allow_origin="https://tools.beletage.ro" + +# Or bucket policy for public read +mc anonymous set download myminio/tiles +``` + +MinIO CORS must expose Range/Content-Range headers: +```json +{ + "CORSRules": [{ + "AllowedOrigins": ["https://your-domain.com"], + "AllowedMethods": ["GET", "HEAD"], + "AllowedHeaders": ["Range", "If-None-Match"], + "ExposeHeaders": ["Content-Range", "Content-Length", "ETag"], + "MaxAgeSeconds": 3600 + }] +} +``` + +--- + +## MapLibre GL JS Integration + +```bash +npm install pmtiles +``` + +```typescript +import maplibregl from 'maplibre-gl'; +import { Protocol } from 'pmtiles'; + +// Register ONCE at app initialization +const protocol = new Protocol(); +maplibregl.addProtocol('pmtiles', protocol.tile); + +// Add source to map +map.addSource('my-tiles', { + type: 'vector', + url: 'pmtiles://https://minio.example.com/tiles/combined.pmtiles', +}); + +// Add layers +map.addLayer({ + id: 'parcels-fill', + type: 'fill', + source: 'my-tiles', + 'source-layer': 'parcels', // layer name from tippecanoe --layer or --named-layer + minzoom: 10, + maxzoom: 16, + paint: { 'fill-color': '#22c55e', 'fill-opacity': 0.15 }, +}); + +// Cleanup on unmount +maplibregl.removeProtocol('pmtiles'); +``` + +--- + +## Hybrid Architecture (PMTiles + Live Tile Server) + +``` +Zoom 0-14: PMTiles from MinIO (pre-generated, ~5ms, zero DB load) +Zoom 14+: Martin from PostGIS (live, always-current, ~50-200ms) +``` + +```typescript +// PMTiles for overview +map.addSource('overview', { + type: 'vector', + url: 'pmtiles://https://minio/tiles/overview.pmtiles', +}); + +// Martin for detail +map.addSource('detail', { + type: 'vector', + tiles: ['https://tiles.example.com/{source}/{z}/{x}/{y}'], + minzoom: 14, + maxzoom: 18, +}); + +// Layers with zoom handoff +map.addLayer({ + id: 'parcels-overview', source: 'overview', 'source-layer': 'parcels', + minzoom: 6, maxzoom: 14, // PMTiles handles low zoom + ... +}); +map.addLayer({ + id: 'parcels-detail', source: 'detail', 'source-layer': 'gis_terenuri', + minzoom: 14, // Martin handles high zoom + ... +}); +``` + +--- + +## Rebuild Strategies + +### Nightly Cron + +```bash +# crontab -e +0 2 * * * /opt/scripts/rebuild-tiles.sh >> /var/log/tile-rebuild.log 2>&1 +``` + +### After Data Sync (webhook/API trigger) + +```bash +# Call from sync completion handler +curl -X POST http://n8n:5678/webhook/rebuild-tiles +``` + +### Partial Rebuild (single layer update) + +```bash +# Rebuild just parcels, then merge with existing layers +tippecanoe -o parcels_new.pmtiles ... parcels.fgb +tile-join -o combined_new.pmtiles --force \ + parcels_new.pmtiles \ + buildings_existing.pmtiles \ + uats_existing.pmtiles +mc cp combined_new.pmtiles myminio/tiles/combined.pmtiles +``` + +--- + +## Build Time Estimates + +| Features | Type | Zoom Range | Time | Output Size | +|---|---|---|---|---| +| 500 | Polygons (UAT) | z0-z12 | <5s | 10-30 MB | +| 100K | Polygons (buildings) | z12-z15 | 30-90s | 100-200 MB | +| 330K | Polygons (parcels) | z6-z15 | 2-5 min | 200-400 MB | +| 1M | Polygons (mixed) | z0-z15 | 8-15 min | 500 MB-1 GB | + +tippecanoe is highly optimized and uses parallel processing. + +--- + +## Common Pitfalls + +1. **tippecanoe only accepts WGS84 (EPSG:4326)** — always reproject with ogr2ogr first +2. **`--detect-shared-borders` is critical for parcels** — without it, gaps appear between adjacent polygons +3. **GeoJSON input is slow** — use FlatGeobuf for 3-5x faster reads +4. **No incremental updates** — must rebuild entire file (use `tile-join` for layer-level replacement) +5. **MinIO needs CORS for browser-direct access** — Range + Content-Range headers must be exposed +6. **Large properties bloat tile size** — use `-y`/`-x` flags to control what goes into tiles +7. **`--no-tile-size-limit` can produce huge tiles** — use with `--drop-densest-as-needed` safety valve +8. **Atomic upload prevents serving partial files** — always upload as temp name then rename diff --git a/geoportal/skill-vector-tile-serving.md b/geoportal/skill-vector-tile-serving.md new file mode 100644 index 0000000..9b1023c --- /dev/null +++ b/geoportal/skill-vector-tile-serving.md @@ -0,0 +1,181 @@ +# Skill: Vector Tile Serving from PostGIS + +## When to Use + +When building a web map that serves vector tiles from PostgreSQL/PostGIS data. Applies to any project using MapLibre GL JS, Mapbox GL JS, or OpenLayers with MVT tiles from a spatial database. + +--- + +## Core Architecture Decision + +**Always use a dedicated tile server over GeoJSON for datasets >20K features.** + +GeoJSON limits: +- 20K polygons: visible jank on `setData()`, 200-400ms freezes +- 50K polygons: multi-second freezes, 500MB+ browser memory +- 100K+ polygons: crashes mobile browsers, 1-2GB memory on desktop +- `JSON.stringify` runs on main thread — blocks UI proportional to data size + +Vector tiles (MVT) solve this: +- Only visible tiles loaded (~50-200KB per viewport) +- Incremental pan/zoom (no re-fetch) +- ~100-200MB client memory regardless of total dataset size +- Works on mobile + +--- + +## Tile Server Rankings (Rechsteiner Benchmark, April 2025) + +| Rank | Server | Language | Speed | Notes | +|---|---|---|---|---| +| 1 | **Martin** | Rust | 1x | Clear winner, 95-122ms range | +| 2 | Tegola | Go | 2-3x slower | Only supports SRID 3857/4326 | +| 3 | BBOX | Rust | ~same as Tegola | Unified raster+vector | +| 4 | pg_tileserv | Go | ~4x slower | Zero-config but limited control | +| 5 | TiPg | Python | Slower | Not for production scale | +| 6 | ldproxy | Java | 4-70x slower | Enterprise/OGC compliance | + +Source: [github.com/FabianRechsteiner/vector-tiles-benchmark](https://github.com/FabianRechsteiner/vector-tiles-benchmark) + +--- + +## Martin: Best Practices + +### Always use explicit config (not auto-discovery) + +Auto-discovery can drop properties, misdetect SRIDs, and behave unpredictably with nested views. + +```yaml +postgres: + connection_string: ${DATABASE_URL} + default_srid: 3844 # your source SRID + auto_publish: false # explicit sources only + tables: + my_layer: + schema: public + table: my_view_name + geometry_column: geom + srid: 3844 + bounds: [20.2, 43.5, 30.0, 48.3] # approximate extent + minzoom: 10 + maxzoom: 18 + properties: + object_id: text # explicit column name: pg_type + name: text + area: float8 +``` + +### Docker deployment + +```yaml +martin: + image: ghcr.io/maplibre/martin:v1.4.0 + command: ["--config", "/config/martin.yaml"] + environment: + - DATABASE_URL=postgresql://user:pass@host:5432/db + volumes: + - ./martin.yaml:/config/martin.yaml:ro + ports: + - "3010:3000" +``` + +### Custom SRID handling + +Martin handles non-4326/3857 SRIDs natively. Set `default_srid` globally or `srid` per source. Martin reprojects to Web Mercator (3857) internally for tile envelope calculations. Your PostGIS spatial indexes on the source SRID are used correctly. + +### Zoom-dependent simplification + +Create separate views per zoom range with `ST_SimplifyPreserveTopology`: + +```sql +-- z0-5: heavy simplification (2000m tolerance) +CREATE VIEW my_layer_z0 AS +SELECT id, name, ST_SimplifyPreserveTopology(geom, 2000) AS geom +FROM my_table; + +-- z8-12: moderate (50m) +CREATE VIEW my_layer_z8 AS +SELECT id, name, ST_SimplifyPreserveTopology(geom, 50) AS geom +FROM my_table; + +-- z12+: full precision +CREATE VIEW my_layer_z12 AS +SELECT * FROM my_table; +``` + +### Performance at 1M+ features + +- Set `minzoom` per source to avoid pathological low-zoom tiles +- Buildings: minzoom 14 (skip at overview levels) +- Use zoom-dependent simplified views for boundaries +- Add HTTP cache (nginx proxy_cache) in front of Martin +- Consider PMTiles for static overview layers + +--- + +## PMTiles: Pre-generated Tile Archives + +Best for: static/rarely-changing layers, overview zoom levels, eliminating DB load. + +### Pipeline + +```bash +# 1. Export from PostGIS, reproject to WGS84 +ogr2ogr -f FlatGeobuf -s_srs EPSG:3844 -t_srs EPSG:4326 \ + layer.fgb "PG:dbname=mydb" \ + -sql "SELECT id, name, geom FROM my_table" + +# 2. Generate PMTiles +tippecanoe -o output.pmtiles \ + --layer="my_layer" layer.fgb \ + --minimum-zoom=0 --maximum-zoom=14 \ + --drop-densest-as-needed \ + --detect-shared-borders \ + --hilbert --force + +# 3. Serve from any HTTP server with Range request support (MinIO, nginx, CDN) +``` + +### MapLibre integration + +```typescript +import { Protocol } from 'pmtiles'; +maplibregl.addProtocol('pmtiles', new Protocol().tile); + +// Add source +map.addSource('overview', { + type: 'vector', + url: 'pmtiles://https://my-server/tiles/overview.pmtiles', +}); +``` + +### Hybrid approach (recommended for large datasets) + +- PMTiles for overview (z0-z14): pre-generated, ~5ms serving, zero DB load +- Martin for detail (z14+): live from PostGIS, always-current data +- Rebuild PMTiles on schedule (nightly) or after data sync + +--- + +## MLT (MapLibre Tiles) — Next-Gen Format (2026) + +- 6x better compression than MVT (column-oriented layout) +- 3.7-4.4x faster client decode (SIMD-friendly) +- Martin v1.3+ supports serving MLT +- MapLibre GL JS 5.x supports decoding MLT +- Spec: [github.com/maplibre/maplibre-tile-spec](https://github.com/maplibre/maplibre-tile-spec) + +--- + +## Common Pitfalls + +1. **Martin auto-discovery drops properties** — always use explicit config with `auto_publish: false` +2. **Nested views lose SRID metadata** — cast geometry: `geom::geometry(Geometry, 3844)` +3. **GisUat.geometry is huge** — always `select` to exclude in list queries +4. **Low-zoom tiles scan entire dataset** — use zoom-dependent simplified views +5. **No tile cache by default** — add nginx/Varnish in front of any tile server +6. **tippecanoe requires WGS84** — reproject from custom SRID before generating PMTiles +7. **PMTiles not incrementally updatable** — full rebuild required on data change +8. **Tegola doesn't support custom SRIDs** — only 3857/4326, requires ST_Transform everywhere +9. **pg_tileserv `ST_Estimated_Extent` fails on views** — use materialized views or function layers +10. **Martin caches source schema at startup** — restart after view DDL changes diff --git a/martin.yaml b/martin.yaml index dd536e8..c68fdfd 100644 --- a/martin.yaml +++ b/martin.yaml @@ -1,4 +1,4 @@ -# Martin v0.15 configuration — optimized tile sources for ArchiTools Geoportal +# Martin v1.4 configuration — optimized tile sources for ArchiTools Geoportal # All geometries are EPSG:3844 (Stereo70). Bounds are approximate Romania extent. # Original table data is NEVER modified — views compute simplification on-the-fly. diff --git a/src/modules/geoportal/components/map-viewer.tsx b/src/modules/geoportal/components/map-viewer.tsx index 98c7e8f..2eed991 100644 --- a/src/modules/geoportal/components/map-viewer.tsx +++ b/src/modules/geoportal/components/map-viewer.tsx @@ -58,6 +58,7 @@ const LAYER_IDS = { terenuriLabel: "l-terenuri-label", cladiriFill: "l-cladiri-fill", cladiriLine: "l-cladiri-line", + cladiriLabel: "l-cladiri-label", selectionFill: "l-selection-fill", selectionLine: "l-selection-line", drawPolygonFill: "l-draw-polygon-fill", @@ -320,7 +321,7 @@ export const MapViewer = forwardRef( ], administrativ: [LAYER_IDS.adminLineOuter, LAYER_IDS.adminLineInner], terenuri: [LAYER_IDS.terenuriFill, LAYER_IDS.terenuriLine, LAYER_IDS.terenuriLabel], - cladiri: [LAYER_IDS.cladiriFill, LAYER_IDS.cladiriLine], + cladiri: [LAYER_IDS.cladiriFill, LAYER_IDS.cladiriLine, LAYER_IDS.cladiriLabel], }; for (const [group, layerIds] of Object.entries(mapping)) { const visible = vis[group] !== false; @@ -438,9 +439,15 @@ export const MapViewer = forwardRef( paint: { "fill-color": "#3b82f6", "fill-opacity": 0.5 } }); map.addLayer({ id: LAYER_IDS.cladiriLine, type: "line", source: SOURCES.cladiri, "source-layer": SOURCES.cladiri, minzoom: 14, paint: { "line-color": "#1e3a5f", "line-width": 0.6 } }); - // TODO: Building body labels (C1, C2...) — disabled pending Martin tile investigation - // Martin MVT tiles don't include cadastral_ref as a property despite the view exposing it. - // Next step: evaluate alternatives (pg_tileserv, GeoJSON source, Martin config). + // Building cadastral_ref labels (e.g. C1, C2...) + map.addLayer({ id: LAYER_IDS.cladiriLabel, type: "symbol", source: SOURCES.cladiri, "source-layer": SOURCES.cladiri, minzoom: 16, + layout: { + "text-field": ["coalesce", ["get", "cadastral_ref"], ""], + "text-font": ["Noto Sans Regular"], + "text-size": 9, "text-anchor": "center", "text-allow-overlap": false, + "text-max-width": 6, + }, + paint: { "text-color": "#1e3a5f", "text-halo-color": "#fff", "text-halo-width": 1 } }); // === Selection highlight === map.addLayer({ id: LAYER_IDS.selectionFill, type: "fill", source: SOURCES.terenuri, "source-layer": SOURCES.terenuri, minzoom: 13,