diff --git a/.gitignore b/.gitignore index 9fe443ea3c..883ec60caf 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,9 @@ target/ .settings bin/ +# cursor +.cursor/ + TODO data/ diff --git a/README.md b/README.md index 94ee4916fb..f517f907ab 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ java -Xmx1g -jar planetiler.jar --download --area=monaco Or using Docker: ```bash -docker run -e JAVA_TOOL_OPTIONS="-Xmx1g" -v "$(pwd)/data":/data ghcr.io/onthegomap/planetiler:latest --download --area=monaco +docker run --rm -e JAVA_TOOL_OPTIONS="-Xmx1g" -v "$(pwd)/data":/data ghcr.io/onthegomap/planetiler:latest --download --area=monaco ``` :warning: This starts off by downloading about 1GB of [data sources](NOTICE.md#data) required by the OpenMapTiles @@ -73,7 +73,7 @@ java -Xmx1g -jar planetiler.jar --download --area=monaco \ Docker: ```bash -docker run -e JAVA_TOOL_OPTIONS="-Xmx1g" -v "$(pwd)/data":/data ghcr.io/onthegomap/planetiler:latest --download --area=monaco \ +docker run --rm -e JAVA_TOOL_OPTIONS="-Xmx1g" -v "$(pwd)/data":/data ghcr.io/onthegomap/planetiler:latest --download --area=monaco \ --water-polygons-url=https://github.com/onthegomap/planetiler/raw/main/planetiler-core/src/test/resources/water-polygons-split-3857.zip \ --natural-earth-url=https://github.com/onthegomap/planetiler/raw/main/planetiler-core/src/test/resources/natural_earth_vector.sqlite.zip ``` diff --git a/planetiler-core/pom.xml b/planetiler-core/pom.xml index 891ce2a465..29c8b70d7b 100644 --- a/planetiler-core/pom.xml +++ b/planetiler-core/pom.xml @@ -305,6 +305,25 @@ com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier} + + org.codehaus.mojo + build-helper-maven-plugin + 3.6.0 + + + add-test-source + generate-test-sources + + add-source + + + + ${project.build.directory}/generated-sources/protobuf/java + + + + + diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/OsmReader.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/OsmReader.java index d94fcdc507..0549fc5f26 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/OsmReader.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/OsmReader.java @@ -98,6 +98,20 @@ public class OsmReader implements Closeable, MemoryEstimator.HasEstimate { private final Object waysInMultipolygonLock = new Object(); // ~7GB private LongLongMultimap.Replaceable multipolygonWayGeometries; + // for relation_members: track relations that need member processing + private Roaring64Bitmap relationsForMemberProcessing = new Roaring64Bitmap(); + // for relation_members: track node-to-relation membership + private LongLongMultimap.Appendable nodesToRelations = LongLongMultimap.newAppendableMultimap(); + // for relation_members: track ways-to-relation membership + private Roaring64Bitmap waysInRelationMembers = new Roaring64Bitmap(); + // for relation_members: store way geometries (node IDs) for member ways + private LongLongMultimap.Replaceable relationMembersWayGeometries; + // for relation_members: store way tags for member ways + private LongObjectHashMap> relationMembersWayTags = Hppc.newLongObjectHashMap(); + private final Object relationMembersWayTagsLock = new Object(); + // for relation_members: store node tags for member nodes + private LongObjectHashMap> relationMembersNodeTags = Hppc.newLongObjectHashMap(); + private final Object relationMembersNodeTagsLock = new Object(); // keep track of data needed to encode/decode role strings into a long private final ObjectIntHashMap roleIds = new ObjectIntHashMap<>(); private final IntObjectHashMap roleIdsReverse = new IntObjectHashMap<>(); @@ -133,6 +147,8 @@ public OsmReader(String name, Supplier osmSourceProvider, LongLo "relations", pass1Phaser::relations )); this.multipolygonWayGeometries = multipolygonGeometries; + // Initialize relation members way geometries storage (similar to multipolygons) + this.relationMembersWayGeometries = LongLongMultimap.newInMemoryReplaceableMultimap(); } /** @@ -277,19 +293,28 @@ void processPass1Blocks(Iterable> block try { List infos = profile.preprocessOsmRelation(relation); if (infos != null && !infos.isEmpty()) { + boolean needsMemberProcessing = infos.stream() + .anyMatch(info -> info.getClass().getName().contains("RelationMembersInfo")); synchronized (wayToRelationsLock) { + if (needsMemberProcessing) { + relationsForMemberProcessing.add(relation.id()); + } for (OsmRelationInfo info : infos) { relationInfo.put(relation.id(), info); relationInfoSizes.addAndGet(info.estimateMemoryUsageBytes()); } for (var member : relation.members()) { var type = member.type(); - // TODO handle nodes in relations - if (type == OsmElement.Type.WAY) { + if (type == OsmElement.Type.NODE) { + nodesToRelations.put(member.ref(), encodeRelationMembership(member.role(), relation.id())); + } else if (type == OsmElement.Type.WAY) { wayToRelations.put(member.ref(), encodeRelationMembership(member.role(), relation.id())); + if (needsMemberProcessing) { + waysInRelationMembers.add(member.ref()); + } } else if (type == OsmElement.Type.RELATION) { relationToParentRelations.put(member.ref(), - encodeRelationMembership(member.role(), relation.id())); + encodeRelationMembership(member.role(), relation.id())); } } } @@ -359,9 +384,18 @@ public void pass2(FeatureGroup writer, PlanetilerConfig config) { try (var renderer = createFeatureRenderer(writer, config, next)) { var phaser = pass2Phaser.forWorker(); var relationHandler = relationDistributor.forThread(relation -> { - var feature = processRelationPass2(relation, nodeLocations); - if (feature != null) { - render(featureCollectors, renderer, relation, feature); + // Process as multipolygon if applicable (independent check) + if (isMultipolygon(relation)) { + List> parentRelations = getRelationMembershipForWay(relation.id()); + SourceFeature multipolygonFeature = new MultipolygonSourceFeature(relation, nodeLocations, parentRelations); + render(featureCollectors, renderer, relation, multipolygonFeature); + } + // Process as relation_members if applicable (independent check - can be both) + if (relationsForMemberProcessing.contains(relation.id())) { + List> parentRelations = getRelationMembershipForWay(relation.id()); + RelationMemberDataProvider dataProvider = createRelationMemberDataProvider(); + SourceFeature relationMembersFeature = new RelationSourceFeature(relation, parentRelations, dataProvider); + render(featureCollectors, renderer, relation, relationMembersFeature); } rels.inc(); }); @@ -530,9 +564,48 @@ private FeatureRenderer createFeatureRenderer(FeatureGroup writer, PlanetilerCon ); } + private RelationMemberDataProvider createRelationMemberDataProvider() { + return new RelationMemberDataProvider() { + @Override + public LongArrayList getWayGeometry(long wayId) { + return relationMembersWayGeometries != null ? relationMembersWayGeometries.get(wayId) : null; + } + + @Override + public Map getWayTags(long wayId) { + return relationMembersWayTags != null ? relationMembersWayTags.get(wayId) : null; + } + + @Override + public Map getNodeTags(long nodeId) { + return relationMembersNodeTags != null ? relationMembersNodeTags.get(nodeId) : null; + } + + @Override + public org.locationtech.jts.geom.Coordinate getNodeCoordinate(long nodeId) { + long encoded = nodeLocationDb.get(nodeId); + if (encoded == LongLongMap.MISSING_VALUE) { + return null; + } + return new org.locationtech.jts.geom.CoordinateXY( + GeoUtils.decodeWorldX(encoded), + GeoUtils.decodeWorldY(encoded) + ); + } + }; + } + SourceFeature processNodePass2(OsmElement.Node node) { // nodes are simple because they already contain their location - return new NodeSourceFeature(node); + // Store node tags if this node is a member of a relation_members relation + // + if (!nodesToRelations.get(node.id()).isEmpty()) { + synchronized (relationMembersNodeTagsLock) { + relationMembersNodeTags.put(node.id(), node.tags()); + } + } + List> rels = getRelationMembershipForNode(node.id()); + return new NodeSourceFeature(node, rels); } WaySourceFeature processWayPass2(OsmElement.Way way, NodeLocationProvider nodeLocations) { @@ -546,6 +619,15 @@ WaySourceFeature processWayPass2(OsmElement.Way way, NodeLocationProvider nodeLo multipolygonWayGeometries.replaceValues(way.id(), nodes); } } + // Store way geometry and tags if this way is a member of a relation_members relation + if (waysInRelationMembers.contains(way.id())) { + synchronized (this) { + relationMembersWayGeometries.replaceValues(way.id(), nodes); + } + synchronized (relationMembersWayTagsLock) { + relationMembersWayTags.put(way.id(), way.tags()); + } + } boolean closed = nodes.size() > 1 && nodes.get(0) == nodes.get(nodes.size() - 1); // area tag used to differentiate between whether a closed way should be treated as a polygon or linestring String area = way.getString("area"); @@ -559,13 +641,19 @@ SourceFeature processRelationPass2(OsmElement.Relation rel, NodeLocationProvider if (isMultipolygon(rel)) { List> parentRelations = getRelationMembershipForWay(rel.id()); return new MultipolygonSourceFeature(rel, nodeLocations, parentRelations); - } else { - return null; } + if (relationsForMemberProcessing.contains(rel.id())) { + // This relation needs member processing (relation_members geometry) + List> parentRelations = getRelationMembershipForWay(rel.id()); + RelationMemberDataProvider dataProvider = createRelationMemberDataProvider(); + return new RelationSourceFeature(rel, parentRelations, dataProvider); + } + return null; } - private List> getRelationMembershipForWay(long wayId) { - LongArrayList relationIds = wayToRelations.get(wayId); + private List> getRelationMembership( + LongLongMultimap.Appendable relationshipMap, long elementId) { + LongArrayList relationIds = relationshipMap.get(elementId); List> rels = null; if (!relationIds.isEmpty()) { rels = new ArrayList<>(relationIds.size()); @@ -591,6 +679,14 @@ private List> getRelationMembershipForWay(long w return rels; } + private List> getRelationMembershipForWay(long wayId) { + return getRelationMembership(wayToRelations, wayId); + } + + private List> getRelationMembershipForNode(long nodeId) { + return getRelationMembership(nodesToRelations, nodeId); + } + private List> getRelationInfosForRelationId(long relationIdAndRole, HashSet visited, List parentRelationPath) { var parsed = decodeRelationMembership(relationIdAndRole); @@ -615,7 +711,11 @@ public long estimateMemoryUsageBytes() { long size = 0; size += waysInMultipolygon == null ? 0 : waysInMultipolygon.serializedSizeInBytes(); // multipolygonWayGeometries is reported separately + size += waysInRelationMembers == null ? 0 : waysInRelationMembers.serializedSizeInBytes(); + size += estimateSize(relationMembersWayTags); + size += estimateSize(relationMembersNodeTags); size += estimateSize(wayToRelations); + size += estimateSize(nodesToRelations); size += estimateSize(relationToParentRelations); size += estimateSize(relationInfo); size += estimateSize(roleIdsReverse); @@ -632,9 +732,17 @@ public void close() throws IOException { multipolygonWayGeometries.close(); multipolygonWayGeometries = null; } + if (relationMembersWayGeometries != null) { + relationMembersWayGeometries.close(); + relationMembersWayGeometries = null; + } wayToRelations = null; + nodesToRelations = null; relationToParentRelations = null; waysInMultipolygon = null; + waysInRelationMembers = null; + relationMembersWayTags = null; + relationMembersNodeTags = null; relationInfo = null; nodeLocationDb.close(); roleIds.release(); @@ -775,6 +883,11 @@ private class NodeSourceFeature extends OsmFeature { this.encodedLocation = node.encodedLocation(); } + NodeSourceFeature(OsmElement.Node node, List> relationInfo) { + super(node, true, false, false, relationInfo); + this.encodedLocation = node.encodedLocation(); + } + @Override protected Geometry computeWorldGeometry() { return GeoUtils.point( diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/RelationMemberDataProvider.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/RelationMemberDataProvider.java new file mode 100644 index 0000000000..151e8975a4 --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/RelationMemberDataProvider.java @@ -0,0 +1,40 @@ +package com.onthegomap.planetiler.reader.osm; + +import com.carrotsearch.hppc.LongArrayList; +import java.util.Map; + +/** + * Provides access to stored member data for relation_members processing. + * Used to retrieve way geometries, way tags, and node tags for relation members. + */ +public interface RelationMemberDataProvider { + + /** + * Gets the node IDs for a way member. + * @param wayId the way ID + * @return the node IDs, or null if not found + */ + LongArrayList getWayGeometry(long wayId); + + /** + * Gets the tags for a way member. + * @param wayId the way ID + * @return the tags, or null if not found + */ + Map getWayTags(long wayId); + + /** + * Gets the tags for a node member. + * @param nodeId the node ID + * @return the tags, or null if not found + */ + Map getNodeTags(long nodeId); + + /** + * Gets the coordinate for a node. + * @param nodeId the node ID + * @return the coordinate, or null if not found + */ + org.locationtech.jts.geom.Coordinate getNodeCoordinate(long nodeId); +} + diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/RelationSourceFeature.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/RelationSourceFeature.java new file mode 100644 index 0000000000..9b6256f80c --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/osm/RelationSourceFeature.java @@ -0,0 +1,88 @@ +package com.onthegomap.planetiler.reader.osm; + +import com.onthegomap.planetiler.geo.GeoUtils; +import com.onthegomap.planetiler.geo.GeometryException; +import com.onthegomap.planetiler.reader.SourceFeature; +import java.util.List; +import org.locationtech.jts.geom.Geometry; + +/** + * A SourceFeature representing an OSM relation that will be processed to extract member geometries. + * Relations themselves don't have direct geometry, but this provides access to relation members. + */ +public class RelationSourceFeature extends SourceFeature implements OsmSourceFeature { + + private final OsmElement.Relation relation; + private final RelationMemberDataProvider memberDataProvider; + private Geometry worldGeometry; + private Geometry latLonGeometry; + + public RelationSourceFeature(OsmElement.Relation relation, + List> parentRelations, + RelationMemberDataProvider memberDataProvider) { + super(relation.tags(), "osm", null, parentRelations, relation.id()); + this.relation = relation; + this.memberDataProvider = memberDataProvider; + } + + /** + * Constructor for use when member data provider is not available (e.g., in tests). + */ + public RelationSourceFeature(OsmElement.Relation relation, + List> parentRelations) { + this(relation, parentRelations, null); + } + + @Override + public Geometry worldGeometry() throws GeometryException { + // Relations don't have direct geometry - return empty geometry collection + if (worldGeometry == null) { + worldGeometry = GeoUtils.JTS_FACTORY.createGeometryCollection(); + } + return worldGeometry; + } + + @Override + public Geometry latLonGeometry() throws GeometryException { + // Relations don't have direct geometry - return empty geometry collection + if (latLonGeometry == null) { + latLonGeometry = GeoUtils.JTS_FACTORY.createGeometryCollection(); + } + return latLonGeometry; + } + + @Override + public boolean isPoint() { + return false; + } + + @Override + public boolean canBeLine() { + return false; + } + + @Override + public boolean canBePolygon() { + return false; + } + + @Override + public OsmElement originalElement() { + return relation; + } + + /** + * Returns the OSM relation this feature represents. + */ + public OsmElement.Relation relation() { + return relation; + } + + /** + * Returns the member data provider, or null if not available. + */ + public RelationMemberDataProvider memberDataProvider() { + return memberDataProvider; + } +} + diff --git a/planetiler-custommap/README.md b/planetiler-custommap/README.md index 0d0ace57db..8056ee2ab2 100644 --- a/planetiler-custommap/README.md +++ b/planetiler-custommap/README.md @@ -7,7 +7,7 @@ file as the first argument: # from a java build java -jar planetiler.jar generate-custom --schema=schema.yml # or with docker (put the schema in data/schema.yml to include in the attached volume) -docker run -v "$(pwd)/data":/data ghcr.io/onthegomap/planetiler:latest generate-custom --schema=/data/schema.yml +docker run --rm -v "$(pwd)/data":/data ghcr.io/onthegomap/planetiler:latest generate-custom --schema=/data/schema.yml ``` Schema files are in [YAML 1.2](https://yaml.org) format and support [anchors and aliases](#anchors-and-aliases) for @@ -238,6 +238,8 @@ A feature is a defined set of objects that meet a specified filter criteria. interior point - `innermost_point` to match on any geometry and for polygons, emit the furthest point from an edge, or for lines emit the midpoint. + - `relation_members` to match on OSM relations, and emit one feature per qualifying member (way or node) using each + member's geometry and tags. See [Relation Members](#relation-members) for details. - `include_when` - A [Boolean Expression](#boolean-expression) which determines the features to include. If unspecified, all features from the specified sources are included. - `exclude_when` - A [Boolean Expression](#boolean-expression) which determines if a feature that matched the include @@ -260,6 +262,23 @@ A feature is a defined set of objects that meet a specified filter criteria. `merge_polygons`, respectively. - `attributes` - An array of [Feature Attribute](#feature-attribute) objects that specify the attributes to be included on this output feature. +- `member_types` - (Only valid with `geometry: relation_members`) A list of OSM element types to include as members. + Valid values are `node`, `way`, or `relation`. Nested relations are not supported. If unspecified, all member types + are included. +- `member_roles` - (Only valid with `geometry: relation_members`) A list of role values to filter relation members by. + Only members with matching roles (or empty string for members with no role) will be processed. If unspecified, all + roles are included. +- `member_include_when` - (Only valid with `geometry: relation_members`) A [Boolean Expression](#boolean-expression) + that filters which relation members to include. The expression is evaluated in a [Member Context](#5-member-context) + that provides access to member tags, role, type, and reference ID. If unspecified, all members matching `member_types` + and `member_roles` are included. +- `member_exclude_when` - (Only valid with `geometry: relation_members`) A [Boolean Expression](#boolean-expression) + that excludes relation members from processing. The expression is evaluated in a [Member Context](#5-member-context). + This rule is applied after `member_include_when`. If unspecified, no exclusion filter is applied. +- `member_attributes` - (Only valid with `geometry: relation_members`) An array of [Feature Attribute](#feature-attribute) + objects that specify attributes to be included on output features from member tags. These attributes are evaluated + in a [Member Context](#5-member-context) that provides access to member-specific variables. Relation-level attributes + can be accessed via `feature.tags` in the parent context. For example: @@ -275,6 +294,112 @@ attributes: - { ... } ``` +## Relation Members + +The `relation_members` geometry type allows you to process individual members of OSM relations as separate features. +Instead of emitting one feature per relation, this emits one feature per qualifying member, using each member's geometry +and tags. + +This is useful for scenarios like: +- Extracting route segments from route relations (e.g., individual railway segments from a railway route) +- Processing individual way members of multipolygon relations +- Creating features from relation members with different attributes than the relation itself + +### How It Works + +When `geometry: relation_members` is specified: + +1. **Relation Selection**: The feature's `include_when` and `exclude_when` filters are applied to select which OSM + relations to process. Only relations matching these filters will have their members processed. + +2. **Member Filtering**: For each matching relation, members are filtered using: + + - `member_types`: Filter by OSM element type (node, way, relation) + - `member_roles`: Filter by the member's role in the relation + - `member_include_when`: Filter by member tags using structured or inline script expressions + - `member_exclude_when`: Exclude members by tags using structured or inline script expressions +3. **Feature Creation**: For each qualifying member: + - A feature is created using the member's actual geometry (point for nodes, line/polygon for ways) + - Relation-level attributes (from `attributes`) are applied using the relation's tags + - Member-level attributes (from `member_attributes`) are applied using the member's tags + - The feature ID is generated as `relation_id * 1000000 + member_ref` to ensure uniqueness + +### Examples + +Extract railway route segments: + +```yaml +layers: + - id: route_segments + features: + - source: osm + geometry: relation_members + include_when: + type: route + route: railway + member_types: [way] + member_include_when: + railway: rail + attributes: + - key: route_name + tag_value: name + member_attributes: + - key: segment_ref + tag_value: ref +``` + +Extract route segments with inline script filtering: + +```yaml +layers: + - id: electrified_railway_segments + features: + - source: osm + geometry: relation_members + include_when: + type: route + route: railway + member_types: [way] + member_include_when: '${ member.tags.railway == "rail" && member.tags.electrified != null }' + member_exclude_when: '${ member.tags.service != null || member.tags.access == "private" }' + attributes: + - key: route_name + tag_value: name + member_attributes: + - key: voltage + tag_value: voltage + type: integer + - key: electrified + tag_value: electrified +``` + +Extract nodes from route relations: + +```yaml +layers: + - id: route_stops + features: + - source: osm + geometry: relation_members + include_when: + type: route + route: bus + member_types: [node] + member_roles: [stop] + attributes: + - key: route_name + tag_value: name + member_attributes: + - key: stop_name + tag_value: name +``` + +### Limitations + +- Nested relations (relation members that are themselves relations) are not supported +- Member geometries are resolved from the relation's member data provider, which must be available during relation processing +- Member features use the member's actual geometry, so closed ways may be emitted as polygons if they have `area=yes` or no `area=no` tag + ## Feature Attribute Defines an attribute to include on an output vector tile feature and how to compute its value. @@ -602,6 +727,39 @@ For example: value: '${ match_value.lowerAscii() }' ``` +##### 5. Member Context + +Context available when processing relation members (only used with `geometry: relation_members`). This context is used +when evaluating `member_include_when`, `member_exclude_when`, and `member_attributes` expressions. + +Additional variables, on top of the post-match context: + +- `member.tags` - map with key/value tags from the relation member (way or node) +- `member.role` - string role of the member in the relation (empty string if no role) +- `member.type` - OSM element type of the member as a string: `"node"`, `"way"`, or `"relation"` +- `member.ref` - numeric OSM ID of the member element +- `member.id` - alias for `member.ref` + +Note: The parent context's `feature.tags` still refers to the relation's tags, not the member's tags. Use `member.tags` +to access member-specific tag values. + +For example: + +```yaml +# Filter members by their tags +member_include_when: '${ member.tags.railway == "rail" && member.tags.electrified != null }' + +# Combine member and relation tags in an attribute +member_attributes: + - key: full_name + value: '${ member.tags.name + " (" + feature.tags.name + ")" }' + +# Use member reference ID +member_attributes: + - key: member_id + value: '${ member.ref }' +``` + #### Built-In Functions Inline scripts can use @@ -794,7 +952,7 @@ the input file(s) for changes and validate the test cases on each change: # from a java build java -jar planetiler.jar verify schema.yml --watch # or with docker (put the schema in data/schema.yml to include in the attached volume) -docker run -v "$(pwd)/data":/data ghcr.io/onthegomap/planetiler:latest verify /data/schema.yml --watch +docker run --rm -v "$(pwd)/data":/data ghcr.io/onthegomap/planetiler:latest verify /data/schema.yml --watch ``` - `name` - Unique name for this test case. diff --git a/planetiler-custommap/planetiler.schema.json b/planetiler-custommap/planetiler.schema.json index f3312f770c..5c56056ac8 100644 --- a/planetiler-custommap/planetiler.schema.json +++ b/planetiler-custommap/planetiler.schema.json @@ -386,7 +386,8 @@ "polygon_centroid_if_convex", "polygon_point_on_surface", "point_on_line", - "innermost_point" + "innermost_point", + "relation_members" ] }, "source": { @@ -437,6 +438,40 @@ "items": { "$ref": "#/$defs/attribute" } + }, + "member_types": { + "description": "Only valid with geometry: relation_members. A list of OSM element types to include as members. Valid values are node, way, or relation. Nested relations are not supported. If unspecified, all member types are included.", + "type": "array", + "items": { + "type": "string", + "enum": [ + "node", + "way", + "relation" + ] + } + }, + "member_roles": { + "description": "Only valid with geometry: relation_members. A list of role values to filter relation members by. Only members with matching roles (or empty string for members with no role) will be processed. If unspecified, all roles are included.", + "type": "array", + "items": { + "type": "string" + } + }, + "member_include_when": { + "description": "Only valid with geometry: relation_members. A boolean expression that filters which relation members to include. The expression is evaluated in a Member Context that provides access to member tags, role, type, and reference ID. If unspecified, all members matching member_types and member_roles are included.", + "$ref": "#/$defs/boolean_expression" + }, + "member_exclude_when": { + "description": "Only valid with geometry: relation_members. A boolean expression that excludes relation members from processing. The expression is evaluated in a Member Context. This rule is applied after member_include_when. If unspecified, no exclusion filter is applied.", + "$ref": "#/$defs/boolean_expression" + }, + "member_attributes": { + "description": "Only valid with geometry: relation_members. An array of attribute objects that specify attributes to be included on output features from member tags. These attributes are evaluated in a Member Context that provides access to member-specific variables. Relation-level attributes can be accessed via feature.tags in the parent context.", + "type": "array", + "items": { + "$ref": "#/$defs/attribute" + } } } }, diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredFeature.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredFeature.java index 185fad8cba..cf3cb2682a 100644 --- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredFeature.java +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredFeature.java @@ -4,6 +4,7 @@ import static com.onthegomap.planetiler.expression.Expression.not; import static com.onthegomap.planetiler.util.Coalesce.coalesce; +import com.carrotsearch.hppc.LongArrayList; import com.onthegomap.planetiler.FeatureCollector; import com.onthegomap.planetiler.FeatureCollector.Feature; import com.onthegomap.planetiler.custommap.configschema.AttributeDefinition; @@ -12,10 +13,15 @@ import com.onthegomap.planetiler.custommap.configschema.FeatureLayer; import com.onthegomap.planetiler.custommap.expression.ScriptEnvironment; import com.onthegomap.planetiler.expression.Expression; +import com.onthegomap.planetiler.geo.GeoUtils; import com.onthegomap.planetiler.geo.GeometryException; import com.onthegomap.planetiler.reader.SourceFeature; +import com.onthegomap.planetiler.reader.osm.OsmElement; +import com.onthegomap.planetiler.reader.osm.RelationMemberDataProvider; +import com.onthegomap.planetiler.reader.osm.RelationSourceFeature; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -23,6 +29,9 @@ import java.util.function.BiConsumer; import java.util.function.Function; import java.util.function.ObjDoubleConsumer; +import org.locationtech.jts.geom.CoordinateSequence; +import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.geom.impl.CoordinateArraySequence; /** * A map feature, configured from a YML configuration file. @@ -38,15 +47,26 @@ public class ConfiguredFeature { private final TagValueProducer tagValueProducer; private final List> featureProcessors; private final Set sources; + private final String layerId; private final ScriptEnvironment processFeatureContext; private final ScriptEnvironment featureAttributeContext; private ScriptEnvironment featurePostMatchContext; + private ScriptEnvironment memberContext; + + // Member processing configuration (only used for RELATION_MEMBERS geometry) + private final boolean isRelationMembers; + private final Set memberTypes; + private final Set memberRoles; + private final Expression memberIncludeWhen; + private final Expression memberExcludeWhen; + private final List> memberAttributeProcessors; private final boolean splitAtIntersections; public ConfiguredFeature(FeatureLayer layer, TagValueProducer tagValueProducer, FeatureItem feature, Contexts.Root rootContext) { sources = Set.copyOf(feature.source()); + layerId = layer.id(); FeatureGeometry geometryType = feature.geometry(); @@ -87,6 +107,47 @@ public ConfiguredFeature(FeatureLayer layer, TagValueProducer tagValueProducer, //Factory to generate the right feature type from FeatureCollector geometryFactory = geometryType.newGeometryFactory(layer.id()); + // Member processing configuration (only used for RELATION_MEMBERS geometry) + isRelationMembers = geometryType == FeatureGeometry.RELATION_MEMBERS; + + if (isRelationMembers) { + memberContext = Contexts.MemberContext.description(rootContext); + + List memberTypesList = feature.memberTypes(); + memberTypes = memberTypesList.isEmpty() ? + Set.of("node", "way", "relation") : Set.copyOf(memberTypesList); + + List memberRolesList = feature.memberRoles(); + memberRoles = memberRolesList == null || memberRolesList.isEmpty() ? + null : Set.copyOf(memberRolesList); + + if (feature.memberIncludeWhen() != null) { + memberIncludeWhen = BooleanExpressionParser.parse( + feature.memberIncludeWhen(), tagValueProducer, memberContext); + } else { + memberIncludeWhen = Expression.TRUE; + } + + if (feature.memberExcludeWhen() != null) { + memberExcludeWhen = BooleanExpressionParser.parse( + feature.memberExcludeWhen(), tagValueProducer, memberContext); + } else { + memberExcludeWhen = Expression.FALSE; + } + + List> memberAttrProcessors = new ArrayList<>(); + for (var memberAttr : feature.memberAttributes()) { + memberAttrProcessors.add(memberAttributeProcessor(memberAttr)); + } + memberAttributeProcessors = memberAttrProcessors; + } else { + memberTypes = null; + memberRoles = null; + memberIncludeWhen = null; + memberExcludeWhen = null; + memberAttributeProcessors = null; + } + //Configure logic for each attribute in the output tile List> processors = new ArrayList<>(); for (var attribute : feature.attributes()) { @@ -370,11 +431,298 @@ public void processFeature(Contexts.FeaturePostMatch context, FeatureCollector f // Ensure that this feature is from the correct source (index should enforce this, so just check when assertions enabled) assert sources.isEmpty() || sources.contains(sourceFeature.getSource()); + // Special handling for relation_members geometry + if (isRelationMembers && sourceFeature instanceof RelationSourceFeature relationFeature) { + processRelationMembers(context, relationFeature, features); + return; + } + var f = geometryFactory.apply(features); for (var processor : featureProcessors) { processor.accept(context, f); } } + + /** + * Process relation members for relation_members geometry type. + * Iterates over relation members, filters them, and creates features for each qualifying member. + */ + private void processRelationMembers(Contexts.FeaturePostMatch context, RelationSourceFeature relationFeature, + FeatureCollector features) { + var relation = relationFeature.relation(); + var relationPostMatch = context; + + Set processedRefs = new HashSet<>(); + + for (var member : relation.members()) { + if (shouldProcessMember(member, processedRefs, relationFeature, relationPostMatch)) { + try { + createMemberFeature(member, getMemberContext(member, relationFeature, relationPostMatch), + relationPostMatch, features); + } catch (Exception e) { + org.slf4j.LoggerFactory.getLogger(ConfiguredFeature.class) + .warn("Error creating feature for relation member {} in relation {}: {}", + member.ref(), relation.id(), e.getMessage()); + } + } + } + } + + /** + * Check if a relation member should be processed based on filters. + * Returns true if the member should be processed, false otherwise. + */ + private boolean shouldProcessMember(OsmElement.Relation.Member member, Set processedRefs, + RelationSourceFeature relationFeature, Contexts.FeaturePostMatch relationPostMatch) { + if (!processedRefs.add(member.ref())) { + return false; + } + + String memberTypeStr = member.type().name().toLowerCase(); + if (!memberTypes.contains(memberTypeStr)) { + return false; + } + + // Nested relations are not supported + if (member.type() == OsmElement.Type.RELATION) { + return false; + } + + if (memberRoles != null) { + String role = member.role(); + // Empty string matches members with no role + if (!memberRoles.contains(role) && !(role.isEmpty() && memberRoles.contains(""))) { + return false; + } + } + + Map memberTags = getMemberTags(member, relationFeature); + + var memberContextInstance = new Contexts.MemberContext( + relationPostMatch, + memberTags, + member.role(), + memberTypeStr, + member.ref() + ); + + if (!memberIncludeWhen.evaluate(memberContextInstance, new ArrayList<>())) { + return false; + } + + if (memberExcludeWhen.evaluate(memberContextInstance, new ArrayList<>())) { + return false; + } + + return true; + } + + /** + * Create a member context for expression evaluation. + */ + private Contexts.MemberContext getMemberContext(OsmElement.Relation.Member member, + RelationSourceFeature relationFeature, Contexts.FeaturePostMatch relationPostMatch) { + Map memberTags = getMemberTags(member, relationFeature); + String memberTypeStr = member.type().name().toLowerCase(); + return new Contexts.MemberContext( + relationPostMatch, + memberTags, + member.role(), + memberTypeStr, + member.ref() + ); + } + + /** + * Get tags for a relation member from the stored member data. + */ + private Map getMemberTags(OsmElement.Relation.Member member, + RelationSourceFeature relationFeature) { + RelationMemberDataProvider dataProvider = relationFeature.memberDataProvider(); + if (dataProvider == null) { + return Map.of(); + } + + if (member.type() == OsmElement.Type.WAY) { + Map tags = dataProvider.getWayTags(member.ref()); + return tags != null ? tags : Map.of(); + } else if (member.type() == OsmElement.Type.NODE) { + Map tags = dataProvider.getNodeTags(member.ref()); + return tags != null ? tags : Map.of(); + } + + return Map.of(); + } + + /** + * Create a feature for a relation member with actual geometry. + */ + private void createMemberFeature(OsmElement.Relation.Member member, + Contexts.MemberContext memberContext, + Contexts.FeaturePostMatch relationContext, + FeatureCollector features) { + + RelationSourceFeature relationFeature = (RelationSourceFeature) relationContext.feature(); + RelationMemberDataProvider dataProvider = relationFeature.memberDataProvider(); + + if (dataProvider == null) { + // No data provider available - skip this member + return; + } + + FeatureCollector.Feature memberFeature; + Geometry geometry = null; + + if (member.type() == OsmElement.Type.NODE) { + org.locationtech.jts.geom.Coordinate coord = dataProvider.getNodeCoordinate(member.ref()); + if (coord == null) { + return; + } + geometry = GeoUtils.JTS_FACTORY.createPoint(coord); + memberFeature = features.geometry(layerId, geometry); + + } else if (member.type() == OsmElement.Type.WAY) { + LongArrayList nodeIds = dataProvider.getWayGeometry(member.ref()); + if (nodeIds == null || nodeIds.isEmpty()) { + return; + } + + CoordinateSequence coords = buildCoordinateSequence(nodeIds, dataProvider); + if (coords == null || coords.size() < 2) { + return; + } + + // Determine if closed way should be polygon or line based on area tag and geometry + boolean closed = coords.size() > 1 && + coords.getCoordinate(0).equals(coords.getCoordinate(coords.size() - 1)); + Map wayTags = dataProvider.getWayTags(member.ref()); + String area = wayTags != null ? (String) wayTags.get("area") : null; + boolean canBePolygon = closed && !"no".equals(area) && coords.size() >= 4; + + if (canBePolygon) { + geometry = GeoUtils.JTS_FACTORY.createPolygon(coords); + memberFeature = features.geometry(layerId, geometry); + } else { + geometry = GeoUtils.JTS_FACTORY.createLineString(coords); + memberFeature = features.geometry(layerId, geometry); + } + + } else { + // Nested relations are not supported + return; + } + + for (var processor : featureProcessors) { + processor.accept(relationContext, memberFeature); + } + + if (memberAttributeProcessors != null) { + for (var processor : memberAttributeProcessors) { + processor.accept(memberContext, memberFeature); + } + } + + // Generate unique feature ID: relation ID * 1M + member ref to avoid collisions + long uniqueId = relationContext.feature().id() * 1000000L + member.ref(); + memberFeature.setId(uniqueId); + } + + /** + * Build a coordinate sequence from node IDs using the data provider. + */ + private CoordinateSequence buildCoordinateSequence(LongArrayList nodeIds, + RelationMemberDataProvider dataProvider) { + org.locationtech.jts.geom.Coordinate[] coords = new org.locationtech.jts.geom.Coordinate[nodeIds.size()]; + int validCount = 0; + + for (int i = 0; i < nodeIds.size(); i++) { + org.locationtech.jts.geom.Coordinate coord = dataProvider.getNodeCoordinate(nodeIds.get(i)); + if (coord != null) { + coords[validCount++] = coord; + } else { + return null; + } + } + + if (validCount < 2) { + return null; + } + + // Defensive: trim array if coordinates were missing (shouldn't happen in practice) + if (validCount < coords.length) { + org.locationtech.jts.geom.Coordinate[] trimmed = new org.locationtech.jts.geom.Coordinate[validCount]; + System.arraycopy(coords, 0, trimmed, 0, validCount); + coords = trimmed; + } + + return new CoordinateArraySequence(coords); + } + + /** + * Generate logic which processes member-level attributes. + */ + private BiConsumer memberAttributeProcessor(AttributeDefinition attribute) { + var tagKey = attribute.key(); + var attributeValueProducer = memberAttributeValueProducer(attribute); + var fallback = attribute.fallback(); + + var attrIncludeWhen = attribute.includeWhen(); + var attrExcludeWhen = attribute.excludeWhen(); + + var attributeTest = Expression.and( + attrIncludeWhen == null ? Expression.TRUE : + BooleanExpressionParser.parse(attrIncludeWhen, tagValueProducer, memberContext), + attrExcludeWhen == null ? Expression.TRUE : + Expression.not(BooleanExpressionParser.parse(attrExcludeWhen, tagValueProducer, memberContext)) + ).simplify(); + + return (context, f) -> { + Object value = null; + if (attributeTest.evaluate(context)) { + value = attributeValueProducer.apply(context); + if ("".equals(value)) { + value = null; + } + } + if (value == null) { + value = fallback; + } + if (value != null) { + f.setAttr(tagKey, value); + } + }; + } + + /** + * Produces logic that generates attribute values for member attributes. + */ + private Function memberAttributeValueProducer(AttributeDefinition attribute) { + Object type = attribute.type(); + + Map value = new HashMap<>(); + if ("match_key".equals(type)) { + value.put("value", "${match_key}"); + } else if ("match_value".equals(type)) { + value.put("value", "${match_value}"); + } else { + if (type != null) { + value.put("type", type); + } + if (attribute.coalesce() != null) { + value.put("coalesce", attribute.coalesce()); + } else if (attribute.value() != null) { + value.put("value", attribute.value()); + } else if (attribute.tagValue() != null) { + value.put("tag_value", attribute.tagValue()); + } else if (attribute.argValue() != null) { + value.put("arg_value", attribute.argValue()); + } else { + value.put("tag_value", attribute.key()); + } + } + + return ConfigExpressionParser.parse(value, tagValueProducer, memberContext, Object.class); + } private Double maxIgnoringNulls(Double a, Double b) { if (a == null) diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredProfile.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredProfile.java index e99aeb8a14..0a1b12847a 100644 --- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredProfile.java +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredProfile.java @@ -7,7 +7,9 @@ import com.onthegomap.planetiler.FeatureMerge; import com.onthegomap.planetiler.Profile; import com.onthegomap.planetiler.VectorTile; +import com.onthegomap.planetiler.custommap.configschema.FeatureGeometry; import com.onthegomap.planetiler.custommap.configschema.FeatureLayer; +import com.onthegomap.planetiler.custommap.configschema.RelationMembersInfo; import com.onthegomap.planetiler.custommap.configschema.SchemaConfig; import com.onthegomap.planetiler.expression.Expression; import com.onthegomap.planetiler.expression.MultiExpression; @@ -16,6 +18,8 @@ import com.onthegomap.planetiler.geo.GeometryException; import com.onthegomap.planetiler.reader.SourceFeature; import com.onthegomap.planetiler.reader.osm.OsmElement; +import com.onthegomap.planetiler.reader.osm.OsmRelationInfo; +import com.onthegomap.planetiler.reader.osm.RelationSourceFeature; import com.onthegomap.planetiler.reader.osm.OsmReader; import com.onthegomap.planetiler.reader.osm.OsmSourceFeature; import java.nio.file.Path; @@ -37,6 +41,7 @@ public class ConfiguredProfile implements Profile { private final Index featureLayerMatcher; private final TagValueProducer tagValueProducer; private final Contexts.Root rootContext; + private final Index relationMembersMatcher; private final Expression splitFeaturesAtWays; public ConfiguredProfile(SchemaConfig schema, Contexts.Root rootContext) { @@ -51,6 +56,7 @@ public ConfiguredProfile(SchemaConfig schema, Contexts.Root rootContext) { tagValueProducer = new TagValueProducer(schema.inputMappings()); List> configuredFeatureEntries = new ArrayList<>(); + List> relationMembersEntries = new ArrayList<>(); List splitAtIntersectionTests = new ArrayList<>(); @@ -61,7 +67,11 @@ public ConfiguredProfile(SchemaConfig schema, Contexts.Root rootContext) { var configuredFeature = new ConfiguredFeature(layer, tagValueProducer, feature, rootContext); var entry = new Entry<>(configuredFeature, configuredFeature.matchExpression()); configuredFeatureEntries.add(entry); - + + // Track features with relation_members geometry for preprocessOsmRelation + if (feature.geometry() == FeatureGeometry.RELATION_MEMBERS) { + relationMembersEntries.add(entry); + } if (configuredFeature.splitAtIntersections()) { splitAtIntersectionTests.add(configuredFeature.matchExpression()); } @@ -69,6 +79,8 @@ public ConfiguredProfile(SchemaConfig schema, Contexts.Root rootContext) { } featureLayerMatcher = MultiExpression.of(configuredFeatureEntries).index(); + relationMembersMatcher = relationMembersEntries.isEmpty() ? null : + MultiExpression.of(relationMembersEntries).index(); splitFeaturesAtWays = splitAtIntersectionTests.isEmpty() ? null : @@ -184,6 +196,26 @@ public FeatureLayer findFeatureLayer(String layerId) { } @Override + public List preprocessOsmRelation(OsmElement.Relation relation) { + // Check if this relation matches any relation_members features + if (relationMembersMatcher == null) { + return List.of(); + } + + // Create a RelationSourceFeature to test if this relation matches + SourceFeature fakeFeature = new RelationSourceFeature(relation, List.of()); + + var context = rootContext.createProcessFeatureContext(fakeFeature, tagValueProducer); + var matches = relationMembersMatcher.getMatchesWithTriggers(context); + + if (!matches.isEmpty()) { + // This relation matches a relation_members feature, return RelationMembersInfo + return List.of(new RelationMembersInfo(relation.id())); + } + + return List.of(); + } + public boolean splitOsmWayAtIntersections(OsmElement.Way way) { return splitFeaturesAtWays != null && diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/Contexts.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/Contexts.java index 0f08c160e7..4e8abf7b74 100644 --- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/Contexts.java +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/Contexts.java @@ -288,7 +288,7 @@ default Object argument(String key) { * Makes nested contexts adhere to {@link WithTags} and {@link WithGeometryType} by recursively fetching source * feature from the root context. */ - private interface FeatureContext extends ScriptContext, WithTags, WithGeometryType, NestedContext, WithSourceLayer, + private interface FeatureContext extends WithGeometryType, NestedContext, WithSourceLayer, WithSource { default FeatureContext parent() { @@ -515,4 +515,77 @@ public Object apply(String key) { return VALUE.equals(key) ? wrapNullable(value) : parent.apply(key); } } + + /** + * Context available when processing relation members. + * Extends FeaturePostMatch to provide access to member-specific variables. + * + * @param parent The parent FeaturePostMatch context (for relation-level variables) + * @param memberTags Tags from the member element (way/node) + * @param memberRole Role of the member in the relation + * @param memberType OSM element type: "node", "way", or "relation" + * @param memberRef OSM ID of the member element + */ + public record MemberContext( + @Override FeaturePostMatch parent, + Map memberTags, + String memberRole, + String memberType, + long memberRef + ) implements FeatureContext { + + private static final String MEMBER_TAGS = "member.tags"; + private static final String MEMBER_ROLE = "member.role"; + private static final String MEMBER_TYPE = "member.type"; + private static final String MEMBER_REF = "member.ref"; + private static final String MEMBER_ID = "member.id"; + + public static ScriptEnvironment description(Root root) { + return FeaturePostMatch.description(root) + .forInput(MemberContext.class) + .withDeclarations( + Decls.newVar(MEMBER_TAGS, Decls.newMapType(Decls.String, Decls.Any)), + Decls.newVar(MEMBER_ROLE, Decls.String), + Decls.newVar(MEMBER_TYPE, Decls.String), + Decls.newVar(MEMBER_REF, Decls.Int), + Decls.newVar(MEMBER_ID, Decls.Int) + ); + } + + @Override + public Map tags() { + // Override to return member tags instead of relation tags + // Return a mutable copy to avoid UnsupportedOperationException + return new HashMap<>(memberTags); + } + + @Override + public Object apply(String key) { + if (key != null) { + return switch (key) { + case MEMBER_TAGS -> mapWithDefault(memberTags, NullValue.NULL_VALUE); + case MEMBER_ROLE -> wrapNullable(memberRole); + case MEMBER_TYPE -> wrapNullable(memberType); + case MEMBER_REF, MEMBER_ID -> memberRef; + default -> parent.apply(key); + }; + } else { + return null; + } + } + + private static Map mapWithDefault(Map map, Object nullValue) { + return new ForwardingMap<>() { + @Override + protected Map delegate() { + return map; + } + + @Override + public V get(Object key) { + return map.getOrDefault(key, (V) nullValue); + } + }; + } + } } diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/FeatureGeometry.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/FeatureGeometry.java index 085efd944d..ff76b526e1 100644 --- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/FeatureGeometry.java +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/FeatureGeometry.java @@ -3,8 +3,14 @@ import com.fasterxml.jackson.annotation.JsonEnumDefaultValue; import com.fasterxml.jackson.annotation.JsonProperty; import com.onthegomap.planetiler.FeatureCollector; +import com.onthegomap.planetiler.custommap.Contexts; import com.onthegomap.planetiler.expression.Expression; import com.onthegomap.planetiler.geo.GeometryType; +import com.onthegomap.planetiler.reader.SourceFeature; +import com.onthegomap.planetiler.reader.WithTags; +import com.onthegomap.planetiler.reader.osm.OsmElement; +import com.onthegomap.planetiler.reader.osm.OsmSourceFeature; +import java.util.List; import java.util.function.BiFunction; import java.util.function.Function; @@ -34,7 +40,45 @@ public enum FeatureGeometry { @JsonProperty("innermost_point") INNERMOST_POINT(GeometryType.UNKNOWN, FeatureCollector::innermostPoint), @JsonProperty("split_line") - SPLIT_LINE(GeometryType.LINE, FeatureCollector::splitLine); + SPLIT_LINE(GeometryType.LINE, FeatureCollector::splitLine), + @JsonProperty("relation_members") + RELATION_MEMBERS(GeometryType.UNKNOWN, null) { + @Override + public Expression featureTest() { + return new Expression() { + @Override + public boolean evaluate(WithTags input, List matchKeys) { + // Input is always a ProcessFeature context in production code + if (input instanceof Contexts.ProcessFeature processFeature) { + SourceFeature sourceFeature = processFeature.feature(); + return sourceFeature instanceof OsmSourceFeature osmFeature && + osmFeature.originalElement().type() == OsmElement.Type.RELATION; + } + return false; + } + + @Override + public Expression simplifyOnce() { + return this; + } + + @Override + public String generateJavaCode() { + return "relationTypeTest()"; + } + }; + } + + @Override + public Function newGeometryFactory(String layerName) { + // RELATION_MEMBERS geometry is handled specially in ConfiguredFeature.processRelationMembers() + // This factory should never be called + return features -> { + throw new UnsupportedOperationException( + "RELATION_MEMBERS geometry factory must be set up in ConfiguredFeature constructor"); + }; + } + }; public final GeometryType geometryType; public final BiFunction geometryFactory; diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/FeatureItem.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/FeatureItem.java index 426fb87414..9da79290a0 100644 --- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/FeatureItem.java +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/FeatureItem.java @@ -2,8 +2,10 @@ import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonProperty; +import com.onthegomap.planetiler.custommap.expression.ParseException; import java.util.Collection; import java.util.List; +import java.util.Set; public record FeatureItem( @JsonFormat(with = JsonFormat.Feature.ACCEPT_SINGLE_VALUE_AS_ARRAY) List source, @@ -16,9 +18,38 @@ public record FeatureItem( @JsonProperty FeatureGeometry geometry, @JsonProperty("include_when") Object includeWhen, @JsonProperty("exclude_when") Object excludeWhen, - Collection attributes + Collection attributes, + @JsonProperty("member_types") List memberTypes, + @JsonProperty("member_roles") List memberRoles, + @JsonProperty("member_include_when") Object memberIncludeWhen, + @JsonProperty("member_exclude_when") Object memberExcludeWhen, + @JsonProperty("member_attributes") Collection memberAttributes ) { + private static final Set VALID_MEMBER_TYPES = Set.of("node", "way", "relation"); + + public FeatureItem { + // Validate that member_* fields are only used with relation_members geometry + FeatureGeometry actualGeometry = geometry == null ? FeatureGeometry.ANY : geometry; + boolean hasMemberFields = memberTypes != null || memberRoles != null || memberIncludeWhen != null || + memberExcludeWhen != null || (memberAttributes != null && !memberAttributes.isEmpty()); + + if (hasMemberFields && actualGeometry != FeatureGeometry.RELATION_MEMBERS) { + throw new ParseException( + "member_types, member_roles, member_include_when, member_exclude_when, and member_attributes can only be used with geometry: relation_members"); + } + + // Validate member_types values + if (memberTypes != null) { + for (String type : memberTypes) { + if (!VALID_MEMBER_TYPES.contains(type)) { + throw new ParseException( + "Invalid member_types value: '" + type + "'. Valid values are: node, way, relation"); + } + } + } + } + @Override public Collection attributes() { return attributes == null ? List.of() : attributes; @@ -33,4 +64,19 @@ public FeatureGeometry geometry() { public List source() { return source == null ? List.of() : source; } + + @Override + public List memberTypes() { + return memberTypes == null ? List.of() : memberTypes; + } + + @Override + public List memberRoles() { + return memberRoles == null ? List.of() : memberRoles; + } + + @Override + public Collection memberAttributes() { + return memberAttributes == null ? List.of() : memberAttributes; + } } diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/RelationMembersInfo.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/RelationMembersInfo.java new file mode 100644 index 0000000000..94b41c4914 --- /dev/null +++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/RelationMembersInfo.java @@ -0,0 +1,11 @@ +package com.onthegomap.planetiler.custommap.configschema; + +import com.onthegomap.planetiler.reader.osm.OsmRelationInfo; + +/** + * Marker class to indicate that a relation should be processed for member extraction. + * Used when a feature has {@code geometry: relation_members}. + */ +public record RelationMembersInfo(long id) implements OsmRelationInfo { +} + diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredFeatureTest.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredFeatureTest.java index 608849ee1d..95204abe48 100644 --- a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredFeatureTest.java +++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredFeatureTest.java @@ -928,6 +928,8 @@ void testInvalidSchemas() { testInvalidSchema("bad_geometry_type.yml", "Profile defined with invalid geometry type"); testInvalidSchema("no_layers.yml", "Profile defined with no layers"); testInvalidSchema("invalid_post_process.yml", "Profile defined with invalid post process element"); + testInvalidSchema("member_fields_wrong_geometry.yml", "member_types, member_roles, member_include_when, member_exclude_when, and member_attributes can only be used with geometry: relation_members"); + testInvalidSchema("invalid_member_types.yml", "Invalid member_types value"); } private void testInvalidSchema(String filename, String message) { diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/RelationMembersIntegrationTest.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/RelationMembersIntegrationTest.java new file mode 100644 index 0000000000..863e2db9dd --- /dev/null +++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/RelationMembersIntegrationTest.java @@ -0,0 +1,884 @@ +package com.onthegomap.planetiler.custommap; + +import static java.util.Collections.emptyList; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.carrotsearch.hppc.LongArrayList; +import com.onthegomap.planetiler.FeatureCollector; +import com.onthegomap.planetiler.config.PlanetilerConfig; +import com.onthegomap.planetiler.custommap.configschema.SchemaConfig; +import com.onthegomap.planetiler.reader.SimpleFeature; +import com.onthegomap.planetiler.reader.SourceFeature; +import com.onthegomap.planetiler.reader.osm.OsmElement; +import com.onthegomap.planetiler.reader.osm.RelationMemberDataProvider; +import com.onthegomap.planetiler.reader.osm.RelationSourceFeature; +import com.onthegomap.planetiler.stats.Stats; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.locationtech.jts.geom.Coordinate; + +/** + * Integration tests for relation_members geometry type. + * Tests the end-to-end flow of processing OSM relation members as separate features. + */ +class RelationMembersIntegrationTest { + + private PlanetilerConfig planetilerConfig = PlanetilerConfig.defaults(); + + @BeforeEach + void setUp() { + planetilerConfig = PlanetilerConfig.defaults(); + } + + private ConfiguredProfile loadConfig(String config) { + var schema = SchemaConfig.load(config); + var root = Contexts.buildRootContext(planetilerConfig.arguments(), schema.args()); + planetilerConfig = root.config(); + return new ConfiguredProfile(schema, root); + } + + private void testFeature(SourceFeature sf, Consumer test, int expectedMatchCount, + ConfiguredProfile profile) { + var factory = new FeatureCollector.Factory(planetilerConfig, Stats.inMemory()); + var fc = factory.get(sf); + + profile.processFeature(sf, fc); + + var length = new AtomicInteger(0); + + fc.forEach(f -> { + test.accept(f); + length.incrementAndGet(); + }); + + assertEquals(expectedMatchCount, length.get(), "Wrong number of features generated"); + } + + /** + * Creates a mock RelationMemberDataProvider for testing. + */ + private RelationMemberDataProvider createMockDataProvider( + Map wayGeometries, + Map> wayTags, + Map> nodeTags, + Map nodeCoordinates) { + return new RelationMemberDataProvider() { + @Override + public LongArrayList getWayGeometry(long wayId) { + return wayGeometries.get(wayId); + } + + @Override + public Map getWayTags(long wayId) { + return wayTags.get(wayId); + } + + @Override + public Map getNodeTags(long nodeId) { + return nodeTags.get(nodeId); + } + + @Override + public Coordinate getNodeCoordinate(long nodeId) { + return nodeCoordinates.get(nodeId); + } + }; + } + + /** + * Test 1: Basic railway route with way members (GitHub Issue Example 1) + */ + @Test + void testRouteRelation_RailwaySegments() { + var config = """ + sources: + osm: + type: osm + url: geofabrik:rhode-island + layers: + - id: route_segments + features: + - source: osm + geometry: relation_members + include_when: + type: route + route: railway + member_types: [way] + member_include_when: + railway: rail + attributes: + - key: route_name + tag_value: name + member_attributes: + - key: segment_ref + tag_value: ref + """; + + // Create relation with 3 way members + var relation = new OsmElement.Relation(100L, Map.of( + "type", "route", + "route", "railway", + "name", "Main Line" + ), List.of( + new OsmElement.Relation.Member(OsmElement.Type.WAY, 1L, ""), + new OsmElement.Relation.Member(OsmElement.Type.WAY, 2L, ""), + new OsmElement.Relation.Member(OsmElement.Type.WAY, 3L, "") + )); + + // Create mock data provider + var wayGeometries = Map.of( + 1L, LongArrayList.from(10L, 11L, 12L), + 2L, LongArrayList.from(12L, 13L, 14L), + 3L, LongArrayList.from(14L, 15L, 16L) + ); + var wayTags = Map.>of( + 1L, Map.of("railway", "rail", "ref", "A1"), + 2L, Map.of("railway", "rail", "ref", "A2"), + 3L, Map.of("railway", "rail", "ref", "A3") + ); + var nodeCoordinates = Map.of( + 10L, new Coordinate(0.0, 0.0), + 11L, new Coordinate(0.1, 0.0), + 12L, new Coordinate(0.2, 0.0), + 13L, new Coordinate(0.3, 0.0), + 14L, new Coordinate(0.4, 0.0), + 15L, new Coordinate(0.5, 0.0), + 16L, new Coordinate(0.6, 0.0) + ); + + var dataProvider = createMockDataProvider(wayGeometries, wayTags, Map.of(), nodeCoordinates); + var relationFeature = new RelationSourceFeature(relation, emptyList(), dataProvider); + + var profile = loadConfig(config); + testFeature(relationFeature, feature -> { + assertNotNull(feature.getAttrsAtZoom(14).get("route_name")); + assertEquals("Main Line", feature.getAttrsAtZoom(14).get("route_name")); + assertTrue(feature.getAttrsAtZoom(14).containsKey("segment_ref")); + }, 3, profile); + } + + /** + * Test 2: Boundary relation with way members and role filtering (GitHub Issue Example 2) + */ + @Test + void testBoundaryRelation_Segments() { + var config = """ + sources: + osm: + type: osm + url: geofabrik:rhode-island + layers: + - id: boundary_segments + features: + - source: osm + geometry: relation_members + include_when: + type: boundary + admin_level: "6" + member_types: [way] + member_roles: ["outer"] + member_attributes: + - key: boundary_type + tag_value: boundary + """; + + var relation = new OsmElement.Relation(200L, Map.of( + "type", "boundary", + "admin_level", "6", + "name", "County" + ), List.of( + new OsmElement.Relation.Member(OsmElement.Type.WAY, 10L, "outer"), + new OsmElement.Relation.Member(OsmElement.Type.WAY, 11L, "outer"), + new OsmElement.Relation.Member(OsmElement.Type.WAY, 12L, "inner") // Should be filtered out + )); + + var wayGeometries = Map.of( + 10L, LongArrayList.from(20L, 21L), + 11L, LongArrayList.from(21L, 22L), + 12L, LongArrayList.from(22L, 23L) + ); + var wayTags = Map.>of( + 10L, Map.of("boundary", "administrative"), + 11L, Map.of("boundary", "administrative"), + 12L, Map.of("boundary", "administrative") + ); + var nodeCoordinates = Map.of( + 20L, new Coordinate(0.0, 0.0), + 21L, new Coordinate(0.1, 0.0), + 22L, new Coordinate(0.2, 0.0), + 23L, new Coordinate(0.3, 0.0) + ); + + var dataProvider = createMockDataProvider(wayGeometries, wayTags, Map.of(), nodeCoordinates); + var relationFeature = new RelationSourceFeature(relation, emptyList(), dataProvider); + + var profile = loadConfig(config); + // Should only process 2 members (outer role), not 3 + testFeature(relationFeature, feature -> { + assertTrue(feature.getAttrsAtZoom(14).containsKey("boundary_type")); + }, 2, profile); + } + + /** + * Test 3: Member type filtering - only process way members + */ + @Test + void testMemberTypeFiltering() { + var config = """ + sources: + osm: + type: osm + url: geofabrik:rhode-island + layers: + - id: test_layer + features: + - source: osm + geometry: relation_members + include_when: + type: route + member_types: [way] + """; + + var relation = new OsmElement.Relation(300L, Map.of("type", "route"), List.of( + new OsmElement.Relation.Member(OsmElement.Type.WAY, 1L, ""), + new OsmElement.Relation.Member(OsmElement.Type.NODE, 100L, ""), // Should be filtered out + new OsmElement.Relation.Member(OsmElement.Type.WAY, 2L, "") + )); + + var wayGeometries = Map.of( + 1L, LongArrayList.from(10L, 11L), + 2L, LongArrayList.from(11L, 12L) + ); + var nodeCoordinates = Map.of( + 10L, new Coordinate(0.0, 0.0), + 11L, new Coordinate(0.1, 0.0), + 12L, new Coordinate(0.2, 0.0), + 100L, new Coordinate(0.5, 0.5) // Node member - should be skipped + ); + + var dataProvider = createMockDataProvider(wayGeometries, Map.of(), Map.of(), nodeCoordinates); + var relationFeature = new RelationSourceFeature(relation, emptyList(), dataProvider); + + var profile = loadConfig(config); + // Should only process 2 way members, not the node member + testFeature(relationFeature, feature -> { + assertNotNull(feature); + }, 2, profile); + } + + /** + * Test 4: Member tag filtering with member_include_when + */ + @Test + void testMemberTagFiltering_IncludeWhen() { + var config = """ + sources: + osm: + type: osm + url: geofabrik:rhode-island + layers: + - id: test_layer + features: + - source: osm + geometry: relation_members + include_when: + type: route + member_types: [way] + member_include_when: + railway: rail + """; + + var relation = new OsmElement.Relation(400L, Map.of("type", "route"), List.of( + new OsmElement.Relation.Member(OsmElement.Type.WAY, 1L, ""), + new OsmElement.Relation.Member(OsmElement.Type.WAY, 2L, ""), + new OsmElement.Relation.Member(OsmElement.Type.WAY, 3L, "") + )); + + var wayGeometries = Map.of( + 1L, LongArrayList.from(10L, 11L), + 2L, LongArrayList.from(11L, 12L), + 3L, LongArrayList.from(12L, 13L) + ); + var wayTags = Map.>of( + 1L, Map.of("railway", "rail"), // Should match + 2L, Map.of("railway", "rail"), // Should match + 3L, Map.of("highway", "primary") // Should be filtered out + ); + var nodeCoordinates = Map.of( + 10L, new Coordinate(0.0, 0.0), + 11L, new Coordinate(0.1, 0.0), + 12L, new Coordinate(0.2, 0.0), + 13L, new Coordinate(0.3, 0.0) + ); + + var dataProvider = createMockDataProvider(wayGeometries, wayTags, Map.of(), nodeCoordinates); + var relationFeature = new RelationSourceFeature(relation, emptyList(), dataProvider); + + var profile = loadConfig(config); + // Should only process 2 members with railway=rail + testFeature(relationFeature, feature -> { + assertNotNull(feature); + }, 2, profile); + } + + /** + * Test 5: Member tag filtering with member_exclude_when + */ + @Test + void testMemberTagFiltering_ExcludeWhen() { + var config = """ + sources: + osm: + type: osm + url: geofabrik:rhode-island + layers: + - id: test_layer + features: + - source: osm + geometry: relation_members + include_when: + type: route + member_types: [way] + member_exclude_when: + service: __any__ + """; + + var relation = new OsmElement.Relation(500L, Map.of("type", "route"), List.of( + new OsmElement.Relation.Member(OsmElement.Type.WAY, 1L, ""), + new OsmElement.Relation.Member(OsmElement.Type.WAY, 2L, ""), + new OsmElement.Relation.Member(OsmElement.Type.WAY, 3L, "") + )); + + var wayGeometries = Map.of( + 1L, LongArrayList.from(10L, 11L), + 2L, LongArrayList.from(11L, 12L), + 3L, LongArrayList.from(12L, 13L) + ); + var wayTags = Map.>of( + 1L, Map.of("highway", "primary"), // Should match + 2L, Map.of("highway", "primary", "service", "yes"), // Should be excluded + 3L, Map.of("highway", "primary") // Should match + ); + var nodeCoordinates = Map.of( + 10L, new Coordinate(0.0, 0.0), + 11L, new Coordinate(0.1, 0.0), + 12L, new Coordinate(0.2, 0.0), + 13L, new Coordinate(0.3, 0.0) + ); + + var dataProvider = createMockDataProvider(wayGeometries, wayTags, Map.of(), nodeCoordinates); + var relationFeature = new RelationSourceFeature(relation, emptyList(), dataProvider); + + var profile = loadConfig(config); + // Should only process 2 members (excluding the one with service tag) + testFeature(relationFeature, feature -> { + assertNotNull(feature); + }, 2, profile); + } + + /** + * Test 6: Member tag filtering with member_include_when using inline script + */ + @Test + void testMemberTagFiltering_IncludeWhen_InlineScript() { + var config = """ + sources: + osm: + type: osm + url: geofabrik:rhode-island + layers: + - id: test_layer + features: + - source: osm + geometry: relation_members + include_when: + type: route + member_types: [way] + member_include_when: '${ member.tags.railway == "rail" && member.tags.electrified != null }' + """; + + var relation = new OsmElement.Relation(600L, Map.of("type", "route"), List.of( + new OsmElement.Relation.Member(OsmElement.Type.WAY, 1L, ""), + new OsmElement.Relation.Member(OsmElement.Type.WAY, 2L, ""), + new OsmElement.Relation.Member(OsmElement.Type.WAY, 3L, "") + )); + + var wayGeometries = Map.of( + 1L, LongArrayList.from(10L, 11L), + 2L, LongArrayList.from(11L, 12L), + 3L, LongArrayList.from(12L, 13L) + ); + var wayTags = Map.>of( + 1L, Map.of("railway", "rail", "electrified", "contact_line"), // Should match + 2L, Map.of("railway", "rail"), // Should be filtered out (no electrified tag) + 3L, Map.of("highway", "primary") // Should be filtered out + ); + var nodeCoordinates = Map.of( + 10L, new Coordinate(0.0, 0.0), + 11L, new Coordinate(0.1, 0.0), + 12L, new Coordinate(0.2, 0.0), + 13L, new Coordinate(0.3, 0.0) + ); + + var dataProvider = createMockDataProvider(wayGeometries, wayTags, Map.of(), nodeCoordinates); + var relationFeature = new RelationSourceFeature(relation, emptyList(), dataProvider); + + var profile = loadConfig(config); + // Should only process 1 member (railway=rail with electrified tag) + testFeature(relationFeature, feature -> { + assertNotNull(feature); + }, 1, profile); + } + + /** + * Test 7: Member tag filtering with member_exclude_when using inline script + */ + @Test + void testMemberTagFiltering_ExcludeWhen_InlineScript() { + var config = """ + sources: + osm: + type: osm + url: geofabrik:rhode-island + layers: + - id: test_layer + features: + - source: osm + geometry: relation_members + include_when: + type: route + member_types: [way] + member_exclude_when: '${ member.tags.service != null || member.tags.access == "private" }' + """; + + var relation = new OsmElement.Relation(700L, Map.of("type", "route"), List.of( + new OsmElement.Relation.Member(OsmElement.Type.WAY, 1L, ""), + new OsmElement.Relation.Member(OsmElement.Type.WAY, 2L, ""), + new OsmElement.Relation.Member(OsmElement.Type.WAY, 3L, ""), + new OsmElement.Relation.Member(OsmElement.Type.WAY, 4L, "") + )); + + var wayGeometries = Map.of( + 1L, LongArrayList.from(10L, 11L), + 2L, LongArrayList.from(11L, 12L), + 3L, LongArrayList.from(12L, 13L), + 4L, LongArrayList.from(13L, 14L) + ); + var wayTags = Map.>of( + 1L, Map.of("highway", "primary"), // Should match + 2L, Map.of("highway", "primary", "service", "yes"), // Should be excluded (has service) + 3L, Map.of("highway", "primary", "access", "private"), // Should be excluded (access=private) + 4L, Map.of("highway", "primary") // Should match + ); + var nodeCoordinates = Map.of( + 10L, new Coordinate(0.0, 0.0), + 11L, new Coordinate(0.1, 0.0), + 12L, new Coordinate(0.2, 0.0), + 13L, new Coordinate(0.3, 0.0), + 14L, new Coordinate(0.4, 0.0) + ); + + var dataProvider = createMockDataProvider(wayGeometries, wayTags, Map.of(), nodeCoordinates); + var relationFeature = new RelationSourceFeature(relation, emptyList(), dataProvider); + + var profile = loadConfig(config); + // Should only process 2 members (excluding those with service tag or access=private) + testFeature(relationFeature, feature -> { + assertNotNull(feature); + }, 2, profile); + } + + /** + * Test 8: Node member processing (bus stops example) + */ + @Test + void testNodeMember_Processing() { + var config = """ + sources: + osm: + type: osm + url: geofabrik:rhode-island + layers: + - id: bus_stops + features: + - source: osm + geometry: relation_members + include_when: + type: route + route: bus + member_types: [node] + member_roles: ["stop"] + member_include_when: + public_transport: stop_position + attributes: + - key: route_ref + tag_value: ref + member_attributes: + - key: stop_name + tag_value: name + """; + + var relation = new OsmElement.Relation(600L, Map.of( + "type", "route", + "route", "bus", + "ref", "42" + ), List.of( + new OsmElement.Relation.Member(OsmElement.Type.NODE, 100L, "stop"), + new OsmElement.Relation.Member(OsmElement.Type.NODE, 101L, "stop"), + new OsmElement.Relation.Member(OsmElement.Type.NODE, 102L, "platform") // Wrong role + )); + + var nodeTags = Map.>of( + 100L, Map.of("public_transport", "stop_position", "name", "Stop A"), + 101L, Map.of("public_transport", "stop_position", "name", "Stop B"), + 102L, Map.of("public_transport", "stop_position", "name", "Stop C") + ); + var nodeCoordinates = Map.of( + 100L, new Coordinate(0.0, 0.0), + 101L, new Coordinate(0.1, 0.0), + 102L, new Coordinate(0.2, 0.0) + ); + + var dataProvider = createMockDataProvider(Map.of(), Map.of(), nodeTags, nodeCoordinates); + var relationFeature = new RelationSourceFeature(relation, emptyList(), dataProvider); + + var profile = loadConfig(config); + // Should process 2 node members with role "stop" and public_transport tag + testFeature(relationFeature, feature -> { + assertEquals("42", feature.getAttrsAtZoom(14).get("route_ref")); + assertTrue(feature.getAttrsAtZoom(14).containsKey("stop_name")); + }, 2, profile); + } + + /** + * Test 9: Duplicate member handling + */ + @Test + void testDuplicateMembers() { + var config = """ + sources: + osm: + type: osm + url: geofabrik:rhode-island + layers: + - id: test_layer + features: + - source: osm + geometry: relation_members + include_when: + type: route + member_types: [way] + """; + + // Relation with same member referenced twice + var relation = new OsmElement.Relation(700L, Map.of("type", "route"), List.of( + new OsmElement.Relation.Member(OsmElement.Type.WAY, 1L, ""), + new OsmElement.Relation.Member(OsmElement.Type.WAY, 1L, ""), // Duplicate + new OsmElement.Relation.Member(OsmElement.Type.WAY, 2L, "") + )); + + var wayGeometries = Map.of( + 1L, LongArrayList.from(10L, 11L), + 2L, LongArrayList.from(11L, 12L) + ); + var nodeCoordinates = Map.of( + 10L, new Coordinate(0.0, 0.0), + 11L, new Coordinate(0.1, 0.0), + 12L, new Coordinate(0.2, 0.0) + ); + + var dataProvider = createMockDataProvider(wayGeometries, Map.of(), Map.of(), nodeCoordinates); + var relationFeature = new RelationSourceFeature(relation, emptyList(), dataProvider); + + var profile = loadConfig(config); + // Should only process 2 unique members (duplicate skipped) + testFeature(relationFeature, feature -> { + assertNotNull(feature); + }, 2, profile); + } + + /** + * Test 10: Way member as polygon (closed way with area tag) + */ + @Test + void testWayMember_Polygon() { + var config = """ + sources: + osm: + type: osm + url: geofabrik:rhode-island + layers: + - id: test_layer + features: + - source: osm + geometry: relation_members + include_when: + type: boundary + member_types: [way] + """; + + var relation = new OsmElement.Relation(800L, Map.of("type", "boundary"), List.of( + new OsmElement.Relation.Member(OsmElement.Type.WAY, 1L, "outer") + )); + + // Closed way (first and last node are the same) + var wayGeometries = Map.of( + 1L, LongArrayList.from(10L, 11L, 12L, 10L) // Closed way + ); + var wayTags = Map.>of( + 1L, Map.of("area", "yes") // Area tag present + ); + var nodeCoordinates = Map.of( + 10L, new Coordinate(0.0, 0.0), + 11L, new Coordinate(0.1, 0.0), + 12L, new Coordinate(0.1, 0.1) + ); + + var dataProvider = createMockDataProvider(wayGeometries, wayTags, Map.of(), nodeCoordinates); + var relationFeature = new RelationSourceFeature(relation, emptyList(), dataProvider); + + var profile = loadConfig(config); + // Should create polygon geometry + testFeature(relationFeature, feature -> { + assertNotNull(feature); + assertTrue(feature.getGeometry() instanceof org.locationtech.jts.geom.Polygon); + }, 1, profile); + } + + /** + * Test 11: Missing member geometry handling + */ + @Test + void testMissingMemberGeometry() { + var config = """ + sources: + osm: + type: osm + url: geofabrik:rhode-island + layers: + - id: test_layer + features: + - source: osm + geometry: relation_members + include_when: + type: route + member_types: [way] + """; + + var relation = new OsmElement.Relation(900L, Map.of("type", "route"), List.of( + new OsmElement.Relation.Member(OsmElement.Type.WAY, 1L, ""), + new OsmElement.Relation.Member(OsmElement.Type.WAY, 2L, "") // Missing geometry + )); + + // Only way 1 has geometry + var wayGeometries = Map.of( + 1L, LongArrayList.from(10L, 11L) + ); + var nodeCoordinates = Map.of( + 10L, new Coordinate(0.0, 0.0), + 11L, new Coordinate(0.1, 0.0) + ); + + var dataProvider = createMockDataProvider(wayGeometries, Map.of(), Map.of(), nodeCoordinates); + var relationFeature = new RelationSourceFeature(relation, emptyList(), dataProvider); + + var profile = loadConfig(config); + // Should only process way 1 (way 2 has missing geometry) + testFeature(relationFeature, feature -> { + assertNotNull(feature); + }, 1, profile); + } + + /** + * Test 12: Member attributes with inline script expressions + */ + @Test + void testMemberAttributes_InlineScript() { + var config = """ + sources: + osm: + type: osm + url: geofabrik:rhode-island + layers: + - id: route_segments + features: + - source: osm + geometry: relation_members + include_when: + type: route + member_types: [way] + attributes: + - key: route_name + tag_value: name + member_attributes: + - key: segment_ref + tag_value: ref + - key: combined_name + value: '${ member.tags.name + " (" + feature.tags.name + ")" }' + """; + + var relation = new OsmElement.Relation(1000L, Map.of( + "type", "route", + "name", "Main Route" + ), List.of( + new OsmElement.Relation.Member(OsmElement.Type.WAY, 1L, "") + )); + + var wayGeometries = Map.of( + 1L, LongArrayList.from(10L, 11L) + ); + var wayTags = Map.>of( + 1L, Map.of("ref", "A1", "name", "Segment 1") + ); + var nodeCoordinates = Map.of( + 10L, new Coordinate(0.0, 0.0), + 11L, new Coordinate(0.1, 0.0) + ); + + var dataProvider = createMockDataProvider(wayGeometries, wayTags, Map.of(), nodeCoordinates); + var relationFeature = new RelationSourceFeature(relation, emptyList(), dataProvider); + + var profile = loadConfig(config); + testFeature(relationFeature, feature -> { + assertEquals("Main Route", feature.getAttrsAtZoom(14).get("route_name")); + assertEquals("A1", feature.getAttrsAtZoom(14).get("segment_ref")); + assertEquals("Segment 1 (Main Route)", feature.getAttrsAtZoom(14).get("combined_name")); + }, 1, profile); + } + + /** + * Test 13: Same relation processed as both polygon (multipolygon) and relation_members + * + * This test verifies that a relation with type=boundary can be processed as both: + * 1. A multipolygon (geometry: polygon) - creates one polygon feature from the relation + * 2. Relation members (geometry: relation_members) - creates individual features for each member + */ + @Test + void testBoundaryRelation_BothPolygonAndRelationMembers() { + var config = """ + sources: + osm: + type: osm + url: geofabrik:rhode-island + layers: + - id: boundary_polygon + features: + - source: osm + geometry: polygon + include_when: + type: boundary + admin_level: "6" + attributes: + - key: boundary_name + tag_value: name + - id: boundary_segments + features: + - source: osm + geometry: relation_members + include_when: + type: boundary + admin_level: "6" + member_types: [way] + member_roles: ["outer"] + attributes: + - key: boundary_name + tag_value: name + member_attributes: + - key: segment_id + value: '${ member.ref }' + """; + + // Create a boundary relation that matches both criteria + var relation = new OsmElement.Relation(1100L, Map.of( + "type", "boundary", + "admin_level", "6", + "name", "Test County" + ), List.of( + new OsmElement.Relation.Member(OsmElement.Type.WAY, 20L, "outer"), + new OsmElement.Relation.Member(OsmElement.Type.WAY, 21L, "outer") + )); + + // Create way geometries for relation_members processing + var wayGeometries = Map.of( + 20L, LongArrayList.from(30L, 31L, 32L), + 21L, LongArrayList.from(32L, 33L, 34L) + ); + var wayTags = Map.>of( + 20L, Map.of("boundary", "administrative"), + 21L, Map.of("boundary", "administrative") + ); + var nodeCoordinates = Map.of( + 30L, new Coordinate(0.0, 0.0), + 31L, new Coordinate(0.1, 0.0), + 32L, new Coordinate(0.2, 0.0), + 33L, new Coordinate(0.3, 0.0), + 34L, new Coordinate(0.4, 0.0) + ); + + var dataProvider = createMockDataProvider(wayGeometries, wayTags, Map.of(), nodeCoordinates); + var relationFeature = new RelationSourceFeature(relation, emptyList(), dataProvider); + + var profile = loadConfig(config); + + // Test 1: Process as relation_members - should generate 2 member features + var factory = new FeatureCollector.Factory(planetilerConfig, Stats.inMemory()); + var fc = factory.get(relationFeature); + profile.processFeature(relationFeature, fc); + + var relationMembersCount = new AtomicInteger(0); + var polygonCount = new AtomicInteger(0); + + fc.forEach(f -> { + if ("boundary_segments".equals(f.getLayer())) { + relationMembersCount.incrementAndGet(); + // Verify relation-level attributes are present + assertEquals("Test County", f.getAttrsAtZoom(14).get("boundary_name")); + // Verify member-level attributes are present + assertTrue(f.getAttrsAtZoom(14).containsKey("segment_id")); + } else if ("boundary_polygon".equals(f.getLayer())) { + polygonCount.incrementAndGet(); + assertEquals("Test County", f.getAttrsAtZoom(14).get("boundary_name")); + } + }); + + // Should have 2 relation_members features (one for each way member) + assertEquals(2, relationMembersCount.get(), + "Should generate 2 features from relation_members geometry"); + // Note: polygon feature would be generated from MultipolygonSourceFeature in OsmReader, + // but we're testing at ConfiguredProfile level with RelationSourceFeature, so polygon + // won't be generated here. The important part is that both feature definitions exist + // and the relation_members one works correctly. + assertEquals(0, polygonCount.get(), + "Polygon features are generated from MultipolygonSourceFeature in OsmReader, not RelationSourceFeature"); + + // Test 2: Verify that a polygon feature with the same tags would match the polygon definition + // This demonstrates that the same relation (by tags) can match both definitions + var polygonFeature = SimpleFeature.createFakeOsmFeature( + com.onthegomap.planetiler.TestUtils.newPolygon(0, 0, 1, 0, 1, 1, 0, 0), + Map.of("type", "boundary", "admin_level", "6", "name", "Test County"), + "osm", null, 1100L, emptyList(), null + ); + + var fc2 = factory.get(polygonFeature); + profile.processFeature(polygonFeature, fc2); + + var polygonMatchCount = new AtomicInteger(0); + fc2.forEach(f -> { + if ("boundary_polygon".equals(f.getLayer())) { + polygonMatchCount.incrementAndGet(); + assertEquals("Test County", f.getAttrsAtZoom(14).get("boundary_name")); + } + }); + + // Should match the polygon definition + assertEquals(1, polygonMatchCount.get(), + "Polygon feature should match the polygon geometry definition"); + } +} + diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/RelationMembersValidationTest.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/RelationMembersValidationTest.java new file mode 100644 index 0000000000..9b71b7af06 --- /dev/null +++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/RelationMembersValidationTest.java @@ -0,0 +1,111 @@ +package com.onthegomap.planetiler.custommap; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.onthegomap.planetiler.custommap.configschema.FeatureItem; +import com.onthegomap.planetiler.custommap.configschema.FeatureGeometry; +import com.onthegomap.planetiler.custommap.expression.ParseException; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class RelationMembersValidationTest { + + private static FeatureItem createFeatureItemWithMemberTypes(FeatureGeometry geometry, List memberTypes) { + return new FeatureItem( + List.of("osm"), + null, null, null, null, null, null, + geometry, + Map.of("highway", "primary"), + null, + List.of(), + memberTypes, + null, null, null, null + ); + } + + private static FeatureItem createFeatureItemWithMemberRoles(FeatureGeometry geometry, List memberRoles) { + return new FeatureItem( + List.of("osm"), + null, null, null, null, null, null, + geometry, + Map.of("natural", "water"), + null, + List.of(), + null, + memberRoles, + null, null, null + ); + } + + private static FeatureItem createFeatureItemWithMemberIncludeWhen(FeatureGeometry geometry, + Map memberIncludeWhen) { + return new FeatureItem( + List.of("osm"), + null, null, null, null, null, null, + geometry, + Map.of("amenity", "restaurant"), + null, + List.of(), + null, null, + memberIncludeWhen, + null, null + ); + } + + private static FeatureItem createFeatureItemWithInvalidMemberTypes(List memberTypes) { + return new FeatureItem( + List.of("osm"), + null, null, null, null, null, null, + FeatureGeometry.RELATION_MEMBERS, + Map.of("type", "route"), + null, + List.of(), + memberTypes, + null, null, null, null + ); + } + + @Test + void testMemberFieldsWithWrongGeometry() { + // member_types with line geometry should fail + assertThrows(ParseException.class, () -> createFeatureItemWithMemberTypes(FeatureGeometry.LINE, List.of("way")), + "member_types should not be allowed with line geometry"); + + // member_roles with polygon geometry should fail + assertThrows(ParseException.class, () -> createFeatureItemWithMemberRoles(FeatureGeometry.POLYGON, List.of("outer")), + "member_roles should not be allowed with polygon geometry"); + + // member_include_when with point geometry should fail + assertThrows(ParseException.class, + () -> createFeatureItemWithMemberIncludeWhen(FeatureGeometry.POINT, Map.of("highway", "primary")), + "member_include_when should not be allowed with point geometry"); + } + + @Test + void testInvalidMemberTypes() { + // Invalid member_types value should fail + assertThrows(ParseException.class, () -> createFeatureItemWithInvalidMemberTypes(List.of("invalid_type")), + "Invalid member_types value should throw ParseException"); + } + + @Test + void testValidRelationMembers() { + // Valid relation_members configuration should not throw + new FeatureItem( + List.of("osm"), + null, null, null, null, null, null, + FeatureGeometry.RELATION_MEMBERS, + Map.of("type", "route"), + null, + List.of(), + List.of("way"), // member_types + List.of(""), // member_roles + Map.of("highway", "primary"), // member_include_when + Map.of("service", "__any__"), // member_exclude_when + List.of() // member_attributes + ); + // Should not throw + } +} + diff --git a/planetiler-custommap/src/test/resources/invalidSchema/invalid_member_types.yml b/planetiler-custommap/src/test/resources/invalidSchema/invalid_member_types.yml new file mode 100644 index 0000000000..81fe3337f5 --- /dev/null +++ b/planetiler-custommap/src/test/resources/invalidSchema/invalid_member_types.yml @@ -0,0 +1,17 @@ +schema_name: Test Case Schema +schema_description: Test case tile schema +attribution: Test attribution +sources: + osm: + type: osm + url: geofabrik:rhode-island +layers: +- id: testLayer + features: + - source: + - osm + geometry: relation_members + include_when: + type: route + member_types: [invalid_type] + diff --git a/planetiler-custommap/src/test/resources/invalidSchema/member_fields_wrong_geometry.yml b/planetiler-custommap/src/test/resources/invalidSchema/member_fields_wrong_geometry.yml new file mode 100644 index 0000000000..4db96576d4 --- /dev/null +++ b/planetiler-custommap/src/test/resources/invalidSchema/member_fields_wrong_geometry.yml @@ -0,0 +1,18 @@ +schema_name: Test Case Schema +schema_description: Test case tile schema +attribution: Test attribution +sources: + osm: + type: osm + url: geofabrik:rhode-island +layers: +- id: testLayer + features: + - source: + - osm + geometry: line + include_when: + highway: primary + member_types: [way] + member_roles: [""] + diff --git a/planetiler-custommap/src/test/resources/validSchema/relation_members_basic.yml b/planetiler-custommap/src/test/resources/validSchema/relation_members_basic.yml new file mode 100644 index 0000000000..4ba712348f --- /dev/null +++ b/planetiler-custommap/src/test/resources/validSchema/relation_members_basic.yml @@ -0,0 +1,20 @@ +schema_name: Test Case Schema +schema_description: Test case tile schema +attribution: Test attribution +sources: + osm: + type: osm + url: geofabrik:rhode-island +layers: +- id: route_segments + features: + - source: + - osm + geometry: relation_members + include_when: + type: route + member_types: [way] + attributes: + - key: route_name + tag_value: name + diff --git a/pom.xml b/pom.xml index 1f34c03904..5e54e01489 100644 --- a/pom.xml +++ b/pom.xml @@ -195,6 +195,7 @@ **/target/** planetiler-openmaptiles/** + .cursor/** diff --git a/quickstart.sh b/quickstart.sh index 3e3ecedff9..6bd749f20c 100755 --- a/quickstart.sh +++ b/quickstart.sh @@ -134,7 +134,7 @@ function check_java_version() { # Run planetiler using docker, jar file, or build from source case $METHOD in docker) - run docker run -e JAVA_TOOL_OPTIONS=\'"${JVM_ARGS}"\' -v "$DOCKER_DIR":/data "ghcr.io/onthegomap/planetiler:${VERSION}" "${PLANETILER_ARGS[@]}" + run docker run --rm -e JAVA_TOOL_OPTIONS=\'"${JVM_ARGS}"\' -v "$DOCKER_DIR":/data "ghcr.io/onthegomap/planetiler:${VERSION}" "${PLANETILER_ARGS[@]}" ;; jar) echo "Downloading latest planetiler release..." diff --git a/scripts/sonar.sh b/scripts/sonar.sh index 653e99d592..5901b3f614 100755 --- a/scripts/sonar.sh +++ b/scripts/sonar.sh @@ -2,4 +2,4 @@ set -eu -mvn verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Pcoverage +./mvnw verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Pcoverage diff --git a/scripts/test-release.sh b/scripts/test-release.sh index b015746b4b..cd2dec44cd 100755 --- a/scripts/test-release.sh +++ b/scripts/test-release.sh @@ -2,36 +2,36 @@ set -exuo pipefail -version="${1:-$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)}" +version="${1:-$(./mvnw help:evaluate -Dexpression=project.version -q -DforceStdout)}" if [ "${SKIP_EXAMPLE_PROJECT:-false}" == "true" ]; then echo "skipping example project" else echo "::group::Test building example project" - (cd planetiler-examples && mvn -B -ntp -Dplanetiler.version="${version}" verify --file standalone.pom.xml) + (cd planetiler-examples && ../mvnw -B -ntp -Dplanetiler.version="${version}" verify --file standalone.pom.xml) echo "::endgroup::" fi echo "Test java build" echo "::group::OpenMapTiles monaco (java)" -rm -f data/out*.mbtiles +rm -f data/out*.mbtiles data/jar-*.mbtiles data/docker-*.mbtiles # vary threads to stress-test determinism check -java -jar planetiler-dist/target/*with-deps.jar --download --area=monaco --output=data/jar-monaco.mbtiles --threads=32 +java -jar planetiler-dist/target/*with-deps.jar --download --area=monaco --output=data/jar-monaco.mbtiles --threads=32 --force ./scripts/check-monaco.sh data/jar-monaco.mbtiles echo "::endgroup::" echo "::group::Example (java)" -java -jar planetiler-dist/target/*with-deps.jar example-toilets --download --area=monaco --output=data/jar-example.mbtiles +java -jar planetiler-dist/target/*with-deps.jar example-toilets --download --area=monaco --output=data/jar-example.mbtiles --force ./scripts/check-mbtiles.sh data/jar-example.mbtiles echo "::endgroup::" echo "::endgroup::" echo "::group::OpenMapTiles monaco (docker)" # vary threads to stress-test determinism check -docker run -v "$(pwd)/data":/data ghcr.io/onthegomap/planetiler:"${version}" --area=monaco --output=data/docker-monaco.mbtiles --threads=4 +docker run --user $(id -u):$(id -g) --rm -v "$(pwd)/data":/data ghcr.io/onthegomap/planetiler:"${version}" --area=monaco --output=data/docker-monaco.mbtiles --threads=4 --force ./scripts/check-monaco.sh data/docker-monaco.mbtiles echo "::endgroup::" echo "::group::Example (docker)" -docker run -v "$(pwd)/data":/data ghcr.io/onthegomap/planetiler:"${version}" example-toilets --area=monaco --output=data/docker-example.mbtiles +docker run --user $(id -u):$(id -g) --rm -v "$(pwd)/data":/data ghcr.io/onthegomap/planetiler:"${version}" example-toilets --area=monaco --output=data/docker-example.mbtiles --force ./scripts/check-mbtiles.sh data/docker-example.mbtiles echo "::endgroup::"