diff --git a/docker/docker-compose.custom.yml b/docker/docker-compose.custom.yml index b7458456..bd4eca43 100644 --- a/docker/docker-compose.custom.yml +++ b/docker/docker-compose.custom.yml @@ -32,4 +32,5 @@ services: file: docker-compose.base.yml service: jore4-mapmatchingdb-base container_name: mapmatchingdb - image: "hsldevcom/jore4-postgres:mapmatching-main--20250425-c07c8632baab554dfbbbb64177ed930282094232" + environment: + DIGIROAD_ROUTING_DUMP_URL: "https://stjore4dev001.blob.core.windows.net/jore4-digiroad/2025-08-29_create_routing_schema_digiroad_r_2025_02_fixup.sql" diff --git a/pom.xml b/pom.xml index 3ff9408b..c3c552bd 100644 --- a/pom.xml +++ b/pom.xml @@ -38,6 +38,7 @@ 2.1.2 1.10 + 34.0 7.0.13 0.26 @@ -46,6 +47,20 @@ false + + + osgeo + OSGeo Release Repository + https://repo.osgeo.org/repository/release/ + + false + + + true + + + + prod @@ -570,6 +585,41 @@ ${geolatte.version} + + + org.geotools + gt-main + ${geotools.version} + + + org.geotools + gt-http + + + + + org.geotools + gt-epsg-hsql + ${geotools.version} + + + org.geotools + gt-geopkg + ${geotools.version} + + + + com.github.doyaaaaaken + kotlin-csv-jvm + 1.5.1 + + + + org.nield + kotlin-statistics + 1.2.1 + + io.arrow-kt arrow-core diff --git a/profiles/dev/config.properties b/profiles/dev/config.properties index f69a0bd1..b87aa50b 100644 --- a/profiles/dev/config.properties +++ b/profiles/dev/config.properties @@ -33,3 +33,11 @@ spring.thymeleaf.cache=false # NOTE! Leave this blank and just populate it in the user-specific .properties # file, which is git-ignored. digitransit.subscription.key= + +# In the jore4-jore3-importer repository (https://github.com/HSLdevcom/jore4-jore3-importer), there +# exists a "create-bulk-map-matching-input" branch containing a script that can be used to generate +# the CSV file. +test.routes.csvfile= + +# The output directory to which the resulting GeoJSON and GeoPackage files are written. +test.output.dir= diff --git a/profiles/prod/config.properties b/profiles/prod/config.properties index 12bb898e..0b753121 100644 --- a/profiles/prod/config.properties +++ b/profiles/prod/config.properties @@ -25,3 +25,11 @@ spring.thymeleaf.cache=true # NOTE! Leave this blank and just populate it in the user-specific .properties # file, which is git-ignored. digitransit.subscription.key= + +# In the jore4-jore3-importer repository (https://github.com/HSLdevcom/jore4-jore3-importer), there +# exists a "create-bulk-map-matching-input" branch containing a script that can be used to generate +# the CSV file. +test.routes.csvfile= + +# The output directory to which the resulting GeoJSON and GeoPackage files are written. +test.output.dir= diff --git a/run-bulk-test.sh b/run-bulk-test.sh new file mode 100755 index 00000000..ac3bb26a --- /dev/null +++ b/run-bulk-test.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +mvn -Pprod -DDB_URL=jdbc:postgresql://localhost:6433/jore4mapmatching?stringtype=unspecified -DDB_USERNAME=mapmatching -DDB_PASSWORD=password clean spring-boot:run diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/MatchRouteViaPointsOnLinksServiceImpl.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/MatchRouteViaPointsOnLinksServiceImpl.kt index 034857fa..ed71b80c 100644 --- a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/MatchRouteViaPointsOnLinksServiceImpl.kt +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/MatchRouteViaPointsOnLinksServiceImpl.kt @@ -191,7 +191,9 @@ class MatchRouteViaPointsOnLinksServiceImpl( targetRoutePointSequenceCandidates: List>, bufferRadiusInMeters: Double ): List? = - targetRoutePointSequenceCandidates.firstNotNullOfOrNull { targetRoutePoints -> + targetRoutePointSequenceCandidates.withIndex().firstNotNullOfOrNull { (index, targetRoutePoints) -> + + val round = index + 1 val bufferAreaRestriction = BufferAreaRestriction.from( @@ -201,8 +203,19 @@ class MatchRouteViaPointsOnLinksServiceImpl( targetRoutePoints.last() ) - routingService - .findRouteViaPointsOnLinks(targetRoutePoints, vehicleType, true, bufferAreaRestriction) - .ifEmpty { null } + val routeLinks: List = + routingService.findRouteViaPointsOnLinks( + targetRoutePoints, + vehicleType, + true, + bufferAreaRestriction + ) + + if (routeLinks.isNotEmpty()) { + if (round > 1) LOGGER.info { "Matched route on attempt #$round." } + routeLinks + } else { + null + } } } diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/ExtractStopToStopSegments.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/ExtractStopToStopSegments.kt new file mode 100644 index 00000000..d0239fe4 --- /dev/null +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/ExtractStopToStopSegments.kt @@ -0,0 +1,381 @@ +package fi.hsl.jore4.mapmatching.service.matching.test + +import fi.hsl.jore4.mapmatching.model.matching.RoutePoint +import fi.hsl.jore4.mapmatching.model.matching.RouteStopPoint +import fi.hsl.jore4.mapmatching.util.LogUtils.joinToLogString +import io.github.oshai.kotlinlogging.KotlinLogging +import org.geolatte.geom.G2D +import org.geolatte.geom.Geometries.mkLineString +import org.geolatte.geom.LineString +import org.geolatte.geom.Point +import org.geolatte.geom.PositionSequence +import org.geolatte.geom.PositionSequenceBuilder +import org.geolatte.geom.PositionSequenceBuilders +import org.geolatte.geom.crs.CoordinateReferenceSystems.WGS84 + +private val LOGGER = KotlinLogging.logger {} + +object ExtractStopToStopSegments { + private data class SegmentInfo( + val routePoints: List, + val referencingRoutes: MutableList + ) { + fun withReferencingRoute(routeId: String): SegmentInfo { + referencingRoutes.add(routeId) + return this + } + } + + private data class RoutePointLocations( + val measuredLocation: Point, + val projectedLocation: Point? = null + ) { + fun hasMeasuredLocationOf(pos: G2D): Boolean = pos == measuredLocation.position + + fun hasProjectedLocationOf(pos: G2D): Boolean = projectedLocation?.run { pos == position } ?: false + + fun hasPosition(pos: G2D): Boolean = hasMeasuredLocationOf(pos) || hasProjectedLocationOf(pos) + + fun hasSharedPositionWith(other: RoutePointLocations): Boolean = + hasPosition(other.measuredLocation.position) || + other.projectedLocation?.run { hasPosition(position) } == true + } + + private fun RoutePoint.toLocations() = + when (this) { + is RouteStopPoint -> RoutePointLocations(location, projectedLocation) + else -> RoutePointLocations(location) + } + + fun extractStopToStopSegments(routes: List): StopToStopSegmentation { + val segmentMap: MutableMap, SegmentInfo>> = mutableMapOf() + val discardedRoutes = mutableListOf() + + routes.forEach { route -> + + try { + splitToStopToStopSegments(route).forEach { (stopToStopSegmentId, geometry, routePoints) -> + + segmentMap + .getOrPut(stopToStopSegmentId) { mutableMapOf() } + .getOrPut(geometry) { SegmentInfo(routePoints, mutableListOf()) } + .withReferencingRoute(route.routeId) + } + } catch (e: IllegalArgumentException) { + LOGGER.info { + "${route.routeId}: Discarding route because of mismatch between number of route point ranges " + + "and LineString segments" + } + e.message?.let(discardedRoutes::add) + } + } + + val stopToStopSegments = + segmentMap + .toSortedMap() + .entries + .flatMap { (stopToStopSegmentId, segmentMap) -> + + segmentMap.entries + .sortedBy { it.key.numPositions } + .withIndex() + .map { (index, segmentEntry) -> + + val geometry: LineString = segmentEntry.key + + val stopToStopRouteId = + if (index == 0) { + "$stopToStopSegmentId" + } else { + "$stopToStopSegmentId-${index + 1}" + } + + val routePoints: List = segmentEntry.value.routePoints + val referencingRoutes: List = segmentEntry.value.referencingRoutes + + StopToStopSegment(stopToStopRouteId, geometry, routePoints, referencingRoutes) + } + } + + return StopToStopSegmentation(stopToStopSegments, discardedRoutes) + } + + private fun splitToStopToStopSegments(route: PublicTransportRoute): List { + val originRoutePoints: List = trimRoutePointStart(route) + + val routePointSegments: List> = splitRoutePoints(originRoutePoints) + + if (routePointSegments.isEmpty()) { + return emptyList() + } + + val terminusOfRoutePointSegments: List> = + routePointSegments + .map { routePoints -> + val first: RoutePoint = routePoints.first() + + if (first !is RouteStopPoint) { + throw IllegalStateException("Expected first point to be a RouteStopPoint") + } + + val last: RoutePoint = routePoints.last() + + if (last !is RouteStopPoint) { + throw IllegalStateException("Expected last point to be a RouteStopPoint") + } + + first to last + } + + val geometrySegments: List> = + splitRouteGeometry( + route.routeId, + route.routeGeometry, + terminusOfRoutePointSegments, + false + ) + + if (geometrySegments.size != routePointSegments.size) { + LOGGER.info { + "${route.routeId}: Stop-to-stop route point ranges (${terminusOfRoutePointSegments.size}): ${ + joinToLogString(terminusOfRoutePointSegments) + }" + } + LOGGER.info { + "${route.routeId}: Split LineStrings (${geometrySegments.size}): ${ + joinToLogString(geometrySegments) + }" + } + + // Re-run LineString splitting with debug prints. + splitRouteGeometry(route.routeId, route.routeGeometry, terminusOfRoutePointSegments, true) + + throw IllegalArgumentException(route.routeId) + // throw IllegalStateException(route.routeId) + } + + fun getStopId(routePoint: RoutePoint): String = + when (routePoint) { + is RouteStopPoint -> { + routePoint.run { passengerId ?: nationalId?.toString() } + ?: routePoint.location.position.run { "($lon,$lat)" } + } + else -> throw IllegalStateException("Expected RouteStopPoint") + } + + return geometrySegments + .zip(routePointSegments) + .map { (lineString, routePoints) -> + + PublicTransportRoute( + "${getStopId(routePoints.first())}-${getStopId(routePoints.last())}", + lineString, + routePoints + ) + } + } + + private fun trimRoutePointStart(route: PublicTransportRoute): List { + // Drop the first route point if the route geometry does not start with it. + return if (route.routePoints + .first() + .toLocations() + .hasPosition(route.routeGeometry.startPosition) + ) { + route.routePoints + } else { + LOGGER.info { + "${route.routeId}: Dropping the first route point because it is not part of route geometry" + } + route.routePoints.drop(1) + } + } + + private fun splitRoutePoints(originRoutePoints: List): List> { + val stopToStopIndexRanges: List> = + extractRoutePointIndexRangesForStopToStopSegments(originRoutePoints) + + if (stopToStopIndexRanges.size < 2) { + return emptyList() + } + + return stopToStopIndexRanges + .map { (startIndex, endIndex) -> + val rps: MutableList = mutableListOf() + + for (index in startIndex..endIndex) { + rps.add(originRoutePoints[index]) + } + + rps + }.filter { routePoints -> + routePoints.run { + size > 2 || !first().toLocations().hasSharedPositionWith(last().toLocations()) + } + } + } + + private fun extractRoutePointIndexRangesForStopToStopSegments(routePoints: List): List> { + if (routePoints.isEmpty()) { + return emptyList() + } + + val routeStopPointIndices: List = + routePoints.mapIndexedNotNull { index, routePoint -> + if (routePoint is RouteStopPoint) index else null + } + + if (routeStopPointIndices.size < 2) { + return emptyList() + } + + val indexRanges: MutableList> = mutableListOf() + + for (i in 0 until routeStopPointIndices.lastIndex) { + val startIndex: Int = routeStopPointIndices[i] + val endIndex: Int = routeStopPointIndices[i + 1] + + indexRanges.add(startIndex to endIndex) + } + + return indexRanges + } + + private fun splitRouteGeometry( + routeId: String, + routeGeometry: LineString, + stopPointRanges: List>, + printDebug: Boolean + ): List> { + if (stopPointRanges.isEmpty()) { + return emptyList() + } + + val firstStopPointOnRoute: RoutePoint = stopPointRanges.first().first + + val trimmedLine: LineString = dropFromStart(routeGeometry, firstStopPointOnRoute.toLocations()) + + val (_, splitLineStrings: List>) = + stopPointRanges + .fold( + trimmedLine to emptyList>() + ) { (remainingLineString, substrings), (_, endStopPoint) -> + + if (printDebug) { + LOGGER.info { "$routeId: Remaining : $remainingLineString" } + } + + if (remainingLineString.isEmpty) { + remainingLineString to substrings + } else { + val endStopLocations: RoutePointLocations = endStopPoint.toLocations() + + val substring: LineString = takeFromStart(remainingLineString, endStopLocations) + + if (printDebug) { + LOGGER.info { "$routeId: LineSubstring: $substring" } + } + + if (substring.isEmpty) { + remainingLineString to substrings + } else { + val newRemaining: LineString = dropFromStart(remainingLineString, endStopLocations) + + newRemaining to substrings + substring + } + } + } + + return splitLineStrings + } + + private fun takeFromStart( + lineString: LineString, + toLocation: RoutePointLocations + ): LineString { + val positionSequenceBuilder = PositionSequenceBuilders.variableSized(G2D::class.java) + + var measuredLocationMatchedAlready = false + var projectedLocationMatchedAlready = false + + lineString.positions + .takeWhile { pos -> + when { + measuredLocationMatchedAlready && projectedLocationMatchedAlready -> false + measuredLocationMatchedAlready -> { + toLocation.hasProjectedLocationOf(pos).also { projectedLocationMatchedAlready = it } + } + projectedLocationMatchedAlready -> { + toLocation.hasMeasuredLocationOf(pos).also { measuredLocationMatchedAlready = it } + } + else -> { + if (toLocation.hasMeasuredLocationOf(pos)) { + measuredLocationMatchedAlready = true + } else if (toLocation.hasProjectedLocationOf(pos)) { + projectedLocationMatchedAlready = true + } + true + } + } + }.forEach(positionSequenceBuilder::add) + + return toLineString(positionSequenceBuilder) + } + + private fun dropFromStart( + lineString: LineString, + untilLocation: RoutePointLocations + ): LineString { + if (untilLocation.hasPosition(lineString.startPosition)) { + return lineString + } + + val linePositions: PositionSequence = lineString.positions + + val firstIndex: Int = linePositions.indexOfFirst(untilLocation::hasPosition) + + if (firstIndex < 0) { + return lineString + } + + val numPositions: Int = linePositions.size() + + val hasAtLeast2StopLocations = + numPositions >= firstIndex + 2 && + untilLocation.hasPosition(linePositions.getPositionN(firstIndex + 1)) + + val hasAtLeast3StopLocations = + hasAtLeast2StopLocations && + numPositions >= firstIndex + 3 && + untilLocation.hasPosition(linePositions.getPositionN(firstIndex + 2)) + + val has4StopLocations = + hasAtLeast3StopLocations && + numPositions >= firstIndex + 4 && + untilLocation.hasPosition(linePositions.getPositionN(firstIndex + 3)) + + val numPositionsToDrop: Int = + when { + has4StopLocations -> firstIndex + 2 + hasAtLeast2StopLocations -> firstIndex + 1 + else -> firstIndex + } + + val positionSequenceBuilder = PositionSequenceBuilders.variableSized(G2D::class.java) + + linePositions + .drop(numPositionsToDrop) + .forEach(positionSequenceBuilder::add) + + return toLineString(positionSequenceBuilder) + } + + private fun toLineString(positionSequenceBuilder: PositionSequenceBuilder): LineString { + val positionSequence: PositionSequence = positionSequenceBuilder.toPositionSequence() + + return when (positionSequence.size()) { + 0, 1 -> LineString(WGS84) + else -> mkLineString(positionSequence, WGS84) + } + } +} diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/GeoPackageUtils.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/GeoPackageUtils.kt new file mode 100644 index 00000000..e38a205a --- /dev/null +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/GeoPackageUtils.kt @@ -0,0 +1,173 @@ +package fi.hsl.jore4.mapmatching.service.matching.test + +import fi.hsl.jore4.mapmatching.util.GeoToolsUtils.transformCRS +import org.geolatte.geom.jts.JTS +import org.geotools.api.data.SimpleFeatureWriter +import org.geotools.api.data.Transaction +import org.geotools.api.feature.simple.SimpleFeature +import org.geotools.api.feature.simple.SimpleFeatureType +import org.geotools.data.DefaultTransaction +import org.geotools.data.collection.ListFeatureCollection +import org.geotools.data.simple.SimpleFeatureIterator +import org.geotools.feature.simple.SimpleFeatureBuilder +import org.geotools.feature.simple.SimpleFeatureTypeBuilder +import org.geotools.geometry.jts.Geometries +import org.geotools.geopkg.FeatureEntry +import org.geotools.geopkg.GeoPackage +import org.geotools.referencing.crs.DefaultGeographicCRS +import org.locationtech.jts.geom.Geometry +import org.locationtech.jts.geom.LineString +import java.io.File +import java.io.IOException + +object GeoPackageUtils { + private const val GEOMETRY_COLUMN_NAME = "geometry" + + fun createGeoPackage( + file: File, + failedSegments: List, + isBufferPolygonInsteadOfLineString: Boolean + ): GeoPackage { + val geoPkg = GeoPackage(file) + geoPkg.init() + + try { + failedSegments.forEach { segment -> + val featureTypeName: String = segment.routeId + val featureType: SimpleFeatureType = + createFeatureTypeForFailedSegment(featureTypeName, isBufferPolygonInsteadOfLineString) + + val featureCollection: ListFeatureCollection = + createFeatureCollection(segment, featureType, isBufferPolygonInsteadOfLineString) + + val entry = FeatureEntry() + // entry.identifier = segment.routeId + entry.geometryColumn = GEOMETRY_COLUMN_NAME + entry.geometryType = + if (isBufferPolygonInsteadOfLineString) Geometries.POLYGON else Geometries.LINESTRING + entry.bounds = featureCollection.bounds + + entry.description = + if (isBufferPolygonInsteadOfLineString) { + "Buffered geometry for failed ${segment.routeId} segment within map-matching" + } else { + "LineString for failed ${segment.routeId} segment within map-matching" + } + + writeEntryToGeoPackage(geoPkg, entry, featureCollection) + + // Not sure, if this is really needed. + geoPkg.createSpatialIndex(entry) + } + + return geoPkg + } catch (ex: Exception) { + geoPkg.close() + throw ex + } + } + + private fun createFeatureTypeForFailedSegment( + name: String, + isBufferPolygonInsteadOfLineString: Boolean + ): SimpleFeatureType { + val builder = SimpleFeatureTypeBuilder() + builder.name = name + + val geomType = if (isBufferPolygonInsteadOfLineString) Geometries.POLYGON else Geometries.LINESTRING + + builder.add(GEOMETRY_COLUMN_NAME, geomType.binding, DefaultGeographicCRS.WGS84) + builder.add("length", Double::class.java) + builder.add("startStopId", String::class.java) + builder.add("endStopId", String::class.java) + builder.add("numberOfGeometryPoints", Integer::class.java) + builder.add("numberOfRoutePoints", Integer::class.java) + builder.add("numberOfRoutesPassingThrough", Integer::class.java) + builder.add("routesPassingThrough", String::class.java) + + return builder.buildFeatureType() + } + + private fun createFeatureCollection( + failedSegment: SegmentMatchFailure, + featureType: SimpleFeatureType, + isBufferPolygonInsteadOfLineString: Boolean + ): ListFeatureCollection { + val feature: SimpleFeature = createFeature(failedSegment, featureType, isBufferPolygonInsteadOfLineString) + + return ListFeatureCollection(featureType, feature) + } + + private fun createFeature( + failedSegment: SegmentMatchFailure, + type: SimpleFeatureType, + isBufferPolygonInsteadOfLineString: Boolean + ): SimpleFeature { + val builder = SimpleFeatureBuilder(type) + + failedSegment.run { + val lineString: LineString = JTS.to(sourceRouteGeometry) + + if (isBufferPolygonInsteadOfLineString) { + val lineString3067: Geometry = transformCRS(lineString, 4326, 3067) + val bufferPolygon3067: Geometry = lineString3067.buffer(bufferRadius.value) + val bufferPolygon: Geometry = transformCRS(bufferPolygon3067, 3067, 4326) + + builder.add(bufferPolygon) + } else { + builder.add(lineString) + } + + builder.add(sourceRouteLength) + builder.add(startStopId) + builder.add(endStopId) + builder.add(sourceRouteGeometry.numPositions) + builder.add(numberOfRoutePoints) + builder.add(referencingRoutes.size) + builder.add(referencingRoutes.joinToString(separator = " ")) + } + + // This works because every feature is put into a separate table. + builder.featureUserData("fid", "1") + + return builder.buildFeature(null) + } + + private fun writeEntryToGeoPackage( + geoPkg: GeoPackage, + entry: FeatureEntry, + featureCollection: ListFeatureCollection + ) { + // Create an SQLite table for FeatureCollection. + geoPkg.create(entry, featureCollection.schema) + + val tx: Transaction = DefaultTransaction() + val it: SimpleFeatureIterator = featureCollection.features() + + try { + val writer: SimpleFeatureWriter = geoPkg.writer(entry, true, null, tx) + + val source = it.next() + val target: SimpleFeature = writer.next() + + target.attributes = source.attributes + + // This if-block is the missing piece not done in the GeoPackage.add() method. + if (source.hasUserData()) { + target.userData.putAll(source.userData) + } + + writer.write() + writer.close() + + tx.commit() + } catch (ex: Exception) { + tx.rollback() + + throw ex as? IOException ?: IOException(ex) + } finally { + tx.close() + it.close() + } + } +} diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/IMapMatchingBulkTestResultsPublisher.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/IMapMatchingBulkTestResultsPublisher.kt new file mode 100644 index 00000000..da414059 --- /dev/null +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/IMapMatchingBulkTestResultsPublisher.kt @@ -0,0 +1,9 @@ +package fi.hsl.jore4.mapmatching.service.matching.test + +interface IMapMatchingBulkTestResultsPublisher { + fun publishMatchResultsForRoutesAndStopToStopSegments( + routeResults: List, + stopToStopSegmentResults: List, + largestBufferRadiusTried: BufferRadius + ) +} diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/IPublicTransportRouteCsvParser.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/IPublicTransportRouteCsvParser.kt new file mode 100644 index 00000000..14d75206 --- /dev/null +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/IPublicTransportRouteCsvParser.kt @@ -0,0 +1,5 @@ +package fi.hsl.jore4.mapmatching.service.matching.test + +interface IPublicTransportRouteCsvParser { + fun parsePublicTransportRoutes(filePath: String): List +} diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTestResultsPublisherImpl.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTestResultsPublisherImpl.kt new file mode 100644 index 00000000..d69744e8 --- /dev/null +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTestResultsPublisherImpl.kt @@ -0,0 +1,466 @@ +package fi.hsl.jore4.mapmatching.service.matching.test + +import com.fasterxml.jackson.databind.ObjectMapper +import fi.hsl.jore4.mapmatching.util.LogUtils.joinToLogString +import io.github.oshai.kotlinlogging.KotlinLogging +import org.geolatte.geom.G2D +import org.geolatte.geom.json.GeoJsonFeature +import org.geolatte.geom.json.GeoJsonFeatureCollection +import org.nield.kotlinstatistics.standardDeviation +import org.springframework.beans.factory.annotation.Value +import org.springframework.stereotype.Component +import java.io.File +import java.util.SortedMap +import kotlin.math.abs +import kotlin.math.roundToInt + +private val LOGGER = KotlinLogging.logger {} + +@Component +class MapMatchingBulkTestResultsPublisherImpl( + val objectMapper: ObjectMapper, + @Value("\${test.output.dir}") val outputDir: String +) : IMapMatchingBulkTestResultsPublisher { + override fun publishMatchResultsForRoutesAndStopToStopSegments( + routeResults: List, + stopToStopSegmentResults: List, + largestBufferRadiusTried: BufferRadius + ) { + publishRouteMatchResults(routeResults) + publishStopToStopSegmentMatchResults(stopToStopSegmentResults, largestBufferRadiusTried) + + LOGGER.info { + "List of IDs of failed routes whose all segments were matched: ${ + joinToLogString( + getRoutesNotMatchedEvenThoughAllSegmentsMatched( + routeResults, + stopToStopSegmentResults + ) + ) + }" + } + } + + fun publishRouteMatchResults(results: List) { + val (succeeded, failed) = partitionBySuccess(results) + + printBasicStatistics(succeeded, failed, "Route") + printBufferStatistics(succeeded, "Route") + + LOGGER.info { + val limit = 10 + val bestResults = + joinToLogString(getBestMatchResults(succeeded, limit)) { + """{"routeId": "${it.routeId}", "lengthDiff": ${ + roundTo2Digits(it.getLengthDifferenceForFirstMatch()) + }, "lengthDiff-%": ${ + roundTo2Digits(it.getLengthDifferencePercentageForFirstMatch()) + }}""" + } + + "Best $limit successful route matches: $bestResults" + } + + LOGGER.info { + val limit = 20 + val worstResults = + joinToLogString(getWorstMatchResults(succeeded, limit)) { + """{"routeId": "${it.routeId}", "lengthDiff": ${ + roundTo2Digits(it.getLengthDifferenceForFirstMatch()) + }, "lengthDiff-%": ${ + roundTo2Digits(it.getLengthDifferencePercentageForFirstMatch()) + }}""" + } + + "Worst $limit successful route matches: $worstResults" + } + + val outputFile: File = writeGeoJsonToFile(getFailedRoutesAsGeoJson(failed), FILENAME_FAILED_ROUTES_GEOJSON) + LOGGER.info { "Wrote failed routes to file: ${outputFile.absolutePath}" } + } + + fun publishStopToStopSegmentMatchResults( + results: List, + largestBufferRadiusTried: BufferRadius + ) { + val (succeeded, failed) = partitionSegmentsBySuccess(results) + + printBasicStatistics(succeeded, failed, "Stop-to-stop segment") + printBufferStatistics(succeeded, "Stop-to-stop segment") + + LOGGER.info { + val limit = 20 + val mostReferencedFailedSegments = + joinToLogString(getMostReferencedFailedSegments(failed, limit)) { + "{\"segmentId\": \"${it.routeId}\", \"numReferencingRoutes\": ${it.referencingRoutes.size}, " + + "\"referencingRoutes\": \"${it.referencingRoutes}\", \"sourceRouteGeometry\": \"${it.sourceRouteGeometry}\"}" + } + + "$limit most referenced failed stop-to-stop segments: $mostReferencedFailedSegments" + } + + LOGGER.info { + val limit = 20 + val worstResults = + joinToLogString(getWorstMatchResults(succeeded, limit)) { + """{"segmentId": "${it.routeId}", "lengthDiff": ${ + roundTo2Digits(it.getLengthDifferenceForFirstMatch()) + }, "lengthDiff-%": ${ + roundTo2Digits(it.getLengthDifferencePercentageForFirstMatch()) + }, "sourceRouteGeometry": "${it.sourceRouteGeometry}"}""" + } + + "Worst $limit successful stop-to-stop segment matches: $worstResults" + } + + val geojsonFile: File = + writeGeoJsonToFile(getFailedSegmentsAsGeoJson(failed), FILENAME_FAILED_SEGMENTS_GEOJSON) + LOGGER.info { "Wrote failed stop-to-stop segments to GeoJSON file: ${geojsonFile.absolutePath}" } + + writeGeoPackageFilesForFailedSegments( + failed, + largestBufferRadiusTried, + getSegmentMatchFailuresOnLowerBufferRadius(succeeded) + ) + } + + private fun writeGeoJsonToFile( + features: GeoJsonFeatureCollection, + filename: String + ): File { + val geojson: String = objectMapper.writeValueAsString(features) + + val outputFile = File(outputDir, filename) + outputFile.writeText(geojson) + return outputFile + } + + private fun writeGeoPackageFilesForFailedSegments( + primaryFailures: List, + largestBufferRadiusTried: BufferRadius, + secondaryFailures: SortedMap> + ) { + val segmentFailureMap = secondaryFailures.toMutableMap() + + // By design, the "secondaryFailures" map should not contain the largest radius tried as + // a key, but in case of a hidden bug, we consider the situation where this assumption is + // not true. + val segmentMatchFailuresOnLargestRadius: List = + primaryFailures + segmentFailureMap.getOrPut(largestBufferRadiusTried) { emptyList() } + + segmentFailureMap.put(largestBufferRadiusTried, segmentMatchFailuresOnLargestRadius) + + segmentFailureMap.entries.forEach { (bufferRadius, segmentMatchFailures) -> + + if (segmentMatchFailures.isNotEmpty()) { + writeGeoPackageFileForFailedSegments( + segmentMatchFailures, + getGeoPackageFilenameForFailedSegments(bufferRadius), + getGeoPackageFilenameForFailedSegmentBuffers(bufferRadius) + ) + } + } + } + + private fun writeGeoPackageFileForFailedSegments( + failed: List, + segmentsFilename: String, + segmentBuffersFilename: String + ) { + var failedSegmentsGeoPackageFile = File(outputDir, segmentsFilename) + + if (failedSegmentsGeoPackageFile.exists()) { + failedSegmentsGeoPackageFile.renameTo(File(outputDir, "$segmentsFilename.bak")) + failedSegmentsGeoPackageFile = File(outputDir, segmentsFilename) + } + + GeoPackageUtils.createGeoPackage(failedSegmentsGeoPackageFile, failed, false) + + LOGGER.info { + "Wrote failed stop-to-stop segments to GeoPackage file: ${ + failedSegmentsGeoPackageFile.absolutePath + }" + } + + var failureBuffersGeoPackageFile = File(outputDir, segmentBuffersFilename) + if (failureBuffersGeoPackageFile.exists()) { + failureBuffersGeoPackageFile.renameTo(File(outputDir, "$segmentBuffersFilename.bak")) + failureBuffersGeoPackageFile = File(outputDir, segmentBuffersFilename) + } + + GeoPackageUtils.createGeoPackage(failureBuffersGeoPackageFile, failed, true) + + LOGGER.info { + "Wrote failed stop-to-stop segment buffers to GeoPackage file: ${ + failureBuffersGeoPackageFile.absolutePath + }" + } + } + + companion object { + private const val FILENAME_FAILED_ROUTES_GEOJSON = "failed_routes.geojson" + private const val FILENAME_FAILED_SEGMENTS_GEOJSON = "failed_segments.geojson" + + private fun getGeoPackageFilenameForFailedSegments(bufferRadius: BufferRadius): String = + "failed_segments_${bufferRadius.value}.gpkg" + + private fun getGeoPackageFilenameForFailedSegmentBuffers(bufferRadius: BufferRadius): String = + "failed_segment_buffers_${bufferRadius.value}.gpkg" + + private fun partitionBySuccess( + results: List + ): Pair, List> { + val succeeded: List = + results.mapNotNull { it as? SuccessfulRouteMatchResult } + + val failed: List = results.mapNotNull { it as? RouteMatchFailure } + + return succeeded to failed + } + + private fun partitionSegmentsBySuccess( + results: List + ): Pair, List> { + val succeeded: List = + results.mapNotNull { it as? SuccessfulSegmentMatchResult } + + val failed: List = + results.mapNotNull { it as? SegmentMatchFailure } + + return succeeded to failed + } + + private fun getFailedRoutesAsGeoJson(failed: List): GeoJsonFeatureCollection = + GeoJsonFeatureCollection( + failed.map { + GeoJsonFeature(it.sourceRouteGeometry, it.routeId, emptyMap()) + } + ) + + private fun getFailedSegmentsAsGeoJson( + failed: List + ): GeoJsonFeatureCollection = + GeoJsonFeatureCollection( + failed.map { + GeoJsonFeature( + it.sourceRouteGeometry, + it.routeId, + mapOf("referencingRoutes" to it.referencingRoutes) + ) + } + ) + + private fun getBestMatchResults( + results: List, + limit: Int + ): List = + results + .sortedBy { abs(it.getLengthDifferenceForFirstMatch()) } + .take(limit) + + private fun getWorstMatchResults( + results: List, + limit: Int + ): List = + results + .sortedByDescending { abs(it.getLengthDifferenceForFirstMatch()) } + .take(limit) + + private fun getMostReferencedFailedSegments( + results: List, + limit: Int + ): List = + results + .filterIsInstance() + .sortedByDescending { it.referencingRoutes.size } + .take(limit) + + private fun roundTo2Digits(n: Double): Double = (n * 100).roundToInt() / 100.0 + + private fun getSegmentMatchFailuresOnLowerBufferRadius( + succeeded: List + ): SortedMap> { + val allUnsuccessfulBufferRadiuses: MutableSet = mutableSetOf() + + succeeded.forEach { + allUnsuccessfulBufferRadiuses.addAll(it.details.unsuccessfulBufferRadiuses) + } + + val alreadyProcessedSegmentIds: MutableSet = mutableSetOf() + val failedSegmentsByRadius: MutableMap> = mutableMapOf() + + allUnsuccessfulBufferRadiuses + .sortedDescending() + .forEach { bufferRadius -> + + succeeded + .filter { bufferRadius in it.details.unsuccessfulBufferRadiuses } + .filter { it.routeId !in alreadyProcessedSegmentIds } + .forEach { matchResult -> + + failedSegmentsByRadius + .getOrPut(bufferRadius) { mutableListOf() } + .add( + SegmentMatchFailure( + matchResult.routeId, + matchResult.sourceRouteGeometry, + matchResult.sourceRouteLength, + bufferRadius, + matchResult.startStopId, + matchResult.endStopId, + matchResult.numberOfRoutePoints, + matchResult.referencingRoutes + ) + ) + + alreadyProcessedSegmentIds.add(matchResult.routeId) + } + } + + return failedSegmentsByRadius.toSortedMap() + } + + private fun getRoutesNotMatchedEvenThoughAllSegmentsMatched( + routeResults: List, + stopToStopSegmentResults: List + ): List { + val idsOfRoutesContainingFailedSegments: Set = + stopToStopSegmentResults + .flatMap(SegmentMatchResult::referencingRoutes) + .toSet() + + return routeResults + .filterIsInstance() + .map(MatchResult::routeId) + .filter { it !in idsOfRoutesContainingFailedSegments } + .sorted() + } + + private fun printBasicStatistics( + succeeded: List, + failed: List, + resultSetName: String + ) { + val numSucceeded = succeeded.size + val numFailed = failed.size + val numTotal: Int = numSucceeded + numFailed + + LOGGER.info { + "$resultSetName match results: all: $numTotal, successful: $numSucceeded (${ + roundTo2Digits(100.0 * numSucceeded / numTotal) + } %), failed: $numFailed (${ + roundTo2Digits(100.0 * numFailed / numTotal) + } %)" + } + + val absLengthDiffs: List = succeeded.map { abs(it.getLengthDifferenceForFirstMatch()) } + + LOGGER.info { + "$resultSetName average absolute length difference: ${ + roundTo2Digits(absLengthDiffs.average()) + } meters, standard deviation: ${ + roundTo2Digits(absLengthDiffs.standardDeviation()) + } meters" + } + + val absLengthDiffPercentages: List = + succeeded.map { abs(it.getLengthDifferencePercentageForFirstMatch()) } + + LOGGER.info { + "$resultSetName average absolute length difference-%: ${ + roundTo2Digits(absLengthDiffPercentages.average()) + } %, standard deviation: ${ + roundTo2Digits(absLengthDiffPercentages.standardDeviation()) + } %" + } + } + + private fun printBufferStatistics( + succeeded: List, + resultSetName: String + ) { + val bufferRadiusSet: MutableSet = mutableSetOf() + + succeeded.forEach { + bufferRadiusSet.addAll(it.details.lengthsOfMatchResults.keys) + } + + val bufferRadiusList: List = bufferRadiusSet.toList().sorted() + + if (bufferRadiusList.size < 2) { + return + } + + val lowestBufferRadiusAppearanceCounts: SortedMap = + bufferRadiusList + .associateWith { bufferRadius -> + succeeded.count { it.getLowestBufferRadius() == bufferRadius } + }.toSortedMap() + + LOGGER.info { + "$resultSetName matches with different buffer radius values: ${ + joinToLogString(lowestBufferRadiusAppearanceCounts.entries) { (bufferRadius, matchCount) -> + "$bufferRadius m: $matchCount pcs" + } + }" + } + + val lengthComparisons: List> = + (0 until bufferRadiusList.size - 1) + .mapNotNull { firstBufferRadiusIndex -> + val remainingBufferRadiusValues: List = + bufferRadiusList.drop( + firstBufferRadiusIndex + ) + + val lowestBufferRadius: BufferRadius = remainingBufferRadiusValues.first() + + val filteredList: List = + succeeded.filter { it.getLowestBufferRadius() == lowestBufferRadius } + + when (filteredList.size) { + 0 -> null + else -> { + remainingBufferRadiusValues.associateWith { bufferRadius -> + val lengthDifferencesOfMatchedRoutes: List = + filteredList + .mapNotNull { match -> + val lengthOfMatchedRoute: Double? = + match.details.lengthsOfMatchResults[bufferRadius] + + lengthOfMatchedRoute?.let { + // Return absolute length difference. + abs(lengthOfMatchedRoute - match.sourceRouteLength) + } ?: run { + LOGGER.warn { + "The length of match result not available for route: ${match.routeId}" + } + null + } + } + + // Return average of all differences. + lengthDifferencesOfMatchedRoutes.average() + } + } + } + }.map { it.toSortedMap() } + + if (lengthComparisons.isNotEmpty()) { + LOGGER.info { + "$resultSetName match length differences compared with regard to buffer radius: ${ + joinToLogString(lengthComparisons) { mapOfDifferences -> + mapOfDifferences.entries.joinToString( + prefix = "[", + transform = { (bufferRadius, avgDiff) -> + "$bufferRadius: ${roundTo2Digits(avgDiff)}" + }, + postfix = "]" + ) + } + }" + } + } + } + } +} diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTester.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTester.kt new file mode 100644 index 00000000..f5b9dc4b --- /dev/null +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTester.kt @@ -0,0 +1,260 @@ +package fi.hsl.jore4.mapmatching.service.matching.test + +import fi.hsl.jore4.mapmatching.model.VehicleType.GENERIC_BUS +import fi.hsl.jore4.mapmatching.model.matching.RoutePoint +import fi.hsl.jore4.mapmatching.service.common.response.LinkTraversalDTO +import fi.hsl.jore4.mapmatching.service.common.response.RouteResultDTO +import fi.hsl.jore4.mapmatching.service.common.response.RoutingResponse +import fi.hsl.jore4.mapmatching.service.matching.IMatchingService +import fi.hsl.jore4.mapmatching.service.matching.PublicTransportRouteMatchingParameters +import fi.hsl.jore4.mapmatching.service.matching.PublicTransportRouteMatchingParameters.JunctionMatchingParameters +import fi.hsl.jore4.mapmatching.util.GeolatteUtils.length +import io.github.oshai.kotlinlogging.KotlinLogging +import jakarta.annotation.PostConstruct +import org.geolatte.geom.G2D +import org.geolatte.geom.LineString +import org.springframework.beans.factory.annotation.Value +import org.springframework.stereotype.Service +import kotlin.time.Duration +import kotlin.time.ExperimentalTime +import kotlin.time.measureTime + +private val LOGGER = KotlinLogging.logger {} + +@Service +class MapMatchingBulkTester( + val csvParser: IPublicTransportRouteCsvParser, + val matchingService: IMatchingService, + val resultsPublisher: IMapMatchingBulkTestResultsPublisher, + @Value("\${test.routes.csvfile}") val csvFile: String +) { + @PostConstruct + @OptIn(ExperimentalTime::class) + fun launchRouteTesting() { + LOGGER.info { "Starting to map-match routes from file..." } + + val duration: Duration = + measureTime { + val (routeMatchResults, stopToStopSegmentMatchResults) = processFile() + + val largestBufferRadiusTried = BufferRadius(DEFAULT_SEGMENT_MATCH_RADIUS_VALUES.max()) + + resultsPublisher.publishMatchResultsForRoutesAndStopToStopSegments( + routeMatchResults, + stopToStopSegmentMatchResults, + largestBufferRadiusTried + ) + } + + LOGGER.info { "Finished map-matching routes in $duration" } + } + + fun processFile(): Pair, List> { + LOGGER.info { "Loading public transport routes from file: $csvFile" } + + val sourceRoutes: List = csvParser.parsePublicTransportRoutes(csvFile) + + LOGGER.info { "Number of source routes: ${sourceRoutes.size}" } + + val (stopToStopSegments: List, discardedRoutes: List) = + ExtractStopToStopSegments.extractStopToStopSegments(sourceRoutes) + + LOGGER.info { "Number of stop-to-stop segments: ${stopToStopSegments.size}" } + LOGGER.info { + "Number of discarded routes within resolution of stop-to-stop segments: ${discardedRoutes.size}" + } + + val routeMatchResults: List = matchRoutes(sourceRoutes, DEFAULT_ROUTE_MATCH_RADIUS_VALUES) + val segmentMatchResults: List = + matchStopToStopSegments(stopToStopSegments, DEFAULT_SEGMENT_MATCH_RADIUS_VALUES) + + return routeMatchResults to segmentMatchResults + } + + private fun matchRoutes( + routes: List, + bufferRadiuses: Set + ): List = + routes.map { (routeId, routeGeometry, routePoints) -> + LOGGER.info { "Starting to match route: $routeId" } + + val result: MatchResult = matchRoute(routeId, routeGeometry, routePoints, bufferRadiuses) + + when (result) { + is SuccessfulMatchResult -> + LOGGER.info { + "Successfully matched route: $routeId (with bufferRadius=${result.getLowestBufferRadius()})" + } + + is RouteMatchFailure -> + LOGGER.info { + "Failed to match route: $routeId (${ + result.errorMessage ?: "" + })" + } + + else -> throw IllegalStateException("Unknown route match result type") + } + + result + } + + private fun matchStopToStopSegments( + segments: List, + bufferRadiuses: Set + ): List = + segments.map { segment -> + val (segmentId, geometry, routePoints, referencingRoutes) = segment + + LOGGER.info { "Starting to match stop-to-stop segment: $segmentId" } + + val result: MatchResult = matchRoute(segmentId, geometry, routePoints, bufferRadiuses) + + if (result is SuccessfulMatchResult) { + LOGGER.info { + "Successfully matched stop-to-stop segment: $segmentId (with bufferRadius=${ + result.getLowestBufferRadius() + })" + } + } else { + LOGGER.info { "Failed to match stop-to-stop segment: $segmentId" } + } + + val numRoutePoints = routePoints.size + + when (result) { + is SuccessfulRouteMatchResult -> + SuccessfulSegmentMatchResult( + segmentId, + geometry, + result.sourceRouteLength, + result.details, + segment.startStopId, + segment.endStopId, + numRoutePoints, + referencingRoutes + ) + + is RouteMatchFailure -> + SegmentMatchFailure( + segmentId, + geometry, + result.sourceRouteLength, + result.bufferRadius, + segment.startStopId, + segment.endStopId, + numRoutePoints, + referencingRoutes + ) + + else -> throw IllegalStateException("Unknown segment match result type") + } + } + + private fun matchRoute( + routeId: String, + geometry: LineString, + routePoints: List, + bufferRadiuses: Set + ): MatchResult { + val sortedBufferRadiuses: List = bufferRadiuses.sorted() + + val lengthOfSourceRoute: Double = length(geometry) + + val lengthsOfMatchedRoutes: MutableList> = mutableListOf() + val unsuccessfulBufferRadiuses: MutableSet = mutableSetOf() + + var errorMessage: String? = null + + sortedBufferRadiuses.withIndex().forEach { (roundIndex, radius) -> + val matchingParams: PublicTransportRouteMatchingParameters = getMatchingParameters(radius) + + val response: RoutingResponse = + matchingService.findMatchForPublicTransportRoute( + routeId, + geometry, + routePoints, + GENERIC_BUS, + matchingParams + ) + + val bufferRadius = BufferRadius(radius) + + when (response) { + is RoutingResponse.RoutingSuccessDTO -> { + val route: RouteResultDTO = response.routes[0] + val lengthOfMatchedRoute: Double = calculateLengthOfRoute(route) + // val lengthOfMatchedRoute: Double = length(route.geometry) + + lengthsOfMatchedRoutes.add(bufferRadius to lengthOfMatchedRoute) + } + + is RoutingResponse.RoutingFailureDTO -> { + unsuccessfulBufferRadiuses.add(bufferRadius) + + if (roundIndex == 0) { + errorMessage = response.message + } + } + } + } + + return when (lengthsOfMatchedRoutes.size) { + 0 -> + RouteMatchFailure( + routeId, + geometry, + lengthOfSourceRoute, + BufferRadius(sortedBufferRadiuses.last()), + errorMessage + ) + + else -> + SuccessfulRouteMatchResult( + routeId, + geometry, + lengthOfSourceRoute, + MatchDetails( + lengthsOfMatchedRoutes.toMap().toSortedMap(), + unsuccessfulBufferRadiuses + ) + ) + } + } + + companion object { + private val DEFAULT_ROUTE_MATCH_RADIUS_VALUES: Set = setOf(55.0) + private val DEFAULT_SEGMENT_MATCH_RADIUS_VALUES: Set = setOf(40.0, 50.0, 90.0) + + private fun getMatchingParameters(bufferRadius: Double): PublicTransportRouteMatchingParameters { + val roadJunctionMatchingParams = + JunctionMatchingParameters(matchDistance = 5.0, clearingDistance = 30.0) + + return PublicTransportRouteMatchingParameters( + bufferRadiusInMeters = bufferRadius, + terminusLinkQueryDistance = bufferRadius, + terminusLinkQueryLimit = 5, + maxStopLocationDeviation = 80.0, + fallbackToViaNodesAlgorithm = true, + roadJunctionMatching = roadJunctionMatchingParams + ) + } + + private fun calculateLengthOfRoute(route: RouteResultDTO): Double { + val linkTraversals: List = route.paths + + return when (linkTraversals.size) { + 0 -> 0.0 + 1, 2 -> route.weight + else -> { + val sumOfInterimLinkLengths: Double = + linkTraversals.drop(1).dropLast(1).map { it.distance }.fold(0.0) { acc, distance -> + acc + distance + } + + linkTraversals.first().weight + sumOfInterimLinkLengths + linkTraversals.last().weight + } + } + } + } +} diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MatchResult.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MatchResult.kt new file mode 100644 index 00000000..446a3548 --- /dev/null +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MatchResult.kt @@ -0,0 +1,126 @@ +package fi.hsl.jore4.mapmatching.service.matching.test + +import org.geolatte.geom.G2D +import org.geolatte.geom.LineString +import java.util.SortedMap +import kotlin.math.abs + +sealed interface MatchResult { + val routeId: String + val sourceRouteGeometry: LineString + val sourceRouteLength: Double + + val matchFound: Boolean +} + +interface SuccessfulMatchResult : MatchResult { + val details: MatchDetails + + fun getLowestBufferRadius(): BufferRadius = details.getBufferRadiusOfFirstMatch() + + fun getLengthOfFirstMatch(): Double = details.getLengthOfFirstMatch() + + fun getLengthOfClosestMatch(): Double = details.getLengthOfClosestMatch(sourceRouteLength) + + fun getLengthDifferenceForFirstMatch(): Double = getLengthOfFirstMatch() - sourceRouteLength + + fun getLengthDifferenceForClosestMatch(): Double = getLengthOfClosestMatch() - sourceRouteLength + + fun getLengthDifferencePercentageForFirstMatch(): Double = + 100.0 * getLengthDifferenceForFirstMatch() / sourceRouteLength + + fun getLengthDifferencePercentageForClosestMatch(): Double = + 100.0 * getLengthDifferenceForClosestMatch() / sourceRouteLength +} + +interface SegmentMatchResult : MatchResult { + val startStopId: String + val endStopId: String + + val numberOfRoutePoints: Int + val referencingRoutes: List +} + +@JvmInline +value class BufferRadius( + val value: Double +) : Comparable { + override fun compareTo(other: BufferRadius): Int = value.compareTo(other.value) + + override fun toString() = value.toString() +} + +data class MatchDetails( + val lengthsOfMatchResults: SortedMap, + val unsuccessfulBufferRadiuses: Set +) { + init { + require(lengthsOfMatchResults.isNotEmpty()) { "lengthsOfMatchResults must not be empty" } + } + + fun getBufferRadiusOfFirstMatch(): BufferRadius = getMapEntryForFirstMatch().key + + fun getBufferRadiusOfClosestMatch(sourceRouteLength: Double): BufferRadius = + getMapEntryForClosestMatch(sourceRouteLength).key + + fun getLengthOfFirstMatch(): Double = getMapEntryForFirstMatch().value + + fun getLengthOfClosestMatch(sourceRouteLength: Double): Double = getMapEntryForClosestMatch(sourceRouteLength).value + + private fun getMapEntryForFirstMatch(): Map.Entry = lengthsOfMatchResults.entries.first() + + private fun getMapEntryForClosestMatch(sourceRouteLength: Double): Map.Entry = + when (lengthsOfMatchResults.size) { + 1 -> getMapEntryForFirstMatch() + else -> + lengthsOfMatchResults.entries.minByOrNull { (bufferRadius, length) -> + abs(length - sourceRouteLength) + }!! + } +} + +data class SuccessfulRouteMatchResult( + override val routeId: String, + override val sourceRouteGeometry: LineString, + override val sourceRouteLength: Double, + override val details: MatchDetails +) : SuccessfulMatchResult { + override val matchFound = true +} + +data class RouteMatchFailure( + override val routeId: String, + override val sourceRouteGeometry: LineString, + override val sourceRouteLength: Double, + val bufferRadius: BufferRadius, + val errorMessage: String? +) : MatchResult { + override val matchFound = false +} + +data class SuccessfulSegmentMatchResult( + override val routeId: String, + override val sourceRouteGeometry: LineString, + override val sourceRouteLength: Double, + override val details: MatchDetails, + override val startStopId: String, + override val endStopId: String, + override val numberOfRoutePoints: Int, + override val referencingRoutes: List +) : SuccessfulMatchResult, + SegmentMatchResult { + override val matchFound = true +} + +data class SegmentMatchFailure( + override val routeId: String, + override val sourceRouteGeometry: LineString, + override val sourceRouteLength: Double, + val bufferRadius: BufferRadius, + override val startStopId: String, + override val endStopId: String, + override val numberOfRoutePoints: Int, + override val referencingRoutes: List +) : SegmentMatchResult { + override val matchFound = false +} diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/PublicTransportRoute.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/PublicTransportRoute.kt new file mode 100644 index 00000000..d2e89b9d --- /dev/null +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/PublicTransportRoute.kt @@ -0,0 +1,14 @@ +package fi.hsl.jore4.mapmatching.service.matching.test + +import fi.hsl.jore4.mapmatching.model.matching.RoutePoint +import org.geolatte.geom.G2D +import org.geolatte.geom.LineString + +/** + * Route source for map-matching bulk tester + */ +data class PublicTransportRoute( + val routeId: String, + val routeGeometry: LineString, + val routePoints: List +) diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/PublicTransportRouteCsvParserImpl.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/PublicTransportRouteCsvParserImpl.kt new file mode 100644 index 00000000..51604c42 --- /dev/null +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/PublicTransportRouteCsvParserImpl.kt @@ -0,0 +1,58 @@ +package fi.hsl.jore4.mapmatching.service.matching.test + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.kotlin.readValue +import com.github.doyaaaaaken.kotlincsv.client.CsvReader +import com.github.doyaaaaaken.kotlincsv.dsl.csvReader +import fi.hsl.jore4.mapmatching.model.matching.RoutePoint +import org.geolatte.geom.G2D +import org.geolatte.geom.LineString +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Component + +@Component +class PublicTransportRouteCsvParserImpl + @Autowired + constructor( + val objectMapper: ObjectMapper + ) : IPublicTransportRouteCsvParser { + override fun parsePublicTransportRoutes(filePath: String): List { + val routes: MutableList = mutableListOf() + + CSV_READER.open(filePath) { + readAllAsSequence() + .drop(1) // skip header line + .forEach { row: List -> + + require(row.size >= COLUMNS) { + "Expected number of columns is $COLUMNS, but actual is: ${row.size}" + } + + routes.add(parseCsvRow(row)) + } + } + + return routes + } + + private fun parseCsvRow(row: List): PublicTransportRoute { + val routeDirectionId: String = row[2] + + val routeGeometry: LineString = objectMapper.readValue(row[4]) + val routePoints: List = objectMapper.readValue(row[5]) + + return PublicTransportRoute(routeDirectionId, routeGeometry, routePoints) + } + + companion object { + private val CSV_READER: CsvReader = + csvReader { + charset = Charsets.UTF_8.name() + quoteChar = '"' + delimiter = ',' + escapeChar = '\\' + } + + private const val COLUMNS = 6 + } + } diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/StopToStopSegment.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/StopToStopSegment.kt new file mode 100644 index 00000000..2090fc60 --- /dev/null +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/StopToStopSegment.kt @@ -0,0 +1,27 @@ +package fi.hsl.jore4.mapmatching.service.matching.test + +import fi.hsl.jore4.mapmatching.model.matching.RoutePoint +import fi.hsl.jore4.mapmatching.model.matching.RouteStopPoint +import org.geolatte.geom.G2D +import org.geolatte.geom.LineString + +data class StopToStopSegment( + val segmentId: String, + val geometry: LineString, + val routePoints: List, + val referencingRoutes: List +) { + val startStopId: String + get() = + when (val startStop = routePoints.first()) { + is RouteStopPoint -> startStop.passengerId + else -> throw IllegalStateException("The first point is not a stop point") + } + + val endStopId: String + get() = + when (val endStop = routePoints.last()) { + is RouteStopPoint -> endStop.passengerId + else -> throw IllegalStateException("The last point is not a stop point") + } +} diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/StopToStopSegmentation.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/StopToStopSegmentation.kt new file mode 100644 index 00000000..4eaccbab --- /dev/null +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/StopToStopSegmentation.kt @@ -0,0 +1,6 @@ +package fi.hsl.jore4.mapmatching.service.matching.test + +data class StopToStopSegmentation( + val segments: List, + val discardedRoutes: List +) diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/util/GeoToolsUtils.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/util/GeoToolsUtils.kt new file mode 100644 index 00000000..04d1ca8e --- /dev/null +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/util/GeoToolsUtils.kt @@ -0,0 +1,41 @@ +package fi.hsl.jore4.mapmatching.util + +import org.geotools.api.referencing.crs.CoordinateReferenceSystem +import org.geotools.geometry.jts.JTS +import org.geotools.referencing.CRS +import org.locationtech.jts.geom.Geometry + +object GeoToolsUtils { + fun transformCRS( + geometry: Geometry, + sourceEpsgCode: String, + targetEpsgCode: String + ): Geometry = + transformCRS( + geometry, + CRS.decode(sourceEpsgCode, true), + CRS.decode(targetEpsgCode, true) + ) + + fun transformCRS( + geometry: Geometry, + sourceSRID: Int, + targetSRID: Int + ): Geometry = + transformCRS( + geometry, + CRS.decode(getEpsgCode(sourceSRID), true), + CRS.decode(getEpsgCode(targetSRID), true) + ) + + fun transformCRS( + geometry: Geometry, + sourceCRS: CoordinateReferenceSystem, + targetCRS: CoordinateReferenceSystem + ): Geometry { + val transform = CRS.findMathTransform(sourceCRS, targetCRS) + return JTS.transform(geometry, transform) + } + + private fun getEpsgCode(srid: Int) = "EPSG:$srid" +} diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/util/GeolatteUtils.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/util/GeolatteUtils.kt index f52dc2b5..c53c974f 100644 --- a/src/main/kotlin/fi/hsl/jore4/mapmatching/util/GeolatteUtils.kt +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/util/GeolatteUtils.kt @@ -2,6 +2,7 @@ package fi.hsl.jore4.mapmatching.util import fi.hsl.jore4.mapmatching.model.LatLng import org.geolatte.geom.ByteBuffer +import org.geolatte.geom.C2D import org.geolatte.geom.G2D import org.geolatte.geom.Geometries.mkLineString import org.geolatte.geom.Geometries.mkPoint @@ -10,15 +11,24 @@ import org.geolatte.geom.GeometryType import org.geolatte.geom.LineString import org.geolatte.geom.Point import org.geolatte.geom.PositionSequenceBuilders +import org.geolatte.geom.ProjectedGeometryOperations import org.geolatte.geom.codec.Wkb import org.geolatte.geom.crs.CoordinateReferenceSystems.WGS84 +import org.geolatte.geom.crs.CrsRegistry +import org.geolatte.geom.crs.Geographic2DCoordinateReferenceSystem +import org.geolatte.geom.crs.ProjectedCoordinateReferenceSystem +import org.geolatte.geom.jts.JTS import java.math.BigDecimal import java.math.RoundingMode.HALF_UP import kotlin.math.max import kotlin.math.pow import kotlin.math.sqrt +@Suppress("MemberVisibilityCanBePrivate") object GeolatteUtils { + val EPSG_3067: ProjectedCoordinateReferenceSystem = + CrsRegistry.getProjectedCoordinateReferenceSystemForEPSG(3067) + // This is in WGS84 units and corresponds to approximately 5.5 millimeters at northern // latitudes. private const val LINE_ENDPOINT_CONNECTION_TOLERANCE = 0.0000001 @@ -96,6 +106,25 @@ object GeolatteUtils { return mkLineString(positionSequenceBuilder.toPositionSequence(), WGS84) } + fun transformFromGeographicToProjected( + geom: Geometry, + sourceCRS: Geographic2DCoordinateReferenceSystem, + targetCRS: ProjectedCoordinateReferenceSystem + ): Geometry { + val jtsPoint: org.locationtech.jts.geom.Geometry = JTS.to(geom) + val jtsGeom: org.locationtech.jts.geom.Geometry = + GeoToolsUtils.transformCRS(jtsPoint, sourceCRS.crsId.code, targetCRS.crsId.code) + + return JTS.from(jtsGeom, targetCRS) + } + + fun length(geographicLineString: LineString): Double { + val lineString3067: LineString = + transformFromGeographicToProjected(geographicLineString, WGS84, EPSG_3067) as LineString + + return ProjectedGeometryOperations.Default.length(lineString3067) + } + private fun calculateDistance( position1: G2D, position2: G2D diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 5e9398a3..8aff7694 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -14,3 +14,5 @@ spring.flyway.create-schemas=true spring.thymeleaf.cache=@spring.thymeleaf.cache@ digitransit.subscription.key=@digitransit.subscription.key@ +test.routes.csvfile=@test.routes.csvfile@ +test.output.dir=@test.output.dir@ diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index d7254cfd..b659c47e 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -6,7 +6,7 @@ - + diff --git a/src/test/kotlin/fi/hsl/jore4/mapmatching/service/matching/MatchingService_FindMatchForPublicTransportRouteTest.kt b/src/test/kotlin/fi/hsl/jore4/mapmatching/service/matching/MatchingService_FindMatchForPublicTransportRouteTest.kt index 20766d4b..68de828b 100644 --- a/src/test/kotlin/fi/hsl/jore4/mapmatching/service/matching/MatchingService_FindMatchForPublicTransportRouteTest.kt +++ b/src/test/kotlin/fi/hsl/jore4/mapmatching/service/matching/MatchingService_FindMatchForPublicTransportRouteTest.kt @@ -165,11 +165,11 @@ class MatchingService_FindMatchForPublicTransportRouteTest assertThat(actualLinkIdsAndForwardTraversals).isEqualTo( listOf( - "440250" to false, - "440764" to false, - "440767" to false, - "440765" to false, - "440750" to true + "90c18933-06f4-4f7b-9df2-9c82b59ecd35:2" to false, + "315a1fa3-fdfa-4340-b581-2a83bb8abfa9:2" to false, + "80f7650b-e63b-4f0b-8159-b04936ce7cdb:2" to false, + "9fbfa4c7-0bc7-490c-99a2-c8524225ba40:2" to false, + "e30555b1-d6bd-43d5-ae94-fc3834d0abef:2" to true ) ) } diff --git a/src/test/kotlin/fi/hsl/jore4/mapmatching/service/routing/RoutingService_FindRouteTest.kt b/src/test/kotlin/fi/hsl/jore4/mapmatching/service/routing/RoutingService_FindRouteTest.kt index 7073b8b3..f39fc7d5 100644 --- a/src/test/kotlin/fi/hsl/jore4/mapmatching/service/routing/RoutingService_FindRouteTest.kt +++ b/src/test/kotlin/fi/hsl/jore4/mapmatching/service/routing/RoutingService_FindRouteTest.kt @@ -189,9 +189,9 @@ class RoutingService_FindRouteTest assertThat(actualLinkIdsAndForwardTraversals).isEqualTo( listOf( - "441679" to true, - "441872" to true, - "441874" to true + "c525b97b-8c30-4aba-bfce-28074d4c08e5:1" to true, + "3113baf5-2120-45d3-8f16-0d94e63644fd:1" to true, + "1d1a650b-5621-40bf-bfb2-73d231849c48:1" to true ) ) } @@ -236,7 +236,7 @@ class RoutingService_FindRouteTest assertThat(actualLinkIdsAndForwardTraversals).isEqualTo( listOf( - "441872" to true + "3113baf5-2120-45d3-8f16-0d94e63644fd:1" to true ) ) } @@ -288,11 +288,11 @@ class RoutingService_FindRouteTest assertThat(actualLinkIdsAndForwardTraversals).isEqualTo( listOf( - "441872" to true, - "441880" to true, - "441870" to false, - "441890" to false, - "441872" to true + "3113baf5-2120-45d3-8f16-0d94e63644fd:1" to true, + "134baafa-fbc0-47e7-8be1-6f25cbd37eff:1" to true, + "2f402bd1-aeeb-4a32-bbf7-36dac212bd14:1" to false, + "bd198e25-1902-44fd-ac89-da19c5115eee:1" to false, + "3113baf5-2120-45d3-8f16-0d94e63644fd:1" to true ) ) } @@ -344,10 +344,10 @@ class RoutingService_FindRouteTest assertThat(actualLinkIdsAndForwardTraversals).isEqualTo( listOf( - "419821" to false, - "419821" to true, - "419827" to true, - "419825" to true + "5e070b32-d8f9-4096-8957-7303f8affe8b:2" to false, + "5e070b32-d8f9-4096-8957-7303f8affe8b:2" to true, + "28c73f2e-0d5e-40e1-9756-5171bb65f39d:2" to true, + "9e19e8ac-55dc-4cfb-b262-9206a957d084:2" to true ) ) } @@ -393,7 +393,7 @@ class RoutingService_FindRouteTest assertThat(actualLinkIdsAndForwardTraversals).isEqualTo( listOf( - "441872" to true + "3113baf5-2120-45d3-8f16-0d94e63644fd:1" to true ) ) } @@ -468,11 +468,11 @@ class RoutingService_FindRouteTest assertThat(actualLinkIdsAndForwardTraversals).isEqualTo( listOf( - "11392370" to true, - "12538103" to true, - "12538103" to false, - "12538103" to false, - "12538103" to true + "4f8aa489-14dd-4061-b197-41db30fc3e98:1" to true, + "88bd12da-4e71-4e32-95f8-f9ee8c276c95:1" to false, + "88bd12da-4e71-4e32-95f8-f9ee8c276c95:1" to true, + "88bd12da-4e71-4e32-95f8-f9ee8c276c95:1" to true, + "88bd12da-4e71-4e32-95f8-f9ee8c276c95:1" to false ) ) } @@ -501,8 +501,8 @@ class RoutingService_FindRouteTest // shorter list than without simplifying, the closed-loop link appears only once assertThat(actualLinkIdsAndForwardTraversals).isEqualTo( listOf( - "11392370" to true, - "12538103" to true + "4f8aa489-14dd-4061-b197-41db30fc3e98:1" to true, + "88bd12da-4e71-4e32-95f8-f9ee8c276c95:1" to false ) ) } diff --git a/src/test/kotlin/fi/hsl/jore4/mapmatching/util/GeolatteUtilsTest.kt b/src/test/kotlin/fi/hsl/jore4/mapmatching/util/GeolatteUtilsTest.kt index 90bb8b51..7488b4ba 100644 --- a/src/test/kotlin/fi/hsl/jore4/mapmatching/util/GeolatteUtilsTest.kt +++ b/src/test/kotlin/fi/hsl/jore4/mapmatching/util/GeolatteUtilsTest.kt @@ -1,11 +1,20 @@ package fi.hsl.jore4.mapmatching.util +import fi.hsl.jore4.mapmatching.util.GeolatteUtils.EPSG_3067 +import fi.hsl.jore4.mapmatching.util.GeolatteUtils.transformFromGeographicToProjected import org.assertj.core.api.Assertions.assertThat +import org.geolatte.geom.C2D import org.geolatte.geom.G2D import org.geolatte.geom.Geometries.mkLineString import org.geolatte.geom.LineString +import org.geolatte.geom.Point import org.geolatte.geom.PositionSequenceBuilders +import org.geolatte.geom.builder.DSL.c +import org.geolatte.geom.builder.DSL.g +import org.geolatte.geom.builder.DSL.point import org.geolatte.geom.crs.CoordinateReferenceSystems.WGS84 +import org.hamcrest.CoreMatchers.equalTo +import org.hamcrest.MatcherAssert.assertThat import org.junit.jupiter.api.Test class GeolatteUtilsTest { @@ -41,4 +50,16 @@ class GeolatteUtilsTest { assertThat(outputLine).isEqualTo(expectedLine) } + + @Test + fun testTransformFromGeographicToProjected() { + val geographicPoint: Point = point(WGS84, g(24.94167, 60.17055)) + + val expectedProjectedPoint: Point = point(EPSG_3067, c(385794.9708466177, 6672185.090453222)) + + assertThat( + transformFromGeographicToProjected(geographicPoint, WGS84, EPSG_3067), + equalTo(expectedProjectedPoint) + ) + } }