From 17e8510a1c922059a3be5b5e857fad92e979b348 Mon Sep 17 00:00:00 2001 From: Jarkko Kaura Date: Wed, 2 Feb 2022 22:13:52 +0200 Subject: [PATCH 01/15] Add initial service for bulk-testing map-matching API. --- pom.xml | 45 ++++++ profiles/dev/config.properties | 5 + profiles/prod/config.properties | 5 + run-bulk-test.sh | 3 + .../IMapMatchingBulkTestResultsPublisher.kt | 5 + .../test/IPublicTransportRouteCsvParser.kt | 5 + ...MapMatchingBulkTestResultsPublisherImpl.kt | 99 +++++++++++++ .../matching/test/MapMatchingBulkTester.kt | 134 ++++++++++++++++++ .../service/matching/test/MatchResult.kt | 80 +++++++++++ .../matching/test/PublicTransportRoute.kt | 14 ++ .../test/PublicTransportRouteCsvParserImpl.kt | 58 ++++++++ .../jore4/mapmatching/util/GeoToolsUtils.kt | 41 ++++++ .../jore4/mapmatching/util/GeolatteUtils.kt | 29 ++++ src/main/resources/application.properties | 1 + src/main/resources/logback.xml | 2 +- .../mapmatching/util/GeolatteUtilsTest.kt | 21 +++ 16 files changed, 546 insertions(+), 1 deletion(-) create mode 100755 run-bulk-test.sh create mode 100644 src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/IMapMatchingBulkTestResultsPublisher.kt create mode 100644 src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/IPublicTransportRouteCsvParser.kt create mode 100644 src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTestResultsPublisherImpl.kt create mode 100644 src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTester.kt create mode 100644 src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MatchResult.kt create mode 100644 src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/PublicTransportRoute.kt create mode 100644 src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/PublicTransportRouteCsvParserImpl.kt create mode 100644 src/main/kotlin/fi/hsl/jore4/mapmatching/util/GeoToolsUtils.kt diff --git a/pom.xml b/pom.xml index 3ff9408b..0e088a93 100644 --- a/pom.xml +++ b/pom.xml @@ -38,6 +38,7 @@ 2.1.2 1.10 + 34.0 7.0.13 0.26 @@ -46,6 +47,20 @@ false + + + osgeo + OSGeo Release Repository + https://repo.osgeo.org/repository/release/ + + false + + + true + + + + prod @@ -570,6 +585,36 @@ ${geolatte.version} + + + org.geotools + gt-main + ${geotools.version} + + + org.geotools + gt-http + + + + + org.geotools + gt-epsg-hsql + ${geotools.version} + + + + com.github.doyaaaaaken + kotlin-csv-jvm + 1.5.1 + + + + org.nield + kotlin-statistics + 1.2.1 + + io.arrow-kt arrow-core diff --git a/profiles/dev/config.properties b/profiles/dev/config.properties index f69a0bd1..e7f95a84 100644 --- a/profiles/dev/config.properties +++ b/profiles/dev/config.properties @@ -33,3 +33,8 @@ spring.thymeleaf.cache=false # NOTE! Leave this blank and just populate it in the user-specific .properties # file, which is git-ignored. digitransit.subscription.key= + +# In the jore4-jore3-importer repository (https://github.com/HSLdevcom/jore4-jore3-importer), there +# exists a "create-bulk-map-matching-input" branch containing a script that can be used to generate +# the CSV file. +test.routes.csvfile= diff --git a/profiles/prod/config.properties b/profiles/prod/config.properties index 12bb898e..f062330b 100644 --- a/profiles/prod/config.properties +++ b/profiles/prod/config.properties @@ -25,3 +25,8 @@ spring.thymeleaf.cache=true # NOTE! Leave this blank and just populate it in the user-specific .properties # file, which is git-ignored. digitransit.subscription.key= + +# In the jore4-jore3-importer repository (https://github.com/HSLdevcom/jore4-jore3-importer), there +# exists a "create-bulk-map-matching-input" branch containing a script that can be used to generate +# the CSV file. +test.routes.csvfile= diff --git a/run-bulk-test.sh b/run-bulk-test.sh new file mode 100755 index 00000000..ac3bb26a --- /dev/null +++ b/run-bulk-test.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +mvn -Pprod -DDB_URL=jdbc:postgresql://localhost:6433/jore4mapmatching?stringtype=unspecified -DDB_USERNAME=mapmatching -DDB_PASSWORD=password clean spring-boot:run diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/IMapMatchingBulkTestResultsPublisher.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/IMapMatchingBulkTestResultsPublisher.kt new file mode 100644 index 00000000..b44d160d --- /dev/null +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/IMapMatchingBulkTestResultsPublisher.kt @@ -0,0 +1,5 @@ +package fi.hsl.jore4.mapmatching.service.matching.test + +interface IMapMatchingBulkTestResultsPublisher { + fun publishResults(matchResults: List) +} diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/IPublicTransportRouteCsvParser.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/IPublicTransportRouteCsvParser.kt new file mode 100644 index 00000000..14d75206 --- /dev/null +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/IPublicTransportRouteCsvParser.kt @@ -0,0 +1,5 @@ +package fi.hsl.jore4.mapmatching.service.matching.test + +interface IPublicTransportRouteCsvParser { + fun parsePublicTransportRoutes(filePath: String): List +} diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTestResultsPublisherImpl.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTestResultsPublisherImpl.kt new file mode 100644 index 00000000..f59bd25f --- /dev/null +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTestResultsPublisherImpl.kt @@ -0,0 +1,99 @@ +package fi.hsl.jore4.mapmatching.service.matching.test + +import fi.hsl.jore4.mapmatching.util.LogUtils.joinToLogString +import io.github.oshai.kotlinlogging.KotlinLogging +import org.nield.kotlinstatistics.standardDeviation +import org.springframework.stereotype.Component +import kotlin.math.abs +import kotlin.math.roundToInt + +private val LOGGER = KotlinLogging.logger {} + +@Component +class MapMatchingBulkTestResultsPublisherImpl : IMapMatchingBulkTestResultsPublisher { + override fun publishResults(matchResults: List) { + val (succeeded, failed) = partitionBySuccess(matchResults) + + LOGGER.info { "Successful: ${succeeded.size}, failed: ${failed.size}" } + + val absLengthDiffs: List = succeeded.map { abs(it.getLengthDifferenceForFirstMatch()) } + + LOGGER.info { + "Average length difference: ${ + roundTo2Digits(absLengthDiffs.average()) + } meters, standard deviation: ${ + roundTo2Digits(absLengthDiffs.standardDeviation()) + } meters" + } + + val absLengthDiffPercentages: List = + succeeded.map { abs(it.getLengthDifferencePercentageForFirstMatch()) } + + LOGGER.info { + "Average length difference-%: ${ + roundTo2Digits(absLengthDiffPercentages.average()) + } %, standard deviation: ${ + roundTo2Digits(absLengthDiffPercentages.standardDeviation()) + } %" + } + + LOGGER.info { + val limit = 10 + val bestResults = + joinToLogString(getBestMatchResults(succeeded, limit)) { + """{"routeId": "${it.routeId}", "lengthDiff": ${ + roundTo2Digits(it.getLengthDifferenceForFirstMatch()) + }, "lengthDiff-%": ${ + roundTo2Digits(it.getLengthDifferencePercentageForFirstMatch()) + }}""" + } + + "Best $limit successful route matches: $bestResults" + } + + LOGGER.info { + val limit = 20 + val worstResults = + joinToLogString(getWorstMatchResults(succeeded, limit)) { + """{"routeId": "${it.routeId}", "lengthDiff": ${ + roundTo2Digits(it.getLengthDifferenceForFirstMatch()) + }, "lengthDiff-%": ${ + roundTo2Digits(it.getLengthDifferencePercentageForFirstMatch()) + }}""" + } + + "Worst $limit successful route matches: $worstResults" + } + } + + companion object { + private fun partitionBySuccess( + results: List + ): Pair, List> { + val succeeded: List = + results.mapNotNull { it as? SuccessfulMatchResult } + + val failed: List = results.mapNotNull { it as? MatchFailure } + + return succeeded to failed + } + + private fun getBestMatchResults( + results: List, + limit: Int + ): List = + results + .sortedBy { abs(it.getLengthDifferenceForFirstMatch()) } + .take(limit) + + private fun getWorstMatchResults( + results: List, + limit: Int + ): List = + results + .sortedByDescending { abs(it.getLengthDifferenceForFirstMatch()) } + .take(limit) + + private fun roundTo2Digits(n: Double): Double = (n * 100).roundToInt() / 100.0 + } +} diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTester.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTester.kt new file mode 100644 index 00000000..e9f467fc --- /dev/null +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTester.kt @@ -0,0 +1,134 @@ +package fi.hsl.jore4.mapmatching.service.matching.test + +import fi.hsl.jore4.mapmatching.model.VehicleType.GENERIC_BUS +import fi.hsl.jore4.mapmatching.model.matching.RoutePoint +import fi.hsl.jore4.mapmatching.service.common.response.RoutingResponse +import fi.hsl.jore4.mapmatching.service.matching.IMatchingService +import fi.hsl.jore4.mapmatching.service.matching.PublicTransportRouteMatchingParameters +import fi.hsl.jore4.mapmatching.service.matching.PublicTransportRouteMatchingParameters.JunctionMatchingParameters +import fi.hsl.jore4.mapmatching.service.matching.test.SuccessfulMatchResult.BufferRadius +import fi.hsl.jore4.mapmatching.service.matching.test.SuccessfulMatchResult.MatchDetails +import fi.hsl.jore4.mapmatching.util.GeolatteUtils.length +import io.github.oshai.kotlinlogging.KotlinLogging +import jakarta.annotation.PostConstruct +import org.geolatte.geom.G2D +import org.geolatte.geom.LineString +import org.springframework.beans.factory.annotation.Value +import org.springframework.stereotype.Service +import kotlin.time.Duration +import kotlin.time.ExperimentalTime +import kotlin.time.measureTime + +private val LOGGER = KotlinLogging.logger {} + +@Service +class MapMatchingBulkTester( + val csvParser: IPublicTransportRouteCsvParser, + val matchingService: IMatchingService, + val resultsPublisher: IMapMatchingBulkTestResultsPublisher, + @Value("\${test.routes.csvfile}") val csvFile: String +) { + @PostConstruct + @OptIn(ExperimentalTime::class) + fun launchRouteTesting() { + LOGGER.info { "Starting to map-match routes from file..." } + + val duration: Duration = + measureTime { + val matchResults: List = processRoutes() + + resultsPublisher.publishResults(matchResults) + } + + LOGGER.info { "Finished map-matching routes in $duration" } + } + + fun processRoutes(): List { + LOGGER.info { "Loading public transport routes from file: $csvFile" } + + val sourceRoutes: List = csvParser.parsePublicTransportRoutes(csvFile) + + val matchResults: List = + sourceRoutes.map { (routeId, routeGeometry, routePoints) -> + LOGGER.info { "Starting to match route: $routeId" } + + val result: MatchResult = matchRoute(routeId, routeGeometry, routePoints) + + if (result.matchFound) { + LOGGER.info { "Successfully matched route: $routeId" } + } else { + LOGGER.info { "Failed to match route: $routeId" } + } + + result + } + + return matchResults + } + + private fun matchRoute( + routeId: String, + geometry: LineString, + routePoints: List + ): MatchResult { + val matchingParams: PublicTransportRouteMatchingParameters = getMatchingParameters(50.0) + + val response: RoutingResponse = + matchingService.findMatchForPublicTransportRoute( + routeId, + geometry, + routePoints, + GENERIC_BUS, + matchingParams + ) + + val lengthOfSourceRoute: Double = length(geometry) + + return when (response) { + is RoutingResponse.RoutingSuccessDTO -> { + SuccessfulMatchResult( + routeId, + geometry, + lengthOfSourceRoute, + createMatchDetails( + response, + geometry, + matchingParams.bufferRadiusInMeters + ) + ) + } + + else -> MatchFailure(routeId, geometry, lengthOfSourceRoute) + } + } + + companion object { + private fun getMatchingParameters(bufferRadius: Double): PublicTransportRouteMatchingParameters { + val roadJunctionMatchingParams = + JunctionMatchingParameters(matchDistance = 5.0, clearingDistance = 30.0) + + return PublicTransportRouteMatchingParameters( + bufferRadiusInMeters = bufferRadius, + terminusLinkQueryDistance = bufferRadius, + terminusLinkQueryLimit = 5, + maxStopLocationDeviation = 80.0, + fallbackToViaNodesAlgorithm = true, + roadJunctionMatching = roadJunctionMatchingParams + ) + } + + private fun createMatchDetails( + routingResponse: RoutingResponse.RoutingSuccessDTO, + sourceGeometry: LineString, + bufferRadius: Double + ): MatchDetails { + val resultGeometry: LineString = routingResponse.routes[0].geometry + + return MatchDetails( + mapOf( + BufferRadius(bufferRadius) to length(resultGeometry) + ).toSortedMap() + ) + } + } +} diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MatchResult.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MatchResult.kt new file mode 100644 index 00000000..bc11b836 --- /dev/null +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MatchResult.kt @@ -0,0 +1,80 @@ +package fi.hsl.jore4.mapmatching.service.matching.test + +import org.geolatte.geom.G2D +import org.geolatte.geom.LineString +import java.util.SortedMap +import kotlin.math.abs + +sealed interface MatchResult { + val routeId: String + val sourceRouteGeometry: LineString + val sourceRouteLength: Double + + val matchFound: Boolean +} + +data class SuccessfulMatchResult( + override val routeId: String, + override val sourceRouteGeometry: LineString, + override val sourceRouteLength: Double, + val details: MatchDetails +) : MatchResult { + override val matchFound = true + + fun getLengthOfFirstMatch(): Double = details.getLengthOfFirstMatch() + + fun getLengthOfClosestMatch(): Double = details.getLengthOfClosestMatch(sourceRouteLength) + + fun getLengthDifferenceForFirstMatch(): Double = getLengthOfFirstMatch() - sourceRouteLength + + fun getLengthDifferenceForClosestMatch(): Double = getLengthOfClosestMatch() - sourceRouteLength + + fun getLengthDifferencePercentageForFirstMatch(): Double = + 100.0 * getLengthDifferenceForFirstMatch() / sourceRouteLength + + fun getLengthDifferencePercentageForClosestMatch(): Double = + 100.0 * getLengthDifferenceForClosestMatch() / sourceRouteLength + + @JvmInline + value class BufferRadius( + val value: Double + ) : Comparable { + override fun compareTo(other: BufferRadius): Int = value.compareTo(other.value) + + override fun toString() = value.toString() + } + + data class MatchDetails( + val lengthsOfMatchResults: SortedMap + ) { + init { + require(lengthsOfMatchResults.isNotEmpty()) { "lengthsOfMatchResults must not be empty" } + } + + fun getBufferRadiusOfFirstMatch(): BufferRadius = getMapEntryForFirstMatch().key + + fun getLengthOfFirstMatch(): Double = getMapEntryForFirstMatch().value + + fun getLengthOfClosestMatch(sourceRouteLength: Double): Double = + getMapEntryForClosestMatch(sourceRouteLength).value + + private fun getMapEntryForFirstMatch(): Map.Entry = lengthsOfMatchResults.entries.first() + + private fun getMapEntryForClosestMatch(sourceRouteLength: Double): Map.Entry = + when (lengthsOfMatchResults.size) { + 1 -> getMapEntryForFirstMatch() + else -> + lengthsOfMatchResults.entries.minByOrNull { (bufferRadius, length) -> + abs(length - sourceRouteLength) + }!! + } + } +} + +data class MatchFailure( + override val routeId: String, + override val sourceRouteGeometry: LineString, + override val sourceRouteLength: Double +) : MatchResult { + override val matchFound = false +} diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/PublicTransportRoute.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/PublicTransportRoute.kt new file mode 100644 index 00000000..d2e89b9d --- /dev/null +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/PublicTransportRoute.kt @@ -0,0 +1,14 @@ +package fi.hsl.jore4.mapmatching.service.matching.test + +import fi.hsl.jore4.mapmatching.model.matching.RoutePoint +import org.geolatte.geom.G2D +import org.geolatte.geom.LineString + +/** + * Route source for map-matching bulk tester + */ +data class PublicTransportRoute( + val routeId: String, + val routeGeometry: LineString, + val routePoints: List +) diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/PublicTransportRouteCsvParserImpl.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/PublicTransportRouteCsvParserImpl.kt new file mode 100644 index 00000000..51604c42 --- /dev/null +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/PublicTransportRouteCsvParserImpl.kt @@ -0,0 +1,58 @@ +package fi.hsl.jore4.mapmatching.service.matching.test + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.kotlin.readValue +import com.github.doyaaaaaken.kotlincsv.client.CsvReader +import com.github.doyaaaaaken.kotlincsv.dsl.csvReader +import fi.hsl.jore4.mapmatching.model.matching.RoutePoint +import org.geolatte.geom.G2D +import org.geolatte.geom.LineString +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Component + +@Component +class PublicTransportRouteCsvParserImpl + @Autowired + constructor( + val objectMapper: ObjectMapper + ) : IPublicTransportRouteCsvParser { + override fun parsePublicTransportRoutes(filePath: String): List { + val routes: MutableList = mutableListOf() + + CSV_READER.open(filePath) { + readAllAsSequence() + .drop(1) // skip header line + .forEach { row: List -> + + require(row.size >= COLUMNS) { + "Expected number of columns is $COLUMNS, but actual is: ${row.size}" + } + + routes.add(parseCsvRow(row)) + } + } + + return routes + } + + private fun parseCsvRow(row: List): PublicTransportRoute { + val routeDirectionId: String = row[2] + + val routeGeometry: LineString = objectMapper.readValue(row[4]) + val routePoints: List = objectMapper.readValue(row[5]) + + return PublicTransportRoute(routeDirectionId, routeGeometry, routePoints) + } + + companion object { + private val CSV_READER: CsvReader = + csvReader { + charset = Charsets.UTF_8.name() + quoteChar = '"' + delimiter = ',' + escapeChar = '\\' + } + + private const val COLUMNS = 6 + } + } diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/util/GeoToolsUtils.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/util/GeoToolsUtils.kt new file mode 100644 index 00000000..04d1ca8e --- /dev/null +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/util/GeoToolsUtils.kt @@ -0,0 +1,41 @@ +package fi.hsl.jore4.mapmatching.util + +import org.geotools.api.referencing.crs.CoordinateReferenceSystem +import org.geotools.geometry.jts.JTS +import org.geotools.referencing.CRS +import org.locationtech.jts.geom.Geometry + +object GeoToolsUtils { + fun transformCRS( + geometry: Geometry, + sourceEpsgCode: String, + targetEpsgCode: String + ): Geometry = + transformCRS( + geometry, + CRS.decode(sourceEpsgCode, true), + CRS.decode(targetEpsgCode, true) + ) + + fun transformCRS( + geometry: Geometry, + sourceSRID: Int, + targetSRID: Int + ): Geometry = + transformCRS( + geometry, + CRS.decode(getEpsgCode(sourceSRID), true), + CRS.decode(getEpsgCode(targetSRID), true) + ) + + fun transformCRS( + geometry: Geometry, + sourceCRS: CoordinateReferenceSystem, + targetCRS: CoordinateReferenceSystem + ): Geometry { + val transform = CRS.findMathTransform(sourceCRS, targetCRS) + return JTS.transform(geometry, transform) + } + + private fun getEpsgCode(srid: Int) = "EPSG:$srid" +} diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/util/GeolatteUtils.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/util/GeolatteUtils.kt index f52dc2b5..c53c974f 100644 --- a/src/main/kotlin/fi/hsl/jore4/mapmatching/util/GeolatteUtils.kt +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/util/GeolatteUtils.kt @@ -2,6 +2,7 @@ package fi.hsl.jore4.mapmatching.util import fi.hsl.jore4.mapmatching.model.LatLng import org.geolatte.geom.ByteBuffer +import org.geolatte.geom.C2D import org.geolatte.geom.G2D import org.geolatte.geom.Geometries.mkLineString import org.geolatte.geom.Geometries.mkPoint @@ -10,15 +11,24 @@ import org.geolatte.geom.GeometryType import org.geolatte.geom.LineString import org.geolatte.geom.Point import org.geolatte.geom.PositionSequenceBuilders +import org.geolatte.geom.ProjectedGeometryOperations import org.geolatte.geom.codec.Wkb import org.geolatte.geom.crs.CoordinateReferenceSystems.WGS84 +import org.geolatte.geom.crs.CrsRegistry +import org.geolatte.geom.crs.Geographic2DCoordinateReferenceSystem +import org.geolatte.geom.crs.ProjectedCoordinateReferenceSystem +import org.geolatte.geom.jts.JTS import java.math.BigDecimal import java.math.RoundingMode.HALF_UP import kotlin.math.max import kotlin.math.pow import kotlin.math.sqrt +@Suppress("MemberVisibilityCanBePrivate") object GeolatteUtils { + val EPSG_3067: ProjectedCoordinateReferenceSystem = + CrsRegistry.getProjectedCoordinateReferenceSystemForEPSG(3067) + // This is in WGS84 units and corresponds to approximately 5.5 millimeters at northern // latitudes. private const val LINE_ENDPOINT_CONNECTION_TOLERANCE = 0.0000001 @@ -96,6 +106,25 @@ object GeolatteUtils { return mkLineString(positionSequenceBuilder.toPositionSequence(), WGS84) } + fun transformFromGeographicToProjected( + geom: Geometry, + sourceCRS: Geographic2DCoordinateReferenceSystem, + targetCRS: ProjectedCoordinateReferenceSystem + ): Geometry { + val jtsPoint: org.locationtech.jts.geom.Geometry = JTS.to(geom) + val jtsGeom: org.locationtech.jts.geom.Geometry = + GeoToolsUtils.transformCRS(jtsPoint, sourceCRS.crsId.code, targetCRS.crsId.code) + + return JTS.from(jtsGeom, targetCRS) + } + + fun length(geographicLineString: LineString): Double { + val lineString3067: LineString = + transformFromGeographicToProjected(geographicLineString, WGS84, EPSG_3067) as LineString + + return ProjectedGeometryOperations.Default.length(lineString3067) + } + private fun calculateDistance( position1: G2D, position2: G2D diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 5e9398a3..6aa3dc1a 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -14,3 +14,4 @@ spring.flyway.create-schemas=true spring.thymeleaf.cache=@spring.thymeleaf.cache@ digitransit.subscription.key=@digitransit.subscription.key@ +test.routes.csvfile=@test.routes.csvfile@ diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index d7254cfd..b659c47e 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -6,7 +6,7 @@ - + diff --git a/src/test/kotlin/fi/hsl/jore4/mapmatching/util/GeolatteUtilsTest.kt b/src/test/kotlin/fi/hsl/jore4/mapmatching/util/GeolatteUtilsTest.kt index 90bb8b51..7488b4ba 100644 --- a/src/test/kotlin/fi/hsl/jore4/mapmatching/util/GeolatteUtilsTest.kt +++ b/src/test/kotlin/fi/hsl/jore4/mapmatching/util/GeolatteUtilsTest.kt @@ -1,11 +1,20 @@ package fi.hsl.jore4.mapmatching.util +import fi.hsl.jore4.mapmatching.util.GeolatteUtils.EPSG_3067 +import fi.hsl.jore4.mapmatching.util.GeolatteUtils.transformFromGeographicToProjected import org.assertj.core.api.Assertions.assertThat +import org.geolatte.geom.C2D import org.geolatte.geom.G2D import org.geolatte.geom.Geometries.mkLineString import org.geolatte.geom.LineString +import org.geolatte.geom.Point import org.geolatte.geom.PositionSequenceBuilders +import org.geolatte.geom.builder.DSL.c +import org.geolatte.geom.builder.DSL.g +import org.geolatte.geom.builder.DSL.point import org.geolatte.geom.crs.CoordinateReferenceSystems.WGS84 +import org.hamcrest.CoreMatchers.equalTo +import org.hamcrest.MatcherAssert.assertThat import org.junit.jupiter.api.Test class GeolatteUtilsTest { @@ -41,4 +50,16 @@ class GeolatteUtilsTest { assertThat(outputLine).isEqualTo(expectedLine) } + + @Test + fun testTransformFromGeographicToProjected() { + val geographicPoint: Point = point(WGS84, g(24.94167, 60.17055)) + + val expectedProjectedPoint: Point = point(EPSG_3067, c(385794.9708466177, 6672185.090453222)) + + assertThat( + transformFromGeographicToProjected(geographicPoint, WGS84, EPSG_3067), + equalTo(expectedProjectedPoint) + ) + } } From e899055787cf446badfeb5f2080a792764174981 Mon Sep 17 00:00:00 2001 From: Jarkko Kaura Date: Tue, 15 Feb 2022 18:37:17 +0200 Subject: [PATCH 02/15] Test stop-to-stop segments extracted from routes. --- .../test/ExtractStopToStopSegments.kt | 381 ++++++++++++++++++ .../IMapMatchingBulkTestResultsPublisher.kt | 5 +- ...MapMatchingBulkTestResultsPublisherImpl.kt | 152 +++++-- .../matching/test/MapMatchingBulkTester.kt | 94 ++++- .../service/matching/test/MatchResult.kt | 106 +++-- .../matching/test/StopToStopSegment.kt | 27 ++ .../matching/test/StopToStopSegmentation.kt | 6 + 7 files changed, 694 insertions(+), 77 deletions(-) create mode 100644 src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/ExtractStopToStopSegments.kt create mode 100644 src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/StopToStopSegment.kt create mode 100644 src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/StopToStopSegmentation.kt diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/ExtractStopToStopSegments.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/ExtractStopToStopSegments.kt new file mode 100644 index 00000000..d0239fe4 --- /dev/null +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/ExtractStopToStopSegments.kt @@ -0,0 +1,381 @@ +package fi.hsl.jore4.mapmatching.service.matching.test + +import fi.hsl.jore4.mapmatching.model.matching.RoutePoint +import fi.hsl.jore4.mapmatching.model.matching.RouteStopPoint +import fi.hsl.jore4.mapmatching.util.LogUtils.joinToLogString +import io.github.oshai.kotlinlogging.KotlinLogging +import org.geolatte.geom.G2D +import org.geolatte.geom.Geometries.mkLineString +import org.geolatte.geom.LineString +import org.geolatte.geom.Point +import org.geolatte.geom.PositionSequence +import org.geolatte.geom.PositionSequenceBuilder +import org.geolatte.geom.PositionSequenceBuilders +import org.geolatte.geom.crs.CoordinateReferenceSystems.WGS84 + +private val LOGGER = KotlinLogging.logger {} + +object ExtractStopToStopSegments { + private data class SegmentInfo( + val routePoints: List, + val referencingRoutes: MutableList + ) { + fun withReferencingRoute(routeId: String): SegmentInfo { + referencingRoutes.add(routeId) + return this + } + } + + private data class RoutePointLocations( + val measuredLocation: Point, + val projectedLocation: Point? = null + ) { + fun hasMeasuredLocationOf(pos: G2D): Boolean = pos == measuredLocation.position + + fun hasProjectedLocationOf(pos: G2D): Boolean = projectedLocation?.run { pos == position } ?: false + + fun hasPosition(pos: G2D): Boolean = hasMeasuredLocationOf(pos) || hasProjectedLocationOf(pos) + + fun hasSharedPositionWith(other: RoutePointLocations): Boolean = + hasPosition(other.measuredLocation.position) || + other.projectedLocation?.run { hasPosition(position) } == true + } + + private fun RoutePoint.toLocations() = + when (this) { + is RouteStopPoint -> RoutePointLocations(location, projectedLocation) + else -> RoutePointLocations(location) + } + + fun extractStopToStopSegments(routes: List): StopToStopSegmentation { + val segmentMap: MutableMap, SegmentInfo>> = mutableMapOf() + val discardedRoutes = mutableListOf() + + routes.forEach { route -> + + try { + splitToStopToStopSegments(route).forEach { (stopToStopSegmentId, geometry, routePoints) -> + + segmentMap + .getOrPut(stopToStopSegmentId) { mutableMapOf() } + .getOrPut(geometry) { SegmentInfo(routePoints, mutableListOf()) } + .withReferencingRoute(route.routeId) + } + } catch (e: IllegalArgumentException) { + LOGGER.info { + "${route.routeId}: Discarding route because of mismatch between number of route point ranges " + + "and LineString segments" + } + e.message?.let(discardedRoutes::add) + } + } + + val stopToStopSegments = + segmentMap + .toSortedMap() + .entries + .flatMap { (stopToStopSegmentId, segmentMap) -> + + segmentMap.entries + .sortedBy { it.key.numPositions } + .withIndex() + .map { (index, segmentEntry) -> + + val geometry: LineString = segmentEntry.key + + val stopToStopRouteId = + if (index == 0) { + "$stopToStopSegmentId" + } else { + "$stopToStopSegmentId-${index + 1}" + } + + val routePoints: List = segmentEntry.value.routePoints + val referencingRoutes: List = segmentEntry.value.referencingRoutes + + StopToStopSegment(stopToStopRouteId, geometry, routePoints, referencingRoutes) + } + } + + return StopToStopSegmentation(stopToStopSegments, discardedRoutes) + } + + private fun splitToStopToStopSegments(route: PublicTransportRoute): List { + val originRoutePoints: List = trimRoutePointStart(route) + + val routePointSegments: List> = splitRoutePoints(originRoutePoints) + + if (routePointSegments.isEmpty()) { + return emptyList() + } + + val terminusOfRoutePointSegments: List> = + routePointSegments + .map { routePoints -> + val first: RoutePoint = routePoints.first() + + if (first !is RouteStopPoint) { + throw IllegalStateException("Expected first point to be a RouteStopPoint") + } + + val last: RoutePoint = routePoints.last() + + if (last !is RouteStopPoint) { + throw IllegalStateException("Expected last point to be a RouteStopPoint") + } + + first to last + } + + val geometrySegments: List> = + splitRouteGeometry( + route.routeId, + route.routeGeometry, + terminusOfRoutePointSegments, + false + ) + + if (geometrySegments.size != routePointSegments.size) { + LOGGER.info { + "${route.routeId}: Stop-to-stop route point ranges (${terminusOfRoutePointSegments.size}): ${ + joinToLogString(terminusOfRoutePointSegments) + }" + } + LOGGER.info { + "${route.routeId}: Split LineStrings (${geometrySegments.size}): ${ + joinToLogString(geometrySegments) + }" + } + + // Re-run LineString splitting with debug prints. + splitRouteGeometry(route.routeId, route.routeGeometry, terminusOfRoutePointSegments, true) + + throw IllegalArgumentException(route.routeId) + // throw IllegalStateException(route.routeId) + } + + fun getStopId(routePoint: RoutePoint): String = + when (routePoint) { + is RouteStopPoint -> { + routePoint.run { passengerId ?: nationalId?.toString() } + ?: routePoint.location.position.run { "($lon,$lat)" } + } + else -> throw IllegalStateException("Expected RouteStopPoint") + } + + return geometrySegments + .zip(routePointSegments) + .map { (lineString, routePoints) -> + + PublicTransportRoute( + "${getStopId(routePoints.first())}-${getStopId(routePoints.last())}", + lineString, + routePoints + ) + } + } + + private fun trimRoutePointStart(route: PublicTransportRoute): List { + // Drop the first route point if the route geometry does not start with it. + return if (route.routePoints + .first() + .toLocations() + .hasPosition(route.routeGeometry.startPosition) + ) { + route.routePoints + } else { + LOGGER.info { + "${route.routeId}: Dropping the first route point because it is not part of route geometry" + } + route.routePoints.drop(1) + } + } + + private fun splitRoutePoints(originRoutePoints: List): List> { + val stopToStopIndexRanges: List> = + extractRoutePointIndexRangesForStopToStopSegments(originRoutePoints) + + if (stopToStopIndexRanges.size < 2) { + return emptyList() + } + + return stopToStopIndexRanges + .map { (startIndex, endIndex) -> + val rps: MutableList = mutableListOf() + + for (index in startIndex..endIndex) { + rps.add(originRoutePoints[index]) + } + + rps + }.filter { routePoints -> + routePoints.run { + size > 2 || !first().toLocations().hasSharedPositionWith(last().toLocations()) + } + } + } + + private fun extractRoutePointIndexRangesForStopToStopSegments(routePoints: List): List> { + if (routePoints.isEmpty()) { + return emptyList() + } + + val routeStopPointIndices: List = + routePoints.mapIndexedNotNull { index, routePoint -> + if (routePoint is RouteStopPoint) index else null + } + + if (routeStopPointIndices.size < 2) { + return emptyList() + } + + val indexRanges: MutableList> = mutableListOf() + + for (i in 0 until routeStopPointIndices.lastIndex) { + val startIndex: Int = routeStopPointIndices[i] + val endIndex: Int = routeStopPointIndices[i + 1] + + indexRanges.add(startIndex to endIndex) + } + + return indexRanges + } + + private fun splitRouteGeometry( + routeId: String, + routeGeometry: LineString, + stopPointRanges: List>, + printDebug: Boolean + ): List> { + if (stopPointRanges.isEmpty()) { + return emptyList() + } + + val firstStopPointOnRoute: RoutePoint = stopPointRanges.first().first + + val trimmedLine: LineString = dropFromStart(routeGeometry, firstStopPointOnRoute.toLocations()) + + val (_, splitLineStrings: List>) = + stopPointRanges + .fold( + trimmedLine to emptyList>() + ) { (remainingLineString, substrings), (_, endStopPoint) -> + + if (printDebug) { + LOGGER.info { "$routeId: Remaining : $remainingLineString" } + } + + if (remainingLineString.isEmpty) { + remainingLineString to substrings + } else { + val endStopLocations: RoutePointLocations = endStopPoint.toLocations() + + val substring: LineString = takeFromStart(remainingLineString, endStopLocations) + + if (printDebug) { + LOGGER.info { "$routeId: LineSubstring: $substring" } + } + + if (substring.isEmpty) { + remainingLineString to substrings + } else { + val newRemaining: LineString = dropFromStart(remainingLineString, endStopLocations) + + newRemaining to substrings + substring + } + } + } + + return splitLineStrings + } + + private fun takeFromStart( + lineString: LineString, + toLocation: RoutePointLocations + ): LineString { + val positionSequenceBuilder = PositionSequenceBuilders.variableSized(G2D::class.java) + + var measuredLocationMatchedAlready = false + var projectedLocationMatchedAlready = false + + lineString.positions + .takeWhile { pos -> + when { + measuredLocationMatchedAlready && projectedLocationMatchedAlready -> false + measuredLocationMatchedAlready -> { + toLocation.hasProjectedLocationOf(pos).also { projectedLocationMatchedAlready = it } + } + projectedLocationMatchedAlready -> { + toLocation.hasMeasuredLocationOf(pos).also { measuredLocationMatchedAlready = it } + } + else -> { + if (toLocation.hasMeasuredLocationOf(pos)) { + measuredLocationMatchedAlready = true + } else if (toLocation.hasProjectedLocationOf(pos)) { + projectedLocationMatchedAlready = true + } + true + } + } + }.forEach(positionSequenceBuilder::add) + + return toLineString(positionSequenceBuilder) + } + + private fun dropFromStart( + lineString: LineString, + untilLocation: RoutePointLocations + ): LineString { + if (untilLocation.hasPosition(lineString.startPosition)) { + return lineString + } + + val linePositions: PositionSequence = lineString.positions + + val firstIndex: Int = linePositions.indexOfFirst(untilLocation::hasPosition) + + if (firstIndex < 0) { + return lineString + } + + val numPositions: Int = linePositions.size() + + val hasAtLeast2StopLocations = + numPositions >= firstIndex + 2 && + untilLocation.hasPosition(linePositions.getPositionN(firstIndex + 1)) + + val hasAtLeast3StopLocations = + hasAtLeast2StopLocations && + numPositions >= firstIndex + 3 && + untilLocation.hasPosition(linePositions.getPositionN(firstIndex + 2)) + + val has4StopLocations = + hasAtLeast3StopLocations && + numPositions >= firstIndex + 4 && + untilLocation.hasPosition(linePositions.getPositionN(firstIndex + 3)) + + val numPositionsToDrop: Int = + when { + has4StopLocations -> firstIndex + 2 + hasAtLeast2StopLocations -> firstIndex + 1 + else -> firstIndex + } + + val positionSequenceBuilder = PositionSequenceBuilders.variableSized(G2D::class.java) + + linePositions + .drop(numPositionsToDrop) + .forEach(positionSequenceBuilder::add) + + return toLineString(positionSequenceBuilder) + } + + private fun toLineString(positionSequenceBuilder: PositionSequenceBuilder): LineString { + val positionSequence: PositionSequence = positionSequenceBuilder.toPositionSequence() + + return when (positionSequence.size()) { + 0, 1 -> LineString(WGS84) + else -> mkLineString(positionSequence, WGS84) + } + } +} diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/IMapMatchingBulkTestResultsPublisher.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/IMapMatchingBulkTestResultsPublisher.kt index b44d160d..5c465374 100644 --- a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/IMapMatchingBulkTestResultsPublisher.kt +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/IMapMatchingBulkTestResultsPublisher.kt @@ -1,5 +1,8 @@ package fi.hsl.jore4.mapmatching.service.matching.test interface IMapMatchingBulkTestResultsPublisher { - fun publishResults(matchResults: List) + fun publishMatchResultsForRoutesAndStopToStopSegments( + routeResults: List, + stopToStopSegmentResults: List + ) } diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTestResultsPublisherImpl.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTestResultsPublisherImpl.kt index f59bd25f..c7df4d02 100644 --- a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTestResultsPublisherImpl.kt +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTestResultsPublisherImpl.kt @@ -11,31 +11,29 @@ private val LOGGER = KotlinLogging.logger {} @Component class MapMatchingBulkTestResultsPublisherImpl : IMapMatchingBulkTestResultsPublisher { - override fun publishResults(matchResults: List) { - val (succeeded, failed) = partitionBySuccess(matchResults) - - LOGGER.info { "Successful: ${succeeded.size}, failed: ${failed.size}" } - - val absLengthDiffs: List = succeeded.map { abs(it.getLengthDifferenceForFirstMatch()) } + override fun publishMatchResultsForRoutesAndStopToStopSegments( + routeResults: List, + stopToStopSegmentResults: List + ) { + publishRouteMatchResults(routeResults) + publishStopToStopSegmentMatchResults(stopToStopSegmentResults) LOGGER.info { - "Average length difference: ${ - roundTo2Digits(absLengthDiffs.average()) - } meters, standard deviation: ${ - roundTo2Digits(absLengthDiffs.standardDeviation()) - } meters" + "List of IDs of failed routes whose all segments were matched: ${ + joinToLogString( + getRoutesNotMatchedEvenThoughAllSegmentsMatched( + routeResults, + stopToStopSegmentResults + ) + ) + }" } + } - val absLengthDiffPercentages: List = - succeeded.map { abs(it.getLengthDifferencePercentageForFirstMatch()) } + fun publishRouteMatchResults(results: List) { + val (succeeded, failed) = partitionBySuccess(results) - LOGGER.info { - "Average length difference-%: ${ - roundTo2Digits(absLengthDiffPercentages.average()) - } %, standard deviation: ${ - roundTo2Digits(absLengthDiffPercentages.standardDeviation()) - } %" - } + printBasicStatistics(succeeded, failed, "Route") LOGGER.info { val limit = 10 @@ -66,14 +64,56 @@ class MapMatchingBulkTestResultsPublisherImpl : IMapMatchingBulkTestResultsPubli } } + fun publishStopToStopSegmentMatchResults(results: List) { + val (succeeded, failed) = partitionSegmentsBySuccess(results) + + printBasicStatistics(succeeded, failed, "Stop-to-stop segment") + + LOGGER.info { + val limit = 20 + val mostReferencedFailedSegments = + joinToLogString(getMostReferencedFailedSegments(failed, limit)) { + "{\"segmentId\": \"${it.routeId}\", \"numReferencingRoutes\": ${it.referencingRoutes.size}, " + + "\"referencingRoutes\": \"${it.referencingRoutes}\", \"sourceRouteGeometry\": \"${it.sourceRouteGeometry}\"}" + } + + "$limit most referenced failed stop-to-stop segments: $mostReferencedFailedSegments" + } + + LOGGER.info { + val limit = 20 + val worstResults = + joinToLogString(getWorstMatchResults(succeeded, limit)) { + """{"segmentId": "${it.routeId}", "lengthDiff": ${ + roundTo2Digits(it.getLengthDifferenceForFirstMatch()) + }, "lengthDiff-%": ${ + roundTo2Digits(it.getLengthDifferencePercentageForFirstMatch()) + }, "sourceRouteGeometry": "${it.sourceRouteGeometry}"}""" + } + + "Worst $limit successful stop-to-stop segment matches: $worstResults" + } + } + companion object { private fun partitionBySuccess( results: List - ): Pair, List> { - val succeeded: List = - results.mapNotNull { it as? SuccessfulMatchResult } + ): Pair, List> { + val succeeded: List = + results.mapNotNull { it as? SuccessfulRouteMatchResult } - val failed: List = results.mapNotNull { it as? MatchFailure } + val failed: List = results.mapNotNull { it as? RouteMatchFailure } + + return succeeded to failed + } + + private fun partitionSegmentsBySuccess( + results: List + ): Pair, List> { + val succeeded: List = + results.mapNotNull { it as? SuccessfulSegmentMatchResult } + + val failed: List = results.mapNotNull { it as? SegmentMatchFailure } return succeeded to failed } @@ -94,6 +134,70 @@ class MapMatchingBulkTestResultsPublisherImpl : IMapMatchingBulkTestResultsPubli .sortedByDescending { abs(it.getLengthDifferenceForFirstMatch()) } .take(limit) + private fun getMostReferencedFailedSegments( + results: List, + limit: Int + ): List = + results + .filterIsInstance() + .sortedByDescending { it.referencingRoutes.size } + .take(limit) + private fun roundTo2Digits(n: Double): Double = (n * 100).roundToInt() / 100.0 + + private fun getRoutesNotMatchedEvenThoughAllSegmentsMatched( + routeResults: List, + stopToStopSegmentResults: List + ): List { + val idsOfRoutesContainingFailedSegments: Set = + stopToStopSegmentResults + .flatMap(SegmentMatchResult::referencingRoutes) + .toSet() + + return routeResults + .filterIsInstance() + .map(MatchResult::routeId) + .filter { it !in idsOfRoutesContainingFailedSegments } + .sorted() + } + + private fun printBasicStatistics( + succeeded: List, + failed: List, + resultSetName: String + ) { + val numSucceeded = succeeded.size + val numFailed = failed.size + val numTotal: Int = numSucceeded + numFailed + + LOGGER.info { + "$resultSetName match results: all: $numTotal, successful: $numSucceeded (${ + roundTo2Digits(100.0 * numSucceeded / numTotal) + } %), failed: $numFailed (${ + roundTo2Digits(100.0 * numFailed / numTotal) + } %)" + } + + val absLengthDiffs: List = succeeded.map { abs(it.getLengthDifferenceForFirstMatch()) } + + LOGGER.info { + "$resultSetName average absolute length difference: ${ + roundTo2Digits(absLengthDiffs.average()) + } meters, standard deviation: ${ + roundTo2Digits(absLengthDiffs.standardDeviation()) + } meters" + } + + val absLengthDiffPercentages: List = + succeeded.map { abs(it.getLengthDifferencePercentageForFirstMatch()) } + + LOGGER.info { + "$resultSetName average absolute length difference-%: ${ + roundTo2Digits(absLengthDiffPercentages.average()) + } %, standard deviation: ${ + roundTo2Digits(absLengthDiffPercentages.standardDeviation()) + } %" + } + } } } diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTester.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTester.kt index e9f467fc..c5b6b72b 100644 --- a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTester.kt +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTester.kt @@ -6,8 +6,6 @@ import fi.hsl.jore4.mapmatching.service.common.response.RoutingResponse import fi.hsl.jore4.mapmatching.service.matching.IMatchingService import fi.hsl.jore4.mapmatching.service.matching.PublicTransportRouteMatchingParameters import fi.hsl.jore4.mapmatching.service.matching.PublicTransportRouteMatchingParameters.JunctionMatchingParameters -import fi.hsl.jore4.mapmatching.service.matching.test.SuccessfulMatchResult.BufferRadius -import fi.hsl.jore4.mapmatching.service.matching.test.SuccessfulMatchResult.MatchDetails import fi.hsl.jore4.mapmatching.util.GeolatteUtils.length import io.github.oshai.kotlinlogging.KotlinLogging import jakarta.annotation.PostConstruct @@ -35,37 +33,95 @@ class MapMatchingBulkTester( val duration: Duration = measureTime { - val matchResults: List = processRoutes() + val (routeMatchResults, stopToStopSegmentMatchResults) = processFile() - resultsPublisher.publishResults(matchResults) + resultsPublisher.publishMatchResultsForRoutesAndStopToStopSegments( + routeMatchResults, + stopToStopSegmentMatchResults + ) } LOGGER.info { "Finished map-matching routes in $duration" } } - fun processRoutes(): List { + fun processFile(): Pair, List> { LOGGER.info { "Loading public transport routes from file: $csvFile" } val sourceRoutes: List = csvParser.parsePublicTransportRoutes(csvFile) - val matchResults: List = - sourceRoutes.map { (routeId, routeGeometry, routePoints) -> - LOGGER.info { "Starting to match route: $routeId" } + LOGGER.info { "Number of source routes: ${sourceRoutes.size}" } - val result: MatchResult = matchRoute(routeId, routeGeometry, routePoints) + val (stopToStopSegments: List, discardedRoutes: List) = + ExtractStopToStopSegments.extractStopToStopSegments(sourceRoutes) - if (result.matchFound) { - LOGGER.info { "Successfully matched route: $routeId" } - } else { - LOGGER.info { "Failed to match route: $routeId" } - } + LOGGER.info { "Number of stop-to-stop segments: ${stopToStopSegments.size}" } + LOGGER.info { + "Number of discarded routes within resolution of stop-to-stop segments: ${discardedRoutes.size}" + } - result - } + val routeMatchResults: List = matchRoutes(sourceRoutes) + val segmentMatchResults: List = matchStopToStopSegments(stopToStopSegments) - return matchResults + return routeMatchResults to segmentMatchResults } + private fun matchRoutes(routes: List): List = + routes.map { (routeId, routeGeometry, routePoints) -> + LOGGER.info { "Starting to match route: $routeId" } + + val result: MatchResult = matchRoute(routeId, routeGeometry, routePoints) + + if (result.matchFound) { + LOGGER.info { "Successfully matched route: $routeId" } + } else { + LOGGER.info { "Failed to match route: $routeId" } + } + + result + } + + private fun matchStopToStopSegments(segments: List): List = + segments.map { segment -> + val (segmentId, geometry, routePoints, referencingRoutes) = segment + + LOGGER.info { "Starting to match stop-to-stop segment: $segmentId" } + + val result: MatchResult = matchRoute(segmentId, geometry, routePoints) + + if (result.matchFound) { + LOGGER.info { "Successfully matched stop-to-stop segment: $segmentId" } + } else { + LOGGER.info { "Failed to match stop-to-stop segment: $segmentId" } + } + + val numRoutePoints = routePoints.size + + when (result) { + is SuccessfulRouteMatchResult -> + SuccessfulSegmentMatchResult( + segmentId, + geometry, + result.sourceRouteLength, + result.details, + segment.startStopId, + segment.endStopId, + numRoutePoints, + referencingRoutes + ) + + else -> + SegmentMatchFailure( + segmentId, + geometry, + result.sourceRouteLength, + segment.startStopId, + segment.endStopId, + numRoutePoints, + referencingRoutes + ) + } + } + private fun matchRoute( routeId: String, geometry: LineString, @@ -86,7 +142,7 @@ class MapMatchingBulkTester( return when (response) { is RoutingResponse.RoutingSuccessDTO -> { - SuccessfulMatchResult( + SuccessfulRouteMatchResult( routeId, geometry, lengthOfSourceRoute, @@ -98,7 +154,7 @@ class MapMatchingBulkTester( ) } - else -> MatchFailure(routeId, geometry, lengthOfSourceRoute) + else -> RouteMatchFailure(routeId, geometry, lengthOfSourceRoute) } } diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MatchResult.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MatchResult.kt index bc11b836..5364f5d2 100644 --- a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MatchResult.kt +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MatchResult.kt @@ -13,13 +13,8 @@ sealed interface MatchResult { val matchFound: Boolean } -data class SuccessfulMatchResult( - override val routeId: String, - override val sourceRouteGeometry: LineString, - override val sourceRouteLength: Double, +interface SuccessfulMatchResult : MatchResult { val details: MatchDetails -) : MatchResult { - override val matchFound = true fun getLengthOfFirstMatch(): Double = details.getLengthOfFirstMatch() @@ -34,47 +29,92 @@ data class SuccessfulMatchResult( fun getLengthDifferencePercentageForClosestMatch(): Double = 100.0 * getLengthDifferenceForClosestMatch() / sourceRouteLength +} + +interface SegmentMatchResult : MatchResult { + val startStopId: String + val endStopId: String + + val numberOfRoutePoints: Int + val referencingRoutes: List +} + +@JvmInline +value class BufferRadius( + val value: Double +) : Comparable { + override fun compareTo(other: BufferRadius): Int = value.compareTo(other.value) - @JvmInline - value class BufferRadius( - val value: Double - ) : Comparable { - override fun compareTo(other: BufferRadius): Int = value.compareTo(other.value) + override fun toString() = value.toString() +} - override fun toString() = value.toString() +data class MatchDetails( + val lengthsOfMatchResults: SortedMap +) { + init { + require(lengthsOfMatchResults.isNotEmpty()) { "lengthsOfMatchResults must not be empty" } } - data class MatchDetails( - val lengthsOfMatchResults: SortedMap - ) { - init { - require(lengthsOfMatchResults.isNotEmpty()) { "lengthsOfMatchResults must not be empty" } - } + fun getBufferRadiusOfFirstMatch(): BufferRadius = getMapEntryForFirstMatch().key - fun getBufferRadiusOfFirstMatch(): BufferRadius = getMapEntryForFirstMatch().key + fun getBufferRadiusOfClosestMatch(sourceRouteLength: Double): BufferRadius = + getMapEntryForClosestMatch(sourceRouteLength).key - fun getLengthOfFirstMatch(): Double = getMapEntryForFirstMatch().value + fun getLengthOfFirstMatch(): Double = getMapEntryForFirstMatch().value - fun getLengthOfClosestMatch(sourceRouteLength: Double): Double = - getMapEntryForClosestMatch(sourceRouteLength).value + fun getLengthOfClosestMatch(sourceRouteLength: Double): Double = getMapEntryForClosestMatch(sourceRouteLength).value - private fun getMapEntryForFirstMatch(): Map.Entry = lengthsOfMatchResults.entries.first() + private fun getMapEntryForFirstMatch(): Map.Entry = lengthsOfMatchResults.entries.first() - private fun getMapEntryForClosestMatch(sourceRouteLength: Double): Map.Entry = - when (lengthsOfMatchResults.size) { - 1 -> getMapEntryForFirstMatch() - else -> - lengthsOfMatchResults.entries.minByOrNull { (bufferRadius, length) -> - abs(length - sourceRouteLength) - }!! - } - } + private fun getMapEntryForClosestMatch(sourceRouteLength: Double): Map.Entry = + when (lengthsOfMatchResults.size) { + 1 -> getMapEntryForFirstMatch() + else -> + lengthsOfMatchResults.entries.minByOrNull { (bufferRadius, length) -> + abs(length - sourceRouteLength) + }!! + } +} + +data class SuccessfulRouteMatchResult( + override val routeId: String, + override val sourceRouteGeometry: LineString, + override val sourceRouteLength: Double, + override val details: MatchDetails +) : SuccessfulMatchResult { + override val matchFound = true } -data class MatchFailure( +data class RouteMatchFailure( override val routeId: String, override val sourceRouteGeometry: LineString, override val sourceRouteLength: Double ) : MatchResult { override val matchFound = false } + +data class SuccessfulSegmentMatchResult( + override val routeId: String, + override val sourceRouteGeometry: LineString, + override val sourceRouteLength: Double, + override val details: MatchDetails, + override val startStopId: String, + override val endStopId: String, + override val numberOfRoutePoints: Int, + override val referencingRoutes: List +) : SuccessfulMatchResult, + SegmentMatchResult { + override val matchFound = true +} + +data class SegmentMatchFailure( + override val routeId: String, + override val sourceRouteGeometry: LineString, + override val sourceRouteLength: Double, + override val startStopId: String, + override val endStopId: String, + override val numberOfRoutePoints: Int, + override val referencingRoutes: List +) : SegmentMatchResult { + override val matchFound = false +} diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/StopToStopSegment.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/StopToStopSegment.kt new file mode 100644 index 00000000..2090fc60 --- /dev/null +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/StopToStopSegment.kt @@ -0,0 +1,27 @@ +package fi.hsl.jore4.mapmatching.service.matching.test + +import fi.hsl.jore4.mapmatching.model.matching.RoutePoint +import fi.hsl.jore4.mapmatching.model.matching.RouteStopPoint +import org.geolatte.geom.G2D +import org.geolatte.geom.LineString + +data class StopToStopSegment( + val segmentId: String, + val geometry: LineString, + val routePoints: List, + val referencingRoutes: List +) { + val startStopId: String + get() = + when (val startStop = routePoints.first()) { + is RouteStopPoint -> startStop.passengerId + else -> throw IllegalStateException("The first point is not a stop point") + } + + val endStopId: String + get() = + when (val endStop = routePoints.last()) { + is RouteStopPoint -> endStop.passengerId + else -> throw IllegalStateException("The last point is not a stop point") + } +} diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/StopToStopSegmentation.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/StopToStopSegmentation.kt new file mode 100644 index 00000000..4eaccbab --- /dev/null +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/StopToStopSegmentation.kt @@ -0,0 +1,6 @@ +package fi.hsl.jore4.mapmatching.service.matching.test + +data class StopToStopSegmentation( + val segments: List, + val discardedRoutes: List +) From d741d686d1c332ad9384bdf5a87729b2d979da70 Mon Sep 17 00:00:00 2001 From: Jarkko Kaura Date: Thu, 14 Apr 2022 18:00:11 +0300 Subject: [PATCH 03/15] Add support for bulk-testing routes with multiple buffer radius values. If matching with one radius value fails, we can try with a larger radius value. --- ...MapMatchingBulkTestResultsPublisherImpl.kt | 91 +++++++++++++++ .../matching/test/MapMatchingBulkTester.kt | 110 +++++++++++------- .../service/matching/test/MatchResult.kt | 5 +- 3 files changed, 160 insertions(+), 46 deletions(-) diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTestResultsPublisherImpl.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTestResultsPublisherImpl.kt index c7df4d02..7667f1b5 100644 --- a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTestResultsPublisherImpl.kt +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTestResultsPublisherImpl.kt @@ -4,6 +4,7 @@ import fi.hsl.jore4.mapmatching.util.LogUtils.joinToLogString import io.github.oshai.kotlinlogging.KotlinLogging import org.nield.kotlinstatistics.standardDeviation import org.springframework.stereotype.Component +import java.util.SortedMap import kotlin.math.abs import kotlin.math.roundToInt @@ -34,6 +35,7 @@ class MapMatchingBulkTestResultsPublisherImpl : IMapMatchingBulkTestResultsPubli val (succeeded, failed) = partitionBySuccess(results) printBasicStatistics(succeeded, failed, "Route") + printBufferStatistics(succeeded, "Route") LOGGER.info { val limit = 10 @@ -68,6 +70,7 @@ class MapMatchingBulkTestResultsPublisherImpl : IMapMatchingBulkTestResultsPubli val (succeeded, failed) = partitionSegmentsBySuccess(results) printBasicStatistics(succeeded, failed, "Stop-to-stop segment") + printBufferStatistics(succeeded, "Stop-to-stop segment") LOGGER.info { val limit = 20 @@ -199,5 +202,93 @@ class MapMatchingBulkTestResultsPublisherImpl : IMapMatchingBulkTestResultsPubli } %" } } + + private fun printBufferStatistics( + succeeded: List, + resultSetName: String + ) { + val bufferRadiusSet: MutableSet = mutableSetOf() + + succeeded.forEach { + bufferRadiusSet.addAll(it.details.lengthsOfMatchResults.keys) + } + + val bufferRadiusList: List = bufferRadiusSet.toList().sorted() + + if (bufferRadiusList.size < 2) { + return + } + + val lowestBufferRadiusAppearanceCounts: SortedMap = + bufferRadiusList + .associateWith { bufferRadius -> + succeeded.count { it.getLowestBufferRadius() == bufferRadius } + }.toSortedMap() + + LOGGER.info { + "$resultSetName matches with different buffer radius values: ${ + joinToLogString(lowestBufferRadiusAppearanceCounts.entries) { (bufferRadius, matchCount) -> + "$bufferRadius m: $matchCount pcs" + } + }" + } + + val lengthComparisons: List> = + (0 until bufferRadiusList.size - 1) + .mapNotNull { firstBufferRadiusIndex -> + val remainingBufferRadiusValues: List = + bufferRadiusList.drop( + firstBufferRadiusIndex + ) + + val lowestBufferRadius: BufferRadius = remainingBufferRadiusValues.first() + + val filteredList: List = + succeeded.filter { it.getLowestBufferRadius() == lowestBufferRadius } + + when (filteredList.size) { + 0 -> null + else -> { + remainingBufferRadiusValues.associateWith { bufferRadius -> + val lengthDifferencesOfMatchedRoutes: List = + filteredList + .mapNotNull { match -> + val lengthOfMatchedRoute: Double? = + match.details.lengthsOfMatchResults[bufferRadius] + + lengthOfMatchedRoute?.let { + // Return absolute length difference. + abs(lengthOfMatchedRoute - match.sourceRouteLength) + } ?: run { + LOGGER.warn { + "The length of match result not available for route: ${match.routeId}" + } + null + } + } + + // Return average of all differences. + lengthDifferencesOfMatchedRoutes.average() + } + } + } + }.map { it.toSortedMap() } + + if (lengthComparisons.isNotEmpty()) { + LOGGER.info { + "$resultSetName match length differences compared with regard to buffer radius: ${ + joinToLogString(lengthComparisons) { mapOfDifferences -> + mapOfDifferences.entries.joinToString( + prefix = "[", + transform = { (bufferRadius, avgDiff) -> + "$bufferRadius: ${roundTo2Digits(avgDiff)}" + }, + postfix = "]" + ) + } + }" + } + } + } } } diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTester.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTester.kt index c5b6b72b..cb51c08d 100644 --- a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTester.kt +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTester.kt @@ -59,20 +59,26 @@ class MapMatchingBulkTester( "Number of discarded routes within resolution of stop-to-stop segments: ${discardedRoutes.size}" } - val routeMatchResults: List = matchRoutes(sourceRoutes) - val segmentMatchResults: List = matchStopToStopSegments(stopToStopSegments) + val routeMatchResults: List = matchRoutes(sourceRoutes, DEFAULT_ROUTE_MATCH_RADIUS_VALUES) + val segmentMatchResults: List = + matchStopToStopSegments(stopToStopSegments, DEFAULT_SEGMENT_MATCH_RADIUS_VALUES) return routeMatchResults to segmentMatchResults } - private fun matchRoutes(routes: List): List = + private fun matchRoutes( + routes: List, + bufferRadiuses: Set + ): List = routes.map { (routeId, routeGeometry, routePoints) -> LOGGER.info { "Starting to match route: $routeId" } - val result: MatchResult = matchRoute(routeId, routeGeometry, routePoints) + val result: MatchResult = matchRoute(routeId, routeGeometry, routePoints, bufferRadiuses) - if (result.matchFound) { - LOGGER.info { "Successfully matched route: $routeId" } + if (result is SuccessfulMatchResult) { + LOGGER.info { + "Successfully matched route: $routeId (with bufferRadius=${result.getLowestBufferRadius()})" + } } else { LOGGER.info { "Failed to match route: $routeId" } } @@ -80,16 +86,23 @@ class MapMatchingBulkTester( result } - private fun matchStopToStopSegments(segments: List): List = + private fun matchStopToStopSegments( + segments: List, + bufferRadiuses: Set + ): List = segments.map { segment -> val (segmentId, geometry, routePoints, referencingRoutes) = segment LOGGER.info { "Starting to match stop-to-stop segment: $segmentId" } - val result: MatchResult = matchRoute(segmentId, geometry, routePoints) + val result: MatchResult = matchRoute(segmentId, geometry, routePoints, bufferRadiuses) - if (result.matchFound) { - LOGGER.info { "Successfully matched stop-to-stop segment: $segmentId" } + if (result is SuccessfulMatchResult) { + LOGGER.info { + "Successfully matched stop-to-stop segment: $segmentId (with bufferRadius=${ + result.getLowestBufferRadius() + })" + } } else { LOGGER.info { "Failed to match stop-to-stop segment: $segmentId" } } @@ -109,7 +122,7 @@ class MapMatchingBulkTester( referencingRoutes ) - else -> + is RouteMatchFailure -> SegmentMatchFailure( segmentId, geometry, @@ -119,46 +132,67 @@ class MapMatchingBulkTester( numRoutePoints, referencingRoutes ) + + else -> throw IllegalStateException("Unknown segment match result type") } } private fun matchRoute( routeId: String, geometry: LineString, - routePoints: List + routePoints: List, + bufferRadiuses: Set ): MatchResult { - val matchingParams: PublicTransportRouteMatchingParameters = getMatchingParameters(50.0) - - val response: RoutingResponse = - matchingService.findMatchForPublicTransportRoute( - routeId, - geometry, - routePoints, - GENERIC_BUS, - matchingParams - ) + val sortedBufferRadiuses: List = bufferRadiuses.sorted() val lengthOfSourceRoute: Double = length(geometry) - return when (response) { - is RoutingResponse.RoutingSuccessDTO -> { + val lengthsOfMatchedRoutes: MutableList> = mutableListOf() + val unsuccessfulBufferRadiuses: MutableSet = mutableSetOf() + + sortedBufferRadiuses.forEach { radius -> + val matchingParams: PublicTransportRouteMatchingParameters = getMatchingParameters(radius) + + val response: RoutingResponse = + matchingService.findMatchForPublicTransportRoute( + routeId, + geometry, + routePoints, + GENERIC_BUS, + matchingParams + ) + + val bufferRadius = BufferRadius(radius) + + if (response is RoutingResponse.RoutingSuccessDTO) { + val resultGeometry: LineString = response.routes[0].geometry + val lengthOfMatchedRoute: Double = length(resultGeometry) + + lengthsOfMatchedRoutes.add(bufferRadius to lengthOfMatchedRoute) + } else { + unsuccessfulBufferRadiuses.add(bufferRadius) + } + } + + return when (lengthsOfMatchedRoutes.size) { + 0 -> RouteMatchFailure(routeId, geometry, lengthOfSourceRoute) + else -> SuccessfulRouteMatchResult( routeId, geometry, lengthOfSourceRoute, - createMatchDetails( - response, - geometry, - matchingParams.bufferRadiusInMeters + MatchDetails( + lengthsOfMatchedRoutes.toMap().toSortedMap(), + unsuccessfulBufferRadiuses ) ) - } - - else -> RouteMatchFailure(routeId, geometry, lengthOfSourceRoute) } } companion object { + private val DEFAULT_ROUTE_MATCH_RADIUS_VALUES: Set = setOf(55.0) + private val DEFAULT_SEGMENT_MATCH_RADIUS_VALUES: Set = setOf(40.0, 50.0, 90.0) + private fun getMatchingParameters(bufferRadius: Double): PublicTransportRouteMatchingParameters { val roadJunctionMatchingParams = JunctionMatchingParameters(matchDistance = 5.0, clearingDistance = 30.0) @@ -172,19 +206,5 @@ class MapMatchingBulkTester( roadJunctionMatching = roadJunctionMatchingParams ) } - - private fun createMatchDetails( - routingResponse: RoutingResponse.RoutingSuccessDTO, - sourceGeometry: LineString, - bufferRadius: Double - ): MatchDetails { - val resultGeometry: LineString = routingResponse.routes[0].geometry - - return MatchDetails( - mapOf( - BufferRadius(bufferRadius) to length(resultGeometry) - ).toSortedMap() - ) - } } } diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MatchResult.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MatchResult.kt index 5364f5d2..76726549 100644 --- a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MatchResult.kt +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MatchResult.kt @@ -16,6 +16,8 @@ sealed interface MatchResult { interface SuccessfulMatchResult : MatchResult { val details: MatchDetails + fun getLowestBufferRadius(): BufferRadius = details.getBufferRadiusOfFirstMatch() + fun getLengthOfFirstMatch(): Double = details.getLengthOfFirstMatch() fun getLengthOfClosestMatch(): Double = details.getLengthOfClosestMatch(sourceRouteLength) @@ -49,7 +51,8 @@ value class BufferRadius( } data class MatchDetails( - val lengthsOfMatchResults: SortedMap + val lengthsOfMatchResults: SortedMap, + val unsuccessfulBufferRadiuses: Set ) { init { require(lengthsOfMatchResults.isNotEmpty()) { "lengthsOfMatchResults must not be empty" } From 951ae7640cfea585f7528de7d57cd3caf6e9fdb3 Mon Sep 17 00:00:00 2001 From: Jarkko Kaura Date: Wed, 12 Oct 2022 20:53:49 +0300 Subject: [PATCH 04/15] Change the way how length of matched route is calculated while bulk-testing. --- .../matching/test/MapMatchingBulkTester.kt | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTester.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTester.kt index cb51c08d..7f35cd4d 100644 --- a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTester.kt +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTester.kt @@ -2,6 +2,8 @@ package fi.hsl.jore4.mapmatching.service.matching.test import fi.hsl.jore4.mapmatching.model.VehicleType.GENERIC_BUS import fi.hsl.jore4.mapmatching.model.matching.RoutePoint +import fi.hsl.jore4.mapmatching.service.common.response.LinkTraversalDTO +import fi.hsl.jore4.mapmatching.service.common.response.RouteResultDTO import fi.hsl.jore4.mapmatching.service.common.response.RoutingResponse import fi.hsl.jore4.mapmatching.service.matching.IMatchingService import fi.hsl.jore4.mapmatching.service.matching.PublicTransportRouteMatchingParameters @@ -165,8 +167,9 @@ class MapMatchingBulkTester( val bufferRadius = BufferRadius(radius) if (response is RoutingResponse.RoutingSuccessDTO) { - val resultGeometry: LineString = response.routes[0].geometry - val lengthOfMatchedRoute: Double = length(resultGeometry) + val route: RouteResultDTO = response.routes[0] + val lengthOfMatchedRoute: Double = calculateLengthOfRoute(route) + // val lengthOfMatchedRoute: Double = length(route.geometry) lengthsOfMatchedRoutes.add(bufferRadius to lengthOfMatchedRoute) } else { @@ -206,5 +209,22 @@ class MapMatchingBulkTester( roadJunctionMatching = roadJunctionMatchingParams ) } + + private fun calculateLengthOfRoute(route: RouteResultDTO): Double { + val linkTraversals: List = route.paths + + return when (linkTraversals.size) { + 0 -> 0.0 + 1, 2 -> route.weight + else -> { + val sumOfInterimLinkLengths: Double = + linkTraversals.drop(1).dropLast(1).map { it.distance }.fold(0.0) { acc, distance -> + acc + distance + } + + linkTraversals.first().weight + sumOfInterimLinkLengths + linkTraversals.last().weight + } + } + } } } From a9ef2b22e9d0fd95dbfd7e9a726dcf5dc0c2c022 Mon Sep 17 00:00:00 2001 From: Jarkko Kaura Date: Fri, 25 Mar 2022 12:16:49 +0200 Subject: [PATCH 05/15] Write failed routes and stop-to-stop segments as GeoJSON FeatureCollection to file. --- profiles/dev/config.properties | 3 ++ profiles/prod/config.properties | 3 ++ ...MapMatchingBulkTestResultsPublisherImpl.kt | 52 ++++++++++++++++++- src/main/resources/application.properties | 1 + 4 files changed, 57 insertions(+), 2 deletions(-) diff --git a/profiles/dev/config.properties b/profiles/dev/config.properties index e7f95a84..98887ee2 100644 --- a/profiles/dev/config.properties +++ b/profiles/dev/config.properties @@ -38,3 +38,6 @@ digitransit.subscription.key= # exists a "create-bulk-map-matching-input" branch containing a script that can be used to generate # the CSV file. test.routes.csvfile= + +# The output directory to which the resulting GeoJSON file is written. +test.output.dir= diff --git a/profiles/prod/config.properties b/profiles/prod/config.properties index f062330b..af5a4160 100644 --- a/profiles/prod/config.properties +++ b/profiles/prod/config.properties @@ -30,3 +30,6 @@ digitransit.subscription.key= # exists a "create-bulk-map-matching-input" branch containing a script that can be used to generate # the CSV file. test.routes.csvfile= + +# The output directory to which the resulting GeoJSON file is written. +test.output.dir= diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTestResultsPublisherImpl.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTestResultsPublisherImpl.kt index 7667f1b5..fb681d95 100644 --- a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTestResultsPublisherImpl.kt +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTestResultsPublisherImpl.kt @@ -1,9 +1,15 @@ package fi.hsl.jore4.mapmatching.service.matching.test +import com.fasterxml.jackson.databind.ObjectMapper import fi.hsl.jore4.mapmatching.util.LogUtils.joinToLogString import io.github.oshai.kotlinlogging.KotlinLogging +import org.geolatte.geom.G2D +import org.geolatte.geom.json.GeoJsonFeature +import org.geolatte.geom.json.GeoJsonFeatureCollection import org.nield.kotlinstatistics.standardDeviation +import org.springframework.beans.factory.annotation.Value import org.springframework.stereotype.Component +import java.io.File import java.util.SortedMap import kotlin.math.abs import kotlin.math.roundToInt @@ -11,7 +17,10 @@ import kotlin.math.roundToInt private val LOGGER = KotlinLogging.logger {} @Component -class MapMatchingBulkTestResultsPublisherImpl : IMapMatchingBulkTestResultsPublisher { +class MapMatchingBulkTestResultsPublisherImpl( + val objectMapper: ObjectMapper, + @Value("\${test.output.dir}") val outputDir: String +) : IMapMatchingBulkTestResultsPublisher { override fun publishMatchResultsForRoutesAndStopToStopSegments( routeResults: List, stopToStopSegmentResults: List @@ -64,6 +73,9 @@ class MapMatchingBulkTestResultsPublisherImpl : IMapMatchingBulkTestResultsPubli "Worst $limit successful route matches: $worstResults" } + + val outputFile: File = writeGeoJsonToFile(getFailedRoutesAsGeoJson(failed), "failed_routes.geojson") + LOGGER.info { "Wrote failed routes to file: ${outputFile.absolutePath}" } } fun publishStopToStopSegmentMatchResults(results: List) { @@ -96,6 +108,21 @@ class MapMatchingBulkTestResultsPublisherImpl : IMapMatchingBulkTestResultsPubli "Worst $limit successful stop-to-stop segment matches: $worstResults" } + + val outputFile: File = + writeGeoJsonToFile(getFailedSegmentsAsGeoJson(failed), "failed_stop-to-stop_segments.geojson") + LOGGER.info { "Wrote failed stop-to-stop segments to file: ${outputFile.absolutePath}" } + } + + private fun writeGeoJsonToFile( + features: GeoJsonFeatureCollection, + filename: String + ): File { + val geojson: String = objectMapper.writeValueAsString(features) + + val outputFile = File(outputDir, filename) + outputFile.writeText(geojson) + return outputFile } companion object { @@ -116,11 +143,32 @@ class MapMatchingBulkTestResultsPublisherImpl : IMapMatchingBulkTestResultsPubli val succeeded: List = results.mapNotNull { it as? SuccessfulSegmentMatchResult } - val failed: List = results.mapNotNull { it as? SegmentMatchFailure } + val failed: List = + results.mapNotNull { it as? SegmentMatchFailure } return succeeded to failed } + private fun getFailedRoutesAsGeoJson(failed: List): GeoJsonFeatureCollection = + GeoJsonFeatureCollection( + failed.map { + GeoJsonFeature(it.sourceRouteGeometry, it.routeId, emptyMap()) + } + ) + + private fun getFailedSegmentsAsGeoJson( + failed: List + ): GeoJsonFeatureCollection = + GeoJsonFeatureCollection( + failed.map { + GeoJsonFeature( + it.sourceRouteGeometry, + it.routeId, + mapOf("referencingRoutes" to it.referencingRoutes) + ) + } + ) + private fun getBestMatchResults( results: List, limit: Int diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 6aa3dc1a..8aff7694 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -15,3 +15,4 @@ spring.thymeleaf.cache=@spring.thymeleaf.cache@ digitransit.subscription.key=@digitransit.subscription.key@ test.routes.csvfile=@test.routes.csvfile@ +test.output.dir=@test.output.dir@ From a9a71be84092620207ab996e756037d67b7f4fad Mon Sep 17 00:00:00 2001 From: Jarkko Kaura Date: Mon, 21 Mar 2022 18:06:55 +0200 Subject: [PATCH 06/15] Create Geopackage file from failed stop-to-stop segments. --- pom.xml | 5 ++ .../service/matching/test/GeoPackageUtils.kt | 78 +++++++++++++++++++ ...MapMatchingBulkTestResultsPublisherImpl.kt | 7 ++ 3 files changed, 90 insertions(+) create mode 100644 src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/GeoPackageUtils.kt diff --git a/pom.xml b/pom.xml index 0e088a93..c3c552bd 100644 --- a/pom.xml +++ b/pom.xml @@ -602,6 +602,11 @@ gt-epsg-hsql ${geotools.version} + + org.geotools + gt-geopkg + ${geotools.version} + com.github.doyaaaaaken diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/GeoPackageUtils.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/GeoPackageUtils.kt new file mode 100644 index 00000000..1e7b7a3b --- /dev/null +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/GeoPackageUtils.kt @@ -0,0 +1,78 @@ +package fi.hsl.jore4.mapmatching.service.matching.test + +import org.geolatte.geom.jts.JTS +import org.geotools.api.feature.simple.SimpleFeature +import org.geotools.api.feature.simple.SimpleFeatureType +import org.geotools.data.collection.ListFeatureCollection +import org.geotools.feature.simple.SimpleFeatureBuilder +import org.geotools.feature.simple.SimpleFeatureTypeBuilder +import org.geotools.geopkg.FeatureEntry +import org.geotools.geopkg.GeoPackage +import org.geotools.referencing.crs.DefaultGeographicCRS +import org.locationtech.jts.geom.LineString +import java.io.File + +object GeoPackageUtils { + fun createGeoPackage( + file: File, + failedStopToStopSegments: List + ): GeoPackage { + val geoPkg = GeoPackage(file) + geoPkg.init() + + val featureType: SimpleFeatureType = createFeatureTypeForFailedStopToStopSegments() + + val entry = FeatureEntry() + entry.description = "Failed stop-to-stop segments" + geoPkg.add(entry, createFeatureCollection(failedStopToStopSegments, featureType)) + geoPkg.createSpatialIndex(entry) + + return geoPkg + } + + private fun createFeatureTypeForFailedStopToStopSegments(): SimpleFeatureType { + val builder = SimpleFeatureTypeBuilder() + builder.name = "FailedStopToStopSegment" + builder.crs = DefaultGeographicCRS.WGS84 + + builder.add("geometry", LineString::class.java) + builder.add("length", Double::class.java) + builder.add("startStopId", String::class.java) + builder.add("endStopId", String::class.java) + builder.add("numberOfGeometryPoints", Integer::class.java) + builder.add("numberOfRoutePoints", Integer::class.java) + builder.add("numberOfReferencingRoutes", Integer::class.java) + builder.add("referencingRoutes", String::class.java) + + return builder.buildFeatureType() + } + + private fun createFeatureCollection( + failedSegments: List, + featureType: SimpleFeatureType + ): ListFeatureCollection { + val features: List = failedSegments.map { createFeature(it, featureType) } + + return ListFeatureCollection(featureType, features) + } + + private fun createFeature( + failedSegment: SegmentMatchFailure, + type: SimpleFeatureType + ): SimpleFeature { + val builder = SimpleFeatureBuilder(type) + + failedSegment.run { + builder.add(JTS.to(sourceRouteGeometry)) + builder.add(sourceRouteLength) + builder.add(startStopId) + builder.add(endStopId) + builder.add(sourceRouteGeometry.numPositions) + builder.add(numberOfRoutePoints) + builder.add(referencingRoutes.size) + builder.add(referencingRoutes.joinToString(separator = ",")) + } + + return builder.buildFeature(failedSegment.routeId) + } +} diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTestResultsPublisherImpl.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTestResultsPublisherImpl.kt index fb681d95..89d1d9f5 100644 --- a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTestResultsPublisherImpl.kt +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTestResultsPublisherImpl.kt @@ -112,6 +112,8 @@ class MapMatchingBulkTestResultsPublisherImpl( val outputFile: File = writeGeoJsonToFile(getFailedSegmentsAsGeoJson(failed), "failed_stop-to-stop_segments.geojson") LOGGER.info { "Wrote failed stop-to-stop segments to file: ${outputFile.absolutePath}" } + + createGeoPackage(failed) } private fun writeGeoJsonToFile( @@ -338,5 +340,10 @@ class MapMatchingBulkTestResultsPublisherImpl( } } } + + private fun createGeoPackage(matchResults: List) { + val file = File.createTempFile("geopkg", "db", File("target")) + GeoPackageUtils.createGeoPackage(file, matchResults) + } } } From 870ffbd317af942dae7bc9dfe61ea1e4dcdab5d7 Mon Sep 17 00:00:00 2001 From: Jarkko Kaura Date: Fri, 8 Apr 2022 17:00:08 +0300 Subject: [PATCH 07/15] Make each failed stop-to-stop segment a separate layer in GeoPackage file. --- .../service/matching/test/GeoPackageUtils.kt | 46 ++++++++++++------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/GeoPackageUtils.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/GeoPackageUtils.kt index 1e7b7a3b..7274b459 100644 --- a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/GeoPackageUtils.kt +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/GeoPackageUtils.kt @@ -15,24 +15,36 @@ import java.io.File object GeoPackageUtils { fun createGeoPackage( file: File, - failedStopToStopSegments: List + failedSegments: List ): GeoPackage { val geoPkg = GeoPackage(file) geoPkg.init() - val featureType: SimpleFeatureType = createFeatureTypeForFailedStopToStopSegments() + try { + failedSegments.forEach { segment -> + val featureTypeName: String = segment.routeId + val featureType: SimpleFeatureType = createFeatureTypeForFailedSegment(featureTypeName) - val entry = FeatureEntry() - entry.description = "Failed stop-to-stop segments" - geoPkg.add(entry, createFeatureCollection(failedStopToStopSegments, featureType)) - geoPkg.createSpatialIndex(entry) + val entry = FeatureEntry() + entry.identifier = segment.routeId + entry.description = "Failed segment within map-matching" - return geoPkg + geoPkg.add(entry, createFeatureCollection(segment, featureType)) + + // Not sure, if this is really needed. + geoPkg.createSpatialIndex(entry) + } + + return geoPkg + } catch (ex: Exception) { + geoPkg.close() + throw ex + } } - private fun createFeatureTypeForFailedStopToStopSegments(): SimpleFeatureType { + private fun createFeatureTypeForFailedSegment(name: String): SimpleFeatureType { val builder = SimpleFeatureTypeBuilder() - builder.name = "FailedStopToStopSegment" + builder.name = name builder.crs = DefaultGeographicCRS.WGS84 builder.add("geometry", LineString::class.java) @@ -41,19 +53,19 @@ object GeoPackageUtils { builder.add("endStopId", String::class.java) builder.add("numberOfGeometryPoints", Integer::class.java) builder.add("numberOfRoutePoints", Integer::class.java) - builder.add("numberOfReferencingRoutes", Integer::class.java) - builder.add("referencingRoutes", String::class.java) + builder.add("numberOfRoutesPassingThrough", Integer::class.java) + builder.add("routesPassingThrough", String::class.java) return builder.buildFeatureType() } private fun createFeatureCollection( - failedSegments: List, + failedSegment: SegmentMatchFailure, featureType: SimpleFeatureType ): ListFeatureCollection { - val features: List = failedSegments.map { createFeature(it, featureType) } + val feature: SimpleFeature = createFeature(failedSegment, featureType) - return ListFeatureCollection(featureType, features) + return ListFeatureCollection(featureType, feature) } private fun createFeature( @@ -63,14 +75,16 @@ object GeoPackageUtils { val builder = SimpleFeatureBuilder(type) failedSegment.run { - builder.add(JTS.to(sourceRouteGeometry)) + val lineString: LineString = JTS.to(sourceRouteGeometry) + + builder.add(lineString) builder.add(sourceRouteLength) builder.add(startStopId) builder.add(endStopId) builder.add(sourceRouteGeometry.numPositions) builder.add(numberOfRoutePoints) builder.add(referencingRoutes.size) - builder.add(referencingRoutes.joinToString(separator = ",")) + builder.add(referencingRoutes.joinToString(separator = " ")) } return builder.buildFeature(failedSegment.routeId) From 7f0245e18eb1bf4a30ed4694593906cb015cb93d Mon Sep 17 00:00:00 2001 From: Jarkko Kaura Date: Mon, 11 Apr 2022 18:25:20 +0300 Subject: [PATCH 08/15] Add GeoPackage file for buffered failed stop-to-stop segments. --- profiles/dev/config.properties | 2 +- profiles/prod/config.properties | 2 +- .../service/matching/test/GeoPackageUtils.kt | 49 ++++++++++++++----- ...MapMatchingBulkTestResultsPublisherImpl.kt | 39 +++++++++++---- .../matching/test/MapMatchingBulkTester.kt | 9 +++- .../service/matching/test/MatchResult.kt | 4 +- 6 files changed, 80 insertions(+), 25 deletions(-) diff --git a/profiles/dev/config.properties b/profiles/dev/config.properties index 98887ee2..b87aa50b 100644 --- a/profiles/dev/config.properties +++ b/profiles/dev/config.properties @@ -39,5 +39,5 @@ digitransit.subscription.key= # the CSV file. test.routes.csvfile= -# The output directory to which the resulting GeoJSON file is written. +# The output directory to which the resulting GeoJSON and GeoPackage files are written. test.output.dir= diff --git a/profiles/prod/config.properties b/profiles/prod/config.properties index af5a4160..0b753121 100644 --- a/profiles/prod/config.properties +++ b/profiles/prod/config.properties @@ -31,5 +31,5 @@ digitransit.subscription.key= # the CSV file. test.routes.csvfile= -# The output directory to which the resulting GeoJSON file is written. +# The output directory to which the resulting GeoJSON and GeoPackage files are written. test.output.dir= diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/GeoPackageUtils.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/GeoPackageUtils.kt index 7274b459..a20c211f 100644 --- a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/GeoPackageUtils.kt +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/GeoPackageUtils.kt @@ -1,21 +1,27 @@ package fi.hsl.jore4.mapmatching.service.matching.test +import fi.hsl.jore4.mapmatching.util.GeoToolsUtils.transformCRS import org.geolatte.geom.jts.JTS import org.geotools.api.feature.simple.SimpleFeature import org.geotools.api.feature.simple.SimpleFeatureType import org.geotools.data.collection.ListFeatureCollection import org.geotools.feature.simple.SimpleFeatureBuilder import org.geotools.feature.simple.SimpleFeatureTypeBuilder +import org.geotools.geometry.jts.Geometries import org.geotools.geopkg.FeatureEntry import org.geotools.geopkg.GeoPackage import org.geotools.referencing.crs.DefaultGeographicCRS +import org.locationtech.jts.geom.Geometry import org.locationtech.jts.geom.LineString import java.io.File object GeoPackageUtils { + private const val GEOMETRY_COLUMN_NAME = "geometry" + fun createGeoPackage( file: File, - failedSegments: List + failedSegments: List, + isBufferPolygonInsteadOfLineString: Boolean ): GeoPackage { val geoPkg = GeoPackage(file) geoPkg.init() @@ -23,13 +29,19 @@ object GeoPackageUtils { try { failedSegments.forEach { segment -> val featureTypeName: String = segment.routeId - val featureType: SimpleFeatureType = createFeatureTypeForFailedSegment(featureTypeName) + val featureType: SimpleFeatureType = + createFeatureTypeForFailedSegment(featureTypeName, isBufferPolygonInsteadOfLineString) val entry = FeatureEntry() entry.identifier = segment.routeId - entry.description = "Failed segment within map-matching" + entry.description = + if (isBufferPolygonInsteadOfLineString) { + "Buffered geometry for failed ${segment.routeId} segment within map-matching" + } else { + "LineString for failed ${segment.routeId} segment within map-matching" + } - geoPkg.add(entry, createFeatureCollection(segment, featureType)) + geoPkg.add(entry, createFeatureCollection(segment, featureType, isBufferPolygonInsteadOfLineString)) // Not sure, if this is really needed. geoPkg.createSpatialIndex(entry) @@ -42,12 +54,16 @@ object GeoPackageUtils { } } - private fun createFeatureTypeForFailedSegment(name: String): SimpleFeatureType { + private fun createFeatureTypeForFailedSegment( + name: String, + isBufferPolygonInsteadOfLineString: Boolean + ): SimpleFeatureType { val builder = SimpleFeatureTypeBuilder() builder.name = name - builder.crs = DefaultGeographicCRS.WGS84 - builder.add("geometry", LineString::class.java) + val geomType = if (isBufferPolygonInsteadOfLineString) Geometries.POLYGON else Geometries.LINESTRING + + builder.add(GEOMETRY_COLUMN_NAME, geomType.binding, DefaultGeographicCRS.WGS84) builder.add("length", Double::class.java) builder.add("startStopId", String::class.java) builder.add("endStopId", String::class.java) @@ -61,23 +77,34 @@ object GeoPackageUtils { private fun createFeatureCollection( failedSegment: SegmentMatchFailure, - featureType: SimpleFeatureType + featureType: SimpleFeatureType, + isBufferPolygonInsteadOfLineString: Boolean ): ListFeatureCollection { - val feature: SimpleFeature = createFeature(failedSegment, featureType) + val feature: SimpleFeature = createFeature(failedSegment, featureType, isBufferPolygonInsteadOfLineString) return ListFeatureCollection(featureType, feature) } private fun createFeature( failedSegment: SegmentMatchFailure, - type: SimpleFeatureType + type: SimpleFeatureType, + isBufferPolygonInsteadOfLineString: Boolean ): SimpleFeature { val builder = SimpleFeatureBuilder(type) failedSegment.run { val lineString: LineString = JTS.to(sourceRouteGeometry) - builder.add(lineString) + if (isBufferPolygonInsteadOfLineString) { + val lineString3067: Geometry = transformCRS(lineString, 4326, 3067) + val bufferPolygon3067: Geometry = lineString3067.buffer(bufferRadius.value) + val bufferPolygon: Geometry = transformCRS(bufferPolygon3067, 3067, 4326) + + builder.add(bufferPolygon) + } else { + builder.add(lineString) + } + builder.add(sourceRouteLength) builder.add(startStopId) builder.add(endStopId) diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTestResultsPublisherImpl.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTestResultsPublisherImpl.kt index 89d1d9f5..8efdf9dd 100644 --- a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTestResultsPublisherImpl.kt +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTestResultsPublisherImpl.kt @@ -74,7 +74,7 @@ class MapMatchingBulkTestResultsPublisherImpl( "Worst $limit successful route matches: $worstResults" } - val outputFile: File = writeGeoJsonToFile(getFailedRoutesAsGeoJson(failed), "failed_routes.geojson") + val outputFile: File = writeGeoJsonToFile(getFailedRoutesAsGeoJson(failed), FILENAME_FAILED_ROUTES_GEOJSON) LOGGER.info { "Wrote failed routes to file: ${outputFile.absolutePath}" } } @@ -109,11 +109,29 @@ class MapMatchingBulkTestResultsPublisherImpl( "Worst $limit successful stop-to-stop segment matches: $worstResults" } - val outputFile: File = - writeGeoJsonToFile(getFailedSegmentsAsGeoJson(failed), "failed_stop-to-stop_segments.geojson") - LOGGER.info { "Wrote failed stop-to-stop segments to file: ${outputFile.absolutePath}" } + val geojsonFile: File = + writeGeoJsonToFile(getFailedSegmentsAsGeoJson(failed), FILENAME_FAILED_SEGMENTS_GEOJSON) + LOGGER.info { "Wrote failed stop-to-stop segments to GeoJSON file: ${geojsonFile.absolutePath}" } - createGeoPackage(failed) + val failedSegmentsGeoPackageFile = File(outputDir, FILENAME_FAILED_SEGMENTS_GPKG) + failedSegmentsGeoPackageFile.delete() + GeoPackageUtils.createGeoPackage(failedSegmentsGeoPackageFile, failed, false) + + LOGGER.info { + "Wrote failed stop-to-stop segments to GeoPackage file: ${ + failedSegmentsGeoPackageFile.absolutePath + }" + } + + val failureBuffersGeoPackageFile = File(outputDir, FILENAME_FAILED_SEGMENT_BUFFERS_GPKG) + failureBuffersGeoPackageFile.delete() + GeoPackageUtils.createGeoPackage(failureBuffersGeoPackageFile, failed, true) + + LOGGER.info { + "Wrote failed stop-to-stop segment buffers to GeoPackage file: ${ + failureBuffersGeoPackageFile.absolutePath + }" + } } private fun writeGeoJsonToFile( @@ -128,6 +146,12 @@ class MapMatchingBulkTestResultsPublisherImpl( } companion object { + private const val FILENAME_FAILED_ROUTES_GEOJSON = "failed_routes.geojson" + private const val FILENAME_FAILED_SEGMENTS_GEOJSON = "failed_segments.geojson" + + private const val FILENAME_FAILED_SEGMENTS_GPKG = "failed_segments.gpkg" + private const val FILENAME_FAILED_SEGMENT_BUFFERS_GPKG = "failed_segment_buffers.gpkg" + private fun partitionBySuccess( results: List ): Pair, List> { @@ -340,10 +364,5 @@ class MapMatchingBulkTestResultsPublisherImpl( } } } - - private fun createGeoPackage(matchResults: List) { - val file = File.createTempFile("geopkg", "db", File("target")) - GeoPackageUtils.createGeoPackage(file, matchResults) - } } } diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTester.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTester.kt index 7f35cd4d..10a1526e 100644 --- a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTester.kt +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTester.kt @@ -129,6 +129,7 @@ class MapMatchingBulkTester( segmentId, geometry, result.sourceRouteLength, + result.bufferRadius, segment.startStopId, segment.endStopId, numRoutePoints, @@ -178,7 +179,13 @@ class MapMatchingBulkTester( } return when (lengthsOfMatchedRoutes.size) { - 0 -> RouteMatchFailure(routeId, geometry, lengthOfSourceRoute) + 0 -> + RouteMatchFailure( + routeId, + geometry, + lengthOfSourceRoute, + BufferRadius(sortedBufferRadiuses.last()) + ) else -> SuccessfulRouteMatchResult( routeId, diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MatchResult.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MatchResult.kt index 76726549..be7b33fb 100644 --- a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MatchResult.kt +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MatchResult.kt @@ -91,7 +91,8 @@ data class SuccessfulRouteMatchResult( data class RouteMatchFailure( override val routeId: String, override val sourceRouteGeometry: LineString, - override val sourceRouteLength: Double + override val sourceRouteLength: Double, + val bufferRadius: BufferRadius ) : MatchResult { override val matchFound = false } @@ -114,6 +115,7 @@ data class SegmentMatchFailure( override val routeId: String, override val sourceRouteGeometry: LineString, override val sourceRouteLength: Double, + val bufferRadius: BufferRadius, override val startStopId: String, override val endStopId: String, override val numberOfRoutePoints: Int, From 540abb1e8e6cd81db9c401aba6b9154e21467bcb Mon Sep 17 00:00:00 2001 From: Jarkko Kaura Date: Fri, 22 Apr 2022 16:58:37 +0300 Subject: [PATCH 09/15] Write GeoPackage files for also segment match failures tried with lower buffer radius values. Also make backups of existing GeoPackage files so that you don't accidentally lose results from previous runs. --- ...MapMatchingBulkTestResultsPublisherImpl.kt | 123 +++++++++++++++--- 1 file changed, 106 insertions(+), 17 deletions(-) diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTestResultsPublisherImpl.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTestResultsPublisherImpl.kt index 8efdf9dd..edeb48ab 100644 --- a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTestResultsPublisherImpl.kt +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTestResultsPublisherImpl.kt @@ -113,8 +113,54 @@ class MapMatchingBulkTestResultsPublisherImpl( writeGeoJsonToFile(getFailedSegmentsAsGeoJson(failed), FILENAME_FAILED_SEGMENTS_GEOJSON) LOGGER.info { "Wrote failed stop-to-stop segments to GeoJSON file: ${geojsonFile.absolutePath}" } - val failedSegmentsGeoPackageFile = File(outputDir, FILENAME_FAILED_SEGMENTS_GPKG) - failedSegmentsGeoPackageFile.delete() + writeGeoPackageFilesForFailedSegments(failed, getSegmentMatchFailuresOnLowerBufferRadius(succeeded)) + } + + private fun writeGeoJsonToFile( + features: GeoJsonFeatureCollection, + filename: String + ): File { + val geojson: String = objectMapper.writeValueAsString(features) + + val outputFile = File(outputDir, filename) + outputFile.writeText(geojson) + return outputFile + } + + private fun writeGeoPackageFilesForFailedSegments( + primaryFailures: List, + secondaryFailures: SortedMap> + ) { + writeGeoPackageFileForFailedSegments( + primaryFailures, + getGeoPackageFilenameForFailedSegments(), + getGeoPackageFilenameForFailedSegmentBuffers() + ) + + secondaryFailures.entries.forEach { (bufferRadius, segmentMatchFailures) -> + + if (segmentMatchFailures.isNotEmpty()) { + writeGeoPackageFileForFailedSegments( + segmentMatchFailures, + getGeoPackageFilenameForFailedSegments(bufferRadius), + getGeoPackageFilenameForFailedSegmentBuffers(bufferRadius) + ) + } + } + } + + private fun writeGeoPackageFileForFailedSegments( + failed: List, + segmentsFilename: String, + segmentBuffersFilename: String + ) { + var failedSegmentsGeoPackageFile = File(outputDir, segmentsFilename) + + if (failedSegmentsGeoPackageFile.exists()) { + failedSegmentsGeoPackageFile.renameTo(File(outputDir, "$segmentsFilename.bak")) + failedSegmentsGeoPackageFile = File(outputDir, segmentsFilename) + } + GeoPackageUtils.createGeoPackage(failedSegmentsGeoPackageFile, failed, false) LOGGER.info { @@ -123,8 +169,12 @@ class MapMatchingBulkTestResultsPublisherImpl( }" } - val failureBuffersGeoPackageFile = File(outputDir, FILENAME_FAILED_SEGMENT_BUFFERS_GPKG) - failureBuffersGeoPackageFile.delete() + var failureBuffersGeoPackageFile = File(outputDir, segmentBuffersFilename) + if (failureBuffersGeoPackageFile.exists()) { + failureBuffersGeoPackageFile.renameTo(File(outputDir, "$segmentBuffersFilename.bak")) + failureBuffersGeoPackageFile = File(outputDir, segmentBuffersFilename) + } + GeoPackageUtils.createGeoPackage(failureBuffersGeoPackageFile, failed, true) LOGGER.info { @@ -134,23 +184,19 @@ class MapMatchingBulkTestResultsPublisherImpl( } } - private fun writeGeoJsonToFile( - features: GeoJsonFeatureCollection, - filename: String - ): File { - val geojson: String = objectMapper.writeValueAsString(features) - - val outputFile = File(outputDir, filename) - outputFile.writeText(geojson) - return outputFile - } - companion object { private const val FILENAME_FAILED_ROUTES_GEOJSON = "failed_routes.geojson" private const val FILENAME_FAILED_SEGMENTS_GEOJSON = "failed_segments.geojson" - private const val FILENAME_FAILED_SEGMENTS_GPKG = "failed_segments.gpkg" - private const val FILENAME_FAILED_SEGMENT_BUFFERS_GPKG = "failed_segment_buffers.gpkg" + private fun getGeoPackageFilenameForFailedSegments(bufferRadius: BufferRadius? = null): String = + bufferRadius + ?.let { "failed_segments_${it.value}.gpkg" } + ?: "failed_segments.gpkg" + + private fun getGeoPackageFilenameForFailedSegmentBuffers(bufferRadius: BufferRadius? = null): String = + bufferRadius + ?.let { "failed_segment_buffers_${it.value}.gpkg" } + ?: "failed_segment_buffers.gpkg" private fun partitionBySuccess( results: List @@ -222,6 +268,49 @@ class MapMatchingBulkTestResultsPublisherImpl( private fun roundTo2Digits(n: Double): Double = (n * 100).roundToInt() / 100.0 + private fun getSegmentMatchFailuresOnLowerBufferRadius( + succeeded: List + ): SortedMap> { + val allUnsuccessfulBufferRadiuses: MutableSet = mutableSetOf() + + succeeded.forEach { + allUnsuccessfulBufferRadiuses.addAll(it.details.unsuccessfulBufferRadiuses) + } + + val alreadyProcessedSegmentIds: MutableSet = mutableSetOf() + val failedSegmentsByRadius: MutableMap> = mutableMapOf() + + allUnsuccessfulBufferRadiuses + .sortedDescending() + .forEach { bufferRadius -> + + succeeded + .filter { bufferRadius in it.details.unsuccessfulBufferRadiuses } + .filter { it.routeId !in alreadyProcessedSegmentIds } + .forEach { matchResult -> + + failedSegmentsByRadius + .getOrPut(bufferRadius) { mutableListOf() } + .add( + SegmentMatchFailure( + matchResult.routeId, + matchResult.sourceRouteGeometry, + matchResult.sourceRouteLength, + bufferRadius, + matchResult.startStopId, + matchResult.endStopId, + matchResult.numberOfRoutePoints, + matchResult.referencingRoutes + ) + ) + + alreadyProcessedSegmentIds.add(matchResult.routeId) + } + } + + return failedSegmentsByRadius.toSortedMap() + } + private fun getRoutesNotMatchedEvenThoughAllSegmentsMatched( routeResults: List, stopToStopSegmentResults: List From f9a9939bfa72a6fc61de666031f661e05ce02e9c Mon Sep 17 00:00:00 2001 From: Jarkko Kaura Date: Mon, 5 May 2025 13:08:47 +0300 Subject: [PATCH 10/15] Change the way GeoPackage files for failed stop-to-stop segments are named. Include the buffer radius in the filenames generated by testing the largest radius. Also, make writing segment failures to GeoPackage files a bit more robust. --- .../IMapMatchingBulkTestResultsPublisher.kt | 3 +- ...MapMatchingBulkTestResultsPublisherImpl.kt | 45 +++++++++++-------- .../matching/test/MapMatchingBulkTester.kt | 5 ++- 3 files changed, 33 insertions(+), 20 deletions(-) diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/IMapMatchingBulkTestResultsPublisher.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/IMapMatchingBulkTestResultsPublisher.kt index 5c465374..da414059 100644 --- a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/IMapMatchingBulkTestResultsPublisher.kt +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/IMapMatchingBulkTestResultsPublisher.kt @@ -3,6 +3,7 @@ package fi.hsl.jore4.mapmatching.service.matching.test interface IMapMatchingBulkTestResultsPublisher { fun publishMatchResultsForRoutesAndStopToStopSegments( routeResults: List, - stopToStopSegmentResults: List + stopToStopSegmentResults: List, + largestBufferRadiusTried: BufferRadius ) } diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTestResultsPublisherImpl.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTestResultsPublisherImpl.kt index edeb48ab..d69744e8 100644 --- a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTestResultsPublisherImpl.kt +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTestResultsPublisherImpl.kt @@ -23,10 +23,11 @@ class MapMatchingBulkTestResultsPublisherImpl( ) : IMapMatchingBulkTestResultsPublisher { override fun publishMatchResultsForRoutesAndStopToStopSegments( routeResults: List, - stopToStopSegmentResults: List + stopToStopSegmentResults: List, + largestBufferRadiusTried: BufferRadius ) { publishRouteMatchResults(routeResults) - publishStopToStopSegmentMatchResults(stopToStopSegmentResults) + publishStopToStopSegmentMatchResults(stopToStopSegmentResults, largestBufferRadiusTried) LOGGER.info { "List of IDs of failed routes whose all segments were matched: ${ @@ -78,7 +79,10 @@ class MapMatchingBulkTestResultsPublisherImpl( LOGGER.info { "Wrote failed routes to file: ${outputFile.absolutePath}" } } - fun publishStopToStopSegmentMatchResults(results: List) { + fun publishStopToStopSegmentMatchResults( + results: List, + largestBufferRadiusTried: BufferRadius + ) { val (succeeded, failed) = partitionSegmentsBySuccess(results) printBasicStatistics(succeeded, failed, "Stop-to-stop segment") @@ -113,7 +117,11 @@ class MapMatchingBulkTestResultsPublisherImpl( writeGeoJsonToFile(getFailedSegmentsAsGeoJson(failed), FILENAME_FAILED_SEGMENTS_GEOJSON) LOGGER.info { "Wrote failed stop-to-stop segments to GeoJSON file: ${geojsonFile.absolutePath}" } - writeGeoPackageFilesForFailedSegments(failed, getSegmentMatchFailuresOnLowerBufferRadius(succeeded)) + writeGeoPackageFilesForFailedSegments( + failed, + largestBufferRadiusTried, + getSegmentMatchFailuresOnLowerBufferRadius(succeeded) + ) } private fun writeGeoJsonToFile( @@ -129,15 +137,20 @@ class MapMatchingBulkTestResultsPublisherImpl( private fun writeGeoPackageFilesForFailedSegments( primaryFailures: List, + largestBufferRadiusTried: BufferRadius, secondaryFailures: SortedMap> ) { - writeGeoPackageFileForFailedSegments( - primaryFailures, - getGeoPackageFilenameForFailedSegments(), - getGeoPackageFilenameForFailedSegmentBuffers() - ) + val segmentFailureMap = secondaryFailures.toMutableMap() + + // By design, the "secondaryFailures" map should not contain the largest radius tried as + // a key, but in case of a hidden bug, we consider the situation where this assumption is + // not true. + val segmentMatchFailuresOnLargestRadius: List = + primaryFailures + segmentFailureMap.getOrPut(largestBufferRadiusTried) { emptyList() } + + segmentFailureMap.put(largestBufferRadiusTried, segmentMatchFailuresOnLargestRadius) - secondaryFailures.entries.forEach { (bufferRadius, segmentMatchFailures) -> + segmentFailureMap.entries.forEach { (bufferRadius, segmentMatchFailures) -> if (segmentMatchFailures.isNotEmpty()) { writeGeoPackageFileForFailedSegments( @@ -188,15 +201,11 @@ class MapMatchingBulkTestResultsPublisherImpl( private const val FILENAME_FAILED_ROUTES_GEOJSON = "failed_routes.geojson" private const val FILENAME_FAILED_SEGMENTS_GEOJSON = "failed_segments.geojson" - private fun getGeoPackageFilenameForFailedSegments(bufferRadius: BufferRadius? = null): String = - bufferRadius - ?.let { "failed_segments_${it.value}.gpkg" } - ?: "failed_segments.gpkg" + private fun getGeoPackageFilenameForFailedSegments(bufferRadius: BufferRadius): String = + "failed_segments_${bufferRadius.value}.gpkg" - private fun getGeoPackageFilenameForFailedSegmentBuffers(bufferRadius: BufferRadius? = null): String = - bufferRadius - ?.let { "failed_segment_buffers_${it.value}.gpkg" } - ?: "failed_segment_buffers.gpkg" + private fun getGeoPackageFilenameForFailedSegmentBuffers(bufferRadius: BufferRadius): String = + "failed_segment_buffers_${bufferRadius.value}.gpkg" private fun partitionBySuccess( results: List diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTester.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTester.kt index 10a1526e..dc4d5def 100644 --- a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTester.kt +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTester.kt @@ -37,9 +37,12 @@ class MapMatchingBulkTester( measureTime { val (routeMatchResults, stopToStopSegmentMatchResults) = processFile() + val largestBufferRadiusTried = BufferRadius(DEFAULT_SEGMENT_MATCH_RADIUS_VALUES.max()) + resultsPublisher.publishMatchResultsForRoutesAndStopToStopSegments( routeMatchResults, - stopToStopSegmentMatchResults + stopToStopSegmentMatchResults, + largestBufferRadiusTried ) } From de954378cf43d63f2d2056fbfa1a6d93092ca3bc Mon Sep 17 00:00:00 2001 From: Jarkko Kaura Date: Tue, 29 Apr 2025 10:58:29 +0300 Subject: [PATCH 11/15] Fix writing failed segments. Let's make a fixed version of the `GeoPackage.add()` method, which originally worked, but at some point broke when the GeoTools libraries were updated. The current version of the method throws a `NullPointException` exception for a missing `fid` property. Debugging the `gt-geopkg` library has revealed that it probably has a bug. The fact that the documentation is notoriously incomplete has not helped in solving the problem. --- .../service/matching/test/GeoPackageUtils.kt | 60 ++++++++++++++++++- 1 file changed, 57 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/GeoPackageUtils.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/GeoPackageUtils.kt index a20c211f..e38a205a 100644 --- a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/GeoPackageUtils.kt +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/GeoPackageUtils.kt @@ -2,9 +2,13 @@ package fi.hsl.jore4.mapmatching.service.matching.test import fi.hsl.jore4.mapmatching.util.GeoToolsUtils.transformCRS import org.geolatte.geom.jts.JTS +import org.geotools.api.data.SimpleFeatureWriter +import org.geotools.api.data.Transaction import org.geotools.api.feature.simple.SimpleFeature import org.geotools.api.feature.simple.SimpleFeatureType +import org.geotools.data.DefaultTransaction import org.geotools.data.collection.ListFeatureCollection +import org.geotools.data.simple.SimpleFeatureIterator import org.geotools.feature.simple.SimpleFeatureBuilder import org.geotools.feature.simple.SimpleFeatureTypeBuilder import org.geotools.geometry.jts.Geometries @@ -14,6 +18,7 @@ import org.geotools.referencing.crs.DefaultGeographicCRS import org.locationtech.jts.geom.Geometry import org.locationtech.jts.geom.LineString import java.io.File +import java.io.IOException object GeoPackageUtils { private const val GEOMETRY_COLUMN_NAME = "geometry" @@ -32,8 +37,16 @@ object GeoPackageUtils { val featureType: SimpleFeatureType = createFeatureTypeForFailedSegment(featureTypeName, isBufferPolygonInsteadOfLineString) + val featureCollection: ListFeatureCollection = + createFeatureCollection(segment, featureType, isBufferPolygonInsteadOfLineString) + val entry = FeatureEntry() - entry.identifier = segment.routeId + // entry.identifier = segment.routeId + entry.geometryColumn = GEOMETRY_COLUMN_NAME + entry.geometryType = + if (isBufferPolygonInsteadOfLineString) Geometries.POLYGON else Geometries.LINESTRING + entry.bounds = featureCollection.bounds + entry.description = if (isBufferPolygonInsteadOfLineString) { "Buffered geometry for failed ${segment.routeId} segment within map-matching" @@ -41,7 +54,7 @@ object GeoPackageUtils { "LineString for failed ${segment.routeId} segment within map-matching" } - geoPkg.add(entry, createFeatureCollection(segment, featureType, isBufferPolygonInsteadOfLineString)) + writeEntryToGeoPackage(geoPkg, entry, featureCollection) // Not sure, if this is really needed. geoPkg.createSpatialIndex(entry) @@ -114,6 +127,47 @@ object GeoPackageUtils { builder.add(referencingRoutes.joinToString(separator = " ")) } - return builder.buildFeature(failedSegment.routeId) + // This works because every feature is put into a separate table. + builder.featureUserData("fid", "1") + + return builder.buildFeature(null) + } + + private fun writeEntryToGeoPackage( + geoPkg: GeoPackage, + entry: FeatureEntry, + featureCollection: ListFeatureCollection + ) { + // Create an SQLite table for FeatureCollection. + geoPkg.create(entry, featureCollection.schema) + + val tx: Transaction = DefaultTransaction() + val it: SimpleFeatureIterator = featureCollection.features() + + try { + val writer: SimpleFeatureWriter = geoPkg.writer(entry, true, null, tx) + + val source = it.next() + val target: SimpleFeature = writer.next() + + target.attributes = source.attributes + + // This if-block is the missing piece not done in the GeoPackage.add() method. + if (source.hasUserData()) { + target.userData.putAll(source.userData) + } + + writer.write() + writer.close() + + tx.commit() + } catch (ex: Exception) { + tx.rollback() + + throw ex as? IOException ?: IOException(ex) + } finally { + tx.close() + it.close() + } } } From c786b3c7b0aba7c060abe464b5df949d8c318a5b Mon Sep 17 00:00:00 2001 From: Jarkko Kaura Date: Sat, 8 Oct 2022 17:02:29 +0300 Subject: [PATCH 12/15] Add more logging while performing bulk testing. --- .../MatchRouteViaPointsOnLinksServiceImpl.kt | 21 ++++++-- .../matching/test/MapMatchingBulkTester.kt | 50 +++++++++++++------ .../service/matching/test/MatchResult.kt | 3 +- 3 files changed, 54 insertions(+), 20 deletions(-) diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/MatchRouteViaPointsOnLinksServiceImpl.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/MatchRouteViaPointsOnLinksServiceImpl.kt index 034857fa..ed71b80c 100644 --- a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/MatchRouteViaPointsOnLinksServiceImpl.kt +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/MatchRouteViaPointsOnLinksServiceImpl.kt @@ -191,7 +191,9 @@ class MatchRouteViaPointsOnLinksServiceImpl( targetRoutePointSequenceCandidates: List>, bufferRadiusInMeters: Double ): List? = - targetRoutePointSequenceCandidates.firstNotNullOfOrNull { targetRoutePoints -> + targetRoutePointSequenceCandidates.withIndex().firstNotNullOfOrNull { (index, targetRoutePoints) -> + + val round = index + 1 val bufferAreaRestriction = BufferAreaRestriction.from( @@ -201,8 +203,19 @@ class MatchRouteViaPointsOnLinksServiceImpl( targetRoutePoints.last() ) - routingService - .findRouteViaPointsOnLinks(targetRoutePoints, vehicleType, true, bufferAreaRestriction) - .ifEmpty { null } + val routeLinks: List = + routingService.findRouteViaPointsOnLinks( + targetRoutePoints, + vehicleType, + true, + bufferAreaRestriction + ) + + if (routeLinks.isNotEmpty()) { + if (round > 1) LOGGER.info { "Matched route on attempt #$round." } + routeLinks + } else { + null + } } } diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTester.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTester.kt index dc4d5def..f5b9dc4b 100644 --- a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTester.kt +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MapMatchingBulkTester.kt @@ -80,12 +80,20 @@ class MapMatchingBulkTester( val result: MatchResult = matchRoute(routeId, routeGeometry, routePoints, bufferRadiuses) - if (result is SuccessfulMatchResult) { - LOGGER.info { - "Successfully matched route: $routeId (with bufferRadius=${result.getLowestBufferRadius()})" - } - } else { - LOGGER.info { "Failed to match route: $routeId" } + when (result) { + is SuccessfulMatchResult -> + LOGGER.info { + "Successfully matched route: $routeId (with bufferRadius=${result.getLowestBufferRadius()})" + } + + is RouteMatchFailure -> + LOGGER.info { + "Failed to match route: $routeId (${ + result.errorMessage ?: "" + })" + } + + else -> throw IllegalStateException("Unknown route match result type") } result @@ -156,7 +164,9 @@ class MapMatchingBulkTester( val lengthsOfMatchedRoutes: MutableList> = mutableListOf() val unsuccessfulBufferRadiuses: MutableSet = mutableSetOf() - sortedBufferRadiuses.forEach { radius -> + var errorMessage: String? = null + + sortedBufferRadiuses.withIndex().forEach { (roundIndex, radius) -> val matchingParams: PublicTransportRouteMatchingParameters = getMatchingParameters(radius) val response: RoutingResponse = @@ -170,14 +180,22 @@ class MapMatchingBulkTester( val bufferRadius = BufferRadius(radius) - if (response is RoutingResponse.RoutingSuccessDTO) { - val route: RouteResultDTO = response.routes[0] - val lengthOfMatchedRoute: Double = calculateLengthOfRoute(route) - // val lengthOfMatchedRoute: Double = length(route.geometry) + when (response) { + is RoutingResponse.RoutingSuccessDTO -> { + val route: RouteResultDTO = response.routes[0] + val lengthOfMatchedRoute: Double = calculateLengthOfRoute(route) + // val lengthOfMatchedRoute: Double = length(route.geometry) - lengthsOfMatchedRoutes.add(bufferRadius to lengthOfMatchedRoute) - } else { - unsuccessfulBufferRadiuses.add(bufferRadius) + lengthsOfMatchedRoutes.add(bufferRadius to lengthOfMatchedRoute) + } + + is RoutingResponse.RoutingFailureDTO -> { + unsuccessfulBufferRadiuses.add(bufferRadius) + + if (roundIndex == 0) { + errorMessage = response.message + } + } } } @@ -187,8 +205,10 @@ class MapMatchingBulkTester( routeId, geometry, lengthOfSourceRoute, - BufferRadius(sortedBufferRadiuses.last()) + BufferRadius(sortedBufferRadiuses.last()), + errorMessage ) + else -> SuccessfulRouteMatchResult( routeId, diff --git a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MatchResult.kt b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MatchResult.kt index be7b33fb..446a3548 100644 --- a/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MatchResult.kt +++ b/src/main/kotlin/fi/hsl/jore4/mapmatching/service/matching/test/MatchResult.kt @@ -92,7 +92,8 @@ data class RouteMatchFailure( override val routeId: String, override val sourceRouteGeometry: LineString, override val sourceRouteLength: Double, - val bufferRadius: BufferRadius + val bufferRadius: BufferRadius, + val errorMessage: String? ) : MatchResult { override val matchFound = false } From 416ad9de2108f0911300ae7385bc48d2bd9f1872 Mon Sep 17 00:00:00 2001 From: Jarkko Kaura Date: Fri, 25 Apr 2025 16:27:50 +0300 Subject: [PATCH 13/15] Use Digiroad 2025/02 vanilla data. --- docker/docker-compose.custom.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/docker-compose.custom.yml b/docker/docker-compose.custom.yml index b7458456..c326a765 100644 --- a/docker/docker-compose.custom.yml +++ b/docker/docker-compose.custom.yml @@ -32,4 +32,5 @@ services: file: docker-compose.base.yml service: jore4-mapmatchingdb-base container_name: mapmatchingdb - image: "hsldevcom/jore4-postgres:mapmatching-main--20250425-c07c8632baab554dfbbbb64177ed930282094232" + environment: + DIGIROAD_ROUTING_DUMP_URL: "https://stjore4dev001.blob.core.windows.net/jore4-digiroad/2025-08-13_create_routing_schema_digiroad_r_2025_02.sql" From e90df4154d88f06e65e6015048d858f8225a7511 Mon Sep 17 00:00:00 2001 From: Jarkko Kaura Date: Tue, 19 Aug 2025 11:06:52 +0300 Subject: [PATCH 14/15] Use fixup data created on 2025-08-29. The data is based on the Digiroad 2025/02 release. --- docker/docker-compose.custom.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-compose.custom.yml b/docker/docker-compose.custom.yml index c326a765..bd4eca43 100644 --- a/docker/docker-compose.custom.yml +++ b/docker/docker-compose.custom.yml @@ -33,4 +33,4 @@ services: service: jore4-mapmatchingdb-base container_name: mapmatchingdb environment: - DIGIROAD_ROUTING_DUMP_URL: "https://stjore4dev001.blob.core.windows.net/jore4-digiroad/2025-08-13_create_routing_schema_digiroad_r_2025_02.sql" + DIGIROAD_ROUTING_DUMP_URL: "https://stjore4dev001.blob.core.windows.net/jore4-digiroad/2025-08-29_create_routing_schema_digiroad_r_2025_02_fixup.sql" From a946a4323ea69d2ffebd4e806ed10a7e86e40b7f Mon Sep 17 00:00:00 2001 From: Janne Bergman Date: Mon, 17 Nov 2025 15:26:05 +0200 Subject: [PATCH 15/15] Use Digiroad 2025/02 link IDs in tests. Fix one test. Test updated due to link's (88bd12da-4e71-4e32-95f8-f9ee8c276c95:1) digitation direction change in source material --- ...ce_FindMatchForPublicTransportRouteTest.kt | 10 ++--- .../routing/RoutingService_FindRouteTest.kt | 42 +++++++++---------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/src/test/kotlin/fi/hsl/jore4/mapmatching/service/matching/MatchingService_FindMatchForPublicTransportRouteTest.kt b/src/test/kotlin/fi/hsl/jore4/mapmatching/service/matching/MatchingService_FindMatchForPublicTransportRouteTest.kt index 20766d4b..68de828b 100644 --- a/src/test/kotlin/fi/hsl/jore4/mapmatching/service/matching/MatchingService_FindMatchForPublicTransportRouteTest.kt +++ b/src/test/kotlin/fi/hsl/jore4/mapmatching/service/matching/MatchingService_FindMatchForPublicTransportRouteTest.kt @@ -165,11 +165,11 @@ class MatchingService_FindMatchForPublicTransportRouteTest assertThat(actualLinkIdsAndForwardTraversals).isEqualTo( listOf( - "440250" to false, - "440764" to false, - "440767" to false, - "440765" to false, - "440750" to true + "90c18933-06f4-4f7b-9df2-9c82b59ecd35:2" to false, + "315a1fa3-fdfa-4340-b581-2a83bb8abfa9:2" to false, + "80f7650b-e63b-4f0b-8159-b04936ce7cdb:2" to false, + "9fbfa4c7-0bc7-490c-99a2-c8524225ba40:2" to false, + "e30555b1-d6bd-43d5-ae94-fc3834d0abef:2" to true ) ) } diff --git a/src/test/kotlin/fi/hsl/jore4/mapmatching/service/routing/RoutingService_FindRouteTest.kt b/src/test/kotlin/fi/hsl/jore4/mapmatching/service/routing/RoutingService_FindRouteTest.kt index 7073b8b3..f39fc7d5 100644 --- a/src/test/kotlin/fi/hsl/jore4/mapmatching/service/routing/RoutingService_FindRouteTest.kt +++ b/src/test/kotlin/fi/hsl/jore4/mapmatching/service/routing/RoutingService_FindRouteTest.kt @@ -189,9 +189,9 @@ class RoutingService_FindRouteTest assertThat(actualLinkIdsAndForwardTraversals).isEqualTo( listOf( - "441679" to true, - "441872" to true, - "441874" to true + "c525b97b-8c30-4aba-bfce-28074d4c08e5:1" to true, + "3113baf5-2120-45d3-8f16-0d94e63644fd:1" to true, + "1d1a650b-5621-40bf-bfb2-73d231849c48:1" to true ) ) } @@ -236,7 +236,7 @@ class RoutingService_FindRouteTest assertThat(actualLinkIdsAndForwardTraversals).isEqualTo( listOf( - "441872" to true + "3113baf5-2120-45d3-8f16-0d94e63644fd:1" to true ) ) } @@ -288,11 +288,11 @@ class RoutingService_FindRouteTest assertThat(actualLinkIdsAndForwardTraversals).isEqualTo( listOf( - "441872" to true, - "441880" to true, - "441870" to false, - "441890" to false, - "441872" to true + "3113baf5-2120-45d3-8f16-0d94e63644fd:1" to true, + "134baafa-fbc0-47e7-8be1-6f25cbd37eff:1" to true, + "2f402bd1-aeeb-4a32-bbf7-36dac212bd14:1" to false, + "bd198e25-1902-44fd-ac89-da19c5115eee:1" to false, + "3113baf5-2120-45d3-8f16-0d94e63644fd:1" to true ) ) } @@ -344,10 +344,10 @@ class RoutingService_FindRouteTest assertThat(actualLinkIdsAndForwardTraversals).isEqualTo( listOf( - "419821" to false, - "419821" to true, - "419827" to true, - "419825" to true + "5e070b32-d8f9-4096-8957-7303f8affe8b:2" to false, + "5e070b32-d8f9-4096-8957-7303f8affe8b:2" to true, + "28c73f2e-0d5e-40e1-9756-5171bb65f39d:2" to true, + "9e19e8ac-55dc-4cfb-b262-9206a957d084:2" to true ) ) } @@ -393,7 +393,7 @@ class RoutingService_FindRouteTest assertThat(actualLinkIdsAndForwardTraversals).isEqualTo( listOf( - "441872" to true + "3113baf5-2120-45d3-8f16-0d94e63644fd:1" to true ) ) } @@ -468,11 +468,11 @@ class RoutingService_FindRouteTest assertThat(actualLinkIdsAndForwardTraversals).isEqualTo( listOf( - "11392370" to true, - "12538103" to true, - "12538103" to false, - "12538103" to false, - "12538103" to true + "4f8aa489-14dd-4061-b197-41db30fc3e98:1" to true, + "88bd12da-4e71-4e32-95f8-f9ee8c276c95:1" to false, + "88bd12da-4e71-4e32-95f8-f9ee8c276c95:1" to true, + "88bd12da-4e71-4e32-95f8-f9ee8c276c95:1" to true, + "88bd12da-4e71-4e32-95f8-f9ee8c276c95:1" to false ) ) } @@ -501,8 +501,8 @@ class RoutingService_FindRouteTest // shorter list than without simplifying, the closed-loop link appears only once assertThat(actualLinkIdsAndForwardTraversals).isEqualTo( listOf( - "11392370" to true, - "12538103" to true + "4f8aa489-14dd-4061-b197-41db30fc3e98:1" to true, + "88bd12da-4e71-4e32-95f8-f9ee8c276c95:1" to false ) ) }