Skip to content

Commit 0386386

Browse files
mmathieummmathieum
authored
Option to verify uniqueness of cleaned IDs (route, stop, trip, service) (#22)
1 parent 4615185 commit 0386386

File tree

5 files changed

+110
-4
lines changed

5 files changed

+110
-4
lines changed

‎src/main/java/org/mtransit/parser/DefaultAgencyTools.java‎

Lines changed: 69 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,12 +41,14 @@
4141
import org.mtransit.parser.mt.data.MDirection;
4242
import org.mtransit.parser.mt.data.MString;
4343
import org.mtransit.parser.mt.data.MStrings;
44+
import org.mtransit.parser.mt.data.MVerify;
4445

4546
import java.text.SimpleDateFormat;
4647
import java.util.ArrayList;
4748
import java.util.Arrays;
4849
import java.util.Calendar;
4950
import java.util.Collections;
51+
import java.util.HashMap;
5052
import java.util.HashSet;
5153
import java.util.List;
5254
import java.util.Locale;
@@ -180,6 +182,7 @@ public void start(@NotNull String[] args) {
180182
return; // DEBUG
181183
}
182184
MGenerator.dumpFiles(this, mSpec, args[0], args[1], args[2], inputUrl, false);
185+
MVerify.verify(mSpec, this); // after dump files to have all values
183186
MTLog.log("Generating data... DONE in %s.", Utils.getPrettyDuration(System.currentTimeMillis() - start));
184187
}
185188

@@ -351,10 +354,14 @@ public boolean excludeAgency(@NotNull GAgency gAgency) {
351354
return KEEP;
352355
}
353356

357+
private final Map<String, String> serviceIdToCleanupServiceId = new HashMap<>();
358+
354359
@NotNull
355360
@Override
356361
public String cleanServiceId(@NotNull String gServiceId) {
357-
return GTFSCommons.cleanOriginalId(gServiceId, getServiceIdCleanupPattern());
362+
final String cleanServiceId = GTFSCommons.cleanOriginalId(gServiceId, getServiceIdCleanupPattern());
363+
serviceIdToCleanupServiceId.put(gServiceId, cleanServiceId);
364+
return cleanServiceId;
358365
}
359366

360367
@Nullable
@@ -379,13 +386,30 @@ public String getServiceIdCleanupRegex() {
379386
return this.serviceIdCleanupPattern;
380387
}
381388

389+
@Override
390+
public boolean verifyServiceIdsUniqueness() {
391+
if (Configs.getAgencyConfig() != null) {
392+
if (Configs.getAgencyConfig().getServiceIdNotUniqueAllowed()) return false;
393+
}
394+
return getServiceIdCleanupRegex() != null; // OPT-IN feature
395+
}
396+
397+
@NotNull
398+
public Map<String, String> getServiceIdToCleanupServiceId() {
399+
return serviceIdToCleanupServiceId;
400+
}
401+
402+
private final Map<String, String> routeIdToCleanupRouteId = new HashMap<>();
403+
382404
@NotNull
383405
@Override
384406
public String cleanRouteOriginalId(@NotNull String gRouteId) {
385407
if (Configs.getRouteConfig().getRouteIdCleanMerged()) {
386408
gRouteId = CleanUtils.cleanMergedID(gRouteId);
387409
}
388-
return GTFSCommons.cleanOriginalId(gRouteId, getRouteIdCleanupPattern());
410+
final String cleanRouteId = GTFSCommons.cleanOriginalId(gRouteId, getRouteIdCleanupPattern());
411+
this.routeIdToCleanupRouteId.put(gRouteId, cleanRouteId);
412+
return cleanRouteId;
389413
}
390414

391415
@Override
@@ -449,6 +473,17 @@ public String getRouteIdCleanupRegex() {
449473
return Configs.getRouteConfig().getRouteIdCleanupRegex();
450474
}
451475

476+
@Override
477+
public boolean verifyRouteIdsUniqueness() {
478+
if (Configs.getRouteConfig().getRouteIdNotUniqueAllowed()) return false;
479+
return getRouteIdCleanupRegex() != null; // OPT-IN feature
480+
}
481+
482+
@NotNull
483+
public Map<String, String> getRouteIdToCleanupRouteId() {
484+
return this.routeIdToCleanupRouteId;
485+
}
486+
452487
@Nullable
453488
private Pattern routeIdCleanupPattern = null;
454489

@@ -657,6 +692,17 @@ public String getTripIdCleanupRegex() {
657692
return Configs.getRouteConfig().getTripIdCleanupRegex();
658693
}
659694

695+
@Override
696+
public boolean verifyTripIdsUniqueness() {
697+
if (Configs.getRouteConfig().getTripIdNotUniqueAllowed()) return false;
698+
return getTripIdCleanupRegex() != null; // OPT-IN feature
699+
}
700+
701+
@NotNull
702+
public Map<String, String> getTripIdToCleanupTripId() {
703+
return this.tripIdToCleanupTripId;
704+
}
705+
660706
@Nullable
661707
private Pattern tripIdCleanupPattern = null;
662708

@@ -671,9 +717,13 @@ private Pattern getTripIdCleanupPattern() {
671717
return this.tripIdCleanupPattern;
672718
}
673719

720+
private final Map<String, String> tripIdToCleanupTripId = new HashMap<>();
721+
674722
@Override
675723
public @NotNull String cleanTripOriginalId(@NotNull String gTripId) {
676-
return GTFSCommons.cleanOriginalId(gTripId, getTripIdCleanupPattern());
724+
final String cleanTripId = GTFSCommons.cleanOriginalId(gTripId, getTripIdCleanupPattern());
725+
this.tripIdToCleanupTripId.put(gTripId, cleanTripId);
726+
return cleanTripId;
677727
}
678728

679729
@Override
@@ -1035,6 +1085,17 @@ public boolean useStopCodeForStopId() {
10351085
return Configs.getRouteConfig().getStopIdCleanupRegex();
10361086
}
10371087

1088+
@Override
1089+
public boolean verifyStopIdsUniqueness() {
1090+
if (Configs.getRouteConfig().getStopIdNotUniqueAllowed()) return false;
1091+
return getStopIdCleanupRegex() != null; // OPT-IN feature
1092+
}
1093+
1094+
@NotNull
1095+
public Map<String, String> getStopIdToCleanupStopId() {
1096+
return this.stopIdToCleanupStopId;
1097+
}
1098+
10381099
@Nullable
10391100
private Pattern stopIdCleanupPattern = null;
10401101

@@ -1049,10 +1110,14 @@ private Pattern getStopIdCleanupPattern() {
10491110
return this.stopIdCleanupPattern;
10501111
}
10511112

1113+
private final Map<String, String> stopIdToCleanupStopId = new HashMap<>();
1114+
10521115
@NotNull
10531116
@Override
10541117
public String cleanStopOriginalId(@NotNull String gStopId) {
1055-
return GTFSCommons.cleanOriginalId(gStopId, getStopIdCleanupPattern());
1118+
final String cleanStopId = GTFSCommons.cleanOriginalId(gStopId, getStopIdCleanupPattern());
1119+
this.stopIdToCleanupStopId.put(gStopId, cleanStopId);
1120+
return cleanStopId;
10561121
}
10571122

10581123
@Override

‎src/main/java/org/mtransit/parser/config/gtfs/data/AgencyConfig.kt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,4 +15,6 @@ data class AgencyConfig(
1515
val defaultColor: String, // REQUIRED
1616
@SerialName("service_id_cleanup_regex")
1717
val serviceIdCleanupRegex: String? = null, // optional
18+
@SerialName("service_id_not_unique_allowed")
19+
val serviceIdNotUniqueAllowed: Boolean = false, // OPT-IN feature
1820
)

‎src/main/java/org/mtransit/parser/config/gtfs/data/RouteConfig.kt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ data class RouteConfig(
1212
val defaultRouteIdEnabled: Boolean = false, // OPT-IN feature
1313
@SerialName("route_id_cleanup_regex")
1414
val routeIdCleanupRegex: String? = null, // optional
15+
@SerialName("route_id_not_unique_allowed")
16+
val routeIdNotUniqueAllowed: Boolean = false, // OPT-IN feature
1517
@SerialName("route_id_clean_merged")
1618
val routeIdCleanMerged: Boolean = false, // OPT-IN feature
1719
@SerialName("use_route_short_name_for_route_id")
@@ -41,13 +43,17 @@ data class RouteConfig(
4143
// DIRECTION
4244
@SerialName("trip_id_cleanup_regex")
4345
val tripIdCleanupRegex: String? = null, // optional
46+
@SerialName("trip_id_not_unique_allowed")
47+
val tripIdNotUniqueAllowed: Boolean = false, // OPT-IN feature
4448
@SerialName("direction_headsign_cleaners")
4549
val directionHeadsignCleaners: List<Cleaner> = emptyList(),
4650
@SerialName("direction_finder_enabled")
4751
val directionFinderEnabled: Boolean = false, // OPT-IN feature
4852
// STOP
4953
@SerialName("stop_id_cleanup_regex")
5054
val stopIdCleanupRegex: String? = null, // optional
55+
@SerialName("stop_id_not_unique_allowed")
56+
val stopIdNotUniqueAllowed: Boolean = false, // OPT-IN feature
5157
@SerialName("use_stop_code_for_stop_id")
5258
val useStopCodeForStopId: Boolean = false, // OPT-IN feature
5359
@SerialName("stop_code_to_stop_id_configs")

‎src/main/java/org/mtransit/parser/gtfs/GAgencyTools.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,8 @@ public interface GAgencyTools {
9292
@Nullable
9393
String getServiceIdCleanupRegex();
9494

95+
boolean verifyServiceIdsUniqueness();
96+
9597
// ROUTE
9698
@NotNull
9799
String cleanRouteOriginalId(@NotNull String routeId);
@@ -116,6 +118,8 @@ public interface GAgencyTools {
116118
@Nullable
117119
String getRouteIdCleanupRegex();
118120

121+
boolean verifyRouteIdsUniqueness();
122+
119123
@NotNull
120124
String getRouteShortName(@NotNull GRoute gRoute);
121125

@@ -163,6 +167,8 @@ public interface GAgencyTools {
163167
@Nullable
164168
String getTripIdCleanupRegex();
165169

170+
boolean verifyTripIdsUniqueness();
171+
166172
@NotNull String cleanTripOriginalId(@NotNull String gTripId);
167173

168174
boolean directionSplitterEnabled(long routeId);
@@ -252,6 +258,8 @@ public interface GAgencyTools {
252258
@Nullable
253259
String getStopIdCleanupRegex();
254260

261+
boolean verifyStopIdsUniqueness();
262+
255263
@NotNull
256264
String cleanStopName(@NotNull String gStopName);
257265

‎src/main/java/org/mtransit/parser/mt/data/MVerify.kt‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
package org.mtransit.parser.mt.data
2+
3+
import org.mtransit.parser.DefaultAgencyTools
4+
import org.mtransit.parser.MTLog
5+
6+
object MVerify {
7+
8+
@JvmStatic
9+
fun verify(@Suppress("unused") mSpec: MSpec, agencyTools: DefaultAgencyTools) {
10+
listOf(
11+
Triple("service IDs", agencyTools.serviceIdToCleanupServiceId, agencyTools.verifyServiceIdsUniqueness()),
12+
Triple("trip IDs", agencyTools.tripIdToCleanupTripId, agencyTools.verifyTripIdsUniqueness()),
13+
Triple("route IDs", agencyTools.routeIdToCleanupRouteId, agencyTools.verifyRouteIdsUniqueness()),
14+
Triple("stop IDs", agencyTools.stopIdToCleanupStopId, agencyTools.verifyStopIdsUniqueness())
15+
).forEach { (idTypeName, idMap, verificationEnabled) ->
16+
if (!verificationEnabled) return@forEach
17+
val valueToKeys = idMap.entries.groupBy({ it.value }, { it.key })
18+
val duplicates = valueToKeys.filter { it.value.size > 1 }
19+
if (duplicates.isNotEmpty()) {
20+
throw MTLog.Fatal("ERROR: $idTypeName are not unique! Duplicates: \n$duplicates")
21+
}
22+
MTLog.logDebug("$idTypeName uniqueness verified.")
23+
}
24+
}
25+
}

0 commit comments

Comments
 (0)