Skip to content

Commit 1872ca2

Browse files
mmathieummmathieum
committed
Index strings
mtransitapps/commons-java#10
1 parent 2eb5822 commit 1872ca2

File tree

12 files changed

+216
-8
lines changed

12 files changed

+216
-8
lines changed

‎src/main/java/org/mtransit/parser/DefaultAgencyTools.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@
3939
import org.mtransit.parser.mt.data.MServiceIds;
4040
import org.mtransit.parser.mt.data.MSpec;
4141
import org.mtransit.parser.mt.data.MDirection;
42+
import org.mtransit.parser.mt.data.MString;
43+
import org.mtransit.parser.mt.data.MStrings;
4244

4345
import java.text.SimpleDateFormat;
4446
import java.util.ArrayList;
@@ -154,7 +156,9 @@ public void start(@NotNull String[] args) {
154156
MTLog.logDebug("Args [%d]: %s.", args.length, Arrays.asList(args));
155157
final List<MServiceDate> lastServiceDates = MReader.loadServiceDates(args[2]);
156158
final List<MServiceId> lastServiceIds = MReader.loadServiceIds(args[2]);
159+
final List<MString> lastStrings = MReader.loadStrings(args[2]);
157160
MServiceIds.addAll(lastServiceIds);
161+
MStrings.addAll(lastStrings);
158162
this.serviceIdInts = extractUsefulServiceIdInts(args, this, true, lastServiceDates);
159163
final String inputUrl = args.length >= 5 ? args[4] : null;
160164
if (excludingAll()) {

‎src/main/java/org/mtransit/parser/db/DumpDbUtils.kt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ object DumpDbUtils {
2727
connection.createStatement().use { statement ->
2828
SQLUtils.execute(statement, "PRAGMA auto_vacuum = NONE")
2929
// DROP IF EXIST
30+
SQLUtils.executeUpdate(statement, GTFSCommons.T_STRINGS_SQL_DROP)
3031
SQLUtils.executeUpdate(statement, GTFSCommons.T_DIRECTION_STOPS_SQL_DROP)
3132
SQLUtils.executeUpdate(statement, GTFSCommons.T_STOP_SQL_DROP)
3233
SQLUtils.executeUpdate(statement, GTFSCommons.T_DIRECTION_SQL_DROP)
@@ -44,6 +45,7 @@ object DumpDbUtils {
4445
SQLUtils.executeUpdate(statement, GTFSCommons.T_SERVICE_IDS_SQL_CREATE)
4546
}
4647
SQLUtils.executeUpdate(statement, GTFSCommons.T_SERVICE_DATES_SQL_CREATE)
48+
SQLUtils.executeUpdate(statement, GTFSCommons.T_STRINGS_SQL_CREATE)
4749
}
4850
}
4951
}

‎src/main/java/org/mtransit/parser/mt/MGenerator.java‎

Lines changed: 54 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@
3636
import org.mtransit.parser.mt.data.MStop;
3737
import org.mtransit.parser.mt.data.MDirection;
3838
import org.mtransit.parser.mt.data.MDirectionStop;
39+
import org.mtransit.parser.mt.data.MString;
40+
import org.mtransit.parser.mt.data.MStrings;
3941

4042
import java.io.BufferedWriter;
4143
import java.io.File;
@@ -176,7 +178,8 @@ public static MSpec generateMSpec(@NotNull GSpec gtfs, @NotNull GAgencyTools age
176178
MTLog.log("- Trips: %d", mTripsList.size());
177179
MTLog.log("- Trip stops: %d", mDirectionStopsList.size());
178180
MTLog.log("- Stops: %d", mStopsList.size());
179-
MTLog.log("- Service Ids: %d", MServiceIds.getAll().size());
181+
MTLog.log("- Service Ids: %d", MServiceIds.count());
182+
MTLog.log("- Strings: %d", MStrings.count());
180183
MTLog.log("- Service Dates: %d", mServiceDatesList.size());
181184
MTLog.log("- Route with Frequencies: %d", mRouteFrequencies.size());
182185
MTLog.log("- First timestamp: %s", MTLog.formatDateTime(firstTimestamp));
@@ -201,6 +204,7 @@ private static void logMerging(@NotNull String msg, long routeId) {
201204
MTLog.logDebug("%s: Generating routes, trips, trip stops & stops objects... (merging %s)", routeId, msg);
202205
}
203206

207+
private static final String GTFS_STRINGS = "gtfs_strings";
204208
private static final String GTFS_SCHEDULE = "gtfs_schedule";
205209
private static final String GTFS_SCHEDULE_SERVICE_DATES = GTFS_SCHEDULE + "_service_dates"; // DB
206210
private static final String GTFS_SCHEDULE_SERVICE_IDS = GTFS_SCHEDULE + "_service_ids"; // DB
@@ -282,6 +286,8 @@ public static void dumpFiles(@NotNull GAgencyTools gAgencyTools,
282286
dumpFrequencyRoutes(gAgencyTools, mSpec, fileBase, deleteAll, rawDirF);
283287
// SERVICE IDS
284288
dumpServiceIds(mSpec, fileBase, deleteAll, dataDirF, rawDirF, dbConnection); // AFTER SCHEDULE STOPS & FREQUENCY ROUTES
289+
// STRINGS
290+
dumpStrings(mSpec, fileBase, deleteAll, dataDirF, rawDirF, dbConnection); // AFTER ALL OTHER TABLE W/ STRINGS
285291
if (deleteAll) {
286292
dumpValues(rawDirF, fileBase, null, null, null, null, null, -1, -1, null, true);
287293
} else {
@@ -568,6 +574,53 @@ private static void dumpServiceIds(
568574
}
569575
}
570576

577+
private static void dumpStrings(
578+
@Nullable MSpec mSpec,
579+
@NotNull String fileBase,
580+
boolean deleteAll,
581+
@NotNull File dataDirF,
582+
@NotNull File rawDirF,
583+
@Nullable Connection dbConnection) {
584+
if (!FeatureFlags.F_EXPORT_SERVICE_ID_INTS) return;
585+
if (!deleteAll
586+
&& (mSpec == null || !mSpec.isValid() || (F_PRE_FILLED_DB && dbConnection == null))) {
587+
throw new MTLog.Fatal("Generated data invalid (agencies: %s)!", mSpec);
588+
}
589+
if (F_PRE_FILLED_DB) {
590+
FileUtils.deleteIfExist(new File(rawDirF, fileBase + GTFS_STRINGS)); // migration from src/main/res/raw to data
591+
}
592+
File file = new File(F_PRE_FILLED_DB ? dataDirF : rawDirF, fileBase + GTFS_STRINGS);
593+
FileUtils.deleteIfExist(file); // delete previous
594+
try (BufferedWriter ow = new BufferedWriter(new FileWriter(file))) {
595+
if (!deleteAll) {
596+
MTLog.logPOINT(); // LOG
597+
Statement dbStatement = null;
598+
String sqlInsert = null;
599+
if (F_PRE_FILLED_DB) {
600+
SQLUtils.setAutoCommit(dbConnection, false); // START TRANSACTION
601+
dbStatement = dbConnection.createStatement();
602+
sqlInsert = GTFSCommons.getT_SERVICE_IDS_SQL_INSERT();
603+
}
604+
for (MString mString : MStrings.getAll()) {
605+
final String stringInsert = mString.toFile();
606+
if (F_PRE_FILLED_DB) {
607+
SQLUtils.executeUpdate(
608+
dbStatement,
609+
String.format(sqlInsert, stringInsert)
610+
);
611+
}
612+
ow.write(stringInsert);
613+
ow.write(Constants.NEW_LINE);
614+
}
615+
if (F_PRE_FILLED_DB) {
616+
SQLUtils.setAutoCommit(dbConnection, true); // END TRANSACTION == commit()
617+
}
618+
}
619+
} catch (Exception ioe) {
620+
throw new MTLog.Fatal(ioe, "I/O Error while writing strings file!");
621+
}
622+
}
623+
571624
@NotNull
572625
private static Pair<Integer, Integer> dumpScheduleServiceDates(@NotNull GAgencyTools gAgencyTools,
573626
@Nullable MSpec mSpec,

‎src/main/java/org/mtransit/parser/mt/MReader.kt‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import org.mtransit.parser.Pair
1414
import org.mtransit.parser.gtfs.data.GFieldTypes
1515
import org.mtransit.parser.mt.data.MServiceDate
1616
import org.mtransit.parser.mt.data.MServiceId
17+
import org.mtransit.parser.mt.data.MString
1718
import java.io.File
1819
import java.util.TimeZone
1920

@@ -164,4 +165,27 @@ object MReader {
164165
}
165166

166167
// endregion
168+
169+
// region strings
170+
171+
private const val GTFS_STRINGS = "gtfs_strings"
172+
173+
@JvmStatic
174+
fun loadStrings(fileBase: String) = try {
175+
File(getResDirName(fileBase) + "/$RAW/${fileBase}$GTFS_STRINGS")
176+
.takeIf { it.exists() }
177+
?.readLines()
178+
?.mapNotNull { line ->
179+
MString.fromFileLine(line)
180+
}
181+
?: run {
182+
MTLog.log("File not found '${"/$RAW/${fileBase}$GTFS_STRINGS"}'!")
183+
null
184+
}
185+
} catch (e: Exception) {
186+
MTLog.logNonFatal(e, "Error while reading '$fileBase' strings!")
187+
null
188+
}
189+
190+
// endregion
167191
}

‎src/main/java/org/mtransit/parser/mt/data/MDirection.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -177,7 +177,7 @@ data class MDirection(
177177
fun toFile() = listOf(
178178
id.toString(), // ID
179179
headsignType.toString(), // HEADSIGN TYPE
180-
headsignValue.quotesEscape(), // HEADSIGN STRING
180+
headsignValue.toStringIds().quotesEscape(), // HEADSIGN STRING
181181
routeId.toString(), // ROUTE ID
182182
).joinToString(Constants.COLUMN_SEPARATOR_)
183183

‎src/main/java/org/mtransit/parser/mt/data/MRoute.kt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,8 @@ data class MRoute(
3737

3838
fun toFile() = buildList {
3939
add(id.toString()) // ID
40-
add((shortName ?: Constants.EMPTY).quotesEscape()) // short name
41-
add(longName.quotesEscape()) // long name
40+
add(shortName.orEmpty().toStringIds().quotesEscape()) // short name
41+
add(longName.toStringIds().quotesEscape()) // long name
4242
add((color?.uppercase() ?: Constants.EMPTY).quotes()) // color
4343
add(originalIdHash.toString()) // original ID hash
4444
add(type.toString())

‎src/main/java/org/mtransit/parser/mt/data/MSchedule.kt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ data class MSchedule(
115115
add(_tripId.quotesEscape())
116116
}
117117
add(headsignType.takeIf { it >= 0 }?.toString() ?: Constants.EMPTY)
118-
add((headsignValue ?: Constants.EMPTY).quotesEscape())
118+
add(headsignValue.orEmpty().toStringIds().toStringIds().quotesEscape())
119119
add(accessible.toString())
120120
}.joinToString(Constants.COLUMN_SEPARATOR_)
121121

@@ -134,7 +134,7 @@ data class MSchedule(
134134
add(Constants.EMPTY.quotes())
135135
} else {
136136
add(headsignType.takeIf { it >= 0 }?.toString() ?: Constants.EMPTY)
137-
add((headsignValue ?: Constants.EMPTY).quotesEscape())
137+
add(headsignValue.orEmpty().toStringIds().quotesEscape())
138138
}
139139
add(accessible.toString())
140140
}.joinToString(Constants.COLUMN_SEPARATOR_)

‎src/main/java/org/mtransit/parser/mt/data/MServiceId.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ data class MServiceId(
2424
).compare(this, other)
2525

2626
companion object {
27-
fun fromFileLine(line: String): MServiceId? =
27+
fun fromFileLine(line: String) =
2828
line.split(Constants.COLUMN_SEPARATOR)
2929
.takeIf { it.size == 2 }
3030
?.let { columns ->

‎src/main/java/org/mtransit/parser/mt/data/MServiceIds.kt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,9 @@ object MServiceIds {
4646
return idToIdInt[serviceId] ?: add(serviceId)
4747
}
4848

49+
@JvmStatic
50+
fun count() = idIntToId.size()
51+
4952
@JvmStatic
5053
fun getAll() = buildList {
5154
idToIdInt.forEach { id, idInt ->

‎src/main/java/org/mtransit/parser/mt/data/MStop.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ data class MStop(
4646
fun toFile() = listOf(
4747
id.toString(), // ID
4848
code.quotesEscape(), // code
49-
name.quotesEscape(), // name
49+
name.toStringIds().quotesEscape(), // name
5050
MDataChangedManager.avoidLatLngChanged(lat), // latitude
5151
MDataChangedManager.avoidLatLngChanged(lng), // longitude
5252
accessible.toString(),

0 commit comments

Comments
 (0)