Generate configs for crowd-sourced realtime components

Author: jbruechert

ansible/hosts

@@ -113,3 +113,9 @@ woodpecker:
         30363331323065643861613364303531653565633833346263313334613032613539386430613364
         3137363965316563650a356631343130653036613838646331363336643265333662346534343163
         33646436316239336363363339386665353532363336636535333139363937373064
+
+crowd-sourced-realtime:
+  hosts:
+    crunchy:
+      ansible_user: root
+      ansible_host: crunchy.spline.de

ansible/roles/crowd-sourced-realtime/files/delay-tracker.service

@@ -0,0 +1,17 @@
+# SPDX-FileCopyrightText: 2025 Jonah Brüchert <[email protected]>
+#
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+[Unit]
+Description=Delay Tracker
+After=network.target
+
+[Service]
+User=crowd-sourced-realtime
+Group=nogroup
+Restart=always
+ExecStart=/usr/local/bin/delay-tracker
+WorkingDirectory=/var/cache/transitous/out/delay-tracker/
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/crowd-sourced-realtime/files/transitous-gps-collector.service

@@ -0,0 +1,17 @@
+# SPDX-FileCopyrightText: 2025 Jonah Brüchert <[email protected]>
+#
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+[Unit]
+Description=GPS Collector
+After=network.target
+
+[Service]
+User=crowd-sourced-realtime
+Group=nogroup
+Restart=always
+ExecStart=/usr/local/bin/transitous-gps-collector
+WorkingDirectory=/var/cache/transitous/out/gps-collector/
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/crowd-sourced-realtime/tasks/main.yml

@@ -0,0 +1,55 @@
+# SPDX-FileCopyrightText: 2025 Jonah Brüchert <[email protected]>
+#
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+- name: Create crowd-sourced-realtime user
+  user:
+    name: crowd-sourced-realtime
+
+- name: Install delay-tracker executable
+  get_url:
+    url: https://cloud.ghsq.de/s/BfAQb86e5qZWrf8/download/delay-tracker-x86_64
+    dest: /usr/local/bin/delay-tracker
+    mode: '0755'
+
+- name: Install delay-tracker service
+  copy:
+    src: delay-tracker.service
+    dest: /etc/systemd/system/delay-tracker.service
+  register: install_delay_tracker_service
+
+- name: Install transitous-gps-collector
+  get_url:
+    url: https://cloud.ghsq.de/s/8FYsw7eXxfStoD5/download/transitous-gps-collector-x86_64
+    dest: /usr/local/bin/transitous-gps-collector
+    mode: '0755'
+
+- name: Install transitous-gps-collector service
+  copy:
+    src: transitous-gps-collector.service
+    dest: /etc/systemd/system/transitous-gps-collector.service
+  register: install_gps_collector_service
+
+- name: Enable delay-tracker
+  systemd_service:
+    name: delay-tracker.service
+    daemon_reload: true
+  when: install_delay_tracker_service.changed
+
+- name: Enable transitous-gps-collector
+  systemd_service:
+    name: transitous-gps-collector.service
+    daemon_reload: true
+  when: install_gps_collector_service.changed
+
+- name: Enable delay-tracker
+  systemd_service:
+    name: delay-tracker.service
+    enabled: true
+    state: "started"
+
+- name: Enable transitous-gps-collector
+  systemd_service:
+    name: transitous-gps-collector.service
+    enabled: true
+    state: "started"

feeds/eu.json

@@ -14,7 +14,8 @@
             "name": "flixbus",
             "type": "transitland-atlas",
             "transitland-atlas-id": "f-u-flixbus",
-            "fix": true
+            "fix": true,
+            "enable-crowd-sourced-realtime": true
         },
         {
             "name": "blablacar-bus",

feeds/lt.json

@@ -100,7 +100,8 @@
         {
             "name": "LTG-Link",
             "type": "http",
-            "url": "https://jbb.ghsq.de/gtfs/lt-ltglink.gtfs.zip"
+            "url": "https://jbb.ghsq.de/gtfs/lt-ltglink.gtfs.zip",
+            "enable-crowd-sourced-realtime": true
         }
     ]
 }

feeds/me.json

@@ -12,7 +12,8 @@
             "license": {
                 "spdx-identifier": "ODbL-1.0"
             },
-            "mdb-id": "mdb-2377"
+            "mdb-id": "mdb-2377",
+            "enable-crowd-sourced-realtime": true
         }
     ]
 }

src/generate-motis-config.py

@@ -5,6 +5,7 @@
 
 import argparse
 import json
+import toml
 import metadata
 import os
 import shutil
@@ -37,10 +38,19 @@ def find_motis_asset(asset_name: str):
     atlas = transitland.Atlas.load(Path("transitland-atlas/"))
     mdb = mobilitydatabase.Database.load()
 
-    gtfs_feeds: list[dict] = []
-    gtfsrt_feeds: list[dict] = []
+    feeds = []
+    regions: list[tuple[str, metadata.Region]] = []
+    if len(arguments.regions) == 0:
+        feeds = feed_dir.glob("*.json")
+    else:
+        for region in arguments.regions:
+            feeds += feed_dir.glob(f"{region}.json")
 
-    with open("motis/config.yml") as f:
+    for feed in sorted(feeds):
+        region_name = feed.name[: feed.name.rfind(".")]
+        regions.append((region_name, metadata.Region(json.load(open(feed, "r")))))
+
+    with open("configs/motis/config.yml") as f:
         yaml = YAML(typ="rt")
 
         config = yaml.load(f)
@@ -66,90 +76,110 @@ def find_motis_asset(asset_name: str):
         config["timetable"]["datasets"] = {}
         config["gbfs"]["feeds"] = {}
 
-        # TODO backward compatibility, remove this in a few months
-        while "full" in arguments.regions:
-            print("Ignoring legacy option 'full', this is the default now.")
-            arguments.regions.remove("full")
-
-        feeds = []
-        if len(arguments.regions) == 0:
-            feeds = feed_dir.glob("*.json")
-        else:
-            for region in arguments.regions:
-                feeds += feed_dir.glob(f"{region}.json")
-
-        for feed in sorted(feeds):
-            with open(feed, "r") as f:
-                parsed = json.load(f)
-                region = metadata.Region(parsed)
-
-                metadata_filename = feed.name
-                region_name = metadata_filename[: metadata_filename.rfind(".")]
-
-                for source in region.sources:
-                    schedule_name = f"{region_name}-{source.name}"
-
-                    if source.skip:
-                        continue
-
-                    match source:
-                        case metadata.TransitlandSource():
-                            resolved_source = atlas.source_by_id(source)
-                            if not resolved_source:
-                                eprint("Error: Could not resolve", source.transitland_atlas_id)
-                                sys.exit(1)
-                            source = resolved_source
-                        case metadata.MobilityDatabaseSource():
-                            resolved_source = mdb.source_by_id(source)
-                            if not resolved_source:
-                                eprint("Error: Could not resolve", source.mdb_id)
-                                sys.exit(1)
-                            source = resolved_source
-
-                    match source.spec:
-                        case source.spec if source.spec in ["gtfs", "gtfs-flex"]:
-                            schedule_file = \
-                                f"{region_name}_{source.name}.gtfs.zip"
-                            name = f"{region_name}-{source.name}"
-                            config["timetable"]["datasets"][name] = \
-                                {
-                                    "path": schedule_file,
-                                    "extend_calendar": source.extend_calendar
-                                }
-                            if source.default_timezone is not None:
-                                config["timetable"]["datasets"][name]["default_timezone"] = source.default_timezone
-
-                        case "gtfs-rt" if isinstance(source, metadata.UrlSource):
-                            name = f"{region_name}-{source.name}"
-                            if name not in config["timetable"]["datasets"]:
-                                eprint(
-                                    "Error: The name of a realtime (gtfs-rt) "
-                                    + "feed needs to match the name of its "
-                                    + "static base feed defined before the "
-                                    + "realtime feed. Found nothing "
-                                    + "belonging to",
-                                    source.name,
-                                )
-                                sys.exit(1)
+        for (region_name, region) in regions:
+            for source in region.sources:
+                schedule_name = f"{region_name}-{source.name}"
+
+                if source.skip:
+                    continue
+
+                match source:
+                    case metadata.TransitlandSource():
+                        resolved_source = atlas.source_by_id(source)
+                        if not resolved_source:
+                            eprint("Error: Could not resolve", source.transitland_atlas_id)
+                            sys.exit(1)
+                        source = resolved_source
+                    case metadata.MobilityDatabaseSource():
+                        resolved_source = mdb.source_by_id(source)
+                        if not resolved_source:
+                            eprint("Error: Could not resolve", source.mdb_id)
+                            sys.exit(1)
+                        source = resolved_source
+
+                match source.spec:
+                    case source.spec if source.spec in ["gtfs", "gtfs-flex"]:
+                        schedule_file = \
+                            f"{region_name}_{source.name}.gtfs.zip"
+                        name = f"{region_name}-{source.name}"
+                        config["timetable"]["datasets"][name] = \
+                            {
+                                "path": schedule_file,
+                                "extend_calendar": source.extend_calendar
+                            }
+                        if source.default_timezone is not None:
+                            config["timetable"]["datasets"][name]["default_timezone"] = source.default_timezone
 
+                        if source.enable_crowd_sourced_realtime:
                             if "rt" not in config["timetable"]["datasets"][name]:
                                 config["timetable"]["datasets"][name]["rt"] = []
 
-                            rt_feed: dict[str, Any] = {
-                                "url": source.url
-                            }
+                            config["timetable"]["datasets"][name]["rt"].append({"url": f"http://localhost:5002/{name}/trip-updates.pb" })
+
+                    case "gtfs-rt" if isinstance(source, metadata.UrlSource):
+                        name = f"{region_name}-{source.name}"
+                        if name not in config["timetable"]["datasets"]:
+                            eprint(
+                                "Error: The name of a realtime (gtfs-rt) "
+                                + "feed needs to match the name of its "
+                                + "static base feed defined before the "
+                                + "realtime feed. Found nothing "
+                                + "belonging to",
+                                source.name,
+                            )
+                            sys.exit(1)
+
+                        if "rt" not in config["timetable"]["datasets"][name]:
+                            config["timetable"]["datasets"][name]["rt"] = []
+
+                        rt_feed: dict[str, Any] = {
+                            "url": source.url
+                        }
 
-                            if source.headers:
-                                rt_feed["headers"] = source.headers
+                        if source.headers:
+                            rt_feed["headers"] = source.headers
 
-                            config["timetable"]["datasets"][name]["rt"] \
-                                .append(rt_feed)
+                        config["timetable"]["datasets"][name]["rt"] \
+                            .append(rt_feed)
 
-                        case "gbfs" if isinstance(source, metadata.UrlSource):
-                            name = f"{region_name}-{source.name}"
-                            config["gbfs"]["feeds"][name] = {"url": source.url}
-                            if source.headers:
-                                config["gbfs"]["feeds"][name]["headers"] = source.headers
+                    case "gbfs" if isinstance(source, metadata.UrlSource):
+                        name = f"{region_name}-{source.name}"
+                        config["gbfs"]["feeds"][name] = {"url": source.url}
+                        if source.headers:
+                            config["gbfs"]["feeds"][name]["headers"] = source.headers
 
         with open("out/config.yml", "w") as fo:
             yaml.dump(config, fo)
+
+    with open("configs/gps-collector/config.toml", "r") as f:
+        config = toml.load(f)
+        config["feeds"] = {}
+
+        for (region_name, region) in regions:
+            for source in region.sources:
+                if source.enable_crowd_sourced_realtime:
+                    config["feeds"][f"{region_name}-{source.name}"] = {}
+
+        if not os.path.exists("out/gps-collector/"):
+            os.makedirs("out/gps-collector/")
+
+        with open("out/gps-collector/config.toml", "w") as fo:
+            toml.dump(config, fo)
+
+    with open("configs/delay-tracker/config.toml", "r") as f:
+        config = toml.load(f)
+        config["feeds"] = {}
+
+        for (region_name, region) in regions:
+            for source in region.sources:
+                if source.enable_crowd_sourced_realtime:  # This can be extended for vehicle-positions feeds later on
+                    config["feeds"][f"{region_name}-{source.name}"] = {
+                        "gtfs_url": f"../{region_name}_{source.name}.gtfs.zip",
+                        "gtfsrt_url": f"http://localhost:5001/gtfsrt/{region_name}-{source.name}/vehicle-positions.pb"
+                    }
+
+        if not os.path.exists("out/delay-tracker/"):
+            os.makedirs("out/delay-tracker/")
+
+        with open("out/delay-tracker/config.toml", "w") as fo:
+            toml.dump(config, fo)

src/metadata.py

@@ -56,6 +56,7 @@ class Source:
     extend_calendar = False
     default_timezone: Optional[str] = None
     keep_additional_fields = True
+    enable_crowd_sourced_realtime = False
 
     def __init__(self, parsed: Optional[dict] = None):
         self.license = License()
@@ -96,6 +97,8 @@ def __init__(self, parsed: Optional[dict] = None):
                 self.default_timezone = parsed["default-timezone"]
             if "keep-additional-fields" in parsed:
                 self.keep_additional_fields = bool(parsed["keep-additional-fields"])
+            if "enable-crowd-sourced-realtime" in parsed:
+                self.enable_crowd_sourced_realtime = bool(parsed["enable-crowd-sourced-realtime"])
 
 
 class HttpOptions: