Add health check endpoint for flight totals refresh task

- Celery task sets cache timestamp on completion
- Health endpoint checks cache instead of querying DB
- Returns 503 if task hasn't run in 20+ minutes

Author: ericholscher

adserver/tasks.py

@@ -7,6 +7,7 @@
 from django.conf import settings
 from django.contrib.sites.shortcuts import get_current_site
 from django.core import mail
+from django.core.cache import cache
 from django.db.models import Count
 from django.db.models import F
 from django.db.models import FloatField
@@ -963,6 +964,13 @@ def refresh_flight_denormalized_totals():
         duration,
     )
 
+    # Update cache with last successful run timestamp
+    cache.set(
+        "flight_totals_last_refresh",
+        timezone.now().isoformat(),
+        timeout=None,  # Never expire
+    )
+
     # Alert if there are significant failures
     if error_count > 0 and error_count / max(total_flights, 1) > 0.1:
         slack_message(

adserver/tests/test_health_endpoints.py

@@ -0,0 +1,57 @@
+"""Tests for health check endpoints."""
+
+from datetime import timedelta
+
+from django.core.cache import cache
+from django.test import TestCase
+from django.urls import reverse
+from django.utils import timezone
+
+
+class FlightTotalsHealthTest(TestCase):
+    """Tests for the flight totals health check endpoint."""
+
+    def setUp(self):
+        """Clear cache before each test."""
+        cache.clear()
+
+    def test_health_check_no_cache(self):
+        """Test health check returns 503 when task has never run."""
+        response = self.client.get(reverse("health-flight-totals"))
+        self.assertEqual(response.status_code, 503)
+        data = response.json()
+        self.assertEqual(data["status"], "error")
+        self.assertIn("never run", data["message"])
+
+    def test_health_check_recent_run(self):
+        """Test health check returns 200 when task ran recently."""
+        # Set cache to indicate task ran 5 minutes ago
+        cache.set("flight_totals_last_refresh", timezone.now().isoformat())
+
+        response = self.client.get(reverse("health-flight-totals"))
+        self.assertEqual(response.status_code, 200)
+        data = response.json()
+        self.assertEqual(data["status"], "ok")
+        self.assertLessEqual(data["minutes_since_refresh"], 1)
+
+    def test_health_check_stale_run(self):
+        """Test health check returns 503 when task hasn't run in a while."""
+        # Set cache to indicate task ran 30 minutes ago (stale)
+        stale_time = timezone.now() - timedelta(minutes=30)
+        cache.set("flight_totals_last_refresh", stale_time.isoformat())
+
+        response = self.client.get(reverse("health-flight-totals"))
+        self.assertEqual(response.status_code, 503)
+        data = response.json()
+        self.assertEqual(data["status"], "error")
+        self.assertGreater(data["minutes_since_refresh"], 20)
+
+    def test_health_check_invalid_timestamp(self):
+        """Test health check returns 503 when cache contains invalid data."""
+        cache.set("flight_totals_last_refresh", "invalid-timestamp")
+
+        response = self.client.get(reverse("health-flight-totals"))
+        self.assertEqual(response.status_code, 503)
+        data = response.json()
+        self.assertEqual(data["status"], "error")
+        self.assertIn("Invalid timestamp", data["message"])

adserver/urls.py

@@ -67,6 +67,7 @@
 from .views import dashboard
 from .views import do_not_track
 from .views import do_not_track_policy
+from .views import flight_totals_health
 
 
 urlpatterns = [
@@ -82,6 +83,12 @@
     # Do not Track
     path(r".well-known/dnt/", do_not_track, name="dnt-status"),
     path(r".well-known/dnt-policy.txt", do_not_track_policy, name="dnt-policy"),
+    # Health checks
+    path(
+        r"health/flight-totals/",
+        flight_totals_health,
+        name="health-flight-totals",
+    ),
     # Ad API
     path(r"api/v1/", include("adserver.api.urls")),
     # Staff interface

adserver/views.py

@@ -159,6 +159,73 @@ def do_not_track(request):
     return JsonResponse(data, content_type="application/tracking-status+json")
 
 
+def flight_totals_health(request):
+    """
+    Health check endpoint for Flight denormalized totals refresh task.
+
+    Returns JSON with task status and HTTP 200 if the task has run recently,
+    or HTTP 503 if the task hasn't run within the expected interval.
+
+    This monitors the periodic Celery task that refreshes denormalized totals
+    (total_views, total_clicks) on the Flight model.
+    """
+    from datetime import timedelta
+
+    from django.core.cache import cache
+    from django.utils.dateparse import parse_datetime
+
+    # Get the last time the task successfully ran from cache
+    last_refresh = cache.get("flight_totals_last_refresh")
+
+    if not last_refresh:
+        return JsonResponse(
+            {
+                "status": "error",
+                "message": "Task has never run or cache was cleared",
+            },
+            status=503,
+        )
+
+    # Parse the ISO timestamp
+    last_refresh_time = parse_datetime(last_refresh)
+    if not last_refresh_time:
+        return JsonResponse(
+            {
+                "status": "error",
+                "message": "Invalid timestamp in cache",
+            },
+            status=503,
+        )
+
+    now = timezone.now() if settings.USE_TZ else datetime.now()
+
+    # Calculate how long ago the task ran
+    time_since_refresh = now - last_refresh_time
+
+    # Task runs every 10 minutes, so alert if it hasn't run in 20 minutes
+    max_staleness = timedelta(minutes=20)
+
+    if time_since_refresh > max_staleness:
+        return JsonResponse(
+            {
+                "status": "error",
+                "last_refresh": last_refresh,
+                "minutes_since_refresh": int(time_since_refresh.total_seconds() / 60),
+                "max_minutes": int(max_staleness.total_seconds() / 60),
+            },
+            status=503,
+        )
+
+    return JsonResponse(
+        {
+            "status": "ok",
+            "last_refresh": last_refresh,
+            "minutes_since_refresh": int(time_since_refresh.total_seconds() / 60),
+        },
+        status=200,
+    )
+
+
 def do_not_track_policy(request):
     """
     Returns the Do Not Track policy.