Merge branch 'dev'

Author: davidpomerenke

backend-python/src/media_impact_monitor/data_loaders/news_online/mediacloud_.py

@@ -1,9 +1,11 @@
 import random
 from datetime import date, timedelta
 from typing import Literal
+from time import sleep
 
 import mediacloud.api
 import pandas as pd
+from tqdm import tqdm
 from dateutil.relativedelta import relativedelta
 from mcmetadata import extract
 from mcmetadata.exceptions import BadContentError
@@ -55,6 +57,7 @@ def get_mediacloud_counts(
 
 @cache
 def _story_list(**kwargs):
+    sleep(30) # undocumented rate limit, see https://github.com/mediacloud/api-client/issues/107#issuecomment-2977041385
     return search.story_list(**kwargs)
 
 
@@ -126,12 +129,7 @@ def func(start_and_end):
         )
 
     label = "Downloading metadata by month"
-    stories_lists = parallel_tqdm(
-        func,
-        _slice_date_range(start_date, end_date),
-        desc=f"{label:<{40}}",
-        n_jobs=8,
-    )
+    stories_lists = [func(start_and_end) for start_and_end in tqdm(_slice_date_range(start_date, end_date), desc=f"{label:<{40}}")]
     stories = [s for sl in stories_lists for s in sl]
     if len(stories) == 0:
         return None

backend-python/src/media_impact_monitor/fulltexts_test.py

@@ -34,8 +34,8 @@ def test_get_fulltexts_for_org(default_start_date, default_end_date):
 
 def test_get_fulltexts_for_event():
     for event_id in [
-        "5599ac04e1fab5e59639d5646ed331ec",  # LG event on 2024-05-17
-        "7c46b5c07afe076ee58ae1c79eccc367",  # LG event on 2024-05-18
+        "36c654cae3ba2fc2ec18d6e537a0f286",
+        "1e9fd86a4b093ee9edb7f807fd30078f",
     ]:
         texts = get_fulltexts(
             FulltextSearch(