From 77008ee6a284c2a00d0fc644fee07c62e5692b06 Mon Sep 17 00:00:00 2001 From: Klarer Sandro Date: Thu, 31 Oct 2024 16:36:13 +0100 Subject: [PATCH 1/2] Speed up parsing the NMEEA messages. Using the pandas concat() function is very time consuming. Changed it according to: https://stackoverflow.com/questions/57000903/what-is-the-fastest-and-most-efficient-way-to-append-rows-to-a-dataframehttps://stackoverflow.com/questions/57000903/what-is-the-fastest-and-most-efficient-way-to-append-rows-to-a-dataframe Parsing 800MB of MNEA messages takes 3500s, now it takes 180s --- CONTRIBUTORS.md | 1 + gnss_lib_py/parsers/nmea.py | 11 ++++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 7e4b0591..bfc573a2 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -7,3 +7,4 @@ Shivam Soni Sriramya Bhamidipati Dalton Vega Daniel Neamati +Sandro Klarer diff --git a/gnss_lib_py/parsers/nmea.py b/gnss_lib_py/parsers/nmea.py index ae858a4e..83685a06 100644 --- a/gnss_lib_py/parsers/nmea.py +++ b/gnss_lib_py/parsers/nmea.py @@ -67,7 +67,7 @@ def __init__(self, input_path, msg_types=None, if msg_types is None: # Use default message types msg_types = ['GGA', 'RMC'] - pd_df = pd.DataFrame() + temporary_dictionary_list_df = [] field_dict = {} prev_timestamp = None @@ -101,8 +101,7 @@ def __init__(self, input_path, msg_types=None, delta_t = datetime.datetime.combine(datestamp(date), timestamp(time)) field_dict['gps_millis'] = datetime_to_gps_millis(delta_t) - new_row = pd.DataFrame([field_dict]) - pd_df = pd.concat([pd_df, new_row]) + temporary_dictionary_list_df.append(field_dict) field_dict = {} prev_timestamp = msg.timestamp if "sentence_type" in msg.__dir__() and msg.sentence_type in msg_types: @@ -134,8 +133,10 @@ def __init__(self, input_path, msg_types=None, date = field_dict.pop('datestamp') delta_t = datetime.datetime.combine(datestamp(date), timestamp(time)) field_dict['gps_millis'] = datetime_to_gps_millis(delta_t) - new_row = pd.DataFrame([field_dict]) - pd_df = pd.concat([pd_df, new_row]) + + temporary_dictionary_list_df.append(field_dict) + + pd_df = pd.DataFrame.from_dict(temporary_dictionary_list_df) # As per `gnss_lib_py` standards, convert the heading from degrees # to radians pd_df['true_course_rad'] = (np.pi/180.)*pd_df['true_course']\ From 24be7e0c76ae44e0ddcc325b6009c91429769fed Mon Sep 17 00:00:00 2001 From: Derek Knowles Date: Sat, 5 Apr 2025 20:01:41 -0700 Subject: [PATCH 2/2] Update nmea.py --- gnss_lib_py/parsers/nmea.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/gnss_lib_py/parsers/nmea.py b/gnss_lib_py/parsers/nmea.py index 83685a06..9e1977e3 100644 --- a/gnss_lib_py/parsers/nmea.py +++ b/gnss_lib_py/parsers/nmea.py @@ -133,9 +133,7 @@ def __init__(self, input_path, msg_types=None, date = field_dict.pop('datestamp') delta_t = datetime.datetime.combine(datestamp(date), timestamp(time)) field_dict['gps_millis'] = datetime_to_gps_millis(delta_t) - temporary_dictionary_list_df.append(field_dict) - pd_df = pd.DataFrame.from_dict(temporary_dictionary_list_df) # As per `gnss_lib_py` standards, convert the heading from degrees # to radians