fixes to the csv output so it produces CSVs identical to the legacy downloader

Author: chatch

simt_emlite/cli/profile_download.py

@@ -119,7 +119,7 @@ def download_single_day(
             serial=downloader.serial,
             output_dir=output_dir,
             readings=readings_a,
-            element_marker="A",
+            element_marker="A" if downloader.is_twin_element else None,
         )
         print(f"Wrote {len(readings_a)} readings to CSV in {output_dir}")
 

simt_emlite/smip/smip_csv.py

@@ -66,24 +66,38 @@ def write(
 
         # Write CSV file
         with open(full_csv_path, "w", newline="") as csvfile:
-            writer = csv.writer(csvfile)
+            writer = csv.writer(csvfile, quoting=csv.QUOTE_ALL)
 
-            # Write header
-            header = ["Timestamp", "Import", "Export"]
-            if element_marker:
-                header.append("Element")
+            # Build column name with optional element marker (e.g., "EML123456789-A")
+            column_name = f"{serial}-{element_marker}" if element_marker else serial
+
+            # Write header with quoted column names
+            header = ["created_at", column_name, f"{column_name}_rev"]
             writer.writerow(header)
 
-            # Write records
+            # Write records (no quoting for data rows)
             for record in records:
-                row = [
-                    record.timestamp.isoformat(),
-                    str(record.import_value) if record.import_value is not None else "",
-                    str(record.export_value) if record.export_value is not None else "",
-                ]
-                if element_marker:
-                    row.append(element_marker)
-                writer.writerow(row)
+                # Format timestamp as "YYYY-MM-DD HH:MM:SS+0000" (no T, no colon in timezone)
+                ts = record.timestamp
+                # Format with strftime to get the base, then handle timezone
+                timestamp_str = ts.strftime("%Y-%m-%d %H:%M:%S%z")
+                # Remove colon from timezone offset (+00:00 -> +0000)
+                # strftime %z already gives +0000 format in Python
+
+                # Format values as integers (no decimal places)
+                import_str = (
+                    str(int(record.import_value))
+                    if record.import_value is not None
+                    else ""
+                )
+                export_str = (
+                    str(int(record.export_value))
+                    if record.export_value is not None
+                    else ""
+                )
+
+                # Write the raw line directly to avoid quoting data rows
+                csvfile.write(f"{timestamp_str},{import_str},{export_str}\n")
 
     @staticmethod
     def write_from_profile_records(

tests/simt_emlite/smip/test_smip_csv.py

@@ -1,14 +1,17 @@
-import unittest
 import os
+import unittest
+
 from simt_emlite.smip.smip_csv import SMIPCSV
 from simt_emlite.smip.smip_csv_record import SMIPCSVRecord
 
-class TestSMIPCSV(unittest.TestCase):
 
+class TestSMIPCSV(unittest.TestCase):
     def test_read_smip_file_with_nas(self):
         """Test reading SMIP file with NA values (should be skipped)"""
         # A number of records with NA values skipped (see unfuddle #410)
-        test_file = os.path.join(os.path.dirname(__file__), 'test_data', 'EML2207642296-20221013.csv')
+        test_file = os.path.join(
+            os.path.dirname(__file__), "test_data", "EML2207642296-20221013.csv"
+        )
 
         records = SMIPCSV.read_from_file(test_file)
 
@@ -27,7 +30,9 @@ def test_read_smip_file_with_nas(self):
 
     def test_read_smip_file_all_nas(self):
         """Test reading SMIP file with all NA values (should return empty list)"""
-        test_file = os.path.join(os.path.dirname(__file__), 'test_data', 'EML2137580799-A-20211210.csv')
+        test_file = os.path.join(
+            os.path.dirname(__file__), "test_data", "EML2137580799-A-20211210.csv"
+        )
 
         records = SMIPCSV.read_from_file(test_file)
 
@@ -36,7 +41,9 @@ def test_read_smip_file_all_nas(self):
 
     def test_read_smip_file_with_neg1s(self):
         """Test reading SMIP file with -1 values (should be skipped)"""
-        test_file = os.path.join(os.path.dirname(__file__), 'test_data', 'EML2137580799-A-20211203.csv')
+        test_file = os.path.join(
+            os.path.dirname(__file__), "test_data", "EML2137580799-A-20211203.csv"
+        )
 
         records = SMIPCSV.read_from_file(test_file)
 
@@ -78,36 +85,35 @@ def test_read_smip_string(self):
 
     def test_write_smip_file(self):
         """Test writing SMIP records to a file"""
-        import tempfile
         import os
+        import tempfile
 
         # Create test records
         from datetime import datetime, timezone
+
         records = [
             SMIPCSVRecord(
                 timestamp=datetime(2023, 12, 10, 14, 30, 0, tzinfo=timezone.utc),
                 import_value=123.45,
-                export_value=56.78
+                export_value=56.78,
             ),
             SMIPCSVRecord(
                 timestamp=datetime(2023, 12, 10, 15, 0, 0, tzinfo=timezone.utc),
                 import_value=234.56,
-                export_value=67.89
+                export_value=67.89,
             ),
             SMIPCSVRecord(
                 timestamp=datetime(2023, 12, 10, 15, 30, 0, tzinfo=timezone.utc),
                 import_value=345.67,
-                export_value=78.90
-            )
+                export_value=78.90,
+            ),
         ]
 
         # Create temporary directory
         with tempfile.TemporaryDirectory() as temp_dir:
             # Write records to file
             SMIPCSV.write(
-                serial="EML123456789",
-                output_file_path=temp_dir,
-                records=records
+                serial="EML123456789", output_file_path=temp_dir, records=records
             )
 
             # Check that file was created
@@ -116,40 +122,41 @@ def test_write_smip_file(self):
             self.assertTrue(os.path.exists(expected_filepath))
 
             # Read the file and verify content
-            with open(expected_filepath, 'r') as f:
+            with open(expected_filepath, "r") as f:
                 content = f.read()
 
-            # Check header
-            self.assertIn("Timestamp,Import,Export", content)
+            # Check header (new format with quoted column names)
+            self.assertIn('"created_at","EML123456789","EML123456789_rev"', content)
 
-            # Check that all records are present
-            self.assertIn("2023-12-10T14:30:00+00:00", content)
-            self.assertIn("2023-12-10T15:00:00+00:00", content)
-            self.assertIn("2023-12-10T15:30:00+00:00", content)
+            # Check that all records are present (new timestamp format without T)
+            self.assertIn("2023-12-10 14:30:00+0000", content)
+            self.assertIn("2023-12-10 15:00:00+0000", content)
+            self.assertIn("2023-12-10 15:30:00+0000", content)
 
-            # Check that values are present (written as-is in kWh)
-            self.assertIn("123.45", content)
-            self.assertIn("234.56", content)
-            self.assertIn("345.67", content)
+            # Check that values are present (written as integers)
+            self.assertIn(",123,", content)
+            self.assertIn(",234,", content)
+            self.assertIn(",345,", content)
 
     def test_write_smip_file_with_element_marker(self):
         """Test writing SMIP records with element marker"""
-        import tempfile
         import os
+        import tempfile
 
         # Create test records
         from datetime import datetime, timezone
+
         records = [
             SMIPCSVRecord(
                 timestamp=datetime(2023, 12, 10, 14, 30, 0, tzinfo=timezone.utc),
                 import_value=123.45,
-                export_value=56.78
+                export_value=56.78,
             ),
             SMIPCSVRecord(
                 timestamp=datetime(2023, 12, 10, 15, 0, 0, tzinfo=timezone.utc),
                 import_value=234.56,
-                export_value=67.89
-            )
+                export_value=67.89,
+            ),
         ]
 
         # Create temporary directory
@@ -159,7 +166,7 @@ def test_write_smip_file_with_element_marker(self):
                 serial="EML123456789",
                 output_file_path=temp_dir,
                 records=records,
-                element_marker="A"
+                element_marker="A",
             )
 
             # Check that file was created with element marker
@@ -168,21 +175,21 @@ def test_write_smip_file_with_element_marker(self):
             self.assertTrue(os.path.exists(expected_filepath))
 
             # Read the file and verify content
-            with open(expected_filepath, 'r') as f:
+            with open(expected_filepath, "r") as f:
                 content = f.read()
 
-            # Check header includes Element column
-            self.assertIn("Timestamp,Import,Export,Element", content)
+            # Check header (new format with serial-element as column names)
+            self.assertIn('"created_at","EML123456789-A","EML123456789-A_rev"', content)
 
-            # Check that element marker is present
-            self.assertIn("A", content)
+            # Check that timestamp format is correct (space-separated, no T)
+            self.assertIn("2023-12-10 14:30:00+0000", content)
 
     def test_write_from_smip_readings(self):
         """Test writing SMIPReading objects to a file"""
-        import tempfile
         import os
-
+        import tempfile
         from datetime import datetime, timezone
+
         from simt_emlite.smip.smip_reading import SMIPReading
 
         # Create test SMIPReading objects
@@ -193,24 +200,24 @@ def test_write_from_smip_readings(self):
                 timestamp=datetime(2023, 12, 10, 14, 30, 0, tzinfo=timezone.utc),
                 imp=123.45,
                 exp=56.78,
-                errorCode=0
+                errorCode=0,
             ),
             SMIPReading(
                 serial="EML123456789",
                 register=1,
                 timestamp=datetime(2023, 12, 10, 15, 0, 0, tzinfo=timezone.utc),
                 imp=234.56,
                 exp=67.89,
-                errorCode=0
+                errorCode=0,
             ),
             SMIPReading(
                 serial="EML123456789",
                 register=1,
                 timestamp=datetime(2023, 12, 10, 15, 30, 0, tzinfo=timezone.utc),
                 imp=345.67,
                 exp=78.90,
-                errorCode=0
-            )
+                errorCode=0,
+            ),
         ]
 
         # Create temporary directory
@@ -220,7 +227,7 @@ def test_write_from_smip_readings(self):
                 serial="EML123456789",
                 output_dir=temp_dir,
                 readings=readings,
-                element_marker="A"
+                element_marker="A",
             )
 
             # Check that file was created with element marker
@@ -229,22 +236,22 @@ def test_write_from_smip_readings(self):
             self.assertTrue(os.path.exists(expected_filepath))
 
             # Read the file and verify content
-            with open(expected_filepath, 'r') as f:
+            with open(expected_filepath, "r") as f:
                 content = f.read()
 
-            # Check header includes Element column
-            self.assertIn("Timestamp,Import,Export,Element", content)
+            # Check header (new format with serial-element as column names)
+            self.assertIn('"created_at","EML123456789-A","EML123456789-A_rev"', content)
 
-            # Check that all records are present
-            self.assertIn("2023-12-10T14:30:00+00:00", content)
-            self.assertIn("2023-12-10T15:00:00+00:00", content)
-            self.assertIn("2023-12-10T15:30:00+00:00", content)
+            # Check that all records are present (new timestamp format without T)
+            self.assertIn("2023-12-10 14:30:00+0000", content)
+            self.assertIn("2023-12-10 15:00:00+0000", content)
+            self.assertIn("2023-12-10 15:30:00+0000", content)
 
-            # Check that values are present (written as-is in kWh)
-            self.assertIn("123.45", content)
-            self.assertIn("234.56", content)
-            self.assertIn("345.67", content)
+            # Check that values are present (written as integers)
+            self.assertIn(",123,", content)
+            self.assertIn(",234,", content)
+            self.assertIn(",345,", content)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     unittest.main()