From 7fb2729146b61c8b09f40674e0e0d65176ca5573 Mon Sep 17 00:00:00 2001 From: linhnam-nguyen <88286063+linhnam-nguyen@users.noreply.github.com> Date: Thu, 11 Sep 2025 11:51:35 +0200 Subject: [PATCH 01/16] Initializing --- File_Engine/Compute/ReadFromCsvFile.cs | 93 ++++++++ File_Engine/Compute/WriteToCsvFile.cs | 310 +++++++++++++++++++++++++ File_oM/Config/CsvSettings.cs | 48 ++++ File_oM/enums/FileFormat.cs | 2 + 4 files changed, 453 insertions(+) create mode 100644 File_Engine/Compute/ReadFromCsvFile.cs create mode 100644 File_Engine/Compute/WriteToCsvFile.cs create mode 100644 File_oM/Config/CsvSettings.cs diff --git a/File_Engine/Compute/ReadFromCsvFile.cs b/File_Engine/Compute/ReadFromCsvFile.cs new file mode 100644 index 0000000..65b468a --- /dev/null +++ b/File_Engine/Compute/ReadFromCsvFile.cs @@ -0,0 +1,93 @@ +/* + * This file is part of the Buildings and Habitats object Model (BHoM) + * Copyright (c) 2015 - 2025, the respective contributors. All rights reserved. + * + * Each contributor holds copyright over their respective contributions. + * The project versioning (Git) records all such contribution source information. + * + * + * The BHoM is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3.0 of the License, or + * (at your option) any later version. + * + * The BHoM is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this code. If not, see . + */ + +using BH.oM.Base; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using BH.Engine.Serialiser; +using BH.Engine.Adapters.File; +using BH.oM.Adapters.File; +using System.Collections; +using BH.oM.Adapter; +using System.ComponentModel; +using BH.oM.Base.Attributes; + +namespace BH.Engine.Adapters.File +{ + public static partial class Compute + { + /***************************************************/ + /**** Public Methods ****/ + /***************************************************/ + + [Description("Read a JSON-serialised file and output any data or object included in it.")] + [Input("filePath", "Path to the file.")] + [Input("active", "Boolean used to trigger the function.")] + public static List ReadFromCsvFile(string filePath, bool active = false) + { + if (!active) + return new List(); + + if (string.IsNullOrWhiteSpace(filePath)) + { + BH.Engine.Base.Compute.RecordError($"The filePath `{filePath}` must not be empty."); + return new List(); + } + + // Make sure no invalid chars are present. + string fullPath = Path.Combine(Path.GetDirectoryName(filePath), Path.GetFileName(filePath)); + + bool fileExisted = System.IO.File.Exists(fullPath); + if (!fileExisted) + { + BH.Engine.Base.Compute.RecordError($"The file `{fullPath}` does not exist."); + return new List(); + } + + string jsonText = System.IO.File.ReadAllText(fullPath); + object converted = BH.Engine.Serialiser.Convert.FromJson(jsonText); + + // Check if there is any ObjectWrapper that was used to allow writing of non-IObjects, like primitive types (numbers/strings). + List convertedList = new List(); + IEnumerable ienum = converted as IEnumerable; + if (ienum != null) + { + foreach (var obj in ienum) + { + ObjectWrapper objectWrapper = obj as ObjectWrapper; + if (objectWrapper != null) + convertedList.Add(objectWrapper.WrappedObject); + else + convertedList.Add(obj); + } + + return convertedList; + } + + return new List() { converted }; + } + } +} + + diff --git a/File_Engine/Compute/WriteToCsvFile.cs b/File_Engine/Compute/WriteToCsvFile.cs new file mode 100644 index 0000000..00299c1 --- /dev/null +++ b/File_Engine/Compute/WriteToCsvFile.cs @@ -0,0 +1,310 @@ +/* + * This file is part of the Buildings and Habitats object Model (BHoM) + * Copyright (c) 2015 - 2025, the respective contributors. All rights reserved. + * + * Each contributor holds copyright over their respective contributions. + * The project versioning (Git) records all such contribution source information. + * + * + * The BHoM is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3.0 of the License, or + * (at your option) any later version. + * + * The BHoM is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this code. If not, see . + */ + +using BH.Engine.Adapters.File; +using BH.Engine.Serialiser; +using BH.oM.Adapter; +using BH.oM.Adapters.File; +using BH.oM.Base; +using BH.oM.Base.Attributes; +using System; +using System.Collections; +using System.Collections.Generic; +using System.ComponentModel; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; + +namespace BH.Engine.Adapters.File +{ + public static partial class Compute + { + /***************************************************/ + /**** Public Methods ****/ + /***************************************************/ + + [Description("Write a JSON-serialised file with the input data or objects.")] + [Input("objects", "Objects to write to the file.")] + [Input("filePath", "Path to the file.")] + [Input("replace", "If the file exists, you need to set this to true in order to allow overwriting it.")] + [Input("active", "Boolean used to trigger the function.")] + public static bool WriteToCsvFile(List lines, string filePath, CsvSettings settings = null, bool replace = false, bool active = false) + { + if (!active || lines == null) + return false; + + if (string.IsNullOrWhiteSpace(filePath)) + { + BH.Engine.Base.Compute.RecordError($"The filePath `{filePath}` must not be empty."); + return false; + } + + // Make sure no invalid chars are present. + filePath = Path.Combine(Path.GetDirectoryName(filePath), Path.GetFileName(filePath)); + + // If the file exists already, stop execution if `replace` is not true. + bool fileExisted = System.IO.File.Exists(filePath); + if (!replace && fileExisted) + { + BH.Engine.Base.Compute.RecordWarning($"The file `{filePath}` exists already. To replace its content, set `{nameof(replace)}` to true."); + return false; + } + + // Serialise to json and create the file and directory. + string table = From2DArray(lines); + System.IO.FileInfo fileInfo = new System.IO.FileInfo(filePath); + fileInfo.Directory.Create(); // If the directory already exists, this method does nothing. + + try + { + System.IO.File.WriteAllText(filePath, table); + } + catch (Exception e) + { + BH.Engine.Base.Compute.RecordError($"Error writing to file:\n\t{e.ToString()}"); + return false; + } + + return true; + } + + /***************************************************/ + /**** Private Methods ****/ + /***************************************************/ + + // Convert a list of objects to a CSV text string, taking care of the following types: List>, List, string[,], string[][], string[] + public static string From2DArray(this object obj, CsvSettings settings = null) + { + if (settings == null) + settings = new CsvSettings(); + + var delim = settings.Delimiter ?? "\t"; + + // Normalize to list of rows (each row = string[]) + var rows = new List(); + + switch (obj) + { + // Rectangular array + case string[,] rect: + { + int r = rect.GetLength(0); + int c = rect.GetLength(1); + for (int i = 0; i < r; i++) + { + var row = new string[c]; + for (int j = 0; j < c; j++) + row[j] = rect[i, j] ?? string.Empty; + rows.Add(row); + } + break; + } + + // Jagged array + case string[][] jagged: + { + foreach (var row in jagged) + rows.Add((row ?? Array.Empty()).Select(v => v ?? string.Empty).ToArray()); + break; + } + + // Any "sequence of sequences of string" (List>, IEnumerable, etc.) + case IEnumerable> seqOfSeq: + { + foreach (var inner in seqOfSeq) + rows.Add((inner ?? Enumerable.Empty()).Select(v => v ?? string.Empty).ToArray()); + break; + } + + // Any "sequence of string" (treated as a single row); exclude string itself + case IEnumerable seq when !(obj is string): + { + rows.Add(seq.Select(v => v ?? string.Empty).ToArray()); + break; + } + + // Single string cell + case string s: + { + rows.Add(new[] { s }); + break; + } + + // Fallback: single cell ToString() + default: + { + rows.Add(new[] { obj?.ToString() ?? string.Empty }); + break; + } + } + + if (rows.Count == 0) + return string.Empty; + + // Pad ragged rows + int maxCols = rows.Max(r => r == null ? 0 : r.Length); + if (maxCols == 0) maxCols = 1; + + for (int i = 0; i < rows.Count; i++) + { + var r = rows[i] ?? Array.Empty(); + if (r.Length < maxCols) + rows[i] = r.Concat(Enumerable.Repeat(string.Empty, maxCols - r.Length)).ToArray(); + } + + // Optional index column (1-based) + if (settings.IncludeIndex) + { + for (int i = 0; i < rows.Count; i++) + { + var newRow = new string[rows[i].Length + 1]; + newRow[0] = (i + 1).ToString(CultureInfo.InvariantCulture); + Array.Copy(rows[i], 0, newRow, 1, rows[i].Length); + rows[i] = newRow; + } + } + + // Numeric rounding (if requested) + if (settings.Digit.HasValue) + { + int digits = settings.Digit.Value; + var format = digits > 0 ? "0." + new string('#', digits) : "0"; + + for (int i = 0; i < rows.Count; i++) + { + for (int j = 0; j < rows[i].Length; j++) + { + var v = rows[i][j]; + double num; + if (TryParseNumber(v, out num)) + { + rows[i][j] = Math.Round(num, digits).ToString(format, CultureInfo.InvariantCulture); + } + } + } + } + + // Serialize with escaping + var sb = new StringBuilder(rows.Count * maxCols * 4); + for (int i = 0; i < rows.Count; i++) + { + var encoded = rows[i].Select(cell => EscapeCsv(cell ?? string.Empty, delim)); + sb.Append(string.Join(delim, encoded)); + if (i < rows.Count - 1) + sb.Append('\n'); + } + + return sb.ToString(); + } + + + // --- Helpers --- + + private static bool TryParseNumber(string s, out double value) + { + // Try strict parse with InvariantCulture; also try trimming spaces. + if (double.TryParse(s, NumberStyles.Float | NumberStyles.AllowThousands, CultureInfo.InvariantCulture, out value)) + return true; + + if (double.TryParse((s ?? string.Empty).Trim(), NumberStyles.Float | NumberStyles.AllowThousands, CultureInfo.InvariantCulture, out value)) + return true; + + // Optional: try replacing comma with dot if it looks like a localized decimal + var swapped = (s ?? string.Empty).Replace(',', '.'); + return double.TryParse(swapped, NumberStyles.Float | NumberStyles.AllowThousands, CultureInfo.InvariantCulture, out value); + } + + private static string EscapeCsv(string input, string delimiter) + { + if (input == null) return string.Empty; + + bool mustQuote = + input.Contains('"') || + input.Contains('\n') || + input.Contains('\r') || + (!string.IsNullOrEmpty(delimiter) && input.Contains(delimiter)); + + if (!mustQuote) + return input; + + // Double quotes inside quoted field + var doubled = input.Replace("\"", "\"\""); + return $"\"{doubled}\""; + } + + + private static bool IsEnumerableOfEnumerableOfString(object x, out IEnumerable> result) + { + result = null; + + if (x == null || x is string) return false; + + var outer = x as IEnumerable; + if (outer == null) return false; + + var rows = new List>(); + + foreach (var inner in outer) + { + if (!IsEnumerableOfString(inner, out var row)) + return false; // any inner not IEnumerable -> fail + rows.Add(row); + } + + result = rows; + return true; + } + + private static bool IsEnumerableOfString(object x, out IEnumerable result) + { + result = null; + + // Exclude string itself (it is IEnumerable) + if (x == null || x is string) return false; + + var enumerable = x as IEnumerable; + if (enumerable == null) return false; + + var list = new List(); + foreach (var item in enumerable) + { + if (item == null) { list.Add(string.Empty); continue; } + var str = item as string; + if (str == null) return false; // not a pure IEnumerable + list.Add(str); + } + + result = list; + return true; + } + + + } +} + + + + + + + diff --git a/File_oM/Config/CsvSettings.cs b/File_oM/Config/CsvSettings.cs new file mode 100644 index 0000000..b752be6 --- /dev/null +++ b/File_oM/Config/CsvSettings.cs @@ -0,0 +1,48 @@ +/* + * This file is part of the Buildings and Habitats object Model (BHoM) + * Copyright (c) 2015 - 2025, the respective contributors. All rights reserved. + * + * Each contributor holds copyright over their respective contributions. + * The project versioning (Git) records all such contribution source information. + * + * + * The BHoM is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3.0 of the License, or + * (at your option) any later version. + * + * The BHoM is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this code. If not, see . + */ + + +using BH.oM.Adapter; +using System.ComponentModel; + +namespace BH.oM.Adapters.File +{ + public class CsvSettings : ActionConfig + { + [Description(" The delimiter to use in the CSV file. Common options are ',' for comma, ';' for semicolon, and '\\t' for tab by default.")] + public string Delimiter { get; set; } = "\t"; + + [Description(" If specified, numerical values will be rounded to this number of decimal places. If null, no rounding is applied.")] + public int? Digit { get; set; } = null; + + [Description(" Whether to include a header row with column names in the CSV file.")] + public bool IncludeHeader { get; set; } = true; + + [Description(" Whether to include an index column at the start of the CSV file.")] + public bool IncludeIndex { get; set; } = false; + } +} + + + + + diff --git a/File_oM/enums/FileFormat.cs b/File_oM/enums/FileFormat.cs index 1615b66..afc7136 100644 --- a/File_oM/enums/FileFormat.cs +++ b/File_oM/enums/FileFormat.cs @@ -33,6 +33,8 @@ public enum FileFormat JSON, BSON, XML, + TXT, + CSV, byteArray } } From 553420f23b2417b1eea1ffb66007610139f6ba0c Mon Sep 17 00:00:00 2001 From: linhnam-nguyen <88286063+linhnam-nguyen@users.noreply.github.com> Date: Fri, 12 Sep 2025 16:17:48 +0200 Subject: [PATCH 02/16] Add DateTime conversion --- File_Engine/Compute/WriteToCsvFile.cs | 254 ++++++++++++++------------ File_oM/Config/CsvSettings.cs | 16 +- File_oM/enums/DateTime.cs | 42 +++++ 3 files changed, 192 insertions(+), 120 deletions(-) create mode 100644 File_oM/enums/DateTime.cs diff --git a/File_Engine/Compute/WriteToCsvFile.cs b/File_Engine/Compute/WriteToCsvFile.cs index 00299c1..263ea69 100644 --- a/File_Engine/Compute/WriteToCsvFile.cs +++ b/File_Engine/Compute/WriteToCsvFile.cs @@ -20,14 +20,9 @@ * along with this code. If not, see . */ -using BH.Engine.Adapters.File; -using BH.Engine.Serialiser; -using BH.oM.Adapter; using BH.oM.Adapters.File; -using BH.oM.Base; using BH.oM.Base.Attributes; using System; -using System.Collections; using System.Collections.Generic; using System.ComponentModel; using System.Globalization; @@ -35,6 +30,7 @@ using System.Linq; using System.Text; + namespace BH.Engine.Adapters.File { public static partial class Compute @@ -48,9 +44,9 @@ public static partial class Compute [Input("filePath", "Path to the file.")] [Input("replace", "If the file exists, you need to set this to true in order to allow overwriting it.")] [Input("active", "Boolean used to trigger the function.")] - public static bool WriteToCsvFile(List lines, string filePath, CsvSettings settings = null, bool replace = false, bool active = false) + public static bool WriteToCsvFile(object data, string filePath, CsvSettings settings = null, bool replace = false, bool active = false) { - if (!active || lines == null) + if (!active || data == null) return false; if (string.IsNullOrWhiteSpace(filePath)) @@ -71,7 +67,7 @@ public static bool WriteToCsvFile(List lines, string filePath, CsvSettin } // Serialise to json and create the file and directory. - string table = From2DArray(lines); + string table = FromObject(data); System.IO.FileInfo fileInfo = new System.IO.FileInfo(filePath); fileInfo.Directory.Create(); // If the directory already exists, this method does nothing. @@ -93,20 +89,20 @@ public static bool WriteToCsvFile(List lines, string filePath, CsvSettin /***************************************************/ // Convert a list of objects to a CSV text string, taking care of the following types: List>, List, string[,], string[][], string[] - public static string From2DArray(this object obj, CsvSettings settings = null) + public static string FromObject(this object obj, CsvSettings settings = null) { if (settings == null) settings = new CsvSettings(); var delim = settings.Delimiter ?? "\t"; - // Normalize to list of rows (each row = string[]) + // 1) Normalize to list of rows (each row = string[]) var rows = new List(); switch (obj) { - // Rectangular array - case string[,] rect: + // --- PRIMARY: Rectangular arrays of objects --- + case object[,] rect: { int r = rect.GetLength(0); int c = rect.GetLength(1); @@ -114,46 +110,61 @@ public static string From2DArray(this object obj, CsvSettings settings = null) { var row = new string[c]; for (int j = 0; j < c; j++) - row[j] = rect[i, j] ?? string.Empty; + row[j] = ToCellString(rect[i, j], settings); rows.Add(row); } break; } - // Jagged array - case string[][] jagged: + // --- PRIMARY: Jagged arrays of objects --- + case object[][] jagged: { - foreach (var row in jagged) - rows.Add((row ?? Array.Empty()).Select(v => v ?? string.Empty).ToArray()); + foreach (var inner in jagged ?? new object[0][]) + { + var src = inner ?? new object[0]; + var row = new string[src.Length]; + for (int j = 0; j < src.Length; j++) + row[j] = ToCellString(src[j], settings); + rows.Add(row); + } break; } - // Any "sequence of sequences of string" (List>, IEnumerable, etc.) - case IEnumerable> seqOfSeq: + // --- SECONDARY: Sequences of sequences of objects --- + case IEnumerable> seqOfSeq: { foreach (var inner in seqOfSeq) - rows.Add((inner ?? Enumerable.Empty()).Select(v => v ?? string.Empty).ToArray()); + { + var items = inner ?? new object[0]; + var rowList = new List(); + foreach (var it in items) + rowList.Add(ToCellString(it, settings)); + rows.Add(rowList.ToArray()); + } break; } - // Any "sequence of string" (treated as a single row); exclude string itself - case IEnumerable seq when !(obj is string): + // --- SECONDARY: Sequence of objects (single row); exclude string itself --- + case IEnumerable seq when !(obj is string): { - rows.Add(seq.Select(v => v ?? string.Empty).ToArray()); + var rowList = new List(); + foreach (var it in seq) + rowList.Add(ToCellString(it, settings)); + rows.Add(rowList.ToArray()); break; } - // Single string cell + // Single string -> one cell case string s: { rows.Add(new[] { s }); break; } - // Fallback: single cell ToString() + // Fallback: single cell from ToString() default: { - rows.Add(new[] { obj?.ToString() ?? string.Empty }); + rows.Add(new[] { ToCellString(obj, settings) }); break; } } @@ -161,50 +172,40 @@ public static string From2DArray(this object obj, CsvSettings settings = null) if (rows.Count == 0) return string.Empty; - // Pad ragged rows - int maxCols = rows.Max(r => r == null ? 0 : r.Length); + // 2) Pad ragged rows to the maximum width + int maxCols = 0; + for (int i = 0; i < rows.Count; i++) + { + var r = rows[i] ?? new string[0]; + if (r.Length > maxCols) maxCols = r.Length; + } if (maxCols == 0) maxCols = 1; for (int i = 0; i < rows.Count; i++) { - var r = rows[i] ?? Array.Empty(); + var r = rows[i] ?? new string[0]; if (r.Length < maxCols) - rows[i] = r.Concat(Enumerable.Repeat(string.Empty, maxCols - r.Length)).ToArray(); + { + var padded = new string[maxCols]; + Array.Copy(r, padded, r.Length); // remaining are null -> treated as empty + rows[i] = padded; + } } - // Optional index column (1-based) + // 3) Optional 1-based index column if (settings.IncludeIndex) { for (int i = 0; i < rows.Count; i++) { - var newRow = new string[rows[i].Length + 1]; + var current = rows[i]; + var newRow = new string[current.Length + 1]; newRow[0] = (i + 1).ToString(CultureInfo.InvariantCulture); - Array.Copy(rows[i], 0, newRow, 1, rows[i].Length); + Array.Copy(current, 0, newRow, 1, current.Length); rows[i] = newRow; } } - // Numeric rounding (if requested) - if (settings.Digit.HasValue) - { - int digits = settings.Digit.Value; - var format = digits > 0 ? "0." + new string('#', digits) : "0"; - - for (int i = 0; i < rows.Count; i++) - { - for (int j = 0; j < rows[i].Length; j++) - { - var v = rows[i][j]; - double num; - if (TryParseNumber(v, out num)) - { - rows[i][j] = Math.Round(num, digits).ToString(format, CultureInfo.InvariantCulture); - } - } - } - } - - // Serialize with escaping + // 4) Serialize with CSV escaping var sb = new StringBuilder(rows.Count * maxCols * 4); for (int i = 0; i < rows.Count; i++) { @@ -217,94 +218,113 @@ public static string From2DArray(this object obj, CsvSettings settings = null) return sb.ToString(); } + /*******************************************/ + /**** Private Methods *****/ + /*******************************************/ - // --- Helpers --- - - private static bool TryParseNumber(string s, out double value) + private static string ToCellString(object value, CsvSettings settings) { - // Try strict parse with InvariantCulture; also try trimming spaces. - if (double.TryParse(s, NumberStyles.Float | NumberStyles.AllowThousands, CultureInfo.InvariantCulture, out value)) - return true; - - if (double.TryParse((s ?? string.Empty).Trim(), NumberStyles.Float | NumberStyles.AllowThousands, CultureInfo.InvariantCulture, out value)) - return true; + if (value == null) + return string.Empty; - // Optional: try replacing comma with dot if it looks like a localized decimal - var swapped = (s ?? string.Empty).Replace(',', '.'); - return double.TryParse(swapped, NumberStyles.Float | NumberStyles.AllowThousands, CultureInfo.InvariantCulture, out value); - } + // string + var s = value as string; + if (s != null) + return s; - private static string EscapeCsv(string input, string delimiter) - { - if (input == null) return string.Empty; + // numeric (round if Digit set) + if (IsNumeric(value)) + { + double d = System.Convert.ToDouble(value, CultureInfo.InvariantCulture); - bool mustQuote = - input.Contains('"') || - input.Contains('\n') || - input.Contains('\r') || - (!string.IsNullOrEmpty(delimiter) && input.Contains(delimiter)); + if (settings.Digit.HasValue) + { + int digits = (int) settings.Digit.Value; + var format = digits > 0 ? "0." + new string('#', digits) : "0"; + return Math.Round(d, digits).ToString(format, CultureInfo.InvariantCulture); + } - if (!mustQuote) - return input; + return d.ToString("G", CultureInfo.InvariantCulture); + } - // Double quotes inside quoted field - var doubled = input.Replace("\"", "\"\""); - return $"\"{doubled}\""; - } + // Date/Time + if (value is DateTime dt) + return dt.FormatDate(settings.DateTimeFormat); - private static bool IsEnumerableOfEnumerableOfString(object x, out IEnumerable> result) - { - result = null; + // Bool + if (value is bool b) + return b ? "true" : "false"; - if (x == null || x is string) return false; + // Enum + var type = value.GetType(); + if (type.IsEnum) + return System.Convert.ToString(value, CultureInfo.InvariantCulture); - var outer = x as IEnumerable; - if (outer == null) return false; + // IFormattable (decimal, Guid, etc.) + var formattable = value as IFormattable; - var rows = new List>(); + if (formattable != null) + return formattable.ToString(null, CultureInfo.InvariantCulture); - foreach (var inner in outer) + if (settings.IncludeObjects) { - if (!IsEnumerableOfString(inner, out var row)) - return false; // any inner not IEnumerable -> fail - rows.Add(row); + // Fallback: use ToString() if overridden + var toString = value.ToString(); + if (toString != null && toString != type.FullName) + return toString; + // Fallback: type name + return $"<{type.Name}>"; } - result = rows; - return true; + // Fallback + return string.Empty; } - private static bool IsEnumerableOfString(object x, out IEnumerable result) + private static bool IsNumeric(object value) { - result = null; + return value is byte || value is sbyte || + value is short || value is ushort || + value is int || value is uint || + value is long || value is ulong || + value is float || value is double || + value is decimal; + } - // Exclude string itself (it is IEnumerable) - if (x == null || x is string) return false; + private static string EscapeCsv(string input, string delimiter) + { + if (input == null) return string.Empty; - var enumerable = x as IEnumerable; - if (enumerable == null) return false; + bool mustQuote = + input.IndexOf('"') >= 0 || + input.IndexOf('\n') >= 0 || + input.IndexOf('\r') >= 0 || + (!string.IsNullOrEmpty(delimiter) && input.IndexOf(delimiter, StringComparison.Ordinal) >= 0); - var list = new List(); - foreach (var item in enumerable) - { - if (item == null) { list.Add(string.Empty); continue; } - var str = item as string; - if (str == null) return false; // not a pure IEnumerable - list.Add(str); - } + if (!mustQuote) + return input; - result = list; - return true; + var doubled = input.Replace("\"", "\"\""); + return "\"" + doubled + "\""; } + private static string FormatDate(this IFormattable date, DateFormatOptions option) + { + switch (option) + { + case DateFormatOptions.ISO8601: + // round-trip, always UTC if DateTime.Kind is UTC + return date.ToString("o", CultureInfo.InvariantCulture); + + case DateFormatOptions.US: + // month/day/year + return date.ToString("MM/dd/yyyy", CultureInfo.InvariantCulture); + case DateFormatOptions.EU: + default: + // day/month/year + return date.ToString("dd/MM/yyyy", CultureInfo.InvariantCulture); + } + } } } - - - - - - - diff --git a/File_oM/Config/CsvSettings.cs b/File_oM/Config/CsvSettings.cs index b752be6..da18bdb 100644 --- a/File_oM/Config/CsvSettings.cs +++ b/File_oM/Config/CsvSettings.cs @@ -31,14 +31,24 @@ public class CsvSettings : ActionConfig [Description(" The delimiter to use in the CSV file. Common options are ',' for comma, ';' for semicolon, and '\\t' for tab by default.")] public string Delimiter { get; set; } = "\t"; - [Description(" If specified, numerical values will be rounded to this number of decimal places. If null, no rounding is applied.")] - public int? Digit { get; set; } = null; - [Description(" Whether to include a header row with column names in the CSV file.")] public bool IncludeHeader { get; set; } = true; [Description(" Whether to include an index column at the start of the CSV file.")] public bool IncludeIndex { get; set; } = false; + + [Description(" Whether to include objects that do not have a string representation. If true, these objects will be included using their ToString() method or a placeholder if not available. If false, such objects will be skipped.")] + public bool IncludeObjects { get; set; } = false; + + [Description(" The character to use as the decimal separator in numerical values. Common options are '.' for dot and ',' for comma. Default is '.'")] + public string DecimalSeparator { get; set; } = "."; + + [Description(" If specified, numerical values will be rounded to this number of decimal places. If null, no rounding is applied.")] + public double? Digit { get; set; } = null; + + [Description(" The format to use for date values. Options include ISO8601 (e.g., 2023-10-05T14:48:00Z), US (e.g., 10/05/2023), and EU (e.g., 05/10/2023). Default is ISO8601.")] + public DateFormatOptions DateTimeFormat { get; set; } = DateFormatOptions.EU; + } } diff --git a/File_oM/enums/DateTime.cs b/File_oM/enums/DateTime.cs new file mode 100644 index 0000000..ef404b5 --- /dev/null +++ b/File_oM/enums/DateTime.cs @@ -0,0 +1,42 @@ +/* + * This file is part of the Buildings and Habitats object Model (BHoM) + * Copyright (c) 2015 - 2025, the respective contributors. All rights reserved. + * + * Each contributor holds copyright over their respective contributions. + * The project versioning (Git) records all such contribution source information. + * + * + * The BHoM is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3.0 of the License, or + * (at your option) any later version. + * + * The BHoM is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this code. If not, see . + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BH.oM.Adapters.File +{ + public enum DateFormatOptions + { + ISO8601, // e.g., 2023-10-05T14:48:00Z + US, // e.g., 10/05/2023 + EU // e.g., 05/10/2023 + } +} + + + + + From 2c74608a57b77077136ffaeabcab988a7fd55767 Mon Sep 17 00:00:00 2001 From: linhnam-nguyen <88286063+linhnam-nguyen@users.noreply.github.com> Date: Fri, 12 Sep 2025 16:52:49 +0200 Subject: [PATCH 03/16] Add decimal separator --- File_Engine/Compute/WriteToCsvFile.cs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/File_Engine/Compute/WriteToCsvFile.cs b/File_Engine/Compute/WriteToCsvFile.cs index 263ea69..8bdd6e4 100644 --- a/File_Engine/Compute/WriteToCsvFile.cs +++ b/File_Engine/Compute/WriteToCsvFile.cs @@ -67,7 +67,7 @@ public static bool WriteToCsvFile(object data, string filePath, CsvSettings sett } // Serialise to json and create the file and directory. - string table = FromObject(data); + string table = FromObject(data,settings); System.IO.FileInfo fileInfo = new System.IO.FileInfo(filePath); fileInfo.Directory.Create(); // If the directory already exists, this method does nothing. @@ -237,14 +237,25 @@ private static string ToCellString(object value, CsvSettings settings) { double d = System.Convert.ToDouble(value, CultureInfo.InvariantCulture); + string raw; if (settings.Digit.HasValue) { int digits = (int) settings.Digit.Value; var format = digits > 0 ? "0." + new string('#', digits) : "0"; return Math.Round(d, digits).ToString(format, CultureInfo.InvariantCulture); } + else + { + raw = d.ToString("G", CultureInfo.InvariantCulture); + } + + // Apply custom decimal separator if needed + if (settings.DecimalSeparator != "." && raw.Contains(".")) + { + raw = raw.Replace(".", settings.DecimalSeparator); + } - return d.ToString("G", CultureInfo.InvariantCulture); + return raw; } // Date/Time From c5f3fe63a899d3c3ae9a700e670cd2ef21a172e0 Mon Sep 17 00:00:00 2001 From: linhnam-nguyen <88286063+linhnam-nguyen@users.noreply.github.com> Date: Fri, 12 Sep 2025 17:15:10 +0200 Subject: [PATCH 04/16] Bugs fixed --- File_Engine/Compute/WriteToCsvFile.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/File_Engine/Compute/WriteToCsvFile.cs b/File_Engine/Compute/WriteToCsvFile.cs index 8bdd6e4..cd4607c 100644 --- a/File_Engine/Compute/WriteToCsvFile.cs +++ b/File_Engine/Compute/WriteToCsvFile.cs @@ -88,7 +88,7 @@ public static bool WriteToCsvFile(object data, string filePath, CsvSettings sett /**** Private Methods ****/ /***************************************************/ - // Convert a list of objects to a CSV text string, taking care of the following types: List>, List, string[,], string[][], string[] + // Convert a list of objects to a CSV text string, taking care of the following types: List>, List, object[,], object[][], object[] public static string FromObject(this object obj, CsvSettings settings = null) { if (settings == null) @@ -242,7 +242,7 @@ private static string ToCellString(object value, CsvSettings settings) { int digits = (int) settings.Digit.Value; var format = digits > 0 ? "0." + new string('#', digits) : "0"; - return Math.Round(d, digits).ToString(format, CultureInfo.InvariantCulture); + raw = Math.Round(d, digits).ToString(format, CultureInfo.InvariantCulture); } else { From 7a4da6b889ccc110b01924ae87297e7d9be343d4 Mon Sep 17 00:00:00 2001 From: linhnam-nguyen <88286063+linhnam-nguyen@users.noreply.github.com> Date: Fri, 12 Sep 2025 17:58:26 +0200 Subject: [PATCH 05/16] Typo fixed --- File_Engine/Compute/WriteToCsvFile.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/File_Engine/Compute/WriteToCsvFile.cs b/File_Engine/Compute/WriteToCsvFile.cs index cd4607c..38016b2 100644 --- a/File_Engine/Compute/WriteToCsvFile.cs +++ b/File_Engine/Compute/WriteToCsvFile.cs @@ -39,9 +39,10 @@ public static partial class Compute /**** Public Methods ****/ /***************************************************/ - [Description("Write a JSON-serialised file with the input data or objects.")] - [Input("objects", "Objects to write to the file.")] + [Description("Write a CSV file with the input data or objects.")] + [Input("data", "Data to write to the file.")] [Input("filePath", "Path to the file.")] + [Input("settings", "Settings to use when writing the CSV file. If null, default settings will be used.")] [Input("replace", "If the file exists, you need to set this to true in order to allow overwriting it.")] [Input("active", "Boolean used to trigger the function.")] public static bool WriteToCsvFile(object data, string filePath, CsvSettings settings = null, bool replace = false, bool active = false) From 8223bef2a6575b19bc5e3541dadb8ffed5b0a32a Mon Sep 17 00:00:00 2001 From: linhnam-nguyen <88286063+linhnam-nguyen@users.noreply.github.com> Date: Fri, 12 Sep 2025 18:11:11 +0200 Subject: [PATCH 06/16] Removed headers --- File_oM/Config/CsvSettings.cs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/File_oM/Config/CsvSettings.cs b/File_oM/Config/CsvSettings.cs index da18bdb..ec642fc 100644 --- a/File_oM/Config/CsvSettings.cs +++ b/File_oM/Config/CsvSettings.cs @@ -31,12 +31,6 @@ public class CsvSettings : ActionConfig [Description(" The delimiter to use in the CSV file. Common options are ',' for comma, ';' for semicolon, and '\\t' for tab by default.")] public string Delimiter { get; set; } = "\t"; - [Description(" Whether to include a header row with column names in the CSV file.")] - public bool IncludeHeader { get; set; } = true; - - [Description(" Whether to include an index column at the start of the CSV file.")] - public bool IncludeIndex { get; set; } = false; - [Description(" Whether to include objects that do not have a string representation. If true, these objects will be included using their ToString() method or a placeholder if not available. If false, such objects will be skipped.")] public bool IncludeObjects { get; set; } = false; From ce135006e7c3a695dafbee707b4936517a62a4a8 Mon Sep 17 00:00:00 2001 From: linhnam-nguyen <88286063+linhnam-nguyen@users.noreply.github.com> Date: Fri, 12 Sep 2025 18:17:08 +0200 Subject: [PATCH 07/16] Clean code --- File_Engine/Compute/WriteToCsvFile.cs | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/File_Engine/Compute/WriteToCsvFile.cs b/File_Engine/Compute/WriteToCsvFile.cs index 38016b2..5e8cb86 100644 --- a/File_Engine/Compute/WriteToCsvFile.cs +++ b/File_Engine/Compute/WriteToCsvFile.cs @@ -94,9 +94,7 @@ public static string FromObject(this object obj, CsvSettings settings = null) { if (settings == null) settings = new CsvSettings(); - - var delim = settings.Delimiter ?? "\t"; - + // 1) Normalize to list of rows (each row = string[]) var rows = new List(); @@ -193,20 +191,8 @@ public static string FromObject(this object obj, CsvSettings settings = null) } } - // 3) Optional 1-based index column - if (settings.IncludeIndex) - { - for (int i = 0; i < rows.Count; i++) - { - var current = rows[i]; - var newRow = new string[current.Length + 1]; - newRow[0] = (i + 1).ToString(CultureInfo.InvariantCulture); - Array.Copy(current, 0, newRow, 1, current.Length); - rows[i] = newRow; - } - } - - // 4) Serialize with CSV escaping + // 3) Serialize with CSV escaping + var delim = settings.Delimiter ?? "\t"; var sb = new StringBuilder(rows.Count * maxCols * 4); for (int i = 0; i < rows.Count; i++) { From cf1a8e8fe9b95e5c12e59c920439c8a72c40f329 Mon Sep 17 00:00:00 2001 From: linhnam-nguyen <88286063+linhnam-nguyen@users.noreply.github.com> Date: Fri, 12 Sep 2025 18:30:25 +0200 Subject: [PATCH 08/16] Adding data formatting --- File_oM/Config/DataSettings.cs | 38 ++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 File_oM/Config/DataSettings.cs diff --git a/File_oM/Config/DataSettings.cs b/File_oM/Config/DataSettings.cs new file mode 100644 index 0000000..ccb15b4 --- /dev/null +++ b/File_oM/Config/DataSettings.cs @@ -0,0 +1,38 @@ +/* + * This file is part of the Buildings and Habitats object Model (BHoM) + * Copyright (c) 2015 - 2025, the respective contributors. All rights reserved. + * + * Each contributor holds copyright over their respective contributions. + * The project versioning (Git) records all such contribution source information. + * + * + * The BHoM is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3.0 of the License, or + * (at your option) any later version. + * + * The BHoM is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this code. If not, see . + */ + + +using BH.oM.Adapter; +using System.Collections.Generic; + +namespace BH.oM.Adapters.File +{ + public class DataSettings : ActionConfig + { + public Dictionary Mappings { get; set; } = new Dictionary(); + } +} + + + + + From fa04c17903ab46a2d609ddd9028a3f79c443ae21 Mon Sep 17 00:00:00 2001 From: linhnam-nguyen <88286063+linhnam-nguyen@users.noreply.github.com> Date: Sat, 13 Sep 2025 23:47:30 +0200 Subject: [PATCH 09/16] Refactor to normalize data format by column --- File_Engine/Compute/WriteToCsvFile.cs | 394 ++++++++++-------- .../Config/{CsvSettings.cs => CsvConfig.cs} | 17 +- .../DataSettings.cs => enums/StringType.cs} | 13 +- 3 files changed, 249 insertions(+), 175 deletions(-) rename File_oM/Config/{CsvSettings.cs => CsvConfig.cs} (72%) rename File_oM/{Config/DataSettings.cs => enums/StringType.cs} (89%) diff --git a/File_Engine/Compute/WriteToCsvFile.cs b/File_Engine/Compute/WriteToCsvFile.cs index 5e8cb86..a558704 100644 --- a/File_Engine/Compute/WriteToCsvFile.cs +++ b/File_Engine/Compute/WriteToCsvFile.cs @@ -20,6 +20,7 @@ * along with this code. If not, see . */ +using BH.Engine.Base; using BH.oM.Adapters.File; using BH.oM.Base.Attributes; using System; @@ -45,7 +46,7 @@ public static partial class Compute [Input("settings", "Settings to use when writing the CSV file. If null, default settings will be used.")] [Input("replace", "If the file exists, you need to set this to true in order to allow overwriting it.")] [Input("active", "Boolean used to trigger the function.")] - public static bool WriteToCsvFile(object data, string filePath, CsvSettings settings = null, bool replace = false, bool active = false) + public static bool WriteToCsvFile(object data, string filePath, CsvConfig settings = null, bool replace = false, bool active = false) { if (!active || data == null) return false; @@ -85,208 +86,281 @@ public static bool WriteToCsvFile(object data, string filePath, CsvSettings sett return true; } - /***************************************************/ - /**** Private Methods ****/ /***************************************************/ - // Convert a list of objects to a CSV text string, taking care of the following types: List>, List, object[,], object[][], object[] - public static string FromObject(this object obj, CsvSettings settings = null) + public static string FromObject(this object obj, CsvConfig settings = null) { if (settings == null) - settings = new CsvSettings(); - - // 1) Normalize to list of rows (each row = string[]) - var rows = new List(); + settings = new CsvConfig(); + + object[,] flatten; - switch (obj) + // Shape normalisation (priority to arrays) + if (obj is object[,] rect) { - // --- PRIMARY: Rectangular arrays of objects --- - case object[,] rect: - { - int r = rect.GetLength(0); - int c = rect.GetLength(1); - for (int i = 0; i < r; i++) - { - var row = new string[c]; - for (int j = 0; j < c; j++) - row[j] = ToCellString(rect[i, j], settings); - rows.Add(row); - } - break; - } - - // --- PRIMARY: Jagged arrays of objects --- - case object[][] jagged: - { - foreach (var inner in jagged ?? new object[0][]) - { - var src = inner ?? new object[0]; - var row = new string[src.Length]; - for (int j = 0; j < src.Length; j++) - row[j] = ToCellString(src[j], settings); - rows.Add(row); - } - break; - } - - // --- SECONDARY: Sequences of sequences of objects --- - case IEnumerable> seqOfSeq: - { - foreach (var inner in seqOfSeq) - { - var items = inner ?? new object[0]; - var rowList = new List(); - foreach (var it in items) - rowList.Add(ToCellString(it, settings)); - rows.Add(rowList.ToArray()); - } - break; - } - - // --- SECONDARY: Sequence of objects (single row); exclude string itself --- - case IEnumerable seq when !(obj is string): - { - var rowList = new List(); - foreach (var it in seq) - rowList.Add(ToCellString(it, settings)); - rows.Add(rowList.ToArray()); - break; - } - - // Single string -> one cell - case string s: - { - rows.Add(new[] { s }); - break; - } - - // Fallback: single cell from ToString() - default: - { - rows.Add(new[] { ToCellString(obj, settings) }); - break; - } + flatten = rect; } - - if (rows.Count == 0) - return string.Empty; - - // 2) Pad ragged rows to the maximum width - int maxCols = 0; - for (int i = 0; i < rows.Count; i++) + else if (obj is object[][] jagged) { - var r = rows[i] ?? new string[0]; - if (r.Length > maxCols) maxCols = r.Length; + flatten = ToRect(jagged); } - if (maxCols == 0) maxCols = 1; - - for (int i = 0; i < rows.Count; i++) + else if (obj is IEnumerable> nested) { - var r = rows[i] ?? new string[0]; - if (r.Length < maxCols) - { - var padded = new string[maxCols]; - Array.Copy(r, padded, r.Length); // remaining are null -> treated as empty - rows[i] = padded; - } + flatten = ToRect(nested); + } + else if (obj is IEnumerable list && !(obj is string)) + { + flatten = ToRect(list); + } + else if (obj != null && obj.GetType().IsPrimitive) + { + // Single primitive -> 1x1 table + flatten = new object[,] { { obj } }; + } + else + { + BH.Engine.Base.Compute.RecordError( + $"The input data of type `{obj?.GetType().Name ?? "null"}` is not supported. " + + "Supported types are: object[,], object[][], IEnumerable>, IEnumerable."); + return string.Empty; } - // 3) Serialize with CSV escaping + // Convert cells to string with formatting rules + string[,] table = ToStringTable(flatten, settings); + + // Serialize to CSV var delim = settings.Delimiter ?? "\t"; - var sb = new StringBuilder(rows.Count * maxCols * 4); - for (int i = 0; i < rows.Count; i++) + int r = table.GetLength(0); + int c = table.GetLength(1); + var sb = new StringBuilder(r * Math.Max(1, c) * 4); + + for (int i = 0; i < r; i++) { - var encoded = rows[i].Select(cell => EscapeCsv(cell ?? string.Empty, delim)); + var encoded = Enumerable.Range(0, c).Select(j => EscapeCsv(table[i, j] ?? string.Empty, delim)); sb.Append(string.Join(delim, encoded)); - if (i < rows.Count - 1) - sb.Append('\n'); + if (i < r - 1) sb.Append('\n'); } return sb.ToString(); } - /*******************************************/ /**** Private Methods *****/ /*******************************************/ - private static string ToCellString(object value, CsvSettings settings) + private static object[,] ToRect(object[][] obj) { - if (value == null) - return string.Empty; - - // string - var s = value as string; - if (s != null) - return s; - - // numeric (round if Digit set) - if (IsNumeric(value)) + if (obj == null || obj.Length == 0) + return new object[0, 0]; + int r = obj.Length; + int c = obj.Max(a => a?.Length ?? 0); + var result = new object[r, c]; + for (int i = 0; i < r; i++) { - double d = System.Convert.ToDouble(value, CultureInfo.InvariantCulture); - - string raw; - if (settings.Digit.HasValue) + var row = obj[i] ?? new object[0]; + for (int j = 0; j < c; j++) { - int digits = (int) settings.Digit.Value; - var format = digits > 0 ? "0." + new string('#', digits) : "0"; - raw = Math.Round(d, digits).ToString(format, CultureInfo.InvariantCulture); + result[i, j] = j < row.Length ? row[j] : null; } - else + } + return result; + } + + private static object[,] ToRect(IEnumerable> obj) + { + if (obj == null) + return new object[0, 0]; + var list = obj.ToList(); + if (list.Count == 0) + return new object[0, 0]; + int r = list.Count; + int c = list.Max(a => a?.Count() ?? 0); + var result = new object[r, c]; + for (int i = 0; i < r; i++) + { + var row = list[i]?.ToList() ?? new List(); + for (int j = 0; j < c; j++) { - raw = d.ToString("G", CultureInfo.InvariantCulture); + result[i, j] = j < row.Count ? row[j] : null; } + } + return result; + } + + private static object[,] ToRect(IEnumerable obj) + { + if (obj == null) + return new object[0, 0]; + var list = obj.ToList(); + if (list.Count == 0) + return new object[0, 0]; + int r = list.Count; + int c = 1; + var result = new object[r, c]; + for (int j = 0; j < c; j++) + { + result[j, 0] = list[j]; + } + return result; + } + + private static string[,] ToStringTable(object[,] obj, CsvConfig settings) + { + if (obj == null) + return new string[0, 0]; + + int r = obj.GetLength(0); + int c = obj.GetLength(1); + var result = new string[r, c]; + + for (int i = 0; i < r; i++) + { + bool isHeader = (i == 0); + for (int j = 0; j < c; j++) + result[i, j] = FormatCell(obj[i, j], settings, j, isHeader); + } + + return result; + } + + private static string FormatCell(object value, CsvConfig settings, int? column, bool isHeader = false) + { + if (settings.ColumnDataFormats != null && column.HasValue && ((settings.IncludeHeader && !isHeader) || !settings.IncludeHeader)) + { + var format = settings.ColumnDataFormats[column.Value]; - // Apply custom decimal separator if needed - if (settings.DecimalSeparator != "." && raw.Contains(".")) + switch(format) { - raw = raw.Replace(".", settings.DecimalSeparator); + case StringType.Boolean: + if(value is bool boolean) + return boolean.FormatBool(settings.BooleanAsNumber); + else + return string.Empty; + + case StringType.Date: + if (value is DateTime date) + return date.FormatDate(settings.DateTimeFormat); + else + return string.Empty; + + case StringType.Numeric: + // Accept only IFormattable numerics here; else fall through to generic branch below + var fnum = value as IFormattable; + return fnum != null + ? fnum.FormatNumeric((int?)settings.Digit, settings.DecimalSeparator) + : string.Empty; + + case StringType.Text: + // Re-run without ColumnDataFormats influence + var noColumnFormats = settings; + noColumnFormats.ColumnDataFormats = null; + return FormatCell(value, noColumnFormats, null, isHeader); } - - return raw; } + // numeric (round if Digit set) + if (value.GetType().IsNumeric(false)) + { + double d = System.Convert.ToDouble(value, CultureInfo.InvariantCulture); + return d.FormatNumeric((int?)settings.Digit, settings.DecimalSeparator); + } // Date/Time if (value is DateTime dt) return dt.FormatDate(settings.DateTimeFormat); - // Bool if (value is bool b) - return b ? "true" : "false"; - - // Enum - var type = value.GetType(); - if (type.IsEnum) - return System.Convert.ToString(value, CultureInfo.InvariantCulture); + return b.FormatBool(settings.BooleanAsNumber); // IFormattable (decimal, Guid, etc.) var formattable = value as IFormattable; - if (formattable != null) - return formattable.ToString(null, CultureInfo.InvariantCulture); + return formattable.FormatIFormattable(); + // Enum + if (value.GetType().IsEnum) + return (value as Enum).FormatEnum(); + + // Other unformattable types if (settings.IncludeObjects) - { - // Fallback: use ToString() if overridden - var toString = value.ToString(); - if (toString != null && toString != type.FullName) - return toString; - // Fallback: type name - return $"<{type.Name}>"; - } + return value.FormatObject(); // Fallback return string.Empty; } - private static bool IsNumeric(object value) + private static string FormatDate(this DateTime date, DateFormatOptions option) { - return value is byte || value is sbyte || - value is short || value is ushort || - value is int || value is uint || - value is long || value is ulong || - value is float || value is double || - value is decimal; + if (date == null) + return string.Empty; + + switch (option) + { + case DateFormatOptions.ISO8601: + // round-trip, always UTC if DateTime.Kind is UTC + return date.ToString("o", CultureInfo.InvariantCulture); + + case DateFormatOptions.US: + // month/day/year + return date.ToString("MM/dd/yyyy", CultureInfo.InvariantCulture); + + case DateFormatOptions.EU: + default: + // day/month/year + return date.ToString("dd/MM/yyyy", CultureInfo.InvariantCulture); + } + } + + private static string FormatNumeric(this IFormattable number, int? digits, string decimalSeparator) + { + if (number == null) + return string.Empty; + string raw; + if (digits.HasValue) + { + int d = Math.Max(0, digits.Value); + var format = d > 0 ? "0." + new string('#', d) : "0"; + raw = number.ToString(format, CultureInfo.InvariantCulture); + } + else + { + raw = number.ToString("G", CultureInfo.InvariantCulture); + } + // Apply custom decimal separator if needed + if (decimalSeparator != "." && raw.Contains(".")) + { + raw = raw.Replace(".", decimalSeparator); + } + return raw; + } + + private static string FormatBool(this bool b, bool asNumber) + { + return asNumber ? (b ? "1" : "0") : (b ? "true" : "false"); + } + + private static string FormatEnum(this Enum e) + { + return System.Convert.ToString(e, CultureInfo.InvariantCulture); + } + + private static string FormatIFormattable(this IFormattable f) + { + return f.ToString(null, CultureInfo.InvariantCulture); + } + + private static string FormatObject(this object obj, string propName = null) + { + if (obj == null) + return string.Empty; + + if (propName != null) + return obj.GetType().GetProperty(propName)?.GetValue(obj).ToString(); + + var toString = obj.ToString(); + if (toString != null && toString != obj.GetType().FullName) + return toString; + + return $"<{obj.GetType().Name}>"; } private static string EscapeCsv(string input, string delimiter) @@ -306,23 +380,5 @@ private static string EscapeCsv(string input, string delimiter) return "\"" + doubled + "\""; } - private static string FormatDate(this IFormattable date, DateFormatOptions option) - { - switch (option) - { - case DateFormatOptions.ISO8601: - // round-trip, always UTC if DateTime.Kind is UTC - return date.ToString("o", CultureInfo.InvariantCulture); - - case DateFormatOptions.US: - // month/day/year - return date.ToString("MM/dd/yyyy", CultureInfo.InvariantCulture); - - case DateFormatOptions.EU: - default: - // day/month/year - return date.ToString("dd/MM/yyyy", CultureInfo.InvariantCulture); - } - } } } diff --git a/File_oM/Config/CsvSettings.cs b/File_oM/Config/CsvConfig.cs similarity index 72% rename from File_oM/Config/CsvSettings.cs rename to File_oM/Config/CsvConfig.cs index ec642fc..0f85e1d 100644 --- a/File_oM/Config/CsvSettings.cs +++ b/File_oM/Config/CsvConfig.cs @@ -22,11 +22,13 @@ using BH.oM.Adapter; +using System.Collections.Generic; using System.ComponentModel; +using System.Runtime.CompilerServices; namespace BH.oM.Adapters.File { - public class CsvSettings : ActionConfig + public class CsvConfig : ActionConfig { [Description(" The delimiter to use in the CSV file. Common options are ',' for comma, ';' for semicolon, and '\\t' for tab by default.")] public string Delimiter { get; set; } = "\t"; @@ -34,6 +36,18 @@ public class CsvSettings : ActionConfig [Description(" Whether to include objects that do not have a string representation. If true, these objects will be included using their ToString() method or a placeholder if not available. If false, such objects will be skipped.")] public bool IncludeObjects { get; set; } = false; + [Description(" If specified, the value of the property representing object will be serialized in the CSV file. If null, object type name will shown.")] + public string PropertyName { get; set; } = null; + + [Description(" Whether to include a header row with column names in the CSV file. Default is true, meaning the first row will contain the property names.")] + public bool IncludeHeader { get; set; } = true; + + [Description("Configuration for formatting datatype for each column. If null, default formatting will be applied based on the data type.")] + public List ColumnDataFormats { get; set; } = null; + + [Description(" Whether to represent boolean values as numbers (1 for true, 0 for false) instead of text (true/false). Default is false, meaning booleans will be represented as text.")] + public bool BooleanAsNumber { get; set; } = false; + [Description(" The character to use as the decimal separator in numerical values. Common options are '.' for dot and ',' for comma. Default is '.'")] public string DecimalSeparator { get; set; } = "."; @@ -42,7 +56,6 @@ public class CsvSettings : ActionConfig [Description(" The format to use for date values. Options include ISO8601 (e.g., 2023-10-05T14:48:00Z), US (e.g., 10/05/2023), and EU (e.g., 05/10/2023). Default is ISO8601.")] public DateFormatOptions DateTimeFormat { get; set; } = DateFormatOptions.EU; - } } diff --git a/File_oM/Config/DataSettings.cs b/File_oM/enums/StringType.cs similarity index 89% rename from File_oM/Config/DataSettings.cs rename to File_oM/enums/StringType.cs index ccb15b4..41832d7 100644 --- a/File_oM/Config/DataSettings.cs +++ b/File_oM/enums/StringType.cs @@ -20,15 +20,20 @@ * along with this code. If not, see . */ - -using BH.oM.Adapter; +using System; using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; namespace BH.oM.Adapters.File { - public class DataSettings : ActionConfig + public enum StringType { - public Dictionary Mappings { get; set; } = new Dictionary(); + Text, + Numeric, + Date, + Boolean, } } From 6e6ddb5b86613df0b8e86e512e6fed8f284d4e5c Mon Sep 17 00:00:00 2001 From: linhnam-nguyen <88286063+linhnam-nguyen@users.noreply.github.com> Date: Sun, 14 Sep 2025 00:01:28 +0200 Subject: [PATCH 10/16] Add support for conversion between double and datetime --- File_Engine/Compute/WriteToCsvFile.cs | 51 ++++++++++++++++++--------- 1 file changed, 35 insertions(+), 16 deletions(-) diff --git a/File_Engine/Compute/WriteToCsvFile.cs b/File_Engine/Compute/WriteToCsvFile.cs index a558704..87da195 100644 --- a/File_Engine/Compute/WriteToCsvFile.cs +++ b/File_Engine/Compute/WriteToCsvFile.cs @@ -237,7 +237,7 @@ private static string FormatCell(object value, CsvConfig settings, int? column, return string.Empty; case StringType.Date: - if (value is DateTime date) + if (value is IFormattable date) return date.FormatDate(settings.DateTimeFormat); else return string.Empty; @@ -284,30 +284,49 @@ private static string FormatCell(object value, CsvConfig settings, int? column, if (settings.IncludeObjects) return value.FormatObject(); - // Fallback + return string.Empty; } - private static string FormatDate(this DateTime date, DateFormatOptions option) + private static string FormatDate(this IFormattable date, DateFormatOptions option) { if (date == null) return string.Empty; - switch (option) + // Handle Excel-style serial number (double) + if (date is double serial) + { + try + { + var dt = DateTime.FromOADate(serial); + return dt.FormatDate(option); + } + catch + { + return serial.ToString(CultureInfo.InvariantCulture); + } + } + + // Handle DateTime + if (date is DateTime dt2) + return dt2.FormatDate(option); + + // Handle DateTimeOffset + if (date is DateTimeOffset dto) { - case DateFormatOptions.ISO8601: - // round-trip, always UTC if DateTime.Kind is UTC - return date.ToString("o", CultureInfo.InvariantCulture); - - case DateFormatOptions.US: - // month/day/year - return date.ToString("MM/dd/yyyy", CultureInfo.InvariantCulture); - - case DateFormatOptions.EU: - default: - // day/month/year - return date.ToString("dd/MM/yyyy", CultureInfo.InvariantCulture); + switch (option) + { + case DateFormatOptions.ISO8601: + return dto.ToString("o", CultureInfo.InvariantCulture); + case DateFormatOptions.US: + return dto.ToString("MM/dd/yyyy", CultureInfo.InvariantCulture); + case DateFormatOptions.EU: + default: + return dto.ToString("dd/MM/yyyy", CultureInfo.InvariantCulture); + } } + + return date.ToString(null, CultureInfo.InvariantCulture); } private static string FormatNumeric(this IFormattable number, int? digits, string decimalSeparator) From fa8253f325c75f1ca7943c56188acd36da4af2eb Mon Sep 17 00:00:00 2001 From: linhnam-nguyen <88286063+linhnam-nguyen@users.noreply.github.com> Date: Sun, 14 Sep 2025 00:42:22 +0200 Subject: [PATCH 11/16] Add support for ReadFromCsvFile --- File_Engine/Compute/ReadFromCsvFile.cs | 324 ++++++++++++++++++++++--- 1 file changed, 286 insertions(+), 38 deletions(-) diff --git a/File_Engine/Compute/ReadFromCsvFile.cs b/File_Engine/Compute/ReadFromCsvFile.cs index 65b468a..533bf5d 100644 --- a/File_Engine/Compute/ReadFromCsvFile.cs +++ b/File_Engine/Compute/ReadFromCsvFile.cs @@ -20,18 +20,12 @@ * along with this code. If not, see . */ -using BH.oM.Base; +using BH.oM.Adapters.File; +using BH.oM.Base.Attributes; using System; using System.Collections.Generic; -using System.IO; -using System.Linq; -using BH.Engine.Serialiser; -using BH.Engine.Adapters.File; -using BH.oM.Adapters.File; -using System.Collections; -using BH.oM.Adapter; using System.ComponentModel; -using BH.oM.Base.Attributes; +using System.Globalization; namespace BH.Engine.Adapters.File { @@ -41,53 +35,307 @@ public static partial class Compute /**** Public Methods ****/ /***************************************************/ - [Description("Read a JSON-serialised file and output any data or object included in it.")] - [Input("filePath", "Path to the file.")] + [Description("Read a CSV file into a 2D object array, parsing each column using CsvConfig.ColumnDataFormats when provided.")] + [Input("filePath", "Path to the CSV file.")] + [Input("settings", "CSV settings including delimiter, decimal separator, and per-column formats. If null, defaults are used.")] [Input("active", "Boolean used to trigger the function.")] - public static List ReadFromCsvFile(string filePath, bool active = false) + public static object[,] ReadFromCsvFile(string filePath, CsvConfig settings = null, bool active = false) { if (!active) - return new List(); + return new object[0, 0]; if (string.IsNullOrWhiteSpace(filePath)) { - BH.Engine.Base.Compute.RecordError($"The filePath `{filePath}` must not be empty."); - return new List(); + BH.Engine.Base.Compute.RecordError("The file path must not be empty."); + return new object[0, 0]; + } + + if (!System.IO.File.Exists(filePath)) + { + BH.Engine.Base.Compute.RecordError($"The file `{filePath}` does not exist."); + return new object[0, 0]; + } + + if (settings == null) + settings = new CsvConfig(); + + var delim = settings.Delimiter ?? "\t"; + string[] lines; + try + { + // Note: ReadAllLines splits on Environment.NewLine (handles \r\n and \n) + lines = System.IO.File.ReadAllLines(filePath); } + catch (Exception e) + { + BH.Engine.Base.Compute.RecordError($"Error reading file:\n\t{e}"); + return new object[0, 0]; + } + + if (lines.Length == 0) + return new object[0, 0]; + + // 1) Parse lines into raw string rows (CSV rules: quotes + escaped quotes) + var rawRows = new List(); + for (int i = 0; i < lines.Length; i++) + { + var line = lines[i]; + // Skip completely empty lines + if (string.IsNullOrEmpty(line)) + continue; + + rawRows.Add(SplitCsvLine(line, delim)); + } + + if (rawRows.Count == 0) + return new object[0, 0]; + + // 2) Normalize columns (pad ragged rows to max width) + int rAll = rawRows.Count; + int c = 0; + for (int i = 0; i < rAll; i++) + if (rawRows[i].Length > c) c = rawRows[i].Length; + + if (c == 0) + return new object[0, 0]; - // Make sure no invalid chars are present. - string fullPath = Path.Combine(Path.GetDirectoryName(filePath), Path.GetFileName(filePath)); - - bool fileExisted = System.IO.File.Exists(fullPath); - if (!fileExisted) + // 3) Decide data start index based on IncludeHeader + int startRow = (settings.IncludeHeader && rAll > 0) ? 1 : 0; + int rData = Math.Max(0, rAll - startRow); + + var table = new object[rData, c]; + + // 4) Parse cells with column-aware formats + for (int i = 0; i < rData; i++) { - BH.Engine.Base.Compute.RecordError($"The file `{fullPath}` does not exist."); - return new List(); + var src = rawRows[i + startRow]; + for (int j = 0; j < c; j++) + { + string cell = j < src.Length ? src[j] : null; + table[i, j] = ParseCell(cell, j, settings); + } } - string jsonText = System.IO.File.ReadAllText(fullPath); - object converted = BH.Engine.Serialiser.Convert.FromJson(jsonText); + return table; + } + + /***************************************************/ + /**** Private Helpers ****/ + /***************************************************/ + + // CSV splitter: handles quotes, escaped quotes (""), and multi-char delimiters (e.g., "\t") + private static string[] SplitCsvLine(string line, string delimiter) + { + if (string.IsNullOrEmpty(line)) + return new string[0]; + + var cells = new List(); + var current = new System.Text.StringBuilder(); + bool inQuotes = false; + int i = 0; + int n = line.Length; + int dlen = string.IsNullOrEmpty(delimiter) ? 0 : delimiter.Length; - // Check if there is any ObjectWrapper that was used to allow writing of non-IObjects, like primitive types (numbers/strings). - List convertedList = new List(); - IEnumerable ienum = converted as IEnumerable; - if (ienum != null) + while (i < n) { - foreach (var obj in ienum) + char ch = line[i]; + + if (ch == '"') { - ObjectWrapper objectWrapper = obj as ObjectWrapper; - if (objectWrapper != null) - convertedList.Add(objectWrapper.WrappedObject); - else - convertedList.Add(obj); + if (inQuotes && i + 1 < n && line[i + 1] == '"') + { + // Escaped quote -> add one quote + current.Append('"'); + i += 2; + continue; + } + inQuotes = !inQuotes; + i++; + continue; } - return convertedList; + if (!inQuotes && dlen > 0 && i + dlen <= n && + string.CompareOrdinal(line, i, delimiter, 0, dlen) == 0) + { + // End of cell + cells.Add(current.ToString()); + current.Length = 0; + i += dlen; + continue; + } + + current.Append(ch); + i++; } - return new List() { converted }; + cells.Add(current.ToString()); + return cells.ToArray(); } - } -} + // Column-aware parsing with fallbacks: + // - If ColumnDataFormats[j] exists, parse by that contract + // - Else, try bool/(1/0), numeric (with DecimalSeparator), date (several formats), else string + private static object ParseCell(string raw, int columnIndex, CsvConfig settings) + { + if (string.IsNullOrEmpty(raw)) + return null; + + bool hasColumnFormat = + settings.ColumnDataFormats != null && + columnIndex >= 0 && + columnIndex < settings.ColumnDataFormats.Count; + + if (hasColumnFormat) + { + switch (settings.ColumnDataFormats[columnIndex]) + { + case StringType.Boolean: + return ParseBool(raw, settings); + + case StringType.Numeric: + { + double num; + if (TryParseNumber(raw, settings.DecimalSeparator, out num)) + return num; + return null; + } + + case StringType.Date: + { + DateTime dt; + if (TryParseDate(raw, settings.DateTimeFormat, out dt)) + return dt; + // If date parsing fails but it's a double OA, try that as a last resort + double serial; + if (double.TryParse(raw.Replace(settings.DecimalSeparator, "."), NumberStyles.Any, CultureInfo.InvariantCulture, out serial)) + { + try + { + return DateTime.FromOADate(serial); + } + catch { /* ignore */ } + } + return null; + } + + case StringType.Text: + return raw; + } + } + + // ---- Heuristic fallback (no explicit column format) ---- + + // Bool + var bParsed = ParseBool(raw, settings); + if (bParsed != null) + return bParsed.Value; + + // Number + double d; + if (TryParseNumber(raw, settings.DecimalSeparator, out d)) + return d; + + // Date (ISO/US/EU tries + OA double fallback) + DateTime when; + if (TryParseDate(raw, settings.DateTimeFormat, out when)) + return when; + + double serial2; + if (double.TryParse(raw.Replace(settings.DecimalSeparator, "."), NumberStyles.Any, CultureInfo.InvariantCulture, out serial2)) + { + try + { + return DateTime.FromOADate(serial2); + } + catch { /* ignore */ } + } + + // Text as-is + return raw; + } + + private static bool? ParseBool(string raw, CsvConfig settings) + { + if (string.Equals(raw, "true", StringComparison.OrdinalIgnoreCase)) return true; + if (string.Equals(raw, "false", StringComparison.OrdinalIgnoreCase)) return false; + + if (settings.BooleanAsNumber) + { + if (raw == "1") return true; + if (raw == "0") return false; + } + return null; + } + + private static bool TryParseNumber(string raw, string decimalSeparator, out double value) + { + value = 0d; + if (string.IsNullOrEmpty(raw)) + return false; + + // Normalise custom decimal separator to "." + var norm = string.IsNullOrEmpty(decimalSeparator) || decimalSeparator == "." + ? raw + : raw.Replace(decimalSeparator, "."); + + return double.TryParse(norm, NumberStyles.Any, CultureInfo.InvariantCulture, out value); + } + + // Date parsing guided by DateFormatOptions (date-only by default), with tolerant fallbacks. + private static bool TryParseDate(string raw, DateFormatOptions option, out DateTime dt) + { + dt = default(DateTime); + if (string.IsNullOrEmpty(raw)) + return false; + + string[] patterns; + switch (option) + { + case DateFormatOptions.ISO8601: + patterns = new[] + { + "o", + "yyyy-MM-ddTHH:mm:ss", + "yyyy-MM-ddTHH:mm:ss.FFFFFFFK", + "yyyy-MM-dd" + }; + break; + + case DateFormatOptions.US: + patterns = new[] + { + "MM/dd/yyyy", + "M/d/yyyy", + "MM/dd/yy" + }; + break; + case DateFormatOptions.EU: + default: + patterns = new[] + { + "dd/MM/yyyy", + "d/M/yyyy", + "dd/MM/yy" + }; + break; + } + + for (int i = 0; i < patterns.Length; i++) + { + DateTime tmp; + if (DateTime.TryParseExact(raw, patterns[i], CultureInfo.InvariantCulture, + DateTimeStyles.None, out tmp)) + { + dt = tmp; + return true; + } + } + + if (DateTime.TryParse(raw, CultureInfo.InvariantCulture, DateTimeStyles.None, out dt)) + return true; + + return false; + } + } +} \ No newline at end of file From 04f6d60a507829d254903f8f6c49e7aec8a8bcbb Mon Sep 17 00:00:00 2001 From: linhnam-nguyen <88286063+linhnam-nguyen@users.noreply.github.com> Date: Sun, 14 Sep 2025 18:11:12 +0200 Subject: [PATCH 12/16] Fixed consitency of output --- File_Engine/Compute/WriteToCsvFile.cs | 113 +++++++++++++++++--------- 1 file changed, 74 insertions(+), 39 deletions(-) diff --git a/File_Engine/Compute/WriteToCsvFile.cs b/File_Engine/Compute/WriteToCsvFile.cs index 87da195..2b697a4 100644 --- a/File_Engine/Compute/WriteToCsvFile.cs +++ b/File_Engine/Compute/WriteToCsvFile.cs @@ -68,7 +68,7 @@ public static bool WriteToCsvFile(object data, string filePath, CsvConfig settin return false; } - // Serialise to json and create the file and directory. + // Serialise to csv and create the file and directory. string table = FromObject(data,settings); System.IO.FileInfo fileInfo = new System.IO.FileInfo(filePath); fileInfo.Directory.Create(); // If the directory already exists, this method does nothing. @@ -95,8 +95,12 @@ public static string FromObject(this object obj, CsvConfig settings = null) object[,] flatten; + if (obj is string || obj.GetType().IsPrimitive || obj is Enum || obj is IFormattable) + { + return FormatCell(obj, settings, null); + } // Shape normalisation (priority to arrays) - if (obj is object[,] rect) + else if (obj is object[,] rect) { flatten = rect; } @@ -112,20 +116,14 @@ public static string FromObject(this object obj, CsvConfig settings = null) { flatten = ToRect(list); } - else if (obj != null && obj.GetType().IsPrimitive) - { - // Single primitive -> 1x1 table - flatten = new object[,] { { obj } }; - } else { BH.Engine.Base.Compute.RecordError( - $"The input data of type `{obj?.GetType().Name ?? "null"}` is not supported. " + - "Supported types are: object[,], object[][], IEnumerable>, IEnumerable."); + $"The input data of type `{obj?.GetType().Name ?? "null"}` is not supported. "); return string.Empty; } - // Convert cells to string with formatting rules + // Convert cells to string array with formatting rules string[,] table = ToStringTable(flatten, settings); // Serialize to CSV @@ -165,6 +163,8 @@ public static string FromObject(this object obj, CsvConfig settings = null) return result; } + /***************************************************/ + private static object[,] ToRect(IEnumerable> obj) { if (obj == null) @@ -186,23 +186,29 @@ public static string FromObject(this object obj, CsvConfig settings = null) return result; } + /***************************************************/ + private static object[,] ToRect(IEnumerable obj) { if (obj == null) return new object[0, 0]; + var list = obj.ToList(); if (list.Count == 0) return new object[0, 0]; + int r = list.Count; - int c = 1; + int c = 0; var result = new object[r, c]; - for (int j = 0; j < c; j++) + for (int i = 0; i < r; i++) { - result[j, 0] = list[j]; + result[i, 0] = list[i]; } return result; } + /***************************************************/ + private static string[,] ToStringTable(object[,] obj, CsvConfig settings) { if (obj == null) @@ -222,38 +228,51 @@ public static string FromObject(this object obj, CsvConfig settings = null) return result; } + /***************************************************/ + private static string FormatCell(object value, CsvConfig settings, int? column, bool isHeader = false) { if (settings.ColumnDataFormats != null && column.HasValue && ((settings.IncludeHeader && !isHeader) || !settings.IncludeHeader)) { - var format = settings.ColumnDataFormats[column.Value]; + StringType? format = settings.ColumnDataFormats[column.Value]; - switch(format) + if (format.HasValue) { - case StringType.Boolean: - if(value is bool boolean) - return boolean.FormatBool(settings.BooleanAsNumber); - else - return string.Empty; - - case StringType.Date: - if (value is IFormattable date) - return date.FormatDate(settings.DateTimeFormat); - else - return string.Empty; - - case StringType.Numeric: - // Accept only IFormattable numerics here; else fall through to generic branch below - var fnum = value as IFormattable; - return fnum != null - ? fnum.FormatNumeric((int?)settings.Digit, settings.DecimalSeparator) - : string.Empty; - - case StringType.Text: - // Re-run without ColumnDataFormats influence - var noColumnFormats = settings; - noColumnFormats.ColumnDataFormats = null; - return FormatCell(value, noColumnFormats, null, isHeader); + switch (format) + { + case StringType.Boolean: + if(value is bool boolean) + return boolean.FormatBool(settings.BooleanAsNumber); + else + return string.Empty; + + case StringType.Date: + if (value is IFormattable date) + return date.FormatDate(settings.DateTimeFormat); + else + return string.Empty; + + case StringType.Numeric: + // Accept only IFormattable numerics here; else fall through to generic branch below + var fnum = value as IFormattable; + return fnum != null + ? fnum.FormatNumeric((int?)settings.Digit, settings.DecimalSeparator) + : string.Empty; + + case StringType.Text: + // Re-run without ColumnDataFormats influence + var noColumnFormats = new CsvConfig + { + IncludeHeader = settings.IncludeHeader, + Digit = settings.Digit, + DecimalSeparator = settings.DecimalSeparator, + DateTimeFormat = settings.DateTimeFormat, + Delimiter = settings.Delimiter, + IncludeObjects = settings.IncludeObjects, + ColumnDataFormats = null + }; + return FormatCell(value, noColumnFormats, null, isHeader); + } } } @@ -288,6 +307,8 @@ private static string FormatCell(object value, CsvConfig settings, int? column, return string.Empty; } + /***************************************************/ + private static string FormatDate(this IFormattable date, DateFormatOptions option) { if (date == null) @@ -329,6 +350,8 @@ private static string FormatDate(this IFormattable date, DateFormatOptions optio return date.ToString(null, CultureInfo.InvariantCulture); } + /***************************************************/ + private static string FormatNumeric(this IFormattable number, int? digits, string decimalSeparator) { if (number == null) @@ -352,21 +375,29 @@ private static string FormatNumeric(this IFormattable number, int? digits, strin return raw; } + /***************************************************/ + private static string FormatBool(this bool b, bool asNumber) { return asNumber ? (b ? "1" : "0") : (b ? "true" : "false"); } + /***************************************************/ + private static string FormatEnum(this Enum e) { return System.Convert.ToString(e, CultureInfo.InvariantCulture); } + /***************************************************/ + private static string FormatIFormattable(this IFormattable f) { return f.ToString(null, CultureInfo.InvariantCulture); } + /***************************************************/ + private static string FormatObject(this object obj, string propName = null) { if (obj == null) @@ -382,6 +413,8 @@ private static string FormatObject(this object obj, string propName = null) return $"<{obj.GetType().Name}>"; } + /***************************************************/ + private static string EscapeCsv(string input, string delimiter) { if (input == null) return string.Empty; @@ -399,5 +432,7 @@ private static string EscapeCsv(string input, string delimiter) return "\"" + doubled + "\""; } + /***************************************************/ + } } From 05598c4c9916ff396dcb986be231ff1624921acd Mon Sep 17 00:00:00 2001 From: linhnam-nguyen <88286063+linhnam-nguyen@users.noreply.github.com> Date: Sun, 14 Sep 2025 18:32:45 +0200 Subject: [PATCH 13/16] Fixed code compliances --- File_Engine/Compute/ReadFromCsvFile.cs | 111 ++++++++++++------------- File_Engine/Compute/WriteToCsvFile.cs | 2 +- 2 files changed, 52 insertions(+), 61 deletions(-) diff --git a/File_Engine/Compute/ReadFromCsvFile.cs b/File_Engine/Compute/ReadFromCsvFile.cs index 533bf5d..095a723 100644 --- a/File_Engine/Compute/ReadFromCsvFile.cs +++ b/File_Engine/Compute/ReadFromCsvFile.cs @@ -63,7 +63,6 @@ public static partial class Compute string[] lines; try { - // Note: ReadAllLines splits on Environment.NewLine (handles \r\n and \n) lines = System.IO.File.ReadAllLines(filePath); } catch (Exception e) @@ -80,7 +79,6 @@ public static partial class Compute for (int i = 0; i < lines.Length; i++) { var line = lines[i]; - // Skip completely empty lines if (string.IsNullOrEmpty(line)) continue; @@ -123,11 +121,10 @@ public static partial class Compute /**** Private Helpers ****/ /***************************************************/ - // CSV splitter: handles quotes, escaped quotes (""), and multi-char delimiters (e.g., "\t") private static string[] SplitCsvLine(string line, string delimiter) { if (string.IsNullOrEmpty(line)) - return new string[0]; + return Array.Empty(); var cells = new List(); var current = new System.Text.StringBuilder(); @@ -144,8 +141,7 @@ private static string[] SplitCsvLine(string line, string delimiter) { if (inQuotes && i + 1 < n && line[i + 1] == '"') { - // Escaped quote -> add one quote - current.Append('"'); + current.Append('"'); // Escaped quote i += 2; continue; } @@ -157,7 +153,6 @@ private static string[] SplitCsvLine(string line, string delimiter) if (!inQuotes && dlen > 0 && i + dlen <= n && string.CompareOrdinal(line, i, delimiter, 0, dlen) == 0) { - // End of cell cells.Add(current.ToString()); current.Length = 0; i += dlen; @@ -172,9 +167,8 @@ private static string[] SplitCsvLine(string line, string delimiter) return cells.ToArray(); } - // Column-aware parsing with fallbacks: - // - If ColumnDataFormats[j] exists, parse by that contract - // - Else, try bool/(1/0), numeric (with DecimalSeparator), date (several formats), else string + /***************************************************/ + private static object ParseCell(string raw, int columnIndex, CsvConfig settings) { if (string.IsNullOrEmpty(raw)) @@ -190,29 +184,27 @@ private static object ParseCell(string raw, int columnIndex, CsvConfig settings) switch (settings.ColumnDataFormats[columnIndex]) { case StringType.Boolean: - return ParseBool(raw, settings); + { + var b = ParseBool(raw, settings); + return b.HasValue ? (object)b.Value : null; + } case StringType.Numeric: { - double num; - if (TryParseNumber(raw, settings.DecimalSeparator, out num)) - return num; - return null; + var num = ParseNumeric(raw, settings.DecimalSeparator); + return num.HasValue ? (object)num.Value : null; } case StringType.Date: { - DateTime dt; - if (TryParseDate(raw, settings.DateTimeFormat, out dt)) - return dt; - // If date parsing fails but it's a double OA, try that as a last resort - double serial; - if (double.TryParse(raw.Replace(settings.DecimalSeparator, "."), NumberStyles.Any, CultureInfo.InvariantCulture, out serial)) + var dt = ParseDate(raw, settings.DateTimeFormat); + if (dt.HasValue) return dt.Value; + + // OA serial fallback (Excel serial date) + var serial = ParseNumeric(raw, settings.DecimalSeparator); + if (serial.HasValue) { - try - { - return DateTime.FromOADate(serial); - } + try { return DateTime.FromOADate(serial.Value); } catch { /* ignore */ } } return null; @@ -223,30 +215,23 @@ private static object ParseCell(string raw, int columnIndex, CsvConfig settings) } } - // ---- Heuristic fallback (no explicit column format) ---- - - // Bool + // Try Bool var bParsed = ParseBool(raw, settings); - if (bParsed != null) - return bParsed.Value; + if (bParsed.HasValue) return bParsed.Value; - // Number - double d; - if (TryParseNumber(raw, settings.DecimalSeparator, out d)) - return d; + // Try Number + var d = ParseNumeric(raw, settings.DecimalSeparator); + if (d.HasValue) return d.Value; - // Date (ISO/US/EU tries + OA double fallback) - DateTime when; - if (TryParseDate(raw, settings.DateTimeFormat, out when)) - return when; + // Try Date + var when = ParseDate(raw, settings.DateTimeFormat); + if (when.HasValue) return when.Value; - double serial2; - if (double.TryParse(raw.Replace(settings.DecimalSeparator, "."), NumberStyles.Any, CultureInfo.InvariantCulture, out serial2)) + // OA serial fallback + var serial2 = ParseNumeric(raw, settings.DecimalSeparator); + if (serial2.HasValue) { - try - { - return DateTime.FromOADate(serial2); - } + try { return DateTime.FromOADate(serial2.Value); } catch { /* ignore */ } } @@ -254,6 +239,8 @@ private static object ParseCell(string raw, int columnIndex, CsvConfig settings) return raw; } + /***************************************************/ + private static bool? ParseBool(string raw, CsvConfig settings) { if (string.Equals(raw, "true", StringComparison.OrdinalIgnoreCase)) return true; @@ -267,28 +254,32 @@ private static object ParseCell(string raw, int columnIndex, CsvConfig settings) return null; } - private static bool TryParseNumber(string raw, string decimalSeparator, out double value) + /***************************************************/ + + private static double? ParseNumeric(string raw, string decimalSeparator) { - value = 0d; if (string.IsNullOrEmpty(raw)) - return false; + return null; - // Normalise custom decimal separator to "." var norm = string.IsNullOrEmpty(decimalSeparator) || decimalSeparator == "." ? raw : raw.Replace(decimalSeparator, "."); - return double.TryParse(norm, NumberStyles.Any, CultureInfo.InvariantCulture, out value); + if (double.TryParse(norm, NumberStyles.Any, CultureInfo.InvariantCulture, out var value)) + return value; + + return null; } - // Date parsing guided by DateFormatOptions (date-only by default), with tolerant fallbacks. - private static bool TryParseDate(string raw, DateFormatOptions option, out DateTime dt) + /***************************************************/ + + private static DateTime? ParseDate(string raw, DateFormatOptions option) { - dt = default(DateTime); if (string.IsNullOrEmpty(raw)) - return false; + return null; string[] patterns; + switch (option) { case DateFormatOptions.ISO8601: @@ -326,16 +317,16 @@ private static bool TryParseDate(string raw, DateFormatOptions option, out DateT DateTime tmp; if (DateTime.TryParseExact(raw, patterns[i], CultureInfo.InvariantCulture, DateTimeStyles.None, out tmp)) - { - dt = tmp; - return true; - } + return tmp; } - if (DateTime.TryParse(raw, CultureInfo.InvariantCulture, DateTimeStyles.None, out dt)) - return true; + DateTime any; + if (DateTime.TryParse(raw, CultureInfo.InvariantCulture, DateTimeStyles.None, out any)) + return any; - return false; + return null; } + + /***************************************************/ } } \ No newline at end of file diff --git a/File_Engine/Compute/WriteToCsvFile.cs b/File_Engine/Compute/WriteToCsvFile.cs index 2b697a4..4c56426 100644 --- a/File_Engine/Compute/WriteToCsvFile.cs +++ b/File_Engine/Compute/WriteToCsvFile.cs @@ -75,7 +75,7 @@ public static bool WriteToCsvFile(object data, string filePath, CsvConfig settin try { - System.IO.File.WriteAllText(filePath, table); + System.IO.File.WriteAllText(filePath, table, Encoding.Unicode); } catch (Exception e) { From 56dca5aa0ab4e1fd7362618a1f29da2abc5dbef2 Mon Sep 17 00:00:00 2001 From: linhnam-nguyen <88286063+linhnam-nguyen@users.noreply.github.com> Date: Tue, 16 Sep 2025 10:01:52 +0200 Subject: [PATCH 14/16] debug stackoverflow bug --- File_Engine/Compute/WriteToCsvFile.cs | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/File_Engine/Compute/WriteToCsvFile.cs b/File_Engine/Compute/WriteToCsvFile.cs index 4c56426..16fc757 100644 --- a/File_Engine/Compute/WriteToCsvFile.cs +++ b/File_Engine/Compute/WriteToCsvFile.cs @@ -232,7 +232,10 @@ public static string FromObject(this object obj, CsvConfig settings = null) private static string FormatCell(object value, CsvConfig settings, int? column, bool isHeader = false) { - if (settings.ColumnDataFormats != null && column.HasValue && ((settings.IncludeHeader && !isHeader) || !settings.IncludeHeader)) + if (settings.ColumnDataFormats != null + && column.HasValue + && column.Value < settings.ColumnDataFormats.Count + && ((settings.IncludeHeader && !isHeader) || !settings.IncludeHeader)) { StringType? format = settings.ColumnDataFormats[column.Value]; @@ -303,6 +306,9 @@ private static string FormatCell(object value, CsvConfig settings, int? column, if (settings.IncludeObjects) return value.FormatObject(); + //String + if (value is string s) + return s; return string.Empty; } @@ -328,22 +334,18 @@ private static string FormatDate(this IFormattable date, DateFormatOptions optio } } - // Handle DateTime - if (date is DateTime dt2) - return dt2.FormatDate(option); - // Handle DateTimeOffset - if (date is DateTimeOffset dto) + if (date is DateTime dt2) { switch (option) { case DateFormatOptions.ISO8601: - return dto.ToString("o", CultureInfo.InvariantCulture); + return dt2.ToString("o", CultureInfo.InvariantCulture); case DateFormatOptions.US: - return dto.ToString("MM/dd/yyyy", CultureInfo.InvariantCulture); + return dt2.ToString("MM/dd/yyyy", CultureInfo.InvariantCulture); case DateFormatOptions.EU: default: - return dto.ToString("dd/MM/yyyy", CultureInfo.InvariantCulture); + return dt2.ToString("dd/MM/yyyy", CultureInfo.InvariantCulture); } } From 0197dcadc954ba85bf646cf315a5383b9c174b3b Mon Sep 17 00:00:00 2001 From: linhnam-nguyen <88286063+linhnam-nguyen@users.noreply.github.com> Date: Tue, 16 Sep 2025 12:53:45 +0200 Subject: [PATCH 15/16] Changed returned type, removed formats on header row --- File_Engine/Compute/ReadFromCsvFile.cs | 41 +++++++++++++++----------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/File_Engine/Compute/ReadFromCsvFile.cs b/File_Engine/Compute/ReadFromCsvFile.cs index 095a723..6c6012a 100644 --- a/File_Engine/Compute/ReadFromCsvFile.cs +++ b/File_Engine/Compute/ReadFromCsvFile.cs @@ -39,21 +39,21 @@ public static partial class Compute [Input("filePath", "Path to the CSV file.")] [Input("settings", "CSV settings including delimiter, decimal separator, and per-column formats. If null, defaults are used.")] [Input("active", "Boolean used to trigger the function.")] - public static object[,] ReadFromCsvFile(string filePath, CsvConfig settings = null, bool active = false) + public static IEnumerable> ReadFromCsvFile(string filePath, CsvConfig settings = null, bool active = false) { if (!active) - return new object[0, 0]; + return Array.Empty>(); if (string.IsNullOrWhiteSpace(filePath)) { BH.Engine.Base.Compute.RecordError("The file path must not be empty."); - return new object[0, 0]; + return Array.Empty>(); } if (!System.IO.File.Exists(filePath)) { BH.Engine.Base.Compute.RecordError($"The file `{filePath}` does not exist."); - return new object[0, 0]; + return Array.Empty>(); } if (settings == null) @@ -68,11 +68,11 @@ public static partial class Compute catch (Exception e) { BH.Engine.Base.Compute.RecordError($"Error reading file:\n\t{e}"); - return new object[0, 0]; + return Array.Empty>(); } if (lines.Length == 0) - return new object[0, 0]; + return Array.Empty>(); // 1) Parse lines into raw string rows (CSV rules: quotes + escaped quotes) var rawRows = new List(); @@ -86,7 +86,7 @@ public static partial class Compute } if (rawRows.Count == 0) - return new object[0, 0]; + return Array.Empty>(); // 2) Normalize columns (pad ragged rows to max width) int rAll = rawRows.Count; @@ -95,26 +95,28 @@ public static partial class Compute if (rawRows[i].Length > c) c = rawRows[i].Length; if (c == 0) - return new object[0, 0]; + return Array.Empty>(); // 3) Decide data start index based on IncludeHeader - int startRow = (settings.IncludeHeader && rAll > 0) ? 1 : 0; - int rData = Math.Max(0, rAll - startRow); + var result = new List>(rAll); - var table = new object[rData, c]; - - // 4) Parse cells with column-aware formats - for (int i = 0; i < rData; i++) + for (int i = 0; i < rAll; i++) { - var src = rawRows[i + startRow]; + bool isHeader = settings.IncludeHeader && i == 0; + + var src = rawRows[i]; + var row = new List(c); + for (int j = 0; j < c; j++) { string cell = j < src.Length ? src[j] : null; - table[i, j] = ParseCell(cell, j, settings); + row.Add(ParseCell(cell, j, settings, isHeader)); } + + result.Add(row); } - return table; + return result; } /***************************************************/ @@ -169,11 +171,14 @@ private static string[] SplitCsvLine(string line, string delimiter) /***************************************************/ - private static object ParseCell(string raw, int columnIndex, CsvConfig settings) + private static object ParseCell(string raw, int columnIndex, CsvConfig settings, bool isHeader = false) { if (string.IsNullOrEmpty(raw)) return null; + if (isHeader) + return raw; + bool hasColumnFormat = settings.ColumnDataFormats != null && columnIndex >= 0 && From 4e47bcd20d5cc1d82f76a3194488cf5af58708d7 Mon Sep 17 00:00:00 2001 From: linhnam-nguyen <88286063+linhnam-nguyen@users.noreply.github.com> Date: Fri, 19 Sep 2025 08:34:56 +0200 Subject: [PATCH 16/16] Added Encoding to CSVConfig --- File_Engine/Compute/WriteToCsvFile.cs | 2 +- File_Engine/Query/Encoding.cs | 2 +- File_oM/Config/CsvConfig.cs | 3 +++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/File_Engine/Compute/WriteToCsvFile.cs b/File_Engine/Compute/WriteToCsvFile.cs index 16fc757..16c5fbe 100644 --- a/File_Engine/Compute/WriteToCsvFile.cs +++ b/File_Engine/Compute/WriteToCsvFile.cs @@ -75,7 +75,7 @@ public static bool WriteToCsvFile(object data, string filePath, CsvConfig settin try { - System.IO.File.WriteAllText(filePath, table, Encoding.Unicode); + System.IO.File.WriteAllText(filePath, table, Query.FromEnum(settings.Encoding)); } catch (Exception e) { diff --git a/File_Engine/Query/Encoding.cs b/File_Engine/Query/Encoding.cs index 22bbdd9..27cc3c5 100644 --- a/File_Engine/Query/Encoding.cs +++ b/File_Engine/Query/Encoding.cs @@ -62,7 +62,7 @@ public static Encoding Encoding(this FSFile file) /***************************************************/ - private static Encoding FromEnum(Encodings encodingEnumValue) + public static Encoding FromEnum(Encodings encodingEnumValue) { switch (encodingEnumValue) { diff --git a/File_oM/Config/CsvConfig.cs b/File_oM/Config/CsvConfig.cs index 0f85e1d..c1c1407 100644 --- a/File_oM/Config/CsvConfig.cs +++ b/File_oM/Config/CsvConfig.cs @@ -56,6 +56,9 @@ public class CsvConfig : ActionConfig [Description(" The format to use for date values. Options include ISO8601 (e.g., 2023-10-05T14:48:00Z), US (e.g., 10/05/2023), and EU (e.g., 05/10/2023). Default is ISO8601.")] public DateFormatOptions DateTimeFormat { get; set; } = DateFormatOptions.EU; + + [Description(" The text encoding to use when reading or writing the CSV file. Default is UTF-8.")] + public Encodings Encoding { get; set; } = Encodings.UTF8; } }