AeroScan · PedroCorcaque · Aug 10, 2023 · Aug 10, 2023 · Aug 12, 2023 · Aug 12, 2023
diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,5 @@
 data
 result
 __pycache__
-files_unprocessed.txt
+files_unprocessed.txt
+logs/*
diff --git a/data_generator.py b/data_generator.py
@@ -18,9 +18,10 @@
     create_dirs,
     list_files,
     compareDictsWithTolerance)
+from lib.logger import Logger
 from lib.generate_gmsh import processGMSH
 from lib.generate_pythonocc import processPythonOCC
-from lib.generate_statistics import generateStatistics, generateStatisticsOld
+from lib.generate_statistics import generateStatistics
 
 from asGeometryOCCWrapper.curves import CurveFactory
 from asGeometryOCCWrapper.surfaces import SurfaceFactory
@@ -41,6 +42,9 @@ def parse_opt():
     parser.add_argument('--use_highest_dim', action='store_true', help='Boolean flag to indicate whether to use the highest dimension of the input CAD as reference or not')
     parser.add_argument('--delete_old_data', action='store_true', help='Boolean flag indicating whether to delete old data in the output directory')
     parser.add_argument('--verbose', action='store_true', help='Boolean flag indicating whether to run the code in debug mode.')
+    parser.add_argument('--log_folder', type=str, default="", help="Path to the folder where the logs will be saved.")
+    parser.add_argument('--log_file', type=str, default="", help="Name of the log file. Default: log_\{timestamp\}.txt")
+    parser.add_argument('--log_level', type=str, default="debug", help="Level fo the log. Possible levels: debug, info, warn, error.")
 
     # Mesh parser general
     mesh_parser = parser.add_argument_group("Mesh arguments")
@@ -74,6 +78,9 @@ def main():
     meta_path = args.meta_path
     delete_old_data = args.delete_old_data
     verbose = args.verbose
+    log_folder = args.log_folder
+    log_file = args.log_file
+    log_level = args.log_level
     # <--- General arguments
 
     # ---> Mesh arguments
@@ -95,6 +102,8 @@ def main():
     only_stats = args.only_stats
     # <--- Stats arguments
 
+    logger = Logger(log_file, log_folder, log_level, verbose)
+
     # ---> Directories verifications
     files = get_files_from_input_path(input_path)
 
@@ -142,7 +151,7 @@ def main():
                 meta_file = os.path.join(meta_path, meta_filename)
                 if os.path.isfile(meta_file):
                     with open(meta_file, "r") as meta_file_object:
-                        print("\n[Normalization] Using meta_file")
+                        logger.log("[Data Generator] Using meta_file", "info")
                         file_info = yaml.load(meta_file_object, Loader=yaml.FullLoader)
 
                         vertical_up_axis = np.array(file_info["vertical_up_axis"]) if \
@@ -155,22 +164,18 @@ def main():
 
             scale_to_mm = 1000/unit_scale
             unit_scale = 1000
-
-            print(f'\nProcessing file - Model {filename} - [{idx+1}/{len(files)}]:')
+            logger.log(f'[Data Generator] Processing file - Model {filename} - [{idx+1}/{len(files)}]:', "info")
 
             shape, geometries_data, mesh = processPythonOCC(file, generate_mesh=(mesh_generator=="occ"), \
                                                             use_highest_dim=use_highest_dim, scale_to_mm=scale_to_mm, \
-                                                            debug=verbose)
-            print("\n[PythonOCC] Done.")
+                                                            debug=False)
             if mesh_generator == "gmsh":
-                print('\n[GMSH]:')
                 features, mesh = processGMSH(input_name=file, mesh_size=mesh_size, \
                                              features=features, mesh_name=mesh_name, \
                                                 shape=shape, use_highest_dim=use_highest_dim, \
-                                                    debug=verbose)
-                print("\n[GMSH] Done.")
+                                                    debug=False)
 
-            print('\n[Normalization]')
+            logger.log('[Data Generator] Normalizing', "info")
             R = np.eye(3)
             t = np.zeros(3)
             s = 1./unit_scale
@@ -209,17 +214,17 @@ def main():
 
                 del geometries_data['surfaces'][face_idx]['mesh_data']
 
-            print("\n[Normalization] Done.")
+            logger.log("[Data Generator] Done.", "info")
 
-            print('\n[Generating statistics]')
+            logger.log('[Data Generator] Generating statistics', "info")
             stats = generateStatistics(geometries_data, o3d_mesh)
-            print("\n[Statistics] Done.")
+            logger.log("[Data Generator] Done.", "info")
 
-            print('\n[Writing meshes]')
+            logger.log('[Data Generator] Writing PLY', "info")
             writeMeshPLY(mesh_name, o3d_mesh)
-            print('\n[Writing meshes] Done.')
+            logger.log('[Data Generator] Done.', "info")
 
-            print('\n[Writing Features]')
+            logger.log('[Data Generator] Writing features', "info")
              # creating features dict
             features = {'curves': [], 'surfaces': []}
             for edge_data in geometries_data['curves']:
@@ -229,24 +234,24 @@ def main():
                 if face_data['geometry'] is not None:
                     features['surfaces'].append(dict(face_data['geometry'].toDict()))
             writeFeatures(features_name=features_name, features=features, tp=features_file_type)
-            print("\n[Writing Features] Done.")
+            logger.log("[Data Generator] Done.", "info")
 
-            print('\n[Writing Statistics]')
+            logger.log('[Data Generator] Writing JSON', "info")
             writeJSON(stats_name, stats)
-            print("\n[Writing Statistics] Done.")
+            logger.log("[Data Generator] Done.", "info")
 
-            print('\n[Generator] Process done.')
+            logger.log('[Data Generator] Process done.', "info")
 
             #del stats
             del features
             del o3d_mesh
             gc.collect()
     else:
-        print("Reading features list...")
+        logger.log("[Data Generator] Reading features list...", "info")
         features = list(set(features_files) - set(statistics_files)) if not delete_old_data else \
                                                                     features_files
         for idx, feature_name in enumerate(features):
-            print(f"\nProcessing file - Model {feature_name} - [{idx+1}/{len(features)}]:")
+            logger.log(f"Processing file - Model {feature_name} - [{idx+1}/{len(features)}]:", "info")
 
             stats_name = os.path.join(stats_folder_dir, feature_name)
             remove_by_filename(stats_name, STATS_FORMATS)
@@ -268,15 +273,15 @@ def main():
                 surface.setMeshByGlobal(mesh)
                 geometries['surfaces'].append({'geometry': surface})
 
-            print("\nGenerating statistics...")
+            logger.log("[Data Generator] Generating statistics...", "info")
             stats = generateStatistics(geometries, mesh)
 
-            print("Writing stats in statistic file...")
+            logger.log("[Data Generator] Writing stats in statistic file...", "info")
             writeJSON(stats_name, stats)
 
             del mesh, features_data, stats
             gc.collect()
-        print(f"\nDone. {len(features)} were processed.")
+        logger.log(f"[Data Generator] Done. {len(features)} were processed.", "info")
     # <--- Main loop
 
 if __name__ == '__main__':

diff --git a/lib/__init__.py b/lib/__init__.py
@@ -2,4 +2,5 @@
 from .generate_gmsh import *
 from .generate_mesh_occ import *
 from .generate_pythonocc import *
-from .generate_statistics import *
+from .generate_statistics import *
+from .logger import Logger
diff --git a/lib/dataset_handler.py b/lib/dataset_handler.py
@@ -0,0 +1,22 @@
+import os
+import logging
+
+class DatasetHandler(logging.Handler):
+
+    _self = None
+    def __new__(cls, filename, foldername):
+        if cls._self is None:
+            cls._self = super().__new__(cls)
+        return cls._self
+
+    def __init__(self, filename: str, foldername: str):
+        super().__init__()
+        self.filename = os.path.join(foldername, filename)
+
+    def emit(self, record):
+        try:
+            log_message = self.format(record)
+            with open(self.filename, "a") as file:
+                file.write(log_message + "\n")
+        except (FileNotFoundError, FileExistsError) as exception:
+            raise Exception() from exception
diff --git a/lib/generate_mesh_occ.py b/lib/generate_mesh_occ.py
@@ -14,6 +14,10 @@
 
 from tqdm import tqdm
 
+from lib.logger import Logger
+
+logger = Logger()
+
 MAX_INT = 2**31 - 1
 
 # def findPointInListWithHashCode(point, points, hash_codes):
@@ -94,7 +98,7 @@ def addEntityToMap(index, entity, map):
 def computeMeshData(vertices, edges, faces, topology):
     vertices_mesh_data = []
     vertices_map = {}
-    print('\n[PythonOCC] Mapping Vertices...')
+    logger.log('[Generate Mesh OCC] Mapping Vertices...', "info")
     for i, vertex in enumerate(tqdm(vertices)):
         vertices_mesh_data.append(-1)
         addEntityToMap(i, vertex, vertices_map)
@@ -103,7 +107,7 @@ def computeMeshData(vertices, edges, faces, topology):
     edges_mesh_data = []
     edge_vertices_map = []
     edges_map = {}
-    print('\n[PythonOCC] Mapping Edges...')
+    logger.log('[Generate Mesh OCC] Mapping Edges...', "info")
     for i, edge in enumerate(tqdm(edges)):
         edges_mesh_data.append({'vert_indices': [], 'vert_parameters': []})
         addEntityToMap(i, edge, edges_map)
@@ -113,18 +117,16 @@ def computeMeshData(vertices, edges, faces, topology):
     faces_mesh_data = []
     face_edges_map = []
     faces_map = {}
-    print('\n[PythonOCC] Mapping Faces...')
+    logger.log('[Generate Mesh OCC] Mapping Faces...', "info")
     for i, face in enumerate(tqdm(faces)):
         faces_mesh_data.append({'vert_indices': [], 'vert_parameters': [], 'face_indices': []})
         addEntityToMap(i, face, faces_map)
         edges_indices = [searchEntityInMap(edge, edges_map) for edge in topology.edges_from_face(face)]
         face_edges_map.append(edges_indices)
     mesh_vertices = []
     mesh_faces = []
-    print('\n[PythonOCC] Generating Mesh Data...')
+    logger.log('[Generate Mesh OCC] Generating Mesh Data...', "info")
     for face_index, face in enumerate(tqdm(faces)):
-        #print('----------------------------------------------------')
-        #print("FACE_INDEX: ", face_index)
 
         face_orientation = face.Orientation()
 
@@ -134,7 +136,7 @@ def computeMeshData(vertices, edges, faces, topology):
         transform = location.Transformation()
 
         if triangulation is None:
-            #WARNING
+            logger.log("[Generate Mesh OCC] The triangulation is None", "warn")
             continue
 
         number_vertices = triangulation.NbNodes()
@@ -160,7 +162,7 @@ def computeMeshData(vertices, edges, faces, topology):
 
             polygon = brep_tool.PolygonOnTriangulation(edge, triangulation, location) # projecting edge in the face triangulation
             if polygon is None:
-                #WARNING
+                logger.log("[Generate Mesh OCC] The polygon is None", "warn")
                 continue
 
             edge_vert_local = np.asarray(polygon.Nodes(), dtype=np.int64) - 1 # map from mesh edge indices to face mesh indices
@@ -169,13 +171,13 @@ def computeMeshData(vertices, edges, faces, topology):
             edge_vert_local_unique = np.unique(edge_vert_local)
             if (len(edge_vert_local) -  len(edge_vert_local_unique)) > 1:
                 has_degenerated_edge = True
-                print(f'WARNING 1: degenerated edge ({edge_index}), canceling face ({face_index})')
+                logger.log(f'[Generate Mesh OCC] degenerated edge ({edge_index}), canceling face ({face_index})', "warn")
                 break
 
             elif (len(edge_vert_local) - len(edge_vert_local_unique)) == 1 and  \
                  (len(edge_vert_local) == 2 or edge_vert_local[0] != edge_vert_local[-1] or len(vertices_index) == 2):
                 has_degenerated_edge = True
-                print(f'WARNING 2: degenerated edge ({edge_index}), canceling face ({face_index})')
+                logger.log(f'[Generate Mesh OCC] degenerated edge ({edge_index}), canceling face ({face_index})', "warn")
                 break
 
             if len(edges_mesh_data[edge_index]['vert_indices']) == 0:
@@ -207,7 +209,7 @@ def computeMeshData(vertices, edges, faces, topology):
                 is_reversed = np.allclose(vertices_array, nodes_array[indices[1]], rtol=0.)
 
                 if is_foward and is_reversed:
-                    print('ERROR')
+                    logger.log('[Generate Mesh OCC] is_foward and is_reversed', "error")
                     continue
 
             bound_indices = [-1, 0] if is_reversed else [0, -1]
@@ -217,7 +219,7 @@ def computeMeshData(vertices, edges, faces, topology):
             diff_mask = current_vertex != vertices_index
             if np.any(np.logical_and(diff_mask, current_vertex != -1)):
                 has_degenerated_edge = True
-                print(f'WARNING 3: degenerated edge ({edge_index}), canceling face ({face_index})')
+                logger.log(f'[Generate Mesh OCC] degenerated edge ({edge_index}), canceling face ({face_index})', "warn")
                 break
             else:
                 face_vertex_node_map[vertex_nodes] = vertices_index
@@ -256,7 +258,6 @@ def computeMeshData(vertices, edges, faces, topology):
             if len(vertices_index) == 1 and edge_vert_local[0] != edge_vert_local[-1]:
                 #triangulation is not closed but the egde is
                 #changing triangulation to be closed too
-                #print(f'ERROR HERE {edge_vert_local} {vertices_index}')
 
                 first_vertex, last_vertex = edge_vert_local[bound_indices]
 
@@ -389,10 +390,10 @@ def computeMeshData(vertices, edges, faces, topology):
                     if not (np.allclose(mesh_vertices[i1_m], np.array(triangulation.Node(i1).Transformed(transform).Coord()), rtol=0.) and \
                            np.allclose(mesh_vertices[i2_m], np.array(triangulation.Node(i2).Transformed(transform).Coord()), rtol=0.) and \
                            np.allclose(mesh_vertices[i3_m], np.array(triangulation.Node(i3).Transformed(transform).Coord()), rtol=0.)):
-                        print(f'Vertices remapping problem.\n' \
+                        logger.log(f'[Generate Mesh OCC] Vertices remapping problem.\n' \
                               f'{mesh_vertices[i1_m]} != {np.array(triangulation.Node(i1).Transformed(transform).Coord())} or \n' \
                               f'{mesh_vertices[i2_m]} != {np.array(triangulation.Node(i2).Transformed(transform).Coord())} or \n' \
-                              f'{mesh_vertices[i3_m]} != {np.array(triangulation.Node(i3).Transformed(transform).Coord())}')
+                              f'{mesh_vertices[i3_m]} != {np.array(triangulation.Node(i3).Transformed(transform).Coord())}', "error")
 
             if face_orientation == 0:
                 verts_of_face = np.array([i1_m, i2_m, i3_m])
@@ -413,10 +414,6 @@ def computeMeshData(vertices, edges, faces, topology):
     #assert np.all(unique_vert_faces == unique_vert), \
     #           f'ERROR: unreferenced vertices in global mesh'
 
-    #print('problematics:', problematics)
-    #print('good:', len(faces) - problematics)
-    #print('locations:', len(locations))
-
     for edge_index in range(len(edges_mesh_data)):
         if type(edges_mesh_data[edge_index]['vert_indices']) is not list:
             edges_mesh_data[edge_index]['vert_indices'] = edges_mesh_data[edge_index]['vert_indices'].tolist()
@@ -426,7 +423,7 @@ def computeMeshData(vertices, edges, faces, topology):
     return mesh_vertices, mesh_faces, edges_mesh_data, faces_mesh_data
 
 def OCCMeshGeneration(shape):
-    print('\n[PythonOCC] Mesh Generation...')
+    logger.log('[Generate Mesh OCC] Mesh Generation...', "info")
     parameters = IMeshTools_Parameters()
 
     #Ref: https://dev.opencascade.org/doc/refman/html/struct_i_mesh_tools___parameters.html#a3027dc569da3d3e3fcd76e0615befb27