diff --git a/.gitignore b/.gitignore index 2cbf4a0..5e72cdb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ data result __pycache__ -files_unprocessed.txt \ No newline at end of file +files_unprocessed.txt +logs/* \ No newline at end of file diff --git a/data_generator.py b/data_generator.py index 0bd5368..dd7faa1 100644 --- a/data_generator.py +++ b/data_generator.py @@ -18,9 +18,10 @@ create_dirs, list_files, compareDictsWithTolerance) +from lib.logger import Logger from lib.generate_gmsh import processGMSH from lib.generate_pythonocc import processPythonOCC -from lib.generate_statistics import generateStatistics, generateStatisticsOld +from lib.generate_statistics import generateStatistics from asGeometryOCCWrapper.curves import CurveFactory from asGeometryOCCWrapper.surfaces import SurfaceFactory @@ -41,6 +42,9 @@ def parse_opt(): parser.add_argument('--use_highest_dim', action='store_true', help='Boolean flag to indicate whether to use the highest dimension of the input CAD as reference or not') parser.add_argument('--delete_old_data', action='store_true', help='Boolean flag indicating whether to delete old data in the output directory') parser.add_argument('--verbose', action='store_true', help='Boolean flag indicating whether to run the code in debug mode.') + parser.add_argument('--log_folder', type=str, default="", help="Path to the folder where the logs will be saved.") + parser.add_argument('--log_file', type=str, default="", help="Name of the log file. Default: log_\{timestamp\}.txt") + parser.add_argument('--log_level', type=str, default="debug", help="Level fo the log. Possible levels: debug, info, warn, error.") # Mesh parser general mesh_parser = parser.add_argument_group("Mesh arguments") @@ -74,6 +78,9 @@ def main(): meta_path = args.meta_path delete_old_data = args.delete_old_data verbose = args.verbose + log_folder = args.log_folder + log_file = args.log_file + log_level = args.log_level # <--- General arguments # ---> Mesh arguments @@ -95,6 +102,8 @@ def main(): only_stats = args.only_stats # <--- Stats arguments + logger = Logger(log_file, log_folder, log_level, verbose) + # ---> Directories verifications files = get_files_from_input_path(input_path) @@ -142,7 +151,7 @@ def main(): meta_file = os.path.join(meta_path, meta_filename) if os.path.isfile(meta_file): with open(meta_file, "r") as meta_file_object: - print("\n[Normalization] Using meta_file") + logger.log("[Data Generator] Using meta_file", "info") file_info = yaml.load(meta_file_object, Loader=yaml.FullLoader) vertical_up_axis = np.array(file_info["vertical_up_axis"]) if \ @@ -155,22 +164,18 @@ def main(): scale_to_mm = 1000/unit_scale unit_scale = 1000 - - print(f'\nProcessing file - Model {filename} - [{idx+1}/{len(files)}]:') + logger.log(f'[Data Generator] Processing file - Model {filename} - [{idx+1}/{len(files)}]:', "info") shape, geometries_data, mesh = processPythonOCC(file, generate_mesh=(mesh_generator=="occ"), \ use_highest_dim=use_highest_dim, scale_to_mm=scale_to_mm, \ - debug=verbose) - print("\n[PythonOCC] Done.") + debug=False) if mesh_generator == "gmsh": - print('\n[GMSH]:') features, mesh = processGMSH(input_name=file, mesh_size=mesh_size, \ features=features, mesh_name=mesh_name, \ shape=shape, use_highest_dim=use_highest_dim, \ - debug=verbose) - print("\n[GMSH] Done.") + debug=False) - print('\n[Normalization]') + logger.log('[Data Generator] Normalizing', "info") R = np.eye(3) t = np.zeros(3) s = 1./unit_scale @@ -209,17 +214,17 @@ def main(): del geometries_data['surfaces'][face_idx]['mesh_data'] - print("\n[Normalization] Done.") + logger.log("[Data Generator] Done.", "info") - print('\n[Generating statistics]') + logger.log('[Data Generator] Generating statistics', "info") stats = generateStatistics(geometries_data, o3d_mesh) - print("\n[Statistics] Done.") + logger.log("[Data Generator] Done.", "info") - print('\n[Writing meshes]') + logger.log('[Data Generator] Writing PLY', "info") writeMeshPLY(mesh_name, o3d_mesh) - print('\n[Writing meshes] Done.') + logger.log('[Data Generator] Done.', "info") - print('\n[Writing Features]') + logger.log('[Data Generator] Writing features', "info") # creating features dict features = {'curves': [], 'surfaces': []} for edge_data in geometries_data['curves']: @@ -229,24 +234,24 @@ def main(): if face_data['geometry'] is not None: features['surfaces'].append(dict(face_data['geometry'].toDict())) writeFeatures(features_name=features_name, features=features, tp=features_file_type) - print("\n[Writing Features] Done.") + logger.log("[Data Generator] Done.", "info") - print('\n[Writing Statistics]') + logger.log('[Data Generator] Writing JSON', "info") writeJSON(stats_name, stats) - print("\n[Writing Statistics] Done.") + logger.log("[Data Generator] Done.", "info") - print('\n[Generator] Process done.') + logger.log('[Data Generator] Process done.', "info") #del stats del features del o3d_mesh gc.collect() else: - print("Reading features list...") + logger.log("[Data Generator] Reading features list...", "info") features = list(set(features_files) - set(statistics_files)) if not delete_old_data else \ features_files for idx, feature_name in enumerate(features): - print(f"\nProcessing file - Model {feature_name} - [{idx+1}/{len(features)}]:") + logger.log(f"Processing file - Model {feature_name} - [{idx+1}/{len(features)}]:", "info") stats_name = os.path.join(stats_folder_dir, feature_name) remove_by_filename(stats_name, STATS_FORMATS) @@ -268,15 +273,15 @@ def main(): surface.setMeshByGlobal(mesh) geometries['surfaces'].append({'geometry': surface}) - print("\nGenerating statistics...") + logger.log("[Data Generator] Generating statistics...", "info") stats = generateStatistics(geometries, mesh) - print("Writing stats in statistic file...") + logger.log("[Data Generator] Writing stats in statistic file...", "info") writeJSON(stats_name, stats) del mesh, features_data, stats gc.collect() - print(f"\nDone. {len(features)} were processed.") + logger.log(f"[Data Generator] Done. {len(features)} were processed.", "info") # <--- Main loop if __name__ == '__main__': diff --git a/lib/__init__.py b/lib/__init__.py index c866732..b11ce2b 100644 --- a/lib/__init__.py +++ b/lib/__init__.py @@ -2,4 +2,5 @@ from .generate_gmsh import * from .generate_mesh_occ import * from .generate_pythonocc import * -from .generate_statistics import * \ No newline at end of file +from .generate_statistics import * +from .logger import Logger \ No newline at end of file diff --git a/lib/dataset_handler.py b/lib/dataset_handler.py new file mode 100644 index 0000000..1e4584d --- /dev/null +++ b/lib/dataset_handler.py @@ -0,0 +1,22 @@ +import os +import logging + +class DatasetHandler(logging.Handler): + + _self = None + def __new__(cls, filename, foldername): + if cls._self is None: + cls._self = super().__new__(cls) + return cls._self + + def __init__(self, filename: str, foldername: str): + super().__init__() + self.filename = os.path.join(foldername, filename) + + def emit(self, record): + try: + log_message = self.format(record) + with open(self.filename, "a") as file: + file.write(log_message + "\n") + except (FileNotFoundError, FileExistsError) as exception: + raise Exception() from exception diff --git a/lib/generate_mesh_occ.py b/lib/generate_mesh_occ.py index cbfbd2d..aa2616f 100644 --- a/lib/generate_mesh_occ.py +++ b/lib/generate_mesh_occ.py @@ -14,6 +14,10 @@ from tqdm import tqdm +from lib.logger import Logger + +logger = Logger() + MAX_INT = 2**31 - 1 # def findPointInListWithHashCode(point, points, hash_codes): @@ -94,7 +98,7 @@ def addEntityToMap(index, entity, map): def computeMeshData(vertices, edges, faces, topology): vertices_mesh_data = [] vertices_map = {} - print('\n[PythonOCC] Mapping Vertices...') + logger.log('[Generate Mesh OCC] Mapping Vertices...', "info") for i, vertex in enumerate(tqdm(vertices)): vertices_mesh_data.append(-1) addEntityToMap(i, vertex, vertices_map) @@ -103,7 +107,7 @@ def computeMeshData(vertices, edges, faces, topology): edges_mesh_data = [] edge_vertices_map = [] edges_map = {} - print('\n[PythonOCC] Mapping Edges...') + logger.log('[Generate Mesh OCC] Mapping Edges...', "info") for i, edge in enumerate(tqdm(edges)): edges_mesh_data.append({'vert_indices': [], 'vert_parameters': []}) addEntityToMap(i, edge, edges_map) @@ -113,7 +117,7 @@ def computeMeshData(vertices, edges, faces, topology): faces_mesh_data = [] face_edges_map = [] faces_map = {} - print('\n[PythonOCC] Mapping Faces...') + logger.log('[Generate Mesh OCC] Mapping Faces...', "info") for i, face in enumerate(tqdm(faces)): faces_mesh_data.append({'vert_indices': [], 'vert_parameters': [], 'face_indices': []}) addEntityToMap(i, face, faces_map) @@ -121,10 +125,8 @@ def computeMeshData(vertices, edges, faces, topology): face_edges_map.append(edges_indices) mesh_vertices = [] mesh_faces = [] - print('\n[PythonOCC] Generating Mesh Data...') + logger.log('[Generate Mesh OCC] Generating Mesh Data...', "info") for face_index, face in enumerate(tqdm(faces)): - #print('----------------------------------------------------') - #print("FACE_INDEX: ", face_index) face_orientation = face.Orientation() @@ -134,7 +136,7 @@ def computeMeshData(vertices, edges, faces, topology): transform = location.Transformation() if triangulation is None: - #WARNING + logger.log("[Generate Mesh OCC] The triangulation is None", "warn") continue number_vertices = triangulation.NbNodes() @@ -160,7 +162,7 @@ def computeMeshData(vertices, edges, faces, topology): polygon = brep_tool.PolygonOnTriangulation(edge, triangulation, location) # projecting edge in the face triangulation if polygon is None: - #WARNING + logger.log("[Generate Mesh OCC] The polygon is None", "warn") continue edge_vert_local = np.asarray(polygon.Nodes(), dtype=np.int64) - 1 # map from mesh edge indices to face mesh indices @@ -169,13 +171,13 @@ def computeMeshData(vertices, edges, faces, topology): edge_vert_local_unique = np.unique(edge_vert_local) if (len(edge_vert_local) - len(edge_vert_local_unique)) > 1: has_degenerated_edge = True - print(f'WARNING 1: degenerated edge ({edge_index}), canceling face ({face_index})') + logger.log(f'[Generate Mesh OCC] degenerated edge ({edge_index}), canceling face ({face_index})', "warn") break elif (len(edge_vert_local) - len(edge_vert_local_unique)) == 1 and \ (len(edge_vert_local) == 2 or edge_vert_local[0] != edge_vert_local[-1] or len(vertices_index) == 2): has_degenerated_edge = True - print(f'WARNING 2: degenerated edge ({edge_index}), canceling face ({face_index})') + logger.log(f'[Generate Mesh OCC] degenerated edge ({edge_index}), canceling face ({face_index})', "warn") break if len(edges_mesh_data[edge_index]['vert_indices']) == 0: @@ -207,7 +209,7 @@ def computeMeshData(vertices, edges, faces, topology): is_reversed = np.allclose(vertices_array, nodes_array[indices[1]], rtol=0.) if is_foward and is_reversed: - print('ERROR') + logger.log('[Generate Mesh OCC] is_foward and is_reversed', "error") continue bound_indices = [-1, 0] if is_reversed else [0, -1] @@ -217,7 +219,7 @@ def computeMeshData(vertices, edges, faces, topology): diff_mask = current_vertex != vertices_index if np.any(np.logical_and(diff_mask, current_vertex != -1)): has_degenerated_edge = True - print(f'WARNING 3: degenerated edge ({edge_index}), canceling face ({face_index})') + logger.log(f'[Generate Mesh OCC] degenerated edge ({edge_index}), canceling face ({face_index})', "warn") break else: face_vertex_node_map[vertex_nodes] = vertices_index @@ -256,7 +258,6 @@ def computeMeshData(vertices, edges, faces, topology): if len(vertices_index) == 1 and edge_vert_local[0] != edge_vert_local[-1]: #triangulation is not closed but the egde is #changing triangulation to be closed too - #print(f'ERROR HERE {edge_vert_local} {vertices_index}') first_vertex, last_vertex = edge_vert_local[bound_indices] @@ -389,10 +390,10 @@ def computeMeshData(vertices, edges, faces, topology): if not (np.allclose(mesh_vertices[i1_m], np.array(triangulation.Node(i1).Transformed(transform).Coord()), rtol=0.) and \ np.allclose(mesh_vertices[i2_m], np.array(triangulation.Node(i2).Transformed(transform).Coord()), rtol=0.) and \ np.allclose(mesh_vertices[i3_m], np.array(triangulation.Node(i3).Transformed(transform).Coord()), rtol=0.)): - print(f'Vertices remapping problem.\n' \ + logger.log(f'[Generate Mesh OCC] Vertices remapping problem.\n' \ f'{mesh_vertices[i1_m]} != {np.array(triangulation.Node(i1).Transformed(transform).Coord())} or \n' \ f'{mesh_vertices[i2_m]} != {np.array(triangulation.Node(i2).Transformed(transform).Coord())} or \n' \ - f'{mesh_vertices[i3_m]} != {np.array(triangulation.Node(i3).Transformed(transform).Coord())}') + f'{mesh_vertices[i3_m]} != {np.array(triangulation.Node(i3).Transformed(transform).Coord())}', "error") if face_orientation == 0: verts_of_face = np.array([i1_m, i2_m, i3_m]) @@ -413,10 +414,6 @@ def computeMeshData(vertices, edges, faces, topology): #assert np.all(unique_vert_faces == unique_vert), \ # f'ERROR: unreferenced vertices in global mesh' - #print('problematics:', problematics) - #print('good:', len(faces) - problematics) - #print('locations:', len(locations)) - for edge_index in range(len(edges_mesh_data)): if type(edges_mesh_data[edge_index]['vert_indices']) is not list: edges_mesh_data[edge_index]['vert_indices'] = edges_mesh_data[edge_index]['vert_indices'].tolist() @@ -426,7 +423,7 @@ def computeMeshData(vertices, edges, faces, topology): return mesh_vertices, mesh_faces, edges_mesh_data, faces_mesh_data def OCCMeshGeneration(shape): - print('\n[PythonOCC] Mesh Generation...') + logger.log('[Generate Mesh OCC] Mesh Generation...', "info") parameters = IMeshTools_Parameters() #Ref: https://dev.opencascade.org/doc/refman/html/struct_i_mesh_tools___parameters.html#a3027dc569da3d3e3fcd76e0615befb27 diff --git a/lib/generate_pythonocc.py b/lib/generate_pythonocc.py index ace5e29..bfba365 100644 --- a/lib/generate_pythonocc.py +++ b/lib/generate_pythonocc.py @@ -9,6 +9,9 @@ from lib.generate_mesh_occ import OCCMeshGeneration, computeMeshData from asGeometryOCCWrapper import CurveFactory, SurfaceFactory +from .logger import Logger + +logger = Logger() MAX_INT = 2**31 - 1 @@ -94,7 +97,7 @@ def addFacesAndAssociatedEdgesToDict(faces, topology, faces_dict, edges_dict, ve return faces_dict, edges_dict, vertices_dict def processHighestDim(topology, generate_mesh): - print('\n[PythonOCC] Using Highest Dim Only, trying with Solids...') + logger.log('[Generate PythonOCC] Using Highest Dim Only, trying with Solids...', "info") faces_dict = {} edges_dict = {} @@ -104,17 +107,17 @@ def processHighestDim(topology, generate_mesh): done = True if not done: - print('\n[PythonOCC] There are no Solids, using Faces as highest dim...') + logger.log('[Generate PythonOCC] There are no Solids, using Faces as highest dim...', "info") faces_dict, edges_dict, vertices_dict = addFacesAndAssociatedEdgesToDict(topology.faces(), topology, faces_dict, edges_dict) done = (faces_dict != {}) if not done == 0: - print('\n[PythonOCC] There are no Faces, using Curves as highest dim...') + logger.log('[Generate PythonOCC] There are no Faces, using Curves as highest dim...', "info") edges_dict, vertices_dict = addEdgesAndAssociatedVerticesToDict(topology.edges(), edges_dict) done = (edges_dict != {}) if not done == 0: - print('\n[PythonOCC] There are no Entities to use...') + logger.log('[Generate PythonOCC] There are no Entities to use...', "info") vertices = [] for key in vertices_dict: @@ -134,7 +137,7 @@ def processHighestDim(topology, generate_mesh): def processNoHighestDim(topology, generate_mesh): - print('\n[PythonOCC] Using all the Shapes') + logger.log('[Generate PythonOCC] Using all the Shapes', "info") vertices = [v for v in topology.vertices()] edges = [e for e in topology.edges()] @@ -146,7 +149,7 @@ def processNoHighestDim(topology, generate_mesh): # Generate features by dimensions def process(shape, generate_mesh=True, use_highest_dim=True): - print('\n[PythonOCC] Topology Exploration to Generate Features by Dimension') + logger.log('[Generate PythonOCC] Topology Exploration to Generate Features by Dimension', "info") topology = TopologyExplorer(shape) @@ -183,9 +186,9 @@ def processPythonOCC(input_name: str, generate_mesh=True, use_highest_dim=True, # healer.Status(extend_status) # if extend_status == ShapeExtend_Status.ShapeExtend_OK: - # print("Shape healing successful.") + # logger.log("Shape healing successful.") # else: - # print("Shape healing failed.") + # logger.log("Shape healing failed.") geometries_data, mesh = process(shape, generate_mesh=generate_mesh, use_highest_dim=use_highest_dim) diff --git a/lib/generate_statistics.py b/lib/generate_statistics.py index c9a9a18..2acee55 100644 --- a/lib/generate_statistics.py +++ b/lib/generate_statistics.py @@ -1,7 +1,9 @@ import numpy as np from tqdm import tqdm import open3d as o3d +from .logger import Logger +logger = Logger() def generate_area_from_surface(surface, vertices: np.array, faces: np.array) -> float: """ This function returns the area from the received surface """ @@ -46,7 +48,7 @@ def generateStatisticsOld(features, mesh, only_stats=False): result['bounding_box'] = mesh_obj.get_min_bound().tolist() + (mesh_obj.get_max_bound() - mesh_obj.get_min_bound()).tolist() - print("Generating for curves: ") + logger.log("[Generate statistics] Generating for curves: ", "info") for curve in tqdm(features["curves"]): if curve is not None: tp = curve["type"] @@ -58,7 +60,7 @@ def generateStatisticsOld(features, mesh, only_stats=False): result['number_void_curves'] = 0 surfaces_dict['area'] = 0.0 total_area_of_surfaces = 0.0 - print("Generating for surfaces: ") + logger.log("[Generate statistics] Generating for surfaces: ", "info") for surface in tqdm(features['surfaces']): if surface is not None: tp = surface["type"] @@ -88,7 +90,7 @@ def generateStatistics(geometries_data, mesh): curves_dict = {} surfaces_dict = {} - print("Generating for curves: ") + logger.log("[Generate statistics] Generating for curves: ", "info") for curve in tqdm(geometries_data["curves"]): if curve['geometry'] is not None: tp = curve['geometry'].getType() @@ -109,7 +111,7 @@ def generateStatistics(geometries_data, mesh): number_void_curves = sum([curves_dict[tp]['number_void_curves'] for tp in curves_dict.keys()]) - print("Generating for surfaces: ") + logger.log("[Generate statistics] Generating for surfaces: ", "info") for surface in tqdm(geometries_data['surfaces']): if surface['geometry'] is not None: tp = surface['geometry'].getType() diff --git a/lib/logger.py b/lib/logger.py new file mode 100644 index 0000000..b816ea1 --- /dev/null +++ b/lib/logger.py @@ -0,0 +1,61 @@ +import os +import logging + +from .tools import get_current_timestamp, get_project_root +from .dataset_handler import DatasetHandler + +class Logger: + + LOG_LEVEL = {"debug": logging.DEBUG, + "info": logging.INFO, + "warn": logging.WARN, + "error": logging.ERROR} + + _self = None + def __new__(cls, filename: str = "", foldername: str = "", level: str = "error", stdout: bool = False): + if cls._self is None: + cls._self = super(Logger, cls).__new__(cls) + return cls._self + + def __init__(self, filename: str = "", foldername: str = "", level: str = "error", stdout: bool = False): + if not len(foldername): + foldername = os.path.join(get_project_root(), "logs") + os.makedirs(foldername, exist_ok=True) + if not len(filename): + filename = f"log_{get_current_timestamp()}" + else: + if os.path.isfile(os.path.join(foldername, filename)): + filename += "_" + str(get_current_timestamp()) + + self.logger = logging.getLogger() + self.logger.setLevel(Logger.LOG_LEVEL[level.lower()]) + + formatter = logging.Formatter("'%(asctime)s - %(levelname)s - %(message)s") + handler = DatasetHandler(filename+".txt", foldername) + handler.setFormatter(formatter) + + if stdout: + stream_handler = logging.StreamHandler() + stream_handler.setFormatter(formatter) + self.logger.addHandler(stream_handler) + self.logger.addHandler(handler) + + def log(self, message, level): + """Shows the log based on log level. + + Arguments + --- + message: data to be shown in the log + level: the log level. Possible levels are: debug, info, warn, error + """ + level = level.lower() + if level == "debug": + self.logger.debug(message) + elif level == "info": + self.logger.info(message) + elif level == "warn": + self.logger.warn(message) + elif level == "error": + self.logger.error(message) + else: + self.logger.warning("Log Level not supported.") diff --git a/lib/tools.py b/lib/tools.py index f000357..2abd33d 100644 --- a/lib/tools.py +++ b/lib/tools.py @@ -1,4 +1,5 @@ import os +import time import numpy as np import pickle import json @@ -8,10 +9,20 @@ from OCC.Core.gp import gp_Trsf, gp_Vec, gp_Quaternion, gp_Mat + + CAD_FORMATS = ['.step', '.stp', '.STEP'] MESH_FORMATS = ['.OBJ', '.obj'] FEATURES_FORMATS = ['.pkl', '.PKL', '.yml', '.yaml', '.YAML', '.json', '.JSON'] +def get_project_root() -> Path: + """Returns the path to project root.""" + return Path(__file__).parent.parent + +def get_current_timestamp() -> float: + """Returns the current timestamp.""" + return time.time() + # Convert a float to string def float2str(number, limit = 10) -> str: if abs(number) >= 10**limit: @@ -143,7 +154,7 @@ def writePKL(features_name: str, features: dict): def writeFeatures(features_name: str, features: dict, tp: str): for feature in features['surfaces']: if feature['face_indices'] is None: - print(feature) + continue if tp.lower() in YAML_NAMES: writeYAML(f'{features_name}', features) elif tp.lower() in PKL_NAMES: