validation through entry point + reorg of the arg space

PantheonCMD/assemblies_nesting.py

@@ -0,0 +1,37 @@
+#!/usr/bin/python3
+
+import subprocess
+from validation.pcmsg import Report, print_report_message
+from validation.pcentrypointvalidator import get_exist, get_level_four_assemblies
+
+
+def get_assemblies():
+    command = ("find rhel-8/assemblies/ -type f -name '*.adoc'")
+    process = subprocess.run(command, stdout=subprocess.PIPE, shell=True).stdout
+    assemblie = process.strip().decode('utf-8').split('\n')
+
+    return assemblie
+
+
+def validate_nesting(report, assemblie_found):
+    existing_entry_points = get_exist(assemblie_found)
+    level_four_assemblies = get_level_four_assemblies(existing_entry_points)
+
+    for item in level_four_assemblies:
+        report.create_report('nesting in assemlies. nesting', item)
+
+
+def validate_assemblies():
+    report = Report()
+    assemblies = get_assemblies()
+    validate_nesting(report, assemblies)
+
+    return report
+
+
+def trytry():
+    a = validate_assemblies()
+    print_report_message(a, 'pantheon2.yml')
+
+
+trytry()

PantheonCMD/current_pcentrypointvalidator.py

@@ -0,0 +1,164 @@
+#!/usr/bin/python3
+
+from pcutil import get_not_exist
+from pcmsg import print_message
+import sys
+import os
+import re
+from pcchecks import Regex
+from pcprvalidator import get_all_assemblies, get_all_modules, get_no_prefix_files, get_undetermined_files
+import glob
+
+
+from pcmsg import print_report_message
+from pcvalidator import validation
+
+
+def get_nonexisting_entry_points(entry_point_list):
+    nonexistent_files = get_not_exist(entry_point_list)
+
+    if nonexistent_files:
+        print_message(nonexistent_files, 'entry point', 'does not exist in your repository')
+        sys.exit(2)
+
+
+def get_full_path_to_includes_with_attributes(files):
+    wildcard_sub = []
+    full_path = []
+
+    for item in files:
+        attribute = re.findall(Regex.ATTRIBUTE, item)
+        if attribute:
+            replace = re.sub(Regex.ATTRIBUTE, "**", item)
+
+            wildcard_sub.append(replace)
+
+    for i in wildcard_sub:
+        full_path.append(glob.glob(i, recursive=True))
+
+    return full_path
+
+
+def get_unique_entries(list):
+    unique = []
+    for file in list:
+        if file in unique:
+            continue
+        if any(os.path.samefile(file, item) for item in unique):
+            continue
+        unique.append(file)
+
+    return unique
+
+
+def get_includes(files):
+    """Retreives full paths to included files from an entry point."""
+    includes_with_attributes = []
+    path_to_includes_with_attributes = []
+    includes_found = []
+    includes_not_found = {}
+    unique_entries_includes_with_attributes = []
+
+    for entry in files:
+        path_to_entry_point = os.path.dirname(os.path.abspath(entry))
+
+        # check existence
+
+        with open(entry, 'r') as file:
+            original = file.read()
+            stripped = Regex.MULTI_LINE_COMMENT.sub('', original)
+            stripped = Regex.SINGLE_LINE_COMMENT.sub('', stripped)
+
+            included_files = re.findall(Regex.INCLUDED_CONTENT, stripped)
+
+            if included_files:
+
+                for include in included_files[:]:
+                    itemized_path = include.split(os.sep)
+
+                    attribute_in_path = False
+                    attribute_file = False
+
+                    for item in itemized_path:
+                        if item.startswith('_'):
+                            attribute_file = True
+                            included_files.remove(include)
+                            break
+                        if item.startswith('{'):
+                            attribute_in_path = True
+                            included_files.remove(include)
+                            includes_with_attributes.append(os.path.join(path_to_entry_point, include))
+                            break
+
+                for include in included_files:
+                    full_path = os.path.join(path_to_entry_point, include)
+                    if os.path.exists(full_path):
+                        includes_found.append(full_path)
+                    else:
+                        includes_not_found.setdefault(entry, {})[include] = 1
+
+    if includes_with_attributes:
+
+        includes_with_attributes = get_full_path_to_includes_with_attributes(includes_with_attributes)
+
+        for i in includes_with_attributes:
+            unique_entries_includes_with_attributes.append(get_unique_entries(i))
+
+    return includes_found, includes_not_found, unique_entries_includes_with_attributes
+
+
+def get_includes_recursively(files):
+
+    lvl_1_includes_found, lvl_1_includes_not_found, lvl_1_includes_with_attributes = get_includes(files)
+
+    lvl_2_includes_found, lvl_2_includes_not_found, lvl_2_includes_with_attributes = get_includes(lvl_1_includes_found)
+
+    lvl_3_includes_found, lvl_3_includes_not_found, lvl_3_includes_with_attributes = get_includes(lvl_2_includes_found)
+
+    lvl_4_includes_found, lvl_4_includes_not_found, lvl_4_includes_with_attributes = get_includes(lvl_3_includes_found)
+
+    includes_found = lvl_1_includes_found + lvl_2_includes_found + lvl_3_includes_found + lvl_4_includes_found
+
+    #includes_not_found = lvl_1_includes_not_found + lvl_2_includes_not_found + lvl_3_includes_not_found + lvl_3_includes_not_found + lvl_4_includes_not_found
+
+    includes_not_found = {**lvl_1_includes_not_found , **lvl_2_includes_not_found, **lvl_3_includes_not_found, **lvl_4_includes_not_found}
+
+    includes_with_attributes = lvl_1_includes_with_attributes + lvl_2_includes_with_attributes + lvl_3_includes_with_attributes + lvl_4_includes_with_attributes
+    includes_with_attributes = [j for i in includes_with_attributes for j in i]
+
+    if includes_with_attributes:
+        includes_with_attributes = get_unique_entries(includes_with_attributes)
+
+
+    # only valid for entry point
+    for file in files:
+        file_name = os.path.basename(file)
+        if not file_name == 'master.adoc':
+            includes_found.append(file)
+
+    return includes_found, includes_not_found, includes_with_attributes
+
+
+def validate_entry_point_files(entry_point_list):
+    get_nonexisting_entry_points(entry_point_list)
+
+    all_includes_found, all_includes_not_found, all_includes_with_attributes = get_includes_recursively(entry_point_list)
+
+    no_prefix_files = get_no_prefix_files(all_includes_found)
+
+    all_assemblies = get_all_assemblies(all_includes_found, no_prefix_files)
+    all_modules = get_all_modules(all_includes_found, no_prefix_files)
+    all_undetermined_files = get_undetermined_files(no_prefix_files)
+
+    for i in all_includes_with_attributes:
+        print(i)
+
+    '''if all_includes_not_found:
+        for key, value in all_includes_not_found.items():
+            print(f'{os.path.basename(key)} contains the following includes that do not exist in your repository:')
+            for v in value:
+                print('\t', v)
+
+    validate = validation(all_includes_found, all_modules, all_assemblies)
+
+    print_report_message(validate, 'entry point')'''

PantheonCMD/old_entry.py

@@ -0,0 +1,126 @@
+#!/usr/bin/python3
+
+import argparse
+import re
+import os
+from pcchecks import Regex
+import sys
+from pcutil import get_exist, get_not_exist
+from pcprvalidator import get_no_prefix_files, get_all_modules, get_all_assemblies, get_undetermined_files
+from pcvalidator import validation
+from pcmsg import print_message, print_report_message
+
+parser = argparse.ArgumentParser()
+
+
+def get_nonexisting_entry_points(entry_point_list):
+    nonexistent_files = get_not_exist(entry_point_list)
+
+    if nonexistent_files:
+        print_message(nonexistent_files, 'entry point', 'does not exist in your repository')
+        sys.exit(2)
+
+
+def get_includes(entry_points):
+    path_to_includes = []
+
+    for entry in entry_points:
+        path_to_entry_point = os.path.dirname(os.path.abspath(entry))
+
+        with open(entry, 'r') as file:
+            original = file.read()
+            stripped = Regex.MULTI_LINE_COMMENT.sub('', original)
+            stripped = Regex.SINGLE_LINE_COMMENT.sub('', stripped)
+
+            included_files = re.findall(Regex.INCLUDED_CONTENT, stripped)
+
+            if included_files:
+
+                for include in included_files[:]:
+                    if include.startswith('_'):
+                        included_files.remove(include)
+
+                for i in included_files:
+                    path_to_includes.append(os.path.join(path_to_entry_point, i))
+
+    return path_to_includes
+
+
+def get_level_one_includes(files):
+    path_to_level_one_includes = get_includes(files)
+
+    return path_to_level_one_includes
+
+
+def get_level_two_includes(files):
+    path_to_level_two_includes = get_includes(files)
+
+    return path_to_level_two_includes
+
+
+def get_level_three_includes(files):
+    path_to_level_three_includes = get_includes(files)
+
+    return path_to_level_three_includes
+
+
+def get_level_four_includes(files):
+    path_to_level_four_includes = get_includes(files)
+
+    return path_to_level_four_includes
+
+
+def get_concatenated_includes(entry_point_list):
+    existing_entry_points = get_exist(entry_point_list)
+    level_one_includes = get_level_one_includes(existing_entry_points)
+    level_two_includes = get_level_two_includes(level_one_includes)
+    level_three_includes = get_level_three_includes(level_two_includes)
+    level_four_includes = get_level_four_includes(level_three_includes)
+    no_prefix_level_four_includes = get_no_prefix_files(level_four_includes)
+    level_four_modules = get_all_modules(level_four_includes, no_prefix_level_four_includes)
+    level_four_assemblies = get_all_assemblies(level_four_includes, no_prefix_level_four_includes)
+
+    all_includes = level_one_includes + level_two_includes + level_three_includes + level_four_modules
+
+    return all_includes, level_four_assemblies
+
+
+def get_level_four_assemblies(entry_point_list):
+    all_includes, level_four_assemblies = get_concatenated_includes(entry_point_list)
+
+    return level_four_assemblies
+
+
+def get_all_includes(entry_point_list):
+    all_includes, level_four_assemblies = get_concatenated_includes(entry_point_list)
+
+    for entry in entry_point_list:
+        if not entry.endswith('master.adoc'):
+            all_includes = all_includes + entry_point_list
+
+    for include in all_includes:
+        if os.path.basename(include).startswith('_'):
+            all_includes.remove(include)
+
+    return all_includes
+
+
+def validate_entry_point_files(entry_point_list):
+    # exit if entry point doesn't exist
+    get_nonexisting_entry_points(entry_point_list)
+    existing_entry_points = get_exist(entry_point_list)
+    includes = get_all_includes(entry_point_list)
+    no_prefix_files = get_no_prefix_files(includes)
+    modules_found = get_all_modules(includes, no_prefix_files)
+    assemblies_found = get_all_assemblies(includes, no_prefix_files)
+    undetermined_file_type = get_undetermined_files(no_prefix_files)
+    level_four_assemblies = get_level_four_assemblies(existing_entry_points)
+
+    if level_four_assemblies:
+        print_message(level_four_assemblies, 'entry point', 'contains unsupported level of nesting for the following files')
+
+    if undetermined_file_type:
+        print_message(undetermined_file_type, 'entry point', 'contains the following files that can not be classified as modules or assemblies')
+
+    validate = validation(includes, modules_found, assemblies_found)
+    print_report_message(validate, 'entry point')

PantheonCMD/pcbuild.py

@@ -117,7 +117,7 @@ def coalesce_document(main_file, attributes=None, depth=0, top_level=True):
     attributes = attributes or {}
     comment_block = False
     lines = []
-    
+
     # Create a copy of global attributes
     if top_level:
         attributes_global = attributes.copy()

PantheonCMD/pcchecks.py

@@ -17,8 +17,11 @@ class Tags:
 class Regex:
     """Define regular expresiions for the checks."""
 
+    ATTRIBUTE = re.compile(r'{.*?}')
     INCLUDE = re.compile(r'include::.*\]\n')
+    INCLUDED_CONTENT = re.compile(r'(?<=include::).*?(?=\[)')
     MODULE_TYPE = re.compile(r':_module-type: (PROCEDURE|CONCEPT|REFERENCE)')
+    CONTENT_TYPE = re.compile(r':_content-type: (PROCEDURE|CONCEPT|REFERENCE|ASSEMBLY)')
     PREFIX_ASSEMBLIES = re.compile(r'.*\/assembly.*\.adoc')
     PREFIX_MODULES = re.compile(r'.*\/con.*\.adoc|.*\/proc.*\.adoc|.*\/ref.*\.adoc')
     # should exclude pseudo vanilla like <<some content>>