diff --git a/doorstop/cli/commands.py b/doorstop/cli/commands.py index 06c4f4d7f..d3743f0d8 100644 --- a/doorstop/cli/commands.py +++ b/doorstop/cli/commands.py @@ -439,49 +439,61 @@ def run_import(args, cwd, error, catch=True, _tree=None): document = item = None attrs = utilities.literal_eval(args.attrs, error) mapping = utilities.literal_eval(args.map, error) + ext = utilities.get_ext(args, error, None, None) + if args.path: - if not args.prefix: + if not args.prefix and not importer.can_import_tree(ext): error("when [path] specified, [prefix] is also required") elif args.document: error("'--document' cannot be used with [path] [prefix]") elif args.item: error("'--item' cannot be used with [path] [prefix]") - ext = utilities.get_ext(args, error, None, None) - elif not (args.document or args.item): + elif not (args.document or args.item or importer.can_import_tree(ext)): error("specify [path], '--document', or '--item' to import") + request_next_number = _request_next_number(args) + tree = _tree or _get_tree( + args, cwd, request_next_number=request_next_number + ) + + documents = [] with utilities.capture(catch=catch) as success: - if args.path: + # import supports importing a tree and file passed in with no --document/--item and no prefix + if args.path and os.path.isfile(args.path) and \ + importer.can_import_tree(ext) and \ + not (args.document or args.item) and \ + not args.prefix: + documents = importer.import_file(args.path, ext=ext, mapping=mapping, tree=tree) + + # passed a path and prefix + elif args.path and args.prefix: # get the document - request_next_number = _request_next_number(args) - tree = _tree or _get_tree( - args, cwd, request_next_number=request_next_number - ) document = tree.find_document(args.prefix) + log.debug(f"Found document: '{document}' for prefix {args.prefix}") # import items into it msg = "importing '{}' into document {}...".format(args.path, document) utilities.show(msg, flush=True) - importer.import_file(args.path, document, ext, mapping=mapping) + documents = importer.import_file(args.path, ext=ext, mapping=mapping, document=document) elif args.document: prefix, path = args.document - document = importer.create_document(prefix, path, parent=args.parent) + documents = [importer.create_document(prefix, path, parent=args.parent)] elif args.item: prefix, uid = args.item request_next_number = _request_next_number(args) item = importer.add_item( prefix, uid, attrs=attrs, request_next_number=request_next_number ) + documents = [item.document] if not success: return False - if document: + for doc in documents: utilities.show( - "imported document: {} ({})".format(document.prefix, document.relpath) + "imported into document: {} ({})".format(doc.prefix, doc.relpath) ) - else: - assert item + if item: utilities.show("imported item: {} ({})".format(item.uid, item.relpath)) return True diff --git a/doorstop/cli/main.py b/doorstop/cli/main.py index eb5756aa4..41a5c1ae0 100644 --- a/doorstop/cli/main.py +++ b/doorstop/cli/main.py @@ -184,7 +184,7 @@ def main(args=None): # pylint: disable=R0915 log.debug(f"command cancelled: {args}") success = False if success: - log.debug("command succeeded: {args}") + log.debug(f"command succeeded: {args}") else: log.debug(f"command failed: {args}") sys.exit(1) diff --git a/doorstop/cli/tests/test_all.py b/doorstop/cli/tests/test_all.py index 7b6d046a6..40598f350 100644 --- a/doorstop/cli/tests/test_all.py +++ b/doorstop/cli/tests/test_all.py @@ -576,7 +576,7 @@ class TestImportFile(MockTestCase): def test_import_file_missing_prefix(self): """Verify 'doorstop import' returns an error with a missing prefix.""" - path = os.path.join(FILES, "exported.xlsx") + path = os.path.join(FILES, "exported.csv") self.assertRaises(SystemExit, main, ["import", path]) def test_import_file_extra_flags(self): @@ -619,7 +619,7 @@ def test_import_csv_to_document_existing(self): self.assertIs(None, main(["import", path, "PREFIX"])) # Assert path = os.path.join(dirpath, "REQ001.yml") - self.assertTrue(os.path.isfile(path)) + self.assertTrue(os.path.isfile(path), f"{path} is not a file") def test_import_tsv_to_document_existing(self): """Verify 'doorstop import' can import TSV to an existing document.""" @@ -630,7 +630,7 @@ def test_import_tsv_to_document_existing(self): self.assertIs(None, main(["import", path, "PREFIX"])) # Assert path = os.path.join(dirpath, "REQ001.yml") - self.assertTrue(os.path.isfile(path)) + self.assertTrue(os.path.isfile(path), f"{path} is not a file") def test_import_xlsx_to_document_existing(self): """Verify 'doorstop import' can import XLSX to an existing document.""" @@ -641,7 +641,7 @@ def test_import_xlsx_to_document_existing(self): self.assertIs(None, main(["import", path, "PREFIX"])) # Assert path = os.path.join(dirpath, "REQ001.yml") - self.assertTrue(os.path.isfile(path)) + self.assertTrue(os.path.isfile(path), f"{path} is not a file") @unittest.skipUnless(os.getenv(ENV), REASON) @@ -702,12 +702,18 @@ def test_export_document_xlsx_error(self): self.assertRaises(SystemExit, main, ["export", "tut", path]) self.assertFalse(os.path.isfile(path)) - def test_export_tree_xlsx(self): - """Verify 'doorstop export' can create an XLSX directory.""" + def test_export_tree_xlsx_dir(self): + """Verify 'doorstop export' can create an XLSX tree export.""" path = os.path.join(self.temp, "all") self.assertIs(None, main(["export", "all", path, "--xlsx"])) self.assertTrue(os.path.isdir(path)) + def test_export_tree_xlsx_file(self): + """Verify 'doorstop export' can create an XLSX tree export.""" + path = os.path.join(self.temp, "all.xlsx") + self.assertIs(None, main(["export", "all", path, "--xlsx"])) + self.assertTrue(os.path.isfile(path)) + def test_export_tree_no_path(self): """Verify 'doorstop export' returns an error with no path.""" self.assertRaises(SystemExit, main, ["export", "all"]) diff --git a/doorstop/core/document.py b/doorstop/core/document.py index 356842ab2..1f0c8874c 100644 --- a/doorstop/core/document.py +++ b/doorstop/core/document.py @@ -10,7 +10,7 @@ import yaml -from doorstop import common, settings +from doorstop import common, settings as dsettings from doorstop.common import ( DoorstopError, DoorstopInfo, @@ -135,7 +135,7 @@ def new( """ # Check separator - if sep and sep not in settings.SEP_CHARS: + if sep and sep not in dsettings.SEP_CHARS: raise DoorstopError("invalid UID separator '{}'".format(sep)) config = os.path.join(path, Document.CONFIG) @@ -192,36 +192,63 @@ def include(self, node): IncludeLoader.filenames = [yamlfile] # type: ignore return self._load(text, yamlfile, loader=IncludeLoader) - def load(self, reload=False): - """Load the document's properties from its file.""" - if self._loaded and not reload: - return - log.debug("loading {}...".format(repr(self))) - data = self._load_with_include(self.config) - # Store parsed data - sets = data.get("settings", {}) - for key, value in sets.items(): - try: - if key == "prefix": + @property + def settings(self): + """ + Document settings as a dict + """ + sets = {} + for key, value in self._data.items(): + if key == "prefix": + sets[key] = str(value) + elif key == "parent": + if value: + sets[key] = value + else: + sets[key] = value + return sets + + @settings.setter + def settings(self, settings): + def fill_key(key, value): + match key: + case "prefix": self._data[key] = Prefix(value) - elif key == "sep": + case "sep": self._data[key] = value.strip() - elif key == "parent": + case "parent": self._data[key] = value.strip() - elif key == "digits": + case "digits": self._data[key] = int(value) # type: ignore - elif key == "itemformat": + case "itemformat": self._data[key] = value.strip() - else: - msg = "unexpected document setting '{}' in: {}".format( - key, self.config - ) + case _: + msg = f"unexpected document setting '{key}' in: {self.config}" raise DoorstopError(msg) + + for key, value in settings.items(): + try: + fill_key(key, value) except (AttributeError, TypeError, ValueError): - msg = "invalid value for '{}' in: {}".format(key, self.config) + msg = f"invalid value for '{key}' in: {self.config}" raise DoorstopError(msg) + + @property + def attributes(self): + """ + Document attributes as a dict + """ + # Save the attributes + attributes = {} + if self._attribute_defaults: + attributes["defaults"] = self._attribute_defaults + if self._extended_reviewed: + attributes["reviewed"] = self._extended_reviewed + return attributes + + @attributes.setter + def attributes(self, attributes): # Store parsed attributes - attributes = data.get("attributes", {}) for key, value in attributes.items(): if key == "defaults": self._attribute_defaults = value @@ -235,8 +262,15 @@ def load(self, reload=False): ) raise DoorstopError(msg) + def load(self, reload=False): + """Load the document's properties from its file.""" + if self._loaded and not reload: + return + log.debug("loading {}...".format(repr(self))) + data = self._load_with_include(self.config) + self.settings = data["settings"] if "settings" in data else {} + self.attributes = data["attributes"] if "attributes" in data else {} self.extensions = data.get("extensions", {}) - # Set meta attributes self._loaded = True if reload: @@ -246,27 +280,13 @@ def load(self, reload=False): def save(self): """Save the document's properties to its file.""" log.debug("saving {}...".format(repr(self))) - # Format the data items + data = {} - sets = {} - for key, value in self._data.items(): - if key == "prefix": - sets[key] = str(value) - elif key == "parent": - if value: - sets[key] = value - else: - sets[key] = value - data["settings"] = sets - # Save the attributes - attributes = {} - if self._attribute_defaults: - attributes["defaults"] = self._attribute_defaults - if self._extended_reviewed: - attributes["reviewed"] = self._extended_reviewed - if attributes: - data["attributes"] = attributes - # Dump the data to YAML + if self.settings: + data["settings"] = self.settings, + if self.attributes: + data["attributes"] = self.attributes + text = self._dump(data) # Save the YAML to file self._write(text, self.config) @@ -313,7 +333,7 @@ def _iter(self, reload=False): except Exception: log.error("Unable to load: %s", item) raise - if settings.CACHE_ITEMS and self.tree: + if dsettings.CACHE_ITEMS and self.tree: self.tree._item_cache[ # pylint: disable=protected-access item.uid ] = item @@ -392,7 +412,7 @@ def sep(self): def sep(self, value): """Set the prefix-number separator to use for new item UIDs.""" # TODO: raise a specific exception for invalid separator characters? - assert not value or value in settings.SEP_CHARS + assert not value or value in dsettings.SEP_CHARS self._data["sep"] = value.strip() # TODO: should the new separator be applied to all items? @@ -480,7 +500,7 @@ def index(self, value): path = os.path.join(self.path, Document.INDEX) log.info("creating {} index...".format(self)) common.write_lines( - self._lines_index(self.items), path, end=settings.WRITE_LINESEPERATOR + self._lines_index(self.items), path, end=dsettings.WRITE_LINESEPERATOR ) @index.deleter @@ -516,7 +536,7 @@ def add_item(self, number=None, level=None, reorder=True, defaults=None, name=No name, self.prefix ) raise DoorstopError(msg) - if self.sep not in settings.SEP_CHARS: + if self.sep not in dsettings.SEP_CHARS: msg = "cannot add item with name '{}' to document '{}' with an invalid separator '{}'".format( name, self.prefix, self.sep ) @@ -608,13 +628,13 @@ def reorder(self, manual=True, automatic=True, start=None, keep=None, _items=Non @staticmethod def _lines_index(items): """Generate (pseudo) YAML lines for the document index.""" - yield "#" * settings.MAX_LINE_LENGTH + yield "#" * dsettings.MAX_LINE_LENGTH yield "# THIS TEMPORARY FILE WILL BE DELETED AFTER DOCUMENT REORDERING" yield "# MANUALLY INDENT, DEDENT, & MOVE ITEMS TO THEIR DESIRED LEVEL" yield "# A NEW ITEM WILL BE ADDED FOR ANY UNKNOWN IDS, i.e. - new: " yield "# THE COMMENT WILL BE USED AS THE ITEM TEXT FOR NEW ITEMS" yield "# CHANGES WILL BE REFLECTED IN THE ITEM FILES AFTER CONFIRMATION" - yield "#" * settings.MAX_LINE_LENGTH + yield "#" * dsettings.MAX_LINE_LENGTH yield "" yield "initial: {}".format(items[0].level if items else 1.0) yield "outline:" @@ -623,8 +643,8 @@ def _lines_index(items): lines = item.text.strip().splitlines() comment = lines[0].replace("\\", "\\\\") if lines else "" line = space + "- {u}: # {c}".format(u=item.uid, c=comment) - if len(line) > settings.MAX_LINE_LENGTH: - line = line[: settings.MAX_LINE_LENGTH - 3] + "..." + if len(line) > dsettings.MAX_LINE_LENGTH: + line = line[: dsettings.MAX_LINE_LENGTH - 3] + "..." yield line @staticmethod @@ -854,9 +874,9 @@ def get_issues( return # Reorder or check item levels - if settings.REORDER: + if dsettings.REORDER: self.reorder(_items=items) - elif settings.CHECK_LEVELS: + elif dsettings.CHECK_LEVELS: yield from self._get_issues_level(items) item_validator = ItemValidator() diff --git a/doorstop/core/exporter.py b/doorstop/core/exporter.py index c70eb5ea9..aee5832b2 100644 --- a/doorstop/core/exporter.py +++ b/doorstop/core/exporter.py @@ -3,7 +3,9 @@ """Functions to export documents and items.""" import datetime +import logging import os + from collections import defaultdict from typing import Any, Dict @@ -12,7 +14,7 @@ from doorstop import common, settings from doorstop.common import DoorstopError -from doorstop.core.types import iter_documents, iter_items +from doorstop.core.types import iter_documents, iter_items, is_tree LIST_SEP = "\n" # string separating list values when joined in a string @@ -25,13 +27,15 @@ def export(obj, path, ext=None, **kwargs): """Export an object to a given format. - The function can be called in two ways: + The function can be called in three ways: 1. document or item-like object + output file path 2. tree-like object + output directory path + 3. tree-like object + output file path + - :param obj: (1) Item, list of Items, Document or (2) Tree - :param path: (1) output file path or (2) output directory path + :param obj: (1) Item, list of Items, Document or (2) (3) Tree + :param path: (1) (3) output file path or (2) output directory path :param ext: file extension to override output extension :raises: :class:`doorstop.common.DoorstopError` for unknown file formats @@ -40,9 +44,15 @@ def export(obj, path, ext=None, **kwargs): """ # Determine the output format + is_dir = (os.path.splitext(path)[-1] == "") ext = ext or os.path.splitext(path)[-1] or ".csv" check(ext) + if can_export_tree(ext) and is_tree(obj) and not is_dir: + log.info("exporting to {}...".format(path)) + export_file(obj, path, ext, **kwargs) + return path + # Export documents count = 0 for obj2, path2 in iter_documents(obj, path, ext): @@ -97,7 +107,7 @@ def export_file(obj, path, ext=None, **kwargs): """ ext = ext or os.path.splitext(path)[-1] func = check(ext, get_file_func=True) - log.debug("converting %s to file format %s...", obj, ext) + log.debug("converting {obj} to file format {ext} using {func}...") try: return func(obj, path, **kwargs) except IOError: @@ -246,14 +256,64 @@ def _file_xlsx(obj, path, auto=False): :return: path of created file """ - workbook = _get_xlsx(obj, auto) + + log.debug(f"xlsx export: _file_xlsx called with {obj}, {path}, auto={auto}") + workbook = _get_xlsx(obj, path, auto) workbook.save(path) return path -def _get_xlsx(obj, auto): - """Create an XLSX workbook object. +def _add_properties_sheet(wb, tree, document_properties): + sheet = wb.create_sheet(title="Document Properties") + sheet.append(["Source Control Status:", tree.vcs.describe()]) + sheet.append([]) + sheet.append([ + "prefix", + "settings key", + "settings value", + "attributes key", + "attributes value", + ]) + for prefix, data in document_properties.items(): + for set_k, set_v in data["settings"].items(): + sheet.append([prefix, repr(set_k), repr(set_v)]) + for attr_k, attr_v in data["attributes"].items(): + sheet.append([prefix, "", "", repr(attr_k), repr(attr_v)]) + + +def _get_xlsx(obj, path, auto): + # Create a new workbook + workbook = openpyxl.Workbook() + # We don't want the default sheet "Sheet" + workbook.remove(workbook.active) + + first_sheet = None + if is_tree(obj): + document_properties = {} + log.debug("xlsx export: exporting tree") + for obj2, path2 in iter_documents(obj, path, ".xlsx"): + sheet = _add_xlsx_sheet(workbook, obj2, auto) + first_sheet = sheet or first_sheet + document_properties[obj2.prefix] = { + "settings": obj2.settings, + "attributes": obj2.attributes + } + if document_properties: + _add_properties_sheet(workbook, obj, document_properties) + else: + log.debug("xlsx export: exporting single Item/Document") + first_sheet = _add_xlsx_sheet(workbook, obj, auto) + + log.debug(f"xlsx export: First sheet is {first_sheet}") + assert first_sheet != None + workbook.active = first_sheet + + return workbook + + +def _add_xlsx_sheet(workbook, obj, auto): + """Add a sheet to XLSX workbook object. :param obj: Item, list of Items, or Document to export :param auto: include placeholders for new items on import @@ -264,13 +324,16 @@ def _get_xlsx(obj, auto): col_widths: Dict[Any, float] = defaultdict(float) col = "A" - # Create a new workbook - workbook = openpyxl.Workbook() - worksheet = workbook.active + worksheet = workbook.create_sheet(title=obj.prefix) + log.debug(f"xls export: Created worksheet {worksheet.title}") + + log.debug(f"xls export: populating cells") # Populate cells for row, data in enumerate(_tabulate(obj, auto=auto), start=1): + log.debug(f"xls export: row={row}, data={data}") for col_idx, value in enumerate(data, start=1): + log.debug(f"xls export: column={col_idx}, value={value}") cell = worksheet.cell(column=col_idx, row=row) # wrap text in every cell @@ -307,8 +370,7 @@ def _get_xlsx(obj, auto): # Freeze top row worksheet.freeze_panes = worksheet.cell(row=2, column=1) - - return workbook + return worksheet def _width(text): @@ -325,6 +387,8 @@ def _width(text): FORMAT_FILE = {".csv": _file_csv, ".tsv": _file_tsv, ".xlsx": _file_xlsx} # Union of format dictionaries FORMAT = dict(list(FORMAT_LINES.items()) + list(FORMAT_FILE.items())) # type: ignore +# Can a given format export a whole tree? +FORMAT_TREE = {".xlsx": True} def check(ext, get_lines_gen=False, get_file_func=False): @@ -368,3 +432,7 @@ def check(ext, get_lines_gen=False, get_file_func=False): raise exc return None + + +def can_export_tree(ext): + return ext in FORMAT_TREE diff --git a/doorstop/core/importer.py b/doorstop/core/importer.py index adbc470e8..693a5d379 100644 --- a/doorstop/core/importer.py +++ b/doorstop/core/importer.py @@ -24,23 +24,24 @@ log = common.logger(__name__) -def import_file(path, document, ext=None, mapping=None, **kwargs): +def import_file(path, ext=None, document=None, tree=None, mapping=None, **kwargs): """Import items from an exported file. :param path: input file location - :param document: document to import items + :param document: document to import items into + :param tree: tree to import documents into :param ext: file extension to override input path's extension :param mapping: dictionary mapping custom to standard attribute names :raise DoorstopError: for unknown file formats - :return: document with imported items + :return: list of Documents created/modified """ - log.info("importing {} into {}...".format(path, document)) + log.info(f"importing {path} into {document if document else tree}...") ext = ext or os.path.splitext(path)[-1] func = check(ext) - func(path, document, mapping=mapping, **kwargs) + return func(path, document, tree=tree, mapping=mapping, **kwargs) def create_document(prefix, path, parent=None, tree=None): @@ -114,6 +115,7 @@ def _file_yml(path, document, **_): :param path: input file location :param document: document to import items + returns: list of Documents created/modified """ # Parse the file log.info("reading items in {}...".format(path)) @@ -129,9 +131,10 @@ def _file_yml(path, document, **_): else: item.delete() add_item(document.prefix, uid, attrs=attrs, document=document) + return [document] -def _file_csv(path, document, delimiter=",", mapping=None): +def _file_csv(path, document, delimiter=",", mapping=None, **_): """Import items from a CSV export to a document. :param path: input file location @@ -139,6 +142,8 @@ def _file_csv(path, document, delimiter=",", mapping=None): :param delimiter: CSV field delimiter :param mapping: dictionary mapping custom to standard attribute names + returns: list of Documents created/modified + """ rows = [] @@ -165,35 +170,40 @@ def _file_csv(path, document, delimiter=",", mapping=None): # Import items from the rows _itemize(header, data, document, mapping=mapping) + return [document] -def _file_tsv(path, document, mapping=None): +def _file_tsv(path, document, mapping=None, **_): """Import items from a TSV export to a document. :param path: input file location :param document: document to import items :param mapping: dictionary mapping custom to standard attribute names + :return: list of Documents created/modified """ _file_csv(path, document, delimiter="\t", mapping=mapping) + return [document] -def _file_xlsx(path, document, mapping=None): - """Import items from an XLSX export to a document. +def _check_doc(tree, worksheet, workbook): + log.debug(f"xlsx import: importing sheet {worksheet.title}") + prefix = worksheet.title + try: + document = tree.find_document(prefix) + except DoorstopError: + log.warn(f"no matching document found for sheet {worksheet.title}. " + f"If you wish to import this, first create a document " + f"with the prefix {prefix}") + return None + return document - :param path: input file location - :param document: document to import items - :param mapping: dictionary mapping custom to standard attribute names - """ +def _load_xlsx(document, worksheet, mapping): + log.info(f"Loading into document {document.path}, prefix {document.prefix}...") + header = [] data = [] - - # Parse the file - log.debug("reading rows in {}...".format(path)) - workbook = openpyxl.load_workbook(path, data_only=True) - worksheet = workbook.active - index = 0 # Extract header and data rows @@ -212,16 +222,55 @@ def _file_xlsx(path, document, mapping=None): msg = "workbook contains the maximum number of rows" warnings.warn(msg, Warning) - # Import items from the rows _itemize(header, data, document, mapping=mapping) +def _file_xlsx(path, document, mapping=None, tree=None, **_): + """Import items from an XLSX export to a document. + + :param path: input file location + :param document: document to import items + :param mapping: dictionary mapping custom to standard attribute names + + :return: list of Documents created/modified + """ + + # Parse the file + log.info(f"reading sheets in {path}...") + workbook = openpyxl.load_workbook(path, data_only=True) + + if document: + if len(workbook.worksheets) == 1: + _load_xlsx(document, workbook.worksheets[0], mapping) + else: + sheet = workbook.get_sheet_by_name(document.prefix) + if sheet: + _load_xlsx(document, sheet, mapping) + else: + raise DoorstopError(f"No sheet matching {document.prefix}") + return [document] + else: + if not tree: + tree = _get_tree() + documents = [] + for worksheet in workbook.worksheets: + if worksheet.title == "Document Properties": + #TODO + continue + log.info(f"checking sheet {worksheet.title}...") + document = _check_doc(tree, worksheet, workbook) + if document: + log.debug(f"importing into {document.prefix} {document.path}") + _load_xlsx(document, worksheet, mapping) + documents.append(document) + return documents + def _itemize(header, data, document, mapping=None): """Conversion function for multiple formats. :param header: list of columns names :param data: list of lists of row values - :param document: document to import items + :param document: document to import items into :param mapping: dictionary mapping custom to standard attribute names """ @@ -318,6 +367,7 @@ def _split_list(value): ".xlsx": _file_xlsx, } +FORMAT_TREE = {".xlsx": True} def check(ext): """Confirm an extension is supported for import. @@ -337,3 +387,7 @@ def check(ext): else: log.debug("found file reader for: {}".format(ext)) return func + + +def can_import_tree(ext): + return ext in FORMAT_TREE diff --git a/doorstop/core/publishers/markdown.py b/doorstop/core/publishers/markdown.py index 7b69c671d..34e008a76 100644 --- a/doorstop/core/publishers/markdown.py +++ b/doorstop/core/publishers/markdown.py @@ -46,6 +46,11 @@ def create_index(self, directory, index=INDEX, extensions=(".md",), tree=None): else: log.warning("no files for {}".format(index)) + # Source control status + common.write_text(" # Source Control Status", path) + common.write_text(, path) + + def _index_tree(self, tree, depth): """Recursively generate markdown index. diff --git a/doorstop/core/tests/files/exported-all.xlsx b/doorstop/core/tests/files/exported-all.xlsx new file mode 100644 index 000000000..c2e7b0127 Binary files /dev/null and b/doorstop/core/tests/files/exported-all.xlsx differ diff --git a/doorstop/core/tests/test_all.py b/doorstop/core/tests/test_all.py index 243dc6cc0..6b2f58b9a 100644 --- a/doorstop/core/tests/test_all.py +++ b/doorstop/core/tests/test_all.py @@ -399,7 +399,7 @@ def test_import_yml(self): _tree = _get_tree() document = _tree.create_document(_path, "REQ") # Act - core.importer.import_file(path, document) + core.importer.import_file(path, document=document) # Assert expected = [item.data for item in self.document.items] actual = [item.data for item in document.items] @@ -414,7 +414,7 @@ def test_import_csv(self): _tree = _get_tree() document = _tree.create_document(_path, "REQ") # Act - core.importer.import_file(path, document) + core.importer.import_file(path, document=document) # Assert expected = [item.data for item in self.document.items] actual = [item.data for item in document.items] @@ -429,7 +429,7 @@ def test_import_tsv(self): _tree = _get_tree() document = _tree.create_document(_path, "REQ") # Act - core.importer.import_file(path, document) + core.importer.import_file(path, document=document) # Assert expected = [item.data for item in self.document.items] actual = [item.data for item in document.items] @@ -445,7 +445,7 @@ def test_import_xlsx(self): _tree = _get_tree() document = _tree.create_document(_path, "REQ") # Act - core.importer.import_file(path, document) + core.importer.import_file(path, document=document) # Assert expected = [item.data for item in self.document.items] actual = [item.data for item in document.items] diff --git a/doorstop/core/tests/test_exporter.py b/doorstop/core/tests/test_exporter.py index 46127db76..ea7abf6dc 100644 --- a/doorstop/core/tests/test_exporter.py +++ b/doorstop/core/tests/test_exporter.py @@ -5,7 +5,9 @@ import os import tempfile import unittest -from unittest.mock import MagicMock, Mock, patch +from unittest.mock import ANY, MagicMock, Mock, patch + +import openpyxl from doorstop.common import DoorstopError from doorstop.core import exporter @@ -132,23 +134,25 @@ def test_file_tsv(self, mock_file_csv): self.item, path, delimiter="\t", auto=False ) - @patch("doorstop.core.exporter._get_xlsx") - def test_file_xlsx(self, mock_get_xlsx): + @patch("doorstop.core.exporter._add_xlsx_sheet") + def test_file_xlsx(self, mock_add_xlsx_sheet): """Verify a (mock) XLSX file can be created.""" + mock_add_xlsx_sheet.side_effect = lambda *args: args[0].create_sheet() temp = tempfile.gettempdir() + path = os.path.join(temp, "exported.xlsx") # Act exporter._file_xlsx(self.item, path) # pylint:disable=W0212 # Assert - mock_get_xlsx.assert_called_once_with(self.item, False) + mock_add_xlsx_sheet.assert_called_once_with(ANY, self.item, False) def test_get_xlsx(self): - """Verify an XLSX object can be created.""" + """Verify an XLSX worksheet object can be created.""" # Act - workbook = exporter._get_xlsx(self.item4, auto=False) # pylint: disable=W0212 + workbook = openpyxl.Workbook() + worksheet = exporter._add_xlsx_sheet(workbook, self.item4, auto=False) # pylint: disable=W0212 # Assert rows = [] - worksheet = workbook.active for data in worksheet.rows: rows.append([cell.value for cell in data]) self.assertIn("long", rows[0]) @@ -157,10 +161,10 @@ def test_get_xlsx(self): def test_get_xlsx_auto(self): """Verify an XLSX object can be created with placeholder rows.""" # Act - workbook = exporter._get_xlsx(self.item4, auto=True) # pylint: disable=W0212 + workbook = openpyxl.Workbook() + worksheet = exporter._add_xlsx_sheet(workbook, self.item4, auto=True) # pylint: disable=W0212 # Assert rows = [] - worksheet = workbook.active for data in worksheet.rows: rows.append([cell.value for cell in data]) self.assertEqual("...", rows[-1][0]) diff --git a/doorstop/core/tests/test_importer.py b/doorstop/core/tests/test_importer.py index 1df09bf83..4e1a381ce 100644 --- a/doorstop/core/tests/test_importer.py +++ b/doorstop/core/tests/test_importer.py @@ -14,6 +14,7 @@ from doorstop.common import DoorstopError from doorstop.core import importer from doorstop.core.builder import _set_tree +from doorstop.core.tests import MockDataMixIn from doorstop.core.tests.test_document import FILES, MockItem from doorstop.core.tree import Tree @@ -51,17 +52,16 @@ ```""" -class TestModule(unittest.TestCase): +class TestModule(MockDataMixIn, unittest.TestCase): """Unit tests for the doorstop.core.importer module.""" maxDiff = None - def test_import_file_unknown(self): """Verify an exception is raised when importing unknown formats.""" mock_document = Mock() - self.assertRaises(DoorstopError, importer.import_file, "a.a", mock_document) + self.assertRaises(DoorstopError, importer.import_file, "a.a", document=mock_document) self.assertRaises( - DoorstopError, importer.import_file, "a.csv", mock_document, ".a" + DoorstopError, importer.import_file, "a.csv", ext=".a", document=mock_document ) @patch("doorstop.core.importer._file_csv") @@ -70,15 +70,15 @@ def test_import_file(self, mock_file_csv): mock_path = "path/to/file.csv" mock_document = Mock() importer.FORMAT_FILE[".csv"] = mock_file_csv - importer.import_file(mock_path, mock_document) - mock_file_csv.assert_called_once_with(mock_path, mock_document, mapping=None) + importer.import_file(mock_path, document=mock_document) + mock_file_csv.assert_called_once_with(mock_path, document=mock_document, mapping=None) @patch("doorstop.core.importer.check") def test_import_file_custom_ext(self, mock_check): """Verify a custom extension can be specified for import.""" mock_path = "path/to/file.ext" mock_document = Mock() - importer.import_file(mock_path, mock_document, ext=".custom") + importer.import_file(mock_path, document=mock_document, ext=".custom") mock_check.assert_called_once_with(".custom") @patch("doorstop.core.importer.add_item") @@ -274,19 +274,7 @@ def test_file_tsv(self, mock_file_csv): mock_path, mock_document, delimiter="\t", mapping=None ) - @patch("doorstop.core.importer._itemize") - def test_file_xlsx(self, mock_itemize): - """Verify a XLSX file can be imported.""" - path = os.path.join(FILES, "exported.xlsx") - mock_document = Mock() - # Act - with catch_warnings(): - importer._file_xlsx(path, mock_document) - # Assert - args, kwargs = mock_itemize.call_args - logging.debug("args: {}".format(args)) - logging.debug("kwargs: {}".format(kwargs)) - header, data, document = args + def _xlsx_check_header(self, header): expected_header = [ "uid", "level", @@ -301,6 +289,26 @@ def test_file_xlsx(self, mock_itemize): "reviewed", ] self.assertEqual(expected_header, header) + + def _xlsx_check_tst(self, data): + expected_data = [ + [ + "tst003", + "1", + "Tutorial", + None, + None, + "", + True, + False, + None, + True, + None, + ] + ] + self.assertEqual(expected_data, data) + + def _xlsx_check_req(self, data): expected_data = [ [ "REQ001", @@ -384,8 +392,56 @@ def test_file_xlsx(self, mock_itemize): ], ] self.assertEqual(expected_data, data) + + @patch("doorstop.core.importer._itemize") + def test_file_xlsx_single(self, mock_itemize): + """Verify a XLSX file can be imported.""" + path = os.path.join(FILES, "exported.xlsx") + mock_document = Mock() + # Act + with catch_warnings(): + importer._file_xlsx(path, mock_document) + # Assert + args, kwargs = mock_itemize.call_args + logging.debug("args: {}".format(args)) + logging.debug("kwargs: {}".format(kwargs)) + header, data, document = args + self._xlsx_check_header(header) + self._xlsx_check_req(data) + self.assertIs(mock_document, document) + @patch("doorstop.core.importer._itemize") + def test_file_xlsx_all(self, mock_itemize): + """Verify a XLSX file can be imported.""" + + def check_itemize(*args, **kwargs): + logging.debug("args: {}".format(args)) + logging.debug("kwargs: {}".format(kwargs)) + header, data, document = args + self._xlsx_check_header(header) + match document.prefix: + case "REQ": + self._xlsx_check_req(data) + case "tst": + self._xlsx_check_tst(data) + case _: + self.fail(f"unexpected sheet {document.prefix}") + + path = os.path.join(FILES, "exported-all.xlsx") + mock_itemize.side_effect = check_itemize + + # Act + with catch_warnings(): + importer._file_xlsx(path, None) + # Assert + args, kwargs = mock_itemize.call_args + logging.debug("args: {}".format(args)) + logging.debug("kwargs: {}".format(kwargs)) + header, data, document = args + self._xlsx_check_header(header) + self._xlsx_check_req(data) + @patch("doorstop.core.importer._itemize") def test_file_xlsx_formula(self, mock_itemize): """Verify a XLSX file with formula can be imported.""" diff --git a/doorstop/core/vcs/__init__.py b/doorstop/core/vcs/__init__.py index 5cc6a417c..fc65fc6ec 100644 --- a/doorstop/core/vcs/__init__.py +++ b/doorstop/core/vcs/__init__.py @@ -52,3 +52,10 @@ def load(path): log.warning("no working copy found at: {}".format(path)) return DEFAULT(path) + +def describe(path): + try: + find_root(path) + except: + return "Not in revision control" + diff --git a/doorstop/core/vcs/base.py b/doorstop/core/vcs/base.py index ed553bfdc..100d17f79 100644 --- a/doorstop/core/vcs/base.py +++ b/doorstop/core/vcs/base.py @@ -66,6 +66,11 @@ def commit(self, message=None): # pragma: no cover (abstract method) """Unlock files, commit, and push.""" raise NotImplementedError + @abstractmethod + def describe(self): # pragma: no cover (abstract method) + """Unlock files, commit, and push.""" + raise NotImplementedError + @property def ignores(self): """Yield glob expressions to ignore.""" diff --git a/doorstop/core/vcs/git.py b/doorstop/core/vcs/git.py index 3c7a804b2..7a7484f54 100644 --- a/doorstop/core/vcs/git.py +++ b/doorstop/core/vcs/git.py @@ -31,3 +31,6 @@ def commit(self, message=None): message = message or input("Commit message: ") self.call("git", "commit", "--all", "--message", message) self.call("git", "push") + + def describe(self): + return "git: " + self.call("git", "describe", "--dirty", return_stdout=True) diff --git a/doorstop/core/vcs/mercurial.py b/doorstop/core/vcs/mercurial.py index e0461912c..233ba359a 100644 --- a/doorstop/core/vcs/mercurial.py +++ b/doorstop/core/vcs/mercurial.py @@ -31,3 +31,8 @@ def commit(self, message=None): message = message or input("Commit message: ") self.call("hg", "commit", "--message", message) self.call("hg", "push") + + def describe(self): + return "mercurial: " + self.call("hg", "log", "-r", ".", "-T", + "{latesttag}{sub('^-0-.*', '', '-{latesttagdistance}-m{node|short}')}", + return_stdout=True) diff --git a/doorstop/core/vcs/mockvcs.py b/doorstop/core/vcs/mockvcs.py index 0e1e6dd7a..373fd19c3 100644 --- a/doorstop/core/vcs/mockvcs.py +++ b/doorstop/core/vcs/mockvcs.py @@ -33,3 +33,7 @@ def delete(self, path): def commit(self, message=None): log.debug("$ simulated commit") + + def describe(self): + log.debug("$ simulated describe") + return "mockvcs: simulated describe" diff --git a/doorstop/core/vcs/subversion.py b/doorstop/core/vcs/subversion.py index 1e37953e5..d39582c22 100644 --- a/doorstop/core/vcs/subversion.py +++ b/doorstop/core/vcs/subversion.py @@ -33,6 +33,9 @@ def commit(self, message=None): message = message or input("Commit message: ") self.call("svn", "commit", "--message", message) + def describe(self): + return "subversion: " + self.call("svnversion", return_stdout=True) + @property def ignores(self): if self._ignores_cache is None: diff --git a/reqs/ext/EXT001.yml b/reqs/ext/EXT001.yml index b86847a72..49450e2d1 100644 --- a/reqs/ext/EXT001.yml +++ b/reqs/ext/EXT001.yml @@ -7,7 +7,6 @@ normative: true ref: '' references: - path: reqs/ext/test.file - sha: 24d9b35c727e6c78676c2ad378a8c3c47cfb539f3583d3cd9e1eafee51d5679d type: file reviewed: n0xxAj0z-SqNcebKEu8p9HDE8jAs5I8Vz4kX-5ZieA4= text: | diff --git a/reqs/ext/EXT002.yml b/reqs/ext/EXT002.yml index 80989fdd2..06af8f8bb 100644 --- a/reqs/ext/EXT002.yml +++ b/reqs/ext/EXT002.yml @@ -7,7 +7,6 @@ normative: true ref: '' references: - path: reqs/ext/test-modified.file - sha: 49ca5d81054fdd20572294b9350b605d05e0df91da09a46fb8bde7fd6c1c172d type: file reviewed: arLWz1tqET94t2j7FG0ncvKpwGe5twDi-jPbBnikxho= text: | diff --git a/reqs/tutorial/TUT001.yml b/reqs/tutorial/TUT001.yml index 88600c665..4a40fdb62 100644 --- a/reqs/tutorial/TUT001.yml +++ b/reqs/tutorial/TUT001.yml @@ -1,4 +1,5 @@ active: true +custom-attrib: null derived: false header: '' level: 1.1 diff --git a/reqs/tutorial/TUT002.yml b/reqs/tutorial/TUT002.yml index 6043467b4..2d9da6e91 100644 --- a/reqs/tutorial/TUT002.yml +++ b/reqs/tutorial/TUT002.yml @@ -1,4 +1,5 @@ active: true +custom-attrib: null derived: false header: '' level: 1.2 diff --git a/reqs/tutorial/TUT003.yml b/reqs/tutorial/TUT003.yml index cc4a78cfb..ef4d4c3d9 100644 --- a/reqs/tutorial/TUT003.yml +++ b/reqs/tutorial/TUT003.yml @@ -1,4 +1,5 @@ active: true +custom-attrib: null derived: false header: '' level: 1 diff --git a/reqs/tutorial/TUT004.yml b/reqs/tutorial/TUT004.yml index a1e6fc7f8..f22e98a2d 100644 --- a/reqs/tutorial/TUT004.yml +++ b/reqs/tutorial/TUT004.yml @@ -1,4 +1,5 @@ active: true +custom-attrib: null derived: false header: '' level: 1.3 diff --git a/reqs/tutorial/TUT005.yml b/reqs/tutorial/TUT005.yml index b705ab427..4632ab317 100644 --- a/reqs/tutorial/TUT005.yml +++ b/reqs/tutorial/TUT005.yml @@ -1,4 +1,5 @@ active: true +custom-attrib: null derived: false header: '' level: 2.0 diff --git a/reqs/tutorial/TUT008.yml b/reqs/tutorial/TUT008.yml index a74a455a9..3222fec81 100644 --- a/reqs/tutorial/TUT008.yml +++ b/reqs/tutorial/TUT008.yml @@ -1,4 +1,5 @@ active: true +custom-attrib: null derived: false header: '' level: 1.4 diff --git a/reqs/tutorial/TUT009.yml b/reqs/tutorial/TUT009.yml index 6e4990d75..12168cd74 100644 --- a/reqs/tutorial/TUT009.yml +++ b/reqs/tutorial/TUT009.yml @@ -1,4 +1,5 @@ active: true +custom-attrib: null derived: false header: '' level: 2.1 diff --git a/reqs/tutorial/TUT010.yml b/reqs/tutorial/TUT010.yml index fda227a80..d9002c4cd 100644 --- a/reqs/tutorial/TUT010.yml +++ b/reqs/tutorial/TUT010.yml @@ -1,4 +1,5 @@ active: true +custom-attrib: null derived: false header: '' level: 2.2 diff --git a/reqs/tutorial/TUT011.yml b/reqs/tutorial/TUT011.yml index 76feee35d..26934c9c8 100644 --- a/reqs/tutorial/TUT011.yml +++ b/reqs/tutorial/TUT011.yml @@ -1,4 +1,5 @@ active: true +custom-attrib: null derived: false header: '' level: 3.0 diff --git a/reqs/tutorial/TUT012.yml b/reqs/tutorial/TUT012.yml index 61c21d5b6..6cb6e51d1 100644 --- a/reqs/tutorial/TUT012.yml +++ b/reqs/tutorial/TUT012.yml @@ -1,4 +1,5 @@ active: true +custom-attrib: null derived: false header: '' level: 3.2 diff --git a/reqs/tutorial/TUT013.yml b/reqs/tutorial/TUT013.yml index 00dd5c01a..b3b2a5548 100644 --- a/reqs/tutorial/TUT013.yml +++ b/reqs/tutorial/TUT013.yml @@ -1,4 +1,5 @@ active: true +custom-attrib: null derived: false header: '' level: 3.3 diff --git a/reqs/tutorial/TUT014.yml b/reqs/tutorial/TUT014.yml index 9ba0249bc..fbb27dfe4 100644 --- a/reqs/tutorial/TUT014.yml +++ b/reqs/tutorial/TUT014.yml @@ -1,4 +1,5 @@ active: true +custom-attrib: null derived: false header: '' level: 4.0 diff --git a/reqs/tutorial/TUT015.yml b/reqs/tutorial/TUT015.yml index 83db715ef..4ee7c7cfd 100644 --- a/reqs/tutorial/TUT015.yml +++ b/reqs/tutorial/TUT015.yml @@ -1,4 +1,5 @@ active: true +custom-attrib: null derived: false header: '' level: 4.1 diff --git a/reqs/tutorial/TUT016.yml b/reqs/tutorial/TUT016.yml index 2d97cee09..9145b9680 100644 --- a/reqs/tutorial/TUT016.yml +++ b/reqs/tutorial/TUT016.yml @@ -1,4 +1,5 @@ active: true +custom-attrib: null derived: false header: '' level: 3.1 diff --git a/reqs/tutorial/TUT017.yml b/reqs/tutorial/TUT017.yml index d85e4d986..91343a7a6 100644 --- a/reqs/tutorial/TUT017.yml +++ b/reqs/tutorial/TUT017.yml @@ -1,4 +1,5 @@ active: true +custom-attrib: null derived: false header: | Lot's of different little examples in a single heading which is very long diff --git a/reqs/tutorial/TUT018.yml b/reqs/tutorial/TUT018.yml index 45bd019c5..f88301406 100644 --- a/reqs/tutorial/TUT018.yml +++ b/reqs/tutorial/TUT018.yml @@ -1,4 +1,5 @@ active: true +custom-attrib: null derived: false header: '' level: 1.6.0 diff --git a/reqs/tutorial/TUT019.yml b/reqs/tutorial/TUT019.yml index 11961e004..8f2adf0fb 100644 --- a/reqs/tutorial/TUT019.yml +++ b/reqs/tutorial/TUT019.yml @@ -1,4 +1,5 @@ active: true +custom-attrib: null derived: false header: '' level: 1.6.1 diff --git a/reqs/tutorial/TUT020.yml b/reqs/tutorial/TUT020.yml index 14dbfab1b..3f663cdc4 100644 --- a/reqs/tutorial/TUT020.yml +++ b/reqs/tutorial/TUT020.yml @@ -1,5 +1,5 @@ -CUSTOM-ATTRIB: true active: true +custom-attrib: true derived: false header: '' level: 2.3 diff --git a/reqs/tutorial/TUT021.yml b/reqs/tutorial/TUT021.yml index 88cbec19f..ea1566d0e 100644 --- a/reqs/tutorial/TUT021.yml +++ b/reqs/tutorial/TUT021.yml @@ -1,4 +1,5 @@ active: true +custom-attrib: null derived: false header: '' level: 5.0 diff --git a/reqs/tutorial/TUT022.yml b/reqs/tutorial/TUT022.yml index 34b5122fc..cd9aab3b1 100644 --- a/reqs/tutorial/TUT022.yml +++ b/reqs/tutorial/TUT022.yml @@ -1,4 +1,5 @@ active: true +custom-attrib: null derived: false header: | Lists diff --git a/reqs/tutorial/TUT023.yml b/reqs/tutorial/TUT023.yml index 46279f0e6..858772470 100644 --- a/reqs/tutorial/TUT023.yml +++ b/reqs/tutorial/TUT023.yml @@ -1,4 +1,5 @@ active: true +custom-attrib: null derived: false header: | Nested list diff --git a/reqs/tutorial/TUT024.yml b/reqs/tutorial/TUT024.yml index 326632d1b..4ee901b9f 100644 --- a/reqs/tutorial/TUT024.yml +++ b/reqs/tutorial/TUT024.yml @@ -1,4 +1,5 @@ active: true +custom-attrib: null derived: false header: | Ordered list with empty items diff --git a/reqs/tutorial/TUT025.yml b/reqs/tutorial/TUT025.yml index f60dab882..ef503c633 100644 --- a/reqs/tutorial/TUT025.yml +++ b/reqs/tutorial/TUT025.yml @@ -1,4 +1,5 @@ active: true +custom-attrib: null derived: false header: | Another list example