implement FileParts

Author: mathiasertl

docs/quickstart.rst

@@ -79,3 +79,30 @@ However, when running, you will just get:
 
     user@host:~$ structured-tutorial docs/tutorials/templates/tutorial.yaml
     real data
+
+***************************
+Tutorials can contain files
+***************************
+
+Tutorials can contain files that are also rendered as templates.
+
+The following example tutorial could be used to instruct the user to create a JSON file in
+``/tmp/example.json`` and then call ``python`` to verify the syntax.
+
+Using this YAML file:
+
+.. literalinclude:: /tutorials/simple-file/tutorial.yaml
+    :caption: docs/tutorials/simple-file/tutorial.yaml
+    :language: yaml
+
+you could render a Tutorial like this:
+
+.. structured-tutorial:: simple-file/tutorial.yaml
+
+Create a configuration file with the following contents:
+
+.. structured-tutorial-part::
+
+and make sure it is valid JSON:
+
+.. structured-tutorial-part::

docs/tutorials/exit_code/tutorial.yaml

@@ -2,7 +2,8 @@ parts:
   - commands:
       - command: "true"
       - command: "true"
-        status_code: 0
+        run:
+          status_code: 0
       # Do not fail the tutorial if this command returns exit code 1.
       - command: "false"
         run:

docs/tutorials/simple-file/tutorial.yaml

@@ -0,0 +1,17 @@
+configuration:
+  doc:
+    context:
+      file_contents: "{\"key\": \"value\"}"
+      file_path: /tmp/example.json
+parts:
+  # Part one: Create configuration file
+  - contents: |
+      {{ file_contents }}
+    destination: "{{ file_path }}"
+    doc:
+      language: json
+  # Part two: make sure the contents are right (as a demo)
+  - commands:
+      - command: python -m json.tool --compact {{ file_path }}
+        doc:
+          output: "{{ file_contents }}"

structured_tutorials/models.py

@@ -3,7 +3,7 @@
 from pathlib import Path
 from typing import Annotated, Any, Literal, Self
 
-from pydantic import BaseModel, Field, field_validator, model_validator
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
 from pydantic_core.core_schema import ValidationInfo
 from yaml import safe_load
 
@@ -19,12 +19,16 @@ def default_cwd_factory(data: dict[str, Any]) -> Path:
 class CommandBaseModel(BaseModel):
     """Base model for commands."""
 
+    model_config = ConfigDict(extra="forbid")
+
     status_code: Annotated[int, Field(ge=0, le=255)] = 0
 
 
 class TestSpecificationMixin:
     """Mixin for specifying tests."""
 
+    model_config = ConfigDict(extra="forbid")
+
     delay: Annotated[float, Field(ge=0)] = 0
     retry: PositiveInt = 0
     backoff_factor: PositiveFloat = 0  # {backoff factor} * (2 ** ({number of previous retries}))
@@ -33,25 +37,33 @@ class TestSpecificationMixin:
 class CleanupCommandModel(CommandBaseModel):
     """Model for cleanup commands."""
 
+    model_config = ConfigDict(extra="forbid")
+
     command: str
 
 
 class TestCommandModel(TestSpecificationMixin, CommandBaseModel):
     """Model for a test command for a normal command."""
 
+    model_config = ConfigDict(extra="forbid")
+
     command: str
 
 
 class TestPortModel(TestSpecificationMixin, BaseModel):
     """Model for testing connectivity after a command is run."""
 
+    model_config = ConfigDict(extra="forbid")
+
     host: str
     port: Annotated[int, Field(ge=0, le=65535)]
 
 
 class CommandRuntimeConfigurationModel(CommandBaseModel):
     """Model for runtime configuration when running a single command."""
 
+    model_config = ConfigDict(extra="forbid")
+
     update_context: dict[str, Any] = Field(default_factory=dict)
     cleanup: tuple[CleanupCommandModel, ...] = tuple()
     test: tuple[TestCommandModel | TestPortModel, ...] = tuple()
@@ -60,13 +72,17 @@ class CommandRuntimeConfigurationModel(CommandBaseModel):
 class CommandDocumentationConfigurationModel(BaseModel):
     """Model for documenting a single command."""
 
+    model_config = ConfigDict(extra="forbid")
+
     output: str = ""
     update_context: dict[str, Any] = Field(default_factory=dict)
 
 
 class CommandModel(BaseModel):
     """Model for a single command."""
 
+    model_config = ConfigDict(extra="forbid")
+
     command: str
     run: CommandRuntimeConfigurationModel = CommandRuntimeConfigurationModel()
     doc: CommandDocumentationConfigurationModel = CommandDocumentationConfigurationModel()
@@ -75,19 +91,35 @@ class CommandModel(BaseModel):
 class CommandsPartModel(BaseModel):
     """Model for a set of commands."""
 
+    model_config = ConfigDict(extra="forbid")
+
     type: Literal["commands"] = "commands"
     commands: tuple[CommandModel, ...]
 
 
+class FileDocumentationConfigurationModel(BaseModel):
+    # sphinx options:
+    language: str = ""
+    caption: str | Literal[False] = ""
+    linenos: bool = False
+    lineno_start: PositiveInt | Literal[False] = False
+    emphasize_lines: str = ""
+    name: str = ""
+
+
 class FilePartModel(BaseModel):
     """Model for a file to be copied."""
 
+    model_config = ConfigDict(extra="forbid")
+
     type: Literal["file"] = "file"
     contents: str | None = None
     source: Path | None = None
     destination: Path
     template: bool = True
 
+    doc: FileDocumentationConfigurationModel = FileDocumentationConfigurationModel()
+
     @field_validator("source", mode="after")
     @classmethod
     def validate_source(cls, value: Path) -> Path:
@@ -99,12 +131,16 @@ def validate_source(cls, value: Path) -> Path:
     def validate_contents_or_source(self) -> Self:
         if self.contents is None and self.source is None:
             raise ValueError("Either contents or source is required.")
+        if self.contents is not None and self.source is not None:
+            raise ValueError("Only one of contents or source is allowed.")
         return self
 
 
 class RuntimeConfigurationModel(BaseModel):
     """Model for configuration at runtime."""
 
+    model_config = ConfigDict(extra="forbid")
+
     context: dict[str, Any] = Field(default_factory=dict)
 
     @model_validator(mode="after")
@@ -117,6 +153,8 @@ def set_default_context(self) -> Self:
 class DocumentationConfigurationModel(BaseModel):
     """Model for configuration of the documentation."""
 
+    model_config = ConfigDict(extra="forbid")
+
     context: dict[str, Any] = Field(default_factory=dict)
 
     @model_validator(mode="after")
@@ -136,13 +174,17 @@ def set_default_context(self) -> Self:
 class ConfigurationModel(BaseModel):
     """Model for the initial configuration of a tutorial."""
 
+    model_config = ConfigDict(extra="forbid")
+
     run: RuntimeConfigurationModel = RuntimeConfigurationModel()
     doc: DocumentationConfigurationModel = DocumentationConfigurationModel()
 
 
 class TutorialModel(BaseModel):
     """Model representing the entire tutorial."""
 
+    model_config = ConfigDict(extra="forbid")
+
     path: Path  # absolute path
     cwd: Path = Field(default_factory=default_cwd_factory)  # absolute path (input: relative to path)
     parts: tuple[CommandsPartModel | FilePartModel, ...]

structured_tutorials/runners/local.py

@@ -1,5 +1,6 @@
 """Runner that runs a tutorial on the local machine."""
 
+import shutil
 import socket
 import subprocess
 import time
@@ -10,6 +11,7 @@
 from structured_tutorials.models import (
     CleanupCommandModel,
     CommandsPartModel,
+    FilePartModel,
     TestCommandModel,
     TestPortModel,
     TutorialModel,
@@ -58,7 +60,7 @@ def run_test(self, test: TestCommandModel | TestPortModel) -> None:
 
         raise RuntimeError("Test did not pass")
 
-    def run_command(self, part: CommandsPartModel) -> None:
+    def run_commands(self, part: CommandsPartModel) -> None:
         for command_config in part.commands:
             # Render the command
             command = self.render(command_config.command)
@@ -83,13 +85,46 @@ def run_command(self, part: CommandsPartModel) -> None:
             for test_command_config in command_config.run.test:
                 self.run_test(test_command_config)
 
+    def write_file(self, part: FilePartModel) -> None:
+        """Write a file."""
+        if part.destination.is_dir() and not part.source:
+            raise RuntimeError("Destination is directory, but no source given to derive filename.")
+
+        if part.destination.is_dir():
+            destination = part.destination / part.source.name
+        else:
+            part.destination.parent.mkdir(parents=True, exist_ok=True)
+            destination = part.destination
+
+        destination = part.destination
+        if destination.is_dir():
+            destination = destination / part.source.name
+
+        # If template=False and source is set, we just copy the file as is, without ever reading it
+        if not part.template and part.source:
+            shutil.copy(part.source, destination)
+            return
+
+        if part.source:
+            with open(part.source) as source_stream:
+                template = source_stream.read()
+        else:
+            template = part.contents
+
+        if part.template:
+            contents = self.render(template)
+        else:
+            contents = template
+
+        with open(destination, "w") as destination_stream:
+            destination_stream.write(contents)
+
     def run_parts(self) -> None:
         for part in self.tutorial.parts:
             if isinstance(part, CommandsPartModel):
-                self.run_command(part)
-            # elif isinstance(part, FilePartModel):  # pragma: no cover
-            #     source = self.tutorial.path.parent / part.source
-            #     print(source)
+                self.run_commands(part)
+            elif isinstance(part, FilePartModel):  # pragma: no cover
+                self.write_file(part)
             else:  # pragma: no cover
                 raise RuntimeError(f"{part} is not a tutorial part")
 

structured_tutorials/sphinx/utils.py

@@ -8,7 +8,7 @@
 from sphinx.config import Config
 from sphinx.errors import ConfigError, ExtensionError
 
-from structured_tutorials.models import CommandsPartModel, TutorialModel
+from structured_tutorials.models import CommandsPartModel, FilePartModel, TutorialModel
 
 
 def validate_configuration(app: Sphinx, config: Config) -> None:
@@ -50,6 +50,9 @@ def from_file(cls, path: Path) -> "TutorialWrapper":
         tutorial = TutorialModel.from_file(path)
         return cls(tutorial)
 
+    def render(self, template: str) -> str:
+        return self.env.from_string(template).render(self.context)
+
     def render_code_block(self, part: CommandsPartModel) -> str:
         """Render a CommandsPartModel as a code-block."""
         commands = []
@@ -80,11 +83,43 @@ def render_code_block(self, part: CommandsPartModel) -> str:
 {% endfor %}"""
         return self.env.from_string(template).render({"commands": commands})
 
+    def render_file(self, part: FilePartModel) -> str:
+        if part.template is False:
+            return ""
+
+        template = part.contents
+        if template is None:
+            with open(part.path) as stream:
+                template = stream.read()
+
+        content = self.render(template)
+
+        # get options
+        if part.doc.caption:
+            caption = self.render(part.doc.caption)
+        elif part.doc.caption is not False:
+            caption = self.render(str(part.destination))
+        else:
+            caption = ""
+
+        directive = """.. code-block:: {{ part.doc.language }}{% if caption %}
+    :caption: {{ caption }}{% endif %}{% if part.doc.linenos or part.doc.lineno_start %}
+    :linenos:{% endif %}{% if part.doc.lineno_start %}
+    :lineno-start: {{ part.doc.lineno_start }}{% endif %}{% if part.doc.emphasize_lines %}
+    :emphasize-lines: {{ part.doc.emphasize_lines }}{% endif %}{% if part.doc.name %}
+    :name: {{ part.doc.name }}{% endif %}
+
+    {{ content }}
+"""
+        return self.env.from_string(directive).render({"part": part, "content": content, "caption": caption})
+
     def render_part(self) -> str:
         """Render the given part of the tutorial."""
         part = self.tutorial.parts[self.next_part]
         if isinstance(part, CommandsPartModel):
             text = self.render_code_block(part)
+        elif isinstance(part, FilePartModel):
+            text = self.render_file(part)
         else:  # pragma: no cover
             raise ValueError("unsupported part type.")
         self.next_part += 1