better error recovery for file encoding and bad json

packages/bolt/src/bolt/ast.py

@@ -75,6 +75,7 @@
     AstChildren,
     AstCommand,
     AstCommandSentinel,
+    AstError,
     AstJson,
     AstLiteral,
     AstNode,
@@ -493,7 +494,7 @@ class AstProcMacroMarker(AstNode):
 class AstProcMacroResult(AstNode):
     """Ast proc macro result node."""
 
-    commands: AstChildren[AstCommand] = required_field()
+    commands: AstChildren[AstCommand|AstError] = required_field()
 
 
 @dataclass(frozen=True, slots=True)

packages/bolt/src/bolt/contrib/sandbox.py

@@ -189,7 +189,11 @@ def activate(self):
         get_attribute_handler = self.runtime.helpers["get_attribute_handler"]
 
         self.runtime.helpers.update(
+<<<<<<< HEAD
             get_attribute_handler=lambda obj: SandboxedAttributeHandler(
+=======
+            get_attribute_handler=lambda obj: SandboxedAttributeHandler(  # type: ignore
+>>>>>>> bolt-fork/migrate-to-beet
                 obj=obj,
                 handler=get_attribute_handler(obj),
                 sandbox=self,

packages/bolt/src/bolt/emit.py

@@ -6,7 +6,7 @@
 from contextlib import contextmanager
 from typing import Any, Callable, Generator, Iterator, List, Optional, ParamSpec, Tuple
 
-from mecha import AstCommand, AstNode, AstRoot
+from mecha import AstCommand, AstError, AstNode, AstRoot
 
 from .utils import internal
 
@@ -16,7 +16,7 @@
 class CommandEmitter:
     """Command emitter."""
 
-    commands: List[AstCommand]
+    commands: List[AstCommand|AstError]
     nesting: List[Tuple[str, Tuple[AstNode, ...]]]
 
     def __init__(self):
@@ -26,8 +26,8 @@ def __init__(self):
     @contextmanager
     def scope(
         self,
-        commands: Optional[List[AstCommand]] = None,
-    ) -> Iterator[List[AstCommand]]:
+        commands: Optional[List[AstCommand|AstError]] = None,
+    ) -> Iterator[List[AstCommand|AstError]]:
         """Create a new scope to gather commands."""
         if commands is None:
             commands = []
@@ -46,7 +46,7 @@ def capture_output(
         f: Callable[P, Any],
         *args: P.args,
         **kwargs: P.kwargs,
-    ) -> List[AstCommand]:
+    ) -> List[AstCommand|AstError]:
         """Invoke a user-defined function and return the list of generated commands."""
         with self.scope() as output:
             result = f(*args, **kwargs)

packages/bolt/src/bolt/parse.py

@@ -76,6 +76,7 @@
     AdjacentConstraint,
     AlternativeParser,
     AstChildren,
+    AstError,
     AstCommand,
     AstJson,
     AstNode,
@@ -968,9 +969,13 @@ def __call__(self, stream: TokenStream) -> Any:
 
     def resolve(self, node: AstRoot) -> AstRoot:
         should_replace = False
-        commands: List[AstCommand] = []
+        commands: List[AstCommand|AstError] = []
 
         for command in node.commands:
+            if isinstance(command, AstError):
+                commands.append(command)
+                continue
+
             stack: List[AstCommand] = [command]
 
             while command.arguments and isinstance(
@@ -1100,9 +1105,13 @@ def __call__(self, stream: TokenStream) -> AstRoot:
         stack: List[AstDecorator] = []
 
         changed = False
-        result: List[AstCommand] = []
+        result: List[AstCommand|AstError] = []
 
         for command in node.commands:
+            if isinstance(command, AstError):
+                result.append(command)
+                continue
+
             if isinstance(command, AstStatement) and isinstance(
                 decorator := command.arguments[0], AstDecorator
             ):
@@ -1185,9 +1194,13 @@ def __call__(self, stream: TokenStream) -> Any:
             return node
 
         changed = False
-        result: List[AstCommand] = []
+        result: List[AstCommand|AstError] = []
 
         for command in node.commands:
+            if isinstance(command, AstError):
+                result.append(command)
+                continue
+
             if command.identifier == "return:value" and command.arguments:
                 changed = True
 
@@ -1222,12 +1235,16 @@ def __call__(self, stream: TokenStream) -> AstRoot:
         node: AstRoot = self.parser(stream)
 
         changed = False
-        result: List[AstCommand] = []
+        result: List[AstError|AstCommand] = []
 
         commands = iter(node.commands)
         previous = ""
 
         for command in commands:
+            if isinstance(command, AstError):
+                result.append(command)
+                continue
+
             if command.identifier in ["elif:condition:body", "else:body"]:
                 if previous not in ["if:condition:body", "elif:condition:body"]:
                     exc = InvalidSyntax(
@@ -1240,6 +1257,9 @@ def __call__(self, stream: TokenStream) -> AstRoot:
                 elif_chain = [command]
 
                 for command in commands:
+                    if isinstance(command, AstError):
+                        continue
+
                     if command.identifier not in ["elif:condition:body", "else:body"]:
                         break
                     elif_chain.append(command)
@@ -1291,6 +1311,9 @@ def __call__(self, stream: TokenStream) -> AstRoot:
 
         if not loop:
             for command in node.commands:
+                if isinstance(command, AstError):
+                    continue
+
                 if command.identifier in ["break", "continue"]:
                     exc = InvalidSyntax(
                         f'Can only use "{command.identifier}" in loops.'
@@ -1350,7 +1373,6 @@ def parse_function_signature(stream: TokenStream) -> AstFunctionSignature:
         stream.expect(("brace", "("))
 
         node = set_location(AstFunctionSignature(name=identifier.value), identifier)
-        lexical_scope.bind_variable(identifier.value, node)
 
         deferred_scope = lexical_scope.deferred(FunctionScope)
 
@@ -1469,6 +1491,7 @@ def parse_function_signature(stream: TokenStream) -> AstFunctionSignature:
                         exc = InvalidSyntax(
                             "Expected at least one named argument after bare variadic marker."
                         )
+
                         raise set_location(exc, argument)
 
             return_type_annotation = None
@@ -1755,11 +1778,15 @@ class ProcMacroExpansion:
 
     def __call__(self, stream: TokenStream) -> AstRoot:
         should_replace = False
-        commands: List[AstCommand] = []
+        commands: List[AstCommand|AstError] = []
 
         node: AstRoot = self.parser(stream)
 
         for command in node.commands:
+            if isinstance(command, AstError):
+                commands.append(command)
+                continue
+
             stack: List[AstCommand] = [command]
 
             while command.arguments and isinstance(
@@ -2074,9 +2101,13 @@ def __call__(self, stream: TokenStream) -> Any:
         node: AstRoot = self.parser(stream)
 
         changed = False
-        result: List[AstCommand] = []
+        result: List[AstCommand|AstError] = []
 
         for command in node.commands:
+            if isinstance(command, AstError):
+                result.append(command)
+                continue
+
             if (
                 isinstance(command, AstStatement)
                 and command.arguments
@@ -2198,9 +2229,13 @@ def __call__(self, stream: TokenStream) -> AstRoot:
 
     def resolve_deferred(self, node: AstRoot, stream: TokenStream) -> AstRoot:
         should_replace = False
-        commands: List[AstCommand] = []
+        commands: List[AstCommand|AstError] = []
 
         for command in node.commands:
+            if isinstance(command, AstError):
+                commands.append(command)
+                continue 
+
             if command.arguments and isinstance(
                 body := command.arguments[-1], AstClassRoot
             ):
@@ -2283,6 +2318,9 @@ def __call__(self, stream: TokenStream) -> AstRoot:
 
         if isinstance(node, AstRoot):
             for command in node.commands:
+                if isinstance(command, AstError):
+                    continue
+
                 if command.identifier in self.command_identifiers:
                     name, _, _ = command.identifier.partition(":")
                     exc = InvalidSyntax(
@@ -2402,7 +2440,6 @@ def __call__(self, stream: TokenStream) -> Any:
         node = AstUnpack(type="dict" if prefix.value == "**" else "list", value=node)
         return set_location(node, prefix, node.value)
 
-
 @dataclass
 class UnpackConstraint:
     """Constraint for unpacking."""

packages/bolt/src/bolt/runtime.py

@@ -4,7 +4,6 @@
     "NonFunctionSerializer",
 ]
 
-
 import builtins
 from dataclasses import dataclass, field
 from functools import partial
@@ -15,6 +14,7 @@
 from beet.core.utils import JsonDict, extra_field, required_field
 from mecha import (
     AstRoot,
+    AstError,
     CommandSpec,
     CommandTree,
     CompilationDatabase,
@@ -30,7 +30,7 @@
 from pathspec import PathSpec
 from tokenstream import set_location
 
-from .ast import AstNonFunctionRoot
+from .ast import AstNonFunctionRoot, AstRoot
 from .codegen import Codegen
 from .emit import CommandEmitter
 from .helpers import get_bolt_helpers
@@ -275,6 +275,10 @@ def non_function_root(self, node: AstNonFunctionRoot, result: List[str]):
             result.append(source)
         if node.commands:
             command = node.commands[0]
+
+            if isinstance(command, AstError):
+                return None
+
             name = command.identifier.partition(":")[0]
             d = Diagnostic(
                 "warn", f'Ignored top-level "{name}" command outside function.'

packages/mecha/src/mecha/api.py

@@ -54,7 +54,7 @@
 from pydantic import BaseModel, field_validator
 from tokenstream import InvalidSyntax, Preprocessor, TokenStream, set_location
 
-from .ast import AstLiteral, AstNode, AstRoot
+from .ast import AstError, AstLiteral, AstNode, AstRoot
 from .config import CommandTree
 from .database import (
     CompilationDatabase,
@@ -396,7 +396,7 @@ def parse(
                 filename=str(filename) if filename else None,
                 file=source,
             )
-            raise DiagnosticError(DiagnosticCollection([set_location(d, exc)])) from exc
+            diagnostics.append(set_location(d, exc))
         else:
             if self.cache and filename and cache_miss:
                 try:
@@ -407,6 +407,24 @@ def parse(
                     pass
             return ast
 
+        if len(diagnostics) > 0:
+            if self.cache and filename and cache_miss:
+                self.cache.invalidate_changes(self.directory / filename)
+
+            raise DiagnosticError(DiagnosticCollection(diagnostics))
+
+    def parse_stream(
+        self,
+        multiline: bool | None,
+        provide: JsonDict | None,
+        parser: str,
+        stream: TokenStream,
+    ):
+        with self.prepare_token_stream(stream, multiline=multiline):
+            with stream.provide(**provide or {}):
+                ast = delegate(parser, stream)
+        return ast
+
     @overload
     def compile(
         self,

packages/mecha/src/mecha/ast.py

@@ -4,6 +4,7 @@
     "AstNode",
     "AstChildren",
     "AstRoot",
+    "AstError",
     "AstCommand",
     "AstCommandSentinel",
     "AstString",
@@ -119,7 +120,9 @@
 from nbtlib import Byte, ByteArray, Compound, CompoundMatch, Double, Int, IntArray
 from nbtlib import List as ListTag
 from nbtlib import ListIndex, LongArray, NamedKey, Numeric, Path, String
-from tokenstream import UNKNOWN_LOCATION, SourceLocation, set_location
+from tokenstream import UNKNOWN_LOCATION, InvalidSyntax, SourceLocation, set_location
+
+from mecha.error import MechaError
 
 from .utils import string_to_number
 
@@ -266,11 +269,18 @@ class AstChildren(AbstractChildren[AstNodeType]):
 class AstRoot(AstNode):
     """Ast root node."""
 
-    commands: AstChildren["AstCommand"] = required_field()
+    commands: AstChildren["AstCommand|AstError"] = required_field()
 
     parser = "root"
 
 
+@dataclass(frozen=True, slots=True)
+class AstError(AstNode):
+    """Ast error node."""
+
+    error: InvalidSyntax = required_field()
+
+
 @dataclass(frozen=True, slots=True)
 class AstCommand(AstNode):
     """Ast command node."""