fix(lint): auto-fix comment periods with godot tool

Run godot -w to automatically add missing periods to comments

Author: jackspirou

llama3/cmd/llama3/decode.go

@@ -12,11 +12,11 @@ import (
 )
 
 var (
-	// Decode command flags
+	// Decode command flags.
 	decSkipSpecial bool
 )
 
-// newDecodeCmd creates the decode subcommand
+// newDecodeCmd creates the decode subcommand.
 func newDecodeCmd() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "decode [token_ids...]",

llama3/cmd/llama3/encode.go

@@ -12,13 +12,13 @@ import (
 )
 
 var (
-	// Encode command flags
+	// Encode command flags.
 	encAddBOS bool
 	encAddEOS bool
 	encOutput string
 )
 
-// newEncodeCmd creates the encode subcommand
+// newEncodeCmd creates the encode subcommand.
 func newEncodeCmd() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "encode [text]",

llama3/cmd/llama3/info.go

@@ -7,7 +7,7 @@ import (
 	"github.com/spf13/cobra"
 )
 
-// newInfoCmd creates the info subcommand
+// newInfoCmd creates the info subcommand.
 func newInfoCmd() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "info",

llama3/cmd/llama3/stream.go

@@ -9,15 +9,15 @@ import (
 )
 
 var (
-	// Stream command flags
+	// Stream command flags.
 	streamBufferSize int
 	streamMaxBuffer  int
 	streamAddBOS     bool
 	streamAddEOS     bool
 	streamOutput     string
 )
 
-// newStreamCmd creates the stream subcommand
+// newStreamCmd creates the stream subcommand.
 func newStreamCmd() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "stream",

llama3/comparison_test.go

@@ -11,19 +11,19 @@ import (
 	"testing"
 )
 
-// ComparisonTestCase represents a test case for comparing Go and JS implementations
+// ComparisonTestCase represents a test case for comparing Go and JS implementations.
 type ComparisonTestCase struct {
 	Input    string `json:"input"`
 	Expected []int  `json:"expected"`
 }
 
-// toJSON converts a value to JSON string
+// toJSON converts a value to JSON string.
 func toJSON(v interface{}) string {
 	b, _ := json.Marshal(v)
 	return string(b)
 }
 
-// generateTestVectors creates a JavaScript file to generate test vectors
+// generateTestVectors creates a JavaScript file to generate test vectors.
 func generateTestVectors(testCases []string) ([]ComparisonTestCase, error) {
 	// Create temporary JS file
 	tmpDir := os.TempDir()
@@ -62,7 +62,7 @@ console.log(JSON.stringify(results, null, 2));
 	return results, nil
 }
 
-// TestComparisonWithJS compares Go implementation with JavaScript implementation
+// TestComparisonWithJS compares Go implementation with JavaScript implementation.
 func TestComparisonWithJS(t *testing.T) {
 	tokenizer, err := New()
 	if err != nil || tokenizer.VocabSize() == 0 {
@@ -208,7 +208,7 @@ func TestComparisonWithJS(t *testing.T) {
 	}
 }
 
-// TestComparisonFromFile tests using test vectors from a file
+// TestComparisonFromFile tests using test vectors from a file.
 func TestComparisonFromFile(t *testing.T) {
 	tokenizer, err := New()
 	if err != nil || tokenizer.VocabSize() == 0 {
@@ -270,7 +270,7 @@ func TestComparisonFromFile(t *testing.T) {
 	}
 }
 
-// generateTestVectorFile creates a file with test vectors for future use
+// generateTestVectorFile creates a file with test vectors for future use.
 func generateTestVectorFile(filename string, count int) error {
 	// Generate diverse test inputs
 	var inputs []string

llama3/compatibility_test.go

@@ -12,7 +12,7 @@ import (
 	testutils "github.com/agentstation/tokenizer/llama3/internal/testing"
 )
 
-// TestCompatibility runs comprehensive compatibility tests with 476 test cases
+// TestCompatibility runs comprehensive compatibility tests with 476 test cases.
 func TestCompatibility(t *testing.T) {
 	tokenizer, err := New()
 	if err != nil || tokenizer.VocabSize() == 0 {
@@ -96,7 +96,7 @@ func TestCompatibility(t *testing.T) {
 		float64(totalPassed)*100/float64(totalPassed+totalFailed))
 }
 
-// getJSTokenization gets tokenization results from JavaScript implementation
+// getJSTokenization gets tokenization results from JavaScript implementation.
 func getJSTokenization(inputs []string, jsPath string) ([][]int, error) {
 	// Create temporary JS file
 	tmpDir := os.TempDir()
@@ -134,7 +134,7 @@ console.log(JSON.stringify(results));
 	return results, nil
 }
 
-// compareTokens compares two token arrays
+// compareTokens compares two token arrays.
 func compareTokens(a, b []int) bool {
 	if len(a) != len(b) {
 		return false
@@ -147,7 +147,7 @@ func compareTokens(a, b []int) bool {
 	return true
 }
 
-// TestTokenizationProperties tests properties that should hold for all inputs
+// TestTokenizationProperties tests properties that should hold for all inputs.
 func TestTokenizationProperties(t *testing.T) {
 	tokenizer, err := New()
 	if err != nil || tokenizer.VocabSize() == 0 {
@@ -185,7 +185,7 @@ func TestTokenizationProperties(t *testing.T) {
 	}
 }
 
-// BenchmarkCases benchmarks various categories of inputs
+// BenchmarkCases benchmarks various categories of inputs.
 func BenchmarkCases(b *testing.B) {
 	tokenizer, err := New()
 	if err != nil || tokenizer.VocabSize() == 0 {

llama3/errors.go

@@ -5,22 +5,22 @@ import (
 	"fmt"
 )
 
-// Common errors
+// Common errors.
 var (
-	// ErrDataNotFound indicates that the tokenizer data files could not be found
+	// ErrDataNotFound indicates that the tokenizer data files could not be found.
 	ErrDataNotFound = errors.New("tokenizer data not found")
 
-	// ErrInvalidToken indicates an invalid token was provided
+	// ErrInvalidToken indicates an invalid token was provided.
 	ErrInvalidToken = errors.New("invalid token")
 
-	// ErrTokenNotFound indicates a token was not found in the vocabulary
+	// ErrTokenNotFound indicates a token was not found in the vocabulary.
 	ErrTokenNotFound = errors.New("token not found")
 
-	// ErrInvalidTokenID indicates an invalid token ID was provided
+	// ErrInvalidTokenID indicates an invalid token ID was provided.
 	ErrInvalidTokenID = errors.New("invalid token ID")
 )
 
-// DataError represents an error related to tokenizer data loading or processing
+// DataError represents an error related to tokenizer data loading or processing.
 type DataError struct {
 	Op   string // Operation that failed
 	Path string // File path if applicable
@@ -38,7 +38,7 @@ func (e *DataError) Unwrap() error {
 	return e.Err
 }
 
-// TokenError represents an error related to token operations
+// TokenError represents an error related to token operations.
 type TokenError struct {
 	Token   string // The token that caused the error
 	TokenID int    // The token ID if applicable
@@ -60,7 +60,7 @@ func (e *TokenError) Unwrap() error {
 	return e.Err
 }
 
-// ConfigError represents an error in tokenizer configuration
+// ConfigError represents an error in tokenizer configuration.
 type ConfigError struct {
 	Field string // Configuration field that has an error
 	Value any    // The invalid value
@@ -77,22 +77,22 @@ func (e *ConfigError) Unwrap() error {
 
 // Helper functions for creating errors
 
-// NewDataError creates a new DataError
+// NewDataError creates a new DataError.
 func NewDataError(op, path string, err error) error {
 	return &DataError{Op: op, Path: path, Err: err}
 }
 
-// NewTokenError creates a new TokenError
+// NewTokenError creates a new TokenError.
 func NewTokenError(op, token string, err error) error {
 	return &TokenError{Op: op, Token: token, Err: err}
 }
 
-// NewTokenIDError creates a new TokenError with a token ID
+// NewTokenIDError creates a new TokenError with a token ID.
 func NewTokenIDError(op string, tokenID int, err error) error {
 	return &TokenError{Op: op, TokenID: tokenID, Err: err}
 }
 
-// NewConfigError creates a new ConfigError
+// NewConfigError creates a new ConfigError.
 func NewConfigError(field string, value any, err error) error {
 	return &ConfigError{Field: field, Value: value, Err: err}
 }

llama3/internal/pretokenizer/state_machine_test.go

@@ -1070,7 +1070,7 @@ func TestCharacterClassification(t *testing.T) {
 	})
 }
 
-// Benchmark tests to ensure performance hasn't regressed
+// Benchmark tests to ensure performance hasn't regressed.
 func BenchmarkTokenizeLongText(b *testing.B) {
 	// Create a realistic long text
 	parts := []string{
@@ -1121,7 +1121,7 @@ func BenchmarkTokenizeWhitespaceHeavyTable(b *testing.B) {
 	}
 }
 
-// TestStateMachineJavaScriptCompatibility tests cases from JavaScript output
+// TestStateMachineJavaScriptCompatibility tests cases from JavaScript output.
 func TestStateMachineJavaScriptCompatibility(t *testing.T) {
 	testCases := []struct {
 		input    string

llama3/internal/tokens/special.go

@@ -7,10 +7,10 @@ import (
 )
 
 var (
-	// SpecialTokenRegex matches Llama 3 special tokens
+	// SpecialTokenRegex matches Llama 3 special tokens.
 	SpecialTokenRegex = regexp.MustCompile(`<\|(?:begin_of_text|end_of_text|start_header_id|end_header_id|eot_id|eom_id|python_tag|finetune_right_pad_id|reserved_special_token_(?:[0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-3][0-9]|24[0-7]))\|>`)
 
-	// OptimisticSpecialTokenRegex matches any pattern that looks like a special token
+	// OptimisticSpecialTokenRegex matches any pattern that looks like a special token.
 	OptimisticSpecialTokenRegex = regexp.MustCompile(`<\|[a-zA-Z0-9_]+\|>`)
 )
 

llama3/internal/vocabulary/decoder.go

@@ -7,7 +7,7 @@ import (
 )
 
 const (
-	// bitsPerMergeID is the number of bits used to encode each merge ID
+	// bitsPerMergeID is the number of bits used to encode each merge ID.
 	bitsPerMergeID = 17
 )