fix(lint): address remaining linting issues

- Remove duplicate/unused constants from constants.go
- Remove unused test helper functions
- Add package comment for internal/testing
- Use exec.CommandContext instead of exec.Command
- Preallocate slice in test vector generation
- Fix unused parameter warnings with _
- Use constants instead of string literals
- Add nolint comment for staticcheck SA6002

Only remaining issue is cyclomatic complexity in scanner.Scan()

Author: jackspirou

cmd/tokenizer/root.go

@@ -44,7 +44,7 @@ Common operations available for tokenizers:
 var versionCmd = &cobra.Command{
 	Use:   "version",
 	Short: "Print version information",
-	Run: func(_ *cobra.Command, args []string) {
+	Run: func(_ *cobra.Command, _ []string) {
 		fmt.Printf("tokenizer version %s\n", version)
 		if commit != "none" {
 			fmt.Printf("  commit:     %s\n", commit)

llama3/cmd/llama3/info.go

@@ -26,7 +26,7 @@ configuration.`,
 	return cmd
 }
 
-func runInfo(_ *cobra.Command, args []string) error {
+func runInfo(_ *cobra.Command, _ []string) error {
 	// Initialize tokenizer
 	tokenizer, err := llama3.New()
 	if err != nil {

llama3/cmd/llama3/stream.go

@@ -64,10 +64,10 @@ Input is read from stdin only.`,
 	return cmd
 }
 
-func runStream(_ *cobra.Command, args []string) error {
+func runStream(_ *cobra.Command, _ []string) error {
 
 	// Validate output format
-	if streamOutput != "space" && streamOutput != "newline" {
+	if streamOutput != outputFormatSpace && streamOutput != outputFormatNewline {
 		return fmt.Errorf("invalid output format %q: must be 'space' or 'newline'", streamOutput)
 	}
 
@@ -98,7 +98,7 @@ func runStream(_ *cobra.Command, args []string) error {
 		switch streamOutput {
 		case outputFormatNewline:
 			fmt.Println(token)
-		case "space":
+		case outputFormatSpace:
 			if !first {
 				fmt.Print(" ")
 			}

llama3/cmd/tools/generate-vectors/main.go

@@ -2,13 +2,15 @@
 package main
 
 import (
+	"context"
 	"encoding/json"
 	"flag"
 	"fmt"
 	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"time"
 )
 
 func main() {
@@ -54,8 +56,10 @@ inputs.forEach(input => {
 		}
 	}()
 
-	// Run the script
-	cmd := exec.Command("node", tmpFile) // #nosec G204 - tmpFile is safely constructed
+	// Run the script with timeout
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	cmd := exec.CommandContext(ctx, "node", tmpFile) // #nosec G204 - tmpFile is safely constructed
 	outputBytes, err := cmd.Output()
 	if err != nil {
 		log.Fatalf("Failed to run JS script: %v", err)

llama3/constants.go

@@ -9,24 +9,6 @@ const (
 	totalVocabSize    = baseVocabSize + specialTokenCount
 )
 
-// Token IDs for special tokens.
-const (
-	beginOfTextTokenID = 128000
-	endOfTextTokenID   = 128001
-)
-
-// Pre-tokenization limits.
-const (
-	maxNumberLength = 3 // Maximum consecutive digits in a single token
-)
-
-// Pool configuration.
-const (
-	defaultStateMachineTokenCapacity = 32   // Initial capacity for state machine tokens
-	defaultTokenBufferCapacity       = 64   // Initial capacity for token buffers
-	maxPooledTokenBufferCapacity     = 1024 // Maximum capacity for pooled token buffers
-)
-
 // Cache configuration.
 const (
 	defaultCacheSize = 0 // 0 means unlimited
@@ -38,22 +20,6 @@ const (
 	bytesPerMerge               = 3 // Number of bytes to read for each merge
 )
 
-// Merge data configuration.
-const (
-	bitsPerMergeID = 17 // Number of bits used to encode each merge ID
-)
-
-// Character mappings for byte-level encoding.
-const (
-	asciiPrintableStart = '!' // First printable ASCII character
-	asciiPrintableEnd   = '~' // Last printable ASCII character
-	extendedASCIIStart1 = '¡' // First extended ASCII range start
-	extendedASCIIEnd1   = '¬' // First extended ASCII range end
-	extendedASCIIStart2 = '®' // Second extended ASCII range start
-	extendedASCIIEnd2   = 'ÿ' // Second extended ASCII range end
-	unicodeOffset       = 256 // Offset for mapping non-printable bytes
-)
-
 // Special token constants.
 const (
 	beginOfTextToken = "<|begin_of_text|>"

llama3/internal/pretokenizer/state_machine.go

@@ -68,7 +68,7 @@ func Tokenize(text string) []string {
 
 	// Return token buffer to pool
 	if cap(sm.tokens) <= 1024 {
-		tokenBufPool.Put(sm.tokens[:0]) // #nosec - slice header is small, not worth pointer optimization
+		tokenBufPool.Put(sm.tokens[:0]) //nolint:staticcheck // slice header is small, not worth pointer optimization
 	}
 
 	// Return state machine to pool
@@ -78,25 +78,6 @@ func Tokenize(text string) []string {
 	return result
 }
 
-// newStateMachine creates a new state machine for tokenizing the given text.
-// This function is primarily used for testing. In production, use Tokenize()
-// which uses pooled state machines for better performance.
-func newStateMachine(text string) *stateMachine {
-	return &stateMachine{
-		input:    []rune(text),
-		position: 0,
-		tokens:   make([]string, 0),
-	}
-}
-
-// tokenizeWithStateMachine processes the input according to the JS regex pattern.
-func (sm *stateMachine) tokenizeWithStateMachine() []string {
-	for sm.position < len(sm.input) {
-		sm.matchNext()
-	}
-	return sm.tokens
-}
-
 // matchNext tries to match the next token according to the pattern.
 func (sm *stateMachine) matchNext() {
 	if sm.position >= len(sm.input) {

llama3/internal/testing/vectors.go

@@ -1,3 +1,4 @@
+// Package testing provides test vector generation utilities.
 package testing
 
 import (
@@ -15,7 +16,7 @@ type TestCase struct {
 
 // GenerateTestCases creates comprehensive test cases.
 func GenerateTestCases() []TestCase {
-	var cases []TestCase
+	cases := make([]TestCase, 0, 200) // Preallocate for typical test case count
 
 	// Edge cases
 	cases = append(cases, []TestCase{