perf: reduce memory and network cost for checkpoint metadata

cmd/entire/cli/checkpoint/blob_resolver.go

@@ -0,0 +1,126 @@
+package checkpoint
+
+import (
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+
+	"github.com/entireio/cli/cmd/entire/cli/agent"
+	"github.com/entireio/cli/cmd/entire/cli/checkpoint/id"
+	"github.com/entireio/cli/cmd/entire/cli/paths"
+
+	"github.com/go-git/go-git/v6/plumbing"
+	"github.com/go-git/go-git/v6/plumbing/object"
+	"github.com/go-git/go-git/v6/plumbing/storer"
+)
+
+// TranscriptBlobRef identifies a blob within a checkpoint tree on the metadata branch.
+// It captures the blob hash from the tree entry without requiring the blob itself to be local.
+type TranscriptBlobRef struct {
+	// SessionIndex is the 0-based session index within the checkpoint.
+	SessionIndex int
+
+	// Hash is the blob's SHA-1 hash from the tree entry.
+	Hash plumbing.Hash
+
+	// Path is the blob's path relative to the checkpoint directory,
+	// e.g. "0/full.jsonl" or "0/full.jsonl.001".
+	Path string
+}
+
+// BlobResolver checks blob existence and reads blobs from go-git's local
+// object store (loose objects + packfiles). It performs no remote operations.
+type BlobResolver struct {
+	storer storer.EncodedObjectStorer
+}
+
+// NewBlobResolver creates a BlobResolver backed by the given object store.
+func NewBlobResolver(s storer.EncodedObjectStorer) *BlobResolver {
+	return &BlobResolver{storer: s}
+}
+
+// HasBlob returns true if the blob exists in the local object store.
+// Checks both loose objects and packfile indices without reading blob content.
+func (r *BlobResolver) HasBlob(hash plumbing.Hash) bool {
+	return r.storer.HasEncodedObject(hash) == nil
+}
+
+// ReadBlob reads a blob's content from the local object store.
+// Returns plumbing.ErrObjectNotFound if the blob is not present locally.
+func (r *BlobResolver) ReadBlob(hash plumbing.Hash) ([]byte, error) {
+	obj, err := r.storer.EncodedObject(plumbing.BlobObject, hash)
+	if err != nil {
+		return nil, err //nolint:wrapcheck // Propagating plumbing.ErrObjectNotFound
+	}
+
+	reader, err := obj.Reader()
+	if err != nil {
+		return nil, fmt.Errorf("blob reader %s: %w", hash, err)
+	}
+	defer reader.Close()
+
+	data, err := io.ReadAll(reader)
+	if err != nil {
+		return nil, fmt.Errorf("read blob %s: %w", hash, err)
+	}
+	return data, nil
+}
+
+// CollectTranscriptBlobHashes walks the metadata branch tree for a checkpoint
+// and returns blob hashes for all transcript files (full.jsonl and chunks)
+// across all sessions. Only reads tree objects — works after a treeless fetch
+// where blobs have not been downloaded.
+//
+// The function navigates the sharded checkpoint directory structure:
+//
+//	<id[:2]>/<id[2:]>/
+//	├── 0/
+//	│   ├── full.jsonl          ← collected
+//	│   ├── full.jsonl.001      ← collected (chunk)
+//	│   └── metadata.json
+//	├── 1/
+//	│   └── full.jsonl          ← collected
+//	└── metadata.json
+func CollectTranscriptBlobHashes(tree *object.Tree, checkpointID id.CheckpointID) ([]TranscriptBlobRef, error) {
+	checkpointTree, err := tree.Tree(checkpointID.Path())
+	if err != nil {
+		return nil, fmt.Errorf("checkpoint tree %s: %w", checkpointID.Path(), err)
+	}
+
+	var refs []TranscriptBlobRef
+
+	// Enumerate session subdirectories (0, 1, 2, ...)
+	for i := 0; ; i++ {
+		sessionDir := strconv.Itoa(i)
+		sessionTree, treeErr := checkpointTree.Tree(sessionDir)
+		if treeErr != nil {
+			break // no more sessions
+		}
+
+		// Collect transcript blob hashes from tree entries.
+		// tree.Entries contains the direct children — no blob reads needed.
+		for _, entry := range sessionTree.Entries {
+			if entry.Name == paths.TranscriptFileName || entry.Name == paths.TranscriptFileNameLegacy {
+				refs = append(refs, TranscriptBlobRef{
+					SessionIndex: i,
+					Hash:         entry.Hash,
+					Path:         sessionDir + "/" + entry.Name,
+				})
+			}
+			// Check for chunk files (full.jsonl.001, full.jsonl.002, etc.)
+			if strings.HasPrefix(entry.Name, paths.TranscriptFileName+".") {
+				idx := agent.ParseChunkIndex(entry.Name, paths.TranscriptFileName)
+				if idx > 0 {
+					refs = append(refs, TranscriptBlobRef{
+						SessionIndex: i,
+						Hash:         entry.Hash,
+						Path:         sessionDir + "/" + entry.Name,
+					})
+				}
+			}
+		}
+	}
+
+	return refs, nil //nolint:nilerr // treeErr from session enumeration loop is used to break, not propagated
+}

cmd/entire/cli/checkpoint/blob_resolver_test.go

@@ -0,0 +1,200 @@
+package checkpoint
+
+import (
+	"context"
+	"testing"
+
+	"github.com/entireio/cli/cmd/entire/cli/checkpoint/id"
+
+	"github.com/go-git/go-git/v6/plumbing"
+)
+
+func TestBlobResolver_HasBlob_Present(t *testing.T) {
+	t.Parallel()
+
+	repo, store, cpID := setupRepoForUpdate(t)
+
+	// Get the metadata branch tree
+	tree, err := store.getSessionsBranchTree()
+	if err != nil {
+		t.Fatalf("getSessionsBranchTree() error = %v", err)
+	}
+
+	// Navigate to the transcript blob via tree entries
+	refs, err := CollectTranscriptBlobHashes(tree, cpID)
+	if err != nil {
+		t.Fatalf("CollectTranscriptBlobHashes() error = %v", err)
+	}
+	if len(refs) == 0 {
+		t.Fatal("expected at least one transcript blob ref")
+	}
+
+	resolver := NewBlobResolver(repo.Storer)
+
+	// Blob should exist — it was written by WriteCommitted
+	if !resolver.HasBlob(refs[0].Hash) {
+		t.Errorf("HasBlob(%s) = false, want true (blob was written locally)", refs[0].Hash)
+	}
+}
+
+func TestBlobResolver_HasBlob_Missing(t *testing.T) {
+	t.Parallel()
+
+	repo, _, _ := setupRepoForUpdate(t)
+	resolver := NewBlobResolver(repo.Storer)
+
+	// Random hash that doesn't exist
+	fakeHash := plumbing.NewHash("deadbeefdeadbeefdeadbeefdeadbeefdeadbeef")
+	if resolver.HasBlob(fakeHash) {
+		t.Error("HasBlob(fake) = true, want false")
+	}
+}
+
+func TestBlobResolver_ReadBlob(t *testing.T) {
+	t.Parallel()
+
+	repo, store, cpID := setupRepoForUpdate(t)
+
+	tree, err := store.getSessionsBranchTree()
+	if err != nil {
+		t.Fatalf("getSessionsBranchTree() error = %v", err)
+	}
+
+	refs, err := CollectTranscriptBlobHashes(tree, cpID)
+	if err != nil {
+		t.Fatalf("CollectTranscriptBlobHashes() error = %v", err)
+	}
+	if len(refs) == 0 {
+		t.Fatal("expected at least one transcript blob ref")
+	}
+
+	resolver := NewBlobResolver(repo.Storer)
+
+	data, err := resolver.ReadBlob(refs[0].Hash)
+	if err != nil {
+		t.Fatalf("ReadBlob() error = %v", err)
+	}
+	if len(data) == 0 {
+		t.Error("ReadBlob() returned empty data")
+	}
+	// The transcript content from setupRepoForUpdate
+	if string(data) != "provisional transcript line 1\n" {
+		t.Errorf("ReadBlob() = %q, want %q", string(data), "provisional transcript line 1\n")
+	}
+}
+
+func TestBlobResolver_ReadBlob_Missing(t *testing.T) {
+	t.Parallel()
+
+	repo, _, _ := setupRepoForUpdate(t)
+	resolver := NewBlobResolver(repo.Storer)
+
+	fakeHash := plumbing.NewHash("deadbeefdeadbeefdeadbeefdeadbeefdeadbeef")
+	_, err := resolver.ReadBlob(fakeHash)
+	if err == nil {
+		t.Error("ReadBlob(fake) should return error")
+	}
+}
+
+func TestCollectTranscriptBlobHashes_SingleSession(t *testing.T) {
+	t.Parallel()
+
+	_, store, cpID := setupRepoForUpdate(t)
+
+	tree, err := store.getSessionsBranchTree()
+	if err != nil {
+		t.Fatalf("getSessionsBranchTree() error = %v", err)
+	}
+
+	refs, err := CollectTranscriptBlobHashes(tree, cpID)
+	if err != nil {
+		t.Fatalf("CollectTranscriptBlobHashes() error = %v", err)
+	}
+
+	if len(refs) != 1 {
+		t.Fatalf("expected 1 transcript ref, got %d", len(refs))
+	}
+
+	ref := refs[0]
+	if ref.SessionIndex != 0 {
+		t.Errorf("SessionIndex = %d, want 0", ref.SessionIndex)
+	}
+	if ref.Hash.IsZero() {
+		t.Error("Hash should not be zero")
+	}
+	if ref.Path != "0/full.jsonl" {
+		t.Errorf("Path = %q, want %q", ref.Path, "0/full.jsonl")
+	}
+}
+
+func TestCollectTranscriptBlobHashes_MultiSession(t *testing.T) {
+	t.Parallel()
+
+	repo, store, cpID := setupRepoForUpdate(t)
+
+	// Write a second session to the same checkpoint
+	err := store.WriteCommitted(context.Background(), WriteCommittedOptions{
+		CheckpointID: cpID,
+		SessionID:    "session-002",
+		Strategy:     "manual-commit",
+		Transcript:   []byte("second session transcript\n"),
+		Prompts:      []string{"second prompt"},
+		AuthorName:   "Test",
+		AuthorEmail:  "test@test.com",
+	})
+	if err != nil {
+		t.Fatalf("WriteCommitted() for second session error = %v", err)
+	}
+
+	tree, err := store.getSessionsBranchTree()
+	if err != nil {
+		t.Fatalf("getSessionsBranchTree() error = %v", err)
+	}
+
+	refs, err := CollectTranscriptBlobHashes(tree, cpID)
+	if err != nil {
+		t.Fatalf("CollectTranscriptBlobHashes() error = %v", err)
+	}
+
+	if len(refs) != 2 {
+		t.Fatalf("expected 2 transcript refs, got %d", len(refs))
+	}
+
+	// Verify session indices
+	if refs[0].SessionIndex != 0 {
+		t.Errorf("refs[0].SessionIndex = %d, want 0", refs[0].SessionIndex)
+	}
+	if refs[1].SessionIndex != 1 {
+		t.Errorf("refs[1].SessionIndex = %d, want 1", refs[1].SessionIndex)
+	}
+
+	// Verify they have different hashes (different transcript content)
+	if refs[0].Hash == refs[1].Hash {
+		t.Error("multi-session refs should have different blob hashes")
+	}
+
+	// Verify all blobs exist locally
+	resolver := NewBlobResolver(repo.Storer)
+	for i, ref := range refs {
+		if !resolver.HasBlob(ref.Hash) {
+			t.Errorf("session %d blob %s should be present locally", i, ref.Hash)
+		}
+	}
+}
+
+func TestCollectTranscriptBlobHashes_NonexistentCheckpoint(t *testing.T) {
+	t.Parallel()
+
+	_, store, _ := setupRepoForUpdate(t)
+
+	tree, err := store.getSessionsBranchTree()
+	if err != nil {
+		t.Fatalf("getSessionsBranchTree() error = %v", err)
+	}
+
+	fakeID := id.MustCheckpointID("ffffffffffff")
+	_, err = CollectTranscriptBlobHashes(tree, fakeID)
+	if err == nil {
+		t.Error("expected error for nonexistent checkpoint")
+	}
+}