diff --git a/go.mod b/go.mod index 652e949..3a9107a 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,9 @@ module github.com/transparency-dev/merkle -go 1.22.7 +go 1.24.0 -require github.com/google/go-cmp v0.7.0 +require ( + github.com/google/go-cmp v0.7.0 + github.com/transparency-dev/formats v0.0.0-20251017110053-404c0d5b696c + golang.org/x/mod v0.31.0 +) diff --git a/go.sum b/go.sum index 40e761a..1185633 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,6 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/transparency-dev/formats v0.0.0-20251017110053-404c0d5b696c h1:5a2XDQ2LiAUV+/RjckMyq9sXudfrPSuCY4FuPC1NyAw= +github.com/transparency-dev/formats v0.0.0-20251017110053-404c0d5b696c/go.mod h1:g85IafeFJZLxlzZCDRu4JLpfS7HKzR+Hw9qRh3bVzDI= +golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI= +golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg= diff --git a/proof/tlog_proof.go b/proof/tlog_proof.go new file mode 100644 index 0000000..7ab72a6 --- /dev/null +++ b/proof/tlog_proof.go @@ -0,0 +1,138 @@ +// Copyright 2025 The Tessera authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package proof + +import ( + "bufio" + "bytes" + "crypto/sha256" + "encoding/base64" + "fmt" + "strconv" + "strings" + + "github.com/transparency-dev/formats/log" + "github.com/transparency-dev/merkle/rfc6962" + "github.com/transparency-dev/merkle/witness" + "golang.org/x/mod/sumdb/note" +) + +// NewTLogProof creates a transparency log proof for a given index, inclusion proof +// and signed checkpoint. +// The format of the returned proof is described at https://c2sp.org/tlog-proof +func NewTLogProof(index uint64, hashes [][sha256.Size]byte, checkpoint []byte) []byte { + return buildTLogProof(index, hashes, checkpoint, nil) +} + +// NewTLogProofWithExtra creates a transparency log proof for a given index, inclusion proof +// and signed checkpoint, along with opaque extra data. +// The format of the returned proof is described at https://c2sp.org/tlog-proof +func NewTLogProofWithExtra(index uint64, hashes [][sha256.Size]byte, checkpoint []byte, extraData []byte) []byte { + return buildTLogProof(index, hashes, checkpoint, extraData) +} + +func buildTLogProof(index uint64, hashes [][sha256.Size]byte, checkpoint []byte, extraData []byte) []byte { + var proof bytes.Buffer + proof.WriteString("c2sp.org/tlog-proof@v1\n") + if extraData != nil { + proof.WriteString("extra ") + fmt.Fprintf(&proof, "%s\n", base64.StdEncoding.EncodeToString(extraData)) + } + fmt.Fprintf(&proof, "index %d\n", index) + for _, h := range hashes { + fmt.Fprintf(&proof, "%s\n", base64.StdEncoding.EncodeToString(h[:])) + } + proof.WriteRune('\n') + proof.Write(checkpoint) + return proof.Bytes() +} + +// VerifyTLogProof verifies a c2sp.org/tlog-proof formatted proof for a given leaf hash. The proof must contain +// a valid inclusion proof for a given leaf hash and a signed checkpoint for a given origin, verified by +// the given log verifier and optionally a witness policy. +func VerifyTLogProof(proof, leafHash []byte, logOrigin string, logVerifier note.Verifier, witnessPolicy []byte) (uint64, []byte, error) { + var err error + b := bufio.NewScanner(bytes.NewReader(proof)) + + if b.Scan(); b.Text() != "c2sp.org/tlog-proof@v1" { + return 0, nil, fmt.Errorf("tlog proof missing expected header") + } + + // Handle optional extra line + var extra []byte + if b.Scan(); strings.HasPrefix(b.Text(), "extra ") { + e, _ := strings.CutPrefix(b.Text(), "extra ") + extra, err = base64.StdEncoding.DecodeString(e) + if err != nil { + return 0, nil, fmt.Errorf("tlog proof extra data not base64 encoded: %w", err) + } + b.Scan() + } + + var idx uint64 + idxStr, ok := strings.CutPrefix(b.Text(), "index ") + if !ok { + return 0, nil, fmt.Errorf("tlog proof missing required index") + } + idx, err = strconv.ParseUint(idxStr, 10, 64) + if err != nil { + return 0, nil, fmt.Errorf("tlog proof index not a valid uint64: %w", err) + } + + var hashes [][]byte + for b.Scan() { + if b.Text() == "" { + break + } + hash, err := base64.StdEncoding.DecodeString(b.Text()) + if err != nil { + return 0, nil, fmt.Errorf("tlog proof hash not base64 encoded: %w", err) + } + if len(hash) != sha256.Size { + return 0, nil, fmt.Errorf("tlog proof hash length was %d, expected %d", len(hash), sha256.Size) + } + hashes = append(hashes, hash) + } + + var checkpoint []byte + for b.Scan() { + checkpoint = append(checkpoint, b.Bytes()...) + checkpoint = append(checkpoint, '\n') + } + + // Verify checkpoint + verifiedCkpt, _, _, err := log.ParseCheckpoint(checkpoint, logOrigin, logVerifier) + if err != nil { + return 0, nil, fmt.Errorf("tlog proof checkpoint could not be verified: %w", err) + } + + // Verify witness signatures + if witnessPolicy != nil { + wg, err := witness.NewWitnessGroupFromPolicy(witnessPolicy) + if err != nil { + return 0, nil, fmt.Errorf("invalid witness policy: %w", err) + } + if !wg.Satisfied(checkpoint) { + return 0, nil, fmt.Errorf("tlog proof checkpoint could not be verified by witness policy") + } + } + + // Verify inclusion proof + if err := VerifyInclusion(rfc6962.DefaultHasher, idx, verifiedCkpt.Size, leafHash, hashes, verifiedCkpt.Hash); err != nil { + return 0, nil, fmt.Errorf("tlog proof inclusion proof not verifiable: %w", err) + } + + return idx, extra, nil +} diff --git a/proof/tlog_proof_test.go b/proof/tlog_proof_test.go new file mode 100644 index 0000000..fe2bb4f --- /dev/null +++ b/proof/tlog_proof_test.go @@ -0,0 +1,243 @@ +// Copyright 2025 The Tessera authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package proof + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "strings" + "testing" + + "github.com/transparency-dev/formats/log" + "golang.org/x/mod/sumdb/note" +) + +func TestNewTLogProof(t *testing.T) { + tests := []struct { + name string + index uint64 + hashes [][sha256.Size]byte + checkpoint []byte + extraData []byte + wantExtra bool + wantIndexStr string + wantCheckpoint string + }{ + { + name: "proof without extra data", + index: 5, + hashes: [][sha256.Size]byte{sha256.Sum256([]byte("hash1")), sha256.Sum256([]byte("hash2"))}, + checkpoint: []byte("test checkpoint\n"), + extraData: nil, + wantExtra: false, + wantIndexStr: "index 5\n", + wantCheckpoint: "test checkpoint", + }, + { + name: "proof with extra data", + index: 10, + hashes: [][sha256.Size]byte{sha256.Sum256([]byte("hash1"))}, + checkpoint: []byte("checkpoint data\n"), + extraData: []byte("extra information"), + wantExtra: true, + wantIndexStr: "index 10\n", + wantCheckpoint: "checkpoint data", + }, + { + name: "proof with empty hashes", + index: 0, + hashes: [][sha256.Size]byte{}, + checkpoint: []byte("checkpoint\n"), + extraData: nil, + wantExtra: false, + wantIndexStr: "index 0\n", + wantCheckpoint: "checkpoint", + }, + { + name: "proof with multiple hashes", + index: 15, + hashes: [][sha256.Size]byte{ + sha256.Sum256([]byte("hash1")), + sha256.Sum256([]byte("hash2")), + sha256.Sum256([]byte("hash3")), + }, + checkpoint: []byte("multi-hash checkpoint\n"), + extraData: nil, + wantExtra: false, + wantIndexStr: "index 15\n", + wantCheckpoint: "multi-hash checkpoint", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var proof []byte + if tt.extraData != nil { + proof = NewTLogProofWithExtra(tt.index, tt.hashes, tt.checkpoint, tt.extraData) + } else { + proof = NewTLogProof(tt.index, tt.hashes, tt.checkpoint) + } + + proofStr := string(proof) + + if !strings.HasPrefix(proofStr, "c2sp.org/tlog-proof@v1\n") { + t.Error("proof missing expected header") + } + + if tt.wantExtra && !strings.Contains(proofStr, "extra ") { + t.Error("proof missing extra data line") + } + + if !tt.wantExtra && strings.Contains(proofStr, "extra ") { + t.Error("proof should not contain extra data line") + } + + if !strings.Contains(proofStr, tt.wantIndexStr) { + t.Errorf("proof missing correct index string: want %q", tt.wantIndexStr) + } + + if !strings.Contains(proofStr, tt.wantCheckpoint) { + t.Errorf("proof missing checkpoint: want %q", tt.wantCheckpoint) + } + + // Verify all hashes are encoded + for i, h := range tt.hashes { + encoded := base64.StdEncoding.EncodeToString(h[:]) + if !strings.Contains(proofStr, encoded) { + t.Errorf("proof missing hash %d: %s", i, encoded) + } + } + + // Verify extra data encoding if present + if tt.extraData != nil { + expectedExtra := base64.StdEncoding.EncodeToString(tt.extraData) + if !strings.Contains(proofStr, expectedExtra) { + t.Error("proof missing encoded extra data") + } + } + }) + } +} + +func TestVerifyTLogProofErrors(t *testing.T) { + tests := []struct { + name string + proof []byte + wantErrSubstr string + }{ + { + name: "missing header", + proof: []byte("wrong-header\nindex 0\n\ncheckpoint\n"), + wantErrSubstr: "missing expected header", + }, + { + name: "invalid extra data encoding", + proof: []byte("c2sp.org/tlog-proof@v1\nextra !!notbase64!!\nindex 0\n\ncheckpoint\n"), + wantErrSubstr: "extra data not base64 encoded", + }, + { + name: "missing index", + proof: []byte("c2sp.org/tlog-proof@v1\n\n\ncheckpoint\n"), + wantErrSubstr: "missing required index", + }, + { + name: "invalid index - not a number", + proof: []byte("c2sp.org/tlog-proof@v1\nindex notanumber\n\ncheckpoint\n"), + wantErrSubstr: "not a valid uint64", + }, + { + name: "invalid index - negative", + proof: []byte("c2sp.org/tlog-proof@v1\nindex -5\n\ncheckpoint\n"), + wantErrSubstr: "not a valid uint64", + }, + { + name: "invalid hash base64", + proof: []byte("c2sp.org/tlog-proof@v1\nindex 0\n!!notbase64!!\n\ncheckpoint\n"), + wantErrSubstr: "hash not base64 encoded", + }, + { + name: "hash too long", + proof: []byte("c2sp.org/tlog-proof@v1\nindex 0\n" + + base64.StdEncoding.EncodeToString(make([]byte, 64)) + "\n\ncheckpoint\n"), + wantErrSubstr: "hash length", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, _, err := VerifyTLogProof(tt.proof, nil, "", nil, nil) + + if err == nil { + t.Fatal("expected error but got none") + } + + if !strings.Contains(err.Error(), tt.wantErrSubstr) { + t.Errorf("error message doesn't contain %q, got: %v", tt.wantErrSubstr, err) + } + }) + } +} + +func TestVerifyTLogProof(t *testing.T) { + origin := "test" + skey, vkey, err := note.GenerateKey(rand.Reader, origin) + if err != nil { + t.Fatalf("unexpected error creating key: %v", err) + } + signer, err := note.NewSigner(skey) + if err != nil { + t.Fatalf("unexpected error creating signer: %v", err) + } + verifier, err := note.NewVerifier(vkey) + if err != nil { + t.Fatalf("unexpected error creating verifier: %v", err) + } + + witnessPolicy := []byte("") + + checkpoint := createSignedCheckpoint(t, signer, 10, []byte("roothash")) + + extraData := []byte("test extra data") + hash := sha256.Sum256([]byte("leaf")) + + proof := NewTLogProofWithExtra(0, [][sha256.Size]byte{}, checkpoint, extraData) + + // This will fail at checkpoint verification stage + // TODO: Provide valid proof + _, _, err = VerifyTLogProof(proof, hash[:], origin, verifier, witnessPolicy) + if err == nil { + t.Errorf("expected verification to fail, but it passed") + } +} + +// Helper function to create a signed checkpoint +func createSignedCheckpoint(t *testing.T, signer note.Signer, size uint64, hash []byte) []byte { + t.Helper() + + checkpoint := log.Checkpoint{ + Origin: "test", + Size: size, + Hash: hash, + } + + checkpointBytes := checkpoint.Marshal() + signed, err := note.Sign(¬e.Note{Text: string(checkpointBytes)}, signer) + if err != nil { + t.Fatalf("failed to sign checkpoint: %v", err) + } + + return signed +} diff --git a/witness/witness.go b/witness/witness.go new file mode 100644 index 0000000..2da89db --- /dev/null +++ b/witness/witness.go @@ -0,0 +1,292 @@ +// Copyright 2025 The Tessera authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package witness + +import ( + "bufio" + "bytes" + "fmt" + "net/url" + "strconv" + "strings" + + "maps" + + f_note "github.com/transparency-dev/formats/note" + "golang.org/x/mod/sumdb/note" +) + +// policyComponent describes a component that makes up a policy. This is either a +// single Witness, or a WitnessGroup. +type policyComponent interface { + // Satisfied returns true if the checkpoint is signed by the quorum of + // witnesses involved in this policy component. + Satisfied(cp []byte) bool + + // Endpoints returns the details required for updating a witness and checking the + // response. The returned result is a map from the URL that should be used to update + // the witness with a new checkpoint, to the value which is the verifier to check + // the response is well formed. + Endpoints() map[string]note.Verifier +} + +// NewWitnessGroupFromPolicy creates a graph of witness objects that represents the +// policy provided, and which can be passed directly to the WithWitnesses +// appender lifecycle option. +// +// The policy structure is as described by [Sigsum's policy format](https://git.glasklar.is/sigsum/core/sigsum-go/-/blob/main/doc/policy.md) +// but with the difference that the configured witness keys MUST be signature type `0x04` `vkey`s as specified +// by C2SP [signed-note](https://github.com/C2SP/C2SP/blob/main/signed-note.md#verifier-keys). +func NewWitnessGroupFromPolicy(p []byte) (WitnessGroup, error) { + scanner := bufio.NewScanner(bytes.NewBuffer(p)) + components := make(map[string]policyComponent) + + var quorumName string + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if i := strings.Index(line, "#"); i >= 0 { + line = line[:i] + } + if line == "" { + continue + } + + switch fields := strings.Fields(line); fields[0] { + case "log": + // This keyword is important to clients who might use the policy file, but we don't need to know about it since + // we _are_ the log, so just ignore it. + case "witness": + // Strictly, the URL is optional so policy files can be used client-side, where they don't care about the URL. + // Given this function is parsing to create the graph structure which will be used by a Tessera log to witness + // new checkpoints we'll ignore that special case here. + if len(fields) != 4 { + return WitnessGroup{}, fmt.Errorf("invalid witness definition: %q", line) + } + name, vkey, witnessURLStr := fields[1], fields[2], fields[3] + if isBadName(name) { + return WitnessGroup{}, fmt.Errorf("invalid witness name %q", name) + } + if _, ok := components[name]; ok { + return WitnessGroup{}, fmt.Errorf("duplicate component name: %q", name) + } + witnessURL, err := url.Parse(witnessURLStr) + if err != nil { + return WitnessGroup{}, fmt.Errorf("invalid witness URL %q: %w", witnessURLStr, err) + } + w, err := NewWitness(vkey, witnessURL) + if err != nil { + return WitnessGroup{}, fmt.Errorf("invalid witness config %q: %w", line, err) + } + components[name] = w + case "group": + if len(fields) < 3 { + return WitnessGroup{}, fmt.Errorf("invalid group definition: %q", line) + } + + name, N, childrenNames := fields[1], fields[2], fields[3:] + if isBadName(name) { + return WitnessGroup{}, fmt.Errorf("invalid group name %q", name) + } + if _, ok := components[name]; ok { + return WitnessGroup{}, fmt.Errorf("duplicate component name: %q", name) + } + var n int + switch N { + case "any": + n = 1 + case "all": + n = len(childrenNames) + default: + i, err := strconv.ParseUint(N, 10, 8) + if err != nil { + return WitnessGroup{}, fmt.Errorf("invalid threshold %q for group %q: %w", N, name, err) + } + n = int(i) + } + if c := len(childrenNames); n > c { + return WitnessGroup{}, fmt.Errorf("group with %d children cannot have threshold %d", c, n) + } + + children := make([]policyComponent, len(childrenNames)) + for i, cName := range childrenNames { + if isBadName(cName) { + return WitnessGroup{}, fmt.Errorf("invalid component name %q", cName) + } + child, ok := components[cName] + if !ok { + return WitnessGroup{}, fmt.Errorf("unknown component %q in group definition", cName) + } + children[i] = child + } + wg := NewWitnessGroup(n, children...) + components[name] = wg + case "quorum": + if len(fields) != 2 { + return WitnessGroup{}, fmt.Errorf("invalid quorum definition: %q", line) + } + quorumName = fields[1] + default: + return WitnessGroup{}, fmt.Errorf("unknown keyword: %q", fields[0]) + } + } + if err := scanner.Err(); err != nil { + return WitnessGroup{}, err + } + + switch quorumName { + case "": + return WitnessGroup{}, fmt.Errorf("policy file must define a quorum") + case "none": + return NewWitnessGroup(0), nil + default: + if isBadName(quorumName) { + return WitnessGroup{}, fmt.Errorf("invalid quorum name %q", quorumName) + } + policy, ok := components[quorumName] + if !ok { + return WitnessGroup{}, fmt.Errorf("quorum component %q not found", quorumName) + } + wg, ok := policy.(WitnessGroup) + if !ok { + // A single witness can be a policy. Wrap it in a group. + return NewWitnessGroup(1, policy), nil + } + return wg, nil + } +} + +var keywords = map[string]struct{}{ + "witness": {}, + "group": {}, + "any": {}, + "all": {}, + "none": {}, + "quorum": {}, + "log": {}, +} + +func isBadName(n string) bool { + _, isKeyword := keywords[n] + return isKeyword +} + +// NewWitness returns a Witness given a verifier key and the root URL for where this +// witness can be reached. +func NewWitness(vkey string, witnessRoot *url.URL) (Witness, error) { + v, err := f_note.NewVerifierForCosignatureV1(vkey) + if err != nil { + return Witness{}, err + } + + u := witnessRoot.JoinPath("/add-checkpoint") + + return Witness{ + Key: v, + URL: u.String(), + }, err +} + +// Witness represents a single witness that can be reached in order to perform a witnessing operation. +// The URLs() method returns the URL where it can be reached for witnessing, and the Satisfied method +// provides a predicate to check whether this witness has signed a checkpoint. +type Witness struct { + Key note.Verifier + URL string +} + +// Satisfied returns true if the checkpoint provided is signed by this witness. +// This will return false if there is no signature, and also if the +// checkpoint cannot be read as a valid note. It is up to the caller to ensure +// that the input value represents a valid note. +func (w Witness) Satisfied(cp []byte) bool { + n, err := note.Open(cp, note.VerifierList(w.Key)) + if err != nil { + return false + } + return len(n.Sigs) == 1 +} + +// Endpoints returns the details required for updating a witness and checking the +// response. The returned result is a map from the URL that should be used to update +// the witness with a new checkpoint, to the value which is the verifier to check +// the response is well formed. +func (w Witness) Endpoints() map[string]note.Verifier { + return map[string]note.Verifier{w.URL: w.Key} +} + +// NewWitnessGroup creates a grouping of Witness or WitnessGroup with a configurable threshold +// of these sub-components that need to be satisfied in order for this group to be satisfied. +// +// The threshold should only be set to less than the number of sub-components if these are +// considered fungible. +func NewWitnessGroup(n int, children ...policyComponent) WitnessGroup { + if n < 0 || n > len(children) { + panic(fmt.Errorf("threshold of %d outside bounds for children %s", n, children)) + } + return WitnessGroup{ + Components: children, + N: n, + } +} + +// WitnessGroup defines a group of witnesses, and a threshold of +// signatures that must be met for this group to be satisfied. +// Witnesses within a group should be fungible, e.g. all of the Armored +// Witness devices form a logical group, and N should be picked to +// represent a threshold of the quorum. For some users this will be a +// simple majority, but other strategies are available. +// N must be <= len(WitnessKeys). +type WitnessGroup struct { + Components []policyComponent + N int +} + +// Satisfied returns true if the checkpoint provided has sufficient signatures +// from the witnesses in this group to satisfy the threshold. +// This will return false if there are insufficient signatures, and also if the +// checkpoint cannot be read as a valid note. It is up to the caller to ensure +// that the input value represents a valid note. +// +// The implementation of this requires every witness in the group to verify the +// checkpoint, which is O(N). If this is called every time a witness returns a +// checkpoint then this algorithm is O(N^2). To support large N, this may require +// some rewriting in order to maintain performance. +func (wg WitnessGroup) Satisfied(cp []byte) bool { + if wg.N <= 0 { + return true + } + satisfaction := 0 + for _, c := range wg.Components { + if c.Satisfied(cp) { + satisfaction++ + } + if satisfaction >= wg.N { + return true + } + } + return false +} + +// Endpoints returns the details required for updating a witness and checking the +// response. The returned result is a map from the URL that should be used to update +// the witness with a new checkpoint, to the value which is the verifier to check +// the response is well formed. +func (wg WitnessGroup) Endpoints() map[string]note.Verifier { + endpoints := make(map[string]note.Verifier) + for _, c := range wg.Components { + maps.Copy(endpoints, c.Endpoints()) + } + return endpoints +}