diff --git a/go.mod b/go.mod index 05a2212..b96a5a8 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/transparency-dev/incubator go 1.25.0 require ( - filippo.io/torchwood v0.9.0 github.com/cockroachdb/pebble v1.1.5 github.com/go-git/go-git/v5 v5.19.0 github.com/google/go-cmp v0.7.0 @@ -14,6 +13,7 @@ require ( golang.org/x/mod v0.36.0 golang.org/x/sync v0.20.0 k8s.io/klog/v2 v2.140.0 + rsc.io/tmp/mpt v0.2.0 ) require ( @@ -31,7 +31,6 @@ require ( github.com/cockroachdb/redact v1.1.5 // indirect github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect github.com/cyphar/filepath-securejoin v0.6.1 // indirect - github.com/dustin/go-humanize v1.0.1 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/getsentry/sentry-go v0.27.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect @@ -42,7 +41,6 @@ require ( github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect @@ -50,16 +48,13 @@ require ( github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/ncruces/go-strftime v0.1.9 // indirect github.com/pjbgf/sha1cd v0.6.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.15.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect - github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/skeema/knownhosts v1.3.1 // indirect @@ -75,9 +70,4 @@ require ( golang.org/x/text v0.36.0 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect - modernc.org/libc v1.65.7 // indirect - modernc.org/mathutil v1.7.1 // indirect - modernc.org/memory v1.11.0 // indirect - modernc.org/sqlite v1.37.1 // indirect - zombiezen.com/go/sqlite v1.4.2 // indirect ) diff --git a/go.sum b/go.sum index fd512e7..1af5b6a 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,5 @@ dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= -filippo.io/torchwood v0.9.0 h1:2W156cI7K3MyxEyNTuS1C9lYEW7y1u7PoHLmvgNsiZc= -filippo.io/torchwood v0.9.0/go.mod h1:zOJguxdmaODUQScAvC80bV6N0SOA9U+bFZG1DwJU6N8= github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= @@ -41,8 +39,6 @@ github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= @@ -78,8 +74,6 @@ github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= @@ -101,12 +95,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= -github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= @@ -126,8 +116,6 @@ github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= @@ -193,7 +181,6 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -228,31 +215,5 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= -lukechampine.com/blake3 v1.4.1 h1:I3Smz7gso8w4/TunLKec6K2fn+kyKtDxr/xcQEN84Wg= -lukechampine.com/blake3 v1.4.1/go.mod h1:QFosUxmjB8mnrWFSNwKmvxHpfY72bmD2tQ0kBMM3kwo= -modernc.org/cc/v4 v4.26.1 h1:+X5NtzVBn0KgsBCBe+xkDC7twLb/jNVj9FPgiwSQO3s= -modernc.org/cc/v4 v4.26.1/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= -modernc.org/ccgo/v4 v4.28.0 h1:rjznn6WWehKq7dG4JtLRKxb52Ecv8OUGah8+Z/SfpNU= -modernc.org/ccgo/v4 v4.28.0/go.mod h1:JygV3+9AV6SmPhDasu4JgquwU81XAKLd3OKTUDNOiKE= -modernc.org/fileutil v1.3.1 h1:8vq5fe7jdtEvoCf3Zf9Nm0Q05sH6kGx0Op2CPx1wTC8= -modernc.org/fileutil v1.3.1/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc= -modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= -modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= -modernc.org/libc v1.65.7 h1:Ia9Z4yzZtWNtUIuiPuQ7Qf7kxYrxP1/jeHZzG8bFu00= -modernc.org/libc v1.65.7/go.mod h1:011EQibzzio/VX3ygj1qGFt5kMjP0lHb0qCW5/D/pQU= -modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= -modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= -modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= -modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= -modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= -modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= -modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= -modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= -modernc.org/sqlite v1.37.1 h1:EgHJK/FPoqC+q2YBXg7fUmES37pCHFc97sI7zSayBEs= -modernc.org/sqlite v1.37.1/go.mod h1:XwdRtsE1MpiBcL54+MbKcaDvcuej+IYSMfLN6gSKV8g= -modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= -modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= -modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= -modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= -zombiezen.com/go/sqlite v1.4.2 h1:KZXLrBuJ7tKNEm+VJcApLMeQbhmAUOKA5VWS93DfFRo= -zombiezen.com/go/sqlite v1.4.2/go.mod h1:5Kd4taTAD4MkBzT25mQ9uaAlLjyR0rFhsR6iINO70jc= +rsc.io/tmp/mpt v0.2.0 h1:EORjerXrh5u8VU0INROMFgclr2HVO2XzSx9j4wmsTak= +rsc.io/tmp/mpt v0.2.0/go.mod h1:aP1ILCaUW9I5rFbt2z0/8KYz/0FDaYfPW2zEArs+OIQ= diff --git a/vindex/api/api.go b/vindex/api/api.go index 8dce265..f2900b1 100644 --- a/vindex/api/api.go +++ b/vindex/api/api.go @@ -44,13 +44,6 @@ type LookupResponse struct { // These values represent the lookup operation in the index at the root hash // committed to by OutputLogLeaf. The values contain all indices for the given // key, and the proof binds these values at this key at the index root hash. - IndexValue []uint64 `json:"index_value"` - IndexProof []IndexNode `json:"index_proof"` -} - -// IndexNode is a node in the Verifiable Index. -type IndexNode struct { - LabelBitLen uint32 `json:"label_bit_len"` - LabelPath []byte `json:"label_path"` - Hash [sha256.Size]byte `json:"hash"` + IndexValue []uint64 `json:"index_value"` + IndexProof []byte `json:"index_proof"` } diff --git a/vindex/client/client.go b/vindex/client/client.go index 4e9821d..2b28b8f 100644 --- a/vindex/client/client.go +++ b/vindex/client/client.go @@ -16,6 +16,7 @@ package client import ( + "bytes" "context" "crypto/sha256" "encoding/binary" @@ -27,7 +28,6 @@ import ( "net/http" "net/url" - "filippo.io/torchwood/prefix" "github.com/transparency-dev/formats/log" "github.com/transparency-dev/incubator/vindex" "github.com/transparency-dev/incubator/vindex/api" @@ -37,15 +37,16 @@ import ( "github.com/transparency-dev/tessera/client" "golang.org/x/mod/sumdb/note" "k8s.io/klog/v2" + "rsc.io/tmp/mpt" ) -func VerifyLookupResponse(keyHash [sha256.Size]byte, resp api.LookupResponse, outV note.Verifier) ([]uint64, []byte, error) { +func VerifyLookupResponse(keyHash [sha256.Size]byte, resp api.LookupResponse, inV, outV note.Verifier) ([]uint64, []byte, error) { // Currently the response contains the RFC6962 style response type; leaf, proof, etc. // What if we flip this all around, and the OutputLog part of the response // only returns an index into the output log, and the client has to look up // that leaf, checkpoint, and generate inclusion proof? - cp, _, _, err := log.ParseCheckpoint(resp.OutputLogCP, outV.Name(), outV) + olcp, _, _, err := log.ParseCheckpoint(resp.OutputLogCP, outV.Name(), outV) if err != nil { return nil, nil, fmt.Errorf("failed to parse output log checkpoint: %v", err) } @@ -54,8 +55,8 @@ func VerifyLookupResponse(keyHash [sha256.Size]byte, resp api.LookupResponse, ou for i := range olp { olp[i] = resp.OutputLogProof[i][:] } - oli := cp.Size - 1 // TODO(mhutchinson): include this in the response? - if err := proof.VerifyInclusion(rfc6962.DefaultHasher, oli, cp.Size, outLeafHash[:], olp, cp.Hash); err != nil { + oli := olcp.Size - 1 // TODO(mhutchinson): include this in the response? + if err := proof.VerifyInclusion(rfc6962.DefaultHasher, oli, olcp.Size, outLeafHash[:], olp, olcp.Hash); err != nil { return nil, nil, fmt.Errorf("failed to verify inclusion in output log: %v", err) } @@ -72,26 +73,26 @@ func VerifyLookupResponse(keyHash [sha256.Size]byte, resp api.LookupResponse, ou } vindexLeafHash := idxLeafHash.Sum(nil) - pns := make([]prefix.ProofNode, len(resp.IndexProof)) - for i, p := range resp.IndexProof { - label, err := prefix.NewLabel(p.LabelBitLen, p.LabelPath) - if err != nil { - return nil, nil, fmt.Errorf("failed to create label: %v", err) - } - pns[i] = prefix.ProofNode{ - Label: label, - Hash: p.Hash, - } + origin := inV.Name() + if origin == "sum.golang.org" { + origin = "go.sum database tree" + } + ilcp, _, _, err := log.ParseCheckpoint(inCp, origin, inV) + if err != nil { + return nil, nil, fmt.Errorf("failed to parse input log checkpoint: %v", err) } - if len(resp.IndexValue) > 0 { - if err := prefix.VerifyMembershipProof(sha256.Sum256, keyHash, [32]byte(vindexLeafHash), pns, mapRoot); err != nil { - return nil, nil, fmt.Errorf("failed to verify membership: %v", err) - } - } else { - if err := prefix.VerifyNonMembershipProof(sha256.Sum256, keyHash, pns, mapRoot); err != nil { - return nil, nil, fmt.Errorf("failed to verify non-membership: %v", err) - } + snap := mpt.Snapshot{ + Version: int64(ilcp.Size), + Hash: mapRoot, + } + kvHash, ok, err := mpt.Verify(snap, keyHash, resp.IndexProof) + if err != nil { + return nil, nil, fmt.Errorf("mpt.Verify(): %v", err) + } + + if ok && !bytes.Equal(kvHash[:], vindexLeafHash) { + return nil, nil, fmt.Errorf("failed to verify membership: %x != %x", kvHash, vindexLeafHash) } return resp.IndexValue, inCp, nil @@ -100,7 +101,7 @@ func VerifyLookupResponse(keyHash [sha256.Size]byte, resp api.LookupResponse, ou // NewVIndexClient returns a client that can perform verified lookups into the index at the // given base URL, using the supplied verifier to check checkpoint signatures on the output // log. -func NewVIndexClient(vindexUrl string, outV note.Verifier) (*VIndexClient, error) { +func NewVIndexClient(vindexUrl string, inV, outV note.Verifier) (*VIndexClient, error) { viu, err := url.Parse(vindexUrl) if err != nil { return nil, fmt.Errorf("failed to parse URL: %v", err) @@ -109,6 +110,7 @@ func NewVIndexClient(vindexUrl string, outV note.Verifier) (*VIndexClient, error return &VIndexClient{ lookupURL: lookupURL, + inV: inV, outV: outV, }, nil } @@ -116,7 +118,7 @@ func NewVIndexClient(vindexUrl string, outV note.Verifier) (*VIndexClient, error // VIndexClient allows verified lookups into a verifiable index. type VIndexClient struct { lookupURL *url.URL - outV note.Verifier + inV, outV note.Verifier } // Lookup returns all indices, in ascending order, where the given key appears in the Input Log. @@ -137,7 +139,7 @@ func (c VIndexClient) Lookup(ctx context.Context, key string) ([]uint64, []byte, return nil, nil, fmt.Errorf("lookup failed: %v", err) } - return VerifyLookupResponse(kh, resp, c.outV) + return VerifyLookupResponse(kh, resp, c.inV, c.outV) } func (c VIndexClient) lookupUnverified(ctx context.Context, kh [sha256.Size]byte) (api.LookupResponse, error) { diff --git a/vindex/cmd/client/client.go b/vindex/cmd/client/client.go index 2a6b669..a0b32ec 100644 --- a/vindex/cmd/client/client.go +++ b/vindex/cmd/client/client.go @@ -104,11 +104,15 @@ func newVIndexClientFromFlags() *client.VIndexClient { if *outLogPubKey == "" { klog.Exitf("out_log_pub_key must be provided") } + inV, err := note.NewVerifier(*inLogPubKey) + if err != nil { + klog.Exitf("failed to construct input log verifier: %v", err) + } outV, err := note.NewVerifier(*outLogPubKey) if err != nil { klog.Exitf("failed to construct output log verifier: %v", err) } - c, err := client.NewVIndexClient(*vindexBaseURL, outV) + c, err := client.NewVIndexClient(*vindexBaseURL, inV, outV) if err != nil { klog.Exitf("failed to construct VIndex Client: %v", err) } diff --git a/vindex/cmd/sumdbverify/client.go b/vindex/cmd/sumdbverify/client.go index e359997..1e390af 100644 --- a/vindex/cmd/sumdbverify/client.go +++ b/vindex/cmd/sumdbverify/client.go @@ -50,6 +50,7 @@ import ( var ( baseURL = flag.String("base_url", "", "The base URL of the server hosting the logs and vindex.") + inLogPubKey = flag.String("in_log_pub_key", "sum.golang.org+033de0ae+Ac4zctda0e5eza+HJyk9SxEdh+s3Ux18htTTAD8OuAn8", "The input log public key.") outLogPubKey = flag.String("out_log_pub_key", "", "The public key to use to verify the output log checkpoint.") modRoot = flag.String("mod_root", "", "The path to a go module checked out locally via git.") ) @@ -472,6 +473,10 @@ func parseLeaf(idx uint64, data []byte) (string, modData, error) { } func newVIndexClientFromFlags() *client.VIndexClient { + inV, err := note.NewVerifier(*inLogPubKey) + if err != nil { + klog.Exitf("failed to construct Input Log verifier: %v", err) + } outV, err := note.NewVerifier(*outLogPubKey) if err != nil { klog.Exitf("failed to construct VIndex verifier: %v", err) @@ -480,7 +485,7 @@ func newVIndexClientFromFlags() *client.VIndexClient { if err != nil { klog.Exitf("failed to construct VIndex URL: %v", err) } - c, err := client.NewVIndexClient(u, outV) + c, err := client.NewVIndexClient(u, inV, outV) if err != nil { klog.Exitf("failed to construct VIndex Client: %v", err) } @@ -488,7 +493,7 @@ func newVIndexClientFromFlags() *client.VIndexClient { } func newInputLogClientFromFlags() *client.InputLogClient { - v, err := note.NewVerifier("sum.golang.org+033de0ae+Ac4zctda0e5eza+HJyk9SxEdh+s3Ux18htTTAD8OuAn8") + v, err := note.NewVerifier(*inLogPubKey) if err != nil { klog.Exitf("failed to construct Input Log verifier: %v", err) } diff --git a/vindex/map.go b/vindex/map.go index c83d8b0..2e3b0dc 100644 --- a/vindex/map.go +++ b/vindex/map.go @@ -28,6 +28,7 @@ import ( "fmt" "io" "iter" + "math" "os" "path" "slices" @@ -35,14 +36,13 @@ import ( "sync" "time" - "filippo.io/torchwood/prefix" - "filippo.io/torchwood/prefix/prefixsqlite" "github.com/cockroachdb/pebble" "github.com/transparency-dev/formats/log" "github.com/transparency-dev/incubator/vindex/api" "github.com/transparency-dev/merkle/compact" "github.com/transparency-dev/merkle/rfc6962" "k8s.io/klog/v2" + "rsc.io/tmp/mpt" ) const ( @@ -157,18 +157,9 @@ func NewVerifiableIndex(ctx context.Context, inputLog InputLog, mapFn MapFn, out return nil, err } - var vtreeStorage prefix.Storage - if opts.PersistIndex { - vtreeStorage, err = prefixsqlite.NewSQLiteStorage(ctx, path.Join(rootDir, "vindex.db")) - } else { - vtreeStorage = prefix.NewMemoryStorage() - } - if err != nil { - return nil, fmt.Errorf("failed to create SQL storage: %v", err) - } - if err := prefix.InitStorage(ctx, sha256.Sum256, vtreeStorage); err != nil { - return nil, fmt.Errorf("InitStorage: %s", err) - } + // TODO(mpt): This needs to be a file-backed tree before submitting + tree := mpt.NewMemTree() + mapper := &inputLogMapper{ inputLog: inputLog, mapFn: mapFn, @@ -181,8 +172,7 @@ func NewVerifiableIndex(ctx context.Context, inputLog InputLog, mapFn MapFn, out walReader: reader, db: db, outputLog: outputLog, - vindex: *prefix.NewTree(sha256.Sum256, vtreeStorage), - vstore: vtreeStorage, + vindex: tree, data: map[[sha256.Size]byte][]uint64{}, } // If we persisted the index then we don't need to rebuild it @@ -351,8 +341,7 @@ type VerifiableIndex struct { outputLog OutputLog indexMu sync.RWMutex // covers vindex and data - vindex prefix.Tree - vstore prefix.Storage + vindex mpt.Tree data map[[sha256.Size]byte][]uint64 // servingSize is the size of the input log we are serving for. @@ -370,11 +359,11 @@ func (b *VerifiableIndex) Close() error { func (b *VerifiableIndex) Lookup(ctx context.Context, key [sha256.Size]byte) (api.LookupResponse, error) { // Scope the lock to be as minimal as possible // This looks up the indices from the in-memory map, and the proof from the vindex. - lookupLocked := func(key [sha256.Size]byte) (bool, []prefix.ProofNode, []uint64, error) { + lookupLocked := func(key [sha256.Size]byte) (mpt.Proof, []uint64, error) { b.indexMu.RLock() defer b.indexMu.RUnlock() - found, viProof, err := b.vindex.Lookup(ctx, key) - return found, viProof, b.data[key], err + proof, err := b.vindex.Prove(key) + return proof, b.data[key], err } result := api.LookupResponse{} @@ -410,7 +399,7 @@ func (b *VerifiableIndex) Lookup(ctx context.Context, key [sha256.Size]byte) (ap result.OutputLogLeaf = data result.OutputLogProof = proof - found, viProof, allIndices, err := lookupLocked(key) + viProof, allIndices, err := lookupLocked(key) if err != nil { return result, fmt.Errorf("failed to get inclusion proof from vindex: %v", err) } @@ -424,17 +413,11 @@ func (b *VerifiableIndex) Lookup(ctx context.Context, key [sha256.Size]byte) (ap } result.IndexValue = allIndices - if expectFound := len(allIndices) > 0; expectFound != found { - return result, fmt.Errorf("found = %t, but expected %t (number of indices: %d)", found, expectFound, len(allIndices)) - } - result.IndexProof = make([]api.IndexNode, len(viProof)) - for i, p := range viProof { - result.IndexProof[i] = api.IndexNode{ - LabelBitLen: p.Label.BitLen(), - LabelPath: p.Label.Bytes(), - Hash: p.Hash, - } - } + // TODO(mpt): Do we want to reimplement this? + // if expectFound := len(allIndices) > 0; expectFound != found { + // return result, fmt.Errorf("found = %t, but expected %t (number of indices: %d)", found, expectFound, len(allIndices)) + // } + result.IndexProof = viProof return result, nil } @@ -451,11 +434,15 @@ func (b *VerifiableIndex) Update(ctx context.Context) error { func (b *VerifiableIndex) publish(ctx context.Context, inCp []byte) error { // Construct the leaf for the output log - rootNode, err := b.vstore.Load(ctx, prefix.RootLabel) + + if b.servingSize > math.MaxInt64 { + return fmt.Errorf("servingSize %d exceeds MaxInt64", b.servingSize) + } + snap, err := b.vindex.Snap(int64(b.servingSize)) if err != nil { - return fmt.Errorf("failed to load vindex root: %v", err) + return fmt.Errorf("Snap(): %v", err) } - outIdx, rawCp, err := b.outputLog.Append(ctx, MarshalLeaf(rootNode.Hash, inCp)) + outIdx, rawCp, err := b.outputLog.Append(ctx, MarshalLeaf(snap.Hash, inCp)) if err != nil { return fmt.Errorf("failed to append to output log: %v", err) } @@ -577,8 +564,7 @@ func (b *VerifiableIndex) buildMap(ctx context.Context, updateIndex bool) error } } - // Finally, we update the vindex - if err := b.vindex.Insert(ctx, h, [sha256.Size]byte(sum.Sum(nil))); err != nil { + if err := b.vindex.Set(h, [sha256.Size]byte(sum.Sum(nil))); err != nil { return fmt.Errorf("Insert(): %s", err) } } diff --git a/vindex/map_test.go b/vindex/map_test.go index 769f28c..e2a6eed 100644 --- a/vindex/map_test.go +++ b/vindex/map_test.go @@ -102,7 +102,7 @@ func TestVerifiableIndex(t *testing.T) { if err != nil { t.Fatal(err) } - indices, _, err := client.VerifyLookupResponse(kh, resp, v) + indices, _, err := client.VerifyLookupResponse(kh, resp, v, v) if err != nil { t.Fatalf("failed to verify vindex response: %v", err) } @@ -115,7 +115,7 @@ func TestVerifiableIndex(t *testing.T) { if err != nil { t.Fatal(err) } - indices, _, err = client.VerifyLookupResponse(kh, resp, v) + indices, _, err = client.VerifyLookupResponse(kh, resp, v, v) if err != nil { t.Fatalf("failed to verify vindex response: %v", err) } @@ -128,7 +128,7 @@ func TestVerifiableIndex(t *testing.T) { if err != nil { t.Fatal(err) } - indices, _, err = client.VerifyLookupResponse(kh, resp, v) + indices, _, err = client.VerifyLookupResponse(kh, resp, v, v) if err != nil { t.Fatalf("failed to verify vindex response: %v", err) } @@ -248,7 +248,7 @@ func TestVerifiableIndex_concurrency(t *testing.T) { if err != nil { return err } - indices, _, err = client.VerifyLookupResponse(kh, resp, v) + indices, _, err = client.VerifyLookupResponse(kh, resp, v, v) if err != nil { return fmt.Errorf("failed to verify vindex response: %v", err) } @@ -261,7 +261,7 @@ func TestVerifiableIndex_concurrency(t *testing.T) { if err != nil { return err } - indices, _, err = client.VerifyLookupResponse(kh, resp, v) + indices, _, err = client.VerifyLookupResponse(kh, resp, v, v) if err != nil { return fmt.Errorf("failed to verify vindex response: %v", err) }