From 4feb7be576b94ddd2b82ec6b8d045f61745f2634 Mon Sep 17 00:00:00 2001 From: Martin Hutchinson Date: Wed, 27 Aug 2025 13:47:51 +0000 Subject: [PATCH 1/2] CT demo for VIndex This doesn't work well at the moment because the size of CT logs is too big for this. In order to make this manageable, the MapFn is restricted to only index domains ending in `.co.uk`, and only outputs the full key, rather than a key for each level in the domain hierarchy. Even with these limitations, this has value because it provides a working base point from which to iterate. --- go.mod | 15 +- go.sum | 35 ++-- vindex/cmd/ct/README.md | 53 ++++++ vindex/cmd/ct/main.go | 365 ++++++++++++++++++++++++++++++++++++++++ vindex/cmd/ct/web.go | 76 +++++++++ 5 files changed, 520 insertions(+), 24 deletions(-) create mode 100644 vindex/cmd/ct/README.md create mode 100644 vindex/cmd/ct/main.go create mode 100644 vindex/cmd/ct/web.go diff --git a/go.mod b/go.mod index 05a2212..4870d30 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/transparency-dev/incubator go 1.25.0 require ( + filippo.io/sunlight v0.8.0 filippo.io/torchwood v0.9.0 github.com/cockroachdb/pebble v1.1.5 github.com/go-git/go-git/v5 v5.19.0 @@ -11,6 +12,7 @@ require ( github.com/transparency-dev/formats v0.1.0 github.com/transparency-dev/merkle v0.0.2 github.com/transparency-dev/tessera v1.0.3-0.20260318145621-a1e0ccb4adf4 + golang.org/x/crypto v0.50.0 golang.org/x/mod v0.36.0 golang.org/x/sync v0.20.0 k8s.io/klog/v2 v2.140.0 @@ -40,8 +42,8 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect - github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect + github.com/google/certificate-transparency-go v1.3.2 // indirect github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect @@ -51,14 +53,14 @@ require ( github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect github.com/pjbgf/sha1cd v0.6.0 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_golang v1.15.0 // indirect - github.com/prometheus/client_model v0.3.0 // indirect - github.com/prometheus/common v0.42.0 // indirect - github.com/prometheus/procfs v0.9.0 // indirect + github.com/prometheus/client_golang v1.22.0 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.62.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect @@ -68,7 +70,6 @@ require ( go.opentelemetry.io/otel v1.42.0 // indirect go.opentelemetry.io/otel/metric v1.42.0 // indirect go.opentelemetry.io/otel/trace v1.42.0 // indirect - golang.org/x/crypto v0.50.0 // indirect golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect golang.org/x/net v0.53.0 // indirect golang.org/x/sys v0.43.0 // indirect diff --git a/go.sum b/go.sum index fd512e7..ade89ab 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +filippo.io/sunlight v0.8.0 h1:7ytoUj2KmU5k4ogDSLwEtCoEjjrTZsh+g++UIfTGpM4= +filippo.io/sunlight v0.8.0/go.mod h1:gJ1qFtjHWqj9j4f5M2fnaER6ZFPUkTrRz4/pTamneDg= filippo.io/torchwood v0.9.0 h1:2W156cI7K3MyxEyNTuS1C9lYEW7y1u7PoHLmvgNsiZc= filippo.io/torchwood v0.9.0/go.mod h1:zOJguxdmaODUQScAvC80bV6N0SOA9U+bFZG1DwJU6N8= github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= @@ -39,8 +41,9 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/cyphar/filepath-securejoin v0.6.1 h1:5CeZ1jPXEiYt3+Z6zqprSAgSWiggmpVyciv8syjIpVE= github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= @@ -70,12 +73,10 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/certificate-transparency-go v1.3.2 h1:9ahSNZF2o7SYMaKaXhAumVEzXB2QaayzII9C8rv7v+A= +github.com/google/certificate-transparency-go v1.3.2/go.mod h1:H5FpMUaGa5Ab2+KCYsxg6sELw3Flkl7pGZzWdBoYLXs= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -103,8 +104,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= @@ -116,16 +117,17 @@ github.com/pjbgf/sha1cd v0.6.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.15.0 h1:5fCgGYogn0hFdhyhLbw7hEsWxufKtY9klyvdNfFlFhM= -github.com/prometheus/client_golang v1.15.0/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= -github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= -github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= -github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= -github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= -github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= -github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= +github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= +github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= @@ -178,7 +180,6 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/vindex/cmd/ct/README.md b/vindex/cmd/ct/README.md new file mode 100644 index 0000000..d993bd3 --- /dev/null +++ b/vindex/cmd/ct/README.md @@ -0,0 +1,53 @@ +## Verifiable Index: CT + +This is a demo of pulling the contents of a tile-based CT log into a [Verifiable Index](../../README.md). + +[tlog-tiles]: https://c2sp.org/tlog-tiles +[Tessera]: https://github.com/transparency-dev/tessera + +The CT Input Log is processed, with each entry being indexed on all common names defined in the cert. +This allows the owner of a domain to look up all certs for their domain, in a way that is fully verified. + +> [!NOTE] +> This demo doesn't map all certificates! +> In order to generate a manageable number of key/values, this only indexes +> final certs, and only domain names ending with `.co.uk`. +> https://github.com/transparency-dev/incubator/issues/64 + +## Running + +The Input Log is expected to be available at a URL provided by the `--static_ct_log_url` flag. +The Verifiable Index and Output Log are constructed locally, persisted to local disk (in the `--storage_dir` directory), and hosted via a web server. + +```shell +OUTPUT_LOG_PRIVATE_KEY=PRIVATE+KEY+example.com/outputlog+07392c46+ATPJ4crkyUbPeaRffN/4NUof3KV0pQznVIPGOQm3SDEJ \ +MY_EMAIL=me@example.com \ +go run ./vindex/cmd/ct \ + --storage_dir ~/vindex-ct/ \ + --origin="arche2026h1.staging.ct.transparency.dev" \ + --public_key="MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEZ+3YKoZTMruov4cmlImbk4MckBNzEdCyMuHlwGgJ8BUrzFLlR5U0619xDDXIXespkpBgCNVQAkhMTTXakM6KMg==" \ + --monitoring_url="https://storage.googleapis.com/static-ct-staging-arche2026h1-bucket/" \ + --user_agent_info=${MY_EMAIL} +``` + +Running the above will run a web server hosting the following URLs: + - `/vindex/lookup` - the provisional [vindex lookup API](./api/api.go) + - `/outputlog/` - the [tlog-tiles][] base URL for the output log + +To inspect the log, you can use the woodpecker tool (using the corresponding public key to the private key used above): + +```shell +# To inspect the Output Log +go run github.com/mhutchinson/woodpecker@main --custom_log_type=tiles --custom_log_url=http://localhost:8088/outputlog/ --custom_log_vkey=example.com/outputlog+07392c46+AWyS8y8ZsRmQnTr6Fr2knaa8+t6CPYFh5Ho3wJEr14B8 +``` + +Use left/right cursor to browse, and `q` to quit. + +A domain indexed by the verifiable map can be looked up using the following command: + +```shell +go run ./vindex/cmd/client \ + --vindex_base_url http://localhost:8088/vindex/ \ + --out_log_pub_key=example.com/outputlog+07392c46+AWyS8y8ZsRmQnTr6Fr2knaa8+t6CPYFh5Ho3wJEr14B8 \ + --lookup=google.co.uk +``` diff --git a/vindex/cmd/ct/main.go b/vindex/cmd/ct/main.go new file mode 100644 index 0000000..9478378 --- /dev/null +++ b/vindex/cmd/ct/main.go @@ -0,0 +1,365 @@ +// Copyright 2025 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// logandmap is a binary that serves as a demo of how to run a log and a map in the +// same process. +// The log is a Tessera POSIX log, and the map is an in-memory verifiable index. +// A web server is hosted that allows lookups in the map to be performed. +// The log is updated periodically with entries of type LogEntry, and the map keys +// each of the module names from that struct to each of the indices in the log where +// an entry for that module is stored. +package main + +import ( + "context" + "crypto/sha256" + "crypto/x509" + "encoding/base64" + "errors" + "flag" + "fmt" + "iter" + "net/http" + "os" + "os/signal" + "path" + "strings" + "syscall" + "time" + + "filippo.io/sunlight" + "filippo.io/torchwood" + "github.com/gorilla/mux" + "github.com/transparency-dev/formats/log" + fnote "github.com/transparency-dev/formats/note" + "github.com/transparency-dev/incubator/vindex" + "golang.org/x/crypto/cryptobyte" + "golang.org/x/mod/sumdb/note" + "golang.org/x/mod/sumdb/tlog" + "k8s.io/klog/v2" +) + +var ( + inputLogUrl = flag.String("monitoring_url", "", "Base URL of the static CT log to index") + origin = flag.String("origin", "", "Origin of the log to check") + pubKey = flag.String("public_key", "", "The log's public key in base64 encoded DER format") + userAgentInfo = flag.String("user_agent_info", "", "Optional string to append to the user agent (e.g. email address for Sunlight logs)") + persistentCacheDir = flag.String("persistent_cache_dir", "", "Optional location of a directory to cache Input Log tiles") + persistIndex = flag.Bool("persist_index", false, "Set to true to use a disk-based implementation of the verifiable index. This can be slow, but useful in situations where memory is constrained.") + + outputLogPrivKeyFile = flag.String("output_log_private_key", "", "Location of private key file. If unset, uses the contents of the OUTPUT_LOG_PRIVATE_KEY environment variable.") + storageDir = flag.String("storage_dir", "", "Root directory in which to store the data for the demo. This will create subdirectories for the Input Log, Output Log, and allocate space to store the verifiable map persistence.") + listen = flag.String("listen", ":8088", "Address to set up HTTP server listening on") +) + +const ( + userAgent = "TrustFabric VerifiableIndex" +) + +func main() { + klog.InitFlags(nil) + flag.Parse() + + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + + if err := run(ctx); err != nil { + klog.Exitf("Run failed: %v", err) + } +} + +func run(ctx context.Context) error { + // Set up storage for the input log, index, and output log. + if *storageDir == "" { + return errors.New("storage_dir must be set") + } + outputLogDir := path.Join(*storageDir, "outputlog") + mapRoot := path.Join(*storageDir, "vindex") + + if err := os.MkdirAll(outputLogDir, 0o755); err != nil { + return fmt.Errorf("failed to create output log directory: %v", err) + } + if err := os.MkdirAll(mapRoot, 0o755); err != nil { + return fmt.Errorf("failed to create vindex directory: %v", err) + } + + outputLog, outputCloser := outputLogOrDie(ctx, outputLogDir) + defer outputCloser() + + inputLog := newStaticCTInputLogFromFlags() + + vi, err := vindex.NewVerifiableIndex(ctx, inputLog, mapFn, outputLog, mapRoot, vindex.Options{PersistIndex: *persistIndex}) + if err != nil { + return fmt.Errorf("failed to create vindex: %v", err) + } + klog.Info("Created verifiable index") + + // Keeps the map synced with the latest published input log state. + go maintainMap(ctx, vi) + + // Run a web server to serve the input log, index, and output log. + go runWebServer(vi, outputLogDir) + <-ctx.Done() + return nil +} + +func cutEntry(tile []byte) (entry []byte, rh tlog.Hash, rest []byte, err error) { + // This implementation is terribly inefficient, parsing the whole entry just + // to re-serialize and throw it away. If this function shows up in profiles, + // let me know and I'll improve it. + e, rest, err := sunlight.ReadTileLeaf(tile) + if err != nil { + return nil, tlog.Hash{}, nil, err + } + rh = tlog.RecordHash(e.MerkleTreeLeaf()) + entry = tile[:len(tile)-len(rest)] + return entry, rh, rest, nil +} + +func newStaticCTInputLogFromFlags() *staticCTInputLog { + ua := userAgent + if *userAgentInfo != "" { + ua = fmt.Sprintf("%s (%s)", userAgent, *userAgentInfo) + } + fetcher, err := torchwood.NewTileFetcher(*inputLogUrl, + torchwood.WithTilePath(sunlight.TilePath), + torchwood.WithUserAgent(ua)) + if err != nil { + klog.Exitf("failed to create client: %v", err) + } + var tileReader torchwood.TileReaderWithContext = fetcher + if *persistentCacheDir != "" { + tileReader, err = torchwood.NewPermanentCache(fetcher, *persistentCacheDir) + if err != nil { + klog.Exitf("failed to create permanent cache: %v", err) + } + } + client, err := torchwood.NewClient(tileReader, torchwood.WithCutEntry(cutEntry)) + if err != nil { + klog.Exitf("failed to create client: %v", err) + } + return &staticCTInputLog{ + c: client, + f: fetcher, + v: verifierFromFlags(), + } +} + +type staticCTInputLog struct { + c *torchwood.Client + f *torchwood.TileFetcher + v note.Verifier + + lastCheckpoint log.Checkpoint +} + +func (l *staticCTInputLog) Checkpoint(ctx context.Context) (checkpoint []byte, err error) { + return l.f.ReadEndpoint(ctx, "checkpoint") +} + +// Parse unmarshals and verifies a checkpoint obtained from GetCheckpoint. +func (l *staticCTInputLog) Parse(checkpoint []byte) (*log.Checkpoint, error) { + cp, _, _, err := log.ParseCheckpoint(checkpoint, l.v.Name(), l.v) + if err != nil { + return nil, err + } + l.lastCheckpoint = *cp + return cp, err +} + +// Leaves returns all the leaves in the range [start, end), outputting them via +// the returned iterator. +func (l *staticCTInputLog) Leaves(ctx context.Context, start, end uint64) iter.Seq2[[]byte, error] { + tree := tlog.Tree{ + N: int64(end), + Hash: tlog.Hash(l.lastCheckpoint.Hash), + } + return func(yield func([]byte, error) bool) { + for _, entry := range l.c.Entries(ctx, tree, int64(start)) { + e, _, err := sunlight.ReadTileLeaf(entry) + if err != nil { + if !yield(nil, err) { + return + } + } + if !yield(e.MerkleTreeLeaf(), nil) { + return + } + } + if err := l.c.Err(); err != nil { + yield(nil, l.c.Err()) + } + } +} + +// outputLogOrDie returns an output log using a POSIX log in the given directory. +func outputLogOrDie(ctx context.Context, outputLogDir string) (log vindex.OutputLog, closer func()) { + s, v := getOutputLogSignerVerifierOrDie() + + l, c, err := vindex.NewOutputLog(ctx, outputLogDir, s, v) + if err != nil { + klog.Exit(err) + } + return l, c +} + +func verifierFromFlags() note.Verifier { + if *origin == "" { + klog.Exitf("Must provide the --origin flag") + } + if *pubKey == "" { + klog.Exitf("Must provide the --pub_key flag") + } + derBytes, err := base64.StdEncoding.DecodeString(*pubKey) + if err != nil { + klog.Exitf("Error decoding public key: %s", err) + } + pub, err := x509.ParsePKIXPublicKey(derBytes) + if err != nil { + klog.Exitf("Error parsing public key: %v", err) + } + + verifierKey, err := fnote.RFC6962VerifierString(*origin, pub) + if err != nil { + klog.Exitf("Error creating RFC6962 verifier string: %v", err) + } + logSigV, err := fnote.NewVerifier(verifierKey) + if err != nil { + klog.Exitf("Error creating verifier: %v", err) + } + + klog.Infof("Using verifier string: %v", verifierKey) + + return logSigV +} + +// maintainMap reads entries from the log and sync them to the vindex. +func maintainMap(ctx context.Context, vi *vindex.VerifiableIndex) { + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + + for { + if err := vi.Update(ctx); err != nil { + klog.Warning(err) + } + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } +} + +func runWebServer(vi *vindex.VerifiableIndex, outLogDir string) { + web := NewServer(vi.Lookup) + + olfs := http.FileServer(http.Dir(outLogDir)) + r := mux.NewRouter() + r.PathPrefix("/outputlog/").Handler(http.StripPrefix("/outputlog/", olfs)) + web.registerHandlers(r) + hServer := &http.Server{ + Addr: *listen, + Handler: r, + } + go func() { + _ = hServer.ListenAndServe() + }() + klog.Infof("Started HTTP server listening on %s", *listen) +} + +// Read output log private key from file or environment variable and generate the +// note Signer and Verifier pair for it. +func getOutputLogSignerVerifierOrDie() (note.Signer, note.Verifier) { + var privKey string + var err error + if len(*outputLogPrivKeyFile) > 0 { + privKey, err = getKeyFile(*outputLogPrivKeyFile) + if err != nil { + klog.Exitf("Unable to get private key: %v", err) + } + } else { + privKey = os.Getenv("OUTPUT_LOG_PRIVATE_KEY") + if len(privKey) == 0 { + klog.Exit("Supply private key file path using --output_log_private_key or set OUTPUT_LOG_PRIVATE_KEY environment variable") + } + } + s, v, err := fnote.NewEd25519SignerVerifier(privKey) + if err != nil { + klog.Exitf("Failed to get signer/verifier: %v", err) + } + return s, v +} + +func getKeyFile(path string) (string, error) { + k, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("failed to read key file: %w", err) + } + return string(k), nil +} + +func mapFn(data []byte) [][sha256.Size]byte { + s := cryptobyte.String(data) + + var version, leafType uint8 + var timestamp uint64 + var certType uint16 + if !s.ReadUint8(&version) || !s.ReadUint8(&leafType) || !s.ReadUint64(×tamp) || !s.ReadUint16(&certType) { + klog.Warningf("Failed to unmarshal headers") + // This should return a sentinel value (e.g. all zero hash) so unprocessable entries can be found + return nil + } + var isPreCert bool + var cert cryptobyte.String + switch certType { + case 0: + // x509 + isPreCert = false + s.ReadUint24LengthPrefixed(&cert) + case 1: + if true { + // Need to support parsing TBS certs + return nil + } + // precert + isPreCert = true + var ikh []byte + s.ReadBytes(&ikh, sha256.Size) + s.ReadUint24LengthPrefixed(&cert) + default: + panic("unknown cert type") + } + + parsedCert, err := x509.ParseCertificate(cert) + if err != nil { + klog.Warningf("failed to parse x509 cert (preCert=%t): %v", isPreCert, err) + // This should return a sentinel value (e.g. all zero hash) so unprocessable entries can be found + return nil + } + if klog.V(2).Enabled() { + klog.V(2).Info(parsedCert.DNSNames) + } + hashes := make([][sha256.Size]byte, 0, len(parsedCert.DNSNames)) + for _, cn := range parsedCert.DNSNames { + // This filtering is simply to make the index manageable for current CT logs + // https://github.com/transparency-dev/incubator/issues/64 + if strings.HasSuffix(cn, ".co.uk") { + // This should output keys for various levels up to the TLD, e.g. + // maps.google.co.uk should have google.co.uk as a secondary key. + h := sha256.Sum256([]byte(cn)) + hashes = append(hashes, h) + } + } + return hashes +} diff --git a/vindex/cmd/ct/web.go b/vindex/cmd/ct/web.go new file mode 100644 index 0000000..68c8c25 --- /dev/null +++ b/vindex/cmd/ct/web.go @@ -0,0 +1,76 @@ +// Copyright 2025 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "crypto/sha256" + _ "embed" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + + "github.com/gorilla/mux" + "github.com/transparency-dev/incubator/vindex/api" + "k8s.io/klog/v2" +) + +func NewServer(lookup func(context.Context, [sha256.Size]byte) (api.LookupResponse, error)) Server { + return Server{ + lookup: lookup, + } +} + +type Server struct { + lookup func(context.Context, [sha256.Size]byte) (api.LookupResponse, error) +} + +// handleLookup handles GET requests for looking up map entries. +func (s Server) handleLookup(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + hashStr, ok := vars["hash"] + if !ok { + http.Error(w, "hash parameter not found", http.StatusBadRequest) + return + } + + h, err := hex.DecodeString(hashStr) + if err != nil { + http.Error(w, fmt.Sprintf("invalid hex hash: %v", err), http.StatusBadRequest) + return + } + if l := len(h); l != sha256.Size { + http.Error(w, fmt.Sprintf("hash wrong length (decoded %d bytes)", l), http.StatusBadRequest) + return + } + + klog.V(2).Infof("Received hash from request: '%s'", h) + + resp, err := s.lookup(r.Context(), [sha256.Size]byte(h)) + if err != nil { + http.Error(w, fmt.Sprintf("lookup failed: %v", err), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + if err := json.NewEncoder(w).Encode(resp); err != nil { + klog.Warningf("failed to encode response: %v", err) + } +} + +func (s Server) registerHandlers(r *mux.Router) { + r.HandleFunc("/vindex/lookup/{hash}", s.handleLookup).Methods("GET") +} From 218eeafa9db8a1c4e3e742433b9bfce816f3840c Mon Sep 17 00:00:00 2001 From: Martin Hutchinson Date: Mon, 11 May 2026 09:42:39 +0000 Subject: [PATCH 2/2] Bringing up to date --- vindex/cmd/ct/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vindex/cmd/ct/main.go b/vindex/cmd/ct/main.go index 9478378..9ee05df 100644 --- a/vindex/cmd/ct/main.go +++ b/vindex/cmd/ct/main.go @@ -138,7 +138,7 @@ func newStaticCTInputLogFromFlags() *staticCTInputLog { if err != nil { klog.Exitf("failed to create client: %v", err) } - var tileReader torchwood.TileReaderWithContext = fetcher + var tileReader torchwood.TileReader = fetcher if *persistentCacheDir != "" { tileReader, err = torchwood.NewPermanentCache(fetcher, *persistentCacheDir) if err != nil { @@ -207,7 +207,7 @@ func (l *staticCTInputLog) Leaves(ctx context.Context, start, end uint64) iter.S func outputLogOrDie(ctx context.Context, outputLogDir string) (log vindex.OutputLog, closer func()) { s, v := getOutputLogSignerVerifierOrDie() - l, c, err := vindex.NewOutputLog(ctx, outputLogDir, s, v) + l, c, err := vindex.NewOutputLog(ctx, outputLogDir, s, v, vindex.OutputLogOpts{}) if err != nil { klog.Exit(err) }