From 4a7c9060fb90622dca8132b56309a0497c8eacbb Mon Sep 17 00:00:00 2001 From: acud <12988138+acud@users.noreply.github.com> Date: Fri, 5 Dec 2025 14:51:17 -0600 Subject: [PATCH 1/3] test: add integration testing for both v1 and v2 feeds --- pkg/bee/api/feed.go | 15 +- pkg/bee/client.go | 5 + pkg/check/feed/feed_v1.go | 183 ++++++++++++++ pkg/check/feed/{feed.go => feed_v2.go} | 35 +-- pkg/check/manifest/manifest_v1.go | 226 ++++++++++++++++++ .../manifest/{manifest.go => manifest_v2.go} | 16 +- pkg/config/check.go | 49 +++- 7 files changed, 492 insertions(+), 37 deletions(-) create mode 100644 pkg/check/feed/feed_v1.go rename pkg/check/feed/{feed.go => feed_v2.go} (85%) create mode 100644 pkg/check/manifest/manifest_v1.go rename pkg/check/manifest/{manifest.go => manifest_v2.go} (93%) diff --git a/pkg/bee/api/feed.go b/pkg/bee/api/feed.go index 76b71de6..c3f6ef04 100644 --- a/pkg/bee/api/feed.go +++ b/pkg/bee/api/feed.go @@ -9,7 +9,9 @@ import ( "net/http" "slices" "strconv" + "time" + "github.com/ethersphere/bee/v2/pkg/cac" "github.com/ethersphere/bee/v2/pkg/crypto" "github.com/ethersphere/bee/v2/pkg/soc" "github.com/ethersphere/bee/v2/pkg/swarm" @@ -68,7 +70,18 @@ func (f *FeedService) CreateRootManifest(ctx context.Context, signer crypto.Sign return &response, nil } -// UpdateWithRootChunk updates a feed with a root chunk +// UpdateWithReference updates a feed with a reference. This is a type v1 feed update. +func (f *FeedService) UpdateWithReference(ctx context.Context, signer crypto.Signer, topic []byte, i uint64, addr swarm.Address, o UploadOptions) (*SocResponse, error) { + ts := make([]byte, 8) + binary.BigEndian.PutUint64(ts, uint64(time.Now().Unix())) + ch, err := cac.New(append(append([]byte{}, ts...), addr.Bytes()...)) + if err != nil { + return nil, err + } + return f.UpdateWithRootChunk(ctx, signer, topic, i, ch, o) +} + +// UpdateWithRootChunk updates a feed with a root chunk. func (f *FeedService) UpdateWithRootChunk(ctx context.Context, signer crypto.Signer, topic []byte, i uint64, ch swarm.Chunk, o UploadOptions) (*SocResponse, error) { ownerHex, err := ownerFromSigner(signer) if err != nil { diff --git a/pkg/bee/client.go b/pkg/bee/client.go index a43acae3..7eee6dc1 100644 --- a/pkg/bee/client.go +++ b/pkg/bee/client.go @@ -1028,6 +1028,11 @@ func (c *Client) UpdateFeedWithRootChunk(ctx context.Context, signer crypto.Sign return c.api.Feed.UpdateWithRootChunk(ctx, signer, topic, i, ch, o) } +// UpdateFeedWithReference updates a feed with a reference +func (c *Client) UpdateFeedWithReference(ctx context.Context, signer crypto.Signer, topic []byte, i uint64, addr swarm.Address, o api.UploadOptions) (*api.SocResponse, error) { + return c.api.Feed.UpdateWithReference(ctx, signer, topic, i, addr, o) +} + // FindFeedUpdate finds the latest update for a feed func (c *Client) FindFeedUpdate(ctx context.Context, signer crypto.Signer, topic []byte, o *api.DownloadOptions) (*api.FindFeedUpdateResponse, error) { return c.api.Feed.FindUpdate(ctx, signer, topic, o) diff --git a/pkg/check/feed/feed_v1.go b/pkg/check/feed/feed_v1.go new file mode 100644 index 00000000..2a61eb4b --- /dev/null +++ b/pkg/check/feed/feed_v1.go @@ -0,0 +1,183 @@ +package feed + +import ( + "bytes" + "context" + "fmt" + "time" + + "github.com/ethersphere/bee/v2/pkg/crypto" + "github.com/ethersphere/bee/v2/pkg/swarm" + "github.com/ethersphere/beekeeper/pkg/bee" + "github.com/ethersphere/beekeeper/pkg/bee/api" + "github.com/ethersphere/beekeeper/pkg/beekeeper" + "github.com/ethersphere/beekeeper/pkg/logging" + "github.com/ethersphere/beekeeper/pkg/orchestration" + "github.com/ethersphere/beekeeper/pkg/random" +) + +// Options represents check options +type Options struct { + PostageTTL time.Duration + PostageDepth uint64 + PostageLabel string + NUpdates int + RootRef string +} + +// NewDefaultOptions returns new default options +func NewDefaultOptions() Options { + return Options{ + PostageTTL: 24 * time.Hour, + PostageDepth: 17, + PostageLabel: "test-label", + NUpdates: 2, + } +} + +// compile check whether Check implements interface +var _ beekeeper.Action = (*CheckV1)(nil) + +// Check instance. +type CheckV1 struct { + logger logging.Logger +} + +// NewCheck returns a new check instance. +func NewCheckV1(logger logging.Logger) beekeeper.Action { + return &CheckV1{ + logger: logger, + } +} + +func (c *CheckV1) Run(ctx context.Context, cluster orchestration.Cluster, opts interface{}) (err error) { + o, ok := opts.(Options) + if !ok { + return fmt.Errorf("invalid options type") + } + + if o.RootRef != "" { + c.logger.Infof("running availability check") + return c.checkAvailability(ctx, cluster, o) + } + return c.feedCheck(ctx, cluster, o) +} + +func (c *CheckV1) checkAvailability(ctx context.Context, cluster orchestration.Cluster, o Options) error { + ref, err := swarm.ParseHexAddress(o.RootRef) + if err != nil { + return fmt.Errorf("invalid root ref: %w", err) + } + + nodeNames := cluster.FullNodeNames() + nodeName := nodeNames[0] + clients, err := cluster.NodesClients(ctx) + if err != nil { + return err + } + + client := clients[nodeName] + _, _, err = client.DownloadFile(ctx, ref, nil) + if err != nil { + return err + } + return nil +} + +// feedCheck creates a root feed manifest, makes a series of updates to the feed +// and verifies that the updates are retrievable via another node. +func (c *CheckV1) feedCheck(ctx context.Context, cluster orchestration.Cluster, o Options) error { + rnd := random.PseudoGenerator(time.Now().UnixNano()) + names := cluster.FullNodeNames() + perm := rnd.Perm(len(names)) + + if len(names) < 2 { + return fmt.Errorf("not enough nodes to run feed check") + } + + clients, err := cluster.NodesClients(ctx) + if err != nil { + return err + } + upClient := clients[names[perm[0]]] + downClient := clients[names[perm[1]]] + + c.logger.Infof("upload client: %s", upClient.Name()) + + batchID, err := upClient.GetOrCreateMutableBatch(ctx, o.PostageTTL, o.PostageDepth, o.PostageLabel) + if err != nil { + return err + } + + privKey, err := crypto.GenerateSecp256k1Key() + if err != nil { + return err + } + + signer := crypto.NewDefaultSigner(privKey) + topic, err := crypto.LegacyKeccak256([]byte("my-topic-v1")) + if err != nil { + return err + } + + // create root + createManifestRes, err := upClient.CreateRootFeedManifest(ctx, signer, topic, api.UploadOptions{BatchID: batchID}) + if err != nil { + return err + } + c.logger.Infof("node %s: manifest created", upClient.Name()) + c.logger.Infof("reference: %s", createManifestRes.Reference) + c.logger.Infof("owner: %s", createManifestRes.Owner) + c.logger.Infof("topic: %s", createManifestRes.Topic) + + // make updates + for i := 0; i < o.NUpdates; i++ { + time.Sleep(3 * time.Second) + data := fmt.Sprintf("update-%d", i) + fName := fmt.Sprintf("file-%d", i) + file := bee.NewBufferFile(fName, bytes.NewBuffer([]byte(data))) + err = upClient.UploadFile(context.Background(), &file, api.UploadOptions{ + BatchID: batchID, + Direct: true, + }) + if err != nil { + return err + } + ref := file.Address() + socRes, err := upClient.UpdateFeedWithReference(ctx, signer, topic, uint64(i), ref, api.UploadOptions{BatchID: batchID}) + if err != nil { + return err + } + c.logger.Infof("node %s: feed updated", upClient.Name()) + c.logger.Infof("soc reference: %s", socRes.Reference) + c.logger.Infof("wrapped reference: %s", file.Address()) + } + time.Sleep(5 * time.Second) + + // fetch update + c.logger.Infof("fetching feed update") + c.logger.Infof("download client: %s", downClient.Name()) + update, err := downClient.FindFeedUpdate(ctx, signer, topic, nil) + if err != nil { + return err + } + + c.logger.Infof("node %s: feed update found", downClient.Name()) + c.logger.Infof("index: %d", update.Index) + c.logger.Infof("next index: %d", update.NextIndex) + + if update.NextIndex != uint64(o.NUpdates) { + return fmt.Errorf("expected next index to be %d, got %d", o.NUpdates, update.NextIndex) + } + + // fetch feed via bzz + d, err := downClient.DownloadFileBytes(ctx, createManifestRes.Reference, nil) + if err != nil { + return fmt.Errorf("download root feed: %w", err) + } + lastUpdateData := fmt.Sprintf("update-%d", o.NUpdates-1) + if string(d) != lastUpdateData { + return fmt.Errorf("expected file content to be %s, got %s", lastUpdateData, string(d)) + } + return nil +} diff --git a/pkg/check/feed/feed.go b/pkg/check/feed/feed_v2.go similarity index 85% rename from pkg/check/feed/feed.go rename to pkg/check/feed/feed_v2.go index b3df02f5..bf5a9b54 100644 --- a/pkg/check/feed/feed.go +++ b/pkg/check/feed/feed_v2.go @@ -17,41 +17,22 @@ import ( "github.com/ethersphere/beekeeper/pkg/random" ) -// Options represents check options -type Options struct { - PostageTTL time.Duration - PostageDepth uint64 - PostageLabel string - NUpdates int - RootRef string -} - -// NewDefaultOptions returns new default options -func NewDefaultOptions() Options { - return Options{ - PostageTTL: 24 * time.Hour, - PostageDepth: 17, - PostageLabel: "test-label", - NUpdates: 2, - } -} - // compile check whether Check implements interface -var _ beekeeper.Action = (*Check)(nil) +var _ beekeeper.Action = (*CheckV2)(nil) // Check instance. -type Check struct { +type CheckV2 struct { logger logging.Logger } // NewCheck returns a new check instance. -func NewCheck(logger logging.Logger) beekeeper.Action { - return &Check{ +func NewCheckV2(logger logging.Logger) beekeeper.Action { + return &CheckV2{ logger: logger, } } -func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts any) (err error) { +func (c *CheckV2) Run(ctx context.Context, cluster orchestration.Cluster, opts any) (err error) { o, ok := opts.(Options) if !ok { return fmt.Errorf("invalid options type") @@ -73,7 +54,7 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts any return nil } -func (c *Check) checkAvailability(ctx context.Context, cluster orchestration.Cluster, o Options) error { +func (c *CheckV2) checkAvailability(ctx context.Context, cluster orchestration.Cluster, o Options) error { ref, err := swarm.ParseHexAddress(o.RootRef) if err != nil { return fmt.Errorf("invalid root ref: %w", err) @@ -98,7 +79,7 @@ func (c *Check) checkAvailability(ctx context.Context, cluster orchestration.Clu // feedCheck creates a root feed manifest, makes a series of updates to the feed // and verifies that the updates are retrievable via another node. -func (c *Check) feedCheck(ctx context.Context, cluster orchestration.Cluster, o Options) error { +func (c *CheckV2) feedCheck(ctx context.Context, cluster orchestration.Cluster, o Options) error { clients, err := cluster.ShuffledFullNodeClients(ctx, random.PseudoGenerator(time.Now().UnixNano())) if err != nil { return fmt.Errorf("node clients: %w", err) @@ -124,7 +105,7 @@ func (c *Check) feedCheck(ctx context.Context, cluster orchestration.Cluster, o } signer := crypto.NewDefaultSigner(privKey) - topic, err := crypto.LegacyKeccak256([]byte("my-topic")) + topic, err := crypto.LegacyKeccak256([]byte("my-topic-v2")) if err != nil { return fmt.Errorf("topic hash: %w", err) } diff --git a/pkg/check/manifest/manifest_v1.go b/pkg/check/manifest/manifest_v1.go new file mode 100644 index 00000000..3271d838 --- /dev/null +++ b/pkg/check/manifest/manifest_v1.go @@ -0,0 +1,226 @@ +package manifest + +import ( + "bytes" + "context" + "encoding/hex" + "fmt" + "math/rand" + "time" + + "github.com/ethersphere/bee/v2/pkg/crypto" + "github.com/ethersphere/bee/v2/pkg/swarm" + "github.com/ethersphere/beekeeper/pkg/bee" + "github.com/ethersphere/beekeeper/pkg/bee/api" + "github.com/ethersphere/beekeeper/pkg/beekeeper" + "github.com/ethersphere/beekeeper/pkg/logging" + "github.com/ethersphere/beekeeper/pkg/orchestration" + "github.com/ethersphere/beekeeper/pkg/random" +) + +// compile check whether Check implements interface +var _ beekeeper.Action = (*CheckV1)(nil) + +// Check instance. +type CheckV1 struct { + logger logging.Logger +} + +// NewCheck returns a new check instance. +func NewCheckV1(logger logging.Logger) beekeeper.Action { + return &CheckV1{ + logger: logger, + } +} + +func (c *CheckV1) Run(ctx context.Context, cluster orchestration.Cluster, opts interface{}) (err error) { + o, ok := opts.(Options) + if !ok { + return fmt.Errorf("invalid options type") + } + + rnd := random.PseudoGenerator(o.Seed) + clients, err := cluster.ShuffledFullNodeClients(ctx, rnd) + if err != nil { + return fmt.Errorf("node clients shuffle: %w", err) + } + + if len(clients) < 2 { + return fmt.Errorf("not enough nodes to run manifest check") + } + upClient := clients[0] + downClient := clients[1] + + err = c.checkWithoutSubDirs(ctx, rnd, o, upClient, downClient) + if err != nil { + return fmt.Errorf("check without subdirs: %w", err) + } + + err = c.checkWithSubDirs(ctx, rnd, o, upClient, downClient) + if err != nil { + return fmt.Errorf("check with subdirs: %w", err) + } + + return nil +} + +func (c *CheckV1) checkWithoutSubDirs(ctx context.Context, rnd *rand.Rand, o Options, upClient *bee.Client, downClient *bee.Client) error { + files, err := generateFiles(rnd, o.FilesInCollection, o.MaxPathnameLength) + if err != nil { + return err + } + + tarReader, err := tarFiles(files) + if err != nil { + return err + } + + tarFile := bee.NewBufferFile("", tarReader) + batchID, err := upClient.GetOrCreateMutableBatch(ctx, o.PostageTTL, o.PostageDepth, o.PostageLabel) + if err != nil { + return fmt.Errorf("node %s: batch id %w", upClient.Name(), err) + } + c.logger.Infof("node %s: batch id %s", upClient.Name(), batchID) + + if err := upClient.UploadCollection(ctx, &tarFile, api.UploadOptions{BatchID: batchID}); err != nil { + return fmt.Errorf("node %d: %w", 0, err) + } + + for _, file := range files { + if err := c.downloadAndVerify(ctx, downClient, tarFile.Address(), &file, bee.File{}); err != nil { + return err + } + } + return nil +} + +func (c *CheckV1) checkWithSubDirs(ctx context.Context, rnd *rand.Rand, o Options, upClient *bee.Client, downClient *bee.Client) error { + privKey, err := crypto.GenerateSecp256k1Key() + if err != nil { + return err + } + + signer := crypto.NewDefaultSigner(privKey) + topic, err := crypto.LegacyKeccak256([]byte("my-website-v1")) + if err != nil { + return err + } + + batchID, err := upClient.GetOrCreateMutableBatch(ctx, o.PostageTTL, o.PostageDepth, o.PostageLabel) + if err != nil { + return fmt.Errorf("node %s: batch id %w", upClient.Name(), err) + } + c.logger.Infof("node %s: batch id %s", upClient.Name(), batchID) + + rootFeedRef, err := upClient.CreateRootFeedManifest(ctx, signer, topic, api.UploadOptions{BatchID: batchID}) + if err != nil { + return err + } + c.logger.Infof("root feed reference: %s", rootFeedRef.Reference) + time.Sleep(3 * time.Second) + + paths := []string{"index.html", "assets/styles/styles.css", "assets/styles/images/image.png", "error.html"} + files, err := generateFilesWithPaths(rnd, paths, int(o.MaxPathnameLength)) + if err != nil { + return err + } + + tarReader, err := tarFiles(files) + if err != nil { + return err + } + tarFile := bee.NewBufferFile("", tarReader) + if err := upClient.UploadCollection(ctx, &tarFile, api.UploadOptions{BatchID: batchID, IndexDocument: "index.html"}); err != nil { + return err + } + c.logger.Infof("collection uploaded: %s", tarFile.Address()) + time.Sleep(3 * time.Second) + + // push first version of website to the feed + ref, err := upClient.UpdateFeedWithReference(ctx, signer, topic, 0, tarFile.Address(), api.UploadOptions{BatchID: batchID}) + if err != nil { + return err + } + c.logger.Infof("feed updated: %s", ref.Reference) + + // download root (index.html) from the feed + err = c.downloadAndVerify(ctx, downClient, rootFeedRef.Reference, nil, files[0]) + if err != nil { + return err + } + + // update website files + files, err = generateFilesWithPaths(rnd, paths, int(o.MaxPathnameLength)) + if err != nil { + return err + } + + tarReader, err = tarFiles(files) + if err != nil { + return err + } + tarFile = bee.NewBufferFile("", tarReader) + if err := upClient.UploadCollection(ctx, &tarFile, api.UploadOptions{BatchID: batchID, IndexDocument: "index.html"}); err != nil { + return err + } + c.logger.Infof("collection uploaded: %s", tarFile.Address()) + time.Sleep(3 * time.Second) + + // push 2nd version of website to the feed + ref, err = upClient.UpdateFeedWithReference(ctx, signer, topic, 1, tarFile.Address(), api.UploadOptions{BatchID: batchID}) + if err != nil { + return err + } + c.logger.Infof("feed updated: %s", ref.Reference) + + // download updated index.html from the feed + err = c.downloadAndVerify(ctx, downClient, rootFeedRef.Reference, nil, files[0]) + if err != nil { + return err + } + + // download other paths and compare + for i := 0; i < len(files); i++ { + err = c.downloadAndVerify(ctx, downClient, tarFile.Address(), &files[i], files[0]) + if err != nil { + return err + } + } + return nil +} + +// downloadAndVerify retrieves a file from the given address using the specified client. +// If the file parameter is nil, it downloads the index file in the collection. +// Then it verifies the hash of the downloaded file against the expected hash. +func (c *CheckV1) downloadAndVerify(ctx context.Context, client *bee.Client, address swarm.Address, file *bee.File, indexFile bee.File) error { + expectedHash := indexFile.Hash() + fName := "" + if file != nil { + fName = file.Name() + expectedHash = file.Hash() + } + c.logger.Infof("downloading file: %s/%s", address, fName) + + for i := 0; i < 10; i++ { + select { + case <-time.After(5 * time.Second): + _, hash, err := client.DownloadManifestFile(ctx, address, fName) + if err != nil { + c.logger.Infof("node %s: error retrieving file: %s", client.Name(), err.Error()) + continue + } + + c.logger.Infof("want hash: %s, got hash: %s", hex.EncodeToString(expectedHash), hex.EncodeToString(hash)) + if !bytes.Equal(expectedHash, hash) { + c.logger.Infof("node %s: file hash does not match.", client.Name()) + continue + } + c.logger.Infof("node %s: file retrieved successfully", client.Name()) + return nil + case <-ctx.Done(): + return ctx.Err() + } + } + + return fmt.Errorf("failed getting manifest file after too many retries") +} diff --git a/pkg/check/manifest/manifest.go b/pkg/check/manifest/manifest_v2.go similarity index 93% rename from pkg/check/manifest/manifest.go rename to pkg/check/manifest/manifest_v2.go index ea447502..493c6e82 100644 --- a/pkg/check/manifest/manifest.go +++ b/pkg/check/manifest/manifest_v2.go @@ -46,21 +46,21 @@ func NewDefaultOptions() Options { } // compile check whether Check implements interface -var _ beekeeper.Action = (*Check)(nil) +var _ beekeeper.Action = (*CheckV2)(nil) // Check instance. -type Check struct { +type CheckV2 struct { logger logging.Logger } // NewCheck returns a new check instance. func NewCheck(logger logging.Logger) beekeeper.Action { - return &Check{ + return &CheckV2{ logger: logger, } } -func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts any) (err error) { +func (c *CheckV2) Run(ctx context.Context, cluster orchestration.Cluster, opts any) (err error) { o, ok := opts.(Options) if !ok { return fmt.Errorf("invalid options type") @@ -91,7 +91,7 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts any return nil } -func (c *Check) checkWithoutSubDirs(ctx context.Context, rnd *rand.Rand, o Options, upClient *bee.Client, downClient *bee.Client) error { +func (c *CheckV2) checkWithoutSubDirs(ctx context.Context, rnd *rand.Rand, o Options, upClient *bee.Client, downClient *bee.Client) error { files, err := generateFiles(rnd, o.FilesInCollection, o.MaxPathnameLength) if err != nil { return fmt.Errorf("generate files: %w", err) @@ -121,14 +121,14 @@ func (c *Check) checkWithoutSubDirs(ctx context.Context, rnd *rand.Rand, o Optio return nil } -func (c *Check) checkWithSubDirs(ctx context.Context, rnd *rand.Rand, o Options, upClient *bee.Client, downClient *bee.Client) error { +func (c *CheckV2) checkWithSubDirs(ctx context.Context, rnd *rand.Rand, o Options, upClient *bee.Client, downClient *bee.Client) error { privKey, err := crypto.GenerateSecp256k1Key() if err != nil { return fmt.Errorf("generate private key: %w", err) } signer := crypto.NewDefaultSigner(privKey) - topic, err := crypto.LegacyKeccak256([]byte("my-website")) + topic, err := crypto.LegacyKeccak256([]byte("my-website-v2")) if err != nil { return fmt.Errorf("topic: %w", err) } @@ -249,7 +249,7 @@ func (c *Check) checkWithSubDirs(ctx context.Context, rnd *rand.Rand, o Options, // downloadAndVerify retrieves a file from the given address using the specified client. // If the file parameter is nil, it downloads the index file in the collection. // Then it verifies the hash of the downloaded file against the expected hash. -func (c *Check) downloadAndVerify(ctx context.Context, client *bee.Client, address swarm.Address, file *bee.File, indexFile bee.File) error { +func (c *CheckV2) downloadAndVerify(ctx context.Context, client *bee.Client, address swarm.Address, file *bee.File, indexFile bee.File) error { expectedHash := indexFile.Hash() fName := "" if file != nil { diff --git a/pkg/config/check.go b/pkg/config/check.go index 13009b50..537142b0 100644 --- a/pkg/config/check.go +++ b/pkg/config/check.go @@ -237,6 +237,31 @@ var Checks = map[string]CheckType{ return opts, nil }, }, + "manifest-v1": { + NewAction: manifest.NewCheckV1, + NewOptions: func(checkGlobalConfig CheckGlobalConfig, check Check) (any, error) { + checkOpts := new(struct { + FilesInCollection *int `yaml:"files-in-collection"` + GasPrice *string `yaml:"gas-price"` + MaxPathnameLength *int32 `yaml:"max-pathname-length"` + PostageTTL *time.Duration `yaml:"postage-ttl"` + PostageDepth *uint64 `yaml:"postage-depth"` + PostageLabel *string `yaml:"postage-label"` + Seed *int64 `yaml:"seed"` + }) + if err := check.Options.Decode(checkOpts); err != nil { + return nil, fmt.Errorf("decoding check %s options: %w", check.Type, err) + } + opts := manifest.NewDefaultOptions() + + if err := applyCheckConfig(checkGlobalConfig, checkOpts, &opts); err != nil { + return nil, fmt.Errorf("applying options: %w", err) + } + + return opts, nil + }, + }, + "peer-count": { NewAction: peercount.NewCheck, NewOptions: func(checkGlobalConfig CheckGlobalConfig, check Check) (any, error) { @@ -633,8 +658,30 @@ var Checks = map[string]CheckType{ return opts, nil }, }, + "feed-v1": { + NewAction: feed.NewCheckV1, + NewOptions: func(checkGlobalConfig CheckGlobalConfig, check Check) (any, error) { + checkOpts := new(struct { + PostageTTL *time.Duration `yaml:"postage-ttl"` + PostageDepth *uint64 `yaml:"postage-depth"` + PostageLabel *string `yaml:"postage-label"` + NUpdates *int `yaml:"n-updates"` + RootRef *string `yaml:"root-ref"` + }) + if err := check.Options.Decode(checkOpts); err != nil { + return nil, fmt.Errorf("decoding check %s options: %w", check.Type, err) + } + opts := feed.NewDefaultOptions() + + if err := applyCheckConfig(checkGlobalConfig, checkOpts, &opts); err != nil { + return nil, fmt.Errorf("applying options: %w", err) + } + + return opts, nil + }, + }, "feed": { - NewAction: feed.NewCheck, + NewAction: feed.NewCheckV2, NewOptions: func(checkGlobalConfig CheckGlobalConfig, check Check) (any, error) { checkOpts := new(struct { PostageTTL *time.Duration `yaml:"postage-ttl"` From 354e62f0257a0d3f31ee8bfbb778538895804483 Mon Sep 17 00:00:00 2001 From: acud <12988138+acud@users.noreply.github.com> Date: Wed, 17 Dec 2025 18:10:44 -0600 Subject: [PATCH 2/3] chore: add to config --- config/local.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/config/local.yaml b/config/local.yaml index 7b0ae478..9391a4a6 100644 --- a/config/local.yaml +++ b/config/local.yaml @@ -199,6 +199,15 @@ checks: postage-label: gc-check timeout: 5m type: gc + ci-manifest-v1: + options: + files-in-collection: 10 + max-pathname-length: 64 + postage-ttl: 24h + postage-depth: 21 + postage-label: test-label + timeout: 30m + type: manifest ci-manifest: options: files-in-collection: 10 @@ -371,6 +380,12 @@ checks: postage-label: test-label timeout: 10m type: gsoc + ci-feed-v1: + options: + postage-ttl: 24h + postage-depth: 21 + postage-label: test-label + type: feed-v1 ci-feed: options: postage-ttl: 24h From ecd32e94df9c93ab265395e7bd698b490848f7d2 Mon Sep 17 00:00:00 2001 From: acud <12988138+acud@users.noreply.github.com> Date: Mon, 29 Dec 2025 09:41:59 -0600 Subject: [PATCH 3/3] chore: instrument errors --- pkg/check/feed/feed_v1.go | 20 +++++++++--------- pkg/check/manifest/manifest_v1.go | 34 +++++++++++++++---------------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/pkg/check/feed/feed_v1.go b/pkg/check/feed/feed_v1.go index 2a61eb4b..f8e940ed 100644 --- a/pkg/check/feed/feed_v1.go +++ b/pkg/check/feed/feed_v1.go @@ -73,13 +73,13 @@ func (c *CheckV1) checkAvailability(ctx context.Context, cluster orchestration.C nodeName := nodeNames[0] clients, err := cluster.NodesClients(ctx) if err != nil { - return err + return fmt.Errorf("nodes clients: %w", err) } client := clients[nodeName] _, _, err = client.DownloadFile(ctx, ref, nil) if err != nil { - return err + return fmt.Errorf("download file: %w", err) } return nil } @@ -97,7 +97,7 @@ func (c *CheckV1) feedCheck(ctx context.Context, cluster orchestration.Cluster, clients, err := cluster.NodesClients(ctx) if err != nil { - return err + return fmt.Errorf("nodes clients: %w", err) } upClient := clients[names[perm[0]]] downClient := clients[names[perm[1]]] @@ -106,24 +106,24 @@ func (c *CheckV1) feedCheck(ctx context.Context, cluster orchestration.Cluster, batchID, err := upClient.GetOrCreateMutableBatch(ctx, o.PostageTTL, o.PostageDepth, o.PostageLabel) if err != nil { - return err + return fmt.Errorf("get or create batch: %w", err) } privKey, err := crypto.GenerateSecp256k1Key() if err != nil { - return err + return fmt.Errorf("secp: %w", err) } signer := crypto.NewDefaultSigner(privKey) topic, err := crypto.LegacyKeccak256([]byte("my-topic-v1")) if err != nil { - return err + return fmt.Errorf("keccak: %w", err) } // create root createManifestRes, err := upClient.CreateRootFeedManifest(ctx, signer, topic, api.UploadOptions{BatchID: batchID}) if err != nil { - return err + return fmt.Errorf("create root feed manifest: %w", err) } c.logger.Infof("node %s: manifest created", upClient.Name()) c.logger.Infof("reference: %s", createManifestRes.Reference) @@ -141,12 +141,12 @@ func (c *CheckV1) feedCheck(ctx context.Context, cluster orchestration.Cluster, Direct: true, }) if err != nil { - return err + return fmt.Errorf("upload: %w", err) } ref := file.Address() socRes, err := upClient.UpdateFeedWithReference(ctx, signer, topic, uint64(i), ref, api.UploadOptions{BatchID: batchID}) if err != nil { - return err + return fmt.Errorf("update feed: %w", err) } c.logger.Infof("node %s: feed updated", upClient.Name()) c.logger.Infof("soc reference: %s", socRes.Reference) @@ -159,7 +159,7 @@ func (c *CheckV1) feedCheck(ctx context.Context, cluster orchestration.Cluster, c.logger.Infof("download client: %s", downClient.Name()) update, err := downClient.FindFeedUpdate(ctx, signer, topic, nil) if err != nil { - return err + return fmt.Errorf("find update: %w", err) } c.logger.Infof("node %s: feed update found", downClient.Name()) diff --git a/pkg/check/manifest/manifest_v1.go b/pkg/check/manifest/manifest_v1.go index 3271d838..655ade81 100644 --- a/pkg/check/manifest/manifest_v1.go +++ b/pkg/check/manifest/manifest_v1.go @@ -67,12 +67,12 @@ func (c *CheckV1) Run(ctx context.Context, cluster orchestration.Cluster, opts i func (c *CheckV1) checkWithoutSubDirs(ctx context.Context, rnd *rand.Rand, o Options, upClient *bee.Client, downClient *bee.Client) error { files, err := generateFiles(rnd, o.FilesInCollection, o.MaxPathnameLength) if err != nil { - return err + return fmt.Errorf("generate files: %w", err) } tarReader, err := tarFiles(files) if err != nil { - return err + return fmt.Errorf("tar files: %w", err) } tarFile := bee.NewBufferFile("", tarReader) @@ -88,7 +88,7 @@ func (c *CheckV1) checkWithoutSubDirs(ctx context.Context, rnd *rand.Rand, o Opt for _, file := range files { if err := c.downloadAndVerify(ctx, downClient, tarFile.Address(), &file, bee.File{}); err != nil { - return err + return fmt.Errorf("download and verify: %w", err) } } return nil @@ -97,13 +97,13 @@ func (c *CheckV1) checkWithoutSubDirs(ctx context.Context, rnd *rand.Rand, o Opt func (c *CheckV1) checkWithSubDirs(ctx context.Context, rnd *rand.Rand, o Options, upClient *bee.Client, downClient *bee.Client) error { privKey, err := crypto.GenerateSecp256k1Key() if err != nil { - return err + return fmt.Errorf("gen secp: %w", err) } signer := crypto.NewDefaultSigner(privKey) topic, err := crypto.LegacyKeccak256([]byte("my-website-v1")) if err != nil { - return err + return fmt.Errorf("keccak: %w", err) } batchID, err := upClient.GetOrCreateMutableBatch(ctx, o.PostageTTL, o.PostageDepth, o.PostageLabel) @@ -114,7 +114,7 @@ func (c *CheckV1) checkWithSubDirs(ctx context.Context, rnd *rand.Rand, o Option rootFeedRef, err := upClient.CreateRootFeedManifest(ctx, signer, topic, api.UploadOptions{BatchID: batchID}) if err != nil { - return err + return fmt.Errorf("create root manifest: %w", err) } c.logger.Infof("root feed reference: %s", rootFeedRef.Reference) time.Sleep(3 * time.Second) @@ -122,16 +122,16 @@ func (c *CheckV1) checkWithSubDirs(ctx context.Context, rnd *rand.Rand, o Option paths := []string{"index.html", "assets/styles/styles.css", "assets/styles/images/image.png", "error.html"} files, err := generateFilesWithPaths(rnd, paths, int(o.MaxPathnameLength)) if err != nil { - return err + return fmt.Errorf("generate files: %w", err) } tarReader, err := tarFiles(files) if err != nil { - return err + return fmt.Errorf("tar files: %w", err) } tarFile := bee.NewBufferFile("", tarReader) if err := upClient.UploadCollection(ctx, &tarFile, api.UploadOptions{BatchID: batchID, IndexDocument: "index.html"}); err != nil { - return err + return fmt.Errorf("upload collection: %w", err) } c.logger.Infof("collection uploaded: %s", tarFile.Address()) time.Sleep(3 * time.Second) @@ -139,29 +139,29 @@ func (c *CheckV1) checkWithSubDirs(ctx context.Context, rnd *rand.Rand, o Option // push first version of website to the feed ref, err := upClient.UpdateFeedWithReference(ctx, signer, topic, 0, tarFile.Address(), api.UploadOptions{BatchID: batchID}) if err != nil { - return err + return fmt.Errorf("update feed: %w", err) } c.logger.Infof("feed updated: %s", ref.Reference) // download root (index.html) from the feed err = c.downloadAndVerify(ctx, downClient, rootFeedRef.Reference, nil, files[0]) if err != nil { - return err + return fmt.Errorf("download and verify: %w", err) } // update website files files, err = generateFilesWithPaths(rnd, paths, int(o.MaxPathnameLength)) if err != nil { - return err + return fmt.Errorf("generate files: %w", err) } tarReader, err = tarFiles(files) if err != nil { - return err + return fmt.Errorf("tar files: %w", err) } tarFile = bee.NewBufferFile("", tarReader) if err := upClient.UploadCollection(ctx, &tarFile, api.UploadOptions{BatchID: batchID, IndexDocument: "index.html"}); err != nil { - return err + return fmt.Errorf("upload collection: %w", err) } c.logger.Infof("collection uploaded: %s", tarFile.Address()) time.Sleep(3 * time.Second) @@ -169,21 +169,21 @@ func (c *CheckV1) checkWithSubDirs(ctx context.Context, rnd *rand.Rand, o Option // push 2nd version of website to the feed ref, err = upClient.UpdateFeedWithReference(ctx, signer, topic, 1, tarFile.Address(), api.UploadOptions{BatchID: batchID}) if err != nil { - return err + return fmt.Errorf("update feed: %w", err) } c.logger.Infof("feed updated: %s", ref.Reference) // download updated index.html from the feed err = c.downloadAndVerify(ctx, downClient, rootFeedRef.Reference, nil, files[0]) if err != nil { - return err + return fmt.Errorf("download and verify: %w", err) } // download other paths and compare for i := 0; i < len(files); i++ { err = c.downloadAndVerify(ctx, downClient, tarFile.Address(), &files[i], files[0]) if err != nil { - return err + return fmt.Errorf("download and verify: %w", err) } } return nil