diff --git a/.github/workflows/markdown-format.yaml b/.github/workflows/markdown-format.yaml index 2346528d5..edf123ab0 100644 --- a/.github/workflows/markdown-format.yaml +++ b/.github/workflows/markdown-format.yaml @@ -16,7 +16,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@v6 with: - node-version: '20' + node-version: '24' cache: 'npm' - name: Install dependencies diff --git a/package-lock.json b/package-lock.json index 1405b75bf..4ed8fb62c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,13 +8,13 @@ "name": "openmina", "version": "1.0.0", "devDependencies": { - "prettier": "^3.0.0" + "prettier": "3.7.4" } }, "node_modules/prettier": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.6.2.tgz", - "integrity": "sha512-I7AIg5boAr5R0FFtJ6rCfD+LFsWHp81dolrFD8S79U9tb8Az2nGrJncnMSnys+bpQJfRUzqs9hnA81OAA3hCuQ==", + "version": "3.7.4", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.7.4.tgz", + "integrity": "sha512-v6UNi1+3hSlVvv8fSaoUbggEM5VErKmmpGA7Pl3HF8V6uKY7rvClBOJlH6yNwQtfTueNkGVpOv/mtWL9L4bgRA==", "dev": true, "license": "MIT", "bin": { diff --git a/package.json b/package.json index dd34e0234..e338d5066 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "check:md": "prettier --check \"**/*.md\" \"**/*.mdx\"" }, "devDependencies": { - "prettier": "^3.0.0" + "prettier": "3.7.4" }, "prettier": { "printWidth": 80, diff --git a/website/docs/researchers/rfcs/0000-template.md b/website/docs/researchers/rfcs/0000-template.md new file mode 100644 index 000000000..05c6037e5 --- /dev/null +++ b/website/docs/researchers/rfcs/0000-template.md @@ -0,0 +1,66 @@ +--- +format: md +title: "RFC 0000: Template" +sidebar_label: "0000 Template" +hide_table_of_contents: false +--- + +> **Original source:** +> [0000-template.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0000-template.md) + +## Summary + +[summary]: #summary + +One paragraph explanation of the feature. + +## Motivation + +[motivation]: #motivation + +Why are we doing this? What use cases does it support? What is the expected +outcome? + +## Detailed design + +[detailed-design]: #detailed-design + +This is the technical portion of the RFC. Explain the design in sufficient +detail that: + +- Its interaction with other features is clear. +- It is reasonably clear how the feature would be implemented. +- Corner cases are dissected by example. + +## Drawbacks + +[drawbacks]: #drawbacks + +Why should we _not_ do this? + +## Rationale and alternatives + +[rationale-and-alternatives]: #rationale-and-alternatives + +- Why is this design the best in the space of possible designs? +- What other designs have been considered and what is the rationale for not + choosing them? +- What is the impact of not doing this? + +## Prior art + +[prior-art]: #prior-art + +Discuss prior art, both the good and the bad, in relation to this proposal. + +## Unresolved questions + +[unresolved-questions]: #unresolved-questions + +- What parts of the design do you expect to resolve through the RFC process + before this gets merged? +- What parts of the design do you expect to resolve through the implementation + of this feature before merge? +- What related issues do you consider out of scope for this RFC that could be + addressed in the future independently of the solution that comes out of this + RFC? diff --git a/website/docs/researchers/rfcs/0001-banlisting.md b/website/docs/researchers/rfcs/0001-banlisting.md new file mode 100644 index 000000000..9f66a514b --- /dev/null +++ b/website/docs/researchers/rfcs/0001-banlisting.md @@ -0,0 +1,131 @@ +--- +format: md +title: "RFC 0001: Banlisting" +sidebar_label: "0001 Banlisting" +hide_table_of_contents: false +--- + +> **Original source:** +> [0001-banlisting.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0001-banlisting.md) + +# Summary + +[summary]: #summary + +Punish peers by refusing to connect to their IP when they misbehave. + +# Motivation + +[motivation]: #motivation + +Our software receives diffs, transactions, merkle tree components, etc from +other peers in the network. Much of this is authenticated, meaning we know what +the resulting hash of some object must be. This happens for the +`syncable_ledger` queries and the staged ledger aux data. When a peer sends us +data that results in a state where the hash doesn't match, we know that the peer +was dishonest or buggy. Some other instances of misbehavior that we want to +punish with a ban: + +- A received SNARK fails to verify +- An external transition was invalid for any of a number of reasons +- A VRF proof fails to verify + +When a node misbehaves, typically it is trying to mount an attack, often a +denial of service attack. We can mitigate the effectiveness of these attempts by +just ignoring that node's messages. + +# Detailed design + +[detailed-design]: #detailed-design + +The basic design is pretty obvious: when a peer misbehaves, notice this, add +their IP to a list, and refuse to communicate with that IP for some amount of +time. + +For this we have a persistent table mapping IPs to ban scores. + +Introduce the banlist: + +```ocaml +module type Banlist_intf = sig + type t + + type ban = + { host: string + ; score: int + ; reason: string option + ; remaining_dur: Time.Span.t } + + val record_misbehavior : t -> host:string -> score:int -> ?reason:string -> unit + + val ban : t -> host:string -> ?reason:string -> dur:Time.Span.t -> unit + + val unban : t -> host:string -> unit + + val bans : t -> ban list + + val lookup_score : t -> host:string -> int option + + val flush : t -> unit Deferred.t +end +``` + +First, where the code currently has `TODO: punish` or `Logger.faulty_peer`, we +should insert a call to `record_misbehavior`. When the score for a host exceeds +some threshold, the banlist will make sure that: + +- The banned hosts won't show up as a result of querying the membership layer + for peers +- RPC connections from those IPs will be rejected. + +The banlist should be persistent, and the CLI should allow manually +adding/removing IPs from the banlist: + +``` +mina client ban add IP duration +mina client ban remove IP +mina client ban list +``` + +By default, bans will last for 1 day. + +# Drawbacks + +[drawbacks]: #drawbacks + +In the case of bugs and not dishonesty, this could really cause chaos. In the +worst case, the network can become totally disconnected and no peer will +willingly communicate with any other for very long. + +# Rationale and alternatives + +[rationale-and-alternatives]: #rationale-and-alternatives + +It's not clear what the ideal punishment for misbehaving nodes is. A 1-day IP +ban limits most opportunities for DoS, and bitcoin does it, so it seems +reasonable. The main alternative is a permaban, but this is unnecessarily harsh. +Someone else is probably going to reuse that IP soon enough. + +A more complex system could haves nodes sharing proofs of peer misbehavior, +which would enable trustless network-wide banning. This would need a fair amount +of work for probably not much gain. + +# Prior art + +[prior-art]: #prior-art + +Bitcoin, when handling network messages from a peer, is careful to check a +variety of conditions to ensure the message conforms to the protocol. When a +check fails, a "ban score" is added to. When the ban score reaches a given +threshold (default 100), the node's IP is banned for a default of 24 hours. Some +checks are insta-bans (most causes of invalid blocks). See in the bitcoin core +source: + +- `src/validation.cpp`, grep for `state.DoS` +- `src/net_processing.cpp`, grep for `Misbehaving` + +# Unresolved questions + +[unresolved-questions]: #unresolved-questions + +- Is there any reason to have a more sophisticated ban policy? diff --git a/website/docs/researchers/rfcs/0002-branch-prefixes.md b/website/docs/researchers/rfcs/0002-branch-prefixes.md new file mode 100644 index 000000000..5f888a736 --- /dev/null +++ b/website/docs/researchers/rfcs/0002-branch-prefixes.md @@ -0,0 +1,79 @@ +--- +format: md +title: "RFC 0002: Branch Prefixes" +sidebar_label: "0002 Branch Prefixes" +hide_table_of_contents: false +--- + +> **Original source:** +> [0002-branch-prefixes.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0002-branch-prefixes.md) + +# Summary + +[summary]: #summary + +Organize branches by categories of work using and standardizing prefixes. + +# Motivation + +[motivation]: #motivation + +As our repository continues to grow, so will our number of branches. As we begin +to move towards having branches for releases, release candidates, bug fixes, and +new features, and rfcs, we will need the ability to quickly understand which of +these categories a given branch falls into. On top of this, categorizing by +prefix will allow us to seperate name conflicts so that branch names will only +conflict if the both the category and the name are the same. + +# Detailed design + +[detailed-design]: #detailed-design + +Moving forward, all branch names, with the exception of master, will be prefixed +by a category identifier. These branch names will all follow the format +"/", where name is a description of what the branch actually +contains. + +These are the categories which branches will fall into: + +- feature (a new feature to the codebase) +- fix (a bug fix of existing features) +- rfc (a new rfc to be discussed) +- release (a branch with the latest version of a specific release target) +- release-candidate (a branch for locking and testing a potential future release + before we finalize it) +- tmp (for ad-hoc, temporary, miscellaneous work) + +These branch categories can be extended as new types of branches are required. +For instance, in the future, we may want to have "environment" branches, where +pushing to an "env/..." branch will kick off some CI to deploy the code at that +branch to an environment. For example: push "feature/some-experiment" to +"env/testbed" to test it in a cluster in the cloud, or push +"release-candidate/beta" to "env/staging" to test a release candidate for +regressions. + +# Drawbacks + +[drawbacks]: #drawbacks + +The main drawback will be the initial migration. We have many branches out in +the wild that are not prefixed yet, so there may be some confusion while we +begin to adopt the new branch naming scheme, and before we can finish cleaning +up the old branches. + +# Rationale and alternatives + +[rationale-and-alternatives]: #rationale-and-alternatives + +I'm not aware of any alternatives for managing this. Releases are managed +through version tags, but the release branches discussed here are really just a +layer on top of that, intended to represent the latest version of a specific +release cycle (version tags should still be used). + +# Unresolved questions + +[unresolved-questions]: #unresolved-questions + +- How should we manage the migration towards using these new branches? Should we + do one mass branch renaming, or should we remove old branches over time, + pruning them as we no longer need them? diff --git a/website/docs/researchers/rfcs/0003-renaming-refactor.md b/website/docs/researchers/rfcs/0003-renaming-refactor.md new file mode 100644 index 000000000..9b72a9779 --- /dev/null +++ b/website/docs/researchers/rfcs/0003-renaming-refactor.md @@ -0,0 +1,108 @@ +--- +format: md +title: "RFC 0003: Renaming Refactor" +sidebar_label: "0003 Renaming Refactor" +hide_table_of_contents: false +--- + +> **Original source:** +> [0003-renaming-refactor.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0003-renaming-refactor.md) + +# Summary + +[summary]: #summary + +Refactor our codebase with consistent, correct, and descriptive names. + +# Motivation + +[motivation]: #motivation + +There are a multitude of naming issues which plague our codebase. Some names are +not consistent (`Statement` vs. `Work`), others are not correct (`Ledger` is not +a ledger; there is no log of transactions). We also have some names that are not +very descriptive (`Statement`, `Work`, `Super_transaction`, `Ledger_builder`, +`Ledger_builder_controller`). Having a standardized and descriptive set of names +in our codebase will increase the initial readability, reduce potential areas of +confusion, and increase our ability to communicate concept consistently (in +person, in code, and in documentation). + +# Detailed design + +[detailed-design]: #detailed-design + +### Merkle trees + +| Current Name | Description | New Name | +| ----------------- | -------------------------------------------- | ----------------------- | +| `Ledger` | Interface into merkle tree of account states | `Account_db` | +| `Ledger_hash` | Root hash of a `Account_db` | `Account_db_root` | +| `Merkle_ledger` | In memory implementation of `Account_db` | `Volatile_account_db` | +| `Merkle_database` | On disk implementation of `Account_db` | `Persistent_account_db` | +| `Syncable_ledger` | Wrapper of `Account_db` to sync over network | `Sync_account_db` | +| `Genesis_ledger` | The initial `Account_db` for the protocol | `Genesis_account_db` | + +### States + +| Current Name | Description | New Name | +| --------------------- | ------------------------------------------------------------------- | -------------------- | +| `Parallel_scan_state` | State of a series of parallel scan trees | " | +| `Ledger_builder` | State of `Parallel_scan_state` + `Transaction_work` | `Pending_account_db` | +| `Ledger_builder_aux` | Auxillary datastructure of `Pending_account_db` | `Work_queue` | +| `Blockchain_state` | State of `Account_db` root and `Pending_account_db` root at a block | `Account_db_state` | +| `Consensus_state` | Consensus mechanism specific state at a block | " | +| `Protocol_state` | The `Account_db_state` and `Consensus_state` at a block | " | +| `Tip` | The `Protocol_state` and `Pending_account_db` at a block | `Full_state` | + +### Transitions + +| Current Name | Description | New Name | +| --------------------- | ---------------------------------------------------- | -------------------------------- | +| `Snark_transition` | Subset of `Full_state_transition` that is snarked | `Provable_full_state_transition` | +| `Internal_transition` | State transition on full states | `Full_state_transition` | +| `External_transition` | State transition on lite states; sent to other nodes | `Protocol_state_transition` | + +### Transactions + +| Current Name | Description | New Name | +| ------------------- | ----------------------------------------------------- | ------------- | +| `Super_transaction` | ADT for all types of account state transitions | `Transaction` | +| `Transaction` | Transaction for payment between accounts | `Payment` | +| `Fee_transfer` | Transaction for distributing work fees | `Fee` | +| `Coinbase` | Transaction for new currency added each `Block_trans` | `Coinbase` | + +### Snarks + +| Current Name | Description | New Name | +| ------------------- | ---------------------------------------------------------------- | ----------------------------- | +| `Statement` | A snark proving the application of a single `Txn` to an `Acc_db` | `Transaction_statement` | +| `Work` | A collection of one or two `Txn_statement`s | `Transcation_work` | +| `Transaction_snark` | The snark which proves `Txn_statement`s | `Transaction_snark` | +| `Blockchain_snark` | The snark which proves `Full_state_trans`s on `Full_state`s | `Full_state_transition_snark` | + +### Primary components + +| Current Name | Description | New Name | +| --------------------------- | ----------------------------------------------------------------------------- | --------------------- | +| `Ledger_builder_controller` | Maintains locked `Full_state` and forks of potential future `Full_state`s | `Full_state_frontier` | +| `Proposer` | Proposes new blocks | " | +| `Snark_worker` | Generates `Transaction_work`s | `Snarker` | +| `Prover` | Proves `Full_state_transition`s | " | +| `Verifier` | Verifies `Full_state_transition_snark`s | " | +| `Micro_client` | A node which only tracks `State_summary` a limited number of account balances | " | + +# Drawbacks + +[drawbacks]: #drawbacks + +This may cause some initial headache while the development team gets used to the +new vocabulary. + +# Rationale and alternatives + +[rationale-and-alternatives]: #rationale-and-alternatives + +There are many alternative naming patterns that we could choose. The important +thing is that we all come to consensus as a team with something we all like (or, +if that's impossible, at least something the majority like and the remaining +don't hate). diff --git a/website/docs/researchers/rfcs/0004-style-guidelines.md b/website/docs/researchers/rfcs/0004-style-guidelines.md new file mode 100644 index 000000000..96fd1d9a0 --- /dev/null +++ b/website/docs/researchers/rfcs/0004-style-guidelines.md @@ -0,0 +1,183 @@ +--- +format: md +title: "RFC 0004: Style Guidelines" +sidebar_label: "0004 Style Guidelines" +hide_table_of_contents: false +--- + +> **Original source:** +> [0004-style-guidelines.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0004-style-guidelines.md) + +# Summary + +[summary]: #summary + +Commit to a semi-formal, extended ocaml style guidelines, with a focus on +defining how scalably use modules in our system. + +# Motivation + +[motivation]: #motivation + +We have been informally following ocamlformat + the janestreet style guidelines +so far. This has been working fairly well, however, there is currently no +definition around module/functor naming and usage standards. By defining that, +we hope to be able to refactor our current module/functor structure to be more +consistent, and hopefully eliminate some of the current headache in the +codebase. + +# Detailed design + +[detailed-design]: #detailed-design + +See `docs/style_guidelines.md` for the style guidelines draft. + +# Rationale and alternatives + +[rationale-and-alternatives]: #rationale-and-alternatives + +Most of the new styleguide follows inline with what we have been doing +informally as a team. However, there are a couple of new points which represent +changes in our codebase moving forward. + +The first is the "no monkeypatching" rule. The rationale for this is explained +already in the style guidelines, so I will not reiterate that here. + +The second is the new functor patterns. Functors which abstract over types need +to have some way to reconcile the equality of those types at different layers of +the codebase. Specifically, if you have two separate functors which return +modules that operate off of the same values, these values are incompatible +unless you make a statement about their equality. Even if you are aware of the +implementation being passed into each functor directly, the type system requires +hints. The tool for doing this is the `with` syntax, but this syntax is very +flexible, and it can be difficult to decide which syntax to use in which +situation. Let's start by reviewing the ways that `with` can be used. + +Each `with` declaration has a two dimensions of decisions, each of which is a +boolean domain. In total, therefore, there are 4 combinations which can be used +in a `with` statement. The first dimension is what the `with` will state an +equality on. A `with` declaration can state an equality between two types, or +two modules. The second dimension is how that equality should effect the newly +composed signature. The equality can either state that the new signature should +still contain the module/value, or it can state that the new signature should +strip the equality and, instead, replace all instances of the module/value in +the signature with the equality. + +For clarity, take a look at this example, which contains all 4 possible +combinations of a `with` statement. + +``` +module Type_equality = struct + module type S = sig + type a + type b + val f : a -> unit + val g : b -> unit + end + + module Inline (X : X.S) : S with type a = X.a and type b = X.b + (* return signature becomes: + * sig + * type a = X.a + * type b = X.b + * val f : a -> unit + * val g : b -> unit + * end + *) + + module Replace (X : X.S) : S with type x := X.a and type b := X.b + (* return signature becomes: + * sig + * val f : X.a -> unit + * val g : X.b -> unit + * end + *) +end + +module Module_equality = struct + module type S = sig + module X : X.S + val f : X.t -> unit + end + + module Inline (X : X.S) : S with module X = X + (* return signature becomes: + * sig + * module X = X + * val f : X.a -> unit + * val g : X.b -> unit + * end + *) + + module Replace (X : X.S) : S with module X := X + (* return signature becomes: + * sig + * val f : X.a -> unit + * val g : X.b -> unit + * end + *) +end +``` + +Each of the options on these dimensions have their own tradeoffs to consider. +Let's begin by comparing type in place equality and replacement equality. In +place equality `=` has the advantage that any equalities are still observable +from the newly generated signature. This can be really useful when you are +composing many modules together with many common dependencies. If replacement +equality is used, equality statements about a set of modules with common +dependencies must be duplicated in all functors which reason about those module +signatures. If the equalities are left in place, then the common dependencies +can be described as equalities in all locations without needing to pass around +any of the common dependencies. To state this more formally, take an example +where there is a module `X : X_intf`, and two functors +`A (X : X_intf) : A_intf with module X = X` and +`B (X : X_intf) : B_intf and module X = X`. If there is a 3rd functor which +operates off of modules generated by `A` and `B` from `X`, we can define it with +an interface that maintains compatiblity between `A.X` and `B.X` by declaring it +as +`C (A : A_intf) (B : B_intf with module X = A.X) : C_intf with module X = A.X`. +If, instead, we had chosen to define the `with` declarations on functors `A` and +`B` with `:=`, the functor `C` would require a definition like +`C (X : X_intf) (A : A_intf with module X := X) (B : B_intf with module X := X) : C_intf with module X := X`. +As we continue to add more common dependencies between `A`, `B`, and `C`, we +will have to keep on adding functor arguments and `with` declarations to `C`. + +Replacement equality `:=` has the advantage that the generated signature is +narrowed in scope. This can be useful whenever you want to compose a smaller +signature from a larger one. In our codebase, however, we only ever need to do +this when we are calling `include` on a module which's signature already defines +a module we have in scope of the current module structure. Outside of this, `=` +appears to be superior, especially if you are always going to talk about a +module signature at a level that allows you to still wire common dependencies +together via equalities (which is how we consistently are using modules and +signatures throghout our codebase). + +Now, let's take a look at type equalities vs module equalities. Type equalities +provide a finer grain of atomicity. Module equalities typically require less +equality statements, but at the cost that the grain of atomicity is defined on a +per signature basis. Let's think about an example to help make this clearer. +Let's say we have 4 modules: `A`, `B`, `C`, and `D`. `B` depends on `A`, and `D` +depends on all three modules `A`, `B`, and `C`. Using type equality, we would +need to say that a functor creating `D` returns +`D_intf with type a = B.a and type b = B.t and type c = C.t` (assuming `B` uses +the same pattern for `A` as `D` does for `A`, `B`, and `C`). However, if we +instead use module equalities, the returning interface of the `D` functor could +be expressed as `D_intf with module B = B and module C = C`. We do not need to +express an equality on `D_intf.A` since `D_intf.B` contains `D_intf.B.A` and +`D_intf.B = B`, therefor `D_intf.B.A = B.A`. This scales much better as more and +more dependencies are nested. With type equality, every layer we go up in +dependencies requires more and more equality declarations. + +As mentioned earlier, the disadvantage of using module equalities is that the +grain of atomicity for a dependency becomes tied to the decomposition of its +signatures. Right now, in our codebase, we don't do much signature +decomposition. However, independent of this issue, we should move towards +decomposing signatures anyway. In fact, it's part of the janestreet styleguide. +Signatures should almost never be written entirely by hand, but instead should +compose smaller, finer grained signatures in order to build up the majority of +what it requires. Using this pattern, we can still limit the surfaced +requirements of dependencies while gaining the advantages of module equality. +When viewed from this perspective, in fact, tying the grain of atomicity to the +signature decomposition of a dependency actually becomes a pro instead of a con, +since we as developers get more control over the level of abstraction our +signatures should have over the underlying structure. diff --git a/website/docs/researchers/rfcs/0005-issue-labels.md b/website/docs/researchers/rfcs/0005-issue-labels.md new file mode 100644 index 000000000..cc9f8f1b2 --- /dev/null +++ b/website/docs/researchers/rfcs/0005-issue-labels.md @@ -0,0 +1,113 @@ +--- +format: md +title: "RFC 0005: Issue Labels" +sidebar_label: "0005 Issue Labels" +hide_table_of_contents: false +--- + +> **Original source:** +> [0005-issue-labels.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0005-issue-labels.md) + +# Summary + +[summary]: #summary + +Currently, our issue labels are ad-hoc and somewhat confusing. This defines +several categories of label, and how to use them. + +# Detailed design + +[detailed-design]: #detailed-design + +Here are the categories of issue label. Each label is composed of the category +name and the label name. For example, `area-snark` or `priority-critical`. + +- `area-`. This narrows general area of the codebase or feature that this issue + impacts. + - `build` + - `catchup` + - `client` + - `consensus` + - `daemon` + - `docs` + - `gossip` + - `monitoring` + - `proposer` + - `protocol` + - `sdk` + - `snark` + - `snarker` + - `testnet` + - `tests` + - `ux` + - `website` + +- `impact-`. This describes the general impact of the issue. + - `bandwidth`: uses too much bandwidth + - `crash`: there's a crash + - `disk`: uses too much disk space + - `dos`: liable to a remote denial of service attack + - `insecure`: an attacker can exploit this for nefarious ends + - `latency`: takes too long + - `memory`: uses too much memory + - `slow`: our code is slower than it could be + +- `effort-`. This estimates how much experience (with OCaml/the + codebase/whatever) is necessary to resolve an issue. + - `easy`: not much experience necessary + - `hard`: a lot of experience necessary, will be challenging + - `medium`: some experience necessary + +- `category-`. General, broad-stroke categories useful for filtering the issue + list. + - `bug` + - `duplicate` + - `enhancement`: not necessarily a feature, and might fix some bugs, but + improves some aspect of the codebase. + - `feature-request` + - `mentored`: this issue has someone who can mentor contributors through + fixing it + - `quick-fix`: the necessary change has already been identified, and it will + take someone already familiar with the codebase less than around half an + hour to implement it. + - `refactor` + - `regression`: this issue was previously already fixed. + - `rfc` + +- `priority-`. How urgent it is to fix the issue. + - `critical` + - `high` + - `low` + +- `status-`. This indicates something about the current status of the issue. + - `cannot-repro`: reproduction attempts failed + - `needs-info`: issue isn't detailed enough to act on + - `needs-repro`: needs to be reproduced + - `needs-rfc`: this change needs nontrivial design work that should be done in + an RFC. + +- `planning-`. When we plan on doing this + - `after-testnet`: after the public testnet is released + - `for-testnet`: before the public testnet is released + +# Rationale and alternatives + +[rationale-and-alternatives]: #rationale-and-alternatives + +The idea with most of these categories is to partition the issues along various +axes to make it easy to filter and track them. The labels within most categories +aren't intended to be mutually exclusive. The main exception is "Priority". + +Keep in mind that only repo collaborators can add/remove issues. Maintainers +should apply labels to new issues as appropriate. + +# Prior art + +[prior-art]: #prior-art + +- Rust: https://github.com/rust-lang/rust/labels +- geth: https://github.com/ethereum/go-ethereum/labels +- aleth: https://github.com/ethereum/aleth/labels +- Electron: https://github.com/electron/electron/labels +- Tezos: https://gitlab.com/tezos/tezos/labels +- ZCash: https://github.com/zcash/zcash/labels diff --git a/website/docs/researchers/rfcs/0006-receipt-chain-proving.md b/website/docs/researchers/rfcs/0006-receipt-chain-proving.md new file mode 100644 index 000000000..c9f9e8e92 --- /dev/null +++ b/website/docs/researchers/rfcs/0006-receipt-chain-proving.md @@ -0,0 +1,147 @@ +--- +format: md +title: "RFC 0006: Receipt Chain Proving" +sidebar_label: "0006 Receipt Chain Proving" +hide_table_of_contents: false +--- + +> **Original source:** +> [0006-receipt-chain-proving.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0006-receipt-chain-proving.md) + +# Summary + +[summary]: #summary + +A method for a client to prove that they sent a transaction by showing a +verifier a Merkle list of their receipt chain from their latest transaction to +the transaction that they are trying to prove. For this RFC, a Merkle list +contains a list of hashes such that the property that for an arbitrary hash in +the list, `h`, there exists a value x, such that $h = hash(x, h.prev) $ + +# Motivation + +[motivation]: #motivation + +We are constructing a cryptocurrency protocol and verifying the state of our +succinct blockchain takes constant time. As a consequence of achieving this +constant-time lookup, we do not keep a record of the transactions that occur +throughout the history of the protocol. + +In some cases, it would be helpful to have a client prove that their transaction +made it into the blockchain. This is particularly useful if a client wants to +prove that they sent a large sum of money to a receiver. + +# Design + +[detailed-design]: #detailed-design + +Each account has a `receipt_chain_hash` field, which is a certificate of all the +transactions that an account has send. If an account has send transactions +`t_n ... t_1` and `p_n ... p_1` is the hash of the payload of these +transactions, then the `receipt_chain_hash`, $r_n$, is equal to the following: + +$$ r*n = h(p_n, h(p*{n - 1}, ...)) $$ + +where `h` is the hash function that determines the `receipt_chain_hash` that +takes as input the hash of a transaction payload and it's preceding +`receipt_chain_hash`. + +For the base case, $ r_0 $ equals to the empty hash for `receipt_chain_hash`. + +A client can prove that they sent transaction `t_k` if they store the Merkle +list of `r_k` to their most recent `receipt_chain_hash`, `r_n`. We do this +because any verifier can look up the current state of the blockchain and see +that their `receipt_chain_hash` is `r_n`. Then, they can recursively show that +$r_i = h (t_{i}, r_{i-1})$ until $r_{k}$. + +The client needs to store these transactions and receipt hashes and the data +should be persisted after the client disconnects from the network. At the same +time, the client needs to easily traverse the transactions and receipt chain +hashes that lead to the current state of the blockchain. A client can easily +achieve this by using a key-value database. The keys of the database are the +receipt hash and the value has the following fields: + +```ocaml +type value = Root | Child of {parent: receipt_chain_hash; value: transaction} +``` + +The value entry can be seen as a tree node structure that refers to the previous +receipt hash that use the key. Note that transactions can fork from other +transactions and the key-value entries can simulate this since multiple tree +nodes can refer to the same `prev_node`. The node also holds the transaction +that was last hashed to produce the key hash. From the hash that we are trying +to prove, `resulting_receipt`, we can iteratively traverse down to `parent` to +find the `receipt` that are trying to prove. Note that these nodes form an +acyclic structure since a hashing function should be collision-free. + +The signature for this structure can look like the following: + +```ocaml +module type Receipt_chain_database_intf = sig + type receipt_chain_hash [@@deriving bin_io, hash] + + type transaction [@@deriving bin_io] + + val prove : proving_receipt:receipt_chain -> resulting_receipt:receipt_chain -> (receipt_chain * transaction) list Or_error.t + + val add : previous:receipt_chain + -> transaction + -> [ `Ok of receipt_chain_hash + | `Duplicate of receipt_chain_hash + | `Error_multiple_previous_receipts of receipt_chain_hash ] +end +``` + +`prove` will provide a merkle list of a proving receipt `h_1` and its +corresponding transaction `t_1` to a resulting*receipt `r_k` and its +corresponding transaction `t_k`, inclusively. Therefore, the output will be in +the form of [(t_1, r_1), ... (t_k, r_k)], where $r_i = h(t*{i}, r\_{i-1})$ for +$i = 2...k$ + +`add` stores a transaction into a client's database as a value. The key is +computed by using the transaction payload and the previous receipt_chain_hash. +This receipt_chain_hash is computed within the `add` function. As a result, the +computed `receipt_chain_hash` is returned + +# Drawbacks + +[drawbacks]: #drawbacks + +The issue with this design is that a client has to keep a record of all the +transactions that they have sent to prove that they sent a certain transaction. +This can be infeasible if the user sends many transactions. On top of this, they +have to show a Merkle list of the transaction that they are interested in +proving to the latest transaction. This can take O(n), where `n` is the number +of transactions a sender has sent. + +Another drawback of this is that it assumes that a peer will send transactions +from one machine. Whenever a node sends transactions on multiple machines, they +can sync up together their machines and share which transactions they have sent. + +The simplest solution to achieve this sync for users is to store their data in a +cloud database, such as Amazon's [RDS](https://aws.amazon.com/rds/) and +[AppSync](https://aws.amazon.com/appsync/). We can create a wrapper for querying +and writing to the database that conforms to the schema of our database +requirements and the user hooks up their cloud database to our wrapper. A user +can also store their data locally as a cache so that they can make fewer calls +to their cloud database. + +# Prior Art + +[prior-art]: #prior-art + +In Bitcoin, a sender can prove that their transaction is in a block. This can be +a confirmation. Once a block is mined on top of another block, there will be 2 +confirmations that this transaction is valid. If there is another block on top +of that block, there will be 3 confirmations for that block. Typically, if a +transaction has 6 confirmation, there is an extremely low chance that the +bitcoin network will fork and ignore the existence of the transaction. + +# Unresolved Questions + +[unresolved-questions]: #unresolved-questions + +What are some other protocols that we can use to sync receipt chain database +from multiple servers? + +How do we prune forked transaction paths? diff --git a/website/docs/researchers/rfcs/0007-delegation-of-stake.md b/website/docs/researchers/rfcs/0007-delegation-of-stake.md new file mode 100644 index 000000000..28f91d504 --- /dev/null +++ b/website/docs/researchers/rfcs/0007-delegation-of-stake.md @@ -0,0 +1,196 @@ +--- +format: md +title: "RFC 0007: Delegation Of Stake" +sidebar_label: "0007 Delegation Of Stake" +hide_table_of_contents: false +--- + +> **Original source:** +> [0007-delegation-of-stake.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0007-delegation-of-stake.md) + +# Summary + +[summary]: #summary + +We want to support the possibility of delegating one's stake to another +public-key, so that the delegate's probability of winning increases (with the +intention, whether it be enforced on chain or not, that they would receive some +of the block reward as in a mining pool). + +Some goals for such a design are the following: + +- We want as much stake used directly or delegated to active stakers as + possible. This is how the security of the network is ensured. +- It should not be too expensive inside the SNARK. +- It should not be too expensive outside the SNARK. + +# Detailed design + +[detailed-design]: #detailed-design + +In general, let's model the "delegation state" of Coda at any given time as a +function `delegate : Public_key -> Public_key`. There are a few different +semantics we could hope to give delegation. + +1. Non-transitive stake delegation. The amount of stake delegated to public-key + `p` is `\sum_{x \in delegate^{-1}(p)} stake(x)`. I.e., people are partitioned + according to their delegate, and that delegate's virtual stake is the sum of + the delegators' stake. +2. Transitive stake delegation. Let's say a key is a "terminal delegate" if + `delegate p = p`. For such `p`, say `q` is a delegator of `p` if + `delegate q = p` or if for some `q'` a delegator of `p` we have + `delegate q = q'`. Then `p`'s virtual stake is defined to be the sum of its + delegates' stakes. + +`2` seems _very_ difficult to implement and it's dubious to me whether it would +be much better than `1`, at least for now. + +As such, here are three non-transitive designs. Discussion on this has converged +on going with design 1 for now. + +## Design 1: Better in the SNARK, but worse everywhere else + +- Add to `Account.t` a field `delegate : Public_key.Compressed.t` +- Create a new transaction type which allows one to set this field (maybe we + stuff a fee transfer in there too so we don't waste the merkle lookup...). +- A proposer maintains a list of all the people who are delegating for them, + incrementally updating it when they see new transactions. +- When the randomness and ledger for the next epoch is locked in, the proposer + performs one VRF evaluation for each account delegating to them. They evaluate + on the concatenation of whatever we were evaluating it on before, plus the + merkle-tree address of the delegator's account (this is just cheaper than + using the public-key). +- When a proposer extends the blockchain, they can use any of the VRF + evaluations made in the prior step. + +The main reason this design is bad is that you have to perform a number of VRF +evaluations proportional to the number of people staking for you, and in +practice it might be costly causing you to not even evaluate the VRF on smaller +accounts which have delegated to you. You also need to maintain a large amount +of additional state (the set of people delegating to you). + +The main thing this design has going for it is that it would barely increase the +size of the circuit. It's also nice that it makes it very easy to delegate your +stake. Just set it and forget it. + +## Design 2: Worse in the SNARK, but better everywhere else. + +- Add to `Account.t` two fields + - `delegate : Public_key.Compressed.t` + - `delegated_stake : Currency.Amount.t` + + The invariant we maintain is that for any account corresponding to public key + $p$, `delegated_stake` is the sum + `\sum_{a : Account.t, a.delegate = p} a.balance`. + +- Create a new transaction type which allows one to set the `delegate` field. It + would also have the following effect: + +```ocaml +apply_transaction (Set_delegate new_delegate) account = + let old_delegate = account.delegate in + let old_delegate_account = find old_delegate in + set account.public_key + { account with delegate = new_delegate }; + set old_degate + { old_delegate_account with delegated_stake -= account.balance }; + set new_delegate + { new_delegate_account with delegated_stake += account.balance }; +``` + +so basically 3 Merkle lookups. + +- Change normal transactions to increment/decrement the `delegated_stake` fields + of delegates of accounts modified as needed. This will require them to do the + following lookups: + - sender + - sender delegate + - receiver + - receiver delegate + + I predict this means that Base transaction SNARKs will be like ~1.5-2x more + costly. + +- Just evaluate the VRF on one's own account, but with the threshold being based + on `delegated_stake` rather than balance. + +The main thing this design has going for it is you only need to do one VRF +evaluation. + +## Design 3: Not worse in the SNARK, with similar advantages to design 2, but delegation is explicit. + +- Add to `Account.t` three fields + - `delegate : Public_key.Compressed.t` + - `delegated_to_me : Currency.Amount.t` + - `delegated_by_me : Currency.Amount.t` + + The invariants we maintain are + - For any account corresponding to public key `p`, `delegated_stake` is the + sum `\sum_{a : Account.t, a.delegated_by_me = p} a.delegated_to_me`. + - For any account `a`, `a.delegated_by_me <= a.balance`. + +- Normal transactions do not affect the new fields. +- We will have a new transaction `Update_delegated_stake (p, delta)` with the + following semantics: + +```ocaml +apply_transaction (Update_delegated_stake (p, delta)) = + let account = find p in + let delegated_by_me = account.delegated_by_me + delta in + assert (delegated_by_me <= account.balance); + set p { account with delegated_by_me = delegated_by_me }; + let delegate = find p.delegate in + set p.delegate { delegate with delegated_to_me += delta }; +``` + +This has two Merkle lookups so we can shoehorn this into the existing base +SNARK. + +- We will have a new transaction `Set_new_delegate (p, new_delegate)` with the + following semantics + +```ocaml +apply_transaction (Set_new_delegate (p, new_delegate)) = + let account = find p in + set p + { account with delegate = new_delegate; delegated_by_me = 0 }; + let old_delegate_account = find account.delegate; + set account.delegate + { old_delegate_account with delegated_to_me -= account.delegated_by_me }; +``` + +- We basically waste one Merkle lookup when switching ones' delegation, but it + isn't so bad. (We do in 4 lookups what we could do in 3.) +- VRF evaluation is checked against the threshold implied by `delegated_to_me`. + +# Rationale and alternatives + +[rationale-and-alternatives]: #rationale-and-alternatives + +This approach has a lot of disadvantages. + +The main alternative (at a high level) is to use transitive stake delegation. +This is hard because checking the `is_delegate` relation seems (in principle) to +require one of + +- touching O(n) accounts whenever someone changes their delegate, +- touching O(n) accounts whenever you want to confirm that someone is a + transitive delegate of someone else. + +Which makes it a non-starter in the SNARK, where we need to do both. + +I think design 3 has acceptable trade-offs in terms of usability (you have to +explcitly update your delegation amount which sucks), but doesn't hurt the +efficiency of the SNARK and we still only have to do one VRF evaluation outside +the SNARK. + +# Prior art + +[prior-art]: #prior-art + +Prior art is pretty scarce, but here are Tezos and Cardano's docs. + +- Tezos: [Docs](https://tezos.gitlab.io/active/proof_of_stake.html#delegation). + As far as I can tell uses non-transitive stake delegation. +- Cardano: [Docs](https://cardanodocs.com/technical/delegation). Uses transitive + stake delegation. diff --git a/website/docs/researchers/rfcs/0008-persistent-ledger-builder-controller.md b/website/docs/researchers/rfcs/0008-persistent-ledger-builder-controller.md new file mode 100644 index 000000000..26ea758ef --- /dev/null +++ b/website/docs/researchers/rfcs/0008-persistent-ledger-builder-controller.md @@ -0,0 +1,147 @@ +--- +format: md +title: "RFC 0008: Persistent Ledger Builder Controller" +sidebar_label: "0008 Persistent Ledger Builder Controller" +hide_table_of_contents: false +--- + +> **Original source:** +> [0008-persistent-ledger-builder-controller.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0008-persistent-ledger-builder-controller.md) + +# Summary + +[summary]: #summary + +This adds a persistent backend for the ledger builder controller. + +# Motivation + +[motivation]: #motivation + +Having a persistent backend will allow nodes that go offline, whether for a +short period or an extended period of time, to catch back up to the network with +as little work as reasonably possible. + +# Detailed design + +[detailed-design]: #detailed-design + +In order to achieve this, we will persist the locked tip using the existing +Merkle database. The backend Merkle database is designed in a way such that +queries for the most commonly performed operations can be batched together as +one request to the database. + +The best tip, materialized by the ledger builder controller, will be represented +as a Merkle ledger mask on top of the persistent Merkle ledger database. This +data structure will only store the modified nodes of the Merkle ledger in +memory, and defer to the contents of the persistent Merkle ledger database for +any information outside of that. The persistent Merkle ledger database will +maintain a reference to all of these data structures derived for it. These +references will be used to garbage collect outdated information in the mask +representations whenever changes are written to the persistent copy. In other +words, information will be removed from the mask whenever it is no longer +different from the information in the underlying database, ensuring that the +data structure will not leak memory. If a mask is empty, it may need to be +retained, in case there are existing references to it. + +There may be several points between the locked tip and the best tip, and we may +need to examine any such point. For performance reasons, we'll daisy-chain masks +on top of one-another all throughout the tree. Whenever an underlying db +updates, changes should only be propagated to the immediate mask-children of +that DB. Because masking is associative, we can merge mask-children in with +there parents without needing to notify any children. In +[RFC 0009](0009-transition-frontier-controller.md), we take advantage of this +property. + +These representation choices involve trade-offs that should be considered. The +Map representation contains just key and value data, while the prefix tree +representation add summary hashes, which could consume more memory. To find +account data with the Map, where the account does not appear in that mask, would +mean first consulting the mask, which fails, then making an additional database +lookup. That could be slower than a lookup in the prefix tree, where summary +hashes could be used to guide the database lookup without a failure step. + +There's an existing Sync functor that takes a database module, to perform +catchup operations. The masks just described work also with a database, so a +MakeMask functor that takes a database module would be a way to make this setup +work. The database module that's passed in is not copyable, but both the Sync +and Diff functors will offer a copy function to copy their local data, leaving +the underlying database unaffected. + +Later, a cache can be introduced onto the Merkle ledger database, allowing us to +perform fewer database queries for commonly looked up information. The details +of caching, however, is not addressed as part of this particular RFC. See +[Use Merkle mask as a proxy](https://github.com/CodaProtocol/coda/issues/1073), +which requests such a feature. + +The module type for the mask could be: + +```ocaml +module type Mask = sig + type t + type key + type value + + type db = Database_intf.S.t + + (* group identity, operations *) + val empty : t + val add : t -> t -> t + val inverse : t -> t + + val insert : key -> value -> t -> t + val remove : key -> t -> t + val from_list : (key * value) list -> t + + (* look up in diff; if key not present, delegate to database for locked tip *) + val get : db -> key -> value option +end +``` + +# Drawbacks + +[drawbacks]: #drawbacks + +The drawback of this is that the persistent Merkle database is not easy to copy. +However, this is by design, as there is only one copy that should be persisted +as the central source of truth. Any other information which we wish to persist +should live outside of this representation. For instance, if we want to persist +the best tip, then only the diff between that tip's database and the locked +tip's database should be stored. This is much more efficient for both storage +and quick synchronizing, in the case of short term downtimes. + +# Rationale and alternatives + +[rationale-and-alternatives]: #rationale-and-alternatives + +An alternative design is to restructure the persistent database so that it can +be cheaply copied. This can be done by keying nodes not by their path in the +Merkle tree and instead use keys that directly identify nodes, and only copy the +nodes that are modified for the copied database. However, there are a few issues +with this. For one, it makes many of the common queries to the data structure +far less efficient. If you want to find a leaf given a path, you would need to +make `k` successive requests to the database, where `k` is the depth of your +tree, versus the single request of bulk lookups required by the other +representation. This concern can be alleviated with more indices into the +database, but those in turn also introduce more request requirements, and also +increase the cost of garbage collection. The second reason is that garbage +collection could prove challenging. The main logic behind when something is +considered garbage is simple: keep a ref count for each node, and when it's +zero, delete it. But, that adds cost to each operation on nodes; or, +alternatively, requires us to perform collection cycles in our scheduler. This +also becomes exponentially more complex as we add in indices to make the +representation more efficient, which will be necessary. Thirdly, this +representation adds even more accesses to the database, since you need to do all +work with ledger through the database. This will make caching more important for +efficiency, and if you are already going to represent the active part of the +ledger in memory via a cache, why not just opt for the more efficient +representation on the persistent side and use an in memory abstraction for +representing the active part of a ledger. + +# An algebra of masks + +It may be useful if there are operations on masks with nice mathematical +properties. For example, there could be an addition operation on masks that +results in the composition of those masks. To "undo" a mask, one could produce +the negation of a mask. These ideas suggests an algebraic group, where the +carrier set is the set of masks. diff --git a/website/docs/researchers/rfcs/0009-transition-frontier-controller.md b/website/docs/researchers/rfcs/0009-transition-frontier-controller.md new file mode 100644 index 000000000..9c3ab614e --- /dev/null +++ b/website/docs/researchers/rfcs/0009-transition-frontier-controller.md @@ -0,0 +1,376 @@ +--- +format: md +title: "RFC 0009: Transition Frontier Controller" +sidebar_label: "0009 Transition Frontier Controller" +hide_table_of_contents: false +--- + +> **Original source:** +> [0009-transition-frontier-controller.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0009-transition-frontier-controller.md) + +## Summary + +[summary]: #summary + +Refactoring the existing ledger-builder-controller to a new simpler set of +components we're calling the transition-frontier-controller. At the top level, +we'll connect these components together in a similar manner to `mina_lib` -- +just wiring together pipes. The new transition-frontier-controller includes the +merkle-mask and a safer history-catchup mechanism in addition to simplifying +asynchronous logic. The goal is to end up with a simple set of components that +will be robust towards future requirements changing and easy to trace dataflow +and test and debug. + +## Motivation + +[motivation]: #motivation + +We are refactoring the existing ledger-builder-controller in order to make it +easier to reason about the async control flow of information in the system. The +old system was not designed carefully taking asynchronous transactions and +network delays into account. These involved lots of concurrency, and so it's +very hard to trace information and there are subtle bugs. + +Since refactoring the control flow requires a rewrite, we decided to think +carefully about more of the other pieces as well now that we know more about the +design of the rest of the protocol code. + +The new transition-frontier-controller is responsible for handling incoming +transitions created by proposers either locally or remotely by: (a) validating +them, (b) adding them to a `Transition_frontier` as described below if possible, +if not, (c) carefully triggering a sync-ledger catchup job to populate more +information in our transition-frontier and completing the add. It is also +responsible for handling sync-ledger answers that catchup jobs started by other +nodes trigger via sending queries. + +The expected outcome is to have something with feature parity to the existing +ledger-builder-controller from the outside. Internally, aside from having rigor +around the async control flow, we also will take advantage of merkle masks to +remove a large chunk of complex logic from the existing +ledger-builder-controller code. + +It should also be much easier to unit-test and trace the flow of information for +debugging purposes. + +## Detailed design + +[detailed-design]: #detailed-design + +### Introduction + +`Ledger_builder_controller` was far too big in its scope. In this rewrite, we're +not only breaking apart the code into more modules, but also breaking it up +across several libraries. + +There will be a new top-level `Transition_frontier_controller` module that will +just wire the pipes together for the new components: + +- [Transition Handler](#transition-handler) +- [Transition Frontier](#transition-frontier) +- [Ledger Catchup](#ledger-catchup) +- [Query Handler](#query-handler) + +Note that all components communicate between each other solely through the use +of pipes. By solely using pipes to connect components we can be more explicit +about the [Async Control Flow](#async-control-flow). + +After describing all the components, we'll cover the specific behavior of the +[Async Control Flow](#async-control-flow) holistically. + +Before covering any of the components it's worth going over some fundamental +changes in behavior and structure in more detail in [big Changes](#big-changes) + +### Big Changes + +Major differences in behavior between this component and the existing ledger- +builder-controller are as follows: + +1. We only sync-ledger to the locked-ledger (if necessary) and do a new + [History sync](#history-sync) for catchup jobs + +Rationale: Without history sync, we are vulnerable to believing an invalid +state: Certain parts of the staged-ledger state are not checked in a SNARK and +so we must get enough info to go back to the locked state (where we know we've +achieved consensus). + +2. There is now a notion of a `Breadcrumb.t` which contains an + `External_transition.t` and a now light-weight `Staged_ledger.t` it also has + a notion of the prior breadcrumb. We take advantage of the merkle-mask to put + these breadcrumbs in each node of the transition-tree. See + [RFC-0008](0008-persistent-ledger-builder-controller.md) for more info on + merkle masks, and [Transition frontier](#transition-frontier) for more info + about how these masks are configured. + +Rationale: The frequent operation of answering sync-ledger queries no longer +require materializing staged-ledgers. We should optimize for operations that +occur frequently. + +3. The `Ktree.t` is mutable and exposes an $O(1)$ + `lookup : State_hash.t -> node_entry` where `node_entry` contains (in our + case) a `Breadcrumb.t`. + +Rationale: We perform `lookup` and `add` frequently on this structure and `add` +can be $O(1)$ in the presence of this `lookup` too. `lookup` also lets us +traverse the tree forwards and backwards without explicit backedges in our +[rose tree](https://en.wikipedia.org/wiki/Rose_tree) -- further simplifying the +`Ktree.t` implementation. + +### Transition Handler + +The transition handler is broken down into two main components: Validator and +Processor + +#### Validator + +- Input: `External_transition.t` +- Output: `External_transition.t` (we should consider making an + `External_transition.Validated.t` for output here) + +The validator receives as input external transitions from the network and +performs a series of checks on the transition before passing it off to be +processed. For simplicity it can also receive transitions created by the local +proposer. In the future, we can consider skipping this step. + +Particularly, we validate transitions with the following checks (in this order): + +1. Checking for existence in the breadcrumb tree (we've already seen it) +2. Checking for consensus errors +3. Verifying the included SNARK + +This order is chosen because it's cheaper computationally to perform (1) and +(2). So if we get lucky we can short-circuit before getting to the SNARK step. + +We also should rebroadcast validated transitions over the network to our +neighbors. + +#### Processor + +- Input: `External_transition.t` (from validator) and `Breadcrumb.t list` (from + catchup) +- Outputs: `External_transition.t` (to ledger catchup); modifying transition + frontier + +The processor receives a single validated external transitions from the +validator and then attempts to add the transition to the breadcrumb tree. The +processor is the only "thread" allowed to write changes to the +[Transition Frontier](#transition-frontier). All other threads must delegate to +the processor. + +[Ledger Catchup](#ledger-catchup) needs to add a batch of transitions all at +once, so it shares some of the add section and constructs the underlying +breadcrumbs in such a way that only requires adding to transition frontier's +table and not applying an staged-ledger-diffs. + +#### Adding to Transition Frontier + +The add process runs in two phases: + +1. Perform a `lookup` on the `Transition_frontier` for the previous + `State_hash.t` of this transition. If it is absent, send to + [catchup scheduler](#catchup-scheduler). If present, continue. +2. Derive a mask from the parent retrieved in (1) and apply the + `Staged_ledger_diff.t` of the breadcrumb to that new mask. See + [Transition Frontier](#transition-frontier) for more. +3. Construct the new `Breadcrumb.t` from the new mask and transition, and + attempt a true mutate-add to the underlying + [Transition Frontier](#transition-frontier) data. + +#### Catchup Scheduler + +The catchup scheduler is responsible for waiting a bit before initiating a long +catchup job. The idea here is to mitigate out-of-order messages since it is much +quicker to avoid such catchups if possible. This will be a module that waits on +a small timeout before yielding to the ledger catchup component. And can be +preempted by some transition/breadcrumb that allows this transition to be +connected to the existing tree. + +### Transition Frontier + +The Transition Frontier is essentially a root `Breadcrumb.t` with some metadata. +The root has a merkle database corresponding to the snarked ledger. This is +needed for consensus with proof-of-stake. It then has the first mask layer +exposing the staged-database and staged-ledger for the locked-tip. Each +subsequent breadcrumb contains a light-weight staged-ledger with a masked merkle +database built off of the breadcrumb prior. We store a hashtable of breadcrumb +children and keep references to the root and best-tip hashes (which we can +lookup later in the table). + +It's two orders of magnitude more computationally expensive to constantly +coalesce masked merkle databases in the worst case than to just keep them +separated (back of napkin math). Moreover, we are able to in $O(1)$ time answer +sync ledger queries if we have staged ledgers at every position. + +### Ledger Catchup + +Input: `External_transition.t` (from catchup scheduler) Output: +`Breadcrumb.t list` (to processor) + +Ledger catchup runs a single catchup worker job at a time. Whenever catchup +scheduler decides it's time for a new catchup job to start, it will send +something on the pipe. + +#### Catchup worker + +A worker is responsible for performing a history sync. + +In the future, we will support multiple catchup workers at once. This prevents a +potential DoS attack where we're forced to catchup between two different forks +and can never complete anything. + +New jobs handed to the same worker cause a retargeting action where we don't +throw out existing data, but start again getting data from the source. + +#### History Sync + +History sync is a new process that is not yet implemented in the current +ledger-builder-controller. + +`history_sync transition` asynchronously walks backwards downloading each +external transition until either (1) it reconnects with some existing breadcrumb +or (2) it passes the locked slot without reconnecting. In either case, we then +walk forwards doing `Path_traversal` logic materializing masked staged-ledgers +all the way up the path. The materialized breadcrumb list gets sent to the +processor. + +The details of this download process are TBD. It will likely make sense to ask +for a manifest of locations for the transitions all at once and to do some sort +of torrenting solution. In the short term, we'll just naively download one +transition at a time. + +If we do pass the locked slot without reconnecting, we need to perform an +additional step of invoking the sync ledger process. + +#### Post Attachment + +We may receive additional external transitions that would connect to the +existing catchup job in process. Rather than dropping those we can buffer them +in a post-attachment pool. When a catchup job finishes we can dispatch a post +attachment job that can breadcrumbify these transitions. + +This should not block catchup breadcrumbs from being added to the +transition-frontier, however. + +### Query Handler + +Input: Sync ledger queries (from network); history sync queries (from network) +Output: Sync answers; history answers + +The query handler is responsible for handling sync ledger queries and history +sync queries coming from other nodes performing catchup. This component reads +state from the transition frontier in order to answer the questions. Given the +new underlying data structure, all answers will occur in $O(1)$ time. + +### Async Control Flow + +As part of this redesign we've carefully considered the asynchronous control +flow of the full system in an attempt to make it very easy to trace data flow. + +Consult the following diagram: + +![](https://github.com/MinaProtocol/mina-resources/blob/main/docs/res/transition_frontier_controller.dot.png) + +Blue arrows represent pipes and asynchronous boundaries of the system. Each +arrow is annotated with the behavior when overflow of the pipe occurs. + +- `exception` means raise an exception if pipe buffer overflows (`write_exn`) +- `blocking` means push back on a deffered if the buffer is full (`write`) +- `drop old` means the buffer should drain oldest first when new data comes in + +Red arrows represent synchronous lines of access. Red components represent data +and not processes. + +### Non-synchronous modification of the Transition Frontier + +This will be fleshed out here later, or in a separate RFC. @bkase has thought +about this, but we're not prioritizing this yet. This is required to prevent +against an adversary forcing us to switch between catching up of two forks. +Always preempting and cancelling one another. + +## Drawbacks + +[drawbacks]: #drawbacks + +This will take a decent amount of engineering time to implement. Additionally, +this design does not consider persisting parts of the frontier to disk. + +We are moving away from a functional representation of the underlying ktree to a +mutable one. This will require more careful managing of changes to the tree. +However, since we are taking care to think carefully about asynchronous +processes we shouldn't have a problem here. + +## Rationale and alternatives + +[rationale-and-alternatives]: #rationale-and-alternatives + +The main rationale for doing this redesign is to ease the difficulty of +debugging asynchronous control. And to avoid debugging all the edge-cases we've +never explored in the existing ledger-builder-controller (what happens in the +presense of particular looking forks). + +One alternative is to instead prioritize in investing in debugging +infrastructure such as visualizations and logging tooling to get through these +asynchronous bugs. However, we will need to do a medium-sized refactor anyway to +integrate the merkle masks properly and the new correct history catchup anyway, +so we're not actually saving much work. + +This seems to be a good point to do a full-rewrite given that we have these new +components we wish to hook in and we also are running into debugging issues and +new bugs. + +## Prior art + +[prior-art]: #prior-art + +The main piece of prior art is the existing Ledger-builder-controller component. +Ledger-builder-controller as is supports the same external interface and a lot +of similar behavior, but in a more adhoc way. The existing +ledger-builder-controller handles incoming transitions by spawning one of two +types of asynchronous jobs, a catchup and a path traversal. As a short-cut, +catchup did not differentiate between the "from nothing" case and the "small +miss" case, and was vulnerable to an attack. Path traversal was the process of +materializing a staged-ledger along a path in the ktree. This had to be +asynchronous because we didn't yet have a notion of a masked ledger. The +ledger-builder-controller kept only one async job running at a time for +simplicity as well, and only cancelled in-progress jobs when another should +replace the existing one. Upon the completion of a job, the +ledger-builder-controller would replace it's ktree and update annotated tip +entities that have materialized staged-ledgers in them. +Ledger-builder-controller also handled sync-ledger answers as it had to access +the path traversal logic and the current transition logic state. + +This component grew organically and did not have any rigorous design thought put +behind it as we did not realize how central such a component would be in our +system until it was too late. + +Moreover, we did not think carefully about the flow of information through this +component and the implications of various async points from getting backed up. + +The bits we'll keep: We have separate and well-unit-tested component for the +`ktree` and have some decent integration-esque tests around the +ledger-builder-controller as a whole that we should be able to mostly reuse. +We're also reusing the only-one-at-a-time shortcut for our async jobs for now. + +## Unresolved questions + +[unresolved-questions]: #unresolved-questions + +### High-throughput + +If we want to support throughput higher than ~10tps we'll need to change +breadcrumb storage to be backed by some sort of disk-backed store. + +This is not expected to be resolved before landing this RFC. + +### History catchup + +We do not specify the mechanism by which efficient downloading should occur. +This is a performance optimization and can be implemented later. We plan on +postponing implementation of this component as long as possible. If everything +is working properly, nodes in integration tests and test networks will not need +this function unless they are late joiners. + +### Miscellanea + +The implementation of this feature before the first merge will omit the mutable +transition frontier implementation (if we can pass integration tests by wrapping +the existing ktree) diff --git a/website/docs/researchers/rfcs/0010-decompose-ledger-builder.md b/website/docs/researchers/rfcs/0010-decompose-ledger-builder.md new file mode 100644 index 000000000..e062a787d --- /dev/null +++ b/website/docs/researchers/rfcs/0010-decompose-ledger-builder.md @@ -0,0 +1,149 @@ +--- +format: md +title: "RFC 0010: Decompose Ledger Builder" +sidebar_label: "0010 Decompose Ledger Builder" +hide_table_of_contents: false +--- + +> **Original source:** +> [0010-decompose-ledger-builder.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0010-decompose-ledger-builder.md) + +## Summary + +[summary]: #summary + +This RFC proposes a decoupling of the staged ledger which is will make the +components of the staged ledger more composable and pave the way for properly +encoding the staged ledger into the new transition frontier data structure. + +## Motivation + +[motivation]: #motivation + +The staged ledger has been suffering from scope creep for some time now. As we +are moving towards the new transition frontier data structure, we are already +decoupling the storage mechanism for the underlying ledger for a staged ledger. +This is going to have a number of difficult ramifications on the existing data +structure, so now seems as good a time as any to fully decouple the staged +ledger. + +## Detailed design + +[detailed-design]: #detailed-design + +[Full architecture](https://github.com/MinaProtocol/mina-resources/blob/main/docs/res/all_data_structures.dot.png) + +![](https://github.com/MinaProtocol/mina-resources/blob/main/docs/res/ledger_builder_data_structures.dot.png) + +### `Merkle_ledger` + +A `Merkle_ledger.t` is a value and a first class module implementing a +`Merkle_ledger_intf`, which is a common subset of all ledgers. Having a first +class module representation of a ledger allows us to build code which can be +generic over an interface without functoring over it. + +### `Merkle_ledger_diff` + +A `Merkle_ledger_diff.t` is a compressed difference from a base ledger to a +target ledger. It can be applied to a base ledger to make it a target ledger. +More formally, a `Merkle_ledger_diff.t` is a base ledger, a target ledger, and a +set of updated accounts, and a set of new accounts. The set of updated accounts +represents all of the accounts that have changed between the base and the +target, while the set of new accounts contains new accounts to add along with +the locations in the tree to add them to. A `Merkle_ledger_diff.t` can be +applied to a ledger by iterating through each set of accounts and writing them +to the ledger. After applying to a ledger, the root of the ledger should be +equal to the target merkle root iff the root before applying was equal to the +base merkle root. + +The long term goal for this data structure is that it will be the serialization +target of `Merkle_mask`. + +### `Parallel_scan_state` + +A `Parallel_scan_state.t` is a generic data structure representing the state of +a parallel scan operation. This state represents as a binary tree of nodes and a +binary operation (`node -> node -> node`) that reduces nodes, with a set of data +leaves at the bottom (which are lifted into tree nodes via a unary operation +`data -> node`). The state keeps track of multiple parallel sets of nodes that +are being reduced at once, all fitted into a single tree. Interaction with the +`Parallel_scan_state` is separated into steps in which both completed binary +operations (or "work") are applied and new nodes are added to the leaves of the +tree. When a set of nodes is reduced to the top, the `Parallel_scan_state` will +emit the final value. The number of new nodes that can be added each step is +limited by the amount of work that was submitted during that step. + +### `Parallel_scan_state_diff` + +A `Parallel_scan_state_diff.t` is a formalization around the difference between +steps of a `Parallel_scan_state.t`. It contains both the set of completed work +and the set of new work to add for a step. + +### `Transaction_snark_work` + +A `Transaction_snark_work.t` represents the work completed by a snark worker +that contributes towards the generation of a single transaction snark proof. +This can either be an initial proof of the application of a transaction to a +ledger, or it can be a recursive composition of these application proofs. + +### `Transaction_snark_scan_state` + +A `Transaction_snark_scan_state.t` is an instantiation of the +`Parallel_scan_state` which defines the nodes and work for generating +transaction snark proofs to be included in the transition snark proof. + +### `Staged_ledger` + +A `Staged_ledger.t` is a combination of a `Merkle_ledger.t` and a +`Transaction_snark_scan_state.t` which represents a state with staged +transactions. The underlying `Merkle_ledger.t` is a representation of a ledger +in which all of the transactions in `Transaction_snark_scan_state.t` are +applied, even though they have not been fully verified yet. The `Staged_ledger` +allows for high level applications of transitions, emitting changes to be +applied back to a fully verified ledger as they are available. + +### `Staged_ledger_diff` + +A `Staged_ledger_diff.t` is a combination of a `Merkle_ledger_diff.t` and a +`Parallel_scan_state_diff.t` instantiated for the `Transaction_snark_scan_state` +(`type data = Transaction.t and type node = Transaction_snark_work.t`). It +provides the difference between two staged ledgers and can be applied to a +`Staged_ledger.t` to transition it. + +## Rationale + +[rationale]: #rationale + +### Why change the `Ledger_builder`? + +The `Ledger_builder` abstraction was too large in scope, which was making tight +asynchronous design of components involving it more difficult. By decomposing it +into various components of smaller scope, designing the asynchronous control +flow the new `Transition_frontier` became simpler and allowed us to be more +thoughtful of computational costs. Furthermore, we want the new abstraction to +be independent of any particular implementation of the `Merkle_ledger` so that +we can take advantage of chained `Merkle_mask`s. + +### `Ledger_builder` -> `Staged_ledger` + +`Ledger_builder` is not a very descriptive name for what the old type +represented, and it is even less so now that it has been decoupled. +`Staged_ledger` is a better descriptor as it represents the process of _staging_ +transactions into the ledger before they are fully verified. Previously, the +term "staged ledger" has been used to refer to the ledger that contains the +state of the unproven transactions in the old staged ledger, so this is a change +of use for the term. Now, the actual ledger that represents the sate is a +`Staged_ledger.ledger`, and the type of `Staged_ledger.t` is the union of a +ledger and a `Transaction_snark_scan_state`. + +## Unresolved questions + +[unresolved-questions]: #unresolved-questions + +A `Staged_ledger_diff` could be more compact than it is. There is some +duplication in information between the `Merkle_ledger_diff` and the +`Parallel_scan_state_diff`. Specifically, the `Merkle_ledger_diff` can be +constructed from the transactions in the `Parallel_scan_state_diff` along with a +base `Merkle_ledger`. The `Merkle_ledger_diff` is a more effecient and compact +format for application to a `Merkle_ledger` however. Still, perhaps there is a +better way to do this. diff --git a/website/docs/researchers/rfcs/0011-txpool-dos-mitigation.md b/website/docs/researchers/rfcs/0011-txpool-dos-mitigation.md new file mode 100644 index 000000000..9efb20dea --- /dev/null +++ b/website/docs/researchers/rfcs/0011-txpool-dos-mitigation.md @@ -0,0 +1,307 @@ +--- +format: md +title: "RFC 0011: Txpool Dos Mitigation" +sidebar_label: "0011 Txpool Dos Mitigation" +hide_table_of_contents: false +--- + +> **Original source:** +> [0011-txpool-dos-mitigation.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0011-txpool-dos-mitigation.md) + +## Mitigations for DOS attacks based on bogus transactions + +We add some rules for deciding whether to store incoming transactions in the +pending transaction pool and gossip them. I'm using the phrase "pending +transaction pool" or "txpool" rather than "mempool" because I think it's +clearer. Lots of things are stored in memory. This is a simple approach that +offers basically equivalent DDoS resistance to existent cryptocurrencies. I have +some ideas for attacks that work against them and this, as well as a more attack +resistant design, but it's much more complicated. See +[the private document](https://docs.google.com/document/d/1FhBThENWdSN6bfT4re_tt78uMjViSan5f212faabBNg/edit?usp=sharing) +for those. + +## Motivation + +Every transaction consumes resources: computation, storage, and bandwidth. In +the case of transactions that are eventually included in a block, those +resources are priced by transaction fees. But transactions that are never +included in a block don't pay transaction fees, so without good design bogus +transactions could be a vector for cheap DoS attacks. We want to make sure it +costs enough to make nodes consume resources that a DoS attack is too expensive +to be worthwhile. + +(The transaction fees for _mined_ transactions may or may not accurately reflect +the total cost to all network participants for processing the transaction, but +that's a problem for another day.) + +We can't control what transactions we receive, but we can control whether we +store them and whether we forward them on, making other nodes deal with them. + +## Detailed design + +As a general principle, we want to accept transactions into the pool iff they +will eventually be mined into a future block. That's the purpose of the pending +transaction pool. For DoS protection, we assume that the cost of including a +transaction in a block is sufficient to deter DoS attacks on proposers, since +the cost of creating a SNARK greatly exceeds the cost of checking and storing an +incoming transaction. Where possible, we prefer to charge for things rather than +banning them. We don't mind if someone uses a lot of resources, so long as those +resources are paid for at a price that network participants would be happy with. + +In this context, we have the following goals: + +1. Support a pool of pending transactions with a maximum size, evicting + transactions when at the maximum size based on fee. + +2. Allow multiple queued transactions from one address. Since payments aren't + instant, users will want this. We could make transaction senders responsible + for queuing, but that wouldn't allow multiple transactions sent from the + same address to be included in the same block. + +3. Allow for transaction replacement, while aligning incentives by charging for + it appropriately. Users can e.g. cancel payments by replacing them with a + no-op transaction (send themselves $0) or resend them with a higher fee if + they're processing too slow. This is important for two reasons. A) a + transaction sent with too low of a fee will get "stuck", and without the + ability to replace it the user's account is bricked until the transaction + ages out of the txpool or is evicted if the pool is full. B) a transaction + sent with too low of a fee may simply take long enough that the user would + rather not have sent it. E.g. if I'm buying a coffee I don't want to wait in + the shop for half an hour while my payment goes through. Or if I'm using a + market based on smart contracts and my trade execution is delayed, the + reason I wanted to make the trade may no longer be valid when it actually + executes. + +### When we first receive a transaction: rules + +When we receive a gossiped transaction, we will check the below constraints, +with respect to our current longest chain. If any of the checks fail, we ignore +the transaction and do not gossip it. + +1. The signature. + +2. The sender's account exists. + +3. The sender's balance (inclusive of any pending txs from the same account + with lower nonce) is >= the transaction amount + fee. + +4. The tx nonce is equal to the sending account's next nonce, inclusive of + transactions already in the pool. (If it conflicts with one in the pool, see + below.) + +#### New punishment scheme + +There are scenarios where an honest node may send us transactions we don't want, +e.g. insufficient fee transactions if their pool is less full than ours or +out-of-date transactions if they're behind us. These consume resources but are +useless to us, and attackers may send lots of them. So we need a way to punish +nodes for sending them, while not banning innocent nodes. We may also be subject +to Sybil attacks - an attacker may create lots of nodes that never do anything +bannable but also don't properly gossip or otherwise contribute. They can +degrade service if the Sybil nodes crowd out real ones. So we want a way to +reward nodes for good conduct, and to punish them for bad but potentially honest +conduct. + +So we modify `banlist.ml` and friends. Rather than a node being either `Normal`, +`Punished`, or `Suspicious`, every node has a continuous, real valued "trust" +score. Nodes that give us useful data gain trust, nodes that give us useless +data lose trust, and nodes that give us invalid data lose a lot of trust. Trust +exponentially decays towards zero over time, with the decay rate set such that +half of it decays in 24 hours. This works out to decay factor of +~0.999991977495368389 as applied every second. We'll update trust scores lazily: +for each node we store the score the last time it was updated, and the time it +was updated. The new score = +`decay_factor**(seconds since last update) * old score`. We'll need to adapt the +existing code that bans/punishes peers. + +If a node's trust goes below -100, we'll ban it for 24 hours. In the future it +may be worth it to prioritize nodes based on trust rather than using a sharp +threshold. This would mitigate Sybil attacks: if we bias peer selection towards +more trusted nodes, then an attacker would have to contribute in order to get +connections, and can't crowd out honest nodes. The best way to do that is to +modify the Kademlia system. The way it works now is that nodes are prioritized +in buckets based on how old they are - on the basis that nodes that have been up +for longer are likelier to continue to be up than newer ones, and that having to +run nodes for a long time makes Sybil attacks expensive. We'd replace age with +trust score. But this is a pretty substantial modification (and maybe we're +replacing kad anyway?), so for now we'll just use the discrete banned/not banned +state. + +If we get any transactions where the signature is invalid, we reduce the peer's +trust by 100, effectively banning it for 24 hours. No honest node sends +transactions like that. If we get transactions that fail for other reasons - the +ones that depend on the current chain state - then we reduce the peer's trust by +`tx_trust_increment`. If we accept the transaction we increase trust by the same +amount. Honest nodes that are out of date might innocently send us transactions +that are not valid, but they won't send us a lot of them. If a node detects it +is substantially behind the network, it should disable gossip until it catches +up. + +There was +[some discussion](https://github.com/minaprotocol/mina/pull/761#issuecomment-424456658) +about score decay when the RFC for banlisting was first proposed. It wasn't +resolved and the current system doesn't implement any decay. A punishment score +system with decay is equivalent to trust scores if you only count bad behavior. +These are pretty close in effect, especially when we're doing discrete bans +rather than prioritization, but there is an important difference. Imagine a very +active peer, an exchange or a payment processor or something. It's not beyond +the realm of possibility for a single node to send 1000txs/hr, especially since +scalability is one of our core goals. If such a peer sends e.g. 2% bad +transactions due to network delay + data corruption + whatever else, it will be +banned if we only track bad behavior and not good - its punishment score will +rise over time, by assumption faster than the decay. In this design with trust +scores, its trust will increase faster than it falls and everything will be +fine. + +#### Replacing transactions + +A useful feature of existing cryptocurrencies is the ability to replace a +pending transaction with a different one by broadcasting a new one with the same +nonce. + +We want to have this feature, but it allows an attacker to make proposers +process transactions which won't eventually get mined (the ones that are +replaced), which violates our first guiding principle. So we require the fee of +the new transaction be at least `min_fee_increment` higher. This is the +"standard" approach. + +#### Multiple pending transactions from the same account + +Since payments aren't instant, users may want to queue multiple outgoing +transactions. This listed rules above cover this case, but things get +complicated when you allow transaction replacement. Imagine Mallory broadcasts +1000 valid transactions with sequential nonces, then replaces the first one with +one that spends all the money in the account. The other 999 of them are now +invalid and won't be mined, but the proposers still had to validate, store and +gossip them, violating our first principle. So the new fee in the example needs +to be at least `min_fee_increment` \* 1000 higher. + +### When txpool size is exceeded: rules + +We have a set limit on txpool size: `max_txpool_size`, in transactions. If an +incoming transaction would cause us to exceed the limit, we evict the lowest fee +transaction from the mempool, or drop the incoming transaction if its fee is <= +the lowest fee transaction in the mempool. + +If an incoming transaction has too low of a fee for us to accept, we count it as +bad for the purposes of trust scores. Except in the case of replacement +transactions. If we punished nodes for sending transactions without the +replacement fee increment, an attacker could induce nodes to banlist each other +by sending them different transactions at the same nonce. + +### Constants + +- `tx_trust_increment`: Let's say we target a max rate of bad transactions of + one per 10 seconds. The maximum rate bad transactions can be sent at without + getting banned is when the peer is just below the ban threshold - exponential + decay means the trust decays fastest when it's absolute value is highest. The + ban threshold is 100, so the algebra comes out to + `100 * decay_rate ** 10 + 100` = 8.022215015188294e-3. + +- `max_txpool_size`: This is interesting. If there were a fixed limit on the + number of transactions per block I'd say set it to an hour's worth or + something. But the limit is set by the parameters of the parallel scan, and + supposedly that'll become dynamic in the future, so I'm not sure. So let's say + 1000? + +## Drawbacks + +This is vulnerable to the attack in the private document. + +## Rationale and alternatives + +- Why is this design the best in the space of possible designs? + +There is virtue in simplicity, especially in a winner-take-all market like ours. +This is simple and relatively easy to implement and not worse than what else +exists. A fancier thing would delay launch further. + +- What other designs have been considered and what is the rationale for not + choosing them? + - Allow pending txs that spend money not (yet) in the sender's account + + Ethereum does this. It's abusable, an attacker can send transactions that + will never be mined. The countermeasure is having a hard limit on pending + transactions per account, which I don't like: + + - Max pending tx per account + + There are legitimate use cases for sending lots of transactions from one + address rapidly, and I strongly prefer charging for things to banning + things. For an example, imagine an exchange. To process withdrawals they may + need to send 100s of transactions per minute from a single address. So long + as that is priced efficiently they should be able to do so. Yes, they should + probably be doing transaction batching in this scenario, but it's better + that doing individual transactions is expensive than if it were impossible. + + - Skip validation for speed + + This is (partly) what we do now, and is vulnerable to all sorts of stuff. + + - Block lookback window for validity. + + Part of Brandon's original plan was to accept transactions that were valid + at any point in the transition frontier, or within some fixed lookback from + a current tip. This is abusable. Mallory can move funds around such that she + can make transactions that were valid recently but aren't now, and consume + resources for free. The attack requires her to move them around at least + once every lookback window blocks, and lets her consume resources for + lookback window blocks, so with a sufficiently small window it's probably + impractical, but I'd rather avoid the headache. I don't see a use case for + it. Since the account holder is the only one who can spend funds from their + account, and since insufficient funds is (almost) the only reason payments + can fail, they should never be sending transactions that used to be valid + but aren't now. + + - Have a minimum transaction fee + + In this design, an attacker can fill the txpool with bogus transactions with + fees too low to ever be included in a block for "free". This is bad, and + imposing a overall minimum fee that is at least as much as SNARKing a + payment costs would prevent it, but figuring out what that minimum should be + is complicated, and the attack is only good so long as the pool isn't full, + so I think it's not worth it. + +- What is the impact of not doing this? + + Various vulnerabilities. + +## Prior art + +Ethereum allows transaction replacement, and multiple pending transactions from +the same account, without requiring they be valid when run sequentially. If a +transaction's smart contract errors out, or runs out of gas, miners get to keep +the fees. So it's sort of a like a deposit. But I don't think they check that +the there's sufficient balance in the account to cover the sum of transaction +fees from all pending transactions before accepting another transaction. That +would be expensive, since the balance of the sending account after a transaction +runs may be higher than when it started due to smart contracts. There's no +explicit replacement fee, and there's definitely no deposit system for it. There +might be a minimum fee increment though. I think they're vulnerable to some of +these attacks. They evict transactions from the mempool on a lowest-fee-first +basis. + +Bitcoin will do what they call "replace-by-fee" which is the transaction +replacement thing. They may have a minimum increment, but not a deposit system. +Transactions are evicted on a lowest-fee-first basis. They allow mempool +transactions to depend on each other, but not on hypothetical future +transactions. When a transaction is evicted from the mempool, they also evict +any transactions that depend on it. + +## Unresolved questions + +- What parts of the design do you expect to resolve through the RFC process + before this gets merged? + + Decide if it's worth it to do the more complicated thing. + +- What parts of the design do you expect to resolve through the implementation + of this feature before merge? + + Nothing comes to mind. + +- What related issues do you consider out of scope for this RFC that could be + addressed in the future independently of the solution that comes out of this + RFC? + + The SNARK pool has similar concerns. diff --git a/website/docs/researchers/rfcs/0012-ban-scoring.md b/website/docs/researchers/rfcs/0012-ban-scoring.md new file mode 100644 index 000000000..22e6de0e3 --- /dev/null +++ b/website/docs/researchers/rfcs/0012-ban-scoring.md @@ -0,0 +1,166 @@ +--- +format: md +title: "RFC 0012: Ban Scoring" +sidebar_label: "0012 Ban Scoring" +hide_table_of_contents: false +--- + +> **Original source:** +> [0012-ban-scoring.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0012-ban-scoring.md) + +## Summary + +We propose a regime for ban scoring to supplement the ban mechanism proposed in +RFC 0001-banlisting. We take Bitcoin's scoring mechanism as a starting point, +since the Bitcoin network is subject to many of the same transgressions as the +Coda network. + +## Motivation + +There are several points in the code marked _TODO:punish_, but to date, there +hasn't been a systematic review of how the behaviors at those points should +contribute to ban scoring. A good scoring system should minimize the bad effects +of malicious or buggy nodes, while allowing honest nodes to remain active in the +network. + +## Detailed design + +### Bitcoin + +In Bitcoin, certain kinds of misbehavior increase a node's ban score. If a nodes +score exceeds a threshold, by default equal to 100, the node is banned from the +network. A node can be trustlisted, exempting it from such banning. A node can +also be manually banlisted even in the absence of observable misbehavior. + +In the Bitcoin C++ implementation, there's an API "Misbehaving" to increment the +ban score, and set a ban flag if the score exceeds the threshold. In some cases, +the ban score is incremented by a fixed amount. In other cases, the score is +incremented by the value of a "DoS" value associated with a peer. There's a +separate API for incrementing the DoS value when rejecting a transaction. + +At the time of writing (commit d6e700e), the Misbehaving API is called for +certain behaviors with fixed ban score increments. These behaviors have a score +of 100, resulting in an immediate ban: + +- invalid block, invalid compact block, nonmatching block transactions +- invalid Bloom filter version, too-large Bloom filter, +- too-large data item to add to Bloom filter, add to missing Bloom filter +- invalid orphan tx, out-of-bounds tx indices + +One misbehavior has a score of 50: + +- message too large for buffer + +For a score of 20: + +- too many unconnecting headers +- too-big message "addr", "inventory", or "getdata" message sizes +- too many headers, non-continuous headers + +And a score of 1: + +- missing "version" or "verack" messages, duplicate "version" message + +In several places in the code, the "DoS" value is checked, and if positive, it's +added to the ban score. The DoS score can be incremented when a transaction is +rejected. Many reasons for rejecting a transaction add 100 to the DoS value (too +many to enumerate here). Those include items like transaction fees out of range, +missing or already-spent inputs, incorrect proof of work, and an invalid Merkle +root. + +Some reasons for rejecting a transaction increment the DoS value a lesser +amount. For instance, and invalid hash such that the proof of work fails, +increments DoS by 50. A previous-block-not-found rejection increments DoS by 10. +In several cases, transactions are rejected, but the DoS is not incremented, +such as the "mempool-full" condition. + +### Coda at the moment + +Bitcoin is a mature codebase, so there are many places where ban scoring has +been used. Nonetheless, Bitcoin uses a relatively coarse ban scoring system; +only a few ban score increment values are used. In Coda, we could reify scores +into a datatype: + +```ocaml + module Ban_score = struct + type t = + | Severe + | Moderate + | Trivial + end +``` + +A slightly finer gradation could be used, if desired. The banlisting system +could translate these constructors into numerical scores. Let's call these +constructors SEV, MOD, and TRV. + +In a number of places, the Coda codebase has comments indicating that a peer +should be punished, either via a `TODO` or call to `Logger.faulty_peer`. Let's +classify those places where punishment has been flagged, and annotate them with +suggested constructors: + +- in `bootstrap_controller.ml`, for bad proofs (SEV), and a validation error + when building a breadcrumb (SEV) +- in `mina_networking.ml`, when an invalid staged ledger hash is received (SEV), + or when a transition sender does not return ancestors (MOD) +- in `ledger_catchup.ml`, when a root hash can't be found (SEV), or a peer + returns an empty list of transitions (instead of `None`) (TRV) +- in `linked_tree.ml`, for peers requesting nonexistent ancestor paths (MOD) +- in `parallel_scan.ml`, in `update_new_job` for unneeded merges (?) (SEV) +- in `staged_ledger.ml`, when a bad signature is encountered when applying a + pre-diff (SEV) +- in `syncable_ledger.ml`, in `num_accounts`, when a content hash doesn't match + a stored root hash (SEV), and in `main_loop`, when a child hash can't be added + (MOD) +- in `catchup_scheduler.ml` and `processor.ml`, when a breadcrumb can't be built + from a transition (SEV) (same failure as in `bootstrap_controller.ml`, above) +- in `ledger_catchup.ml`, a transition could not be validated (SEV) +- in `transaction_pool.ml`, a payment check fails (SEV) + +At these points in the code, the banlist API would be called with these +constructors. The API should take a severity argument and a string indicating +the nature of the bad behavior. + +### Integration with trust system + +RFC 0010 proposes a trust system where peers can earn positive trust through +good actions, and lose trust through bad actions. The ban scoring proposed here +can be integrated with the trust system, by decrementing trust upon bad actions, +according to the severities mentioned above. + +The events classified as SEV should result in an immediate ban. Therefore, trust +should be capped so that a single SEV event decrements the trust to the level +resulting in a ban. + +## Drawbacks + +A banning system in necessary to preserve the integrity of the network. The only +drawback would be if the system is ineffective. + +In this RFC, we don't consider how to make the ban scoring persistent, which +will be required, since nodes stop and re-start. + +## Rationale and alternatives + +The locations in the code mentioned above are only those already flagged with +`TODO` or `faulty_peer`. There may be other code locations where punishment is +warranted. + +In the current code, there are often calls to the logger where punishment is +mentioned in a `TODO`. There could be an API that calls the logger and the +banlist API, to guarantee there's an inspectible history leading to a ban score. + +## Prior art + +There is existing code in Coda to maintain a set of peers banned by IP address +in `banlist_lib`. It will be superseded when this RFC and RFC 0010 are +implemented. + +See the discussion above of how Bitcoin computes ban scores. + +## Unresolved questions + +- Is there a principled way to decide the severity of transgressions by peers? +- Can a peer too-easily circumvent the system? +- What are the numerical values associated with the constructors above? +- What is the ban threshold? diff --git a/website/docs/researchers/rfcs/0013-rpc-versioning.md b/website/docs/researchers/rfcs/0013-rpc-versioning.md new file mode 100644 index 000000000..81bb44526 --- /dev/null +++ b/website/docs/researchers/rfcs/0013-rpc-versioning.md @@ -0,0 +1,157 @@ +--- +format: md +title: "RFC 0013: Rpc Versioning" +sidebar_label: "0013 Rpc Versioning" +hide_table_of_contents: false +--- + +> **Original source:** +> [0013-rpc-versioning.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0013-rpc-versioning.md) + +## Summary + +We propose a mechanism to allow updating the versions of remote procedure calls +(RPCs) between Coda protocol nodes. + +## Motivation + +Coda uses a remote procedure call (RPC) mechanism for nodes to query other +nodes, and to broadcast messages to the gossip network. As the codebase evolves, +the structure of those RPC messages may evolve, and new kinds of messages may be +added. Nodes running different versions of the software need to be able to +communicate. + +When querying, the caller and callee nodes may be using different versions of +RPC calls. The caller can be running the newer version, and the callee the older +version, or vice-versa. Both scenarios have to be accommodated. + +## Detailed design + +The Jane Street Async library contains a module `Versioned_rpc` with the +machinery to allow evolution of RPC call versions. + +In the `coda_network` library, the `Versioned_rpc` library is used in two ways, +for queries and for broadcasting. + +### Queries + +For queries, the pattern is (simplifying somewhat): + +```ocaml + module Query = struct + + module T = struct + let name = ... + module T = struct + type query = ... + type response = ... option + end + module Caller = T + module Callee = T + end + + include Versioned_rpc.Both_convert.Plain.Make (T) + + module V1 = struct + module T = struct + type query = ... + type response = ... option + let version = 1 + let query_of_caller_model = Fn.id + let callee_model_of_query = Fn.id + let response_of_callee_model = Fn.id + let caller_model_of_response = Fn.id + end + include Register (T) + end + + end +``` + +The name identifies the particular RPC query. Calling the functor `Plain.Make` +creates the other functor `Register` called within `V1`. The module `T.T` offers +types for a query and response, and in this code, both the caller and callee +agree on those types (they could differ, in theory). + +The four functions implemented here with `Fn.id`, the identity function, are +coercions between the query and response types in `T.T` and `V1.T`. In the +existing RPC queries, those types are are the same, so we can use the identity +function. + +For a given query, if we wish to update the protocol, we'd add: + +```ocaml + module V2 = struct + module T = struct + type query = ... + type response = ... option + let version = 2 + let query_of_caller_model : T.Caller.query -> query = ... + let callee_model_of_query : query -> T.Callee.query = ... + let response_of_callee_model : T.Callee.response -> response = ... + let caller_model_of_response : reponse -> T.Caller.response = ... + end + include Register (T) + end +``` + +The types of the coercions are shown. For each coercion, the input and output +types could differ. + +There could be additional new modules for subsequent versions. Eventually, +versions could be pruned from the code, to encourage nodes to upgrade their +software. When a new query version is created, the `Vn` module for the previous +version could have an annotation: + +```ocaml + [@@remove_after "20200702"] +``` + +where the annotation is implemented via a ppx. Compiling after the given date +results in a warning. A year or so past the introduction of the new version +might be a suitable date for removing the previous version. + +The query modules are used in a list of "implementations". To define an +implementation, we need a function of type: + +```ocaml + Host_and_port.t -> version:int -> T.Caller.query -> T.Callee.response option Deferred.t +``` + +which does the work within the node to respond to the query. The host and port +represent the "connection state" of the TCP connection between the nodes, which +is the host and ephemeral port of the caller. The version passed is the +caller's. In theory, these functions could dispatch on the version. Instead, the +version should be considered informative, and the real accomodation between +versions should happen in the coercions. Therefore, the implementation functions +do not need to change between versions. + +### Broadcasting + +The RPC versioning mechanism for broadcasting is similar, except that instead of +query and response types, there is a "msg" type. The versioning module defines +coercions + +```ocaml + val msg_of_caller_model : Caller.msg -> msg + val callee_model_of_msg : msg -> Callee.msg +``` + +In the `V1` module, those are both `Fn.id`. For a new version, we'd create a new +`Vn` module with a new version number and appropriate coercions. As for queries, +we'd want to indicate a removal date for earlier-version modules. + +## Drawbacks + +Nodes using a version implemented by a versioning module cannot communicate with +nodes where that module has been removed. That's a feature, really, although +perhaps a temporary inconvenience for nodes that haven't upgraded their +software. + +## Prior art + +The Jane Street version RPC library is already in the Coda codebase. + +## Unresolved questions + +The versioning mechanism described here has not been tested locally. diff --git a/website/docs/researchers/rfcs/0014-address-encoding.md b/website/docs/researchers/rfcs/0014-address-encoding.md new file mode 100644 index 000000000..be87a5441 --- /dev/null +++ b/website/docs/researchers/rfcs/0014-address-encoding.md @@ -0,0 +1,88 @@ +--- +format: md +title: "RFC 0014: Address Encoding" +sidebar_label: "0014 Address Encoding" +hide_table_of_contents: false +--- + +> **Original source:** +> [0014-address-encoding.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0014-address-encoding.md) + +## Summary + +[summary]: #summary + +Change address encoding from Base64 to Base58. + +## Motivation + +[motivation]: #motivation + +First, it's a standard format across other well known cryptocurrencies (Bitcoin, +Zcash, Monero). That means most entities in the ecosystem, from end-users to +exchanges, will be more familiar and comfortable with Base58 encodings than what +we have currently. + +Additionally, quoting from the +[Bitcoin Wiki](https://en.bitcoin.it/wiki/Base58Check_encoding#Background): + +``` +// Why base-58 instead of standard base-64 encoding? +// - Don't want 0OIl characters that look the same in some fonts and +// could be used to create visually identical looking account numbers. +// - A string with non-alphanumeric characters is not as easily accepted as an account number. +// - E-mail usually won't line-break if there's no punctuation to break at. +// - Doubleclicking selects the whole number as one word if it's all alphanumeric. + +``` + +## Detailed design + +[detailed-design]: #detailed-design + +I recommend we follow the Bitcoin design, as laid out +[here](https://en.bitcoin.it/wiki/Base58Check_encoding#Base58_symbol_chart). + +We should also add a unique prefix to prevent Coda from being accidentally sent +to Bitcoin addresses or vice versa. For example, we can add the following prefix +before hashing: + +`hash addr = SHA256("Coda address " ^ addr)` + +## Drawbacks + +[drawbacks]: #drawbacks + +- Addresses are longer than Base64 +- It will take some time to implement, perhaps we can use the + [Tezos implementation](https://github.com/vbmithr/ocaml-base58) + +## Rationale and alternatives + +[rationale-and-alternatives]: #rationale-and-alternatives + +We could continue to use Base64, change to hex, or use a custom encoding. + +I'd recommend Base58 because it is the standard, is more compact than hex (which +matters given our key size), and it has the benefits enumerated above. I don't +see many benefits with developing a custom encoding. + +## Prior art + +[prior-art]: #prior-art + +- [Bitcoin](https://en.bitcoin.it/wiki/Base58Check_encoding#Base58_symbol_chart) +- [Ethereum](https://github.com/ethereum/EIPs/blob/master/EIPS/eip-55.md) +- [Cardano](https://cardanodocs.com/cardano/addresses/) +- [Monero](https://monero.stackexchange.com/questions/1502/what-do-monero-addresses-have-in-common) + +## Unresolved questions + +[unresolved-questions]: #unresolved-questions + +1. Should we use a big-endian or little-endian byte order? Both Ethereum and + Bitcoin use big-endian encodings. +2. Is it worth spending more time thinking about this or should we just + implement what seems to be a standard practice? +3. How can we make sure our addresses look visually distinct from + bitcoin/monero/zcash addresses? diff --git a/website/docs/researchers/rfcs/0015-transition-frontier-extensions.md b/website/docs/researchers/rfcs/0015-transition-frontier-extensions.md new file mode 100644 index 000000000..96d8ac956 --- /dev/null +++ b/website/docs/researchers/rfcs/0015-transition-frontier-extensions.md @@ -0,0 +1,123 @@ +--- +format: md +title: "RFC 0015: Transition Frontier Extensions" +sidebar_label: "0015 Transition Frontier Extensions" +hide_table_of_contents: false +--- + +> **Original source:** +> [0015-transition-frontier-extensions.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0015-transition-frontier-extensions.md) + +## Summary + +[summary]: #summary + +Allow properties based the transition frontier to be efficiently tracked. + +## Motivation + +[motivation]: #motivation + +There are several properties that we might want to obtain from the data in the +transition frontier that are too expensive to calculate by iterating over all +its transitions. However, we don't want to add complexity to the transition +frontier by having it keep track of these values that are not strictly necessary +for its operation and are only used externally. + +As an example, the snark pool never has elements removed and requires some +system of garbage collection. One way to do this would be to track a reference +count for each piece of `Work` in the table of proofs, but this table of +references probably shouldn't be part the transition frontier itself as it's +completely unrelated. + +We'd like to have a way for other parts of the system to incrementally build up +datastructures based on events that are happening in the frontier. + +Other uses for such an abstraction: + +- managing the transaction pool +- tracking information for consensus optimization +- Handling persistence for the transition frontier + +## Detailed design + +[detailed-design]: #detailed-design + +This design is based on the introduction of the following event type to +`Protocols.coda_transition_frontier`: + +```ocaml +module Transition_frontier_diff = struct + type 'a t = + | New_breadcrumb of 'a + (** Triggered when a new breadcrumb is added without changing the root or best_tip *) + | New_best_tip of + { old_root: 'a + ; new_root: 'a (** Same as old root if the root doesn't change *) + ; new_best_tip: 'a + ; old_best_tip: 'a + ; garbage: 'a list } + (** Triggered when a new breadcrumb is added, causing a new best_tip *) + [@@deriving sexp] +end +``` + +The `Transition_frontier` will hold a record full of extensions, and will call +`handle_diff` on all extensions whenever a frontier diff event (defined above) +is triggered by `add_breadcrumb_exn`, passing in the diff event. The lifetime of +the extension is tied to the `Transition_frontier`, so when the frontier is torn +down and rebuilt, so are the extensions. + +External users of the extensions can subscribe to a version of the +`Transition_frontier` `MVar` that will broadcast the creation of a new +`Transition_frontier` to all listeners. When they receive a new +`Transition_frontier`, they can also start reading from a `Pipe` that is exposed +by whichever extension they care about. + +For instance, in the snark pool example, the reference count could be +incremented for all Work referenced by the added breadcrumb when +`Extend_best_tip` is triggered, and the reference count for the removed work +could be decremented when `New_root` fires. Whenever the reference count for a +piece of work goes to zero, an event gets dispatched into the Pipe for the +`Snark_pool` to remove that work from the pool. + +## Drawbacks + +[drawbacks]: #drawbacks + +- If the `add/remove_breadcrumb` are slow, this could slow down the transition + frontier. +- Adding the calculation to the frontier itself would avoid adding an + abstraction, though this listener is fairly simple as described. + +## Rationale and alternatives + +[rationale-and-alternatives]: #rationale-and-alternatives + +- This design allows for more data to be incrementally calculated based on + activity in the transition frontier while adding minimal complexity to the + frontier itself. +- Alternative: Calculate values on-demand by iterating over all transitions in + the frontier -- this is more expensive +- Alternative: Add incremental calculation to the transition frontier itself -- + this adds unrelated complexity to the frontier code + +An alternative implementation of this "Listener" solution would have the +transition frontier hold a list of listener functions that have signature +`Breadcrumb.t -> unit`, so the `t` is embedded in the closure of the listener. +This is potentially simpler but less explicit. + +## Prior art + +[prior-art]: #prior-art + +- The old `Ledger_builder_controller` was more complex than the current + transition frontier design, and we'd like to avoid adding more complexity to + this component. + +## Unresolved questions + +[unresolved-questions]: #unresolved-questions + +- Potentially out of scope of this listener api RFC, but in the snark pool + manager, it is unclear how to obtain the work from the breadcrumb. diff --git a/website/docs/researchers/rfcs/0016-transition-frontier-persistence.md b/website/docs/researchers/rfcs/0016-transition-frontier-persistence.md new file mode 100644 index 000000000..2f53516df --- /dev/null +++ b/website/docs/researchers/rfcs/0016-transition-frontier-persistence.md @@ -0,0 +1,200 @@ +--- +format: md +title: "RFC 0016: Transition Frontier Persistence" +sidebar_label: "0016 Transition Frontier Persistence" +hide_table_of_contents: false +--- + +> **Original source:** +> [0016-transition-frontier-persistence.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0016-transition-frontier-persistence.md) + +# Transition Frontier Persistence + +## Summary + +[summary]: #summary + +This RFC proposes a new system for persisting the transition frontier's state to +the filesystem efficiently. + +## Motivation + +[motivation]: #motivation + +The Transition Frontier is too large of a data structure to just blindly +serialize and write to disk. Under non-optimal network scenarios, we expect the +upper bound of the data structure to be >100Gb. Even if the structure were +smaller, we cannot write the structure out to disk every time we mutate it as +the speed of the transition frontier data structure is critical to the systems +ability to prevent DDoS attacks. Therefore, a more robust and effecient system +is required to persist the Transition Frontier to disk without negatively +effecting the speed of operations on the in memory copy of the Transition +Frontier. + +## Detailed design + +[detailed-design]: #detailed-design + +### Persistent Transition Frontier + +[detailed-design-persistent-transition-frontier]: + #detailed-design-persistent-transition-frontier + +The persistent copy of the Transition Frontier is stored in a RocksDB database. +Overall, the structure is similar to that of the full in memory Transition +Frontier in that the data for each state is stored in a key value fashion where +the key is the protocol state hash of that state. However, the persistent +version does not store as much information at each state as the in memory one. +In particular, the in memory data structure stores a staged ledger at each +state, but this takes a lot of space and can be recomputed from the root if +needed. Therefore, each state in the persistent Transition Frontier only needs +to store the external transition, along with a single staged ledger scan state +at the root state. When the persistent Transition Frontier is loaded from disk, +the necessary intermediate data is reconstructed using the staged ledger diffs +from the external transitions, the root snarked ledger, and the root staged +ledger scan state. + +Updates to the persistent Transition Frontier happen in bulk over a time +interval. As such, the `Persistent_transition_frontier` module provides an +interface for flushing Transition Frontier diffs to the RocksDB database. The +details of the mechanism which performs this action are discussed in the next +section. + +### Transition Frontier Persistence Extension + +[detailed-design-transition-frontier-extension]: + #detailed-design-transition-frontier-extension + +As actions are performed on the Transition Frontier, diffs are emitted and +stored in a buffer. More specifically, an extension will be added to the +frontier which performs these diff buffering and flushing activities called +`Transition_frontier_persistence_ext`. Once the buffer of diffs is either full +or a timeout interval has occurred, the `Transition_frontier_persistence_ext` +will flush all of its diffs to the `Persistent_transition_frontier`. + +### Incremental Hash + +[detailed-design-incremental-hash]: #detailed-design-incremental-hash + +Having two different mechanisms for writing the same data can be tricky as there +can be bugs in one of the two mechanisms that would cause the data structures to +become desynchronized. In order to help prevent this, we can introduce an +incremental hash on top of the Transition Frontier which can be updated upon +each diff application. This hash will give a direct and easy way to compare the +structural equality of the two data structures. Being incremental, however, also +means that the order of diff application needs to be the same across both data +structures, so care needs to be taken with that ordering. Therefore, in a sense, +this hash will represent the structure and content of the data structure, as +well as the order in which actions were taken to get there. We only care about +the former in our case, and the latter is just a consequence of the hash being +incremental. + +In order to calculate this hash correctly, we need to introduce a new concept to +a diff, which is that of a diff mutant. Each diff represents some mutation to +perform on the Transition Frontier, however not every diff will contain the +enough information by itself to encapsulate the state of the data structure +after the mutation occurs. For example, setting a balance on an account in two +implementations of the data structure does not guarantee that the accounts in +each an equal as there are other fields on the account besides that. This is +where the concept of a diff mutant comes in. The mutant of a diff is the set of +all modified values in the data structure after the diff has been applied. Using +this, we can create a proper incremental diff which will truly ensure our data +structures are in sync. + +These hashes will be Sha256 as there is no reason to use the Pedersen hashing +algorithm we use in the rest of our code since none of this information needs to +be snarked. The formula for calculating a new hash `h'` given an old hash `h` +and a diff `diff` is as follows: `h' = sha256 h diff (Diff.mutant diff)`. + +### Diff Application and Incremental Hash Pseudo-code + +[detailed-design-diff-application-and-incremental-hash-pseudo-code]: + #detailed-design-diff-application-and-incremental-hash-pseudo-code + +Here is some pseudo code which models the `Diff.t` type as a GADT where the +parameter type is the type of the mutant the diff returns. This only models a +few of the diff types and most likely needs to be updated as this was first +drafted before the Transition Frontier Extension RFC landed. + +```ocaml +module Diff = struct + module T = struct + type 'mutant t = + | AddAccount : Public_key.Compressed.t * Account.t -> (Path.t * Account.t) t + | SetHash : Path.t * Hash.t -> Hash.t t + | SetBalance : Public_key.Compressed.t * Balance.t -> Account.t t + [@@deriving hash, eq, sexp] + end + + include T + + module Hlist = Hlist.Make (T) + + let map_mutant + (diff : 'mutant t) + (mutant : 'mutant) + ~(add_account : Path.t -> Account.t -> 'result) + ~(set_hash : Hash.t -> 'result) + ~(set_balance : Account.t -> 'result) + : 'result + = fun diff mutant ~add_account ~set_hash ~set_balance -> + match diff, mutant with + | AddAccount _, (path, account) -> add_account path account + | SetHash _, hash -> set_hash hash + | SetBalance _, account -> set_balance account +end + +let apply_diff (t : t) (diff : 'mutant Diff.t) : 'mutant = + match diff with + | AddAccount (pk, account) -> + let path = allocate_account t pk in + set t path account; + (path, account) + | SetHash (path, h) -> + set t path h; + h + | SetBalance (pk, balance) -> + let path = path_of_public_key t pk in + let account = Account.set_balance (get t path) balance in + set t path account; + account + +let update (t : t) (ls : 'a Diff.Hlist.t) : unit = + Diff.Hlist.fold_left ls ~init:(get_incremental_hash t) ~f:(fun ihash diff -> + let mutant = apply_diff t diff in + let diff_hash = Diff.hash diff in + let mutant_hash = + Diff.map_mutant diff mutant + ~add_account:(fun path account -> + Hash.merge (Path.hash path) (Account.hash account)) + ~set_hash:Fn.id + ~set_balance:Account.hash + in + Hash.merge ihash (Hash.merge diff mutant_hash) +``` + +## Drawbacks + +[drawbacks]: #drawbacks + +The primary drawback to this system is that it creates additional complexity in +that we have 2 different methods of representing the same data structure and +content. This can lead to odd bugs that can be difficult to trace. However, +introducing the incremental hash helps to mitigate this issue. + +## Prior art + +[prior-art]: #prior-art + +In the past, before we replaced the Ledger Builder Controller with the +Transition Frontier, we would `bin_io` the entire Ledger Builder Controller out +to the filesystem on every update. This was slow and wasteful and would have +needed to be replaced anyway. + +## Unresolved questions + +[unresolved-questions]: #unresolved-questions + +Is the incremental hash overkill? Could we restructure the diffs such that the +`diff_mutant` is not required for hashing? Is there something else we can do +that doesn't tie the order of diff application to the value of the hash? diff --git a/website/docs/researchers/rfcs/0017-module-versioning.md b/website/docs/researchers/rfcs/0017-module-versioning.md new file mode 100644 index 000000000..e96d0f0fb --- /dev/null +++ b/website/docs/researchers/rfcs/0017-module-versioning.md @@ -0,0 +1,258 @@ +--- +format: md +title: "RFC 0017: Module Versioning" +sidebar_label: "0017 Module Versioning" +hide_table_of_contents: false +--- + +> **Original source:** +> [0017-module-versioning.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0017-module-versioning.md) + +## Summary + +We describe ways to version modules containing types with serialized +representations. + +## Motivation + +Within the Coda codebase, modules contain (ideally) one main type. As the +software evolves, that type can change. When compiling, the OCaml type +discipline enforces consistent use of such types. Sometimes we'd like to export +representations of those types, perhaps by persisting them, or by sending them +to other software. Such other software may be written in languages other than +OCaml. + +## Detailed design + +There are three main serialized representations of data used in the Coda +codebase: `bin_io`, `sexp`, and `yojson`. The readers and writers of those +representations are typically created using the `derive` annotation on types. + +### Explicit versioning + +The `bin_io` representation and associated `bin_prot` library are claimed to be +type-safe when used in OCaml (see +[Jane Street bin_prot](https://github.com/janestreet/bin_prot/). Moreover, other +programming languages don't appear to be able to produce or consume this +representation. Therefore, type and versioning information need not be stated +explicitly in this representation. Nonetheless, it is still important to use +versioning, so that we may convert older representations to the current module +version's type. + +Many programming languages can produce and consume the `sexp` and `yojson` +representations. Given an OCaml type, the derived writers for those +representation do not mention the type or a version. So non-OCaml consumers of +these representations cannot easily detect the type distinctions in OCaml that +gave rise to the representations. + +When a new version of a type is created, it can be worthwhile to retain the +older version and associated code. That allows producing and consuming +representations for older versions of the software. We can also take older +representations to produce a value of the type associated with the latest +version of a module. + +Here is a proposed discipline for creating a module with a name and a version: + +```ocaml +module Some_data = struct + + module Stable = struct + + module V1 = struct + module T = struct + let version = 1 + + type t = ... [@@deriving bin_io,...] + end + + include T + + type latest = t + + let to_latest t = t + + include Make_version (T) + end + + module Latest = V1 + + (* declare module *) + + module Module_decl = struct + let name = "some_data" + type latest = Latest.t + end + + (* register versions *) + + module Registrar = Make (Module_decl) + module Registered_V1 = Registrar.Register (V1) +end +``` + +The `Make_version` functor generates boilerplate code for `bin_io` that shadows +the code generated by `deriving`. The shadowing code adds a version number in +serializations. The `Registrar` module maintains a set of registered modules, +indexed by version numbers. A `Registrar` can take a serialization containing a +version number and deserialize it to an instance of the type for that version +number. By applying the `to_latest` for that version number, we get a value of +the type for the latest module version: + +Example: + +```ocaml + let buf = ... in + ignore (Registered_Vn.bin_write_t ~pos:0 buf some_Vn_value); + match Registrar.deserialize_binary_opt buf with + | None -> failwith "shouldn't happen" + | Some _ -> printf "got latest version of some_Vn_value" +``` + +A new module version `V2` consists of a new version number, a new type, which is +also the type `latest`, and the function `to_latest`. + +So we'd have: + +```ocaml +module Some_data = struct + + module Stable = struct + + module V2 = struct + + module T = struct + let version = 2 + + type t = ... [@@deriving bin_io,...] + end + + include T + + include Make_latest_version (T) + + end + + module Latest = V2 (* changed, was V1 *) + + module V1 = struct + + module T = struct + let version = 1 + type t = ... [@@deriving bin_io,...] + end + + include T + + type latest = Latest.t (* changed, was t *) + + let to_latest = ... (* changed, was the identity *) + + include Make_version (T) + end + + module Module_decl = struct + let name = "some_data" + type latest = Latest.t + end + + module Registrar = Make (Module_decl) + module Registered_V1 = Registrar.Register (V1) + module Registered_V2 = Registrar.Register (V2) (* new *) + module Registered_Latest = Registered_V2 (* changed, was Registered_V1 *) +``` + +The `Make_latest_version` functor is like `Make_version`, except that it gives +definitions for `latest` (it's `t`) and `to_latest` (the identity function on +`t`). + +Clients of a versioned module should always use the `Stable.Latest` version. + +### Serialization restricted to Stable, versioned modules + +Serialization should be allowed only for modules following this discipline, +including the use of the `Stable` submodule containing versioned modules. + +An exception can be carved out for using `sexp` serialization of data from +modules without `Stable` and versioning. That kind of serialization is useful +for logging and other printing from within Coda, and is less likely to be used +with other software. + +### Coordination with RPC versioning + +RFC 0012 indicates how to version types used with the Jane Street RPC mechanism. +The types specified in versioned modules here can also be used in the query, +response, and message types used for RPC. In that case, if a new version of a +module is created, the RPC version should be updated at the same time. + +### Embracing this discipline; static enforcement + +As of this writing, there are over 600 type definitions with `deriving bin_io` +in the Coda codebase, even more with `sexp`. Only about 60 type definitions are +versioned using the `Stable.Vn` naming discipline (and without the explicit +version information suggested here). To embrace this discipline fully will take +some effort. + +We can make awareness of the existing recommendation, or the extended discipline +suggested here a checklist item for Github pull requests. + +In some critical situations, such as when using versioned RPC (RFC 0012), we'd +like static assurance that types are versioned. By writing a suitable ppx, we +can add an annotation `versioned` to type `deriving` lists. That ppx will +generate a definition, perhaps called `__versioned`, if all types contributing +to a type also have the annotation. It can also check that the type is named +`t`, and occurs in the module hierarchy `Stable.Vn.T` (or in the module +hierarchy for versioned RPC types). The functors `Make_version` and +`Make_latest_version` can also require their argument to have the annotation, to +localize the error when the annotation is omitted. + +### Using versioned types in other versioned types + +When mentioning a versioned type in the definition of another versioned type, +such as in a record, a specific version of the included type must be used. That +is, do not use version `Latest`, which may refer to different modules over time. +With this restriction in place, the serialization of the including type has a +fixed format. Otherwise, multiple, incompatible serializations of the including +type could be generated. + +### Type parameters + +The mechanism described here does not handle the case where a module type has +parameters. Serialization and deserialization can only be done when the +parameters have been instantiated. The solution is to define a type where the +parameters are known. + +For example, if type `my_type` takes a type parameter: + +```ocaml +(* can use Make_version on T *) +module T = struct + type t = string my_type +end +``` + +## Drawbacks + +We may not need the versioning for `sexp` and `yojson`, if these representations +are not, in fact, used by other software. + +## Rationale and alternatives + +The main choice is between versioning and not versioning. If we don't use +versioning, given a representation, it becomes unclear what type it's associated +with. + +We could extend the discipline here to string representations, though those are +not automatically derivable. + +## Prior art + +The file `docs/style-guide.md` mentions versioning of stable module types. + +PR #1645, already merged, partially implements a module registration mechanism. +That implementation deals only with the `bin_io` representation, not `yojson` or +`sexp`. + +PR #1653, already merged, more completely implements a module registration +mechanism like the one described here. + +PR #1633 added a checklist item for versioned modules to the PR template. diff --git a/website/docs/researchers/rfcs/0018-better-logging.md b/website/docs/researchers/rfcs/0018-better-logging.md new file mode 100644 index 000000000..af3768984 --- /dev/null +++ b/website/docs/researchers/rfcs/0018-better-logging.md @@ -0,0 +1,242 @@ +--- +format: md +title: "RFC 0018: Better Logging" +sidebar_label: "0018 Better Logging" +hide_table_of_contents: false +--- + +> **Original source:** +> [0018-better-logging.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0018-better-logging.md) + +# Better Logging + +## Summary + +[summary]: #summary + +This RFC contains a proposal for a new logging system which will provide a +better and more flexible format for our logging which can be easily processed by +programs and is easy to construct into a human readable format. + +## Motivation + +[motivation]: #motivation + +Our logs in a state where they are both hard to read and hard to process. To +break down the individual issues with our logs: + +- String escaping is broken +- Messages contain lots of data in the form of large serialized sexp, reducing + readability +- No metadata is programmatically available without parsing the messages +- Existing tools (jq, rq) are very slow at processing the raw format of our logs +- Logs are very verbose, often containing redundant and unecessary information + (long "paths", all logs are stamped with same info of host+pid which never + changes across a single node) +- Lack of standard format or restrictions on length + +This design attempts to address each of these issues in turn. If done correctly, +we should be left with a logging format this is easily parsed by machines and +can be easily formatted for parsing by humans. + +## Detailed design + +[detailed-design]: #detailed-design + +### Format + +[detailed-design-format]: #detailed-design-format + +```json +{ + "timestamp": timestamp, + "level": level, + "source": { + "module": string, + "location": string + }, + "message": format_string, + "metadata": {{...}} +} +``` + +The new format elides some of the previous fields, most notably host and pid. +These can easily be decorated externally on a per log basis as these are really +attributes of the node and never change across log messages. The new format also +condenses the previous path section, which was used to store both the location +based context and logger attributes. Instead, a single source is stored in the +form of a module name and source location. The message is changed from a raw +string into a special format string which can interpolate metadata stored inside +of the logger message. The metadata field is an arbitrary json object which maps +identifiers to any json value. Logging context is stored inside of metadata and +can optionally be embeded into the message format string. + +The format string for the message will support interpolation of the form `$` +where `` is some key in `metadata`. A log processor should decide whether to +embed or elide this information based on the length of the interpolation and the +input of the user. More details on this are documented in the log processor +section below. + +### Logger Interface + +[detailed-design-logger-interface]: #detailed-design-logger-interface + +The logger interface will be changed slightly to support the new format. Most +noticably, it will take an associative list of json values for the metadata +field. It will also support the ability to derive a new logger with some added +context, much like the old `Logger.child` system. However, rather than tracking +source, this will allow for passing metadata down to log statements without +explicitly providing them to the log function each time. The logger will check +the correctness of the format string at compile time. + +### Logging PPX + +[detailed-design-logging-ppx]: #detailed-design-logging-ppx + +The new logger interface is not as clean as the old one, requiring some of the +same fields to be passed in over and over again in order to get source +information available. It also only performs metadata interpolation checks at +run time, where as some checks could be done at compile time. Furthermore, it +now requires manual calls to json serialization functions. In order to alleviate +these and make the interface nice, a PPX can be built. + +For instance: + +```ocaml +[%log_error logger "Error adding breadcrumb ${breadcrumb:Breadcrumb} to transition frontier: %s" [breadcrumb]] + (Error.to_string err) +``` + +would translate to + +```ocaml +Logger.error logger + ~module:__MODULE__ + ~location:__LOC__ + ~metadata:[("breadcrumb", Breadcrumb.to_yojson breadcrumb)] + "Error adding breadcrumb $breadcrumb to transition frontier: %s" + (Error.to_string err) +``` + +which would produce a log like (where the `<>`'s are replaced) + +```json +{ + "timestamp": , + "level": "error", + "source": { + "module": "<__MODULE__>", + "location": "<__loc__>" + }, + "message": "Error adding breadcrumb $breadcrumb to transition frontier: ", + "metadata": { + "breadcrumb": {}, + + } +} +``` + +and would be formatted as + +``` +Error adding breadcrumb {...} to transition frontier: +``` + +### Log Processor + +[detailed-design-log-processor]: #detailed-design-log-processor + +While the logging format is still in json, meaning that programs such as `jq` et +al. can be used to process logs, this has been too slow in practice. +Specifically, in the case of `jq`, there is limited support for handling files +full of multiple json objects (statically or streaming), so `jq` needs to be +reinvoked for every line in the log, which involves re-parsing and processing +filter expressions for each log entry. Instead, we will go back to writing our +own log processor utility. This makes additional sense in the context that we +want to perform and configure interpolation for our log messages. + +The log processor should support some basic configuration to control how +interpolation of the message string is performed. For instance, the default +could be to attempt to keep all message under a specific length (say `< 80`) and +to elide interpolation otherwise. Another option could be to constrain a max +length of the interpolated value, or even constrain interpolation by the type of +the value (for instance, elide all objects and interpolate other values). And +yet another would be to just always interpolate the entire values, or print +messages in a raw format with the relevant metadata displayed below the message. +These various options should be added as they are needed by developers, so I +will not lay out the specific requirements here. + +The log processor should support a simple `jq`-esque filter language. The scope +of this is _significantly_ smaller than that of `jq` as it only needs the +ability to form simple predicates on a json object. An example of what this +language could look like is speced out below in BNF. It mostly follows +javascript syntax with little divergence, except that it is much more +constrained in scope. The toplevel rule is `` since this is a filter +language. + +```bnf + ::= "true" | "false" + + ::= "0".."9" + ::= | + + ::= "a".."z" | "A".."Z" | "_" + ::= | "0" .. "9" + ::= '"' | '\' | '/' | 'b' | 'n' | 'r' | 't' + ::= "" | '\' | + ::= '"' '"' + + ::= + + ::= | | + ::= "," | + ::= "[" "]" + ::= "." | "[" "]" | "[" "]" + ::= | + ::= | + ::= | | "(" ")" + + ::= "\/" | + ::= "/" "/" + ::= "==" | "!=" | "in" + ::= | "match" + + ::= "&&" | "||" + ::= | | "!" | | "(" ")" +``` + +Unsupported features: + +- unicode escapes (`"\u235f"`) +- non strict equality (`==`, `!=`) +- object/array literals (`{some: "object"}`, `[1, 2, 3]`) +- bindings (`let x = ... in x + 1` or some more js style form) + +_NOTES: syntax currently does not support single quoted strings_ + +## Drawbacks + +[drawbacks]: #drawbacks + +This format is much more verbose than raw text logs, though this was true with +our old logging format as well. I believe we are more concerned about the +machine parsability of logs than we are about smaller logs. + +## Prior art + +[prior-art]: #prior-art + +Before our current format, we had a sexp style format with a custom log +processor. That system was not perfect for a few reasons, and this RFC attempts +to take the good parts of that setup and fix some of the nasty parts. In +particular, this RFC attempts to layout a more approachable filter language and +an easier to process form of metadata (formerly known as attributes). + +## Unresolved questions + +[unresolved-questions]: #unresolved-questions + +- How much work will the PPX be to create? Is there a way we can fully validate + that a message only interpolates known values at any point in the context + (perhaps by modeling metadata inheritence)? +- Is the filter language sufficient enough for our needs? diff --git a/website/docs/researchers/rfcs/0019-epoch-ledger-sync.md b/website/docs/researchers/rfcs/0019-epoch-ledger-sync.md new file mode 100644 index 000000000..d1e71ca5d --- /dev/null +++ b/website/docs/researchers/rfcs/0019-epoch-ledger-sync.md @@ -0,0 +1,210 @@ +--- +format: md +title: "RFC 0019: Epoch Ledger Sync" +sidebar_label: "0019 Epoch Ledger Sync" +hide_table_of_contents: false +--- + +> **Original source:** +> [0019-epoch-ledger-sync.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0019-epoch-ledger-sync.md) + +## Summary + +[summary]: #summary + +This RFC proposes and compares a number of methodologies for synchronizing and, +in some cases, persisting the epoch ledgers for proof of stake consensus +participation. + +## Motivation + +[motivation]: #motivation + +When starting a proposer on an active network, we currently have no way to +acquire epoch ledgers for the current state of the network. Right now, a +proposer must wait 2 full epochs in order to record the necessary epoch ledgers +in order to participate in consensus. This is suboptimal and increases our +bootstrapping time so significantly that it likely breaks some of the +assumptions of the Ouroboros papers and opens up new angles of attack on our +network. Therefore, there needs to be someway to at least synchronize this +information, acquiring it from other nodes on the network. Local persistence is +also good for mitigating the need to synchronize when a node goes offline for a +short period of time (< 1 epoch). + +## Unresolved questions + +[unresolved-questions]: #unresolved-questions + +- Is it ok to force non proposers to store the epoch ledgers as well? Snark + workers and "lite" clients do not actually _need_ this information in order to + be active, only proposers. This requirement, in particular, will be cumbersome + to "lite" clients running in the browser or on phones. +- Does anything stop nodes from just turning around and requesting this + information from another node in the network? It could be foreseeable that + someone who does not want to store and serve all of this information would + just redispatch the requests to other nodes on the network and proxy those + answers, bypassing the "forced participation" mechanism + +## Detailed design + +[detailed-design]: #detailed-design + +There are a number of options for implementing this, ranging from least correct +and shortest time to implement, to most correct but longest time to implement. +This RFC is layed out this way as there have been some questions as to what the +true, mainnet-ready implementation of this system should look like, or if it +should even exist internal to the protocol at all. Let's begin by reviewing the +problem at a high level, and then jumping into the various levels of +implementation. + +### General idea + +[detailed-design-general-idea]: #detailed-design-general-idea + +Epoch ledgers are "snapshot" at the beginning of each epoch. More specifically, +the epoch ledger for some epoch `n` is the snarked ledger of the most recent +block in epoch `n-1` (TODO: confirm with @evan). Within an epoch `n`, the ledger +we want to calculate our VRF threshold from is the epoch ledger of epoch `n-1`. +As such, we must keep two epoch ledgers around while participating in the +protocol: The epoch ledger for epoch `n`, and the epoch ledger for epoch `n-1`, +referred to as the `curr_epoch` and `last_epoch`, respectively. In a local +context, the only information we care about from an epoch ledger is the total +amount of currency in the ledger and the balance of any accounts our node can +propose for (right now, the proposers account + its delegated accounts). +However, in order to allow other nodes to synchronize the information they need, +nodes will be forced to store the entire epoch ledger at both these points. When +a node becomes active (is participating) and it finds that it does not have the +necessary epoch ledger information to properly participate, it will request the +`curr_epoch` and `last_epoch` epoch ledgers from its peers so that it can +participate properly. A node can identify what the correct merkle hashes for +these epoch ledgers should be by inspecting any protocol state within the +`curr_epoch`. + +#### Dumbest implementation (no persistence, high memory usage, high individual network answer size) + +[detailed-design-dumbest-implementation]: + #detailed-design-dumbest-implementation + +The easiest (and dumbest) way to implement this is to just have each ledger be a +full, in-memory copy of the ledger at that state in time, have no disk +persistence, and request the entire serialized ledger from peers to synchronize. +This will have a high memory usage as it will require us to store 2 complete in +memory copies of the ledger. Since the entire serialized ledger is served up +during synchronization, this method will also require nodes to potentially send +large amounts of data over the network when responding to queries. Therefore, +this option is not scalable at all. However, this option requires an absolute +minimum amount of work and would work fine for small ledgers, it will just break +our network when we have lots of accounts in our ledgers. + +#### Dumb implementation (no persistence, high memory usage, low individual network answer size) + +[detailed-design-dumb-implementation]: #detailed-design-dumb-implementation + +A slightly better way to do this would be to keep the same memory model for +representing the ledger, but using the sync ledger abstraction in order to +synchronize the contents of the ledger. This would have the effect of splitting +up the requests between multiple peers, where each individual answer will stay a +reasonable size. It will likely take slightly longer to sync using this method +on small networks, but on larger networks the sync ledger is likely to be faster +since it works more like a torrent system. This has some increased difficulty in +implementation, however, as there are no in memory ledger implementations +currently hooked into the sync ledger. This option would scale a little better +than the last one in the sense that it would still work with large numbers of +accounts as long as nodes have enough ram to store the 2 ledger copies in +memory. It's also a little more forward thinking as, when we implement the +participation logic, the epoch ledger will likely mutate into a rolling target +since each account will track an additional bit which needs to be synchronized +across proposers, though it is unclear whether or not that would fit directly +into the abstraction since different proposers on different forks will have +differing views of that information. + +#### Simple persistent implementation (persistence w/ high disk usage, lowest memory usage, low individual network answer size) + +[detailed-design-simple-persistent-implementation]: + #detailed-design-simple-persistent-implementation + +Instead of keeping the full ledger copy in memory, we could keep on disk copies. +Write speed is not a concern on epoch ledgers at all, and lookup speed a fairly +unimportant as well since the information required locally from the ledger can +be easily cached, so a node really only needs to lookup information in an epoch +ledger when serving queries to other nodes on the network. Since the snarked +ledger is already stored in RocksDB, it would be simple to just copy the entire +database over to another location for the epoch ledger. This would also simplify +the synchronization implementation as the persistent ledger is the version we +already use with the sync ledger. However, there would be some concerns in +ensuring we do this correctly. We need to make sure we can safely copy the +ledger while we have an active RocksDB instance (I believe RocksDB already has +support for this), and make sure that we write code that won't leak old epoch +ledger onto the filesystem. In particular, care needs to be taken when booting +up to ensure that you properly invalidate your old epoch ledgers if you need to +sync to new ones. + +#### Simple mask implementation (no persistence, medium memory usage, low individual network answer size) + +[detailed-design-simple-mask-implementation]: + #detailed-design-simple-mask-implementation + +When storing epoch ledgers in memory, the memory footprint can be greatly +reduced by using backwards chained masks. This would mean that we would only +store the diff between each epoch ledger and the diff between the `curr_epoch` +epoch ledger and the snarked ledger at the point of finality. There are two +downsides here from an implementation perspective though: firstly, we will need +to add support for backwards chained masks. This shouldn't be hard in theory, +but the mask code has been notoriously difficult to modify correctly. Secondly, +we will need to make the sync ledger work with a mask. This should be roughly +the same amount of effort that is required for other in-memory ledger +representations, though. + +#### Persistent mask implementation (persistence w/ medium disk usage, (lowest or medium) memory usage, low individual network answer size) + +[detailed-design-persistent-mask-implementation]: + #detailed-design-persistent-mask-implementation + +The most robust solution would be to use masks while persisting to the +filesystem. This would require building out a persistent mask layer. If this +route is taking, it is optional whether or not we want to even store the masks +in memory, and that decision could be made based on whether or not we need the +lookup performance (which we likely won't). However, this will mean that we have +to perform more disk io whenever we update the root snarked ledger in the +transition frontier (about 2-3x disk io per root snarked ledger mutations). This +could be mitigated in some ways, but is mostly unavoidable if we choose to +persist masks. + +### Forced participation + +[detailed-design-forced-participation]: #detailed-design-forced-participation + +Participation in this mechanism will be enforced by punishing peers who fail to +answer queries related to epoch ledger synchronization. This comes with a few +potential future issues, which are detailed in the +[unresolved questions section](#unresolved-questions). + +## Drawbacks + +[drawbacks]: #drawbacks + +This overall approach has a number of drawbacks. For one, it enforces a hard +requirement for nodes to store nearly 3x the data they would otherwise need to, +and in that sense, is very wasteful. Furthermore, this information we are +forcing all nodes to store has little benefit to each node directly, and +therefore feels more like a burden of the protocol design than a necessity to +correctness of the consensus mechanism. It also has implications on the scope of +what "lite" clients need to store in order to participate, as well as snark +workers and other "non-proposer" nodes. Some of the dumber options listed here +will certainly need to be rewritten or upgraded in the future, so in that sense, +our choice here could be taking an a decent amount of technical debt. + +## Rationale and alternatives + +[rationale-and-alternatives]: #rationale-and-alternatives + +An alternative to this would be to pull the responsibility for providing this +information out into a 3rd party service outside of the network protocol. If +this was done, it would lift the need for every node to store this large amount +of data locally and enable them to also synchronize more quickly as they would +not need to download the entire epoch ledger but, rather, could just download +the accounts and associated merkle proofs they are interested in evaluating VRFs +for. However, this comes with a number of other issues, mostly related to high +level concerns about the protocol's ability to maintain itself without external +3rd party services, and I cannot speak on those much as I cannot properly weight +the implications. diff --git a/website/docs/researchers/rfcs/0020-transition-frontier-extensions-2.md b/website/docs/researchers/rfcs/0020-transition-frontier-extensions-2.md new file mode 100644 index 000000000..062919e1e --- /dev/null +++ b/website/docs/researchers/rfcs/0020-transition-frontier-extensions-2.md @@ -0,0 +1,171 @@ +--- +format: md +title: "RFC 0020: Transition Frontier Extensions 2" +sidebar_label: "0020 Transition Frontier Extensions 2" +hide_table_of_contents: false +--- + +> **Original source:** +> [0020-transition-frontier-extensions-2.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0020-transition-frontier-extensions-2.md) + +## Summary + +[summary]: #summary + +The RFC proposes a new design for Transition Frontier Extensions (previously +known as the Transition Frontier Listener). The design in code has changed +drastically as newer development needs arose. This RFC intends to improve upon +the current design by consolidating concerns and providing a stricter guideline +to what should and should not be an extension. + +## Motivation + +[motivation]: #motivation + +Transition Frontier Extensions, as they are right now in the code, are fairly +messy. There are multiple representations of what a Transition Frontier Diff is, +some extensions exist only to resurface diff information, and extensions are not +provided some necessary information at initialization. New rules/guidelines are +necessary for determining what should and should not be an extension, as well as +what the scope of extensions should be. + +## Prior art + +[prior-art]: #prior-art + +See [#1585](https://github.com/CodaProtocol/coda/pull/1585) for early +discussions about Transition Frontier Extensions (then referred to as the +Transition Frontier Listener). + +## Detailed design + +[detailed-design]: #detailed-design + +### Direct List of Modification + +- pass Transition Frontier root into Extension's `initial_view` function +- remove Root_diff extension +- remove New_root diff +- replace diffs to micro-diffs w/ mutant types +- remove persistence diffs (and just use micro-diffs) +- rewrite Transition Frontier to use micro-diff pattern internally for + representing mutations +- add a diff pipe for tests + +### Extensions Redefined + +A Transition Frontier Extension is a stateful, incremental view on the state of +a Transiton Frontier. When a Transition Frontier is initialized, all of its +extensions are also initialized using the Transition Frontier's root. Every +mutation performed is represented as a list of diffs, and when the Transition +Frontier updates, each Extension is notified of this list of diffs +synchronously. Transition Frontier Extensions will notify the Transition +Frontier if there was a update to the Extension's view when handling the diffs. +If an Extension's view is updated, then a synchronous event is broadcast +internally with the new view of that Extension. A Transition Frontier Extension +has access to the Transition Frontier so that it can query and calculate +information it requires when it handles diffs. + +### Extension Guidelines + +An extension's only input should be Transition Frontier Diffs. An extension +should only be used if there is some incrementally computable view of +information on top of the Transition Frontier that cannot be queried in with +reasonable computational complexity on the fly. As an example, an extension +which just provides the best tip is useless since the best tip can just be +queried in O(1) time from the underlying Transition Frontier. An example of what +makes a good extension, however, is tracking information such as the set of +removed/added breadcrumbs from the best tip. With the exception of the +Persistence Buffer Extension, Transiton Frontier Extensions should not resurface +diffs they receives as that would be considered an abstraction leak. Diffs +should (ideally) only be surfaced as part of tests. As part of this RFC, the +Root_diff extension will be removed as this extension violates this last rule, +causing a leak of diff information. + +### New Transition Frontier Micro-diffs + +Transition Frontier Diffs will now be represented as smaller, composable +micro-diffs rather than monolithic diffs like before. The primary advantage of +this is composability, however it also helps to unify diffs with persistent +diffs, allowing us to more easily implement the incremental hash computation in +transition frontier and removes a layer of translation between diff formats. It +also more easily allows us to defer extra computation into the Transition +Frontier Extensions themselves by keeping the individual diffs light. Below is +an psuedo-code implementation of the new micro-diffs. + +```ocaml +type 'mutant diff = + | Breadcrumb_added : Breadcrumb.t -> {added: Breadcrumb.t; parent: Breadcrumb.t} diff + | Root_transitioned : {new: State_hash.t; garbage: Breadcrumb.t list} -> {previous: Root.t; new: Root.t; garbage: Breadcrumb.t list} diff + | Best_tip_changed : State_hash.t -> {previous: Breadcrumb.t; new: Breadcrumb.t} diff +``` + +Using these micro diffs, the Transition Frontier will return a list of diffs for +every mutation. For instance, it is possible for the Transition Frontier to just +have one diff `[Breadcrumb_added ...]`, or it could have upwards of three diffs +`[Breadcrumb_added ...; Root_transitioned ...; Best_tip_changed ...]`. + +NOTE: The New_root diff is no longer necessary as Transition Frontier roots are +now provided to Extensions at initialization. + +### In-Code Documentation and Notices + +As part of this RFC, we will add some documentation comments to the code related +to Extensions describing the guidelines for adding new extensions and what the +concerns of an extension should be. This will mostly just be regurgating +information from the section in here laying out those guidelines. + +### Implementation Details + +In order to break a dependency cycle between the Transition Frontier and its +Extensions, the Transition Frontier will be split up into a base implementation +and a final wrapper that glues the base implementation together with the +Extensions. The base Transition Frontier will include nearly all of the +Transition Frontier data structure internals (Breadcrumbs, Nodes, queries), but +it will not include the final full function for adding breadcrumbs to the tree. +Instead, it will provide two key functions that are part of adding breadcrumbs +with the following signature: + +```ocaml +(* [Diff.E.t] is the existential wrapper for a ['a Diff.t] *) +val calculate_diffs : t -> Breadcrumb.t -> Diff.E.t list +val process_diff : type mutant. t -> mutant Diff.t -> mutant +``` + +Extensions can then be defined using this base module. Then the final module can +glue everything together, as shown in this psuedo-code: + +```ocaml +module Extension = struct + module type S = sig + ... + end + + let update (module Ext : S) t diffs = + let opt_deferred = + let open Deferred.Option.Let_syntax in + let%bind view = Ext.handle_diffs t diffs in + Ext.broadcast t view + in + Deferred.map opt_deferred ignore +end + +module Extensions = struct + type t = + { snark_pool_refcount: Extension.Snark_pool_refcount + ; ... } + [@@deriving fields] + + let update_all t diffs = + fold t + ~snark_pool_refcount:(Extension.update (module Extension.Snark_pool_refcount)) + ... +end + +let add_breadcrumb t breadcrumb = + let diffs = calculate_diffs breadcrumb in + t.incremental_hash <- + List.fold_left diffs ~init:t.incremental_hash ~f:(fun hash (Diff.E.T diff) -> + Incremental_hash.hash hash (process_diff t diff)); + Extensions.update_all t.extensions diffs +``` diff --git a/website/docs/researchers/rfcs/0021-graphql-api.md b/website/docs/researchers/rfcs/0021-graphql-api.md new file mode 100644 index 000000000..76f2a12a4 --- /dev/null +++ b/website/docs/researchers/rfcs/0021-graphql-api.md @@ -0,0 +1,492 @@ +--- +format: md +title: "RFC 0021: Graphql Api" +sidebar_label: "0021 Graphql Api" +hide_table_of_contents: false +--- + +> **Original source:** +> [0021-graphql-api.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0021-graphql-api.md) + +## Summary + +[summary]: #summary + +A GraphQL api design to allow the wallet app (and future similar applications) +to communicate with a full node. + +## Motivation + +[motivation]: #motivation + +Currently, the main use case for this is to support the development of a wallet +application which can communicate with a full node which is running either +remotely or locally in another process. + +Related Goals: + +- Make app development using the api relatively easy and fast. +- Make it easier for people to communicate programmatically with a full node. +- Consumers of the api should not be required to write in a specific language. +- Eventually be able to rewrite the command-line client to use the same api. + +## Detailed design + +[detailed-design]: #detailed-design + +★ Note: Included below is the initial design which describes some of the goals +and thoughts associated with creating this api. The full schema (under +development) is specified at +https://github.com/CodaProtocol/coda/blob/master/frontend/wallet/schema.graphql. + +This design is heavily influenced by which data is stored on the node and which +is intended to be stored by the client. Most importantly, we assume the node +knows about some private keys which it associates to public keys for which it +will try to store related transactions indefinitely (where "related" means being +sent to or from the public key). We may also need to store information about +blocks that these keys have created. We will likely want to at least optionally +cache the history of the blocks/payments on the client side as well, just to +avoid needing to query it all every time. + +Note: There are several custom scalars defined at the top for readability. These +will be `String`s in the final implementation due to the complications involved +in the encoding of custom scalars not being expressed in the schema and needing +to be implemented symmetrically on the client and server. + +Note: Public keys will all be the "compressed" public keys that are used +elsewhere in the node. + +```graphql +# Note: this will all be strings in the actual api. +scalar Date +scalar PublicKey +scalar PrivateKey +scalar UInt64 +scalar UInt32 + +enum ConsensusStatus { + SUBMITTED + INCLUDED # Included in any block + FINALIZED + SNARKED + FAILED +} + +type ConsensusState { + status: ConsensusStatus! + estimatedPercentConfirmed: Float! +} + +type Payment { + nonce: Int! + submittedAt: Date! + includedAt: Date + from: PublicKey! + to: PublicKey! + amount: UInt64! + fee: UInt32! + memo: String +} + +type PaymentUpdate { + payment: Payment + consensus: ConsensusState! +} + +enum SyncStatus { + ERROR + BOOTSTRAP # Resyncing + STALE # You haven't seen any activity recently + SYNCED +} + +type SyncUpdate { + status: SyncStatus! + estimatedPercentSynced: Float! + description: String +} + +type SnarkWorker { + key: PublicKey! + fee: UInt32! +} + +type SnarkFee { + snarkCreator: PublicKey! + fee: UInt32! +} + +type SnarkFeeUpdate { + fee: SnarkFee + consensus: ConsensusState! +} + +type Block { + coinbase: UInt32! + creator: PublicKey! + payments: [Payment]! + snarkFees: [SnarkFee]! +} + +type BlockUpdate { + block: Block! + consensus: ConsensusState! +} + +type Balance { + total: UInt64! + unknown: UInt64! +} + +type Wallet { + publicKey: PublicKey! + balance: Balance! +} + +type NodeStatus { + network: String +} + +## Input types + +input AddWalletInput { + public: PublicKey + private: PrivateKey +} + +input DeleteWalletInput { + public: PublicKey +} + +input AddPaymentReceiptInput { + receipt: String +} + +input SetNetworkInput { + address: String +} + +input SetSnarkWorkerInput { + worker: PublicKey! + fee: UInt32! +} + +input CreatePaymentInput { + from: PublicKey! + to: PublicKey! + amount: UInt64! + fee: UInt32! + memo: String +} + +input PaymentFilterInput { + toOrFrom: PublicKey +} + +input BlockFilterInput { + creator: PublicKey +} + +## Payload types + +type CreatePaymentPayload { + payment: Payment +} + +type SetSnarkWorkerPayload { + worker: SnarkWorker +} + +type SetNetworkPayload { + address: String +} + +type AddPaymentReceiptPayload { + payment: Payment +} + +type AddWalletPayload { + publicKey: PublicKey +} + +type DeleteWalletPayload { + publicKey: PublicKey +} + +# Pagination types + +type PageInfo { + hasPreviousPage: Boolean! + hasNextPage: Boolean! +} + +type PaymentEdge { + cursor: String + node: PaymentUpdate +} + +type PaymentConnection { + edges: [PaymentEdge] + nodes: [PaymentUpdate] + pageInfo: PageInfo! + totalCount: Int +} + +type BlockEdge { + cursor: String + node: BlockUpdate +} + +type BlockConnection { + edges: [BlockEdge] + nodes: [BlockUpdate] + pageInfo: PageInfo! + totalCount: Int +} + +type Query { + # List of wallets for which the node knows the private key + ownedWallets: [Wallet!]! + + wallet(publicKey: PublicKey!): Wallet + + payments( + filter: PaymentFilterInput + first: Int + after: String + last: Int + before: String + ): PaymentConnection + + blocks( + filter: BlockFilterInput + first: Int + after: String + last: Int + before: String + ): BlockConnection + + # Null if node isn't performing snark work + currentSnarkWorker: SnarkWorker + + # Current sync status of the node + syncState: SyncUpdate! + + # version of the node (commit hash or version #) + version: String! + + # Network that the node is connected to + network: String + status: NodeStatus +} + +type Mutation { + createPayment(input: CreatePaymentInput!): CreatePaymentPayload + + setSnarkWorker(input: SetSnarkWorkerInput!): SetSnarkWorkerPayload + + # Configure which network your node is connected to + setNetwork(input: SetNetworkInput!): SetNetworkPayload + + # Adds transaction to the node (note: Not sure how we want to represent this yet) + addPaymentReceipt(input: AddPaymentReceiptInput!): AddPaymentReceiptPayload + + # Tell server to track a private key and all associated transactions + addWallet(input: AddWalletInput!): AddWalletPayload + + # Deletes private key associated with `key` and all related information + deleteWallet(input: DeleteWalletInput!): DeleteWalletPayload +} + +type Subscription { + # Subscribe to sync status of the node + newSyncUpdate: SyncUpdate! + + # Subscribe to payments for which this key is the sender or receiver + newPaymentUpdate(filterBySenderOrReceiver: PublicKey!): PaymentUpdate! + + # Subscribe all blocks created by `key` + newBlock(key: PublicKey): BlockUpdate! + + # Subscribe to fees earned by key + newSnarkFee(key: PublicKey): SnarkFee! +} + +schema { + query: Query + mutation: Mutation + subscription: Subscription +} +``` + +Staking is a little bit tricky because of the different states involved. These +queries are pulled out for clarity, but you can imagine them being simply added +to the above schema. + +You can either perform staking yourself, or delegate your stake to another +public key + +- Changing your delegation status is a transaction, which will experience the + same consensus flow as other transactions, with the addition of having to wait + an additional epoch (?) for it to actually come into effect. +- Doing staking work yourself doesn't involve a transaction unless you need to + cancel an existing delegation first. + +This means that at any given moment, there could be a number of pending +delegation transactions all awaiting consensus. + +```graphql +type Delegation { + nonce: UInt32! + submittedAt: Date! + includedAt: Date + from: PublicKey! + to: PublicKey! + fee: UInt32! + memo: String +} + +type DelegationUpdate { + status: Delegation! + + # We may have reached consensus but still be waiting for the correct epoch + active: Boolean! + + consensus: ConsensusState! +} + +type Query { + stakingStatus(key: PublicKey): Boolean! + + # Most recent status for each relevant delegation transaction + delegationStatus(key: PublicKey!): [DelegationUpdate]! +} + +type Subscription { + newDelegationUpdate(publicKey: PublicKey): DelegationUpdate! +} + +input SetStakingInput { + on: Boolean +} + +input SetDelegationInput { + from: PublicKey! + to: PublicKey! + fee: UInt32! + memo: String +} + +type SetStakingPayload { + on: Boolean +} + +type SetDelegationPayload { + delegation: Delegation +} + +type Mutation { + setStaking(input: SetStakingInput!): SetStakingPayload + setDelegation(input: SetDelegationInput!): SetDelegationPayload +} +``` + +Pagination is done in the relay "connections" style +(https://facebook.github.io/relay/graphql/connections.htm). This means that a +paginated api will expose an "edges" field, each of which wraps a node (the +element being paginated over) with its corresponding cursor. This allows you to +pass the cursor to the "after" argument of the query along with a "first" +(describing how many elements to return), which in the case below, will result +in the "first" 10 elements "after" `"cursor"` to be returned. `hasNextPage` lets +you know whether or not you need to query for another page. As an example, use +of paginated endpoints might look something like this: + +```graphql +{ + payments(first: 10, after: "cursor") { + edges { + cursor + node { + payment { + amount + } + } + } + pageInfo { + hasNextPage + } + } +} +``` + +## Drawbacks + +[drawbacks]: #drawbacks + +We could potentially make a wallet by making queries though the existing +command-line client, which might involve less work. This brings in a dependency +on graphql and reimplements/replaces some of the work done for the RPC +interface. + +## Rationale and alternatives + +[rationale-and-alternatives]: #rationale-and-alternatives + +- The payment and block history for the wallets could be only stored on the + client. This would make the node only be responsible for storing the private + keys of the wallets. This results in a somewhat simpler/lighter full node, + with the obvious cost that whenever you close your wallet app, you're missing + any transactions that might be happening, which isn't great for the wallet + experience. +- The wallet could be responsible for storing the private keys as well, but then + the node would have to ask the wallet for them whenever it needed them for + snarking etc. Probably not what we want. + +Why GraphQL? + +- Defines a typed interface for communication between client and server. +- Interface is explorable/discoverable with great existing tools, making + development easier. +- Strong OCaml/Reason support without requiring that consumers of the api be + written in a specific language. + +Alternatives to GraphQL: + +- Falcor (or similar) +- Completely custom alternative. + +We considered a custom alternative as a single wallet app connected to a single +node instance isn't a very standard setup for graphql app, which we saw as more +associated with public web apis with many consumers. However, all the work in +this case would obviously have to be from scratch, representing potentially +significant time investment to get stable for questionable gain. Client +libraries would also have to be written for any languages that wanted to +interface, whereas GraphQL already supports many and seems to have pretty good +community buy-in. + +## Prior art + +[prior-art]: #prior-art + +- RPC interface that the commandline client uses: This has inspired the graphql + api though in some cases we have tried to simplify the interface and avoid any + binary serialization that might rely on having ocaml at both ends. +- REST server in the node: A simple interface that uses rest to deliver some + status pages. + +## Unresolved questions + +[unresolved-questions]: #unresolved-questions + +- Several apis involve objects that get incrementally updated during consensus + It's important that we have a reliable way to associate updates with their + corresponding objects in memory. We don't have a concept of IDs for most of + these objects, but if we could create one by hashing together enough + properties of the objects, it might help with this and avoid accidents + comparing fields in an insufficient way. +- Authentication +- Potentially out of scope: How will this evolve and be used in the future, when + most wallets will just run a light node locally? +- Future work: Create a utility to allow for hardware wallets to be used. This + will involve calling the addWallet mutation without a private key and having + the node talk to another process that interacts with the hardware wallet. diff --git a/website/docs/researchers/rfcs/0022-postake-naming-conventions.md b/website/docs/researchers/rfcs/0022-postake-naming-conventions.md new file mode 100644 index 000000000..34974af2a --- /dev/null +++ b/website/docs/researchers/rfcs/0022-postake-naming-conventions.md @@ -0,0 +1,51 @@ +--- +format: md +title: "RFC 0022: Postake Naming Conventions" +sidebar_label: "0022 Postake Naming Conventions" +hide_table_of_contents: false +--- + +> **Original source:** +> [0022-postake-naming-conventions.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0022-postake-naming-conventions.md) + +## Summary + +[summary]: #summary + +This RFC proposes a new standard for names related to our implementation proof +of stake. + +## Motivation + +[motivation]: #motivation + +There has been a great deal of misunderstandings and miscommunications around +proof of stake in the past, and nearly all of this has been due to non-unique +and non-specific names. By fixing this, the hope would be that the team is able +to communicate more effectively about concepts in proof of stake without +relaying a ground base of information at the beginning of every meeting. + +## Detailed design + +[detailed-design]: #detailed-design + +The biggest source of confusion is the ambiguous and non-unique use of "prev" +and "next". The goal with these names is to remove those names and keep all +names unambiguous. These name changes would be reflected in the code after we +agree upon them. + +``` +Epoch Ledger: the ledger for VRF evaluations (associated with a specific epoch) +Current Epoch: the epoch which the blockchain is currently in +Staking Epoch: the epoch before the current epoch (which provides information for VRF evaluations within the current epoch) +Staking Epoch Ledger: the epoch ledger from the staking epoch (actively used for VRF evaluations within the current epoch) +Next Epoch Ledger: the epoch ledger from the current epoch (will be used for VRF evaluations in the next epoch) +Blockchain Length: the total number of blocks in the blockchain +Epoch Count: the total number of epochs that contain blocks +Epoch Length: the number of blocks in an epoch +Epoch Seed Update Range: the middle third of an epoch (where the epoch seed is calculated) +Epoch Seed: the input for the VRF message which is calculated in the seed update range +Checkpoint: a pointer to a state hash that is in the blockchain +Epoch Start Checkpoint: a pointer to the state hash immediately preceding the first block of an epoch +Epoch Lock Checkpoint: a pointer to the state hash immediately preceding the first block in the last third of an epoch (the last block in the seed update range) +``` diff --git a/website/docs/researchers/rfcs/0023-glossary-terms.md b/website/docs/researchers/rfcs/0023-glossary-terms.md new file mode 100644 index 000000000..b9656266e --- /dev/null +++ b/website/docs/researchers/rfcs/0023-glossary-terms.md @@ -0,0 +1,259 @@ +--- +format: md +title: "RFC 0023: Glossary Terms" +sidebar_label: "0023 Glossary Terms" +hide_table_of_contents: false +--- + +> **Original source:** +> [0023-glossary-terms.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0023-glossary-terms.md) + +## Summary + +[summary]: #summary + +This RFC proposes standardized glossary terms for various concepts in the Coda +Protocol and how they are communicated to the external world in documentation, +logging, user interfaces, and public communications. This is not intended to +cover the existing codebase in scope, as it may be too cumbersome at this point +to rename everything to fit these proposed conventions. + +However, it is advisable that over time the codebase also migrates to this +shared terminology so that future community members can on-board seamlessly from +high level documentation to contributing code. + +## Motivation + +[motivation]: #motivation + +Currently, multiple terms exist for several concepts in the Coda protocol, and +they are used interchangeably when communicating to the world. This has some +downsides as there is a lack of clarity on which terms to use, and could +potentially cause confusion for people unfamiliar with the protocol. + +As such, the motivation of this RFC is to standardize the naming convention for +various concepts into a unified glossary that can be used in documentation, +user-facing copy, and marketing channels. The expected outcome is unified +language when communicating to the public and in developing Coda related +products. This can then lead to more clarity and easier onboarding. + +## Detailed design + +[detailed-design]: #detailed-design + +The proposal contains the suggested term with it's associated concept it +encapsulates, as well as alternative terms and the rationale for the suggestion. + +The terms proposed are below: + +### Coda vs coda + +**Concept:** "Coda" is the network / protocol. "coda" is the token. + +**Rationale:** Aligns with other networks, and makes it easy to differentiate +context based on casing. + +**Downsides:** More things to remember. + +**Usage:** The native token of the Coda network is coda. + +**Alternatives:** + +- Use "Coda" everywhere. + +### Block Producer + +**Concept:** A node that participates in a process to determine what blocks it +is allowed to produce, and then produces blocks that can be broadcast to the +network. + +**Rationale:** This term is used by other protocols, clearly describes the role +duties, and is unambiguous. + +**Downsides:** EOS is the most popular chain that uses "block producers" and +there is a chance of confusion if people think Coda also has only 21 block +producers - but this is a tail risk. + +**Usage:** A block producer or block producing node stakes its coda and coda +delegated to it in order to participate in consensus. + +**Alternatives:** + +- Validator - used by most other protocols, but not suitable for Coda, as every + node can be a "verifier" or "validator" of SNARKs. +- Staker - tied to a specific consensus model, and doesn't explain what the node + does for the network. It also has an edit distance of 2 from snarker, which + can cause some confusion. +- Proposer - currently used in codebase, but unused by other protocols, and also + sounds less firm / tied to generating blocks (relative to producer). +- Prover - currently used to describe the action of generating the block SNARK, + but if this that job moves to snark workers, then this term is irrelevant. +- Miner, Baker, etc - not relevant to Coda's consensus mechanism. + +### Block + +**Concept:** A set of transactions and consensus information that extend the +state of the network. Includes a proof that the current state of the network is +fully valid. + +**Rationale:** This term aligns with the rest of the industry, and it makes it +easier to onboard other blockchain users. + +**Downsides:** Perhaps it is not as technically accurate as a transition. + +**Usage:** Blocks enable the Coda network's state to be updated. Blocks include +transactions to be applied to the ledger as well as various pieces of consensus +information that allow the network to eventually agree on what blocks will stay +in the blockchain. + +**Alternatives:** + +- External transition - maybe this is more precise, but has the heavy lifting of + having to explain a new term for a mostly similar concept. Additionally, the + word transition is very similar to transaction, causing confusion at times. + +### Snark worker + +**Concept:** A Coda node that produces SNARKs to compress data in the network. + +**Rationale:** The community likes SNARKs and it is a point of differentiation +for Coda, so Snark worker is an engaging term. + +**Downsides:** Gets invalidated if Coda switches to another type of +zero-knowledge proof. NOTE - There is some concern about using the progressive +tense of "snark" as a verb, eg. "snarking" - as this can be confused with +staking. However, if the community enjoys "snarking", it would make sense to +continue using it as a verb. + +Additionally, the usage of "work" in snark worker can be misconstrued as a +connection to Proof-of-Work, but this will addressed in documentation under the +FAQ section. + +**Usage:** + +- Anyone can join the coda network and become a snark worker. +- Snark workers help compress the blockchain by generating SNARKs. + +**Alternatives:** + +- Compressor - this term was considered initially, but given the community + excitement about SNARKs, it makes sense to include the more specific term in + the lexicon. +- Snarker - this term is used interchangeably with "snark worker", but it is + suggested to converge on snark worker. + +### Full node + +**Concept:** A Coda node that is able to verify the state of the network - +however it may need to request paths for its accounts from other nodes that have +all the accounts state. + +**Rationale:** By calling this type of node a full node, Coda distinguishes +itself from other networks, as all nodes are technically full nodes if they can +verify SNARKs - these nodes are not required to trust other nodes. + +**Downsides:** Because these nodes still need to request state from other nodes, +there is an argument to be made that they are not full nodes. Furthermore, +Bitcoin classifies full nodes as nodes that "download every block and +transaction and check them against Bitcoin's consensus rules." Therefore, there +may be some community pushback against calling these nodes full nodes, even +though they are trustless. + +**Usage:** On the Coda network, even phones can run full nodes! + +**Alternatives:** + +- Fully verifying nodes - more accurate, but a mouthful, and have to create a + new term. +- Trustless nodes - again, some lifting required. +- Light node / light client - has a negative connotation of not being able to + validate chain state. + +### User Transaction + +**Concept:** A transaction issued by a user - currently a payment or a +delegation change. + +**Rationale:** This term clearly describes the concept, and leaves room for +further types of user issued transactions. + +**Downsides:** This is a subset of "Transactions", so it is a bit redundant. + +**Usage:** There are three types of transactions in the Coda network currently - +user transactions, fee transfers, and coinbases. User transactions allow users +to manage accounts and send money. + +**Alternatives:** + +- User command - this term is confusing, especially in docs, as it can be + conflated with a CLI command that a user issues. + +### Protocol Transaction + +**Concept:** A transaction issued by the protocol - currently a fee transfer or +coinbase (both are structurally identical). + +**Rationale:** In discussions with parties that were not familiar with the +protocol, fee transfers were misunderstood as representations of fees associated +with a user transaction. As soon as it was explained that these were +transactions programmatically issued by the protocol rather than a user, it +became clear. As such, it is recommend to partition transactions into _user +transactions_ and _protocol transactions_. + +**Downsides:** If there will ever be fee transfers issued by users, this will +break the proposed structure. However, there currently doesn't seem to be any +plans currently to do that. + +**Usage:** Snark workers are compensated for their snark work by protocol +transactions. + +**Alternatives:** + +- Fee transfers - current usage, confusing to unfamiliar users. + +### Block Hash + +**Concept:** A hash that serves as an identifier for a specific block in the +blockchain. + +**Rationale:** Most major cryptocurrency protocols use the term block hash, and +it is helpful to align with existing terms when possible. + +**Downsides:** This didn't make sense when blocks were referred to as external +transitions. Now that blocks is the accepted terminology, it should follow that +the updated name for the reference is block hash. + +**Usage:** Each block contains a unique block hash that is used as an +indentifier. + +**Alternatives:** + +- State hash - the previous term -- perhaps more accurate, but doesn't conform + to broader industry terms. + +## Drawbacks + +[drawbacks]: #drawbacks + +The drawback to aligning on language is that we lose some specificity that comes +with each specific term, relative to the others. However, this concern is minor, +and is superceded by the need for consistent language. + +## Rationale and alternatives + +[rationale-and-alternatives]: #rationale-and-alternatives + +See [design section](#detailed-design) for rationale and alternatives. + +## Prior art + +[prior-art]: #prior-art + +- Previous RFC regarding nomenclature in code: + https://github.com/CodaProtocol/coda/blob/develop/rfcs/0018-postake-naming-conventions.md + +## Unresolved questions + +[unresolved-questions]: #unresolved-questions + +There will likely be other terms we will need to converge on. diff --git a/website/docs/researchers/rfcs/0024-memos-with-arbitrary-bytes.md b/website/docs/researchers/rfcs/0024-memos-with-arbitrary-bytes.md new file mode 100644 index 000000000..befbabef1 --- /dev/null +++ b/website/docs/researchers/rfcs/0024-memos-with-arbitrary-bytes.md @@ -0,0 +1,72 @@ +--- +format: md +title: "RFC 0024: Memos With Arbitrary Bytes" +sidebar_label: "0024 Memos With Arbitrary Bytes" +hide_table_of_contents: false +--- + +> **Original source:** +> [0024-memos-with-arbitrary-bytes.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0024-memos-with-arbitrary-bytes.md) + +## Summary + +Allow arbitrary bytes in the user command memo field as alternative to Blake2 +digests. + +## Motivation + +When creating a transaction, users may wish to add a meaningful description. The +memo field is a convenient place for such a description. The description could +be viewed in a wallet app. + +## Detailed design + +Memos that contain Blake2 digests should still be available. To distinguish +memos with digests from memos of bytes, we can prepend a tag byte to the memo +string. + +Memos are still of fixed length, but the bytes provided may not fill up the +memo. The length is given in another prepended byte. For digests, that length is +always 32. For memos of bytes where the length is less than 32, the memo is +right-padded with null bytes (the OCaml character '\x00'). + +To create a memo of bytes, the input can be provided as an OCaml string or a +value of type "bytes". The length of that input is verified to be no greater +than 32. + +## Drawbacks + +The tag and length bytes increase the size of the memo field, so that the size +of data processed by transaction SNARKs increases, increasing the times for +proving. + +Giving users full control of the memo field allows them to put illegal or +morally dubious content there, which could harm the reputation of Coda. + +## Rationale and alternatives + +We could distinguish wholly-arbitrary bytes from human-readable byte sequences, +by requiring the input to be UTF-8 in the latter case. A third tag could be used +for that purpose. That would add a small amount of complexity to the API. + +The current memo data size of 32, which is derived from the Blake2 digest size, +could be increased, at the expense of more work for the SNARK. + +Of course, we could leave the memo contain as-is, containing only Blake2 +digests, which would avoid the drawbacks mentioned. + +## Prior art + +RFC #2708 implements the strategy described here. The current wallet design +allows reading and viewing of memos in transactions. + +## Unresolved questions + +The performance impact of this change has not been evaluated. + +Should we enforce full memo validity, as implemented in the "is_valid" function +in the implementation, in the SNARK? If not, what aspects of the memo should be +enforced? Arguably, the structure of the memo bytes don't affect the protocol. + +It's unknown how eventual Coda users will make use of this facility beyond what +the wallet implementation provides. diff --git a/website/docs/researchers/rfcs/0025-time-locked-accounts.md b/website/docs/researchers/rfcs/0025-time-locked-accounts.md new file mode 100644 index 000000000..61b39a880 --- /dev/null +++ b/website/docs/researchers/rfcs/0025-time-locked-accounts.md @@ -0,0 +1,105 @@ +--- +format: md +title: "RFC 0025: Time Locked Accounts" +sidebar_label: "0025 Time Locked Accounts" +hide_table_of_contents: false +--- + +> **Original source:** +> [0025-time-locked-accounts.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0025-time-locked-accounts.md) + +## Time-locked and time-vesting accounts + +Accounts may contain funds that are available to send according to a vesting +schedule. + +## Motivation + +Time-locked accounts are useful to delay the availability of funds. Such +accounts have been used in other cryptocurrencies to implement features such as +payment channels. The locking feature can be used to create incentives; if the +account holder performs a certain action, the account can be funded so that the +locked funds are released. + +Vesting schedules are a generalization of the time-locking mechanism, where +funds become increasingly available over time. There are two times specified, +one that indicates when some amount of funds becomes available to send, and +another, later time that specifies when all funds become available to send. The +amount available to send increases between those two times. + +## Detailed design + +Accounts may have no timing restrictions, have a time lock, or a vesting +schedule. In Coda, the type `Account.t` is a record type with several fields. An +additional field `timing` could contain an element of a sum type with three +alternatives: `Untimed` and `Timed`. + +`Untimed` means there are no time restrictions on sending funds. + +A `Timed` value contains an `initial_minimum_balance`, `cliff` time, a +`vesting_period` time, and a `vesting_increment`. Until the cliff time has +passed, the account can send only funds from its balance that are in excess of +the initial minimum balance. After the cliff time has passed, more funds are +available, by calculating a current minimum balance which decreases over time. +The minimum balance decreases by the vesting increment for each vesting period: + +``` + current_minimum_balance = + if global_slot < cliff_time + then + initial_minimum_balance + else + max 0 (initial_minimum_balance - ((global_slot - cliff_time) / vesting_period) * vesting_increment)) +``` + +(where / is integer division). + +If a transaction amount would make the account balance less than the current +minimum, the transaction is disallowed. + +Nothing prevents sending funds to timed accounts. When timed accounts are +created, the balance must be at least the initial minimum balance. The account +can receive additional funds, and funds can be spent, as long as the minimum +balance invariant is maintained. + +A time-locked account time, can be created by using a vesting increment equal to +the initial minimum balance and a vesting period of one slot. (In the +calculation above, using a vesting period of zero would result in a division by +zero.) + +# Implementation + +The restrictions on sending funds need to be enforced in the transaction SNARK +and out-of-SNARK. If the restrictions are violated, an error occurs. + +For the out-of-SNARK transaction, the relevant code location to enforce the +restrictions appears to be in `Transaction_logic.apply_user_command_unchecked`. + +For the in-SNARK transaction, the restriction would be enforced in +`Transaction_snark.apply_tagged_transaction`. + +## Drawbacks + +Adding this feature makes the account data slightly larger, slows down the +validation of transactions, and makes the implementation a bit more complex. +Those considerations should be weighed against the utility of the feature to +users. + +## Rationale and alternatives + +Instead of using a global slot time, we could use block height. The global slot +time is available via the protocol state, which will be made available to +transactions. + +## Prior art + +Bitcoin uses a block field `nLockTime`, denoting a block time, which can be used +to time-lock individual transactions. There are many articles about time-locked +accounts for Ethereum by programming smart contracts. See, for example: +https://medium.com/bitfwd/time-locked-wallets-an-introduction-to-ethereum-smart-contracts-3dfccac0673c. +Beyond cryptocurrencies, vesting of benefits is a well-established idea in +employment. + +## Unresolved questions + +Do timed accounts interact with staking in any way? diff --git a/website/docs/researchers/rfcs/0026-transition-caching.md b/website/docs/researchers/rfcs/0026-transition-caching.md new file mode 100644 index 000000000..258bc316f --- /dev/null +++ b/website/docs/researchers/rfcs/0026-transition-caching.md @@ -0,0 +1,91 @@ +--- +format: md +title: "RFC 0026: Transition Caching" +sidebar_label: "0026 Transition Caching" +hide_table_of_contents: false +--- + +> **Original source:** +> [0026-transition-caching.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0026-transition-caching.md) + +# Transition Caching + +## Summary + +[summary]: #summary + +A new transition caching logic for the transition router which aims to track +transitions which are still being processed as well as transitions which have +failed validation. + +## Motivation + +[motivation]: #motivation + +Within the transition router system, the only check for duplicate transitions is +performed by the transition validator, and each transition is only checked +against the transitions which are currently in the transition frontier. However, +there are two types of duplicate transitions which are not being checked for: +transitions which are still being processed by the system (either in the +processor pipe or in the catchup scheduler and catchup thread), and transitions +which have been determined to be invalid. In the case of the former, the system +ends up processing more transitions than necessary, and the number of duplicated +processing increases along with the networks size. In the case of the latter, +the system is opened up for DDoS attacks since an adversary could continuously +send transitions with valid proofs but invalid staged ledger diffs, causing each +node to spend a significant enough amount of time before invalidating the +transition each time it recieves it. + +NOTE: This RFC has been re-scoped to only address duplicate transitions already +being processed and not transitions which were previously determined to be +invalid. + +## Detailed design + +[detailed-design]: #detailed-design + +The goal here is to introduce a new cache to the system: the +`Unprocessed_transition_cache`. + +`Unprocessed_transition_cache` is scoped explicitly to the +`Transition_frontier_controller`. The set stored in this cache represents the +set of transitions which have been read from the network but have not yet been +processed and added to the transition frontier. Since the lifetime of elements +in the set are finite, the `Unprocessed_transition_cache` can be represented as +a hash set. It will be the responsibility of the transition validator to add +items to this cache, and the responsibility of the processor to invalidate the +cache once transitions are added to the transition frontier. Transitions which +are determined to be invalid need to also be invalidated. + +In order to assist in ensuring that items in the cache are properly invalidated, +I recommend the introduction of a `'a Cached.t` type which will track the state +of the item in one or more caches. The `Cached` module would provide an +interface for performing cache related actions and would track a boolean value +representing whether or not the required actions have been performed. What's +special about the `'a Cached.t` type is that it will have a custom finalization +handler which will throw an exception if no cache actions have been performed by +the time it is garbage collected. This exception is toggled by a debug flag. +When the debug flag is off, the finalization handler will nearly log a message. + +## Drawbacks + +[drawbacks]: #drawbacks + +- Proper cache invalidation is difficult to maintain full state on. Hopefully + the `Cached` abstraction will be enough to alleviate this pain. + +## Rationale and alternatives + +[rationale-and-alternatives]: #rationale-and-alternatives + +- This cache is small and simple in its scope. +- `Cached` enforces that our architecture correctly invalidates the + `Unprocessed_transition_cache`, avoiding cache leaks. + +## Unresolved questions + +[unresolved-questions]: #unresolved-questions + +- `Cached` will throw errors only at run time. Perhaps a GADT would be better, + but I'm unsure how to model this in a way with a GADT that would 100% assert + that we perform some kind of cache action. diff --git a/website/docs/researchers/rfcs/0027-wallet-internationalization.md b/website/docs/researchers/rfcs/0027-wallet-internationalization.md new file mode 100644 index 000000000..3193f526f --- /dev/null +++ b/website/docs/researchers/rfcs/0027-wallet-internationalization.md @@ -0,0 +1,160 @@ +--- +format: md +title: "RFC 0027: Wallet Internationalization" +sidebar_label: "0027 Wallet Internationalization" +hide_table_of_contents: false +--- + +> **Original source:** +> [0027-wallet-internationalization.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0027-wallet-internationalization.md) + +## Summary + +--- + +Integrate an internationalization library into the Coda wallet for +multi-language support. + +## Motivation + +--- + +The Coda wallet currently has only English text, making it more difficult for +non-English speaking developers and users to easily contribute and use. + +The wallet should support multiple languages to reach a wide-ranging audience. + +## Detailed design + +--- + +React-intl is currently one of the most well-supported internationalization +library for React. It is built upon the native +**[Internationalization API](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Intl)** +browser API, and neatly wraps them for performance and usability enhancements. + +Lib: +[https://github.com/formatjs/react-intl](https://github.com/formatjs/react-intl) + +Bs-bindings: +[https://github.com/reasonml-community/bs-react-intl](https://github.com/reasonml-community/bs-react-intl) + +General steps to implement react-intl + +1. Create a JSON of IDs mapped to their text. Each new language will have their + own JSON file. + +The best option to load the localization text would be to package the JSON files +as static assets and load the language on-demand. The V1 iteration doesn't need +this though. A couple languages to start in-memory is okay until the assets +become too large (over <100KB of text). + +```json +// en.json +[ + { + "id": "page.hello", + "defaultMessage": "Hello", + "message": "" + } +] +``` + +Discussion: Am I converting _all_ texts in this grant? It's within capacity. +Just not sure if the Coda team is comfortable with having an app-wide change +immediately due to some potential internal conflicts. + +2. Wrap the app with the react-intl provider. + +```javascript + // ReactApp.re + let make = () => { + let settingsValue = AddressBookProvider.createContext(); + let onboardingValue = OnboardingProvider.createContext(); + // Determine how to pass the correct locale and translation messages to IntlProvider + // locale state is ideally loaded async from local storage if users have set a preference + let (locale, dispatch) = reducer->React.useReducer(initialState); + module Locale { + type locale = + | En + | Es; + let toString = (locale) => { + switch (locale) { + | En => "en" + | Es => "es" + }; + } + // https://github.com/reasonml-community/bs-react-intl/blob/master/examples/Locale.re + // let toTranslation => ... + // The toTranslation util from the example is too long to insert here as well + } + Locale.toString} + messages={locale->Locale.toTranslations}> + + {...rest of the app} + + ; + }; +``` + +3. Replace any hardcoded English text with `FormattedMessage` + +```javascript +// some react file + +``` + +4. Potential work to format time and date + +React-Intl provides `FormattedDate` and `FormattedTime`, even +`FormattedRelativeTime` for those awesome "1 hour ago" calculations. + +## Drawbacks + +--- + +Without strong support of any given language, mistranslations may occur and +continual support may drop. Lack of ongoing support means the application +becomes a mix of the default language (English) with translated bits. + +## Rationale and alternatives + +--- + +Besides being the most well supported library in the space, react-intl already +has the bindings supported by the Reason community. Having to write the bindings +ourselves adds more unnecessary complexity. + +Alternatives to react-intl are listed below, although Bucklescript-bindings do +not yet exist for them. + +- lingui.js - + [https://github.com/lingui/js-lingui](https://github.com/lingui/js-lingui) + - Lingui is a 5kb framework, and offers both React and non-React APIs. +- react i18next - + [https://github.com/i18next/react-i18next](https://github.com/i18next/react-i18next) + - Offers a more templating style API, instead of the many stringed + `ReactIntl.formattedMessage`components in react-intl. + +## Prior art + +--- + +The modern browser has a built-in Internationalization API, which inspired many +of the current JS intl libraries. + +## Unresolved questions + +--- + +- What percentage of the wallet should be internationalized for this scope of + work? + +- How we do continue to support additional languages beyond English? What will + be the process for adding additional translations? Who vets the accuracy? + +- [bs-react-intl-extractor](https://github.com/cknitt/bs-react-intl-extractor) + can be used to extract all the coded localization IDs into JSON files. This + can let anyone figure out what IDs are used map out translations text for + every used ID. diff --git a/website/docs/researchers/rfcs/0028-frontier-synchronization.md b/website/docs/researchers/rfcs/0028-frontier-synchronization.md new file mode 100644 index 000000000..a963a9aee --- /dev/null +++ b/website/docs/researchers/rfcs/0028-frontier-synchronization.md @@ -0,0 +1,319 @@ +--- +format: md +title: "RFC 0028: Frontier Synchronization" +sidebar_label: "0028 Frontier Synchronization" +hide_table_of_contents: false +--- + +> **Original source:** +> [0028-frontier-synchronization.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0028-frontier-synchronization.md) + +## Summary + +[summary]: #summary + +This RFC proposes a new access system for the transition frontier, designed to +ensure that transition frontier reads are safe from race conditions that cause +various pieces of state tied to the transition frontier to desynchronize. + +## Motivation + +[motivation]: #motivation + +We have recently identified a few (>1) bugs in our code related to +desynchronization of the frontier's breadcrumbs, the frontier's persistent root, +and the frontier's extensions. These bugs are caused by the fact that the +current frontier's interface synchronously and immediately access the frontier's +data structure, even if we may be in the middle of updating the frontier. +Similarly, arbitrary reads of extensions may have a similar issue in that the +frontier may have just been updated but the extensions have not been fully +updated yet. This is a systemic issue in the code base, not something that +should be fixed on a case by case basis, so a mild reworking of the frontier +logic is required to address it not just now, but into the future as well. + +## Detailed design + +[detailed-design]: #detailed-design + +This design centers around a new synchronization cycle for the frontier which +creates a sort of read/write lock system around the contents of the frontier. +Additionally, some extra improvements are made around references to data that is +inside (or related to) the frontier's state, so as to bugs in later accesses of +data structures read from the frontier. + +#### Diagram + +![](https://github.com/MinaProtocol/mina-resources/blob/main/rfcs/res/frontier_synchronization.conv.tex.png) + +#### Frontier Synchronization Cycle + +While the frontier is active, there is a single "thread" (chain of deferreds) +which runs through a synchronization cycle for the frontier. The algorithm for +this cycles goes: + +1. Execute all reads available. Continue executing reads until a write is + available. +2. Execute a single write from the write queue. This step can be done in + parallel to executing reads in #1, but once a write is being processed, no + more reads should be added to the current batch. +3. Once all the deferreds in #1 resolve, transmute the write action returned by + the write deferred into a list of diffs +4. Apply all the diffs from #2 to the full frontier +5. Write the diffs from #2 into the persistent frontier sync diff buffer + asynchronously +6. Update each frontier extension with the diffs from #2 +7. Notify frontier extension subscribers of any frontier extension changes + +#### Full/Partial Breadcrumbs + Any Refs + +The new synchronization cycle addresses synchronization issues between the state +of the full frontier datastructure, the persistent root database, and the +transition frontier extensions, but there is still a remaining synchronization +issue regarding breadcrumbs. Reading breadcrumbs from the transition frontier is +synchronized within the new synchronization cycle, but staged ledger mask +accesses cannot be sanely synchronized within that same cycle. In order to +address this issue, breadcrumbs are now separated into two types: full +breadcrumbs (which contain staged ledger's with masks attached to the frontier's +persistent root, directly or indirectly) and partial breadcrumbs (which only +contain the staged ledger's scan state and not the full data structure with the +mask). Reading a breadcrumb from the frontier will return neither a full or +partial breadcrumb, but instead will return a `Breadcrumb.Any.Ref.t`, which is a +reference to either a full or partial breadcrumb. This value can be safely +passed back from outside the `'a Transition_frontier.Read.t` monad and the +"staged state" (either the staged ledger or scan state depending on the status +of the breadcrumb) can be safely queried at an arbitrary time in the future. The +downside of this technique is that only immediate (and synchronous) reads from +the breadcrumb's staged ledger mask are safe under this interface, disallowing +certain actions. For instance, under this system, it is not safe to use a +breacrumb's staged ledger mask as a target for the syncable ledger (which +probably isn't safe anyway for other reasons). + +#### Detached Mask State Management + +In the current system, when a mask is destroyed, all of it's children masks are +also destroyed. Masks will now additionally provide the ability to attach +"destruction hooks". This will be used to cleanup full breadcrumbs and downgrade +them to partial breadcrumbs if the breadcrumb's parent mask chain becomes +inaccessible. This allows code outside of synchronized frontier reads to +continue to query breadcrumbs from the frontier and build successor breadcrumbs +off of them safely. This is important in the context of catchup since new +subtrees of breadcrumbs are built asynchronously to the frontier being updated. +If a subtree is built off of a breadcrumb which is later removed from the +frontier (thus having it's mask destroyed), then that subtree will subsequently +be marked as partial and any masks in it will be destroyed and inaccessible. + +IMPLEMENTATION NOTE: The entire chain of mask destruction hooks needs to be +executed synchronously within a single async cycle in order to update breadcrumb +subtrees without fear of race conditions. + +#### Frontier Read Monad + +A new monad is introduced for specifying reads from the transition frontier (see +[alternatives section](#alternatives) for explanation on why). All existing read +functions on the transition frontier will instead by turned into functions which +do not take in a transition frontier and return a result wrapped in the +`'a Transition_frontier.Read.t` monad. This monad is used to build up a list of +computations which will be performed during the read phase. The monad is +designed to interact with deferreds so that async programming is still +accessible during reads when necessary. + +#### New Frontier Interface + +Below is a partial signature for what the new transition frontier interface +would look like. Note that all base read/write functions from the transition +frontier are removed, and instead, only the read monad and write actions are +accessible. The only way to interact with the transition frontier thus becomes +the `read` and `write` functions. + +```ocaml +module Transition_frontier : sig + module Breadcrumb : sig + module Full : sig + type t + val block : t -> Block.Validated.t + val staged_ledger : t -> Staged_ledger.t + end + + module Partial : sig + type t + val block : t -> Block.Validated.t + val scan_state : t -> Transaction_snark_scan_state.t + end + + type full + type partial + + type 'typ t = + | Full : Full.t -> full t + | Partial : Partial.t -> partial t + type 'typ breadcrumb = 'typ t + + val external_transition : t -> External_transition.Validated.t + + module Any : sig + type t = T : _ breadcrumb -> t + type any = t + + val wrap : _ breadcrumb -> t + + module Ref : sig + (* intentionally left abstract to enforce synchronous access *) + type t + + val wrap : any -> t + (* downgrades a full to a partial; returns error if already a partial *) + val downgrade : t -> unit Or_error.t + + val external_transition : t -> External_transition.Validated.t + val staged_state : + t + -> [ `Staged_ledger of Staged_ledger.t + | `Scan_state of Transaction_snark_scan_state.t ] + end + end + end + + module Extensions : sig + type t + + (* ... *) + end + + module Read : sig + include Monad.S + + (* to wrap computations in the monad *) + val deferred : 'a Deferred.t -> 'a t + + val find : State_hash.t -> Breadcrumb.Any.Ref.t option t + + (* necessary extension access will be provided here, Extensions.t will not be directly accessible here *) + + (* ... *) + end + + type t + + (* Actions are representations of mutations (writes) to perform on + * the frontier. There is only one action that can be performed on + * a frontier right now, adding breadcrumbs, but we can add either + * a single breadcrumb, or a subtree of breadcrumbs. Future actions + * can also be added as needed *) + type action = + | Add_breadcrumb of Breadcrumb.Full.t + | Add_subtree of Breadcrumb.Full.t Rose_tree.t + + val read : + t + -> f:('a Read.t) + -> 'a Deferred.t + + val write : + t + -> f:(Action.t Read.t) + -> unit Deferred.t +end +``` + +#### Usage Notes + +It's important that computations described in `'a Read.t` monad need to be short +(in terms of execution time). Having long cycles occur when handling reads or +writes will have a significant effect on the overall delay for updating the +frontier. As such, code should be designed to return whatever values you want +from the `'a Read.t` monad as early as possible, and to continue using those +values outside of calls to `Transition_frontier.read`. + +## Drawbacks + +[drawbacks]: #drawbacks + +- current design does not address breadcrumb specific desync bugs fully +- adds more complexity overhead to all frontier interactions +- will have performance impact on the speed at which we update the frontier +- may open up new vectors for adversarial DDoS attacks + - for instance, adversary selectively delays rebroadcasting blocks to target, + then triggers multiple catchups (enqueuing many writes at once when the + catchups jobs finish), and then flood the target read requests to make the + writes as slow as possible (all this combined could take the target's stake + offline for a period of time) +- places new importance on reads being short (ideally synchronous) operations +- read monad needs to be updated to expose new read functionalities for + extensions + - preferably, adding new extensions or functions to access extensions would + not require changes to the read monad to expose them, but without higher + kinded types, it's can't be done in a reasonable way + - e.g. w/ generalized polymorphic variants, extension ref type could be + `type 'intf extension = ... constraint 'intf = _ [> ]` and then the + function to access could be + `val read_extension : ('result 'intf) extension -> 'result 'intf -> 'result t` + and could be read like `let%bind x = read_extension Root_history (`Lookup + y) in ...` + +## Alternatives + +[alternatives]: #alternatives + +#### Full Breadcrumb Pool + Ledger Mask Locks + +An alternative approach to mask destruction hooks which downgrade subtrees of +full breadcrumbs as necessary would be to maintain a ref counted pool of full +breadcrumbs which lock the ledger masks they control. This would require adding +backwards chaining (copy on write) as a capability to masks as the frontier +would need to support masks which are behind the persistent root. This method +could also easily introduce memory leaks. + +#### Alternative Misread Protection (no read monad) + +[alternative-misread]: #alternative-misread + +To protect against misreads, all reads have to be expressed in a monad +(`'a Transition_frontier.Read.t`). The `Transition_frontier.read` function +interprets this monad when the read is performed. + +An alternative approach would be to have a `Transition_frontier.Readable.t` type +which and have the type signature of `Transition_frontier.read` be +`t -> f:(Readable.t -> 'a Deferred.t) -> 'a Deferred.t`. Every instance of +`Readable.t` will be uniquely created for each pass of reads. When the pass of +reads completes, the `Readable.t` instance will be marked as "closed". If any +reads are attempted on a `Readable.t` instance, a runtime exception will be +thrown. This helps prevent code like the following from being written: + +```ocaml +let open Deferred.Let_syntax in +let%bind readable = Transition_frontier.read frontier ~f:(fun r _ -> return r) in +(* ... *) +``` + +But, the cost of this strategy is that misuse of the interface will cause +runtime exceptions. As such, while the monad is slightly more complex, it is +worthwhile to avoid the possible bugs that can be introduced by +misunderstandings. + +#### Alternative Read Monad Design + +The read monad interfaces with `'a Deferred.t` computations by providing the +function `val deferred : 'a Deferred.t -> 'a t`. This allows the monad to mix +both non-deferred and eferred computations, reducing the task scheduling +overhead (and thus the overall delay caused by reads) at the cost of making the +code a bit uglier. A prettier way to do this would be to give the bind (and map) +functions for the `'a Read.t` monad a type signature like +`'a Deferred.t Read.t -> f:('a -> 'b Read.t Deferred.t) -> 'b Deferred.t Read.t`. +I think the overall cost of having every segment of the `'a Read.t` monad +computations be put into their individual async scheduler job cycles to be not +worth this minor change in the DSL. + +## Unresolved questions + +[unresolved-questions]: #unresolved-questions + +- Brought up by @cmr: should queue reads be bucketed by most recent write at the + time of the read call? + - For instance, say some write (WA) is the most recently enqueue write. We + enqueue 2 reads, (RA) and (RB), then another write, (WB), and some more + reads, (RC) and (RD). Only reads (RA) and (RB) would trigger when write (WA) + is handled, and reads (RC) and (RD) would not trigger until after (WB) is + handled. +- Is the write queue necessary? How does it interact with the current + architecture of the Transition Frontier Processor? The Processor already acts + as a write queue, with more logic in place for how to write. diff --git a/website/docs/researchers/rfcs/0029-libp2p.md b/website/docs/researchers/rfcs/0029-libp2p.md new file mode 100644 index 000000000..20bdf3835 --- /dev/null +++ b/website/docs/researchers/rfcs/0029-libp2p.md @@ -0,0 +1,449 @@ +--- +format: md +title: "RFC 0029: Libp2P" +sidebar_label: "0029 Libp2P" +hide_table_of_contents: false +--- + +> **Original source:** +> [0029-libp2p.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0029-libp2p.md) + +# libp2p for coda + +## Summary + +[summary]: #summary + +Coda's networking today uses an +[largely-unmaintained Kademlia library written in Haskell](https://github.com/codaprotocol/kademlia). +We've been intending to replace it with some other maintained S/Kademlia +implementation. We use Kademlia for peer discovery: finding IP address and ports +of other protocol participants. On top of this, we build a "gossip net" for +broadcasting messages to all participants with a simple flooding protocol. +libp2p offers these same primitives plus some useful features like NAT traversal +and connection relaying. It also has a Javascript implementation that works in +the browser, which will be useful for our SDK. + +## Detailed Design + +[detailed-design]: #detailed-design + +libp2p itself is a large, flexible, extensible network stack with multiple +implementations and multiple options for various functionality. I propose this +minimal initial configuration: + +- On the daemon side, we will use `go-libp2p`. `rust-libp2p` lacks some features + (pubsub validation) that we want. +- For the transport, secio over TCP (optionally with WebSocket). secio is + libp2p's custom transport security protocol. I believe the libp2p developers + intend to replace secio with TLS 1.3 in the future. Transport security is not + yet essential for us, but it's nice to have and will become more important + once we start using relays. +- libp2p multiplexes several protocols over a single transport connection. For + the multiplexer, we will use the libp2p mplex protocol. We can easily add more + in the future, but mplex is the only one implemented by `js-libp2p` at the + moment. +- Kademlia for peer discovery and routing. +- "floodsub" for pubsub +- A custom protocol for encapsulating Coda's Jane Street RPCs (possibly + temporary). + +This basic configuration improves on our network stack in three major ways: + +1. Multiplexing over a single connection should slightly reduce our connection + establishment overhead (we open a separate TCP stream per RPC right now). +2. Transport security means connections are authenticated with the NodeID. + Currently our only notion of node identity is an IP address. +3. Browsers can join the DHT and connect to daemons. + +It still has some limitations: no NAT traversal, no browser↔browser connections, +no message relaying. + +The signature of the new networking code: + +```ocaml +(** An interface to limited libp2p functionality for Coda to use. + +A subprocess is spawned to run the go-libp2p code. This module communicates +with that subprocess over an ad-hoc RPC protocol. + +TODO: separate internal helper errors from underlying libp2p errors. + +In general, functions in this module return ['a Deferred.Or_error.t]. Unless +otherwise mentioned, the deferred is resolved immediately once the RPC action +to the libp2p helper is finished. Unless otherwise mentioned, everything can +throw an exception due to an internal helper error. These indicate a bug in +this module/the helper, and not misuse. + +Some errors can arise from calling certain functions before [configure] has been +called. In general, anything that returns an [Or_error] can fail in this manner. + +A [Mina_net2.t] has the following lifecycle: + +- Fresh: the result of [Mina_net2.create]. This spawns the helper process but + does not connect to any network. Few operations can be done on fresh nets, + only [Keypair.random] for now. + +- Configured: after calling [Mina_net2.configure]. Configure creates the libp2p + objects and can start listening on network sockets. This doesn't join any DHT + or attempt peer connections. Configured networks can do everything but any + pubsub messages may have very limited reach without being in the DHT. + +- Active: after calling [Mina_net2.begin_advertising]. This joins the DHT, + announcing our existence to our peers and initiating local mDNS discovery. + +- Closed: after calling [Mina_net2.shutdown]. This flushes all the pending RPC + +TODO: consider encoding the network state in the types. + +A note about connection limits: + +In the original coda_net, connection limits were enforced synchronously on +every received connection. Right now with mina_net2, connection management is +asynchronous and post-hoc. In the background, once per minute it checks the +connection count. If it is above the "high water mark", it will close +("trim") eligible connections until it reaches the "low water mark". All +connections start with a "grace period" where they won't be closed. Peer IDs +can be marked as "protected" which prevents them being trimmed. Ember believes this +is vulnerable to resource exhaustion by opening many new connections. + +*) + +open Base +open Async +open Pipe_lib +open Network_peer + +(** Handle to all network functionality. *) +type net + +module Keypair : sig + [%%versioned: + module Stable : sig + module V1 : sig + type t + end + end] + + type t = Stable.Latest.t + + (** Securely generate a new keypair. *) + val random : net -> t Deferred.t + + (** Formats this keypair to a comma-separated list of public key, secret key, and peer_id. *) + val to_string : t -> string + + (** Undo [to_string t]. + + Only fails if the string has the wrong format, not if the embedded + keypair data is corrupt. *) + val of_string : string -> t Core.Or_error.t + + val to_peer_id : t -> Peer.Id.t +end + +(** A "multiaddr" is libp2p's extensible encoding for network addresses. + + They generally look like paths, and are read left-to-right. Each protocol + type defines how to decode its address format, and everything leftover is + encapsulated inside that protocol. + + Some example multiaddrs: + + - [/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC] + - [/ip4/127.0.0.1/tcp/1234/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC] + - [/ip6/2601:9:4f81:9700:803e:ca65:66e8:c21] + *) +module Multiaddr : sig + type t + + val to_string : t -> string + + val of_string : string -> t +end + +type discovered_peer = {id: Peer.Id.t; maddrs: Multiaddr.t list} + +module Pubsub : sig + (** A subscription to a pubsub topic. *) + module Subscription : sig + type 'a t + + (** Publish a message to this pubsub topic. + * + * Returned deferred is resolved once the publish is enqueued locally. + * This function continues to work even if [unsubscribe t] has been called. + * It is exactly [Pubsub.publish] with the topic this subscription was + * created for, and fails in the same way. *) + val publish : 'a t -> 'a -> unit Deferred.t + + (** Unsubscribe from this topic, closing the write pipe. + * + * Returned deferred is resolved once the unsubscription is complete. + * This can fail if already unsubscribed. *) + val unsubscribe : _ t -> unit Deferred.Or_error.t + + (** The pipe of messages received about this topic. *) + val message_pipe : 'a t -> 'a Envelope.Incoming.t Strict_pipe.Reader.t + end + + (** Publish a message to a topic. + * + * Returned deferred is resolved once the publish is enqueued. + * This can fail if signing the message failed. + * *) + val publish : net -> topic:string -> data:string -> unit Deferred.t + + (** Subscribe to a pubsub topic. + * + * Fails if already subscribed. If it succeeds, incoming messages for that + * topic will be written to the [Subscription.message_pipe t]. Returned deferred + * is resolved with [Ok sub] as soon as the subscription is enqueued. + * + * [should_forward_message] will be called once per new message, and will + * not be called again until the deferred it returns is resolved. The helper + * process waits 5 seconds for the result of [should_forward_message] to be + * reported, otherwise it will not forward it. + *) + val subscribe : + net + -> string + -> should_forward_message:(string Envelope.Incoming.t -> bool Deferred.t) + -> string Subscription.t Deferred.Or_error.t + + (** Like [subscribe], but knows how to stringify/destringify + * + * Fails if already subscribed. If it succeeds, incoming messages for that + * topic will be written to the [Subscription.message_pipe t]. Returned deferred + * is resolved with [Ok sub] as soon as the subscription is enqueued. + * + * [should_forward_message] will be called once per new message, and will + * not be called again until the deferred it returns is resolved. The helper + * process waits 5 seconds for the result of [should_forward_message] to be + * reported, otherwise it will not forward it. + *) + val subscribe_encode : + net + -> string + -> should_forward_message:('a Envelope.Incoming.t -> bool Deferred.t) + -> bin_prot:'a Bin_prot.Type_class.t + -> on_decode_failure:[ `Ignore + | `Call of + string Envelope.Incoming.t -> Error.t -> unit ] + -> 'a Subscription.t Deferred.Or_error.t +end + +(** [create ~logger ~conf_dir] starts a new [net] storing its state in [conf_dir] + * + * The new [net] isn't connected to any network until [configure] is called. + * + * This can fail for a variety of reasons related to spawning the subprocess. +*) +val create : logger:Logger.t -> conf_dir:string -> net Deferred.Or_error.t + +(** Configure the network connection. + * + * Listens on each address in [maddrs]. + * + * This will only connect to peers that share the same [network_id]. [on_new_peer], if present, + * will be called for each peer we discover. [unsafe_no_trust_ip], if true, will not attempt to + * report trust actions for the IPs of observed connections. + * + * This fails if initializing libp2p fails for any reason. +*) +val configure : + net + -> me:Keypair.t + -> external_maddr:Multiaddr.t + -> maddrs:Multiaddr.t list + -> network_id:string + -> on_new_peer:(discovered_peer -> unit) + -> unsafe_no_trust_ip:bool + -> unit Deferred.Or_error.t + +(** The keypair the network was configured with. + * + * Resolved once configuration succeeds. + *) +val me : net -> Keypair.t Deferred.t + +(** List of all peers we know about. *) +val peers : net -> Peer.t list Deferred.t + +(** Try to connect to a peer ID, returning a [Peer.t]. *) +val lookup_peerid : net -> Peer.Id.t -> Peer.t Deferred.Or_error.t + +(** An open stream. + + Close the write pipe when you are done. This won't close the reading end. + The reading end will be closed when the remote peer closes their writing + end. Once both write ends are closed, the stream ends. + + Long-lived connections are likely to get closed by the remote peer if + they reach their connection limit. See the module-level notes about + connection limiting. + + IMPORTANT NOTE: A single write to the stream will not necessarily result + in a single read on the other side. libp2p may fragment messages arbitrarily. + *) +module Stream : sig + type t + + (** [pipes t] returns the reader/writer pipe for our half of the stream. *) + val pipes : t -> string Pipe.Reader.t * string Pipe.Writer.t + + (** [reset t] informs the other peer to close the stream. + + The returned [Deferred.Or_error.t] is fulfilled with [Ok ()] immediately + once the reset is performed. It does not wait for the other host to + acknowledge. + *) + val reset : t -> unit Deferred.Or_error.t + + val remote_peer : t -> Peer.t +end + +(** [Protocol_handler.t] is the rough equivalent to [Tcp.Server.t]. + + This lets one stop handling a protocol. + *) +module Protocol_handler : sig + type t + + (** Returns the protocol string being handled. *) + val handling_protocol : t -> string + + (** Whether [close t] has been called. *) + val is_closed : t -> bool + + (** Stop handling new streams on this protocol. + + [reset_existing_streams] controls whether open streams for this protocol + will be reset, and defaults to [false]. + *) + val close : ?reset_existing_streams:bool -> t -> unit Deferred.t +end + +(** Opens a stream with a peer on a particular protocol. + + Close the write pipe when you are done. This won't close the reading end. + The reading end will be closed when the remote peer closes their writing + end. Once both write ends are closed, the connection terminates. + + This can fail if the peer isn't reachable, doesn't implement the requested + protocol, and probably for other reasons. + *) +val open_stream : + net -> protocol:string -> Peer.Id.t -> Stream.t Deferred.Or_error.t + +(** Handle incoming streams for a protocol. + + [on_handler_error] determines what happens if the handler throws an + exception. If an exception is raised by [on_handler_error] (either explicitly + via [`Raise], or in the function passed via [`Call]), [Protocol_handler.close] will + be called. + + The function in `Call will be passed the stream that faulted. +*) +val handle_protocol : + net + -> on_handler_error:[`Raise | `Ignore | `Call of Stream.t -> exn -> unit] + -> protocol:string + -> (Stream.t -> unit Deferred.t) + -> Protocol_handler.t Deferred.Or_error.t + +(** Try listening on a multiaddr. +* +* If successful, returns the list of all addresses this net is listening on +* For example, if listening on ["/ip4/127.0.0.1/tcp/0"], it might return +* ["/ip4/127.0.0.1/tcp/35647"] after the OS selects an available listening +* port. +* +* This can be called many times. +*) +val listen_on : net -> Multiaddr.t -> Multiaddr.t list Deferred.Or_error.t + +(** The list of addresses this net is listening on. + + This returns the same thing that [listen_on] does, without listening + on an address. +*) +val listening_addrs : net -> Multiaddr.t list Deferred.Or_error.t + +(** Connect to a peer, ensuring it enters our peerbook and DHT. + + This can fail if the connection fails. *) +val add_peer : net -> Multiaddr.t -> unit Deferred.Or_error.t + +(** Join the DHT and announce our existence. + + Call this after using [add_peer] to add any bootstrap peers. *) +val begin_advertising : net -> unit Deferred.Or_error.t + +(** Stop listening, close all connections and subscription pipes, and kill the subprocess. *) +val shutdown : net -> unit Deferred.t + +(** Ban an IP from connecting to the helper. + + This ban is in place until [unban_ip] is called or the helper restarts. + After the deferred resolves, no new incoming streams will involve that IP. + TODO: does this forbid explicitly dialing them? *) +val ban_ip : + net -> Unix.Inet_addr.t -> [`Ok | `Already_banned] Deferred.Or_error.t + +(** Unban an IP, allowing connections from it. *) +val unban_ip : + net -> Unix.Inet_addr.t -> [`Ok | `Not_banned] Deferred.Or_error.t + +(** List of currently banned IPs. *) +val banned_ips : net -> Unix.Inet_addr.t list Deferred.t +``` + +Concretely, this will be implemented by spawning a Go child process that speaks +a simple JSON protocol. This will let us use `go-libp2p` (which seems to be the +most robust libp2p implementation at the moment) without figuring out how to get +Go and async working in the same process. + +`Gossip_net` will gain a new backend that uses this module, replacing `real.ml`. +Additional gossip topics are easy to add without modifying the `Message` type. +The raw pubsub messages are uninterpreted bytes; a `subscribe_encode` is +provided that knows how to use bin_prot instances for gossip messages. + +## Drawbacks + +[drawbacks]: #drawbacks + +libp2p is a pretty large dependency. It isn't super mature: only recently have +the three main implementations achieved DHT interoperatbility. floodsub is, +algorithmically, no better than what we have today. + +## Rationale and alternatives + +[rationale-and-alternatives]: #rationale-and-alternatives + +One easier thing to do would be just replacing the `membership` implementation +with a new DHT. However, this doesn't help browsers join the DHT. There aren't +many browser peer to peer networking libraries. The main one seems to be PeerJS. + +## Prior art + +[prior-art]: #prior-art + +Tezos +[implements their own](https://gitlab.com/tezos/tezos/tree/master/src/lib_p2p). +Cardano +[implements their own](https://github.com/input-output-hk/ouroboros-network/tree/master/ouroboros-network). +Ethereum 2.0 +[is using `libp2p`](https://github.com/ethereum/consensus-specs/blob/2c632c0087f0a692ab74987229524edbb941eeb3/specs/phase0/p2p-interface.md). +Parity created `rust-libp2p` and is using it in substrate and polkadot. Bitcoin +[implements their own](https://github.com/bitcoin/bitcoin/blob/master/src/net.cpp). +Stellar +[implements their own](https://github.com/stellar/stellar-core/tree/master/src/overlay). +Nimiq, a "browser-based blockchain", +[implements their own](https://github.com/nimiq-network/core/tree/master/src/main/generic/network). + +## Unresolved questions + +[unresolved-questions]: #unresolved-questions + +None at this time. diff --git a/website/docs/researchers/rfcs/0030-fork-signalling.md b/website/docs/researchers/rfcs/0030-fork-signalling.md new file mode 100644 index 000000000..70d8a4e17 --- /dev/null +++ b/website/docs/researchers/rfcs/0030-fork-signalling.md @@ -0,0 +1,121 @@ +--- +format: md +title: "RFC 0030: Fork Signalling" +sidebar_label: "0030 Fork Signalling" +hide_table_of_contents: false +--- + +> **Original source:** +> [0030-fork-signalling.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0030-fork-signalling.md) + +## Summary + +[summary]: #summary + +Nodes can signal to other nodes that they intend to switch from the current fork +to a new fork. This RFC is scoped to changes needed for mainnet launch. + +## Motivation + +[motivation]: #motivation + +Each node in the network has a notion of the current fork it's running on. A +node may wish to change forks, perhaps because it's running new software, or to +create a new subnet of nodes with some common interest. Such an ability is a +limited form of on-chain governance. When a node proposes a new fork, other +nodes need to be informed of the proposal. + +## Detailed design + +[detailed-design]: #detailed-design + +A fork is denoted by a protocol version, which is a semantic version. That is, +the protocol version has the form `m.n.p`, where `m` is an integer denoting a +major version, `n` is a minor version number, and `p` is a patch number. Blocks +have two protocol version fields, one to indicate the current fork, another to +indicate a proposed fork. Because there will not always be a proposed fork, that +field has an option type. + +A change to the patch number represents a software upgrade, and does not require +signalling. A change to the minor version is signalled, but nodes can continue +to run existing software (a soft fork). A change to the major version number is +signalled, and requires that nodes upgrade their software (a hard fork). + +The compile-time configuration includes a current protocol version. For testing +and debugging, that protocol version can be overridden via a command-line flag, +which is saved to the node's dynamic configuration. If the dynamic configuration +includes the protocol version, the next time the node is started, that protocol +version will be used. It's an error to start the node with a protocol version +from the command line that's different from one stored in the dynamic +configuration. The command-line flag feature should be removed by the time of +mainnet. + +For RPCs between nodes, if the response contains a block with a different major +current protocol version, that response is ignored, and the sender punished. The +next protocol version is ignored for these RPCs. + +For gossiped blocks, if the block's major protocol version differs from the +node's major protocol version that block is ignored, and because the protocol +has been violated, the sender is punished. + +To signal a proposed fork change, we need mechanisms for a node to set its +proposed protocol version, to be included in the blocks it gossips subsequently. +An optional command-line flag indicates the proposed protocol version to be +used, which also stores the proposed protocol version in the dynamic +configuration, to be used the next time the node is run. Additionally, we'll +provide a GraphQL endpoint to set the proposed protocol version, which would +also store the proposed protocol version in the dynamic configuration. + +Much of this design appears in PR #4565, which has not been merged as of this +writing. The punishment implemented there for mismatched current protocol +versions is a trust decrease of 0.25, allowing a small number of mismatches in +blocks before banning a peer. + +## Drawbacks + +[drawbacks]: #drawbacks + +If switching the protocol version is under manual control, a possible outcome of +this mechanism would be extreme fragmentation of the network into sub-networks, +each sharing a common current protocol version. Limiting the current protocol +version to a compile-time value in mainnet should remove this concern. + +## Rationale and alternatives + +[rationale-and-alternatives]: #rationale-and-alternatives + +Semantic versioning allows us to distinguish hard and soft works. An alternative +would be to limit fork signalling to hard forks, since soft forks maintain +compatibility with old code. Signalling soft forks is nonetheless useful, so +that nodes are certain of the protocol being used. + +## Prior art + +[prior-art]: #prior-art + +There are related issues and PRs. Issue #4199 mentions adding the "fork ID" +fields to blocks (the type `External_transition.t`). Issue #4200 mentions +examining those fields. PR #4347 was meant to address both of those issues. +Issue #4201 mentions reporting statistics of fork IDs seen in gossiped blocks, +although details are still needed. The protocol version notion in this RFC +subsumes fork IDs. + +Bitcoin introduced a mechanism called `Miner-activated soft forks`, where miners +could increment a version number in blocks. Nodes would be required to accept +blocks with the new version number once they'd seen a certain frequency of them +within a number-of-blocks window, and reject lower-versioned blocks upon a +higher frequency threshold. The new version number signalled an `activation`, a +change in consensus rules. The Bitcoin mechanism does not handle hard forks. The +design here is inspired by that mechanism, although our design is intended for +hard forks in the first instance. + +There are other soft fork mechanisms in Bitcoin. See [this article] +(https://medium.com/@elombrozo/forks-signaling-and-activation-d60b6abda49a) for +a discussion of them. + +## Unresolved questions + +[unresolved-questions]: #unresolved-questions + +Is the trust decrease of 0.25 the right level of punishment for a peer that +sends a mismatched current fork ID? diff --git a/website/docs/researchers/rfcs/0031-sentry-architecture.md b/website/docs/researchers/rfcs/0031-sentry-architecture.md new file mode 100644 index 000000000..40d90b408 --- /dev/null +++ b/website/docs/researchers/rfcs/0031-sentry-architecture.md @@ -0,0 +1,115 @@ +--- +format: md +title: "RFC 0031: Sentry Architecture" +sidebar_label: "0031 Sentry Architecture" +hide_table_of_contents: false +--- + +> **Original source:** +> [0031-sentry-architecture.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0031-sentry-architecture.md) + +## Summary + +[summary]: #summary + +Introduce a new "hidden" and "sentry" mode to the daemon for building +DoS-resilient network topologies. + +## Motivation + +[motivation]: #motivation + +Block producer availability is important both for healthy consensus (blocks have +a limited time to be broadcast before they are no longer useful) and also for +node operator profits. Block producers (and, to a lesser extent, SNARK workers) +need to be powerful enough to produce their proofs in a reasonable amount of +time, and have operational security requirements around protecting the private +key. If a DoS takes a node offline such that they can't broadcast their work, or +put the CPUs to use, the operator loses money and consensus is weakened. It is +also risky to directly expose a machine with sensitive key material to untrusted +internet traffic. Sensitive nodes can be protected from these risks by running +them in a new "hidden mode". These hidden nodes will communicate via nodes +running in "sentry mode". + +The recommended way to deploy this would be to have several sentries, possibly +in different cloud providers/datacenters, configured per hidden node. + +## Detailed design + +[detailed-design]: #detailed-design + +New RPCs: + +- `Sentry_forward`: sent from a hidden node to a sentry node with some new work, + to be broadcast over the gossip net. +- `Please_sentry`: sent from a hidden node to its sentries periodically as a + sort of "keepalive" to ensure the sentry node continues forwarding messages + even if it crashes and forgets about us. These should be sent at least once + per slot. +- `Hidden_forward`: sent from a sentry node to a hidden node on all new gossip. + +### Hidden Mode + +Nodes in hidden mode will not participate in discovery at all, by never calling +`begin_advertising`. They will also filter out all connections not from their +configured sentries. These are "soft" mitigations - hidden nodes, when deployed, +should not be publicly routable at all, or otherwise have the firewall carefully +configured to only allow communications with sentries. + +Because hidden nodes are not well connected, we can't rely on the usual +guarantees of the pubsub implementation to receive block gossip. Thus +`Hidden_forward`: instead of using `subscribe_encode`, new messages will be +received from this RPC. + +The sentries are configured with a new `--sentry-address MULTIADDR` daemon flag. +On startup and on a timer, hidden nodes will send `Please_sentry` to their +configured sentries. + +### Sentry Mode + +Sentry nodes are normal participants in networking. When they receive a new +message, they forward it to connected hidden nodes using `Hidden_forward`. + +Hidden nodes can be configured with a `--sentry-for MULTIADDR` daemon flag, +which will ensure they always receive new messages. In addition, sentries will +accept `Please_sentry` RPCs from any IP address in an [RFC1918][1918] private +address range and add them to the `sentry-for` list. This allows hidden nodes to +be on unstable private addresses without having to reconfigure the sentries. + +## Drawbacks + +[drawbacks]: #drawbacks + +- Not using pubsub requires some additional code. +- This unavoidably adds (at least) one hop of latency before the block producers + will see new blocks from the network. + +## Rationale and alternatives + +[rationale-and-alternatives]: #rationale-and-alternatives + +- Instead of having a hidden mode, we could simply operate normally and + configure the network filter to not allow non-sentry connections. This, + however, requires a custom pubsub (see below), and is somewhat wasteful: + hidden nodes shouldn't need to care about the DHT at all. +- We could have a custom pubsub implementation, which preferentially forwards to + hidden nodes before hitting (eg) randomsub. This requires writing some Go + code, versus regular OCaml RPCs as described. +- Instead of `Please_sentry`, we could monitor disconnects and then busy poll + until the sentry comes back online. We probably should do this, at some point, + as it's more precise. + +## Prior art + +[prior-art]: #prior-art + +Similar to the +[Cosmos Hub architecture](https://forum.cosmos.network/t/sentry-node-architecture-overview/454). + +## Unresolved questions + +[unresolved-questions]: #unresolved-questions + +Should we use local mDNS discovery to find sentries? + +[1918]: https://tools.ietf.org/html/rfc1918 diff --git a/website/docs/researchers/rfcs/0032-automated-validation.md b/website/docs/researchers/rfcs/0032-automated-validation.md new file mode 100644 index 000000000..12437e298 --- /dev/null +++ b/website/docs/researchers/rfcs/0032-automated-validation.md @@ -0,0 +1,499 @@ +--- +format: md +title: "RFC 0032: Automated Validation" +sidebar_label: "0032 Automated Validation" +hide_table_of_contents: false +--- + +> **Original source:** +> [0032-automated-validation.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0032-automated-validation.md) + +## Summary + +[summary]: #summary + +This RFC describes the design for an automated validation service which will be +deployed alongside testnets. The goal is to define a system which can be quickly +iterated on, from both the perspective of adding new validations and maintaining +this service. As such it is important that this service supports the +capabilities required to define a wide variety of different validations we would +like to perform against the network on a regular basis. + +## Motivation + +[motivation]: #motivation + +There are many things we want to validate about our testnets on a regular basis +(and always before a release). Most of these validations need to be performed at +various points during the network's execution. Currently, we are doing all of +this validation manually (responsibility of the testnet guardian), which takes a +significant amount of engineering time and distracts from other tasks. +Furthermore validation errors are often caught later than they could have been +since engineers are (unfortunately) not computers and won't necessarily check +the each validation criteria as soon as it is available to verify. This also +extends towards validations which need to be checked throughout the execution of +the network, such as network forks. + +The primary motivations behind building an automated validation service are to +reduce validation costs to engineering time, tighten the iteration loop around +stabilizing our network, give us more confidence in future public testnet +releases, and to provide a route to regular CI network validations. + +## Detailed design + +[detailed-design]: #detailed-design + +Observing the +[list of validations we require for stable networks](https://www.notion.so/codaprotocol/Testnet-Validation-1705d5bf03e04d03b21a1f5724a17597), +the majority of validations can be answered through a combination of log +processing and graphql queries. As such, we will initially scope the design of +the validation service to focus on those two aspects, while trying to create a +design that isn't over-fitted to these two aspects so that we can refit parts of +the architecture for validations which are outside this scope after the initial +implementation is complete. + +### Architecture + +[detailed-design-architecture]: #detailed-design-architecture + +In order to discuss the architecture for the automated validation service, we +should first define how we want to interact with the automated validation +service from a high level. A naive approach to the validation service would be +to always enable all validations, but in order to make this service reusable in +various environments (qa net, public testnet, mainnet), allow it to be +horizontally scalable, and perhaps even use it as part of the integration test +framework, the validation service should be configurable. The inputs to the +validation service should include the following information: + +- credentials to talk to various APIs (gcloud, kubernetes, discord, etc...) +- information about which network to validate +- the set of resources deployed on the network (see below for more detail on + resources) +- a list of "validation queries"; each "validation query" includes: + - the name of the validation to perform + - a query on which resources it should be performed on + +From the list of validation queries, the automated validation can solve for +other information it needs to actually run, such as what log filters it needs to +pull from stackdriver, what information to compute from events and state changes +on the network, and what resources are scoped to each of these concepts. The +architecture of the automated validation service is thus broken up and modeled +as a series of concurrent "processes", which we will classify into various roles +(providers, statistics, and validators), which are responsible for one piece of +the entire service. + +Below is a diagram which displays how the system is initialized given the +validation queries and resource database. From the information computed during +this initialization, the automated validation service can create the various +processes and external resources (such as log sinks on google cloud) necessary +to perform the requested validations on the requested resources. + +![](https://github.com/MinaProtocol/mina-resources/blob/main/docs/res/automated_validation_init_architecture.conv.tex.png) + +The following diagram shows the runtime architecture of the processes in the +system once initialization has been completed. Note how this diagram shows that +statistics may consume multiple providers, and validations may consume multiple +statistics. + +![](https://github.com/MinaProtocol/mina-resources/blob/main/docs/res/automated_validation_runtime_architecture.conv.tex.png) + +#### Resources + +[detailed-design-architecture-resources]: + #detailed-design-architecture-resources + +A deployment consists of a series of pods and containers. We define a universal +abstraction for reasoning about these artifacts of a deployment, which +generically call "resources". Resources are classified into different categories +in an subtyping model. For example, the main resource we are interested in is a +`CodaNode`. There are multiple subclasses of `CodaNode`, such as +`BlockProducer`, `SnarkCoordinator`, or `Seed`. Each of these classes may have +different metadata associate with instances of them. For instance, all +`CodaNode` instances have a `pod_name`, and `BlockProducer` instances have +additional metadata for what block producer role they are and their integer id +(eg in "fish-block-producer-1", role="fish" and id=1). + +Resources can be collected into queriable datasets which allow selection of +resources by metadata filters or classification. + +#### Providers + +[detailed-design-architecture-providers]: + #detailed-design-architecture-providers + +Providers provide network state to the rest of the system. There are multiple +kinds of providers, such as `GraphQLProviders`, which provide data from GraphQL +queries on a periodic basis, and `EventProviders`/`LogProviders`, which extract +events from logs. Logs can be streamed back to the automated validation service +fairly easily from StackDriver by creating a log sink, pub sub topic, and one or +more filter subscriptions in Google Cloud (I have tested this functionality for +pull-based subscriptions, and push-based subscriptions are an easy extension of +this to setup). Providers are dispatched for specific resources, and provide an +internal subscription to state updates for those resources so that statistics +may receive updates. + +#### Statistics + +[detailed-design-architecture-statistics]: + #detailed-design-architecture-statistics + +Statistics consume network state information from one or more providers and +incrementally compute a rolling state (representing the computed statistic). +Statistics are (at least in the initial design) individually scoped to resources +(meaning that each statistic has a singular instance computed for each resource +that is being monitored by the system). Statistics broadcast their state updates +so that vlaidations can be performed on one or more statistic as values change. +Statistics can also be forwarded to as prometheus metrics. Doing this would +allow us to compute more meaningful network wide statistics that we cannot +easily compute today without writing a program to scrape the logs after the +network runs. + +#### Validations + +[detailed-design-architecture-validations]: + #detailed-design-architecture-validations + +Validations subscribe to and perform checks against one or more statistics. The +result of a validation is either "this state is valid" or "this state is invalid +because of X". If a validation fails, the alert system is notified of the +specific validation error. Since validations can subscribe to more than one +statistic at a time, they are checked concurrently to computing the values of +statistics. When any of the statistics that a validation is subscribed to +update, the validation is reperformed on the set of most-up-to-date statistics. + +#### Alerts + +[detailed-design-architecture-alerts]: #detailed-design-architecture-alerts + +When validations fail, alerts can be triggered. In theory, there can be multiple +backends for how alerts are surfaced to the team. As an initial scope for the +project, I think that focusing on discord alerts to an internal channel which is +monitored by the testnet guardian and other engineers should suffice for now. +One important note here is that this service should be _loud_, but not _noisy_. +This means that the service should send a notification somewhere people see it, +but it should contain some logic to help limit the noise level of these alerts. +We don't want to have this alerting system spam too much and make uneccessary +noise, because that will just lead people to ignore it like we do much of the +noise from failing CI tests. As such, a basic validation error rate limiter and +timeout logic seems to be a must. By default, we should limit errors to +(roughly) 5 per hour, and we can tweak this individually for each validation. + +### Language Choice + +[detailed-design-language-choice]: #detailed-design-language-choice + +Language choice is a key component for the automated validation service design +which has large ramifications on it's implementation and maintenance costs. In +particular, we want to optimize for the maintenance costs of the automated +validation service. There are 2 important aspects to consider when optimizing +for the maintenance cost of this service: scalability and reusability. +Scalability, in the context of this service, directly relates to how concurrent +this system can be, since this service will be monitoring networks in the size +order of 100s of nodes. Reusability relates to our ability to configure and +resuse abstractions for the individual components involved in the service +(resources, providers, statistics, validations). In addition to these, we also +would like to choose a language which either engineers already know, or a +language which would be easy for engineers to pick up without much learning +overhead. + +#### Languages Not Chosen + +With the requirement that this system be highly concurrent (so that we don't +need to spend a lot of engineering effort trying to optimize this tool), that +immediately hurts the case for the main language we use currently, OCaml. OCaml +comes with great benefits, such as it already being our standard language on the +team and it having a good static type system, but the single threaded +concurrency story in OCaml is poor, and the solutions and tooling around them +(pthreads & rpc_parallel) are either too low level and tedious to get correct or +too bulky to use, making concurrenty OCaml code quite difficult to iterate on. +As such, OCaml seems like the wrong choice for this system as we would spend a +lot of engineering effort on just the architecture for this system (and would +likely need to revisit that as we intend to scale this system up to more and +more validations and larger network sizes). + +Python is another language which we use, but also suffers from a poor +single-threaded concurrency model. Python is a little bit easier to get +concurrency going in than OCaml, but is less principled than OCaml and fully +lacks type safety. Python has type annotations, but the tooling for analyzing +them is poor due to they way the type annotations we designed, so most type +checking can only happen as a runtime assertion. Furthermore, it's concurrency +model requires manual thread yielding (or to setup your code to auto yeild on +every function call), making thread starvation a much harsher and likely reality +compared to OCaml. + +Rust has very good concurrency and parallelism support (debatably one of the +best out there today). It has strong type safety and a fantastic borrow checker +model to help avoid typical memory errors when writing concurrent or parallel +code. However, Rust has a very high learning curve compared to other languages +due to it's borrow checker model and layer of abstraction. Being a low-level +language by nature, Rust programs tend to be more verbose and difficult to +design, and designing a program incorrectly up front can cause wasted work when +the compiler informs you that your code isn't written correctly. Given that our +engineering team is more focused to abstract functional programming than low +level programming, the learning curve and barrier to entry seems even steeper +since part of that includes becoming comfortable with low level memory +management practices and gotchas. Due to this, and the much slower iteration +cycle that Rust has compared to other languages (unless you are very good at +Rust), Rust seems like the incorrect choice for this system. + +#### The Ideal Platform + +I believe the best language for this system is a language which runs on the +BeamVM. The BeamVM is a virtual machine which is part of the Erlang Open Telecom +Platform (OTP) distribution. OTP is a technology suite and set of standards +which was developed by Ericsson in the late 90s. While OTP has "telecom" in the +name, nothing about it is actually specific to telecom as an industry. Rather, +it was built by Ericsson in the late 90s in order to solve their problems of +needing a highly concurrent software system for a new ATM switch they were +building. Since it was open sourced in the late 90s, Erlang OTP has seen a wide +variety of uses throughout concurrenty distributed systems, most notably making +strides in web development in recent years. Some examples of large projects +built on the BeamVM include RabbitMQ, Facebook's chat service, WhatsApp's +backend, and Amazon's EC2 database services, just to name a few. + +The main advantage of the BeamVM is the concurrency model it supports, along +with the efficiency it achieves when evaluating this model at runtime. Inside +the BeamVM, code is separated into "processes" (not to be confused with OS-level +processes), each of which is represented by a call stack and a mailbox. +Processes may send messages to other processes, and block and receive messages +through their mailbox. An important thing to note here is that the only state a +process has is it's call stack and mailbox. This means that there is no mutable +state internal to any process, immediately eliminating a large amount of bugs +related to non-synchronized state access. Messages are safe and efficient to +share between processes, and (without going into detail) the model by which +processes block and pull information out of their incoming mailbox prevents a +number of traditional "starvation" or "deadlocking" cases. Processes are +organized into "supervision trees", where parent processes (called supervisors) +are responsible for monitoring and restarting their children. OTP includes a set +of standard abstractions around processes that fit into this "supervision tree" +model, with the intention to provide fault tolerance to your concurrent +architecture _by default_. This system does not eliminate concurrent bugs all +together, but it limits the kind of bugs you can encounter, and provides tools +to help make the system robust to errors that may occur at runtime. The BeamVM +is very efficient with how it actually schedules these processes, and is +typically able to evenly saturate as many CPU cores as you want it to (so long +as you follow proper design patterns, which OTP makes easy). + +#### The Chosen Language + +Erlang is the language which was originally developed for working on the BeamVM +(and as such, the two projects are heavily intertwined). Erlang is a weird +language though, pulling syntatical inspiration from Prolog into a purely +immutable dynamically typed functional programming language with a unique +concurrency model. Since Erlang's introduction as part of the OTP system, many +other languages targetting the same virtual machine have cropped up, attempting +to make Erlang's programming model more approachable and easier to pick up and +learn for developers. Some projects have added ML-like languages on top of +Erlang and added static types along the way. However, the most popular and +widely used language (which has basically superceded Erlang at this point) is +Elixir. + +Elixir provides an easier syntax (ruby-esque syntax), a module system, macros, +and reusable mixins for code generation on top of the Erlang OTP system. It also +comes with a type specification system which is built into the language and +tooling which can statically analyze the type specs during compilation (not as +good as proper static types, but one of the better static anlysis type systems; +better than flowjs, for example, and all libraries, including stdlib and erlang +libs, come with typespecs). It has a low learning curve and lots of resources +for learning both the language and the concurrency model it sits on top of (the +BeamVM process model). There is no mutability, it's very easy to build DSLs and +OCaml functor-like patterns. This provides the ability to easily create and +maintain abstractions for defining the various pieces of this system (resources, +providers, statistics, validations). The concurrency model of the BeamVM means +that iteration over concurrent optimizations is short and easy. Since the BeamVM +unifies the entire concurrency model into one abstraction (processes), +refactoring to optimize the amount of concurrency the system gets is simple +since you only rewrite the code around your process and typically don't need to +change details about how your process does it's job. For these reasons, Elixir +seems like the best choice for building and maintaining this system in. + +Elixir is easy to install on Linux and OSX (available through all major package +managers). It comes with a build tool, called `mix`, which is the main way that +you interact with a project. It downloads dependencies, handles configurations, +runs scripts, compiles code, and executes static analysis (formatting, type +checking). Running Elixir applications is pretty simple as well. All that's +needed is the built BeamVM artifact and a docker container with the BeamVM +installed. We don't need to consider it anytime soon for our usecases, but there +is also a microkernel image for the BeamVM which supports SSH'ing directly into +an interactive Elixir shell. + +## Proof of Concept + +[proof-of-concept]: #proof-of-concept + +##### NB: The proof of concept is nearly done and will soon be put up so the implementation can be reviewed. For now, in the interest of getting this RFC in front of people, I am documenting the interface the proof of concept exposes for defining new providers, statistics, and validations. + +I have fleshed out a PoC for what this system would look like in Elixir. In the +PoC, we only focus on LogProviders rather than GraphQLProviders, however the +latter type of Provider should be arbitrary to build. The alerting system and +validation specification interface is also not fleshed out in the interest of +keeping proof of concept simple. The PoC includes mixins for easily defining new +resources, providers, statistics, and validations in the system. These mixins +(which are included into modules using the `use` macro) expect specific +functions to be implemented on the module (called "callbacks" in Erlang/Elixir), +and the mixins can add additional boilerplate code to the module. For OCaml +developers, Elixir's concept of a "mixin" can be thought of similarly to how we +think about "functors" in OCaml (though they work differently). + +In the PoC, LogProviders can be easily added by specifying the resource class +and filter the LogProvider is associated with. For instance, a LogProvider which +provides events for whenever new blocks are produced is defined as follows: + +```elixir +defmodule LogProviders.BlockProduced do + use Architecture.LogProvider + def resource_class, do: Resources.BlockProducer + def filter, do: ["Successfully produced a new block: $breadcrumb"] +end +``` + +Adding new statistics involves defining providers the statistic consumes, what +subset of resources the statistic is applicable to, and a state for the +statistic along with how to initialize/update that state. There are two kinds of +update functions that are defined for a statistic: one which updates on a +regular time interval (just called `update` right now; useful for time based +statistics), and another one which handles subscription updates from providers +the statistic consumes. Below is an example of a statistic which computes the +block production rate of a block producer. Keep in mind that this interface is +highly subject to change and will likely be cleaned up some to make it less +verbose. + +##### NB: some types and attributes were removed from this definition to avoid confusing people who are new to Elixir and focus the conversation on the logic + +```elixir +defmodule Statistics.BlockProductionRate do + use Architecture.Statistic + + def log_providers, do: [LogProviders.BlockProduced] + def resources(resource_db), do: Resources.Database.all(resource_db, Resources.BlockProducer) + + defmodule State do + use Class + + defclass( + start_time: Time.t(), + elapsed_time: Time.t(), + last_updated: Time.t(), + blocks_produced: pos_integer() + ) + end + + def init(_resource) do + start_time = Time.utc_now() + + %State{ + start_time: start_time, + elapsed_time: 0, + last_updated: start_time, + blocks_produced: 0 + } + end + + defp update_time(state) do + now = :time.now_ns() + us_since_last_update = Time.diff(now, state.last_updated, :microsecond) + elapsed_time = Time.add(state.elapsed_time, us_since_last_update, :microsecond) + %State{state | last_updated: now, elapsed_time: elapsed_time} + end + + def update(_resource, state), do: update_time(state) + + def handle_log(_resource, state, LogProviders.BlockProduced, _log) do + %State{state | blocks_produced: state.blocks_produced + 1} + end +end +``` + +Validations are added by listing the statistics required to preform a +validation, along with a function which will validate the state of these +statistics every time they update. Below is an untested (so probably incorrect) +example of defining a validation which ensures block producers are producing an +expected number of blocks. + +```elixir +defmodule Validations.BlockProductionRate do + use Architecture.Validation + + defp slot_time, do: 3 * 60 * 1000 + defp grace_window(_state), do: 20 * 60 * 1000 + defp acceptable_margin, do: 0.05 + + defp win_rate(_), do: raise("TODO") + + # current interface only supports validations computed from a single statistic, for simplicity of PoC + def statistic, do: Statistics.BlockProductionRate + + def validate(_resource, state) do + # implication + if state.elapsed_ns < grace_window(state) do + :valid + else + slots_elapsed = state.elapsed_ns / slot_time() + slot_production_ratio = state.blocks_produced / slots_elapsed + + # control flow structure like an if-else or a case-switch + cond do + slot_production_ratio >= 1 -> + {:invalid, "wow, something is *really* broken"} + + slot_production_ratio < win_rate(state.stake_ratio) - acceptable_margin() -> + {:invalid, "not producing enough blocks"} + + slot_production_ratio > win_rate(state.stake_ratio) + acceptable_margin() -> + {:invalid, "producing more blocks than expected"} + + # default case + true -> + :valid + end + end + end +end +``` + +## Drawbacks + +[drawbacks]: #drawbacks + +- introducing a new language to the stack adds learning and onboarding overhead + - (I believe the benefit here outways this cost, but would like to hear other + engineer's opinions) +- this design adds a new component to maintain alongside daemon feature + development and network releases + - (still seems better than divorcing validation scripts from protocol + development) + +## Rationale and alternatives + +[rationale-and-alternatives]: #rationale-and-alternatives + +- the primary alternative would be to focus on producing and maintaining a + series of validation scripts which run against the network + - this approach makes more complex validations more difficult + - implementation would be cheaper for this, but the maintenance would probably + be higher + - determining how to deploy this and test this seems harder (cron jobs with + storage volumes to track data across executions? ew) + - this approach makes it difficult to rely on the automated validation wrt + stability against ongoing protocol changes + +## Prior art + +[prior-art]: #prior-art + +- Conner has written a few Python scripts in the `coda-automation` repo which we + currently use to generate graphics that we manually validate + - much of this work will probably be ported into this tool at some point, so + this work is still important and provides a basis for some of this services + functionality + +## Unresolved questions + +[unresolved-questions]: #unresolved-questions + +- is the protocol team comfortable with maintaining a new service that is not + written in OCaml? +- should this service be used as part of the integration testing framework + (there is a large overlap in what and how this validation service will monitor + the health of the network and what the new integration test framework project + intends to achieve) diff --git a/website/docs/researchers/rfcs/0033-blockchain-in-hard-fork.md b/website/docs/researchers/rfcs/0033-blockchain-in-hard-fork.md new file mode 100644 index 000000000..c1846d5d8 --- /dev/null +++ b/website/docs/researchers/rfcs/0033-blockchain-in-hard-fork.md @@ -0,0 +1,199 @@ +--- +format: md +title: "RFC 0033: Blockchain In Hard Fork" +sidebar_label: "0033 Blockchain In Hard Fork" +hide_table_of_contents: false +--- + +> **Original source:** +> [0033-blockchain-in-hard-fork.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0033-blockchain-in-hard-fork.md) + +## Summary + +[summary]: #summary + +We present alternatives for handling the blockchain in a hard fork. The focus is +on hard forks in response to a network failure, though the mechanisms may be +applicable in more general scenarios. + +This RFC does not consider how the transaction SNARK and scan state are handled +in hard forks. There will be another RFC for those, and an RFC to describe the +entire hard fork plan. + +## Motivation + +[motivation]: #motivation + +For the launch of the Coda main net, we wish to have a hard fork plan and +implementation in place, in case there's a network failure. One of the parts of +such a plan is how to update the blockchain. + +## Summary description of alternatives + +[summary-descriptions]: #summary-descriptions + +After the fork, do one of these: + +1. New genesis timestamp and genesis ledger; assert a new starting epoch and + slot + +2. Retain the original genesis timestamp, with no time offset, with rules to + maintain chain strength + +3. Retain the original genesis timestamp, use a time offset for new blocks + +## Detailed description of alternatives + +[detailed-descriptions]: #detailed-descriptions + +Terminology: + +- pause point: the time of the last good block before a network failure; it's a + chosen time + +# Alternative 1 + +- choose a root and ledger at the pause point + +- the initial epoch, slot are the special case of 0,0 + - otherwise, increment the slot from the pause point + - provide as a compile-time parameter + +- if we have a proof of the protocol state, provide that + +- subsequent blocks may use different proof, verification keys + +- force a software upgrade by bumping the protocol major version + +## Notes + +- nodes will bootstrap from the new genesis state + +- time-locked accounts and transactions with expiration should continue to work, + because they rely on global slot numbers, and the slot is incremented from the + pause point + +## Issues + +- what is the source of randomness (epoch seed) for the post-fork chain? + +- is it acceptable to erase history (but we already have a succinct blockchain) + +- specifically, is there unfairness to the chains discarded past the pause point + +- what is allowed to change after the fork, with what effect: + - protocol state structure + - ledger structure/depth, + - account structure + - consensus constants + +- do we have to maintain pre-, post-fork code/data structures to verify the new + genesis state, ledger, and new blocks + +- if there's no proof of the ledger, how to convince the world that the new + genesis ledger and ledger are valid + +# Alternative 2 + +- choose a root and ledger at a pause point + +- nodes have a notion of "last fork time", an epoch-and-slot + +- upon a fork, gossip a special block to update the last fork time is issued, + also referring to the protocol state and proof at the pause point, and + resetting the chain strength + +- if items affecting keys, ledger hashes, and proofs haven't changed, a software + upgrade may not be required; otherwise, force an upgrade + +## Issues + +- there may be empty slots or even empty epochs between the pause point and the + restart; the reset of the chain strength may be sufficient to handle the + consequences + +- could we provide a new genesis block and proof, instead of the special block + +- As in Alternative (1), items may change after the fork, so there may be pre-, + post-fork proofs, coda, data structures; each additional fork may introduce + more complexity + +- If the ledger/account structure changes, we won't have a proof for the + post-fork state, also mentioned as an issue for Alternative (1) + +- If the epoch, slot widths change, computations for epochs or slots may become + more complex; subsequent forks may introduce more complexity + - possible solution: a most-recent-fork timestamp (an earthly time, not the + "last fork time") might be used for epoch and slot calculations + +# Alternative 3 + +- as Alternative (2), except that a time offset is added to the special block + - eliminates the empty slot/epoch issue + +## Issues + +- as in Alternative (2), except that the chain strength issue is resolved + +# Resolution + +After discussion in the PR associated with this RFC (#5019), there was agreement +of the acceptability of Alternative 2. There remains some concern about the +weakening of chain strength, but attacks based on that weakening don't seem to +open a plausible attack to an adversary. + +In addition to the above description of Alternative 2, the discussion proposed +and accepted another feature, unsafety bits contained in the protocol state, to +indicate which parts of the protocol state may have changed. + +If no unsafe bits are set, the fork is safe, in the sense that the protocol +state may be proved with blockchain SNARKs in effect at the pause point. + +If an unsafe bit is set, some component of the protocol state has changed, so +the protocol state cannot be proved. For example, the protocol state contains a +blockchain state, which includes a hash of a SNARKed ledger and a hash of a +staged ledger. If the structure of accounts changes, that will affect both +ledger hashes. Therefore, the protocol state might have unsafe bits specifically +for ledger hashes. + +Unsafety bits indicate unsafety relative to the previous block on the chain. +Blocks produced after a block with some unsafety bits set will not have any +unsafety bits set, at least, not until the next hard fork. + +## Drawbacks + +[drawbacks]: #drawbacks + +There's no compelling reason not to do this. We need to be prepared to perform a +hard fork if the main net fails. + +## Rationale and alternatives + +[rationale-and-alternatives]: #rationale-and-alternatives + +Unlike most RFCs, this RFC provides design alternatives, rather than proposing a +particular design. + +## Prior art + +[prior-art]: #prior-art + +Echo Nolan created a branch, `rfc/hard-forks`, that was not merged into the code +base. There is a section in the RFC there, "Technical considerations", that +mentions issues related to the blockchain. + +The core technical idea for the blockchain in that RFC was to add `era ids` in +blocks corresponding to slot ranges. An era id denotes some set of features, and +the type system verifies feature flags for code using such features. The +blockchain SNARK verifies the era id in blocks. + +The alternatives presented here don't mention feature flags, but do raise the +possibility of pre- and post-fork code. + +## Unresolved questions + +[unresolved-questions]: #unresolved-questions + +See the issues for the alternatives presented in this RFC. Undoubtedly, new +issues will become apparent through implementation and testing of any of the +alternatives. diff --git a/website/docs/researchers/rfcs/0034-reduce-scan-state-memory-usage.md b/website/docs/researchers/rfcs/0034-reduce-scan-state-memory-usage.md new file mode 100644 index 000000000..ae02d8b9b --- /dev/null +++ b/website/docs/researchers/rfcs/0034-reduce-scan-state-memory-usage.md @@ -0,0 +1,258 @@ +--- +format: md +title: "RFC 0034: Reduce Scan State Memory Usage" +sidebar_label: "0034 Reduce Scan State Memory Usage" +hide_table_of_contents: false +--- + +> **Original source:** +> [0034-reduce-scan-state-memory-usage.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0034-reduce-scan-state-memory-usage.md) + +## Summary + +[summary]: #summary + +This RFC analyzes current scan state memory usage and proposes a series of +optimizations which can be peformed in order to allow us to meet our target TPS +goals without consuming an astronomical amount of RAM. + +## Motivation + +[motivation]: #motivation + +In order to motivate these optimizations, we first need to analyze the expected +memory usage of the scan state at our target TPS and slot time parameters. Below +are the relevant calculations for computing the total memory usage (in bytes) of +a single scan state, given 3 parameters of the scan state. `M` is the depth of +each tree in the scan state, `T` is the total number of trees per scan state, +and `D` is the scan state delay (we will discuss these in more detail later). + +![](https://latex.codecogs.com/gif.latex?%5Cdpi%7B150%7D%20%5Cbegin%7Balign*%7D%20Base%20%26%5Ctriangleq%20%5Bomitted%5D%20%5C%5C%20Merge%20%26%5Ctriangleq%20%5Bomitted%5D%20%5C%5C%20FullBranch%20%26%5Ctriangleq%202%20%5Ccdot%20Merge%20+%207%20%5Ccdot%20Word%20%5C%5C%20EmptyBranch%20%26%5Ctriangleq%205%20%5Ccdot%20Word%20%5C%5C%20FullLeaf%20%26%5Ctriangleq%20Base%20+%205%20%5Ccdot%20Word%20%5C%5C%20EmptyLeaf%20%26%5Ctriangleq%203%20%5Ccdot%20Word%20%5C%5C%20NumberOfBranches%20%26%5Ctriangleq%20T%20%282%5E%7BM%7D-1%29%20%5C%5C%20NumberOfFullBranches%20%26%5Ctriangleq%20%5Csum_%7Bi%3D1%7D%5E%7BM%7D%20%5Csum_%7Bj%3D1%7D%5E%7Bi%7D%202%5E%7BM-j%7D%20%28D+1%29%20%5C%5C%20NumberOfEmptyBranches%20%26%5Ctriangleq%20NumberOfBranches%20-%20NumberOfFullBranches%20%5C%5C%20NumberOfFullLeaves%20%26%5Ctriangleq%20%28T-1%29%202%5E%7BM%7D%20%5C%5C%20NumberOfEmptyLeaves%20%26%5Ctriangleq%202%5E%7BM%7D%20%5C%5C%20TreeStructureOverhead%20%26%5Ctriangleq%20T%20%28%282M-1%29%20Word%29%20%5C%5C%20ScanState%20%26%5Ctriangleq%20TreeStructureOverhead%20%5C%5C%20%26%5Cphantom%7B%5Ctriangleq%7D+%20NumberOfFullBranches%20%5Ccdot%20FullBranch%20%5C%5C%20%26%5Cphantom%7B%5Ctriangleq%7D+%20NumberOfEmptyBranches%20%5Ccdot%20EmptyBranch%20%5C%5C%20%26%5Cphantom%7B%5Ctriangleq%7D+%20NumberOfFullLeaves%20%5Ccdot%20FullLeaf%20%5C%5C%20%26%5Cphantom%7B%5Ctriangleq%7D+%20NumberOfEmptyLeaves%20%5Ccdot%20EmptyLeaf%20%5Cend%7Balign*%7D) + + + +##### TODO: fix indentation of NumberOfEmptyBranches computation; got rate limited by the API :( + +For convenience, I have created a +[python script](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/res/scan_state_memory_usage.py) +which computes the scan state size in relation to the scan state depth and +delay. As per +[scan_state_constants.ml](https://github.com/MinaProtocol/mina/blob/8f4f05b50764a09fb748a590a7c50cd89bbed94d/src/lib/snark_params/scan_state_constants.ml), +the scan state depth is computed by +`1 + ceil(log2(MaxUserCommandsPerBlock + 2))`, and the maximum number of user +commands per a block is set by `MaxTPS * W / 1000` (where `W` is the length of a +slot in ms). Thus, at block windows of 30 seconds and a target `MaxTPS` of 1, +the scan state depth would be +`1 + ceil(log2((1 * 30000 / 1000) + 2)) == 1 + ceil(log2(32)) == 6`, and at 3 +minute block windows, it would be +`1 + ceil(log2((1 * 180000 / 1000) + 2)) == 1 + ceil(log2(182)) == 9`. The +number of trees in a scan state is effected by both the depth of the scan state +and the scan state delay. Scan state delay is a parameter which we will select +closer to mainnet launch based on how many active snark workers we expect to be +online (as well as the ratio of transaction snark proving time to slot length). +Below is a graph of what the scan state size would be at various +parameterizations of delay, for both 30 second slots and 3 minute slots. + +![](https://github.com/MinaProtocol/mina-resources/blob/main/docs/res/scan_state_memory_usage.png) + +For convenience, the 30 second (depth 6) graph is shown alone below so that the +values are easier to inspect. + +![](https://github.com/MinaProtocol/mina-resources/blob/main/docs/res/scan_state_memory_usage_depth_6.png) + +[Raw CSV data](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/res/scan_state_memory_usage.csv) + +What these graphs show is that the dominate factor in reducing scan state memory +usage is actually length of a slot time. However, the length of a slot time is, +in turn, dependent on blockchain proving time. Furthermore, if slot time is +reduced, but transaction proving time increases (due to, i.e., transaction snark +bundling), then delay will need to be increased. Still, lowering the slot time +(to reduce the scan state depth) and increasing delay is preferrable to keeping +the slot time higher to have a lower delay. + +Still, even if we assume the best case of being able to achieve 30 second slots +and only needing a scan state delay of 2, a single scan state in the current +representation will take up a total of 21.58mb. This is unacceptable considering +that we store a scan state at every single breadcrumb in the transition +frontier. The transition frontier will always have at minimum `k` breadcrumbs +(for a node that has been participating for at least `k` blocks on the network), +and there exists some constant (let's call it `g` for now), which is greater +than 1, that, when multiplied by `k`, gives us an average expected number of +breadcrumbs in the transition frontier at any point in time. Calculating `g` is +not possible until we have fully determined the consensus parameters for `f` and +`Δ`, so as a dirty reason about the size of a transition frontier, we will +somewhat conservatively assume `g = 2`. With this assumption, at `k = 1024` (the +`k` value used by Cardano in the Ouroboros papers), all the scan states in the +frontier will take up 22.09gb in the best case scenario, and 44.19gb in the +average scenario. Therefore, the representation and storage rules for scan +states needs to be modified in order to allow TPS to scale to a reasonable +target for mainnet. + +## Detailed design + +[detailed-design]: #detailed-design + +### Full vs. Partial Scan States + +Scan states will be separated into 2 representations: full scan states, and +partial scan states. A full scan state will is guaranteed to contain all the +transaction witnesses for the base satements, where as a partial scan state has +no such guarantee. As we will discuss below in the section "Global Transaction +Witness Cache", the scan states will no longer store direct copies of the +transaction witnesses. As such, the actual difference between a full and partial +scan state is primarily that the transaction witnesses of a scan state will not +have any guarantee of storage in the global transaction witness cache. Thus, +full and partial scan states will individually have the exact same memory +footprint, but indirectly, the memory footprint of a partial scan state is less +since it only maintains weak references to transaction witnesses stored in the +global transaction witness cache. + +### Downgrading Full Scan States During Blockchain Extensions + +Full scan states are only required at the tips of the transition frontier. These +are the only blocks in the frontier where snark workers need the transaction +witnesses to exist. In reality, snark workers only need the transaction +witnesses at the best tip, but keeping the transaction witneses at all tips +helps us cache transaction witnesses between blockchain extensions, which helps +reduce the amount of transaction witnesses that need to be constructed when a +reorg occurs. This means that the transition frontier will downgrade a full scan +state into a partial scan state once a block extends the block that scan state +belongs to. Said another way, when a block is added to a tip of the frontier, +that tip becomes a branch. Tips will store full scan states (potential +exceptions to this mentioned the section below), and branches will store partial +scan states. Therefore, when a tip becomes a branch, the scan state for that +block will be downgraded. + +### Pruning Probalistically Dead Frontier Tips + +Above we discuss that full scan states are only needed at the tips. This is +true, but it's also true that as a tip is further back from the current best tip +of the frontier, the probability that this tip (and thus the transaction +witnesses contained in this tip) will be relevant at some point in the future +goes down. Because of this, after some number of blocks are added to a better +chain than a tip, its scan state can be downgraded. Downgrading this scan state +is safe since, as long as we maintain the parent block's breadcrumb mask, we can +always upgrade this scan state again if we need to in the future. Doing this +incorrectly could potentially open up a denial of service attack (see +[drawbacks](#drawbacks)), so it is important to take this into consideration +when designing the function that will determine when to frontier tip is "dead", +and thus, should have its scan state downgraded. + +### Global Transaction Witness Cache + +In the current implementation, identical transaction witnesses can exist +duplicated across multiple scan states. Transaction witnesses which are +inherited from the previous scan state will share the same pointer in the new +scan state, but any transaction witnesses that are built up as part of applying +a staged ledger diff are uniquely constructed in each scan state. It is rather +common that multiple forks from the same parent block will share transaction +witnesses, so long as the coinbase and fee transfers in a staged ledger diff +exist at the end of the sequence of transactions. Duplication of these +transaction witnesses can be identified faster than the actual construction of +the transaction witness itself, so transaction witnesses can be stored in a +global ref-counted cache instead of directly on the scan state itself. More +formally, we compute a transaction witness `src -> dst` from the source merkle +root `src` and the transaction `txn`. Building the transaction witness involves +updating 2 accounts in a merkle tree, which, at the worst case, involves `57` +merge hashes and `2` account hashes at ledger depth `30`. Given the `src` and +`txn`, there is only one transaction witness `src -> dst` which can be +constructed. Therefore, the value `H(src || H(txn))` uniquely identifies a +transaction witness `src -> dst` without having to compute the `dst`. If a +global transaction cache witness exists, then the staged ledger diff application +algorithm can first check if `H(src || H(txn))` exists in the cache before +building `src -> dst`. Under this scheme, the base statements of the scan state +would no longer directly embed a reference to the transaction witness, but would +instead store just the `H(src || H(txn))` for that witness, which will be used +to find the actual transaction witness from the global transaction witness cache +when it is required. + +### Scan State Icebox + +Similar to how the likelihood of a tip being extended decreases as the dominate +chain in the transition frontier grows in length relative to that tip, the same +is true for branches as well. The intermediate scan states in the transition +frontier branches are needed to build new scan states which extend those +branches. However, if we don't believe that a block will be extended (at least +probalistically), we can persist the scan state of that block to disk, and load +it back from disk if we ever need it again in the future. This "scan state +icebox" where we persist the scan states to will need to be garbage collected as +blocks are evicted from the transition frontier during root transitions. + +## Work Breakdown/Prioritization + +##### Must Do (definitely required to mitigate memory usage to acceptable levels) + +1. Implement Global Transaction Witness Cache +2. Remove Transaction Witness Scan States Embedding +3. Implement Full/Partial Scan State Logic +4. Downgrade Branching Scan States in Frontier + +##### Might Do (extra work which will further reduce memory usage) + +1. Frontier Dead Tip Pruning +2. Scan State Icebox + +## Drawbacks + +[drawbacks]: #drawbacks + +### Dead Frontier Tip Scan State Reconstruction Attack + +Pruning probalistically dead frontier tips opens up nodes for potential denial +of service attack. Pruning scan states is seen as "safe" because there is no +actual loss in local data availability as long as the node maintains the ledger +mask of the previous staged ledger in the transition frontier. This is because +the scan state can be restored to its full format by recomputing the transaction +witnesses from the previous staged ledger's target ledger. However, it is +possible for an adversary to generate a fork off of an old tip in the frontier, +then broadcast that, forcing any nodes which have pruned that scan state to +reconstruct the transaction witnesses for it (under the current logic). This +could be prevented by either adding a better heuristic for whether or not +something is worth adding to our frontier (as in, if it's such a bad block, +maybe we should skip it and only download it during catchup if later there is a +good chain built off of it). Alternatively, this can be mitigated by only +constructing transaction witnesses for tips which are "close enough" to our best +tip in terms of strength. It would be a hard attack to execute in practice, but +it is theoretically possible for an adversary to wait until a series of their +accounts get VRF wins in nearby slots, and then make nodes perform a whole bunch +of scan state witness reconstructions at the same time. All this said, it's not +clear whether this attack could cause enough work to actually take nodes offline +or lag behind the network enough to do anything bad. + +### Incorrect Implementation of Scan State Icebox Leads to High Disk I/O + +If the scan state icebox is not implemented correctly, it could potentially +greatly increase the amount of disk I/O performed during participation. As such, +it is very important that this is well instrumented and tested so that it does +not accidentally vastly reduce performance in order to reduce the memory usage +of the scan states some. + +## Unresolved questions + +[unresolved-questions]: #unresolved-questions + +- can the exact gains from this be calculated before implementing these + optimizations, so that we can use that as a framework for informing whether or + not the optimizations were successful? diff --git a/website/docs/researchers/rfcs/0035-scan-state-hard-fork.md b/website/docs/researchers/rfcs/0035-scan-state-hard-fork.md new file mode 100644 index 000000000..cc59e41ae --- /dev/null +++ b/website/docs/researchers/rfcs/0035-scan-state-hard-fork.md @@ -0,0 +1,192 @@ +--- +format: md +title: "RFC 0035: Scan State Hard Fork" +sidebar_label: "0035 Scan State Hard Fork" +hide_table_of_contents: false +--- + +> **Original source:** +> [0035-scan-state-hard-fork.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0035-scan-state-hard-fork.md) + +## Summary + +[summary]: #summary + +We describe how to handle the transaction SNARK scan state across hard forks. + +## Motivation + +[motivation]: #motivation + +For the launch of the Coda main net, we wish to have a hard fork plan and +implementation in place, in case there's a network failure. One of the parts of +such a plan is how to update the transaction SNARK scan state. + +The scan state consists of an in-core binary tree of transactions and proofs +using the transaction SNARK. The work to produce those proofs, provided by +"SNARK workers", is computationally intensive, and paid for by block producers. +Therefore, it's desirable to use the information in the scan state across forks, +if possible. + +## Detailed design + +[detailed-design]: #detailed-design + +For the blockchain, we distinguish between the "safe" case, where the validity +and structure of the protocol state do not change across the fork, and the +"unsafe" case, where that condition does not hold. + +For the safe case, we can make use of the information in the scan state at the +pause point. Some nodes in the scan may have proofs, while others await proofs. + +# Safe case + +## Choosing a scan state; what to do with it + +Suppose we've chosen a root protocol state from a node at the pause point. In +that node's transition frontier, there is a breadcrumb associated with that +root, which contains a staged ledger with a scan state. The hash of that staged +ledger is contained in the root protocol state (in the body's blockchain state). + +## Alternative: completing the scan state + +Given the root scan state, it's possible to complete the proofs offline to +produce a new protocol state. An advantage would be that the scan state would be +empty after the fork, simpler in engineering terms, and likely similar to what +we'd do in the unsafe case. But the proofs would be produced outside of the +usual SNARK worker mechanism, and so outside the view of the ordinary consensus +mechanism, perhaps lessening trust in the result. There would also need to be a +tool to complete the proofs, requiring additional engineering work. + +If the scan state does not have a full complement of leaves, the leaves can be +padded with "dummy" transactions to assure that we can propagate proofs up to +the root. + +The blockchain-in-hard-fork RFC mentions a "special block" that gets gossipped +to indicate a fork. In the case that we complete the scan state, That block can +contain a new protocol state and proof derived from the new SNARKed ledger. We +would need code similar to what's in `Block_producer` for producing an ordinary +block (`External_transition`) to generate the special block, starting with the +chosen root protocol state. When a node receives a special block, it creates an +empty scan state and empty pending coinbase. + +## Alternative: baking in the scan state + +If we don't complete the scan state, we can persist the root breadcrumb, and +place it in the transition frontier when restarting the network. This +alternative requires some additional engineering to save the breadcrumb, get it +into the new binary, and load it. Those tasks are relatively simple, though. + +## Rescuing transactions + +Both alternatives for the safe case make use of the root breadcrumb across the +fork, but we're explicitly discarding transactions in other breadcrumbs. We have +transaction SNARK proofs for them, and it's wasteful, and may be upsetting if +those transactions are rolled back. + +The other breadcrumbs in this node's transition frontier reflect a particular +view of a best tip; other nodes may have different best tips. The node can query +peers to get their transition frontier breadcrumbs, and we can make such queries +recursively to some or all reachable nodes. From those breadcrumbs, we can +calculate a longest common prefix, which can be persisted, and placed in the +transition frontier after the fork. + +# Unsafe case + +In this case, the proofs in the scan state are not of consequence to the +post-fork chain. Those proofs are for transactions not yet reflected in SNARKed +ledger. + +## Alternative: discard the scan state + +We can simply discard the scan state at the pause point. Both the transactions +and their proofs in the scan state are lost. + +Discarding proofs in the scan state means also discarding the fee transfers +associated with them. Since these fee transfers have already been applied to the +staged ledger, we will have to discard the staged ledger at the root, and when +the chain resumes, using the SNARKed ledger at the root as the staged ledger, +and an empty scan state. The fees that would have accrued to SNARK workers for +proofs, and the fees for block producers, are not transferred. Operators should +be made aware of this fee-loss risk. + +Discarding the staged ledger means that we're discarding the finalized +transactions reflected in that ledger. Also, the staged ledger hash at the pause +point is no longer valid, and needs to be recomputed from the SNARKed ledger +used as the staged ledger. + +Although the unsafe case in a disaster should be a rare event, rolling back +otherwise-finalized transactions may be upsetting to users. + +## Alternative: re-prove the scan state(s) + +We could retain the scan state and staged ledger, by discarding the original +proofs, and generating new proofs. + +The new proofs could be created online, by carrying over the scan state, but +with the proofs discarded, when the network resumes. + +The new proofs could be created offline, to replace the existing proofs. This +approach introduces some of the same distaste of "behind-the-scenes" proving +mentioned in the scan state completion for the safe case. + +Whether the new transaction proofs are created online or offline, the fee +transfers in the scan state would be retained, so that SNARK workers who +generated the original proofs will still get paid. + +With the online approach, the scan state needs to be persisted, so it can be +used by the post-fork binary. + +With the offline approach, we could also rescue transactions as described above, +but reprove those transactions contained in them to generate new breadcrumbs to +be saved and loaded to the transition frontier after the fork. + +# Choices for mainnet + +For mainnet, we're focusing on the unsafe case. And for that case, the +engineering team's choice is to carry over the scan state, with proofs removed, +to the post-fork binary (the `online` case). The new proofs can be provided +without cost by O(1) or other altruistic parties. + +## Drawbacks + +[drawbacks]: #drawbacks + +There's no compelling reason not to do this. We need to be prepared to perform a +hard fork if the main net fails. + +## Rationale and alternatives + +[rationale-and-alternatives]: #rationale-and-alternatives + +The alternatives and their rationales are described above. + +## Prior art + +[prior-art]: #prior-art + +Echo Nolan created a hard fork RFC (branch `rfc/hard-forks`) that describes an +online process to drain the scan state in phases if the SNARK changes. This +approach may be especially suitable for planned forks. In that case, we want to +maintain suitable incentives as the fork time approaches, so we'd like to avoid +the possibility of discarding proofs. + +## Unresolved questions + +[unresolved-questions]: #unresolved-questions + +Is it practical to download breadcrumbs, seeing how there's been concern about +the size of scan states? + +If we do offline proving: + +- How long will it take to produce the needed proofs to finish off the scan + state? What's a reasonable time? + +- Would it be reasonable to pay SNARK workers to produce such proofs, either + exclusively or in tandem with locally-generated proofs? + +- Will the world accept this approach? + +How much of the current code used to produce a block can be reused to produce a +special block? diff --git a/website/docs/researchers/rfcs/0036-hard-fork-disaster-recovery.md b/website/docs/researchers/rfcs/0036-hard-fork-disaster-recovery.md new file mode 100644 index 000000000..fe9e0fa99 --- /dev/null +++ b/website/docs/researchers/rfcs/0036-hard-fork-disaster-recovery.md @@ -0,0 +1,261 @@ +--- +format: md +title: "RFC 0036: Hard Fork Disaster Recovery" +sidebar_label: "0036 Hard Fork Disaster Recovery" +hide_table_of_contents: false +--- + +> **Original source:** +> [0036-hard-fork-disaster-recovery.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0036-hard-fork-disaster-recovery.md) + +## Summary + +[summary]: #summary + +This RFC explains how to a create a hard fork in response to a severe failure in +the Coda network. It draws on the strategies described in earlier RFCs that +describe handling of the blockchain and scan state for hard forks. + +## Motivation + +[motivation]: #motivation + +The Coda network may get to a state where the blockchain can no longer make +progress. Symptoms might include many blockchain forks, or blocks not being +added at all, or repeated crashes of nodes. To continue, a hard fork of the +blockchain can be created, using an updated version of the Coda software. + +## Detailed design + +[detailed-design]: #detailed-design + +When it becomes evident that the network is failing, the Coda developers will +perform the following tasks: + +- on some node, run a CLI command to persist enough state to re-start the + network; to choose a node, we may wish to query a set of nodes to find the + best state, and use a representative node + +- run a tool to transform the persisted state into data needed for the Coda + binary + +- create a new Coda binary with a new protocol version + +- notify node operators of the change, in a manner to be determined, and provide + access to the new binary + +To remedy the problems that led to a failure, the Coda software will likely +change in some significant way when the network is restarted. Using a new +protocol version, with a new major/minor or minor version, will require node +operators to upgrade their software. + +## CLI command to save state + +The Coda developers will choose a node with a root to represent the starting +point of the hard fork. The choice of node is beyond the scope of this RFC. + +The CLI command can be in the `internal` group of commands, since it's meant for +use in extraordinary circumstances. A suggested name is `save-hard-fork-data`. +That command communicates with the running node daemon via the daemon-RPC +mechanism used in other client commands. + +Let `frontier` be the current transition frontier. When the CLI command is run, +the daemon saves the following data: + +- its root + + this is an instance of `Protocol_state.value`, retrievable via + + ```ocaml + let full = Transition_frontier.full_frontier frontier in + let root = full.root in + root |> find_protocol_state + ``` + +- the SNARK proof for that root + + this is an instance of `Proof.t`, retrievable via + + ```ocaml + let full = Transition_frontier.full_frontier frontier in + let transition_with_hash,_ = Full_frontier.(root_data full).transition in + let transition = With_hash.data transition_with_hash in + transition.protocol_state_proof + ``` + +- the SNARKed ledger corresponding to the root + + this is an instance of `Mina_ledger.Ledger.Any_ledger.witness`, retrievable + via + + ```ocaml + let full = Transition_frontier.full_frontier frontier in + full.root_ledger + ``` + + Note: There appears to be a mechanism in `Persistent_root` for saving the root + ledger, but it appears only to store a ledger hash, and not the ledger itself. + +- two epoch ledgers + + there is pending PR #4115 which allows saving epoch ledgers to RocksDB + databases + + which two epoch ledgers needed depends on whether the root is in the epoch + current at the time of the network pause, or in the previous one: + - if the root is in the current epoch, the two ledgers needed are + `staking_epoch_snapshot` and `next_epoch_snapshot`, as in the PR + - if the root is in the previous epoch, the two ledger needed are + `staking_epoch_snapshot` and `previous_epoch_snapshot` (not implemented in + the PR) + +- the protocol states for the scan state at the root + + this is a list of `Protocol_state.value`, retrievable via + + ```ocaml + let full = Transition_frontier.full_frontier frontier in + let state_map = full.protocol_states_for_root_scan_state in + State_hash.Map.data state_map + ``` + +- the breadcrumb at the root + + this is an instance of `Breadcrumb.t`, retrievable via + + ```ocaml + let full = Transition_frontier.full_frontier frontier in + let root = full.root in + Full_frontier.find_exn full root + ``` + + The breadcrumb contains a validated block and a staged ledger. + + From the breadcrumb, we need to save: + - the scan state and pending coinbase (both part of the contained staged + ledger) + - the root transition + + The staged ledger can be reconstructed from the SNARKed ledger, the scan + state, and the protocol states. See + `Persistent_frontier.construct_staged_ledger_at_root`. + +- optionally, a chain of breadcrumbs between the root and best tip + + over some reachable nodes, find the common prefix of breadcrumbs because the + scan states contained in breadcrumbs can be large, do this computation lazily: + - find a common prefix of breadcrumb hashes - obtain the breadcrumbs + corresponding to those hashes from a representative node N.B.: it is + possible that there is no common prefix beyond the root breadcrumb + +The in-memory values (that is, those other than the epoch ledgers) can be +serialized as JSON or S-expressions to some particular location, say +`recovery_data` in the Coda configuration directory. The epoch ledgers can be +copied to that same location. + +## Preparing the data for inclusion in a binary + +Operators should be able to install a new package containing a binary and all +data needed to join the resumed network. + +The SNARKed ledger can be stored in serialized format, stored as a value in a +generated OCaml module, which can be loaded when creating the full transition +frontier: + +```ocaml + module Forked_ledger = struct + let ledger = ... (* Bin_prot serialization *) + end +``` + +The ledger can be passed as the `~root_ledger` argument to +`Full_frontier.create`. + +The epoch ledgers can be compiled into the binary, or, if epoch ledger +persistence is available, included as files from the install package. In the +latter case, the operator may need to copy the installed epoch ledgers to the +Coda config directory. + +If the fork is safe, then like the SNARKed ledger, we can compile the saved root +breadcrumb into the binary. The breadcrumb data would be passed in the +`~root_data` argument to `Full_frontier.create`. + +If provided, the breadcrumb chain can be used to populate an initial transaction +pool in the binary. Each breadcrumb contains an block; there's a function +`External_transition.transactions` to get its transactions. + +It might be that the SNARKed ledger and breadcrumbs are too large to include in +the binary. In that case, we could provide serialized versions of them, to be +loaded on daemon startup. + +## Gossipping a hard fork block + +When the hard fork occurs, a restarted daemon gossips a special block containing +a new hard fork time, an epoch and slot. The type `Gossip_net.Latest.T.msg` can +be updated with a new alternative, say `Last_fork_time`. Like an ordinary block, +the special block contains a protocol state, to be verified by the blockchain +SNARK. The unsafe bits in an ordinary block are always `false`. In the special +block, some of those bits may be `true`. + +In the case of a "safe" hard fork, where no unsafe bits are set, the hard fork +block contains the root protocol state we saved and its proof. In the case of an +unsafe hard fork, the unsafe bits indicate which parts of the protocol state can +be bypassed in the proof. In the unsafe case, different proof and verification +keys may be needed across the fork. + +Like an ordinary block, the special block contains a current protocol version. +In the safe case, the patch version may be updated. In the unsafe case, the +major version or minor versions must be updated, forcing a software upgrade. + +Currently, verifying the blockchain for ordinary blocks is done using `update` +in the functor `Blockchain_snark.Blockchain_snark_state.Make`, which relies on a +`Snark_transition.t` input derived from a block. For a hard fork, we'd write a +new function that verifies that the protocol state is the same as the old state, +except for those pieces denoted by unsafe bits. + +Nodes running the new software won't accept other blocks until they've received +the special block, and time has reached the designated epoch and slot. + +## Drawbacks + +[drawbacks]: #drawbacks + +In the best case, the network will run smoothly, making preparations for a hard +fork gratuitious, and the software unnecessarily complex. That said, the cost of +forgoing those preparations is high. + +By starting from the transition frontier root, we are explicitly discarding +blocks in the transition frontier past the root (k such blocks, from the +consensus parameters). The transactions in those blocks are not finalized. We +avoid that loss if we add these transactions back to the transaction pool. + +## Rationale and alternatives + +[rationale-and-alternatives]: #rationale-and-alternatives + +This design is for hard forks made necessary by a network failure. + +Other designs may be needed for planned hard forks, if we change features of the +protocol. For example, we can save a breadcrumb, as in the safe fork case, but +drain the scan state online after the fork, so that existing proofs are used, +before switching to a new transaction SNARK. See the unmerged branch +`rfc/hard-forks` for details. That way, SNARK workers who may be aware of the +planned fork will continue to produce SNARKs, without risking lost fees when the +planned fork occurs. + +## Prior art + +[prior-art]: #prior-art + +See RFCs 0032 and 0033 for how to handle the blockchain and scan state across +hard forks. + +## Unresolved questions + +[unresolved-questions]: #unresolved-questions + +What unsafe bits are there in the protocol state, and what do they denote? + +Will the network actually resume, if this plan is followed? + +Will users trust the network after an unsafe fork? diff --git a/website/docs/researchers/rfcs/0037-github-merging-strategy.md b/website/docs/researchers/rfcs/0037-github-merging-strategy.md new file mode 100644 index 000000000..3ae5729e4 --- /dev/null +++ b/website/docs/researchers/rfcs/0037-github-merging-strategy.md @@ -0,0 +1,193 @@ +--- +format: md +title: "RFC 0037: Github Merging Strategy" +sidebar_label: "0037 Github Merging Strategy" +hide_table_of_contents: false +--- + +> **Original source:** +> [0037-github-merging-strategy.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0037-github-merging-strategy.md) + +## Summary + +[summary]: #summary + +This RFC proposes changing our git merge strategy from squash to merge commits. + +## Motivation + +[motivation]: #motivation + +We would like for pull-requests (PRs) to merge more quickly and automatically. +We would also like to encourage smaller PRs, to reduce the burden on reviewers +and to reduce the amount of work that has to be done keeping up with upstream +changes in the `develop` branch. + +In addition, we have begun to use GitHub's 'PR chains', where a series of +smaller changes can be linked together in a chain, but with each change in its +own PR. These allow us to continue to work on larger features even when earlier +changes haven't yet been included in the `develop` branch, and without combining +the changes into a single large PR. PR #5093 added support for the +`ready-to-merge-into-develop` label, which automates the process of merging +these PR chains correctly. + +When multiple PRs modify the same lines of code and one of them is merged into +`develop`, the other will have 'merge conflicts' and will need manual +intervention to fix the conflict before it can be merged. We would like to +minimise this because + +- PRs with conflicts do not merge automatically +- the PR process is slowed down by waiting for these manual interventions +- developer time is wasted in resolving these conflicts. + +Our current merge strategy is 'squash and merge', which discards the +individually committed changes within each PR. This hampers `git`'s ability to +reduce or eliminate merge conflicts by combining the changes from individual +commits, unnecessarily creating more merge conflicts to be handled manually. + +These unnecessary conflicts are particularly prevelant in PR chains. For +example, + +``` +PR 1 + develop +-> [Add line 1 in A.txt] +-> [Modify line 1 in B.txt] + +PR 2 (chained to PR 1) + develop +-> [Add line 1 in A.txt] +-> [Modify line 1 in B.txt] +-> [Modify line 1 in A.txt] +-> ... +``` + +when `PR 1` is squashed and merged into develop, `git` is unable to recognise +that the initial changes made in `PR 2` match with those from `PR 1` and reports +a merge conflict. + +If PRs in chains have review comments that result in changes, these will usually +also result in a merge conflict. For example, + +``` +PR 1 + develop +-> [Add line 1 in A.txt] +-> [Modify line 1 in B.txt] +-> [Modify line 1 in A.txt in response to review comments] + +PR 2 (chained to PR 1) + develop +-> [Add line 1 in A.txt] +-> [Modify line 1 in B.txt] +-> [Add line 1 in C.txt] +-> ... +``` + +when `PR 1` is squashed and merged, `git` will again be unable to merge `PR 2` +automatically. + +## Detailed design + +[detailed-design]: #detailed-design + +We can use `git`'s merge commits when merging to avoid the above issues. These +preserve the individual commit information that `git` needs to automatically +resolve these conflicts. + +This is a simple configuration change in the GitHub settings for the repo. + +## Drawbacks + +[drawbacks]: #drawbacks + +#### We lose a linear `git` history (from `git log` and friends) + +Locally, developers can use `git log -m --first-parent` to view only the merge +commits. `git log -m --first-parent --patch` shows the changes from commits +beneath the merge commit, as expected. + +To generate a linear history from any branch between commits `aaaaaa` and +`ffffff`, run the commands + +```bash +git checkout aaaaaa +for commit_id in $(git rev-list --reverse --topo-order --first-parent aaaaaa..ffffff); do + git read-tree $commit_id && git checkout-index -f -a && git update-index -q --refresh && git commit --no-verify -a -C $commit_id; +done +``` + +#### `git blame` identifies the commit within PRs instead of the merge commit + +Similar to the above, developers can locally use `git blame -m --first-parent`, +or blame on the 'prettified' branch. + +#### `git bisect` will explore some non-merge commits + +We can set `git bisect` to ignore any non-merge commits by running a script +`bisect-non-merge.sh bad_commit good_commit` with + +```bash +#!/bin/bash +set -euv +git bisect start $1 $2 +git bisect skip $(git rev-list --no-merges $2..$1) +``` + +To include commits from before the switch from squash to merge, this can be +modified to + +```bash +#!/bin/bash +set -euv +git bisect start $1 $2 +git bisect skip $(git rev-list --no-merges ^ffffff $2..$1) +``` + +where `ffffff` is the last commit where we used the squash merge strategy. + +##### Manual bisection + +For any scripts where we want to implement bisect ourselves, we can instead use +`git rev-list --first-parent aaaaaa..ffffff` to get the full list of commits +from `aaaaaa` (non-inclusive) to `ffffff` that we want to bisect over. + +#### The history will contain bad or useless commit messages + +This can be mostly ignored when using the above mitigations. Changing commit +message culture is out of the scope of this RFC. + +## Rationale and alternatives + +[rationale-and-alternatives]: #rationale-and-alternatives + +- Why is this design the best in the space of possible designs? + - The alternatives (squashing and rebasing) both throw away the information + that `git` needs to handle merges most effectively. +- What other designs have been considered and what is the rationale for not + choosing them? + - Squashing and rebasing, as above. +- What is the impact of not doing this? + - Slower, manual PR merging and wasted developer effort. + +## Prior art + +[prior-art]: #prior-art + +`git` was designed with merges using these kinds of merge commits in mind. The +Linux kernel has used merge commits since it switched over to `git` in 2005. + +## Unresolved questions + +[unresolved-questions]: #unresolved-questions + +- What parts of the design do you expect to resolve through the RFC process + before this gets merged? + - None +- What parts of the design do you expect to resolve through the implementation + of this feature before merge? + - None +- What related issues do you consider out of scope for this RFC that could be + addressed in the future independently of the solution that comes out of this + RFC? + - Commit message culture. diff --git a/website/docs/researchers/rfcs/0038-rosetta-construction-api.md b/website/docs/researchers/rfcs/0038-rosetta-construction-api.md new file mode 100644 index 000000000..58a6bb7ac --- /dev/null +++ b/website/docs/researchers/rfcs/0038-rosetta-construction-api.md @@ -0,0 +1,664 @@ +--- +format: md +title: "RFC 0038: Rosetta Construction Api" +sidebar_label: "0038 Rosetta Construction Api" +hide_table_of_contents: false +--- + +> **Original source:** +> [0038-rosetta-construction-api.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0038-rosetta-construction-api.md) + +## Summary + +[summary]: #summary + +The +[Rosetta Construction API](https://www.rosetta-api.org/docs/construction_api_introduction.html) +is the write-half of the Rosetta API. This RFC discusses the different chunks of +work that need to get done to make this half a reality. Note that discussion of +the Data API is out-of-scope for this RFC (and is already fully implemented and +partially tested). + +## Motivation + +[motivation]: #motivation + +We wish to support Rosetta as it enables clients to build once and support +multiple chains. Many vendors that wish to build on top of our protocol are +asking for full Rosetta support. Not just the read-half (the Data API) but also +the write-half. + +The desired outcome is a full to-spec Construction API implementation. + +## Detailed design + +[detailed-design]: #detailed-design + +The following flow chart is pulled from the +[Rosetta documentation](https://www.rosetta-api.org/docs/construction_api_introduction.html), +but useful for understanding the different pieces necessary for the +implementation: + +``` + Caller (i.e. Coinbase) + Construction API Implementation + +-------------------------------------------------------------------------------------------+ + | + Derive Address +----------------------------> /construction/derive + from Public Key | + | + X | + X Create Metadata Request +---------------------> /construction/preprocess + X (array of operations) | + + Get metadata needed X | | + to construct transaction X +-----------------------------------------------+ + X v | + X Fetch Online Metadata +-----------------------> /construction/metadata (online) + X | + | + X | + X Construct Payloads to Sign +------------------> /construction/payloads + X (array of operations) | + + X | | + Create unsigned transaction X +------------------------------------------------+ + X v | + X Parse Unsigned Transaction +------------------> /construction/parse + X to Confirm Correctness | + X | + | + X | + X Sign Payload(s) +-----------------------------> /construction/combine + X (using caller's own detached signer) | + + X | | + Create signed transaction X +-----------------------------------------------+ + X v | + X Parse Signed Transaction +--------------------> /construction/parse + X to Confirm Correctness | + X | + | + X | + X Get hash of signed transaction +--------------> /construction/hash +Broadcast Signed Transaction X to monitor status | + X | + X Submit Transaction +--------------------------> /construction/submit (online) + X | + + +``` + +This flow chart will guide our explanation of what is needed to build a full +to-spec implementation of the API. Afterwards, we'll list out each proposed +piece of work with details of how it should be implemented. Upon mergin of this +RFC, each work item will become an issue on GitHub. + +The initial version of the construction API will _only_ support payments. We can +quickly follow with a version that will support delgation as well. This RFC will +talk about all the tasks assuming both payments and delegation are supported. +All token-specific transactions (and any future transactions) are not yet +supported and out-of-scope for this RFC. + +### Flow chart + +#### Before Derivation + +Before the derivation step, we need to generate a keypair. We'll use the private +key to sign the payment and the public key to tell others who the sender is. + +#### Derivation + +[derivation]: #derivation + +Derivation demands that the public key expected as input be a hex-encoded +byte-array value. So we'll [add functionality](#marshal-keys) to the +[client-sdk](#marshal-keys), the [generate-keypair binary](#marshal-keys), and +the offcial [Mina CLI](#marshal-keys) to marshall the `Fq.t * Fq.t` pair (the +native representation of an uncompressed public key). + +The [derivation endpoint](#derivation-endpoint) would be responsible for reading +in the uncompressed public key bytes which +[requires adjusting the Rosetta spec](#add-curves), compressing the public key, +and base58-encoding it inline with how we currently represent public keys in +serialized form. + +#### Preprocess + +[preprocess]: #preprocess + +The [preprocess endpoint](#preprocess-endpoint) takes a proposed set of +operations for which we'll need to +[clearly specify examples for different transactions](#operations-docs). It +assures they can be [converted into transactions](#inverted-operations-map) and +it returns an input that is needed for the [metadata](#metadata) phase during +which we can gather info on-chain. In our case, this is just the sender's public +key+token_id. + +#### Metadata + +[metadata]: #metadata + +The [metadata endpoint](#metadata-endpoint) takes the senders public +key+token_id and returns which nonce to use for transaction construction. + +#### Payloads + +[payloads]: #payloads + +The [payloads endpoint](#payloads-endpoint) takes the metadata and the +operations and returns an +[encoded unsigned transaction](#unsigned-transaction-encoding). + +#### After Payloads + +[after-payloads]: #after-payloads + +After the payloads endpoint, folks must sign the transaction. In the future, we +should build support for this natively, but for now our client-sdk's signing +mechanism suffices. As such, we don't need to do much here other than +[encode the signed transaction properly](#signed-transaction-encoding). + +#### Parse + +[parse]: #parse + +The [parse endpoint](#parse-endpoint) takes a possibly signed transaction and +parses it into operations. + +#### Combine + +[combine]: #combine + +The [combine endpoint](#combine-endpoint) takes an unsigned transaction and the +signature and returns an +[encoded signed transaction](#signed-transaction-encoding). + +#### Hash + +[hash]: #hash + +The [hash endpoint](#hash-endpoint) takes the signed transaction and returns the +hash. + +#### Submit + +[submit]: #submit + +The [submit endpoint](#submit-endpoint) takes a signed transaction and +broadcasts it over the network. We also should +[audit broadcast behavior to ensure errors are returned when mempool add fails](#audit-transaction-broadcast). + +#### Testing + +We should [integrate the construction calls](#test-integrate-construction) into +the existing `test-agent`. By doing this, we don't need to worry about getting +this into CI since it is already there (or will be by the time this RFC lands, +thanks @lk86 !). + +We also will want to [integrate the official rosetta-cli](#test-rosetta-cli) to +verify our implementation. + +In addition, we'll manually test on subsequent QA and Testnets. + +### Work items + +Think of these as the tasks necessary to complete this project. Each item here +will turn into a GitHub issue when this RFC lands. + +#### Marshal Keys + +[marshal-keys]: #marshal-keys + +Add support for creating/marshalling public keys ([via Derivation](#derivation)) + +**Format** + +Compressed public keys are accepted of the following form: + +Field elements are expected to be backed by a 32-byte array where the highest +bits of the field are stored in arr[31]. + +Presented is a hex encoded 32-byte array where the highest bit of arr[31] is the +`is_odd` parity bit. + +``` +|----- pk : Fq.t (32 bytes) ------{is_odd}--| + +``` + +Example: + +The encoding `fad1d3e31aede102793fb2cce62b4f1e71a214c94ce18ad5756eba67ef398390` + +Decodes to the field represented by the number +`fad1d3e31aede102793fb2cce62b4f1e71a214c94ce18ad5756eba67ef398310`. That's the +same as the encoding, except that the 9 representing the high nybble of the +final byte is replaced by 1, by zeroing the high bit. Because the high bit was +set, is_odd is true. + +**Name** + +We'll call this the "raw" format for our public keys. In most places, we can get +away with just adding a `-raw` flag in some form to support this new kind of +representation. + +a. Change the Client-SDK + +i. Add a `rawPublicKeyOfPrivateKey` method to the exposed `client_sdk.ml` module +that returns `of_private_key_exn s` which is then marshalled to a string +according to the above specification. + +ii. Add a new `rawPublicKey : publickey -> string` function to `CodaSDK`. + +iii. Add new documentation for this change. + +b. Change the generate-keypair binary + +i. Also print out the raw representation after generating the keypair on a new +line: + +`Raw public key: ...01E0F3...0392FA` + +ii. Add a new subcommand `show-public-key` which takes the private key file as +input and prints the same output as running the generate command. + +c. Change coda cli + +i. Add a new subcommand `show-public-key` as a subcommand to `mina accounts` +(reuse the implementation in (b.ii) + +#### Derivation endpoint + +[derivation-endpoint]: #derivation-endpoint + +[via Derivation](#derivation) + +Read in the bytes, compress the public key, and base58-encoding it inline with +how we currently represent public keys in serialized form. Adding errors +appropriately for malformed keys. + +#### Add curves + +[addcurves]: #addcurves + +Add support for our curves and signature to Rosetta +([via Derivation](#derivation)) + +Follow the instructions on +[this forum post](https://community.rosetta-api.org/t/add-secp256r1-to-curvetype/130/2) +to add support for the +[tweedle curves and schnorr signatures](https://github.com/CodaProtocol/signer-reference). +This entails updating the rosetta specification with documentation about this +curve, and changing the rosetta-sdk-go implementation to recognize the new curve +and signature types. Do not worry about adding the implementation to the keys +package of rosetta-cli for now. + +#### Operations docs + +[operations-docs]: #operations-docs + +Add examples of each kind of transaction that one may want to construct as JSON +files. Eventually we'd want one for each type of transaction, but for now it +suffices to just include a payment. + +For example: The following expressrion would be saved in `payment.json` + +``` +[{ + "operation_identifier": ..., + "amount": ..., + "type": "Payment_source_dec" +}, +{ + "operation_identifier": ..., + "amount": ..., + "type": "Payment_receiver_inc" +}, +... +] +``` + +This is useful for manual testing purposes and sets us up for integration with +the +[construction-portion of rosetta-cli integration](https://community.rosetta-api.org/t/feedback-request-automated-construction-api-testing-improvements/146/4). + +#### Inverted operations map + +[inverted-operations-map]: #inverted-operations-map + +[via Preprocess](#preprocess) + +Write a function that recovers the transactions that are associated with some +set of operations. We should create a test +`forall (t : Transaction). op^-1(op(t)) ~= t` which enumerates all the kinds of +transactions. For an initial release it suffices to test this for payments. + +#### Preprocess Endpoint + +[preprocess-endpoint]: #preprocess-endpoint + +[via Preprocess](#preprocess) + +First [invert the operations](#inverted-operations-map) into a transaction, we +find the sender and include the address in the response (note that on our +network an address is made up of a token id and a sender). The options type will +be defined as follows: + +```ocaml +module Options = struct + type t = + { sender : string (* base58-ecoded compressed public key *) + ; token_id : string (* uint64 encoded as string *) + } + [@@deriving yojson] +end +``` + +#### Metadata Endpoint + +[preprocess-endpoint]: #preprocess-endpoint + +[via Metadata](#metadata) + +This is a simple GraphQL query. This endpoint should be easy to implement. + +#### Unsigned transaction encoding + +[unsigned-transaction-encoding]: #unsigned-transaction-encoding + +[via Payloads](#payloads) + +The Rosetta spec leaves the encoding of unsigned transactions +implementation-defined. Since we want to make it easy for alternate signers to +be created (eg. the ledger), we'll want this encoding to be some faithful +representation of the bytes upon which the signature operation acts. + +Specifically this is the user command having been transformed into a +`Transaction_union_payload.t` and then hashed into a +`(field, bool) Random_oracle_input.t`. We will serialize the Random_oracle_input +in two ways as defined below and send that byte-buffer as hex-encoded ascii. + +``` +// Serialization schema for Random oracle input (1) + +00 00 00 05 # 4-byte prefix for length of array (little endian) + # +xx xx ... # each field encoded as a 32-bytes each one for each of the length +yy yy ... # + # Field elements are represented by laying out their bits from high + # to low (adding a padding zero at the highest bit in the front) + # and then grouping by 8 and converting to bytes: + # + # (always zero) Bit254 Bit253 Bit252 ... Bit2 Bit1 Bit0 + # |----groups of 8---|--groups of 8---| + # + # +00 00 34 D4 # 4-byte prefix for length of bits in the bitstring (little endian) + # +A4 43 D4 ... # the bool list compacted into a bitstring, pad the last 1 byte with + # extra zeros on the right if necessary + +// Note: Edited on 8/18 to include 4-byte length of bits in the bitstring to remove any ambiguity between the zero-padding and true zeros in the bitstring +``` + +``` +// Serialization schema for Random oracle input (2) +// This is denoted as "signerInput" in the output +// +// The prefix and suffix can be used by a signer more easily + +SignerInput (JSON): +{ + prefix: [field], + suffix: [field] +} + +// where the fields are encoded as strings like above +// example: + +{ + prefix: [ "000000000000000000000000000000000000000000000000000000000001E0F3", ... ], + suffix: [ "000000000000000000000000000000000000000000000000000000000001E0F3", ... ] +} + +A signer would take the prefix and suffix and use it during `derive` (which doesn't necessarily need to be exactly the same as the implementation Mina (it just needs to be "random"). And use `px`, `py`, and `r` in between prefix and suffix for hash. +``` + +Another important property of the unsigned-transaction and signed-transaction +representations is that they are invertible. The `unsigned_transaction_string` +is then a `JSON` input (stringified) conforming to the following schema: + +``` +{ randomOracleInput : string (* Random_oracle_input.t |> to_bytes |> to_hex *) +, signerInput : SignerInput +, payment: Payment? +, stakeDelegation: StakeDelegation? +} +// where stakeDelegation and payemnt are currently defined in the client-sdk shown below +// it is an error to treat stakeDelegation / payment in any way other than a variant, but it is encoded unsafely like this becuase JSON is garbage-fire and can't represent sum types ergonomically +``` + +```reasonml +// Taken from Client-SDK code + +type stakeDelegation = { + [@bs.as "to"] + to_: publicKey, + from: publicKey, + fee: uint64, + nonce: uint32, + memo: option(string), + validUntil: option(uint32), +}; + +type payment = { + [@bs.as "to"] + to_: publicKey, + from: publicKey, + fee: uint64, + amount: uint64, + nonce: uint32, + memo: option(string), + validUntil: option(uint32), +}; +``` + +Note that our client-sdk only has support for signing payments and delegations +but this version of the construction API only supports those transactions types +as well. We'll default to the client-sdk for now. + +Additionally, we should expose a new method in the client-sdk to feed the raw +`Random_oracle.Input.t` to signer logic in addition to going through the +existing signPayment and signDelegation ones. The Client-sdk implementation +should enforce that these two implementations agree with an adjustment to our CI +unit test. + +#### Payloads Endpoint + +[payloads-endpoint]: #payloads-endpoint + +[via Payloads](#payloads) + +First [convert the operations](#inverted-operations-map) embedding the correct +sender nonce from the metadata. Return an +[encoded unsigned transaction](#unsigned-transaction-encoding) as described +above. + +This endpoint will also accept a query parameter `?plain_random_oracle` + +#### Signed transaction encoding + +[signed-transaction-encoding]: #signed-transaction-encoding + +[via After Payloads](#after-payloads) + +Since we'll later be broadcasting the signed transaction via GraphQL, our signed +transaction encoding is precicesly the union of the format required for the +sendPayment mutation and the sendDelegation mutation (stringified): + +``` +{ + signature: string (* Signature hex bytes as described below *), + payment: payment?, + stakeDelegation: stakeDelegation? +} +``` + +**Format** + +``` +// Signature encoding + +a signature is a field and a scalar +|---- field 32bytes (Fp) ---|----- scalar 32bytes (Fq) ----| +Use the same hex-encoded representation as described above for the public keys for each of the 32byte chunks. +``` + +#### Parse Endpoint + +[parse-endpoint]: #parse-endpoint + +[via Parse](#parse) + +The parse endpoint takes the transaction and needs to return the operations. The +implementation will use the same logic as the Data API transaction -> operations +logic and so we do not need an extra task to make this happen. + +Importantly, we've ensured that our unsigned and signed transaction serialized +representations have enough information in them for us to recreate the +transaction full values at parse time (ie. we don't only store the hash needed +for signatures). + +#### Combine Endpoint + +[combine-endpoint]: #combine-endpoint + +[via Combine](#combine) + +The combine endpoint +[encodes the signed transaction](#signed-transaction-encoding) according to the +schema defined above. + +#### Hash Endpoint + +[hash-endpoint]: #hash-endpoint + +[via Hash](#hash) + +The hash endpoint takes the signed transaction and returns the hash. This can be +done by pulling in `Mina_base` into Rosetta and calling hash on the transaction. + +#### Audit transaction broadcast + +[audit-transaction-broadcast]: #audit-transaction-broadcast + +[via Submit](#submit) + +Upon skimming our GraphQL implementation, it seems like it is already succeeding +only if the transaction is successfully added to the mempool, but it important +we more carefully audit the implementation to ensure this is the case as it's an +explicit requirement in the spec. + +#### Submit Endpoint + +[submit-endpoint]: #submit-endpoint + +[via Submit](#submit) + +The submit endpoint takes a signed transaction and broadcasts it over the +network. We can do this by calling the `sendPayment` or `sendDelegation` +mutation depending on the state of the input after parsing the given +transaction. + +#### Test integrate construction + +[test-integrate-construction]: #test-integrate-construction + +[via Testing](#testing) + +The existing Rosetta test-agent tests our Data API implementation by running a +demo instance of Coda and mutating its state with GraphQL mutations and then +querying with the Data API to see if the data that comes out is equivalent to +what we put in. + +We can extend the test-agent to also send construction API requests. We should +at least add behavior to send a payment and delegation constructed using this +API. We can shell out to a subprocess to handle the "off-api" pieces of keypair +generation and signing. + +We also should include logic that verifies the following: + +1. The unsigned transaction output by payloads parses into the same operations + provided +2. The signed transaction output by combine parses into the same operations + provided +3. After the signed transaction is in the mempool, the result from the data api + is a superset of the operations provided originally +4. After the signed transaction is in a block, the result from the data api is a + superset of the operations provided orginally + +#### Test Rosetta CLI + +[test-rosetta-cli]: #test-rosetta-cli + +[via Testing](#testing) + +The [rosetta-cli](https://github.com/coinbase/rosetta-cli) is used to verify +correctness of implementations of the rosetta spec. This should be run in CI +against our demo node and against live qa and testnets. We can release a version +on a testnet before we've fully verified the implementation against rosetta-cli, +but the project is not considered "done" until we've done this properly. + +It's worth noting that the rosetta-cli is +[about to get new features to be more flexible at testing other construction API scenarios](https://community.rosetta-api.org/t/feedback-request-automated-construction-api-testing-improvements/146/4). +These changes will certainly support payments and delegations. We don't need to +wait for the implementation of this new system to support payments today. + +## Drawbacks + +[drawbacks]: #drawbacks + +It's extra work, but we really wish to enable folks to build on our protocol in +this way. + +## Rationale and alternatives + +[rationale-and-alternatives]: #rationale-and-alternatives + +Decisions were made here to limit scope where possible to enable shipping an MVP +as soon as possible. This is why we are explicitly not supporting extra +transactions on top of payments (initially) and payments+delegations (closely +afterwards). + +Luckily Rosetta has a very clear specification, so our designs are mostly +constrained by the decisions made in that API. + +In [marshal keys (c)](#marshal-keys), we could also change commands that accept +public keys to also accept this new format. Additionally, we could change the +GraphQL API to support this new format too. I think both of these changes are +unnecessary to prioritize as the normal flows will still be fine and we'll still +encourage folks to pass around the standard base58-encoded compressed public +keys as they are shorter. + +In the sections about +[encoding unsigned transactions](#unsigned-transaction-encoding) and +[encoding signed transactions](#signed-transaction-encoding), we make an +explicit decision to pick a format that supports arbitrary signers. There is +minimal change involved with the client-sdk to make that supported; +additionally, this was done to improve implementation velocity and because we +did conciously choose that interface with usability in mind. JSON is chosen to +pack products of data as using a readable JSON string makes it easy to audit, +debug, and understand our implementation. + +## Prior art + +[prior-art]: #prior-art + +The [spec](https://www.rosetta-api.org/docs/construction_api_introduction.html) + +## Unresolved questions + +[unresolved-questions]: #unresolved-questions + +There are no unresolved questions at this time that I'd like to answer before +merging this RFC. + +As stated above, explicitly out-of-scope is any future changes to the Data API +portion of Rosetta and Construction API support for transactions other than +payments and delegations. diff --git a/website/docs/researchers/rfcs/0039-snark-keys-management.md b/website/docs/researchers/rfcs/0039-snark-keys-management.md new file mode 100644 index 000000000..beaef609a --- /dev/null +++ b/website/docs/researchers/rfcs/0039-snark-keys-management.md @@ -0,0 +1,244 @@ +--- +format: md +title: "RFC 0039: Snark Keys Management" +sidebar_label: "0039 Snark Keys Management" +hide_table_of_contents: false +--- + +> **Original source:** +> [0039-snark-keys-management.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0039-snark-keys-management.md) + +## Summary + +[summary]: #summary + +This RFC proposes some strategies to make our snark key management more +flexible, and to allow us to be more explicit about which keys to use and when +to generate them. + +## Motivation + +[motivation]: #motivation + +The current key management strategy is brittle and hard to understand or +interact with. + +- Every change that affects the snark keys requires a recompile of the majority + of our libraries, and it is not possible to specify changes to the constraint + system and the corresponding proving/verification keypairs at runtime. +- Snark keys are generated as part of the build process for some of the + compile-time configurations, which slows down compiles and obfuscates the + particular keys that any build is using, as well as preventing us from using a + single, unified binary for our different configurations. +- Every set of keys that have been generated for a CI job are stored in the S3 + keys bucket, with no easy way to identify them. +- Keys in the S3 bucket are always preferred to locally-generated keys during + the build process. + - During local builds from developers' machines, internet connection problems + may result in either partially-downloaded keyfiles or long download waits + while the keys are fetched from S3. +- It is difficult for the infrastructure team to find and package the correct + keys for a particular binaries when deploying. +- The caching behavior for keys is implicit and entangled with the pickles proof + system interface, which makes it hard for the team to inspect or adjust the + behavior. + +The goal of this RFC is to resolve these issues. At a high-level, the proposed +steps to do this are: + +- Make keys identifiable from the file contents. +- Add a tool for generating keys outside of the build process. +- Allow new keys to be specified explicitly. +- Make key generation explicit, removing it from the build process. + +## Detailed design + +[detailed-design]: #detailed-design + +### Make keys identifiable from the file contents + +In order to confirm that loaded keys are compatible with a given configuration, +we need to have some representation of the configuration embedded with the keys +files. We propose that every key (and URS) file contains a header in minified +JSON format before the actual contents, specifying + +- the constraint constants, as defined in the runtime configuration +- the SHA commit identifiers for the current commit in the mina and marlin repos + used when generating the keys + - the identifying hash depends on the marlin repo SHA, so including it lets us + recompute/verify the identifying hash, even though we could technically + derive it from the mina SHA and its git repository. +- the length of the binary data following the header, so that we can validate + that the file was successfully downloaded in full, where applicable +- the date associated with the current commit + - we don't chose the file generation time, in order to make the header + reproducible + - it is helpful to have some notion of date, so that we can build a simple + `ls`-style tool that we can use to identify out-of-date keys in the local + cache etc. +- the type of the file's contents +- the constraint system hash (for keys) or domain size (for URS) +- any other input information + - e.g. constraint system hash for the transaction snark wrapped by a + particular blockchain snark +- a version number, so that we can add or modify fields +- the identifying hash, which should match the hash part of the filename for + generated files + +For example, a header for the transaction snark key file might look like: + +```json +{ + "header_version": 1, + "kind": { + "type": "step_proving_key", + "identifier": "transaction_snark_base" + }, + "constraint_constants": { + "c": 8, + "ledger_depth": 14, + "work_delay": 2, + "block_window_duration_ms": 180000, + "transaction_capacity": { "txns_per_second_x10": "2" }, + "coinbase_amount": "200", + "supercharged_coinbase_factor": 2, + "account_creation_fee": "0.001" + }, + "commits": { "mina": "COMMIT_SHA_HERE", "marlin": "COMMIT_SHA_HERE" }, + "length": 1000000000, + "commit_date": "1970-01-01 00:00:00", + "constraint_system_hash": "MD5_HERE", + "identifying_hash": "HASH_HERE" +} +``` + +This will allow us to identify files from S3 or in a cache directory by +examining only the header, making it easier to remove outdated keys and to find +or enumerate keys. + +The majority of the work here is adjusting the rust code so that the header can +be generated from OCaml and written to the same file as its contents. + +### Add a tool for generating keys outside of the build process + +This tool could be an `internal`/`advanced` subcommand of the main executable to +begin with. The interface may be extremely basic, reusing the current runtime +configuration files: + +``` +mina.exe advanced generate-snark-keys -config-file path/to/config.json +``` + +Suggested optional arguments are: + +- `-key-directory` - specify the directory to place the keys in +- `-output-config-file` - write a config file with the keys path information + included +- `-list-missing-headers` - do not generate files, instead dump the JSON headers + of key files that do not already exist in the given key directory. + - This is for use by the infrastructure team, so that CI and deployment + processes can identify which keys to look for in S3 or another cache without + explicitly building S3 code into the process. + - This should probably have an exit status of 1 (rather than the normal 0) if + e.g. transaction snark keys are missing, to signal to the CI tool that some + files are missing but their headers could not be inferred because they + depend on other files. + +This tool should be implemented as a library at the level of the current +`Snark_keys` library, so that we can also create a standalone executable for +easier use by the infrastructure team. + +Most of this is calling existing functions; the only particular difficulty is +modifying `Pickles` and `Cache_dir` to allow finer-grained control of cache +locations for keys. + +### Allow new keys to be specified explicitly. + +In order to allow different keys to be used with the same binary, we need a way +to specify the files to load or the directories to search in. We should add +fields to the runtime configuration `config.json` file for + +- `snark_key_directories` - the list of directories to search in for keys +- `snark_keys` - a list where the filenames of individual keys can be specified, + identified by a `kind` field matching the one in the key's header. + +For example, + +```json +{ ... +, "snark_key_directories": ["~/.coda_cache_dir/snark_keys"] +, "snark_keys": + [ { "kind": + { "type": "step_proving_key" + , "identifier": "blockchain_snark" } + , "path": "~/Downloads/blockchain_snark_pk" } + , { "kind": + { "type": "step_verification_key" + , "identifier": "blockchain_snark" } + , "path": "~/Downloads/blockchain_snark_vk" } + , { "kind": + { "type": "wrap_proving_key" + , "identifier": "blockchain_snark" } + , "path": "~/Downloads/blockchain_snark_wrap_pk" } + , { "kind": + { "type": "wrap_verification_key" + , "identifier": "blockchain_snark" } + , "path": "~/Downloads/blockchain_snark_wrap_vk" } ] } +``` + +Integrating this with the `Pickles` library will involve restructuring its +`compile` function, probably converting it to a functor that exposes a module +with the hidden load and store operations for keys, as well as the other +functions that will need to be called explicitly once there is no implicit +caching. Once this is done, the paths can be passed to the relevant subsystems, +can load the files on-demand from the Pickles interfaces. + +### Make key generation explicit, removing it from the build process. + +Once the other stages have been completed, the CI, deployment, and release +processes will need to be updated to ensure that the keys are correctly placed/ +packaged, and that the configuration file specifies the correct locations. The +specific details of these are not clear yet and may vary, so these details are +considered out of the scope of this RFC. + +In most cases, it should be possible to use the +`mina.exe advanced generate-snark-keys -config-file path/to/input_config.json -output-config-file path/to/output_config.json` +form of the `generate-snark-keys` function above to do most or all of the +necessary setup work. + +When the switchover is complete, removing the `Snark_keys` library from the repo +will stop keys from being generated at build time with no other changes needed. + +## Drawbacks + +[drawbacks]: #drawbacks + +- This adds an extra step to building a working, proof-enabled binary. +- This adds more features to the `config.json` configuration files that most + users will/should not use. + +## Rationale and alternatives + +[rationale-and-alternatives]: #rationale-and-alternatives + +Rationale: + +- This design adds useful information that has been missing to the place where + it would be used. +- This design separates the distinct concerns of code compilation and + cryptographic artifact generation. +- This design is simpler and more flexible than the current system. +- This design makes it possible to change caching solutions in response to cost + and complexity changes. +- This design moves a poorly-understood part of our infrastructure to the + configuration level of the other infrastructure components. + +## Prior art + +[prior-art]: #prior-art + +The current system. + +## Unresolved questions + +[unresolved-questions]: #unresolved-questions diff --git a/website/docs/researchers/rfcs/0040-rosetta-timelocking.md b/website/docs/researchers/rfcs/0040-rosetta-timelocking.md new file mode 100644 index 000000000..186d7820c --- /dev/null +++ b/website/docs/researchers/rfcs/0040-rosetta-timelocking.md @@ -0,0 +1,183 @@ +--- +format: md +title: "RFC 0040: Rosetta Timelocking" +sidebar_label: "0040 Rosetta Timelocking" +hide_table_of_contents: false +--- + +> **Original source:** +> [0040-rosetta-timelocking.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0040-rosetta-timelocking.md) + +## Summary + +[summary]: #summary + +This is a proposal for supporting time-locked account tracking in Rosetta. +Though, really, it boils down to supporting historical balance lookups for +accounts through the archive node (some of which may have time locks on them). +We are still considering adding support for time-locked account creation after +the genesis block, this proposal aims to describe what changes we could make to +the design and implementation in the scenario that we do wish to support this +feature. + +## Motivation + +[motivation]: #motivation + +Right now, Rosetta can not handle accounts with time locks on them. The +specification demands we present the liquid balance in the accounts. +Unfortunately, in the protocol we sample the liquidity curve at the moment funds +are attempting to be transferred which is at odds with how Rosetta attempts to +understand the world. We wish to support this. + +## Detailed design + +[detailed-design]: #detailed-design + +### Balance exemptions and historical balance lookups + +Rosetta supports the notion of a balance exempt account. These are accounts that +one should not use operations alone to monitor changes in balance. The +specification details that this should be used sparingly, but goes on to suggest +that vesting accounts are one such example. If we only support time-locked +accounts in the genesis ledger, all we need to do is at +Rosetta-server-startup-time is pull those time-locked account addresses and fill +the balance exemption field. If we don't, we'll need to update this field +dynamically, it remains to be seen if this is allowable by the Rosetta +specification, see unresolved question 1. + +This is not a solution alone -- the specification goes on to say that in the +case that one or more accounts are balance exempt, you must support historical +balance lookups. This is difficult for us because Mina's constant-sized +blockchain does not contain historical data. We use the archive node to store +extra data for us -- we're already using it to store all the blocks that our +nodes see. In order to support historical balance lookups, we have to add extra +information so we can compute this data. Specifically, we'll add information +about the current balance of any accounts touched during a transaction whenever +we store in the archive node. We can use this and the genesis ledger from +Rosetta to reconstruct the current balance of any account at any block by +crawling backward from that block until we see a transaction (or fallback to the +genesis ledger if no such transaction exists). Additionally, we can use the +extra timing information in time-locked accounts in the genesis ledger to sample +the liquidity curve at any moment as well. In the universe where we need to +support time-locked account creation after genesis, we should be able to extract +the timing information from the relevant transaction that creates the account +via the archive node as well. + +### Implementation details + +**Protocol/Archive** + +- Add the following columns to the `blocks_user_commands` table in SQL + +``` +fee_payer_balance : bigint NOT NULL, +sender_balance : bigint NOT NULL, +receiver_balance : bigint NOT NULL +``` + +These represent the amounts of tokens in the accounts (measured in nanomina) +after applying the user_command referenced by that block at that moment. + +- Add the following columns to the `blocks_internal_commands` table in SQL + +``` +receiver_balance : bigint NOT NULL +``` + +This represent the amounts of tokens in the account (measured in nanomina) after +applying the internal_command referenced by that block at that moment. + +- Add a new table `timing_info` to the SQL database with the following schema: + +``` +public_key, + +``` + +- Change the SQL schema to add relevant indexes to support the SQL query we'll + be performing from Rosetta +- Populate the new tables with the relevant info every time we add transactions + to the archive node (remember to look at fee, sender, and receiver) +- Pull the genesis ledger and add timing information to the database to the + `timing_info` table +- Every time we create a new time locked account, add to the `timing_info` table + +**Rosetta** + +- Set historical-balance-lookups to true +- Use the time-locked accounts to populate the balance exemption field. +- Change `/account/balance` queries to support the block-identifier parameter + and use it to perform the following SQL queries against the archive node: + +1. Recursively traverse the canonical chain until you find the starting block + that the identifier points to (we already have a similar query in the + `/block` section, use this as a starting point) +2. Recursively traverse the chain backward from that point until you find the + first transaction that involves the public key (either via fee, sender, + receiver) specified in the `/account/balance` parameter +3. Use the join-tables to find the relevant data in the `balances` table and + look at the amount + +- For `/account/balance` queries involving the time-locked accounts, use the + time-locking functions that already exist + https://github.com/MinaProtocol/mina/blob/92ea2c06523559b9980658d15b9e5271400ac856/src/lib/coda_base/account.ml#L561 + using the timing info in the database and the balance we found above. + +## Drawbacks + +[drawbacks]: #drawbacks + +This will increase the size of our archived data by a factor of around two. This +seems acceptable. + +## Rationale and alternatives + +[rationale-and-alternatives]: #rationale-and-alternatives + +### Use a Rosetta operation to unlock tokens each block + +The nice thing about this approach is we wouldn't have to change our protocol, +and we use operations -- the atomic unit of change in Rosetta -- to model tokens +vesting. + +This approach is not ideal from a scalability perspective because we would need +to generate synthetic operations adding the liquid balances to every single +currently vesting account. + +There is also another more serious reason that this approach is unacceptable: +Floating-point rounding issues will cause the sum of the parts to not equal the +whole. In other words, summing each of the synthetic operations growing the +liquid balance up until block `b`, would not be equal to querying the liquid +balance at block `b` itself. + +### Change the protocol + +Other protocols that have similar time-locked accounts require an explicit +on-chain transaction to move liquid funds out of the vesting account before they +are actually usable. If we changed our protocol to support such a transaction, +it would be trivial to model this in Rosetta. + +However, this provides a worse experience for users. Even though they know their +account has liquid funds, and even though the _protocol_ knows their account has +liquid funds, a separate transaction is required before they're usable. + +Additionally, we want to avoid changing the protocol this close to a looming +mainnet launch. + +## Prior art + +[prior-art]: #prior-art + +Celo's and Solana's Rosetta implementation is similar to the "Change the +protocol" section in Rationale and Alternatives. + +## Unresolved questions + +[unresolved-questions]: #unresolved-questions + +1. Will the Rosetta team officially be okay with this approach, even in the + world where we need time locked account creation after genesis (and thus need + a non-static balance exemption list) . See this discourse thread for more + info: + https://community.rosetta-api.org/t/representing-minas-vesting-accounts-using-balance-exemptions/317 diff --git a/website/docs/researchers/rfcs/0041-infra-testnet-persistence.md b/website/docs/researchers/rfcs/0041-infra-testnet-persistence.md new file mode 100644 index 000000000..a471f0e95 --- /dev/null +++ b/website/docs/researchers/rfcs/0041-infra-testnet-persistence.md @@ -0,0 +1,584 @@ +--- +format: md +title: "RFC 0041: Infra Testnet Persistence" +sidebar_label: "0041 Infra Testnet Persistence" +hide_table_of_contents: false +--- + +> **Original source:** +> [0041-infra-testnet-persistence.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0041-infra-testnet-persistence.md) + +## Summary + +[summary]: #summary + +This RFC proposes a dynamic storage solution capable of being leveraged by Mina +blockchain components to persist both application and chain state across +redeployments, infrastructure and application level network updates, +container/pod failures and other such destructive events and operations. The +proposal entails integrating a native Kubernetes mechanism for persisting all +testnet state as necessary into Mina's existing infrastructure setup built on +top of Google Cloud Platform and its hosted Kubernetes service offering, GKE. + +## Motivation + +[motivation]: #motivation + +Mina blockchain currently consists of several components or agent types each +with a particular role or function; and all of which depend on either +pre-compiled or runtime state in the form of genesis ledgers, wallet keys, +static and runtime configurations and intermediary chain states (for syncing +with and participating in block production protocols) for example. The +infrastructure, however, on which these components are deployed, at least in its +existing form, does not support stateful applications in the traditional sense +in that state is not persisted across restarts, redeployments or generally +speaking destructive operations of any kind on any of the component processes. +While by design and implementation through the use of _k8s_ `emptyDir` volumes, +historically these two points have resulted in considerable inefficiencies if +not blockers in application/code iteration while developers test feature and +protocol/product changes, infrastructure adjusts resource and cloud environment +settings and/or especially when community dynamics warrant operational tweaks +during public testnet releases. + +At a high level, the desired outcome of implementing such a solution should +result in: + +- a simple and automated process for both affiliating & persisting testnet + component resources and artifacts to individual testnet deployments (manual as + well automated) +- a flexible and persistent application storage framework to be leveraged by + infrastructure across multiple cloud providers +- a layer of abstraction over Helm/K8s storage and persistent-volume primitives + to be utilized by developers for integration testing, experimentation and of + course persisting and sharing network state + +## Detailed design + +[detailed-design]: #detailed-design + +Storage or state persistence for Kubernetes applications is largley a solved +problem and can generally be thought as consisting of two cloud-provider +agnostic, orthogonal though more often than not tightly interconnected +concepts: 1. the physical storage layer at which application data is remotely +persisted, and 2. the orchestration layer which regulates application "claims" +to this underlying storage provider. The former, more commonly referred to as +`Storage Classes` with respect to Kubernetes +[primitives](https://kubernetes.io/docs/concepts/storage/storage-classes/), +represents an abstraction over the type or "classes" of storage offered by a +particular administrator to describe different characteristics of the options +provided. These characteristics generally relate to storage I/O performance, +quality-of-service levels, backup policies, or perhaps arbitrary policies +customized for specific cluster profiles. The +[latter](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims), +known as `PersistentVolumeClaims`, is manifested as a specification and status +of claims to these storage classes and typically encapsulates storage size, +application binding and input/output access properties. + +### Storage Classes + +Managed at the infrastructure level and constrained to either a self-hosted or +Google Cloud Platform hosted storage provider service (due to infrastructure's +current use of Google Kubernetes Engine or GKE), implementing and integrating +storage classes within Mina's infrastructure entails installing _k8s_ +`StorageClass` objects, like the following, utilizing either a stand-alone Helm +chart, integrated into a common/core component Helm chart (to what amounts to +something like the following _yaml_ definition): + +``` +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: +provisioner: kubernetes.io/gce-pd +parameters: + type: + fstype: ext4 + replication-type: none +volumeBindingMode: WaitForFirstConsumer +``` + +or explicitly defined as a Terraform `kubernetes_storage_class` +[resource](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/storage_class): + +``` +resource "kubernetes_storage_class" "infra_testnet_ssd" { + metadata { + name = "testnet-ssd | testnet-standard" + } + storage_provisioner = "kubernetes.io/gce-pd" + parameters = { + type = + fstype = "ext4" + replication-type = "none" + } + volume_binding_Mode = "WaitForFirstConsumer" + allow_volume_expansion = true +} +``` + +The Terraform resource definition approach is preferred since the resources are +themselves singular infrastructure components and can be implemented within +existing infrastructure terraform definition files, organized by region. +Moreover there doesn't currently exist a fitting location for common Helm +objects and their associated Kubernetes resources and its debatable whether a +common Helm application (vs. library) is desirable. The initial thinking is that +a single storage class will need to be defined for all of infrastructure's +storage purposes per supported type (standard + ssd) per testnet deploy region. +This should facilitate initial experimentation as the classes' reliability and +performance are vetted in practice for different use-cases as well as provide +resiliency in the event of regional outages. Note that storage provided by a +class is replicated between availability zones of the installation region which +is itself scoped to a _k8s_ context when applying via `terraform apply` or +either `helm or kubectl` directly (hence the lack of a specification of zone or +region within the above definitions). It may be worth creating separate storage +classes for manually vs. automatically deployed testnets for +book-keeping/organizational purposes though it doesn't quite seem worth it at +the moment considering the availability of object metadata, labeling and +annotations involved at the volume claim level to help with this organization. + +#### Cost and Performance + +According to GCP's disk pricing documentation, each GB of persistent storage +costs approximately $0.17 on provisioned SSD disks and about $0.04 for standard +HDD/SCSI disks per month. While the amount of storage a single testnet would +claim is variable in nature and dependent on its scale (i.e. the number of +wallet keys, snark keys, static/runtime configs and chain intermediary states), +a significant portion of the storage space is expected to be consumed by either +shared or singular resources (e.g. genesis and potentially epoch ledgers, shared +keysets and archive-node postgres DBs). A reasonable, albeit rough, estimation +currently exists of ~10GB per deployment (with a bit of variance forseeable once +in practice). With this mind, a single testnet deployment maintained over an +entire month would likely increase infrastructure costs by about $17 for SSD +performance and $4 for standard performance. Considering that the average +life-expectancy of a testnet generally is far from a month, even for public +testnets (more on the order of a few days to a couple of weeks for manual +deploys and minutes to hours for automated deployments for testing), the cost +impact of this solution should be relatively negligible as long as infra +sustains the default policy of delete volumes on pod/claim cleanup and proper +hygiene and practices are followed for cleaning up obsolete testnets. **Note:** +recent reports show between ~$300-$410 per month of Google Cloud Storage costs. +We also propose taking advantage of GCP Monitoring +[Alerting Policies](https://console.cloud.google.com/monitoring/alerting/policies/create?project=o1labs-192920) +to monitor and alert on defined thresholds being met for various categories +including but not limited to overarching totals, per testnet allocations, +automated testing runtime and idle usage. + +With regards to performance and as previously mentioned, the current plan of +record is to provision both standard and ssd storage types, leverage only ssd +initially for testnet storage use-cases and at the same time gradually +benchmark/test different use-cases on both ssd and standard types across +regions. This should help us identify candidates to migrate from ssd to standard +with very little to no impact on performance without compromising on performance +in the present and potentially limiting engineering velocity anymore than +necessary. + +### Persistent Volume Claims + +Defined and dynamically generated at the application level, _k8s_ +`PersistenceVolumeClaims` objects will be implemented as _Helm_ named templates +which, as demonstrated with the pending Healthcheck work, can be pre-configured +with sane defaults and yet overridden at run or deploy time with custom values. +These templates will be scoped to each particular use-case, whether providing a +claim to persistence for a single entity within a testnet deployment or a shared +testnet resource - again taking into consideration of the idea and goal of being +able to provide defaults for the majority of cases and customization where +deemed necessary. + +#### Component specific state + +State specific to individual testnet components will be set within each +component's Helm chart, likely as an additional YAML doc included within +deployments similar to the following: + +``` + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + testnet: {{ $.Release.Name }} + name: "block-producer-{{ $config.name }}-runtime-state" +spec: + accessModes: + - ReadWriteMany + volumeMode: Filesystem + resources: + requests: + storage: {{ $.Values.storage.runtime.capacity }} + storageClassName: {{ $.Values.storage.runtime.class } +``` + +Considering the use of named templates and the minimal single line footprint of +their inclusion into files, we think it makes sense to avoid creating a separate +chart file for PVCs at least for core volume claims. + +#### Testnet shared state + +For shared resources, it makes sense to define within a Terraform +`kubernetes_persistent_volume_claim` +[resource](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/persistent_volume_claim) +implemented a part of the Terraform `testnet` module as opposed to a particular +Helm chart. For example, something like the following would be generated once +and then levereaged as mount volumes within all relevant pods within a testnet: + +``` +resource "kubernetes_persistent_volume_claim" "genesis_ledger" { + metadata { + name = "${testnet_name}-genesis-ledger" + } + spec { + access_modes = ["ReadWriteOnce"] + resources { + requests = { + storage = "${var.storage.genesis_ledger} + } + } + storage_class_name = "" + } +} +``` + +#### Persistent Volume Claim named templates + +In order to minimize code redundancy and promote consistency across Mina Helm +charts with regards to defining persistent volume claims for various testnet +component storage needs, we propose leveraging Helm's +[named templates](https://helm.sh/docs/chart_template_guide/named_templates/) +feature. Named templates are a mechanism introduced in version 3 of _Helm_ that +are basically Helm templates defined in a file and namespaced. They enable +re-use of a single persistent-volume claim (PVC) definition and allow for the +setting of sane/standard defaults with added variability/customization through +embedded values scoped to input template rendering arguments. The choice is +based on this concept and the following: + +- enables single claim definitions and re-use throughout source chart and all + others (external dependency charts, subcharts) within a Helm operation runtime + (e.g. a PVC for each testnet's `genesis_ledger` and each component's chain + intermediary state would be need to be set within each deployment though only + defined once in a shared named template) +- provides a single and consistent source of truth for standard PVCs throughout + Mina's Helm charts +- template definitions can be and are recommended to be namespaced by source + chart (though namespace collisions are handled with last to load taking + precedence) + +Moreover, we also propose the creation of a common Mina Helm +[library chart](https://helm.sh/docs/topics/library_charts/) to define standard +Mina PVC types in as well as to be imported by component charts dependent on +them. This libary chart would be versioned and included within the Helm CI +pipeline, as all other active Mina charts, allowing for proper linting/testing +when changed and intentional upgrades by dependencies. The following shows an +example of the PVC type named templates to be implemented and included within +this chart. + +``` +{{/* +Mina daemon wallet-keys PVC settings +*/}} +{{- define "testnet.pvc.walletKeys" }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + testnet: {{ $.Release.name }} + name: "wallet-keys-{{ $.id }}" +spec: + accessModes: + - ReadWriteMany + volumeMode: Filesystem + resources: + requests: + storage: {{ $.capacity }} + storageClassName: {{ $.Values.storage.walletKeys.storageClass }} +{{- end }} +``` + +#### Role Specific Volume Mounts + +Of the available testnet roles, there are generally both common and +role-specific persistent storage concerns which are candidates for application +of the proposed persistence solution. We attempt to enumerate all roles and +potential persistent storage applications currently of interest within this +section in order to demonstrate the extent of the integration. + +| Role | Description | Storage Requirements | +| ------------------------ | ------------------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------- | +| Archive Node | archival service tracking chain dynamics and state | `{ 1. genesis ledger, 2. daemon config, 3. runtime state, 4. postgres database state }` | +| Block Producer | active participant in a chain's block production protocol | `{ 1. genesis ledger, 2. wallet keys, 3. daemon config, 4. runtime state }` | +| Seed Node | standalone client acting as a bootstrapping and peer discovery source for network participants | `{ 1. genesis ledger, 2. daemon config, 3. runtime state }` | +| SNARK coordinator/worker | entities responsible for producing SNARK proofs used in validating chain transactions | `{ 1. genesis ledger, 2. wallet keys, 3. daemon config, 4. runtime state }` | +| User agent | (test) bot tasked with sending chain transactions as a means of fabricating network/chain dynamics | `{ 1. genesis ledger, 2. wallet keys, 3. daemon config, 4. runtime state }` | +| Faucet | (test) bot tasked with dispersing testnet funds to participants as means of enabling network activity | `{ 1. genesis ledger, 2. wallet keys, 3. daemon config, 4. runtime state }` | +| Echo Service | (test) bot tasked with echo'ing participant transactions for simulating an end-to-end transaction experience | `{ 1. genesis ledger, 2. wallet keys, 3. daemon config, 4. runtime state }` | + +##### Genesis Ledger + +_persistence type:_ common _mount path:_ +`/root/.coda-config/genesis/genesis_ledger*` _expected size:_ xx _access mode:_ +ReadOnlyMany + +##### Genesis Proof + +_persistence type:_ common _mount path:_ +`/root/.coda-config/genesis/genesis_proof*` _expected size:_ xx _access mode:_ +ReadOnlyMany + +##### Epoch Ledger + +_persistence type:_ common _mount path:_ `/root/.coda-config/epoch_ledger.json` +_expected size:_ xx _access mode:_ ReadOnlyMany + +##### Daemon Config + +_persistence type:_ common _mount path:_ `/config/daemon.json` _expected size:_ +xx _access mode:_ ReadOnlyMany + +##### Wallet Keys + +_persistence type:_ individual or common (keysets) _mount path:_ +`/root/wallet-keys` _expected size:_ _12Ki_ _access mode:_ ReadOnlyMany + +##### Runtime State + +_persistence type:_ individual _mount path:_ `/root/.coda-config/*` _expected +size:_ _2Gi_ _access mode:_ ReadWriteOnce + +##### Example Runtime (block-producer) Filehandles: + +``` +coda 11 root 7u REG 8,1 373572603 656496 /root/.coda-config/coda.log [33/881] +coda 11 root 8u REG 8,1 2800287 656427 /root/.coda-config/mina-best-tip.log +coda 11 root 9w REG 0,152 15760 1702339 /tmp/coda_cache_dir/c0e9260e-2dcc-3355-ad60-2ffc0c4bb94d/LOG +coda 11 root 10r DIR 0,152 4096 1702338 /tmp/coda_cache_dir/c0e9260e-2dcc-3355-ad60-2ffc0c4bb94d +coda 11 root 11uW REG 0,152 0 1702340 /tmp/coda_cache_dir/c0e9260e-2dcc-3355-ad60-2ffc0c4bb94d/LOCK +coda 11 root 12r DIR 0,152 4096 1702338 /tmp/coda_cache_dir/c0e9260e-2dcc-3355-ad60-2ffc0c4bb94d +coda 11 root 13w REG 0,152 32565 1702344 /tmp/coda_cache_dir/c0e9260e-2dcc-3355-ad60-2ffc0c4bb94d/000003.log +coda 11 root 18w REG 8,1 8956787 656451 /root/.coda-config/trust/000003.log +coda 11 root 19w REG 8,1 15664 656468 /root/.coda-config/receipt_chain/LOG +coda 11 root 20r DIR 8,1 4096 656467 /root/.coda-config/receipt_chain +coda 11 root 21uW REG 8,1 0 656469 /root/.coda-config/receipt_chain/LOCK +coda 11 root 23w REG 8,1 0 656473 /root/.coda-config/receipt_chain/000003.log + +coda 11 root 24w REG 8,1 15656 656476 /root/.coda-config/transaction/LOG +coda 11 root 28w REG 8,1 0 656481 /root/.coda-config/transaction/000003. + +coda 11 root 29w REG 8,1 15724 656484 /root/.coda-config/external_transition_database/LOG +coda 11 root 31uW REG 8,1 0 656485 /root/.coda-config/external_transition_database/LOCK +coda 11 root 33w REG 8,1 22792019 656489 /root/.coda-config/external_transition_database/000003.log + +coda 11 root 49w REG 8,1 692991 656520 /root/.coda-config/frontier/LOG +coda 11 root 51uW REG 8,1 0 656504 /root/.coda-config/frontier/LOCK +coda 11 root 54w REG 8,1 26641 656522 /root/.coda-config/frontier/MANIFEST-000005 +coda 11 root 543r REG 8,1 15839261 656529 /root/.coda-config/frontier/000370.sst +coda 11 root 548r REG 8,1 76433064 656537 /root/.coda-config/frontier/000365.sst +coda 11 root 552r REG 8,1 15711505 656534 /root/.coda-config/frontier/000368.sst +coda 11 root 555w REG 8,1 49954439 656521 /root/.coda-config/frontier/000371.log +coda 11 root 556r REG 8,1 187430 656542 /root/.coda-config/frontier/000366.sst +coda 11 root 558r REG 8,1 16822452 656533 /root/.coda-config/frontier/000372.sst + +coda 11 root 56w REG 8,1 16774 656508 /root/.coda-config/root/snarked_ledger/LOG +coda 11 root 58uW REG 8,1 0 656513 /root/.coda-config/root/snarked_ledger/LOCK +coda 11 root 60r REG 8,1 34928 656524 /root/.coda-config/root/snarked_ledger/000004.sst +coda 11 root 61w REG 8,1 112 656525 /root/.coda-config/root/snarked_ledger/MANIFEST-000005 +coda 11 root 62w REG 8,1 370077 656515 /root/.coda-config/root/snarked_ledger/000006.log +``` + +##### Postgres DB State + +_persistence type:_ individual _mount path:_ xx _expected size:_ xx _access +mode:_ ReadWriteOnce + +#### Labels and Organization + +As previously mentioned, each of the referenced storage persistence use-cases +would make use of the Kubernetes `PersistentVolumeClaim` resource which +represents an individual claim or reservation to a volume specification (e.g. +storage capacity, list of mount options, custom identifier/name) from an +underlying storage class. Properties of this resource are demonstrated below +though we highlight several pertaining to the organization of this +testnet(-role):`pvc` mapping. + +`pvc.metadata.name`: - identifier of the persistent volume claim +resource. Must be unique within a single namespace and referenced within a pod's +`volumes` list. e.g. _pickles-nightly-wallets-11232020_ + +`pvc.metadata.namespace`: - testnet from which a persistent volume +claim is requested. e.g. _pickles-nightly-11232020_ + +`pvc.spec.volumeName`: - custom name identifier of the volume to issue +a claim to. **Note:** Claims to volumes can be requested across namespaces +allowing for shared storage resources between testnets provided the volume's +retain policy permits retainment or rebinding. e.g. +_pickles-nightly-wallets-11232020-volume_ + +**Persistent Volume Claim (PVC) Properties:** + +``` +FIELDS: + apiVersion + kind + metadata + annotations + clusterName + labels + + name + namespace + + spec + accessModes <[]string> + dataSource + apiGroup + kind name + resources + limits + requests + selector + matchExpressions <[]Object> + key + operator + values <[]string> + matchLabels + storageClassName + volumeMode + volumeName + status + accessModes <[]string> + capacity + conditions <[]Object> + lastProbeTime + lastTransitionTime + message + reason + status + type + phase +``` + +Within each pod specification, the expectation is that each testnet +role/deployment will include additional `volume` entries within the +`pod.spec.volumes` listing indicating all `pvcs` generated within a particular +testnet to be available to containers within the pod. The additional volume +listings generally amount to slight modifications to the current `emptyDir` +volumes found throughout certain testnet role charts. So far example, the +following `yaml`: + +```yaml +- name: wallet-keys + emptyDir: {} +- name: config-dir + emptyDir: {} +``` + +would become something like: + +```yaml +- name: wallet-keys + persistentVolumeClaim: pickles-nightly-wallets-11232020 +- name: config-dir + persistentVolumeClaim: pickles-nightly-config-11232020 +``` + +Note that individual container volume mount specifications should not have to +change as they are agnostic to the underlying volume type by design. + +#### Testing + +There currently exists linting and dry-run tests executed on each Helm chart +change triggered through the Buildkite CI Mina pipeline. To ensure proper +persistence of testnet resources on deployment in practice though, we propose +leveraging Protocol's integration test framework to generate minimal +Terraform-based test cases involving persistence variable injection (to be +consumed by named templates), Helm testnet role releases and ideally Kubernetes +job resources for enacting test setup and verification steps. This could amount +to an additional configuration object within _mina_'s +`integration_test_cloud_engine` library representing job configs. Similar to how +Network_configs are defined though significantly lesser in scope, `Job_configs` +would basically amount to a pointer to a script (within source or to be +downloaded) to execute the setup and verification steps. + +## Drawbacks + +[drawbacks]: #drawbacks + +- further dependency on single cloud resource vendor (GCP) + +## Rationale and alternatives + +[rationale-and-alternatives]: #rationale-and-alternatives + +Why is this design the best in the space of possible designs? + +We believe this to be the best design in the space due to its flexibility in +application (due largely to the use of `named templates`) and simplicity of +integration into the existing infrastructure setup based on our current affinity +for `GCP` as a provider for cloud resources. + +What other designs have been considered and what is the rationale for not +choosing them? + +### Migration to other Cloud Provider K8s solutions + +Considering the current project's time boundedness, development costs to migrate +existing infrastructure setups in addition to potential increases in operational +costs and the cognitive overhead of familiarizing developers with new cloud +provider tools and user-interfaces, the benefits of migrating to another cloud +provider k8s solution (e.g. AWS EKS or Azure's AKS), which generally consist of +both potential cost + performance savings in the form of organization credits +and/or more economical and efficient resource offerings, the benefits don't +appear to outweigh the costs. Though considering the relatively portable nature +of most infrastructure components, further investigation and analysis is +suggested and planned as the project progresses. + +### Stateful Sets + +The decision away from StatefulSets comes down to flexibility (in both pod +naming scheme and persistent-volume claim binding) and avoiding the added +overhead of having to create a headless service to associate with each +StatefulSet. StatefulSets are considerably bespoke when it comes to provisioned +pod names (based on the ordinal naming scheme) and persistent-volume claim names +as well as lack the same degree of customizability that we get by merely using +templated PVCs and allowing per pod volume mounting scoped to common defaults or +overridden if desired with specific values as expressed within Values.yaml. This +allows for the scenario involving unique artifacts per testnet in addition to +cases where one would want to launch a testnet making use of other +artifacts/volumes for troubleshooting/experimentation purposes. Again the goal +here is to allow flexibility while providing an out-of-the-box solution that +works for most cases. + +The overhead of having to manually create "headless" services for each +StatefulSet isn't massive though again unnecessary and also requires that we +migrate from using Deployments with additional overhead there without much gain +and added constraints. It's probably worth noting that StatefulSets also don't +automatically clean up storage/volume resources created during provisioning - +that cleanup is left as a manual task and again speaks to the lack of reasoning +to make the switch. Really what they provide is naming/identifier stickiness +across nodes for the pod-pvc relationship though, again, PVCs are just resources +that can be bound to without really worrying about which entity is requesting +the bind or what the pods name is. + +What is the impact of not doing this? + +The impact of not implementing a persistent-storage solution such as what's +proposed would amount to the continued experience of the aforementioned +pain-points which result in considerable inefficiences and limitations when it +comes to the robustness of testnets in the event of unexpected or in some cases +expected destructive operations in addition to the ability for testnet +operations to iterate on deployments in place without losing state. + +## Prior art + +[prior-art]: #prior-art + + + +- [Refactor](https://github.com/MinaProtocol/coda-automation/issues/352) Block +Producer Helm Chart should use StatefulSet + +- Persistence investigations for + [testing various chain scenarios](https://github.com/MinaProtocol/coda-automation/issues/391) + +## Unresolved questions + +[unresolved-questions]: #unresolved-questions + +TBD diff --git a/website/docs/researchers/rfcs/0042-node-status-collection.md b/website/docs/researchers/rfcs/0042-node-status-collection.md new file mode 100644 index 000000000..039dbf836 --- /dev/null +++ b/website/docs/researchers/rfcs/0042-node-status-collection.md @@ -0,0 +1,133 @@ +--- +format: md +title: "RFC 0042: Node Status Collection" +sidebar_label: "0042 Node Status Collection" +hide_table_of_contents: false +--- + +> **Original source:** +> [0042-node-status-collection.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0042-node-status-collection.md) + +## Summary + +[summary]: #summary + +This RFC proposes a system that collects various stats related to node health to +better understand the health of the network participants. This node status +collection system would collect stats from the node and send those stats to a +url (by default those info would be send to o1labs google cloud backend). And if +those stats were sent to the o1labs backend, then those would be persisted and +be used for debugging and monitoring purpose. + +## Motivation + +[motivation]: #motivation + +In current version of the network, we saw some performance and scaling issues. +By collecting more stats from the network, we hope to improve the speed of the +mina protocol by analysize those stats. + +## Protocol between the node status collection service and o1labs backend + +[protocol]: #protocol + +1. Node submits a payload to the backend using `POST /submit` every 5 slots +2. Server saves the request +3. Server replies with `200` with `{"status": "OK"}` if the payload is valid + +## Interface design + +[interface-design]: #interface-design + +The node status collection system would make the following `post` request to the +designated url every 5 slots. + +- `POST /submit` to submit a json payload containing the following data: + +``` +{ "data": + { "peer_id": "" + , "ip_address": "" + , "mina_version": "" + , "git_hash": "" + , "timestamp": "" + , "libp2p_input_bandwidth": "" + , "libp2p_output_bandwidth": "" + , "libp2p_cpu_usage": "" + , "pubsub_msg_received": { "new_state": { "count": "" + , "bandwidth": "" } + , "transaction_pool_diff": { "count": "" + , "bandwidth": "" } + , "snark_pool_diff": { "count": "" + , "bandwidth": "" } + } + , "pubsub_msg_broadcasted": { "new_state": { "count": "" + , "bandwidth": "" } + , "transaction_pool_diff": { "count": "" + , "bandwidth": "" } + , "snark_pool_diff": { "count": "" + , "bandwidth": "" } + } + , "rpc_received": { "get_transition_chain_proof": { "count": "" + , "bandwidth": "" } + , ... + } + , "rpc_sent": { "get_transition_chain_proof": { "count": "" + , "bandwidth": "integer, bandwidth in bytes>" } + , ... + } + , "block_height_at_best_tip": "" + , "max_observed_block_height": "" + , "max_observed_unvalidated_block_height": "" + , "slot_and_epoch_number_at_best_tip": { "epoch": "" + , "slot": "" + } + , "sync_status": "" + , "catchup_job_states": { "To_initial_validate": "" + , "Finished": "" + , "To_verify": "" + , "To_download": "" + } + , "uptime_of_node": "" + , "blocks_received": [ { "hash": "" + ,"peer_id": ""} + , "received_at": "" + , "is_valid": "" + , "reason_for_rejection": "" + } +} +``` + +If the `post` request is made against o1labs' backend service, then there would +be the following possible responses: + +- `400 Bad Request` with `{"error": "//.json` + +`` is the hash of the ip address of the submitter +`` is the base64 encoding of the libp2p peer id of the submitter The +json file contains the content of the request if the request is valid diff --git a/website/docs/researchers/rfcs/0043-node-error-collection.md b/website/docs/researchers/rfcs/0043-node-error-collection.md new file mode 100644 index 000000000..b0e4f3eda --- /dev/null +++ b/website/docs/researchers/rfcs/0043-node-error-collection.md @@ -0,0 +1,107 @@ +--- +format: md +title: "RFC 0043: Node Error Collection" +sidebar_label: "0043 Node Error Collection" +hide_table_of_contents: false +--- + +> **Original source:** +> [0043-node-error-collection.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0043-node-error-collection.md) + +## Summary + +[summary]: #summary + +This RFC proposes a system that automates the collection of errors from Mina +block producer nodes. This system would send an error report when + +1. an exception occurs; +2. a fatal error occurs. By default those error report would be sent to o1labs + google cloud backend to facilitate prioritization of fixes for improving the + stability of nodes. + +## Motivation + +[motivation]: #motivation + +This system would help o1labs to identify the severity and the frequency of +errors. This would greatly facitate the discovery and analysis of the bugs. + +## Protocol between the node error collection service and o1labs backend + +[protocol]: #protocol + +1. Node submits a payload to the backend using `POST /submit` whenever an + exception or a fatal error occurs +2. Server saves the request +3. Server replies with `200` with `{"status": "OK"}` if the payload is valid + +## Interface design + +[interface-design]: #interface-design + +The node error collection system would make the following `post` request to the +designated url. + +- `POST /submit` to submit a json payload containing the following data: + +``` +{ "data": + { "peer_id": "" + , "ip_address": "" + , "public_key": "" + , "git_branch": "" + , "commit_hash": "" + , "chain_id": "" + , "contact_info": "" + , "timestamp": "" + , "level": "" + , "id": "" + , "error": "" + , "stacktrace": "" + , "cpu": "" + , "ram": "" + , "catchup_job_states": { "To_initial_validate": "" + , "Finished": "" + , "To_verify": "" + , "To_download": "" + } + , "sync_status": "" + , "block_height_at_best_tip": "" + , "max_observed_block_height": "" + , "max_observed_unvalidated_block_height": "" + , "uptime_of_node": "" + , "location": "" + } +} +``` + +If the `post` request is made against o1labs' backend service, the there would +be the following possible responses: + +- `400 Bad Request` with `{"error": "///.json` + +`` is the hash string taken from the corresponding field of the +request `` is the hash of the ip address of the submitter +`` is the base64 encoding of the libp2p peer id of the submitter The +json file contains the content of the request if the request is valid diff --git a/website/docs/researchers/rfcs/0044-node-status-and-node-error-backend.md b/website/docs/researchers/rfcs/0044-node-status-and-node-error-backend.md new file mode 100644 index 000000000..c620c5806 --- /dev/null +++ b/website/docs/researchers/rfcs/0044-node-status-and-node-error-backend.md @@ -0,0 +1,211 @@ +--- +format: md +title: "RFC 0044: Node Status And Node Error Backend" +sidebar_label: "0044 Node Status And Node Error Backend" +hide_table_of_contents: false +--- + +> **Original source:** +> [0044-node-status-and-node-error-backend.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0044-node-status-and-node-error-backend.md) + +## Summary + +[summary]: #summary + +This RFC proposes that we use Google Cloud Logging service to be the backend of +the node status/error system. The logs would be stored in the gcloud buckets; +and we could use the Log Explorer to search around logs. Metrics can also be +created against the logs. For visualization and graphing, we would use Kibana. + +## Motivation + +[motivation]: #motivation + +We need a backend for node staus/error systems. Candidates for the backend +include GCloud Logging, AWS stack and Grafana Loki and LogDNA. + +## Implementation + +[implementation]: #implementation + +### Google Cloud Logging + +Google Cloud Loggind provides storage and data searching/visualization of logs. +It provides a handy command line sdk tool. But unfortunately there's no official +support for ocaml binding. The architecture of the Google Cloud Logging is +depicted as following: + +![](https://github.com/MinaProtocol/mina-resources/blob/main/docs/res/gcloud_logging.png) + +For the frontend, each nodes would have 2 command line options to allow them to +sed node status/error reports to any specific url: `--node-status-url URL` and +`--node-error-url URL`. By default users would send their reports to our +backend. They could change the destination by providing their own destination +`URL` to the command options. Those setting could also be changed in the +daemon.json file for the corresponding field. + +We would setup micro-services under the corresponding subdomain: +https://node-status-report.minaprotocol.com and +https://node-error-report.minaprotocol.com. The micro-service would be +implemented using `Google Functions`. It already has the environment setup for +us which is very convenient. Here's the pseudo-code that demonstrates how it +would work: + +```js +exports.nodeStatus = (req, res) => { + if (req.body.payload.version != 1) { + res.status(400); + res.render("error", { error: "Version Mismatch" }); + } else if (Buffer.byteLength(req.body, "utf8") > 1000000) { + res.status(413); + res.render("error", { error: "Payload Too Large" }); + } else { + console.log(req.body); + res.end(); + } +}; +``` + +We would setup an alert that tells us if user's input is over the designated +limit (~1mb). This way we could tune the upper bound of the message size later. + +For storage of the logs, we can setup customized buckets that can be configured +to have 3650 days of log retentions. + +For visualization and plotting, logs can be passed to the elastic cloud on GCP +through Pub/Sub message sharing. We decided to hold the elastic stack by +ourselves on our k8s clusters. + +Most of the setup would be capture in a terraform modules, except the management +of secret and service accounts since it's not safe to expose those in terraform +states. + +In summary, we need to setup a micro-service for each corresponding system and +we also need to setup separate log buckets and log sinks for them. + +## Other choices + +[other-choices]: #other-choices + +### AWS Stack + +The AWS stack provides the log storage and data searching/visualization. + +![](https://github.com/MinaProtocol/mina-resources/blob/main/docs/res/aws_stack.png) + +Like the diagram shown above, we would setup a public Kenesis firehose data +stream where the mina client can push to. And this Kenesis data stream would be +connected to the S3 bucket, the OpenSearch and Kibana. (Splunk and Redshift are +potential tools that we could utilize but we would not use them in this +project.) + +We would directly push the status/error reports to AWS Kenesis firehose data +stream. There would be no server maintained by us at the backend. When writing +this RFC, the frontend of the node status collection service has already been +merged +(https://github.com/MinaProtocol/mina/blob/compatible/src/lib/node_status_service/node_status_service.ml). +In the current implementation we send the report to the url that provided by +users through `--node-status-url`. In the new design, what we would do is to let +the mina client by default send the report directly to our Kenesis data stream +with the help of ocaml-aws library. The user still have the option to send their +reports elsewhere through the `--node-status-url`. + +In summary, For the AWS stack we need to setup 1 Kenesis firehose data stream to +receive the logs, and 1 S3 storage bucket to store the logs, and 1 OpenSearch +service that provides the search ability for the logs, and 1 Kibana service that +provides the visualization for the data The communication between different +components are through Kenesis data stream, what we need to do is to setup +things correctly. + +The same setup also applies to the node error system backend. + +### Grafana Loki + +Grafana Loki functions as basically a log aggregation system for the logs. For +log storage, we could choose between different cloud storage backend like S3 or +GCS. It uses an agent to send logs to the loki server. This means we need to +setup a micro-service that listening on https://node-status.minaprotocol.com and +redirect the data to Loki. They provide a query language called LogQL which is +similar to the prometheus query language. Another upside of this choice is that +it has good integration with Grafana which is already used and loved by us. One +thing to notice is that loki is "label" based, if we want to get the most of it, +we need to find a good way to label stuff. + +### LogDNA + +LogDNA provides both data storage and data visualization and alerting +functionality. Besides the usual log collecting agent like Loki, LogDNA also +provides the option to sends logs directly to their API which could save us the +work to implement a micro-service by ourselves (depending on whether we feel +safe to give users the log-uploading keys). The alert service they provide is +also handy in node error system. + +## Prices + +0. Elastic Cloud on GCP for 0.0001/Count +1. S3, $0.023 for 1GB/month, would be a little cheaper if we use more than 50GB +2. OpenSearch, Free usage for the first 12 months for 750 hrs per month +3. Loki, depending on the storage we choose. And we need to run the loki + instance somewhere. We could choose to use the grafana cloud. But it seems to + have 30 days of log retention. The prices $49/month for 100GB of logs. (I + think we already use their service, so the log storage is already paid) +4. LogDNA, $3/GB, logs also have 30 days of retention. + +## Rationale Behind the choices + +### Rationale Behind the micro-service + +I personally think that the best option is to setup a micro-service that handles +the traffic to our selected backend. The reasons are the following: + +1. This decouples the choice of the backend from the mina client. If we ever + want to make any change to the backend, we won't need to update the client + code. +2. Hiding the choice of backend would also prevent us from exposing any + credential files/configs to the users. This is safer. +3. Having a micro-service sitting in the middle would gives us the room to add + DOS protection in the future if that's ever needed. +4. Having a micro-service would make the entire thing more decentralized in the + sense that @Jason Borseth has talked about. It means that the mina client + would always push the reports to any URL they specified (by default to us). + This is more uniform than having the mina client to push to a certain backend + in the default case or push to a URL if they choose not to send the report to + us. +5. The current implementation would just be simple bash script that redirects + the reports to the GCloud Logging API, so it's really easy to implement. If + we ever need to do any DOS protection, we could switch to a python script or + any other languages that have GCloud SDK. This gives us a lot of flexibility + for changes and upgrades. +6. Since the traffic on node error/status service won't be high in the recent + future, we won't need to worry about the scalability of this micro-service + for now. If it ever becomes a problem, we can change this design then. For + now, this design should be enough. + +### Rationale Behind the Google Cloud Logging + +The reason that we choose Google Cloud Logging fo our backend is that 0. Google +Cloud Logging meets the requirements of both node status/error system. + +1. We have most of service held by Google Cloud Platform. It's already familiar + for most of the engineers. This is the main reason that we choose it over AWS + stack + +2. For LogDNA, it has a 30 days log retention limit which clearly doesn't suit + our needs. Plus, LogDNA is much expensive than the other 2. + +3. For Grafana Loki, it features a "label"-indexed log compression system. This + would shine if the log that it process has a certain amount of static labels. + For our system, this is not the case. Plus that, none of us are familiar with + Loki. And finally Grafana's cloud system also has a 30 days' limit on log + retention. This limitation implies that if we want to use Loki then we have + to set up our own Loki service which adds some more additional maintenance + complexity. + +## The decision to make our Kibana ingress private + +We choose Kibana as our tool to visualize and search through the reports. And we +decide to make the Kibana ingress private. The reason is that the reports that +users send us contains stack traces that might contains sensitive information +that could expose vulnerability to malicious users. So we decided not to make +those reports public. But we would create some grafana charts that highlights +the stats we collected from those reports. diff --git a/website/docs/researchers/rfcs/0045-zkapp-balance-data-in-archive.md b/website/docs/researchers/rfcs/0045-zkapp-balance-data-in-archive.md new file mode 100644 index 000000000..86d8f8d94 --- /dev/null +++ b/website/docs/researchers/rfcs/0045-zkapp-balance-data-in-archive.md @@ -0,0 +1,299 @@ +--- +format: md +title: "RFC 0045: Zkapp Balance Data In Archive" +sidebar_label: "0045 Zkapp Balance Data In Archive" +hide_table_of_contents: false +--- + +> **Original source:** +> [0045-zkapp-balance-data-in-archive.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0045-zkapp-balance-data-in-archive.md) + +## Summary + +[summary]: #summary + +For zkApps, balances and account creation fees of the affected parties are not +being stored in the database. The data sent from the daemon to the archive +process does not contain such information. The archive database schema does not +have a place to store all such information, were it provided. + +Add additional information to the RPC data sent from the daemon, so that account +creation information and balances are provided for each block. Update the schema +to allow storing the information. + +Currently, balance and nonce information is stored for each transaction, which +is more fine-grained than needed. Instead, store complete account information, +including balances and nonces, for each account with a transaction in a block, +just once for that block. That should simplify both the code to store that +information, and queries for that information, as done by the Rosetta +implementation. + +We also wish to store account index information (ledger leaf order), to allow +predicting block winners from archive database information. + +## Motivation + +[motivation]: #motivation + +The archive processor has no code to store balance and account creation fee +information for zkApp transactions. The replayer needs such information in the +archive database to verify balances and nonces, and Rosetta needs it to respond +to account queries. + +## Detailed design + +[detailed-design]: #detailed-design + +### Changes to the archive database schema + +Balances are no longer per-transaction, but per-block. + +Table `blocks_internal_commands`, remove the column: + +`receiver_account_creation_fee_paid` `receiver_balance` + +Table `blocks_user_commands`, remove the columns: + +`fee_payer_account_creation_fee_paid` `receiver_account_creation_fee_paid` +`fee_payer_balance` `source_balance` `receiver_balance` + +where the array contains account creation fees for each of the `other_parties`. +While the array is not nullable, array elements may be `NULL`. + +Rename the existing `zkapp_account` to `zkapp_precondition_accounts`, and in the +table `zkapp_predicate`, rename `account_id` to `precondition_account_id`, and +modify the foreign key reference accordingly. + +Add new table `zkapp_accounts`: + +``` + app_state_id int NOT NULL REFERENCES zkapp_states(id) + verification_key_id int NOT NULL REFERENCES zkapp_verification_keys(id) + zkapp_version bigint NOT NULL + sequence_state_id int NOT NULL REFERENCES zkapp_sequence_states(id) + last_sequence_slot bigint NOT NULL + proved_state bool NOT NULL + zkapp_uri_id int NOT NULL REFERENCES zkapp_uris(id) +``` + +The new table `zkapp_uris` is: + +``` + id serial PRIMARY_KEY + uri text NOT NULL UNIQUE +``` + +The table `balances` is replaced by a new table `accounts_accessed`, with +columns: + +``` + ledger_index int NOT NULL + block_id int NOT NULL REFERENCES blocks(id) + account_id int NOT NULL REFERENCES account_ids(id) + token_symbol text NOT NULL + balance bigint NOT NULL + nonce bigint NOT NULL + receipt_chain_hash text NOT NULL + delegate int REFERENCES public_keys(id) + voting_for text NOT NULL + timing int REFERENCES timing_info(id) + permissions int NOT NULL REFERENCES zkapp_permissions(id) + zkapp int REFERENCES zkapp_accounts(id) +``` + +In order to include the genesis ledger accounts in this table, we may need a +separate app to populate it. Alternatively, we could use an app to dump the SQL +needed to populate the table, and keep that SQL in the Mina repository. + +The new table `account_ids`: + +``` + id serial PRIMARY_KEY + public_key_id int NOT NULL REFERENCES public_keys(id) + token text NOT NULL + token_owner int REFERENCES account_ids(id) +``` + +A `NULL` entry for the `token_owner` indicates that this account owns the token. + +The new table `zkapp_sequence_states` has the same definition as the existing +`zkapp_states`; it represents a vector of field elements. We probably don't want +to commingle sequence states with app states in a single table, because they +contain differing numbers of elements. + +Add a new table `accounts_created`: + +``` + block_id int NOT NULL REFERENCES blocks(id) + account_id_id int NOT NULL REFERENCES account_ids(id) + creation_fee bigint NOT NULL +``` + +There should be an entry in this table for every account, other than those in +the genesis ledger. + +Delete the unused table `zkapp_party_balances`. + +### Changes to the daemon-to-archive-process RPC + +Blocks are sent from the daemon to the archive process via an RPC named +`Send_archive_diff`. There are several kinds of messages that can be sent using +that RPC, but the one we're interested in here is the `Breadcrumb_added` +message, which contains a block. + +Currently, the per-transaction balances (for user commands and internal commands +only, not for zkApps) are contained in the transaction statuses of transactions +contained in a block. The transactions are contained in the +`Staged_ledger_diff.t` part of the block, which is an instance of the type +`External_transition.t`. + +We'll still want to send blocks using this RPC, but remove some information, and +add some other information. + +The RPC type is unversioned, because the daemon and archive process should come +from the same build. Hence, the changes proposed here don't require adding +versioning. + +Information to be removed: + +- In `Transaction.Status.t`, the `Applied` constructor is applied to a pair + consisting of `Auxiliary_data.t` containing account creation fees, and + `Balance_data.t`, containing balances. Make the constructor nullary. + +- In `Transaction.Status.t`, the `Failed` constructor is applied to a pair, + consisting of instances of `Failure.Collection.t` and `Balance_data.t`. Make + the constructor unary by omitting the second element of the pair. + +- Delete the types `Auxiliary_data.t` and `Balance_data.t`. + +- For internal commands, the types `Coinbase_balance_data.t` and + `Fee_transfer_balance_data.t` are used in the archive processor to convert the + transaction status balance data to a suitable form for those commands. Because + per-transaction balances won't be stored, delete those types. + +Information to be added: + +- The `Breadcrumb_added` message can contain a list of accounts affected by the + block. Specifically, the list contains a list of `int`, `Account.t` pairs (or + perhaps a record with two fields), where the integer is the ledger index, and + the account is its ledger state when the block is created. From this list, we + can populate the new `accounts_accessed` table, and for new accounts, the + `public_keys` table. The account information is available in the function + `Archive_lib.Diff.Builder.breadcrumb_added`. The staged ledger is contained in + the breadcrumb argument, and the block contains transactions, so the accounts + affected by the block can be queried from the staged ledger. + +- The same message can contain a list of records representing account creation + fees burned for the block, with fields for the public key of the created + account and the fee amount. That information can be extracted from the scan + state of the staged ledger in the breadcrumb. + +Removing the information from transaction statuses changes the structure of +blocks, which use a versioned type. While the version number need not be +changed, because the new type will be deployed at a hard fork, the change +affects the serialization of blocks, so a new `Bin_prot.t` layout will be needed +for `External_transition.t`. (The change in PR #10684, which removes the +validation callback from blocks (serialized as the unit value), would also +require a new layout, independently of the changes here.) + +## Changes to the archive processor + +The archive processor will need to be updated to add the account information +that's changed for each block. There is no existing code to add entries to the +`blocks_zkapps_commands` join table, it needs to be added. + +Because the `Breadcrumb_added` message will contain account creation fee +information, the processor will no longer require the temporizing hack to +calculate that information. The creation fee information will no longer be +written to join tables, instead it will be written to the new `accounts_created` +table. + +### Changes to archive blocks + +For extensional blocks (the type `Extensional.t`), a field `zkapps_cmds` needs +to be added, and the function `add_from_extensional` needs to use that field to +populate the tables `zkapp_commands` and `blocks_zkapp_commands`. Also, there +needs to be a field `accounts` with the account information used to populate the +new `accounts_accessed` table. + +Likewise, precomputed blocks (type `External_transition.Precomputed_block.t`) +will need an `accounts` field with account information. + +For both kinds of blocks, we'll also need a list of account creation fees. + +These archive block types do not contain version information in their JSON +serialization. Therefore, blocks exported with changed types will require code +using the same types, and old exported blocks will not be importable by new +code. + +## Changes to the replayer + +The replayer currently verifies balances, nonces, and account creation fees +after replaying each transaction. With the proposed changes, it can perform +those verifications after each block. + +## Changes to Rosetta + +Currently, the `account` endpoint in the Rosetta implementation makes at least +one SQL query to find a balance and a nonce. If the initial query fails, a +fallback query is made using two subqueries: at a given block height, the +balance comes from the most recent transaction (user or internal command), while +the nonce comes from the most recent user command. With the changes here, a +single query will always return both the balance and nonce, because both items +always appear in the `accounts_accessed` table. + +We'll still need to distinguish canonical from pending block heights, and in the +latter case, use the Postgresql recursion mechanism to find a path back to a +canonical block. + +Placing the account creation fees in a separate table will require changes to +the SQL queries for the `block` endpoint, which currently rely on those fees' +presence in the `blocks_internal_commands` and `blocks_user_commands` tables. + +There is no current support for zkApps in the Rosetta code, that will need to be +added. + +## Drawbacks + +[drawbacks]: #drawbacks + +Storing complete account information for each block likely increases the +database storage requirements, even though per-transaction balances will be +removed. + +Because balances and nonces are stored per-block, rather than per-transaction, +related bugs may be more difficult to difficult to rectify, when flagged by the +replayer. + +## Rationale and alternatives + +[rationale-and-alternatives]: #rationale-and-alternatives + +A simpler alternative would be to use the existing `balances` table, but without +the sequence information, rather than storing all the account information. That +would be enough to allow the replayer and Rosetta to work after zkApps are +running on a network. But that would limit some possible use cases for the +archive data. For example, with the complete account information, it's possible +to construct the ledger corresponding to any block, without running the +replayer. (We'd still want to the replayer available, to verify ledger hashes, +balances, and nonces in the archive database.) + +## Prior art + +[prior-art]: #prior-art + +The proposal here is an extension of existing mechanisms used to store and query +transaction data in an archive database. + +## Unresolved questions + +[unresolved-questions]: #unresolved-questions + +There are some minor questions posed in the text above. Besides those: + +- The new database schema has information not present in the current schema, + such as the `accounts_accessed` table. It won't be possible to write a simple + migration program, that simply shuffles around existing information, if + migration is needed. It may be possible to do a migration by modifying the + replayer, which maintains a ledger, to write out account information. Is there + a better way? diff --git a/website/docs/researchers/rfcs/0046-version-other-serializations.md b/website/docs/researchers/rfcs/0046-version-other-serializations.md new file mode 100644 index 000000000..a4b23b1a3 --- /dev/null +++ b/website/docs/researchers/rfcs/0046-version-other-serializations.md @@ -0,0 +1,144 @@ +--- +format: md +title: "RFC 0046: Version Other Serializations" +sidebar_label: "0046 Version Other Serializations" +hide_table_of_contents: false +--- + +> **Original source:** +> [0046-version-other-serializations.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0046-version-other-serializations.md) + +## Summary + +[summary]: #summary + +We can version serializations used in Mina, besides `Bin_prot` serializations, +by extending the existing versioning mechanism. + +## Motivation + +[motivation]: #motivation + +Some of the data persisted by Mina nodes will change structure from +time-to-time. For example, the structure of "precomputed blocks" will change at +the hard fork. We wish to have a mechanism to distinguish different versions of +such structures, and allow using older versions in current code. + +## Detailed design + +[detailed-design]: #detailed-design + +### Top-tagged JSON + +In RFC 0047, there is a suggestion to allow "top-tagging" of `Bin_prot` +serializations. For JSON serializations, that approach can be the default. In a +versioned module `Vn`, we would shadow the generated Yojson functions: + +```ocaml + let to_yojson item = `Assoc [("version",`Int n) + ;("data",to_yojson item) + ] + + let of_yojson json = + match json with + | `Assoc [("version",`Int version) + ;("data",data_json) + ] -> + if version = n then + of_yojson data_json + else + Error (sprintf "In JSON, expected version %d, got %d" n version) + | _ -> Error "Expected versioned JSON" +``` + +For `Bin_prot`-serialized data, we already generate: + +```ocaml + val bin_read_to_latest_opt : Bin_prot.Common.buf -> pos_ref:(int ref) -> Stable.Latest.t option +``` + +which allows reading serialized data of any version, and converting to the +latest version. (RFC 0047 proposes generating that function in a slightly +different way.) + +For JSON, we can have: + +```ocaml + val of_yojson_to_latest_opt : Yojson.Safe.t -> Stable.Latest.t Or_error.t +``` + +The returned value can indicate an error when the JSON has an invalid version, +is missing a version field, or is otherwise ill-formatted. + +We wish to generated top-tagged JSON only for selected types. In the usual case, +we do not shadow the functions generated by `deriving yojson`. When we do want +top-tagging, in the `Stable` module, we can add the annotation: + +``` + [@@@version_tag_yojson] +``` + +### Top-tagged S-expressions + +If needed, we could follow an approach to version-tag S-expressions similar to +the one proposed here for JSON. + +### Legacy precomputed blocks + +There is a cache of precomputed blocks stored in Google Cloud in JSON format, +without versioning. At the hard fork, `Precomputed_block` will have a +stable-versioned module `V2`. We'd like to add the version-tagging mechanism for +the stable version, while also being able to read older blocks, so tools like +`archive_blocks` can use them. + +In `Precomputed_block`, we can define a module: + +```ocaml + module Legacy = struct + type t = ... [@@deriving yojson] + end +``` + +where `t` uses the original definition of the precomputed block type. + +Then, in `Precomputed_block`, define: + +```ocaml + let of_yojson_legacy_or_versioned json = + match json with + | `Assoc [("version",_); ("data", _)] -> of_yojson_to_latest_opt json + | _ -> Legacy.of_yojson json +``` + +## Drawbacks + +[drawbacks]: #drawbacks + +There is a modest implementation effort required. + +## Rationale and alternatives + +[rationale-and-alternatives]: #rationale-and-alternatives + +Instead of defining the `Legacy` module for precomputed blocks, it's possible to +rewrite the cache with version tags added to the JSON. That can be done with an +automation script. Once that's done, we could treat precomputed block JSON in a +uniform way, without special handling for legacy blocks. + +## Prior art + +[prior-art]: #prior-art + +The existing versioning system is prior art, and RFC 0047 describes top-tagging +for `Bin_prot`-serialized data. + +## Unresolved questions + +[unresolved-questions]: #unresolved-questions + +Besides precomputed blocks, are there other types that would benefit from +versioning their JSON? + +Is it worth version-tagging the existing precomputed block on Google Cloud? + +Are there any use cases for versioning S-expression data? diff --git a/website/docs/researchers/rfcs/0047-versioning-changes-for-hard-fork.md b/website/docs/researchers/rfcs/0047-versioning-changes-for-hard-fork.md new file mode 100644 index 000000000..af6b92cb0 --- /dev/null +++ b/website/docs/researchers/rfcs/0047-versioning-changes-for-hard-fork.md @@ -0,0 +1,240 @@ +--- +format: md +title: "RFC 0047: Versioning Changes For Hard Fork" +sidebar_label: "0047 Versioning Changes For Hard Fork" +hide_table_of_contents: false +--- + +> **Original source:** +> [0047-versioning-changes-for-hard-fork.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0047-versioning-changes-for-hard-fork.md) + +## Summary + +[summary]: #summary + +Remove unneeded version tags from `Bin_prot`-serialized data produced by +`ppx_version`. Generate version tags on an as-needed basis where they're useful. +Detect versioning violations in the CI version linter in a more useful way. + +## Motivation + +[motivation]: #motivation + +Versioned data types are built from other versioned types and primitive OCaml +types. When generating `Bin_prot`-serialized data from instances of those type, +a version tag is generated not only for the containing type, but also for each +constituent type. That strategy allows determining the version of any piece of +the serialized data. + +The Jane Street `Async` versioned RPC mechanism used in `Mina_networking` +already provides versioning of data sent between nodes. Therefore, +version-tagging of the serialized data is not needed for that use case. Removing +the version tags will reduce the amount of data sent over the network. + +Some precomputed blocks from Mainnet show the amount of data currently used by +version tags: + +Bin_io size Number of tags Proportion + +--- + + 9956 930 9% + +1682720 121776 7% 1826633 131088 7% 1803712 129340 7% + +We may wish to save serialized data and be able to determine its version. For +that use case, a single "top" tag for the containing type suffices. + +Some user-visible data, such as public keys, are formatted using the current +serialization mechanism, where all contained types have version tags. For those +few cases, allow use of the existing mechanism. + +The CI version linter currently compares changes to versioned types against the +PR base branch. That's more fine-grained than needed, if the goal of preventing +version changes is to allow interoperability with deployed software. Instead, +compare the types against the branch of the latest released software, as well as +the base branch. That strategy will reduce the amount of noise produced by the +linter, and provide more useful results. + +## Detailed design + +[detailed-design]: #detailed-design + +### Version-tagging changes + +Currently, tagging a `Stable` module with `%versioned` produces shadowing +`bin_read_t` and `bin_write_t` functions that read and write version tags, +positive integers, prepended to serialized data, via the `ppx_version` library. +The proposal is not to shadow those functions, and instead use those generated +by `deriving bin_io`, directly. It is still useful to use `%versioned` to assure +that serializations do not change for a given versioned type. + +For backwards-compatibility, we can add the annotation + +``` + [@@@with_all_version_tags] +``` + +to the legacy `Stable.Vn` modules. For each such module, generate a module +within it: + +```ocaml + module With_all_version_tags = struct + type t = ... + + let bin_read_t = ... bin_read_t ... + let bin_write_t = ... bin_write_t ... + ... + end +``` + +where the shadowing functions are generated as they are today. For this case, +the constituent types must also have the same annotation, which is enforced by +the construction of the type `t`: the constituent types of `t` are of the form +`M.Stable.Vn.With_all_version_tags.t`. + +Within `Stable` modules, we can have the annotation + +``` + [@@@with_top_version_tag] +``` + +to generate: + +```ocaml + module With_top_version_tag = struct + type t = ... + + let bin_read_t = ... bin_read_t ... + let bin_write_t = ... bin_write_t ... + ... + end +``` + +where the shadowing functions handle a single version tag to the serializations +of outermost type instances. The constituent types need not have the annotation: +the type `t` will be identical to the type `t` in the containing `Vn` module. + +The existing versioning system generates a function in `Stable` modules: + +```ocaml + val bin_read_to_latest_opt : Bin_prot.Common.buf -> pos_ref:(int ref) -> Stable.Latest.t option +``` + +That function deserializes data by dispatching on version tags. Serializations +may or may not have version tags. If there are neither all-tagged or top-tagged +serializations, generate nothing. If there are all-tagged serializations, +generate: + +```ocaml + val bin_read_all_tagged_to_latest : Bin_prot.Common.buf -> pos_ref:(int ref) -> Stable.Latest.t Or_error.t +``` + +This function will know only about the version in the legacy `Vn` modules where +we've maintained all-tagging. + +If there are top-tagged serializations, generate: + +```ocaml + val bin_read_top_tagged_to_latest : Bin_prot.Common.buf -> pos_ref:(int ref) -> Stable.Latest.t Or_error.t +``` + +This function will know about all the versions within a `Stable` module. + +Because the default serialization does not contain version tags, it no longer +makes sense to generate such a function in `Stable`. Instead, generate it inside +`With_all_version_tags` and `With_top_version_tag`, if they're created. The +return type should be changed to an `Or_error.t` type, to describe errors, +instead of using an option type. + +### Version linting changes + +The CI version linter looks for changes to versioned types in a PR branch by +comparing those types with counterparts in the base branch. The linter will +complain if a new version is added, and later modified, even if there are no +deployed nodes using the new version. + +Instead, the linter should use both the PR base branch and a release branch for +comparison, according to this table: + +Base diff Release diff Significance Warn? + +--- + +N N No version changes N Y Y Version changed Y N Y Previous force merge N Y N +Modified new version N + +Under this regime, a new version can be added and modified as needed, without +triggering a linter warning. If a PR contains an innocuous change to a versioned +type, such as a field renaming, that can be accomplished with a force-merge, +without triggering warnings in later PRs. + +It should be enough to change the script `ci_diff_types.sh` to compare the PR +branch against the release branch, in addition to the comparison made with +`$BASE_BRANCH_NAME`. There may need to be a mechanism to automate that change +when releases become available, which is beyond the scope of this RFC. + +The version linter had been using the text of type definitions for comparisons, +and the "Binable" linter had been using the text of the uses of the `Binable` +functors. We can instead use `Bin_prot` shape digests for comparisons in both +cases. For a type `t`, the digest is available as a string: + +```ocaml + Bin_prot.Shape.eval_to_digest_string bin_shape_t +``` + +With this approach to comparisons, we can use a single linter task in CI, rather +than separate version and Binable linters. + +Using shape digests to detect changes to serializations also means we no longer +need unit tests for version-asserted types. `ppx_version` was checking for a +`For_tests` module in that case, to remind that such tests were needed; that +check can be removed. + +Shape digests will also allow detecting changes to serializations provided by +hand-written `bin_io` code. We may wish to add a module annotation like +`%versioned_custom` for that use case. If the `bin_shape_t` for a type with a +custom serialization is built with `Bin_prot.Shape.basetype`, the shape should +use a new UUID if the serialization changes (effectively, a version), for change +detection to be effective. + +## Drawbacks + +[drawbacks]: #drawbacks + +The existing versioning mechanism is working acceptably well. The only drawback +of making changes to the version-tagging is the effort to update `ppx_version`. +That should not be a major effort, since it's mostly reorganizing existing code. + +## Rationale and alternatives + +[rationale-and-alternatives]: #rationale-and-alternatives + +If these changes are not made, network performance will be less than it could +be. + +We could omit the top-tagging mechanism, since there isn't a specific use case +for it now. Adding it later is possible, if it's needed for some purpose. Then +again, adding that feature now while working on other code changes may be easier +than deferring it. + +We could phase out the all-tagged mechanism, by allowing two syntaxes for public +keys and other affected types, and eventually disallowing the old syntax. That +strategy may be too drastic to impose upon users. + +## Prior art + +[prior-art]: #prior-art + +The relevant prior art is the existing versioning mechanism. + +## Unresolved questions + +[unresolved-questions]: #unresolved-questions + +- Do we need top-tagging? +- What types will require all-tagging? Any user-visible Base58Check-encoded data + created from `Bin_prot`-serializations will need that. One way to enforce this + would be to have `Codable.Make_base58_check` take a module containing a value + `all_tagged`, which would be contained in the generated + `With_all_version_tags` modules. diff --git a/website/docs/researchers/rfcs/0048-rosetta-zkapps.md b/website/docs/researchers/rfcs/0048-rosetta-zkapps.md new file mode 100644 index 000000000..82e35b11b --- /dev/null +++ b/website/docs/researchers/rfcs/0048-rosetta-zkapps.md @@ -0,0 +1,182 @@ +--- +format: md +title: "RFC 0048: Rosetta Zkapps" +sidebar_label: "0048 Rosetta Zkapps" +hide_table_of_contents: false +--- + +> **Original source:** +> [0048-rosetta-zkapps.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0048-rosetta-zkapps.md) + +## Summary + +[summary]: #summary + +In this RFC, we describe the changes needed for Rosetta to support the upcoming +(pending community approval) Mina hardfork which contains zkApps. + +The work entails supporting the existing legacy transactions with the new +archive schema, adding support for the hardfork and zkapp transactions, and +testing. + +## Motivation + +[motivation]: #motivation + +We wish for [Rosetta](https://www.rosetta-api.org/) to support zkApps and other +changes present in the (pending community approval) Mina hardfork. + +The desired outcome is fully updating the Rosetta implementation to support this +hardfork and sufficiently test it. + +## Detailed design + +[detailed-design]: #detailed-design + +The work encompassed in this upgrading can be broken down into four sections: + +1. Supporting existing legacy transactions with the new archive schema +2. Handling the new zkApp compatible transactions +3. Supporting hardforks +4. Testing + +### Legacy Transaction Support + +The new archive schema has made a fundamental design shift to record balance +changes at the per-block level of granularity rather than the per-transaction +one. To read more about this change, see the +[associated RFC, 0045](./0045-zkapp-balance-data-in-archive.md). + +The changes needed for Rosetta to support these changes are outlined in the +[0045 RFC](./0045-zkapp-balance-data-in-archive.md) under the "Changes to +Rosetta" sub-heading. + +To summarize, for the `Account` endpoint, the SQL queries can be simplified to +merely a pending and canonical query to the new `accounts_accessed` table; we no +longer need to inspect individual transactions. + +For the `Block` endpoint, adjust the queries to not rely on the bits that are no +longer present in the existing tables, and add a new SQL query to use the +`accounts_accessed` table to pull the account-creation-fee parts. + +### Handling zkApp transactions + +zkApp Transactions will introduce many additions to our Rosetta implementation, +but in an effort to keep the scope down, there are a few areas of Rosetta that +will not support them. + +Rosetta's spec demands that it be able to track all balance changes that result +from transactions on the network; however, we don't need to be able to construct +all such balances nor do we need to track balance changes that only involve +custom tokens. In order to support custom tokens built on top of Mina we would +need to both be able to broadcast zkApp transactions and also keep track of +token changes. + +#### Construction API + +As such, custom tokens are _out of scope_ and thus the Rosetta Construction flow +will _not_ need to change. Instead, we will continue to support the existing +legacy transactions. + +#### Data API + +The [Data API](https://www.rosetta-api.org/docs/data_api_introduction.html) will +need to change to support zkApps. + +Most of the changes will be localized to supporting new zkApps related +[Rosetta operations](https://www.rosetta-api.org/docs/models/Operation.html) and +putting them in a new type of +[Rosetta transaction](https://www.rosetta-api.org/docs/models/Transaction.html). + +These new transactions will be present in both the `/block` and `/mempool` +endpoints. + +Note: GraphQL queries for zkApp transactions can be +[generated programmatically](https://github.com/MinaProtocol/mina/blob/develop/src/lib/mina_base/parties.ml#L1431) +as they are quite large. + +**Operations for zkApps** + +[Operation types](https://github.com/MinaProtocol/mina/blob/35ed5e191af9cfa2709f567f6fe85d96dabfafef/src/lib/rosetta_lib/operation_types.ml) +should be extended with new operations for all kinds of ways zkApp transactions +can manipulate account balances (for each one-sided changes). + +Luckily this is made extremely clear by the shape of the zkApps transaction +structure. + +The following intends to be an exhaustive list of those sorts of operations: + +1. `Zkapp_fee_payer_dec` +2. `Zkapp_balance_change` + +Note that only balance changes that correspond with the default MINA token +(`token_id` = 'wSHV2S4qX9jFsLjQo8r1BsMLH2ZRKsZx6EJd1sbozGPieEC4Jf' -- the +base58check representation of the Field element "one") should be considered as +changes here. + +There are many new types of failures and they're per-party in zkApps +transactions. These are enumerated in +[`transaction_status.ml`](https://github.com/MinaProtocol/mina/blob/a6e5f182855b3f4b4afb0ea8636760e618e2f7a0/src/lib/mina_base/transaction_status.ml). +They can be pulled from the `zkapp_party_failures` table in the archive +database. Since the errors are already per-party, they're easily associated with +operations on a one-to-one basis. These can be added verbatim to as +["reason metadata"](https://github.com/MinaProtocol/mina/blob/a6e5f182855b3f4b4afb0ea8636760e618e2f7a0/src/lib/rosetta_lib/user_command_info.ml#L449) +to those operations with a `Failed` label. + +### Supporting hardforks + +Changes will need to be made to `/network/status` such that the genesis block is +the first block of the hardfork network (so it's index will be greater than 1; +to be determined when we fork). This is important to be the true block height or +else the archive data queries will be incorrect. + +`/network/list` will remain unchanged as in the hardfork world the old network +"doesn't exist". + +### Testing + +#### Rosetta CLI + +Rosetta has a [cli testing tool](https://github.com/coinbase/rosetta-cli) that +must pass on this network. + +See the +[Rosetta README.md](https://github.com/MinaProtocol/mina/blob/develop/src/app/rosetta/README.md) +for information on how to run this tool. + +Ideally we can connect this CI as we've done in the past. + +#### Test-curl + +There are a series of shell scripts that can be manipulated to use the +transaction construction tool in the `src/app/rosetta` folder. These should +continue to pass: The +[README.md](https://github.com/MinaProtocol/mina/blob/develop/src/app/rosetta/README.md) +also contains information on these scripts. + +## Drawbacks + +[drawbacks]: #drawbacks + +More effort, but needed for tooling support + +## Rationale and alternatives + +[rationale-and-alternatives]: #rationale-and-alternatives + +We considered including custom tokens, but it will be extra work and not really +worth the effort at the moment as there are no widely used custom tokens (yet). + +## Prior art + +[prior-art]: #prior-art + +Our existing implementation of Rosetta and archive schemas. + +Also see the [Rosetta spec](https://www.rosetta-api.org/). + +## Unresolved questions + +[unresolved-questions]: #unresolved-questions + +None at this time. diff --git a/website/docs/researchers/rfcs/0049-protocol-testing.md b/website/docs/researchers/rfcs/0049-protocol-testing.md new file mode 100644 index 000000000..d7ff126bd --- /dev/null +++ b/website/docs/researchers/rfcs/0049-protocol-testing.md @@ -0,0 +1,356 @@ +--- +format: md +title: "RFC 0049: Protocol Testing" +sidebar_label: "0049 Protocol Testing" +hide_table_of_contents: false +--- + +> **Original source:** +> [0049-protocol-testing.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0049-protocol-testing.md) + +# TODO LIST + +- document basic actions +- document crash expectations per each DSL function +- generalized log searching with finds/filters/maps +- detail how assertions are performed at the end of tests +- determine how rolling assertions will work + +# Protocol Testing + +## Summary + +[summary]: #summary + +##### TODO + +## Motivation + +[motivation]: #motivation + +##### TODO + +As we march towards mainnet + +#### Tests are not abstract over execution environment + +The current integration test suite sits on top of a local process test harness +which it uses to create a local network of nodes on the current machine. Tests +can control and inspect these nodes by using RPCs. There is poor separation +between the integration tests and the local process test harness, meaning that +if we want to write tests that execute in a different envionment (e.g. in the +cloud instead of local), we need to write new tests for that environment. Having +our tests only support the local execution environment means that we cannot use +our tests to benchmark aspects of the protocol (e.g. bootstrapping time, block +propagation on real networks, TPS, etc...). We also can't run very large tests +since all the processes have to share the same machine (and often we choose to +run our tests without SNARKs on for this same reason). Having an abstraction +over the execution environment will allow us to not only run large, distributed +tests that we can bechmark with the same test description, but also it would +allow us to reuse our testing infrastructure for other things, such as testnet +deployment and validation. Afterall, what is a validated deployment other than a +test that keeps the network running after success? + +#### Tests are tightly coupled to build profile configuration + +Our current tests are very tightly coupled to the build configuration set in the +dune build profile associated with each test. This causes a variety of issues, +from the mild annoyance of having to keep an external script (`scripts/test.py`) +to keep track of the correct set of `profile:test` permutations, to more serious +ones that have plagued us for a while now. Most notably, the issue that has +repeatedly bitten us related to this is having difficult to detect logical +mismatches between hard coded timeouts in the integration test code and the more +realistic range of possible execution times created through a combination of +build profile configuration values (delta, slot interval, k, etc...). This has +caused everything from flakey tests (such as the infamous holy grail test, which +Jiawei recently determined a bug from after it being disabled for months), to +sudden CI blockers that get introduced to develop by small +configuration/efficiency changes to the protocol (since it can pass on the +branch where it's introduced even though it fails a majority of the time). This +can be addressed by having a more principled DSL for describing tests which +provides tools for more interactive and abstract ways to wait for things, and +avoiding any kind of hard timeouts for tests all together. + +#### Tests are difficult to debug + +##### TODO: CLEANME + +The integration tests we are using right now are very difficult to debug. Even +when they find issues, it can take a long time to identify those issues. This is +the case because errors thrown from integrations tests can often be seemingly +unrelated to the changes that cause them. Obviously, these errors eventually +make sense to us, but it takes a large amount of diagnosing and developer effort +to get there in many cases. There are a few reasons for this, some of which are +unavoidable. For instance, it is somewhat expected that there will be tracing to +do to connect error messages with root causes in something as complex as a Coda. +However, this is made more difficult by some things which are avoidable that we +can fix with some effort. + +1. Our Coda processes test harness contains many boilerplate assertions, where + as better isolated tests can make it easier to find a minimum test case that + fails to help narrow a lot of possibilities to check and track. +2. We do not have much information into what the test was doing when it failed + out of the box. It's common practice to add temporary custom logging to + identify where the test was. +3. Logs for all nodes and the test runner are all squished in a way that is + annoying to deal with and debug. Logproc makes it easier, but it is not a + commonplace tool among developers and is missing some important features to + help alleviate this. +4. Timeouts in tests result in immediate cancellation of the test. Timeouts + should fail a test, but letting a test run for a bit past a fail timeout is + helpful in filtering out tests which start failing due to decreases in + efficiency and not due to errors in the code. This is key as timeout + adjustments are a very common fix to our tests, but currently take a lot of + time to diagnose before making the decision to bump the timeout. + +#### Tests require too much internal knowledge to write + +Writing tests right now requires a lot of internal knowledge regarding the +protocol. Interaction with nodes is done over a custom RPC interface, timeouts +are raw millisecond values which require knowledge of consensus parameters, +ouroboros proof of stake, and general network delay/interaction in order to +determine the correct values, etc. The downside of this is that, while tests +should be primarily written by Protocol Engineers, other teams in the +organization, such as Product Engineers and Protocol Reliability Engineers +(PREs) should be able to write tests when necessary. For instance, Product may +want to add tests to ensure the API interacts as they expect it to under +specific protocol conditions, and the PRE team may want to write a test to +validate a bug they encountered in a testnet or to add a new validation step to +the release process. This can be addressed by using a thoughtful DSL which +focuses on abstracting the test description too a layer which requires as +minimal internal knowledge as possible. If designed well, this DSL should even +be approachable for non-OCaml developers to learn without having to learn OCaml +in too much depth first. + +## Requirements + +[requirements]: #requirements + +##### TODO: intro for reqs + +### Benchmarks + +- **TPS** +- **Bootstrap** +- Ledger Catchup +- SNARK Bubble Delay +- VRF w/ Delegation +- **Ledger Commit** +- Disk Usage + +### Tests + +- Existing +- **Bootstrap Bombardment** +- Better Bootstrap +- Better Catchup +- Persistence +- Multichain Tests (ensure different runtime parameters create incompatible + networks) +- **Partition Rejoin** +- Doomsday Recovery +- **Hard Fork** + - Protocol Upgrades + - SNARK Upgrades +- **Adversarial Testing** + - ... + +### Validation + +- sending all txn types +- all nodes are synced +- nodes can produce blocks? (if distribution allows this condition) +- network topology gut-checks +- services health checks + - api + - archive + - etc... + +### Unit Benchmarks + +- app size +- mem usage (et al) +- algorithm timings (et al) +- disk usage + +## User Stories + +[user-stories]: #user-stories + +##### TODO: rephrase in general context (talk about web app and CI auto dispatching missing tests/metermaid) + +As an example query, let's say we wanted to validate a branch that intends to +decrease the bootstrapping time of the network with relation to ledger size. We +would build a query like "give me all bootstrap measurements for develop and +\ for the configuations +`num_initial_accounts=[10k, 100k, 1m], network_size=[5, 10, 15]`". + +## Prerequisites + +[prerequisites]: #prerequisites + +- runtime configuration +- artificial neighbor population +- generalized firehose access (optional) + +## Detailed design + +[detailed-design]: #detailed-design + +### Cross-Build Measurements Storage System + +Measurements refers to the whole category of various benchmarks, metrics, and +compute properties we want to record and view from various builds of our daemon. +We will need some place to store measurements associated with different builds +and configurations which is accessible to CI and our developers (at minimum). +This storage system should support querying measurements by time, builds (git +SHAs), and configuration matrices. + +I'm not certain yet what the best tool is to use for this use case, but I +imagine that a simple cloud hosted NoSQL database such as DynamoDB would work +pretty well here. A time series database could be useful for tracking +improvements across develop over time (where the timestamp for everything is the +timestamp of the git commit, not when the test was run), but the win here seems +minimal for the added cost of obscurity. It's ok if queries on this storage +system are not super fast. + +### Unit Benchmarks + +Some of the metrics we want can be expressed as unit benchmarks, which are easy +setup and begin recording today with minimal effort. The +[benchmark runner script](https://github.com/CodaProtocol/coda/blob/develop/scripts/benchmarks.sh) +can be extended to not only run the benchmarks, but to also collect the timing +information from the output of the benchmarks and publish these numbers to the +measurement storage system. + +### Integration Test Architecture + +Below is a diagram of the proposed testing architecture, showing a high level +dataflow of the components involved in executing tests. Details are left +abstract over where the tests will be running for now as the architecture is +intended to remain the same, primarily swapping out the orchestra backend to +change testing environments. + +![](https://github.com/MinaProtocol/mina-resources/blob/main/docs/res/abstract_testing_architecture.png) + +##### Orchestrator/Orchestra + +The orchestrator is some process which allocates, monitors, and destroys nodes +in an orchestra. It provides some interface to control when and what nodes are +allocated, how to configure those nodes, and when to destroy them. During the +cycle of a single test, the test executive will register a new orchestra with +the orchestrator, which will only live so long as the test is executing. The +orchestrator will automatically clean this orchestra up when the test is +completed, unless it is told to do so earlier. Orchestrators support a number of +configurations for nodes, most of which are mapped down to the runtime +configuration fed to the daemon for that node. One important feature of the +orchestrator's node configurations is the ability to optional specify a specific +network topology for that node, which is to say, the orchestrator can control +precisely which peers a node will see as neighbors on the gossip network. + +The orchestrator does not necessarily need to be a separate process from the +test executive, but it is separate in the architecture so that custom local +process management ochestrators can be swapped out for cloud orchestrators with +no changes to tests. + +##### Test Executive + +The test executive is the primary process which initializes, coordinates, and +records the test's execution. It interprets the test specification DSL, sending +messages to various other processes in the system in order to perform the +necessary actions to run the test. It begins the test by registering a new +orchestra with the orchestrator, then spawning the necessary metrics +infrastructure and initial nodes in the test orchestra. Depending on the +specification of the test, it may send various API messages to various nodes in +the test network, or wait for certain events/conditions by subscribing to the +event streams of various node, or some combination of the two. Eventually, the +executive will terminate the test (either by failure/timeout, or by reaching an +expected terminating network state). Once this happens, the executive will +determine whether the test was successful and collect any relevant metrics for +the test by parsing through the metrics and logs for the test. Finally, the +orchestra will be torn down and the executive will record the final results for +the test to a database, where we can observe and compare test results from +multiple test runs at once. + +### Orchestra Backends + +TODO: prune first paragraph to update for Kubernetes decision + +The new integration tests have the ability to support multiple implementations +of the orchestrator which can be swapped out in place to execute the same test +description in different testing environments. The primary orechestra backend +that would be used in CI for many of the basic integration tests would be +similar to the existing test harness in that it would create all the nodes +locally on the machine. For running larger tests and tests we want to collect +measurements from, we would use a cloud based backend for spawning individual +virtual machine instances for all the nodes. One thinking on this is that we +could use Kubernetes for this, but really, we can use any tool, Kubernetes just +might save some work since it already fits many of the requirements for an +orchestrator, meaning we would just need to write a thin wrapper to set it up as +an orchestrator. In the future, we could also run more distributed tests by +having an orchestrator which communicates with multiple cloud providers in +multiple regions at once. If this is a strongly desirable capability now, it may +be worth implementing the single cloud provider backend using something like +terraform instead of Kubernetes so that we don't need to do extra work in the +future. + +Kubernetes based backend allows us to write 1 backend and run both live in the +cloud and locally through the use of Minikube. + +https://kubernetes.io/blog/2018/05/01/developing-on-kubernetes/ + +### Test Description Interface + +This section details the full scope of the design for the new test description +interface. Note that this will all be scoped down to a MVP subset of this +interface that gives us what we need immediately for our goals before mainnet. + +[Test DSL Specification](https://github.com/MinaProtocol/mina/blob/compatible/docs/test_dsl_spec.md) + +#### Requirements + +- concurrent programming support +- automated action logging +- abstract waiting facilities with soft timeouts (don't fail tests too early) +- different runtime configurations per node +- explicit node topology descriptions +- end of test log filtering and metrics collection +- errors are collected and bundled together in a useful way + +### Result Collection/Surfacing + +### Development Workflow Integration + +## Work Breakdown/Prioritization + +- generalized flesh out log processing interface +- testing DSL + - monad w/ non-fatal error accumulation and fatal error short circuiting + - spawning/destroying nodes + - interacting with nodes + - `wait_for` + - ... +- local orchestrator process backend +- cloud orchestrator backend +- convert existing tests into new DSL + +## Drawbacks + +[drawbacks]: #drawbacks + +- likely have an increased maintenence cost short term due to complexity of + moving parts +- may add additional time overhead to run tests due to docker + kubernetes + (although using docker + kubernetes allows us to decouple builds from tests) + +## Rationale and alternatives + +[rationale-and-alternatives]: #rationale-and-alternatives + +## Prior art + +[prior-art]: #prior-art + +## Unresolved questions + +[unresolved-questions]: #unresolved-questions diff --git a/website/docs/researchers/rfcs/0050-genesis-ledger-export.md b/website/docs/researchers/rfcs/0050-genesis-ledger-export.md new file mode 100644 index 000000000..29a698b80 --- /dev/null +++ b/website/docs/researchers/rfcs/0050-genesis-ledger-export.md @@ -0,0 +1,174 @@ +--- +format: md +title: "RFC 0050: Genesis Ledger Export" +sidebar_label: "0050 Genesis Ledger Export" +hide_table_of_contents: false +--- + +> **Original source:** +> [0050-genesis-ledger-export.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0050-genesis-ledger-export.md) + +## Summary + +This RFC describes the procedure to generate a genesis ledger from a running +network, using a node connected to that network. + +## Motivation + +The procedure described here is a part of the hard fork procedure, which aims at +spawning a new network, being a direct continuation of the mainnet (or any other +Mina network for that matter). To enable this, the ledger of the old network +must be exported in some form and then fed into the newly created network. +Because the new network's initial state can be fed into nodes in a configuration +file, it makes sense to generate that file directly from the old node. Then +necessary updates can be made to it manually to update various protocol +constants, and then the new configuration file can be handed over to node +operators. + +## Detailed design + +The genesis ledger export is achieved using a GraphQL field named `fork_config`. +Asking for this field requires providing a slot or a state hash of the block +that we want to base the exported ledger on. This field, if asked for, contains +a new runtime configuration, automatically updated with: + +- the dump of the **staged ledger** at the fork point +- updated values of `Fork_config`, i.e. previous state hash, previous blockchain + length and previous global slot; +- Current epoch ledger; +- Current epoch data (total currency and seed); +- Next epoch ledger; +- Next epoch data (total currency and seed); +- Protocol state at the fork point; + +**IMPORTANT**: as of now the `genesis_ledger_timestamp` is **not** being updated +and must be manually set to the right value (which is at the moment unknown). + +By the fork point above we mean the last block before the slot where no more +transactions were accepted (transaction-stop slot). + +Thus generated configuration can be saved to a file, modified if needed and fed +directly into a new node, running a different protocol version, using +`--config-file` flag. As of the moment of writing this, `compatible` and +`berkeley` branches' configuration files are compatible with each other (see: +[PR #13768](https://github.com/MinaProtocol/mina/pull/13768)). Sadly since then +that compatibility has been broken by +[PR #14014](https://github.com/MinaProtocol/mina/pull/14014). We need to either +port this change back to `compatible` or create a migration script which will +adapt a `mainnet` config file to the format required by `berkeley`. The former +solution would probably be better. + +The `fork_config` field has been added to GraphQL in +[PR #13787](https://github.com/MinaProtocol/mina/pull/13787). It needs to be +extended to return the blockchain state for a given block (height or state hash) +so that we can export the desired ledger after the blockchain has moved on. + +## Drawbacks + +This RFC provides a simple enough procedure to generate the genesis ledger for +the new network. However, it's not without its problems. + +### File size + +At the moment the mainnet has more than 100 000 accounts created. Each account +takes at least 4 lines in the configuration, which adds up to around 600kB of +JSON data. The daemon can take considerable time at startup to parse it and load +its contents into memory. If we move on with this approach, it might be +desirable to make a dedicated effort to improving the configuration parsing +speed, as these files will only grow larger in subsequent hard forks. +Alternatively, we might want to devise a better (less verbose) storage mechanism +for the genesis ledger. + +### Security concerns + +The generated genesis ledger is prone to malevolent manual modifications. Beyond +containing the hash of the previous ledger, it's unprotected from tampering +with. + +One way to improve this is to provide an external program, capable of computing +hash of the ledger as it will be after the config is loaded into a node. Users +will be able to obtain a raw fork config file from their nodes. Later, given the +official config for the new network, they will be able to run the program +against both files and compute ledger hashes. The reason why this is needed is +that the configuration file will likely contain some manual updates. For +instance the genesis ledger timestamp will need to be updated manually when the +start time of the new network is known. Further changes may concern genesis +constants and other network configuration. All these changes should be ignored +during the hash computation and only the genesis ledger itself should be taken +into consideration. This way a user seeing that the configuration file is not +identical to the one they computed, still does not contain any changes to the +genesis ledger. + +Further protection against tampering with the ledger we gain from the fact that +all the nodes must use the same one, or they'll be kicked out from the network. + +## Rationale and alternatives + +The presented way of handling the ledger export is the simplest one and the +easiest to implement. The security concern indicated above cannot be mitigated +with any method currently available. In order to overcome it, we would have to +re-think the whole procedure and somehow continue the existing network with the +changed protocol instead of creating a new one. + +It seems reasonable to export the ledger in binary form instead, but currently +the node does not persist the staged ledger in any way that could survive the +existing node and could be loaded by another one. Even if we had such a process, +the encoding of the ledger would have to be compatible between `compatible` and +`berkeley`, which could be difficult to maintain in any binary format. + +Otherwise there's no reasonable alternative to the process described. + +## Prior art + +Some of the existing blockchains, like Tezos, deal with the protocol upgrade +problem, avoiding hard-forking entirely, and therefore avoiding the ledger +export in particular. They achieve it by careful software design in which the +protocol (containing in particular the consensus mechanism and transaction +logic) consists in a plugin to the daemon, which can be loaded and unloaded at +runtime. Thus the protocol update is as simple as loading another plugin at +runtime and does not even require a node restart. + +It would certainly be beneficial to Mina to implement a similar solution, but +this is obviously a huge amount of work (involving redesigning the whole code +base), which makes it infeasible for the moment. + +## Unresolved questions + +The genesis timestamp of the new network needs to be specified in the runtime +configuration, but it is as of now (and will probably remain for some time +still) unknown. This makes it hard to put it into the configuration in any +automated fashion. Relying on personnel performing the hard fork to update it is +far from ideal, but there seems to be no better solution available at the +moment. + +Also epoch seeds from mainnet are incompatible with those on berkeley. When +epoch ledgers are being exported from a compatible node and transferred into a +berkeley node, the latter cannot load them, because Base58check fails to decode +them. This is a problem we need to overcome or decide that we won't export the +epoch ledgers and assume they're the same as the genesis ledger for the purpose +of hard fork. + +## Testing + +An automatic integration test will be written to check that the data is being +exported properly. The procedure is to start a fresh network and generate a +couple of transactions. Then the transactions are stopped. Finally the ledger +export is performed and the test compares the exported state to the current +state of the blockchain as obtained through GraphQL. These checks must take into +account the fact, that it has changed slightly since the transaction stop (a +couple additional blocks might have been produced). However, all balances should +definitely be the same (after the transaction stop no transactions are allowed, +there are no fees of coinbase rewards anymore). + +The procedure can also be tested manually as follows: + +- Sync up with the mainnet. +- Export the genesis ledger at any point in time. +- The program mentioned in a previous section can be used to verify the exported + ledger. +- Possibly add an account you control and change everyone's delegation to point + at that account so that you can produce blocks. +- Start a new network with the exported state. +- The new network should be able to produce blocks. +- All the accounts should have the same balances and delegates as on the mainnet + at the moment of export. diff --git a/website/docs/researchers/rfcs/0051-protocol-versioning.md b/website/docs/researchers/rfcs/0051-protocol-versioning.md new file mode 100644 index 000000000..2d1469f7a --- /dev/null +++ b/website/docs/researchers/rfcs/0051-protocol-versioning.md @@ -0,0 +1,116 @@ +--- +format: md +title: "RFC 0051: Protocol Versioning" +sidebar_label: "0051 Protocol Versioning" +hide_table_of_contents: false +--- + +> **Original source:** +> [0051-protocol-versioning.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0051-protocol-versioning.md) + +# Protocol Versioning + +Protocol versioning is the system by which we identify different versions of the +blockchain protocol and the software the operates it. + +## Summary + +There are multiple dimensions of compatibility between software operating a +decentralized protocol. In this RFC, we concretely breakdown those dimensions of +compatibility into a hierarchy, and then propose a semver-inspired verisioning +scheme that utilizes that hierarchy. + +## Motivation + +The motivation for this comes from a few angles. Firstly, having a versioning +scheme for the protocol itself allows developers in the ecosystem to more +accurately discuss compatibility of different software. For instance, if any 2 +separate implementations of the daemon exist, then the protocol version can +identify if those 2 implementations are compatible. Tools that process data from +the network can also identify the versions of the protocol they are compatible +with, or even dynamically support multiple versions of the protocol via +configuration. + +Besides this, we also want a way to programmatically reason about different +versions of the protocol from within our software, including having the ability +to be aware of protocol versions within the snark proofs that Mina uses. + +The solution in this RFC is optimizing for simplicity and clarity. + +## Detailed design + +At a protocol level, there are 2 important dimensions of compatibility: +compatibility of the transaction system, and compatibility of the networking +protocol. Transaction system compatibility determines support for the +transaction format and the logical details of how transactions are processed. +Compatibility of the networking protocol determines the participation rules +(consensus et al) and wire format. + +Compatibility of the transaction system can be thought of as the mapping of +ledger states into sets of all valid applicable transactions for each ledger +state. This ensures that, if we have any 2 different implementations of the +transaction logic, that those 2 implementations are only considered compatible +if they will always accept, reject, or fail transactions in an equivalent way. + +Compatibility of the networking protocol can be thought of as the set of all RPC +messages, wires types, gossip topics, p2p feature set, and participation rules. +Participation rules include any rule in the protocol that regards certain +behavior as malicious or otherwise not permitted. This ranges from consensus +details to bannable offenses and even verification logic for gossip messages. +It's important the capture all of this under the umbrella of networking protocol +compatibility since divergences in these details can lead to unintended forks on +the chain. + +To label versions of our daemon software, we will use the following versioning +convention, inspired by semver: `..`. In +this setup, the version of the transaction system is the most dominant version, +since any updates to it necessitate a hardfork, and when reasoning about the +logic of the chain state, it is the most significant version number. The prefix +of the `.` uniquely identifies a particular hard fork +version, since the pair of those 2 versions determines the full compatibility of +an implementation. The leftover `` is retained for the usage of +individual daemon implementations to use however they see fit. We leave this +detail to each implementor of the daemon, as the meaning of these versions is +intended to be specific to the implementation. + +For the existing daemon implementation in OCaml, which is maintained in the +`MinaProtocol/mina` repository, we will use the following versioning convention +for the ``: `-`. Here, the +`` will be used to denotate any breaking API changes for +user-facing APIs the daemon supports (CLI, GraphQL, Archive Format). Whenever we +add, remove, deprecate, or modify the interface of existing APIs, this version +must be incremented. The `` is used in the same way the patch +version is utilized in semver: to signify that there are new backwards +compatible bug fixes or improvements. We may also add an additional suffix to +the `` if there is a different variant of the artifact for that +version. For instance, if we were testing some optimizations behind a feature +flag, but weren't ready to ship it to the stable version of the software (or +didn't want to for some reason), then we could maintain a variant build for that +artifact by appending a `-opt` prefix to the version. + +## Drawbacks + +[drawbacks]: #drawbacks + +The main drawback of this plan is that it requires additional review when +tagging the version for a new release of the software. However, this appears to +be a necessary and an important step in the process, and hopefully will lead us +to a world where we are developing the protocol separately from the +implementation, rather than assigning a protocol version to an implementation +retroactively upon release. + +## Rationale and alternatives + +One alternative to this is following semver directly, and not specifying more +specific meanings behind the version number. This would make the versioning +scheme more standard, but it would no longer allow us to use the `` +and the `` as separate values when reasoning about their relative +compatibility in tooling (and in the snarks). + +## Prior art + +The daemon already supports a semver protocol version, but does not specify how +it should be set over time. This Berkeley hard fork is an opportunity to set the +rules for it in place. + +## Unresolved questions diff --git a/website/docs/researchers/rfcs/0052-verification-key-permissions.md b/website/docs/researchers/rfcs/0052-verification-key-permissions.md new file mode 100644 index 000000000..952a8c6a1 --- /dev/null +++ b/website/docs/researchers/rfcs/0052-verification-key-permissions.md @@ -0,0 +1,166 @@ +--- +format: md +title: "RFC 0052: Verification Key Permissions" +sidebar_label: "0052 Verification Key Permissions" +hide_table_of_contents: false +--- + +> **Original source:** +> [0052-verification-key-permissions.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0052-verification-key-permissions.md) + +# Verification Key Permissions + +This RFC describes the permission scheme for zkApps account verification keys +that will be supported at the launch of the Berkeley network. + +## Summary + +Verification keys control the proofs that can be used when interacting with a +zkApp account on-chain. Mina's zkApps allow for specifying permissions when +updating various account fields, including the verification key. Updating the +verification key is analagous to updating the code of a smart contract on other +chains. + +In this RFC, we describe a configuration for the verification key permissions +that allows developers to make contracts immutable while protecting such +contracts from deadlocking upon future upgrades of the protocol. + +## Motivation + +At the launch of the Berkeley network, the Mina Protocol does not yet guarantee +backwards compatibility of zkApps in future upgrades. Due to this, it is +possible for an immutable contract to break -- any interactions with it that +require proofs to be verified against it's verification key will no longer be +accepted by the chain. Similarly, the on-chain features that the contract relied +on may have been removed or modified. + +We wish for zkApps developers to be able to make their contract immutable +without putting their contract at risk to breaking upon a future release, +rendering any funds locked up in the contract inaccessible. + +## Detailed design + +NB: This RFC relies on the +[protocol versioning RFC](./0051-protocol-versioning.md). + +In order to prevent a zkApp from breaking upon a future incompatible upgrade of +the protocol, we will put special rules in place for the `Impossible` and +`Proof` permissions on the verification key. The verification key permission +will be represented as a tuple `(Control.t * txn_version)`, and +`Proof`/`Impossible` controllers will be reinterpreted as `Signature` if the +specified `txn_version` differs from the current protocol's transaction version. +User interfaces may provide a less-detailed representation for `None` and +`Either`, but the snark and over-the-wire protocol must accept a tuple for all +variants. + +To ensure that contracts do not become soft-locked, both the transaction logic +and the snark must only accept transactions with verification key permission +`txn_version`s equal to the current version. Any other versions must be rejected +by the transaction pool, block validation logic, and by the snark. We will +accomplish this by adding a `txn_version` check to the set of well-formedness +checks we do against transactions. + +Because setting the verification key permission requires specifying the +`txn_version`, the `txn_version` will be included in the transactions hash, +ensuring that the update cannot be replayed to 're-lock' the account to a newer, +incompatible version. + +When the `txn_version` stored in the account's verification key permission +matches the current hard fork version of the protocol, the `Impossible` and +`Proof` permissions act exactly like their normal counterparts (`Proof` fields +can only be updated with a valid proof, `Impossible` fields can never be +updated). When the `txn_version` stored within an account's verification key +permission is older than (less than) the current hard fork version, then both of +these permissions fallback to the `Signature` case, so that the now broken +zkApps can be updated for the new hard fork. + +The details for updating an old version account to a new version account are +elided in this proposal and will be determined on a per upgrade basis. As such, +we will keep the existing `zkapp_version` on accounts, storing both the +`zkapp_version` of an account and the verificatin key permission `txn_version` +separately. This separation means that the migration of an account's format +happens separately from the account's smart contract migration. The migration of +the account format can flexibly be done either on-chain, upon first interaction +with an account, or off-chain, during the hard fork package generation step (but +the decision of which route to take is left until we know what we are upgrading +to). + +## Test plan and functional requirements + +Unit tests will be written to test these new permission rules, and a new test +case will be added to the hard fork integration test. + +The unit tests are to be written against the transaction snark, testing various +account updates as inputs. The tests are broken into categories by the expected +result: that the statement is unprovable, that the statement was proven to be +failed, or that the statement was proven to be successful. + +- Unprovable account updates + - updates which set the `verification_key` permission to `Proof` or + `Impossible` with a `txn_version` other than the current hard fork version +- Failed account updates + - updates which modify the `verification_key` permission in a way that + violates the `set_permission` setting while the account's verification key + permission's `txn_version` is equal to the current hard fork version + - updates which modify the `verification_key` using `Signature` or `None` + authorizations when the `verification_key` permission is set to `Proof` or + `Impossible` while the account's verification key permission's `txn_version` + is equal to the current hard fork version + - updates which modify the `verification_key` using a `Proof` authorization + when the `verification_key` permission is set to `Impossible` while the + account's verification key permission's `txn_version` is equal to the + current hard fork version +- Successful account updates + - updates which set the `verification_key` permission to `Proof` or + `Impossible` with a `txn_version` equal to the current hard fork version, + given `set_permission` allows it + - updates that modify the `verification_key` using a `Proof` authorization + when the `verification_key` permission is set to `Proof` while the account's + verification key permission's `txn_version` is equal to the current hard + fork version + - updates that modify the `verification_key` using a `Signature` authorization + when the `verification_key` permission is set to `Proof` or `Impossible` + while the account's verification key permission's `txn_version` is less than + the current hard fork version + - updates that modify the `verification_key` permission using a `Signature` + authorization whenever the `verification_key` permission is set to `Proof` + or `Impossible` while the account's verification key permission's + `txn_version` is less than the current hard fork version (even if + `set_permission` disagrees) + +The new test cases in the hard fork integration tests will utilize 2 accounts in +the ledger which we will name A and B. The new test cases are as follows: + +- Before the hard fork + - Attempt to set any account's `verification_key` permission to `Proof` for + the wrong hard fork + - Attempt to set any account's `verification_key` permission to `Impossible` + for the wrong hard fork + - Set A's `verification_key` permission to `Proof` for the current hard fork + - Set B's `verification_key` permission to `Impossible` for the current hard + fork + - Check that you can still update A's `verification_key` using the `Proof` + authorization + - Check that neither A nor B can have their `verification_key` updated using + the `Signature` authorization +- After the hard fork + - Check that you can update both A's and B's `verification_key` field using + the `Signature` authorization + - Check that you can update both A's and B's `verification_key` permission + using the `Signature` authorization + +## Drawbacks + +[drawbacks]: #drawbacks + +## Rationale and alternatives + +## Unresolved questions + +- Should we require that any outdated permissions must be reset by the first + account update sent to it? + - At a glance, this seems to make sense, but it also seems unnecessarily + restrictive. + - DECISION: it is unnecessary to enforce this, especially given we are + tracking the verification key permission's `txn_version` separately from the + `zkapp_version` of the underlying account diff --git a/website/docs/researchers/rfcs/0053-hard-fork-package-generation.md b/website/docs/researchers/rfcs/0053-hard-fork-package-generation.md new file mode 100644 index 000000000..f84ae1254 --- /dev/null +++ b/website/docs/researchers/rfcs/0053-hard-fork-package-generation.md @@ -0,0 +1,292 @@ +--- +format: md +title: "RFC 0053: Hard Fork Package Generation" +sidebar_label: "0053 Hard Fork Package Generation" +hide_table_of_contents: false +--- + +> **Original source:** +> [0053-hard-fork-package-generation.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0053-hard-fork-package-generation.md) + +# Hard Fork Package Generation + +Generating new software packages for hard fork releases on the Mina daemon by +bundling artifacts for the new hard fork genesis state. + +## Summary + +Hard forks of Mina are implemented by restarting a new chain from a genesis +state that was captured from the prior chain. In order to facilitate this, we +need tooling for migrating the prior chain state and packaging that new genesis +state up with the new software release. Such tooling should be usable in the +case both where there is a planned hard fork, as well as the case where there is +an unplanned emergency hard fork. This RFC proposes process and tooling for +generating hard fork software packages under both scenarios (planned and +unplanned hard forks). + +## Motivation + +In order to control the data flow and parallelization of generating the hard +fork package, we utilize the Buildkite CI platform, which has great support for +specifying arbitrary jobs and dependencies between jobs, and these jobs can be +scheduled onto Builkite agents that can have any resources we need available +(such as a certain number of CPU cores or a certain amount of RAM). + +By setting up the hard fork package generation pipeline to have a simple ledger +data input, we can utilize the pipeline for generating hard fork test packages +from custom ledgers, and for generating release hard fork packages for both +planned and unplanned from historical ledgers. + +## Terms + +`Network Configuration`: the combination of compile-time mlh configuration and +pre-package runtime json configuration that determines the chain id + +`Genesis State`: all state related to the genesis of a chain (genesis block, +genesis block proof, genesis snarked ledger, genesis staged ledger, genesis +epoch ledgers (x2)) + +## Detailed design + +The general process for generating a hard fork package is outlined as follows: + +1. Capture the unmigrated genesis state from the prior chain. +2. Migrate the genesis state to the new version of the protocol. +3. Build the new version of the daemon that support the upgraded protocol. +4. Package together the new build with the migrated genesis state and the new + network configuration. +5. Test the package. +6. Release the package, announce to the community, and deploy. + +The details of step 1 change based on whether or not the hard fork is planned +(in which case this step can be automated) or is unplanned (in which case this +step is manual). Steps 2-4 can be automated (provided input from step 1). Step 5 +can also be automated, but could include manual steps if we wish to make it more +robust. Step 6 is manual and must be done in coordination with ecosystem +partners. We will not discuss the details of step 6 in the scope of this RFC. + +There is a requirement from Mina Foundation that the hard fork package can be +prepared in 6 hours or less, so all of these steps need to fit within that +timeframe. + + + +### Capturing Genesis State for Planned Hard Forks + +Under the current hard fork plan, a soft fork update will be shipped to the +prior chain which bakes in shutdown logic at specific slot heights. At some slot +height, the prior network will stop accepting transactions, and then at a +subsequent slot height after that, the network will shutdown entirely (will stop +producing/accepting/broadcasting blocks). At this point in time, we take the +strongest chain, and within that chain, take the final block before the slot +height we stopped accepting transactions at. This block is where we will dump +the genesis state from. + +Rather than waiting to dump the genesis block data at the network halt slot, we +will proactively dump every candidate hard fork genesis state, and select the +correct candidate hard fork genesis state manually once the network halt slot is +reached. There will be a special CLI flag that enables dumping the candidate +hard fork genesis block states to disk automatically, which will be enabled on +some nodes connected to the network. To ensure that we capture this data and do +not lose it, a cron job will be executed on the nodes dumping this state which +will attempt to upload any dumped candidate hard fork genesis block states to +the cloud. + +**IMPORTANT NOTE:** dumping ledgers back to back needs to not break a node (this +may require additional work). + +### Capturing Genesis State for Unplanned Hard Forks + +In the case of an unplanned (emergency) hard fork, we do not have the liberty of +allowing the daemon to automatically dump the state that we will start the new +chain from. Additionally, the state we want to take could be arbitrarily far +back in history (obviously we won't take something that is too far back in +history since we don't want to undo much history, but we can't make assumptions +about how far back we need to look). + +In such a circumstance, we must dump this state from the archive db rather than +from an actively running daemon. The replayer tool is capable of materializing +staged ledgers, snarked ledgers, and epoch ledgers for arbitrary blocks by +replaying transactions from the archive db. It is important that this tool +should take at most a couple of hours to execute, as the time spent executing +this tool is a further delay in generating a new hard fork package for the +emergency release. + +### Generating the Hard Fork Package + +The hard fork package will be generated in a buildkite pipeline. The buildkite +pipeline will accept environment variables providing URLs to download the +unmigrated genesis state from. Because different jobs require different parts of +the genesis state to run, we will split up the unmigrated genesis state inputs +into the following files: + +- `block.json` (the full protocol state of the dumped block) +- `staged_ledger.json` (the contents of the staged ledger of the dumped block) +- `next_staking_ledger.json` (the contents of the next staking ledger of the + dumped block) +- `staking_ledger.json` (the contents of the staking ledger of the dumped block) + +Here is a diagram of the various buildkite jobs required to generate the hard +fork package: + +![](https://github.com/MinaProtocol/mina-resources/blob/main/docs/res/hard-fork-package-generation-buildkite-pipeline.dot.png) + +The `build_daemon` job will build the daemon with the correct compile time +configuration for mainnet (`mainnet.mlh`). The hard fork specific configuration +will be provided via a static runtime configuration that is bundled with the +hard fork release. + +The `generate_network_config` job will generate the runtime configuration for +the hard fork network, embedding the correct ledger hashes and correct fields +for `fork_previous_length`, `fork_previous_state_hash` and +`fork_previous_global_slot`. + +- IMPORTANT: when does the genesis_timestamp get written in? has is it + configured? + +The `migrate_*_ledger` jobs will perform any necessary migrations on the input +ledgers, and will generate rocksdb ledgers that will be packaged with the hard +fork release. These jobs run in parallel. + +The `package_deb` and `package_dockerhub` jobs are our standard artifact +bundling jobs that we use for releases, with some modifications to pull in the +correct data from the hard fork build pipeline. The `package_deb` step will +bundle together that artifacts from all the prior steps into a `.deb` package. +The `package_dockerhub` installs this `.deb` package onto a Ubuntu docker image. + +In the future, we will add additional jobs to the pipeline that will also +perform automated tests against the release candidate. + +### Testing the Hard Fork Package + +In the future, we wish to automate the process of testing a hard fork release +candidate. However for the upcoming hard fork, we will instead rely on a manual +testing process in order to verify that the hard fork release candidate is of +sufficient quality to release. The goal of this testing at this layer is to +determine that the new package is configured to the proper genesis state, and +that a network can be initialized using the package. We do not perform wider +scope functional test suites at this stage, as the version of the software being +built when deploying a hard fork will have already been tested thoroughly; we +are only interested in testing workflows introduced or affected by the hard fork +package generation process. + +Ideally, we deploy a very short testnet with community members involved to +verify the hard fork package. However, this is not possible to do without +modifying the ledger, unless we were to involve the larger Mina staking +community. Instead, we will test the hard fork package by initializing seed +nodes and checking some very specific criteria. This is considered ok as our +goal here is not to test networking, but the initialization states of a daemon +when it hits the genesis timestamp, and the state we read from it once it has +fully initialized the genesis chain. + +The test plan for the seed node would go as follows: + +1. Run a seed node with a custom runtime config that sets the genesis timestamp + to be 10 minutes in the future. +2. Monitor the seed node logs to ensure that it boots up and begins sleeping + until genesis timestamp. +3. Once the seed has passed the genesis timestamp, monitor logs to ensure the + node goes through bootstrap only to pass through into participation with the + genesis block as its best tip. +4. Run a script against the node that will sample random accounts from each of + the genesis ledgers and compare the data served by the daemon against the + data in the genesis state we captured from the old chain. + +These tests will ensure that the hard fork package is setup properly to +initialize the new chain with the correct state. + +In order to enable these tests, we will need to ensure we have GraphQL or CLI +endpoints available for each of the ledgers (snarked ledger, staged ledger, and +the epoch ledgers). Then we will need to write a script that reads the ledger +files input to the hard fork package generation timeline and performs the +sampling test described in step (4). + + + +### Package Generation Time Estimates + +- `build_daemon` and `package_deb` together take about 11 minutes in CI today +- `package_dockerhub` takes about 4 minutes in CI today +- the `migrate_*_ledger` jobs take 1-2 minutes each (measured against current +mainnet ledger size) + + +## Test plan and functional requirements + +In order to ensure our hard fork generation tooling is sufficient both in the +case of a planned hard fork and an unplanned hard fork, we must perform +end-to-end tests for each workflow. Automating these tests will be important in +the long term, but it will also be a difficult challenge. As such, we will +describe only the testing requirements here, which can be applied to either +manual or automated execution of these tests. + +- Hard fork initialization + - When a daemon is configured with a genesis timestamp after the current time, + it sleeps until genesis. + - When a daemon is configured with genesis state ledger hashes only (no + accounts in config), it can successfully load those ledger from disk, + assuming they have been packaged with the software. +- Planned hard fork + - Daemons are able to dump ledger data without long async cycles or increased + rates of validation timeout errors. + - Daemons are able to dump ledgers containing up to 1 million accounts (there + is currently a bug with this due to RPC size limits) + - Infrastructure for deploying nodes is guaranteed to dump the required + genesis ledger states. + - Such a test should run a few nodes, and occasionally kill various nodes. + We want to ensure this system has high reliability. + - Buildkite pipeline is able to produce hard fork packages within the allowed + time window (6 hours) even for large ledgers. + - We will use a multiple of the current mainnet ledger size when performing + this test to ensure that the system meets requirements for future ledger + sizes that we may have to perform unplanned hard forks from. +- Unplanned hard fork + - The replayer tool is capable of materializing staged ledgers, snarked + ledgers, and epoch ledgers from arbitrary blocks. + - The replayer tool is able to materialize any ledger at an arbitrary block + within 4 hours. + +## Drawbacks + +[drawbacks]: #drawbacks + +This approach does not currently include a plan for migrating the scan state +data. This means that there will be some transaction history missing from the +final blockchain proof of the prior network. It would be an additional and +non-trivial engineering project to actually extend this proof retroactively to +cover the missing transactions that we are not migrating into the new proof +system. This leads to a weakening of Mina's trustlessness, and adds new +requirements onto clients that want to truly interact with the Mina protocol's +state in a trustless fashion. + +## Rationale and alternatives + +- ALTERNATIVE TO CLI-FLAG FOR DATA DUMPS: cron job that runs a script against + graphql api to find candidates, then dumps via CLI +- ALTERNATIVE TO PRE-DUMPING DATA: keep node alive after the network stop slot, + and dump the data at that point + - this requires putting the node into a new state, and introduces new risk + since the node could crash in that state and lose data + +## Unresolved questions + +- For emergency hard forks, do we need to consider the world in which we only + take the snarked ledger an not the staged ledger? +- Is JSON actually the best format for dumping, migrating, and loading ledgers? + It's slow to serialize/deserialize, the human readability isn't important for + this scope, and it has the potential to introduce data translation errors in + ways that other serialization formats don't. Should we just use a `bin_prot` + representation instead? diff --git a/website/docs/researchers/rfcs/0054-limit-zkapp-cmds-per-block.md b/website/docs/researchers/rfcs/0054-limit-zkapp-cmds-per-block.md new file mode 100644 index 000000000..f33b1af09 --- /dev/null +++ b/website/docs/researchers/rfcs/0054-limit-zkapp-cmds-per-block.md @@ -0,0 +1,181 @@ +--- +format: md +title: "RFC 0054: Limit Zkapp Cmds Per Block" +sidebar_label: "0054 Limit Zkapp Cmds Per Block" +hide_table_of_contents: false +--- + +> **Original source:** +> [0054-limit-zkapp-cmds-per-block.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0054-limit-zkapp-cmds-per-block.md) + +## Summary + +During the ITN stress testing it was noticed that daemon's memory consumption +tends to increase dramatically after a block containing a large number of zkApp +commands. Before appropriate optimizations can be developed, we need a temporary +solution to prevent nodes crashing due to insufficient memory. The idea is to +limit the number of zkApp commands that can be included in any single block. + +## Motivation + +By limiting the number of zkApp commands going into blocks we avoid the +aforementioned issue until a proper solution can be devised and implemented. The +root cause of the issue is that proofs contained within these commands are +stored in the scan state and tend to occupy a lot of space. Fixing these storage +issues won't affect the protocol, so ideally we want a workaround that doesn't +affect the protocol either, so that at the convenient time we can turn it off +without making a fork. + +## Detailed design + +Since the solution should not affect the protocol, it should be implemented at +the mempool/block producer boundary. In the mempool there is `transactions` +function, which returns a sequence of transactions from the mempool in the order +of decreasing transaction fees. The `create_diff` function in `Staged_ledger` +then takes that sequence and tries to apply as many transactions from it as can +fit into the block. In the latter function it is possible to simply count +successfully applied zkApp commands and filter out any transactions which: + +- would violate the set zkApp command limit +- or depend on any previously filtered transactions because of a nonce increase. + +The exact number of zkApps allowed in each block should be set dynamically, so +that we can adjust it without redeploying nodes. Therefore we are going to +provide an authorised GraphQL mutation to alter the setting at runtime. A +sensible default will be compiled into the binary as well. + +The setting can be stored in the Mina_lib configuration and initialized when the +mempool is being created at startup. The limit will also be controllable through +an authenticated GraphQL mutation, which will update the setting in the +configuration at runtime. + +## Drawbacks + +Any non-protocol-level solution to this issue has a drawback that a malicious +node operator could modify their node to turn off the safeguard. However, +because the safeguard only affects block production, it doesn't really matter +unless the malicious agent is going to produce blocks. If so, their chance of +conducting a successful DoS attack against the network is proportional to their +stake, but their incentive to do so is **inversely** proportional to their +stake, which means the more capable one is to conduct the attack, the more they +are going to lose in case of success. + +With the safeguard turned on, if the zkApps are coming in faster than they can +be processed, they will stack up in nodes' mempools. Mempools **will** +eventually overflow, which means that either some of these zkApp commands or +some regular user commands will start to drop. This will likely inflate +transaction fees as users will attempt to get their transactions into +increasingly crowded mempools. Also a lot of transactions will be lost in the +process due to mempool overflow. + +Some payments and delegations may wait a long time for inclusion or even get +dropped if they are created by the same fee payer as a zkApp command waiting for +inclusion due to the limit. This cannot be helped, unfortunately. + +Another risk arises when we decide to turn of the limitation, because the +underlying issue is fixed. In order to safely turn the limit off, a node needs +to be updated with the fix. Because this will be a non-breaking change, nodes +may be slow to adopt it. According to rough estimates, if 16% of the stake +upgrades and turns the limit off, they're capable of taking the non-upgraded +nodes down with memory over-consumption and taking over the network. To prevent +this we have to ensure that at least the majority of the stakeholder upgrades as +quickly as possible. + +Finally, the limit introduces an attack vector, where a malicious party can +submit `limit + 1` zkApp commands and arbitrarily many more commands depending +on them, so that they are guaranteed not to be included. They can set up +arbitrarily high fees on these commands which won't be included in order to kick +out other users' transactions from the mempool and increase the overall fees on +the network. An attacker would have to pay the fees for all their included zkApp +commands, but not for the skipped ones Then they can use another account to kick +out their expensive transactions form the mempool. So conducting such an attack +will still be costly, but not as costly as it should be. + +## Rationale and alternatives + +This is a temporary solution until the scan state storage can be optimised to +accommodate storing proofs more efficiently. Therefore it is more important that +it's simple and easy to implement than to solve the problem in a robust manner. +Because the issue endangers the whole network, some smaller drawbacks are +acceptable as long as the main issue is prevented from happening. + +An alternative would be to assign more precise measurement of memory occupied to +each command and limit the amount of the total memory occupied by commands +within a block. Better still, we could compute the difference in memory occupied +by the scan state before and after each block and make sure it does not go above +certain limit. This would, however, complicate the solution and require more +time to develop it, while it still wouldn't properly solve the problem. +Therefore we should strive for a quick solution which already improves the +situation and wait for the proper fix to come. + +## Prior art + +The problem of blockchain networks being unable to process incoming transactions +fast enough is a well-known one and there are several techniques of dealing with +it. + +One solution is to limit the block size (and hence indirectly the number of +transactions fitting in a single block). The most notable example here is +Bitcoin, which has a hard block size limit of 1MB. This is often criticized for +limiting the network's throughput severely, but the restriction remains in place +nonetheless, because the consequences of lifting it would be even worse. + +Mina also has its own block size limit, however, the problem we are dealing with +here is different in that we've got two distinct categories of commands, only +one of which is affected. Unfortunately, unless we move zkApp commands to a +separate mempool, any limit set on zkApp commands throughput will also affect +user commands by occupying mempool space (see Drawbacks above). + +Another solution is more related to execution time, especially that of smart +contracts, which can - in principle - run indefinitely without termination and +there is no easy way of preventing this without hindering expressiveness of a +smart contract language significantly (due to insolvability of the halting +problem). Major blockchains like Ethereum or Tezos, instead of limiting block +size directly, restrict the number of computational steps (defined by some VM +model) necessary to replay a block. A block which cannot be replayed in the +specified number of steps is automatically considered invalid. + +The operation's execution time is modelled with gas. Each atomic computation is +assigned a gas cost roughly proportional to the time the VM takes to execute +that computation. Simultaneously, a block is given a hard gas limit and the +total gas required by all the transactions within the block must be below that +limit. + +Translating this solution to the discussed problem would involve modelling +memory occupied by each operation in the scan state with some measure (analogous +to gas) and then limiting the maximum value of operations (expressed in that +measure) fitting in a block. This is a more complex solution than the one +proposed here and probably requires significant time to devise the right model. +It wouldn't also remove the problem of zkApp commands stacking in the mempool, +although it might make it less severe by setting a more fine-grained limit. +However, considering that it would still be a temporary solution, it's probably +not worth the effort. + +## Unresolved questions + +Are the drawbacks described in this document an acceptable trade-off for +preventing crashes due to out-of-memory issues? Is the alternative, more +fine-grained solution viable? + +## Testing + +The part of the code responsible for applying transactions from the mempool to +the ledger is not properly isolated from the surrounding code, but it can be +isolated and then unit-tested relatively easily. This is being done in a loop, +which gives us an opportunity to test either any single step of that loop or the +loop as a whole (ideally both). In such tests the most important properties to +check would include: + +- if the limit is disabled, zkApp commands are applied normally. +- no zkApp command is applied when the limit is reached. +- no transaction depending on a skipped zkApp command is ever applied. +- list of applied transactions contains at most the limit of zkApp commands. +- if there's less than limit of zkApp commands in the mempool, more signed + commands can be applied instead. + +Additionally an integration test checking inclusion of transactions from the +mempool in a newly created block could be written. Such a test should in +particular ensure that the limit does not affect block validation. Note that the +limit can be changed dynamically, so we can initialise a network with all nodes +having the same settings and then change it for some of them, thus examining +different configurations. diff --git a/website/docs/researchers/rfcs/0055-stop-transaction-processing.md b/website/docs/researchers/rfcs/0055-stop-transaction-processing.md new file mode 100644 index 000000000..6ede5f2ad --- /dev/null +++ b/website/docs/researchers/rfcs/0055-stop-transaction-processing.md @@ -0,0 +1,217 @@ +--- +format: md +title: "RFC 0055: Stop Transaction Processing" +sidebar_label: "0055 Stop Transaction Processing" +hide_table_of_contents: false +--- + +> **Original source:** +> [0055-stop-transaction-processing.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0055-stop-transaction-processing.md) + +# Stop processing transactions / stop the network after a certain slot + +This PR describes the feature to stop processing transactions and to stop the +network after a certain slot, to be used in the Berkeley hard fork. + +## Summary + +Transactions come from a client or the gossip network and are processed by BPs +and SNARK workers to be included in blocks. These blocks are propagated through +the network and validated by other participants. + +In this RFC, the procedure to stop processing any new transactions and to stop +the network after a certain slot is described. This is, we define two slots: the +first one is the slot after which any blocks produced will include no +transaction at all and no fee payments, and the second one is the slot after +which no blocks are produced and blocks received are rejected. + +## Motivation + +In a hard fork scenario, we want to halt the preceding network and produce a new +genesis ledger for the succeeding network. This new genesis ledger should be +produced from a stabilised staged ledger from the preceding network. This is, we +define a point in time (slot) where the network continues to operate but with no +"activity". In detail, after this slot, the network continues to produce blocks +but without including any transactions, sets coinbase fees to zero, and ensures +there are no fees for snark work, by including no snark work in the block. This +will run for a certain number of slots, after which the network will stop +producing blocks. This will allow the network to stabilise and produce a new +genesis ledger from the last ledger produced by the network. + +This feature enables part of this procedure, by adding the definition of the +slots and the mechanisms to stop the node from processing transactions and to +stop the networks after those slots. + +## Detailed design + +The procedure to stop processing transactions and producing/validating empty +blocks after a certain slot will be as follows: + +- There will be a configuration parameter set at compile-time that will define + the slot at which the node will stop processing transactions. +- The previous configuration cannot be overridable at runtime, by design, as + this compromises the safety of the daemon software. +- The node (daemon) will stop accepting new transactions from clients after the + configured slot. +- After the configured slot, the block producer will stop including transactions + in blocks, as well as any snark work and coinbase fee will be set to zero. +- The block validator will reject blocks produced after the stop slot that + contain any transaction, snark work or a non-zero coinbase fee. +- The node should start notifying the user every 60 slots (3 hours) when + transaction processing halts in less than 480 slots (24 hours). + +To stop the network after a certain slot, the procedure will be as described +next: + +- There will be a configuration parameter set at compile-time that will define + the slot at which the node will stop the network. +- After the configured slot, the block producer will stop producing any blocks. +- The block validator will reject any blocks received after the stop network +- slot. +- The node should start notifying the user every 60 slots (3 hours) when block + production/validation halts in less than 480 slots (24 hours). + +Each of these procedures will be described in detail in the following sections. + +### Compile-time configuration + +The configuration parameters `slot_tx_end` and `slot_chain_end` will be set at +compile-time and will define the slot at which the node will stop processing +transactions and the slot at which the network stops, respectively. These +configuration parameters will be optional and will default to `None`. If +`slot_tx_end` is set to `None`, the node will not stop processing transactions. +If `slot_chain_end` is set to `None`, the node will not stop producing or +validating blocks. + +### Client submits transaction + +When a client sends a transaction to the node daemon, the node will check if the +stop transaction slot configuration is set. If so, and the current global slot +is less than the configured stop slot, the transaction will be accepted by the +node and processed as usual. If the current global slot is equal or greater than +the configured stop slot, the transaction will be rejected. The client will be +notified of the rejection and the reason why the transaction was rejected. This +improves user UX by rejecting transactions that will not be included in the +ledger in the preceding network. + +This can be done by adding these checks and subsequent rejection messages to the +GraphQL functions that receive and submit user commands. + +### Block producer + +When the block producer is producing a block, it will check if the stop network +slot configuration is set. If so, and the current global slot is equal or +greater than the configured stop slot the block producer will not produce a +block. If the configured stop slot is not set or it's greater than the current +global slot, the block producer will then check if the stop transaction slot +configuration is set. If so, and the current global slot is equal or greater +than the configured stop slot the block producer will produce a block without +any transactions nor snark work and with a coinbase fee of zero. If the +configured stop slot is not set or is greater than the current global slot, the +block producer will produce blocks as usual. + +This can be done by adding these checks to block production logic. First, decide +whether or not blocks should be produced. If the stop network slot is set and +the current global slot is equal or greater than it doesn't produce blocks. If +the previous is false, return an empty staged ledger diff instead of the +generated one whenever the stop transaction slot is defined and the current +global slot is equal or greater than it, ultimately resulting in a block +produced with no transactions, no internal commands, no completed snark work, +and a coinbase fee of zero. When doing these checks, the node will also check +for the conditions to emit the info log messages at the timings and conditions +expressed earlier. + +### Block validator + +When the block validator is validating a block, it will check if the stop +network slot configuration is set. If so, and the current global slot is equal +or greater than the configured stop slot, the block validator will reject the +block. If the stop network slot is not set or is greater than the current global +slot, the block validator will then check if the stop transaction slot +configuration is set. If so, and the global slot at which the block was produced +is less than the configured stop slot, the block validator will validate the +block as usual. If the stop transaction slot configuration is not set or is +greater than the global slot of the block, the block validator will reject +blocks that define a staged ledger diff different than the empty one. + +This can be done by adding these checks to the transition handler logic. First, +reject any blocks if the stop network slot value is set and the current global +slot is greater than it. Second, and if the previous is not true, check the +staged ledger diff of the transition against the empty staged ledger diff +instead doing the usual verification process when the configured stop +transaction slot is defined and the global slot for which the block was produced +is equal or greater than it. When doing these checks, the node will also check +for the conditions to emit the info log messages at the timings and conditions +expressed earlier. + +## Test plan and functional requirements + +Integration tests will be added to test the behavior of the block producer and +the block validator. The following requirements should be tested: + +- Block producer + - When the stop network slot configuration is set to `None`, the block + producer should produce blocks. + - When the stop network slot configuration is set to `` and the current + global slot is less than ``, the block producer should produce blocks. + - When the stop network slot configuration is set to `` and the current + global slot is greater or equal to ``, the block producer should not + produce blocks. + - When the stop transaction slot configuration is set to `None`, the block + producer processes transactions, snark work and coinbase fee as usual. + - When the stop transaction slot configuration is set to `` and the + current global slot is less than ``, the block producer processes + transactions, snark work and coinbase fee as usual. + - When the stop transaction slot configuration is set to `` and the + current global slot is greater or equal to ``, the block producer + produces empty blocks (blocks with an empty staged ledger diff). +- Block validator + - When the stop network slot configuration is set to `None`, the block + validator validates blocks as usual. + - When the stop network slot configuration is set to `` and the current + global slot is less than ``, the block validator validates blocks as + usual. + - When the stop network slot configuration is set to `` and the current + global slot is greater or equal to ``, the block validator rejects all + blocks. + - When the stop transaction slot configuration is set to `None`, the block + validator validates blocks as usual. + - When the stop transaction slot configuration is set to `` and the + global slot of the block is less than ``, the block validator + validates blocks as usual. + - When the stop transaction slot configuration is set to `` and the + global slot of the block is greater or equal to ``, the block + validator rejects blocks that define a staged ledger diff differently than + the empty one. +- Node/client + - When the stop transaction slot configuration is set to `None`, the node + processes transactions from clients as usual. + - When the stop transaction slot configuration is set to `` and the + current global slot is less than ``, the node processes transactions + from clients as usual. + - When the stop transaction slot configuration is set to `` and the + current global slot is greater or equal to ``, the node rejects + transactions from clients. + +## Drawbacks + +Non-patched nodes or nodes with the configuration overridden will still be able +to send transactions to the network. These transactions will be included in the +transaction pool but will not be processed by the patched block producers, +alongside other transactions that may have arrived at the transaction pool +before the configured stop slot but haven't been included in a block as of that +slot. This will result in a transaction pool that will not be emptied until the +network stops and those transactions will not be included in the succeeding +network unless there's a mechanism to port them over to the new network to be +processed and included there. This might result in a bad UX, especially for +users who send transactions to the network before the configured stop slot and +don't see them included in the ledger and disappear from the transaction pool +when the network restarts. Moreover, non-patched nodes will produce and process +transactions as usual after the transaction stop slot, resulting in these nodes +constantly attempting to fork. + +## Rationale and alternatives + +## Prior art + +## Unresolved questions diff --git a/website/docs/researchers/rfcs/0056-hard-fork-data-migration.md b/website/docs/researchers/rfcs/0056-hard-fork-data-migration.md new file mode 100644 index 000000000..93c9d3a2d --- /dev/null +++ b/website/docs/researchers/rfcs/0056-hard-fork-data-migration.md @@ -0,0 +1,156 @@ +--- +format: md +title: "RFC 0056: Hard Fork Data Migration" +sidebar_label: "0056 Hard Fork Data Migration" +hide_table_of_contents: false +--- + +> **Original source:** +> [0056-hard-fork-data-migration.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0056-hard-fork-data-migration.md) + +## Summary + +[summary]: #summary + +This document describes a strategy for migrating mainnet archive data to a new +archive data for use at the hard fork. + +## Motivation + +[motivation]: #motivation + +We wish to have archive data available from mainnet, so that the archive +database at the hard fork contains a complete history of blocks and +transactions. + +## Detailed design + +[detailed-design]: #detailed-design + +There are significant differences between the mainnet and proposed hard fork +database schemas. Most notably, the `balances` table in the mainnet schema no +longer exists, and is replaced in the new schema with the table +`accounts_accessed`. The data in `accounts_accessed` cannot be determined +statically from the mainnet data. There is also a new table `accounts_created`, +which might be determinable statically The new schema also has the columns +`min_window_density` and `sub_window_densities` in the `blocks` table; those +columns do not exist in the `blocks` table for mainnet. + +To populate the new database, there can be two applications: + +- The first application migrates as much data as possible from the mainnet + database, and downloads precomputed blocks to get the window density data. The + `accounts_accessed` and `accounts_created` tables are not populated in this + step. This application runs against the mainnet and the new database. + +- The second application, based on the replayer app, replays the transactions in + the partially-migrated database, and populates the `accounts_accessed` and + `accounts_created` tables. This application also performs the checks performed + by the standard replayer, except that ledger hashes are not checked, because + the hard fork ledger has greater depth, which results in different hashes. + This application runs only against the new database. + +These applications can be run in sequence to get a fully-migrated database. They +should be able to work incrementally, so that part of the mainnet database can +be migrated and, as new blocks are added on mainnet, the new data in the +database can be migrated. + +To obtain that incrementality, the first application can look at the migrated +database, and determine the most recent migrated block. It can continue +migrating starting at the next block in the mainnet data. The second application +can use the checkpointing mechanism already in place for the replayer. A +checkpoint file indicates the global slot since genesis for starting the replay, +and the ledger to use for that replay. The application writes new checkpoint +files as it proceeds. + +To take advantage of such incrementality, there can be a cron job that migrates +a day's worth of data at a time (or some other interval). With the cron job in +place, at the time of the actual hard fork, only a small amount of data will +need to be migrated. + +The cron job will need Google Cloud buckets (or other storage): + +- a bucket to store migrated-so-far database dumps +- a bucket to store checkpoint files + +To prime the cron job, upload an initial database dump, and an initial +checkpoint file. Those can be created via these steps, run locally: + +- download a mainnet archive dump, and loading it into PostgreSQL +- create a new, empty database using the new archive schema +- run the first migration app against the mainnet and new databases +- run the second migration app with the `--checkpoint-interval` set to some + suitable value (perhaps 100), and starting with the original mainnet ledger in + the input file +- use `pg_dump` to dump the migrated database, upload it +- upload the most recent checkpoint file + +The cron job will perform these same steps in an automated fashion: + +- pull latest mainnet archive dump, load into PostgresQL +- pull latest migrated database, load into PostgreSQL +- pull latest checkpoint file +- run first migration app against the two databases +- run second migration app, using the downloaded checkpoint file; checkpoint + interval should be smaller (perhaps 50), because there are typically only 200 + or so blocks in a day +- upload migrated database +- upload most recent checkpoint file + +There should be monitoring of the cron job, in case there are errors. + +Just before the hard fork, the last few blocks can be migrated by running +locally: + +- download the mainnet archive data directly from the k8s PostgreSQL node, not + from the archive dump, load it into PostgreSQL +- download the most recent migrated database, load it into PostgresQL +- download the most recent checkpoint file +- run the first migration application against the two database +- run the second migration application using the most recent checkpoint file + +It is worthwhile to perform these last steps as a `dry run` to make sure all +goes well. Those steps can be run as many times as needed. + +## Drawbacks + +[drawbacks]: #drawbacks + +If we want mainnet data to be available after the hard fork, there needs to be +migration of that data. + +## Rationale and alternatives + +[rationale-and-alternatives]: #rationale-and-alternatives + +It may be possible to add or delete columns in the original schema to perform +some of the migration without transferring data between databases. It would +still be necessary to add the windowing data from precomputed blocks, and to +have a separate pass to populate the `accounts...` tables. + +## Prior art + +[prior-art]: #prior-art + +There are preliminary implementations of the two applications: + +- The first application is in branch `feature/berkeley-db-migrator`. Downloading + precomputed blocks appears to be the main bottleneck there, so those blocks + are downloaded in batches, which helps considerably. + +- The second application is in branch `feature/add-berkeley-accounts-tables`. + +There has been some local testing of these applications. + +The replayer cron jobs for mainnet, devnet, and berkeley can serve as a starting +point for the implementation of the cron job described here. + +## Unresolved questions + +[unresolved-questions]: #unresolved-questions + +The second application populates the `accounts_created` table, but the first +application could do so, by examining the `...account_creation_fee_paid` columns +of the `blocks_user_commands` table in the mainnet schema. The current +implementation relies on dynamic behavior, rather than static data, which +overcomes potential errors in that data. diff --git a/website/docs/researchers/rfcs/0057-hardcap-zkapp-commands.md b/website/docs/researchers/rfcs/0057-hardcap-zkapp-commands.md new file mode 100644 index 000000000..d8228c146 --- /dev/null +++ b/website/docs/researchers/rfcs/0057-hardcap-zkapp-commands.md @@ -0,0 +1,49 @@ +--- +format: md +title: "RFC 0057: Hardcap Zkapp Commands" +sidebar_label: "0057 Hardcap Zkapp Commands" +hide_table_of_contents: false +--- + +> **Original source:** +> [0057-hardcap-zkapp-commands.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0057-hardcap-zkapp-commands.md) + +## Summary + +Blocks containing a large number of zkApp commands have caused memory issues in +the ITN. A _soft_ solution has already been released (see +`rfcs/0054-limit-zkapp-cmds-per-block.md`) which causes a BP node to reject +zkApp transactions from its block candidate that exceed a preconfigured limit +(set on either start-up, or through an authenticated GraphQL endpoint). However, +we wish for a _hard_ solution that will cause a node to reject any incoming +block that has zkApp commands which exceed the limit. + +## Motivation + +Previously, there was a zkApp Softcap Limit that could be configured either on +start-up of the mina node, or through an authenticated GraphQL endpoint. +However, this is not safe enough as any block-producer running a node could just +recompile the code and change the configuration, circumventing the zkApp command +limit. Furthermore, the limit is _soft_ in the sense that a mina node will still +accept blocks which exceed the configured zkApp command limit. Therefore, +another mechanism is required to ensure that any block producers who attempt to +bypass the limit will not have their blocks accepted. + +## Detailed design + +The limit should be specified in the runtime config for maintainability and ease +of release. Unlike in the softcap case, the limit needs to be implemented at the +block application level, rather than the block production level, as this change +impacts non-BP mina nodes, as well. One candidate for the location is the +`create_diff` function in the `staged_ledger.ml`. There is already a +`Validate_and_apply_transactions` section in the function that could be +co-opted. + +## Testing + +A simple test would be to run two nodes in a local network, with different +configurations. Have the first node be a BP without this fix, and another be a +non-BP node with this fix (set the limit to zero). Firing an excessive amout of +zkApp command transactions at the BP node will cause it to produce a block which +exceeds the zkApp command limit. Consequently, the non-BP node should stay +constant at its initial block-height. diff --git a/website/docs/researchers/rfcs/0058-disable-zkapp-commands.md b/website/docs/researchers/rfcs/0058-disable-zkapp-commands.md new file mode 100644 index 000000000..d8e45e3bd --- /dev/null +++ b/website/docs/researchers/rfcs/0058-disable-zkapp-commands.md @@ -0,0 +1,33 @@ +--- +format: md +title: "RFC 0058: Disable Zkapp Commands" +sidebar_label: "0058 Disable Zkapp Commands" +hide_table_of_contents: false +--- + +> **Original source:** +> [0058-disable-zkapp-commands.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0058-disable-zkapp-commands.md) + +## Summary + +_Soft_ and _hard_ limits for zkApp commands have already been implemented (see +`rfcs/0054-limit-zkapp-cmds-per-block.md` and `0057-hardcap-zkapp-commands.md`). +However, both of these changes still permit the inclusion of zkApp commands into +the Mina node's mempool, and their dissemination via gossiping. If we wish to +truly disable zkApp commands in the network then a more exhaustive exclusion is +required. + +## Detailed design + +The change should sit behind a compile-time flag (similar to the ITN +`itn_features`). Changing +[this code](https://github.com/MinaProtocol/mina/blob/03c403e2c1e57a36de4e5b92f75856c825cb7e7e/src/lib/mina_base/user_command.ml#L405) +so that all zkApp commands are treated as malformed will prevent them from being +added to the mempool. + +## Testing + +The change can be tested by switching on the flag and firing zkApp commands at a +node. The node should not accept any of the zkApp commands, nor should any be +gossiped to other nodes in the network, which can be checked by querying the +GraphQL endpoints. diff --git a/website/docs/researchers/rfcs/0059-new-transaction-model.md b/website/docs/researchers/rfcs/0059-new-transaction-model.md new file mode 100644 index 000000000..d4ac25e2f --- /dev/null +++ b/website/docs/researchers/rfcs/0059-new-transaction-model.md @@ -0,0 +1,237 @@ +--- +format: md +title: "RFC 0059: New Transaction Model" +sidebar_label: "0059 New Transaction Model" +hide_table_of_contents: false +--- + +> **Original source:** +> [0059-new-transaction-model.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0059-new-transaction-model.md) + +# Redesign of the transaction execution model + +## Summary + +This proposes refactoring the transaction execution model, primarily to make it +easy to implement Snapps transactions involving an arbitrary number of parties +without having circuits that verify corresponding numbers of proofs. I.e., this +model will make it possible to e.g., have a transaction involving 10 snapp +accounts while only having a base transaction SNARK circuit that verifies a +single proof. + +## Introduction + +Currently, the transaction execution part of the protocol can be thought of as +running a state machine where the state consists of various things, roughly + +- current staged ledger +- a signed amount (the fee excess) +- the pending coinbase stack +- the next available token ID + +We propose extending this state to include an optional field called +`current_transaction_local_state` consisting of + +- a hash, called `id` +- a signed amount, called `excess` +- an optional token ID, called `current_token_id` +- a non-empty stack of "parties" (described below), called `remaining_parties` + +## Transactions + +Under this approach, a transaction would semantically be a sequence of +"parties". A "party" is a sequence of tuples +`(authorization, predicate, update)` where + +- An authorization is one of + - a proof + - a signature + - nothing +- A predicate is an encoding of a function `Account.t -> bool` +- An update is an encoding of a function `Account.t -> Account.t` + +For example, a normal payment transaction from an account at nonce `nonce` for +amount `amount` with fee `fee` would be (in pseduocaml) the sequence + +```ocaml +[ { authorization= Signature ... + ; predicate= (fun a -> a.nonce == nonce) + ; update= (fun a -> {a with balance= a.balance - (amount + fee)}) + } +; { authorization= Nothing + ; predicate= (fun _ -> true) + ; update= (fun a -> {a with balance= a.balance + amount}) + } +] +``` + +A token swap trading `n` of token `A` from `sender_A` for `m` of token `B` from +`sender_B`, plus a fee payment of `fee` from `fee_payer` would look like + +```ocaml +[ { authorization= Signature ... + ; predicate= (fun a -> a.nonce == nonce_fee_payer) + ; update= (fun a -> {a with balance= a.balance - fee}) + } +; { authorization= Signature ... + ; predicate= (fun a -> a.nonce == nonce_A) + ; update= (fun a -> {a with balance= a.balance - n}) + } +; { authorization= Nothing + ; predicate= (fun _ -> a.token_id == A && a.public_key == sender_B) + ; update= (fun a -> {a with balance= a.balance + n}) + } +; { authorization= Signature ... + ; predicate= (fun _ -> a.token_id == B) + ; update= (fun a -> {a with balance= a.balance - m}) + } +; { authorization= Nothing + ; predicate= (fun _ -> a.token_id == B && a.public_key == sender_A) + ; update= (fun a -> {a with balance= a.balance + m}) + } +] +``` + +The authorizations will be verified against the hash of the whole list of +"parties". + +When actually broadcast, transactions would be in an elaborated form containing +witness information needed to actually execute them (for example, the account_id +of each party), rather than the mere functions that constrain their execution, +but this information is not needed inside the SNARK. + +### How transaction execution would work semantically + +Currently, the transitions in our state machine are individual transactions. +This proposes extending that with the transitions + +``` +type transition = + | Transaction of Transaction.t + | Step_or_start_party_sequence of step_or_start + +type step_or_start = + | Step + | Start of party list +``` + +It remains to explain how to execute a "party" as a state transition. In +pseudocaml/snarky, it will work as follows + +```ocaml +let apply_step_or_start + (e : execution_state) (instr : step_or_start) + : execution_state + = + let local_state = + match e.current_transaction_local_state, instr with + | None, Step + | Some _, Start _ -> assert false + | None, Start ps -> + {default_local_state with parties=ps; id= hash ps} + | Some s, Step -> s + in + let {authorization; predicate; update}, remaining = + Non_empty_list.uncons local_state.remaining_parties + in + let a, merkle_path = exists ~request:Party_account in + assert (implied_root a merkle_path e.ledger_hash = e.ledger_hash) ; + assert (verify_authorization authorization a s.id) ; + assert (verify_predicate predicate a) ; + assert (auth_sufficient_given_permissions a.permissions authorization update) ; + let fee_excess_change = + match s.current_token_id with + | None -> a.token_id + | Some curr -> + if curr == a.token_id + then Currency.Fee.zero + else ( + (* If we are switching tokens, then we cannot have printed money out of thin air. *) + assert (s.excess >= 0); + if curr == Token_id.default + then s.excess + else 0 (* burn the excess of this non-default token *) + ) + in + let a' = perform_update update a in + let excess = current_transaction_local_state.excess + (a.balance - a'.balance) in + let new_ledger_hash = implied_root a' merkle_path in + match remaining with + | [] -> + assert (excess >= 0); + let fee_excess_change = + fee_excess_change + + if a.token_id == Token_id.default then excess else 0 + in + { e with current_transaction_local_state= None + ; ledger_hash= new_ledger_hash + ; fee_excess= e.fee_excess + fee_excess_change } + | _::_ -> + { e with current_transaction_local_state= + Some + { local_state with parties= remaining + ; excess= local_state.excess + excess } + ; ledger_hash= new_ledger_hash + ; fee_excess= e.fee_excess + excess_change } +``` + +### How this would boil down into "base" transaction SNARKs + +The idea would be to have 3 new base transaction SNARKs corresponding to the 3 +forms of authentication. Each would implement the above `apply_step_or_start` +function but with the `verify_authorization` specialized to either signature +verification, SNARK verification, or nothing. + +Under this model, executing a transaction works as follows. Let +`t = [t1; t2; t3]` + +### Fees and proofs required + +Instead of 2 proofs per transaction as is required now, we will switch to 2 +proofs per "party". + +Similarly, we should switch to ordering transactions in the transaction pool by +`fee / number of parties`. + +## Benefits of this approach + +The main benefit of this approach is that we can have a small number of base +circuits, each of which has at most one verifier inside of it, while still +supporting transactions containing arbitrary numbers of parties and proofs. This +enables such applications as multi-token swaps and snapp interactions involving +arbitrarily many accounts. + +Another benefit is the simplified and unified implementation for transaction +application logic (both inside and outside the SNARK). + +## Eliminating all other user-command types + +Ideally, eventually, for simplicity, we will replace the implementation of +transaction logic and transaction SNARK for the existing "special case" +transactions (of payments and stake delegations) into sequences of "parties" as +above. We can still keep the special case variants in the transaction type if +desired. + +If we do this in the most straightforward way, a payment would go from occupying +one leaf in the scan state to either 2 or 3 (if there is a separate fee payer). +However, the proofs corresponding to these leaves would be correspondingly +simpler. That said, there probably would be some efficiency loss and so if we +want to avoid that, we can make circuits that "unroll the loop" and execute +several parties per circuit. + +Specifically, for any sequence of authorization types, we can make a +corresponding circuit to execute a sequence of parties with those authorization +types. For example, it might be worth having a special circuit for the +authorization type sequence `[ Signature; None ]` for a simple payment +transaction that executes one party with a Signature authorization (the sender), +and then one with no authorization (the receiver). + +## Potential issues + +- Backwards compatibility + - Before changing the special case transactions into the above, we will have + to make sure all signers are updated as the signed payload will change. +- Transaction pool sorting + - Currently, transactions in the transaction pool are sorted by sender by + nonce. If general sequences of parties are allowed as transactions, this + will not work and we will have to figure out another way to order things. diff --git a/website/docs/researchers/rfcs/0060-networking-refactor.md b/website/docs/researchers/rfcs/0060-networking-refactor.md new file mode 100644 index 000000000..ae15ba883 --- /dev/null +++ b/website/docs/researchers/rfcs/0060-networking-refactor.md @@ -0,0 +1,621 @@ +--- +format: md +title: "RFC 0060: Networking Refactor" +sidebar_label: "0060 Networking Refactor" +hide_table_of_contents: false +--- + +> **Original source:** +> [0060-networking-refactor.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0060-networking-refactor.md) + +# Mina Networking Layer Refactor + +## Summary + +[summary]: #summary + +This RFC proposes an overhauling refactor for how our libp2p helper and daemon +processes interface. This document will cover a new IPC interface, model for +separation of concerns, and code abstraction details that should give us more +performance options and flexibility as we continue to build on top of our +existing gossip network implementation. + +NOTE: This RFC is kept abstract of IPC details related to moving towards +bitswap. Additions to this IPC design will be discussed in a separate RFC for +bitswap after this RFC is completed and agreed upon. + +## Motivation + +[motivation]: #motivation + +Over the development lifecycle of Mina, we have migrated between various gossip +network systems, and while we are now settled on libp2p as our gossip network +toolkit, we are continuing to improve the way in which we use it by utilizing +more features to optimize our network traffic and reliability. These +improvements will bring even more changes in how our existing OCaml codebase +will interact with our gossip network layer. However, at the moment, due to our +regular migrations and changes to networking, our gossip network interface +inside of OCaml is factured into 3 layers. There is quite a bit of code that is +entirely outdated. Furthermore, the protocol we use to communicate between our +Go and OCaml processes has become rather muddy, and as we have learned more +about the performance characteristics of the 2 processes, we have realized that +we need to make some serious updates to this protocol in order to prevent it +from being a bottleneck in our blockchain software. + +## Detailed design + +[detailed-design]: #detailed-design + +In order to achieve our goals, this RFC introduces an updated libp2p helper IPC +protocol and details the new abstraction/structure of the OCaml side networking +interface. + +In service of removing our existing bottlenecks around the IPC protocol, we will +be removing stream state awareness from the OCaml side of the code, preferring +to have the Go helper processes be the only one that dynamically manages streams +(including opening, closing, and reusing multiplexed streams). In this world, +the OCaml process will be fully abstracted to a request/response level of +interacting with other peers on the network. + +We will also be moving the peer management logic outside of OCaml and into the +Go helper process. This means the Go process is now responsible for seed +management, trustlist/banlist management, and even peer selection for most +requests. The OCaml process will still be the main director of peer scoring, but +will no longer manage the state of peers itself (and some of the more basic peer +scoring properties, such as overall message rate limiting, can just live on the +Go side). There will still be edge cases in which the OCaml process will +instruct the Go helper process to send requests to specific peers, but for all +requests where the OCaml process does not need a specific peer to respond (eg +bootstrap), the Go helper process will manage the selection logic for those +peers. + +NOTE: The scope of the design discussed in this document is the full refactor, +including parts which would require a hard fork to implement properly. There is +a section at the end of the design section which details how we can develop this +design in 2 stages such that we will be able to isolate the network-interface +breaking changes that would need to be shipped in a hard fork. + +### Security + +Up front, let's identify the security aspects we are aiming to achieve in this +RFC. This RFC will not cover security issues relating to rate limiting (which +will be covered by the trust scoring RFC), nor issues relating to the maximum +message size (which will be covered by the bitswap RFC). Our main security goals +we are considering in this RFC are focused around the IPC protocol between the +daemon and helper processes. Specifically, the design proposed in this RFC +intentially avoids situations in which adversarial nodes could control the +number of IPC messages sent between the daemon and the helper (independent of +the number of messages sent over the network). In other words, this design is +such that the number of IPC messages exchanged between the processes is O(1) in +relation to the number of incoming network messages (this is not true of the +existing design). In addition, this design limits synchronized state between the +2 processes, which prevents vulnerabilities in which an adversary may be able to +desynchronize state between the daemon and helper processes. + +### IPC + +The new libp2p helper IPC protocol improves upon the previous design in a number +of ways. It updates the serialization format so that there is no longer a need +to base64 encode/decode messages on each side of the protocol, and it also +replaces the singular bidirectional data stream over stdin/stdout with a system +with multiple concurrent data streams between the two processes. In order to +achieve the latter of these, the libp2p helper process will now be need to be +aware of message types for messages it receives over the network (see +[#8725](https://github.com/MinaProtocol/mina/pull/8725)). + +#### Data Streams + +In order to facilitate staging this work into both soft-fork and hard-fork +changesets, we will abstract over the concept of a "data stream" for any +unidirectional stream of communication between the helper and daemon processes. +Doing so, we can discuss the long-term communication architecture for the IPC +interface, but be able to write the code in a way such that we can easily swap +out this architecture. The code should be implemented such that it is easy to +change which messages are expected and sent over which data streams without +modifying the protocol logic itself. This allows us to implement a partial +version of the architecture until we are able to take the hard-fork changes. +Data streams should also be implemented abstract from transport mechanism, so +that we can more easily consider upgrades to our transport layer in the future +(such as supporting TCP sockets and remote helpers processes). The remainder of +this section will only focus on the long-term architecture (more details about +how this work will be broken up and staged is available in the "Staging the +Compatible and Hard Fork Changes" section of this RFC). + +#### Communication Architecture + +The helper and daemon will now exchange messages over a variety of data streams, +allowing each process to prioritize data streams differently. Correctly +optimizing this prioritization on the daemon side is important, since OCaml is +single threaded by nature (for now). In particular, the daemon needs to be able +to priotize processing and validating certain network messages in order to +ensure that the node keeps in sync with the network and forwards relevant +information for others to stay in sync. Specifically, the daemon needs to +prioritize processing and validating new block gossips so that they can be +forwarded to other nodes on the network in a timely manor. + +The transport layer we will use for these data streams will be Unix pipes. The +parent daemon process can create all the required Unix pipes for the various +data streams, and pass the correct file descriptors (either the write or read +descriptors depending on the direction of the pipe) to the child helper process +when it initializes. Pipes are considered preferable over shared memory for the +data streams since they already provide synchronization primitives for +reading/writing and are easier to implement correctly, though shared memory +would be likely be slightly more optimized. + +Below is a proposed set of data streams we would setup for the helper IPC +protocol. Keep in mind that some of these pipes require some form of message +type awareness in order to be implemented. We have ongoing work that adds +message type awareness to the helper process, but this work requires a hard +fork. If we want to split up message-specific pipes before a hard fork, we would +need to add support for message peeking to the helper (which would involve +making the helper aware of at least part of the encoding format for RPC +messages). + +- stdin (used only for initialization message, then closed) +- stdout (used only for helper logging) +- stderr (used only for helper logging) +- stats_in (publishes helper stats to daemon on an interval) +- block_gossip_in (incoming block gossip messages) +- mempool_gossip_in (other incoming gossip messages, related to mempool state) +- response_in (incoming RPC responses) +- request_in (incoming RPC requests) +- validation_out (all validations except request validations, which are bundled + with responses) +- response_out (outgoing RPC responses) +- broadcast_out (outgoing broadcast messages) + +The rough priorities for reading the incoming pipes from the daemon process +would be: + +- block_gossip_in +- response_in +- request_in +- mempool_gossip_in + +NOTE: It is critical in the implementation of this that the prioritization +scheme we choose here does not allow the mempool gossip pipe to be starved. The +main thing to keep in mind to avoid this is to ensure that we do not over weight +reading the incoming requests, so that another node on the network cannot delay +(or potentially censor) txns and snarks we are receiving over gossip. One +approach towards this could be to limit the parallelism per pipe while keeping +the maximum parallel messages we handle from IPC high enough such that we can +always schedule new mempool gossip jobs even when there are a lot of requests. + +CONSIDER: Is it important that IPC messages include timestamps so that the +daemon and helper processes can perform staleness checks as they read messages? +For example: if we haven't read a mempool gossip in a while, and read one, +discovering that the first message on the pipe is rather old, should we further +prioritize this pipe for a bit until we catchup? A potential risk of this system +is that it would be hard to guarantee that none of the data streams aren't +succeptible to starvation attacks. + +#### Serialization Format + +The new serialization format will be [Cap'N Proto](https://capnproto.org/). +There are already [Go](https://github.com/capnproto/go-capnproto2) and +[OCaml](https://github.com/capnproto/capnp-ocaml) libraries for the Cap'N Proto +serialization format, which generate code for each language based on a common +schema definition of the protocol. Using Cap'N Proto instead of JSON will allow +us to embed raw binary data in our IPC messages, which will avoid the rather +costly and constant base64 encoding/decoding we currently do for all binary data +we transfer between the processes. It's possible to keep some pipes in JSON if +preferable, but within the current plan, all messages would be converted over to +Cap'N Proto to avoid having to support tooling for keeping both serialization +formats in sync between the processes. + +NOTE: The [OCaml Cap'N Proto library](https://github.com/capnproto/capnp-ocaml) +currently has an inefficient way of handling binary data embeded in Cap'N Proto +messages. It uses `Bytes.t` as the backing type for the packed data, and +`string` as the type for representing the unpacked data. @mrmr1993 pointed out +in the RFC review that we would save 2 memory copies if we used `Bigstring.t` as +the backing type for packed data, and slices of that `Bigstring.t` for the +unpacked data. These changes are fairly straightforward to make, and can be done +in a fork of the library we maintain. + +#### Entrypoints + +The new libp2p helper interface would support separate entrypoints for specific +libp2p tasks, which will simplify some of the IPC interface by removing one-off +RPC calls from OCaml to Go. Now, there will be 3 entrypoints, 2 of which will +briefly run some computation and exit the process with a result over stdout, and +the last of which starts the actual helper process we will use to connect to the +network. These interfaces will be accessed directly via CLI arguments rather +than being triggered by IPC messages. In otherwords, the IPC system is only +active when the helper is run in `gossip_network` mode. + +The supported entrypoints will be: + +- `generate_keypair` +- `validate_keypair --keypair={keypair}` +- `gossip_network` + +#### Protocol + +When the lip2p helper process is first started by the Daemon (in +`gossip_network` mode), the daemon will send an `init(config Config)` is written +once over stdin. The information sent in this message could theoretically be +passed via the CLI arguments, but doing this would lose some type safety, so we +prefer to send this data over as an IPC message. Once this message has been +received by the helper process, the helper process will open ports, join the +network, and begin participating in the main protocol loop. In this main +protocol loop, either process should expect to receive any IPC messages over any +data streams at any time. + +Here is a list of the IPC messages that will be supported in each direction, +along with some relevant type definitions in Go: + +```txt +Daemon -> Helper + // it's possible to just remove this message as it is just a specialized case of `sendRequests` + sendRequestToPeer(requestId RequestId, to Peer, msgType MsgType, rawData []byte) + sendRequests(requestId RequestId, to AbstractPeerGraph, msgType MsgType, rawData []byte) + sendResponse(requestId RequestId, status ValidationStatus, rawData []byte) + broadcast(msgType MsgType, rawData []byte) + validate(validation ValidationHandle, status ValidationStatus) + +Helper -> Daemon + handleRequest(requestId RequestId, from Peer, rawData []byte) + handleResponse(requestId RequestId, validation ValidationHandle, rawData []byte) + handleGossip(from Peer, validation ValidationHandle, rawData []byte) + stats(stats Stats) +``` + +```go +// == The `Config` struct is sent with the `init` message at the start of the protocol. + +// The following old fields have been completely removed: +// - `metricsPort` (moving over to push-based stats syncing, where we will sync any metrics we want to expose) +// - `unsafeNotTrustIp` (only used as a hack in old integration test framework; having it makes p2p code harder to reason about) +// - `gaterConfig` (moving towards more abstracted interface in which Go manages gating state data) +type Config struct { + networkId string // unique network identifier + privateKey string // libp2p id private key + stateDirectory string // directory to store state in (peerstore and dht will be stored/loaded from here) + listenOn []Multiaddr // interfaces we listen on + externalAddr Multiaddr // interface we advertise for other nodes to connect to + floodGossip bool // enables gossip flooding (should only be turned on for protected nodes hidden behind a sentry node) + directPeers []Multiaddr // forces the node to maintain connections with peers in this list (typically only used for sentry node setups and other specific networking scenarios; these peers are automatically trustlisted) + seedPeers []Multiaddr // list of seed peers to connect to initially (seeds are automatically trustlisted) + maxConnections int // maximum number of connections allowed before the connection manager begins trimming open connections + validationQueueSize int // size of the queue of active pending validation messages + // TODO: peerExchange bool vs minaPeerExchange bool (seems like at least one of these should be deprecated) + // - peerExchange == enable libp2p's concept of peer exchange in the pubsub options + // - minaPeerExchange == write random peers to connecting peers +} + +// == The `Stats` struct is sent on an interval via the `stats` message. +// == It contains metrics and statistics relevant to the helper process, +// == to be further exposed by the daemon as prometheus metrics. + +type MinMaxAvg struct { + min float64 + max float64 + avg float64 +} + +type InOut struct { + in float64 + out float64 +} + +type Stats struct { + storedPeerCount int + connectedPeerCount int + messageSize MinMaxAvg + latency MinMaxAvg + totalBandwidthUsage InOut + totalBandwidthRate InOut +} + +// == A `ValidationStatus` notifies the helper process of whether or not +// == a message was valid (or relevant). + +type ValidationStatus int +const ( + VALIDATION_ACCEPT ValidationStatus = iota + VALIDATION_REJECT + VALIDATION_IGNORE +) + +// == These types define the concept of an `AbstractPeerGraph`, which +// == describes a peer traversal algorithm for the helper to perform +// == as when finding a successful response to an RPC query. + +// Alternative (safer) representations are possible, but this is +// simplest to encode in the Cap'N Proto shared schema language. + +// Each node of the graph either allows any peer to query, or it +// identifies a specific node to query.. +type AbstractPeerType int +const ( + ANY_PEER AbstractPeerType = iota + SPECIFIC_PEER +) + +// This is essentially an ADT, but we cannot encode an ADT directly +// in Go (though we can use tagged unions when we describe this in +// Cap'N Proto). An equivalent ADT definition would be: +// type abstract_peer_node = +// | AnyPeer +// | SpecificPeer of Peer +type AbstractPeerNode struct { + typ AbstractPeerType + peer Peer // nil unless type == SPECIFIC_PEER +} + +type AbstractPeerEdge struct { + src int + dst int +} + +// A graph is interpreted by starting (in parallel) at the source +// nodes. When a node is interpreted, a request is sent to the peer +// identified by the node. If the request for a node fails, then the +// algorithm begins interpreting the successors of that node (also in +// parallel). Interpretation halts when either a single request is +// successful, or all requests fail after traversing the entire graph. +type AbstractPeerGraph struct { + sources []int + nodes []AbstractPeerNode + edges []AbstractPeerEdge +} +``` + +#### Query Control Flow + +In contrast to the prior implementation, the query control flow in the new +protocol always follows the following pattern: + +1. The daemon sends 1 message to the helper to begin the query (this message may + instruct the helper to begin sending out 1 or more requests, with control + over maximum parallelism). +2. The helper continuously and concurrently runs the following protocol until a + successful response is found: 2.a) The helper picks a peer it has not already + queried based on the daemon's request and sends a request to this peer. 2.b) + The helper streams the response back to the daemon. 2.c) The daemon sends a + validation callback to the helper. + +Keeping the peer selection logic on the helper side allows the daemon to avoid +asking the helper for peers before it sends the request. Since the trust scoring +state is also already on the helper process, the helper can also select peers +based on their score (more details on this to come in the trust scoring RFC). +The daemon can still instruct the helper process to query specifc peers, in +which cases the daemon will already know of the specific peer and will not need +to ask the helper for any additional information. + +#### Validation Control Flow + +The daemon has to validate incoming network messages of all types (gossip, +requests, responses). As such, each IPC message from the helper process that is +communicating an incoming gossip message or response includes a +`ValidationHandle`, and the daemon is expected to send a `validate` message back +to the helper to acknowledge the network message with a `ValidationStatus`. +Incoming RPC requests are a special case, however. Since the daemon will already +send a message back to the helper in the form of a `response` to the incoming +RPC request, the `ValidationStatus` is provided there instead. In this case, a +specific `ValidationHandle` is not required, since there is already a +`RequestId` that uniquely identifies the response with the request we are +validating. + +In summary, the new validation control flow is: + +- gossip and response validation + - `handle{Gossip,Response}` message is sent to daemon + - `validate` is sent back to helper +- request validation + - `handleRequest` message is sent to daemon + - `sendResponse` message is sent back to helper, which contains both the + response and validation state + +### Staging the Compatible and Hard Fork Changes + +Some of the key changes proposed in this RFC require a hard fork in order to be +released to the network. However, the next hard fork may be a while out. We +could just implement this work off of `develop` and wait until the next hard +fork to ship it, but this would mean that any immediate improvements we make to +the networking code on `compatible` will conflict with our refactor work on +`develop`. Overall, it is still a benefit to have this refactor on `compatible` +so that we can benefit from it immediately in the soft fork world while keeping +the code more or less in line with the future hard fork we want to take. + +Accordingly, in order to break this work up, we can do this refactor in 2 +passes: first, perform the main module and organization refactor off of +`compatible`, then once that is complete and merged, perform the hard fork +specific refactor off of `develop`. The `compatible` portion of the refactor can +include all changes that do not effect or rely on changes to messages sent over +the gossip network. Below is a detailed list of what would be included in each +phase of the refactor. + +- `compatible` + - Transport layer abstraction + - OCaml module conslidation w/ new interface + - Daemon/Libp2p protocol refactor (peer abstraction et al) + - Validation, Response, Gossip, and Request pipe split +- `develop` + - Message type awareness + - Message type based pipe split + - Trust scoring based peer-selection (requires trust system) + +### OCaml Implementation + +Structure wise, the OCaml implementation will continue to model the network +interface abstractly so that a dummy implementation may be used for unit testing +purposes. We will continue to have an +[interface](https://github.com/MinaProtocol/mina/blob/compatible/src/lib/gossip_net/intf.ml) +along with a +[existential wrapper](https://github.com/MinaProtocol/mina/blob/compatible/src/lib/gossip_net/any.ml) +that provides indirection over the selected gossip network implementation. A +[stub implementation](https://github.com/MinaProtocol/mina/blob/compatible/src/lib/gossip_net/fake.ml) +will also continue to exist. + +In order to maintain reasonable separation of concerns, the libp2p helper +implementation of the gossip network interface will be split into 2 main +modules. + +- `Libp2p` :: direct low-level access to `libp2p_helper` process management and + protocol +- `Mina_net` :: high-level networking interface which defines supported RPCs and + exposes networking functionality to the rest of the code (publicly exposed to + the rest of the code) + +The RPC interface would continue to be defined under the +[current GADT based setup](https://github.com/MinaProtocol/mina/blob/compatible/src/lib/mina_networking/mina_networking.ml). +This type setup will also be extended so that `Rpc.implementation` modules can +be passed in when the gossip network subsystem is initialized. This will be an +improvement to the current system in which ad-hoc functions are defined at the +[Mina_lib](https://github.com/MinaProtocol/mina/blob/compatible/src/lib/mina_lib/mina_lib.ml) +layer. This module based approach will also provide a mechanism through which we +can define global validation logic for RPC query responses that will +automatically be applied to all RPC queries of that type. RPC queries will still +be able to provide their own per-request validation logic in addition to this. + +Below is an example of what the new gossip network interface would look like +from the perspective of the rest of the daemon code. Note that it is much more +abstract than before, modeling our new design choices regarding migrating state +from OCaml to Go. + +```ocaml +module Mina_net : sig + module Config : sig + type t = (* omitted *) + end + + module Gossip_pipes : sig + type t = + { blocks: External_transition.t Strict_pipe.Reader.t + ; txn_pool_diffs: Transaction_pool.Diff.t Strict_pipe.Reader.t + ; snark_pool_diffs: Snark_pool.Diff.t Strict_pipe.Reader.t } + end + + module Stats : sig + type t = (* omitted *) + end + + module Abstract_peer_graph = + Graph.Persistent.Digraph.ConcreteBidirectional (struct + type t = + | AnyPeer + | SpecificPeer of + [@@deriving equal, hash] + end) + + type t + + (* We can construct the gossip network subsystem using the configuration, + * the set of RPC implementations, a state which is shared with the + * handlers of the provided RPC implementations. Once constructed, the + * gossip network handle will be returned along with pipes for reading + * incoming gossip network messages. *) + val create : + Config.t + -> ('state Rpc_intf.t_with_implementation) list + -> 'state + -> (t * Gossip_pipes.t) Deferred.Or_error.t + + val stats : t -> Stats.t + + (* Query operations now have the ability to express additional validation + * logic on a per-request basis, in addition to RPC-wide validation logic + * that is defined *) + val query_peer : + t + -> Peer.t + -> ('q, 'r) Rpc_intf.rpc + -> 'q + -> ?f:('r Envelope.Incoming.t Deferred.t -> Validation_status.t Deferred.t) + -> 'r Envelope.Incoming.t Deferred.Or_error.t + + val query_peers : + t + -> Abstract_peer_graph.t + -> ('q, 'r) Rpc_intf.rpc + -> 'q + -> ?f:('r Envelope.Incoming.t Deferred.t -> Validation_status.t Deferred.t) + -> 'r Envelope.Incoming.t Deferred.Or_error.t + + val broadcast : t -> Gossip_message.t -> unit Deferred.t + + val ban_peer : t -> Peer.t -> unit Deferred.t +end +``` + +### Go Implementation + +The Go implementation will be fairly similar to how it's structured today. The +scope of the state it maintains is more or less the same, the biggest changes +introduced in this RFC effect the OCaml code more. The main work in Go will just +be switching it to use Cap'N Proto and use the new message format instead of the +old one. + +One notable change that can be made is that, since we are moving to a push-based +model for libp2p helper metrics, we no longer need to host a prometheus server +from Go. However, we will still want the ability to optionally host an http +server that exposes the [pprof](https://golang.org/pkg/net/http/pprof/) debugger +interface, which we currently support in the metrics server we run. + +## Execution + +[execution]: #execution + +In order to execute on this refactor in a fashion where we can make incremental +improvements on the networking layer, we will break the work up as follows: + +1. Migrate existing IPC messages to Cap'N Proto. +2. Migrate to Unix pipes; split data streams up, except for per-gossip message + data streams (which requires message type awareness). +3. Migrate IPC messages to new protocol design. +4. Add message type awareness, and split up per-gossip message data streams. + +## Test Plan + +[test-plan]: #test-plan + +In order to test this thoroughly, we need to run the software in a realistic +networking scenario and exercise all IPC messages. This would involve connecting +a node running this upgrade to a testnet, and monitoring the types of IPC +messages we transmit while the node is running to ensure we hit them all. We +would want to run this on a block producer with some stake, a snark coordinator, +and some node that we send transactions through so that we properly test the +broadcast logic. Additionally, we should exercise some bans in order to verify +that our gating reconfiguration logic works as expected. Otherwise, we will use +the monitoring output to inform us of any missed surface area in testing. + +## Drawbacks + +[drawbacks]: #drawbacks + +- a refactor of this scope will take some time to test (given historical context + for libp2p work, this could be significant) + - COUNTER: we will have to do something like this eventually anyway, better to + do it now than later + +## Rationale and alternatives + +[rationale-and-alternatives]: #rationale-and-alternatives + +- instead of refactoring our current go process integration, we could replace + our go helper with a rust process now that there is better libp2p support in + rust + - would alleviate us of our current go issues, and move to a better language + that more people on the team know and can contribute to + - certainly less bugs, but certainly harder to build + - this would be a lot more work and would likely take even longer to test + - more risk associated with this route +- [ZMQ](https://zeromq.org/) could be an alternative for bounded-queue IPC + - benchmarks seem promising, but more research needs to be done +- Unix sockets could be an alternative to Unix pipes + - has the advantage that we can move processes across devices and the IPC will + still work + - more overhead than Unix pipes + - with the data stream generalization, we can always swap this in if and when + we decide to move processes around +- [flatbuffers](https://google.github.io/flatbuffers/) could be an alternative + serialization format (with some advantages and tradeoffs vs Cap'N Proto) + - there are no existing OCaml libraries for this + +## Unresolved questions + +[unresolved-questions]: #unresolved-questions + +- how should reconfiguration work? we currently support that, but should we just + restart the helper process instead? diff --git a/website/docs/researchers/rfcs/0061-solidity-snapps.md b/website/docs/researchers/rfcs/0061-solidity-snapps.md new file mode 100644 index 000000000..bce9a247c --- /dev/null +++ b/website/docs/researchers/rfcs/0061-solidity-snapps.md @@ -0,0 +1,338 @@ +--- +format: md +title: "RFC 0061: Solidity Snapps" +sidebar_label: "0061 Solidity Snapps" +hide_table_of_contents: false +--- + +> **Original source:** +> [0061-solidity-snapps.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0061-solidity-snapps.md) + +# Overview of solidity features for snapps + +This document aims to examine the features of the solidity smart contract +language, to describe how they can be simulated by snapps, and proposing changes +to the snapp transaction model for those that it currently cannot simulate. + +This document refers to the features of v0.8.5 of the solidity language, and +makes reference to the snapp parties transaction RFC at MinaProtocol/mina#8068 +(version 95e148b4eef01c6104de21e4c6c7c7465536b9d8 at time of writing). + +## Basic features + +### State variables + +Solidity uses +[state variables](https://docs.soliditylang.org/en/v0.8.5/structure-of-a-contract.html#state-variables) +to manage the internal state of a contract. We intend to simulate this state +with a 'snapp state' formed of +[8 field elements](https://github.com/MinaProtocol/mina/blob/b137fbd750d9de1b5dfe009c12de134de0eb7200/src/lib/mina_base/snapp_state.ml#L17). +Where the state holds more data than will fit in 8 field elements, we can +simulate this larger storage by holding a hash of some of these variables in +place of their contents. + +In solidity, 'public' variables can be referenced by other contracts via a +function. We propose using the same method for snapps; see the function section +below for details. + +#### Off-chain storage and snapp state accessibility + +When the variables do not fit within the field elements, the data for the snapp +will not be directly available on-chain, and must be computed or retrieved from +some off-chain source. It is important to provide primitives for revealing the +updated states, otherwise updating a snapp's state may only reveal a hash, and +the new underlying data may be rendered inaccessible. + +To this end, it may be useful to add support for the poseidon hash used by mina +to IPFS, so that this data can be stored (ephemerally) in IPFS. We will also +discuss a proposal to expose data for state transitions as 'events' associated +with the parties; see the events section below for details. + +### Functions + +[Functions](https://docs.soliditylang.org/en/v0.8.5/structure-of-a-contract.html#functions) +are the primary interface of solidity contracts; in order to interact with a +smart contract, you submit a transaction that calls one of the functions the +contract exposes. These may call other functions from the same contract or from +other contracts. + +We propose simulating functions with snark proofs, where each function call +corresponds to a single snark proof. Our current snark model uses a 'wrapping' +primitive, which allows a single 'verification key' to verify wrapped proofs +witnessing one (or more) of several different 'circuits' (here, function +declarations). Function calls against different snapps require separate +'parties', although multiple calls to functions in the same snapp may be merged +into a single proof and issued as a single 'party' (either by proof composition +or inlining, depending on the use case). + +#### Arguments and returned values + +In order to simulate calling functions with arguments, and returning values from +functions, snapp parties must be able to expose some 'witness' to these values. +The format is determined by the circuit statement, but usually this will be +`hash(arguments, returned_values)`. + +_This is currently not supported by the snapp transaction model RFC._ + +**Proposal:** add an additional field element (`aux_data`) that is passed as +part of the input to the snapp proof, which may be used to bind the function's +input and returned values. + +#### Function calls between snapps + +In order for a snapp to verify that function calls are executed, snapp proofs +must be able to interrogate the other parties included in a transaction. The +current RFC doesn't identify what the proof inputs should be, but describes a +stack of parties (`parties`) and the current state of the stack when a +transaction is reached in the stack (`remaining_parties`). + +**Proposal:** pass `parties`, the stack of parties, as part of the snapp input. + +We should also consider nested function calls, each of which may result in one +or more parties (e.g. to pay one or more receivers, or to make other further +function calls). We can make these conceptually simpler and more composable by +grouping the transactions nested below a party's snapp together, in a hierarchy +of snapps. This will be particularly helpful for snapps which make recursive +calls or deeply nested calls, by letting them avoid walking arbitrarily far +along the stack of parties to find the one they care about. + +**Proposal:** use a stack of stacks for the parties involved in a transaction, +allowing each snapp to access its inner transactions by examining its stack. For +example, a snapp which calls other snapps might have a stack that looks like + +```ocaml +[ transfer_for_fee +; [ snapp1 + ; transfer1 (* Sent by snapp1 *) + ; [ snapp2 (* Called by snapp1 *) + ; transfer2 (* Sent by snapp2 *) + ; [snapp3] ] (* Called by snapp2 *) + ; transfer3 (* Sent by snapp1 *) + ; [snapp4] ] ] (* Called by snapp1 *) +``` + +Concretely, this allows snapp1 to access `transfer3` and `snapp4` without +needing to know or care about the transfers and snapps executed by `snapp2`. In +the implementation, this could look something like: + +```ocaml +let get_next_party + current_stack (* The stack for the most recent snapp *) + call_stack (* The partially-completed parent stacks *) + = + let next_stack, next_call_stack = + if call_stack = empty_stack then + empty_stack, empty_stack + else + call_stack.pop() + in + (* If the current stack is complete, 'return' to the previous + partially-completed one. + *) + let current_stack, call_stack = + if current_stack = empty_stack then + next_stack, next_call_stack + else + stack, call_stack + in + let stack_or_party, next_stack = current_stack.pop() in + let party, remaining_stack = + let stack = + if stack_or_party.is_party() then + (* dummy value for circuit *) + current_stack + else + stack_or_party.as_stack() + in + let popped_value, remaining_stack = stack.pop() in + if stack_or_party.is_party() then + stack_or_party.as_party(), empty_stack + else + popped_value, remaining_stack + in + let party, current_stack, next_stack = + if remaining_stack = empty_stack then + party, next_stack, empty_stack + else + party, remaining_stack, next_stack + in + let call_stack = + if next_stack = empty_stack then call_stack + else call_stack.push(next_stack) + in + party, current_stack, call_stack +``` + +This increases the number of stack operations per party from 1 to 4. + +### Function modifiers + +[Function modifiers](https://docs.soliditylang.org/en/v0.8.5/structure-of-a-contract.html#function-modifiers) +are a language level feature of solidity, and exist solely to avoid unnecessary +function calls. This requires no features at the transaction model level. + +### Events + +[Events](https://docs.soliditylang.org/en/v0.8.5/structure-of-a-contract.html#events) +in solidity are emitted by a smart contract, but are not available for use by +the contracts themselves. They are used to signal state transitions or other +information about the contract, and can be used to expose information without +the need to replay all past contract executions to discover the current state. + +_This is currently not supported by the snapp transaction model RFC._ + +**Proposal:** add an additional field to each party that contains a list of +events generated by executing a snapp, or none if it is a non-snapp party. This +event stack should be passed as part of the input to the snapp, as the output of +hash-consing the list in reverse order. (_TODO-protocol-team: decide on the +maximum number / how the number affects the txn fee / etc. to avoid abuse._) + +#### Exposing internal state variables + +As mentioned in the 'state variables' section above, the contents of a snapp's +internal state becomes unavailable on-chain if that state is larger than the +available 8 field elements. Events give us the opportunity to re-expose this +data on chain, by e.g. emitting a `Set_x(1)` event when updating the value of +the internal variable `x` to `1`, so that the current state of the snapp can be +recovered without off-chain communication with the previous party sending the +snapp. + +This is likely to be an important feature: it's not possible to execute a snapp +without knowing the internal state, and this appears to be the easiest and most +reliable way to ensure that it is available. Without such support, it's possible +and relatively likely for a snapp's state to become unknown / unavailable, +effectively locking the snapp. + +### Errors + +Solidity +[errors](https://docs.soliditylang.org/en/v0.8.5/structure-of-a-contract.html#errors) +are triggered by a `revert` statement, and are able to carry additional +metadata. + +In the current snapp transaction RFC, this matches the behaviour of invalid +proofs, where the errors correspond to an unsatisfiable statement for a circuit. +In this model, we lose the ability to expose the additional metadata on-chain; +however, execution is not on-chain, so the relevant error metadata can be +exposed at proof-generation time instead. + +### Struct and enum types + +[Struct types](https://docs.soliditylang.org/en/v0.8.5/structure-of-a-contract.html#struct-types) +are a language feature of solidity, which is already supported by snarky. +[Enum types](https://docs.soliditylang.org/en/v0.8.5/structure-of-a-contract.html#enum-types) +have also had long-lived support in snarky, although have seen little use in +practice. + +## Types + +### Value types + +All of the +[value types](https://docs.soliditylang.org/en/v0.8.5/types.html#value-types) +supported by solidity are also supported by snarky. + +### Reference types + +[Reference types](https://docs.soliditylang.org/en/v0.8.5/types.html#reference-types) +in solidity refer to blocks of memory. These don't have a direct analog in +snarks, but can be simulated -- albeit at a much higher computational cost -- by +cryptographic primitives. Many of these primitives are already implemented in +snarky. + +### Mapping types + +[Mapping types](https://docs.soliditylang.org/en/v0.8.5/types.html#mapping-types) +are similar to reference types, but associate some 'key' value with each data +entry. This can be implemented naively on top of a primitive for +`[(key, value)]`, an array of key-value pairs, which is already available in +snarky. + +We currently do not support a primitive for de-duplication: a key may appear +multiple times in a `[(key, value)]` implementation and the prover for a snapp +could choose any of the values associated with the given key in a particular +map. This will require some research into the efficiency of the different +primitives available for this, but has no impact upon the transaction model. + +## Standard library + +### Block / transaction properties + +The +[block and transaction properties](https://docs.soliditylang.org/en/v0.8.5/units-and-global-variables.html#block-and-transaction-properties) +available in solidity are, at time of writing: + +- `blockhash(uint blockNumber)` + - Can be easily supported using the protocol state hash, although the timing + is tight for successful execution (receive a block, create the snapp proof, + send it, have it included in the next block). + - **Proposal:** support the previous `protocol_state_hash` in snapp + predicates. +- `block.chainid` + - **Proposal:** expose the chain ID in the genesis constants, allow it to be + used in the snapp predicate. +- `block.coinbase` + - Snapp proofs are generated before the block producer is known. Not possible + to support. +- `block.difficulty` + - Not applicable, block difficulty is fixed. +- `block.gaslimit` + - Not applicable, we don't have a gas model. +- `block.number` + - Available by using `blockchain_length` in the snapp predicate. +- `block.timestamp` + - Available by using `timestamp` in the snapp predicate. +- `gasleft()` + - Not applicable, we don't have a gas model. +- `msg.data` + - Available as part of the snapp input +- `msg.sender` + - **Proposal:** Expose the party at the head of the parent stack as part of + the snapp input. +- `msg.sig` + - As above. +- `msg.value` + - As above. +- `tx.gasprice` + - Not applicable, we don't have a gas model. +- `tx.origin` + - Can be retrieved from the `parties` stack of parties. May be one or more + parties, depending on the structure of the transaction. + +### Account look-ups + +Solidity supports the following +[accessors on addresses](https://docs.soliditylang.org/en/v0.8.5/units-and-global-variables.html#members-of-address-types): + +- `balance` + - Could use the `staged_ledger_hash` snapp predicate (currently disabled) and + perform a merkle lookup. However, this doesn't account for changes made by + previous parties in the same transaction, or previous transactions in the + block. + - **Proposal:** Add a 'lookup' transaction kind that returns the state of an + account using `aux_data`, with filters to select only the relevant data. + Snapps can then 'call' this by including this party as one of their parties. + - Note: using this will make the snapp transaction fail if the balance of the + account differs from the one used to build the proof at proving time. +- `code`, `codehash` + - Snapp equivalent is the `verification_key`. + - Options are as above for `balance`. If this key is for one of the + snapp-permissioned parties, the key can assumed to be statically known, + since their snapp proof will be rejected and the transaction reverted if the + key has changed. +- `transfer(uint256 amount)` + - Executed by including a transfer to the party as part of the snapp's party + stack. +- `call`, `delegatecall`, `staticcall` + - Executed by including a snapp party as part of the snapp's party stack. + +### Contract-specific functions + +Solidity supports +[reference to `this` and a `selfdestruct` call](https://docs.soliditylang.org/en/v0.8.5/units-and-global-variables.html#members-of-address-types). + +We can support `this` by checking the address of the party for a particular +snapp proof, or by otherwise including its address in the snapp input. + +We currently do not support account deletion, so it is not possible to implement +an equivalent to `selfdestruct`. diff --git a/website/docs/researchers/rfcs/0062-bitswap.md b/website/docs/researchers/rfcs/0062-bitswap.md new file mode 100644 index 000000000..763de9d31 --- /dev/null +++ b/website/docs/researchers/rfcs/0062-bitswap.md @@ -0,0 +1,474 @@ +--- +format: md +title: "RFC 0062: Bitswap" +sidebar_label: "0062 Bitswap" +hide_table_of_contents: false +--- + +> **Original source:** +> [0062-bitswap.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0062-bitswap.md) + +# Summary + +[summary]: #summary + +This RFC proposes adding Bitswap to our libp2p networking stack in order to +address issues related to our current gossip network pub/sub layer. + +# Motivation + +[motivation]: #motivation + +Mina has very large messages that are broadcast over the gossip net pub/sub +layer. This incurs a high bandwidth cost due to the nature of our pub/sub +rebroadcast cycles work in order to consistently broadcast messages throughout +the network. For example, we observer blocks on mainnet as large as ~2mb. This +would represent only a single block, and each block broadcast message has a +multiplicative cost on bandwidth as it's being broadcast throughout the network. +This bandwidth cost also translates into CPU cost due to the cost of hashing +incoming messages to check against the de-duplication cache before processing +them. We currently observe behaviors where the libp2p helper process can be +pegged at 100% CPU on certain hardware setups when there is high pub/sub +throughput on the network. Since gossip pub/sub is used not only for blocks, but +also transactions and snark work, the broadcasting of each of these +simultaneously ends up compounding the issue. + +Implementing Bitswap in our libp2p layer will address this issue by allowing us +to immediately reduce our pub/sub message size, while making the larger data +referenced by pub/sub messages available upon request. It provides a mechanism +for breaking up large data into chunks that can be distributed throughout the +network (streamed back from multiple peers), and a system for finding peers on +the network who are able to serve that data. + +# Detailed design + +[detailed-design]: #detailed-design + +[Bitswap](https://docs.ipfs.io/concepts/bitswap/) is a module provided by +[libp2p](https://libp2p.io/) that enables distributed data synchronization over +a p2p network, somewhat comparable to how +[BitTorrent](https://en.wikipedia.org/wiki/BitTorrent) works. It works by +splitting up data into chunks called blocks (we will explicitly refer to these +as "Bitswap blocks" to disambiguate them from "blockchain blocks"), which are +structured into a DAG with a single root. When a node on the network wants to +download to some data, it asks it's peers to see which (if any) have the root +Bitswap block corresponding to that data. If none of the peers have the data, it +falls back to querying the gossip network's +[DHT](https://docs.ipfs.io/concepts/dht/#kademlia) to find a suitable node that +can serve the data. + +In this design, we will lay out an architecture to support Bitswap in our Mina +implementation, along with a strategy for migrating Mina blocks into Bitswap to +reduce current gossip pub/sub pressure. We limit the scope of migrating Mina +data to Bitswap only to blocks for the context of this RFC, but in the future, +we will also investigate moving snark work, transactions, and ledger data into +Bitswap. Snark work and transactions will likely be modeled similarly to Mina +blocks with respect to Bitswap, but ledger data will require some special +thought since it's Bitswap block representation will have overlapping Bitswap +blocks across different ledgers. + +## Bitswap Block Format + +Bitswap blocks are chunks of arbitrary binary data which are content addressed +by +[IPFS CIDs](https://docs.ipfs.io/concepts/content-addressing/#cid-conversion). +There is no pre-defined maximum size of each Bitswap block, but IPFS uses 256kb, +and the maximum recommended size of a Bitswap block is 1mb. Realistically, we +want Bitswap blocks to be as small as possible, so we should start at 256kb for +our maximum size, but keep the size of Bitswap blocks as a parameter we can tune +so that we can optimize for block size vs block count. + +While the Bitswap specification does not care about what data is stored in each +block, we do require each block have a commonly-defined format: + +1. `[2 bytes]` count of links n +2. `[n * 32 bytes]` links (each link is a 256-bit hash) +3. `[up to (maxBlockSize - 2 - 32 * n) bytes]` data + +Hence, data blob is converted to a tree of blocks. We advertise the "root" block +of the tree as the initial block to download for each resource we store in +Bitswap, and the libp2p helper process will automatically explore all the child +blocks referenced throughout the tree. To construct the full binary blob out of +this tree, breadth-first search (BFS) algorithm should be utilized to traverse +the tree. BFS is a more favourable approach to DFS (another traversal order) as +it allows to lazily load the blob by each time following nodes links to which we +already have from the root block (counter to the order induced by DFS where one +has to go to the deepest level before emitting a chunk of data). + +For links a 256-bit version of Blake2b hash is to be used. Packing algorithm can +be implemented in the way that no padding is used in blocks and there are +maximum `n = (blobSize - 32) / (maxBlockSize - 34)` blocks generated with +`n - 1` blocks of exactly `maxBlockSize` bytes. + +## Bitswap Block Database + +There exist some key constraints in choosing a good solution for the Bitswap +block db. Importantly, the block database needs to support a concurrent readers +and a writer at the same time. Additionally, the database should be optimized +for existence checks and reads, as these are the operations we will perform the +most frequently against it. Some consistency guarantees (ability to rely on +happens-before relation between write and read events) are also required. The +database would ideally be persisted, so that we can quickly reload the data in +the event of node crash (we want to avoid increasing bootstrap time for a node, +as that keeps stake offline after crashes). + +Given these constraints, [LMDB](http://www.lmdb.tech/doc/) is a good choice for +the Bitswap block storage. It meets all of the above criteria, including +persistence. As a bonus, it's light on extra RAM usage. + +### Database Schema + +| Key | Value | Key bytes | +| ------------------- | ------------------- | ----------------------------- | +| `status/` | integer: 0..2 | `<1><32-byte blake2b digest>` | +| `block/` | bitswap block bytes | `<0><32-byte blake2b digest>` | + +Status is an integer taking one of the values: + +- `0` (not all descendant blocks present) +- `1` (all descendant blocks present) +- `2` (delete in process) + +### Additional data to store in the Daemon db + +In addition to data already stored in the DB controlled by the Daemon, we would +need to store: + +- `headerHashToRootCid`: relation between header and associated root cid (if + known) +- `rootCidsToDelete`: list of root cids marked for deletion (i.e. root cids + related to blocks which were completely removed from frontier) +- `recentlyCreatedBlocks`: list of recently created blocks for which we didn't + receive confirming upcall + +### Invariants + +The following invariants are maintained for the block storage: + +- For each `status/{rcid}` in DB, there is an entry for `{rcid}` in the + `headerHashToRootCid` +- For each cid for which `block/{cid}` exists in DB, either holds: + - There exists `status/{cid}` + - There exists `cid'` such that `cid` is in the link list of bitswap block at + `block/{cid'}` +- `status/{rc}` can progress strictly in the order: + `null -> 0 -> 1 -> 2 -> null` + +### Initialization + +Daemon initialization: + +1. Remove keys `k` from `rootCidsToDelete` for which `status/{k}` is absent +2. Remove blocks `b` from `recentlyCreatedBlocks` for which + `status/{b.root_cid}` is present and is not equal to `0` +3. Start Helper +4. Send bulk delete request to Helper for keys from `rootCidsToDelete` +5. Send bulk download request to Helper for blocks that are known to frontier + but which do not have `status/{b.root_cid} == 1` +6. Send add resource request for each block in `recentlyCreatedBlocks` + +Helper has no initialization at all. + +### Helper to Daemon interface + +Helper receives requests of kind: + +- Delete resources with root cid in list `[cid1, cid2, ...]` + - Sends a single upcall upon deletion of all of these resources +- Download resources with root cid in list `[cid1, cid2, ...]` + - Sends an upcall upon full retrieval of each resource (one per resource) +- Add resource with root cid `{root_cid}` and bitswap blocks + `[block1, ..., blockN]` (no checks for hashes are made) + - Sends an upcall confirming the resource was successfully added + +### Synchronization + +Block creation: + +1. Block is added to `recentlyCreatedBlocks` +2. Add resource request is sent to Helper +3. Upon add resource confirmation upcall, block is removed from + `recentlyCreatedBlocks` + +Frontier is moved forward and some old blocks get removed: + +1. Remove record for block from `headerHashToRootCid` +2. Add block to `rootCidsToDelete` +3. Delete resource request is sent to Helper +4. Upon delete resource confirmation upcall, block is removed from + `rootCidsToDelete` + +A gossip for the new header is received and Daemon decides that the block body +corresponding to the header has to be fetched: + +1. Store record in `headerHashToRootCid` for the header +2. Send download request to Helper +3. Upon download upcall is received, block and header are added to frontier a. + In case downloaded block came late and the block is not more of an interest, + launch the deletion flow as described above + +(We assume root cid is received along with the header in the gossip) + +## Migrating Mina Blocks to Bitswap + +To migrate Mina block propagation to Bitswap, we will separate a Mina block into +2 portions: a block header, and a block body. Most of the data in a Mina block +is stored inside of the `staged_ledger_diff`. The common data in every Mina +block is ~8.06kb (including the `protocol_state_proof`), so using everything +**except** for the `staged_ledger_diff` as the block header seems natural. The +`staged_ledger_diff` would then act as the block body for Mina blocks, and would +be downloaded/made available via Bitswap rather than broadcast over pub/sub. + +When blocks are broadcast through the network now, only the block header and a +root CID for the `staged_ledger_diff` are in the message. When a node receives a +new block header, the node will first verify the `protocol_state_proof` (all +public information that needs to be fed in for proof verification will be +available in the block header). Once the proof is checked, a node would then +download the `staged_ledger_diff` via Bitswap. Once that is downloaded, the node +would follow the same pattern right now for generating a breadcrumb by expanding +the `staged_ledger` from the parent breadcrumb and the new `staged_ledger_diff`, +after which the Mina block will be fully validated. At this point, the +breadcrumb is added to the frontier. + +In summation, the proposed changes in order to move Mina blocks into Bitswap +are: + +1. Define separate block header (block w/o `staged_ledger_diff` with new field + `staged_ledger_diff_root_cid`). +2. Verify `staged_ledger_diff_root_cid` relation to the header. +3. Rebroadcast block headers after proofs are checked, but before + `staged_ledger_diff`s are verified and the breadcrumb is added to the + frontier. +4. Punish block producer public keys if they submit an invalid + `staged_ledger_diff` by ignoring all future block headers from that producer + within the epoch (do not punish senders, as they may not have banned or + checked the `staged_ledger_diff` yet). + +Punishing is done only within the epoch as otherwise punish lists would be +accumulating without boundary and real adversaries will nevertheless find the +way around by mnoving stake for to other address for the next epoch. + +## Verifying relation between root CID and header + +One large difference from before is that nodes will rebroadcast the block header +to other nodes on the network before the `staged_ledger_diff` is downloaded and +verified, in order to avoid increasing block propagation time on the network +with the new addition of Bitswap. This change brings some unique problems that +we need to solve now, as previously, we wouldn't forward Mina blocks to other +nodes until we knew the block was fully valid. + +In the new world, an adversary could broadcast around the same block header and +proof, but swap out the `staged_ledger_diff` root Bitswap block CID with +different values to attack the network. An adversary can do that both being a +block producer and as a man-in-the-middle (MITM). + +To rule out the MITM attack vector, a signature is to be carried along with the +header. The signature is made of a pair of root CID and header hash with block +producer's public key. No MITM actor can forge the signature, hence the attack +becomes infeasible. + +To mitigate adversary the block producer attack case, the following tactic is +employed: if a node ever downloads a `staged_ledger_diff` which does not achieve +the target staged ledger hash after application to the parent staged ledger, +that node will ban the block producer public key associated with the block. This +significantly raises the cost of attack and makes it in effect pointless. + +# Shipping plan + +[shipping]: #shipping + +Shipping of the feature is two-staged with first stage being shipped as a +soft-fork release and the second stage being shipped as part of a hard-fork. + +## Soft-fork + +For soft-fork stage, here is the anticipated changeset: + +1. Support for sharing blocks via Bitswap +2. New pub/sub topics: + - `mina/block-body/1.0.0` + - `mina/tx/1.0.0` + - `mina/snark-work/1.0.0` +3. Engine to rebroadcast blocks from the new topics to old + `coda/consensus-messages/0.0.1` and vice versa + +### Legacy topic management + +Most new nodes will support both old and new topics for broadcast. Nodes are +able to filter subscriptions from other nodes based on what they subscribe to, +configured using the +[`WithSubscriptionFilter` option](https://github.com/libp2p/go-libp2p-pubsub/blob/55d412efa7f5a734d2f926e0c7c948f0ab4def21/subscription_filter.go#L36). +Utilizing this, nodes that support the new topics can filter out the old topic +from nodes that support both topics. By filtering the topics like this, nodes +running the new version can broadcast new blocks over both topics while avoiding +sending the old message format to other nodes which support the new topic. + +In particular, following method of the filter is to be implemented: + +``` +type SubscriptionFilter interface { + ... + FilterIncomingSubscriptions(peer.ID, []*pb.RPC_SubOpts) ([]*pb.RPC_SubOpts, error) +} +``` + +On receiving of these incoming subscription for the old topic, we check whether +the same peer is already subscribed to all three new topics and if so, +subscriptions is filtered out. Node configuration code shall be implemented +accordingly, subscribing to the new topics first. + +Over time, when most of the network participants adopt the newer version, only a +few specifically configured nodes (including some seeds) will continue servicing +the old topic, while most of the network will live entirely on the new topic. + +Mina node will take two additional arguments: + +- `--pubsub-v1`: `rw`, `ro` or default `none` +- `--pubsub-v2`: default `rw`, `ro` or `none` + +Daemon runs message propagation for legacy topic as follows: + +1. If `--pubsub-v2=rw` or `--pubsub-v2=ro`, listen to `mina/tx/1.0.0` and + `mina/snark-work/1.0.0` + 1. For each valid message if `--pubsub-v1=rw` resend it to + `coda/consensus-messages/0.0.1` +2. If `--pubsub-v2=rw` or `--pubsub-v2=ro`, listen to `mina/block/1.0.0` and add + headers to frontier +3. For each new block in frontier, publish it to `coda/consensus-messages/0.0.1` + if `--pubsub-v1=rw` +4. If `--pubsub-v1=rw` or `--pubsub-v1=ro`, listen to + `coda/consensus-messages/0.0.1` + 1. For each valid block message if `--pubsub-v2=rw` resend corresponding + header to `mina/block/1.0.0` + 2. For each valid transaction message if `--pubsub-v2=rw` resend it to + `mina/tx/1.0.0` + 3. For each valid snark work message if `--pubsub-v2=rw` resend it to + `mina/snark-work/1.0.0` + +Releasing of Bitswap will happen in two stages: + +1. Release with default parameters `--pubsub-v1=rw` and `--pubsub-v2=rw` +2. After most block producers adopt new version, change to default + `--pubsub-v1=none` +3. Launch a network of "pubsub relays" which will service both `--pubsub-v1=rw` + and `--pubsub-v2=rw` until the next hardfork + +### New block topic + +New block topic presents a new message type comprising: + +1. Block header (as defined above) +2. Block body certificate + +Block body certificate in turn is the data structure with the following fields: + +1. 32-byte block body root +2. Signature + +Block body root is a hash of a root bitswap block containing bytes of block +body. + +Signature is a digital signature of pair +`(block body root hash, block header hash)` made with secret key corresponding +to block producer's key specified in the block's header. + +Frontier absorbs block headers along with the corresponding block body roots and +keeps them as 1:1 relation. Whenever the same block header is received with a +different block body root (and a valid block producer signature), block producer +is banned from producing a block for the given slot. + +Additionally, if bitswap block referenced by block body root is broken, block +producer is banned from producing a block for the given slot. + +In case of ban for a slot, all of the corresponding data is cleaned out, ban is +local. + +## Hard-fork + +In the next hardfork old topic becomes entirely abandoned. + +Also, consider migrating to hash-based message ids as described in +[issue #9876](https://github.com/MinaProtocol/mina/issues/9876). + +# Drawbacks + +[drawbacks]: #drawbacks + +This adds significant complexity to how the protocol gossips around information. +The control flow for validating blocks is more complex than before, and there is +new state to synchronize between the processes in the architecture. It also adds +new delays to when the full block data will be available to each node (but the +tradeoff here is that we are able to more consistently gossip block s around the +network within the same slot those blocks are produced). + +# Rationale and alternatives + +[rationale-and-alternatives]: #rationale-and-alternatives + +- it would be possible to download larger data from peers via RPC and still + reduce the pub/sub message size, though there are some issues with this + approach + - it is not guaranteed that any of your peers will have the data you need, in + which case you need some alternative mechanism to discover who does have it + - puts a lot of bandwidth pressure on individual peers rather than spreading + the load between multiple peers (which helps with both bandwidth pressure + and data redundancy for increase availability) +- alternatives to using LMDB as the Bitswap cache + - use [SQLite](https://www.sqlite.org/index.html) + - even though all we need is a key/value db, not a relational db, SQLite is + portable and performant + - would require us to enable both the + [write-ahead logging](https://sqlite.org/wal.html) and use + [memory-mapped I/O](https://www.sqlite.org/mmap.html) features in order to + use it the way we would like to + - use raw Linux filesystem (from @georgeee) + - would use a lot of inodes and file descriptors if we do not build a + mechanism that stores multiple key-value pairs in shared files, which + could prove tricky to implement + - would need to solve concurrency problems related to concurrent + readers/writers, which could be tricky to get correct and have confidence + in + +## Verifying root CID to header relation + +There is another option to verify the relation: include a commitment in the +snark not only to the target staged ledger hash, but also the root Bitswap block +CID of the `staged_ledger_diff` that brings us to that state. + +Two options compare with one other in the following way: + +| | Snark commitment | Signature | +| -------------------------- | ---------------- | ---------- | +| _Snark proving time_ | Increased | Unchanged | +| _Computational complexity_ | High | Low | +| _Soft-fork_ | Incompatible | Compatible | + +For the _Snark commitment_ option, adversary needs to generate a snark proof for +each `staged_ledger_diff` they want to broadcast to the network, hence the high +computational complexity of an attack, which is a desirable property. However, +_Signature_ option is preferred due to the fact that it's softfork-compatible +and doesn't increase the complexity of circuit. + +# Appendix + +_For reference on the above computation of ~8.06kb for a block without a staged +ledger diff, here is a snippet of OCaml code that can be run in +`dune utop src/lib/mina_transition`_ + +```ocaml +let open Core in +let open Mina_transition in +let precomputed_block = External_transition.Precomputed_block.t_of_sexp @@ Sexp.of_string External_transition_sample_precomputed_block.sample_block_sexp in +let small_precomputed_block = {precomputed_block with staged_ledger_diff = Staged_ledger_diff.empty_diff} in +let conv (t : External_transition.Precomputed_block.t) = + External_transition.create + ~protocol_state:t.protocol_state + ~protocol_state_proof:t.protocol_state_proof + ~staged_ledger_diff:t.staged_ledger_diff + ~delta_transition_chain_proof:t.delta_transition_chain_proof + ~validation_callback:(Mina_net2.Validation_callback.create_without_expiration ()) + () +in +Protocol_version.set_current (Protocol_version.create_exn ~major:0 ~minor:0 ~patch:0) ; +External_transition.Stable.Latest.bin_size_t (conv small_precomputed_block) ;; +``` diff --git a/website/docs/researchers/rfcs/0063-reducing-daemon-memory-usage.md b/website/docs/researchers/rfcs/0063-reducing-daemon-memory-usage.md new file mode 100644 index 000000000..d7463c339 --- /dev/null +++ b/website/docs/researchers/rfcs/0063-reducing-daemon-memory-usage.md @@ -0,0 +1,374 @@ +--- +format: md +title: "RFC 0063: Reducing Daemon Memory Usage" +sidebar_label: "0063 Reducing Daemon Memory Usage" +hide_table_of_contents: false +--- + +> **Original source:** +> [0063-reducing-daemon-memory-usage.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0063-reducing-daemon-memory-usage.md) + +## Summary + +[summary]: #summary + +This RFC proposes changes to the Berkeley release of the daemon which will bring +daemon's maximum possible memory usage within the range of our current hardware +memory requirements of the Mainnet release. + +## Motivation + +[motivation]: #motivation + +With zkApps enabled, the maximum memory usage of the daemon now exceeds the +current hardware requirements (`16GB`), in the event that the network is fully +saturated with max-cost zkApps transactions in blocks for an extended period of +time. With the planned parameters for Berkeley, we estimate that the maximum +memory usage of a daemon is around `58.144633GB`. + +## Detailed design + +[detailed-design]: #detailed-design + +In order to reduce the memory usage of the daemon, we will offload some of the +memory allocated data to an on-disk cache. The +[memory analysis section](#memory-analysis) will show that the majority of +memory in the fully saturated max-cost transaction environment comes from ledger +proofs, zkApps proofs, and zkApps verification keys. Each of these pieces of +data are accessed infrequently by the daemon, and, as such, do not need to be +stored in RAM for fast access. + +Below is a list of all the interactions the daemon has with ledger proofs, +zkApps proofs, and zkApps verification keys: + +- ledger proofs are stored upon receipt of snark work from the gossip network +- zkApps proofs and newly deployed zkApps verification keys are stored upon + receipt of zkApps transactions from the gossip network +- ledger proofs, zkApps proofs, and zkApps verification keys are read when a + block is produced, in order for them to be included into the staged ledger + diff +- ledger proofs, zkApps proofs, and zkApps verification keys are stored upon + receipt of new blocks from the gossip network +- ledger proofs, zkApps proofs, and zkApps verification keys are read when a + block is applied to the ledger +- ledger proofs, zkApps proofs, and zkApps verification keys are read when + serving bootstrap requests to nodes joining the network + +In order to write this data to disk, we will use the +[Lightning Memory-Mapped Database](http://www.lmdb.tech/doc/) (LMDB for short). +LMDB is a lightweight, portable, and performant memory map backed key-value +storage database. We will demonstrate that the performance of this database is +more than sufficient for our use case in the +[impact analysis section](#impact-analysis). + +For this optimization, it is not important that the on-disk cache is persisted +across daemon runs. Such a feature can be added in the future, and this storage +layer can double as a way to better persist data from the snark pool and data +references from the persistent frontier. However, for now, we will set up the +daemon so that it wipes the existing on-disk cache between every restart, in +order to simplify the implementation and avoid having to deal with potentially a +corrupted on-disk cache (in the event the daemon or operating system did not +shut down properly). This is particularly important given our choice of LMDB +does not provide complete guarantees against data corruption out of the box, due +to the fact memory maps can lead to partial writes if the kernel panics or is +otherwise interrupted before the system is gracefully shutdown. + +To prevent disk leaks in the cache, we will use GC finalizers on cache +references to count the active references the daemon has to information written +to the cache. Since the daemon is always starting with a fresh on-disk cache, +this will give an accurate reference count to any data cached on-disk. When a GC +finalizer decrements the total reference count of an item stored on the cache to +0, it will delete that item from the cache. With this setup, the on-disk cache +can only leak if there is a memory leak within the daemon itself, in which the +daemon is leaking references to the cache. + +For identifying cached values by their content, we will use the Sha256 hash +function, as provided by the `digestif` OCaml library (which is an existing +dependency of the daemon). Sha256 is a performant an sufficiently collision +resistant hashing algorithm for this use case. The usage of Sha256 here means +that cache keys will be 8 bytes long, since Sha256 digests are 256 bits in +length. When tracking the refcounts of cache references using the GC, we will +track a hash table with the on-disk cache, which will map from Sha256 digests +into active reference counts for that digest. The GC finalizers for cache +references will decrement the refcount for their respective digests, removing +the refcount entry and deleting the underlying item from the on-disk cache if +the refcount becomes 0. + +## Memory Analysis + +[memory-analysis]: #memory-analysis + +In order to accurately estimate the memory usage of the daemon in the fully +saturated max-cost transaction environment, we have written a program +`src/app/disk_caching_stats/disk_caching_stats.exe`. See the +[README.md](https://github.com/MinaProtocol/mina/blob/compatible/src/app/disk_caching_stats/README.md) +for more information on how the program calculates these estimates. + +Below is the output of the script when all of the parameters are tuned to what +we have planned for the Berkeley release. + +``` +baseline = 3.064569GB +scan_states = 15.116044GB +ledger_masks = 2.292873GB +staged_ledger_diffs = 32.345724GB +snark_pool = 4.453962GB +transaction_pool = 0.871383GB +TOTAL: 58.144555GB +``` + +In this output, the baseline is made up of static measurements taken from the +daemon, which represents the overhead of running a daemon regardless of +throughput or transaction cost. We have then estimated the expected worst-case +memory usage of the scan states, ledger masks, and staged ledger diffs, which +are the overwhelming majority of data allocated for the frontier. And finally, +we estimate the memory footprint of the snark pool and transaction pool. + +Now, if we adjust the estimates for proposed optimizations to store proofs and +verification keys to disk (by subtracting their sizes from the memory footprint +and replacing them with cache references), we get the following output. + +``` +baseline = 3.064569GB +scan_states = 3.658435GB +ledger_masks = 0.562126GB +staged_ledger_diffs = 9.202166GB +snark_pool = 0.014414GB +transaction_pool = 0.247903GB +TOTAL: 16.749612GB +``` + +As we can see, this brings the estimation down much closer to the current `16GB` +hardware requirement. From here, we can look into additional optimizations to +bring it down even further to fit within the current hardware requirements. Such +optimizations could include: + +- Sharing the prover key allocation across the daemon's subprocesses, reducing + the baseline to nearly 1/3rd of what it is now. +- Persisting the entire staged ledger diff of each block to disk, given we + rarely need to read the commands contained within a staged ledger diff after + the diff has been applied. The diffs only need to be sent to peers after + they've been applied to the ledger, so we can just store the diffs in + `bin_prot` format and not bother deserializing them when we serve them to + other nodes. + +### Impact Analysis + +[impact-analysis]: #impact-analysis + +There is a space-time tradeoff here in the sense that, by excising data from RAM +to disk, we are now needing to perform disk I/O and deserialize/serialize data +in order to perform reads/writes. So part as part of this design, it is +important to show that the performance hit we take for reading/writing data +cached to disk is relatively insignificant in the operation of the daemon. + + + +LMDB provides benchmarks against other similar databases +[on their website](http://www.lmdb.tech/bench/microbench/). The important piece +of data here is that, with a 128MB cache and values of `100,000` bytes in size, +LMDB is benched at being capable of performing `1,718,213` random reads per +second (about `582` nanoseconds per read). Given the amount of reads, and +frequencey of reads, a daemon will be performing from this on-disk cache, these +benchmarks show that reading from LMDB will have a negligible effect on daemon +performance. All proofs and verification keys we would read/write to disk are +under this `100,000` benchmark size, so the actual performance should be better +than this. + + + +Bin_prot serialization benchmarks for the proofs and verification keys have been +added to the same program that does the memory impact analysis presented above. +In this program, we run 10_000 trials of each operation, and take an average of +the elapsed time for the entire execution. Below are the results from this +program (as run on my local machine). + +``` +========================================================================================== +SERIALIZATION BENCHMARKS Pickles.Side_loaded.Proof.t +========================================================================================== +write: 32.424211502075195us (total: 324.24211502075195ms) +read: 46.872687339782715us (total: 468.72687339782715ms) + +========================================================================================== +SERIALIZATION BENCHMARKS Mina_base.Verification_key_wire.t +========================================================================================== +write: 6.0153961181640625us (total: 60.153961181640625ms) +read: 1.0202760457992552ms (total: 10.202760457992554s) + +========================================================================================== +SERIALIZATION BENCHMARKS Ledger_proof.t +========================================================================================== +write: 36.144065856933594us (total: 361.44065856933594ms) +read: 51.637029647827148us (total: 516.37029647827148ms) +``` + +Taking these numbers, we can estimate the relative impact +deserialization/serialization will have on the important operations of the +daemon. + +``` +========================================================================================== +SERIALIZATION OVERHEAD ESTIMATES +========================================================================================== +zkapp command ingest = 457.58285522460938us +snark work ingest = 142.36927032470706us +block ingest = 76.7938720703125ms +block production = 844.14577026367192ms +``` + +The estimates for zkapp command and snark work ingest represent the overhead to +add a single new max cost zkapp command or snark work bundle to the on-disk +cache. The block ingest represents the cost to add resources from max cost block +to disk. This is all computed under the assumption that we take the easy-route +to implementation and just serialize all values before hashing them (so that we +can compute the hash from the serialized format), but the design of the +implementation actually allows us to avoid doing this. The block production +overhead is the amount of time the daemon would spend loading all relevant +resources from disk in order to produce a max cost block. + +### Implementation + +In order to implement this change, we will create a new `disk_cache_lib` +library. There already exists a `cache_lib` library in the codebase, but it's +design constraints and intent are different (it represents a shared in-memory +cache with explicit rules about "consuming" items from the cache). Still, the +`disk_cache_lib` library will follow a similar abstractions as `cache_lib`, +without the concerns for requiring consumption of cache items. + +A new `Disk_cache.t` type will be added, representing access to the on-disk +cache. This value will be initialized globally at daemon startup, but will only +be accessible within `disk_cache_lib` library itself. A type `'a Disk_cached.t` +will be used to represent items that are stored in the on-disk cache, where the +type parameter will is the underlying type of the value stored on-disk. A +`'a Disk_cached.t` is initialized from a first-class module that defines the +serialization, deserialization, and hashing functionality for the type it +stores, with a helper `Disk_cached.Make` functor being provided to abstract over +this (the same pattern utilized by `'a Hashtbl.t` and `'a Map.t` from `core`). + +Below is a mockup of what the `disk_cache_lib` library would look like, with +some commented code detailing the internals of the library which are not +exposed. + +```ocaml +open Core +open Async + +module Disk_cache : sig + (** Initialize the on-disk cache explicitly before interactions with it take place. *) + val initialize : unit -> unit Deferred.t + + (* type t = Lmdb.t *) + + (** Increment the cache ref count, saving a value if the ref count was 0. *) + (* val incr : t -> Disk_cached.id -> 'a Bin_prot.Type_class.t -> 'a -> unit *) + + (** Decrement the cache ref count, deleting the value if the ref count becomes 0. *) + (* val decr : t -> Disk_cached.id -> unit *) + + (** Read from the cache, crashing if the value cannot be found. *) + (* val read : t -> Disk_cached.id -> 'a Bin_prot.Type_class.t -> 'a *) +end + +module Disk_cached : sig + module type Element_intf = sig + include Binable.S + + val digest : (module Digestif.S with type ctx = 'ctx) -> 'ctx -> t -> 'ctx + end + + (* type id = Digestif.Sha256.t *) + + type 'a t (* = T : (module Element_intf with type t = 'a) * id -> 'a t *) + + (** Create a new cached reference from a value. Hashes the incoming value to check if it is already + stored in the cache, and stores it in the cache if not. This function does not keep references + to the value passed into it so that the value can be garbage collected. The caching library + tracks the total number of references created in OCaml to each value stored in the cache, and + will automatically delete the value from the cache when all references are garbage collected. + *) + val create : (module Element_intf with type t = 'a) -> 'a -> 'a t + + (** Reads from the on-disk cache. It is important that the caller does not hold onto the returned + value, otherwise they could leak the values read from the cache. + *) + val get : 'a t -> 'a + + (** Helper functor for wrapping the first-class module logic. *) + module Make : functor (Element : Element_intf) -> sig + type nonrec t = Element.t t + + val create : 'a -> t + + val get : t -> 'a + end +end +``` + +It is important that calls to `Disk_cached.get` do not hold onto the returned +value after they are done reading it. This could be somewhat enforced through a +more complex design, but it doesn't seem necessary at this time. + +To use this library, types that we want to cache on disk merely need to +implement a +`val digest : (module Digestif.S with type ctx = 'ctx) -> 'ctx -> t -> 'ctx` +function in addition to the `bin_prot` functions they already implement. The +`Disk_cached.Make` helper can (optionally) be called to create a module which +abstracts over the details of working with the polymorphic `'a Disk_cached.t` +type. This new `X.Disk_cached.t` (`X.t Disk_cache_lib.Disk_cached.t` for long +hand) type can be put in-place of the `X.t` type in any in-memory structures to +remove the normal in-memory references we would be maintaining to an `X.t`. When +we deserialize an `X.t` from the network, we must call `X.Disk_cached.create` on +that value to transfer it into an `X.Disk_cached.t`, and then throw away the +`X.t` we deserialized initially. + +## Drawbacks + +[drawbacks]: #drawbacks + +This approach requires us to perform additional disk I/O in order to offload the +data from RAM to disk. Our above analysis shows this will have a negligible +impact on the performance of a daemon, but this approach will mean we will use +more disk space than before. Based on the estimates presented above, the worst +case additional disk usage will be `~42GB`. With our initial approach, we +mitigate the risk of a disk usage leak by wiping out the on-disk cache when the +daemon restarts. + +## Rationale and alternatives + +[rationale-and-alternatives]: #rationale-and-alternatives + +The only real alternative to this approach would be to find some way to optimize +the memory impact of the proofs and verification keys without writing them to +disk, which would require some form of compression. Given proofs and +verification keys are cryptographic data, and thus have a necessarily high +degree of "randomness" in the values they contain, they are not easily +compressed via normal techniques. Even still, the space/time tradeoff of +compressing this data will be harder to argue for, given that we need to be able +to read this data during critical operations of the daemon (block production, +block ingest, snark work ingest, and zkapp command ingest). + +## Prior art + +[prior-art]: #prior-art + +We do not have prior art in the direction of on-disk caching for relief of +memory usage. We do have prior art for the LMDB implementation of OCaml, as we +already have integrated LMDB into the Bitswap work we plan to release in a soft +fork work after Berkeley. We can lean on this prior work here since we will also +be using LMDB for this on-disk cache. + +## Unresolved questions + +[unresolved-questions]: #unresolved-questions + +- Can we increase the hardware requirements to `32GB` at the point of the + Berkeley release? +- Which of the additional recommended memory optimizations should we take first + in order to bring the estimated memory usage well below the current `16GB` + memory requirement? +- How much additional buffer should we leave between our memory estimate and the + actual hardware requirements (accounting for RAM spikes and other processes on + the system)? diff --git a/website/docs/researchers/rfcs/0064-deriving-with-generics-snapps.md b/website/docs/researchers/rfcs/0064-deriving-with-generics-snapps.md new file mode 100644 index 000000000..964e084ea --- /dev/null +++ b/website/docs/researchers/rfcs/0064-deriving-with-generics-snapps.md @@ -0,0 +1,300 @@ +--- +format: md +title: "RFC 0064: Deriving With Generics Snapps" +sidebar_label: "0064 Deriving With Generics Snapps" +hide_table_of_contents: false +--- + +> **Original source:** +> [0064-deriving-with-generics-snapps.md](https://github.com/MinaProtocol/mina/blob/compatible/rfcs/0064-deriving-with-generics-snapps.md) + +## Summary + +[summary]: #summary + +This RFC introduces a mechanism for datatype generic programming (see +[motivation](#motivation) for a definition) that is easier to maintain than ppx +macros but still powerful enough for innumerable use cases in our codebase +(graphql, bridges between languages, random oracle inputs, etc.). This document +additionally decribes its specific application to Snapp parties transactions. +While this RFC will describe all ways this approach simplifies our Snapp parties +implementation, the scope of work on this project will leave some derivers for +future work. + +## Motivation + +[motivation]: #motivation + +Datatype generic programming sometimes referred to as reflection refers to a +form of abstraction for creating reusable logic that acts on a wide variety of +datatypes. Typically this is used to fold and unfold over data structures to +"automatically" implement toJson, toString, hash, equals, etc. + +Specifically with respect to Snapps transactions: We have complex nested +structures to define the new parties transaction and at the moment, we redeclare +JSON, GraphQL, JS/OCaml bridges, specifications, and TypeScript definitions in +SnarkyJS completely separately. This is error-prone, hard to maintain, and has +already introduced bugs in our Snapps implementation even before we've shipped +v1. Using datatype generic programming, we can define all of these only once, +atomically on each primitive of the datatype, never forget to update when +definitions change (the compiler will yell at us), and rely on the datatype +generic machinery to perform the structural fold and unfold for us. + +In OCaml, we'd look for some form of datatype generic programming that allows us +to fold and unfold over algebraic datatypes and records. Typically, in Mina's +codebase we have used ppx deriving macros. `[@@deriving yojson]` derives a JSON +serializer and deserializer and `[@@deriving sexp]` derives an S-expression +parser and printer. + +While PPX macros are very powerful, writing custom PPX macros is extremely +difficult in OCaml, unfortunately, and very hard to maintain. + +Luckily, folks at Jane Street have implemented a mechanism to mechanically, +generically, define folds/unfolds on arbitrary data types in OCaml using the +`[@@deriving fields]` macros that is written with "common OCaml" and anyone +familiary with Jane Street libraries in OCaml, ie. most contributors to the Mina +OCaml core, can maintain. + +```ocaml +(* to_string from the ppx_fields_conv docs *) +type t = { + dir : [ `Buy | `Sell ]; + quantity : int; + price : float; + mutable cancelled : bool; + symbol : string; +} [@@deriving fields] + +let to_string t = + let conv to_s = fun acc f -> + (sprintf "%s: %s" (Field.name f) (to_s (Field.get f t))) :: acc + in + let fs = + Fields.fold ~init:[] + ~dir:(conv (function `Buy -> "Buy" | `Sell -> "Sell")) + ~quantity:(conv Int.to_string) + ~price:(conv Float.to_string) + ~cancelled:(conv Bool.to_string) + in + String.concat fs ~sep:", " +``` + +This is a step in the right direction but we can make this more succinct with +terser combinators -- (note we use `Fields.make_creator` so we can compose +decoders with encoders) + +```ocaml +type t = { + dir : [ `Buy | `Sell ]; + quantity : int; + price : float; + mutable cancelled : bool; + symbol : string; +} [@@deriving fields] + +let to_string t = + let open To_string.Prim in + String.concat ~sep:", " @@ + (Fields.make_creator ~init:(To_string.init ()) + ~dir + ~quantity:int + ~price:float + ~cancelled:bool |> To_string.finish ()) +``` + +Further we can build combinators for horizontally composing derivers such that: + +```ocaml +(* pseudocode *) +type t = { + dir : [ `Buy | `Sell ]; + quantity : int; + price : float; + mutable cancelled : bool; + symbol : string; +} [@@deriving fields] + +let to_string, equal = + let module D = Derive.Make2(To_string)(To_equal) in + let open D.Prim in + let dir = both Dir.to_string Dir.to_yojson in + Fields.make_creator + ~init:(D.init ()) ~dir ~quantity:int ~price:float ~cancelled:bool + |> D.finish +``` + +Coupled with combinators these custom folds are almost powerful enough. + +However, sometimes we need to add metadata to our data types in order to +faithfully implement some fold. For example, we may want to provide a custom +field name in a JSON serializer or documentation for a GraphQL schema. + +Rather than pollute our data types we can settle for one extra relatively simple +companion macro that I propose we call `[@@deriving ann]` we can pull out all +the custom annotations on the datatype and finally have enough machinery for us +to cleanly implement JSON, GraphQL, specifications, random oracle inputs, +typescript definitions, and more. + +## Detailed design + +[detailed-design]: #detailed-design + +### General Framework + +See #10132 for a proposed imlementation of the machinery in the `fields_deriver` +library. + +#### Deriving Annotations + +We need to implement a `[@@deriving ann]` macro that takes a structure that +looks something like: + +```ocaml +type t = + { foo : int (** foo must be greater than zero *) + ; reserved : string [@key "_reserved"] + } +[@@deriving ann] +``` + +and produces something like (sketch): + +```ocaml +let t_ann : Ann.t Field.Map.t +``` + +where there is helper code: + +```ocaml +(* helper util code that we only need once *) +module Ann = struct + type t = + { ocaml_doc : string + ; key : string + (* any other annotations we want to capture *) + } + + module Field = struct + module T = struct + type 'a t = | E : ('a, 'b) Field.t -> 'a t + (* ... with a sort function based on the Field name *) + end + + module Map = Core_kernel.Map.Make(T) + end + + (* wraps your field in the existential wrapper for you and then does a map + lookup *) + val get : ('a, 'b) Field.t -> Ann.t Field.Map.t -> Ann.t option +end +``` + +Now we can build combinators on our field folds/unfolds + +#### Combinators + +The combinators look something like this: + +```ocaml + val int_ + val string_ + val bool_ + val list_ + + module Prim = struct + val int + val string + val bool + val list + end +``` + +The derivers in `Prim` are intended to be used directly by the `make_creator` +fold/unfold. Example: + +```ocaml +let open Prim in +Fields.make_creator (init ()) ~foo:int ~bar:string |> finish +``` + +The underscore-suffixed versions of the derivers are used whenenever types need +to be composed -- for example, when using `list` + +```ocaml +let open Prim in +Fields.make_creator (init ()) ~foo:int ~bar:(list D.string_) |> finish +``` + +More examples are present in the first PR #10132. Suggestions on naming scheme +for these is appreciated, either here or on that PR. + +### Applications for Snapps Parties (minimal) + +A minimal application of this mechanism applied to snapps transactions would be +to apply these derivers to all the types involved in the Snapps parties +transactions. + +The derivers we need at a minimum are: `To_json`, `Of_json`, `Graphql_fields`, +`Graphql_args` + +With these four derivers we can decode and encode JSON and send and receive JSON +in GraphQL requests. + +When we bridge the `to_json` over to SnarkyJS we can generate a Snapp +transaction and we'll know that it will be accepted by the GraphQL server +generated via the `Graphql_fields/args` schema. + +### Applications for Snapps Parties (phase2) + +Create derivers for TypeScript `.d.ts` parties interface types and the +OCaml/JavaScript bridge for these data types. + +### Other Applications + +Other applications to explore are specification deriving using ocaml doccomments +on data structures and random oracle input `to_input` derivation rather than +relying on HLists. + +## Drawbacks + +[drawbacks]: #drawbacks + +To fully adopt this vision, we'll need to rewrite a lot of different pieces +within Mina. Luckily, this can be done piecemeal and whenever we decide to +allocate effort toward individual sections. + +## Rationale and alternatives + +[rationale-and-alternatives]: #rationale-and-alternatives + +We could stick with PPX macros but they are too hard to write. Other sorts of +code genreators don't fit into our workfflow. + +We could also not take any sort of datatype generic approach of dealing with +this issue and instead write the same thing manually or stick with what we've +done so far -- however, as mentioned above we are already running into bugs and +are concerned about maintainability of the current implementation. + +In an effort to avoid digging ourselves further in a tech debt hole, this RFC +proposes we adopt this generic programming approach immediately. + +## Prior art + +[prior-art]: #prior-art + +TODO + +In Haskell, generic programming + +In Swift, generic programming + +Discuss prior art, both the good and the bad, in relation to this proposal. + +## Unresolved questions + +[unresolved-questions]: #unresolved-questions + +To resolve during implementation: + +- To what extent do we re-write the datastructures in the bridge using this + mechanism vs keep it scoped to GraphQL for now? diff --git a/website/docs/researchers/rfcs/index.md b/website/docs/researchers/rfcs/index.md new file mode 100644 index 000000000..881ab2a66 --- /dev/null +++ b/website/docs/researchers/rfcs/index.md @@ -0,0 +1,171 @@ +--- +sidebar_position: 1 +title: Mina Protocol RFCs +description: Request for Comments documents for the Mina Protocol +slug: /researchers/rfcs +--- + +# Mina Protocol RFCs + +This section contains all Request for Comments (RFC) documents from the +[Mina Protocol OCaml implementation](https://github.com/MinaProtocol/mina). +These RFCs document design decisions, protocol changes, and architectural +proposals for the Mina blockchain. + +RFCs serve as the primary mechanism for proposing new features, collecting +community input, and documenting design decisions. They provide valuable context +for understanding why certain architectural choices were made in the protocol. + +The original RFCs are maintained in the +[MinaProtocol/mina repository](https://github.com/MinaProtocol/mina/tree/compatible/rfcs). + +## RFC categories + +### Protocol and consensus + +Core protocol design, consensus mechanisms, and blockchain state management. + +| RFC | Title | Description | +| --------------------------------------- | --------------------- | --------------------------------------------- | +| [0006](./0006-receipt-chain-proving.md) | Receipt chain proving | Proof mechanism for transaction receipts | +| [0007](./0007-delegation-of-stake.md) | Delegation of stake | Stake delegation mechanics and implementation | +| [0019](./0019-epoch-ledger-sync.md) | Epoch ledger sync | Synchronization of epoch ledgers | +| [0030](./0030-fork-signalling.md) | Fork signalling | Mechanism for signaling protocol forks | +| [0051](./0051-protocol-versioning.md) | Protocol versioning | Versioning scheme for protocol compatibility | +| [0059](./0059-new-transaction-model.md) | New transaction model | Redesigned transaction model | + +### State management + +Transition frontier, ledger, and state persistence. + +| RFC | Title | Description | +| ------------------------------------------------------ | ------------------------------------ | ----------------------------------------- | +| [0008](./0008-persistent-ledger-builder-controller.md) | Persistent ledger builder controller | Controller for persistent ledger building | +| [0009](./0009-transition-frontier-controller.md) | Transition frontier controller | Managing the transition frontier | +| [0010](./0010-decompose-ledger-builder.md) | Decompose ledger builder | Modular ledger builder design | +| [0015](./0015-transition-frontier-extensions.md) | Transition frontier extensions | Extensions to transition frontier | +| [0016](./0016-transition-frontier-persistence.md) | Transition frontier persistence | Persisting frontier state | +| [0020](./0020-transition-frontier-extensions-2.md) | Transition frontier extensions 2 | Additional frontier extensions | +| [0026](./0026-transition-caching.md) | Transition caching | Caching strategies for transitions | +| [0028](./0028-frontier-synchronization.md) | Frontier synchronization | Synchronizing frontiers across nodes | +| [0034](./0034-reduce-scan-state-memory-usage.md) | Reduce scan state memory usage | Memory optimization for scan state | + +### Networking + +P2P communication, libp2p integration, and network architecture. + +| RFC | Title | Description | +| ------------------------------------- | ------------------- | --------------------------------- | +| [0029](./0029-libp2p.md) | libp2p | libp2p integration for networking | +| [0031](./0031-sentry-architecture.md) | Sentry architecture | Sentry node architecture design | +| [0060](./0060-networking-refactor.md) | Networking refactor | Overhauling the networking layer | +| [0062](./0062-bitswap.md) | Bitswap | Bitswap protocol integration | + +### APIs and interfaces + +GraphQL, RPC, Rosetta, and external interfaces. + +| RFC | Title | Description | +| ------------------------------------------ | ------------------------ | ------------------------------------ | +| [0013](./0013-rpc-versioning.md) | RPC versioning | Versioning scheme for RPC interfaces | +| [0021](./0021-graphql-api.md) | GraphQL API | GraphQL API for wallet communication | +| [0038](./0038-rosetta-construction-api.md) | Rosetta Construction API | Rosetta API construction endpoints | +| [0040](./0040-rosetta-timelocking.md) | Rosetta timelocking | Timelocking support in Rosetta | +| [0048](./0048-rosetta-zkapps.md) | Rosetta zkApps | zkApps support in Rosetta API | + +### Hard forks + +Hard fork procedures, disaster recovery, and data migration. + +| RFC | Title | Description | +| -------------------------------------------------- | -------------------------------- | ---------------------------------- | +| [0033](./0033-blockchain-in-hard-fork.md) | Blockchain in hard fork | Blockchain state during hard forks | +| [0035](./0035-scan-state-hard-fork.md) | Scan state hard fork | Scan state handling in hard forks | +| [0036](./0036-hard-fork-disaster-recovery.md) | Hard fork disaster recovery | Recovery procedures for hard forks | +| [0047](./0047-versioning-changes-for-hard-fork.md) | Versioning changes for hard fork | Version management during forks | +| [0053](./0053-hard-fork-package-generation.md) | Hard fork package generation | Generating hard fork packages | +| [0056](./0056-hard-fork-data-migration.md) | Hard fork data migration | Data migration during hard forks | + +### zkApps + +Zero-knowledge application features and constraints. + +| RFC | Title | Description | +| ----------------------------------------------- | ------------------------------ | --------------------------------------- | +| [0045](./0045-zkapp-balance-data-in-archive.md) | zkApp balance data in archive | Archive storage for zkApp balances | +| [0052](./0052-verification-key-permissions.md) | Verification key permissions | Permission system for verification keys | +| [0054](./0054-limit-zkapp-cmds-per-block.md) | Limit zkApp commands per block | Block-level zkApp command limits | +| [0057](./0057-hardcap-zkapp-commands.md) | Hardcap zkApp commands | Hard limits on zkApp commands | +| [0058](./0058-disable-zkapp-commands.md) | Disable zkApp commands | Mechanism to disable zkApp commands | +| [0061](./0061-solidity-snapps.md) | Solidity SNAPPs | Solidity integration for SNAPPs | +| [0064](./0064-deriving-with-generics-snapps.md) | Deriving with generics SNAPPs | Generic derivation for SNAPPs | + +### Security and validation + +Transaction pool security, ban scoring, and validation mechanisms. + +| RFC | Title | Description | +| --------------------------------------------- | ------------------------------- | ------------------------------------ | +| [0001](./0001-banlisting.md) | Banlisting | Peer banlisting mechanism | +| [0011](./0011-txpool-dos-mitigation.md) | Transaction pool DoS mitigation | Preventing DoS attacks on mempool | +| [0012](./0012-ban-scoring.md) | Ban scoring | Scoring system for peer bans | +| [0032](./0032-automated-validation.md) | Automated validation | Automated transaction validation | +| [0049](./0049-protocol-testing.md) | Protocol testing | Testing framework for protocol | +| [0055](./0055-stop-transaction-processing.md) | Stop transaction processing | Emergency transaction halt mechanism | + +### Serialization and encoding + +Data encoding, versioning, and serialization formats. + +| RFC | Title | Description | +| ---------------------------------------------- | ---------------------------- | ------------------------------------ | +| [0014](./0014-address-encoding.md) | Address encoding | Address encoding format | +| [0017](./0017-module-versioning.md) | Module versioning | Versioning for serialization modules | +| [0024](./0024-memos-with-arbitrary-bytes.md) | Memos with arbitrary bytes | Arbitrary byte support in memos | +| [0046](./0046-version-other-serializations.md) | Version other serializations | Versioning additional serializations | + +### Account features + +Time-locked accounts, delegations, and account management. + +| RFC | Title | Description | +| --------------------------------------- | --------------------- | ----------------------------- | +| [0025](./0025-time-locked-accounts.md) | Time-locked accounts | Time-based account locking | +| [0050](./0050-genesis-ledger-export.md) | Genesis ledger export | Exporting genesis ledger data | + +### Infrastructure and operations + +Node status, logging, and operational tooling. + +| RFC | Title | Description | +| ---------------------------------------------------- | ---------------------------------- | ----------------------------------- | +| [0018](./0018-better-logging.md) | Better logging | Improved logging infrastructure | +| [0039](./0039-snark-keys-management.md) | SNARK keys management | Managing SNARK proving keys | +| [0041](./0041-infra-testnet-persistence.md) | Infrastructure testnet persistence | Persistent testnet infrastructure | +| [0042](./0042-node-status-collection.md) | Node status collection | Collecting node status data | +| [0043](./0043-node-error-collection.md) | Node error collection | Collecting node error data | +| [0044](./0044-node-status-and-node-error-backend.md) | Node status and error backend | Backend for status/error collection | +| [0063](./0063-reducing-daemon-memory-usage.md) | Reducing daemon memory usage | Memory optimization for daemon | + +### Development processes + +Style guides, naming conventions, and development workflows. + +| RFC | Title | Description | +| --------------------------------------------- | --------------------------- | ------------------------------- | +| [0000](./0000-template.md) | Template | RFC template for new proposals | +| [0002](./0002-branch-prefixes.md) | Branch prefixes | Git branch naming conventions | +| [0003](./0003-renaming-refactor.md) | Renaming refactor | Code renaming guidelines | +| [0004](./0004-style-guidelines.md) | Style guidelines | Code style guidelines | +| [0005](./0005-issue-labels.md) | Issue labels | GitHub issue labeling scheme | +| [0022](./0022-postake-naming-conventions.md) | PoStake naming conventions | Naming conventions for PoS code | +| [0023](./0023-glossary-terms.md) | Glossary terms | Protocol terminology glossary | +| [0027](./0027-wallet-internationalization.md) | Wallet internationalization | i18n support for wallet | +| [0037](./0037-github-merging-strategy.md) | GitHub merging strategy | Git merge workflow | + +## Additional resources + +- [RFC Repository](https://github.com/MinaProtocol/mina/tree/compatible/rfcs) - + Original RFC source files on GitHub +- [Mina Protocol Documentation](https://docs.minaprotocol.com/) - Official + protocol documentation diff --git a/website/docusaurus.config.ts b/website/docusaurus.config.ts index 7a75cdff3..5e05d6e10 100644 --- a/website/docusaurus.config.ts +++ b/website/docusaurus.config.ts @@ -30,6 +30,7 @@ const config: Config = { // Markdown configuration markdown: { + format: 'detect', hooks: { onBrokenMarkdownLinks: 'throw', }, diff --git a/website/sidebars.ts b/website/sidebars.ts index ee70026b0..df4ce8106 100644 --- a/website/sidebars.ts +++ b/website/sidebars.ts @@ -200,6 +200,20 @@ const sidebars: SidebarsConfig = { 'researchers/specs/schnorr-signatures', ], }, + { + type: 'category', + label: 'RFCs', + link: { + type: 'doc', + id: 'researchers/rfcs/index', + }, + items: [ + { + type: 'autogenerated', + dirName: 'researchers/rfcs', + }, + ], + }, ], // Appendix sidebar - general reference material