Feature: add rkyv support

Cargo.toml

@@ -32,6 +32,7 @@ pretty_assertions  = { version = "1.0.0" }
 proc-macro2        = { version = "1.0" }
 quote              = { version = "1.0" }
 rand               = { version = "0.9" }
+rkyv               = { version = "0.8.15" }
 semver             = { version = "1.0.14" }
 serde              = { version = "1.0.114", features = ["derive", "rc"] }
 serde_json         = { version = "1.0.57" }
@@ -82,10 +83,13 @@ exclude = [
 
     "examples/log-mem",
     "examples/sm-mem",
+    "examples/sm-mem-rkyv",
     "examples/rocksstore",
     "examples/types-kv",
+    "examples/types-kv-rkyv",
 
     "examples/raft-kv-memstore",
+    "examples/raft-kv-memstore-rkyv",
     "examples/raft-kv-memstore-grpc",
     "examples/raft-kv-memstore-single-threaded",
     "examples/raft-kv-memstore-network-v2",

examples/README.md

@@ -17,6 +17,7 @@ This directory contains example applications demonstrating different implementat
 | Example | Log | State Machine | RaftNetwork Impl | RaftNetwork | Client | Server | Special Features |
 |---------|-----|---------------|------------------|-------------|--------|--------|------------------|
 | [raft-kv-memstore] | [log-mem] | [sm-mem] | HTTP/reqwest | RaftNetwork | reqwest | actix-web | Basic example |
+| [raft-kv-memstore-rkyv] | [log-mem] | [sm-mem-rkyv] | HTTP/reqwest | RaftNetwork | reqwest | actix-web | `rkyv` snapshots + zero-copy `access` |
 | [raft-kv-rocksdb] | [rocksstore] | [rocksstore] | HTTP/reqwest([network-v1]) | RaftNetwork | reqwest | actix-web | Persistent storage |
 | [raft-kv-memstore-network-v2] | [log-mem] | [sm-mem] | HTTP/reqwest | RaftNetworkV2 | reqwest | actix-web | Network V2 interface |
 | [multi-raft-kv] | [log-mem] | [sm-mem] | HTTP/channel | GroupRouter | channel | in-memory | Multi-Raft groups |
@@ -30,6 +31,7 @@ This directory contains example applications demonstrating different implementat
 ### Storage Implementations
 - **[log-mem]** - In-memory Raft Log Store using `std::collections::BTreeMap`
 - **[sm-mem]** - In-memory KV State Machine implementation
+- **[sm-mem-rkyv]** - In-memory KV State Machine implementation with `rkyv` snapshots
 - **[rocksstore]** - RocksDB-based persistent storage using `rocksdb` crate
 
 ### Backward Compatibility (since 0.10)
@@ -44,10 +46,12 @@ The following symbolic links are provided for backward compatibility:
 
 ### Utilities
 - **[types-kv]** - Shared KV request/response types for example crates
+- **[types-kv-rkyv]** - Shared KV request/response types serialized with `rkyv`
 - **[utils]** - Shared type declarations and utilities
 
 <!-- Reference Links -->
 [raft-kv-memstore]: raft-kv-memstore/
+[raft-kv-memstore-rkyv]: raft-kv-memstore-rkyv/
 [raft-kv-rocksdb]: raft-kv-rocksdb/
 [raft-kv-memstore-network-v2]: raft-kv-memstore-network-v2/
 [raft-kv-memstore-grpc]: raft-kv-memstore-grpc/
@@ -56,9 +60,11 @@ The following symbolic links are provided for backward compatibility:
 [multi-raft-kv]: multi-raft-kv/
 [log-mem]: log-mem/
 [sm-mem]: sm-mem/
+[sm-mem-rkyv]: sm-mem-rkyv/
 [rocksstore]: rocksstore/
 [network-v1]: network-v1-http/
 [types-kv]: types-kv/
+[types-kv-rkyv]: types-kv-rkyv/
 [utils]: utils/
 
 [memstore]: memstore/

examples/raft-kv-memstore-rkyv/.gitignore

@@ -0,0 +1,5 @@
+target
+vendor
+.idea
+
+/*.log

examples/raft-kv-memstore-rkyv/Cargo.toml

@@ -0,0 +1,47 @@
+[package]
+name = "raft-kv-memstore-rkyv"
+version = "0.1.0"
+readme = "README.md"
+
+edition = "2024"
+authors = [
+    "drdr xp <drdr.xp@gmail.com>",
+    "Pedro Paulo de Amorim <pepa.amorim@gmail.com>",
+]
+categories = ["algorithms", "asynchronous", "data-structures"]
+description = "An example distributed key-value store built upon `openraft` with rkyv snapshots."
+homepage = "https://github.com/databendlabs/openraft"
+keywords = ["raft", "consensus"]
+license = "MIT OR Apache-2.0"
+repository = "https://github.com/databendlabs/openraft"
+
+[[bin]]
+name = "raft-key-value-rkyv"
+path = "src/bin/main.rs"
+
+[dependencies]
+client-http        = { path = "../client-http" }
+log-mem            = { path = "../log-mem" }
+network-v1-http    = { path = "../network-v1-http" }
+sm-mem-rkyv        = { path = "../sm-mem-rkyv" }
+openraft           = { path = "../../openraft", features = ["serde", "rkyv", "type-alias"] }
+openraft-legacy    = { path = "../../legacy" }
+types-kv           = { path = "../types-kv" }
+
+actix-web          = { version = "4.0.0-rc.2" }
+clap               = { version = "4.1.11", features = ["derive", "env"] }
+futures            = { version = "0.3" }
+reqwest            = { version = "0.12.5", features = ["json"] }
+serde              = { version = "1.0.114", features = ["derive"] }
+serde_json         = { version = "1.0.57" }
+tracing            = { version = "0.1.29" }
+tracing-subscriber = { version = "0.3.0", features = ["env-filter"] }
+
+[dev-dependencies]
+anyhow = { version = "1.0.63" }
+maplit = { version = "1.0.2" }
+
+[features]
+
+[package.metadata.docs.rs]
+all-features = true

examples/raft-kv-memstore-rkyv/README.md

@@ -0,0 +1,145 @@
+# Example distributed key-value store built upon openraft (rkyv snapshots).
+
+It is an example of how to build a real-world key-value store with `openraft`.
+Includes:
+- An in-memory `RaftLogStorage` and `RaftStateMachine` implementation [store](./src/store/mod.rs).
+- Snapshot serialization with `rkyv`, including zero-copy validation/access via
+  [`rkyv::access`](https://docs.rs/rkyv/latest/rkyv/fn.access.html) in
+  [`sm-mem-rkyv`](../sm-mem-rkyv/src/lib.rs).
+
+- A server is based on [actix-web](https://docs.rs/actix-web/4.0.0-rc.2).
+  All service endpoints accept a `Json<I>` input argument,
+  and return a JSON-encoded `Result<T, E>` response,
+  where `T` and `E` are API-specific types.
+  For example, the `/write` endpoint accepts a user-defined `Request`
+  and returns a `Result<WriteResponse, ClientWriteError>`.
+
+  Includes:
+  - raft-internal network APIs for replication and voting.
+  - Admin APIs to add nodes, change-membership etc.
+  - Application APIs to write a value by key or read a value by key.
+  - Linearizable read implementations including follower reads.
+
+- Client and `RaftNetwork`([rpc](./src/network/raft.rs)) are built upon [reqwest](https://docs.rs/reqwest).
+
+  [ExampleClient](./src/client.rs) is a minimal raft client in rust to talk to a raft cluster.
+  - It includes application API `write()`, `read()`, `linearizable_read()`, `follower_read()`, and administrative API `init()`, `add_learner()`, `change_membership()`, `metrics()`.
+  - This client tracks the last known leader id, a write operation(such as `write()` or `change_membership()`) will be redirected to the leader on client side.
+
+## Run it
+
+There is a example in bash script and an example in rust:
+
+- [test-cluster.sh](./test-cluster.sh) shows a simulation of 3 nodes running and sharing data,
+  It only uses `curl` and shows the communication between a client and the cluster in plain HTTP messages.
+  You can run the cluster demo with:
+
+  ```shell
+  ./test-cluster.sh
+  ```
+
+- [test_cluster.rs](./tests/cluster/test_cluster.rs) does almost the same as `test-cluster.sh` but in rust
+  with the `ExampleClient`.
+
+  Run it with `cargo test`.
+
+
+if you want to compile the application, run:
+
+```shell
+cargo build
+```
+
+(If you append `--release` to make it compile in production, but we don't recommend to use
+this project in production yet.)
+
+## What the test script does
+
+To run it, get the binary `raft-key-value-rkyv` inside `target/debug` and run:
+
+```shell
+./raft-key-value-rkyv --id 1 --http-addr 127.0.0.1:21001
+```
+
+It will start a node.
+
+To start the following nodes:
+
+```shell
+./raft-key-value-rkyv --id 2 --http-addr 127.0.0.1:21002
+```
+
+You can continue replicating the nodes by changing the `id` and `http-addr`.
+
+After that, call the first node created:
+
+```
+POST - 127.0.0.1:21001/init
+```
+
+It will define the first node created as the leader.
+
+Then you need to inform to the leader that these nodes are learners:
+
+```
+POST - 127.0.0.1:21001/add-learner '[2, "127.0.0.1:21002"]'
+POST - 127.0.0.1:21001/add-learner '[3, "127.0.0.1:21003"]'
+```
+
+Now you need to tell the leader to add all learners as members of the cluster:
+
+```
+POST - 127.0.0.1:21001/change-membership  "[1, 2, 3]"
+```
+
+Write some data in any of the nodes:
+
+```
+POST - 127.0.0.1:21001/write  "{"Set":{"key":"foo","value":"bar"}}"
+```
+
+Read the data from any node:
+
+```
+POST - 127.0.0.1:21002/read  "foo"
+```
+
+You should be able to read that on the another instance even if you did not sync any data!
+
+For linearizable reads, use the `/linearizable_read` endpoint on the leader, or `/follower_read` on any node to distribute read load:
+
+```
+POST - 127.0.0.1:21001/linearizable_read  "foo"
+POST - 127.0.0.1:21002/follower_read  "foo"
+```
+
+
+## How it's structured.
+
+The application is separated in 4 modules:
+
+ - `bin`: You can find the `main()` function in [main](./src/bin/main.rs) the file where the setup for the server happens.
+ - `network`: You can find the [api](./src/network/api.rs) that implements the endpoints used by the public API and [rpc](./src/network/raft.rs) where all the raft communication from the node happens. [management](./src/network/management.rs) is where all the administration endpoints are present, those are used to add orremove nodes, promote and more. [raft](./src/network/raft.rs) is where all the communication are received from other nodes.
+ - `store`: You can find the file [store](./src/store/mod.rs) where all the key-value implementation is done. Here is where your data application will be managed.
+
+## Where is my data?
+
+The data is store inside state machines, each state machine represents a point of data and
+raft enforces that all nodes have the same data in synchronization. You can have a look of
+the struct [ExampleStateMachine](./src/store/mod.rs)
+
+## Cluster management
+
+The raft itself does not store node addresses.
+But in a real-world application, the implementation of `RaftNetwork` needs to know the addresses.
+
+Thus, in this example application:
+
+- The storage layer has to store nodes' information.
+- The network layer keeps a reference to the store so that it is able to get the address of a target node to send RPC to.
+
+To add a node to a cluster, it includes 3 steps:
+
+- Write a `node` through raft protocol to the storage.
+- Add the node as a `Learner` to let it start receiving replication data from the leader.
+- Invoke `change-membership` to change the learner node to a member.

examples/raft-kv-memstore-rkyv/src/app.rs

@@ -0,0 +1,12 @@
+use crate::NodeId;
+use crate::Raft;
+use crate::StateMachineStore;
+
+// Representation of an application state. This struct can be shared around to share
+// instances of raft, store and more.
+pub struct App {
+    pub id: NodeId,
+    pub addr: String,
+    pub raft: Raft,
+    pub state_machine_store: StateMachineStore,
+}

examples/raft-kv-memstore-rkyv/src/bin/main.rs

@@ -0,0 +1,30 @@
+use clap::Parser;
+use raft_kv_memstore_rkyv::start_example_raft_node;
+use tracing_subscriber::EnvFilter;
+
+#[derive(Parser, Clone, Debug)]
+#[clap(author, version, about, long_about = None)]
+pub struct Opt {
+    #[clap(long)]
+    pub id: u64,
+
+    #[clap(long)]
+    pub http_addr: String,
+}
+
+#[actix_web::main]
+async fn main() -> std::io::Result<()> {
+    // Setup the logger
+    tracing_subscriber::fmt()
+        .with_target(true)
+        .with_thread_ids(true)
+        .with_level(true)
+        .with_ansi(false)
+        .with_env_filter(EnvFilter::from_default_env())
+        .init();
+
+    // Parse the parameters passed by arguments.
+    let options = Opt::parse();
+
+    start_example_raft_node(options.id, options.http_addr).await
+}