diff --git a/README.md b/README.md index d9e7293..8e1cae1 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ The packages in this library implement the various components of the CONIKS syst - ``client``: Client-side protocol cgo hooks - ``crypto``: Cryptographic algorithms and operations - ``merkletree``: Merkle prefix tree and related data structures +- ``storage``: DB hooks for storage backend (currently the library supports key-value db only) - ``utils``: Utility functions - ``storage``: DB hooks for storage backend (currently the library supports key-value db only) diff --git a/client/cgo/cgotest.go b/client/cgo/cgotest.go index 36c3024..1971b95 100644 --- a/client/cgo/cgotest.go +++ b/client/cgo/cgotest.go @@ -103,7 +103,7 @@ func testVerifyHashChain(t *testing.T) { if err != nil { t.Fatal(err) } - pad, err := merkletree.NewPAD(merkletree.NewPolicies(2, vrfPrivKey), signKey, 10) + pad, err := merkletree.NewPAD(merkletree.NewPolicies(2, vrfPrivKey), nil, signKey, 10) if err != nil { t.Fatal(err) } @@ -133,7 +133,7 @@ func testVerifyAuthPath(t *testing.T) { if err != nil { t.Fatal(err) } - pad, err := merkletree.NewPAD(merkletree.NewPolicies(2, vrfPrivKey), signKey, 10) + pad, err := merkletree.NewPAD(merkletree.NewPolicies(2, vrfPrivKey), nil, signKey, 10) if err != nil { t.Fatal(err) } @@ -209,7 +209,7 @@ func testVerifyProofOfAbsenceSamePrefix(t *testing.T) { if err != nil { t.Fatal(err) } - pad, err := merkletree.NewPAD(merkletree.NewPolicies(2, vrfPrivKey), signKey, 10) + pad, err := merkletree.NewPAD(merkletree.NewPolicies(2, vrfPrivKey), nil, signKey, 10) if err != nil { t.Fatal(err) } diff --git a/merkletree/merkletree.go b/merkletree/merkletree.go index d0bd6aa..3035bca 100644 --- a/merkletree/merkletree.go +++ b/merkletree/merkletree.go @@ -9,12 +9,18 @@ import ( ) var ( - ErrorInvalidTree = errors.New("[merkletree] invalid tree") + ErrorInvalidTree = errors.New("[merkletree] Invalid tree") ) const ( - EmptyBranchIdentifier = 'E' - LeafIdentifier = 'L' + EmptyBranchIdentifier = 'E' + LeafIdentifier = 'L' + InteriorNodeIdentifier = 'I' + NodeKeyIdentifier = 'N' + STRIdentifier = 'S' + PoliciesIdentifier = 'P' + EpochIdentifier = 'O' + TreeNonceIdentifier = 'T' ) type MerkleTree struct { diff --git a/merkletree/merkletreekv.go b/merkletree/merkletreekv.go new file mode 100644 index 0000000..4e5ccc4 --- /dev/null +++ b/merkletree/merkletreekv.go @@ -0,0 +1,104 @@ +package merkletree + +import ( + "errors" + + "github.com/coniks-sys/coniks-go/crypto" + "github.com/coniks-sys/coniks-go/storage/kv" + "github.com/coniks-sys/coniks-go/utils" +) + +var ( + ErrorBadTreeNonce = errors.New("[merkletree] Bad tree nonce") +) + +func NewMerkleTreeFromKV(db kv.DB, epoch uint64) (*MerkleTree, error) { + nonceKey := append([]byte{TreeNonceIdentifier}, util.ULongToBytes(epoch)...) + val, err := db.Get(nonceKey) + var nonce []byte + if err != nil { + return nil, err + } else if len(val) != crypto.HashSizeByte { + return nil, ErrorBadTreeNonce + } else { + nonce = val + } + m := new(MerkleTree) + m.nonce = nonce + root, err := m.reconstructTree(db, nil, epoch, []bool{}) + if err != nil { + return nil, err + } + m.root = root.(*interiorNode) + m.hash = m.root.Hash(m) + return m, nil +} + +func (m *MerkleTree) reconstructTree(db kv.DB, parent MerkleNode, epoch uint64, prefixBits []bool) (MerkleNode, error) { + n, err := loadNode(db, epoch, prefixBits) + if err != nil { + return nil, err + } + + if _, ok := n.(*emptyNode); ok { + n.(*emptyNode).parent = parent + return n, nil + } + if _, ok := n.(*userLeafNode); ok { + n.(*userLeafNode).parent = parent + return n, nil + } + n.(*interiorNode).parent = parent + n.(*interiorNode).leftChild, err = m.reconstructTree(db, n, epoch, append(prefixBits, false)) + if err != nil { + return nil, err + } + n.(*interiorNode).rightChild, err = m.reconstructTree(db, n, epoch, append(prefixBits, true)) + if err != nil { + return nil, err + } + return n, nil +} + +func ReconstructBranch(db kv.DB, epoch uint64, index []byte) (*MerkleTree, error) { + indexBits := util.ToBits(index) + m := new(MerkleTree) + root, err := loadNode(db, epoch, []bool{}) + if err != nil { + return nil, err + } + m.root = root.(*interiorNode) + var parent = root +loadingLoop: + for depth := 1; depth < len(indexBits); depth++ { + n, err := loadNode(db, epoch, indexBits[:depth]) + if err != nil { + return nil, err + } + if indexBits[depth-1] { + parent.(*interiorNode).rightChild = n + } else { + parent.(*interiorNode).leftChild = n + } + switch n.(type) { + case *userLeafNode: + n.(*userLeafNode).parent = parent + break loadingLoop + case *emptyNode: + n.(*emptyNode).parent = parent + break loadingLoop + case *interiorNode: + n.(*interiorNode).parent = parent + } + parent = n + } + + return m, nil +} + +func (m *MerkleTree) StoreToKV(epoch uint64, wb kv.Batch) { + // store tree nodes + m.root.storeToKV(epoch, []bool{}, wb) + // store tree nonce + wb.Put(append([]byte{TreeNonceIdentifier}, util.ULongToBytes(epoch)...), m.nonce) +} diff --git a/merkletree/merkletreekv_test.go b/merkletree/merkletreekv_test.go new file mode 100644 index 0000000..c2131f5 --- /dev/null +++ b/merkletree/merkletreekv_test.go @@ -0,0 +1,128 @@ +package merkletree + +import ( + "bytes" + "testing" + + "github.com/coniks-sys/coniks-go/storage/kv" + "github.com/coniks-sys/coniks-go/utils" +) + +func TestTreeStore(t *testing.T) { + util.WithDB(func(db kv.DB) { + key1 := "key1" + val1 := []byte("value1") + key2 := "key2" + val2 := []byte("value2") + + m1, err := NewMerkleTree() + if err != nil { + t.Fatal(err) + } + index1 := vrfPrivKey1.Compute([]byte(key1)) + if err := m1.Set(index1, key1, val1); err != nil { + t.Fatal(err) + } + index2 := vrfPrivKey1.Compute([]byte(key2)) + if err := m1.Set(index2, key2, val2); err != nil { + t.Fatal(err) + } + m1.recomputeHash() + + wb := db.NewBatch() + m1.StoreToKV(1, wb) + err = db.Write(wb) + if err != nil { + t.Fatal(err) + } + + m2, err := NewMerkleTreeFromKV(db, 1) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(m2.nonce, m1.nonce) || + !bytes.Equal(m2.root.Hash(m1), m1.hash) { + t.Fatal("Bad tree construction") + } + + ap := m2.Get(index1) + if ap.Leaf.Value() == nil { + t.Error("Cannot find key:", key1) + return + } + if !bytes.Equal(ap.Leaf.Value(), val1) { + t.Error(key1, "value mismatch") + } + + ap = m2.Get(index2) + if ap.Leaf.Value() == nil { + t.Error("Cannot find key:", key2) + return + } + if !bytes.Equal(ap.Leaf.Value(), val2) { + t.Error(key2, "value mismatch") + } + }) +} + +func TestReconstructBranch(t *testing.T) { + util.WithDB(func(db kv.DB) { + key1 := "key1" + val1 := []byte("value1") + key2 := "key3" + val2 := []byte("value2") + + m1, err := NewMerkleTree() + if err != nil { + t.Fatal(err) + } + index1 := vrfPrivKey1.Compute([]byte(key1)) + if err := m1.Set(index1, key1, val1); err != nil { + t.Fatal(err) + } + index2 := vrfPrivKey1.Compute([]byte(key2)) + if err := m1.Set(index2, key2, val2); err != nil { + t.Fatal(err) + } + m1.recomputeHash() + + wb := db.NewBatch() + m1.StoreToKV(1, wb) + err = db.Write(wb) + if err != nil { + t.Fatal(err) + } + + if err != nil { + t.Fatal(err) + } + + m2_1, err := ReconstructBranch(db, 1, index1) + if err != nil { + t.Fatal(err) + } + ap := m2_1.Get(index1) + if ap.Leaf.Value() == nil { + t.Error("Cannot find key:", key1) + return + } + if !bytes.Equal(ap.Leaf.Value(), val1) { + t.Error(key1, "value mismatch", + "want", val1, + "get", ap.Leaf.Value()) + } + + m2_2, err := ReconstructBranch(db, 1, index2) + if err != nil { + t.Fatal(err) + } + ap = m2_2.Get(index2) + if ap.Leaf.Value() == nil { + t.Error("Cannot find key:", key2) + return + } + if !bytes.Equal(ap.Leaf.Value(), val2) { + t.Error(key2, "value mismatch") + } + }) +} diff --git a/merkletree/node.go b/merkletree/node.go index 73db757..8172ccd 100644 --- a/merkletree/node.go +++ b/merkletree/node.go @@ -2,6 +2,7 @@ package merkletree import ( "github.com/coniks-sys/coniks-go/crypto" + "github.com/coniks-sys/coniks-go/storage/kv" "github.com/coniks-sys/coniks-go/utils" ) @@ -70,6 +71,7 @@ type MerkleNode interface { isEmpty() bool Hash(*MerkleTree) []byte Clone(*interiorNode) MerkleNode + storeToKV(epoch uint64, prefixBits []bool, wb kv.Batch) } var _ MerkleNode = (*userLeafNode)(nil) @@ -147,11 +149,11 @@ func (n *emptyNode) Clone(parent *interiorNode) MerkleNode { } } -func (n *userLeafNode) isEmpty() bool { +func (n *interiorNode) isEmpty() bool { return false } -func (n *interiorNode) isEmpty() bool { +func (n *userLeafNode) isEmpty() bool { return false } diff --git a/merkletree/nodekv.go b/merkletree/nodekv.go new file mode 100644 index 0000000..72bf213 --- /dev/null +++ b/merkletree/nodekv.go @@ -0,0 +1,132 @@ +package merkletree + +import ( + "encoding/binary" + "errors" + + "github.com/coniks-sys/coniks-go/crypto" + "github.com/coniks-sys/coniks-go/crypto/vrf" + "github.com/coniks-sys/coniks-go/storage/kv" + "github.com/coniks-sys/coniks-go/utils" +) + +var ( + ErrorBadNodeIdentifier = errors.New("[merkletree] Bad node identifier") +) + +func serializeKVKey(epoch uint64, prefixBits []bool) []byte { + // NodeKeyIdentifier + epoch + len(prefixBits) + index + index := util.ToBytes(prefixBits) + key := make([]byte, 0, 1+8+4+len(index)) + key = append(key, NodeKeyIdentifier) + key = append(key, util.ULongToBytes(epoch)...) + key = append(key, util.IntToBytes(len(prefixBits))...) + key = append(key, index...) + return key +} + +// storeToKV stores an interiorNode into db as following scheme: +// [identifier, level, leftHash, rightHash] +func (n *interiorNode) storeToKV(epoch uint64, prefixBits []bool, wb kv.Batch) { + buf := make([]byte, 0, 1+4+crypto.HashSizeByte*2) + buf = append(buf, InteriorNodeIdentifier) + buf = append(buf, util.IntToBytes(n.level)...) + buf = append(buf, n.leftHash...) + buf = append(buf, n.rightHash...) + + wb.Put(serializeKVKey(epoch, prefixBits), buf) + n.leftChild.storeToKV(epoch, append(prefixBits, false), wb) + n.rightChild.storeToKV(epoch, append(prefixBits, true), wb) +} + +// storeToKV stores a userLeafNode into db as following scheme: +// [identifier, level, key+len, value+len, salt, index, commitment] +func (n *userLeafNode) storeToKV(epoch uint64, prefixBits []bool, wb kv.Batch) { + buf := make([]byte, 0, 1+4+crypto.HashSizeByte*2+vrf.Size+len(n.key)+len(n.value)+4+4) + buf = append(buf, LeafIdentifier) + buf = append(buf, util.IntToBytes(n.level)...) + buf = append(buf, util.IntToBytes(len(n.key))...) + buf = append(buf, []byte(n.key)...) + buf = append(buf, util.IntToBytes(len(n.value))...) + buf = append(buf, n.value...) + buf = append(buf, n.salt...) + buf = append(buf, n.index...) + buf = append(buf, n.commitment...) + + wb.Put(serializeKVKey(epoch, prefixBits), buf) +} + +// storeToKV stores an emptyNode into db as following scheme: +// [identifier, level, index] +func (n *emptyNode) storeToKV(epoch uint64, prefixBits []bool, wb kv.Batch) { + buf := make([]byte, 0, 1+4+len(n.index)) + buf = append(buf, EmptyBranchIdentifier) + buf = append(buf, util.IntToBytes(n.level)...) + buf = append(buf, n.index...) + + wb.Put(serializeKVKey(epoch, prefixBits), buf) +} + +func loadNode(db kv.DB, epoch uint64, prefixBits []bool) (MerkleNode, error) { + nodeBytes, err := db.Get(serializeKVKey(epoch, prefixBits)) + if err == db.ErrNotFound() { + return nil, nil + } else if err != nil { + return nil, err + } + + return deserializeNode(nodeBytes), nil +} + +func deserializeNode(buf []byte) MerkleNode { + switch buf[0] { + case InteriorNodeIdentifier: + buf = buf[1:] + n := new(interiorNode) + n.level = int(binary.LittleEndian.Uint32(buf[:4])) + buf = buf[4:] + n.leftHash = append([]byte{}, buf[:crypto.HashSizeByte]...) + buf = buf[crypto.HashSizeByte:] + n.rightHash = append([]byte{}, buf[:crypto.HashSizeByte]...) + buf = buf[crypto.HashSizeByte:] + if len(buf) != 0 { + panic(kv.ErrorBadBufferLength) + } + return n + case LeafIdentifier: + buf = buf[1:] + n := new(userLeafNode) + n.level = int(binary.LittleEndian.Uint32(buf[:4])) + buf = buf[4:] + keyLen := int(binary.LittleEndian.Uint32(buf[:4])) + buf = buf[4:] + n.key = string(buf[:keyLen]) + buf = buf[keyLen:] + valueLen := int(binary.LittleEndian.Uint32(buf[:4])) + buf = buf[4:] + n.value = buf[:valueLen] + buf = buf[valueLen:] + n.salt = buf[:crypto.HashSizeByte] + buf = buf[crypto.HashSizeByte:] + n.index = buf[:crypto.HashSizeByte] + buf = buf[crypto.HashSizeByte:] + n.commitment = buf[:crypto.HashSizeByte] + buf = buf[crypto.HashSizeByte:] + if len(buf) != 0 { + panic(kv.ErrorBadBufferLength) + } + return n + case EmptyBranchIdentifier: + buf = buf[1:] + n := new(emptyNode) + n.level = int(binary.LittleEndian.Uint32(buf[:4])) + buf = buf[4:] + n.index = buf[:] + buf = buf[len(n.index):] + if len(buf) != 0 { + panic(kv.ErrorBadBufferLength) + } + return n + } + panic(ErrorBadNodeIdentifier) +} diff --git a/merkletree/nodekv_test.go b/merkletree/nodekv_test.go new file mode 100644 index 0000000..a9e967f --- /dev/null +++ b/merkletree/nodekv_test.go @@ -0,0 +1,113 @@ +package merkletree + +import ( + "bytes" + "testing" + + "github.com/coniks-sys/coniks-go/storage/kv" + "github.com/coniks-sys/coniks-go/utils" +) + +func TestNodeSerializationAndDeserialization(t *testing.T) { + util.WithDB(func(db kv.DB) { + m, err := NewMerkleTree() + if err != nil { + t.Fatal(err) + } + + key1 := "key1" + val1 := []byte("value1") + key2 := "key3" + val2 := []byte("value2") + index1 := vrfPrivKey1.Compute([]byte(key1)) + if err := m.Set(index1, key1, val1); err != nil { + t.Fatal(err) + } + + index2 := vrfPrivKey1.Compute([]byte(key2)) + if err := m.Set(index2, key2, val2); err != nil { + t.Fatal(err) + } + + m.recomputeHash() + + ap := m.Get(index1) + if ap.Leaf.Value() == nil { + t.Fatal("Cannot find key:", key1) + } + + enWant, ok := m.root.leftChild.(*emptyNode) + if !ok { + t.Fatal("Bad type insertion") + } + inWant, ok := m.root.rightChild.(*interiorNode) + if !ok { + t.Fatal("Bad type insertion") + } + lnWant := ap.Leaf.(*userLeafNode) + + wb := db.NewBatch() + enWant.storeToKV(1, []bool{false}, wb) + inWant.storeToKV(1, []bool{true}, wb) + lnWant.storeToKV(1, util.ToBits(lnWant.index)[:lnWant.level], wb) + err = db.Write(wb) + if err != nil { + t.Fatal(err) + } + + // test empty node + n, err := loadNode(db, 1, []bool{false}) + if err != nil { + t.Fatal(err) + } + enGot, ok := n.(*emptyNode) + if !ok { + t.Fatal("Bad type assertion") + } + if enGot.level != enWant.level || + !bytes.Equal(enGot.index, enWant.index) { + t.Fatal("Bad de/serialization", + "expect", enWant.level, + "got", enGot.level, + "expect", enWant.index, + "got", enGot.index) + } + + // test interior node + n, err = loadNode(db, 1, []bool{true}) + if err != nil { + t.Fatal(err) + } + inGot, ok := n.(*interiorNode) + if !ok { + t.Fatal("Bad type assertion") + } + if inGot.level != inWant.level || + !bytes.Equal(inGot.leftHash, inWant.leftHash) || + !bytes.Equal(inGot.rightHash, inWant.rightHash) { + t.Fatal("Bad de/serialization", + "expect", inWant, + "got", inGot) + } + + // test leaf node + n, err = loadNode(db, 1, util.ToBits(lnWant.index)[:lnWant.level]) + if err != nil { + t.Fatal(err) + } + lnGot, ok := n.(*userLeafNode) + if !ok { + t.Fatal("Bad type assertion") + } + if lnGot.level != lnWant.level || + !bytes.Equal(lnGot.index, lnWant.index) || + lnGot.key != lnWant.key || + !bytes.Equal(lnGot.value, lnWant.value) || + !bytes.Equal(lnGot.salt, lnWant.salt) || + !bytes.Equal(lnGot.commitment, lnWant.commitment) { + t.Fatal("Bad de/serialization", + "expect", inWant, + "got", inGot) + } + }) +} diff --git a/merkletree/pad.go b/merkletree/pad.go index 4cdbadb..4e205e7 100644 --- a/merkletree/pad.go +++ b/merkletree/pad.go @@ -7,6 +7,7 @@ import ( "github.com/coniks-sys/coniks-go/crypto" "github.com/coniks-sys/coniks-go/crypto/sign" "github.com/coniks-sys/coniks-go/crypto/vrf" + "github.com/coniks-sys/coniks-go/storage/kv" ) var ( @@ -22,11 +23,12 @@ type PAD struct { loadedEpochs []uint64 // slice of epochs in snapshots latestSTR *SignedTreeRoot policies Policies // the current policies in place + db kv.DB } // NewPAD creates new PAD consisting of an array of hash chain -// indexed by the epoch and its maximum length is len -func NewPAD(policies Policies, signKey sign.PrivateKey, len uint64) (*PAD, error) { +// indexed by the epoch and its maximum length is length +func NewPAD(policies Policies, db kv.DB, signKey sign.PrivateKey, length uint64) (*PAD, error) { if policies == nil { panic(ErrorNilPolicies) } @@ -37,9 +39,10 @@ func NewPAD(policies Policies, signKey sign.PrivateKey, len uint64) (*PAD, error if err != nil { return nil, err } + pad.db = db pad.policies = policies - pad.snapshots = make(map[uint64]*SignedTreeRoot, len) - pad.loadedEpochs = make([]uint64, 0, len) + pad.snapshots = make(map[uint64]*SignedTreeRoot, length) + pad.loadedEpochs = make([]uint64, 0, length) pad.updateInternal(nil, 0) return pad, nil } @@ -78,6 +81,8 @@ func (pad *PAD) updateInternal(policies Policies, epoch uint64) { pad.reshuffle() } } + + pad.StoreToKV(epoch) } func (pad *PAD) Update(policies Policies) { @@ -117,7 +122,17 @@ func (pad *PAD) GetSTR(epoch uint64) *SignedTreeRoot { if epoch >= pad.latestSTR.Epoch { return pad.latestSTR } - return pad.snapshots[epoch] + if pad.snapshots[epoch] != nil { + return pad.snapshots[epoch] + } + // look through persistent storage + // str := new(SignedTreeRoot) + // err := str.LoadFromKV(pad.db, pad.key, epoch) + // if err != nil { + // return nil + // } + // return str + return nil // util we have a better way to construct the tree partially based on the lookup index. } func (pad *PAD) GetLatestSTR() *SignedTreeRoot { diff --git a/merkletree/pad_test.go b/merkletree/pad_test.go index f30956f..3757685 100644 --- a/merkletree/pad_test.go +++ b/merkletree/pad_test.go @@ -7,8 +7,9 @@ import ( "crypto/rand" "errors" "fmt" - "github.com/coniks-sys/coniks-go/crypto/sign" "io" + + "github.com/coniks-sys/coniks-go/crypto/sign" ) var signKey sign.PrivateKey @@ -37,7 +38,7 @@ func TestPADHashChain(t *testing.T) { treeHashes := make(map[uint64][]byte) - pad, err := NewPAD(NewPolicies(2, vrfPrivKey1), signKey, 10) + pad, err := NewPAD(NewPolicies(2, vrfPrivKey1), nil, signKey, 10) if err != nil { t.Fatal(err) } @@ -143,7 +144,7 @@ func TestPADHashChain(t *testing.T) { func TestHashChainExceedsMaximumSize(t *testing.T) { var hashChainLimit uint64 = 4 - pad, err := NewPAD(NewPolicies(2, vrfPrivKey1), signKey, hashChainLimit) + pad, err := NewPAD(NewPolicies(2, vrfPrivKey1), nil, signKey, hashChainLimit) if err != nil { t.Fatal(err) } @@ -183,7 +184,7 @@ func TestPoliciesChange(t *testing.T) { key3 := "key3" val3 := []byte("value3") - pad, err := NewPAD(NewPolicies(3, vrfPrivKey1), signKey, 10) + pad, err := NewPAD(NewPolicies(3, vrfPrivKey1), nil, signKey, 10) if err != nil { t.Fatal(err) } @@ -246,7 +247,7 @@ func TestNewPADMissingPolicies(t *testing.T) { t.Fatal("Expected NewPAD to panic if policies are missing.") } }() - if _, err := NewPAD(nil, signKey, 10); err != nil { + if _, err := NewPAD(nil, nil, signKey, 10); err != nil { t.Fatal("Expected NewPAD to panic but got error.") } } @@ -272,7 +273,7 @@ func TestNewPADErrorWhileCreatingTree(t *testing.T) { origRand := mockRandReadWithErroringReader() defer unMockRandReader(origRand) - pad, err := NewPAD(NewPolicies(3, vrfPrivKey1), signKey, 3) + pad, err := NewPAD(NewPolicies(3, vrfPrivKey1), nil, signKey, 3) if err == nil || pad != nil { t.Fatal("NewPad should return an error in case the tree creation failed") } @@ -378,7 +379,7 @@ func benchPADLookup(b *testing.B, entries uint64) { // `updateEvery`. If `updateEvery > N` createPAD won't update the STR. func createPad(N uint64, keyPrefix string, valuePrefix []byte, snapLen uint64, updateEvery uint64) (*PAD, error) { - pad, err := NewPAD(NewPolicies(3, vrfPrivKey1), signKey, snapLen) + pad, err := NewPAD(NewPolicies(3, vrfPrivKey1), nil, signKey, snapLen) if err != nil { return nil, err } diff --git a/merkletree/padkv.go b/merkletree/padkv.go new file mode 100644 index 0000000..15a1de0 --- /dev/null +++ b/merkletree/padkv.go @@ -0,0 +1,76 @@ +package merkletree + +import ( + "encoding/binary" + "errors" + + "github.com/coniks-sys/coniks-go/crypto/sign" + "github.com/coniks-sys/coniks-go/storage/kv" + "github.com/coniks-sys/coniks-go/utils" +) + +var ( + ErrorBadEpochLength = errors.New("[merkletree] Bad epoch length") +) + +// NewPADFromKV creates new PAD with a latest tree stored in the KV db +func NewPADFromKV(db kv.DB, policies Policies, signKey sign.PrivateKey, length uint64) (*PAD, error) { + var err error + pad := new(PAD) + pad.signKey = signKey + pad.snapshots = make(map[uint64]*SignedTreeRoot, length) + pad.loadedEpochs = make([]uint64, 0, length) + pad.db = db + + // get latest epoch from db + epBytes, err := db.Get([]byte{EpochIdentifier}) + if err != nil { + return nil, err + } + if len(epBytes[:]) != 8 { + panic(ErrorBadEpochLength) + } + ep := uint64(binary.LittleEndian.Uint64(epBytes[:8])) + + // reconstruct tree from db + pad.tree, err = NewMerkleTreeFromKV(db, ep) + if err != nil { + return nil, err + } + + // get policies from db + err = policies.LoadFromKV(db, ep) + if err != nil { + return nil, err + } + + // get str from db + str := new(SignedTreeRoot) + str.tree = pad.tree.Clone() + str.Policies = policies + err = str.LoadFromKV(db, signKey, ep) + if err != nil { + return nil, err + } + pad.latestSTR = str + pad.policies = policies + pad.snapshots[ep] = str + pad.loadedEpochs = append(pad.loadedEpochs, ep) + + return pad, nil +} + +func (pad *PAD) StoreToKV(epoch uint64) error { + if pad.db == nil { + return nil + } + wb := pad.db.NewBatch() + pad.latestSTR.StoreToKV(wb) + // and store latest STR's epoch to db + wb.Put([]byte{EpochIdentifier}, util.ULongToBytes(epoch)) + err := pad.db.Write(wb) + if err != nil { + return err + } + return nil +} diff --git a/merkletree/padkv_test.go b/merkletree/padkv_test.go new file mode 100644 index 0000000..56bb5b1 --- /dev/null +++ b/merkletree/padkv_test.go @@ -0,0 +1,179 @@ +package merkletree + +import ( + "bytes" + "testing" + + "github.com/coniks-sys/coniks-go/storage/kv" + "github.com/coniks-sys/coniks-go/utils" +) + +func TestPadStore(t *testing.T) { + util.WithDB(func(db kv.DB) { + key1 := "key" + val1 := []byte("value") + + key2 := "key2" + val2 := []byte("value2") + + key3 := "key3" + val3 := []byte("value3") + + pad, err := NewPAD(NewPolicies(2, vrfPrivKey1), db, signKey, 10) + if err != nil { + t.Fatal(err) + } + if err := pad.Set(key1, val1); err != nil { + t.Fatal(err) + } + if err := pad.Set(key2, val2); err != nil { + t.Fatal(err) + } + if err := pad.Set(key3, val3); err != nil { + t.Fatal(err) + } + pad.Update(nil) + + policies := new(ConiksPolicies) + padGot, err := NewPADFromKV(db, policies, signKey, 10) + if err != nil { + t.Fatal(err) + } + ap, err := padGot.Lookup(key1) + if ap.Leaf.Value() == nil { + t.Fatalf("Cannot find key: %v", key1) + } + if !bytes.Equal(pad.tree.hash, padGot.tree.hash) || + !bytes.Equal(pad.latestSTR.Serialize(), padGot.latestSTR.Serialize()) { + t.Fatal("Malformed PAD construction from db", + "want", pad.latestSTR.Serialize(), + "got", padGot.latestSTR.Serialize()) + } + padGot.Update(nil) // just to make sure everything is okay + }) +} + +//TODO: need to be fixed with GetSTR method. +func _TestGetOldSTR(t *testing.T) { + util.WithDB(func(db kv.DB) { + key1 := "key" + val1 := []byte("value") + + key2 := "key2" + val2 := []byte("value2") + + key3 := "key3" + val3 := []byte("value3") + + pad, err := NewPAD(NewPolicies(2, vrfPrivKey1), db, signKey, 2) + if err != nil { + t.Fatal(err) + } + if err := pad.Set(key1, val1); err != nil { + t.Fatal(err) + } + pad.Update(nil) // epoch = 1 + if err := pad.Set(key2, val2); err != nil { + t.Fatal(err) + } + pad.Update(nil) // epoch = 2 + if err := pad.Set(key3, val3); err != nil { + t.Fatal(err) + } + pad.Update(nil) // epoch = 3 + + ap, err := pad.LookupInEpoch(key1, 0) + if err != nil { + t.Fatal(err) + } + if ap.Leaf.Value() != nil { + t.Fatal("Unexpected key lookup at epoch", 0) + } + + ap, err = pad.LookupInEpoch(key2, 1) + if err != nil { + t.Fatal(err) + } + if ap.Leaf.Value() != nil { + t.Fatal("Unexpected key lookup at epoch", 1) + } + ap, err = pad.LookupInEpoch(key1, 1) + if err != nil { + t.Fatal(err) + } + if ap.Leaf.Value() == nil { + t.Fatal("Cannot find key", key1, "at epoch", 1) + } + + ap, err = pad.LookupInEpoch(key3, 2) + if err != nil { + t.Fatal(err) + } + if ap.Leaf.Value() != nil { + t.Fatal("Unexpected key lookup at epoch", 2) + } + ap, err = pad.LookupInEpoch(key2, 2) + if err != nil { + t.Fatal(err) + } + if ap.Leaf.Value() == nil { + t.Fatal("Cannot find key", key2, "at epoch", 2) + } + + ap, err = pad.LookupInEpoch(key3, 3) + if err != nil { + t.Fatal(err) + } + if ap.Leaf.Value() == nil { + t.Fatal("Cannot find key", key3, "at epoch", 3) + } + + }) +} + +func BenchmarkStorePAD100K(b *testing.B) { + // total number of entries in tree: + NumEntries := uint64(100000) + + keyPrefix := "key" + valuePrefix := []byte("value") + snapLen := uint64(10) + noUpdate := uint64(NumEntries + 1) + pad, err := createPad(NumEntries, keyPrefix, valuePrefix, snapLen, noUpdate) + if err != nil { + b.Fatal(err) + } + util.WithDB(func(db kv.DB) { + pad.db = db + b.ResetTimer() + for i := 0; i < b.N; i++ { + pad.Update(nil) + } + }) +} + +func BenchmarkLoadPAD100K(b *testing.B) { + // total number of entries in tree: + NumEntries := uint64(100000) + + keyPrefix := "key" + valuePrefix := []byte("value") + snapLen := uint64(10) + noUpdate := uint64(NumEntries + 1) + pad, err := createPad(NumEntries, keyPrefix, valuePrefix, snapLen, noUpdate) + if err != nil { + b.Fatal(err) + } + util.WithDB(func(db kv.DB) { + pad.db = db + pad.Update(nil) + b.ResetTimer() + for i := 0; i < b.N; i++ { + p := new(ConiksPolicies) + _, err := NewPADFromKV(db, p, signKey, snapLen) + if err != nil { + b.Fatal(err) + } + } + }) +} diff --git a/merkletree/policy.go b/merkletree/policy.go index 4a7284b..d4e6de6 100644 --- a/merkletree/policy.go +++ b/merkletree/policy.go @@ -5,6 +5,7 @@ import ( "github.com/coniks-sys/coniks-go/crypto" "github.com/coniks-sys/coniks-go/crypto/vrf" + "github.com/coniks-sys/coniks-go/storage/kv" "github.com/coniks-sys/coniks-go/utils" ) @@ -14,13 +15,17 @@ type Policies interface { Iterate() map[string][]byte Serialize() []byte vrfPrivate() *vrf.PrivateKey + + // storage interface + StoreToKV(uint64, kv.Batch) + LoadFromKV(kv.DB, uint64) error } type ConiksPolicies struct { LibVersion string HashID string - vrfPrivateKey *vrf.PrivateKey EpochDeadline TimeStamp + vrfPrivateKey *vrf.PrivateKey } var _ Policies = (*ConiksPolicies)(nil) @@ -29,8 +34,8 @@ func NewPolicies(epDeadline TimeStamp, vrfPrivKey *vrf.PrivateKey) Policies { return &ConiksPolicies{ LibVersion: Version, HashID: crypto.HashID, - vrfPrivateKey: vrfPrivKey, EpochDeadline: epDeadline, + vrfPrivateKey: vrfPrivKey, } } diff --git a/merkletree/policykv.go b/merkletree/policykv.go new file mode 100644 index 0000000..6704fbe --- /dev/null +++ b/merkletree/policykv.go @@ -0,0 +1,54 @@ +package merkletree + +import ( + "encoding/binary" + + "github.com/coniks-sys/coniks-go/crypto/vrf" + "github.com/coniks-sys/coniks-go/storage/kv" + "github.com/coniks-sys/coniks-go/utils" +) + +func (p *ConiksPolicies) serializeKVKey(epoch uint64) []byte { + buf := make([]byte, 0, 1+8) + buf = append(buf, PoliciesIdentifier) + buf = append(buf, util.ULongToBytes(epoch)...) + return buf +} + +// StoreToKV stores ConiksPolicies instance into an array with following scheme: +// [p.LibVersion+len, p.HashID+len, epochDeadline, vrfPrivateKey] +func (p *ConiksPolicies) StoreToKV(epoch uint64, wb kv.Batch) { + var buf []byte + buf = append(buf, util.IntToBytes(len([]byte(p.LibVersion)))...) + buf = append(buf, []byte(p.LibVersion)...) + buf = append(buf, util.IntToBytes(len([]byte(p.HashID)))...) + buf = append(buf, []byte(p.HashID)...) + buf = append(buf, util.ULongToBytes(uint64(p.EpochDeadline))...) + buf = append(buf, p.vrfPrivateKey[:]...) + wb.Put(p.serializeKVKey(epoch), buf) +} + +func (p *ConiksPolicies) LoadFromKV(db kv.DB, epoch uint64) error { + buf, err := db.Get(p.serializeKVKey(epoch)) + if err != nil { + return err + } + l := int(binary.LittleEndian.Uint32(buf[:4])) + buf = buf[4:] + p.LibVersion = string(buf[:l]) + buf = buf[l:] + l = int(binary.LittleEndian.Uint32(buf[:4])) + buf = buf[4:] + p.HashID = string(buf[:l]) + buf = buf[l:] + p.EpochDeadline = TimeStamp(binary.LittleEndian.Uint64(buf[:8])) + buf = buf[8:] + vrfKey := new(vrf.PrivateKey) + copy(vrfKey[:], buf[:vrf.PrivateKeySize]) + p.vrfPrivateKey = vrfKey + buf = buf[vrf.PrivateKeySize:] + if len(buf) != 0 { + panic(kv.ErrorBadBufferLength) + } + return nil +} diff --git a/merkletree/strkv.go b/merkletree/strkv.go new file mode 100644 index 0000000..03f4bad --- /dev/null +++ b/merkletree/strkv.go @@ -0,0 +1,55 @@ +package merkletree + +import ( + "encoding/binary" + + "github.com/coniks-sys/coniks-go/crypto" + "github.com/coniks-sys/coniks-go/crypto/sign" + "github.com/coniks-sys/coniks-go/storage/kv" + "github.com/coniks-sys/coniks-go/utils" +) + +func (str *SignedTreeRoot) serializeKVKey(epoch uint64) []byte { + buf := make([]byte, 0, 1+8) + buf = append(buf, STRIdentifier) + buf = append(buf, util.ULongToBytes(epoch)...) + return buf +} + +// StoreToKV stores a STR into db as following scheme: +// [epoch, prevEpoch, prevStrHash, Signature] +func (str *SignedTreeRoot) StoreToKV(wb kv.Batch) { + buf := make([]byte, 0, 8+8+crypto.HashSizeByte+sign.SignatureSize) + buf = append(buf, util.ULongToBytes(str.Epoch)...) + if str.Epoch > 0 { + buf = append(buf, util.ULongToBytes(str.PreviousEpoch)...) + } + buf = append(buf, str.PreviousSTRHash...) + buf = append(buf, str.Signature...) + wb.Put(str.serializeKVKey(str.Epoch), buf) + + str.tree.StoreToKV(str.Epoch, wb) + str.Policies.StoreToKV(str.Epoch, wb) +} + +func (str *SignedTreeRoot) LoadFromKV(db kv.DB, key sign.PrivateKey, epoch uint64) error { + buf, err := db.Get(str.serializeKVKey(epoch)) + if err != nil { + return err + } + str.Epoch = uint64(binary.LittleEndian.Uint64(buf[:8])) + buf = buf[8:] + if str.Epoch > 0 { + str.PreviousEpoch = uint64(binary.LittleEndian.Uint64(buf[:8])) + buf = buf[8:] + } + str.PreviousSTRHash = buf[:crypto.HashSizeByte] + buf = buf[crypto.HashSizeByte:] + str.Signature = buf[:sign.SignatureSize] + buf = buf[sign.SignatureSize:] + if len(buf) != 0 { + panic(kv.ErrorBadBufferLength) + } + + return nil +} diff --git a/merkletree/strkv_test.go b/merkletree/strkv_test.go new file mode 100644 index 0000000..29ee3fb --- /dev/null +++ b/merkletree/strkv_test.go @@ -0,0 +1,72 @@ +package merkletree + +import ( + "bytes" + "testing" + + "github.com/coniks-sys/coniks-go/crypto" + "github.com/coniks-sys/coniks-go/storage/kv" + "github.com/coniks-sys/coniks-go/utils" +) + +func TestSTRStore(t *testing.T) { + util.WithDB(func(db kv.DB) { + var epoch uint64 = 1 + + p := NewPolicies(2, vrfPrivKey1) + m, err := NewMerkleTree() + if err != nil { + t.Fatal(err) + } + m.recomputeHash() + str1 := NewSTR(signKey, p, m, epoch, make([]byte, crypto.HashSizeByte)) + wb := db.NewBatch() + str1.StoreToKV(wb) + err = db.Write(wb) + if err != nil { + t.Fatal(err) + } + + strGot := new(SignedTreeRoot) + err = strGot.LoadFromKV(db, signKey, epoch) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(str1.Signature, strGot.Signature) || + !bytes.Equal(str1.PreviousSTRHash, strGot.PreviousSTRHash) || + str1.PreviousEpoch != strGot.PreviousEpoch || + str1.Epoch != strGot.Epoch { + t.Fatal("Bad de/serialization", + "expect", str1, + "got", strGot) + } + + str2 := NewSTR(signKey, p, m, epoch+1, crypto.Digest(str1.Signature)) + wb = db.NewBatch() + str2.StoreToKV(wb) + err = db.Write(wb) + if err != nil { + t.Fatal(err) + } + strGot = new(SignedTreeRoot) + err = strGot.LoadFromKV(db, signKey, epoch+1) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(str2.Signature, strGot.Signature) || + !bytes.Equal(str2.PreviousSTRHash, strGot.PreviousSTRHash) || + str2.PreviousEpoch != strGot.PreviousEpoch || + str2.Epoch != strGot.Epoch { + t.Fatal("Bad de/serialization", + "expect", str2, + "got", strGot) + } + + // test get non-exist str from db + strGot = new(SignedTreeRoot) + err = strGot.LoadFromKV(db, signKey, epoch+2) + if err != db.ErrNotFound() { + t.Fatal("Got unexpected str from db") + } + }) +} diff --git a/merkletree/tb_test.go b/merkletree/tb_test.go index bf039e7..cbb1b53 100644 --- a/merkletree/tb_test.go +++ b/merkletree/tb_test.go @@ -9,7 +9,7 @@ func TestTB(t *testing.T) { key := "key" val := []byte("value") - pad, err := NewPAD(NewPolicies(3, vrfPrivKey1), signKey, 3) + pad, err := NewPAD(NewPolicies(3, vrfPrivKey1), nil, signKey, 3) if err != nil { t.Fatal(err) } diff --git a/utils/coname.go b/utils/coname.go index 69e7fd3..f6f4aca 100644 --- a/utils/coname.go +++ b/utils/coname.go @@ -14,6 +14,15 @@ package util +import ( + "io/ioutil" + "os" + + "github.com/coniks-sys/coniks-go/storage/kv" + "github.com/coniks-sys/coniks-go/storage/kv/leveldbkv" + "github.com/syndtr/goleveldb/leveldb" +) + // In each byte, the bits are ordered MSB to LSB func ToBytes(bits []bool) []byte { bs := make([]byte, (len(bits)+7)/8) @@ -33,3 +42,17 @@ func ToBits(bs []byte) []bool { } return bits } + +func WithDB(f func(kv.DB)) { + dir, err := ioutil.TempDir("", "merkletree") + if err != nil { + panic(err) + } + defer os.RemoveAll(dir) + db, err := leveldb.OpenFile(dir, nil) + if err != nil { + panic(err) + } + defer db.Close() + f(leveldbkv.Wrap(db)) +}