From 94a6911f267379a7d0fb33be594e22a1e1565155 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@cloudfounders.com>
Date: Fri, 10 Jul 2015 12:18:57 +0200
Subject: [PATCH 001/373] point to opam's OCAML_LIBDIR

---
 pylabs/test/server/quick/test_client_lib.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/pylabs/test/server/quick/test_client_lib.py b/pylabs/test/server/quick/test_client_lib.py
index 0c788240..3c48cf24 100644
--- a/pylabs/test/server/quick/test_client_lib.py
+++ b/pylabs/test/server/quick/test_client_lib.py
@@ -15,19 +15,30 @@
 """
 
 from Compat import X
+import os
 
 def test_client_lib():
     my_temp = '/tmp/client_lib_test'
+    OCAML_LIBDIR = X.subprocess.check_output('ocamlfind printconf destdir',
+                                             shell=True)
+    OCAML_LIBDIR = OCAML_LIBDIR.strip()
+    env = os.environ.copy()
+    env['OCAML_LIBDIR'] = OCAML_LIBDIR
     cmds = [
         (['make', 'uninstall_client'], None),
         (['make', 'install_client'], None),
         (['mkdir', '-p',  my_temp], None),
-        (['cp', './examples/ocaml/demo.ml', './examples/ocaml/_tags', my_temp], None),
+        (['cp', './examples/ocaml/demo.ml',
+          './examples/ocaml/_tags', my_temp], None),
         (['ocamlbuild', '-use-ocamlfind', 'demo.native'], my_temp),
         (['make', 'uninstall_client'], None),
     ]
     for cmd, cwd in cmds:
         if cwd == None:
             cwd = '../..'
-        r = X.subprocess.check_output(cmd, cwd = cwd)
+        print cmd
+        print env
+        r = X.subprocess.check_output(cmd,
+                                      cwd = cwd,
+                                      env = env)
         print r

From 1ee61299370b22e86f280439bffaeef91c6e9ba5 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@cloudfounders.com>
Date: Fri, 10 Jul 2015 13:23:56 +0200
Subject: [PATCH 002/373] install opam packages in 1 step

---
 jenkins/008_install_dev_env.sh | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/jenkins/008_install_dev_env.sh b/jenkins/008_install_dev_env.sh
index 24d4b4c9..afd3ab80 100755
--- a/jenkins/008_install_dev_env.sh
+++ b/jenkins/008_install_dev_env.sh
@@ -7,12 +7,12 @@ opam update -y
 opam switch 4.02.1
 eval `opam config env`
 
-opam install -y "ssl.0.4.7"
-opam install -y conf-libev
-opam install -y camlbz2
-opam install -y snappy
-opam install -y "lwt.2.4.7"
-opam remove -y camltc
-opam install -y "camltc.999"
-opam install -y bisect
-opam install -y quickcheck
+# do this in 1 step, otherwise, you might not get what you want
+opam install -y "ssl.0.4.7" \
+  conf-libev \
+  camlbz2 \
+  snappy \
+  "lwt.2.4.8" \
+  "camltc.999" \
+  bisect \
+  quickcheck

From 22294255313a3ade6f4f34d5214211c73b34a758 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@cloudfounders.com>
Date: Fri, 10 Jul 2015 13:55:55 +0200
Subject: [PATCH 003/373] nocrypto is needed

---
 jenkins/008_install_dev_env.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/jenkins/008_install_dev_env.sh b/jenkins/008_install_dev_env.sh
index afd3ab80..8d125b02 100755
--- a/jenkins/008_install_dev_env.sh
+++ b/jenkins/008_install_dev_env.sh
@@ -15,4 +15,5 @@ opam install -y "ssl.0.4.7" \
   "lwt.2.4.8" \
   "camltc.999" \
   bisect \
-  quickcheck
+  quickcheck \
+  nocrypto.0.4.0

From 3d335cff0c3098e8d7e53324d46a73a7e016809c Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@cloudfounders.com>
Date: Fri, 10 Jul 2015 15:32:51 +0200
Subject: [PATCH 004/373] don't print env

---
 pylabs/test/server/quick/test_client_lib.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/pylabs/test/server/quick/test_client_lib.py b/pylabs/test/server/quick/test_client_lib.py
index 3c48cf24..b21be28c 100644
--- a/pylabs/test/server/quick/test_client_lib.py
+++ b/pylabs/test/server/quick/test_client_lib.py
@@ -37,7 +37,6 @@ def test_client_lib():
         if cwd == None:
             cwd = '../..'
         print cmd
-        print env
         r = X.subprocess.check_output(cmd,
                                       cwd = cwd,
                                       env = env)

From b885bce5197f62f8fc282d634536b787fe72729a Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@cloudfounders.com>
Date: Fri, 10 Jul 2015 17:12:23 +0200
Subject: [PATCH 005/373] travis fiddling

---
 .travis.yml | 2 +-
 travis.sh   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 44e32bb8..725ce8b3 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,6 +1,6 @@
 language: c
 env:
-  - PPA=avsm/ocaml42+opam11
+  - PPA=avsm/ocaml42+opam12
 
 before_install: ./travis.sh before_install
 install: ./travis.sh install
diff --git a/travis.sh b/travis.sh
index 0e044bc0..b27aa830 100755
--- a/travis.sh
+++ b/travis.sh
@@ -2,7 +2,7 @@
 
 APT_DEPENDS="libev-dev libssl-dev libsnappy-dev libgmp3-dev"
 APT_OCAML_DEPENDS="ocaml ocaml-native-compilers camlp4-extra opam"
-OPAM_DEPENDS="ssl conf-libev lwt camlbz2 camltc.999 snappy quickcheck ocplib-endian uuidm nocrypto"
+OPAM_DEPENDS="ssl.0.4.7 conf-libev lwt.2.4.8 camlbz2 camltc.999 snappy quickcheck ocplib-endian uuidm nocrypto.0.4.0"
 
 export OPAMYES=1
 export OPAMVERBOSE=1

From d763f320167428e6b3b85a05a732dedbf434776e Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@cloudfounders.com>
Date: Fri, 10 Jul 2015 17:23:11 +0200
Subject: [PATCH 006/373] sudo required

---
 .travis.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.travis.yml b/.travis.yml
index 725ce8b3..59d7c812 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,4 +1,5 @@
 language: c
+sudo: required
 env:
   - PPA=avsm/ocaml42+opam12
 

From b9b16491fa1a09d69b65210bb52fde81e0d8193f Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@cloudfounders.com>
Date: Fri, 10 Jul 2015 17:33:32 +0200
Subject: [PATCH 007/373] opam 1.2

---
 travis.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/travis.sh b/travis.sh
index b27aa830..c5ad920e 100755
--- a/travis.sh
+++ b/travis.sh
@@ -32,7 +32,7 @@ install () {
     echo "Running 'install' phase"
 
     opam init
-    eval `opam config -env`
+    eval `opam config env`
 
     opam remote add incubaid-devel -k git git://github.com/Incubaid/opam-repository-devel.git
     opam update
@@ -42,7 +42,7 @@ install () {
 
 script () {
     echo "Running 'script' phase"
-    eval `opam config -env`
+    eval `opam config env`
 
     echo "Building 'arakoon.native'"
     ocamlbuild -use-ocamlfind -classic-display arakoon.native

From 3e0f040bffe65d9c6d4f285d17a8e4d0ca1b099b Mon Sep 17 00:00:00 2001
From: Wim Provoost <wim@openvstorage.com>
Date: Fri, 10 Jul 2015 17:46:36 +0200
Subject: [PATCH 008/373] Update README.md

---
 README.md | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/README.md b/README.md
index d5288f67..02c90506 100644
--- a/README.md
+++ b/README.md
@@ -53,8 +53,4 @@ Of course you want to use Arakoon from within a program. We provide OCaml, C and
 
 Releases
 --------
-Arakoon releases are available from the [releases](https://github.com/Incubaid/arakoon/releases) page. Any `1.X` branch corresponds to the development of releases in the `1.X` series.
-
-Have fun,
-
-The Arakoon team
+Arakoon releases are available from the [releases](https://github.com/openvstorage/arakoon/releases) page. 

From 98d4a2d8d7bee9db7bd970d90d8cc4149ab2d86b Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@cloudfounders.com>
Date: Fri, 10 Jul 2015 17:41:49 +0200
Subject: [PATCH 009/373] point to openvstorage iso incubaid

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index d5288f67..18e94f3b 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
 Arakoon
 =======
-![](https://travis-ci.org/Incubaid/arakoon.svg?branch=1.8)
+![](https://travis-ci.org/openvstorage/arakoon.svg?branch=1.8)
 
 Arakoon is a distributed key/value store with preference for consistency. More technically, it's a [Multi-Paxos](http://en.wikipedia.org/wiki/Paxos_%28computer_science%29#Multi-Paxos) implementation (written in [OCaml](http://ocaml.org/)) running on top of [TokyoCabinet](http://fallabs.com/tokyocabinet/).
 
@@ -53,7 +53,7 @@ Of course you want to use Arakoon from within a program. We provide OCaml, C and
 
 Releases
 --------
-Arakoon releases are available from the [releases](https://github.com/Incubaid/arakoon/releases) page. Any `1.X` branch corresponds to the development of releases in the `1.X` series.
+Arakoon releases are available from the [releases](https://github.com/openvstorage/arakoon/releases) page. Any `1.X` branch corresponds to the development of releases in the `1.X` series.
 
 Have fun,
 

From 9c563c6f5972b23ba269e599bf6839f33cbeff5d Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@cloudfounders.com>
Date: Fri, 10 Jul 2015 18:31:45 +0200
Subject: [PATCH 010/373] link points to build, not image

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index ecfd421a..da8efd45 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
 Arakoon
 =======
-![](https://travis-ci.org/openvstorage/arakoon.svg?branch=1.8)
+[![](https://travis-ci.org/openvstorage/arakoon.svg?branch=1.8)](https://travis-ci.org/openvstorage/arakoon)
 
 Arakoon is a distributed key/value store with preference for consistency. More technically, it's a [Multi-Paxos](http://en.wikipedia.org/wiki/Paxos_%28computer_science%29#Multi-Paxos) implementation (written in [OCaml](http://ocaml.org/)) running on top of [TokyoCabinet](http://fallabs.com/tokyocabinet/).
 

From 556412296b3a42379c7f25d0edd66eee45820352 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@cloudfounders.com>
Date: Mon, 13 Jul 2015 15:58:12 +0200
Subject: [PATCH 011/373] set versions for all opam packages.

---
 jenkins/008_install_dev_env.sh | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/jenkins/008_install_dev_env.sh b/jenkins/008_install_dev_env.sh
index 8d125b02..45c6dc8c 100755
--- a/jenkins/008_install_dev_env.sh
+++ b/jenkins/008_install_dev_env.sh
@@ -1,19 +1,19 @@
 #!/bin/bash -xue
 
 which opam > /dev/null || { echo 'opam not found!'; exit 1; }
-{ opam remote list | grep Incubaid/opam-repository-devel > /dev/null; } || { opam remote add incubaid-devel -k git git://github.com/Incubaid/opam-repository-devel.git; }
 
-opam update -y
 opam switch 4.02.1
 eval `opam config env`
 
 # do this in 1 step, otherwise, you might not get what you want
-opam install -y "ssl.0.4.7" \
-  conf-libev \
-  camlbz2 \
-  snappy \
-  "lwt.2.4.8" \
-  "camltc.999" \
-  bisect \
-  quickcheck \
-  nocrypto.0.4.0
+opam install -y ssl.0.5.0 \
+  conf-libev.4-11 \
+  camlbz2.0.6.0 \
+  snappy.0.1.0 \
+  lwt.2.4.8 \
+  camltc.0.9.2 \
+  bisect.1.3 \
+  quickcheck.1.0.2 \
+  nocrypto.0.4.0 \
+  sexplib.112.35.00 \
+  uuidm.0.9.5

From b6aac93d7d83133f5c7b88db15993e52a321ec5e Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@cloudfounders.com>
Date: Mon, 13 Jul 2015 16:31:16 +0200
Subject: [PATCH 012/373] fix versions of opam packages for deb/rpm

---
 jenkins/package_deb/008_install_dev_env.sh | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/jenkins/package_deb/008_install_dev_env.sh b/jenkins/package_deb/008_install_dev_env.sh
index 8c28e8f0..45c6dc8c 100755
--- a/jenkins/package_deb/008_install_dev_env.sh
+++ b/jenkins/package_deb/008_install_dev_env.sh
@@ -1,16 +1,19 @@
 #!/bin/bash -xue
 
 which opam > /dev/null || { echo 'opam not found!'; exit 1; }
-{ opam remote list | grep Incubaid/opam-repository-devel > /dev/null; } || { opam remote add incubaid-devel -k git git://github.com/Incubaid/opam-repository-devel.git; }
 
-opam update -y
 opam switch 4.02.1
 eval `opam config env`
 
-opam install -y \
-    conf-libev.4-11 camlbz2.0.6.0 lwt.2.4.8 camltc.0.9.2 \
-    ssl.0.5.0 bisect.1.3 quickcheck.1.0.2 \
-    ounit.2.0.0 \
-    nocrypto.0.4.0 zarith.1.3 cstruct.1.6.0 ctypes.0.4.1 \
-    snappy.0.1.0 \
-    ocplib-endian.0.8 uuidm.0.9.5
+# do this in 1 step, otherwise, you might not get what you want
+opam install -y ssl.0.5.0 \
+  conf-libev.4-11 \
+  camlbz2.0.6.0 \
+  snappy.0.1.0 \
+  lwt.2.4.8 \
+  camltc.0.9.2 \
+  bisect.1.3 \
+  quickcheck.1.0.2 \
+  nocrypto.0.4.0 \
+  sexplib.112.35.00 \
+  uuidm.0.9.5

From 044893f2aa544fbdf49d19dd00029f7525787cff Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@cloudfounders.com>
Date: Mon, 13 Jul 2015 16:36:10 +0200
Subject: [PATCH 013/373] also fix ctypes and friends

---
 jenkins/package_deb/008_install_dev_env.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/jenkins/package_deb/008_install_dev_env.sh b/jenkins/package_deb/008_install_dev_env.sh
index 45c6dc8c..f3a3c589 100755
--- a/jenkins/package_deb/008_install_dev_env.sh
+++ b/jenkins/package_deb/008_install_dev_env.sh
@@ -12,6 +12,9 @@ opam install -y ssl.0.5.0 \
   snappy.0.1.0 \
   lwt.2.4.8 \
   camltc.0.9.2 \
+  cstruct.1.7.0 \
+  ctypes.0.4.1 \
+  ctypes-foreign.0.4.0 \
   bisect.1.3 \
   quickcheck.1.0.2 \
   nocrypto.0.4.0 \

From c6104b1f197bce390d5fdc4a5c077e22fac326f8 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@cloudfounders.com>
Date: Mon, 13 Jul 2015 16:41:52 +0200
Subject: [PATCH 014/373] ocplib-endian,zarith

---
 jenkins/package_deb/008_install_dev_env.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/jenkins/package_deb/008_install_dev_env.sh b/jenkins/package_deb/008_install_dev_env.sh
index f3a3c589..40068c07 100755
--- a/jenkins/package_deb/008_install_dev_env.sh
+++ b/jenkins/package_deb/008_install_dev_env.sh
@@ -16,7 +16,9 @@ opam install -y ssl.0.5.0 \
   ctypes.0.4.1 \
   ctypes-foreign.0.4.0 \
   bisect.1.3 \
+  ocplib-endian.0.8 \
   quickcheck.1.0.2 \
   nocrypto.0.4.0 \
   sexplib.112.35.00 \
-  uuidm.0.9.5
+  uuidm.0.9.5 \
+  zarith.1.3 \

From 12fa5b2958458e1f4931d5a8061bd9d88e3254ff Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@cloudfounders.com>
Date: Mon, 13 Jul 2015 17:29:49 +0200
Subject: [PATCH 015/373] change sexplib version & alignment of 2 dev_env.sh
 scripts

---
 jenkins/008_install_dev_env.sh             | 25 +++++++++++++---------
 jenkins/package_deb/008_install_dev_env.sh |  4 ++--
 2 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/jenkins/008_install_dev_env.sh b/jenkins/008_install_dev_env.sh
index 45c6dc8c..16ecf072 100755
--- a/jenkins/008_install_dev_env.sh
+++ b/jenkins/008_install_dev_env.sh
@@ -7,13 +7,18 @@ eval `opam config env`
 
 # do this in 1 step, otherwise, you might not get what you want
 opam install -y ssl.0.5.0 \
-  conf-libev.4-11 \
-  camlbz2.0.6.0 \
-  snappy.0.1.0 \
-  lwt.2.4.8 \
-  camltc.0.9.2 \
-  bisect.1.3 \
-  quickcheck.1.0.2 \
-  nocrypto.0.4.0 \
-  sexplib.112.35.00 \
-  uuidm.0.9.5
+     conf-libev.4-11 \
+     camlbz2.0.6.0 \
+     snappy.0.1.0 \
+     lwt.2.4.8 \
+     camltc.0.9.2 \
+     cstruct.1.7.0 \
+     ctypes.0.4.1 \
+     ctypes-foreign.0.4.0 \
+     bisect.1.3 \
+     ocplib-endian.0.8 \
+     quickcheck.1.0.2 \
+     nocrypto.0.4.0 \
+     sexplib.112.24.01 \
+     uuidm.0.9.5 \
+     zarith.1.3
diff --git a/jenkins/package_deb/008_install_dev_env.sh b/jenkins/package_deb/008_install_dev_env.sh
index 40068c07..0d1c3370 100755
--- a/jenkins/package_deb/008_install_dev_env.sh
+++ b/jenkins/package_deb/008_install_dev_env.sh
@@ -19,6 +19,6 @@ opam install -y ssl.0.5.0 \
   ocplib-endian.0.8 \
   quickcheck.1.0.2 \
   nocrypto.0.4.0 \
-  sexplib.112.35.00 \
+  sexplib.112.24.01 \
   uuidm.0.9.5 \
-  zarith.1.3 \
+  zarith.1.3

From e06efa9ff4a724bd1b0274e0135718b8faf05533 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@cloudfounders.com>
Date: Thu, 16 Jul 2015 20:40:43 +0200
Subject: [PATCH 016/373] cstruct 1.6.0 just like 1.8.6 package

---
 jenkins/008_install_dev_env.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jenkins/008_install_dev_env.sh b/jenkins/008_install_dev_env.sh
index 16ecf072..9a889673 100755
--- a/jenkins/008_install_dev_env.sh
+++ b/jenkins/008_install_dev_env.sh
@@ -12,7 +12,7 @@ opam install -y ssl.0.5.0 \
      snappy.0.1.0 \
      lwt.2.4.8 \
      camltc.0.9.2 \
-     cstruct.1.7.0 \
+     cstruct.1.6.0 \
      ctypes.0.4.1 \
      ctypes-foreign.0.4.0 \
      bisect.1.3 \

From 1d971124259808e786e1e3837119f799d4afb445 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@cloudfounders.com>
Date: Wed, 22 Jul 2015 16:41:14 +0200
Subject: [PATCH 017/373] in case of error, return exit code 1

---
 src/tlog/tlc2.ml | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/src/tlog/tlc2.ml b/src/tlog/tlc2.ml
index 807bf3a0..9cd3b3c6 100644
--- a/src/tlog/tlc2.ml
+++ b/src/tlog/tlc2.ml
@@ -897,7 +897,7 @@ let make_tlc2 ~compressor tlog_dir tlf_dir head_dir ~fsync node_id ~fsync_tlog_d
 let truncate_tlog filename =
   let skipper () _entry = Lwt.return ()
   in
-  let t =
+  let t () =
     begin
       let folder,extension,index = folder_for filename None in
       if extension <> ".tlog"
@@ -923,4 +923,13 @@ let truncate_tlog filename =
           Lwt_io.with_file ~mode:Lwt_io.input filename do_it
         end
     end
-  in Lwt_main.run t
+  in
+  let t' =
+    Lwt.catch
+      t
+      (fun ex ->
+       Lwt_io.printlf "problem:%s%!" (Printexc.to_string ex) >>= fun () ->
+       Lwt.return 1
+      )
+  in
+  Lwt_main.run t'

From 3cad626de3bb4de48b73ab87b2c43670fafc29c8 Mon Sep 17 00:00:00 2001
From: "cflap0019\\Wim Provoost" <wim@openvstorage.com>
Date: Thu, 30 Jul 2015 17:13:42 +0200
Subject: [PATCH 018/373] Change to Open vStorage

---
 doc/_templates/layout.html | 21 +++++++++------------
 doc/getting_started.rst    |  2 +-
 doc/installing_arakoon.rst |  6 +++---
 doc/introduction.rst       |  2 +-
 doc/make_release.rst       |  2 +-
 5 files changed, 15 insertions(+), 18 deletions(-)

diff --git a/doc/_templates/layout.html b/doc/_templates/layout.html
index b268ec54..20ed314d 100644
--- a/doc/_templates/layout.html
+++ b/doc/_templates/layout.html
@@ -19,17 +19,14 @@
     <script src="http://html5shim.googlecode.com/svn/trunk/html5.js"></script>
   <![endif]-->
 
-  <script type="text/javascript">
-    var _gaq = _gaq || [];
-    _gaq.push(['_setAccount', 'UA-27966654-1']);
-    _gaq.push(['_trackPageview']);
-
-    (function() {
-      var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
-      ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
-      var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
-    })();
-  </script>
+  <script>
+        (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+        (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
+        m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+        })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
+        ga('create', 'UA-33858052-10', 'auto');
+        ga('send', 'pageview');
+    </script>
 
   <style type='text/css'>
     footer {
@@ -74,7 +71,7 @@
     <footer>
       <div class='row'>
         <div class='span-one-third'>
-          <p>&copy; 2012, Incubaid BVBA</p>
+          <p>&copy; 2010-2015, Open vStorage</p>
         </div>
         <div class='span-two-thirds'>
           <p class='pull-right'>
diff --git a/doc/getting_started.rst b/doc/getting_started.rst
index 3a02151a..e43aa609 100644
--- a/doc/getting_started.rst
+++ b/doc/getting_started.rst
@@ -13,7 +13,7 @@ The simplest is to download and install the debian archive. It's not difficult t
 
 ::
 
-    $> wget https://github.com/Incubaid/arakoon/releases/download/1.7.1/arakoon_1.7.1_amd64.deb
+    $> wget https://github.com/openvstorage/arakoon/releases/download/1.7.1/arakoon_1.7.1_amd64.deb
     ...
     $> sudo dpkg -i ./arakoon_1.7.1_amd64.deb
     ...
diff --git a/doc/installing_arakoon.rst b/doc/installing_arakoon.rst
index cdc82a6e..58f6105e 100644
--- a/doc/installing_arakoon.rst
+++ b/doc/installing_arakoon.rst
@@ -1,7 +1,7 @@
 ==================
 Installing Arakoon
 ==================
-This section explains you how you can install Arakoon, Incubaid's distributed
+This section explains you how you can install Arakoon, Open vStorage's distributed
 key-value store. You can also install a Python or OCaml client to interact with
 Arakoon.
 
@@ -66,6 +66,6 @@ To install the Arakoon OCaml Client, you only have to install the
 
 Other Clients
 =============
-Arakoon has also a C and PHP client, available on bitbucket.org_.
+Arakoon has also a python available on github.com_.
 
-.. _bitbucket.org: https://bitbucket.org/despiegk/arakoon/src/ada5840d1b78/src/client/
+.. _github.com: https://github.com/openvstorage/arakoon/tree/1.8/src/client
diff --git a/doc/introduction.rst b/doc/introduction.rst
index 28aa2534..f4def526 100644
--- a/doc/introduction.rst
+++ b/doc/introduction.rst
@@ -2,7 +2,7 @@
 Introduction
 ============
 Arakoon is a key-value store that guarantees consistency above anything else.
-It was created by the Incubaid team due to a lack of existing solutions
+It was created by the Open vStorage team due to a lack of existing solutions
 fitting our requirements, and is available as Open Source software.
 
 Goals
diff --git a/doc/make_release.rst b/doc/make_release.rst
index 49792882..f4086e65 100644
--- a/doc/make_release.rst
+++ b/doc/make_release.rst
@@ -34,5 +34,5 @@ How to make a release
 
  * wait until github published it.
 
- * send mail to arakoon_internal@incubaid.com
+ * send mail to arakoon@openvstorage.com
 

From a90dd7f2a2df442d55e13f688e346ac56c2775bf Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Mon, 3 Aug 2015 10:37:00 +0200
Subject: [PATCH 019/373] prevent scenario where no master would be chosen
 because nodes got stuck in a loop

---
 src/lib/std.ml               |  4 ++--
 src/paxos/multi_paxos.ml     | 15 ++++++++++++---
 src/paxos/multi_paxos_fsm.ml | 20 +++++++++++---------
 3 files changed, 25 insertions(+), 14 deletions(-)

diff --git a/src/lib/std.ml b/src/lib/std.ml
index 022c0ad7..4f5cf605 100644
--- a/src/lib/std.ml
+++ b/src/lib/std.ml
@@ -229,11 +229,11 @@ module List = struct
   @param key Key to look-up
   @param default Optional default value if key isn't found
   @param f Update function *)
-  let alter ~list ~key ?default f =
+  let alter ?(equals=(=)) ~list ~key ?default f =
     let rec loop found acc = function
       | [] -> (found, acc)
       | ((k, v) as p :: r) ->
-          if k = key
+          if equals k key
             then match f v with
               | None -> loop true acc r
               | Some v' -> loop true ((k, v') :: acc) r
diff --git a/src/paxos/multi_paxos.ml b/src/paxos/multi_paxos.ml
index 7cea3226..c7a91a42 100644
--- a/src/paxos/multi_paxos.ml
+++ b/src/paxos/multi_paxos.ml
@@ -72,9 +72,18 @@ let network_of_messaging (m:messaging) =
 let update_votes (nones, somes) = function
   | None -> (nones + 1, somes)
   | Some key ->
-      List.alter ~list:somes ~key ~default:1 (fun v -> Some (v + 1))
-      |> List.sort (fun (_, fa) (_, fb) -> fb - fa)
-      |> fun s -> (nones, s)
+    List.alter
+      ~equals:(fun v1 v2 ->
+               v1 = v2
+               || (match v1, v2 with
+                   (* ignore the timestamp when comparing a masterset,
+                      otherwise in some situations electing the master
+                      might get stuck in a loop *)
+                   | Value.Vm(m1, _), Value.Vm(m2, _) -> m1 = m2
+                   | _ -> false))
+      ~list:somes ~key ~default:1 (fun v -> Some (v + 1))
+    |> List.sort (fun (_, fa) (_, fb) -> fb - fa)
+    |> fun s -> (nones, s)
 
 type paxos_event =
   | FromClient of ((Update.Update.t) * (Store.update_result -> unit Lwt.t)) list
diff --git a/src/paxos/multi_paxos_fsm.ml b/src/paxos/multi_paxos_fsm.ml
index a384cc73..8f54b5be 100644
--- a/src/paxos/multi_paxos_fsm.ml
+++ b/src/paxos/multi_paxos_fsm.ml
@@ -32,11 +32,14 @@ let election_suggest (type s) constants (n, counter) () =
   let me = constants.me in
   let v_lims, msg =
     match vo with
-      | None ->
-        (1,[]) , "None"
-      | Some x -> (0,[(x,1)]) , "Some _"
+      | None ->   (1,[]), "None"
+      | Some x -> (0,[(x,1)]), Value.value2s x
   in
-  Logger.info_f_ "%s: election_suggest: n=%s i=%s %s" me  (Sn.string_of n) (Sn.string_of i) msg >>= fun () ->
+  Logger.info_f_
+    "%s: election_suggest: n=%s i=%s %s"
+    me
+    (Sn.string_of n) (Sn.string_of i)
+    msg >>= fun () ->
   start_election_timeout constants n i >>= fun () ->
   let delay =
     match constants.master with
@@ -69,11 +72,10 @@ let promises_check_done constants state () =
   let me = constants.me in
   let nnones, v_s = v_lims in
   let bv,bf =
-    begin
-      match v_s with
-        | [] ->  Value.create_master_value me, 0
-        | hd::_ -> hd
-    end in
+    match v_s with
+    | [] ->  Value.create_master_value me, 0
+    | hd::_ -> hd
+  in
   let nnodes = List.length constants.others + 1 in
   let needed = constants.quorum_function nnodes in
   let nvoted = List.length who_voted in

From 3460be5bf7a9c890043a05e169df8224ba98e987 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Mon, 3 Aug 2015 16:48:18 +0200
Subject: [PATCH 020/373] make_release: 1.8.7

---
 META                      | 2 +-
 debian/changelog          | 5 +++++
 redhat/SPECS/arakoon.spec | 4 +++-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/META b/META
index 8b47f533..7880392d 100644
--- a/META
+++ b/META
@@ -1,7 +1,7 @@
 # -*- conf -*-
 
 description = "Arakoon client"
-version = "1.8.6"
+version = "1.8.7"
 exists_if = "arakoon_client.cma,arakoon_client.cmxa,arakoon_client.cmxs"
 requires = "threads lwt lwt.unix str oUnit quickcheck lwt.ssl ocplib-endian nocrypto cstruct"
 archive(byte) = "arakoon_client.cma"
diff --git a/debian/changelog b/debian/changelog
index 1e4313a1..0af95ed9 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,8 @@
+arakoon (1.8.7) unstable; urgency=low
+  * new upstream version
+
+ -- Jan Doms <jan.doms@gmail.com>  Mon, 03 Aug 2015 16:48:18 +0100
+
 arakoon (1.8.6) unstable; urgency=low
   * new upstream version
 
diff --git a/redhat/SPECS/arakoon.spec b/redhat/SPECS/arakoon.spec
index bfb8f378..6e27251a 100644
--- a/redhat/SPECS/arakoon.spec
+++ b/redhat/SPECS/arakoon.spec
@@ -2,7 +2,7 @@
 
 Summary: Arakoon
 Name: arakoon
-Version: 1.8.6
+Version: 1.8.7
 Release: 3%{?dist}
 License: Apache 2.0
 Requires: libev >= 4
@@ -33,6 +33,8 @@ cp arakoon.native %{buildroot}%{_bindir}/arakoon
 %{_bindir}/arakoon
 
 %changelog
+* Mon Aug 03 2015 Jan Doms <jan.doms@gmail.com> - 1.8.7
+- Create arakoon 1.8.7 RPM package
 * Fri Jun 26 2015 Jan Doms <jan.doms@gmail.com> - 1.8.6
 - Create arakoon 1.8.6 RPM package
 * Fri Jun 12 2015 Jan Doms <jan.doms@gmail.com> - 1.8.5

From 4593d18dceca7b7aee54b2fa6dcfaff009134474 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Wed, 5 Aug 2015 09:39:31 +0200
Subject: [PATCH 021/373] update dependency versions

---
 jenkins/008_install_dev_env.sh             | 4 ++--
 jenkins/package_deb/008_install_dev_env.sh | 4 ++--
 src/tlog/compression.ml                    | 2 +-
 travis.sh                                  | 2 +-
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/jenkins/008_install_dev_env.sh b/jenkins/008_install_dev_env.sh
index 9a889673..1cf7d3c4 100755
--- a/jenkins/008_install_dev_env.sh
+++ b/jenkins/008_install_dev_env.sh
@@ -10,7 +10,7 @@ opam install -y ssl.0.5.0 \
      conf-libev.4-11 \
      camlbz2.0.6.0 \
      snappy.0.1.0 \
-     lwt.2.4.8 \
+     lwt.2.5.0 \
      camltc.0.9.2 \
      cstruct.1.6.0 \
      ctypes.0.4.1 \
@@ -19,6 +19,6 @@ opam install -y ssl.0.5.0 \
      ocplib-endian.0.8 \
      quickcheck.1.0.2 \
      nocrypto.0.4.0 \
-     sexplib.112.24.01 \
+     sexplib.112.35.00 \
      uuidm.0.9.5 \
      zarith.1.3
diff --git a/jenkins/package_deb/008_install_dev_env.sh b/jenkins/package_deb/008_install_dev_env.sh
index 0d1c3370..dd5ff5f4 100755
--- a/jenkins/package_deb/008_install_dev_env.sh
+++ b/jenkins/package_deb/008_install_dev_env.sh
@@ -10,7 +10,7 @@ opam install -y ssl.0.5.0 \
   conf-libev.4-11 \
   camlbz2.0.6.0 \
   snappy.0.1.0 \
-  lwt.2.4.8 \
+  lwt.2.5.0 \
   camltc.0.9.2 \
   cstruct.1.7.0 \
   ctypes.0.4.1 \
@@ -19,6 +19,6 @@ opam install -y ssl.0.5.0 \
   ocplib-endian.0.8 \
   quickcheck.1.0.2 \
   nocrypto.0.4.0 \
-  sexplib.112.24.01 \
+  sexplib.112.35.00 \
   uuidm.0.9.5 \
   zarith.1.3
diff --git a/src/tlog/compression.ml b/src/tlog/compression.ml
index 55d1f626..01b9b24c 100644
--- a/src/tlog/compression.ml
+++ b/src/tlog/compression.ml
@@ -66,7 +66,7 @@ let _compress_tlog
     | No -> failwith "compressor is 'No'"
   in
   Lwt_io.with_file
-    ~buffer_size:131072
+    ~buffer:(Lwt_bytes.create 131072)
     ~mode:Lwt_io.input tlog_name
     (fun ic ->
        let tmp_file = archive_name ^ ".part" in
diff --git a/travis.sh b/travis.sh
index c5ad920e..22cf819f 100755
--- a/travis.sh
+++ b/travis.sh
@@ -2,7 +2,7 @@
 
 APT_DEPENDS="libev-dev libssl-dev libsnappy-dev libgmp3-dev"
 APT_OCAML_DEPENDS="ocaml ocaml-native-compilers camlp4-extra opam"
-OPAM_DEPENDS="ssl.0.4.7 conf-libev lwt.2.4.8 camlbz2 camltc.999 snappy quickcheck ocplib-endian uuidm nocrypto.0.4.0"
+OPAM_DEPENDS="ssl.0.5.0 conf-libev lwt.2.5.0 camlbz2 camltc.999 snappy quickcheck ocplib-endian uuidm nocrypto.0.4.0"
 
 export OPAMYES=1
 export OPAMVERBOSE=1

From 3350adc8603da91e470a5d7beac5b4e2df019680 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Wed, 5 Aug 2015 14:32:14 +0200
Subject: [PATCH 022/373] bump nocrypto version

---
 jenkins/008_install_dev_env.sh             | 2 +-
 jenkins/package_deb/008_install_dev_env.sh | 2 +-
 travis.sh                                  | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/jenkins/008_install_dev_env.sh b/jenkins/008_install_dev_env.sh
index 1cf7d3c4..9591141f 100755
--- a/jenkins/008_install_dev_env.sh
+++ b/jenkins/008_install_dev_env.sh
@@ -18,7 +18,7 @@ opam install -y ssl.0.5.0 \
      bisect.1.3 \
      ocplib-endian.0.8 \
      quickcheck.1.0.2 \
-     nocrypto.0.4.0 \
+     nocrypto.0.5.0 \
      sexplib.112.35.00 \
      uuidm.0.9.5 \
      zarith.1.3
diff --git a/jenkins/package_deb/008_install_dev_env.sh b/jenkins/package_deb/008_install_dev_env.sh
index dd5ff5f4..77857e1f 100755
--- a/jenkins/package_deb/008_install_dev_env.sh
+++ b/jenkins/package_deb/008_install_dev_env.sh
@@ -18,7 +18,7 @@ opam install -y ssl.0.5.0 \
   bisect.1.3 \
   ocplib-endian.0.8 \
   quickcheck.1.0.2 \
-  nocrypto.0.4.0 \
+  nocrypto.0.5.0 \
   sexplib.112.35.00 \
   uuidm.0.9.5 \
   zarith.1.3
diff --git a/travis.sh b/travis.sh
index 22cf819f..a52a3c17 100755
--- a/travis.sh
+++ b/travis.sh
@@ -2,7 +2,7 @@
 
 APT_DEPENDS="libev-dev libssl-dev libsnappy-dev libgmp3-dev"
 APT_OCAML_DEPENDS="ocaml ocaml-native-compilers camlp4-extra opam"
-OPAM_DEPENDS="ssl.0.5.0 conf-libev lwt.2.5.0 camlbz2 camltc.999 snappy quickcheck ocplib-endian uuidm nocrypto.0.4.0"
+OPAM_DEPENDS="ssl.0.5.0 conf-libev lwt.2.5.0 camlbz2 camltc.999 snappy quickcheck ocplib-endian uuidm nocrypto.0.5.0"
 
 export OPAMYES=1
 export OPAMVERBOSE=1

From 29eb0cf1983d3c23248c3a35f1504fc13ea3c6f0 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Wed, 5 Aug 2015 14:41:42 +0200
Subject: [PATCH 023/373] fix travis

---
 travis.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/travis.sh b/travis.sh
index a52a3c17..2bf7b881 100755
--- a/travis.sh
+++ b/travis.sh
@@ -2,7 +2,7 @@
 
 APT_DEPENDS="libev-dev libssl-dev libsnappy-dev libgmp3-dev"
 APT_OCAML_DEPENDS="ocaml ocaml-native-compilers camlp4-extra opam"
-OPAM_DEPENDS="ssl.0.5.0 conf-libev lwt.2.5.0 camlbz2 camltc.999 snappy quickcheck ocplib-endian uuidm nocrypto.0.5.0"
+OPAM_DEPENDS="ssl.0.5.0 conf-libev lwt.2.5.0 camlbz2 camltc.999 snappy quickcheck ocplib-endian uuidm nocrypto.0.5.0 ctypes.0.4.1 ctypes-foreign.0.4.0"
 
 export OPAMYES=1
 export OPAMVERBOSE=1

From 0f2b289436c4b7ad8125f91c5384a9dee48de68f Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Wed, 5 Aug 2015 14:50:54 +0200
Subject: [PATCH 024/373] nocrypto 0.5.1

---
 jenkins/008_install_dev_env.sh             | 2 +-
 jenkins/package_deb/008_install_dev_env.sh | 2 +-
 travis.sh                                  | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/jenkins/008_install_dev_env.sh b/jenkins/008_install_dev_env.sh
index 9591141f..433db692 100755
--- a/jenkins/008_install_dev_env.sh
+++ b/jenkins/008_install_dev_env.sh
@@ -18,7 +18,7 @@ opam install -y ssl.0.5.0 \
      bisect.1.3 \
      ocplib-endian.0.8 \
      quickcheck.1.0.2 \
-     nocrypto.0.5.0 \
+     nocrypto.0.5.1 \
      sexplib.112.35.00 \
      uuidm.0.9.5 \
      zarith.1.3
diff --git a/jenkins/package_deb/008_install_dev_env.sh b/jenkins/package_deb/008_install_dev_env.sh
index 77857e1f..fe62438c 100755
--- a/jenkins/package_deb/008_install_dev_env.sh
+++ b/jenkins/package_deb/008_install_dev_env.sh
@@ -18,7 +18,7 @@ opam install -y ssl.0.5.0 \
   bisect.1.3 \
   ocplib-endian.0.8 \
   quickcheck.1.0.2 \
-  nocrypto.0.5.0 \
+  nocrypto.0.5.1 \
   sexplib.112.35.00 \
   uuidm.0.9.5 \
   zarith.1.3
diff --git a/travis.sh b/travis.sh
index 2bf7b881..0cea9d39 100755
--- a/travis.sh
+++ b/travis.sh
@@ -2,7 +2,7 @@
 
 APT_DEPENDS="libev-dev libssl-dev libsnappy-dev libgmp3-dev"
 APT_OCAML_DEPENDS="ocaml ocaml-native-compilers camlp4-extra opam"
-OPAM_DEPENDS="ssl.0.5.0 conf-libev lwt.2.5.0 camlbz2 camltc.999 snappy quickcheck ocplib-endian uuidm nocrypto.0.5.0 ctypes.0.4.1 ctypes-foreign.0.4.0"
+OPAM_DEPENDS="ssl.0.5.0 conf-libev lwt.2.5.0 camlbz2 camltc.999 snappy quickcheck ocplib-endian uuidm nocrypto.0.5.1 ctypes.0.4.1 ctypes-foreign.0.4.0"
 
 export OPAMYES=1
 export OPAMVERBOSE=1

From 4b89fa8518d047c8739cd99089560e57e53ce6d9 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Wed, 5 Aug 2015 15:16:43 +0200
Subject: [PATCH 025/373] cstruct.1.7.0

---
 jenkins/008_install_dev_env.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jenkins/008_install_dev_env.sh b/jenkins/008_install_dev_env.sh
index 433db692..7f065a61 100755
--- a/jenkins/008_install_dev_env.sh
+++ b/jenkins/008_install_dev_env.sh
@@ -12,7 +12,7 @@ opam install -y ssl.0.5.0 \
      snappy.0.1.0 \
      lwt.2.5.0 \
      camltc.0.9.2 \
-     cstruct.1.6.0 \
+     cstruct.1.7.0 \
      ctypes.0.4.1 \
      ctypes-foreign.0.4.0 \
      bisect.1.3 \

From c70b466f488f4dde950650de0105dd1250294793 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Wed, 5 Aug 2015 15:30:24 +0200
Subject: [PATCH 026/373] make_release: 1.8.8

---
 META                      | 2 +-
 debian/changelog          | 5 +++++
 redhat/SPECS/arakoon.spec | 4 +++-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/META b/META
index 7880392d..800a8f66 100644
--- a/META
+++ b/META
@@ -1,7 +1,7 @@
 # -*- conf -*-
 
 description = "Arakoon client"
-version = "1.8.7"
+version = "1.8.8"
 exists_if = "arakoon_client.cma,arakoon_client.cmxa,arakoon_client.cmxs"
 requires = "threads lwt lwt.unix str oUnit quickcheck lwt.ssl ocplib-endian nocrypto cstruct"
 archive(byte) = "arakoon_client.cma"
diff --git a/debian/changelog b/debian/changelog
index 0af95ed9..d28c2fb1 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,8 @@
+arakoon (1.8.8) unstable; urgency=low
+  * new upstream version
+
+ -- Jan Doms <jan.doms@gmail.com>  Wed, 05 Aug 2015 15:30:24 +0100
+
 arakoon (1.8.7) unstable; urgency=low
   * new upstream version
 
diff --git a/redhat/SPECS/arakoon.spec b/redhat/SPECS/arakoon.spec
index 6e27251a..8f2deda1 100644
--- a/redhat/SPECS/arakoon.spec
+++ b/redhat/SPECS/arakoon.spec
@@ -2,7 +2,7 @@
 
 Summary: Arakoon
 Name: arakoon
-Version: 1.8.7
+Version: 1.8.8
 Release: 3%{?dist}
 License: Apache 2.0
 Requires: libev >= 4
@@ -33,6 +33,8 @@ cp arakoon.native %{buildroot}%{_bindir}/arakoon
 %{_bindir}/arakoon
 
 %changelog
+* Wed Aug 05 2015 Jan Doms <jan.doms@gmail.com> - 1.8.8
+- Create arakoon 1.8.8 RPM package
 * Mon Aug 03 2015 Jan Doms <jan.doms@gmail.com> - 1.8.7
 - Create arakoon 1.8.7 RPM package
 * Fri Jun 26 2015 Jan Doms <jan.doms@gmail.com> - 1.8.6

From 20dc0cc56877767cb69d2c096b9a510bc7cccdd5 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Mon, 10 Aug 2015 17:16:43 +0200
Subject: [PATCH 027/373] remove unneeded O_CLOEXEC flag so we don't run into
 https://github.com/ocsigen/lwt/issues/170

---
 src/node/node_cfg.ml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/node/node_cfg.ml b/src/node/node_cfg.ml
index 5dd2acc5..ebef9cc0 100644
--- a/src/node/node_cfg.ml
+++ b/src/node/node_cfg.ml
@@ -748,7 +748,7 @@ module Node_cfg = struct
             let go () =
               Logger.debug_f_ "Touching %S" fn >>= fun () ->
               Lwt_unix.openfile fn
-                [Lwt_unix.O_RDWR; Lwt_unix.O_CLOEXEC; Lwt_unix.O_CREAT; Lwt_unix.O_EXCL]
+                [Lwt_unix.O_RDWR; Lwt_unix.O_CREAT; Lwt_unix.O_EXCL]
                 0o600 >>= fun fd ->
 
               Lwt.finalize

From fe49c8b764130307a4384c5215d366fb8d5bc06b Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Fri, 21 Aug 2015 17:43:43 +0200
Subject: [PATCH 028/373] timeout for Client_helper.find_master'

---
 src/client/client_helper.ml | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/src/client/client_helper.ml b/src/client/client_helper.ml
index b4f694e0..dc253a7c 100644
--- a/src/client/client_helper.ml
+++ b/src/client/client_helper.ml
@@ -181,11 +181,15 @@ let find_master' ~tls cluster_cfg =
         Logger.debug_f_ "Client_main.find_master': Trying %S" node_name >>= fun () ->
         Lwt.catch
           (fun () ->
-            with_client'
-              ~tls
-              cfg cluster_cfg.cluster_id
-              (fun client ->
-                client # who_master ()) >>= function
+           Lwt.choose
+             [ (Lwt_unix.sleep 1. >>= fun () ->
+                Lwt.return None);
+               with_client'
+                 ~tls
+                 cfg cluster_cfg.cluster_id
+                 (fun client ->
+                  client # who_master ()); ]
+           >>= function
                 | None -> begin
                     Logger.debug_f_
                       "Client_main.find_master': %S doesn't know" node_name >>= fun () ->

From a8fe19c79b35783ebd5f6a5333c9593ace4ed342 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@cloudfounders.com>
Date: Tue, 25 Aug 2015 11:14:28 +0200
Subject: [PATCH 029/373] no exit hook processing.

---
 src/main/main.ml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/main/main.ml b/src/main/main.ml
index 531e9d84..d3092ece 100644
--- a/src/main/main.ml
+++ b/src/main/main.ml
@@ -530,4 +530,12 @@ let main () =
       | ServerAction sa -> do_server sa
   in
   (* let () = Printf.printf "[rc=%i]\n" rc in *)
+  let () =
+    (* no exit hook processing needed: we've done everything we needed to do *)
+    let rec loop () =
+      let _ = Lwt_sequence.take_l Lwt_main.exit_hooks in
+      loop ()
+    in
+    try loop () with Lwt_sequence.Empty -> ()
+  in
   exit exit_code

From a73725c309c96afbbb181ffde1c6f2c25b4fb83c Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Tue, 25 Aug 2015 12:51:56 +0200
Subject: [PATCH 030/373] make_release: 1.8.9

---
 META                      | 2 +-
 debian/changelog          | 5 +++++
 redhat/SPECS/arakoon.spec | 4 +++-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/META b/META
index 800a8f66..c4611da2 100644
--- a/META
+++ b/META
@@ -1,7 +1,7 @@
 # -*- conf -*-
 
 description = "Arakoon client"
-version = "1.8.8"
+version = "1.8.9"
 exists_if = "arakoon_client.cma,arakoon_client.cmxa,arakoon_client.cmxs"
 requires = "threads lwt lwt.unix str oUnit quickcheck lwt.ssl ocplib-endian nocrypto cstruct"
 archive(byte) = "arakoon_client.cma"
diff --git a/debian/changelog b/debian/changelog
index d28c2fb1..d5e782c7 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,8 @@
+arakoon (1.8.9) unstable; urgency=low
+  * new upstream version
+
+ -- Jan Doms <jan.doms@gmail.com>  Tue, 25 Aug 2015 12:51:56 +0100
+
 arakoon (1.8.8) unstable; urgency=low
   * new upstream version
 
diff --git a/redhat/SPECS/arakoon.spec b/redhat/SPECS/arakoon.spec
index 8f2deda1..5628ff7a 100644
--- a/redhat/SPECS/arakoon.spec
+++ b/redhat/SPECS/arakoon.spec
@@ -2,7 +2,7 @@
 
 Summary: Arakoon
 Name: arakoon
-Version: 1.8.8
+Version: 1.8.9
 Release: 3%{?dist}
 License: Apache 2.0
 Requires: libev >= 4
@@ -33,6 +33,8 @@ cp arakoon.native %{buildroot}%{_bindir}/arakoon
 %{_bindir}/arakoon
 
 %changelog
+* Tue Aug 25 2015 Jan Doms <jan.doms@gmail.com> - 1.8.9
+- Create arakoon 1.8.9 RPM package
 * Wed Aug 05 2015 Jan Doms <jan.doms@gmail.com> - 1.8.8
 - Create arakoon 1.8.8 RPM package
 * Mon Aug 03 2015 Jan Doms <jan.doms@gmail.com> - 1.8.7

From 9e999350bcd048c0f9e350bc292da2e3d1a3124c Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Tue, 1 Sep 2015 11:21:24 +0200
Subject: [PATCH 031/373] add Lwt.async_exception_hook handler

---
 src/main/main.ml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/main/main.ml b/src/main/main.ml
index d3092ece..87a3f3a9 100644
--- a/src/main/main.ml
+++ b/src/main/main.ml
@@ -158,6 +158,9 @@ let main () =
   Ssl_threads.init ();
   Ssl.init ~thread_safe:true ();
 
+  Lwt.async_exception_hook :=
+    (fun exn -> Logger.ign_info_f_ ~exn "Caught async exception")
+
   let _ = Bz2.version in
   let () = Sys.set_signal Sys.sigpipe Sys.Signal_ignore in
   let () = Random.self_init () in

From 450b77cc2f70646a0340117a9d8f798ec8d64774 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Fri, 9 Oct 2015 16:41:43 +0200
Subject: [PATCH 032/373] a bit more flushing of output channels

---
 src/tools/server.ml | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/tools/server.ml b/src/tools/server.ml
index 8bc93b09..53fac93f 100644
--- a/src/tools/server.ml
+++ b/src/tools/server.ml
@@ -34,8 +34,7 @@ let close = function
 let deny_max (_ic,oc,_cid) =
   Logger.warning_ "max connections reached, denying this one" >>= fun () ->
   Llio.output_int32 oc (Arakoon_exc.int32_of_rc Arakoon_exc.E_MAX_CONNECTIONS) >>= fun () ->
-  Llio.output_string oc "too many clients" >>= fun () ->
-  Lwt_io.flush oc
+  Llio.output_string oc "too many clients"
 
 let deny_closing (_ic,oc,_cid) =
   Logger.warning_ "closing socket, denying this one" >>= fun () ->
@@ -54,7 +53,12 @@ let session_thread (sid:string) cid protocol fd =
              let ic = Lwt_ssl.in_channel_of_descr fd'
              and oc = Lwt_ssl.out_channel_of_descr fd' in
              (ic, oc)
-       in protocol (ic,oc,cid)
+       in
+       Lwt.finalize
+         (fun () -> protocol (ic,oc,cid))
+         (fun () ->
+          Lwt_io.close oc >>= fun () ->
+          Lwt_io.close ic)
     )
     (function
       | FOOBAR as foobar->

From a34373bef76a4ce1011699d953ef8d8a2b923da1 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Tue, 20 Oct 2015 17:46:50 +0200
Subject: [PATCH 033/373] fix ssl tests (basically, add a semi-colon)

---
 src/main/main.ml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/main.ml b/src/main/main.ml
index 87a3f3a9..e4cca68c 100644
--- a/src/main/main.ml
+++ b/src/main/main.ml
@@ -158,9 +158,9 @@ let main () =
   Ssl_threads.init ();
   Ssl.init ~thread_safe:true ();
 
-  Lwt.async_exception_hook :=
+  let () =Lwt.async_exception_hook :=
     (fun exn -> Logger.ign_info_f_ ~exn "Caught async exception")
-
+  in
   let _ = Bz2.version in
   let () = Sys.set_signal Sys.sigpipe Sys.Signal_ignore in
   let () = Random.self_init () in

From 112486346b1631e6d9657373a415bdf4c225d47b Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Mon, 26 Oct 2015 14:24:59 +0100
Subject: [PATCH 034/373] ocaml 4.02.3 and recent libraries

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 59d7c812..a0e50226 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,7 +1,7 @@
 language: c
 sudo: required
 env:
-  - PPA=avsm/ocaml42+opam12
+  - PPA=ppa:avsm/ocaml42+opam12
 
 before_install: ./travis.sh before_install
 install: ./travis.sh install

From 62d412378e77a862a182db820c4889936e95c5a5 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Mon, 26 Oct 2015 14:38:57 +0100
Subject: [PATCH 035/373] add docker file

---
 .travis.yml                                   |  2 +-
 docker/arakoon_debian/Dockerfile              | 73 +++++++++++++++++++
 jenkins/008_install_dev_env.sh                |  4 +-
 .../005_install_ubuntu_dependencies.sh        | 10 ---
 jenkins/package_deb/seed                      |  1 -
 5 files changed, 76 insertions(+), 14 deletions(-)
 create mode 100644 docker/arakoon_debian/Dockerfile
 delete mode 100755 jenkins/package_deb/005_install_ubuntu_dependencies.sh
 delete mode 120000 jenkins/package_deb/seed

diff --git a/.travis.yml b/.travis.yml
index a0e50226..59d7c812 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,7 +1,7 @@
 language: c
 sudo: required
 env:
-  - PPA=ppa:avsm/ocaml42+opam12
+  - PPA=avsm/ocaml42+opam12
 
 before_install: ./travis.sh before_install
 install: ./travis.sh install
diff --git a/docker/arakoon_debian/Dockerfile b/docker/arakoon_debian/Dockerfile
new file mode 100644
index 00000000..0668effe
--- /dev/null
+++ b/docker/arakoon_debian/Dockerfile
@@ -0,0 +1,73 @@
+FROM ubuntu:15.04
+RUN apt-get -y update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+        build-essential m4 apt-utils \
+        libffi-dev libssl-dev \
+        libbz2-dev \
+        libgmp3-dev \
+        libev-dev \
+        libsnappy-dev \
+        libxen-dev \
+        help2man \
+        pkg-config \
+        time \
+        aspcud \
+        wget \
+        rsync \
+        darcs \
+        git \
+        unzip \
+        protobuf-compiler \
+        libgcrypt20-dev \
+        libjerasure-dev \
+        yasm \
+        automake \
+        python-dev \
+        python-pip \
+        debhelper \
+        psmisc \
+        strace \
+        curl \
+        g++ \
+        libgflags-dev \
+        sudo \
+        libtool \
+        libboost-all-dev \
+        fuse
+
+RUN useradd jenkins -u 1001 -g root
+
+RUN pip install fabric junit-xml
+
+RUN wget https://raw.github.com/ocaml/opam/master/shell/opam_installer.sh
+RUN sh ./opam_installer.sh /usr/local/bin 4.02.3
+RUN mkdir /home/opam
+ENV opam_env='opam config env --root=/home/opam/OPAM'
+RUN opam init --root=/home/opam/OPAM --comp 4.02.3
+
+RUN eval `${opam_env}` && opam install -y \
+        ocamlfind \
+        ssl \
+        camlbz2 \
+        snappy \
+        sexplib \
+        bisect \
+        lwt \
+        camltc \
+        cstruct \
+        ctypes \
+        ctypes-foreign \
+        uuidm \
+        zarith \
+        mirage-no-xen.1 \
+        quickcheck \
+        nocrypto \
+        cmdliner \
+        conf-libev \
+        depext \
+        kinetic-client \
+        tiny_json \
+        ppx_deriving_yojson \
+        core_kernel
+
+RUN echo "jenkins ALL=NOPASSWD: ALL" >/etc/sudoers.d/jenkins
+ENV LANG=C
diff --git a/jenkins/008_install_dev_env.sh b/jenkins/008_install_dev_env.sh
index 7f065a61..4edfe4c1 100755
--- a/jenkins/008_install_dev_env.sh
+++ b/jenkins/008_install_dev_env.sh
@@ -2,7 +2,7 @@
 
 which opam > /dev/null || { echo 'opam not found!'; exit 1; }
 
-opam switch 4.02.1
+opam switch 4.02.3
 eval `opam config env`
 
 # do this in 1 step, otherwise, you might not get what you want
@@ -19,6 +19,6 @@ opam install -y ssl.0.5.0 \
      ocplib-endian.0.8 \
      quickcheck.1.0.2 \
      nocrypto.0.5.1 \
-     sexplib.112.35.00 \
+     sexplib.113.00.00 \
      uuidm.0.9.5 \
      zarith.1.3
diff --git a/jenkins/package_deb/005_install_ubuntu_dependencies.sh b/jenkins/package_deb/005_install_ubuntu_dependencies.sh
deleted file mode 100755
index 8482fe69..00000000
--- a/jenkins/package_deb/005_install_ubuntu_dependencies.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/bash -xue
-
-sudo aptitude update || true
-
-for PKG in libssl-dev \
- texlive texlive-latex-extra \
- git help2man python-epydoc graphviz \
- libsnappy-dev rpm; do
- sudo aptitude install -yVDq $PKG
-done
diff --git a/jenkins/package_deb/seed b/jenkins/package_deb/seed
deleted file mode 120000
index 85b4efcc..00000000
--- a/jenkins/package_deb/seed
+++ /dev/null
@@ -1 +0,0 @@
-../base/seed
\ No newline at end of file

From d035acb9bd4e942e161cea6e83a378ce2ee4d686 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Mon, 26 Oct 2015 16:25:49 +0100
Subject: [PATCH 036/373] docker image for rpm

---
 _tags                             |  2 +
 docker/arakoon_centos7/Dockerfile | 86 +++++++++++++++++++++++++++++++
 docker/arakoon_debian/Dockerfile  |  2 +-
 jenkins/package_rpm/010_build.sh  | 23 +++++----
 4 files changed, 102 insertions(+), 11 deletions(-)
 create mode 100644 docker/arakoon_centos7/Dockerfile

diff --git a/_tags b/_tags
index d108f409..18015af9 100644
--- a/_tags
+++ b/_tags
@@ -38,3 +38,5 @@ true: package(nocrypto)
 <examples/ocaml/*.{byte,native}>: is_main
 "logger_macro.cmo" : use_camlp4, camlp4orf
 <src/**/*.ml> : camlp4of, use_log_macro, maybe_use_bisect, unsafe_string
+"rpmbuild": -traverse
+"arakoon_docker_rpm": -traverse
diff --git a/docker/arakoon_centos7/Dockerfile b/docker/arakoon_centos7/Dockerfile
new file mode 100644
index 00000000..f3884fa2
--- /dev/null
+++ b/docker/arakoon_centos7/Dockerfile
@@ -0,0 +1,86 @@
+FROM centos:7
+
+RUN rpm --import file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
+RUN yum -y install wget
+RUN yum -y install curl
+RUN yum -y install make m4 gcc patch unzip
+RUN yum -y install git rsync mercurial
+RUN yum -y install gcc-c++
+RUN yum -y install bzip2-devel libffi-devel snappy-devel libev-devel
+RUN yum -y install python-devel
+
+# TODO: What about darcs ?
+
+RUN useradd jenkins -u 1001 -g root
+
+
+RUN wget https://raw.github.com/ocaml/opam/master/shell/opam_installer.sh
+RUN sh ./opam_installer.sh /usr/local/bin 4.02.3
+RUN mkdir /home/opam
+ENV opam_env='opam config env --root=/home/opam/OPAM'
+RUN opam init --root=/home/opam/OPAM --comp 4.02.3
+
+
+
+RUN rpm -iUvh http://dl.fedoraproject.org/pub/epel/7/x86_64/e/epel-release-7-5.noarch.rpm
+RUN yum -y update
+RUN yum -y install python-pip
+RUN yum -y install openssl-devel gmp-devel which
+RUN yum -y install zlib-devel
+
+# protobuf
+RUN yum -y install bzip2
+RUN wget https://github.com/google/protobuf/releases/download/v2.6.1/protobuf-2.6.1.tar.bz2 -O protobuf-2.6.1.tar.bz2 \
+    && tar -jxvf protobuf-2.6.1.tar.bz2 \
+    && cd protobuf-2.6.1 \
+    && ./configure && make && make install
+
+#RUN rpm -ivh http://cbs.centos.org/kojifiles/packages/protobuf/2.5.0/10.el7.centos/src/protobuf-2.5.0-10.el7.centos.src.rpm
+
+RUN pip install fabric junit-xml
+
+RUN wget http://cudf-solvers.irill.org/cudf_remote_proxy
+RUN chmod u+x cudf_remote_proxy
+RUN mv cudf_remote_proxy /usr/local/bin/
+
+ENV OPAMEXTERNALSOLVER="cudf_remote_proxy %{input}% %{output}% %{criteria}%"
+
+RUN eval `${opam_env}` && opam install -y \
+        ocamlfind \
+        ssl \
+        camlbz2 \
+        snappy \
+        sexplib \
+        bisect \
+        lwt \
+        camltc \
+        cstruct \
+        ctypes \
+        ctypes-foreign \
+        uuidm \
+        zarith \
+        mirage-no-xen.1 \
+        quickcheck \
+        nocrypto \
+        cmdliner \
+        conf-libev \
+        depext \
+        kinetic-client \
+        tiny_json \
+        ppx_deriving_yojson \
+        core_kernel
+
+
+RUN yum -y install autoconf help2man perl-Thread-Queue
+
+RUN yum -y install sudo
+#
+# Disable "ssh hostname sudo <cmd>", because it will show the password in clear.
+#         You have to run "ssh -t hostname sudo <cmd>".
+#
+# Defaults    requiretty          # is line 56
+RUN awk 'NR == 56 {next} {print}' /etc/sudoers >/tmp/__sudoers && mv /tmp/__sudoers /etc/sudoers
+
+RUN yum -y install rpm-build
+
+CMD eval `${opam_env}` && opam list
diff --git a/docker/arakoon_debian/Dockerfile b/docker/arakoon_debian/Dockerfile
index 0668effe..71893f54 100644
--- a/docker/arakoon_debian/Dockerfile
+++ b/docker/arakoon_debian/Dockerfile
@@ -36,7 +36,7 @@ RUN apt-get -y update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
 
 RUN useradd jenkins -u 1001 -g root
 
-RUN pip install fabric junit-xml
+RUN pip install fabric junit-xml nose
 
 RUN wget https://raw.github.com/ocaml/opam/master/shell/opam_installer.sh
 RUN sh ./opam_installer.sh /usr/local/bin 4.02.3
diff --git a/jenkins/package_rpm/010_build.sh b/jenkins/package_rpm/010_build.sh
index 55fd6cf0..a1ee1319 100755
--- a/jenkins/package_rpm/010_build.sh
+++ b/jenkins/package_rpm/010_build.sh
@@ -1,12 +1,15 @@
 #!/bin/bash -xue
 
-eval `opam config env`
-
-ln -s '../../' ./redhat/SOURCES/arakoon | true
-ARAKOON_HOME=`pwd`
-cd ~/
-rm rpmbuild | true
-ln -s $ARAKOON_HOME/redhat rpmbuild
-cd rpmbuild
-rpmbuild -ba SPECS/arakoon.spec
-
+eval `${opam_env}`
+export START=${PWD}
+echo START=${START}
+rm -rf ${START}/rpmbuild/
+make clean
+mkdir -p ${START}/rpmbuild/SOURCES
+cd ${START}/rpmbuild/SOURCES/
+ln -f -s ${START} ./arakoon
+cd ${START}
+ls -lR .
+#chown root:root ./redhat/SPECS/arakoon.spec
+#make
+rpmbuild --define "_topdir ${START}/rpmbuild" -bb ./redhat/SPECS/arakoon.spec

From 85a69967cef1aadd3fc5acb9cf227bd94b01b98e Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Tue, 27 Oct 2015 09:32:37 +0100
Subject: [PATCH 037/373] removed plenty of decommissioned scripts

---
 jenkins/008_install_dev_env.sh                | 24 -------
 jenkins/030_prepare_system_tests.sh           |  5 --
 jenkins/050_run_pending_configuration.sh      |  5 --
 jenkins/060_install_nose.sh                   | 68 -------------------
 jenkins/B/005_install_ubuntu_dependencies.sh  |  1 -
 jenkins/B/008_install_dev_env.sh              |  1 -
 jenkins/B/080_archive_version.sh              |  1 -
 jenkins/B/seed                                |  1 -
 jenkins/C/005_install_ubuntu_dependencies.sh  |  1 -
 jenkins/C/008_install_dev_env.sh              |  1 -
 jenkins/C/080_archive_version.sh              |  1 -
 jenkins/C/seed                                |  1 -
 jenkins/D/005_install_ubuntu_dependencies.sh  |  1 -
 jenkins/D/008_install_dev_env.sh              |  1 -
 jenkins/D/080_archive_version.sh              |  1 -
 jenkins/D/seed                                |  1 -
 .../base/005_install_ubuntu_dependencies.sh   |  9 ---
 jenkins/base/008_install_dev_env.sh           |  1 -
 jenkins/base/seed                             |  1 -
 jenkins/kwik/seed                             |  1 -
 jenkins/package_rpm/seed                      |  1 -
 jenkins/seed                                  |  1 -
 .../005_install_ubuntu_dependencies.sh        |  1 -
 .../system_tests_quick/008_install_dev_env.sh |  1 -
 .../030_prepare_system_tests.sh               |  4 --
 .../040_build_in_sandbox.sh                   |  1 -
 .../050_run_pending_configuration.sh          |  5 --
 .../system_tests_quick/060_install_nose.sh    | 68 -------------------
 jenkins/system_tests_quick/070_run_tests.sh   | 33 ---------
 .../system_tests_quick/080_archive_version.sh |  1 -
 jenkins/system_tests_quick/seed               |  1 -
 .../005_install_ubuntu_dependencies.sh        |  1 -
 .../008_install_dev_env.sh                    |  1 -
 .../030_prepare_system_tests.sh               |  1 -
 .../040_build_in_sandbox.sh                   |  1 -
 .../050_run_pending_configuration.sh          |  1 -
 .../060_install_nose.sh                       |  1 -
 .../system_tests_slow_left/070_run_tests.sh   | 34 ----------
 .../080_archive_version.sh                    |  1 -
 jenkins/system_tests_slow_left/seed           |  1 -
 .../005_install_ubuntu_dependencies.sh        |  1 -
 .../008_install_dev_env.sh                    |  1 -
 .../030_prepare_system_tests.sh               |  1 -
 .../040_build_in_sandbox.sh                   |  1 -
 .../050_run_pending_configuration.sh          |  1 -
 .../060_install_nose.sh                       |  1 -
 .../system_tests_slow_right/070_run_tests.sh  | 31 ---------
 .../080_archive_version.sh                    |  1 -
 jenkins/system_tests_slow_right/seed          |  1 -
 jenkins/system_tests_slow_shaky/seed          |  1 -
 50 files changed, 325 deletions(-)
 delete mode 100755 jenkins/008_install_dev_env.sh
 delete mode 100755 jenkins/030_prepare_system_tests.sh
 delete mode 100755 jenkins/050_run_pending_configuration.sh
 delete mode 100755 jenkins/060_install_nose.sh
 delete mode 120000 jenkins/B/005_install_ubuntu_dependencies.sh
 delete mode 120000 jenkins/B/008_install_dev_env.sh
 delete mode 120000 jenkins/B/080_archive_version.sh
 delete mode 120000 jenkins/B/seed
 delete mode 120000 jenkins/C/005_install_ubuntu_dependencies.sh
 delete mode 120000 jenkins/C/008_install_dev_env.sh
 delete mode 120000 jenkins/C/080_archive_version.sh
 delete mode 120000 jenkins/C/seed
 delete mode 120000 jenkins/D/005_install_ubuntu_dependencies.sh
 delete mode 120000 jenkins/D/008_install_dev_env.sh
 delete mode 120000 jenkins/D/080_archive_version.sh
 delete mode 120000 jenkins/D/seed
 delete mode 100755 jenkins/base/005_install_ubuntu_dependencies.sh
 delete mode 120000 jenkins/base/008_install_dev_env.sh
 delete mode 100644 jenkins/base/seed
 delete mode 100644 jenkins/kwik/seed
 delete mode 120000 jenkins/package_rpm/seed
 delete mode 100644 jenkins/seed
 delete mode 120000 jenkins/system_tests_quick/005_install_ubuntu_dependencies.sh
 delete mode 120000 jenkins/system_tests_quick/008_install_dev_env.sh
 delete mode 100755 jenkins/system_tests_quick/030_prepare_system_tests.sh
 delete mode 120000 jenkins/system_tests_quick/040_build_in_sandbox.sh
 delete mode 100755 jenkins/system_tests_quick/050_run_pending_configuration.sh
 delete mode 100755 jenkins/system_tests_quick/060_install_nose.sh
 delete mode 100755 jenkins/system_tests_quick/070_run_tests.sh
 delete mode 120000 jenkins/system_tests_quick/080_archive_version.sh
 delete mode 120000 jenkins/system_tests_quick/seed
 delete mode 120000 jenkins/system_tests_slow_left/005_install_ubuntu_dependencies.sh
 delete mode 120000 jenkins/system_tests_slow_left/008_install_dev_env.sh
 delete mode 120000 jenkins/system_tests_slow_left/030_prepare_system_tests.sh
 delete mode 120000 jenkins/system_tests_slow_left/040_build_in_sandbox.sh
 delete mode 120000 jenkins/system_tests_slow_left/050_run_pending_configuration.sh
 delete mode 120000 jenkins/system_tests_slow_left/060_install_nose.sh
 delete mode 100755 jenkins/system_tests_slow_left/070_run_tests.sh
 delete mode 120000 jenkins/system_tests_slow_left/080_archive_version.sh
 delete mode 120000 jenkins/system_tests_slow_left/seed
 delete mode 120000 jenkins/system_tests_slow_right/005_install_ubuntu_dependencies.sh
 delete mode 120000 jenkins/system_tests_slow_right/008_install_dev_env.sh
 delete mode 120000 jenkins/system_tests_slow_right/030_prepare_system_tests.sh
 delete mode 120000 jenkins/system_tests_slow_right/040_build_in_sandbox.sh
 delete mode 120000 jenkins/system_tests_slow_right/050_run_pending_configuration.sh
 delete mode 120000 jenkins/system_tests_slow_right/060_install_nose.sh
 delete mode 100755 jenkins/system_tests_slow_right/070_run_tests.sh
 delete mode 120000 jenkins/system_tests_slow_right/080_archive_version.sh
 delete mode 120000 jenkins/system_tests_slow_right/seed
 delete mode 120000 jenkins/system_tests_slow_shaky/seed

diff --git a/jenkins/008_install_dev_env.sh b/jenkins/008_install_dev_env.sh
deleted file mode 100755
index 4edfe4c1..00000000
--- a/jenkins/008_install_dev_env.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/bin/bash -xue
-
-which opam > /dev/null || { echo 'opam not found!'; exit 1; }
-
-opam switch 4.02.3
-eval `opam config env`
-
-# do this in 1 step, otherwise, you might not get what you want
-opam install -y ssl.0.5.0 \
-     conf-libev.4-11 \
-     camlbz2.0.6.0 \
-     snappy.0.1.0 \
-     lwt.2.5.0 \
-     camltc.0.9.2 \
-     cstruct.1.7.0 \
-     ctypes.0.4.1 \
-     ctypes-foreign.0.4.0 \
-     bisect.1.3 \
-     ocplib-endian.0.8 \
-     quickcheck.1.0.2 \
-     nocrypto.0.5.1 \
-     sexplib.113.00.00 \
-     uuidm.0.9.5 \
-     zarith.1.3
diff --git a/jenkins/030_prepare_system_tests.sh b/jenkins/030_prepare_system_tests.sh
deleted file mode 100755
index 0fb86def..00000000
--- a/jenkins/030_prepare_system_tests.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/bash -xue
-
-/opt/qbase3/qshell ./jenkins/prepareSystemTests.py
-
-mkdir -p ${WORKSPACE}/testresults
diff --git a/jenkins/050_run_pending_configuration.sh b/jenkins/050_run_pending_configuration.sh
deleted file mode 100755
index 1a910448..00000000
--- a/jenkins/050_run_pending_configuration.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/bash -xue
-
-/opt/qbase3/qshell -c "
-q.qp._runPendingReconfigeFiles()
-"
diff --git a/jenkins/060_install_nose.sh b/jenkins/060_install_nose.sh
deleted file mode 100755
index a4713ee6..00000000
--- a/jenkins/060_install_nose.sh
+++ /dev/null
@@ -1,68 +0,0 @@
-#!/bin/bash -xue
-
-echo "Install nose utilities in QBase"
-
-test -f /tmp/coverage-3.4b2.tar.gz || wget -q -O /tmp/coverage-3.4b2.tar.gz http://pypi.python.org/packages/source/c/coverage/coverage-3.4b2.tar.gz
-/opt/qbase3/bin/easy_install /tmp/coverage-3.4b2.tar.gz
-
-test -f /tmp/nosexcover-1.0.4.tar.gz || wget -q -O /tmp/nosexcover-1.0.4.tar.gz http://pypi.python.org/packages/source/n/nosexcover/nosexcover-1.0.4.tar.gz
-/opt/qbase3/bin/easy_install /tmp/nosexcover-1.0.4.tar.gz
-
-rm -f ${WORKSPACE}/coverage.xml
-rm -f /opt/qbase3/var/tests/coverage.xml
-
-cd ${WORKSPACE}
-mkdir -p _tools
-cd _tools
-rm -f fix_coverage.py
-
-cat > fix_coverage.py << EOF
-import sys
-import os
-import os.path
-import xml.sax
-import xml.sax.saxutils
-
-class CoverageFixHandler(xml.sax.saxutils.XMLGenerator):
-    def __init__(self, base, prefixes, *args, **kwargs):
-        xml.sax.saxutils.XMLGenerator.__init__(self, *args, **kwargs)
-
-        self._base = base
-        self._prefixes = tuple(
-            sorted(prefixes, key=lambda s: len(s), reverse=True))
-
-    def startElement(self, name, attrs):
-        attrs = dict(attrs.items())
-
-        # Note: can't use 'in' !
-        if 'filename' in attrs:
-            filename = attrs['filename']
-
-            if filename.startswith('/'):
-                # Convert into relative path
-                relative_path = os.path.relpath(filename, self._base)
-
-                attrs['filename'] = relative_path
-
-        if name == 'package' and 'name' in attrs:
-            package = attrs['name']
-
-            for prefix in self._prefixes:
-                if package.startswith(prefix):
-                    attrs['name'] = package[len(prefix):]
-                    break
-
-        return xml.sax.saxutils.XMLGenerator.startElement(self, name, attrs)
-
-
-def fix(base, prefixes, in_, out):
-    xml.sax.parse(in_, CoverageFixHandler(base, prefixes, out))
-
-if __name__ == '__main__':
-    if len(sys.argv) < 2:
-        sys.stderr.write('Missing base argument\n')
-        sys.exit(1)
-
-    fix(sys.argv[1], sys.argv[2:], sys.stdin, sys.stdout)
-
-EOF
diff --git a/jenkins/B/005_install_ubuntu_dependencies.sh b/jenkins/B/005_install_ubuntu_dependencies.sh
deleted file mode 120000
index b5b3d4d8..00000000
--- a/jenkins/B/005_install_ubuntu_dependencies.sh
+++ /dev/null
@@ -1 +0,0 @@
-../kwik/005_install_ubuntu_dependencies.sh
\ No newline at end of file
diff --git a/jenkins/B/008_install_dev_env.sh b/jenkins/B/008_install_dev_env.sh
deleted file mode 120000
index 284bc908..00000000
--- a/jenkins/B/008_install_dev_env.sh
+++ /dev/null
@@ -1 +0,0 @@
-../kwik/008_install_dev_env.sh
\ No newline at end of file
diff --git a/jenkins/B/080_archive_version.sh b/jenkins/B/080_archive_version.sh
deleted file mode 120000
index e90c6e5d..00000000
--- a/jenkins/B/080_archive_version.sh
+++ /dev/null
@@ -1 +0,0 @@
-../kwik/080_archive_version.sh
\ No newline at end of file
diff --git a/jenkins/B/seed b/jenkins/B/seed
deleted file mode 120000
index 8c02f798..00000000
--- a/jenkins/B/seed
+++ /dev/null
@@ -1 +0,0 @@
-../seed
\ No newline at end of file
diff --git a/jenkins/C/005_install_ubuntu_dependencies.sh b/jenkins/C/005_install_ubuntu_dependencies.sh
deleted file mode 120000
index b5b3d4d8..00000000
--- a/jenkins/C/005_install_ubuntu_dependencies.sh
+++ /dev/null
@@ -1 +0,0 @@
-../kwik/005_install_ubuntu_dependencies.sh
\ No newline at end of file
diff --git a/jenkins/C/008_install_dev_env.sh b/jenkins/C/008_install_dev_env.sh
deleted file mode 120000
index 284bc908..00000000
--- a/jenkins/C/008_install_dev_env.sh
+++ /dev/null
@@ -1 +0,0 @@
-../kwik/008_install_dev_env.sh
\ No newline at end of file
diff --git a/jenkins/C/080_archive_version.sh b/jenkins/C/080_archive_version.sh
deleted file mode 120000
index e90c6e5d..00000000
--- a/jenkins/C/080_archive_version.sh
+++ /dev/null
@@ -1 +0,0 @@
-../kwik/080_archive_version.sh
\ No newline at end of file
diff --git a/jenkins/C/seed b/jenkins/C/seed
deleted file mode 120000
index 8c02f798..00000000
--- a/jenkins/C/seed
+++ /dev/null
@@ -1 +0,0 @@
-../seed
\ No newline at end of file
diff --git a/jenkins/D/005_install_ubuntu_dependencies.sh b/jenkins/D/005_install_ubuntu_dependencies.sh
deleted file mode 120000
index b5b3d4d8..00000000
--- a/jenkins/D/005_install_ubuntu_dependencies.sh
+++ /dev/null
@@ -1 +0,0 @@
-../kwik/005_install_ubuntu_dependencies.sh
\ No newline at end of file
diff --git a/jenkins/D/008_install_dev_env.sh b/jenkins/D/008_install_dev_env.sh
deleted file mode 120000
index 284bc908..00000000
--- a/jenkins/D/008_install_dev_env.sh
+++ /dev/null
@@ -1 +0,0 @@
-../kwik/008_install_dev_env.sh
\ No newline at end of file
diff --git a/jenkins/D/080_archive_version.sh b/jenkins/D/080_archive_version.sh
deleted file mode 120000
index e90c6e5d..00000000
--- a/jenkins/D/080_archive_version.sh
+++ /dev/null
@@ -1 +0,0 @@
-../kwik/080_archive_version.sh
\ No newline at end of file
diff --git a/jenkins/D/seed b/jenkins/D/seed
deleted file mode 120000
index 27568078..00000000
--- a/jenkins/D/seed
+++ /dev/null
@@ -1 +0,0 @@
-../kwik/seed
\ No newline at end of file
diff --git a/jenkins/base/005_install_ubuntu_dependencies.sh b/jenkins/base/005_install_ubuntu_dependencies.sh
deleted file mode 100755
index 912928d0..00000000
--- a/jenkins/base/005_install_ubuntu_dependencies.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/bin/bash -xue
-
-sudo aptitude update || true
-
-for PKG in libssl-dev \
- texlive texlive-latex-extra \
- git python-epydoc graphviz libsnappy-dev symlinks; do
-    sudo aptitude install -yVDq $PKG
-done
diff --git a/jenkins/base/008_install_dev_env.sh b/jenkins/base/008_install_dev_env.sh
deleted file mode 120000
index dcf1c6b1..00000000
--- a/jenkins/base/008_install_dev_env.sh
+++ /dev/null
@@ -1 +0,0 @@
-../008_install_dev_env.sh
\ No newline at end of file
diff --git a/jenkins/base/seed b/jenkins/base/seed
deleted file mode 100644
index 418ec803..00000000
--- a/jenkins/base/seed
+++ /dev/null
@@ -1 +0,0 @@
-arakoon-1.x-ubuntu-12.10-quantal-20140617
diff --git a/jenkins/kwik/seed b/jenkins/kwik/seed
deleted file mode 100644
index 6ca0a618..00000000
--- a/jenkins/kwik/seed
+++ /dev/null
@@ -1 +0,0 @@
-arakoon-1.x-qbase3-ubuntu-12.10-quantal-20140310
diff --git a/jenkins/package_rpm/seed b/jenkins/package_rpm/seed
deleted file mode 120000
index 85b4efcc..00000000
--- a/jenkins/package_rpm/seed
+++ /dev/null
@@ -1 +0,0 @@
-../base/seed
\ No newline at end of file
diff --git a/jenkins/seed b/jenkins/seed
deleted file mode 100644
index 54009052..00000000
--- a/jenkins/seed
+++ /dev/null
@@ -1 +0,0 @@
-arakoon-1.x-qbase3-ubuntu-12.10-quantal-20140617
diff --git a/jenkins/system_tests_quick/005_install_ubuntu_dependencies.sh b/jenkins/system_tests_quick/005_install_ubuntu_dependencies.sh
deleted file mode 120000
index 782d9ce3..00000000
--- a/jenkins/system_tests_quick/005_install_ubuntu_dependencies.sh
+++ /dev/null
@@ -1 +0,0 @@
-../base/005_install_ubuntu_dependencies.sh
\ No newline at end of file
diff --git a/jenkins/system_tests_quick/008_install_dev_env.sh b/jenkins/system_tests_quick/008_install_dev_env.sh
deleted file mode 120000
index b535b265..00000000
--- a/jenkins/system_tests_quick/008_install_dev_env.sh
+++ /dev/null
@@ -1 +0,0 @@
-../base/008_install_dev_env.sh
\ No newline at end of file
diff --git a/jenkins/system_tests_quick/030_prepare_system_tests.sh b/jenkins/system_tests_quick/030_prepare_system_tests.sh
deleted file mode 100755
index 00191beb..00000000
--- a/jenkins/system_tests_quick/030_prepare_system_tests.sh
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/bin/bash -xue
-
-/opt/qbase3/qshell ./jenkins/prepareSystemTests.py
-mkdir -p ${WORKSPACE}/testresults
diff --git a/jenkins/system_tests_quick/040_build_in_sandbox.sh b/jenkins/system_tests_quick/040_build_in_sandbox.sh
deleted file mode 120000
index 715c5650..00000000
--- a/jenkins/system_tests_quick/040_build_in_sandbox.sh
+++ /dev/null
@@ -1 +0,0 @@
-../build_in_shandbox.sh
\ No newline at end of file
diff --git a/jenkins/system_tests_quick/050_run_pending_configuration.sh b/jenkins/system_tests_quick/050_run_pending_configuration.sh
deleted file mode 100755
index 1a910448..00000000
--- a/jenkins/system_tests_quick/050_run_pending_configuration.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/bash -xue
-
-/opt/qbase3/qshell -c "
-q.qp._runPendingReconfigeFiles()
-"
diff --git a/jenkins/system_tests_quick/060_install_nose.sh b/jenkins/system_tests_quick/060_install_nose.sh
deleted file mode 100755
index a4713ee6..00000000
--- a/jenkins/system_tests_quick/060_install_nose.sh
+++ /dev/null
@@ -1,68 +0,0 @@
-#!/bin/bash -xue
-
-echo "Install nose utilities in QBase"
-
-test -f /tmp/coverage-3.4b2.tar.gz || wget -q -O /tmp/coverage-3.4b2.tar.gz http://pypi.python.org/packages/source/c/coverage/coverage-3.4b2.tar.gz
-/opt/qbase3/bin/easy_install /tmp/coverage-3.4b2.tar.gz
-
-test -f /tmp/nosexcover-1.0.4.tar.gz || wget -q -O /tmp/nosexcover-1.0.4.tar.gz http://pypi.python.org/packages/source/n/nosexcover/nosexcover-1.0.4.tar.gz
-/opt/qbase3/bin/easy_install /tmp/nosexcover-1.0.4.tar.gz
-
-rm -f ${WORKSPACE}/coverage.xml
-rm -f /opt/qbase3/var/tests/coverage.xml
-
-cd ${WORKSPACE}
-mkdir -p _tools
-cd _tools
-rm -f fix_coverage.py
-
-cat > fix_coverage.py << EOF
-import sys
-import os
-import os.path
-import xml.sax
-import xml.sax.saxutils
-
-class CoverageFixHandler(xml.sax.saxutils.XMLGenerator):
-    def __init__(self, base, prefixes, *args, **kwargs):
-        xml.sax.saxutils.XMLGenerator.__init__(self, *args, **kwargs)
-
-        self._base = base
-        self._prefixes = tuple(
-            sorted(prefixes, key=lambda s: len(s), reverse=True))
-
-    def startElement(self, name, attrs):
-        attrs = dict(attrs.items())
-
-        # Note: can't use 'in' !
-        if 'filename' in attrs:
-            filename = attrs['filename']
-
-            if filename.startswith('/'):
-                # Convert into relative path
-                relative_path = os.path.relpath(filename, self._base)
-
-                attrs['filename'] = relative_path
-
-        if name == 'package' and 'name' in attrs:
-            package = attrs['name']
-
-            for prefix in self._prefixes:
-                if package.startswith(prefix):
-                    attrs['name'] = package[len(prefix):]
-                    break
-
-        return xml.sax.saxutils.XMLGenerator.startElement(self, name, attrs)
-
-
-def fix(base, prefixes, in_, out):
-    xml.sax.parse(in_, CoverageFixHandler(base, prefixes, out))
-
-if __name__ == '__main__':
-    if len(sys.argv) < 2:
-        sys.stderr.write('Missing base argument\n')
-        sys.exit(1)
-
-    fix(sys.argv[1], sys.argv[2:], sys.stdin, sys.stdout)
-
-EOF
diff --git a/jenkins/system_tests_quick/070_run_tests.sh b/jenkins/system_tests_quick/070_run_tests.sh
deleted file mode 100755
index 5243edae..00000000
--- a/jenkins/system_tests_quick/070_run_tests.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/bash -xue
-
-/opt/qbase3/qshell -c "
-
-q.testrunner.list()
-
-test_spec = '${TEST_SPEC-arakoon_system_tests.server.quick}'
-#test_spec = 'arakoon_system_tests.server.system_tests_long'
-#test_spec = 'arakoon_system_tests'
-#test_spec = 'arakoon_system_tests.server.TestCmdTools'
-
-output_format = q.enumerators.TestRunnerOutputFormat.XML
-output_folder = '${WORKSPACE}/testresults'
-
-arguments = q.testrunner._parseArgs('%s_results' % test_spec, output_format, output_folder, False)
-#arguments = q.testrunner._parseArgs(test_spec, output_format, output_folder, False)
-arguments.extend([
-    '--with-coverage',
-    '--cover-package=arakoon',
-    '--cover-erase',
-    '--with-xcoverage',
-])
-arguments.append(q.testrunner._convertTestSpec(test_spec))
-q.testrunner._runTests(arguments)
-"
-
-RESULT=$?
-
-test -f /opt/qbase3/var/tests/coverage.xml && mv /opt/qbase3/var/tests/coverage.xml ${WORKSPACE}/coverage.xml
-
-cat ${WORKSPACE}/coverage.xml | python ${WORKSPACE}/_tools/fix_coverage.py "${WORKSPACE}" ".opt.qbase3.lib.python.site-packages." ".var.hudson.workspace.pyrakoon-tip-system-tests.python_env.pyrakoon." > ${WORKSPACE}/coverage_fixed.xml
-
-if test $RESULT -eq 0; then true; else false; fi
diff --git a/jenkins/system_tests_quick/080_archive_version.sh b/jenkins/system_tests_quick/080_archive_version.sh
deleted file mode 120000
index 80e718fb..00000000
--- a/jenkins/system_tests_quick/080_archive_version.sh
+++ /dev/null
@@ -1 +0,0 @@
-../archive_version.sh
\ No newline at end of file
diff --git a/jenkins/system_tests_quick/seed b/jenkins/system_tests_quick/seed
deleted file mode 120000
index 8c02f798..00000000
--- a/jenkins/system_tests_quick/seed
+++ /dev/null
@@ -1 +0,0 @@
-../seed
\ No newline at end of file
diff --git a/jenkins/system_tests_slow_left/005_install_ubuntu_dependencies.sh b/jenkins/system_tests_slow_left/005_install_ubuntu_dependencies.sh
deleted file mode 120000
index c14d1ca0..00000000
--- a/jenkins/system_tests_slow_left/005_install_ubuntu_dependencies.sh
+++ /dev/null
@@ -1 +0,0 @@
-../system_tests_quick/005_install_ubuntu_dependencies.sh
\ No newline at end of file
diff --git a/jenkins/system_tests_slow_left/008_install_dev_env.sh b/jenkins/system_tests_slow_left/008_install_dev_env.sh
deleted file mode 120000
index b535b265..00000000
--- a/jenkins/system_tests_slow_left/008_install_dev_env.sh
+++ /dev/null
@@ -1 +0,0 @@
-../base/008_install_dev_env.sh
\ No newline at end of file
diff --git a/jenkins/system_tests_slow_left/030_prepare_system_tests.sh b/jenkins/system_tests_slow_left/030_prepare_system_tests.sh
deleted file mode 120000
index 3b70c916..00000000
--- a/jenkins/system_tests_slow_left/030_prepare_system_tests.sh
+++ /dev/null
@@ -1 +0,0 @@
-../030_prepare_system_tests.sh
\ No newline at end of file
diff --git a/jenkins/system_tests_slow_left/040_build_in_sandbox.sh b/jenkins/system_tests_slow_left/040_build_in_sandbox.sh
deleted file mode 120000
index 715c5650..00000000
--- a/jenkins/system_tests_slow_left/040_build_in_sandbox.sh
+++ /dev/null
@@ -1 +0,0 @@
-../build_in_shandbox.sh
\ No newline at end of file
diff --git a/jenkins/system_tests_slow_left/050_run_pending_configuration.sh b/jenkins/system_tests_slow_left/050_run_pending_configuration.sh
deleted file mode 120000
index fa418df6..00000000
--- a/jenkins/system_tests_slow_left/050_run_pending_configuration.sh
+++ /dev/null
@@ -1 +0,0 @@
-../050_run_pending_configuration.sh
\ No newline at end of file
diff --git a/jenkins/system_tests_slow_left/060_install_nose.sh b/jenkins/system_tests_slow_left/060_install_nose.sh
deleted file mode 120000
index 2b3eeaa2..00000000
--- a/jenkins/system_tests_slow_left/060_install_nose.sh
+++ /dev/null
@@ -1 +0,0 @@
-../060_install_nose.sh
\ No newline at end of file
diff --git a/jenkins/system_tests_slow_left/070_run_tests.sh b/jenkins/system_tests_slow_left/070_run_tests.sh
deleted file mode 100755
index 4f582377..00000000
--- a/jenkins/system_tests_slow_left/070_run_tests.sh
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/bin/bash -xue
-
-/opt/qbase3/qshell -c "
-q.testrunner.list()
-
-test_spec = '${TEST_SPEC-arakoon_system_tests.server.left}'
-#test_spec = 'arakoon_system_tests.server.system_tests_long'
-#test_spec = 'arakoon_system_tests'
-#test_spec = 'arakoon_system_tests.server.TestCmdTools'
-
-output_format = q.enumerators.TestRunnerOutputFormat.XML
-output_folder = '${WORKSPACE}/testresults'
-
-arguments = q.testrunner._parseArgs('%s_results' % test_spec, output_format, output_folder, False)
-
-arguments.extend([
-    '--logging-format=%(asctime)-15s %(message)s',
-    '--with-coverage',
-    '--cover-package=arakoon',
-    '--cover-erase',
-    '--with-xcoverage',
-])
-arguments.append(q.testrunner._convertTestSpec(test_spec))
-
-q.testrunner._runTests(arguments)
-"
-
-RESULT=$?
-
-test -f /opt/qbase3/var/tests/coverage.xml && mv /opt/qbase3/var/tests/coverage.xml ${WORKSPACE}/coverage.xml
-
-cat ${WORKSPACE}/coverage.xml | python ${WORKSPACE}/_tools/fix_coverage.py "${WORKSPACE}" ".opt.qbase3.lib.python.site-packages." ".var.hudson.workspace.pyrakoon-tip-system-tests.python_env.pyrakoon." > ${WORKSPACE}/coverage_fixed.xml
-
-if test $RESULT -eq 0; then true; else false; fi
diff --git a/jenkins/system_tests_slow_left/080_archive_version.sh b/jenkins/system_tests_slow_left/080_archive_version.sh
deleted file mode 120000
index 80e718fb..00000000
--- a/jenkins/system_tests_slow_left/080_archive_version.sh
+++ /dev/null
@@ -1 +0,0 @@
-../archive_version.sh
\ No newline at end of file
diff --git a/jenkins/system_tests_slow_left/seed b/jenkins/system_tests_slow_left/seed
deleted file mode 120000
index 8c02f798..00000000
--- a/jenkins/system_tests_slow_left/seed
+++ /dev/null
@@ -1 +0,0 @@
-../seed
\ No newline at end of file
diff --git a/jenkins/system_tests_slow_right/005_install_ubuntu_dependencies.sh b/jenkins/system_tests_slow_right/005_install_ubuntu_dependencies.sh
deleted file mode 120000
index c14d1ca0..00000000
--- a/jenkins/system_tests_slow_right/005_install_ubuntu_dependencies.sh
+++ /dev/null
@@ -1 +0,0 @@
-../system_tests_quick/005_install_ubuntu_dependencies.sh
\ No newline at end of file
diff --git a/jenkins/system_tests_slow_right/008_install_dev_env.sh b/jenkins/system_tests_slow_right/008_install_dev_env.sh
deleted file mode 120000
index b535b265..00000000
--- a/jenkins/system_tests_slow_right/008_install_dev_env.sh
+++ /dev/null
@@ -1 +0,0 @@
-../base/008_install_dev_env.sh
\ No newline at end of file
diff --git a/jenkins/system_tests_slow_right/030_prepare_system_tests.sh b/jenkins/system_tests_slow_right/030_prepare_system_tests.sh
deleted file mode 120000
index 3b70c916..00000000
--- a/jenkins/system_tests_slow_right/030_prepare_system_tests.sh
+++ /dev/null
@@ -1 +0,0 @@
-../030_prepare_system_tests.sh
\ No newline at end of file
diff --git a/jenkins/system_tests_slow_right/040_build_in_sandbox.sh b/jenkins/system_tests_slow_right/040_build_in_sandbox.sh
deleted file mode 120000
index 715c5650..00000000
--- a/jenkins/system_tests_slow_right/040_build_in_sandbox.sh
+++ /dev/null
@@ -1 +0,0 @@
-../build_in_shandbox.sh
\ No newline at end of file
diff --git a/jenkins/system_tests_slow_right/050_run_pending_configuration.sh b/jenkins/system_tests_slow_right/050_run_pending_configuration.sh
deleted file mode 120000
index fa418df6..00000000
--- a/jenkins/system_tests_slow_right/050_run_pending_configuration.sh
+++ /dev/null
@@ -1 +0,0 @@
-../050_run_pending_configuration.sh
\ No newline at end of file
diff --git a/jenkins/system_tests_slow_right/060_install_nose.sh b/jenkins/system_tests_slow_right/060_install_nose.sh
deleted file mode 120000
index 2b3eeaa2..00000000
--- a/jenkins/system_tests_slow_right/060_install_nose.sh
+++ /dev/null
@@ -1 +0,0 @@
-../060_install_nose.sh
\ No newline at end of file
diff --git a/jenkins/system_tests_slow_right/070_run_tests.sh b/jenkins/system_tests_slow_right/070_run_tests.sh
deleted file mode 100755
index 5bac6691..00000000
--- a/jenkins/system_tests_slow_right/070_run_tests.sh
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/bin/bash -xue
-
-/opt/qbase3/qshell -c "
-q.testrunner.list()
-
-test_spec = '${TEST_SPEC-arakoon_system_tests.server.right}'
-
-output_format = q.enumerators.TestRunnerOutputFormat.XML
-output_folder = '${WORKSPACE}/testresults'
-
-arguments = q.testrunner._parseArgs('%s_results' % test_spec, output_format, output_folder, False)
-
-arguments.extend([
-    '--logging-format=%(asctime)-15s %(message)s',
-    '--with-coverage',
-    '--cover-package=arakoon',
-    '--cover-erase',
-    '--with-xcoverage',
-])
-arguments.append(q.testrunner._convertTestSpec(test_spec))
-
-q.testrunner._runTests(arguments)
-"
-
-RESULT=$?
-
-test -f /opt/qbase3/var/tests/coverage.xml && mv /opt/qbase3/var/tests/coverage.xml ${WORKSPACE}/coverage.xml
-
-cat ${WORKSPACE}/coverage.xml | python ${WORKSPACE}/_tools/fix_coverage.py "${WORKSPACE}" ".opt.qbase3.lib.python.site-packages." ".var.hudson.workspace.pyrakoon-tip-system-tests.python_env.pyrakoon." > ${WORKSPACE}/coverage_fixed.xml
-
-if test $RESULT -eq 0; then true; else false; fi
diff --git a/jenkins/system_tests_slow_right/080_archive_version.sh b/jenkins/system_tests_slow_right/080_archive_version.sh
deleted file mode 120000
index 80e718fb..00000000
--- a/jenkins/system_tests_slow_right/080_archive_version.sh
+++ /dev/null
@@ -1 +0,0 @@
-../archive_version.sh
\ No newline at end of file
diff --git a/jenkins/system_tests_slow_right/seed b/jenkins/system_tests_slow_right/seed
deleted file mode 120000
index 8c02f798..00000000
--- a/jenkins/system_tests_slow_right/seed
+++ /dev/null
@@ -1 +0,0 @@
-../seed
\ No newline at end of file
diff --git a/jenkins/system_tests_slow_shaky/seed b/jenkins/system_tests_slow_shaky/seed
deleted file mode 120000
index 8c02f798..00000000
--- a/jenkins/system_tests_slow_shaky/seed
+++ /dev/null
@@ -1 +0,0 @@
-../seed
\ No newline at end of file

From 1389789f1dec0da65c5b65be3a59a33d517f3fa5 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Tue, 27 Oct 2015 10:03:41 +0100
Subject: [PATCH 038/373] need iptables; don't need fuse

---
 docker/arakoon_debian/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/arakoon_debian/Dockerfile b/docker/arakoon_debian/Dockerfile
index 71893f54..886d76aa 100644
--- a/docker/arakoon_debian/Dockerfile
+++ b/docker/arakoon_debian/Dockerfile
@@ -32,7 +32,7 @@ RUN apt-get -y update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
         sudo \
         libtool \
         libboost-all-dev \
-        fuse
+        iptables
 
 RUN useradd jenkins -u 1001 -g root
 

From b42f4c2beda1ef465ed9748df08337d080e48aae Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Tue, 27 Oct 2015 11:34:43 +0100
Subject: [PATCH 039/373] cleanup

---
 .gitignore                                    |  1 +
 buildInSandbox.sh                             | 15 --------
 jenkins/{B/070_run_tests.sh => B.sh}          |  3 +-
 jenkins/B/040_build.sh                        |  1 -
 jenkins/{C/070_run_tests.sh => C.sh}          |  3 +-
 jenkins/C/040_build.sh                        |  1 -
 jenkins/{D/070_run_tests.sh => D.sh}          |  3 +-
 jenkins/D/040_build.sh                        |  1 -
 jenkins/archive_version.sh                    |  3 --
 jenkins/base/010_build.sh                     | 34 -------------------
 jenkins/base/020_build_docs.sh                | 20 -----------
 jenkins/build_in_shandbox.sh                  | 16 ---------
 jenkins/job_template.sh                       |  9 -----
 jenkins/{kwik/070_run_tests.sh => kwik.sh}    |  4 +--
 .../kwik/005_install_ubuntu_dependencies.sh   |  9 -----
 jenkins/kwik/008_install_dev_env.sh           |  1 -
 jenkins/kwik/040_build.sh                     | 14 --------
 jenkins/kwik/080_archive_version.sh           |  3 --
 jenkins/package_deb.sh                        |  5 +++
 jenkins/package_deb/008_install_dev_env.sh    | 24 -------------
 jenkins/package_deb/010_build.sh              |  7 ----
 .../010_build.sh => package_rpm.sh}           |  0
 jenkins/package_rpm/008_install_dev_env.sh    |  1 -
 jenkins/prepareSystemTests.py                 | 13 -------
 .../005_install_ubuntu_dependencies.sh        |  1 -
 .../008_install_dev_env.sh                    |  1 -
 .../030_prepare_system_tests.sh               |  1 -
 .../040_build_in_sandbox.sh                   |  1 -
 .../050_run_pending_configuration.sh          |  1 -
 .../060_install_nose.sh                       |  1 -
 .../system_tests_slow_shaky/070_run_tests.sh  | 31 -----------------
 .../080_archive_version.sh                    |  1 -
 pylabs/test/server/system_tests_common.py     |  5 ++-
 run_docker_debian.sh                          |  6 ++++
 34 files changed, 24 insertions(+), 216 deletions(-)
 delete mode 100755 buildInSandbox.sh
 rename jenkins/{B/070_run_tests.sh => B.sh} (81%)
 delete mode 120000 jenkins/B/040_build.sh
 rename jenkins/{C/070_run_tests.sh => C.sh} (82%)
 delete mode 120000 jenkins/C/040_build.sh
 rename jenkins/{D/070_run_tests.sh => D.sh} (82%)
 delete mode 120000 jenkins/D/040_build.sh
 delete mode 100755 jenkins/archive_version.sh
 delete mode 100755 jenkins/base/010_build.sh
 delete mode 100755 jenkins/base/020_build_docs.sh
 delete mode 100755 jenkins/build_in_shandbox.sh
 delete mode 100644 jenkins/job_template.sh
 rename jenkins/{kwik/070_run_tests.sh => kwik.sh} (75%)
 delete mode 100755 jenkins/kwik/005_install_ubuntu_dependencies.sh
 delete mode 120000 jenkins/kwik/008_install_dev_env.sh
 delete mode 100755 jenkins/kwik/040_build.sh
 delete mode 100755 jenkins/kwik/080_archive_version.sh
 create mode 100755 jenkins/package_deb.sh
 delete mode 100755 jenkins/package_deb/008_install_dev_env.sh
 delete mode 100755 jenkins/package_deb/010_build.sh
 rename jenkins/{package_rpm/010_build.sh => package_rpm.sh} (100%)
 delete mode 120000 jenkins/package_rpm/008_install_dev_env.sh
 delete mode 100644 jenkins/prepareSystemTests.py
 delete mode 120000 jenkins/system_tests_slow_shaky/005_install_ubuntu_dependencies.sh
 delete mode 120000 jenkins/system_tests_slow_shaky/008_install_dev_env.sh
 delete mode 120000 jenkins/system_tests_slow_shaky/030_prepare_system_tests.sh
 delete mode 120000 jenkins/system_tests_slow_shaky/040_build_in_sandbox.sh
 delete mode 120000 jenkins/system_tests_slow_shaky/050_run_pending_configuration.sh
 delete mode 120000 jenkins/system_tests_slow_shaky/060_install_nose.sh
 delete mode 100755 jenkins/system_tests_slow_shaky/070_run_tests.sh
 delete mode 120000 jenkins/system_tests_slow_shaky/080_archive_version.sh
 create mode 100755 run_docker_debian.sh

diff --git a/.gitignore b/.gitignore
index 4e62b590..9ae0e40b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -29,3 +29,4 @@ redhat/BUILD
 /_virtualenv
 /examples/ocaml/_build/
 /examples/ocaml/demo.native
+/testresults.xml
diff --git a/buildInSandbox.sh b/buildInSandbox.sh
deleted file mode 100755
index 1a03bc97..00000000
--- a/buildInSandbox.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/bin/bash -xue
-
-ocamlbuild -use-ocamlfind arakoon.native
-
-mkdir -p /opt/qbase3/apps/arakoon/bin/
-cp arakoon.native /opt/qbase3/apps/arakoon/bin/arakoon
-cp -r pylabs/extensions/* /opt/qbase3/lib/pymonkey/extensions/
-cp pylabs/Compat.py /opt/qbase3/lib/pymonkey/extensions/Compat.py
-
-mkdir -p /opt/qbase3/var/tests/arakoon_system_tests/
-touch /opt/qbase3/var/tests/arakoon_system_tests/__init__.py
-cp -r pylabs/test/* /opt/qbase3/var/tests/arakoon_system_tests/
-mkdir -p /opt/qbase3/lib/python/site-packages/arakoon/
-cp src/client/python/*.py /opt/qbase3/lib/python/site-packages/arakoon/
-
diff --git a/jenkins/B/070_run_tests.sh b/jenkins/B.sh
similarity index 81%
rename from jenkins/B/070_run_tests.sh
rename to jenkins/B.sh
index f301886e..56cc80c6 100755
--- a/jenkins/B/070_run_tests.sh
+++ b/jenkins/B.sh
@@ -1,5 +1,6 @@
 #!/bin/bash -xue
 
 echo $PWD
-
+eval `${opam_env}`
+make
 python test_it.py --with-xunit --xunit-file=${PWD}/testresults.xml ./server/left
diff --git a/jenkins/B/040_build.sh b/jenkins/B/040_build.sh
deleted file mode 120000
index b80cc7fd..00000000
--- a/jenkins/B/040_build.sh
+++ /dev/null
@@ -1 +0,0 @@
-../kwik/040_build.sh
\ No newline at end of file
diff --git a/jenkins/C/070_run_tests.sh b/jenkins/C.sh
similarity index 82%
rename from jenkins/C/070_run_tests.sh
rename to jenkins/C.sh
index f1add4c1..b34d5f56 100755
--- a/jenkins/C/070_run_tests.sh
+++ b/jenkins/C.sh
@@ -1,5 +1,6 @@
 #!/bin/bash -xue
 
 echo $PWD
-
+eval `${opam_env}`
+make
 python test_it.py --with-xunit --xunit-file=${PWD}/testresults.xml ./server/right
diff --git a/jenkins/C/040_build.sh b/jenkins/C/040_build.sh
deleted file mode 120000
index b80cc7fd..00000000
--- a/jenkins/C/040_build.sh
+++ /dev/null
@@ -1 +0,0 @@
-../kwik/040_build.sh
\ No newline at end of file
diff --git a/jenkins/D/070_run_tests.sh b/jenkins/D.sh
similarity index 82%
rename from jenkins/D/070_run_tests.sh
rename to jenkins/D.sh
index 107a8f2c..1516cec0 100755
--- a/jenkins/D/070_run_tests.sh
+++ b/jenkins/D.sh
@@ -1,5 +1,6 @@
 #!/bin/bash -xue
 
 echo $PWD
-
+eval `${opam_env}`
+make
 python test_it.py --with-xunit --xunit-file=${PWD}/testresults.xml ./server/shaky
diff --git a/jenkins/D/040_build.sh b/jenkins/D/040_build.sh
deleted file mode 120000
index b80cc7fd..00000000
--- a/jenkins/D/040_build.sh
+++ /dev/null
@@ -1 +0,0 @@
-../kwik/040_build.sh
\ No newline at end of file
diff --git a/jenkins/archive_version.sh b/jenkins/archive_version.sh
deleted file mode 100755
index 22e90e6b..00000000
--- a/jenkins/archive_version.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/bash -xue
-
-./arakoon.native --version > arakoonqpackage.version.txt
diff --git a/jenkins/base/010_build.sh b/jenkins/base/010_build.sh
deleted file mode 100755
index c1b4e279..00000000
--- a/jenkins/base/010_build.sh
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/bin/bash -xue
-
-eval `opam config env`
-
-ocamlfind printconf
-ocamlfind list | grep bz2
-ocamlfind list | grep lwt
-ocamlfind list | grep camltc
-
-ocamlbuild -clean
-rm -f arakoon.byte
-rm -f arakoon.native
-ocamlbuild -use-ocamlfind arakoon.native arakoon.byte
-
-#./arakoon.d.byte --run-all-tests-xml foobar.xml
-./arakoon.native --run-all-tests-xml testresults.xml || true
-
-#make coverage
-#./arakoon.d.byte --run-all-tests
-#./report.sh
-
-# redo this for artifacts...
-#ocamlbuild -use-ocamlfind arakoon.native arakoon.byte
-
-echo "Fixup symlinks (absolute to relative)"
-symlinks -c .
-
-mkdir -p doc/python/client
-epydoc --html --output ./doc/python/client --name Arakoon --url http://www.arakoon.org --inheritance listed --graph all src/client/python
-mkdir -p doc/python/client/extension
-epydoc --html --output ./doc/python/client/extension --name "Arakoon PyLabs client extension" --url http://www.arakoon.org --inheritance listed --graph all pylabs/extensions/arakoon_ext/client/*.py
-mkdir -p doc/python/server/extension
-epydoc --html --output ./doc/python/server/extension --name "Arakoon PyLabs server extension" --url http://www.arakoon.org --inheritance listed --graph all pylabs/extensions/arakoon_ext/server/*.py
-cp arakoon.native arakoon
diff --git a/jenkins/base/020_build_docs.sh b/jenkins/base/020_build_docs.sh
deleted file mode 100755
index 4fb73807..00000000
--- a/jenkins/base/020_build_docs.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/bin/bash -ex
-
-VENV=_virtualenv
-
-if [ ! -d $VENV ]; then
-        virtualenv --no-site-packages $VENV
-        source $VENV/bin/activate
-
-        pip install sphinx
-
-        deactivate
-fi
-
-source $VENV/bin/activate
-
-cd doc
-make html
-make latexpdf
-
-deactivate
diff --git a/jenkins/build_in_shandbox.sh b/jenkins/build_in_shandbox.sh
deleted file mode 100755
index c09c3cfb..00000000
--- a/jenkins/build_in_shandbox.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/bin/bash -xue
-
-echo WORKSPACE=${WORKSPACE}
-eval `opam config env`
-
-
-ocamlfind printconf
-ocamlfind list | grep bz2
-ocamlfind list | grep lwt
-ocamlfind list | grep camltc
-ocamlfind list | grep snappy
-
-ocamlbuild -clean
-ocamlbuild -use-ocamlfind arakoon.native
-
-./buildInSandbox.sh
diff --git a/jenkins/job_template.sh b/jenkins/job_template.sh
deleted file mode 100644
index fd232321..00000000
--- a/jenkins/job_template.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/bin/bash -xue
-
-#TODO Fill in job name, e.g. 'base' or 'php'
-JOB=...
-
-for script in `ls jenkins/$JOB/* | grep -e "\/[0-9]\{3\}_"`; do
-    cd "$WORKSPACE"
-    $script
-done
diff --git a/jenkins/kwik/070_run_tests.sh b/jenkins/kwik.sh
similarity index 75%
rename from jenkins/kwik/070_run_tests.sh
rename to jenkins/kwik.sh
index 22b68a36..f8680100 100755
--- a/jenkins/kwik/070_run_tests.sh
+++ b/jenkins/kwik.sh
@@ -1,6 +1,6 @@
 #!/bin/bash -xue
 
 echo $PWD
-#source ./_virtualenv/bin/activate
-
+eval `${opam_env}`
+make
 python test_it.py --with-xunit --xunit-file=$PWD/testresults.xml ./server/quick
diff --git a/jenkins/kwik/005_install_ubuntu_dependencies.sh b/jenkins/kwik/005_install_ubuntu_dependencies.sh
deleted file mode 100755
index 1c1a19d0..00000000
--- a/jenkins/kwik/005_install_ubuntu_dependencies.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/bin/bash -xue
-
-sudo aptitude update || true
-
-for PKG in libssl-dev \
- texlive texlive-latex-extra \
- git python-nose python-epydoc graphviz libsnappy-dev; do
-    sudo aptitude install -yVDq $PKG
-done
diff --git a/jenkins/kwik/008_install_dev_env.sh b/jenkins/kwik/008_install_dev_env.sh
deleted file mode 120000
index dcf1c6b1..00000000
--- a/jenkins/kwik/008_install_dev_env.sh
+++ /dev/null
@@ -1 +0,0 @@
-../008_install_dev_env.sh
\ No newline at end of file
diff --git a/jenkins/kwik/040_build.sh b/jenkins/kwik/040_build.sh
deleted file mode 100755
index 5feb6110..00000000
--- a/jenkins/kwik/040_build.sh
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/bash -xue
-
-echo WORKSPACE=${WORKSPACE}
-eval `opam config env`
-
-
-ocamlfind printconf
-ocamlfind list | grep bz2
-ocamlfind list | grep lwt
-ocamlfind list | grep camltc
-ocamlfind list | grep snappy
-
-make clean
-make
diff --git a/jenkins/kwik/080_archive_version.sh b/jenkins/kwik/080_archive_version.sh
deleted file mode 100755
index 22e90e6b..00000000
--- a/jenkins/kwik/080_archive_version.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/bash -xue
-
-./arakoon.native --version > arakoonqpackage.version.txt
diff --git a/jenkins/package_deb.sh b/jenkins/package_deb.sh
new file mode 100755
index 00000000..9f957901
--- /dev/null
+++ b/jenkins/package_deb.sh
@@ -0,0 +1,5 @@
+#!/bin/bash -xue
+
+eval `${opam_env}` \
+DEB_BUILD_OPTIONS="nostrip nocheck" fakeroot debian/rules clean build binary
+cp ../*.deb .
diff --git a/jenkins/package_deb/008_install_dev_env.sh b/jenkins/package_deb/008_install_dev_env.sh
deleted file mode 100755
index fe62438c..00000000
--- a/jenkins/package_deb/008_install_dev_env.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/bin/bash -xue
-
-which opam > /dev/null || { echo 'opam not found!'; exit 1; }
-
-opam switch 4.02.1
-eval `opam config env`
-
-# do this in 1 step, otherwise, you might not get what you want
-opam install -y ssl.0.5.0 \
-  conf-libev.4-11 \
-  camlbz2.0.6.0 \
-  snappy.0.1.0 \
-  lwt.2.5.0 \
-  camltc.0.9.2 \
-  cstruct.1.7.0 \
-  ctypes.0.4.1 \
-  ctypes-foreign.0.4.0 \
-  bisect.1.3 \
-  ocplib-endian.0.8 \
-  quickcheck.1.0.2 \
-  nocrypto.0.5.1 \
-  sexplib.112.35.00 \
-  uuidm.0.9.5 \
-  zarith.1.3
diff --git a/jenkins/package_deb/010_build.sh b/jenkins/package_deb/010_build.sh
deleted file mode 100755
index ad7b256b..00000000
--- a/jenkins/package_deb/010_build.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash -xue
-
-eval `opam config env`
-
-env DEB_BUILD_OPTIONS=nocheck fakeroot debian/rules clean
-env DEB_BUILD_OPTIONS=nocheck fakeroot debian/rules build
-env DEB_BUILD_OPTIONS=nocheck fakeroot debian/rules binary
diff --git a/jenkins/package_rpm/010_build.sh b/jenkins/package_rpm.sh
similarity index 100%
rename from jenkins/package_rpm/010_build.sh
rename to jenkins/package_rpm.sh
diff --git a/jenkins/package_rpm/008_install_dev_env.sh b/jenkins/package_rpm/008_install_dev_env.sh
deleted file mode 120000
index 434fd61d..00000000
--- a/jenkins/package_rpm/008_install_dev_env.sh
+++ /dev/null
@@ -1 +0,0 @@
-../package_deb/008_install_dev_env.sh
\ No newline at end of file
diff --git a/jenkins/prepareSystemTests.py b/jenkins/prepareSystemTests.py
deleted file mode 100644
index 7bc45904..00000000
--- a/jenkins/prepareSystemTests.py
+++ /dev/null
@@ -1,13 +0,0 @@
-from pymonkey.InitBase import i,q
-import os
-import subprocess
-
-packages = [
-            ('qbase_extra','3.2'),
-            ('testrunner','1.1'),
-           ]
-
-for (p,v) in packages:
-    pkg = i.qp.find(p,version = v)
-    pkg.install()
-
diff --git a/jenkins/system_tests_slow_shaky/005_install_ubuntu_dependencies.sh b/jenkins/system_tests_slow_shaky/005_install_ubuntu_dependencies.sh
deleted file mode 120000
index c14d1ca0..00000000
--- a/jenkins/system_tests_slow_shaky/005_install_ubuntu_dependencies.sh
+++ /dev/null
@@ -1 +0,0 @@
-../system_tests_quick/005_install_ubuntu_dependencies.sh
\ No newline at end of file
diff --git a/jenkins/system_tests_slow_shaky/008_install_dev_env.sh b/jenkins/system_tests_slow_shaky/008_install_dev_env.sh
deleted file mode 120000
index b535b265..00000000
--- a/jenkins/system_tests_slow_shaky/008_install_dev_env.sh
+++ /dev/null
@@ -1 +0,0 @@
-../base/008_install_dev_env.sh
\ No newline at end of file
diff --git a/jenkins/system_tests_slow_shaky/030_prepare_system_tests.sh b/jenkins/system_tests_slow_shaky/030_prepare_system_tests.sh
deleted file mode 120000
index 3b70c916..00000000
--- a/jenkins/system_tests_slow_shaky/030_prepare_system_tests.sh
+++ /dev/null
@@ -1 +0,0 @@
-../030_prepare_system_tests.sh
\ No newline at end of file
diff --git a/jenkins/system_tests_slow_shaky/040_build_in_sandbox.sh b/jenkins/system_tests_slow_shaky/040_build_in_sandbox.sh
deleted file mode 120000
index 715c5650..00000000
--- a/jenkins/system_tests_slow_shaky/040_build_in_sandbox.sh
+++ /dev/null
@@ -1 +0,0 @@
-../build_in_shandbox.sh
\ No newline at end of file
diff --git a/jenkins/system_tests_slow_shaky/050_run_pending_configuration.sh b/jenkins/system_tests_slow_shaky/050_run_pending_configuration.sh
deleted file mode 120000
index fa418df6..00000000
--- a/jenkins/system_tests_slow_shaky/050_run_pending_configuration.sh
+++ /dev/null
@@ -1 +0,0 @@
-../050_run_pending_configuration.sh
\ No newline at end of file
diff --git a/jenkins/system_tests_slow_shaky/060_install_nose.sh b/jenkins/system_tests_slow_shaky/060_install_nose.sh
deleted file mode 120000
index 2b3eeaa2..00000000
--- a/jenkins/system_tests_slow_shaky/060_install_nose.sh
+++ /dev/null
@@ -1 +0,0 @@
-../060_install_nose.sh
\ No newline at end of file
diff --git a/jenkins/system_tests_slow_shaky/070_run_tests.sh b/jenkins/system_tests_slow_shaky/070_run_tests.sh
deleted file mode 100755
index 54d20cb9..00000000
--- a/jenkins/system_tests_slow_shaky/070_run_tests.sh
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/bin/bash -xue
-
-/opt/qbase3/qshell -c "
-q.testrunner.list()
-
-test_spec = '${TEST_SPEC-arakoon_system_tests.server.shaky}'
-
-output_format = q.enumerators.TestRunnerOutputFormat.XML
-output_folder = '${WORKSPACE}/testresults'
-
-arguments = q.testrunner._parseArgs('%s_results' % test_spec, output_format, output_folder, False)
-
-arguments.extend([
-    '--logging-format=%(asctime)-15s %(message)s',
-    '--with-coverage',
-    '--cover-package=arakoon',
-    '--cover-erase',
-    '--with-xcoverage',
-])
-arguments.append(q.testrunner._convertTestSpec(test_spec))
-
-q.testrunner._runTests(arguments)
-"
-
-RESULT=$?
-
-test -f /opt/qbase3/var/tests/coverage.xml && mv /opt/qbase3/var/tests/coverage.xml ${WORKSPACE}/coverage.xml
-
-cat ${WORKSPACE}/coverage.xml | python ${WORKSPACE}/_tools/fix_coverage.py "${WORKSPACE}" ".opt.qbase3.lib.python.site-packages." ".var.hudson.workspace.pyrakoon-tip-system-tests.python_env.pyrakoon." > ${WORKSPACE}/coverage_fixed.xml
-
-if test $RESULT -eq 0; then true; else false; fi
diff --git a/jenkins/system_tests_slow_shaky/080_archive_version.sh b/jenkins/system_tests_slow_shaky/080_archive_version.sh
deleted file mode 120000
index 80e718fb..00000000
--- a/jenkins/system_tests_slow_shaky/080_archive_version.sh
+++ /dev/null
@@ -1 +0,0 @@
-../archive_version.sh
\ No newline at end of file
diff --git a/pylabs/test/server/system_tests_common.py b/pylabs/test/server/system_tests_common.py
index 407ad728..ce21b458 100644
--- a/pylabs/test/server/system_tests_common.py
+++ b/pylabs/test/server/system_tests_common.py
@@ -1068,7 +1068,10 @@ def assert_running_nodes ( n ):
     except subprocess.CalledProcessError:
         count = 0
 
-    assert_equals(count, n, "Number of expected running nodes mismatch")
+    assert_equals(
+        count, n,
+        "Number of expected running nodes mismatch: expected=%s, actual=%s" % (n,count)
+    )
 
 def assert_value_list ( start_suffix, list_size, list ) :
     assert_list( value_format_str, start_suffix, list_size, list )
diff --git a/run_docker_debian.sh b/run_docker_debian.sh
new file mode 100755
index 00000000..048a355f
--- /dev/null
+++ b/run_docker_debian.sh
@@ -0,0 +1,6 @@
+PROJECT_HOME=/home/romain/workspace/ARAKOON/arakoon
+INSIDE=/home/jenkins/workspace/the_job/
+docker run --rm=true --interactive=true --privileged -t -v \
+       ${PROJECT_HOME}:${INSIDE} \
+       --env "SUITE=$1" \
+       -w ${INSIDE} arakoon_debian $@

From 1cf99d63d90ad5d4563d162dfa77d9504f9bca3b Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Tue, 27 Oct 2015 11:55:09 +0100
Subject: [PATCH 040/373] add test_expand_shrink_cluster

---
 .../test/server/shaky/test_sso_deployment.py  | 26 +++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/pylabs/test/server/shaky/test_sso_deployment.py b/pylabs/test/server/shaky/test_sso_deployment.py
index 1bbdefb7..4b728e9a 100644
--- a/pylabs/test/server/shaky/test_sso_deployment.py
+++ b/pylabs/test/server/shaky/test_sso_deployment.py
@@ -84,3 +84,29 @@ def non_retrying_write_loop ():
     assert_false ( test_failed )
 
     Common.assert_running_nodes( 3 )
+
+@Common.with_custom_setup( Common.default_setup, Common.basic_teardown )
+def test_expand_shrink_cluster():
+    # test to see the effect of a stranger talking to a cluster
+
+    # add a fourth node
+    Common.add_node(3)
+
+    # regenerate configs & restart all four
+    Common.regenerateClientConfig(Common.cluster_id)
+    Common.restart_nodes_wf_sim(4)
+
+    # remove the fourth node, restart the rest
+    cl = Common._getCluster()
+    cl.removeNode(Common.node_names[3])
+    Common.regenerateClientConfig(Common.cluster_id)
+    Common.restart_nodes_wf_sim(3)
+
+    # now there should a cluster of 3 nodes that know each other
+    # and 1 stranger which will want to talk with this cluster
+    time.sleep(20)
+    Common.assert_running_nodes(4)
+    client = Common.get_client()
+    master = client.whoMaster()
+    assert_true ( master in Common.node_names[0:3] )
+    client.nop()

From e9574f1a25234b11b36cd81328f0e2e712516e30 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Tue, 27 Oct 2015 16:49:17 +0100
Subject: [PATCH 041/373] run tests verbose; tighten assert_running_nodes

---
 docker/arakoon_debian/Dockerfile              | 10 +++++-
 jenkins/B.sh                                  |  6 +++-
 jenkins/C.sh                                  |  6 +++-
 jenkins/D.sh                                  |  6 +++-
 jenkins/kwik.sh                               |  5 ++-
 pylabs/Compat.py                              | 21 ++++++------
 .../test/server/quick/system_tests_basic.py   |  2 +-
 pylabs/test/server/quick/test_client_lib.py   | 17 +++++++---
 pylabs/test/server/system_tests_common.py     | 32 +++++++++++++++----
 test_it.py                                    |  9 +++++-
 10 files changed, 88 insertions(+), 26 deletions(-)

diff --git a/docker/arakoon_debian/Dockerfile b/docker/arakoon_debian/Dockerfile
index 886d76aa..72c13515 100644
--- a/docker/arakoon_debian/Dockerfile
+++ b/docker/arakoon_debian/Dockerfile
@@ -32,7 +32,8 @@ RUN apt-get -y update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
         sudo \
         libtool \
         libboost-all-dev \
-        iptables
+        iptables \
+        net-tools
 
 RUN useradd jenkins -u 1001 -g root
 
@@ -40,6 +41,12 @@ RUN pip install fabric junit-xml nose
 
 RUN wget https://raw.github.com/ocaml/opam/master/shell/opam_installer.sh
 RUN sh ./opam_installer.sh /usr/local/bin 4.02.3
+
+RUN mkdir /home/tests
+RUN chmod 777 /home/tests
+ENV TEST_HOME='/home/tests'
+ENV PATH=${PATH}:/home/tests/apps/arakoon/bin
+
 RUN mkdir /home/opam
 ENV opam_env='opam config env --root=/home/opam/OPAM'
 RUN opam init --root=/home/opam/OPAM --comp 4.02.3
@@ -71,3 +78,4 @@ RUN eval `${opam_env}` && opam install -y \
 
 RUN echo "jenkins ALL=NOPASSWD: ALL" >/etc/sudoers.d/jenkins
 ENV LANG=C
+RUN chmod 777 /home/opam/OPAM/4.02.3/lib
diff --git a/jenkins/B.sh b/jenkins/B.sh
index 56cc80c6..0ee8c6d0 100755
--- a/jenkins/B.sh
+++ b/jenkins/B.sh
@@ -2,5 +2,9 @@
 
 echo $PWD
 eval `${opam_env}`
+echo TEST_HOME=${TEST_HOME}
+export PATH=${PATH}:${TEST_HOME}/apps/arakoon/bin
+
+make clean
 make
-python test_it.py --with-xunit --xunit-file=${PWD}/testresults.xml ./server/left
+python test_it.py -v --with-xunit --xunit-file=${PWD}/testresults.xml ./server/left
diff --git a/jenkins/C.sh b/jenkins/C.sh
index b34d5f56..5e60f6e1 100755
--- a/jenkins/C.sh
+++ b/jenkins/C.sh
@@ -2,5 +2,9 @@
 
 echo $PWD
 eval `${opam_env}`
+echo TEST_HOME=${TEST_HOME}
+export PATH=${PATH}:${TEST_HOME}/apps/arakoon/bin
+
+make clean
 make
-python test_it.py --with-xunit --xunit-file=${PWD}/testresults.xml ./server/right
+python test_it.py -v --with-xunit --xunit-file=${PWD}/testresults.xml ./server/right
diff --git a/jenkins/D.sh b/jenkins/D.sh
index 1516cec0..87a67cbc 100755
--- a/jenkins/D.sh
+++ b/jenkins/D.sh
@@ -2,5 +2,9 @@
 
 echo $PWD
 eval `${opam_env}`
+echo TEST_HOME=${TEST_HOME}
+export PATH=${PATH}:${TEST_HOME}/apps/arakoon/bin
+
+make clean
 make
-python test_it.py --with-xunit --xunit-file=${PWD}/testresults.xml ./server/shaky
+python test_it.py -v --with-xunit --xunit-file=${PWD}/testresults.xml ./server/shaky
diff --git a/jenkins/kwik.sh b/jenkins/kwik.sh
index f8680100..03657004 100755
--- a/jenkins/kwik.sh
+++ b/jenkins/kwik.sh
@@ -2,5 +2,8 @@
 
 echo $PWD
 eval `${opam_env}`
+echo TEST_HOME=${TEST_HOME}
+export PATH=${PATH}:${TEST_HOME}/apps/arakoon/bin
+
 make
-python test_it.py --with-xunit --xunit-file=$PWD/testresults.xml ./server/quick
+python test_it.py -v --with-xunit --xunit-file=$PWD/testresults.xml ./server/quick
diff --git a/pylabs/Compat.py b/pylabs/Compat.py
index 4bef7cb3..c4744255 100644
--- a/pylabs/Compat.py
+++ b/pylabs/Compat.py
@@ -73,13 +73,13 @@ def _writeConfig(self, p, h):
     logging.debug("writing (%i)%s:\n%s", self._count, fn, self.cfg2str(p))
     with open(fn,'w') as cfg:
         p.write(cfg)
-        
+
 
 def _run(self,cmd):
     #['mount', '-t', 'tmpfs', '-o', 'size=20m', 'tmpfs', '/opt/qbase3/var/tmp//arakoon_system_tests/test_disk_full_on_slave/sturdy_0/db']
     proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr = subprocess.PIPE)
     proc.wait()
-    output = proc.stdout.read()    
+    output = proc.stdout.read()
     err = proc.stderr.read()
     return (proc.returncode, output,err)
 
@@ -138,8 +138,11 @@ def copyDirTree(self,src,dst):
 
     subprocess = subprocess
 
+    root = os.environ.get('TEST_HOME')
+    if root is None:
+        root = os.environ['HOME']
 
-    _base = '/tmp/X'
+    _base = root
     tmpDir = _base + '/tmp'
     appDir = _base + '/apps'
     cfgDir = _base + '/cfg'
@@ -150,7 +153,7 @@ def copyDirTree(self,src,dst):
 class _LOGGING:
     def __init__(self,q):
         self._q = q
-        
+
     def info(self,template,*rest):
         message = template % rest
         self._q.logger.log(message,level = 10)
@@ -158,7 +161,7 @@ def info(self,template,*rest):
     def debug(self,template,*rest):
         message = template % rest
         self._q.logger.log(message,level = 5)
-            
+
 
 class Q: # (Compat)
     def __init__(self):
@@ -178,7 +181,7 @@ def __init__(self):
         self.AppStatusType = q.enumerators.AppStatusType
         self.listFilesInDir = q.system.fs.listFilesInDir
 
-        self.subprocess = subprocess 
+        self.subprocess = subprocess
         def check_output(cmd, **kwargs):
             return subprocess.Popen(cmd, stdout=subprocess.PIPE, **kwargs).communicate()[0]
         self.subprocess.check_output = check_output
@@ -188,7 +191,7 @@ def fileExists(self,fn):
 
     getConfig = _getConfig
     writeConfig = _writeConfig
-            
+
     def removeDirTree(self,path):
         return self._q.system.fs.removeDirTree(path)
 
@@ -197,7 +200,7 @@ def createDir(self,path):
 
     def removeFile(self,path):
         os.unlink(path)
-    
+
     def getFileContents(self, path):
         data = self._q.system.fs.fileGetContents(path)
         return data
@@ -209,7 +212,7 @@ def copyDirTree(self, source, destination):
         self._q.system.fs.copyDirTree(source,destination)
     cfg2str = _cfg2str
     run = _run
-    
+
 
 def which_compat():
     print "which_compat"
diff --git a/pylabs/test/server/quick/system_tests_basic.py b/pylabs/test/server/quick/system_tests_basic.py
index fcfd53f2..848a31bf 100644
--- a/pylabs/test/server/quick/system_tests_basic.py
+++ b/pylabs/test/server/quick/system_tests_basic.py
@@ -680,7 +680,7 @@ def test_download_db():
 
     C.flush_store(n0)
     C.stop_all()
-    C.compare_stores(n0,n1)
+    C.compare_stores(n0, n1)
 
 @C.with_custom_setup(C.setup_3_nodes, C.basic_teardown)
 def test_delete_prefix():
diff --git a/pylabs/test/server/quick/test_client_lib.py b/pylabs/test/server/quick/test_client_lib.py
index b21be28c..6cc7396f 100644
--- a/pylabs/test/server/quick/test_client_lib.py
+++ b/pylabs/test/server/quick/test_client_lib.py
@@ -16,6 +16,8 @@
 
 from Compat import X
 import os
+import logging
+import subprocess
 
 def test_client_lib():
     my_temp = '/tmp/client_lib_test'
@@ -37,7 +39,14 @@ def test_client_lib():
         if cwd == None:
             cwd = '../..'
         print cmd
-        r = X.subprocess.check_output(cmd,
-                                      cwd = cwd,
-                                      env = env)
-        print r
+        try:
+            r = X.subprocess.check_output(cmd,
+                                          cwd = cwd,
+                                          env = env,
+                                          stderr= X.subprocess.STDOUT
+            )
+            print r
+        except subprocess.CalledProcessError as ex:
+            logging.info("ex:%s" % ex)
+            logging.info("output=%s" % ex.output)
+            raise ex
diff --git a/pylabs/test/server/system_tests_common.py b/pylabs/test/server/system_tests_common.py
index ce21b458..b615a652 100644
--- a/pylabs/test/server/system_tests_common.py
+++ b/pylabs/test/server/system_tests_common.py
@@ -165,11 +165,17 @@ def dump_store( node_id ):
     try:
         dump_fd = open( dump_file, 'w' )
         logging.info( "Dumping store of %s to %s" % (node_id, dump_file) )
-        stdout= X.subprocess.check_output( cmd , captureOutput=True, stdout=dump_fd )
+        stdout= X.subprocess.check_output(cmd)
+        dump_fd.write(stdout)
         dump_fd.close()
-    except:
+    except OSError as ose:
+        if ose.errno == 2:
+            logging.info("store :%s is empty" % db_file)
+        else:
+            raise ose
+    except Exception as ex:
         logging.info("Unexpected error: %s" % sys.exc_info()[0])
-
+        raise ex
     return dump_file
 
 def flush_store(node_name):
@@ -193,8 +199,14 @@ def get_i(node_id, head=False):
         db_file = head_dir + "/head.db"
     else:
         db_file = get_node_db_file(node_id)
+
+    logging.debug("%s exists: %s" % (db_file, os.path.exists(db_file)))
+
     cmd = [get_arakoon_binary(), "--dump-store", db_file]
+    logging.debug("get_i:cmd = %s" % str(cmd))
     stdout=  X.subprocess.check_output(cmd)
+    print "\nstdout=\n"
+    print stdout
     i_line = stdout.split("\n") [0]
     logging.info("i_line='%s'", i_line)
 
@@ -1062,10 +1074,18 @@ def assert_last_i_in_sync ( node_1, node_2 ):
         pass
 
 
-def assert_running_nodes ( n ):
+def assert_running_nodes(n):
     try:
-        count = int(X.subprocess.check_output(['pgrep', '-c', daemon_name]))
-    except subprocess.CalledProcessError:
+        output = X.subprocess.check_output(['pgrep',
+                                            '-a',
+                                            '-f', "%s --node" % (daemon_name)],
+                                           stderr = X.subprocess.STDOUT
+        )
+        lines = output.strip().split('\n')
+        logging.info(lines)
+        count = len(lines)
+    except subprocess.CalledProcessError,ex:
+        logging.info("ex:%s => count = 0" % ex)
         count = 0
 
     assert_equals(
diff --git a/test_it.py b/test_it.py
index 6d055ff7..56fbfabb 100644
--- a/test_it.py
+++ b/test_it.py
@@ -3,7 +3,10 @@
 import subprocess
 import sys
 
-root = "/tmp/X"
+root = os.environ.get('TEST_HOME')
+if root is None:
+    root = os.environ['HOME']
+
 bin_dir = '%s/apps/arakoon/bin' % root
 bin = bin_dir +'/arakoon'
 
@@ -12,6 +15,7 @@ def prologue():
         os.makedirs(bin_dir)
     if not os.path.exists(bin):
         subprocess.call(['cp','./arakoon.native', bin])
+        print "=> copying exec to: %s" % bin
     else:
         version = subprocess.check_output(['./arakoon.native','--version'])
         version2 = subprocess.check_output([bin,'--version'])
@@ -19,6 +23,8 @@ def prologue():
             print version,version2
             print "=> copying exec"
             subprocess.call(['cp','./arakoon.native', bin])
+        else:
+            print "not copying arakoon executable"
 
 
 prologue()
@@ -34,6 +40,7 @@ def prologue():
     print "LD_LIBRARY_PATH=%s" % env[ldlib]
 rest = sys.argv[1:]
 cmd.extend(rest)
+
 subprocess.call(cmd,
                 cwd = './pylabs',
                 env = env

From b1a0264b364d2b3a1f5f71456e6b77580715250b Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 28 Oct 2015 17:15:01 +0100
Subject: [PATCH 042/373] fix bug caused by tight assert_running_nodes

---
 pylabs/test/server/shaky/test_272.py | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/pylabs/test/server/shaky/test_272.py b/pylabs/test/server/shaky/test_272.py
index e4dae8f9..3c25cd6b 100644
--- a/pylabs/test/server/shaky/test_272.py
+++ b/pylabs/test/server/shaky/test_272.py
@@ -43,18 +43,12 @@ def test_272():
     for i in range(100):
         Common.rotate_log(node, 1, False)
         time.sleep(0.2)
-        # Note:
-        # At this point, we expect one node to be running, as well as the
-        # benchmark process. The `assert_running_nodes` procedure uses
-        # 'pgrep -c' which counts the number of 'arakoon' processes running, so
-        # we should pass *2* here, despite only a single node process should be
-        # running.
-        Common.assert_running_nodes(2)
+        Common.assert_running_nodes(1)
         rc = bench.returncode
         if rc <> None:
             raise Exception ("benchmark should not have stopped")
 
-    Common.assert_running_nodes(2)
+    Common.assert_running_nodes(1)
     logging.info("now wait for benchmark to finish")
     rc = bench.wait()
     if rc <> 0:

From dc6d20b57da93a84deb19622badb790e80b94987 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 28 Oct 2015 23:09:50 +0100
Subject: [PATCH 043/373] flush when closing

---
 src/node/node_main.ml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/node/node_main.ml b/src/node/node_main.ml
index 55e302e6..59958456 100644
--- a/src/node/node_main.ml
+++ b/src/node/node_main.ml
@@ -843,7 +843,7 @@ let _main_2 (type s)
                  inner (succ i) in
              inner 0, stop in
            let count_close_store, stop = count_thread "Closing store (%is)" in
-           Lwt.pick [ S.close ~flush:false ~sync:true store ;
+           Lwt.pick [ S.close ~flush:true ~sync:true store ;
                       count_close_store ] >>= fun () ->
            stop := true;
            Logger.fatal_f_

From 996cccb2ae7eab78d32838f13a1b4ac091504b21 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Thu, 29 Oct 2015 15:07:53 +0100
Subject: [PATCH 044/373] log size together with time in on_accept

---
 src/node/catchup.ml            | 14 ++++++++++----
 src/node/catchup_test.ml       | 10 +++++-----
 src/node/node_main.ml          |  6 +++---
 src/system/startup.ml          |  2 +-
 src/tlog/mem_tlogcollection.ml |  2 +-
 src/tlog/tlc2.ml               |  4 ++--
 src/tlog/tlc2_test.ml          | 25 ++++++++++++++-----------
 src/tlog/tlog_main.ml          |  8 ++++++--
 src/tlog/tlogcollection.ml     |  6 +++---
 src/tlog/tlogcommon.ml         |  5 ++++-
 src/tlog/tlogwriter.ml         |  4 ++--
 11 files changed, 51 insertions(+), 35 deletions(-)

diff --git a/src/node/catchup.ml b/src/node/catchup.ml
index 8c9e652e..e5344fa6 100644
--- a/src/node/catchup.ml
+++ b/src/node/catchup.ml
@@ -313,7 +313,9 @@ let verify_n_catchup_store (type s) ~stop me ?(apply_last_tlog_value=false) ((mo
 
 let last_entries
       (type s) (module S : Store.STORE with type t = s)
-      store tlog_collection (start_i:Sn.t) (oc:Lwt_io.output_channel)
+      store
+      (tlog_collection:Tlogcollection.tlog_collection)
+      (start_i:Sn.t) (oc:Lwt_io.output_channel)
   =
   (* This one is kept (for how long?)
      for x-version clusters during upgrades
@@ -368,7 +370,8 @@ let last_entries
             let i = Entry.i_of entry
             and v = Entry.v_of entry
             in
-            Tlogcommon.write_entry oc i v
+            Tlogcommon.write_entry oc i v >>= fun _total_size ->
+            Lwt.return ()
           in
           Lwt.catch
             (fun () -> tlog_collection # iterate start_i3 too_far_i f)
@@ -384,7 +387,9 @@ let last_entries
 
 let last_entries2
       (type s) (module S : Store.STORE with type t = s)
-      store tlog_collection (start_i:Sn.t) (oc:Lwt_io.output_channel)
+      store
+      (tlog_collection:Tlogcollection.tlog_collection)
+      (start_i:Sn.t) (oc:Lwt_io.output_channel)
   =
   Logger.debug_f_ "last_entries2 %s" (Sn.string_of start_i) >>= fun () ->
   let consensus_i = S.consensus_i store in
@@ -401,7 +406,8 @@ let last_entries2
             let i = Entry.i_of entry
             and v = Entry.v_of entry
             in
-            Tlogcommon.write_entry oc i v
+            Tlogcommon.write_entry oc i v >>= fun _total_size ->
+            Lwt.return_unit
           in
           Llio.output_int oc 1 >>= fun () ->
           Lwt.catch
diff --git a/src/node/catchup_test.ml b/src/node/catchup_test.ml
index 1a408fcf..d0fc0713 100644
--- a/src/node/catchup_test.ml
+++ b/src/node/catchup_test.ml
@@ -39,7 +39,7 @@ let _fill tlog_coll n =
         and v = Printf.sprintf "value%i" i in
         let u = Update.Set (k,v) in
         let value = Value.create_client_value [u] sync in
-        tlog_coll # log_value (Sn.of_int i) value >>= fun () ->
+        tlog_coll # log_value (Sn.of_int i) value >>= fun _total_size ->
         _loop (i+1)
       end
   in
@@ -61,8 +61,8 @@ let _fill2 tlog_coll n =
         let value = Value.create_client_value [u] sync in
         let value2 = Value.create_client_value [u2] sync in
         let sni = Sn.of_int i in
-        tlog_coll # log_value  sni value  >>= fun () ->
-        tlog_coll # log_value  sni value2 >>= fun () ->
+        tlog_coll # log_value  sni value  >>= fun _ ->
+        tlog_coll # log_value  sni value2 >>= fun _ ->
         _loop (i+1)
       end
   in
@@ -87,8 +87,8 @@ let _fill3 tlog_coll n =
         let value = Value.create_client_value [u; u3] sync in
         let value2 = Value.create_client_value [u2; u3] sync in
         let sni = Sn.of_int i in
-        tlog_coll # log_value sni value  >>= fun () ->
-        tlog_coll # log_value sni value2 >>= fun () ->
+        tlog_coll # log_value sni value  >>= fun _ ->
+        tlog_coll # log_value sni value2 >>= fun _ ->
         _loop (i+1)
       end
   in
diff --git a/src/node/node_main.ml b/src/node/node_main.ml
index 59958456..e1cacc10 100644
--- a/src/node/node_main.ml
+++ b/src/node/node_main.ml
@@ -332,7 +332,7 @@ module X = struct
     let t0 = Unix.gettimeofday () in
     Logger.debug_f_ "on_accept: n:%s i:%s" (Sn.string_of n) (Sn.string_of i)
     >>= fun () ->
-    tlog_coll # accept i v >>= fun () ->
+    tlog_coll # accept i v >>= fun total_size ->
     begin
       match v with
         | Value.Vc (us,_)     ->
@@ -345,9 +345,9 @@ module X = struct
     let d = t1 -. t0 in
     if d >= 1.0
     then
-      Logger.info_f_ "T:on_accept took: %f" d
+      Logger.info_f_ "T:on_accept took: %f (%i B)" d total_size
     else
-      Logger.debug_f_ "T:on_accept took: %f" d
+      Logger.debug_f_ "T:on_accept took: %f (%i B) " d total_size
 
   let reporting period backend () =
     let fp = float period in
diff --git a/src/system/startup.ml b/src/system/startup.ml
index 770e7be5..4a01c5a2 100644
--- a/src/system/startup.ml
+++ b/src/system/startup.ml
@@ -81,7 +81,7 @@ let _make_tlog_coll ~compressor tlcs values tlc_name tlf_dir head_dir
     | [] -> Lwt.return ()
     | v :: vs ->
       begin
-        tlc # log_value i v >>= fun () ->
+        tlc # log_value i v >>= fun _ ->
         loop (Sn.succ i) vs
       end
   in
diff --git a/src/tlog/mem_tlogcollection.ml b/src/tlog/mem_tlogcollection.ml
index 1a06f1d7..e83ed6e3 100644
--- a/src/tlog/mem_tlogcollection.ml
+++ b/src/tlog/mem_tlogcollection.ml
@@ -74,7 +74,7 @@ class mem_tlog_collection _name =
       let entry = Entry.make i v 0L marker in
       let () = data <- entry::data in
       let () = last_entry <- (Some entry) in
-      Lwt.return ()
+      Lwt.return 0
 
     method log_value i v = self #log_value_explicit i v ~sync:false None
 
diff --git a/src/tlog/tlc2.ml b/src/tlog/tlc2.ml
index 9cd3b3c6..9f3124b5 100644
--- a/src/tlog/tlc2.ml
+++ b/src/tlog/tlc2.ml
@@ -512,7 +512,7 @@ class tlc2
              self # _prelude i >>= fun file ->
              let p = F.file_pos file in
              let oc = F.oc_of file in
-             Tlogcommon.write_entry oc i value >>= fun () ->
+             Tlogcommon.write_entry oc i value >>= fun total_size ->
              Lwt_io.flush oc >>= fun () ->
              begin
                if sync
@@ -528,7 +528,7 @@ class tlc2
              let entry = Entry.make i value p marker in
              _previous_entry <- Some entry;
              Index.note entry _index;
-             Lwt.return ()
+             Lwt.return total_size
            end)
 
     method log_value i value =
diff --git a/src/tlog/tlc2_test.ml b/src/tlog/tlc2_test.ml
index d9dffa99..4936eeeb 100644
--- a/src/tlog/tlc2_test.ml
+++ b/src/tlog/tlc2_test.ml
@@ -34,11 +34,11 @@ let prepare_tlog_scenarios (dn,factory) =
   Tlogcommon.tlogEntriesPerFile := 5 ;
   factory dn "node_name" >>= fun (tlog_coll:tlog_collection) ->
   let value = Value.create_master_value ~lease_start:0. "me" in
-  tlog_coll # log_value  0L value  >>= fun () ->
-  tlog_coll # log_value  1L value  >>= fun () ->
-  tlog_coll # log_value  2L value  >>= fun () ->
-  tlog_coll # log_value  3L value  >>= fun () ->
-  tlog_coll # log_value  4L value  >>= fun () ->
+  tlog_coll # log_value  0L value  >>= fun _ ->
+  tlog_coll # log_value  1L value  >>= fun _ ->
+  tlog_coll # log_value  2L value  >>= fun _ ->
+  tlog_coll # log_value  3L value  >>= fun _ ->
+  tlog_coll # log_value  4L value  >>= fun _ ->
   tlog_coll # close () >>= fun _ ->
   Lwt.return old_tlog_entries_value
 
@@ -48,7 +48,7 @@ let test_interrupted_rollover (dn, tlx_dir, factory) =
     Unix.unlink fn; *)
   factory dn "node_name" >>= fun tlog_coll ->
   let value = Value.create_master_value ~lease_start:0. "me" in
-  tlog_coll # log_value 5L value >>= fun () ->
+  tlog_coll # log_value 5L value >>= fun _ ->
   tlog_coll # close () >>= fun _ ->
   Tlc2.get_tlog_names dn tlx_dir >>= fun tlog_names ->
   let n = List.length tlog_names in
@@ -204,10 +204,11 @@ let test_compression_bug (dn, tlx_dir, factory) =
       let key = Printf.sprintf "test_compression_bug_%i" i in
       let value = Value.create_client_value [Update.Set(key, v)] sync in
       let sni = Sn.of_int i in
-      tlc # log_value sni value >>= fun () ->
+      tlc # log_value sni value >>= fun _ ->
       loop (i+1)
   in
-  tlc # log_value 0L (Value.create_client_value [Update.Set("xxx","XXX")] false) >>= fun () ->
+  tlc # log_value 0L (Value.create_client_value [Update.Set("xxx","XXX")] false)
+  >>= fun _total_size ->
   loop 1 >>= fun () ->
   tlc # close ~wait_for_compression:true () >>= fun () ->
   File_system.stat (tlx_dir ^ "/000.tlx") >>= fun stat ->
@@ -239,11 +240,13 @@ let test_compression_previous (dn, tlx_dir, factory) =
       let key = Printf.sprintf "test_compression_bug_%i" i in
       let value = Value.create_client_value [Update.Set(key, v)] sync in
       let sni = Sn.of_int i in
-      tlc # log_value sni value >>= fun () ->
+      tlc # log_value sni value >>= fun _ ->
       loop (i+1)
   in
-  tlc # log_value 0L (Value.create_client_value [Update.Set("xxx","XXX")] false) >>= fun () ->
-  loop 1 >>= fun () ->
+  tlc # log_value 0L (Value.create_client_value [Update.Set("xxx","XXX")] false)
+  >>= fun _ ->
+  loop 1
+  >>= fun () ->
   tlc # close ~wait_for_compression:true () >>= fun () ->
 
   (* mess around : uncompress tlxs to tlogs again, put some temp files in the way *)
diff --git a/src/tlog/tlog_main.ml b/src/tlog/tlog_main.ml
index cf897c59..8dd9e723 100644
--- a/src/tlog/tlog_main.ml
+++ b/src/tlog/tlog_main.ml
@@ -112,8 +112,12 @@ let mark_tlog file_name node_name =
 let make_tlog tlog_name (i:int) =
   let sni = Sn.of_int i in
   let t =
-    let f oc = Tlogcommon.write_entry oc sni
-                 (Value.create_client_value [Update.Update.Nop] false)
+    let f oc =
+      Tlogcommon.write_entry
+        oc sni
+        (Value.create_client_value [Update.Update.Nop] false)
+        >>= fun _ -> Lwt.return_unit
+
     in
     Lwt_io.with_file ~mode:Lwt_io.output tlog_name f
   in
diff --git a/src/tlog/tlogcollection.ml b/src/tlog/tlogcollection.ml
index 28ce0268..4ee53bcb 100644
--- a/src/tlog/tlogcollection.ml
+++ b/src/tlog/tlogcollection.ml
@@ -20,10 +20,10 @@ open Tlogreader2
 class type tlog_collection = object
   method validate_last_tlog: unit -> (tlogValidity * Entry.t option * Index.index) Lwt.t
   method iterate: Sn.t -> Sn.t -> (Entry.t -> unit Lwt.t) -> unit Lwt.t
-  method accept : Sn.t -> Value.t -> unit Lwt.t
-  method log_value : Sn.t -> Value.t -> unit Lwt.t
+  method accept : Sn.t -> Value.t    -> int Lwt.t
+  method log_value : Sn.t -> Value.t -> int Lwt.t
   method log_value_explicit : Sn.t -> Value.t ->
-                              sync:bool -> string option -> unit Lwt.t
+                              sync:bool -> string option -> int Lwt.t
   method get_last_i: unit -> Sn.t
   method get_last_value: Sn.t -> Value.t option (* Lwt.t *)
   method get_last: unit -> (Value.t * Sn.t) option
diff --git a/src/tlog/tlogcommon.ml b/src/tlog/tlogcommon.ml
index 7950efcb..8de11d6c 100644
--- a/src/tlog/tlogcommon.ml
+++ b/src/tlog/tlogcommon.ml
@@ -132,7 +132,10 @@ let write_entry oc i value =
   let cmd = Buffer.contents b in
   let chksum = Crc32c.calculate_crc32c cmd 0 (String.length cmd) in
   Llio.output_int32 oc chksum >>= fun() ->
-  Llio.output_string oc cmd
+  Llio.output_string oc cmd >>= fun () ->
+  let extra_size = 12 (* i:8 + chksum:4 + string:4 *) in
+  let total_size = Bytes.length cmd + extra_size in
+  Lwt.return total_size
 
 let write_marker oc i value m =
   Sn.output_sn oc i >>= fun () ->
diff --git a/src/tlog/tlogwriter.ml b/src/tlog/tlogwriter.ml
index 7ba393fd..567b9a3e 100644
--- a/src/tlog/tlogwriter.ml
+++ b/src/tlog/tlogwriter.ml
@@ -28,10 +28,10 @@ class tlogWriter oc lastI =
     method log_value i value =
       if isValidSuccessor i lastWrittenI  then
         begin
-          write_entry oc i value >>= fun () ->
+          write_entry oc i value >>= fun total_size ->
           Lwt_io.flush oc >>= fun () ->
           let () = lastWrittenI <- i in
-          Lwt.return ()
+          Lwt.return total_size
         end
       else
         Llio.lwt_failfmt "invalid successor %s" (Sn.string_of i)

From dc721276b4b3b09963080d56fdacf87405537225 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Thu, 29 Oct 2015 15:26:20 +0100
Subject: [PATCH 045/373] don't flush automatically keep previous behaviour

---
 src/node/node_main.ml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/node/node_main.ml b/src/node/node_main.ml
index e1cacc10..9fe50214 100644
--- a/src/node/node_main.ml
+++ b/src/node/node_main.ml
@@ -843,7 +843,7 @@ let _main_2 (type s)
                  inner (succ i) in
              inner 0, stop in
            let count_close_store, stop = count_thread "Closing store (%is)" in
-           Lwt.pick [ S.close ~flush:true ~sync:true store ;
+           Lwt.pick [ S.close ~flush:false ~sync:true store ;
                       count_close_store ] >>= fun () ->
            stop := true;
            Logger.fatal_f_

From 157f8cc91445f24e76267916e53a8f783f32738a Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Thu, 29 Oct 2015 16:02:42 +0100
Subject: [PATCH 046/373] cleaner intent

---
 src/node/batched_store.ml | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/src/node/batched_store.ml b/src/node/batched_store.ml
index 28b863a5..362f4310 100644
--- a/src/node/batched_store.ml
+++ b/src/node/batched_store.ml
@@ -99,19 +99,23 @@ struct
     match s._ls_tx with
       | None -> ()
       | Some _ ->
-        S._trancommit s.s;
-        s._ls_tx <- None;
-        s._entries <- 0;
-        s._size <- 0
+         begin
+           S._trancommit s.s;
+           s._ls_tx <- None;
+           s._entries <- 0;
+           s._size <- 0
+         end
 
   let _abort_ls_tx_if_any s =
     match s._ls_tx with
       | None -> ()
       | Some _ ->
-        S._tranabort s.s;
-        s._ls_tx <- None;
-        s._entries <- 0;
-        s._size <- 0
+         begin
+           S._tranabort s.s;
+           s._ls_tx <- None;
+           s._entries <- 0;
+           s._size <- 0
+         end
 
   let with_transaction s f =
     Lwt_mutex.with_lock s._tx_lock (fun () ->
@@ -187,7 +191,7 @@ struct
 
   let flush s =
     _commit_ls_tx_if_any s;
-    Lwt.return ()
+    Lwt.return_unit
 
   let close s ~flush ~sync =
     begin

From 69e968ec99dd0ceed43d15584e72b7a9d9bbba1e Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Thu, 29 Oct 2015 17:25:11 +0100
Subject: [PATCH 047/373] 16

---
 src/tlog/tlogcommon.ml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tlog/tlogcommon.ml b/src/tlog/tlogcommon.ml
index 8de11d6c..83a7ecb4 100644
--- a/src/tlog/tlogcommon.ml
+++ b/src/tlog/tlogcommon.ml
@@ -133,7 +133,7 @@ let write_entry oc i value =
   let chksum = Crc32c.calculate_crc32c cmd 0 (String.length cmd) in
   Llio.output_int32 oc chksum >>= fun() ->
   Llio.output_string oc cmd >>= fun () ->
-  let extra_size = 12 (* i:8 + chksum:4 + string:4 *) in
+  let extra_size = 16 (* i:8 + chksum:4 + string:4 *) in
   let total_size = Bytes.length cmd + extra_size in
   Lwt.return total_size
 

From 3947e1e1cd5f6b98ea8661053968b90da30440ae Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Thu, 29 Oct 2015 17:30:01 +0100
Subject: [PATCH 048/373] apt-key update (attempt)

---
 travis.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/travis.sh b/travis.sh
index 0cea9d39..73c0e5eb 100755
--- a/travis.sh
+++ b/travis.sh
@@ -14,6 +14,8 @@ before_install () {
     echo "yes" | sudo add-apt-repository ppa:$PPA
     echo "Updating Apt cache"
     sudo apt-get update -qq
+    echo "updating keys"
+    sudo apt-key update
     echo "Installing general dependencies"
     sudo apt-get install -qq ${APT_DEPENDS}
     echo "Installing dependencies"

From ab377f40f8e494c0ed625c4a375ce167362f7712 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Fri, 30 Oct 2015 15:59:38 +0100
Subject: [PATCH 049/373] fix building of .deb package

---
 jenkins/package_deb.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jenkins/package_deb.sh b/jenkins/package_deb.sh
index 9f957901..986f5a50 100755
--- a/jenkins/package_deb.sh
+++ b/jenkins/package_deb.sh
@@ -1,5 +1,5 @@
 #!/bin/bash -xue
 
-eval `${opam_env}` \
+eval `${opam_env}`
 DEB_BUILD_OPTIONS="nostrip nocheck" fakeroot debian/rules clean build binary
 cp ../*.deb .

From 785965e0f044f0203667a9985d62b046ad572a5c Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Mon, 2 Nov 2015 10:50:46 +0100
Subject: [PATCH 050/373] only close file descriptors once (in _socket_closer,
 not by the channels)

---
 src/tools/server.ml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/tools/server.ml b/src/tools/server.ml
index 53fac93f..547f5c8f 100644
--- a/src/tools/server.ml
+++ b/src/tools/server.ml
@@ -46,8 +46,8 @@ let session_thread (sid:string) cid protocol fd =
     (fun () ->
        let (ic, oc) = match fd with
          | Plain fd' ->
-             let ic = Lwt_io.of_fd ~mode:Lwt_io.input fd'
-             and oc = Lwt_io.of_fd ~mode:Lwt_io.output fd' in
+             let ic = Lwt_io.of_fd ~close:Lwt.return ~mode:Lwt_io.input fd'
+             and oc = Lwt_io.of_fd ~close:Lwt.return ~mode:Lwt_io.output fd' in
              (ic, oc)
          | TLS fd' ->
              let ic = Lwt_ssl.in_channel_of_descr fd'

From f4a3c3d4e6edd148a2c215270f2ff6892007a189 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Mon, 2 Nov 2015 11:25:40 +0100
Subject: [PATCH 051/373] make_release: 1.8.10

---
 META                      | 2 +-
 debian/changelog          | 5 +++++
 redhat/SPECS/arakoon.spec | 4 +++-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/META b/META
index c4611da2..dd940f04 100644
--- a/META
+++ b/META
@@ -1,7 +1,7 @@
 # -*- conf -*-
 
 description = "Arakoon client"
-version = "1.8.9"
+version = "1.8.10"
 exists_if = "arakoon_client.cma,arakoon_client.cmxa,arakoon_client.cmxs"
 requires = "threads lwt lwt.unix str oUnit quickcheck lwt.ssl ocplib-endian nocrypto cstruct"
 archive(byte) = "arakoon_client.cma"
diff --git a/debian/changelog b/debian/changelog
index d5e782c7..ce8ed0b2 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,8 @@
+arakoon (1.8.10) unstable; urgency=low
+  * new upstream version
+
+ -- Romain Slootmaekers <romain.slootmaekers@openvstorage.com>  Mon, 02 Nov 2015 11:25:40 +0100
+
 arakoon (1.8.9) unstable; urgency=low
   * new upstream version
 
diff --git a/redhat/SPECS/arakoon.spec b/redhat/SPECS/arakoon.spec
index 5628ff7a..ac10f6c3 100644
--- a/redhat/SPECS/arakoon.spec
+++ b/redhat/SPECS/arakoon.spec
@@ -2,7 +2,7 @@
 
 Summary: Arakoon
 Name: arakoon
-Version: 1.8.9
+Version: 1.8.10
 Release: 3%{?dist}
 License: Apache 2.0
 Requires: libev >= 4
@@ -33,6 +33,8 @@ cp arakoon.native %{buildroot}%{_bindir}/arakoon
 %{_bindir}/arakoon
 
 %changelog
+* Mon Nov 02 2015 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.8.10
+- Create arakoon 1.8.10 RPM package
 * Tue Aug 25 2015 Jan Doms <jan.doms@gmail.com> - 1.8.9
 - Create arakoon 1.8.9 RPM package
 * Wed Aug 05 2015 Jan Doms <jan.doms@gmail.com> - 1.8.8

From 725fc490d7b7cfd2798052a2ae4ddfed0c4f0173 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Fri, 27 Nov 2015 14:21:54 +0100
Subject: [PATCH 052/373] flush stderr and stdout for client_main

---
 src/client/client_main.ml |  7 ++++---
 src/node/dump_store.ml    |  4 ++--
 src/tlog/tlog_main.ml     | 12 ++++++------
 src/tools/lwt_extra.ml    |  9 +++++++++
 4 files changed, 21 insertions(+), 11 deletions(-)

diff --git a/src/client/client_main.ml b/src/client/client_main.ml
index 85d9e8d2..84991e39 100644
--- a/src/client/client_main.ml
+++ b/src/client/client_main.ml
@@ -35,6 +35,8 @@ let with_remote_nodestream ~tls node_cfg cluster_id f =
   in
   with_connection' ~tls addrs do_it
 
+let run f = Lwt_extra.run f ; 0
+
 let ping ~tls ip port cluster_id =
   let do_it connection =
     let t0 = Unix.gettimeofday () in
@@ -46,8 +48,8 @@ let ping ~tls ip port cluster_id =
     Lwt_io.printlf "%s\ntook\t%f" s d
   in
   let sa = make_address ip port in
-  let t = with_connection ~tls sa do_it in
-  Lwt_main.run t; 0
+  let t () = with_connection ~tls sa do_it in
+  run t
 
 
 
@@ -62,7 +64,6 @@ let find_master ~tls cluster_cfg =
     | Unknown_node (n, _) -> return n (* Keep original behaviour *)
     | Exception exn -> Lwt.fail exn
 
-let run f = Lwt_main.run (f ()); 0
 
 let with_master_client ~tls cfg_name f =
   let open Node_cfg in
diff --git a/src/node/dump_store.ml b/src/node/dump_store.ml
index 6bf295b2..bd96c9bf 100644
--- a/src/node/dump_store.ml
+++ b/src/node/dump_store.ml
@@ -59,7 +59,7 @@ let dump_store filename =
     summary store >>= fun () ->
     S.close store ~sync:false ~flush:false
   in
-  Lwt_main.run (t());
+  Lwt_extra.run t;
   0
 
 exception ExitWithCode of int;;
@@ -151,7 +151,7 @@ let inject_as_head fn node_id cfg_fn ~force ~in_place =
 
   in
   try
-    Lwt_main.run (t ())
+    Lwt_extra.run t
   with
     | ExitWithCode i -> i
     | exn -> raise exn
diff --git a/src/tlog/tlog_main.ml b/src/tlog/tlog_main.ml
index 8dd9e723..9cbe62d5 100644
--- a/src/tlog/tlog_main.ml
+++ b/src/tlog/tlog_main.ml
@@ -29,7 +29,7 @@ let dump_tlog filename ~values=
   in
   let folder,_,index = Tlc2.folder_for filename None in
 
-  let t =
+  let t () =
     begin
       let do_it ic =
 
@@ -43,7 +43,7 @@ let dump_tlog filename ~values=
       Lwt_io.with_file ~mode:Lwt_io.input filename do_it
     end
   in
-  Lwt_main.run t
+  Lwt_extra.run t
 
 let _last_entry filename =
   let last = ref None in
@@ -85,11 +85,11 @@ let strip_tlog filename =
             Lwt.return 1
           end
   in
-  let t =
+  let t () =
     _last_entry filename >>= fun last ->
     maybe_truncate last
   in
-  Lwt_main.run t
+  Lwt_extra.run t
 
 let mark_tlog file_name node_name =
   let t =
@@ -135,11 +135,11 @@ let compress_tlog tlu archive_type=
   let () = if not (Sys.file_exists tlu) then failwith "Input file %s does not exist" tlu in
   let tlx = Tlc2.to_archive_name compressor tlu in
   let () = if Sys.file_exists tlx then failwith "Can't compress %s as %s already exists" tlu tlx in
-  let t =
+  let t () =
     Compression.compress_tlog ~cancel:(ref false) tlu tlx compressor >>= fun () ->
     File_system.unlink tlu
   in
-  Lwt_main.run t;
+  Lwt_extra.run t;
   0
 
 let uncompress_tlog tlx =
diff --git a/src/tools/lwt_extra.ml b/src/tools/lwt_extra.ml
index 1786e2a9..bd719fe6 100644
--- a/src/tools/lwt_extra.ml
+++ b/src/tools/lwt_extra.ml
@@ -71,3 +71,12 @@ end = struct
                     else ();
                 Lwt.return ()))
 end
+
+let run f =
+  Lwt_main.run (
+      Lwt.finalize
+        f
+        (fun () ->
+         Lwt_io.flush Lwt_io.stdout >>= fun () ->
+         Lwt_io.flush Lwt_io.stderr)
+  )

From f0511649406813b2c384670f230650da121863af Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Mon, 30 Nov 2015 14:32:31 +0100
Subject: [PATCH 053/373] some catchup only improvements

---
 src/node/node_main.ml | 48 ++++++++++++++++++++++++++++++++-----------
 1 file changed, 36 insertions(+), 12 deletions(-)

diff --git a/src/node/node_main.ml b/src/node/node_main.ml
index 9fe50214..24d6afb7 100644
--- a/src/node/node_main.ml
+++ b/src/node/node_main.ml
@@ -184,22 +184,43 @@ let log_prelude cluster_cfg =
 
 let full_db_name me = me.home ^ "/" ^ me.node_name ^ ".db"
 
-let only_catchup (type s) (module S : Store.STORE with type t = s) ~tls_ctx ~name ~cluster_cfg ~make_tlog_coll =
+let only_catchup
+      (type s) (module S : Store.STORE with type t = s)
+      ~tls_ctx
+      ~name
+      ~cluster_cfg
+      ~make_tlog_coll
+      get_snapshot_name
+  =
   Logger.info_ "ONLY CATCHUP" >>= fun () ->
   let me, other_configs = _split name cluster_cfg.cfgs in
   let cluster_id = cluster_cfg.cluster_id in
   let db_name = full_db_name me in
+  let snapshot_name = get_snapshot_name() in
+  let full_snapshot_path = Filename.concat me.head_dir snapshot_name in
+  Lwt.catch
+    (fun () ->
+     S.copy_store2 full_snapshot_path db_name
+                   ~overwrite:false
+                   ~throttling:Node_cfg.default_head_copy_throttling
+    )
+    (function
+      | Not_found -> Lwt.return ()
+      | e -> raise e
+    )
+  >>= fun () ->
+
+  S.make_store ~lcnum:cluster_cfg.lcnum
+               ~ncnum:cluster_cfg.ncnum
+               db_name >>= fun store ->
+  let compressor = me.compressor in
+  make_tlog_coll ~compressor
+                 me.tlog_dir me.tlx_dir me.head_dir
+                 ~fsync:false name ~fsync_tlog_dir:false
+  >>= fun tlc ->
   let try_catchup mr_name =
     Lwt.catch
       (fun () ->
-       S.make_store ~lcnum:cluster_cfg.lcnum
-                    ~ncnum:cluster_cfg.ncnum
-                    db_name >>= fun store ->
-       let compressor = me.compressor in
-       make_tlog_coll ~compressor
-                      me.tlog_dir me.tlx_dir me.head_dir
-                      ~fsync:false name ~fsync_tlog_dir:false
-       >>= fun tlc ->
        Catchup.catchup ~tls_ctx ~stop:(ref false)
                        me.Node_cfg.Node_cfg.node_name other_configs ~cluster_id
                        ((module S),store,tlc) mr_name >>= fun _ ->
@@ -399,9 +420,12 @@ let _main_2 (type s)
   if catchup_only
   then
     begin
-      only_catchup (module S) ~tls_ctx:ssl_context ~name ~cluster_cfg ~make_tlog_coll
-      >>= fun _ -> (* we don't need that here as there is no continuation *)
-
+      only_catchup
+        (module S)
+        ~tls_ctx:ssl_context
+        ~name ~cluster_cfg ~make_tlog_coll
+        get_snapshot_name
+      >>= fun () ->
       Lwt.return 0
     end
   else

From 3fc2ed19b38b7eb304665481c8e6d8ba32e0c582 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Thu, 10 Dec 2015 21:24:11 +0100
Subject: [PATCH 054/373] option to autofix a node in some scenarios

---
 .../arakoon_ext/server/ArakoonManagement.py   | 12 ++-
 .../test/server/quick/system_tests_basic.py   | 94 +++++++++++++++++--
 src/main/main.ml                              |  4 +-
 src/node/node_main.ml                         | 84 +++++++++++++++--
 src/system/startup.ml                         | 23 +++--
 src/tlog/tlc2.ml                              | 58 ++++++------
 src/tlog/tlog_main.ml                         | 31 +++---
 7 files changed, 228 insertions(+), 78 deletions(-)

diff --git a/pylabs/extensions/arakoon_ext/server/ArakoonManagement.py b/pylabs/extensions/arakoon_ext/server/ArakoonManagement.py
index e7b927ce..1a18290c 100644
--- a/pylabs/extensions/arakoon_ext/server/ArakoonManagement.py
+++ b/pylabs/extensions/arakoon_ext/server/ArakoonManagement.py
@@ -1085,14 +1085,14 @@ def _requireLocal(self, nodeName):
         if not nodeName in self.listLocalNodes():
             raise ArakoonNodeNotLocal( nodeName)
 
-    def startOne(self, nodeName):
+    def startOne(self, nodeName, extraParams = None):
         """
         Start the node with a given name
         @param nodeName The name of the node
-
+        @param extraParams an optional list of extra parameters for the command line
         """
         self._requireLocal(nodeName)
-        return self._startOne(nodeName)
+        return self._startOne(nodeName, extraParams)
 
 
     def catchupOnly(self, nodeName):
@@ -1272,7 +1272,7 @@ def _cmdLine(self, name):
         cmdLine = string.join(cmd, ' ')
         return cmdLine
 
-    def _startOne(self, name):
+    def _startOne(self, name, extraParams = None):
         if self._getStatusOne(name) == X.AppStatusType.RUNNING:
             return
 
@@ -1284,8 +1284,10 @@ def _startOne(self, name):
 
         command = self._cmd(name)
         cmd.extend(command)
+        if extraParams:
+            cmd.extend(extraParams)
         cmd.append('-daemonize')
-        logging.debug('calling: %s', str(cmd))
+        logging.debug('calling: %s', ' '.join(cmd))
         return subprocess.call(cmd, close_fds = True)
 
     def _getIp(self,ip_mess):
diff --git a/pylabs/test/server/quick/system_tests_basic.py b/pylabs/test/server/quick/system_tests_basic.py
index 848a31bf..63c09710 100644
--- a/pylabs/test/server/quick/system_tests_basic.py
+++ b/pylabs/test/server/quick/system_tests_basic.py
@@ -106,22 +106,98 @@ def test_max_value_size_tinkering ():
     assert_raises (X.arakoon_client.ArakoonException, client.set, key, value)
 
 
-@C.with_custom_setup(C.setup_1_node,C.basic_teardown)
-def test_marker_presence_required ():
+def _remove_marker(cluster, nn):
+    cfg = cluster.getNodeConfig(nn)
+    home = cfg['home']
+    tlog = home + '/000.tlog'
     _arakoon = CONFIG.binary_full_path
-    C.assert_running_nodes(1)
+    subprocess.call([_arakoon,'--strip-tlog', tlog])
+
+def _truncate_tlog(cluster, nn, n_bytes):
+    cfg = cluster.getNodeConfig(nn)
+    home = cfg['home']
+    tlog = home + '/000.tlog'
+    _arakoon = CONFIG.binary_full_path
+    subprocess.call(['truncate', tlog, '-s', str(n_bytes)])
+
+def _mini_fill(cluster):
     client = C.get_client()
     for x in xrange(100):
         client.set("x%i" % x,"X%i" % x)
+
+def _wait_for_master(cluster):
+    start = time.time()
+    end = start + 10
+    interval = 1
+    cfg = "%s.cfg" % cluster._getConfigFileName()
+    print cfg
+    args = [
+        CONFIG.binary_full_path,
+        '-config', cfg,
+        '--who-master'
+    ]
+    while time.time() < end:
+        logging.info('Retrieving master')
+
+        proc = subprocess.Popen(args, close_fds=True,
+            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+        rc = proc.wait()
+
+        logging.info('rc: %s', rc)
+
+        if rc == 0:
+            return proc.stdout.read().strip()
+        else:
+            e = proc.stderr.read()
+            logging.info('`--who-master` returned %r: %r', rc, e)
+
+        time.sleep(interval)
+
+    raise Exception('Unable to find master')
+
+@C.with_custom_setup(C.setup_1_node, C.basic_teardown)
+def test_autofix_missing_marker():
+    cluster = C._getCluster()
+    C.assert_running_nodes(1)
+
+    _mini_fill(cluster)
+
+    cluster.stop()
+    nn = C.node_names[0]
+    _remove_marker(cluster, nn)
+    cluster.startOne(nn, ["-autofix"])
+    _wait_for_master(cluster)
+
+@C.with_custom_setup(C.setup_1_node,C.basic_teardown)
+def test_autofix_unexpected_end():
+    cluster = C._getCluster()
+    C.assert_running_nodes(1)
+
+    _mini_fill(cluster)
+
+    cluster.stop()
+    nn = C.node_names[0]
+    _truncate_tlog(cluster, nn, -1)
+
+    cluster.startOne(nn, ["-autofix"])
+    _wait_for_master(cluster)
+
+
+
+@C.with_custom_setup(C.setup_1_node,C.basic_teardown)
+def test_marker_presence_required ():
+
+    cluster = C._getCluster()
+    C.assert_running_nodes(1)
+    _mini_fill(cluster)
     cluster = C._getCluster()
     cluster.stop()
 
     # remove the marker
     nn = C.node_names[0]
-    cfg = cluster.getNodeConfig(nn)
-    home = cfg['home']
-    tlog = home + '/000.tlog'
-    subprocess.call([_arakoon,'--strip-tlog', tlog])
+    _remove_marker(cluster, nn)
+
 
     # this will fail
     cluster.start()
@@ -131,12 +207,16 @@ def test_marker_presence_required ():
     #check the exit code:
     cfgp = "%s.cfg" % (cluster._getConfigFileName()) # OMG
     logging.debug("cfgp=%s",cfgp)
+    _arakoon = CONFIG.binary_full_path
     try:
        subprocess.check_call([_arakoon, '--node', nn, '-config', cfgp])
     except subprocess.CalledProcessError,e:
         assert_equals(e.returncode,42)
 
     # add the marker and start again:
+    cfg = cluster.getNodeConfig(nn)
+    home = cfg['home']
+    tlog = home + '/000.tlog'
     subprocess.call([_arakoon,'--mark-tlog', tlog, 'closed:%s' % nn])
     cluster.start()
     time.sleep(1.0)
diff --git a/src/main/main.ml b/src/main/main.ml
index e4cca68c..168e5942 100644
--- a/src/main/main.ml
+++ b/src/main/main.ml
@@ -215,6 +215,7 @@ let main () =
   and right = ref None
   and rinc = ref false
   and scenario = ref (String.concat ", " Benchmark.default_scenario)
+  and autofix = ref false
   in
   let set_action a = Arg.Unit (fun () -> action := a) in
   let set_laction a = set_action (LocalAction a) in
@@ -315,6 +316,7 @@ let main () =
     (* ("-port", Arg.Set_int port, "specifies server port"); *)
     ("-config", Arg.Set_string config_file,
      "specifies config file (default = cfg/arakoon.ini )");
+    ("-autofix", Arg.Set autofix, "attempt to autofix (option to -node)");
     ("-catchup-only", Arg.Set catchup_only,
      "will only do a catchup of the node, without actually starting it (option to --node)");
     ("-daemonize", Arg.Set daemonize,
@@ -489,7 +491,7 @@ let main () =
           let make_config () = Node_cfg.read_config canonical in
           Daemons.maybe_daemonize !daemonize make_config;
           let main_t = (Node_main.main_t make_config
-                          !node_id !daemonize !catchup_only)
+                          !node_id !daemonize !catchup_only !autofix)
           in
           (* Lwt_engine.set (new Lwt_engine.select :> Lwt_engine.t); *)
           Lwt_main.run main_t
diff --git a/src/node/node_main.ml b/src/node/node_main.ml
index 24d6afb7..400b6800 100644
--- a/src/node/node_main.ml
+++ b/src/node/node_main.ml
@@ -387,7 +387,10 @@ let _main_2 (type s)
       (module S : Store.STORE with type t = s)
       make_tlog_coll
       make_config get_snapshot_name ~name
-      ~daemonize ~catchup_only ~stop : int Lwt.t =
+      ~daemonize ~catchup_only
+      ~autofix
+      ~stop : int Lwt.t
+  =
   Lwt_io.set_default_buffer_size 32768;
   let cluster_cfg = make_config () in
   let cfgs = cluster_cfg.cfgs in
@@ -415,6 +418,7 @@ let _main_2 (type s)
   in
 
   log_prelude cluster_cfg >>= fun () ->
+  Logger.info_f_ "autofix:%b" autofix >>= fun () ->
   Plugin_loader.load me.home cluster_cfg.plugins >>= fun () ->
 
   if catchup_only
@@ -525,10 +529,12 @@ let _main_2 (type s)
         (quorum_function n_nodes) n_nodes >>= fun () ->
       Logger.info_f_ "DAEMONIZATION=%b" daemonize >>= fun () ->
 
-      let build_startup_state () =
-        begin
+      let open_tlog_collection_and_store ~autofix me =
+        Logger.info_f_ "open_tlog_collection_and_store ~autofix:%b" autofix >>= fun () ->
+        let db_name = full_db_name me in
+        let head_copied = ref false in
+        let _open_tlc_and_store () =
           Node_cfg.Node_cfg.validate_dirs me >>= fun () ->
-          let db_name = full_db_name me in
           let snapshot_name = get_snapshot_name() in
           let full_snapshot_path = Filename.concat me.head_dir snapshot_name in
           Lwt.catch
@@ -536,10 +542,13 @@ let _main_2 (type s)
              S.copy_store2 full_snapshot_path db_name
                            ~overwrite:false
                            ~throttling:Node_cfg.default_head_copy_throttling
+             >>= fun () ->
+             head_copied := true;
+             Lwt.return_unit
             )
             (function
-              | Not_found -> Lwt.return ()
-              | e -> raise e
+              | Not_found -> Lwt.return_unit
+              | e -> Lwt.fail e
             )
           >>= fun () ->
           let compressor = me.compressor in
@@ -551,6 +560,54 @@ let _main_2 (type s)
           and ncnum = cluster_cfg.ncnum
           in
           S.make_store ~lcnum ~ncnum db_name >>= fun (store:S.t) ->
+          Lwt.return (tlog_coll, store)
+        in
+        Lwt.catch
+        (fun () -> _open_tlc_and_store ())
+        (fun exn ->
+         if autofix
+         then
+           begin
+             let clean_up_retry fn =
+               (if not !head_copied
+               then File_system.unlink db_name
+               else Lwt.return_unit)
+               >>= fun () ->
+               Tlog_main._mark_tlog fn (Tlc2._make_close_marker me.node_name) >>= fun _ ->
+               _open_tlc_and_store()
+             in
+             match exn with
+             | Tlc2.TLCNotProperlyClosed _ ->
+                begin
+                  Logger.warning_ ~exn "AUTOFIX: improperly closed tlog"
+                  >>= fun ()->
+                  Tlc2.get_last_tlog me.tlog_dir me.tlx_dir
+                  >>= fun (_, fn) ->
+                  clean_up_retry fn
+                end
+             | Tlogcommon.TLogUnexpectedEndOfFile _ ->
+                begin
+                  Logger.warning_ ~exn "AUTOFIX: unexpected end of tlog"
+                  >>= fun () ->
+                  Tlc2.get_last_tlog me.tlog_dir me.tlx_dir
+                  >>= fun (_, fn) ->
+                  Tlc2._truncate_tlog fn >>= fun ()->
+                  clean_up_retry fn
+                end
+
+             | _ ->
+                Logger.fatal_f_ ~exn "AUTOFIX: can't autofix this" >>= fun () ->
+                Lwt.fail exn
+           end
+         else
+           Lwt.fail exn
+        )
+      in
+      let build_startup_state () =
+        begin
+
+          open_tlog_collection_and_store me ~autofix
+          >>= fun (tlog_coll, store) ->
           let last_i = tlog_coll # get_last_i () in
           begin
             if me.is_witness
@@ -585,6 +642,8 @@ let _main_2 (type s)
           let act_not_preferred = ref false in
           let sb =
             let test = Node_cfg.Node_cfg.test cluster_cfg in
+            let snapshot_name = get_snapshot_name() in
+            let full_snapshot_path = Filename.concat me.head_dir snapshot_name in
             new SB.sync_backend me
               (client_push: (Update.t * (Store.update_result -> unit Lwt.t)) -> unit Lwt.t)
               inject_push
@@ -939,11 +998,14 @@ let _main_2 (type s)
     end
 
 
-let main_t make_config name daemonize catchup_only : int Lwt.t =
+let main_t make_config name daemonize catchup_only autofix : int Lwt.t =
   let module S = (val (Store.make_store_module (module Batched_store.Local_store))) in
   let make_tlog_coll = Tlc2.make_tlc2 in
   let get_snapshot_name = Tlc2.head_name in
-  _main_2 (module S) make_tlog_coll make_config get_snapshot_name ~name ~daemonize ~catchup_only ~stop:(ref false)
+  _main_2 (module S)
+          make_tlog_coll
+          make_config get_snapshot_name
+          ~name ~daemonize ~catchup_only ~autofix ~stop:(ref false)
 
 let test_t make_config name ~stop =
   let module S = (val (Store.make_store_module (module Mem_store))) in
@@ -953,5 +1015,7 @@ let test_t make_config name ~stop =
   in
   let get_snapshot_name = fun () -> "DUMMY" in
   let daemonize = false
-  and catchup_only = false in
-  _main_2 (module S) make_tlog_coll make_config get_snapshot_name ~name ~daemonize ~catchup_only ~stop
+  and catchup_only = false
+  and autofix = false in
+  _main_2 (module S) make_tlog_coll make_config get_snapshot_name
+          ~name ~daemonize ~catchup_only ~stop ~autofix
diff --git a/src/system/startup.ml b/src/system/startup.ml
index 4a01c5a2..39f2bca3 100644
--- a/src/system/startup.ml
+++ b/src/system/startup.ml
@@ -104,16 +104,19 @@ let _make_run ~stores ~tlcs ~now ~values ~get_cfgs name () =
       Hashtbl.add stores db_name store;
       Lwt.return store
   end in
-  let t = Node_main._main_2
-    (module S)
-    (_make_tlog_coll tlcs values)
-    get_cfgs
-    (fun () -> "DUMMY")
-    ~name
-    ~daemonize:false
-    ~catchup_only:false
-    ~stop:!stop
-          >>= fun _ -> Lwt.return () in
+  let t =
+    Node_main._main_2
+      (module S)
+      (_make_tlog_coll tlcs values)
+      get_cfgs
+      (fun () -> "DUMMY")
+      ~name
+      ~daemonize:false
+      ~catchup_only:false
+      ~autofix:false
+      ~stop:!stop
+    >>= fun _ -> Lwt.return ()
+  in
   node_ts := t :: !node_ts;
   t
 
diff --git a/src/tlog/tlc2.ml b/src/tlog/tlc2.ml
index 9f3124b5..53b42fe0 100644
--- a/src/tlog/tlc2.ml
+++ b/src/tlog/tlc2.ml
@@ -893,40 +893,38 @@ let make_tlc2 ~compressor tlog_dir tlf_dir head_dir ~fsync node_id ~fsync_tlog_d
   >>= fun () ->
   Lwt.return col
 
-
-let truncate_tlog filename =
-  let skipper () _entry = Lwt.return ()
-  in
-  let t () =
+let _truncate_tlog filename =
+  let skipper () _entry = Lwt.return () in
+  let folder,extension,index = folder_for filename None in
+  if extension <> ".tlog"
+  then Lwt.fail (Failure "Cannot truncate a compressed tlog")
+  else
     begin
-      let folder,extension,index = folder_for filename None in
-      if extension <> ".tlog"
-      then Lwt.fail (Failure "Cannot truncate a compressed tlog")
-      else
-        begin
-          let do_it ic =
-            let lowerI = Sn.start
-            and higherI = None
-            and first = Sn.of_int 0
-            in
-            Lwt.catch
-              (fun() -> folder ic ~index lowerI higherI ~first () skipper)
-              (function
-                | Tlogcommon.TLogCheckSumError pos
-                | Tlogcommon.TLogUnexpectedEndOfFile pos ->
-                  let fd = Unix.openfile filename [ Unix.O_RDWR ] 0o600 in
-                  Lwt.return ( Unix.ftruncate fd (Int64.to_int pos) )
-                | ex -> Lwt.fail ex
-              ) >>= fun () ->
-            Lwt.return 0
-          in
-          Lwt_io.with_file ~mode:Lwt_io.input filename do_it
-        end
+      let do_it ic =
+        let lowerI = Sn.start
+        and higherI = None
+        and first = Sn.of_int 0
+        in
+        Lwt.catch
+          (fun() -> folder ic ~index lowerI higherI ~first () skipper)
+          (function
+            | Tlogcommon.TLogCheckSumError pos
+            | Tlogcommon.TLogUnexpectedEndOfFile pos ->
+               let fd = Unix.openfile filename [ Unix.O_RDWR ] 0o600 in
+               Lwt.return ( Unix.ftruncate fd (Int64.to_int pos) )
+            | ex -> Lwt.fail ex
+          )
+      in
+      Lwt_io.with_file ~mode:Lwt_io.input filename do_it
     end
-  in
+
+let truncate_tlog filename =
   let t' =
     Lwt.catch
-      t
+      (fun () ->
+       _truncate_tlog filename
+       >>= fun () ->
+       Lwt.return 0)
       (fun ex ->
        Lwt_io.printlf "problem:%s%!" (Printexc.to_string ex) >>= fun () ->
        Lwt.return 1
diff --git a/src/tlog/tlog_main.ml b/src/tlog/tlog_main.ml
index 9cbe62d5..0d9551b1 100644
--- a/src/tlog/tlog_main.ml
+++ b/src/tlog/tlog_main.ml
@@ -91,22 +91,23 @@ let strip_tlog filename =
   in
   Lwt_extra.run t
 
+let _mark_tlog file_name node_name =
+  _last_entry file_name >>= fun last ->
+  match last with
+  | None -> Lwt.return 0
+  | Some e ->
+     let i     = Entry.i_of e
+     and value = Entry.v_of e
+     in
+     let f oc = Tlogcommon.write_marker oc i value (Some node_name) in
+     Lwt_io.with_file
+       ~mode:Lwt_io.output
+       ~flags:[Unix.O_APPEND;Unix.O_WRONLY]
+       file_name f
+     >>= fun () -> Lwt.return 0
+
 let mark_tlog file_name node_name =
-  let t =
-    _last_entry file_name >>= fun last ->
-    match last with
-      | None -> Lwt.return 0
-      | Some e ->
-        let i     = Entry.i_of e
-        and value = Entry.v_of e
-        in
-        let f oc = Tlogcommon.write_marker oc i value (Some node_name) in
-        Lwt_io.with_file
-          ~mode:Lwt_io.output
-          ~flags:[Unix.O_APPEND;Unix.O_WRONLY]
-          file_name f
-        >>= fun () -> Lwt.return 0
-  in
+  let t = _mark_tlog file_name node_name in
   Lwt_main.run t
 
 let make_tlog tlog_name (i:int) =

From 8b1053ab2426be9341f36baaea6a79eac1dc7fac Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Fri, 11 Dec 2015 10:33:41 +0100
Subject: [PATCH 055/373] make_release: 1.8.11

---
 META                      | 2 +-
 debian/changelog          | 5 +++++
 redhat/SPECS/arakoon.spec | 4 +++-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/META b/META
index dd940f04..f3fb6116 100644
--- a/META
+++ b/META
@@ -1,7 +1,7 @@
 # -*- conf -*-
 
 description = "Arakoon client"
-version = "1.8.10"
+version = "1.8.11"
 exists_if = "arakoon_client.cma,arakoon_client.cmxa,arakoon_client.cmxs"
 requires = "threads lwt lwt.unix str oUnit quickcheck lwt.ssl ocplib-endian nocrypto cstruct"
 archive(byte) = "arakoon_client.cma"
diff --git a/debian/changelog b/debian/changelog
index ce8ed0b2..51fd4172 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,8 @@
+arakoon (1.8.11) unstable; urgency=low
+  * new upstream version
+
+ -- Romain Slootmaekers <romain.slootmaekers@openvstorage.com>  Fri, 11 Dec 2015 10:33:41 +0100
+
 arakoon (1.8.10) unstable; urgency=low
   * new upstream version
 
diff --git a/redhat/SPECS/arakoon.spec b/redhat/SPECS/arakoon.spec
index ac10f6c3..0db7d05b 100644
--- a/redhat/SPECS/arakoon.spec
+++ b/redhat/SPECS/arakoon.spec
@@ -2,7 +2,7 @@
 
 Summary: Arakoon
 Name: arakoon
-Version: 1.8.10
+Version: 1.8.11
 Release: 3%{?dist}
 License: Apache 2.0
 Requires: libev >= 4
@@ -33,6 +33,8 @@ cp arakoon.native %{buildroot}%{_bindir}/arakoon
 %{_bindir}/arakoon
 
 %changelog
+* Fri Dec 11 2015 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.8.11
+- Create arakoon 1.8.11 RPM package
 * Mon Nov 02 2015 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.8.10
 - Create arakoon 1.8.10 RPM package
 * Tue Aug 25 2015 Jan Doms <jan.doms@gmail.com> - 1.8.9

From db64429ac773c4a6b4683d87f7ad716c9ff0b77a Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Fri, 11 Dec 2015 14:06:33 +0100
Subject: [PATCH 056/373] pin versions

---
 docker/arakoon_debian/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/arakoon_debian/Dockerfile b/docker/arakoon_debian/Dockerfile
index 72c13515..e3bfaca2 100644
--- a/docker/arakoon_debian/Dockerfile
+++ b/docker/arakoon_debian/Dockerfile
@@ -64,7 +64,7 @@ RUN eval `${opam_env}` && opam install -y \
         ctypes \
         ctypes-foreign \
         uuidm \
-        zarith \
+        zarith.1.3 \
         mirage-no-xen.1 \
         quickcheck \
         nocrypto \

From cba4594148285faba1ad51149ffb57cbcb7348b4 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Thu, 17 Dec 2015 15:07:27 +0100
Subject: [PATCH 057/373] add local collapse

---
 src/main/main.ml      | 27 +++++++++++++++++++++------
 src/node/collapser.ml | 32 ++++++++++++++++++++++++++++++++
 src/node/node_cfg.ml  | 15 +++++++++++++++
 src/node/node_main.ml | 18 ++----------------
 4 files changed, 70 insertions(+), 22 deletions(-)

diff --git a/src/main/main.ml b/src/main/main.ml
index 168e5942..f4b8825f 100644
--- a/src/main/main.ml
+++ b/src/main/main.ml
@@ -48,6 +48,7 @@ type local_action =
   | STATISTICS
   | PREFIX
   | USER_FUNCTION
+  | Collapse_local
   | Collapse_remote
   | Backup_db
   | Optimize_db
@@ -335,6 +336,10 @@ let main () =
        !scenario);
     ("-max_results", Arg.Set_int max_results, "max size of the result (for --prefix)");
     ("--test-repeat", Arg.Set_int test_repeat_count, "<repeat_count>");
+    ("--collapse-local", Arg.Tuple[set_laction Collapse_local;
+                                   Arg.Set_string node_id;
+                                   Arg.Set_int n_tlogs;],
+     "<node_id> <n> collapse all but <n> tlogs into the head database");
     ("--collapse-remote", Arg.Tuple[set_laction Collapse_remote;
                                     Arg.Set_string cluster_id;
                                     Arg.Set_string ip;
@@ -420,6 +425,15 @@ let main () =
   let options = [] in
   let interface = actions @ options in
 
+  let make_config =
+    let canonical =
+      if !config_file.[0] = '/'
+      then !config_file
+      else Filename.concat (Unix.getcwd()) !config_file
+    in
+    fun () -> Node_cfg.read_config canonical
+  in
+
   let do_local ~tls = function
     | ShowUsage -> print_endline usage;0
     | RunAllTests -> run_all_tests ()
@@ -453,6 +467,13 @@ let main () =
     | WHO_MASTER -> Client_main.who_master ~tls !config_file ()
     | EXPECT_PROGRESS_POSSIBLE -> Client_main.expect_progress_possible ~tls !config_file
     | STATISTICS -> Client_main.statistics ~tls !config_file
+    | Collapse_local ->
+       Lwt_main.run
+         begin
+           let cfg = make_config () in
+           Collapser.collapse_out_of_band cfg !node_id !n_tlogs
+         end;
+       0
     | Collapse_remote -> Collapser_main.collapse_remote
                            ~tls !ip !port !cluster_id !n_tlogs
     | Backup_db -> Nodestream_main.get_db ~tls !ip !port !cluster_id !location
@@ -483,12 +504,6 @@ let main () =
     match node with
       | Node ->
         begin
-          let canonical =
-            if !config_file.[0] = '/'
-            then !config_file
-            else Filename.concat (Unix.getcwd()) !config_file
-          in
-          let make_config () = Node_cfg.read_config canonical in
           Daemons.maybe_daemonize !daemonize make_config;
           let main_t = (Node_main.main_t make_config
                           !node_id !daemonize !catchup_only !autofix)
diff --git a/src/node/collapser.ml b/src/node/collapser.ml
index 45a03577..28f5d30a 100644
--- a/src/node/collapser.ml
+++ b/src/node/collapser.ml
@@ -238,3 +238,35 @@ let collapse_many
       tlog_coll # remove_oldest_tlogs tlogs_to_collapse >>= fun () ->
       cb (Sn.div g_too_far_i (Sn.of_int !Tlogcommon.tlogEntriesPerFile))
     end
+
+let collapse_out_of_band cfg name tlogs_to_keep =
+  let open Lwt.Infix in
+  let open Node_cfg.Node_cfg in
+
+  let me, _ = split name cfg.cfgs in
+  let head_location = Filename.concat me.head_dir Tlc2.head_fname in
+
+  let module S = (val (Store.make_store_module (module Batched_store.Local_store))) in
+  S.make_store
+    ~lcnum:cfg.lcnum
+    ~ncnum:cfg.ncnum
+    ~read_only:true head_location >>= fun head ->
+
+  Tlc2.make_tlc2 ~compressor:me.compressor
+                 me.tlog_dir me.tlx_dir me.head_dir
+                 ~fsync:true name ~fsync_tlog_dir:true
+  >>= fun tlog_coll ->
+
+  collapse_many
+    tlog_coll
+    (module S)
+    (S.copy_store2,
+     head_location,
+     me.head_copy_throttling)
+    tlogs_to_keep
+    (fun sn -> Lwt_log.debug_f "sn' = %i" sn)
+    (fun sn -> Lwt_log.debug_f "sn = %Li" sn)
+    me.collapse_slowdown
+  >>= fun () ->
+  Lwt.return ()
+
diff --git a/src/node/node_cfg.ml b/src/node/node_cfg.ml
index ebef9cc0..cdf17b51 100644
--- a/src/node/node_cfg.ml
+++ b/src/node/node_cfg.ml
@@ -783,4 +783,19 @@ module Node_cfg = struct
         verify t.head_dir "Head dir" (InvalidHeadDir t.head_dir)
 
       end
+
+  let split node_name cfgs =
+    let rec loop me_o others = function
+      | [] -> me_o, others
+      | cfg :: rest ->
+         if cfg.node_name = node_name then
+           loop (Some cfg) others rest
+         else
+           loop me_o (cfg::others) rest
+    in
+    let me_o, others = loop None [] cfgs in
+    match me_o with
+    | None -> failwith (node_name ^ " is not known in config")
+    | Some me -> me,others
+
 end
diff --git a/src/node/node_main.ml b/src/node/node_main.ml
index 400b6800..5237cdaa 100644
--- a/src/node/node_main.ml
+++ b/src/node/node_main.ml
@@ -25,20 +25,6 @@ open Statistics
 
 let section = Logger.Section.main
 
-let _split node_name cfgs =
-  let rec loop me_o others = function
-    | [] -> me_o, others
-    | cfg :: rest ->
-      if cfg.node_name = node_name then
-        loop (Some cfg) others rest
-      else
-        loop me_o (cfg::others) rest
-  in
-  let me_o, others = loop None [] cfgs in
-  match me_o with
-    | None -> failwith (node_name ^ " is not known in config")
-    | Some me -> me,others
-
 
 let _config_logging me get_cfgs =
   let cluster_cfg = get_cfgs () in
@@ -193,7 +179,7 @@ let only_catchup
       get_snapshot_name
   =
   Logger.info_ "ONLY CATCHUP" >>= fun () ->
-  let me, other_configs = _split name cluster_cfg.cfgs in
+  let me, other_configs = Node_cfg.Node_cfg.split name cluster_cfg.cfgs in
   let cluster_id = cluster_cfg.cluster_id in
   let db_name = full_db_name me in
   let snapshot_name = get_snapshot_name() in
@@ -394,7 +380,7 @@ let _main_2 (type s)
   Lwt_io.set_default_buffer_size 32768;
   let cluster_cfg = make_config () in
   let cfgs = cluster_cfg.cfgs in
-  let me, others = _split name cfgs in
+  let me, others = Node_cfg.Node_cfg.split name cfgs in
   _config_logging me.node_name make_config >>= fun dump_crash_log ->
   let maybe_dump_crash_log () =
     match dump_crash_log with

From d45f7438ed63b78a4368a79554d19b9a1115b5bc Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Thu, 17 Dec 2015 15:15:42 +0100
Subject: [PATCH 058/373] test for local collapse

---
 pylabs/test/server/left/test_collapse.py  | 48 ++++++++++++++++++++++-
 pylabs/test/server/system_tests_common.py |  7 ++++
 src/main/main.ml                          |  4 +-
 3 files changed, 55 insertions(+), 4 deletions(-)

diff --git a/pylabs/test/server/left/test_collapse.py b/pylabs/test/server/left/test_collapse.py
index 51386898..2a201a5e 100644
--- a/pylabs/test/server/left/test_collapse.py
+++ b/pylabs/test/server/left/test_collapse.py
@@ -27,7 +27,7 @@
 
 
 @Common.with_custom_setup(Common.setup_2_nodes_forced_master, Common.basic_teardown)
-def test_collapse_witness_node():
+def test_remote_collapse_witness_node():
     master = Common.node_names[0]
     witness = Common.node_names[1]
     n = 298765
@@ -49,7 +49,7 @@ def test_collapse_witness_node():
     logging.info("catchup from collapsed node finished")
 
 @Common.with_custom_setup(Common.setup_2_nodes, Common.basic_teardown)
-def test_collapse():
+def test_remote_collapse():
     zero = Common.node_names[0]
     one = Common.node_names[1]
     n = 298765
@@ -69,3 +69,47 @@ def test_collapse():
         counter = counter + 1
         up2date = cli.expectProgressPossible()
     logging.info("catchup from collapsed node finished")
+
+@Common.with_custom_setup(Common.setup_2_nodes_forced_master, Common.basic_teardown)
+def test_local_collapse_witness_node():
+    master = Common.node_names[0]
+    witness = Common.node_names[1]
+    n = 298765
+    Common.iterate_n_times(n, Common.simple_set)
+    logging.info("did %i sets, now going into collapse scenario" % n)
+    Common.local_collapse(witness,1)
+    logging.info("collapsing done")
+    Common.stopOne(master)
+    Common.wipe(master)
+    Common.startOne(master)
+    cli = Common.get_client()
+    assert_false(cli.expectProgressPossible())
+    up2date = False
+    counter = 0
+    while not up2date and counter < 100:
+        time.sleep(1.0)
+        counter = counter + 1
+        up2date = cli.expectProgressPossible()
+    logging.info("catchup from collapsed node finished")
+
+@Common.with_custom_setup(Common.setup_2_nodes, Common.basic_teardown)
+def test_local_collapse():
+    zero = Common.node_names[0]
+    one = Common.node_names[1]
+    n = 298765
+    Common.iterate_n_times(n, Common.simple_set)
+    logging.info("did %i sets, now going into collapse scenario" % n)
+    Common.local_collapse(zero,1)
+    logging.info("collapsing done")
+    Common.stopOne(one)
+    Common.wipe(one)
+    Common.startOne(one)
+    cli = Common.get_client()
+    assert_false(cli.expectProgressPossible())
+    up2date = False
+    counter = 0
+    while not up2date and counter < 100:
+        time.sleep(1.0)
+        counter = counter + 1
+        up2date = cli.expectProgressPossible()
+    logging.info("catchup from collapsed node finished")
diff --git a/pylabs/test/server/system_tests_common.py b/pylabs/test/server/system_tests_common.py
index b615a652..0d525043 100644
--- a/pylabs/test/server/system_tests_common.py
+++ b/pylabs/test/server/system_tests_common.py
@@ -530,6 +530,13 @@ def collapse(name, n = 1):
     rc = X.subprocess.call([CONFIG.binary_full_path, '--collapse-remote',cluster_id,ip,port,str(n)])
     return rc
 
+def local_collapse(name, n=1):
+    cluster = _getCluster()
+    cfg = "%s.cfg" % cluster._getConfigFileName()
+    rc = X.subprocess.call([ CONFIG.binary_full_path, '--collapse-local', name, str(n),
+                             '-config', cfg ])
+    return rc
+
 def add_node ( i ):
     ni = node_names[i]
     logging.info( "Adding node %s to config", ni )
diff --git a/src/main/main.ml b/src/main/main.ml
index f4b8825f..705adfaf 100644
--- a/src/main/main.ml
+++ b/src/main/main.ml
@@ -425,13 +425,13 @@ let main () =
   let options = [] in
   let interface = actions @ options in
 
-  let make_config =
+  let make_config () =
     let canonical =
       if !config_file.[0] = '/'
       then !config_file
       else Filename.concat (Unix.getcwd()) !config_file
     in
-    fun () -> Node_cfg.read_config canonical
+    Node_cfg.read_config canonical
   in
 
   let do_local ~tls = function

From fc2e17890d5c35b6e7adc2b75116d639b465067b Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Fri, 18 Dec 2015 14:00:29 +0100
Subject: [PATCH 059/373] enable SO_KEEPALIVE for connections on client_port

---
 src/tools/server.ml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/tools/server.ml b/src/tools/server.ml
index 547f5c8f..510964dc 100644
--- a/src/tools/server.ml
+++ b/src/tools/server.ml
@@ -209,6 +209,7 @@ let make_server_thread
         Lwt.catch
           (fun () ->
            Lwt_unix.accept listening_socket >>= fun (plain_fd, cl_socket_address) ->
+           Lwt_unix.setsockopt listening_socket Unix.SO_KEEPALIVE true;
            let cid = name ^ "_" ^ Int64.to_string (connection_counter ()) in
            let finalize () =
              Hashtbl.remove client_threads cid;

From 7e3d0a6e98f172bcc938ee4fe11b5bcfeb116baa Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Mon, 21 Dec 2015 13:23:23 +0100
Subject: [PATCH 060/373] allow configuring tcp keepalive values

---
 src/arakoon_client.mllib                |  1 +
 src/libcutil.clib                       |  1 +
 src/msg/tcp_messaging.ml                |  2 +
 src/msg/tcp_messaging_test.ml           |  6 ++-
 src/node/node_cfg.ml                    | 45 +++++++++++++++++++++-
 src/node/node_main.ml                   | 19 ++++++---
 src/paxos/multi_paxos_test.ml           |  6 ++-
 src/system/startup.ml                   |  4 ++
 src/tools/network.ml                    | 14 ++++++-
 src/tools/server.ml                     |  3 +-
 src/tools/server_test.ml                |  2 +
 src/tools/tcp_keepalive.ml              | 42 ++++++++++++++++++++
 src/tools/tcp_keepalive_sockopt_stubs.c | 51 +++++++++++++++++++++++++
 13 files changed, 184 insertions(+), 12 deletions(-)
 create mode 100644 src/tools/tcp_keepalive.ml
 create mode 100644 src/tools/tcp_keepalive_sockopt_stubs.c

diff --git a/src/arakoon_client.mllib b/src/arakoon_client.mllib
index 30a19bb4..eeafd50e 100644
--- a/src/arakoon_client.mllib
+++ b/src/arakoon_client.mllib
@@ -31,3 +31,4 @@ Arakoon_client_config
 Lwt_extra
 Network
 Typed_ssl
+Tcp_keepalive
diff --git a/src/libcutil.clib b/src/libcutil.clib
index 030f3b91..55d0acc9 100644
--- a/src/libcutil.clib
+++ b/src/libcutil.clib
@@ -2,3 +2,4 @@ arakoon_crc32c.o
 bsd_crc32c.o
 arakoon_limits.o
 tcp_info_stubs.o
+tcp_keepalive_sockopt_stubs.o
diff --git a/src/msg/tcp_messaging.ml b/src/msg/tcp_messaging.ml
index e1f172cb..eb964671 100644
--- a/src/msg/tcp_messaging.ml
+++ b/src/msg/tcp_messaging.ml
@@ -66,6 +66,7 @@ let run_report connections =
 class tcp_messaging
   ?(timeout=60.0)
   ?(client_ssl_context:[> `Client ] Typed_ssl.t option)
+  ~tcp_keepalive_settings
   my_addresses my_cookie (drop_it: drop_function)
   max_buffer_size ~stop =
 
@@ -401,6 +402,7 @@ class tcp_messaging
                     ~name ~setup_callback
                     ~teardown_callback ip my_port protocol
                     ~scheme ?ssl_context
+                    ~tcp_keepalive_settings
                     ~stop
           in
           s ()
diff --git a/src/msg/tcp_messaging_test.ml b/src/msg/tcp_messaging_test.ml
index b53e5a1f..5b625638 100644
--- a/src/msg/tcp_messaging_test.ml
+++ b/src/msg/tcp_messaging_test.ml
@@ -94,7 +94,11 @@ class player id (m:messaging) =
   end
 
 let make_transport addresses =
-  let tcp_transport = new tcp_messaging addresses "yummy" (fun _ _ _-> false) Node_cfg.default_max_buffer_size ~stop:(ref false) in
+  let tcp_transport =
+    new tcp_messaging
+        addresses "yummy" (fun _ _ _-> false) Node_cfg.default_max_buffer_size ~stop:(ref false)
+        ~tcp_keepalive_settings:Node_cfg.default_tcp_keepalive
+  in
   (tcp_transport :> messaging)
 
 let eventually_die ?(t=10.0) () =
diff --git a/src/node/node_cfg.ml b/src/node/node_cfg.ml
index ebef9cc0..9d6527b5 100644
--- a/src/node/node_cfg.ml
+++ b/src/node/node_cfg.ml
@@ -27,6 +27,21 @@ let default_client_buffer_capacity = 32
 let default_lcnum = 16384
 let default_ncnum = 8192
 let default_head_copy_throttling = 0.0
+
+type tcp_keepalive_cfg = Tcp_keepalive.t = {
+      enable_tcp_keepalive : bool;
+      tcp_keepalive_time : int;
+      tcp_keepalive_intvl : int;
+      tcp_keepalive_probes : int;
+    }
+
+let default_tcp_keepalive = {
+    enable_tcp_keepalive = true;
+    tcp_keepalive_time = 20;
+    tcp_keepalive_intvl = 20;
+    tcp_keepalive_probes = 3;
+  }
+
 open Master_type
 open Client_cfg
 
@@ -231,7 +246,8 @@ module Node_cfg = struct
       lcnum : int; (* tokyo cabinet: leaf nodes in cache *)
       ncnum : int; (* tokyo cabinet: internal nodes in cache *)
       tls : TLSConfig.Cluster.t option;
-}
+      tcp_keepalive : tcp_keepalive_cfg;
+    }
 
   let node_cfg_to_node_client_cfg (cfg : t) =
     { Arakoon_client_config.ips = cfg.ips;
@@ -328,6 +344,7 @@ module Node_cfg = struct
       lcnum = default_lcnum;
       ncnum = default_ncnum;
       tls = None;
+      tcp_keepalive = default_tcp_keepalive;
     }
     in
     cluster_cfg
@@ -659,6 +676,26 @@ module Node_cfg = struct
           in
           Some cfg
     in
+    let enable_tcp_keepalive =
+      Ini.get
+        inifile "global" "enable_tcp_keepalive"
+        Ini.p_bool (Ini.default true)
+    in
+    let tcp_keepalive_time =
+      Ini.get
+        inifile "global" "tcp_keepalive_time"
+        Ini.p_int (Ini.default 20)
+    in
+    let tcp_keepalive_intvl =
+      Ini.get
+        inifile "global" "tcp_keepalive_intvl"
+        Ini.p_int (Ini.default 20)
+    in
+    let tcp_keepalive_probes =
+      Ini.get
+        inifile "global" "tcp_keepalive_probes"
+        Ini.p_int (Ini.default 3)
+    in
     let cluster_cfg =
       { cfgs;
         log_cfgs;
@@ -676,6 +713,12 @@ module Node_cfg = struct
         lcnum;
         ncnum;
         tls;
+        tcp_keepalive = {
+            enable_tcp_keepalive;
+            tcp_keepalive_time;
+            tcp_keepalive_intvl;
+            tcp_keepalive_probes;
+          };
       }
     in
     cluster_cfg
diff --git a/src/node/node_main.ml b/src/node/node_main.ml
index 400b6800..68fff846 100644
--- a/src/node/node_main.ml
+++ b/src/node/node_main.ml
@@ -113,6 +113,7 @@ let _config_messaging
       ?ssl_context me others
       cookie laggy
       lease_period max_buffer_size
+      ~tcp_keepalive_settings
       ~stop =
   let drop_it = match laggy with
     | true -> let count = ref 0 in
@@ -134,14 +135,16 @@ let _config_messaging
       [] others
   in
   let client_ssl_context = ssl_context in
-  let messaging = new tcp_messaging
-    (me.ips, me.messaging_port) cookie drop_it max_buffer_size ~timeout:(lease_period *. 2.5)
-    ?client_ssl_context ~stop in
+  let messaging =
+    new tcp_messaging
+        (me.ips, me.messaging_port) cookie drop_it max_buffer_size ~timeout:(lease_period *. 2.5)
+        ~tcp_keepalive_settings
+        ?client_ssl_context ~stop in
   messaging # register_receivers mapping;
   (messaging :> Messaging.messaging)
 
 
-let _config_service ?ssl_context cfg stop backend =
+let _config_service ?ssl_context ~tcp_keepalive_settings cfg stop backend =
   let port = cfg.client_port in
   let hosts = cfg.ips in
   let max_connections =
@@ -155,6 +158,7 @@ let _config_service ?ssl_context cfg stop backend =
                       Server.make_server_thread ~name host port
                         (Client_protocol.protocol stop backend)
                         ~scheme ?ssl_context ~stop
+                        ~tcp_keepalive_settings
                    )
                    hosts
   in
@@ -509,8 +513,10 @@ let _main_2 (type s)
         end
       in
       Lwt.ignore_result ( upload_cfg_to_keeper () ) ;
+      let tcp_keepalive_settings = cluster_cfg.tcp_keepalive in
       let messaging  = _config_messaging
                          ?ssl_context
+                         ~tcp_keepalive_settings
                          me cfgs cookie me.is_laggy
                          (float me.lease_period)
                          cluster_cfg.max_buffer_size
@@ -659,7 +665,10 @@ let _main_2 (type s)
           in
           let backend = (sb :> Backend.backend) in
 
-          let service = _config_service ?ssl_context:service_ssl_context me stop backend in
+          let service = _config_service
+                          ?ssl_context:service_ssl_context
+                          ~tcp_keepalive_settings
+                          me stop backend in
 
           let send, _run, _register, is_alive =
             Multi_paxos.network_of_messaging messaging in
diff --git a/src/paxos/multi_paxos_test.ml b/src/paxos/multi_paxos_test.ml
index 34a5bb70..e842d5b0 100644
--- a/src/paxos/multi_paxos_test.ml
+++ b/src/paxos/multi_paxos_test.ml
@@ -347,8 +347,10 @@ let build_perfect () =
   get_buffer, (send, run, register, is_alive)
 
 let build_tcp () =
-  let (m : messaging) = new Tcp_messaging.tcp_messaging (["127.0.0.1"], 7777) "yummie"
-    (fun _ _ _ -> false) Node_cfg.default_max_buffer_size ~stop:(ref false)
+  let (m : messaging) =
+    new Tcp_messaging.tcp_messaging (["127.0.0.1"], 7777) "yummie"
+        (fun _ _ _ -> false) Node_cfg.default_max_buffer_size ~stop:(ref false)
+        ~tcp_keepalive_settings:Node_cfg.default_tcp_keepalive
   in
   let network = network_of_messaging m in
   m # get_buffer, network
diff --git a/src/system/startup.ml b/src/system/startup.ml
index 39f2bca3..97d09c0d 100644
--- a/src/system/startup.ml
+++ b/src/system/startup.ml
@@ -157,6 +157,7 @@ let post_failure () =
     lcnum = 8192;
     ncnum = 4096;
     tls = None;
+    tcp_keepalive = Node_cfg.default_tcp_keepalive;
   }
   in
   let get_cfgs () = cluster_cfg in
@@ -217,6 +218,7 @@ let restart_slaves () =
      lcnum = Node_cfg.default_lcnum;
      ncnum = Node_cfg.default_ncnum;
      tls = None;
+     tcp_keepalive = Node_cfg.default_tcp_keepalive;
     }
   in
   let get_cfgs () = cluster_cfg in
@@ -275,6 +277,7 @@ let ahead_master_loses_role () =
      lcnum = 8192;
      ncnum = 4096;
      tls = None;
+     tcp_keepalive = Node_cfg.default_tcp_keepalive;
     }
   in
   let get_cfgs () = cluster_cfg in
@@ -346,6 +349,7 @@ let interrupted_election () =
      lcnum = 8192;
      ncnum = 4096;
      tls = None;
+     tcp_keepalive = Node_cfg.default_tcp_keepalive;
     }
   in
   let get_cfgs () = cluster_cfg in
diff --git a/src/tools/network.ml b/src/tools/network.ml
index 9c77ff8c..25d4c76f 100644
--- a/src/tools/network.ml
+++ b/src/tools/network.ml
@@ -28,6 +28,17 @@ let a2s = function
   | Unix.ADDR_INET (sa,p) -> Printf.sprintf "(%s,%i)" (Unix.string_of_inet_addr sa) p
   | Unix.ADDR_UNIX s      -> Printf.sprintf "ADDR_UNIX(%s)" s
 
+let lwt_unix_fd_to_fd
+      (fd : Lwt_unix.file_descr) : int =
+  Obj.magic (Obj.field (Obj.repr fd) 0)
+
+let lwt_unix_fd_to_unix_fd
+      (fd : Lwt_unix.file_descr) : Unix.file_descr =
+  Obj.magic (Obj.field (Obj.repr fd) 0)
+
+let unix_fd_to_fd (fd : Unix.file_descr) : int =
+  Obj.magic fd
+
 let __open_connection ?(ssl_context : [> `Client ] Typed_ssl.t option)
                       socket_address =
   (* Lwt_io.open_connection socket_address *)
@@ -45,8 +56,7 @@ let __open_connection ?(ssl_context : [> `Client ] Typed_ssl.t option)
          else Lwt.return ()
        end
        >>= fun () ->
-       let fd_field = Obj.field (Obj.repr socket) 0 in
-       let (fdi:int) = Obj.magic (fd_field) in
+       let fdi = lwt_unix_fd_to_fd socket in
        let peer_s = a2s peer in
        Logger.info_f_ "__open_connection SUCCEEDED (fd=%i) %s %s" fdi
          (a2s a2) peer_s
diff --git a/src/tools/server.ml b/src/tools/server.ml
index 510964dc..bf280ea1 100644
--- a/src/tools/server.ml
+++ b/src/tools/server.ml
@@ -150,6 +150,7 @@ let make_server_thread
       ?(setup_callback=no_callback)
       ?(teardown_callback = no_callback)
       ?(ssl_context : [> `Server ] Typed_ssl.t option)
+      ~tcp_keepalive_settings
       ~scheme ~stop
       host port protocol =
 
@@ -209,7 +210,7 @@ let make_server_thread
         Lwt.catch
           (fun () ->
            Lwt_unix.accept listening_socket >>= fun (plain_fd, cl_socket_address) ->
-           Lwt_unix.setsockopt listening_socket Unix.SO_KEEPALIVE true;
+           Tcp_keepalive.apply plain_fd tcp_keepalive_settings;
            let cid = name ^ "_" ^ Int64.to_string (connection_counter ()) in
            let finalize () =
              Hashtbl.remove client_threads cid;
diff --git a/src/tools/server_test.ml b/src/tools/server_test.ml
index d818ca64..20142b7d 100644
--- a/src/tools/server_test.ml
+++ b/src/tools/server_test.ml
@@ -46,6 +46,7 @@ let test_echo () =
                  ~setup_callback host port echo_protocol
                  ~teardown_callback
                  ~scheme ~stop
+                 ~tcp_keepalive_settings:Node_cfg.default_tcp_keepalive
   in
   let client () =
     Logger.debug_ "sleeping until server socket started" >>= fun () ->
@@ -99,6 +100,7 @@ let test_max_connections () =
   let server = Server.make_server_thread ~scheme
                  ~setup_callback ~teardown_callback
                  host port echo_protocol2 ~stop
+                 ~tcp_keepalive_settings:Node_cfg.default_tcp_keepalive
   in
   let n_problems = ref 0 in
   let rc_x = ref None in
diff --git a/src/tools/tcp_keepalive.ml b/src/tools/tcp_keepalive.ml
new file mode 100644
index 00000000..7211fd10
--- /dev/null
+++ b/src/tools/tcp_keepalive.ml
@@ -0,0 +1,42 @@
+(*
+Copyright 2015 iNuron NV
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*)
+
+type socket = Unix.file_descr
+
+type t = {
+    (* see http://www.tldp.org/HOWTO/html_single/TCP-Keepalive-HOWTO/#usingkeepalive
+     * for the meaning of these values
+     *)
+    enable_tcp_keepalive : bool;
+    tcp_keepalive_time : int;
+    tcp_keepalive_intvl : int;
+    tcp_keepalive_probes : int;
+  }
+
+external set_tcp_keepalive_time : socket -> int -> unit = "arakoon_tcp_set_tcp_keepalive_time"
+external set_tcp_keepalive_intvl : socket -> int -> unit = "arakoon_tcp_set_tcp_keepalive_intvl"
+external set_tcp_keepalive_probes : socket -> int -> unit = "arakoon_tcp_set_tcp_keepalive_probes"
+
+let apply fd t =
+  if t.enable_tcp_keepalive
+  then
+    begin
+      Lwt_unix.setsockopt fd Unix.SO_KEEPALIVE true;
+      let unix_fd = Network.lwt_unix_fd_to_unix_fd fd in
+      set_tcp_keepalive_time unix_fd t.tcp_keepalive_time;
+      set_tcp_keepalive_intvl unix_fd t.tcp_keepalive_intvl;
+      set_tcp_keepalive_probes unix_fd t.tcp_keepalive_probes;
+    end
diff --git a/src/tools/tcp_keepalive_sockopt_stubs.c b/src/tools/tcp_keepalive_sockopt_stubs.c
new file mode 100644
index 00000000..3afeb725
--- /dev/null
+++ b/src/tools/tcp_keepalive_sockopt_stubs.c
@@ -0,0 +1,51 @@
+/*
+  Copyright 2015 iNuron NV
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+ */
+
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netinet/tcp.h>
+
+#include <caml/alloc.h>
+#include <caml/memory.h>
+#include <caml/mlvalues.h>
+#include <caml/unixsupport.h>
+
+CAMLprim value arakoon_tcp_set_tcp_sock_opt(value fd, value v, int option) {
+  CAMLparam2(fd, v);
+
+  int c_fd = Int_val(fd);
+  int c_v  = Int_val(v);
+
+  int ret = setsockopt(c_fd, SOL_TCP, option, &c_v, sizeof(c_v));
+
+  if(ret == -1) {
+    uerror("arakoon_tcp_set_tcp_sock_opt", Nothing);
+  }
+
+  CAMLreturn(Val_unit);
+}
+
+CAMLprim value arakoon_tcp_set_tcp_keepalive_time(value fd, value v) {
+  return arakoon_tcp_set_tcp_sock_opt(fd, v, TCP_KEEPIDLE);
+}
+
+CAMLprim value arakoon_tcp_set_tcp_keepalive_intvl(value fd, value v) {
+  return arakoon_tcp_set_tcp_sock_opt(fd, v, TCP_KEEPINTVL);
+}
+
+CAMLprim value arakoon_tcp_set_tcp_keepalive_probes(value fd, value v) {
+  return arakoon_tcp_set_tcp_sock_opt(fd, v, TCP_KEEPCNT);
+}

From 4bbd5a8007ba4f376c702a9c74faf14d9b3f61a9 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Mon, 21 Dec 2015 14:15:34 +0100
Subject: [PATCH 061/373] configure tcp keepalive for some of the client
 connections

---
 src/client/arakoon_remote_client_test.ml |  2 +-
 src/client/client_helper.ml              | 20 ++++++++--------
 src/client/client_main.ml                | 23 +++++++++++--------
 src/client/nodestream_main.ml            | 19 +++++++++-------
 src/main/main.ml                         | 25 +++++++++++++++-----
 src/msg/tcp_messaging.ml                 |  4 ++--
 src/msg/tcp_messaging_test.ml            |  2 +-
 src/node/collapser_main.ml               |  4 ++--
 src/node/node_main.ml                    | 15 ++++++------
 src/paxos/multi_paxos_test.ml            |  2 +-
 src/system/single.ml                     | 29 ++++++++++++++++++------
 src/system/startup.ml                    |  3 ++-
 src/tools/server.ml                      |  4 ++--
 src/tools/server_test.ml                 |  4 ++--
 14 files changed, 97 insertions(+), 59 deletions(-)

diff --git a/src/client/arakoon_remote_client_test.ml b/src/client/arakoon_remote_client_test.ml
index 1218bdff..039387fb 100644
--- a/src/client/arakoon_remote_client_test.ml
+++ b/src/client/arakoon_remote_client_test.ml
@@ -63,7 +63,7 @@ let __client_server_wrapper__ (real_test:real_test) =
         (List.filter (fun cfg -> cfg.Node_cfg.node_name = master_name) cluster_cfg.Node_cfg.cfgs)
     in
     Client_main.with_client
-      ~tls:None
+      ~tls:None ~tcp_keepalive:default_tcp_keepalive
       master_cfg
       cluster_cfg.Node_cfg.cluster_id
       real_test
diff --git a/src/client/client_helper.ml b/src/client/client_helper.ml
index dc253a7c..1cef6a14 100644
--- a/src/client/client_helper.ml
+++ b/src/client/client_helper.ml
@@ -35,12 +35,13 @@ let default_create_client_context ~ca_cert ~creds ~protocol =
   ctx
 
 
-let with_connection ~tls sa do_it = match tls with
+let with_connection ~tls ~tcp_keepalive sa do_it = match tls with
   | None -> Lwt_io.with_connection sa do_it
   | Some ctx ->
     let fd = Lwt_unix.socket (Unix.domain_of_sockaddr sa) Unix.SOCK_STREAM 0 in
     Lwt_unix.set_close_on_exec fd;
     Lwt_unix.connect fd sa >>= fun () ->
+    Tcp_keepalive.apply fd tcp_keepalive;
     Typed_ssl.Lwt.ssl_connect fd ctx >>= fun (_, sock) ->
     let ic = Lwt_ssl.in_channel_of_descr sock
     and oc = Lwt_ssl.out_channel_of_descr sock in
@@ -51,7 +52,7 @@ let with_connection ~tls sa do_it = match tls with
 
 exception No_connection
 
-let with_connection' ~tls addrs f =
+let with_connection' ~tls ~tcp_keepalive addrs f =
   let count = List.length addrs in
   let res = Lwt_mvar.create_empty () in
   let err = Lwt_mvar.create None in
@@ -61,7 +62,7 @@ let with_connection' ~tls addrs f =
   let f' addr =
     Lwt.catch
       (fun () ->
-        with_connection ~tls addr (fun c ->
+        with_connection ~tls ~tcp_keepalive addr (fun c ->
           if Lwt_mutex.is_locked l
           then Lwt.return ()
           else
@@ -147,7 +148,7 @@ end
     applicable) exceptions, except {! Lwt.Canceled } which is passed through
     (you most likely don't want to catch that anyway).
 *)
-let find_master' ~tls cluster_cfg =
+let find_master' ~tls ~tcp_keepalive cluster_cfg =
   let open Arakoon_client_config in
 
   let lookup_cfg n =
@@ -186,6 +187,7 @@ let find_master' ~tls cluster_cfg =
                 Lwt.return None);
                with_client'
                  ~tls
+                 ~tcp_keepalive
                  cfg cluster_cfg.cluster_id
                  (fun client ->
                   client # who_master ()); ]
@@ -242,10 +244,10 @@ let find_master' ~tls cluster_cfg =
     In some circumstances, this action could loop forever, so it's mostly
     useful in combination with {! Lwt_unix.with_timeout } or something related.
 *)
-let find_master_loop ~tls cluster_cfg =
+let find_master_loop ~tls ~tcp_keepalive cluster_cfg =
   let open MasterLookupResult in
   let rec loop () =
-    find_master' ~tls cluster_cfg >>= fun r -> match r with
+    find_master' ~tls ~tcp_keepalive cluster_cfg >>= fun r -> match r with
       | No_master
       | Too_many_nodes_down -> begin
           Logger.debug_f_ "Client_main.find_master_loop: %s" (to_string r) >>=
@@ -257,10 +259,10 @@ let find_master_loop ~tls cluster_cfg =
   in
   loop ()
 
-let with_master_client' ~tls cluster_cfg f =
+let with_master_client' ~tls ~tcp_keepalive cluster_cfg f =
   let open MasterLookupResult in
-  find_master' ~tls cluster_cfg >>= function
+  find_master' ~tls ~tcp_keepalive cluster_cfg >>= function
   | Found (master_name, master_cfg) ->
-    with_client' ~tls master_cfg cluster_cfg.Arakoon_client_config.cluster_id f
+    with_client' ~tls ~tcp_keepalive master_cfg cluster_cfg.Arakoon_client_config.cluster_id f
   | master_result ->
     Lwt.fail (Error master_result)
diff --git a/src/client/client_main.ml b/src/client/client_main.ml
index 84991e39..5379cab3 100644
--- a/src/client/client_main.ml
+++ b/src/client/client_main.ml
@@ -24,20 +24,20 @@ let default_create_client_context = default_create_client_context
 let with_connection = with_connection
 let with_connection' = with_connection'
 
-let with_client ~tls node_cfg cluster_id f =
-  with_client' ~tls (node_cfg_to_node_client_cfg node_cfg) cluster_id f
+let with_client ~tls ~tcp_keepalive node_cfg cluster_id f =
+  with_client' ~tls ~tcp_keepalive (node_cfg_to_node_client_cfg node_cfg) cluster_id f
 
-let with_remote_nodestream ~tls node_cfg cluster_id f =
+let with_remote_nodestream ~tls ~tcp_keepalive node_cfg cluster_id f =
   let open Node_cfg in
   let addrs = List.map (fun ip -> make_address ip node_cfg.client_port) node_cfg.ips in
   let do_it _addr connection =
     Remote_nodestream.make_remote_nodestream cluster_id connection >>= f
   in
-  with_connection' ~tls addrs do_it
+  with_connection' ~tls ~tcp_keepalive addrs do_it
 
 let run f = Lwt_extra.run f ; 0
 
-let ping ~tls ip port cluster_id =
+let ping ~tls ~tcp_keepalive ip port cluster_id =
   let do_it connection =
     let t0 = Unix.gettimeofday () in
     Arakoon_remote_client.make_remote_client cluster_id connection
@@ -48,7 +48,7 @@ let ping ~tls ip port cluster_id =
     Lwt_io.printlf "%s\ntook\t%f" s d
   in
   let sa = make_address ip port in
-  let t () = with_connection ~tls sa do_it in
+  let t () = with_connection ~tls ~tcp_keepalive sa do_it in
   run t
 
 
@@ -56,8 +56,9 @@ let ping ~tls ip port cluster_id =
 
 let find_master ~tls cluster_cfg =
   let cluster_cfg' = to_client_cfg cluster_cfg in
+  let tcp_keepalive = cluster_cfg.tcp_keepalive in
   let open MasterLookupResult in
-  find_master' ~tls cluster_cfg' >>= function
+  find_master' ~tls ~tcp_keepalive cluster_cfg' >>= function
     | Found (node_name, _) -> return node_name
     | No_master -> Lwt.fail (Failure "No Master")
     | Too_many_nodes_down -> Lwt.fail (Failure "too many nodes down")
@@ -70,7 +71,7 @@ let with_master_client ~tls cfg_name f =
   let ccfg = read_config cfg_name in
   find_master ~tls ccfg >>= fun master_name ->
   let master_cfg = List.hd (List.filter (fun cfg -> cfg.node_name = master_name) ccfg.cfgs) in
-  with_client ~tls master_cfg ccfg.cluster_id f
+  with_client ~tls ~tcp_keepalive:ccfg.tcp_keepalive master_cfg ccfg.cluster_id f
 
 let set ~tls cfg_name key value =
   let t () = with_master_client ~tls cfg_name (fun client -> client # set key value)
@@ -225,12 +226,13 @@ let _cluster_and_node_cfg node_name' cfg_name =
 let node_state ~tls node_name' cfg_name =
   let open Node_cfg in
   let cluster_cfg,node_cfg = _cluster_and_node_cfg node_name' cfg_name in
+  let tcp_keepalive = cluster_cfg.tcp_keepalive in
   let cluster = cluster_cfg.cluster_id in
   let f client =
     client # current_state () >>= fun state ->
     Lwt_io.printl state
   in
-  let t () = with_client ~tls node_cfg cluster f in
+  let t () = with_client ~tls ~tcp_keepalive node_cfg cluster f in
   run t
 
 
@@ -238,9 +240,10 @@ let node_state ~tls node_name' cfg_name =
 let node_version ~tls node_name' cfg_name =
   let open Node_cfg in
   let cluster_cfg, node_cfg = _cluster_and_node_cfg node_name' cfg_name in
+  let tcp_keepalive = cluster_cfg.tcp_keepalive in
   let cluster = cluster_cfg.cluster_id in
   let t () =
-    with_client ~tls node_cfg cluster
+    with_client ~tls ~tcp_keepalive node_cfg cluster
       (fun client ->
          client # version () >>= fun (major,minor,patch, info) ->
          Lwt_io.printlf "%i.%i.%i" major minor patch >>= fun () ->
diff --git a/src/client/nodestream_main.ml b/src/client/nodestream_main.ml
index 71b0ce8b..f513b3b6 100644
--- a/src/client/nodestream_main.ml
+++ b/src/client/nodestream_main.ml
@@ -17,51 +17,54 @@ open Remote_nodestream
 open Lwt
 open Network
 
-let optimize_db ~tls ip port cluster_id =
+let optimize_db ~tls ~tcp_keepalive ip port cluster_id =
   let do_it connection =
     make_remote_nodestream cluster_id connection >>= fun client ->
     client # optimize_db ()
   in
   let address = make_address ip port in
   let t () =
-    Client_main.with_connection ~tls address do_it  >>= fun () ->
+    Client_main.with_connection
+      ~tls
+      ~tcp_keepalive
+      address do_it  >>= fun () ->
     Lwt.return 0
   in
   Lwt_main.run( t())
 
 
-let defrag_db ~tls ip port cluster_id =
+let defrag_db ~tls ~tcp_keepalive ip port cluster_id =
   let do_it connection =
     make_remote_nodestream cluster_id connection >>= fun client ->
     client # defrag_db ()
   in
   let address = make_address ip port in
   let t () =
-    Client_main.with_connection ~tls address do_it >>= fun () ->
+    Client_main.with_connection ~tls ~tcp_keepalive address do_it >>= fun () ->
     Lwt.return 0
   in
   Lwt_main.run (t ())
 
-let get_db ~tls ip port cluster_id location =
+let get_db ~tls ~tcp_keepalive ip port cluster_id location =
   let do_it connection =
     make_remote_nodestream cluster_id connection >>= fun client ->
     client # get_db location
   in
   let address = make_address ip port in
   let t () =
-    Client_main.with_connection ~tls address do_it  >>= fun () ->
+    Client_main.with_connection ~tls ~tcp_keepalive address do_it  >>= fun () ->
     Lwt.return 0
   in
   Lwt_main.run( t())
 
-let drop_master ~tls ip port cluster_id =
+let drop_master ~tls ~tcp_keepalive ip port cluster_id =
   let do_it connection =
     make_remote_nodestream cluster_id connection >>= fun client ->
     client # drop_master ()
   in
   let address = make_address ip port in
   let t () =
-    Client_main.with_connection ~tls address do_it >>= fun () ->
+    Client_main.with_connection ~tls ~tcp_keepalive address do_it >>= fun () ->
     Lwt.return 0
   in
   Lwt_main.run(t ())
diff --git a/src/main/main.ml b/src/main/main.ml
index 168e5942..49a45e51 100644
--- a/src/main/main.ml
+++ b/src/main/main.ml
@@ -420,6 +420,8 @@ let main () =
   let options = [] in
   let interface = actions @ options in
 
+  let tcp_keepalive = default_tcp_keepalive in
+
   let do_local ~tls = function
     | ShowUsage -> print_endline usage;0
     | RunAllTests -> run_all_tests ()
@@ -454,21 +456,32 @@ let main () =
     | EXPECT_PROGRESS_POSSIBLE -> Client_main.expect_progress_possible ~tls !config_file
     | STATISTICS -> Client_main.statistics ~tls !config_file
     | Collapse_remote -> Collapser_main.collapse_remote
-                           ~tls !ip !port !cluster_id !n_tlogs
-    | Backup_db -> Nodestream_main.get_db ~tls !ip !port !cluster_id !location
-    | Optimize_db -> Nodestream_main.optimize_db ~tls !ip !port !cluster_id
-    | Defrag_db   -> Nodestream_main.defrag_db ~tls !ip !port !cluster_id
+                           ~tls ~tcp_keepalive
+                           !ip !port !cluster_id !n_tlogs
+    | Backup_db -> Nodestream_main.get_db
+                     ~tls ~tcp_keepalive
+                     !ip !port !cluster_id !location
+    | Optimize_db -> Nodestream_main.optimize_db
+                       ~tls ~tcp_keepalive
+                       !ip !port !cluster_id
+    | Defrag_db   -> Nodestream_main.defrag_db
+                       ~tls ~tcp_keepalive
+                       !ip !port !cluster_id
     | NumberOfValues -> Client_main.get_key_count ~tls !config_file ()
     | InitNursery -> Nursery_main.init_nursery !config_file !cluster_id
     | MigrateNurseryRange -> Nursery_main.migrate_nursery_range
                                !config_file !left_cluster !separator !right_cluster
     | DeleteNurseryCluster -> Nursery_main.delete_nursery_cluster !config_file !cluster_id !separator
-    | PING -> Client_main.ping ~tls !ip !port !cluster_id
+    | PING -> Client_main.ping
+                ~tls ~tcp_keepalive
+                !ip !port !cluster_id
     | NODE_VERSION -> Client_main.node_version ~tls !node_id !config_file
     | NODE_STATE   -> Client_main.node_state ~tls !node_id !config_file
     | InjectAsHead -> Dump_store.inject_as_head !filename !node_id !config_file
                         ~force:(!force) ~in_place:(!in_place)
-    | Drop_master -> Nodestream_main.drop_master ~tls !ip !port !cluster_id
+    | Drop_master -> Nodestream_main.drop_master
+                       ~tls ~tcp_keepalive
+                       !ip !port !cluster_id
     | RANGE_ENTRIES ->
        Client_main.range_entries
          ~tls !config_file
diff --git a/src/msg/tcp_messaging.ml b/src/msg/tcp_messaging.ml
index eb964671..4b845364 100644
--- a/src/msg/tcp_messaging.ml
+++ b/src/msg/tcp_messaging.ml
@@ -66,7 +66,7 @@ let run_report connections =
 class tcp_messaging
   ?(timeout=60.0)
   ?(client_ssl_context:[> `Client ] Typed_ssl.t option)
-  ~tcp_keepalive_settings
+  ~tcp_keepalive
   my_addresses my_cookie (drop_it: drop_function)
   max_buffer_size ~stop =
 
@@ -402,7 +402,7 @@ class tcp_messaging
                     ~name ~setup_callback
                     ~teardown_callback ip my_port protocol
                     ~scheme ?ssl_context
-                    ~tcp_keepalive_settings
+                    ~tcp_keepalive
                     ~stop
           in
           s ()
diff --git a/src/msg/tcp_messaging_test.ml b/src/msg/tcp_messaging_test.ml
index 5b625638..06c9d3b9 100644
--- a/src/msg/tcp_messaging_test.ml
+++ b/src/msg/tcp_messaging_test.ml
@@ -97,7 +97,7 @@ let make_transport addresses =
   let tcp_transport =
     new tcp_messaging
         addresses "yummy" (fun _ _ _-> false) Node_cfg.default_max_buffer_size ~stop:(ref false)
-        ~tcp_keepalive_settings:Node_cfg.default_tcp_keepalive
+        ~tcp_keepalive:Node_cfg.default_tcp_keepalive
   in
   (tcp_transport :> messaging)
 
diff --git a/src/node/collapser_main.ml b/src/node/collapser_main.ml
index a4c71b31..d9346bd3 100644
--- a/src/node/collapser_main.ml
+++ b/src/node/collapser_main.ml
@@ -19,7 +19,7 @@ limitations under the License.
 open Lwt
 
 
-let collapse_remote ~tls ip port cluster_id n =
+let collapse_remote ~tls ~tcp_keepalive ip port cluster_id n =
   let t () =
     begin
       if n < 1
@@ -34,7 +34,7 @@ let collapse_remote ~tls ip port cluster_id n =
       Lwt.return 0
     in
     Lwt.catch
-      (fun () -> Client_main.with_connection ~tls address collapse)
+      (fun () -> Client_main.with_connection ~tls ~tcp_keepalive address collapse)
       (fun exn -> Logger.fatal Logger.Section.main ~exn "remote_collapsing_failed"
         >>= fun () -> Lwt.return (-1)
       )
diff --git a/src/node/node_main.ml b/src/node/node_main.ml
index 68fff846..8681385d 100644
--- a/src/node/node_main.ml
+++ b/src/node/node_main.ml
@@ -113,7 +113,7 @@ let _config_messaging
       ?ssl_context me others
       cookie laggy
       lease_period max_buffer_size
-      ~tcp_keepalive_settings
+      ~tcp_keepalive
       ~stop =
   let drop_it = match laggy with
     | true -> let count = ref 0 in
@@ -138,13 +138,13 @@ let _config_messaging
   let messaging =
     new tcp_messaging
         (me.ips, me.messaging_port) cookie drop_it max_buffer_size ~timeout:(lease_period *. 2.5)
-        ~tcp_keepalive_settings
+        ~tcp_keepalive
         ?client_ssl_context ~stop in
   messaging # register_receivers mapping;
   (messaging :> Messaging.messaging)
 
 
-let _config_service ?ssl_context ~tcp_keepalive_settings cfg stop backend =
+let _config_service ?ssl_context ~tcp_keepalive cfg stop backend =
   let port = cfg.client_port in
   let hosts = cfg.ips in
   let max_connections =
@@ -158,7 +158,7 @@ let _config_service ?ssl_context ~tcp_keepalive_settings cfg stop backend =
                       Server.make_server_thread ~name host port
                         (Client_protocol.protocol stop backend)
                         ~scheme ?ssl_context ~stop
-                        ~tcp_keepalive_settings
+                        ~tcp_keepalive
                    )
                    hosts
   in
@@ -513,10 +513,10 @@ let _main_2 (type s)
         end
       in
       Lwt.ignore_result ( upload_cfg_to_keeper () ) ;
-      let tcp_keepalive_settings = cluster_cfg.tcp_keepalive in
+      let tcp_keepalive = cluster_cfg.tcp_keepalive in
       let messaging  = _config_messaging
                          ?ssl_context
-                         ~tcp_keepalive_settings
+                         ~tcp_keepalive
                          me cfgs cookie me.is_laggy
                          (float me.lease_period)
                          cluster_cfg.max_buffer_size
@@ -667,7 +667,7 @@ let _main_2 (type s)
 
           let service = _config_service
                           ?ssl_context:service_ssl_context
-                          ~tcp_keepalive_settings
+                          ~tcp_keepalive
                           me stop backend in
 
           let send, _run, _register, is_alive =
@@ -707,6 +707,7 @@ let _main_2 (type s)
                  let addrs = List.map (fun ip -> Network.make_address ip master_cfg.client_port) master_cfg.ips in
                  Client_main.with_connection'
                    ~tls:ssl_context
+                   ~tcp_keepalive
                    addrs
                    (fun _addr conn ->
                     Arakoon_remote_client.make_remote_client cluster_id conn >>= fun client ->
diff --git a/src/paxos/multi_paxos_test.ml b/src/paxos/multi_paxos_test.ml
index e842d5b0..08b1e7b5 100644
--- a/src/paxos/multi_paxos_test.ml
+++ b/src/paxos/multi_paxos_test.ml
@@ -350,7 +350,7 @@ let build_tcp () =
   let (m : messaging) =
     new Tcp_messaging.tcp_messaging (["127.0.0.1"], 7777) "yummie"
         (fun _ _ _ -> false) Node_cfg.default_max_buffer_size ~stop:(ref false)
-        ~tcp_keepalive_settings:Node_cfg.default_tcp_keepalive
+        ~tcp_keepalive:Node_cfg.default_tcp_keepalive
   in
   let network = network_of_messaging m in
   m # get_buffer, network
diff --git a/src/system/single.ml b/src/system/single.ml
index d90a6644..c89b1d50 100644
--- a/src/system/single.ml
+++ b/src/system/single.ml
@@ -37,7 +37,9 @@ let all_same_master (_tn, cluster_cfg, all_t) =
     let master_cfg = List.hd (List.filter (fun cfg -> cfg.node_name = master_name)
                                 cluster_cfg.cfgs)
     in
-    Client_main.with_client ~tls:None master_cfg cluster_cfg.cluster_id set_one >>= fun () ->
+    Client_main.with_client
+      ~tls:None ~tcp_keepalive:Node_cfg.default_tcp_keepalive
+      master_cfg cluster_cfg.cluster_id set_one >>= fun () ->
     let masters = ref [] in
     let do_one cfg =
       let nn = node_name cfg in
@@ -47,7 +49,9 @@ let all_same_master (_tn, cluster_cfg, all_t) =
           masters := master :: !masters;
           Logger.info_f_ "Client:%s got: %s" nn (Log_extra.string_option2s master)
       in
-      Client_main.with_client ~tls:None cfg cluster_cfg.cluster_id f
+      Client_main.with_client
+        ~tls:None ~tcp_keepalive:Node_cfg.default_tcp_keepalive
+        cfg cluster_cfg.cluster_id f
     in
     let cfgs = cluster_cfg.cfgs in
     Lwt_list.iter_s do_one cfgs >>= fun () ->
@@ -110,7 +114,9 @@ let nothing_on_slave (_tn, cluster_cfg, all_t) =
       with_client delete_on_slave >>= fun () ->
       with_client test_and_set_on_slave
     in
-    f (fun client_action -> Client_main.with_client ~tls:None cfg cluster_id client_action)
+    f (fun client_action -> Client_main.with_client
+                              ~tls:None ~tcp_keepalive:Node_cfg.default_tcp_keepalive
+                              cfg cluster_id client_action)
   in
   let test_slaves ccfg =
     find_slaves cfgs >>= fun slave_cfgs ->
@@ -131,7 +137,9 @@ let dirty_on_slave (_tn, cluster_cfg,_) =
   let master_cfg = List.hd (List.filter (fun cfg -> cfg.node_name = master_name)
                               cluster_cfg.cfgs)
   in
-  Client_main.with_client ~tls:None master_cfg cluster_cfg.cluster_id
+  Client_main.with_client
+    ~tls:None ~tcp_keepalive:Node_cfg.default_tcp_keepalive
+    master_cfg cluster_cfg.cluster_id
     (fun client -> client # set "xxx" "xxx")
   >>= fun () ->
 
@@ -150,7 +158,8 @@ let dirty_on_slave (_tn, cluster_cfg,_) =
         "dirty get should fail with not found"
         "dirty get failed with not found as intended"
     in
-    Client_main.with_client ~tls:None cfg cluster_id dirty_get
+    Client_main.with_client ~tls:None ~tcp_keepalive:Node_cfg.default_tcp_keepalive
+                            cfg cluster_id dirty_get
   in
   let do_slaves ccfg =
     find_slaves cfgs >>= fun slave_cfgs ->
@@ -553,7 +562,10 @@ let find_master ~tls cluster_cfg =
   let timeout = 3 * lp in
   let go () =
     let open Client_helper.MasterLookupResult in
-    Client_helper.find_master_loop ~tls (Node_cfg.Node_cfg.to_client_cfg cluster_cfg) >>= function
+    Client_helper.find_master_loop
+      ~tls
+      ~tcp_keepalive:Node_cfg.default_tcp_keepalive
+      (Node_cfg.Node_cfg.to_client_cfg cluster_cfg) >>= function
       | Found (name, cfg) -> return name
       | No_master -> Lwt.fail (Failure "No Master")
       | Too_many_nodes_down -> Lwt.fail (Failure "too many nodes down")
@@ -572,7 +584,10 @@ let _with_master ((_tn:string), cluster_cfg, _) f =
     List.hd
       (List.filter (fun cfg -> cfg.node_name = master_name) cluster_cfg.cfgs)
   in
-  Client_main.with_client ~tls:None master_cfg cluster_cfg.cluster_id f
+  Client_main.with_client
+    ~tls:None
+    ~tcp_keepalive:Node_cfg.default_tcp_keepalive
+    master_cfg cluster_cfg.cluster_id f
 
 
 let trivial_master tpl =
diff --git a/src/system/startup.ml b/src/system/startup.ml
index 97d09c0d..cad635b6 100644
--- a/src/system/startup.ml
+++ b/src/system/startup.ml
@@ -367,7 +367,8 @@ let interrupted_election () =
    *)
   let get_master_from cfg =
     Client_main.with_client
-      cfg ~tls:None cluster_id
+      cfg ~tls:None ~tcp_keepalive:Node_cfg.default_tcp_keepalive
+      cluster_id
       (fun client -> client # who_master ())
   in
   let rec phase1 () =
diff --git a/src/tools/server.ml b/src/tools/server.ml
index bf280ea1..c89fc90a 100644
--- a/src/tools/server.ml
+++ b/src/tools/server.ml
@@ -150,7 +150,7 @@ let make_server_thread
       ?(setup_callback=no_callback)
       ?(teardown_callback = no_callback)
       ?(ssl_context : [> `Server ] Typed_ssl.t option)
-      ~tcp_keepalive_settings
+      ~tcp_keepalive
       ~scheme ~stop
       host port protocol =
 
@@ -210,7 +210,7 @@ let make_server_thread
         Lwt.catch
           (fun () ->
            Lwt_unix.accept listening_socket >>= fun (plain_fd, cl_socket_address) ->
-           Tcp_keepalive.apply plain_fd tcp_keepalive_settings;
+           Tcp_keepalive.apply plain_fd tcp_keepalive;
            let cid = name ^ "_" ^ Int64.to_string (connection_counter ()) in
            let finalize () =
              Hashtbl.remove client_threads cid;
diff --git a/src/tools/server_test.ml b/src/tools/server_test.ml
index 20142b7d..aa7cccaa 100644
--- a/src/tools/server_test.ml
+++ b/src/tools/server_test.ml
@@ -46,7 +46,7 @@ let test_echo () =
                  ~setup_callback host port echo_protocol
                  ~teardown_callback
                  ~scheme ~stop
-                 ~tcp_keepalive_settings:Node_cfg.default_tcp_keepalive
+                 ~tcp_keepalive:Node_cfg.default_tcp_keepalive
   in
   let client () =
     Logger.debug_ "sleeping until server socket started" >>= fun () ->
@@ -100,7 +100,7 @@ let test_max_connections () =
   let server = Server.make_server_thread ~scheme
                  ~setup_callback ~teardown_callback
                  host port echo_protocol2 ~stop
-                 ~tcp_keepalive_settings:Node_cfg.default_tcp_keepalive
+                 ~tcp_keepalive:Node_cfg.default_tcp_keepalive
   in
   let n_problems = ref 0 in
   let rc_x = ref None in

From c9abf14365b3359672048b7ea97548619c859626 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Mon, 21 Dec 2015 17:45:04 +0100
Subject: [PATCH 062/373] tcp keepalive for the rest of the client connections

---
 src/arakoon_client.mllib      |  1 +
 src/client/client_helper.ml   |  2 +-
 src/msg/tcp_messaging.ml      |  2 +-
 src/node/catchup.ml           | 66 ++++++++---------------------------
 src/node/node_main.ml         |  6 ++--
 src/nursery/nursery.ml        | 13 +++----
 src/nursery/nursery_main.ml   |  2 +-
 src/paxos/multi_paxos.ml      |  4 ++-
 src/paxos/multi_paxos_test.ml |  3 ++
 src/paxos/slave.ml            |  1 +
 src/tools/network.ml          | 17 +++------
 src/tools/tcp_keepalive.ml    |  2 +-
 src/tools/unix_fd.ml          | 26 ++++++++++++++
 13 files changed, 67 insertions(+), 78 deletions(-)
 create mode 100644 src/tools/unix_fd.ml

diff --git a/src/arakoon_client.mllib b/src/arakoon_client.mllib
index eeafd50e..5f16c0e6 100644
--- a/src/arakoon_client.mllib
+++ b/src/arakoon_client.mllib
@@ -31,4 +31,5 @@ Arakoon_client_config
 Lwt_extra
 Network
 Typed_ssl
+Unix_fd
 Tcp_keepalive
diff --git a/src/client/client_helper.ml b/src/client/client_helper.ml
index 1cef6a14..a51c4cfc 100644
--- a/src/client/client_helper.ml
+++ b/src/client/client_helper.ml
@@ -39,9 +39,9 @@ let with_connection ~tls ~tcp_keepalive sa do_it = match tls with
   | None -> Lwt_io.with_connection sa do_it
   | Some ctx ->
     let fd = Lwt_unix.socket (Unix.domain_of_sockaddr sa) Unix.SOCK_STREAM 0 in
+    Tcp_keepalive.apply fd tcp_keepalive;
     Lwt_unix.set_close_on_exec fd;
     Lwt_unix.connect fd sa >>= fun () ->
-    Tcp_keepalive.apply fd tcp_keepalive;
     Typed_ssl.Lwt.ssl_connect fd ctx >>= fun (_, sock) ->
     let ic = Lwt_ssl.in_channel_of_descr sock
     and oc = Lwt_ssl.out_channel_of_descr sock in
diff --git a/src/msg/tcp_messaging.ml b/src/msg/tcp_messaging.ml
index 4b845364..89820eac 100644
--- a/src/msg/tcp_messaging.ml
+++ b/src/msg/tcp_messaging.ml
@@ -151,7 +151,7 @@ class tcp_messaging
       Logger.info_f_ "establishing connection to (%s,%i) (timeout = %f)" host_ip port timeout
       >>= fun () ->
       Lwt_unix.with_timeout timeout
-        (fun () ->__open_connection ?ssl_context socket_address)
+        (fun () -> __open_connection ?ssl_context ~tcp_keepalive socket_address)
       >>= fun (socket, ic, oc) ->
       Llio.output_int64 oc _MAGIC >>= fun () ->
       Llio.output_int oc _VERSION >>= fun () ->
diff --git a/src/node/catchup.ml b/src/node/catchup.ml
index e5344fa6..65499c2f 100644
--- a/src/node/catchup.ml
+++ b/src/node/catchup.ml
@@ -26,54 +26,12 @@ open Tlogcommon
 exception StoreAheadOfTlogs of (Int64.t * Sn.t)
 exception StoreCounterTooLow of string
 
-let with_connection ~tls_ctx address f = match tls_ctx with
-  | None -> Lwt_io.with_connection address f
-  | Some ctx ->
-      let fd = Lwt_unix.socket (Unix.domain_of_sockaddr address) Unix.SOCK_STREAM 0 in
-      Lwt_unix.set_close_on_exec fd;
-      Lwt_unix.connect fd address >>= fun () ->
-      Typed_ssl.Lwt.ssl_connect fd ctx >>= fun (_, sock) ->
-      let ic = Lwt_ssl.in_channel_of_descr sock
-      and oc = Lwt_ssl.out_channel_of_descr sock in
-      Lwt.catch
-        (fun () ->
-          f (ic, oc) >>= fun r ->
-          Lwt_ssl.close sock >>= fun () ->
-          Lwt.return r)
-        (fun exc ->
-          Lwt.catch
-            (fun () ->
-              Lwt_ssl.close sock)
-            (fun _ -> Lwt.return ()) >>= fun () ->
-          Lwt.fail exc)
-
-let _with_client_connection ~tls_ctx (ips,port) f =
-  let sl2s ss = list2s (fun s -> s) ss in
-  let rec loop = function
-    | [] -> Llio.lwt_failfmt "None of the ips: %s can be reached" (sl2s ips)
-    | ip :: rest ->
-      Lwt.catch
-        (fun () ->
-           let address = Network.make_address ip port in
-           with_connection ~tls_ctx address f)
-        (fun exn ->
-           Logger.info_f_ ~exn "ip = %s " ip >>= fun () ->
-           match exn with
-             | Unix.Unix_error(error,_,_) ->
-               begin
-                 match error with
-                   | Unix.ECONNREFUSED
-                   | Unix.ENETUNREACH
-                   | Unix.EHOSTUNREACH
-                   | Unix.EHOSTDOWN ->
-                     loop rest
-                   | _ -> Lwt.fail exn
-               end
-             | Lwt_unix.Timeout ->
-               loop rest
-             | _ -> Lwt.fail exn)
-  in
-  loop ips
+let _with_client_connection ~tls_ctx ~tcp_keepalive (ips,port) f =
+  Client_helper.with_connection'
+    ~tls:tls_ctx
+    ~tcp_keepalive
+    (List.map (fun ip -> Network.make_address ip port) ips)
+    (fun _ -> f)
 
 
 let head_saved_epilogue hfn tlog_coll =
@@ -105,7 +63,7 @@ let stop_fuse stop =
   else
     Lwt.return ()
 
-let catchup_tlog (type s) ~tls_ctx ~stop other_configs ~cluster_id  mr_name ((module S : Store.STORE with type t = s),store,tlog_coll)
+let catchup_tlog (type s) ~tls_ctx ~tcp_keepalive ~stop other_configs ~cluster_id  mr_name ((module S : Store.STORE with type t = s),store,tlog_coll)
   =
   let current_i = tlog_coll # get_last_i () in
   Logger.info_f_ "catchup_tlog %s" (Sn.string_of current_i) >>= fun () ->
@@ -138,7 +96,7 @@ let catchup_tlog (type s) ~tls_ctx ~stop other_configs ~cluster_id  mr_name ((mo
 
   Lwt.catch
     (fun () ->
-      _with_client_connection  ~tls_ctx mr_addresses copy_tlog >>= fun () ->
+      _with_client_connection  ~tls_ctx ~tcp_keepalive mr_addresses copy_tlog >>= fun () ->
       Logger.info_f_ "catchup_tlog completed"
     )
     (fun exn -> Logger.warning_ ~exn "catchup_tlog failed")
@@ -273,11 +231,15 @@ let catchup_store ~stop (type s) me
       end
     end
 
-let catchup ~tls_ctx ~stop me other_configs ~cluster_id ((_,_,tlog_coll) as dbt) mr_name =
+let catchup
+      ~tls_ctx ~tcp_keepalive
+      ~stop me other_configs ~cluster_id ((_,_,tlog_coll) as dbt) mr_name =
   Logger.info_f_ "CATCHUP start: I'm @ %s and %s is more recent"
     (Sn.string_of (tlog_coll # get_last_i ())) mr_name
   >>= fun () ->
-  catchup_tlog ~tls_ctx ~stop other_configs ~cluster_id mr_name dbt >>= fun until_i ->
+  catchup_tlog
+    ~tls_ctx ~tcp_keepalive
+    ~stop other_configs ~cluster_id mr_name dbt >>= fun until_i ->
   Logger.info_f_ "CATCHUP phase 1 done (until_i = %s); now the store"
     (Sn.string_of until_i) >>= fun () ->
   catchup_store ~stop me dbt (Sn.succ until_i) >>= fun () ->
diff --git a/src/node/node_main.ml b/src/node/node_main.ml
index 8681385d..0fba15fe 100644
--- a/src/node/node_main.ml
+++ b/src/node/node_main.ml
@@ -225,7 +225,7 @@ let only_catchup
   let try_catchup mr_name =
     Lwt.catch
       (fun () ->
-       Catchup.catchup ~tls_ctx ~stop:(ref false)
+       Catchup.catchup ~tls_ctx ~tcp_keepalive:cluster_cfg.tcp_keepalive ~stop:(ref false)
                        me.Node_cfg.Node_cfg.node_name other_configs ~cluster_id
                        ((module S),store,tlc) mr_name >>= fun _ ->
        S.close store ~flush:true ~sync:true >>= fun () ->
@@ -825,7 +825,9 @@ let _main_2 (type s)
                   else None
           in
           let constants =
-            Multi_paxos.make ~catchup_tls_ctx:catchup_tls_ctx my_name
+            Multi_paxos.make
+              ~catchup_tls_ctx:catchup_tls_ctx my_name
+              ~tcp_keepalive
               me.is_learner
               other_names send
               get_last_value
diff --git a/src/nursery/nursery.ml b/src/nursery/nursery.ml
index 7e8c40a8..51c6e78e 100644
--- a/src/nursery/nursery.ml
+++ b/src/nursery/nursery.ml
@@ -27,11 +27,11 @@ let section = Logger.Section.main
 
 let so2s = Log_extra.string_option2s
 
-let try_connect (ips, port) =
+let try_connect ~tcp_keepalive (ips, port) =
   Lwt.catch
     (fun () ->
        let sa = Network.make_address ips port in
-       Network.__open_connection sa >>= fun (_, ic, oc) ->
+       Network.__open_connection ~tcp_keepalive sa >>= fun (_, ic, oc) ->
        let r = Some (ic,oc) in
        Lwt.return r
     )
@@ -50,12 +50,13 @@ module NC = struct
 
   type t = {
     rc : NCFG.t;
+    tcp_keepalive : Tcp_keepalive.t;
     keeper_cn: string;
     connections : (nn ,lc) Hashtbl.t;
     masters: (string, string option) Hashtbl.t;
   }
 
-  let make rc keeper_cn =
+  let make rc keeper_cn tcp_keepalive =
     let masters = Hashtbl.create 5 in
     let () = NCFG.iter_cfgs rc (fun k _ -> Hashtbl.add masters k None) in
     let connections = Hashtbl.create 13 in
@@ -66,7 +67,7 @@ module NC = struct
                       let a = Address (ip,port) in
                       Hashtbl.add connections nn a) v)
     in
-    {rc; connections;masters;keeper_cn}
+    { rc; connections; masters; keeper_cn; tcp_keepalive; }
 
   let _get_connection t nn =
     let (cn,_node) = nn in
@@ -74,7 +75,7 @@ module NC = struct
       | Address (ips,port) ->
         begin
           let ip = List.hd ips in
-          try_connect (ip,port) >>= function
+          try_connect ~tcp_keepalive:t.tcp_keepalive (ip,port) >>= function
           | Some conn ->
             Protocol_common.prologue cn conn >>= fun () ->
             let () = Hashtbl.add t.connections nn (Connection conn) in
@@ -435,7 +436,7 @@ let nursery_test_main () =
   let () = NCFG.add_cluster nursery_cfg "left" left_cfg in
   let () = NCFG.add_cluster nursery_cfg "right" right_cfg in
   let keeper = "left" in
-  let nc = NC.make nursery_cfg keeper in
+  let nc = NC.make nursery_cfg keeper Node_cfg.default_tcp_keepalive in
   (*
   let test k v =
     NC.set client k v >>= fun () ->
diff --git a/src/nursery/nursery_main.ml b/src/nursery/nursery_main.ml
index afc2cbf5..fb84cf8a 100644
--- a/src/nursery/nursery_main.ml
+++ b/src/nursery/nursery_main.ml
@@ -90,7 +90,7 @@ let get_keeper_config config =
 let get_nursery_client keeper_id cli_cfg =
   let get_nc client =
     client # get_nursery_cfg () >>= fun ncfg ->
-    Lwt.return ( Nursery.NC.make ncfg keeper_id )
+    Lwt.return ( Nursery.NC.make ncfg keeper_id default_tcp_keepalive)
   in
   with_master_remote_stream keeper_id cli_cfg get_nc
 
diff --git a/src/paxos/multi_paxos.ml b/src/paxos/multi_paxos.ml
index c7a91a42..11bfda8c 100644
--- a/src/paxos/multi_paxos.ml
+++ b/src/paxos/multi_paxos.ml
@@ -127,6 +127,7 @@ type 'a constants =
    is_learner: bool;
    stop : bool ref;
    catchup_tls_ctx : [ `Client | `Server ] Typed_ssl.t option;
+   tcp_keepalive : Tcp_keepalive.t;
    mutable election_timeout : (Sn.t * Sn.t * float) option;
    mutable lease_expiration_id : int;
    mutable respect_run_master : (string * float) option;
@@ -142,7 +143,7 @@ let is_election constants =
     | Elected | Preferred _ -> true
     | ReadOnly | Forced _ -> false
 
-let make (type s) ~catchup_tls_ctx me is_learner others send get_value
+let make (type s) ~catchup_tls_ctx ~tcp_keepalive me is_learner others send get_value
       on_accept on_consensus on_witness
       last_witnessed quorum_function (master:master) (module S : Store.STORE with type t = s) store tlog_coll
       other_cfgs lease_expiration inject_event is_alive ~cluster_id
@@ -169,6 +170,7 @@ let make (type s) ~catchup_tls_ctx me is_learner others send get_value
     cluster_id;
     stop;
     catchup_tls_ctx;
+    tcp_keepalive;
     election_timeout = None;
     lease_expiration_id = 0;
     respect_run_master = None;
diff --git a/src/paxos/multi_paxos_test.ml b/src/paxos/multi_paxos_test.ml
index 08b1e7b5..8f5d5e01 100644
--- a/src/paxos/multi_paxos_test.ml
+++ b/src/paxos/multi_paxos_test.ml
@@ -86,6 +86,7 @@ let test_generic network_factory n_nodes () =
               lease_expiration_id = 0;
               respect_run_master = None;
               catchup_tls_ctx = None;
+              tcp_keepalive = Node_cfg.default_tcp_keepalive;
              }
   in
   let all_happy = build_names (n_nodes -1) in
@@ -283,6 +284,7 @@ let test_master_loop network_factory ()  =
                    lease_expiration_id = 0;
                    respect_run_master = None;
                    catchup_tls_ctx = None;
+                   tcp_keepalive = Node_cfg.default_tcp_keepalive;
                   } in
   let continue = ref 2 in
   let c0_t () =
@@ -422,6 +424,7 @@ let test_simulation filters () =
     lease_expiration_id = 0;
     respect_run_master = None;
     catchup_tls_ctx = None;
+    tcp_keepalive = Node_cfg.default_tcp_keepalive;
   } in
   let c0_t () =
     let expected prev_key key =
diff --git a/src/paxos/slave.ml b/src/paxos/slave.ml
index d4f6b878..370508e7 100644
--- a/src/paxos/slave.ml
+++ b/src/paxos/slave.ml
@@ -290,6 +290,7 @@ let slave_discovered_other_master (type s) constants state () =
         | Some (_, ls) -> ls +. (float_of_int constants.lease_expiration) <= Unix.gettimeofday () in
       Catchup.catchup
         ~tls_ctx
+        ~tcp_keepalive:constants.tcp_keepalive
         ~stop:constants.stop
         me other_cfgs ~cluster_id ((module S), store, tlog_coll) master >>= fun () ->
 
diff --git a/src/tools/network.ml b/src/tools/network.ml
index 25d4c76f..16acdf6e 100644
--- a/src/tools/network.ml
+++ b/src/tools/network.ml
@@ -28,25 +28,16 @@ let a2s = function
   | Unix.ADDR_INET (sa,p) -> Printf.sprintf "(%s,%i)" (Unix.string_of_inet_addr sa) p
   | Unix.ADDR_UNIX s      -> Printf.sprintf "ADDR_UNIX(%s)" s
 
-let lwt_unix_fd_to_fd
-      (fd : Lwt_unix.file_descr) : int =
-  Obj.magic (Obj.field (Obj.repr fd) 0)
-
-let lwt_unix_fd_to_unix_fd
-      (fd : Lwt_unix.file_descr) : Unix.file_descr =
-  Obj.magic (Obj.field (Obj.repr fd) 0)
-
-let unix_fd_to_fd (fd : Unix.file_descr) : int =
-  Obj.magic fd
 
 let __open_connection ?(ssl_context : [> `Client ] Typed_ssl.t option)
+                      ~tcp_keepalive
                       socket_address =
-  (* Lwt_io.open_connection socket_address *)
   let domain = Unix.domain_of_sockaddr socket_address in
   let socket = Lwt_unix.socket domain Unix.SOCK_STREAM 0  in
-  let () = Lwt_unix.setsockopt socket Lwt_unix.TCP_NODELAY true in
   Lwt.catch
     (fun () ->
+       let () = Lwt_unix.setsockopt socket Lwt_unix.TCP_NODELAY true in
+       let () = Tcp_keepalive.apply socket tcp_keepalive in
        Lwt_unix.connect socket socket_address >>= fun () ->
        let a2 = Lwt_unix.getsockname socket in
        let peer = Lwt_unix.getpeername socket in
@@ -56,7 +47,7 @@ let __open_connection ?(ssl_context : [> `Client ] Typed_ssl.t option)
          else Lwt.return ()
        end
        >>= fun () ->
-       let fdi = lwt_unix_fd_to_fd socket in
+       let fdi = Unix_fd.lwt_unix_fd_to_fd socket in
        let peer_s = a2s peer in
        Logger.info_f_ "__open_connection SUCCEEDED (fd=%i) %s %s" fdi
          (a2s a2) peer_s
diff --git a/src/tools/tcp_keepalive.ml b/src/tools/tcp_keepalive.ml
index 7211fd10..de35616e 100644
--- a/src/tools/tcp_keepalive.ml
+++ b/src/tools/tcp_keepalive.ml
@@ -35,7 +35,7 @@ let apply fd t =
   then
     begin
       Lwt_unix.setsockopt fd Unix.SO_KEEPALIVE true;
-      let unix_fd = Network.lwt_unix_fd_to_unix_fd fd in
+      let unix_fd = Unix_fd.lwt_unix_fd_to_unix_fd fd in
       set_tcp_keepalive_time unix_fd t.tcp_keepalive_time;
       set_tcp_keepalive_intvl unix_fd t.tcp_keepalive_intvl;
       set_tcp_keepalive_probes unix_fd t.tcp_keepalive_probes;
diff --git a/src/tools/unix_fd.ml b/src/tools/unix_fd.ml
new file mode 100644
index 00000000..720553c9
--- /dev/null
+++ b/src/tools/unix_fd.ml
@@ -0,0 +1,26 @@
+(*
+Copyright 2015 iNuron NV
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*)
+
+let lwt_unix_fd_to_fd
+      (fd : Lwt_unix.file_descr) : int =
+  Obj.magic (Obj.field (Obj.repr fd) 0)
+
+let lwt_unix_fd_to_unix_fd
+      (fd : Lwt_unix.file_descr) : Unix.file_descr =
+  Obj.magic (Obj.field (Obj.repr fd) 0)
+
+let unix_fd_to_fd (fd : Unix.file_descr) : int =
+  Obj.magic fd

From 877e30e283cadb1bb5f39e6246c8b3b4e5e925d5 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Mon, 21 Dec 2015 17:57:14 +0100
Subject: [PATCH 063/373] fix file descriptor leaks

---
 src/msg/tcp_messaging.ml | 18 ++++++++++++------
 src/nursery/nursery.ml   | 13 ++++++++++---
 2 files changed, 22 insertions(+), 9 deletions(-)

diff --git a/src/msg/tcp_messaging.ml b/src/msg/tcp_messaging.ml
index 89820eac..ba65d99c 100644
--- a/src/msg/tcp_messaging.ml
+++ b/src/msg/tcp_messaging.ml
@@ -153,14 +153,20 @@ class tcp_messaging
       Lwt_unix.with_timeout timeout
         (fun () -> __open_connection ?ssl_context ~tcp_keepalive socket_address)
       >>= fun (socket, ic, oc) ->
-      Llio.output_int64 oc _MAGIC >>= fun () ->
-      Llio.output_int oc _VERSION >>= fun () ->
-      Llio.output_string oc my_cookie >>= fun () ->
-      Llio.output_string oc my_ip >>= fun () ->
-      Llio.output_int oc my_port  >>= fun () ->
-      Lwt.return (socket, ic, oc)
     (* open_connection can also fail with Unix.Unix_error (63, "connect",_)
        on local host *)
+      Lwt.catch
+        (fun () ->
+         Llio.output_int64 oc _MAGIC >>= fun () ->
+         Llio.output_int oc _VERSION >>= fun () ->
+         Llio.output_string oc my_cookie >>= fun () ->
+         Llio.output_string oc my_ip >>= fun () ->
+         Llio.output_int oc my_port  >>= fun () ->
+         Lwt.return (socket, ic, oc))
+        (fun exn ->
+         Lwt_io.close ic >>= fun () ->
+         Lwt_io.close oc >>= fun () ->
+         Lwt.fail exn)
 
     method private _get_connection (addresses:RR.t) =
       let (address:address) = RR.current addresses in
diff --git a/src/nursery/nursery.ml b/src/nursery/nursery.ml
index 51c6e78e..36446e2e 100644
--- a/src/nursery/nursery.ml
+++ b/src/nursery/nursery.ml
@@ -77,9 +77,16 @@ module NC = struct
           let ip = List.hd ips in
           try_connect ~tcp_keepalive:t.tcp_keepalive (ip,port) >>= function
           | Some conn ->
-            Protocol_common.prologue cn conn >>= fun () ->
-            let () = Hashtbl.add t.connections nn (Connection conn) in
-            Lwt.return conn
+             Lwt.catch
+               (fun () ->
+                Protocol_common.prologue cn conn >>= fun () ->
+                let () = Hashtbl.add t.connections nn (Connection conn) in
+                Lwt.return conn)
+               (fun exn ->
+                let ic, oc = conn in
+                Lwt_io.close ic >>= fun () ->
+                Lwt_io.close oc >>= fun () ->
+                Lwt.fail exn)
           | None -> Llio.lwt_failfmt "Connection to (%s,%i) failed" ip port
         end
       | Connection conn -> Lwt.return conn

From eae0e7b112393193dc54931c951a974d3c73d4fd Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Tue, 22 Dec 2015 10:29:45 +0100
Subject: [PATCH 064/373] expose Unix_fd & Tcp_keepalive in client lib

---
 META     | 4 ++--
 Makefile | 3 +++
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/META b/META
index f3fb6116..8276a53d 100644
--- a/META
+++ b/META
@@ -4,6 +4,6 @@ description = "Arakoon client"
 version = "1.8.11"
 exists_if = "arakoon_client.cma,arakoon_client.cmxa,arakoon_client.cmxs"
 requires = "threads lwt lwt.unix str oUnit quickcheck lwt.ssl ocplib-endian nocrypto cstruct"
-archive(byte) = "arakoon_client.cma"
-archive(native) = "arakoon_client.cmxa"
+archive(byte) = "arakoon_client.cma tcp_keepalive_sockopt_stubs.o"
+archive(native) = "arakoon_client.cmxa tcp_keepalive_sockopt_stubs.o"
 
diff --git a/Makefile b/Makefile
index bd5eef06..e077dba2 100644
--- a/Makefile
+++ b/Makefile
@@ -63,6 +63,9 @@ install_client:
           _build/src/tools/lwt_extra.cmi \
           _build/src/tools/typed_ssl.mli \
           _build/src/tools/typed_ssl.cmi \
+          _build/src/tools/unix_fd.cmi \
+          _build/src/tools/tcp_keepalive_sockopt_stubs.o \
+          _build/src/tools/tcp_keepalive.cmi \
           _build/src/arakoon_client.a
 
 uninstall_client:

From c53406da92d17bb037703e9d788865fc3531b209 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Tue, 22 Dec 2015 17:28:50 +0100
Subject: [PATCH 065/373] prune a few dependencies, update others

---
 META                              |  2 +-
 _tags                             |  2 --
 docker/arakoon_centos7/Dockerfile | 23 ++++++-----------------
 docker/arakoon_debian/Dockerfile  | 23 ++++++-----------------
 myocamlbuild.ml                   |  9 ---------
 travis.sh                         |  2 +-
 6 files changed, 14 insertions(+), 47 deletions(-)

diff --git a/META b/META
index 8276a53d..ddf620d8 100644
--- a/META
+++ b/META
@@ -3,7 +3,7 @@
 description = "Arakoon client"
 version = "1.8.11"
 exists_if = "arakoon_client.cma,arakoon_client.cmxa,arakoon_client.cmxs"
-requires = "threads lwt lwt.unix str oUnit quickcheck lwt.ssl ocplib-endian nocrypto cstruct"
+requires = "threads lwt lwt.unix str oUnit quickcheck lwt.ssl ocplib-endian"
 archive(byte) = "arakoon_client.cma tcp_keepalive_sockopt_stubs.o"
 archive(native) = "arakoon_client.cmxa tcp_keepalive_sockopt_stubs.o"
 
diff --git a/_tags b/_tags
index 18015af9..42545d8a 100644
--- a/_tags
+++ b/_tags
@@ -14,8 +14,6 @@ true: package(quickcheck)
 true: pp(camlp4of)
 true: package(snappy)
 true: package(ocplib-endian)
-true: package(uuidm)
-true: package(nocrypto)
 "src":include
 "src/lib":include
 "src/tools":include
diff --git a/docker/arakoon_centos7/Dockerfile b/docker/arakoon_centos7/Dockerfile
index f3884fa2..3b9c1ca7 100644
--- a/docker/arakoon_centos7/Dockerfile
+++ b/docker/arakoon_centos7/Dockerfile
@@ -45,30 +45,19 @@ RUN mv cudf_remote_proxy /usr/local/bin/
 
 ENV OPAMEXTERNALSOLVER="cudf_remote_proxy %{input}% %{output}% %{criteria}%"
 
-RUN eval `${opam_env}` && opam install -y \
+RUN eval `${opam_env}` && \
+    opam update && \
+    opam install -y \
         ocamlfind \
-        ssl \
+        ssl.0.5.2 \
         camlbz2 \
         snappy \
-        sexplib \
         bisect \
-        lwt \
+        lwt.2.5.1 \
         camltc \
-        cstruct \
-        ctypes \
-        ctypes-foreign \
-        uuidm \
-        zarith \
-        mirage-no-xen.1 \
         quickcheck \
-        nocrypto \
-        cmdliner \
         conf-libev \
-        depext \
-        kinetic-client \
-        tiny_json \
-        ppx_deriving_yojson \
-        core_kernel
+        depext
 
 
 RUN yum -y install autoconf help2man perl-Thread-Queue
diff --git a/docker/arakoon_debian/Dockerfile b/docker/arakoon_debian/Dockerfile
index e3bfaca2..08d4859e 100644
--- a/docker/arakoon_debian/Dockerfile
+++ b/docker/arakoon_debian/Dockerfile
@@ -51,30 +51,19 @@ RUN mkdir /home/opam
 ENV opam_env='opam config env --root=/home/opam/OPAM'
 RUN opam init --root=/home/opam/OPAM --comp 4.02.3
 
-RUN eval `${opam_env}` && opam install -y \
+RUN eval `${opam_env}` && \
+    opam update && \
+    opam install -y \
         ocamlfind \
-        ssl \
+        ssl.0.5.2 \
         camlbz2 \
         snappy \
-        sexplib \
         bisect \
-        lwt \
+        lwt.2.5.1 \
         camltc \
-        cstruct \
-        ctypes \
-        ctypes-foreign \
-        uuidm \
-        zarith.1.3 \
-        mirage-no-xen.1 \
         quickcheck \
-        nocrypto \
-        cmdliner \
         conf-libev \
-        depext \
-        kinetic-client \
-        tiny_json \
-        ppx_deriving_yojson \
-        core_kernel
+        depext
 
 RUN echo "jenkins ALL=NOPASSWD: ALL" >/etc/sudoers.d/jenkins
 ENV LANG=C
diff --git a/myocamlbuild.ml b/myocamlbuild.ml
index cf09bd83..0a928aff 100644
--- a/myocamlbuild.ml
+++ b/myocamlbuild.ml
@@ -6,12 +6,7 @@ let dependencies = [ "lwt";
                      "ssl";
                      "bz2";
                      "quickcheck";
-                     "nocrypto";
-                     "zarith";
-                     "cstruct";
-                     "ctypes";
                      "ocplib-endian";
-                     "uuidm";
                    ]
 
 (* Enabled compiler warnings, argument for '-w', see `man ocamlc` *)
@@ -139,10 +134,6 @@ let _ = dispatch & function
         S[A"-linkpkg"; A"src/libcutil.a";
          ]);
 
-      flag ["ocaml";"link";"is_main"](
-        S[A"-package"; A"nocrypto";]
-      );
-
       flag ["ocaml";"byte";"link"] (S[A"-custom";]);
 
       flag ["ocaml";"compile";"warn_error"]
diff --git a/travis.sh b/travis.sh
index 73c0e5eb..c6430306 100755
--- a/travis.sh
+++ b/travis.sh
@@ -2,7 +2,7 @@
 
 APT_DEPENDS="libev-dev libssl-dev libsnappy-dev libgmp3-dev"
 APT_OCAML_DEPENDS="ocaml ocaml-native-compilers camlp4-extra opam"
-OPAM_DEPENDS="ssl.0.5.0 conf-libev lwt.2.5.0 camlbz2 camltc.999 snappy quickcheck ocplib-endian uuidm nocrypto.0.5.1 ctypes.0.4.1 ctypes-foreign.0.4.0"
+OPAM_DEPENDS="ssl.0.5.2 conf-libev lwt.2.5.1 camlbz2 camltc.999 snappy quickcheck ocplib-endian"
 
 export OPAMYES=1
 export OPAMVERBOSE=1

From 7ce4cd2bfda14e614cc686a665801d42f96c128d Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Wed, 23 Dec 2015 10:09:44 +0100
Subject: [PATCH 066/373] make_release: 1.8.12

---
 META                      | 2 +-
 debian/changelog          | 5 +++++
 redhat/SPECS/arakoon.spec | 4 +++-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/META b/META
index ddf620d8..58b87837 100644
--- a/META
+++ b/META
@@ -1,7 +1,7 @@
 # -*- conf -*-
 
 description = "Arakoon client"
-version = "1.8.11"
+version = "1.8.12"
 exists_if = "arakoon_client.cma,arakoon_client.cmxa,arakoon_client.cmxs"
 requires = "threads lwt lwt.unix str oUnit quickcheck lwt.ssl ocplib-endian"
 archive(byte) = "arakoon_client.cma tcp_keepalive_sockopt_stubs.o"
diff --git a/debian/changelog b/debian/changelog
index 51fd4172..bcbc8975 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,8 @@
+arakoon (1.8.12) unstable; urgency=low
+  * new upstream version
+
+ -- Jan Doms <jan.doms@gmail.com>  Wed, 23 Dec 2015 10:09:44 +0100
+
 arakoon (1.8.11) unstable; urgency=low
   * new upstream version
 
diff --git a/redhat/SPECS/arakoon.spec b/redhat/SPECS/arakoon.spec
index 0db7d05b..e0169922 100644
--- a/redhat/SPECS/arakoon.spec
+++ b/redhat/SPECS/arakoon.spec
@@ -2,7 +2,7 @@
 
 Summary: Arakoon
 Name: arakoon
-Version: 1.8.11
+Version: 1.8.12
 Release: 3%{?dist}
 License: Apache 2.0
 Requires: libev >= 4
@@ -33,6 +33,8 @@ cp arakoon.native %{buildroot}%{_bindir}/arakoon
 %{_bindir}/arakoon
 
 %changelog
+* Wed Dec 23 2015 Jan Doms <jan.doms@gmail.com> - 1.8.12
+- Create arakoon 1.8.12 RPM package
 * Fri Dec 11 2015 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.8.11
 - Create arakoon 1.8.11 RPM package
 * Mon Nov 02 2015 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.8.10

From 9bec05f5b73608569ae5c5e42e62a6e43e76704b Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Wed, 23 Dec 2015 10:42:03 +0100
Subject: [PATCH 067/373] need ocplib-endian in dockerfile

---
 docker/arakoon_centos7/Dockerfile | 1 +
 docker/arakoon_debian/Dockerfile  | 1 +
 2 files changed, 2 insertions(+)

diff --git a/docker/arakoon_centos7/Dockerfile b/docker/arakoon_centos7/Dockerfile
index 3b9c1ca7..207621b1 100644
--- a/docker/arakoon_centos7/Dockerfile
+++ b/docker/arakoon_centos7/Dockerfile
@@ -57,6 +57,7 @@ RUN eval `${opam_env}` && \
         camltc \
         quickcheck \
         conf-libev \
+        ocplib-endian \
         depext
 
 
diff --git a/docker/arakoon_debian/Dockerfile b/docker/arakoon_debian/Dockerfile
index 08d4859e..b1609e10 100644
--- a/docker/arakoon_debian/Dockerfile
+++ b/docker/arakoon_debian/Dockerfile
@@ -63,6 +63,7 @@ RUN eval `${opam_env}` && \
         camltc \
         quickcheck \
         conf-libev \
+        ocplib-endian \
         depext
 
 RUN echo "jenkins ALL=NOPASSWD: ALL" >/etc/sudoers.d/jenkins

From b56e62a1eaa8e8a146cde6b4382ed51537656a93 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Wed, 30 Dec 2015 11:36:03 +0100
Subject: [PATCH 068/373] bugfixes for local collapse - don't throw away too
 many tlfs/tlogs - take "entries per tlog" into account

---
 src/node/collapser.ml          | 20 +++++++++++++++-----
 src/tlog/mem_tlogcollection.ml |  2 --
 src/tlog/tlc2.ml               | 11 -----------
 src/tlog/tlogcollection.ml     |  1 -
 4 files changed, 15 insertions(+), 19 deletions(-)

diff --git a/src/node/collapser.ml b/src/node/collapser.ml
index 28f5d30a..6dad59a8 100644
--- a/src/node/collapser.ml
+++ b/src/node/collapser.ml
@@ -213,11 +213,15 @@ let collapse_many
   tlog_coll # get_tlog_count () >>= fun total_tlogs ->
   Logger.debug_f_ "total_tlogs = %i; tlogs_to_keep=%i" total_tlogs tlogs_to_keep >>= fun () ->
   let (_,(head_location:string), _) = store_fs in
-  _head_i (module S) head_location >>= fun head_io ->
   let last_i = tlog_coll # get_last_i () in
-  Logger.debug_f_ "head @ %s : last_i %s " (Log_extra.option2s Sn.string_of head_io) (Sn.string_of last_i)
-  >>= fun () ->
-  let head_i = match head_io with None -> Sn.start | Some i -> i in
+  let get_head_i () =
+    _head_i (module S) head_location >>= fun head_io ->
+    Logger.debug_f_ "head @ %s : last_i %s " (Log_extra.option2s Sn.string_of head_io) (Sn.string_of last_i)
+    >>= fun () ->
+    let head_i = match head_io with None -> Sn.start | Some i -> i in
+    Lwt.return head_i
+  in
+  get_head_i () >>= fun head_i ->
   let lag = Sn.to_int (Sn.sub last_i head_i) in
   let npt = !Tlogcommon.tlogEntriesPerFile in
   let tlog_lag = (lag +npt - 1)/ npt in
@@ -235,7 +239,8 @@ let collapse_many
       (* +2 because before X goes to the store, you need to have seen X+1 and thus too_far = X+2 *)
       Logger.debug_f_ "g_too_far_i = %s" (Sn.string_of g_too_far_i) >>= fun () ->
       collapse_until tlog_coll (module S) store_fs g_too_far_i cb slowdown >>= fun () ->
-      tlog_coll # remove_oldest_tlogs tlogs_to_collapse >>= fun () ->
+      get_head_i () >>= fun head_i ->
+      tlog_coll # remove_below head_i >>= fun () ->
       cb (Sn.div g_too_far_i (Sn.of_int !Tlogcommon.tlogEntriesPerFile))
     end
 
@@ -246,6 +251,11 @@ let collapse_out_of_band cfg name tlogs_to_keep =
   let me, _ = split name cfg.cfgs in
   let head_location = Filename.concat me.head_dir Tlc2.head_fname in
 
+  let () = match cfg.overwrite_tlog_entries with
+    | None -> ()
+    | Some i ->  Tlogcommon.tlogEntriesPerFile := i
+  in
+
   let module S = (val (Store.make_store_module (module Batched_store.Local_store))) in
   S.make_store
     ~lcnum:cfg.lcnum
diff --git a/src/tlog/mem_tlogcollection.ml b/src/tlog/mem_tlogcollection.ml
index e83ed6e3..fbe555a3 100644
--- a/src/tlog/mem_tlogcollection.ml
+++ b/src/tlog/mem_tlogcollection.ml
@@ -91,8 +91,6 @@ class mem_tlog_collection _name =
         let () = ignore wait_for_compression in
         Lwt.return ()
 
-    method remove_oldest_tlogs _count = Lwt.return ()
-
     method remove_below _i = Lwt.return ()
   end
 
diff --git a/src/tlog/tlc2.ml b/src/tlog/tlc2.ml
index 53b42fe0..f50f57cf 100644
--- a/src/tlog/tlc2.ml
+++ b/src/tlog/tlc2.ml
@@ -808,17 +808,6 @@ class tlc2
       File_system.rename tmp canon
 
 
-    method remove_oldest_tlogs count =
-      get_tlog_names tlog_dir tlf_dir >>= fun existing ->
-      let rec remove_one l = function
-        | 0 -> Lwt.return ()
-        | n ->
-          let oldest = get_full_path tlog_dir tlf_dir (List.hd l) in
-          Logger.debug_f_ "Unlinking %s" oldest >>= fun () ->
-          File_system.unlink (oldest) >>= fun () ->
-          remove_one (List.tl l) (n-1)
-      in remove_one existing count
-
     method remove_below i =
       get_tlog_names tlog_dir tlf_dir >>= fun existing ->
       let maybe_remove fn =
diff --git a/src/tlog/tlogcollection.ml b/src/tlog/tlogcollection.ml
index 4ee53bcb..02b59816 100644
--- a/src/tlog/tlogcollection.ml
+++ b/src/tlog/tlogcollection.ml
@@ -36,7 +36,6 @@ class type tlog_collection = object
   method get_head_name : unit -> string
   method get_tlog_from_i : Sn.t -> Sn.t
   method get_tlog_count: unit -> int Lwt.t
-  method remove_oldest_tlogs : int -> unit Lwt.t
   method remove_below : Sn.t -> unit Lwt.t
   method which_tlog_file : Sn.t -> (string option) Lwt.t
 end

From 0049f735a886e322bf8969bba7a45f09a36bd379 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 6 Jan 2016 15:36:22 +0100
Subject: [PATCH 069/373] test illustrating diff between python client &
 pyrakoon

---
 .gitmodules                                   |  2 +-
 pylabs/pyrakoon                               |  2 +-
 .../test/server/quick/system_tests_basic.py   | 34 +++++++++++++++++++
 3 files changed, 36 insertions(+), 2 deletions(-)

diff --git a/.gitmodules b/.gitmodules
index 9a40a52c..d5ae7b7b 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,3 +1,3 @@
 [submodule "pylabs/pyrakoon"]
 	path = pylabs/pyrakoon
-	url = git://github.com/Incubaid/pyrakoon.git
+	url = https://github.com/openvstorage/pyrakoon.git
diff --git a/pylabs/pyrakoon b/pylabs/pyrakoon
index 49ae1341..4772bd2e 160000
--- a/pylabs/pyrakoon
+++ b/pylabs/pyrakoon
@@ -1 +1 @@
-Subproject commit 49ae1341e1d1e0e577484469fdf824f4e984ba97
+Subproject commit 4772bd2e44156acacbc835b41d13b531be07cc5e
diff --git a/pylabs/test/server/quick/system_tests_basic.py b/pylabs/test/server/quick/system_tests_basic.py
index 63c09710..20f0e32e 100644
--- a/pylabs/test/server/quick/system_tests_basic.py
+++ b/pylabs/test/server/quick/system_tests_basic.py
@@ -810,6 +810,40 @@ def test_multi_get_option():
             assert_true(v is None)
     logging.debug("done")
 
+
+@C.with_custom_setup(C.setup_3_nodes, C.basic_teardown)
+def test_client_when_master_dies():
+    client = C.get_client ()
+    master = client.whoMaster()
+    print "master=", master
+    v0 = "the_value"
+    key = "they_key"
+    client[key] = v0
+
+    C.stopOne(master)
+    cluster = C._getCluster()
+    _wait_for_master(cluster)
+    master = client.whoMaster()
+    print "new master:", master
+    try:
+        v1 = client[key]
+        print "got value",v1
+    except Exception as ex:
+        print "EXCEPTION:"
+        print ex
+        print ex.__class__
+        assert_true(isinstance(ex, X.arakoon_client.ArakoonException))
+        assert_true(False, "should not throw")
+
+    master = client.whoMaster()
+    print "master", master
+    client2 = C.get_client ()
+    v2 = client2[key]
+    print "so far so good"
+    v1 = client[key]
+    print v1
+
+
 @C.with_custom_setup(C.setup_3_nodes_ipv6, C.basic_teardown)
 def test_ipv6():
     cli = C.get_client()

From e1bae9ad5c765d5e7c2db5abe528c750ac939fc1 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Thu, 7 Jan 2016 13:59:33 +0100
Subject: [PATCH 070/373] updated pyrakoon + allow running jenkins tests with
 pyrakoon

---
 jenkins/B.sh      | 2 ++
 jenkins/C.sh      | 2 ++
 jenkins/D.sh      | 2 ++
 jenkins/common.sh | 8 ++++++++
 jenkins/kwik.sh   | 2 ++
 jenkins/run.sh    | 2 ++
 pylabs/pyrakoon   | 2 +-
 7 files changed, 19 insertions(+), 1 deletion(-)
 create mode 100755 jenkins/common.sh

diff --git a/jenkins/B.sh b/jenkins/B.sh
index 0ee8c6d0..793a45c8 100755
--- a/jenkins/B.sh
+++ b/jenkins/B.sh
@@ -1,5 +1,7 @@
 #!/bin/bash -xue
 
+./jenkins/common.sh
+
 echo $PWD
 eval `${opam_env}`
 echo TEST_HOME=${TEST_HOME}
diff --git a/jenkins/C.sh b/jenkins/C.sh
index 5e60f6e1..7a609bda 100755
--- a/jenkins/C.sh
+++ b/jenkins/C.sh
@@ -1,5 +1,7 @@
 #!/bin/bash -xue
 
+./jenkins/common.sh
+
 echo $PWD
 eval `${opam_env}`
 echo TEST_HOME=${TEST_HOME}
diff --git a/jenkins/D.sh b/jenkins/D.sh
index 87a67cbc..bd1e733f 100755
--- a/jenkins/D.sh
+++ b/jenkins/D.sh
@@ -1,5 +1,7 @@
 #!/bin/bash -xue
 
+./jenkins/common.sh
+
 echo $PWD
 eval `${opam_env}`
 echo TEST_HOME=${TEST_HOME}
diff --git a/jenkins/common.sh b/jenkins/common.sh
new file mode 100755
index 00000000..7f0214af
--- /dev/null
+++ b/jenkins/common.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -e
+
+env | sort
+
+git submodule init
+git submodule update
diff --git a/jenkins/kwik.sh b/jenkins/kwik.sh
index 03657004..7c1cb7b0 100755
--- a/jenkins/kwik.sh
+++ b/jenkins/kwik.sh
@@ -1,5 +1,7 @@
 #!/bin/bash -xue
 
+./jenkins/common.sh
+
 echo $PWD
 eval `${opam_env}`
 echo TEST_HOME=${TEST_HOME}
diff --git a/jenkins/run.sh b/jenkins/run.sh
index 81404702..7b77dd05 100755
--- a/jenkins/run.sh
+++ b/jenkins/run.sh
@@ -2,6 +2,8 @@
 
 set -e
 
+./jenkins/common.sh
+
 for f in $(ls ./jenkins/$1);
 do
     extension="${f##*.}"
diff --git a/pylabs/pyrakoon b/pylabs/pyrakoon
index 4772bd2e..ac1386c2 160000
--- a/pylabs/pyrakoon
+++ b/pylabs/pyrakoon
@@ -1 +1 @@
-Subproject commit 4772bd2e44156acacbc835b41d13b531be07cc5e
+Subproject commit ac1386c2e1777b9e3222350c510581c9e0a39cbc

From aeca98e7efc5658702cb586777e46ae5b143957f Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 13 Jan 2016 09:20:35 +0100
Subject: [PATCH 071/373] adding url capabilities

---
 Makefile                                 |   2 +
 debian/changelog                         |   5 +
 src/all_test.ml                          |   1 +
 src/arakoon_client.mllib                 |   2 +
 src/client/arakoon_remote_client_test.ml |  22 +-
 src/client/client_main.ml                |  49 +-
 src/client/remote_nodestream_test.ml     |  19 +-
 src/inifiles/inifiles.ml                 |  66 ++-
 src/inifiles/inifiles.mli                |   7 +-
 src/inifiles/inifiles_test.ml            |  52 ++
 src/main/arakoon.ml                      | 557 +++++++++++++++++++++-
 src/main/main.ml                         | 573 -----------------------
 src/node/dump_store.ml                   |  26 +-
 src/node/node_cfg.ml                     | 216 +++++----
 src/node/node_cfg_test.ml                |  10 +-
 src/node/node_main.ml                    |  11 +-
 src/nursery/client_cfg.ml                |  17 +-
 src/nursery/client_cfg_test.ml           |  60 +--
 src/nursery/nursery_main.ml              |  19 +-
 src/system/drop_master.ml                |  57 ++-
 src/system/single.ml                     |  25 +-
 src/system/startup.ml                    |   8 +-
 src/tools/arakoon_etcd.ml                |  33 ++
 src/tools/arakoon_url.ml                 |  31 ++
 src/tools/lwt_extra.ml                   |  13 +
 25 files changed, 1034 insertions(+), 847 deletions(-)
 create mode 100644 src/inifiles/inifiles_test.ml
 delete mode 100644 src/main/main.ml
 create mode 100644 src/tools/arakoon_etcd.ml
 create mode 100644 src/tools/arakoon_url.ml

diff --git a/Makefile b/Makefile
index e077dba2..e7f81acd 100644
--- a/Makefile
+++ b/Makefile
@@ -64,6 +64,8 @@ install_client:
           _build/src/tools/typed_ssl.mli \
           _build/src/tools/typed_ssl.cmi \
           _build/src/tools/unix_fd.cmi \
+	  _build/src/tools/arakoon_etcd.cmi \
+	  _build/src/tools/arakoon_url.cmi \
           _build/src/tools/tcp_keepalive_sockopt_stubs.o \
           _build/src/tools/tcp_keepalive.cmi \
           _build/src/arakoon_client.a
diff --git a/debian/changelog b/debian/changelog
index bcbc8975..2aed312d 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,8 @@
+arakoon (1.8.12+etcd) unstable; urgency=low
+  * new upstream version
+
+ -- Romain Slootmaekers <romain.slootmaekers@openvstorage.com>  Thu, 14 Jan 2016 16:00:00 +0100
+
 arakoon (1.8.12) unstable; urgency=low
   * new upstream version
 
diff --git a/src/all_test.ml b/src/all_test.ml
index b75fa2ed..16077424 100644
--- a/src/all_test.ml
+++ b/src/all_test.ml
@@ -31,6 +31,7 @@ let configure_logging () =
 let tools_tests = "tools" >::: [
     Server_test.suite;
     Backoff_test.suite;
+    Inifiles_test.suite;
   ]
 
 let client_tests = "client" >::: [
diff --git a/src/arakoon_client.mllib b/src/arakoon_client.mllib
index 5f16c0e6..dde6c419 100644
--- a/src/arakoon_client.mllib
+++ b/src/arakoon_client.mllib
@@ -16,6 +16,8 @@ Routing
 Interval
 Ncfg
 Client_cfg
+Arakoon_url
+Arakoon_etcd
 Ini
 Inifiles
 Parseini
diff --git a/src/client/arakoon_remote_client_test.ml b/src/client/arakoon_remote_client_test.ml
index 039387fb..26d55324 100644
--- a/src/client/arakoon_remote_client_test.ml
+++ b/src/client/arakoon_remote_client_test.ml
@@ -44,15 +44,20 @@ type real_test = Arakoon_client.client -> unit Lwt.t
 let __client_server_wrapper__ (real_test:real_test) =
   let stop = ref false in
   let lease_period = 2 in
-  let make_config () = Node_cfg.make_test_config
-                         ~cluster_id:_CLUSTER
-                         ~node_name:(Printf.sprintf "sweety_%i")
-                         1 (Master_type.Elected) lease_period
+  let make_config () =
+    let r =
+      Node_cfg.make_test_config
+      ~cluster_id:_CLUSTER
+      ~node_name:(Printf.sprintf "sweety_%i")
+      1 (Master_type.Elected) lease_period
+    in
+    Lwt.return r
+  in
+  let t0 =
+    Node_main.test_t make_config "sweety_0" ~stop >>= fun _ -> Lwt.return ()
   in
-  let cluster_cfg = make_config () in
-  let t0 = Node_main.test_t make_config "sweety_0" ~stop >>= fun _ -> Lwt.return () in
 
-  let client_t () =
+  let client_t cluster_cfg =
     let sp = float(lease_period) *. 0.5 in
     Lwt_unix.sleep sp >>= fun () -> (* let the cluster reach stability *)
     Logger.info_ "cluster should have reached stability" >>= fun () ->
@@ -69,8 +74,9 @@ let __client_server_wrapper__ (real_test:real_test) =
       real_test
   in
   let main () =
+    make_config () >>= fun cluster_cfg ->
     Lwt.pick [(t0 >>= fun () -> Lwt.return false);
-              (client_t () >>= fun () -> Lwt.return true);] >>= fun succeeded ->
+              (client_t cluster_cfg >>= fun () -> Lwt.return true);] >>= fun succeeded ->
     stop := true;
     OUnit.assert_bool "client thread should finish before server" succeeded;
     Lwt.return ()
diff --git a/src/client/client_main.ml b/src/client/client_main.ml
index 5379cab3..07272d4a 100644
--- a/src/client/client_main.ml
+++ b/src/client/client_main.ml
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 *)
 
-open Node_cfg.Node_cfg
+open Node_cfg
 open Network
 open Statistics
 open Client_helper
@@ -25,7 +25,7 @@ let with_connection = with_connection
 let with_connection' = with_connection'
 
 let with_client ~tls ~tcp_keepalive node_cfg cluster_id f =
-  with_client' ~tls ~tcp_keepalive (node_cfg_to_node_client_cfg node_cfg) cluster_id f
+  with_client' ~tls ~tcp_keepalive (Node_cfg.node_cfg_to_node_client_cfg node_cfg) cluster_id f
 
 let with_remote_nodestream ~tls ~tcp_keepalive node_cfg cluster_id f =
   let open Node_cfg in
@@ -55,8 +55,8 @@ let ping ~tls ~tcp_keepalive ip port cluster_id =
 
 
 let find_master ~tls cluster_cfg =
-  let cluster_cfg' = to_client_cfg cluster_cfg in
-  let tcp_keepalive = cluster_cfg.tcp_keepalive in
+  let cluster_cfg' = Node_cfg.to_client_cfg cluster_cfg in
+  let tcp_keepalive = cluster_cfg.Node_cfg.tcp_keepalive in
   let open MasterLookupResult in
   find_master' ~tls ~tcp_keepalive cluster_cfg' >>= function
     | Found (node_name, _) -> return node_name
@@ -66,9 +66,9 @@ let find_master ~tls cluster_cfg =
     | Exception exn -> Lwt.fail exn
 
 
-let with_master_client ~tls cfg_name f =
+let with_master_client ~tls cfg_url f =
   let open Node_cfg in
-  let ccfg = read_config cfg_name in
+  retrieve_cfg cfg_url >>= fun ccfg ->
   find_master ~tls ccfg >>= fun master_name ->
   let master_cfg = List.hd (List.filter (fun cfg -> cfg.node_name = master_name) ccfg.cfgs) in
   with_client ~tls ~tcp_keepalive:ccfg.tcp_keepalive master_cfg ccfg.cluster_id f
@@ -200,49 +200,52 @@ let statistics ~tls cfg_name =
   let t () = with_master_client ~tls cfg_name f
   in run t
 
-let who_master ~tls cfg_name () =
-  let cluster_cfg = read_config cfg_name in
+let who_master ~tls cfg_url () =
+
   let t () =
+    Node_cfg.retrieve_cfg cfg_url >>= fun cluster_cfg ->
     find_master ~tls cluster_cfg >>= fun master_name ->
     Lwt_io.printl master_name
   in
   run t
 
-let _cluster_and_node_cfg node_name' cfg_name =
-  let open Node_cfg in
-  let cluster_cfg = read_config cfg_name in
+let _cluster_and_node_cfg node_name' cfg_url =
+  Node_cfg.retrieve_cfg cfg_url >>= fun cluster_cfg ->
   let _find cfgs =
     let rec loop = function
-      | [] -> failwith (node_name' ^ " is not known in config " ^ cfg_name)
+      | [] -> failwith (node_name' ^ " is not known in config " ^ (Arakoon_url.to_string cfg_url))
       | cfg :: rest ->
-        if cfg.node_name = node_name' then cfg
+        if cfg.Node_cfg.node_name = node_name' then cfg
         else loop rest
     in
     loop cfgs
   in
-  let node_cfg = _find cluster_cfg.cfgs in
-  cluster_cfg, node_cfg
+  let node_cfg = _find cluster_cfg.Node_cfg.cfgs in
+  Lwt.return (cluster_cfg, node_cfg)
 
 let node_state ~tls node_name' cfg_name =
-  let open Node_cfg in
-  let cluster_cfg,node_cfg = _cluster_and_node_cfg node_name' cfg_name in
-  let tcp_keepalive = cluster_cfg.tcp_keepalive in
-  let cluster = cluster_cfg.cluster_id in
   let f client =
     client # current_state () >>= fun state ->
     Lwt_io.printl state
   in
-  let t () = with_client ~tls ~tcp_keepalive node_cfg cluster f in
+
+  let t () =
+    _cluster_and_node_cfg node_name' cfg_name
+    >>= fun (cluster_cfg, node_cfg) ->
+    let open Node_cfg in
+    let cluster = cluster_cfg.cluster_id in
+    let tcp_keepalive = cluster_cfg.tcp_keepalive in
+    with_client ~tls ~tcp_keepalive node_cfg cluster f in
   run t
 
 
 
 let node_version ~tls node_name' cfg_name =
   let open Node_cfg in
-  let cluster_cfg, node_cfg = _cluster_and_node_cfg node_name' cfg_name in
-  let tcp_keepalive = cluster_cfg.tcp_keepalive in
-  let cluster = cluster_cfg.cluster_id in
   let t () =
+    _cluster_and_node_cfg node_name' cfg_name >>= fun (cluster_cfg, node_cfg) ->
+    let tcp_keepalive = cluster_cfg.tcp_keepalive in
+    let cluster = cluster_cfg.cluster_id in
     with_client ~tls ~tcp_keepalive node_cfg cluster
       (fun client ->
          client # version () >>= fun (major,minor,patch, info) ->
diff --git a/src/client/remote_nodestream_test.ml b/src/client/remote_nodestream_test.ml
index d44254dc..2f5b8e46 100644
--- a/src/client/remote_nodestream_test.ml
+++ b/src/client/remote_nodestream_test.ml
@@ -33,16 +33,18 @@ let cluster_id = "baby1"
 let __wrap__ port conversation =
   let stop = ref false in
   let lease_period = 2 in
-  let make_config () = Node_cfg.make_test_config
-                         ~cluster_id
-                         ~node_name:(Printf.sprintf "sweety_%i")
-                         ~base:port
-                         1 (Master_type.Elected) lease_period
+  let make_config () =
+    Node_cfg.make_test_config
+      ~cluster_id
+      ~node_name:(Printf.sprintf "sweety_%i")
+      ~base:port
+      1 (Master_type.Elected) lease_period
+    |> Lwt.return
   in
-  let cluster_cfg = make_config () in
+
   let t0 = Node_main.test_t make_config "sweety_0" ~stop >>= fun _ -> Lwt.return () in
 
-  let client_t () =
+  let client_t cluster_cfg =
     let sp = float(lease_period) *. 0.5 in
     Lwt_unix.sleep sp >>= fun () -> (* let the cluster reach stability *)
     Logger.info_ "cluster should have reached stability" >>= fun () ->
@@ -60,7 +62,8 @@ let __wrap__ port conversation =
     Lwt_io.close oc >>= fun () ->
     Lwt.return ()
   in
-  Lwt.pick [client_t ();t0] >>= fun () ->
+  make_config () >>= fun cluster_cfg ->
+  Lwt.pick [client_t cluster_cfg;t0] >>= fun () ->
   stop := true;
   Lwt.return ()
 
diff --git a/src/inifiles/inifiles.ml b/src/inifiles/inifiles.ml
index 5c76f3af..1dbd4d1f 100644
--- a/src/inifiles/inifiles.ml
+++ b/src/inifiles/inifiles.ml
@@ -39,10 +39,9 @@ limitations under the License.
    - removed pcre dependency
    - fixed little bug on parsing of empty lines with spaces
    - little change to fold signature to accomodate ocaml 4.0
+   - removed IO from ini file parsing so it can be used with Lwt too
 *)
 
-open Parseini
-open Inilexer
 
 exception Invalid_section of string
 exception Invalid_element of string
@@ -95,15 +94,10 @@ let rec filterfile ?(buf=Buffer.create 500) f fd =
       filterfile ~buf f fd
   with End_of_file -> Buffer.contents buf
 
-let read_inifile file fd tbl =
-  let lxbuf =
-    Lexing.from_string
-      (filterfile
-         (fun line -> not (Str.string_match comment line 0))
-         fd)
-  in
+let parse_inifile txt tbl =
+  let lxbuf = Lexing.from_string (txt ^ "\n") in (* otherwise, it might fail for a silly reason *)
   try
-    let parsed_file = inifile lexini lxbuf in
+    let parsed_txt = Parseini.inifile Inilexer.lexini lxbuf in
     List.iter
       (fun (section, values) ->
          Hashtbl.add tbl section
@@ -111,31 +105,32 @@ let read_inifile file fd tbl =
               (fun tbl (key, value) -> Hashtbl.add tbl key value;tbl)
               (Hashtbl.create 10)
               values))
-      parsed_file
+      parsed_txt
   with Parsing.Parse_error | Failure "lexing: empty token" ->
-    raise (Ini_parse_error (lxbuf.Lexing.lex_curr_p.Lexing.pos_lnum, file))
-
-let write_inifile fd tbl =
-  Hashtbl.iter
-    (fun k v ->
-       output_string fd "[";output_string fd k;output_string fd "]\n";
-       (Hashtbl.iter
-          (fun k v ->
-             output_string fd k;output_string fd " = ";output_string fd v;
-             output_string fd "\n") v);
-       output_string fd "\n")
-    tbl
-
-class inifile ?(spec=[]) file =
+    raise (Ini_parse_error (lxbuf.Lexing.lex_curr_p.Lexing.pos_lnum, txt))
+
+let write_inifile oc tbl =
+  let open Lwt.Infix in
+  let sections = Hashtbl.fold (fun k s acc -> (k,s)::acc) tbl [] |> List.rev in
+  Lwt_list.iter_s
+    (fun (k,section) ->
+     Lwt_io.fprintlf oc "[%s]" k >>= fun () ->
+     let kvs = Hashtbl.fold (fun k v acc -> (k,v) :: acc) section [] |> List.rev in
+     Lwt_list.iter_s
+       (fun (k,v) ->
+        Lwt_io.fprintlf oc "%s = %s\n" k v
+       )
+       kvs
+     >>= fun () ->
+     Lwt_io.write oc "\n"
+    ) sections
+
+class inifile ?(spec=[]) txt =
   object (self)
-    val file = file
     val data = Hashtbl.create 50
 
     initializer
-      let inch = open_in file in
-      (try read_inifile file inch data
-      with exn -> close_in inch;raise exn);
-      close_in inch;
+      let () = parse_inifile txt data in
       self#validate
 
     method private validate =
@@ -245,10 +240,10 @@ class inifile ?(spec=[]) file =
                with Not_found -> raise (Invalid_section sec))
                [])))
 
-    method save ?(file = file) () =
-      let outch = open_out file in
-      write_inifile outch data;
-      flush outch;
+    method save file =
+      Lwt_io.with_file
+        Lwt_io.Output file
+        (fun oc -> write_inifile oc data)
   end
 
 let readdir path =
@@ -269,7 +264,7 @@ let fold ?(spec=[]) func path initial =
         if (Unix.stat path).Unix.st_kind = Unix.S_REG
         then Filename.check_suffix path ".ini"
         else false
-      with Unix.Unix_error (_,_,_) -> false in  
+      with Unix.Unix_error (_,_,_) -> false in
   (List.fold_left
      func
      initial
@@ -280,4 +275,3 @@ let fold ?(spec=[]) func path initial =
            (List.rev_map
               (fun p -> (path ^ "/" ^ p))
               (readdir path)))))
-
diff --git a/src/inifiles/inifiles.mli b/src/inifiles/inifiles.mli
index 890c8b76..2a507baf 100644
--- a/src/inifiles/inifiles.mli
+++ b/src/inifiles/inifiles.mli
@@ -52,8 +52,7 @@ type section_specification = {
 (** The type of a secification *)
 type specification = section_specification list
 
-(** send the name of an ini file to the constructor the file must
-    exist, but can be empty *)
+(** parse text into inifile object. *)
 class inifile : ?spec:specification -> string ->
   object
     (** get a value from the config object raise Invalid_section, or
@@ -72,13 +71,13 @@ class inifile : ?spec:specification -> string ->
     method setval : string -> string -> string -> unit
 
     (** delete the topmost binding (the one returned by getval) from the
-        section sec. Possibly exposeing another binding.  raise
+        section sec. Possibly exposing another binding.  raise
         Invalid_section on error.  delval sec elt *)
     method delval : string -> string -> unit
 
     (** save the changes you have made
         optionally save to a different file *)
-    method save : ?file:string -> unit -> unit
+    method save : Lwt_io.file_name -> unit Lwt.t
 
     (** iterates across a section. passes all key valu pairs to f
         exactly once.*)
diff --git a/src/inifiles/inifiles_test.ml b/src/inifiles/inifiles_test.ml
new file mode 100644
index 00000000..f5ad9f2b
--- /dev/null
+++ b/src/inifiles/inifiles_test.ml
@@ -0,0 +1,52 @@
+let test_parsing () =
+  let txt =
+    let lines =[
+      "[global]"
+      ; "cluster = abm_0, abm_1, abm_2\n"
+      ; "cluster_id = abm"
+      ; "plugins = albamgr_plugin nsm_host_plugin"
+      ; "\n"
+
+      ; "[abm_0]"
+      ; "ip = 127.0.0.1"
+      ; "client_port = 4000"
+      ; "messaging_port = 4010"
+      ; "home = /home/romain/workspace/tmp/arakoon/abm/abm_0"
+      ; "log_level = debug"
+      ; "fsync = false"
+      ; "\n"
+
+      ; "[abm_1]"
+      ; "ip = 127.0.0.1"
+      ; "client_port = 4001"
+      ; "messaging_port = 4011"
+      ; "home = /home/romain/workspace/tmp/arakoon/abm/abm_1"
+      ; "log_level = debug"
+      ; "fsync = false"
+      ; "\n"
+
+      ;"[abm_2]";"ip = 127.0.0.1"
+      ;"client_port = 4002"
+      ;"messaging_port = 4012"
+      ;"home = /home/romain/workspace/tmp/arakoon/abm/abm_2"
+      ;"log_level = debug"
+      ;"fsync = false"
+      ]
+    in
+    (*let lines = ["[global]\ncluster = x,y,z\n";] in*)
+    String.concat "\n" lines
+  in
+  let () =
+    try
+      let _cfg = Node_cfg.Node_cfg._retrieve_cfg_from_txt txt in ()
+    with (Inifiles.Ini_parse_error(lnum,txt) as exn)->
+      let () =
+        Printf.printf "lnum:%i\n%s" lnum txt in
+      raise exn
+  in
+  ()
+
+open OUnit
+let suite = "inifiles" >:::[
+      "parsing" >:: test_parsing;
+    ]
diff --git a/src/main/arakoon.ml b/src/main/arakoon.ml
index 3acb4110..d53b3634 100644
--- a/src/main/arakoon.ml
+++ b/src/main/arakoon.ml
@@ -14,4 +14,559 @@ See the License for the specific language governing permissions and
 limitations under the License.
 *)
 
-Main.main ();;
+open OUnit_XML
+open OUnit
+open Node_cfg
+
+type local_action =
+  | ShowUsage
+  | RunAllTests
+  | RunAllTestsXML
+  | RunSomeTests
+  | ListTests
+  | SystemTests
+  | ShowVersion
+  | DumpTlog
+  | StripTlog
+  | ReplayTlogs
+  | DumpStore
+  | MakeTlog
+  | MarkTlog
+  | CloseTlog
+  | TruncateTlog
+  | CompressTlog
+  | UncompressTlog
+  | BENCHMARK
+  | LOAD
+  | SET
+  | GET
+  | DELETE
+  | DELETE_PREFIX
+  | NOP
+  | WHO_MASTER
+  | EXPECT_PROGRESS_POSSIBLE
+  | STATISTICS
+  | PREFIX
+  | USER_FUNCTION
+  | Collapse_local
+  | Collapse_remote
+  | Backup_db
+  | Optimize_db
+  | Defrag_db
+  | NumberOfValues
+  | InitNursery
+  | MigrateNurseryRange
+  | DeleteNurseryCluster
+  | PING
+  | InjectAsHead
+  | NODE_VERSION
+  | Drop_master
+  | NODE_STATE
+  | RANGE_ENTRIES
+  | REV_RANGE_ENTRIES
+
+type server_action =
+  | Node
+  | TestNode
+
+type action =
+  | LocalAction of local_action
+  | ServerAction of server_action;;
+
+let do_path p = Printf.printf "%s\n" (OUnit.string_of_path p)
+
+let list_tests() =
+  List.iter do_path (OUnit.test_case_paths All_test.suite)
+
+let show_version ()=
+  let open Arakoon_version in
+  let () =
+    Printf.printf "version: %i.%i.%i\n" major minor patch;
+    Printf.printf "git_revision: %S\n" git_revision;
+    Printf.printf "compiled: %S\n" compile_time;
+    Printf.printf "machine: %S\n" machine;
+    Printf.printf "compiler_version : %S\n" compiler_version;
+    Printf.printf "tlogEntriesPerFile: %i\n" !Tlogcommon.tlogEntriesPerFile;
+    Printf.printf "dependencies:\n%s\n" dependencies in
+  ()
+
+let interpret_test_results =
+  List.fold_left
+    (fun acc r ->
+     match r with
+     | RSuccess _ -> acc
+     | RFailure _
+     | RError _
+     | RSkip _
+     | RTodo _ -> 1)
+    0
+
+let run_all_tests () =
+  All_test.configure_logging();
+  let tests = All_test.suite in
+  let results = run_test_tt tests in
+  interpret_test_results results
+
+let run_all_tests_xml filename =
+  All_test.configure_logging();
+  let tests = All_test.suite in
+  let result = { result_list = [];
+                 total_time = 0.0;
+                 total_success = 0;
+                 total_failures = 0;
+                 total_errors = 0;
+                 total_disabled = 0;
+                 tmp_start = 0.0;
+                 tmp_result = RSuccess []
+               } in
+  let results = perform_timed_tests result tests in
+  let () = print_xml result filename in
+  interpret_test_results results
+
+let run_system_tests () =
+  All_test.configure_logging();
+  let tests = System_tests.suite in
+  let results = OUnit.run_test_tt tests in
+  interpret_test_results results
+
+
+let run_some_tests repeat_count filter =
+  All_test.configure_logging();
+  Printf.printf "running tests matching '%s'\n" filter;
+  let parts = Str.split_delim (Str.regexp "\\.") filter in
+  let show_count num =
+    if repeat_count > 1 then Printf.printf "-- iteration %d\n" (repeat_count - num + 1)
+  in
+  let rec do_n_times f results = function
+    | 0 -> results
+    | n ->
+      let () = show_count n in
+      let results' = f () in
+      do_n_times f (List.append results' results) (n-1)
+  in
+  match OUnit.test_filter parts All_test.suite  with
+    | Some test ->
+      begin
+        List.iter do_path (OUnit.test_case_paths test);
+        let results = do_n_times (fun () -> OUnit.run_test_tt_main test) [] repeat_count in
+        interpret_test_results results
+      end
+
+    | None -> failwith (Printf.sprintf "no test matches '%s'" filter);;
+
+
+let main () =
+  Ssl_threads.init ();
+  Ssl.init ~thread_safe:true ();
+
+  let () =Lwt.async_exception_hook :=
+    (fun exn -> Logger.ign_info_f_ ~exn "Caught async exception")
+  in
+  let _ = Bz2.version in
+  let () = Sys.set_signal Sys.sigpipe Sys.Signal_ignore in
+  let () = Random.self_init () in
+  let () = Client_log.enable_lwt_logging_for_client_lib_code () in
+  let usage_buffer = Buffer.create 1024 in
+  let app = Buffer.add_string usage_buffer in
+  let usage =
+    let bin = Sys.argv.(0) in
+    app ("usage: " ^ bin ^ " --<command>\n");
+    app ("\nIf you're clueless, try " ^ bin ^ " --help\n");
+    Buffer.contents usage_buffer
+  in
+  let action = ref (LocalAction ShowUsage)
+  and filename = ref "toy.db"
+  and xml_filename = ref "output.xml"
+  and filter = ref ""
+  and node_id = ref ""
+  and key = ref ""
+  and value = ref ""
+  and value_option = ref None
+  and ip = ref "127.0.0.1"
+  and port = ref 4000
+  and cluster_id = ref "<none>"
+  and value_size = ref 10
+  and key_size = ref 10
+  and tx_size = ref 100
+  and max_n = ref (1000 * 1000)
+  and daemonize = ref false
+  and test_repeat_count = ref 1
+  and counter = ref 0
+  and n_tlogs = ref 1
+  and n_clients = ref 1
+  and catchup_only = ref false
+  and dump_values = ref false
+  and max_results = ref 1000
+  and location = ref ""
+  and left_cluster = ref ""
+  and separator = ref ""
+  and right_cluster = ref ""
+  and tlog_dir = ref ""
+  and tlf_dir = ref ""
+  and end_i = ref None
+  and tls_ca_cert = ref ""
+  and tls_cert = ref ""
+  and tls_key = ref ""
+  and tls_version = ref "1.0"
+  and force = ref false
+  and in_place = ref false
+  and archive_type = ref ".tlf"
+  and left = ref None
+  and linc = ref true
+  and right = ref None
+  and rinc = ref false
+  and scenario = ref (String.concat ", " Benchmark.default_scenario)
+  and autofix = ref false
+  in
+  let set_action a = Arg.Unit (fun () -> action := a) in
+  let set_laction a = set_action (LocalAction a) in
+  let actions = [
+    ("--node", Arg.Tuple [set_action (ServerAction Node);
+                          Arg.Set_string node_id;
+                         ],
+     "runs a node");
+    (* tempory test node, TODO: remove later *)
+    ("--test-node", Arg.Tuple [set_action (ServerAction TestNode);
+                               Arg.Set_string node_id;
+                              ],
+     "runs a node");
+    ("--list-tests", set_laction ListTests, "lists all possible tests");
+    ("--run-all-tests", set_laction RunAllTests, "runs all tests");
+
+    ("--run-all-tests-xml", Arg.Tuple [set_laction RunAllTestsXML;
+                                       Arg.Set_string xml_filename],
+     "<filename> : runs all tests with XML output to file");
+
+    ("--run-some-tests", Arg.Tuple [set_laction RunSomeTests;
+                                    Arg.Set_string filter],
+     "run tests matching filter");
+    ("--truncate-tlog", Arg.Tuple[ set_laction TruncateTlog;
+                                   Arg.Set_string filename],
+     "<filename> : truncate a tlog after the last valid entry");
+    ("--dump-tlog", Arg.Tuple[ set_laction DumpTlog;
+                               Arg.Set_string filename],
+     "<filename> : dump a tlog file in readable format");
+    ("--strip-tlog", Arg.Tuple[ set_laction StripTlog;
+                                Arg.Set_string filename],
+     "<filename> : remove the marker of a tlog (development)");
+    ("--mark-tlog", Arg.Tuple[ set_laction MarkTlog;
+                               Arg.Set_string filename;
+                               Arg.Set_string key;
+                             ],
+     "<filename> <key>: add a marker to a tlog");
+    ("--unsafe-close-tlog", Arg.Tuple[ set_laction CloseTlog;
+                               Arg.Set_string filename;
+                               Arg.Set_string node_id;
+                             ],
+     "<filename> <node_name>: marks the tlog with 'closed:node_name'. Note when a tlog marker is missing, the node Tokyo Cabinet database could be silently corrupted, which is not fixed by this command.");
+    ("--replay-tlogs", Arg.Tuple[ set_laction ReplayTlogs;
+                                  Arg.Set_string tlog_dir;
+                                  Arg.Set_string tlf_dir;
+                                  Arg.Set_string filename;
+                                  Arg.Rest (fun is ->
+                                      if String.length is > 0
+                                      then end_i := (Some (Scanf.sscanf is "%Li" (fun i -> i))))
+                                ],
+     "<tlog_dir> <tlf_dir> <path-to-db> [<end-i>]");
+    ("-dump-values", Arg.Set dump_values, "also dumps values (in --dump-tlog)");
+    ("--make-tlog", Arg.Tuple[ set_laction MakeTlog;
+                               Arg.Set_string filename;
+                               Arg.Set_int counter;],
+     "<filename> <counter> : make a tlog file with 1 NOP entry @ <counter>");
+    ("--dump-store", Arg.Tuple [ set_laction DumpStore;
+                                 Arg.Set_string filename],
+     "<filename> : dump a store");
+    ("--compress-tlog", Arg.Tuple[set_laction CompressTlog;
+                                  Arg.Set_string filename],
+     "<filename> : compress a tlog file");
+    ("-archive", Arg.Set_string archive_type,
+     "either '.tlf' or '.tls'");
+    ("--uncompress-tlog", Arg.Tuple[set_laction UncompressTlog;
+                                    Arg.Set_string filename],
+     "<filename> : uncompress a tlog file");
+    ("--set", Arg.Tuple [set_laction SET;
+                         Arg.Set_string key;
+                         Arg.Set_string value;
+                        ], "<key> <value> : arakoon[<key>] = <value>");
+    ("--get", Arg.Tuple [set_laction GET;
+                         Arg.Set_string key
+                        ],"<key> : arakoon[<key>]");
+    ("--delete", Arg.Tuple[set_laction DELETE;
+                           Arg.Set_string key;
+                          ], "<key> : delete arakoon[<key>]");
+    ("--delete-prefix", Arg.Tuple[set_laction DELETE_PREFIX;
+                                  Arg.Set_string key;
+                                 ], "<prefix> : delete all entries where the key matches <prefix>");
+    ("--nop", set_laction NOP, "perform a no-op going through paxos");
+    ("--prefix", Arg.Tuple[set_laction PREFIX;
+                           Arg.Set_string key;
+                          ], "<prefix>: all starting with <prefix>");
+    ("--user-function", Arg.Tuple[set_laction USER_FUNCTION;
+                                  Arg.Set_string key;], "<name>: invoke user function 'name'");
+    ("-value", Arg.String (fun s -> value_option := Some s), "specify a value to pass to the user-function");
+    ("--benchmark", set_laction BENCHMARK, "run a benchmark on an existing Arakoon cluster");
+    ("--load", Arg.Tuple [set_laction LOAD;Arg.Set_int n_clients],
+     "<n> clients that generate load on a cluster");
+    ("--who-master", Arg.Tuple[set_laction WHO_MASTER;], "tells you who's the master");
+    ("--expect-progress-possible", Arg.Tuple[set_laction EXPECT_PROGRESS_POSSIBLE;],
+     "tells you if the master thinks progress is possible");
+    ("--statistics", set_laction STATISTICS, "returns some master statistics");
+    ("--run-system-tests", set_laction SystemTests,
+     "run system tests (you need a running installation)");
+    ("--version", set_laction ShowVersion, "shows version");
+    (* ("-port", Arg.Set_int port, "specifies server port"); *)
+    ("-config", Arg.String (fun s -> config_url := Arakoon_url.make s),
+     "specifies config url (if protocol is absent, 'file' is assumed; default = cfg/arakoon.ini )");
+    ("-autofix", Arg.Set autofix, "attempt to autofix (option to -node)");
+    ("-catchup-only", Arg.Set catchup_only,
+     "will only do a catchup of the node, without actually starting it (option to --node)");
+    ("-daemonize", Arg.Set daemonize,
+     "add if you want the process to daemonize (only for --node)");
+    ("-start", Arg.Unit (fun () -> ()),
+     "no-op for process-matching purposes");
+    ("-value_size", Arg.Set_int value_size, "size of the values (only for --benchmark)");
+    ("-key_size", Arg.Set_int key_size,
+     "size of the keys (only for --benchmark)");
+    ("-tx_size", Arg.Set_int tx_size, "size of transactions (only for --benchmark)");
+    ("-max_n", Arg.Set_int max_n,     "<benchmark size> (only for --benchmark)");
+    ("-n_clients", Arg.Set_int n_clients, "<n_clients>  (only for --benchmark)");
+    ("-scenario", Arg.Set_string scenario,
+     Printf.sprintf
+       "(only for --benchmark) which scenario to run, default being %S"
+       !scenario);
+    ("-max_results", Arg.Set_int max_results, "max size of the result (for --prefix)");
+    ("--test-repeat", Arg.Set_int test_repeat_count, "<repeat_count>");
+    ("--collapse-local", Arg.Tuple[set_laction Collapse_local;
+                                   Arg.Set_string node_id;
+                                   Arg.Set_int n_tlogs;],
+     "<node_id> <n> collapse all but <n> tlogs into the head database");
+    ("--collapse-remote", Arg.Tuple[set_laction Collapse_remote;
+                                    Arg.Set_string cluster_id;
+                                    Arg.Set_string ip;
+                                    Arg.Set_int port;
+                                    Arg.Set_int n_tlogs;
+                                   ],
+     "<cluster_id> <ip> <port> <n> tells node to collapse all but <n> tlogs into its head database");
+    ("--nursery-init", Arg.Tuple[set_laction InitNursery;
+                                 Arg.Set_string cluster_id;
+                                ],
+     "<cluster_id> Initialize the routing to contain a single cluster");
+    ("--nursery-migrate", Arg.Tuple[set_laction MigrateNurseryRange;
+                                    Arg.Set_string left_cluster;
+                                    Arg.Set_string separator;
+                                    Arg.Set_string right_cluster; ],
+     "<left_cluster> <separator> <right_cluster> migrate a range by either adding a new cluster or modifying an existing separator between two cluster ranges");
+    ("--nursery-delete", Arg.Tuple[set_laction DeleteNurseryCluster;
+                                   Arg.Set_string cluster_id;
+                                   Arg.Rest (fun s -> separator := s);
+                                  ],
+     "<cluster_id> <separator> removes <cluster_id> from the nursery, if the cluster is a boundary cluster no separator is required");
+    ("--backup-db", Arg.Tuple[set_laction Backup_db;
+                              Arg.Set_string cluster_id;
+                              Arg.Set_string ip;
+                              Arg.Set_int port;
+                              Arg.Set_string location;
+                             ],
+     "<cluster_id> <ip> <port> <location> requests the node to stream over its database (only works on slaves)");
+    ("--optimize-db", Arg.Tuple[set_laction Optimize_db;
+                                Arg.Set_string cluster_id;
+                                Arg.Set_string ip;
+                                Arg.Set_int port;
+                               ],
+     "<cluster_id> <ip> <port> requests the node to optimize its database (only works on slaves)");
+    ("--defrag-db", Arg.Tuple[set_laction Defrag_db;
+                              Arg.Set_string cluster_id;
+                              Arg.Set_string ip;
+                              Arg.Set_int port;
+                             ],
+     "<cluster_id> <ip> <port> requests the node to defragment its database");
+    ("--n-values", set_laction NumberOfValues,
+     "returns the number of values in the store");
+    ("--ping", Arg.Tuple[set_laction PING;
+                         Arg.Set_string cluster_id;
+                         Arg.Set_string ip;
+                         Arg.Set_int port;
+                        ],
+     "<cluster_id> <ip> <port> sends a ping to the node");
+    ("--node-version", Arg.Tuple[set_laction NODE_VERSION;
+                                 Arg.Set_string node_id],
+     "<node> : returns the version of <node>");
+    ("--node-state", Arg.Tuple [set_laction NODE_STATE;
+                                Arg.Set_string node_id],
+     "<node> : returns the state of <node>");
+    ("--inject-as-head", Arg.Tuple [set_laction InjectAsHead;
+                                    Arg.Set_string filename;
+                                    Arg.Set_string node_id],
+     "<head.db> <node_id>"
+    );
+    ("--force", Arg.Set force,
+     "force injection of the new head, even when the current head is corrupted (only for --inject-as-head)");
+    ("--inplace", Arg.Set in_place,
+     "perform an in-place inject-as-head using rename (only for --inject-as-head)");
+    ("--drop-master", Arg.Tuple [set_laction Drop_master;
+                                 Arg.Set_string cluster_id;
+                                 Arg.Set_string ip;
+                                 Arg.Set_int port;
+                                ],
+     "<cluster_id> <ip> <port> requests the node to drop its master role");
+    ("-tls-ca-cert", Arg.Set_string tls_ca_cert, "<path> TLS CA certificate");
+    ("-tls-cert", Arg.Set_string tls_cert, "<path> Certificate to use for TLS connections");
+    ("-tls-key", Arg.Set_string tls_key, "<path> Key to use for TLS connections");
+    ("-tls-version", Arg.Set_string tls_version, "<[1.0]|1.1|1.2> TLS version to use for connections");
+    ("--range-entries", Arg.Tuple [set_laction RANGE_ENTRIES;],
+     "list entries within range");
+    ("--rev-range-entries", Arg.Tuple [set_laction REV_RANGE_ENTRIES;],
+     "reverse list entries within range");
+    ("-left", Arg.String  (fun s -> left  := Some s), "left boundary (range query)");
+    ("-right", Arg.String (fun s -> right := Some s), "right boundary (range query)");
+
+  ] in
+
+  let options = [] in
+  let interface = actions @ options in
+
+  let make_config () = Node_cfg.retrieve_cfg !config_url in
+  let tcp_keepalive = default_tcp_keepalive in
+
+  let do_local ~tls = function
+    | ShowUsage -> print_endline usage;0
+    | RunAllTests -> run_all_tests ()
+    | RunAllTestsXML -> run_all_tests_xml !xml_filename
+    | RunSomeTests -> run_some_tests !test_repeat_count !filter
+    | ListTests -> list_tests ();0
+    | SystemTests -> run_system_tests()
+    | ShowVersion -> show_version();0
+    | DumpTlog -> Tlog_main.dump_tlog !filename ~values:!dump_values
+    | StripTlog -> Tlog_main.strip_tlog !filename
+    | MakeTlog -> Tlog_main.make_tlog !filename !counter
+    | MarkTlog -> Tlog_main.mark_tlog !filename !key
+    | CloseTlog -> Tlog_main.mark_tlog !filename (Tlc2._make_close_marker !node_id)
+    | ReplayTlogs -> Replay_main.replay_tlogs !tlog_dir !tlf_dir !filename !end_i
+    | DumpStore -> Dump_store.dump_store !filename
+    | TruncateTlog -> Tlc2.truncate_tlog !filename
+    | CompressTlog -> Tlog_main.compress_tlog !filename !archive_type
+    | UncompressTlog -> Tlog_main.uncompress_tlog !filename
+    | SET -> Client_main.set ~tls !config_url !key !value
+    | GET -> Client_main.get ~tls !config_url !key
+    | PREFIX -> Client_main.prefix ~tls !config_url !key !max_results
+    | DELETE_PREFIX -> Client_main.delete_prefix ~tls !config_url !key
+    | NOP -> Client_main.nop ~tls !config_url
+    | BENCHMARK ->Client_main.benchmark ~tls !config_url
+      !key_size !value_size !tx_size !max_n
+      !n_clients !scenario
+    | LOAD -> Load_client.main ~tls !config_url
+                !n_clients
+    | DELETE -> Client_main.delete ~tls !config_url !key
+    | USER_FUNCTION -> Client_main.user_function ~tls !config_url
+                                                 !key !value_option
+    | WHO_MASTER -> Client_main.who_master ~tls !config_url ()
+    | EXPECT_PROGRESS_POSSIBLE -> Client_main.expect_progress_possible ~tls !config_url
+    | STATISTICS -> Client_main.statistics ~tls !config_url
+    | Collapse_local ->
+       Lwt_main.run
+         begin
+           let open Lwt.Infix in
+           make_config () >>= fun cfg ->
+           Collapser.collapse_out_of_band cfg !node_id !n_tlogs
+         end;
+       0
+    | Collapse_remote -> Collapser_main.collapse_remote
+                           ~tls ~tcp_keepalive
+                           !ip !port !cluster_id !n_tlogs
+    | Backup_db -> Nodestream_main.get_db
+                     ~tls ~tcp_keepalive
+                     !ip !port !cluster_id !location
+    | Optimize_db -> Nodestream_main.optimize_db
+                       ~tls ~tcp_keepalive
+                       !ip !port !cluster_id
+    | Defrag_db   -> Nodestream_main.defrag_db
+                       ~tls ~tcp_keepalive
+                       !ip !port !cluster_id
+    | NumberOfValues -> Client_main.get_key_count ~tls !config_url ()
+    | InitNursery -> Nursery_main.init_nursery !config_url !cluster_id
+    | MigrateNurseryRange -> Nursery_main.migrate_nursery_range
+                               !config_url !left_cluster !separator !right_cluster
+    | DeleteNurseryCluster -> Nursery_main.delete_nursery_cluster !config_url !cluster_id !separator
+    | PING -> Client_main.ping
+                ~tls ~tcp_keepalive
+                !ip !port !cluster_id
+    | NODE_VERSION -> Client_main.node_version ~tls !node_id !config_url
+    | NODE_STATE   -> Client_main.node_state ~tls !node_id !config_url
+    | InjectAsHead -> Dump_store.inject_as_head !filename !node_id !config_url
+                        ~force:(!force) ~in_place:(!in_place)
+    | Drop_master -> Nodestream_main.drop_master
+                       ~tls ~tcp_keepalive
+                       !ip !port !cluster_id
+    | RANGE_ENTRIES ->
+       Client_main.range_entries
+         ~tls !config_url
+         !left !linc !right !rinc !max_results
+    | REV_RANGE_ENTRIES ->
+       Client_main.rev_range_entries
+         ~tls !config_url
+         !left !linc !right !rinc !max_results
+
+  in
+  let do_server node =
+    match node with
+      | Node ->
+        begin
+          Daemons.maybe_daemonize !daemonize make_config;
+          let main_t =
+            Node_main.main_t
+              make_config
+              !node_id !daemonize !catchup_only !autofix
+          in
+          (* Lwt_engine.set (new Lwt_engine.select :> Lwt_engine.t); *)
+          Lwt_main.run main_t
+        end
+      | TestNode ->
+        begin
+          let lease_period = 60 in
+          let node = Master_type.Forced "t_arakoon_0" in
+          let make_config () = Node_cfg.make_test_config 3 node lease_period |> Lwt.return in
+          let main_t = (Node_main.test_t make_config !node_id ~stop:(ref false)) in
+          Lwt_main.run main_t
+        end
+  in
+  Arg.parse
+    interface
+    (fun x -> raise (Arg.Bad ("Bad argument : " ^ x)))
+    usage;
+
+  let tls =
+    if !tls_ca_cert = ""
+      then None
+      else begin
+        let ca_cert = !tls_ca_cert
+        and protocol = match !tls_version with
+          | "1.0" -> Ssl.TLSv1
+          | "1.1" -> Ssl.TLSv1_1
+          | "1.2" -> Ssl.TLSv1_2
+          | _ -> failwith "Invalid \"tls-version\" value"
+        and creds = match !tls_cert with
+          | "" -> None
+          | s -> Some (s, !tls_key)
+        in
+        let ctx = Client_main.default_create_client_context ~ca_cert ~creds ~protocol in
+        Some ctx
+      end
+  in
+
+  let exit_code =
+    match !action with
+      | LocalAction la -> do_local ~tls la
+      | ServerAction sa -> do_server sa
+  in
+  (* let () = Printf.printf "[rc=%i]\n" rc in *)
+  let () =
+    (* no exit hook processing needed: we've done everything we needed to do *)
+    let rec loop () =
+      let _ = Lwt_sequence.take_l Lwt_main.exit_hooks in
+      loop ()
+    in
+    try loop () with Lwt_sequence.Empty -> ()
+  in
+  exit exit_code
+
+let () = main ()
diff --git a/src/main/main.ml b/src/main/main.ml
deleted file mode 100644
index aef59a23..00000000
--- a/src/main/main.ml
+++ /dev/null
@@ -1,573 +0,0 @@
-(*
-Copyright (2010-2014) INCUBAID BVBA
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*)
-
-open OUnit_XML
-open OUnit
-open Node_cfg
-
-type local_action =
-  | ShowUsage
-  | RunAllTests
-  | RunAllTestsXML
-  | RunSomeTests
-  | ListTests
-  | SystemTests
-  | ShowVersion
-  | DumpTlog
-  | StripTlog
-  | ReplayTlogs
-  | DumpStore
-  | MakeTlog
-  | MarkTlog
-  | CloseTlog
-  | TruncateTlog
-  | CompressTlog
-  | UncompressTlog
-  | BENCHMARK
-  | LOAD
-  | SET
-  | GET
-  | DELETE
-  | DELETE_PREFIX
-  | NOP
-  | WHO_MASTER
-  | EXPECT_PROGRESS_POSSIBLE
-  | STATISTICS
-  | PREFIX
-  | USER_FUNCTION
-  | Collapse_local
-  | Collapse_remote
-  | Backup_db
-  | Optimize_db
-  | Defrag_db
-  | NumberOfValues
-  | InitNursery
-  | MigrateNurseryRange
-  | DeleteNurseryCluster
-  | PING
-  | InjectAsHead
-  | NODE_VERSION
-  | Drop_master
-  | NODE_STATE
-  | RANGE_ENTRIES
-  | REV_RANGE_ENTRIES
-
-type server_action =
-  | Node
-  | TestNode
-
-type action =
-  | LocalAction of local_action
-  | ServerAction of server_action;;
-
-let do_path p = Printf.printf "%s\n" (OUnit.string_of_path p)
-
-let list_tests() =
-  List.iter do_path (OUnit.test_case_paths All_test.suite)
-
-let show_version ()=
-  let open Arakoon_version in
-  let () =
-    Printf.printf "version: %i.%i.%i\n" major minor patch;
-    Printf.printf "git_revision: %S\n" git_revision;
-    Printf.printf "compiled: %S\n" compile_time;
-    Printf.printf "machine: %S\n" machine;
-    Printf.printf "compiler_version : %S\n" compiler_version;
-    Printf.printf "tlogEntriesPerFile: %i\n" !Tlogcommon.tlogEntriesPerFile;
-    Printf.printf "dependencies:\n%s\n" dependencies in
-  ()
-
-let interpret_test_results =
-  List.fold_left
-    (fun acc r ->
-     match r with
-     | RSuccess _ -> acc
-     | RFailure _
-     | RError _
-     | RSkip _
-     | RTodo _ -> 1)
-    0
-
-let run_all_tests () =
-  All_test.configure_logging();
-  let tests = All_test.suite in
-  let results = run_test_tt tests in
-  interpret_test_results results
-
-let run_all_tests_xml filename =
-  All_test.configure_logging();
-  let tests = All_test.suite in
-  let result = { result_list = [];
-                 total_time = 0.0;
-                 total_success = 0;
-                 total_failures = 0;
-                 total_errors = 0;
-                 total_disabled = 0;
-                 tmp_start = 0.0;
-                 tmp_result = RSuccess []
-               } in
-  let results = perform_timed_tests result tests in
-  let () = print_xml result filename in
-  interpret_test_results results
-
-let run_system_tests () =
-  All_test.configure_logging();
-  let tests = System_tests.suite in
-  let results = OUnit.run_test_tt tests in
-  interpret_test_results results
-
-
-let run_some_tests repeat_count filter =
-  All_test.configure_logging();
-  Printf.printf "running tests matching '%s'\n" filter;
-  let parts = Str.split_delim (Str.regexp "\\.") filter in
-  let show_count num =
-    if repeat_count > 1 then Printf.printf "-- iteration %d\n" (repeat_count - num + 1)
-  in
-  let rec do_n_times f results = function
-    | 0 -> results
-    | n ->
-      let () = show_count n in
-      let results' = f () in
-      do_n_times f (List.append results' results) (n-1)
-  in
-  match OUnit.test_filter parts All_test.suite  with
-    | Some test ->
-      begin
-        List.iter do_path (OUnit.test_case_paths test);
-        let results = do_n_times (fun () -> OUnit.run_test_tt_main test) [] repeat_count in
-        interpret_test_results results
-      end
-
-    | None -> failwith (Printf.sprintf "no test matches '%s'" filter);;
-
-
-let main () =
-  Ssl_threads.init ();
-  Ssl.init ~thread_safe:true ();
-
-  let () =Lwt.async_exception_hook :=
-    (fun exn -> Logger.ign_info_f_ ~exn "Caught async exception")
-  in
-  let _ = Bz2.version in
-  let () = Sys.set_signal Sys.sigpipe Sys.Signal_ignore in
-  let () = Random.self_init () in
-  let () = Client_log.enable_lwt_logging_for_client_lib_code () in
-  let usage_buffer = Buffer.create 1024 in
-  let app = Buffer.add_string usage_buffer in
-  let usage =
-    let bin = Sys.argv.(0) in
-    app ("usage: " ^ bin ^ " --<command>\n");
-    app ("\nIf you're clueless, try " ^ bin ^ " --help\n");
-    Buffer.contents usage_buffer
-  in
-  let action = ref (LocalAction ShowUsage)
-  and filename = ref "toy.db"
-  and xml_filename = ref "output.xml"
-  and filter = ref ""
-  and node_id = ref ""
-  and key = ref ""
-  and value = ref ""
-  and value_option = ref None
-  and ip = ref "127.0.0.1"
-  and port = ref 4000
-  and cluster_id = ref "<none>"
-  and value_size = ref 10
-  and key_size = ref 10
-  and tx_size = ref 100
-  and max_n = ref (1000 * 1000)
-  and daemonize = ref false
-  and test_repeat_count = ref 1
-  and counter = ref 0
-  and n_tlogs = ref 1
-  and n_clients = ref 1
-  and catchup_only = ref false
-  and dump_values = ref false
-  and max_results = ref 1000
-  and location = ref ""
-  and left_cluster = ref ""
-  and separator = ref ""
-  and right_cluster = ref ""
-  and tlog_dir = ref ""
-  and tlf_dir = ref ""
-  and end_i = ref None
-  and tls_ca_cert = ref ""
-  and tls_cert = ref ""
-  and tls_key = ref ""
-  and tls_version = ref "1.0"
-  and force = ref false
-  and in_place = ref false
-  and archive_type = ref ".tlf"
-  and left = ref None
-  and linc = ref true
-  and right = ref None
-  and rinc = ref false
-  and scenario = ref (String.concat ", " Benchmark.default_scenario)
-  and autofix = ref false
-  in
-  let set_action a = Arg.Unit (fun () -> action := a) in
-  let set_laction a = set_action (LocalAction a) in
-  let actions = [
-    ("--node", Arg.Tuple [set_action (ServerAction Node);
-                          Arg.Set_string node_id;
-                         ],
-     "runs a node");
-    (* tempory test node, TODO: remove later *)
-    ("--test-node", Arg.Tuple [set_action (ServerAction TestNode);
-                               Arg.Set_string node_id;
-                              ],
-     "runs a node");
-    ("--list-tests", set_laction ListTests, "lists all possible tests");
-    ("--run-all-tests", set_laction RunAllTests, "runs all tests");
-
-    ("--run-all-tests-xml", Arg.Tuple [set_laction RunAllTestsXML;
-                                       Arg.Set_string xml_filename],
-     "<filename> : runs all tests with XML output to file");
-
-    ("--run-some-tests", Arg.Tuple [set_laction RunSomeTests;
-                                    Arg.Set_string filter],
-     "run tests matching filter");
-    ("--truncate-tlog", Arg.Tuple[ set_laction TruncateTlog;
-                                   Arg.Set_string filename],
-     "<filename> : truncate a tlog after the last valid entry");
-    ("--dump-tlog", Arg.Tuple[ set_laction DumpTlog;
-                               Arg.Set_string filename],
-     "<filename> : dump a tlog file in readable format");
-    ("--strip-tlog", Arg.Tuple[ set_laction StripTlog;
-                                Arg.Set_string filename],
-     "<filename> : remove the marker of a tlog (development)");
-    ("--mark-tlog", Arg.Tuple[ set_laction MarkTlog;
-                               Arg.Set_string filename;
-                               Arg.Set_string key;
-                             ],
-     "<filename> <key>: add a marker to a tlog");
-    ("--unsafe-close-tlog", Arg.Tuple[ set_laction CloseTlog;
-                               Arg.Set_string filename;
-                               Arg.Set_string node_id;
-                             ],
-     "<filename> <node_name>: marks the tlog with 'closed:node_name'. Note when a tlog marker is missing, the node Tokyo Cabinet database could be silently corrupted, which is not fixed by this command.");
-    ("--replay-tlogs", Arg.Tuple[ set_laction ReplayTlogs;
-                                  Arg.Set_string tlog_dir;
-                                  Arg.Set_string tlf_dir;
-                                  Arg.Set_string filename;
-                                  Arg.Rest (fun is ->
-                                      if String.length is > 0
-                                      then end_i := (Some (Scanf.sscanf is "%Li" (fun i -> i))))
-                                ],
-     "<tlog_dir> <tlf_dir> <path-to-db> [<end-i>]");
-    ("-dump-values", Arg.Set dump_values, "also dumps values (in --dump-tlog)");
-    ("--make-tlog", Arg.Tuple[ set_laction MakeTlog;
-                               Arg.Set_string filename;
-                               Arg.Set_int counter;],
-     "<filename> <counter> : make a tlog file with 1 NOP entry @ <counter>");
-    ("--dump-store", Arg.Tuple [ set_laction DumpStore;
-                                 Arg.Set_string filename],
-     "<filename> : dump a store");
-    ("--compress-tlog", Arg.Tuple[set_laction CompressTlog;
-                                  Arg.Set_string filename],
-     "<filename> : compress a tlog file");
-    ("-archive", Arg.Set_string archive_type,
-     "either '.tlf' or '.tls'");
-    ("--uncompress-tlog", Arg.Tuple[set_laction UncompressTlog;
-                                    Arg.Set_string filename],
-     "<filename> : uncompress a tlog file");
-    ("--set", Arg.Tuple [set_laction SET;
-                         Arg.Set_string key;
-                         Arg.Set_string value;
-                        ], "<key> <value> : arakoon[<key>] = <value>");
-    ("--get", Arg.Tuple [set_laction GET;
-                         Arg.Set_string key
-                        ],"<key> : arakoon[<key>]");
-    ("--delete", Arg.Tuple[set_laction DELETE;
-                           Arg.Set_string key;
-                          ], "<key> : delete arakoon[<key>]");
-    ("--delete-prefix", Arg.Tuple[set_laction DELETE_PREFIX;
-                                  Arg.Set_string key;
-                                 ], "<prefix> : delete all entries where the key matches <prefix>");
-    ("--nop", set_laction NOP, "perform a no-op going through paxos");
-    ("--prefix", Arg.Tuple[set_laction PREFIX;
-                           Arg.Set_string key;
-                          ], "<prefix>: all starting with <prefix>");
-    ("--user-function", Arg.Tuple[set_laction USER_FUNCTION;
-                                  Arg.Set_string key;], "<name>: invoke user function 'name'");
-    ("-value", Arg.String (fun s -> value_option := Some s), "specify a value to pass to the user-function");
-    ("--benchmark", set_laction BENCHMARK, "run a benchmark on an existing Arakoon cluster");
-    ("--load", Arg.Tuple [set_laction LOAD;Arg.Set_int n_clients],
-     "<n> clients that generate load on a cluster");
-    ("--who-master", Arg.Tuple[set_laction WHO_MASTER;], "tells you who's the master");
-    ("--expect-progress-possible", Arg.Tuple[set_laction EXPECT_PROGRESS_POSSIBLE;],
-     "tells you if the master thinks progress is possible");
-    ("--statistics", set_laction STATISTICS, "returns some master statistics");
-    ("--run-system-tests", set_laction SystemTests,
-     "run system tests (you need a running installation)");
-    ("--version", set_laction ShowVersion, "shows version");
-    (* ("-port", Arg.Set_int port, "specifies server port"); *)
-    ("-config", Arg.Set_string config_file,
-     "specifies config file (default = cfg/arakoon.ini )");
-    ("-autofix", Arg.Set autofix, "attempt to autofix (option to -node)");
-    ("-catchup-only", Arg.Set catchup_only,
-     "will only do a catchup of the node, without actually starting it (option to --node)");
-    ("-daemonize", Arg.Set daemonize,
-     "add if you want the process to daemonize (only for --node)");
-    ("-start", Arg.Unit (fun () -> ()),
-     "no-op for process-matching purposes");
-    ("-value_size", Arg.Set_int value_size, "size of the values (only for --benchmark)");
-    ("-key_size", Arg.Set_int key_size,
-     "size of the keys (only for --benchmark)");
-    ("-tx_size", Arg.Set_int tx_size, "size of transactions (only for --benchmark)");
-    ("-max_n", Arg.Set_int max_n,     "<benchmark size> (only for --benchmark)");
-    ("-n_clients", Arg.Set_int n_clients, "<n_clients>  (only for --benchmark)");
-    ("-scenario", Arg.Set_string scenario,
-     Printf.sprintf
-       "(only for --benchmark) which scenario to run, default being %S"
-       !scenario);
-    ("-max_results", Arg.Set_int max_results, "max size of the result (for --prefix)");
-    ("--test-repeat", Arg.Set_int test_repeat_count, "<repeat_count>");
-    ("--collapse-local", Arg.Tuple[set_laction Collapse_local;
-                                   Arg.Set_string node_id;
-                                   Arg.Set_int n_tlogs;],
-     "<node_id> <n> collapse all but <n> tlogs into the head database");
-    ("--collapse-remote", Arg.Tuple[set_laction Collapse_remote;
-                                    Arg.Set_string cluster_id;
-                                    Arg.Set_string ip;
-                                    Arg.Set_int port;
-                                    Arg.Set_int n_tlogs;
-                                   ],
-     "<cluster_id> <ip> <port> <n> tells node to collapse all but <n> tlogs into its head database");
-    ("--nursery-init", Arg.Tuple[set_laction InitNursery;
-                                 Arg.Set_string cluster_id;
-                                ],
-     "<cluster_id> Initialize the routing to contain a single cluster");
-    ("--nursery-migrate", Arg.Tuple[set_laction MigrateNurseryRange;
-                                    Arg.Set_string left_cluster;
-                                    Arg.Set_string separator;
-                                    Arg.Set_string right_cluster; ],
-     "<left_cluster> <separator> <right_cluster> migrate a range by either adding a new cluster or modifying an existing separator between two cluster ranges");
-    ("--nursery-delete", Arg.Tuple[set_laction DeleteNurseryCluster;
-                                   Arg.Set_string cluster_id;
-                                   Arg.Rest (fun s -> separator := s);
-                                  ],
-     "<cluster_id> <separator> removes <cluster_id> from the nursery, if the cluster is a boundary cluster no separator is required");
-    ("--backup-db", Arg.Tuple[set_laction Backup_db;
-                              Arg.Set_string cluster_id;
-                              Arg.Set_string ip;
-                              Arg.Set_int port;
-                              Arg.Set_string location;
-                             ],
-     "<cluster_id> <ip> <port> <location> requests the node to stream over its database (only works on slaves)");
-    ("--optimize-db", Arg.Tuple[set_laction Optimize_db;
-                                Arg.Set_string cluster_id;
-                                Arg.Set_string ip;
-                                Arg.Set_int port;
-                               ],
-     "<cluster_id> <ip> <port> requests the node to optimize its database (only works on slaves)");
-    ("--defrag-db", Arg.Tuple[set_laction Defrag_db;
-                              Arg.Set_string cluster_id;
-                              Arg.Set_string ip;
-                              Arg.Set_int port;
-                             ],
-     "<cluster_id> <ip> <port> requests the node to defragment its database");
-    ("--n-values", set_laction NumberOfValues,
-     "returns the number of values in the store");
-    ("--ping", Arg.Tuple[set_laction PING;
-                         Arg.Set_string cluster_id;
-                         Arg.Set_string ip;
-                         Arg.Set_int port;
-                        ],
-     "<cluster_id> <ip> <port> sends a ping to the node");
-    ("--node-version", Arg.Tuple[set_laction NODE_VERSION;
-                                 Arg.Set_string node_id],
-     "<node> : returns the version of <node>");
-    ("--node-state", Arg.Tuple [set_laction NODE_STATE;
-                                Arg.Set_string node_id],
-     "<node> : returns the state of <node>");
-    ("--inject-as-head", Arg.Tuple [set_laction InjectAsHead;
-                                    Arg.Set_string filename;
-                                    Arg.Set_string node_id],
-     "<head.db> <node_id>"
-    );
-    ("--force", Arg.Set force,
-     "force injection of the new head, even when the current head is corrupted (only for --inject-as-head)");
-    ("--inplace", Arg.Set in_place,
-     "perform an in-place inject-as-head using rename (only for --inject-as-head)");
-    ("--drop-master", Arg.Tuple [set_laction Drop_master;
-                                 Arg.Set_string cluster_id;
-                                 Arg.Set_string ip;
-                                 Arg.Set_int port;
-                                ],
-     "<cluster_id> <ip> <port> requests the node to drop its master role");
-    ("-tls-ca-cert", Arg.Set_string tls_ca_cert, "<path> TLS CA certificate");
-    ("-tls-cert", Arg.Set_string tls_cert, "<path> Certificate to use for TLS connections");
-    ("-tls-key", Arg.Set_string tls_key, "<path> Key to use for TLS connections");
-    ("-tls-version", Arg.Set_string tls_version, "<[1.0]|1.1|1.2> TLS version to use for connections");
-    ("--range-entries", Arg.Tuple [set_laction RANGE_ENTRIES;],
-     "list entries within range");
-    ("--rev-range-entries", Arg.Tuple [set_laction REV_RANGE_ENTRIES;],
-     "reverse list entries within range");
-    ("-left", Arg.String  (fun s -> left  := Some s), "left boundary (range query)");
-    ("-right", Arg.String (fun s -> right := Some s), "right boundary (range query)");
-
-  ] in
-
-  let options = [] in
-  let interface = actions @ options in
-
-  let make_config () =
-    let canonical =
-      if !config_file.[0] = '/'
-      then !config_file
-      else Filename.concat (Unix.getcwd()) !config_file
-    in
-    Node_cfg.read_config canonical
-  in
-  let tcp_keepalive = default_tcp_keepalive in
-
-  let do_local ~tls = function
-    | ShowUsage -> print_endline usage;0
-    | RunAllTests -> run_all_tests ()
-    | RunAllTestsXML -> run_all_tests_xml !xml_filename
-    | RunSomeTests -> run_some_tests !test_repeat_count !filter
-    | ListTests -> list_tests ();0
-    | SystemTests -> run_system_tests()
-    | ShowVersion -> show_version();0
-    | DumpTlog -> Tlog_main.dump_tlog !filename ~values:!dump_values
-    | StripTlog -> Tlog_main.strip_tlog !filename
-    | MakeTlog -> Tlog_main.make_tlog !filename !counter
-    | MarkTlog -> Tlog_main.mark_tlog !filename !key
-    | CloseTlog -> Tlog_main.mark_tlog !filename (Tlc2._make_close_marker !node_id)
-    | ReplayTlogs -> Replay_main.replay_tlogs !tlog_dir !tlf_dir !filename !end_i
-    | DumpStore -> Dump_store.dump_store !filename
-    | TruncateTlog -> Tlc2.truncate_tlog !filename
-    | CompressTlog -> Tlog_main.compress_tlog !filename !archive_type
-    | UncompressTlog -> Tlog_main.uncompress_tlog !filename
-    | SET -> Client_main.set ~tls !config_file !key !value
-    | GET -> Client_main.get ~tls !config_file !key
-    | PREFIX -> Client_main.prefix ~tls !config_file !key !max_results
-    | DELETE_PREFIX -> Client_main.delete_prefix ~tls !config_file !key
-    | NOP -> Client_main.nop ~tls !config_file
-    | BENCHMARK ->Client_main.benchmark ~tls !config_file
-      !key_size !value_size !tx_size !max_n
-      !n_clients !scenario
-    | LOAD -> Load_client.main ~tls !config_file
-                !n_clients
-    | DELETE -> Client_main.delete ~tls !config_file !key
-    | USER_FUNCTION -> Client_main.user_function ~tls !config_file !key !value_option
-    | WHO_MASTER -> Client_main.who_master ~tls !config_file ()
-    | EXPECT_PROGRESS_POSSIBLE -> Client_main.expect_progress_possible ~tls !config_file
-    | STATISTICS -> Client_main.statistics ~tls !config_file
-    | Collapse_local ->
-       Lwt_main.run
-         begin
-           let cfg = make_config () in
-           Collapser.collapse_out_of_band cfg !node_id !n_tlogs
-         end;
-       0
-    | Collapse_remote -> Collapser_main.collapse_remote
-                           ~tls ~tcp_keepalive
-                           !ip !port !cluster_id !n_tlogs
-    | Backup_db -> Nodestream_main.get_db
-                     ~tls ~tcp_keepalive
-                     !ip !port !cluster_id !location
-    | Optimize_db -> Nodestream_main.optimize_db
-                       ~tls ~tcp_keepalive
-                       !ip !port !cluster_id
-    | Defrag_db   -> Nodestream_main.defrag_db
-                       ~tls ~tcp_keepalive
-                       !ip !port !cluster_id
-    | NumberOfValues -> Client_main.get_key_count ~tls !config_file ()
-    | InitNursery -> Nursery_main.init_nursery !config_file !cluster_id
-    | MigrateNurseryRange -> Nursery_main.migrate_nursery_range
-                               !config_file !left_cluster !separator !right_cluster
-    | DeleteNurseryCluster -> Nursery_main.delete_nursery_cluster !config_file !cluster_id !separator
-    | PING -> Client_main.ping
-                ~tls ~tcp_keepalive
-                !ip !port !cluster_id
-    | NODE_VERSION -> Client_main.node_version ~tls !node_id !config_file
-    | NODE_STATE   -> Client_main.node_state ~tls !node_id !config_file
-    | InjectAsHead -> Dump_store.inject_as_head !filename !node_id !config_file
-                        ~force:(!force) ~in_place:(!in_place)
-    | Drop_master -> Nodestream_main.drop_master
-                       ~tls ~tcp_keepalive
-                       !ip !port !cluster_id
-    | RANGE_ENTRIES ->
-       Client_main.range_entries
-         ~tls !config_file
-         !left !linc !right !rinc !max_results
-    | REV_RANGE_ENTRIES ->
-       Client_main.rev_range_entries
-         ~tls !config_file
-         !left !linc !right !rinc !max_results
-
-  in
-  let do_server node =
-    match node with
-      | Node ->
-        begin
-          Daemons.maybe_daemonize !daemonize make_config;
-          let main_t = (Node_main.main_t make_config
-                          !node_id !daemonize !catchup_only !autofix)
-          in
-          (* Lwt_engine.set (new Lwt_engine.select :> Lwt_engine.t); *)
-          Lwt_main.run main_t
-        end
-      | TestNode ->
-        begin
-          let lease_period = 60 in
-          let node = Master_type.Forced "t_arakoon_0" in
-          let make_config () = Node_cfg.make_test_config 3 node lease_period in
-          let main_t = (Node_main.test_t make_config !node_id ~stop:(ref false)) in
-          Lwt_main.run main_t
-        end
-  in
-  Arg.parse
-    interface
-    (fun x -> raise (Arg.Bad ("Bad argument : " ^ x)))
-    usage;
-
-  let tls =
-    if !tls_ca_cert = ""
-      then None
-      else begin
-        let ca_cert = !tls_ca_cert
-        and protocol = match !tls_version with
-          | "1.0" -> Ssl.TLSv1
-          | "1.1" -> Ssl.TLSv1_1
-          | "1.2" -> Ssl.TLSv1_2
-          | _ -> failwith "Invalid \"tls-version\" value"
-        and creds = match !tls_cert with
-          | "" -> None
-          | s -> Some (s, !tls_key)
-        in
-        let ctx = Client_main.default_create_client_context ~ca_cert ~creds ~protocol in
-        Some ctx
-      end
-  in
-
-  let exit_code =
-    match !action with
-      | LocalAction la -> do_local ~tls la
-      | ServerAction sa -> do_server sa
-  in
-  (* let () = Printf.printf "[rc=%i]\n" rc in *)
-  let () =
-    (* no exit hook processing needed: we've done everything we needed to do *)
-    let rec loop () =
-      let _ = Lwt_sequence.take_l Lwt_main.exit_hooks in
-      loop ()
-    in
-    try loop () with Lwt_sequence.Empty -> ()
-  in
-  exit exit_code
diff --git a/src/node/dump_store.ml b/src/node/dump_store.ml
index bd96c9bf..9f355fac 100644
--- a/src/node/dump_store.ml
+++ b/src/node/dump_store.ml
@@ -64,22 +64,18 @@ let dump_store filename =
 
 exception ExitWithCode of int;;
 
-let inject_as_head fn node_id cfg_fn ~force ~in_place =
-  let canonical =
-    if cfg_fn.[0] = '/'
-    then cfg_fn
-    else Filename.concat (Unix.getcwd()) cfg_fn
-  in
-  let cluster_cfg = read_config canonical in
-  let node_cfgs = List.filter
-                    (fun ncfg -> node_name ncfg = node_id)
-                    cluster_cfg.cfgs
-  in
-  let node_cfg = match node_cfgs with
-    | [] -> failwith (Printf.sprintf "unknown node: %S" node_id)
-    | x :: _ -> x
-  in
+let inject_as_head fn node_id cfg_url ~force ~in_place =
+
   let t () =
+    retrieve_cfg cfg_url >>= fun cluster_cfg ->
+    let node_cfgs = List.filter
+                      (fun ncfg -> node_name ncfg = node_id)
+                      cluster_cfg.cfgs
+    in
+    let node_cfg = match node_cfgs with
+      | [] -> failwith (Printf.sprintf "unknown node: %S" node_id)
+      | x :: _ -> x
+    in
     let tlog_dir = node_cfg.tlog_dir in
     let tlx_dir = node_cfg.tlx_dir in
     let head_dir = node_cfg.head_dir in
diff --git a/src/node/node_cfg.ml b/src/node/node_cfg.ml
index a12d19ed..73b88135 100644
--- a/src/node/node_cfg.ml
+++ b/src/node/node_cfg.ml
@@ -18,7 +18,7 @@ limitations under the License.
 
 let section = Logger.Section.main
 
-let config_file = ref "cfg/arakoon.ini"
+let config_url = ref (Arakoon_url.make "cfg/arakoon.ini")
 
 let default_lease_period = 10
 let default_max_value_size = 8 * 1024 * 1024
@@ -130,7 +130,7 @@ module TLSConfig = struct
 end
 
 module Node_cfg = struct
-
+  open Lwt.Infix
   type t = {node_name:string;
             ips: string list;
             client_port:int;
@@ -423,14 +423,14 @@ module Node_cfg = struct
     in
     master
 
-  let get_nursery_cfg inifile filename =
+  let get_nursery_cfg inifile cfg_url =
     try
       begin
         let n_cluster_id = Ini.get inifile "nursery" "cluster_id" Ini.p_string Ini.required in
-        let cfg =  ClientCfg.from_file "nursery" filename in
-        Some (n_cluster_id, cfg)
+        ClientCfg.from_url "nursery" cfg_url >>= fun cfg ->
+        Some (n_cluster_id, cfg) |> Lwt.return
       end
-    with _ -> None
+    with _ -> Lwt.return_none
 
   let _get_cluster_id inifile =
     try
@@ -612,118 +612,128 @@ module Node_cfg = struct
     }
 
 
-  let read_config config_file =
-    let inifile = new Inifiles.inifile config_file in
-    let fm = _startup_mode inifile in
-    let nodes = _node_names inifile in
-    let plugin_names = _plugins inifile in
-    let cfgs, remaining = List.fold_left
-                            (fun (a,remaining) section ->
+
+    let _retrieve_cfg_from_txt txt =
+      let inifile = new Inifiles.inifile txt in
+      let fm = _startup_mode inifile in
+      let nodes = _node_names inifile in
+      let plugin_names = _plugins inifile in
+      let cfgs, remaining = List.fold_left
+                              (fun (a,remaining) section ->
                                if List.mem section nodes || _get_bool inifile section "learner"
                                then
                                  let cfg = _node_config inifile section fm in
                                  let new_remaining = List.filter (fun x -> x <> section) remaining in
                                  (cfg::a, new_remaining)
                                else (a,remaining))
-                            ([],nodes) (inifile # sects) in
-    let log_cfg_names = List.map (fun cfg -> cfg.log_config) cfgs in
-    let log_cfgs = List.fold_left
-                     (fun a section ->
+                              ([],nodes) (inifile # sects) in
+      let log_cfg_names = List.map (fun cfg -> cfg.log_config) cfgs in
+      let log_cfgs = List.fold_left
+                       (fun a section ->
                         if List.mem (Some section) log_cfg_names
                         then
                           let log_cfg = _log_config inifile section in
                           (section, log_cfg)::a
                         else
                           a)
-                     [] (inifile # sects) in
-    let batched_transaction_cfg_names = List.map (fun cfg -> cfg.batched_transaction_config) cfgs in
-    let batched_transaction_cfgs = List.fold_left
-                                     (fun a section ->
+                       [] (inifile # sects) in
+      let batched_transaction_cfg_names = List.map (fun cfg -> cfg.batched_transaction_config) cfgs in
+      let batched_transaction_cfgs = List.fold_left
+                                       (fun a section ->
                                         if List.mem (Some section) batched_transaction_cfg_names
                                         then
                                           let batched_transaction_cfg = _batched_transaction_config inifile section in
                                           (section, batched_transaction_cfg)::a
                                         else
                                           a)
-                                     [] (inifile # sects) in
-    let () = if List.length remaining > 0 then
-        failwith ("Can't find config section for: " ^ (String.concat "," remaining))
-    in
-    let quorum_function = _get_quorum_function inifile in
-    let lease_period = _get_lease_period inifile in
-    let cluster_id = _get_cluster_id inifile in
-    let m_n_cfg = get_nursery_cfg inifile config_file in
-    let overwrite_tlog_entries = _tlog_entries_overwrite inifile in
-    let max_value_size = _max_value_size inifile in
-    let max_buffer_size = _max_buffer_size inifile in
-    let client_buffer_capacity = _client_buffer_capacity inifile in
-    let lcnum = _lcnum inifile in
-    let ncnum = _ncnum inifile in
-    let tls_ca_cert = _tls_ca_cert inifile in
-    let tls_service = _tls_service inifile in
-    let tls_service_validate_peer = _tls_service_validate_peer inifile in
-    let tls_version = _tls_version inifile in
-    let tls_cipher_list = _tls_cipher_list inifile in
-    let tls = match tls_ca_cert with
-      | None -> None
-      | Some ca_cert ->
-          let cfg = TLSConfig.Cluster.make
-                      ~ca_cert
-                      ~service:tls_service
-                      ~service_validate_peer:tls_service_validate_peer
-                      ~protocol:tls_version
-                      ~cipher_list:tls_cipher_list
-          in
-          Some cfg
-    in
-    let enable_tcp_keepalive =
-      Ini.get
-        inifile "global" "enable_tcp_keepalive"
-        Ini.p_bool (Ini.default true)
-    in
-    let tcp_keepalive_time =
-      Ini.get
-        inifile "global" "tcp_keepalive_time"
-        Ini.p_int (Ini.default 20)
-    in
-    let tcp_keepalive_intvl =
-      Ini.get
-        inifile "global" "tcp_keepalive_intvl"
-        Ini.p_int (Ini.default 20)
-    in
-    let tcp_keepalive_probes =
-      Ini.get
-        inifile "global" "tcp_keepalive_probes"
-        Ini.p_int (Ini.default 3)
-    in
-    let cluster_cfg =
-      { cfgs;
-        log_cfgs;
-        batched_transaction_cfgs;
-        nursery_cfg = m_n_cfg;
-        _master = fm;
-        quorum_function;
-        _lease_period = lease_period;
-        cluster_id = cluster_id;
-        plugins = plugin_names;
-        overwrite_tlog_entries;
-        max_value_size;
-        max_buffer_size;
-        client_buffer_capacity;
-        lcnum;
-        ncnum;
-        tls;
-        tcp_keepalive = {
-            enable_tcp_keepalive;
-            tcp_keepalive_time;
-            tcp_keepalive_intvl;
-            tcp_keepalive_probes;
-          };
-      }
-    in
-    cluster_cfg
+                                       [] (inifile # sects) in
+      let () = if List.length remaining > 0 then
+                 failwith ("Can't find config section for: " ^ (String.concat "," remaining))
+      in
+      let quorum_function = _get_quorum_function inifile in
+      let lease_period = _get_lease_period inifile in
+      let cluster_id = _get_cluster_id inifile in
+      let m_n_cfg = None (* TODO: get_nursery_cfg inifile config_file *) in
+      let overwrite_tlog_entries = _tlog_entries_overwrite inifile in
+      let max_value_size = _max_value_size inifile in
+      let max_buffer_size = _max_buffer_size inifile in
+      let client_buffer_capacity = _client_buffer_capacity inifile in
+      let lcnum = _lcnum inifile in
+      let ncnum = _ncnum inifile in
+      let tls_ca_cert = _tls_ca_cert inifile in
+      let tls_service = _tls_service inifile in
+      let tls_service_validate_peer = _tls_service_validate_peer inifile in
+      let tls_version = _tls_version inifile in
+      let tls_cipher_list = _tls_cipher_list inifile in
+      let tls = match tls_ca_cert with
+        | None -> None
+        | Some ca_cert ->
+           let cfg = TLSConfig.Cluster.make
+                       ~ca_cert
+                       ~service:tls_service
+                       ~service_validate_peer:tls_service_validate_peer
+                       ~protocol:tls_version
+                       ~cipher_list:tls_cipher_list
+           in
+           Some cfg
+      in
+      let enable_tcp_keepalive =
+        Ini.get
+          inifile "global" "enable_tcp_keepalive"
+          Ini.p_bool (Ini.default true)
+      in
+      let tcp_keepalive_time =
+        Ini.get
+          inifile "global" "tcp_keepalive_time"
+          Ini.p_int (Ini.default 20)
+      in
+      let tcp_keepalive_intvl =
+        Ini.get
+          inifile "global" "tcp_keepalive_intvl"
+          Ini.p_int (Ini.default 20)
+      in
+      let tcp_keepalive_probes =
+        Ini.get
+          inifile "global" "tcp_keepalive_probes"
+          Ini.p_int (Ini.default 3)
+      in
+      let cluster_cfg =
+        { cfgs;
+          log_cfgs;
+          batched_transaction_cfgs;
+          nursery_cfg = m_n_cfg;
+          _master = fm;
+          quorum_function;
+          _lease_period = lease_period;
+          cluster_id = cluster_id;
+          plugins = plugin_names;
+          overwrite_tlog_entries;
+          max_value_size;
+          max_buffer_size;
+          client_buffer_capacity;
+          lcnum;
+          ncnum;
+          tls;
+          tcp_keepalive = {
+              enable_tcp_keepalive;
+              tcp_keepalive_time;
+              tcp_keepalive_intvl;
+              tcp_keepalive_probes;
+            };
+        }
+      in
+      cluster_cfg
+
+  let _retrieve_cfg_from_file config_file =
+    Lwt_extra.read_file config_file >>= fun txt ->
+    let cfg = _retrieve_cfg_from_txt txt in
+    Lwt.return cfg
+
+  let _retrieve_cfg_from_etcd peers path =
+    Arakoon_etcd.retrieve_value peers path >>= fun txt ->
+    let cfg = _retrieve_cfg_from_txt txt in
+    Lwt.return cfg
 
-  let read_client_config fn = to_client_cfg (read_config fn)
 
   let node_name t = t.node_name
   let home t = t.home
@@ -732,7 +742,13 @@ module Node_cfg = struct
 
   let get_master t = t.master
 
-  let get_node_cfgs_from_file () = read_config !config_file
+  let retrieve_cfg url =
+    match url with
+    | Arakoon_url.File f            -> _retrieve_cfg_from_file f
+    | Arakoon_url.Etcd (peers,path) -> _retrieve_cfg_from_etcd peers path
+
+
+  let retrieve_client_cfg url = retrieve_cfg url >>= fun cfg -> Lwt.return (to_client_cfg cfg)
 
   let test ccfg ~cluster_id = ccfg.cluster_id = cluster_id
 
diff --git a/src/node/node_cfg_test.ml b/src/node/node_cfg_test.ml
index 6c21cc35..45331aa2 100644
--- a/src/node/node_cfg_test.ml
+++ b/src/node/node_cfg_test.ml
@@ -17,10 +17,16 @@ limitations under the License.
 open OUnit
 open Node_cfg
 open Node_cfg
+open Lwt.Infix
 let test_correctness () =
 
-  let cfg = Node_cfg.read_config !config_file in
-  OUnit.assert_equal cfg.cluster_id "ricky"
+  let t =
+    Node_cfg.retrieve_cfg !config_url >>= fun cfg ->
+    let r = OUnit.assert_equal cfg.cluster_id "ricky" in
+    Lwt.return r
+  in
+  Lwt_main.run t
+
 
 
 let suite = "node_cfg" >::: ["correctness" >:: test_correctness]
diff --git a/src/node/node_main.ml b/src/node/node_main.ml
index 19b15f1d..1b546443 100644
--- a/src/node/node_main.ml
+++ b/src/node/node_main.ml
@@ -27,7 +27,7 @@ let section = Logger.Section.main
 
 
 let _config_logging me get_cfgs =
-  let cluster_cfg = get_cfgs () in
+  get_cfgs () >>= fun cluster_cfg ->
   let cfg =
     try List.find (fun c -> c.node_name = me) cluster_cfg.cfgs
     with Not_found -> failwith ("Could not find config for node " ^ me )
@@ -376,13 +376,14 @@ end
 let _main_2 (type s)
       (module S : Store.STORE with type t = s)
       make_tlog_coll
-      make_config get_snapshot_name ~name
+      (make_config: unit -> 'a Lwt.t)
+      get_snapshot_name ~name
       ~daemonize ~catchup_only
       ~autofix
       ~stop : int Lwt.t
   =
   Lwt_io.set_default_buffer_size 32768;
-  let cluster_cfg = make_config () in
+  make_config () >>= fun cluster_cfg ->
   let cfgs = cluster_cfg.cfgs in
   let me, others = Node_cfg.Node_cfg.split name cfgs in
   _config_logging me.node_name make_config >>= fun dump_crash_log ->
@@ -996,7 +997,9 @@ let _main_2 (type s)
     end
 
 
-let main_t make_config name daemonize catchup_only autofix : int Lwt.t =
+let main_t (make_config: unit -> 'a Lwt.t)
+           name daemonize catchup_only autofix : int Lwt.t
+  =
   let module S = (val (Store.make_store_module (module Batched_store.Local_store))) in
   let make_tlog_coll = Tlc2.make_tlc2 in
   let get_snapshot_name = Tlc2.head_name in
diff --git a/src/nursery/client_cfg.ml b/src/nursery/client_cfg.ml
index 46f77635..e5c1e69b 100644
--- a/src/nursery/client_cfg.ml
+++ b/src/nursery/client_cfg.ml
@@ -69,8 +69,8 @@ module ClientCfg = struct
   let get (t:t) name = Hashtbl.find t name
 
 
-  let from_file section fn =  (* This is the format as defined in the extension *)
-    let inifile = new Inifiles.inifile fn in
+  let _from_txt section txt =
+    let inifile = new Inifiles.inifile txt in
     let cfg = make () in
     let _ips node_name = Ini.get inifile node_name "ip" Ini.p_string_list Ini.required in
     let _get s n p = Ini.get inifile s n p Ini.required in
@@ -84,4 +84,17 @@ module ClientCfg = struct
     in
     cfg
 
+  let _from_file section filename =
+    Lwt_extra.read_file filename >>= fun txt ->
+    _from_txt section txt |> Lwt.return
+
+  let _from_etcd section peers path =
+    Arakoon_etcd.retrieve_value peers path >>= fun txt ->
+    _from_txt section txt |> Lwt.return
+
+  let from_url section url =  (* This is the format as defined in the extension *)
+    match url with
+    | Arakoon_url.File f -> _from_file section f
+    | Arakoon_url.Etcd (peers,path) -> _from_etcd section peers path
+
 end
diff --git a/src/nursery/client_cfg_test.ml b/src/nursery/client_cfg_test.ml
index 88df31d0..41dd4fda 100644
--- a/src/nursery/client_cfg_test.ml
+++ b/src/nursery/client_cfg_test.ml
@@ -20,34 +20,40 @@ open Client_cfg
 open OUnit
 
 let test_parsing () =
-  let lines = [
-    "[global]";
-    "cluster_id = sturdy";
-    "cluster = sturdy_0,sturdy_1, sturdy_2  ";
-    "\n";
-    "[sturdy_0]";
-    "client_port = 7080";
-    "name = sturdy_0";
-    "ip = 127.0.0.1";
-    "\n";
-    "[sturdy_1]";
-    "client_port = 7081";
-    "name = sturdy_1";
-    "ip = 127.0.0.1";
-    "\n";
-    "[sturdy_2]";
-    "ip = 127.0.0.1";
-    "client_port = 7082";
-    "name = sturdy_2"]
+  let open Lwt.Infix in
+  let t () =
+    let lines = [
+        "[global]";
+        "cluster_id = sturdy";
+        "cluster = sturdy_0,sturdy_1, sturdy_2  ";
+        "\n";
+        "[sturdy_0]";
+        "client_port = 7080";
+        "name = sturdy_0";
+        "ip = 127.0.0.1";
+        "\n";
+        "[sturdy_1]";
+        "client_port = 7081";
+        "name = sturdy_1";
+        "ip = 127.0.0.1";
+        "\n";
+        "[sturdy_2]";
+        "ip = 127.0.0.1";
+        "client_port = 7082";
+        "name = sturdy_2"]
+    in
+    let contents = List.fold_left (fun acc v -> acc ^ "\n" ^ v) "" lines in
+    let fn = "/tmp/client_cfg_test.ml" in
+    Lwt_io.with_file
+      Lwt_io.Output fn
+      (fun oc -> Lwt_io.write oc contents)
+    >>= fun () ->
+    ClientCfg.from_url "global" (Arakoon_url.File fn) >>= fun cfg ->
+    let sa0 = ClientCfg.get cfg "sturdy_0" in
+    let r = OUnit.assert_equal sa0 (["127.0.0.1"],7080) in
+    Lwt.return r
   in
-  let contents = List.fold_left (fun acc v -> acc ^ "\n" ^ v) "" lines in
-  let fn = "/tmp/client_cfg_test.ml" in
-  let oc = open_out "/tmp/client_cfg_test.ml" in
-  let () = output_string oc contents in
-  let () = close_out oc in
-  let cfg = ClientCfg.from_file "global" fn in
-  let sa0 = ClientCfg.get cfg "sturdy_0" in
-  OUnit.assert_equal sa0 (["127.0.0.1"],7080)
+  Lwt_extra.run t
 
 let suite = "client_cfg" >:::[
     "parsing" >:: test_parsing;
diff --git a/src/nursery/nursery_main.ml b/src/nursery/nursery_main.ml
index fb84cf8a..4a729a9e 100644
--- a/src/nursery/nursery_main.ml
+++ b/src/nursery/nursery_main.ml
@@ -77,16 +77,21 @@ let setup_logger file_name =
   Lwt_log.default := file_logger;
   Lwt.return ()
 
-let get_keeper_config config =
-  let inifile = new Inifiles.inifile config in
-  let m_cfg = Node_cfg.get_nursery_cfg inifile config in
+let _get_keeper_config_from_file fn url=
+  Lwt_extra.read_file fn >>= fun txt ->
+  let inifile = new Inifiles.inifile txt in
+  Node_cfg.get_nursery_cfg inifile url >>= fun m_cfg ->
   begin
     match m_cfg with
       | None -> failwith "No nursery keeper specified in config file"
       | Some (keeper_id, cli_cfg) ->
-        keeper_id, cli_cfg
+        Lwt.return (keeper_id, cli_cfg)
   end
 
+let get_keeper_config cfg_url =
+  match cfg_url with
+  | Arakoon_url.File f -> _get_keeper_config_from_file f cfg_url
+
 let get_nursery_client keeper_id cli_cfg =
   let get_nc client =
     client # get_nursery_cfg () >>= fun ncfg ->
@@ -97,13 +102,13 @@ let get_nursery_client keeper_id cli_cfg =
 
 let __migrate_nursery_range config left sep right =
   Logger.debug_ "=== STARTING MIGRATE ===" >>= fun () ->
-  let keeper_id, cli_cfg = get_keeper_config config in
+  get_keeper_config config >>= fun (keeper_id, cli_cfg) ->
   get_nursery_client keeper_id cli_cfg >>= fun nc ->
   Nursery.NC.migrate nc left sep right
 
 let __init_nursery config cluster_id =
   Logger.info_ "=== STARTING INIT ===" >>= fun () ->
-  let (keeper_id, cli_cfg) = get_keeper_config config in
+  get_keeper_config config >>= fun (keeper_id, cli_cfg) ->
   let set_routing client =
     Lwt.catch( fun () ->
         client # get_routing () >>= fun _cur ->
@@ -127,7 +132,7 @@ let __delete_from_nursery config cluster_id sep =
       else Some sep
     end
   in
-  let (keeper_id, cli_cfg) = get_keeper_config config in
+  get_keeper_config config >>= fun (keeper_id, cli_cfg) ->
   get_nursery_client keeper_id cli_cfg >>= fun nc ->
   Nursery.NC.delete nc cluster_id m_sep
 
diff --git a/src/system/drop_master.ml b/src/system/drop_master.ml
index 8b917ace..d5e35787 100644
--- a/src/system/drop_master.ml
+++ b/src/system/drop_master.ml
@@ -26,12 +26,15 @@ let stop = ref (ref false)
 let setup tn master base () =
   let lease_period = 2 in
   let stop = !stop in
-  let make_config () = Node_cfg.Node_cfg.make_test_config ~base 3 master lease_period in
+  let make_config () =
+    Node_cfg.Node_cfg.make_test_config ~base 3 master lease_period
+    |> Lwt.return
+  in
   let t0 = Node_main.test_t make_config "t_arakoon_0" ~stop >>= fun _ -> Lwt.return () in
   let t1 = Node_main.test_t make_config "t_arakoon_1" ~stop >>= fun _ -> Lwt.return () in
   (* let t2 = Node_main.test_t make_config "t_arakoon_2" stop >>= fun _ -> Lwt.return () in *)
   let all_t = [t0;t1(* ;t2 *)] in
-  Lwt.return (tn, make_config (), all_t)
+  Lwt.return (tn, make_config , all_t)
 
 let find_master cluster_cfg =
   Lwt.catch
@@ -64,7 +67,10 @@ let teardown (_tn, _, all_t) =
   stop := ref false;
   Lwt.join all_t
 
-let _drop_master do_maintenance (_tn, cluster_cfg, _) =
+let _drop_master do_maintenance
+                 (_tn, (make_cluster_cfg: unit -> cluster_cfg Lwt.t), _)
+  =
+  make_cluster_cfg () >>= fun cluster_cfg ->
   let lease_period = cluster_cfg._lease_period in
   let sp = (float lease_period) *. 1.2 in
   Lwt_unix.sleep sp >>= fun () -> (* let the cluster reach stability *)
@@ -93,26 +99,29 @@ let _drop_master do_maintenance (_tn, cluster_cfg, _) =
   let sa = Network.make_address host port in
   let cid = cluster_cfg.cluster_id in
   Lwt.pick
-    [(Lwt_io.with_connection sa
-                             (fun conn ->
-                              Remote_nodestream.make_remote_nodestream cid conn >>= fun client ->
-                              Logger.info_ "drop_master scenario" >>= fun () ->
-                              client # drop_master () >>= fun () ->
-                              if not do_maintenance
-                              then
-                                begin
-                                  Client_main.find_master ~tls:None cluster_cfg >>= fun new_master ->
-                                  Logger.info_f_ "new? master = %s" new_master >>= fun () ->
-                                  OUnit.assert_bool "master should have been changed" (new_master <> master_name);
-                                  Lwt.return ()
-                                end
-                              else
-                                  wait_until_master cluster_cfg >>= fun new_master ->
-                                  Logger.info_f_ "new? master = %s" new_master >>= fun () ->
-                                  OUnit.assert_bool "master should be the same" (new_master = master_name);
-                                  Lwt.return ()));
+    [(Lwt_io.with_connection
+        sa
+        (fun conn ->
+         Remote_nodestream.make_remote_nodestream cid conn >>= fun client ->
+         Logger.info_ "drop_master scenario" >>= fun () ->
+         client # drop_master () >>= fun () ->
+         if not do_maintenance
+         then
+           begin
+             Client_main.find_master ~tls:None cluster_cfg >>= fun new_master ->
+             Logger.info_f_ "new? master = %s" new_master >>= fun () ->
+             OUnit.assert_bool "master should have been changed" (new_master <> master_name);
+             Lwt.return ()
+           end
+         else
+           wait_until_master cluster_cfg >>= fun new_master ->
+           Logger.info_f_ "new? master = %s" new_master >>= fun () ->
+           OUnit.assert_bool "master should be the same" (new_master = master_name);
+           Lwt.return ()));
      (Lwt_unix.sleep 60. >>= fun () ->
-      Lwt.fail (Failure "drop master did not terminate quickly enough"))] >>= fun () ->
+      Lwt.fail (Failure "drop master did not terminate quickly enough"))
+    ]
+  >>= fun () ->
   if do_maintenance
   then
     Lwt_condition.signal Mem_store.defrag_condition ();
@@ -122,8 +131,8 @@ let _drop_master do_maintenance (_tn, cluster_cfg, _) =
   find_master cluster_cfg >>= fun _ ->
   Lwt.return ()
 
-let drop_master tpl = _drop_master false tpl
-let drop_master_while_maintenance tpl = _drop_master true tpl
+let drop_master                   tpl = _drop_master false tpl
+let drop_master_while_maintenance tpl = _drop_master true  tpl
 
 
 let make_suite base name w =
diff --git a/src/system/single.ml b/src/system/single.ml
index c89b1d50..832f8986 100644
--- a/src/system/single.ml
+++ b/src/system/single.ml
@@ -27,8 +27,9 @@ let should_fail = Arakoon_remote_client_test.should_fail
 let _start tn =
   Logger.info_f_ "---------------------%s--------------------" tn
 
-let all_same_master (_tn, cluster_cfg, all_t) =
+let all_same_master (_tn, make_cluster_cfg, all_t) =
   let scenario () =
+    make_cluster_cfg () >>= fun cluster_cfg ->
     let q = float (cluster_cfg._lease_period) *. 1.5 in
     Lwt_unix.sleep q >>= fun () ->
     Logger.debug_ "start of scenario" >>= fun () ->
@@ -37,6 +38,7 @@ let all_same_master (_tn, cluster_cfg, all_t) =
     let master_cfg = List.hd (List.filter (fun cfg -> cfg.node_name = master_name)
                                 cluster_cfg.cfgs)
     in
+
     Client_main.with_client
       ~tls:None ~tcp_keepalive:Node_cfg.default_tcp_keepalive
       master_cfg cluster_cfg.cluster_id set_one >>= fun () ->
@@ -78,7 +80,8 @@ let all_same_master (_tn, cluster_cfg, all_t) =
             scenario () ]
 
 
-let nothing_on_slave (_tn, cluster_cfg, all_t) =
+let nothing_on_slave (_tn, make_cluster_cfg, all_t) =
+  make_cluster_cfg () >>= fun cluster_cfg ->
   let cfgs = cluster_cfg.cfgs in
   let find_slaves cfgs =
     Client_main.find_master ~tls:None cluster_cfg >>= fun m ->
@@ -129,7 +132,8 @@ let nothing_on_slave (_tn, cluster_cfg, all_t) =
   Lwt.pick [Lwt.join all_t;
             Lwt_unix.sleep 5.0 >>= fun () -> test_slaves cluster_cfg]
 
-let dirty_on_slave (_tn, cluster_cfg,_) =
+let dirty_on_slave (_tn, make_cluster_cfg,_) =
+  make_cluster_cfg () >>= fun cluster_cfg ->
   Lwt_unix.sleep (float (cluster_cfg._lease_period)) >>= fun () ->
   Logger.debug_ "dirty_on_slave" >>= fun () ->
   let cfgs = cluster_cfg.cfgs in
@@ -574,7 +578,8 @@ let find_master ~tls cluster_cfg =
   in
   Lwt_unix.with_timeout (float timeout) go
 
-let _with_master ((_tn:string), cluster_cfg, _) f =
+let _with_master ((_tn:string), make_cluster_cfg, _) f =
+  make_cluster_cfg () >>= fun cluster_cfg ->
   let sp = float(cluster_cfg._lease_period) *. 0.5 in
   Lwt_unix.sleep sp >>= fun () -> (* let the cluster reach stability *)
   Logger.info_ "cluster should have reached stability" >>= fun () ->
@@ -657,17 +662,19 @@ let setup make_master tn base () =
   let lease_period = 10 in
   let cluster_id = Printf.sprintf "%s_%i" tn base in
   let master = make_master tn 0 in
-  let make_config () = Node_cfg.Node_cfg.make_test_config
-                         ~base ~cluster_id
-                         ~node_name:(_node_name tn)
-                         3 master lease_period
+  let make_config () =
+    Node_cfg.Node_cfg.make_test_config
+      ~base ~cluster_id
+      ~node_name:(_node_name tn)
+      3 master lease_period
+    |> Lwt.return
   in
   let stop = !stop in
   let t0 = Node_main.test_t ~stop make_config (_node_name tn 0) >>= fun _ -> Lwt.return () in
   let t1 = Node_main.test_t ~stop make_config (_node_name tn 1) >>= fun _ -> Lwt.return () in
   let t2 = Node_main.test_t ~stop make_config (_node_name tn 2) >>= fun _ -> Lwt.return () in
   let all_t = [t0;t1;t2] in
-  Lwt.return (tn, make_config (), all_t)
+  Lwt.return (tn, make_config, all_t)
 
 let teardown (tn, _, all_t) =
   !stop := true;
diff --git a/src/system/startup.ml b/src/system/startup.ml
index cad635b6..e5161d97 100644
--- a/src/system/startup.ml
+++ b/src/system/startup.ml
@@ -160,7 +160,7 @@ let post_failure () =
     tcp_keepalive = Node_cfg.default_tcp_keepalive;
   }
   in
-  let get_cfgs () = cluster_cfg in
+  let get_cfgs () = cluster_cfg |> Lwt.return in
   let v0 = Value.create_master_value ~lease_start:0. node0 in
   let v1 = Value.create_client_value [Update.Set("x","y")] false in
   let tlcs = Hashtbl.create 5 in
@@ -221,7 +221,7 @@ let restart_slaves () =
      tcp_keepalive = Node_cfg.default_tcp_keepalive;
     }
   in
-  let get_cfgs () = cluster_cfg in
+  let get_cfgs () = cluster_cfg |> Lwt.return in
   let v0 = Value.create_master_value ~lease_start:0. node2 in
   let v1 = Value.create_client_value [Update.Set("xxx","xxx")] false in
   let tlcs = Hashtbl.create 5 in
@@ -280,7 +280,7 @@ let ahead_master_loses_role () =
      tcp_keepalive = Node_cfg.default_tcp_keepalive;
     }
   in
-  let get_cfgs () = cluster_cfg in
+  let get_cfgs () = cluster_cfg |> Lwt.return in
   let v0 = Value.create_master_value ~lease_start:0. node0 in
   let v1 = Value.create_client_value [Update.Set("xxx","xxx")] false in
   let v2 = Value.create_client_value [Update.Set("invalidkey", "shouldnotbepresent")] false in
@@ -352,7 +352,7 @@ let interrupted_election () =
      tcp_keepalive = Node_cfg.default_tcp_keepalive;
     }
   in
-  let get_cfgs () = cluster_cfg in
+  let get_cfgs () = cluster_cfg |> Lwt.return in
   let v0 = Value.create_master_value ~lease_start:0. wannabe_master in
   let tlcs = Hashtbl.create 5 in
   let stores = Hashtbl.create 5 in
diff --git a/src/tools/arakoon_etcd.ml b/src/tools/arakoon_etcd.ml
new file mode 100644
index 00000000..b4023b7f
--- /dev/null
+++ b/src/tools/arakoon_etcd.ml
@@ -0,0 +1,33 @@
+open Lwt.Infix
+
+let parse_url url =
+  let host,port,path =
+    Scanf.sscanf url "%s@:%i/%s" (fun h i p -> h,i,p)
+  in
+  let peers = [host,port] in
+  peers, path
+
+let retrieve_value peers path =
+  (*
+    curl -X GET "http://127.0.0.1.:5000/v2/keys/path_to/asd/055a61c0_0"
+    or
+    etcdctl --peers=127.0.0.1:5000 get path_to/asd/055a61c0_0
+   *)
+  let peers_s = String.concat "," (List.map (fun (h,p) -> Printf.sprintf "%s:%i" h p) peers) in
+  let cmd = ["etcdctl";
+             "--peers=" ^ peers_s;
+             "get";
+             path
+            ]
+  in
+  let cmd_s = String.concat " " cmd in
+  Lwt_io.eprintlf "cmd_s: %s" cmd_s >>= fun () ->
+  let command = Lwt_process.shell cmd_s in
+  Lwt_process.with_process_in
+    command
+    (fun p ->
+     let stream = Lwt_io.read_lines (p # stdout) in
+     Lwt_stream.to_list stream >>= fun lines ->
+     let txt = String.concat "\n" lines in
+     Lwt.return txt
+    )
diff --git a/src/tools/arakoon_url.ml b/src/tools/arakoon_url.ml
new file mode 100644
index 00000000..6cb8096c
--- /dev/null
+++ b/src/tools/arakoon_url.ml
@@ -0,0 +1,31 @@
+type url =
+  File of string
+  | Etcd of (((string * int) list) * string)
+
+let to_string = function
+  | File f -> f
+  | _ -> "??"
+
+let make string =
+  let protocol, rest =
+    try
+      Scanf.sscanf string "%s@://%s" (fun p r -> p,r)
+    with End_of_file ->
+      "file", string
+  in
+  match protocol with
+  | "file" ->
+      let canonical =
+        if rest.[0] = '/'
+        then rest
+        else Filename.concat (Unix.getcwd()) rest
+      in
+      File canonical
+  | "etcd" ->
+     let host,port,path =
+       Scanf.sscanf rest "%s@:%i/%s" (fun h i p -> h,i,p)
+     in
+     let peers = [host,port]
+     in
+     Etcd (peers, path)
+  | _ -> failwith (Printf.sprintf "unknown protocol:%s" protocol)
diff --git a/src/tools/lwt_extra.ml b/src/tools/lwt_extra.ml
index bd719fe6..dec38562 100644
--- a/src/tools/lwt_extra.ml
+++ b/src/tools/lwt_extra.ml
@@ -72,6 +72,19 @@ end = struct
                 Lwt.return ()))
 end
 
+let read_file fn =
+  Lwt_io.with_file Lwt_io.Input fn
+    (fun ic ->
+     let stream = Lwt_io.read_lines ic in
+     Lwt_stream.to_list stream >>= fun lines ->
+     let buf = Buffer.create 4096 in
+     List.iter (fun line ->
+                Buffer.add_string buf line;
+                Buffer.add_char buf '\n';
+               ) lines;
+     Lwt.return (Buffer.contents buf)
+    )
+
 let run f =
   Lwt_main.run (
       Lwt.finalize

From 19fa0ea61a299534e6fb28afe2db119ee9b7dc98 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Tue, 19 Jan 2016 16:31:07 +0100
Subject: [PATCH 072/373] remove unneeded Lwt_preemptive.detach

---
 src/tlog/compression.ml |  3 +--
 src/tlog/tlogreader2.ml | 14 ++++++--------
 2 files changed, 7 insertions(+), 10 deletions(-)

diff --git a/src/tlog/compression.ml b/src/tlog/compression.ml
index 01b9b24c..5b28a718 100644
--- a/src/tlog/compression.ml
+++ b/src/tlog/compression.ml
@@ -103,8 +103,7 @@ let _compress_tlog
           let compress_and_write last_i buffer =
             let contents = Buffer.contents buffer in
             let t0 = Unix.gettimeofday () in
-            Lwt_preemptive.detach (fun () -> compress contents) ()
-            >>= fun output ->
+            let output = compress contents in
             begin
               if !cancel
               then Lwt.fail Canceled
diff --git a/src/tlog/tlogreader2.ml b/src/tlog/tlogreader2.ml
index 76013f81..45cb8df3 100644
--- a/src/tlog/tlogreader2.ml
+++ b/src/tlog/tlogreader2.ml
@@ -22,12 +22,10 @@ let section = Logger.Section.main
 let _uncompress_bz2 compressed =
   (* it exploded on the monkey when I did a Lwt_preemptive detach here *)
   let lc = String.length compressed in
-  let r = Bz2.uncompress compressed 0 lc in
-  Lwt.return r
+  Bz2.uncompress compressed 0 lc
 
 let _uncompress_snappy compressed =
-  let u = Snappy.uncompress compressed in
-  Lwt.return u
+  Snappy.uncompress compressed
 
 module Index = struct
   type index_r = { filename: string;
@@ -247,7 +245,7 @@ module C = struct
         begin
           Logger.debug_f_ "uncompressing: %i bytes"
             (String.length compressed) >>= fun () ->
-          inflate compressed >>= fun buffer ->
+          let buffer = inflate compressed in
           Logger.debug_f_ "uncompressed size: %i bytes"
             (String.length buffer) >>= fun () ->
           _fold_block a buffer 0 >>= fun a' ->
@@ -256,7 +254,7 @@ module C = struct
     in
     _skip_blocks () >>= fun compressed ->
     Logger.debug_f_ "... to _skip_in_block %i" (String.length compressed) >>= fun () ->
-    inflate compressed >>= fun buffer ->
+    let buffer = inflate compressed in
     Logger.debug_f_ "uncompressed (size=%i)" (String.length buffer) >>= fun () ->
     let maybe_first, pos = _skip_in_block buffer 0 in
     begin
@@ -353,14 +351,14 @@ module O = struct (* correct but slow folder for .tlc (aka Old) format *)
       | Some compressed ->
         begin
           Logger.debug_f_ "compressed: %i" (String.length compressed) >>= fun () ->
-          inflate compressed >>= fun buffer ->
+          let buffer = inflate compressed in
           _fold_block a buffer 0 >>= fun a' ->
           _fold_blocks a'
         end
     in
     _read_block () >>= fun compressed ->
     Logger.debug_f_ "... to _skip_in_block %i" (String.length compressed) >>= fun () ->
-    inflate compressed >>= fun buffer ->
+    let buffer = inflate compressed in
     Logger.debug_f_ "uncompressed (size=%i)" (String.length buffer) >>= fun () ->
     let maybe_first, pos = _skip_in_block buffer 0 in
     begin

From db3a24e3380e1d7d26c289ff77a9881bb2d6295c Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Tue, 19 Jan 2016 17:14:13 +0100
Subject: [PATCH 073/373] adding test suite for etcd

---
 Makefile                             |   2 +
 docker/arakoon_debian/Dockerfile     |   8 +-
 pylabs/test/server/quick/TestEtcd.py | 214 +++++++++++++++++++++++++++
 src/tools/arakoon_etcd.ml            |   1 +
 4 files changed, 224 insertions(+), 1 deletion(-)
 create mode 100644 pylabs/test/server/quick/TestEtcd.py

diff --git a/Makefile b/Makefile
index e7f81acd..ddcbce2c 100644
--- a/Makefile
+++ b/Makefile
@@ -10,6 +10,8 @@ OCAML_FIND ?= ocamlfind
 all: build
 
 clean:
+	rm -rf ./debian/tmp/*
+	rm -rf ./debian/libarakoon-ocaml-dev/*
 	ocamlbuild -clean
 
 build:
diff --git a/docker/arakoon_debian/Dockerfile b/docker/arakoon_debian/Dockerfile
index b1609e10..400fb41b 100644
--- a/docker/arakoon_debian/Dockerfile
+++ b/docker/arakoon_debian/Dockerfile
@@ -37,7 +37,7 @@ RUN apt-get -y update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
 
 RUN useradd jenkins -u 1001 -g root
 
-RUN pip install fabric junit-xml nose
+RUN pip install fabric junit-xml nose simplejson python-etcd
 
 RUN wget https://raw.github.com/ocaml/opam/master/shell/opam_installer.sh
 RUN sh ./opam_installer.sh /usr/local/bin 4.02.3
@@ -69,3 +69,9 @@ RUN eval `${opam_env}` && \
 RUN echo "jenkins ALL=NOPASSWD: ALL" >/etc/sudoers.d/jenkins
 ENV LANG=C
 RUN chmod 777 /home/opam/OPAM/4.02.3/lib
+
+# Install etcd:
+RUN curl -L  https://github.com/coreos/etcd/releases/download/v2.2.4/etcd-v2.2.4-linux-amd64.tar.gz -o etcd-v2.2.4-linux-amd64.tar.gz
+RUN tar xzvf etcd-v2.2.4-linux-amd64.tar.gz
+RUN cp ./etcd-v2.2.4-linux-amd64/etcd /usr/bin \
+    && cp ./etcd-v2.2.4-linux-amd64/etcdctl /usr/bin
diff --git a/pylabs/test/server/quick/TestEtcd.py b/pylabs/test/server/quick/TestEtcd.py
new file mode 100644
index 00000000..7fd5eb0e
--- /dev/null
+++ b/pylabs/test/server/quick/TestEtcd.py
@@ -0,0 +1,214 @@
+from Compat import X
+from .. import system_tests_common as C
+from nose.tools import *
+import etcd
+import logging
+import subprocess
+import ConfigParser
+from unittest import TestCase
+import time
+import os
+
+class EtcdClient:
+    def __init__(self,addresses, prefix):
+        logging.debug("etcd: __init__(%s,%s)", addresses, prefix)
+        self._addresses = addresses
+        self._prefix = prefix
+
+    def _get_client(self):
+        host,port = self._addresses[0]
+        client = etcd.client.Client( host = host,
+                                     port = port
+        )
+        return client
+
+    def _real_key(self,key):
+        return self._prefix + key
+
+    def get(self, key):
+        client = self._get_client()
+        k = self._real_key(key)
+        logging.debug("etcd: GET %s",k)
+        v = client.get(k)
+        return v
+
+    def set(self, key,value):
+        client = self._get_client()
+        k = self._real_key(key)
+        logging.debug("etcd: SET %s %s", k, value)
+        client.set(k,value)
+
+    def delete(self, key):
+        k = self._real_key(key)
+        logging.debug("etcd: DELETE %s",k)
+        client.delete(k)
+
+class EtcdServer:
+    def __init__(self, home, host,port, prefix):
+        self._home = home
+        self._port = port
+        self._host = host
+        self._prefix = prefix
+
+    def start(self):
+        logging.debug("ETCD:start on %s", self._home)
+        X.removeDirTree(self._home)
+        X.createDir(self._home)
+        address = "http://%s:%i" % (self._host, self._port)
+        real_cmd = [
+               'nohup', 'etcd',
+               '-advertise-client-urls=%s' % address,
+               '-listen-client-urls=%s' % address,
+               '-data-dir', self._home,
+               '>> %s/stdout' % self._home ,
+               '2>&1 &'
+               ]
+        real_cmd_s = ' '.join(real_cmd)
+        fn = '%s/start.sh' % self._home
+        with open(fn, 'w') as f:
+            print >>f, real_cmd_s
+        os.chmod(fn, 0755)
+        os.system(fn)
+
+        time.sleep(5)
+
+    def stop(self):
+        self.kill()
+
+    def url(self, key):
+        return "etcd://%s:%i/%s%s" % (self._host,
+                                      self._port,
+                                      self._prefix,key)
+    def kill(self):
+        cmd = ["fuser -n tcp %i -k" % self._port ]
+        subprocess.call(cmd, shell = True)
+
+
+class TestEtcd(TestCase):
+
+    def __init__(self, x):
+        TestCase.__init__(self,x)
+        host = "127.0.0.1"
+        port = 5000
+        prefix = "arakoon/test_etcd/"
+        self._root ='/'.join([X.tmpDir,'arakoon_system_tests', 'test_etcd' ])
+        self._etcd_home = '%s/%s' % (self._root, 'etcd_server')
+        self._server = EtcdServer(self._etcd_home, host, port, prefix)
+        self._etcdClient = EtcdClient([(host, port)], prefix)
+        self._cluster_id = "etcd_based_cluster"
+        self._arakoon_port = 4000
+        self._nodes = ['etcd_ara0']
+
+
+        ConfigParser.ConfigParser()
+
+    def _config(self):
+        cfg = ConfigParser.ConfigParser()
+
+        cfg.add_section('global')
+
+        cfg.set('global', 'cluster_id', self._cluster_id)
+        cfg.set('global', 'cluster', ','.join(self._nodes))
+        index = 0
+        for node in self._nodes:
+            cfg.add_section(node)
+            cfg.set(node, 'ip','127.0.0.1')
+            cfg.set(node, 'client_port', self._arakoon_port + index)
+            cfg.set(node, 'messaging_port', self._arakoon_port + 10 + index)
+            cfg.set(node, 'log_level', 'debug')
+            home = '%s/%s' % (self._root, node)
+            logging.debug("home=%s", home)
+            cfg.set(node, 'home', home)
+            index = index + 1
+        return cfg
+
+    def setUp(self):
+        logging.debug("setUp")
+        for node in self._nodes:
+            home = '%s/%s' % (self._root, node)
+            X.removeDirTree(home)
+            X.createDir(home)
+
+        self._server.start()
+        cfg = self._config()
+        cfg_s = X.cfg2str(cfg)
+        logging.debug("cfg_s=%s",cfg_s)
+        self._etcdClient.set(self._cluster_id,cfg_s)
+
+
+    def tearDown(self):
+        logging.debug("tearDown")
+        try:
+            subprocess.call("pkill arakoon", shell = True)
+            self._server.stop()
+        except Exception, e:
+            print e
+            try :
+                subprocess.call("pkill -9 etcd", shell = True)
+            except:
+                pass
+        time.sleep(5)
+
+    def testEtcdUsage(self):
+        logging.debug("testUsage")
+        url = self._server.url(self._cluster_id)
+        subprocess.call("pgrep -a arakoon", shell = True)
+
+        # startup
+        def _arakoon_cli(x, tail = None):
+            cmd = [C.get_arakoon_binary()]
+            cmd.extend(x)
+            cmd.extend(["-config", url])
+            if tail <> None:
+                cmd.extend(tail)
+            return cmd
+
+        cmd = _arakoon_cli(["--node", 'etcd_ara0'], tail = ["-daemonize"])
+        logging.debug('calling: %s', ' '.join(cmd))
+        r = subprocess.call(cmd, close_fds = True)
+
+        time.sleep(1.)
+
+        # cli node-state, & verify server's running
+
+        cmd = _arakoon_cli(['--node-state','etcd_ara0'])
+        logging.debug('calling: %s', ' '.join(cmd))
+        r = subprocess.check_output(cmd).strip()
+        logging.debug("node state:%s", r)
+        self.assertEquals(r, 'Stable_master')
+        # cli crud
+
+        key = "some_key"
+        value = "some_value"
+        cmd = _arakoon_cli(['--set', key, value])
+        subprocess.call(cmd)
+
+        cmd = _arakoon_cli(['--get', key])
+        v2 = subprocess.check_output(cmd).strip()
+        self.assertEquals(v2, '"%s"' % value)
+
+        cmd = _arakoon_cli(['--delete', key])
+        subprocess.call(cmd)
+        def get() :
+            cmd = _arakoon_cli(['--get', key])
+            return subprocess.check_output(cmd)
+        self.assertRaises(Exception, get)
+
+        # log_level:
+        cfg = self._config()
+        node = 'etcd_ara0'
+        cfg.set(node, 'log_level','debug')
+        cfg_s = X.cfg2str(cfg)
+        logging.debug("cfg_s= %s", cfg_s)
+        self._etcdClient.set(self._cluster_id, cfg_s)
+        cmd = ["fuser","-n","tcp", "4000","-k", "-%s" % 'USR1' ]
+        subprocess.call(cmd)
+        time.sleep(5)
+        # check if we see stuff in logging
+        log_file = '%s/%s/%s.log' % (self._root, node, node)
+        tail = subprocess.check_output("tail -40 %s" % log_file, shell = True)
+        logging.debug("tail of %s = %s", log_file, tail)
+        cmd = "grep ETCD %s | wc" % log_file
+        r = subprocess.check_output(cmd, shell =True).strip().split()[0]
+        logging.debug("#ETCD counts:%s", r)
+        self.assertTrue(int(r)>= 1) # note:first retrieval dumps to stderr
diff --git a/src/tools/arakoon_etcd.ml b/src/tools/arakoon_etcd.ml
index b4023b7f..6bd329c3 100644
--- a/src/tools/arakoon_etcd.ml
+++ b/src/tools/arakoon_etcd.ml
@@ -22,6 +22,7 @@ let retrieve_value peers path =
   in
   let cmd_s = String.concat " " cmd in
   Lwt_io.eprintlf "cmd_s: %s" cmd_s >>= fun () ->
+  Lwt_log.info_f  "ETCD: %s" cmd_s >>= fun () ->
   let command = Lwt_process.shell cmd_s in
   Lwt_process.with_process_in
     command

From c1abff67d5659ace6370cd4c1b1fc2a967d9e7ca Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Thu, 21 Jan 2016 13:32:04 +0100
Subject: [PATCH 074/373] added generic retrieve

---
 pylabs/test/server/quick/TestEtcd.py | 16 ++++++++++++++++
 src/inifiles/inifiles_test.ml        | 16 ++++++++++++++++
 src/node/node_cfg.ml                 | 20 +++-----------------
 src/nursery/nursery_main.ml          | 12 ++++++++----
 src/tools/arakoon_etcd.ml            | 22 ++++++++++++++++++++++
 src/tools/arakoon_url.ml             | 16 ++++++++++++++++
 src/tools/lwt_extra.ml               | 22 +++++++++++-----------
 7 files changed, 92 insertions(+), 32 deletions(-)

diff --git a/pylabs/test/server/quick/TestEtcd.py b/pylabs/test/server/quick/TestEtcd.py
index 7fd5eb0e..25a55c15 100644
--- a/pylabs/test/server/quick/TestEtcd.py
+++ b/pylabs/test/server/quick/TestEtcd.py
@@ -1,3 +1,19 @@
+"""
+Copyright 2016 iNuron NV
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
 from Compat import X
 from .. import system_tests_common as C
 from nose.tools import *
diff --git a/src/inifiles/inifiles_test.ml b/src/inifiles/inifiles_test.ml
index f5ad9f2b..31feb84f 100644
--- a/src/inifiles/inifiles_test.ml
+++ b/src/inifiles/inifiles_test.ml
@@ -1,3 +1,19 @@
+(*
+Copyright 2016 iNuron NV
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+ *)
+
 let test_parsing () =
   let txt =
     let lines =[
diff --git a/src/node/node_cfg.ml b/src/node/node_cfg.ml
index 73b88135..2716ab1a 100644
--- a/src/node/node_cfg.ml
+++ b/src/node/node_cfg.ml
@@ -722,31 +722,17 @@ module Node_cfg = struct
             };
         }
       in
-      cluster_cfg
-
-  let _retrieve_cfg_from_file config_file =
-    Lwt_extra.read_file config_file >>= fun txt ->
-    let cfg = _retrieve_cfg_from_txt txt in
-    Lwt.return cfg
-
-  let _retrieve_cfg_from_etcd peers path =
-    Arakoon_etcd.retrieve_value peers path >>= fun txt ->
-    let cfg = _retrieve_cfg_from_txt txt in
-    Lwt.return cfg
-
+      cluster_cfg |> Lwt.return
 
   let node_name t = t.node_name
+
   let home t = t.home
 
   let client_addresses t = (t.ips, t.client_port)
 
   let get_master t = t.master
 
-  let retrieve_cfg url =
-    match url with
-    | Arakoon_url.File f            -> _retrieve_cfg_from_file f
-    | Arakoon_url.Etcd (peers,path) -> _retrieve_cfg_from_etcd peers path
-
+  let retrieve_cfg url = Arakoon_etcd.retrieve_cfg _retrieve_cfg_from_txt url
 
   let retrieve_client_cfg url = retrieve_cfg url >>= fun cfg -> Lwt.return (to_client_cfg cfg)
 
diff --git a/src/nursery/nursery_main.ml b/src/nursery/nursery_main.ml
index 4a729a9e..b168a4a3 100644
--- a/src/nursery/nursery_main.ml
+++ b/src/nursery/nursery_main.ml
@@ -77,8 +77,8 @@ let setup_logger file_name =
   Lwt_log.default := file_logger;
   Lwt.return ()
 
-let _get_keeper_config_from_file fn url=
-  Lwt_extra.read_file fn >>= fun txt ->
+let _get_keeper_config_generic fetcher url=
+  fetcher() >>= fun txt ->
   let inifile = new Inifiles.inifile txt in
   Node_cfg.get_nursery_cfg inifile url >>= fun m_cfg ->
   begin
@@ -89,8 +89,12 @@ let _get_keeper_config_from_file fn url=
   end
 
 let get_keeper_config cfg_url =
-  match cfg_url with
-  | Arakoon_url.File f -> _get_keeper_config_from_file f cfg_url
+  let fetcher =
+    match cfg_url with
+    | Arakoon_url.File f             -> (fun () -> Lwt_extra.read_file f)
+    | Arakoon_url.Etcd (peers, path) -> (fun () -> Arakoon_etcd.retrieve_value peers path)
+  in
+  _get_keeper_config_generic fetcher cfg_url
 
 let get_nursery_client keeper_id cli_cfg =
   let get_nc client =
diff --git a/src/tools/arakoon_etcd.ml b/src/tools/arakoon_etcd.ml
index 6bd329c3..0e8910c2 100644
--- a/src/tools/arakoon_etcd.ml
+++ b/src/tools/arakoon_etcd.ml
@@ -1,3 +1,19 @@
+(*
+Copyright 2016 iNuron NV
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+ *)
+
 open Lwt.Infix
 
 let parse_url url =
@@ -32,3 +48,9 @@ let retrieve_value peers path =
      let txt = String.concat "\n" lines in
      Lwt.return txt
     )
+
+let retrieve_cfg (p:string -> 'a Lwt.t) = function
+  | Arakoon_url.File cfg_file ->
+     Lwt_extra.read_file cfg_file >>= p
+  | Arakoon_url.Etcd (peers,path)->
+     retrieve_value peers path >>= p
diff --git a/src/tools/arakoon_url.ml b/src/tools/arakoon_url.ml
index 6cb8096c..c77ccd4b 100644
--- a/src/tools/arakoon_url.ml
+++ b/src/tools/arakoon_url.ml
@@ -1,3 +1,19 @@
+(*
+Copyright 2016 iNuron NV
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*)
+
 type url =
   File of string
   | Etcd of (((string * int) list) * string)
diff --git a/src/tools/lwt_extra.ml b/src/tools/lwt_extra.ml
index dec38562..71e1a019 100644
--- a/src/tools/lwt_extra.ml
+++ b/src/tools/lwt_extra.ml
@@ -72,18 +72,18 @@ end = struct
                 Lwt.return ()))
 end
 
-let read_file fn =
-  Lwt_io.with_file Lwt_io.Input fn
-    (fun ic ->
-     let stream = Lwt_io.read_lines ic in
-     Lwt_stream.to_list stream >>= fun lines ->
-     let buf = Buffer.create 4096 in
-     List.iter (fun line ->
-                Buffer.add_string buf line;
-                Buffer.add_char buf '\n';
-               ) lines;
-     Lwt.return (Buffer.contents buf)
+let read_txt ic =
+  let stream = Lwt_io.read_lines ic in
+  let buf = Buffer.create 4096 in
+  Lwt_stream.iter
+    (fun line ->
+     Buffer.add_string buf line;
+     Buffer.add_char buf '\n'
     )
+    stream >>= fun () ->
+  Lwt.return (Buffer.contents buf)
+
+let read_file fn = Lwt_io.with_file Lwt_io.Input fn read_txt
 
 let run f =
   Lwt_main.run (

From e3f7545ce1fd42def43f46983b3c9e8eb033a53b Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Fri, 22 Jan 2016 09:14:23 +0100
Subject: [PATCH 075/373] works again for inifiles with comments

---
 src/inifiles/inifiles.ml      | 12 +++++++++++-
 src/inifiles/inifiles_test.ml | 32 +++++++++++++++++++++-----------
 src/nursery/client_cfg.ml     | 15 ++-------------
 3 files changed, 34 insertions(+), 25 deletions(-)

diff --git a/src/inifiles/inifiles.ml b/src/inifiles/inifiles.ml
index 1dbd4d1f..c3586c77 100644
--- a/src/inifiles/inifiles.ml
+++ b/src/inifiles/inifiles.ml
@@ -95,7 +95,17 @@ let rec filterfile ?(buf=Buffer.create 500) f fd =
   with End_of_file -> Buffer.contents buf
 
 let parse_inifile txt tbl =
-  let lxbuf = Lexing.from_string (txt ^ "\n") in (* otherwise, it might fail for a silly reason *)
+  let lxbuf =
+    let lines = Str.split (Str.regexp "\n") txt in
+    let lines' = List.filter (fun line -> not (Str.string_match comment line 0)) lines in
+    let buf = Buffer.create (String.length txt) in
+    let () =List.iter
+      (fun line -> Buffer.add_string buf line;
+                   Buffer.add_char buf '\n'
+      ) lines' in
+    let txt' = Buffer.contents buf in
+    Lexing.from_string txt'
+  in
   try
     let parsed_txt = Parseini.inifile Inilexer.lexini lxbuf in
     List.iter
diff --git a/src/inifiles/inifiles_test.ml b/src/inifiles/inifiles_test.ml
index 31feb84f..cf4cd791 100644
--- a/src/inifiles/inifiles_test.ml
+++ b/src/inifiles/inifiles_test.ml
@@ -14,7 +14,15 @@ See the License for the specific language governing permissions and
 limitations under the License.
  *)
 
-let test_parsing () =
+let _parse txt =
+  try
+    let _cfg = Node_cfg.Node_cfg._retrieve_cfg_from_txt txt in ()
+  with (Inifiles.Ini_parse_error(lnum,txt) as exn)->
+    let () =
+      Printf.printf "lnum:%i\n%s" lnum txt in
+    raise exn
+
+let test_parsing1 () =
   let txt =
     let lines =[
       "[global]"
@@ -49,20 +57,22 @@ let test_parsing () =
       ;"fsync = false"
       ]
     in
-    (*let lines = ["[global]\ncluster = x,y,z\n";] in*)
     String.concat "\n" lines
   in
-  let () =
-    try
-      let _cfg = Node_cfg.Node_cfg._retrieve_cfg_from_txt txt in ()
-    with (Inifiles.Ini_parse_error(lnum,txt) as exn)->
-      let () =
-        Printf.printf "lnum:%i\n%s" lnum txt in
-      raise exn
+  _parse txt
+
+let test_parsing2 () =
+  let txt =
+    let open Lwt.Infix in
+    Lwt_extra.run
+      (fun () ->
+       Lwt_extra.read_file "./cfg/arakoon.ini"
+      )
   in
-  ()
+  _parse txt
 
 open OUnit
 let suite = "inifiles" >:::[
-      "parsing" >:: test_parsing;
+      "parsing1" >:: test_parsing1;
+      "parsing2" >:: test_parsing2;
     ]
diff --git a/src/nursery/client_cfg.ml b/src/nursery/client_cfg.ml
index e5c1e69b..b41a271b 100644
--- a/src/nursery/client_cfg.ml
+++ b/src/nursery/client_cfg.ml
@@ -82,19 +82,8 @@ module ClientCfg = struct
                   add cfg n (ips,port)
                ) nodes
     in
-    cfg
+    cfg |> Lwt.return
 
-  let _from_file section filename =
-    Lwt_extra.read_file filename >>= fun txt ->
-    _from_txt section txt |> Lwt.return
-
-  let _from_etcd section peers path =
-    Arakoon_etcd.retrieve_value peers path >>= fun txt ->
-    _from_txt section txt |> Lwt.return
-
-  let from_url section url =  (* This is the format as defined in the extension *)
-    match url with
-    | Arakoon_url.File f -> _from_file section f
-    | Arakoon_url.Etcd (peers,path) -> _from_etcd section peers path
+  let from_url section url = Arakoon_etcd.retrieve_cfg (_from_txt section) url
 
 end

From c0c521d32f900b555065792aedce587c81d3613f Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Fri, 22 Jan 2016 09:35:21 +0100
Subject: [PATCH 076/373] don't refetch inifile that's already in our
 possession

---
 src/node/node_cfg.ml        | 10 +++++-----
 src/nursery/client_cfg.ml   | 13 +++++++++----
 src/nursery/nursery_main.ml |  2 +-
 3 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/src/node/node_cfg.ml b/src/node/node_cfg.ml
index 2716ab1a..dc694fcb 100644
--- a/src/node/node_cfg.ml
+++ b/src/node/node_cfg.ml
@@ -423,14 +423,14 @@ module Node_cfg = struct
     in
     master
 
-  let get_nursery_cfg inifile cfg_url =
+  let get_nursery_cfg inifile =
     try
       begin
         let n_cluster_id = Ini.get inifile "nursery" "cluster_id" Ini.p_string Ini.required in
-        ClientCfg.from_url "nursery" cfg_url >>= fun cfg ->
-        Some (n_cluster_id, cfg) |> Lwt.return
+        let cfg = ClientCfg.from_ini "nursery" inifile in
+        Some (n_cluster_id, cfg)
       end
-    with _ -> Lwt.return_none
+    with _ -> None
 
   let _get_cluster_id inifile =
     try
@@ -653,7 +653,7 @@ module Node_cfg = struct
       let quorum_function = _get_quorum_function inifile in
       let lease_period = _get_lease_period inifile in
       let cluster_id = _get_cluster_id inifile in
-      let m_n_cfg = None (* TODO: get_nursery_cfg inifile config_file *) in
+      let m_n_cfg = get_nursery_cfg inifile in
       let overwrite_tlog_entries = _tlog_entries_overwrite inifile in
       let max_value_size = _max_value_size inifile in
       let max_buffer_size = _max_buffer_size inifile in
diff --git a/src/nursery/client_cfg.ml b/src/nursery/client_cfg.ml
index b41a271b..7723daf2 100644
--- a/src/nursery/client_cfg.ml
+++ b/src/nursery/client_cfg.ml
@@ -69,8 +69,7 @@ module ClientCfg = struct
   let get (t:t) name = Hashtbl.find t name
 
 
-  let _from_txt section txt =
-    let inifile = new Inifiles.inifile txt in
+  let from_ini section inifile =
     let cfg = make () in
     let _ips node_name = Ini.get inifile node_name "ip" Ini.p_string_list Ini.required in
     let _get s n p = Ini.get inifile s n p Ini.required in
@@ -82,8 +81,14 @@ module ClientCfg = struct
                   add cfg n (ips,port)
                ) nodes
     in
-    cfg |> Lwt.return
+    cfg
+
+  let _from_txt section txt =
+    let inifile = new Inifiles.inifile txt in
+    from_ini section inifile
+
+  let _from_txt_lwt section txt =_from_txt section txt |> Lwt.return
 
-  let from_url section url = Arakoon_etcd.retrieve_cfg (_from_txt section) url
+  let from_url section url = Arakoon_etcd.retrieve_cfg (_from_txt_lwt section) url
 
 end
diff --git a/src/nursery/nursery_main.ml b/src/nursery/nursery_main.ml
index b168a4a3..809c4680 100644
--- a/src/nursery/nursery_main.ml
+++ b/src/nursery/nursery_main.ml
@@ -80,7 +80,7 @@ let setup_logger file_name =
 let _get_keeper_config_generic fetcher url=
   fetcher() >>= fun txt ->
   let inifile = new Inifiles.inifile txt in
-  Node_cfg.get_nursery_cfg inifile url >>= fun m_cfg ->
+  let m_cfg = Node_cfg.get_nursery_cfg inifile in
   begin
     match m_cfg with
       | None -> failwith "No nursery keeper specified in config file"

From 5318959700b7df73b517cb1051e0dae408a2c05a Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Fri, 22 Jan 2016 11:07:10 +0100
Subject: [PATCH 077/373] Etcd url string repr

---
 src/tools/arakoon_url.ml | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/tools/arakoon_url.ml b/src/tools/arakoon_url.ml
index c77ccd4b..e0bc2c9f 100644
--- a/src/tools/arakoon_url.ml
+++ b/src/tools/arakoon_url.ml
@@ -20,7 +20,14 @@ type url =
 
 let to_string = function
   | File f -> f
-  | _ -> "??"
+  | Etcd (peers,path) ->
+     let peers_s =
+       List.map
+         (fun (host,port) -> Printf.sprintf "%s:%i" host port)
+         peers
+       |> String.concat ","
+     in
+     Printf.sprintf "etcd://%s/%s" peers_s path
 
 let make string =
   let protocol, rest =

From 5bd79eb1546b44d474e0516ef635bee379cc0d39 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Mon, 25 Jan 2016 11:24:48 +0100
Subject: [PATCH 078/373] make_release: 1.8.13

---
 META                      | 2 +-
 debian/changelog          | 5 +++++
 redhat/SPECS/arakoon.spec | 4 +++-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/META b/META
index 58b87837..fd732960 100644
--- a/META
+++ b/META
@@ -1,7 +1,7 @@
 # -*- conf -*-
 
 description = "Arakoon client"
-version = "1.8.12"
+version = "1.8.13"
 exists_if = "arakoon_client.cma,arakoon_client.cmxa,arakoon_client.cmxs"
 requires = "threads lwt lwt.unix str oUnit quickcheck lwt.ssl ocplib-endian"
 archive(byte) = "arakoon_client.cma tcp_keepalive_sockopt_stubs.o"
diff --git a/debian/changelog b/debian/changelog
index 2aed312d..ba1f7a0b 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,8 @@
+arakoon (1.8.13) unstable; urgency=low
+  * new upstream version
+
+ -- Romain Slootmaekers <romain.slootmaekers@openvstorage.com>  Mon, 25 Jan 2016 11:24:48 +0100
+
 arakoon (1.8.12+etcd) unstable; urgency=low
   * new upstream version
 
diff --git a/redhat/SPECS/arakoon.spec b/redhat/SPECS/arakoon.spec
index e0169922..756d4195 100644
--- a/redhat/SPECS/arakoon.spec
+++ b/redhat/SPECS/arakoon.spec
@@ -2,7 +2,7 @@
 
 Summary: Arakoon
 Name: arakoon
-Version: 1.8.12
+Version: 1.8.13
 Release: 3%{?dist}
 License: Apache 2.0
 Requires: libev >= 4
@@ -33,6 +33,8 @@ cp arakoon.native %{buildroot}%{_bindir}/arakoon
 %{_bindir}/arakoon
 
 %changelog
+* Mon Jan 25 2016 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.8.13
+- Create arakoon 1.8.13 RPM package
 * Wed Dec 23 2015 Jan Doms <jan.doms@gmail.com> - 1.8.12
 - Create arakoon 1.8.12 RPM package
 * Fri Dec 11 2015 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.8.11

From a2227b751dcbf3a8fff422b6a490cc0bbb017b94 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Tue, 26 Jan 2016 09:58:30 +0100
Subject: [PATCH 079/373] tlog_map

---
 COMPILING                       |   5 +-
 cfg/arakoon.ini                 |  12 +-
 src/main/arakoon.ml             |   9 +-
 src/main/replay_main.ml         |  13 +-
 src/node/catchup.ml             | 102 ++-----
 src/node/catchup_test.ml        |   6 +-
 src/node/collapser.ml           |  34 +--
 src/node/collapser_test.ml      |  30 +-
 src/node/dump_store.ml          |  12 +-
 src/node/node_cfg.ml            |  93 ++++---
 src/node/node_main.ml           |  30 +-
 src/node/sync_backend.ml        |  11 +-
 src/system/startup.ml           |  49 ++--
 src/tlog/mem_tlogcollection.ml  |   9 +-
 src/tlog/tlc2.ml                | 415 ++++++----------------------
 src/tlog/tlc2_test.ml           | 123 +++++----
 src/tlog/tlog_main.ml           |   4 +-
 src/tlog/tlog_map.ml            | 466 ++++++++++++++++++++++++++++++++
 src/tlog/tlogcollection.ml      |  19 +-
 src/tlog/tlogcollection_test.ml | 105 +++----
 src/tlog/tlogcommon.ml          |   3 -
 21 files changed, 903 insertions(+), 647 deletions(-)
 create mode 100644 src/tlog/tlog_map.ml

diff --git a/COMPILING b/COMPILING
index d30d90a1..cc426352 100644
--- a/COMPILING
+++ b/COMPILING
@@ -28,9 +28,10 @@ RUNNING THE TESTS:
 	./arakoon.native --run-all-tests
 
 COVERAGE:
-Bisect needs to be installed. (get it from http://bisect.x9c.fr/)
+Bisect needs to be installed. 
 
-	ocamlbuild -tag use_bisect -tag use_macro arakoon.d.byte
+	make coverage
+        ./arakoon.d.byte --run-all-tests
 	./report.sh
 
 DEBIAN packages:
diff --git a/cfg/arakoon.ini b/cfg/arakoon.ini
index e7e9ef66..1fa9cb19 100644
--- a/cfg/arakoon.ini
+++ b/cfg/arakoon.ini
@@ -1,6 +1,6 @@
 [global]
 #comma separated list of node names that are part of the cluster
-cluster = arakoon_0 , arakoon_1, witness_0
+cluster = arakoon_0 
 #optional:
 #master = arakoon_0
 # modifier for master, if preferred, the master is no longer forced
@@ -67,6 +67,12 @@ cluster_id = ricky
 #
 # tls_cipher_list =
 
+
+# when to rotate a tlog
+tlog_max_entries =    10_000    
+tlog_max_size    = 5_000_000 # in bytes
+
+
 [default_log_config]
 # available levels are: debug info notice warning error fatal
 client_protocol = debug
@@ -79,6 +85,7 @@ tcp_messaging = debug
 #max_size    = 100000 # in bytes
 
 
+
 [arakoon_0]
 #comma separated list of all ip addresses used to identify that node
 ip = 127.0.0.1
@@ -88,7 +95,7 @@ home = /tmp/arakoon/arakoon_0
 #optional different directory for .tlog, .tlc and friends
 #tlog_dir = /tmp/arakoon/arakoon_0
 #log_dir = /tmp/arakoon/arakoon_0
-#tlog_compression = bz2 # default is bz2; valid choices are: none snappy bz2
+#tlog_compression = snappy # default is snappy; valid choices are: none snappy bz2
 
 fsync=true
 
@@ -142,7 +149,6 @@ log_level = info
 log_config = default_log_config
 
 
-
 [arakoon_2]
 ip = 127.0.0.1
 client_port = 4002
diff --git a/src/main/arakoon.ml b/src/main/arakoon.ml
index d53b3634..e0c87a61 100644
--- a/src/main/arakoon.ml
+++ b/src/main/arakoon.ml
@@ -86,7 +86,6 @@ let show_version ()=
     Printf.printf "compiled: %S\n" compile_time;
     Printf.printf "machine: %S\n" machine;
     Printf.printf "compiler_version : %S\n" compiler_version;
-    Printf.printf "tlogEntriesPerFile: %i\n" !Tlogcommon.tlogEntriesPerFile;
     Printf.printf "dependencies:\n%s\n" dependencies in
   ()
 
@@ -440,7 +439,7 @@ let main () =
     | StripTlog -> Tlog_main.strip_tlog !filename
     | MakeTlog -> Tlog_main.make_tlog !filename !counter
     | MarkTlog -> Tlog_main.mark_tlog !filename !key
-    | CloseTlog -> Tlog_main.mark_tlog !filename (Tlc2._make_close_marker !node_id)
+    | CloseTlog -> Tlog_main.mark_tlog !filename (Tlog_map._make_close_marker !node_id)
     | ReplayTlogs -> Replay_main.replay_tlogs !tlog_dir !tlf_dir !filename !end_i
     | DumpStore -> Dump_store.dump_store !filename
     | TruncateTlog -> Tlc2.truncate_tlog !filename
@@ -518,7 +517,11 @@ let main () =
               !node_id !daemonize !catchup_only !autofix
           in
           (* Lwt_engine.set (new Lwt_engine.select :> Lwt_engine.t); *)
-          Lwt_main.run main_t
+          try
+            Lwt_main.run main_t
+          with Failure msg ->
+            (* don't bubble stacktrace: the msg is the explaination we need *)
+            Printf.eprintf "problem: %s\n%!" msg; 66
         end
       | TestNode ->
         begin
diff --git a/src/main/replay_main.ml b/src/main/replay_main.ml
index da1cf4d4..e522183b 100644
--- a/src/main/replay_main.ml
+++ b/src/main/replay_main.ml
@@ -18,7 +18,7 @@ open Lwt
 
 module S = (val (Store.make_store_module (module Batched_store.Local_store)))
 
-let replay_tlogs tlog_dir tlf_dir db_name end_i =
+let replay_tlogs tlog_dir tlx_dir db_name end_i =
   let console x = Lwt_io.eprintlf x in
   let maybe_log_i start_i too_far_i pi =
     let (+:) = Sn.add
@@ -37,8 +37,10 @@ let replay_tlogs tlog_dir tlf_dir db_name end_i =
       | _ -> Lwt.return ()
   in
   let t () =
+    let open Tlog_map in
     Lwt.catch
       (fun () ->
+         TlogMap.make tlog_dir tlx_dir "node_name???" >>= fun tlog_map ->
          S.make_store ~lcnum:1024 ~ncnum:512 db_name >>= fun store ->
          let cio = S.consensus_i store in
          let cios = Log_extra.option2s Sn.string_of cio in
@@ -47,9 +49,10 @@ let replay_tlogs tlog_dir tlf_dir db_name end_i =
            match
              end_i with
              | None ->
-               begin
-                 Tlc2.get_last_tlog tlog_dir tlf_dir >>= fun (_new_c,fn) ->
-                 Tlc2._validate_one fn "" ~check_marker:false >>= fun (last, _index) ->
+                begin
+
+                 TlogMap.get_last_tlog tlog_map >>= fun (_new_c,fn) ->
+                 Tlog_map._validate_one fn "" ~check_marker:false >>= fun (last, _index, _pos) ->
                  let i =
                    match last with
                      | None -> Sn.start
@@ -72,7 +75,7 @@ let replay_tlogs tlog_dir tlf_dir db_name end_i =
              let acc = ref None in
              let log_i pi = maybe_log_i start_i too_far_i pi in
              let f = Catchup.make_f ~stop:(ref false) (module S) db_name log_i acc store in
-             Tlc2.iterate_tlog_dir tlog_dir tlf_dir ~index:None
+             TlogMap.iterate_tlog_dir tlog_map ~index:None
                start_i
                too_far_i
                f
diff --git a/src/node/catchup.ml b/src/node/catchup.ml
index 65499c2f..1a764a4e 100644
--- a/src/node/catchup.ml
+++ b/src/node/catchup.ml
@@ -210,17 +210,16 @@ let catchup_store ~stop (type s) me
               Printf.sprintf
                 "Catchup store failed. Store counter is too low: %s < %s"
                 (Sn.string_of si) (Sn.string_of pred_too_far_i)
-            and n_entries = Sn.of_int !Tlogcommon.tlogEntriesPerFile in
+            and n_entries = failwith "todo" (*Sn.of_int !Tlogcommon.tlogEntriesPerFile *) in
             if Sn.rem si n_entries = Sn.pred n_entries
             then
               let ssi = Sn.succ si in
               tlog_coll # which_tlog_file ssi >>= function
               | None ->
                 let n = tlog_coll # get_tlog_from_i ssi in
-                let ni = Sn.to_int n in
                 Lwt.return
                   (Printf.sprintf
-                     "%s (found tlog nor archive for %3i)" msg ni)
+                     "%s (found tlog nor archive for %3i)" msg n)
               | Some _ -> Lwt.return msg
             else
               Lwt.return msg
@@ -245,7 +244,10 @@ let catchup
   catchup_store ~stop me dbt (Sn.succ until_i) >>= fun () ->
   Logger.info_ "CATCHUP end"
 
-let verify_n_catchup_store (type s) ~stop me ?(apply_last_tlog_value=false) ((module S : Store.STORE with type t = s), store, tlog_coll) =
+let verify_n_catchup_store
+      (type s) ~stop me ?(apply_last_tlog_value=false)
+      ((module S : Store.STORE with type t = s), store, tlog_coll)
+  =
   let current_i = tlog_coll # get_last_i () in
   let too_far_i =
     if apply_last_tlog_value
@@ -255,15 +257,17 @@ let verify_n_catchup_store (type s) ~stop me ?(apply_last_tlog_value=false) ((mo
       current_i in
   let io_s = Log_extra.option2s Sn.string_of  in
   let si_o = S.consensus_i store in
-  Logger.info_f_ "verify_n_catchup_store; too_far_i=%s current_i=%s si_o:%s"
-    (Sn.string_of too_far_i) (Sn.string_of current_i) (io_s si_o) >>= fun () ->
-   match too_far_i, si_o with
+
+  Logger.info_f_
+    "verify_n_catchup_store; too_far_i=%s current_i=%s si_o:%s"
+    (Sn.string_of too_far_i) (Sn.string_of current_i) (io_s si_o)
+  >>= fun () ->
+
+  match too_far_i, si_o with
     | i, None when i <= 0L -> Lwt.return ()
     | i, Some j when i = j -> Lwt.return ()
-    | i, Some j when i > j ->
-      catchup_store ~stop me ((module S),store,tlog_coll) too_far_i
-    | _, None ->
-      catchup_store ~stop me ((module S),store,tlog_coll) too_far_i
+    | i, Some j when i > j -> catchup_store ~stop me ((module S),store,tlog_coll) too_far_i
+    | _, None              -> catchup_store ~stop me ((module S),store,tlog_coll) too_far_i
     | _,_ ->
       let msg = Printf.sprintf
         "too_far_i:%s, si_o:%s should not happen: tlogs have been removed?"
@@ -281,71 +285,8 @@ let last_entries
   =
   (* This one is kept (for how long?)
      for x-version clusters during upgrades
-  *)
-  Logger.warning_f_ "DEPRECATED : last_entries " >>= fun () ->
-  Logger.debug_f_ "last_entries %s" (Sn.string_of start_i) >>= fun () ->
-  let consensus_i = S.consensus_i store in
-  begin
-    match consensus_i with
-      | None -> Lwt.return ()
-      | Some ci ->
-        begin
-          tlog_collection # get_infimum_i () >>= fun inf_i ->
-          let too_far_i = Sn.succ ci in
-          Logger.debug_f_
-            "inf_i:%s too_far_i:%s" (Sn.string_of inf_i)
-            (Sn.string_of too_far_i)
-          >>= fun () ->
-          begin
-            if start_i < inf_i
-            then
-              begin
-                Llio.output_int oc 2 >>= fun () ->
-                tlog_collection # dump_head oc
-              end
-            else
-              Lwt.return start_i
-          end
-          >>= fun start_i2->
-
-
-          let step = Sn.of_int (!Tlogcommon.tlogEntriesPerFile) in
-          let rec loop_parts (start_i2:Sn.t) =
-            if Sn.rem start_i2 step = Sn.start &&
-               Sn.add start_i2 step < too_far_i
-            then
-              begin
-                Logger.debug_f_ "start_i2=%Li < %Li" start_i2 too_far_i
-                >>= fun () ->
-                Llio.output_int oc 3 >>= fun () ->
-                tlog_collection # dump_tlog_file start_i2 oc
-                >>= fun start_i2' ->
-                loop_parts start_i2'
-              end
-            else
-              Lwt.return start_i2
-          in
-          loop_parts start_i2
-          >>= fun start_i3 ->
-          Llio.output_int oc 1 >>= fun () ->
-          let f entry =
-            let i = Entry.i_of entry
-            and v = Entry.v_of entry
-            in
-            Tlogcommon.write_entry oc i v >>= fun _total_size ->
-            Lwt.return ()
-          in
-          Lwt.catch
-            (fun () -> tlog_collection # iterate start_i3 too_far_i f)
-            (function
-              | Tlogcommon.TLogUnexpectedEndOfFile _ -> Lwt.return ()
-              | ex -> Lwt.fail ex) >>= fun () ->
-          Sn.output_sn oc (-1L)
-        end
-  end
-  >>= fun () ->
-  Logger.info_f_ "done with_last_entries"
-
+   *)
+  Lwt.fail_with "no longer supported"
 
 let last_entries2
       (type s) (module S : Store.STORE with type t = s)
@@ -360,9 +301,6 @@ let last_entries2
     match consensus_i with
       | None -> Lwt.return ()
       | Some ci ->
-
-        let step = Sn.of_int (!Tlogcommon.tlogEntriesPerFile) in
-
         let stream_entries start_i too_far_i =
           let _write_entry entry =
             let i = Entry.i_of entry
@@ -381,9 +319,11 @@ let last_entries2
           Sn.output_sn oc (-1L)
         in
         let too_far_i = Sn.succ ci in
+        let step = failwith "todo" (*Sn.of_int (!Tlogcommon.tlogEntriesPerFile) *) in
         let rec loop_files (start_i2:Sn.t) =
-          if Sn.rem start_i2 step = Sn.start &&
-             Sn.add start_i2 step < too_far_i
+          if (* there are complete log files *)
+            Sn.rem start_i2 step = Sn.start &&
+              Sn.add start_i2 step < too_far_i
           then
             begin
               Logger.debug_f_ "start_i2=%Li < %Li" start_i2 too_far_i
diff --git a/src/node/catchup_test.ml b/src/node/catchup_test.ml
index d0fc0713..c8c18055 100644
--- a/src/node/catchup_test.ml
+++ b/src/node/catchup_test.ml
@@ -137,9 +137,11 @@ let teardown () =
   Logger.debug_ "end of teardown"
 
 let _tic filler_function n name verify_store =
-  Tlogcommon.tlogEntriesPerFile := 101;
+  let open Tlog_map in
+  let node_id = "node_name" in
+  TlogMap.make ~tlog_max_entries:101 _dir_name _tlx_dir node_id >>= fun tlog_map ->
   Tlc2.make_tlc2 ~compressor:Compression.Snappy  _dir_name _tlx_dir _tlx_dir
-                 ~fsync:false "node_name" ~fsync_tlog_dir:true >>= fun tlog_coll ->
+                 ~fsync:false node_id ~fsync_tlog_dir:true >>= fun tlog_coll ->
   filler_function tlog_coll n >>= fun () ->
   let db_name = _dir_name ^ "/" ^ name ^ ".db" in
   S.make_store ~lcnum:1024 ~ncnum:512 db_name >>= fun store ->
diff --git a/src/node/collapser.ml b/src/node/collapser.ml
index 6dad59a8..6b9b50e1 100644
--- a/src/node/collapser.ml
+++ b/src/node/collapser.ml
@@ -19,6 +19,7 @@ limitations under the License.
 open Lwt
 type 'a store_maker= ?read_only:bool -> string -> 'a Lwt.t
 open Tlogcommon
+open Tlog_map
 
 let section = Logger.Section.main
 
@@ -30,7 +31,7 @@ let collapse_until (type s) (tlog_coll:Tlogcollection.tlog_collection)
      (throttling:float)
     )
     (too_far_i:Sn.t)
-    (cb: Sn.t -> unit Lwt.t)
+    (cb: int -> unit Lwt.t)
     (slowdown : float option)=
 
   let new_location = head_location ^ ".clone" in
@@ -55,10 +56,9 @@ let collapse_until (type s) (tlog_coll:Tlogcollection.tlog_collection)
   Lwt.finalize (
     fun () ->
       tlog_coll # get_infimum_i () >>= fun min_i ->
-      let first_tlog = (Sn.to_int min_i) /  !Tlogcommon.tlogEntriesPerFile in
+      let first_tlog = tlog_coll # get_tlog_from_i  min_i in
       let store_i = S.consensus_i new_store in
       let tfs = Sn.string_of too_far_i in
-      let tlog_entries_per_file = Sn.of_int (!Tlogcommon.tlogEntriesPerFile) in
       let processed = ref 0 in
       let start_i =
         begin
@@ -120,10 +120,10 @@ let collapse_until (type s) (tlog_coll:Tlogcollection.tlog_collection)
                     begin
                       maybe_log pi >>= fun () ->
                       begin
-                        if Sn.rem pi tlog_entries_per_file = 0L
+                        if tlog_coll # is_rollover_point pi
                         then
 
-                          cb (Sn.of_int (first_tlog + !processed)) >>= fun () ->
+                          cb (first_tlog + !processed) >>= fun () ->
                           Lwt.return( processed := !processed + 1 )
                         else
                           Lwt.return ()
@@ -207,7 +207,7 @@ let collapse_many
       (type s) tlog_coll
       (module S : Store.STORE with type t = s)
       (store_fs: 'b * string * float)
-      tlogs_to_keep cb' cb slowdown =
+      tlogs_to_keep cb' (cb:int -> unit Lwt.t) slowdown =
 
   Logger.debug_f_ "collapse_many" >>= fun () ->
   tlog_coll # get_tlog_count () >>= fun total_tlogs ->
@@ -222,11 +222,8 @@ let collapse_many
     Lwt.return head_i
   in
   get_head_i () >>= fun head_i ->
-  let lag = Sn.to_int (Sn.sub last_i head_i) in
-  let npt = !Tlogcommon.tlogEntriesPerFile in
-  let tlog_lag = (lag +npt - 1)/ npt in
-  let tlogs_to_collapse = tlog_lag - tlogs_to_keep - 1 in
-  Logger.debug_f_ "tlog_lag = %i; tlogs_to_collapse = %i" tlog_lag tlogs_to_collapse >>= fun () ->
+  let tlogs_to_collapse = tlog_coll # tlogs_to_collapse head_i last_i tlogs_to_keep in
+  Logger.debug_f_ "tlogs_to_collapse = %i" tlogs_to_collapse >>= fun () ->
   if tlogs_to_collapse <= 0
   then
     Logger.info_f_ "Nothing to collapse..." >>= fun () ->
@@ -235,13 +232,16 @@ let collapse_many
     begin
       Logger.info_f_ "Going to collapse %d tlogs" tlogs_to_collapse >>= fun () ->
       cb' (tlogs_to_collapse+1) >>= fun () ->
-      let g_too_far_i = Sn.add (Sn.of_int 2) (Sn.add head_i (Sn.of_int (tlogs_to_collapse * npt))) in
+      let g_too_far_i =
+        failwith "Sn.add (Sn.of_int 2) (Sn.add head_i (Sn.of_int (tlogs_to_collapse * npt)))"
+      in
       (* +2 because before X goes to the store, you need to have seen X+1 and thus too_far = X+2 *)
       Logger.debug_f_ "g_too_far_i = %s" (Sn.string_of g_too_far_i) >>= fun () ->
       collapse_until tlog_coll (module S) store_fs g_too_far_i cb slowdown >>= fun () ->
       get_head_i () >>= fun head_i ->
       tlog_coll # remove_below head_i >>= fun () ->
-      cb (Sn.div g_too_far_i (Sn.of_int !Tlogcommon.tlogEntriesPerFile))
+      let x = failwith "(Sn.div g_too_far_i (Sn.of_int !Tlogcommon.tlogEntriesPerFile))" in
+      cb x
     end
 
 let collapse_out_of_band cfg name tlogs_to_keep =
@@ -251,11 +251,6 @@ let collapse_out_of_band cfg name tlogs_to_keep =
   let me, _ = split name cfg.cfgs in
   let head_location = Filename.concat me.head_dir Tlc2.head_fname in
 
-  let () = match cfg.overwrite_tlog_entries with
-    | None -> ()
-    | Some i ->  Tlogcommon.tlogEntriesPerFile := i
-  in
-
   let module S = (val (Store.make_store_module (module Batched_store.Local_store))) in
   S.make_store
     ~lcnum:cfg.lcnum
@@ -275,8 +270,7 @@ let collapse_out_of_band cfg name tlogs_to_keep =
      me.head_copy_throttling)
     tlogs_to_keep
     (fun sn -> Lwt_log.debug_f "sn' = %i" sn)
-    (fun sn -> Lwt_log.debug_f "sn = %Li" sn)
+    (fun n -> Lwt_log.debug_f "n = %i" n)
     me.collapse_slowdown
   >>= fun () ->
   Lwt.return ()
-
diff --git a/src/node/collapser_test.ml b/src/node/collapser_test.ml
index 0a356d38..9b93dd26 100644
--- a/src/node/collapser_test.ml
+++ b/src/node/collapser_test.ml
@@ -19,6 +19,7 @@ limitations under the License.
 open Lwt
 open OUnit
 open Update
+open Tlog_map
 
 let section = Logger.Section.main
 let compressor = Compression.Snappy
@@ -54,11 +55,13 @@ let _make_values tlc n =
   in
   loop 0
 
-let test_collapse_until (dn, tlf_dir, head_dir) =
-  let () = Tlogcommon.tlogEntriesPerFile := 1000 in
-  Logger.debug_f_ "dn=%s, tlf_dir=%s, head_dir=%s" dn tlf_dir head_dir >>= fun () ->
-  Tlc2.make_tlc2 ~compressor dn tlf_dir head_dir
-                 ~fsync:false "node_name" ~fsync_tlog_dir:false >>= fun tlc ->
+let test_collapse_until (dn, tlx_dir, head_dir) =
+  let node_id = "node_name" in
+  TlogMap.make ~tlog_max_entries:1000 dn tlx_dir node_id >>= fun tlog_map ->
+  Logger.debug_f_ "dn=%s, tlf_dir=%s, head_dir=%s" dn tlx_dir head_dir >>= fun () ->
+  Tlc2.make_tlc2 ~compressor dn tlx_dir head_dir
+                 ~fsync:false node_id ~fsync_tlog_dir:false
+  >>= fun tlc ->
   _make_values tlc 1111 >>= fun () ->
   tlc # close () >>= fun () ->
   Lwt_unix.sleep 5.0 >>= fun () -> (* give it time to generate the .tlc *)
@@ -87,17 +90,20 @@ let test_dn = "/tmp/collapser"
 let _tlf_dir = "/tmp/collapser_tlf"
 let _head_dir = "/tmp/collapser_head"
 
-let test_collapse_many (dn, tlf_dir, head_dir) =
-  let () = Tlogcommon.tlogEntriesPerFile := 100 in
-  Logger.debug_f_ "test_collapse_many_regime dn=%s, tlf_dir=%s, head_dir=%s" dn tlf_dir head_dir >>= fun () ->
-  Tlc2.make_tlc2 ~compressor dn tlf_dir head_dir
-                 ~fsync:false "node_name" ~fsync_tlog_dir:false >>= fun tlc ->
+let test_collapse_many (dn, tlx_dir, head_dir) =
+  let node_id = "node_name" in
+  TlogMap.make ~tlog_max_entries:100 dn tlx_dir node_id >>= fun tlog_map ->
+
+  Logger.debug_f_ "test_collapse_many_regime dn=%s, tlx_dir=%s, head_dir=%s" dn tlx_dir head_dir
+  >>= fun () ->
+  Tlc2.make_tlc2 ~compressor dn tlx_dir head_dir
+                 ~fsync:false node_id ~fsync_tlog_dir:false >>= fun tlc ->
   _make_values tlc 632 >>= fun () ->
   tlc # close () >>= fun () ->
   Lwt_unix.sleep 5.0 >>= fun () -> (* compression finished ? *)
   let storename = Filename.concat test_dn "head.db" in
-  let cb fn = Logger.debug_f_ "collapsed %s" (Sn.string_of fn) in
-  let cb' = fun _n -> Lwt.return () in
+  let cb n = Logger.debug_f_ "collapsed %03i" n in
+  let cb' = fun _ -> Lwt.return () in
   File_system.unlink storename >>= fun () ->
   let store_methods = (Batched_store.Local_store.copy_store2, storename, 0.0) in
   Collapser.collapse_many tlc (module S) store_methods 5 cb' cb None >>= fun () ->
diff --git a/src/node/dump_store.ml b/src/node/dump_store.ml
index 9f355fac..e111fa57 100644
--- a/src/node/dump_store.ml
+++ b/src/node/dump_store.ml
@@ -18,6 +18,7 @@ open Lwt
 open Routing
 open Node_cfg.Node_cfg
 open Interval
+open Tlog_map
 
 module S = (val (Store.make_store_module (module Batched_store.Local_store)))
 
@@ -76,8 +77,7 @@ let inject_as_head fn node_id cfg_url ~force ~in_place =
       | [] -> failwith (Printf.sprintf "unknown node: %S" node_id)
       | x :: _ -> x
     in
-    let tlog_dir = node_cfg.tlog_dir in
-    let tlx_dir = node_cfg.tlx_dir in
+    TlogMap.make node_cfg.tlog_dir node_cfg.tlx_dir node_id >>= fun tlog_map ->
     let head_dir = node_cfg.head_dir in
     let old_head_name = Filename.concat head_dir Tlc2.head_fname  in
 
@@ -118,7 +118,7 @@ let inject_as_head fn node_id cfg_url ~force ~in_place =
     if not ok then failwith "new head is not an improvement";
     let bottom_n = match new_head_i with
       | None -> failwith "can't happen"
-      | Some i -> Sn.to_int (Tlc2.get_file_number i)
+      | Some i -> failwith "todo" (*Sn.to_int (Tlc2.get_file_number i)*)
     in
     begin
       if (not in_place)
@@ -134,11 +134,11 @@ let inject_as_head fn node_id cfg_url ~force ~in_place =
     >>= fun () ->
     Lwt_io.printlf "# [OK]">>= fun () ->
     Lwt_io.printlf "# remove superfluous .tlx files" >>= fun () ->
-    Tlc2.get_tlog_names tlog_dir tlx_dir >>= fun tlns ->
-    let old_tlns = List.filter (fun tln -> let n = Tlc2.get_number tln in n < bottom_n) tlns in
+    TlogMap.get_tlog_names tlog_map >>= fun tlns ->
+    let old_tlns = List.filter (fun tln -> let n = Tlog_map.get_number tln in n < bottom_n) tlns in
     Lwt_list.iter_s
       (fun old_tln ->
-         let canonical = Tlc2.get_full_path tlog_dir tlx_dir old_tln in
+         let canonical = TlogMap.get_full_path tlog_map old_tln in
          Lwt_io.printlf "rm %s" canonical >>= fun () ->
          File_system.unlink canonical
       ) old_tlns >>= fun () ->
diff --git a/src/node/node_cfg.ml b/src/node/node_cfg.ml
index dc694fcb..2bcd7b84 100644
--- a/src/node/node_cfg.ml
+++ b/src/node/node_cfg.ml
@@ -239,7 +239,8 @@ module Node_cfg = struct
       cluster_id : string;
       plugins: string list;
       nursery_cfg : (string*ClientCfg.t) option;
-      overwrite_tlog_entries: int option;
+      tlog_max_size : int ;
+      tlog_max_entries: int;
       max_value_size: int;
       max_buffer_size: int;
       client_buffer_capacity: int;
@@ -273,7 +274,8 @@ module Node_cfg = struct
            ; "cluster_id", string c.cluster_id
            ; "plugins", list string c.plugins
            ; "nursery_cfg", option (pair string ClientCfg.to_string) c.nursery_cfg
-           ; "overwrite_tlog_entries", option int c.overwrite_tlog_entries
+           ; "tlog_max_entries", int c.tlog_max_entries
+           ; "tlog_max_size", int c.tlog_max_size
            ; "max_value_size", int c.max_value_size
            ; "max_buffer_size", int c.max_buffer_size
            ; "client_buffer_capacity", int c.client_buffer_capacity
@@ -326,7 +328,7 @@ module Node_cfg = struct
     let batched_transaction_cfgs = [("default_batched_transaction_config", get_default_batched_transaction_config ())] in
     let cfgs = loop [] n_nodes in
     let quorum_function = Quorum.quorum_function in
-    let overwrite_tlog_entries = None in
+
     let cluster_cfg = {
       cfgs;
       log_cfgs;
@@ -337,7 +339,8 @@ module Node_cfg = struct
       _lease_period = default_lease_period;
       cluster_id;
       plugins = [];
-      overwrite_tlog_entries;
+      tlog_max_entries = 10_000 ;
+      tlog_max_size  = 100_000;
       max_value_size = default_max_value_size;
       max_buffer_size = default_max_buffer_size;
       client_buffer_capacity = default_client_buffer_capacity;
@@ -350,7 +353,8 @@ module Node_cfg = struct
     cluster_cfg
 
 
-
+  let _global_int inifile attribute d =
+    Ini.get inifile "global" attribute Ini.p_int (Ini.default d)
 
   let _node_names inifile =
     Ini.get inifile "global" "cluster" Ini.p_string_list Ini.required
@@ -358,6 +362,9 @@ module Node_cfg = struct
   let _ips inifile node_name =
     Ini.get inifile node_name "ip" Ini.p_string_list Ini.required
 
+  let _tlog_max_entries inifile = _global_int inifile "tlog_max_entries"    10_000
+  let _tlog_max_size    inifile = _global_int inifile "tlog_max_size"    5_000_000
+
   let _tlog_entries_overwrite inifile =
     Ini.get inifile "global" "__tainted_tlog_entries_per_file"
       (Ini.p_option Ini.p_int )
@@ -507,6 +514,7 @@ module Node_cfg = struct
     let client_port = get_int "client_port" in
     let messaging_port = get_int "messaging_port" in
     let home = get_string "home" in
+
     let tlog_dir =
       try get_string "tlog_dir"
       with _ -> home
@@ -536,7 +544,7 @@ module Node_cfg = struct
       else
         begin
           let s = Ini.get inifile node_name "tlog_compression" Ini.p_string
-                          (Ini.default "bz2")
+                          (Ini.default "snappy")
           in
           match String.lowercase s with
           | "snappy"  -> Compression.Snappy
@@ -618,35 +626,41 @@ module Node_cfg = struct
       let fm = _startup_mode inifile in
       let nodes = _node_names inifile in
       let plugin_names = _plugins inifile in
-      let cfgs, remaining = List.fold_left
-                              (fun (a,remaining) section ->
-                               if List.mem section nodes || _get_bool inifile section "learner"
-                               then
-                                 let cfg = _node_config inifile section fm in
-                                 let new_remaining = List.filter (fun x -> x <> section) remaining in
-                                 (cfg::a, new_remaining)
-                               else (a,remaining))
-                              ([],nodes) (inifile # sects) in
+      let cfgs, remaining =
+        List.fold_left
+          (fun (a,remaining) section ->
+           if List.mem section nodes || _get_bool inifile section "learner"
+           then
+             let cfg = _node_config inifile section fm in
+             let new_remaining = List.filter (fun x -> x <> section) remaining in
+             (cfg::a, new_remaining)
+           else (a,remaining))
+          ([],nodes) (inifile # sects)
+      in
       let log_cfg_names = List.map (fun cfg -> cfg.log_config) cfgs in
-      let log_cfgs = List.fold_left
-                       (fun a section ->
-                        if List.mem (Some section) log_cfg_names
-                        then
-                          let log_cfg = _log_config inifile section in
-                          (section, log_cfg)::a
-                        else
-                          a)
-                       [] (inifile # sects) in
+      let log_cfgs =
+        List.fold_left
+          (fun a section ->
+           if List.mem (Some section) log_cfg_names
+           then
+             let log_cfg = _log_config inifile section in
+             (section, log_cfg)::a
+           else
+             a)
+          [] (inifile # sects)
+      in
       let batched_transaction_cfg_names = List.map (fun cfg -> cfg.batched_transaction_config) cfgs in
-      let batched_transaction_cfgs = List.fold_left
-                                       (fun a section ->
-                                        if List.mem (Some section) batched_transaction_cfg_names
-                                        then
-                                          let batched_transaction_cfg = _batched_transaction_config inifile section in
-                                          (section, batched_transaction_cfg)::a
-                                        else
-                                          a)
-                                       [] (inifile # sects) in
+      let batched_transaction_cfgs =
+        List.fold_left
+          (fun a section ->
+           if List.mem (Some section) batched_transaction_cfg_names
+           then
+             let batched_transaction_cfg = _batched_transaction_config inifile section in
+             (section, batched_transaction_cfg)::a
+           else
+             a)
+          [] (inifile # sects)
+      in
       let () = if List.length remaining > 0 then
                  failwith ("Can't find config section for: " ^ (String.concat "," remaining))
       in
@@ -655,6 +669,16 @@ module Node_cfg = struct
       let cluster_id = _get_cluster_id inifile in
       let m_n_cfg = get_nursery_cfg inifile in
       let overwrite_tlog_entries = _tlog_entries_overwrite inifile in
+      let tlog_max_entries = _tlog_max_entries inifile in
+      let tlog_max_size    = _tlog_max_size inifile in
+      let () = match overwrite_tlog_entries with
+        | None -> ()
+        | Some _ ->
+           let msg = "deprecated:__tainted_tlog_entries_per_file will be ignored;"^
+                       " please use 'tlog_max_entries' "
+           in
+           Logger.ign_warning_ msg
+      in
       let max_value_size = _max_value_size inifile in
       let max_buffer_size = _max_buffer_size inifile in
       let client_buffer_capacity = _client_buffer_capacity inifile in
@@ -707,7 +731,8 @@ module Node_cfg = struct
           _lease_period = lease_period;
           cluster_id = cluster_id;
           plugins = plugin_names;
-          overwrite_tlog_entries;
+          tlog_max_entries;
+          tlog_max_size;
           max_value_size;
           max_buffer_size;
           client_buffer_capacity;
diff --git a/src/node/node_main.ml b/src/node/node_main.ml
index 1b546443..017e22c7 100644
--- a/src/node/node_main.ml
+++ b/src/node/node_main.ml
@@ -166,7 +166,7 @@ let log_prelude cluster_cfg =
   Logger.info_f_ "version: %i.%i.%i" Arakoon_version.major
     Arakoon_version.minor Arakoon_version.patch   >>= fun () ->
   Logger.info_f_ "NOFILE: %i" (Limits.get_rlimit Limits.NOFILE Limits.Soft)      >>= fun () ->
-  Logger.info_f_ "tlogEntriesPerFile: %i" (!Tlogcommon.tlogEntriesPerFile)       >>= fun () ->
+
   Logger.info_f_ "cluster_cfg=%s" (string_of_cluster_cfg cluster_cfg)            >>= fun () ->
   Logger.info_f_ "Batched_store.max_entries = %i" !Batched_store.max_entries    >>= fun () ->
   Logger.info_f_ "Batched_store.max_size = %i" !Batched_store.max_size
@@ -179,7 +179,7 @@ let only_catchup
       ~tls_ctx
       ~name
       ~cluster_cfg
-      ~make_tlog_coll
+      ~(make_tlog_coll:Tlogcollection.tlc_factory)
       get_snapshot_name
   =
   Logger.info_ "ONLY CATCHUP" >>= fun () ->
@@ -205,6 +205,8 @@ let only_catchup
                db_name >>= fun store ->
   let compressor = me.compressor in
   make_tlog_coll ~compressor
+                 ~tlog_max_entries:cluster_cfg.tlog_max_entries
+                 ~tlog_max_size:cluster_cfg.tlog_max_size
                  me.tlog_dir me.tlx_dir me.head_dir
                  ~fsync:false name ~fsync_tlog_dir:false
   >>= fun tlc ->
@@ -375,7 +377,7 @@ end
 
 let _main_2 (type s)
       (module S : Store.STORE with type t = s)
-      make_tlog_coll
+      (make_tlog_coll:Tlogcollection.tlc_factory)
       (make_config: unit -> 'a Lwt.t)
       get_snapshot_name ~name
       ~daemonize ~catchup_only
@@ -403,11 +405,6 @@ let _main_2 (type s)
   let ssl_context = build_ssl_context me cluster_cfg in
   let service_ssl_context = build_service_ssl_context me cluster_cfg in
 
-  let () = match cluster_cfg.overwrite_tlog_entries with
-    | None -> ()
-    | Some i ->  Tlogcommon.tlogEntriesPerFile := i
-  in
-
   log_prelude cluster_cfg >>= fun () ->
   Logger.info_f_ "autofix:%b" autofix >>= fun () ->
   Plugin_loader.load me.home cluster_cfg.plugins >>= fun () ->
@@ -546,6 +543,8 @@ let _main_2 (type s)
           >>= fun () ->
           let compressor = me.compressor in
           make_tlog_coll ~compressor
+                         ~tlog_max_entries:cluster_cfg.tlog_max_entries
+                         ~tlog_max_size:cluster_cfg.tlog_max_size
                          me.tlog_dir me.tlx_dir me.head_dir
                          ~fsync:me.fsync name ~fsync_tlog_dir:me._fsync_tlog_dir
           >>= fun (tlog_coll:Tlogcollection.tlog_collection) ->
@@ -566,15 +565,17 @@ let _main_2 (type s)
                then File_system.unlink db_name
                else Lwt.return_unit)
                >>= fun () ->
-               Tlog_main._mark_tlog fn (Tlc2._make_close_marker me.node_name) >>= fun _ ->
+               Tlog_main._mark_tlog fn (Tlog_map._make_close_marker me.node_name) >>= fun _ ->
                _open_tlc_and_store()
              in
+             let open Tlog_map in
              match exn with
-             | Tlc2.TLCNotProperlyClosed _ ->
+             | Tlog_map.TLCNotProperlyClosed _ ->
                 begin
                   Logger.warning_ ~exn "AUTOFIX: improperly closed tlog"
                   >>= fun ()->
-                  Tlc2.get_last_tlog me.tlog_dir me.tlx_dir
+                  TlogMap.make me.tlog_dir me.tlx_dir me.node_name >>= fun tlog_map ->
+                  TlogMap.get_last_tlog tlog_map
                   >>= fun (_, fn) ->
                   clean_up_retry fn
                 end
@@ -582,7 +583,8 @@ let _main_2 (type s)
                 begin
                   Logger.warning_ ~exn "AUTOFIX: unexpected end of tlog"
                   >>= fun () ->
-                  Tlc2.get_last_tlog me.tlog_dir me.tlx_dir
+                  TlogMap.make me.tlog_dir me.tlx_dir me.node_name >>= fun tlog_map ->
+                  TlogMap.get_last_tlog tlog_map
                   >>= fun (_, fn) ->
                   Tlc2._truncate_tlog fn >>= fun ()->
                   clean_up_retry fn
@@ -950,7 +952,7 @@ let _main_2 (type s)
             Logger.fatal_f_ "[rc=%i] Store counter too low: %s" rc msg >>= fun () ->
             maybe_dump_crash_log () >>= fun () ->
             Lwt.return rc
-          | Tlc2.TLCNotProperlyClosed msg ->
+          | Tlog_map.TLCNotProperlyClosed msg ->
             let rc = 42 in
             Logger.fatal_f_ "[rc=%i] tlog not properly closed %s" rc msg >>= fun () ->
             Lwt.return rc
@@ -1001,7 +1003,7 @@ let main_t (make_config: unit -> 'a Lwt.t)
            name daemonize catchup_only autofix : int Lwt.t
   =
   let module S = (val (Store.make_store_module (module Batched_store.Local_store))) in
-  let make_tlog_coll = Tlc2.make_tlc2 in
+  let (make_tlog_coll :Tlogcollection.tlc_factory) = Tlc2.make_tlc2 in
   let get_snapshot_name = Tlc2.head_name in
   _main_2 (module S)
           make_tlog_coll
diff --git a/src/node/sync_backend.ml b/src/node/sync_backend.ml
index a45d7ada..db3a1bfa 100644
--- a/src/node/sync_backend.ml
+++ b/src/node/sync_backend.ml
@@ -679,16 +679,17 @@ struct
             (* remove all but tlogs_to_keep last tlogs *)
             Collapser._head_i (module S) head_path >>= fun head_io ->
 
-            let head_n = match head_io with
+            let head_i, head_n = match head_io with
               | None -> failwith "impossible i for copied head"
-              | Some i -> Tlc2.get_file_number i
+              | Some i -> i, tlog_collection # get_tlog_from_i i
             in
-            let keep_bottom_n = Sn.succ (Sn.sub head_n (Sn.of_int tlogs_to_keep)) in
-            if Sn.compare keep_bottom_n Sn.zero = 1
+            let keep_bottom_n = head_n -  tlogs_to_keep + 1 in
+            if keep_bottom_n > 0 
             then
               begin
                 self # wait_for_tlog_release keep_bottom_n >>= fun () ->
-                tlog_collection # remove_below (Tlc2.get_tlog_i keep_bottom_n)
+                let keep_i = tlog_collection # get_start_i keep_bottom_n in
+                tlog_collection # remove_below keep_i
               end
             else
               Lwt.return ()
diff --git a/src/system/startup.ml b/src/system/startup.ml
index e5161d97..0c75e447 100644
--- a/src/system/startup.ml
+++ b/src/system/startup.ml
@@ -73,10 +73,17 @@ let _make_cfg name n lease_period =
     head_copy_throttling = 0.0;
   }
 
-let _make_tlog_coll ~compressor tlcs values tlc_name tlf_dir head_dir
-                    ~fsync node_id ~fsync_tlog_dir =
+let _make_tlog_coll
+      ~tlcs ~values
+      ~compressor ?(tlog_max_entries:int option) ?(tlog_max_size:int option)
+      tlc_name tlf_dir head_dir
+      ~(fsync:bool) (node_id:string) ~(fsync_tlog_dir:bool)
+  =
   let () = ignore compressor in
-  Mem_tlogcollection.make_mem_tlog_collection tlc_name tlf_dir head_dir ~fsync node_id ~fsync_tlog_dir >>= fun tlc ->
+  Mem_tlogcollection.make_mem_tlog_collection
+    ?tlog_max_entries ?tlog_max_size
+    tlc_name tlf_dir head_dir ~fsync node_id ~fsync_tlog_dir
+  >>= fun tlc ->
   let rec loop i = function
     | [] -> Lwt.return ()
     | v :: vs ->
@@ -94,20 +101,22 @@ let node_ts = ref []
 
 let _make_run ~stores ~tlcs ~now ~values ~get_cfgs name () =
   let module S =
-  struct
-    include LS
-    let make_store
-        ~lcnum ~ncnum
-        ?(read_only=false) (db_name:string) =
-      LS.make_store ~lcnum ~ncnum ~read_only db_name >>= fun store ->
-      LS.with_transaction store (fun tx -> LS.set_master store tx name now) >>= fun () ->
-      Hashtbl.add stores db_name store;
-      Lwt.return store
-  end in
+    struct
+      include LS
+      let make_store
+            ~lcnum ~ncnum
+            ?(read_only=false) (db_name:string) =
+        LS.make_store ~lcnum ~ncnum ~read_only db_name >>= fun store ->
+        LS.with_transaction store (fun tx -> LS.set_master store tx name now) >>= fun () ->
+        Hashtbl.add stores db_name store;
+        Lwt.return store
+    end
+  in
+  let (tlc_factory:Tlogcollection.tlc_factory) = _make_tlog_coll ~tlcs ~values in
   let t =
     Node_main._main_2
       (module S)
-      (_make_tlog_coll tlcs values)
+      (tlc_factory)
       get_cfgs
       (fun () -> "DUMMY")
       ~name
@@ -150,7 +159,8 @@ let post_failure () =
     cluster_id = "ricky";
     plugins = [];
     nursery_cfg = None;
-    overwrite_tlog_entries = None;
+    tlog_max_entries = 10_000;
+    tlog_max_size = 100_000;
     max_value_size = Node_cfg.default_max_value_size;
     max_buffer_size = Node_cfg.default_max_buffer_size;
     client_buffer_capacity = Node_cfg.default_client_buffer_capacity;
@@ -211,7 +221,8 @@ let restart_slaves () =
      cluster_id = "ricky";
      plugins = [];
      nursery_cfg = None;
-     overwrite_tlog_entries = None;
+     tlog_max_entries = 10_000;
+     tlog_max_size = 100_000;
      max_value_size = Node_cfg.default_max_value_size;
      max_buffer_size = Node_cfg.default_max_buffer_size;
      client_buffer_capacity = Node_cfg.default_client_buffer_capacity;
@@ -270,7 +281,8 @@ let ahead_master_loses_role () =
      cluster_id = "ricky";
      plugins = [];
      nursery_cfg = None;
-     overwrite_tlog_entries = None;
+     tlog_max_entries = 10_000;
+     tlog_max_size = 100_000;
      max_value_size = Node_cfg.default_max_value_size;
      max_buffer_size = Node_cfg.default_max_buffer_size;
      client_buffer_capacity = Node_cfg.default_client_buffer_capacity;
@@ -342,7 +354,8 @@ let interrupted_election () =
      cluster_id;
      plugins = [];
      nursery_cfg = None;
-     overwrite_tlog_entries = None;
+     tlog_max_entries = 10_000;
+     tlog_max_size = 100_000;
      max_value_size = Node_cfg.default_max_value_size;
      max_buffer_size = Node_cfg.default_max_buffer_size;
      client_buffer_capacity = Node_cfg.default_client_buffer_capacity;
diff --git a/src/tlog/mem_tlogcollection.ml b/src/tlog/mem_tlogcollection.ml
index fbe555a3..b93954c1 100644
--- a/src/tlog/mem_tlogcollection.ml
+++ b/src/tlog/mem_tlogcollection.ml
@@ -70,6 +70,9 @@ class mem_tlog_collection _name =
 
     method which_tlog_file _start_i = failwith "which_tlog_file not supported"
 
+    method tlogs_to_collapse _ _ _ = failwith "tlogs_to_collapse not supported"
+    method is_rollover_point _ = failwith "is_rollover_point not supported"
+
     method log_value_explicit i (v:Value.t) ~sync marker =
       let entry = Entry.make i v 0L marker in
       let () = data <- entry::data in
@@ -85,7 +88,8 @@ class mem_tlog_collection _name =
 
     method get_head_name () = failwith "get_head_name not implemented"
 
-    method get_tlog_from_i _ = Sn.start
+    method get_tlog_from_i _ = 0
+    method get_start_i n = Sn.zero
 
     method close ?(wait_for_compression = false) () =
         let () = ignore wait_for_compression in
@@ -94,7 +98,8 @@ class mem_tlog_collection _name =
     method remove_below _i = Lwt.return ()
   end
 
-let make_mem_tlog_collection _tlog_dir _tlf_dir _head_dir ~fsync name ~fsync_tlog_dir =
+let make_mem_tlog_collection
+      ?tlog_max_entries ?tlog_max_size _tlog_dir _tlf_dir _head_dir ~fsync name ~fsync_tlog_dir =
   let () = ignore fsync in
   let () = ignore fsync_tlog_dir in
   let x = new mem_tlog_collection name in
diff --git a/src/tlog/tlc2.ml b/src/tlog/tlc2.ml
index f50f57cf..7acae737 100644
--- a/src/tlog/tlc2.ml
+++ b/src/tlog/tlc2.ml
@@ -21,26 +21,11 @@ open Unix.LargeFile
 open Lwt_buffer
 
 open Tlogreader2
+open Tlog_map
 
-exception TLCNotProperlyClosed of string
 
-let section = Logger.Section.main
-
-let file_regexp = Str.regexp "^[0-9]+\\.tl\\(og\\|f\\|c\\|x\\)$"
-
-let tlog_regexp = Str.regexp "^[0-9]+\\.tlog$"
-let file_name c = Printf.sprintf "%03i.tlog" c
-
-let extension comp =
-  let open Compression in
-  match comp with
-    | Snappy -> ".tlx"
-    | Bz2 -> ".tlf"
-    | No -> ".tlog"
 
-let archive_name comp c =
-  let x = extension comp in
-  Printf.sprintf "%03i%s" c x
+let section = Logger.Section.main
 
 
 let to_archive_name comp fn =
@@ -57,38 +42,8 @@ let to_archive_name comp fn =
 
 let head_fname = "head.db"
 
-let get_full_path tlog_dir tlf_dir name =
-  if Filename.check_suffix name ".tlx"
-     || Filename.check_suffix name ".tlx.part"
-     || Filename.check_suffix name ".tlf"
-     || Filename.check_suffix name ".tlf.part"
-  then
-    Filename.concat tlf_dir name
-  else
-    Filename.concat tlog_dir name
-
-
-let find_tlog_file tlog_dir tlf_dir c =
-  let open Compression in
-  let f0 = archive_name Snappy c in
-  let f1 = archive_name Bz2 c in
-  let f2 = file_name c in
-  let rec _find = function
-    | [] -> Lwt.fail (Failure (Printf.sprintf "no tlog file for %3i" c))
-    | f :: fs ->
-       let full = get_full_path tlog_dir tlf_dir f in
-       File_system.exists full >>= fun ok ->
-       if ok
-       then Lwt.return full
-       else _find fs
-  in
-  _find [f0;f1;f2]
-
 let head_name () = head_fname
 
-let get_file_number i = Sn.div i (Sn.of_int !Tlogcommon.tlogEntriesPerFile)
-let get_tlog_i n = Sn.mul n (Sn.of_int !Tlogcommon.tlogEntriesPerFile)
-
 
 
 let to_tlog_name fn =
@@ -102,73 +57,6 @@ let to_tlog_name fn =
     root ^ ".tlog"
   else failwith (Printf.sprintf "to_tlog_name:extension is '%s' and should be one of .tlc .tlf .tlx" extension)
 
-let get_number fn =
-  let dot_pos = String.index fn '.' in
-  let pre = String.sub fn 0 dot_pos in
-  int_of_string pre
-
-let extension_of filename =
-  let len = String.length filename in
-  let dot_pos = String.rindex filename '.' in
-  String.sub filename dot_pos (len - dot_pos)
-
-let get_tlog_names tlog_dir tlx_dir =
-  let get_entries dir invalid_dir_exn =
-    Lwt.catch
-      (fun () ->
-         File_system.lwt_directory_list dir
-      )
-      (function
-        | Unix.Unix_error(Unix.EIO, fn, _) as exn ->
-          if fn = "readdir"
-          then Lwt.fail invalid_dir_exn
-          else Lwt.fail exn
-        | exn -> Lwt.fail exn
-      ) in
-  get_entries tlog_dir (Node_cfg.InvalidTlogDir tlog_dir) >>= fun tlog_entries ->
-  (if tlog_dir = tlx_dir
-   then
-     Lwt.return []
-   else
-     get_entries tlx_dir (Node_cfg.InvalidTlxDir tlx_dir)) >>= fun tlf_entries ->
-  let entries = List.rev_append tlf_entries tlog_entries in
-  let filtered = List.filter
-                   (fun e -> Str.string_match file_regexp e 0)
-                   entries
-  in
-  let my_compare fn1 fn2 =
-    let n1 = get_number fn1
-    and n2 = get_number fn2 in
-    if n1 = n2
-       (* make .tlx < .tlf < .tlog , so we can filter later*)
-    then
-      let fn1x = extension_of fn1
-      and fn2x = extension_of fn2
-         in
-         let order x = match x with
-           | ".tlx"  -> 1
-           | ".tlf"  -> 2
-           | ".tlog" -> 3
-           | _       -> 4
-         in compare (order fn1x) (order fn2x)
-    else compare n1 n2
-  in
-  let sorted = List.sort my_compare filtered in
-  let filtered2 = List.fold_left
-                    (fun acc name ->
-                       match acc with
-                         | [] -> name :: acc
-                         | prev :: rest ->
-                           if get_number prev = get_number name
-                           then prev :: rest (* smaller is better *)
-                           else name :: acc
-                    ) [] sorted
-  in
-  let sorted2 = List.rev filtered2 in
-  let log_e e = Logger.debug_f_ "entry %s %i" e (get_number e) in
-  Lwt_list.iter_s log_e sorted2 >>= fun () ->
-  Lwt.return sorted2
-
 
 let folder_for filename index =
   let extension = extension_of filename in
@@ -179,105 +67,27 @@ let folder_for filename index =
   | ".tlc"  -> Tlogreader2.AO.fold, extension, None
   | _       -> failwith (Printf.sprintf "no folder for '%s'" extension)
 
-let fold_read tlog_dir tlf_dir file_name
-      ~index
-      (lowerI:Sn.t)
-      (too_far_i:Sn.t option)
-      ~first
-      (a0:'a)
-      (f:'a -> Entry.t -> 'a Lwt.t) =
-  let full_name = get_full_path tlog_dir tlf_dir file_name in
-  let folder, extension, index' = folder_for file_name index in
-  Logger.debug_f_ "fold_read extension=%s => index':%s" extension (Index.to_string index') >>= fun () ->
-  let ic_f ic = folder ic ~index:index' lowerI too_far_i ~first a0 f in
-  Lwt.catch
-    (fun () -> Lwt_io.with_file ~mode:Lwt_io.input full_name ic_f)
-    (function
-      | Unix.Unix_error(_,"open",_) as exn ->
-        if extension = ".tlog"
-        then (* file might have moved meanwhile *)
-          begin
-            Logger.debug_f_ "%s became archive" file_name
-            >>= fun () ->
-            let c = get_number file_name in
-            find_tlog_file tlog_dir tlf_dir c >>= fun full_an ->
-            let real_folder,_,_ = folder_for full_an None in
-            Logger.debug_f_ "folding compressed %s" full_an >>= fun () ->
-            Lwt_io.with_file ~mode:Lwt_io.input full_an
-              (fun ic -> real_folder ic ~index:None lowerI too_far_i ~first a0 f)
-          end
-        else
-          Lwt.fail exn
-      | exn -> Lwt.fail exn
-    )
-
-type validation_result = (Entry.t option * Index.index)
-
-let _make_close_marker node_id = "closed:" ^ node_id
-let _make_open_marker node_id =  "opened:" ^ node_id
-
-let _validate_one tlog_name node_id ~check_marker : validation_result Lwt.t =
-  Logger.debug_f_ "Tlc2._validate_one %s" tlog_name >>= fun () ->
-  let e2s e = let i = Entry.i_of e in Printf.sprintf "(%s,_)" (Sn.string_of i) in
-  let prev_entry = ref None in
-  let new_index = Index.make tlog_name in
-  Lwt.catch
-    (fun () ->
-       let first = Sn.of_int 0 in
-       let folder, _, index = folder_for tlog_name None in
-
-       let do_it ic = folder ic ~index Sn.start None ~first None
-                        (fun _a0 entry ->
-                           let () = Index.note entry new_index in
-                           let r = Some entry in
-                           let () = prev_entry := r in
-                           Lwt.return r)
-       in
-       Lwt_io.with_file tlog_name ~mode:Lwt_io.input do_it
-       >>= fun eo ->
-       begin
-         if not check_marker
-         then Lwt.return eo
-         else
-           let eo' =
-             match eo with
-               | None -> None
-               | Some e ->
-                 let s = Some (_make_close_marker node_id) in
-                 if Entry.check_marker e s
-                 then !prev_entry
-                 else raise (TLCNotProperlyClosed (Entry.entry2s e))
-           in
-           Lwt.return eo'
-       end
-       >>= fun eo' ->
-       Logger.debug_f_ "XX:a=%s" (Log_extra.option2s e2s eo') >>= fun () ->
-       Logger.debug_f_ "After validation index=%s" (Index.to_string new_index) >>= fun () ->
-       Lwt.return (eo', new_index)
-    )
-    (function
-      | Unix.Unix_error(Unix.ENOENT,_,_) -> Lwt.return (None, new_index)
-      | exn -> Lwt.fail exn
-    )
 
 
 let _validate_list tlog_names node_id ~check_marker=
-  Lwt_list.fold_left_s (fun _ tn -> _validate_one tn node_id ~check_marker) (None,None) tlog_names >>= fun (eo,index) ->
+  Lwt_list.fold_left_s
+    (fun _ tn -> _validate_one tn node_id ~check_marker)
+    (None,None,0)
+    tlog_names
+  >>= fun (eo,index, pos) ->
   Logger.info_f_ "_validate_list %s => %s" (String.concat ";" tlog_names) (Index.to_string index) >>= fun () ->
   Lwt.return (TlogValidIncomplete, eo, index)
 
 
-
-
-let validate_last tlog_dir tlf_dir node_id ~check_marker=
+let validate_last tlog_map node_id ~check_marker=
   Logger.debug_f_ "validate_last ~check_marker:%b" check_marker >>= fun () ->
-  get_tlog_names tlog_dir tlf_dir >>= fun tlog_names ->
+  TlogMap.get_tlog_names tlog_map >>= fun tlog_names ->
   match tlog_names with
     | [] -> _validate_list [] node_id ~check_marker
     | _ ->
       let n = List.length tlog_names in
       let last = List.nth tlog_names (n-1) in
-      let fn = get_full_path tlog_dir tlf_dir last in
+      let fn = TlogMap.get_full_path tlog_map last in
       _validate_list [fn] node_id ~check_marker>>= fun r ->
       let (_validity, last_eo, _index) = r in
       match last_eo with
@@ -286,7 +96,7 @@ let validate_last tlog_dir tlf_dir node_id ~check_marker=
             if n > 1
             then
               let prev_last = List.nth tlog_names (n-2) in
-              let last_non_empty = get_full_path tlog_dir tlf_dir prev_last in
+              let last_non_empty = TlogMap.get_full_path tlog_map prev_last in
               _validate_list [last_non_empty] node_id ~check_marker
             else
               Lwt.return r
@@ -295,16 +105,7 @@ let validate_last tlog_dir tlf_dir node_id ~check_marker=
 
 
 
-let get_count tlog_names =
-  match List.length tlog_names with
-    | 0 -> 0
-    | n -> let last = List.nth tlog_names (n-1) in
-      let length = String.length last in
-      let postfix = String.sub last (length -4) 4 in
-      let number = get_number last in
-      if postfix = "tlog"
-      then number
-      else number + 1
+
 
 module F = struct
   type t = {oc:Lwt_io.output Lwt_io.channel;
@@ -324,9 +125,9 @@ module F = struct
 end
 
 
-let _init_file fsync_dir tlog_dir tlf_dir c =
+let _init_file fsync_dir tlog_map c =
   let fn = file_name c in
-  let full_name = get_full_path tlog_dir tlf_dir fn in
+  let full_name = TlogMap.get_full_path tlog_map fn in
   Lwt_unix.openfile full_name [Unix.O_CREAT;Unix.O_APPEND;Unix.O_WRONLY] 0o644 >>= fun fd ->
   begin
     if fsync_dir
@@ -343,71 +144,20 @@ let _init_file fsync_dir tlog_dir tlf_dir c =
   Lwt.return f
 
 
-let iterate_tlog_dir tlog_dir tlf_dir ~index start_i too_far_i f =
-  let tfs = Sn.string_of too_far_i in
-  Logger.debug_f_ "Tlc2.iterate_tlog_dir tlog_dir:%s start_i:%s too_far_i:%s ~index:%s"
-    tlog_dir (Sn.string_of start_i) tfs (Index.to_string index)
-  >>= fun () ->
-  get_tlog_names tlog_dir tlf_dir >>= fun tlog_names ->
-  let acc_entry (_i0:Sn.t) entry = f entry >>= fun () -> let i = Entry.i_of entry in Lwt.return i in
-  let num_tlogs = List.length tlog_names in
-  let maybe_fold acc fn =
-    let cnt,low = acc in
-    let factor = Sn.of_int (!Tlogcommon.tlogEntriesPerFile) in
-    let linf = Sn.of_int (get_number fn) in
-    let ( * ) = Sn.mul in
-    let test0 = linf * factor in
-    let test1 = (Sn.succ linf) * factor in
-    let low_s = Sn.string_of low in
-    let lowp = Sn.succ low in
-    let lowp_s = Sn.string_of lowp in
-    let test_result =
-      (test0 <= lowp) &&
-      (low   <  test1) &&
-      low <= too_far_i
-    in
-    Logger.debug_f_ "%s <?= lowp:%s &&  low:%s <? %s && (low:%s <?= %s) yields:%b"
-      (Sn.string_of test0 ) lowp_s
-      low_s (Sn.string_of test1)
-      low_s tfs
-      test_result
-    >>= fun () ->
-    if test_result
-    then
-      begin
-        Logger.debug_f_ "fold_read over: %s (test0=%s)" fn
-          (Sn.string_of test0) >>= fun () ->
-        let first = test0 in
-        Logger.info_f_ "Replaying tlog file: %s (%d/%d)" fn cnt num_tlogs  >>= fun () ->
-        let t1 = Unix.gettimeofday () in
-        fold_read tlog_dir tlf_dir fn ~index low (Some too_far_i) ~first low acc_entry >>= fun x ->
-        Logger.info_f_ "Completed replay of %s, took %f seconds, %i to go" fn (Unix.gettimeofday () -. t1) (num_tlogs - cnt) >>= fun () ->
-        Lwt.return (cnt+1,x)
-      end
-    else Lwt.return (cnt+1,low)
-  in
-  Lwt_list.fold_left_s maybe_fold (1,start_i) tlog_names
-  >>= fun _x ->
-  Lwt.return ()
-
-
-
-
-
 class tlc2
         ?(compressor=Compression.Snappy)
         (tlog_dir:string) (tlf_dir:string) (head_dir:string) (new_c:int)
         (last:Entry.t option) (index:Index.index)
+        (tlog_map: TlogMap.t)
         (node_id:string) ~(fsync:bool) ~(fsync_tlog_dir:bool)
   =
   let inner =
-    match last with
-      | None -> 0
-      | Some e ->
-        let i = Entry.i_of e in
-        let step = (Sn.of_int !Tlogcommon.tlogEntriesPerFile) in
-        (Sn.to_int (Sn.rem i step)) + 1
-  in
+    let io = match last with
+      | None -> None
+      | Some e -> Some (Entry.i_of e)
+    in
+    TlogMap.inner_of_i tlog_map io in
+
   object(self: # tlog_collection)
     val mutable _file = (None:F.t option)
     val mutable _index = (None: Index.index)
@@ -417,6 +167,7 @@ class tlc2
     val mutable _compression_q = Lwt_buffer.create_fixed_capacity 5
     val mutable _compression_thread = None
     val mutable _compressing = false
+
     val _closing = ref false
     val _write_lock = Lwt_mutex.create ()
     val _jc = (Lwt_condition.create () : unit Lwt_condition.t)
@@ -424,9 +175,9 @@ class tlc2
 
 
     method validate_last_tlog () =
-      validate_last tlog_dir tlf_dir node_id ~check_marker:false >>= fun r ->
+      validate_last tlog_map node_id ~check_marker:false >>= fun r ->
       let (_validity, _entry, new_index) = r in
-      let tlu = get_full_path tlog_dir tlf_dir (file_name _outer) in
+      let tlu = TlogMap.get_full_path tlog_map (file_name _outer) in
       let matches = Index.match_filename tlu new_index in
       Logger.debug_f_ "tlu=%S new_index=%s index=%s => matches=%b"
         tlu
@@ -528,6 +279,7 @@ class tlc2
              let entry = Entry.make i value p marker in
              _previous_entry <- Some entry;
              Index.note entry _index;
+             let () = TlogMap.new_entry tlog_map total_size in
              Lwt.return total_size
            end)
 
@@ -540,15 +292,15 @@ class tlc2
       self # log_value_explicit i v ~sync marker
 
     method private _prelude i =
-      let ( *: ) = Sn.mul in
       match _file with
         | None ->
           begin
-            Logger.debug_f_ "prelude %s" (Sn.string_of i) >>= fun () ->
-            let outer = Sn.div i (Sn.of_int !Tlogcommon.tlogEntriesPerFile) in
-            _outer <- Sn.to_int outer;
+
+            let outer = TlogMap.outer_of_i tlog_map i in
+            _outer <- outer;
+            Logger.debug_f_ "prelude i:%s: outer=%i" (Sn.string_of i) outer>>= fun () ->
             begin
-              if (_outer <> 0) && (i = Sn.mul outer (Sn.of_int !Tlogcommon.tlogEntriesPerFile))
+              if (_outer <> 0) && (TlogMap.is_first_of_new tlog_map i )
               then
                 (* the first thing logged falls exactly into a new tlog,
                    the compression job for the previous one thus is not yet picked up *)
@@ -556,16 +308,22 @@ class tlc2
               else
                 Lwt.return ()
             end >>= fun () ->
-            _init_file fsync_tlog_dir tlog_dir tlf_dir _outer >>= fun file ->
+            let () =
+              if TlogMap.should_roll tlog_map _outer i
+              then
+                let new_outer = TlogMap.new_outer tlog_map i in
+                _outer <- new_outer
+            in
+            _init_file fsync_tlog_dir tlog_map _outer >>= fun file ->
             _file <- Some file;
             _index <- Index.make (F.fn_of file);
             Lwt.return file
           end
         | Some (file:F.t) ->
-          if i >= (Sn.of_int !Tlogcommon.tlogEntriesPerFile) *: (Sn.of_int (_outer + 1))
+          if TlogMap.should_roll tlog_map _outer i
           then
             let do_jump () =
-              let new_outer = Sn.to_int (Sn.div i (Sn.of_int !Tlogcommon.tlogEntriesPerFile)) in
+              let new_outer = TlogMap.new_outer tlog_map i in
               Logger.info_f_ "i= %s & outer = %i => jump to %i" (Sn.string_of i) _outer new_outer
               >>= fun () ->
               self # _jump_to_tlog file new_outer
@@ -579,9 +337,8 @@ class tlc2
       match compressor with
       | Bz2 | Snappy ->
          begin
-           let tlu = get_full_path tlog_dir tlf_dir (file_name old_outer) in
-           let tlc = get_full_path tlog_dir tlf_dir
-                                   (archive_name compressor old_outer) in
+           let tlu = TlogMap.get_full_path tlog_map (file_name old_outer) in
+           let tlc = TlogMap.get_full_path tlog_map (archive_name compressor old_outer) in
            let job = (tlu,tlc,compressor) in
            Logger.info_f_ "adding new task to compression queue %s,%s"
                           tlu tlc >>= fun () ->
@@ -594,11 +351,11 @@ class tlc2
       let old_outer = _outer in
       _inner <- 0;
       _outer <- new_outer;
-      _init_file fsync_tlog_dir tlog_dir tlf_dir new_outer >>= fun new_file ->
+      _init_file fsync_tlog_dir tlog_map new_outer >>= fun new_file ->
       _file <- Some new_file;
       let index =
         let fn = file_name _outer in
-        let full_name = get_full_path tlog_dir tlf_dir fn in
+        let full_name = TlogMap.get_full_path tlog_map fn in
         Index.make full_name
       in
       _index <- index;
@@ -610,20 +367,10 @@ class tlc2
     method iterate (start_i:Sn.t) (too_far_i:Sn.t) (f:Entry.t -> unit Lwt.t) =
       let index = _index in
       Logger.debug_f_ "tlc2.iterate : index=%s" (Index.to_string index) >>= fun () ->
-      iterate_tlog_dir tlog_dir tlf_dir ~index start_i too_far_i f
+      TlogMap.iterate_tlog_dir tlog_map ~index start_i too_far_i f
 
 
-    method get_infimum_i () =
-      get_tlog_names tlog_dir tlf_dir >>= fun names ->
-      let f = !Tlogcommon.tlogEntriesPerFile in
-      let v =
-        match names with
-          | [] -> Sn.of_int (-f)
-          | h :: _ -> let n = Sn.of_int (get_number h) in
-            Sn.mul (Sn.of_int f) n
-      in
-      Logger.debug_f_ "Tlc2.infimum=%s" (Sn.string_of v) >>= fun () ->
-      Lwt.return v
+    method get_infimum_i () = TlogMap.infimum tlog_map
 
     method get_head_name () =
       Filename.concat head_dir (head_name())
@@ -696,6 +443,9 @@ class tlc2
         | None -> None
         | Some pe ->  Some (Entry.v_of pe, Entry.i_of pe)
 
+    method tlogs_to_collapse head_i last_i tlogs_to_keep =
+      TlogMap.tlogs_to_collapse tlog_map head_i last_i tlogs_to_keep
+
     method close ?(wait_for_compression=false) () =
       Lwt_mutex.lock _write_lock >>= fun () ->
       Logger.debug_ "tlc2::close()" >>= fun () ->
@@ -727,7 +477,7 @@ class tlc2
       Logger.debug_ "tlc2::closes () (part2)" >>= fun () ->
       let last_file () =
         match _file with
-          | None -> _init_file fsync_tlog_dir tlog_dir tlf_dir _outer
+          | None -> _init_file fsync_tlog_dir tlog_map _outer
           | Some file -> Lwt.return file
       in
       last_file () >>= fun file ->
@@ -745,22 +495,25 @@ class tlc2
       end >>= fun () ->
       F.close file
 
-    method get_tlog_from_i (i:Sn.t) = get_file_number i
+    method get_tlog_from_i (i:Sn.t) : int = TlogMap.outer_of_i tlog_map i
+
+    method get_start_i (n:int) : Sn.t = TlogMap.get_start_i tlog_map n
+    method is_rollover_point i = TlogMap.is_rollover_point tlog_map i
 
     method get_tlog_count () =
-      get_tlog_names tlog_dir tlf_dir >>= fun tlogs ->
+      TlogMap.get_tlog_names tlog_map >>= fun tlogs ->
       Lwt.return (List.length tlogs)
 
     method which_tlog_file (start_i : Sn.t) =
       let open Compression in
-      let n = Sn.to_int (get_file_number start_i) in
+      let n = self # get_tlog_from_i start_i in
       let an0 = archive_name Snappy n
       and an1 = archive_name Bz2 n
       and fn = file_name n
       in
-      let an0_c = get_full_path tlog_dir tlf_dir an0 in
-      let an1_c = get_full_path tlog_dir tlf_dir an1 in
-      let fn_c = get_full_path tlog_dir tlf_dir fn
+      let an0_c = TlogMap.get_full_path tlog_map an0
+      and an1_c = TlogMap.get_full_path tlog_map an1
+      and fn_c  = TlogMap.get_full_path tlog_map fn
       in
       let rec loop = function
         | []     -> Lwt.return None
@@ -776,9 +529,9 @@ class tlc2
       self # which_tlog_file start_i >>= fun co ->
       let canonical = match co with
         | Some fn -> fn
-        | None -> let n = Sn.to_int (get_file_number start_i) in
+        | None -> let n = self # get_tlog_from_i start_i in
                   let fn = file_name n in
-                  get_full_path tlog_dir tlf_dir fn
+                  TlogMap.get_full_path tlog_map fn
       in
       Logger.debug_f_ "start_i = %Li => canonical=%s" start_i canonical >>= fun () ->
       let name = Filename.basename canonical in
@@ -796,41 +549,24 @@ class tlc2
       >>= fun () ->
       Logger.debug_f_ "dumped:%s (%Li) bytes" canonical length
       >>= fun () ->
-      let next_i = Sn.add start_i (Sn.of_int !tlogEntriesPerFile) in
+      let next_i = failwith "Sn.add start_i (Sn.of_int !tlogEntriesPerFile)" in
       Lwt.return next_i
 
     method save_tlog_file name length ic =
       (* what with rotation (jump to new tlog), open streams, ...*)
-      let canon = get_full_path tlog_dir tlf_dir name in
+      let canon = TlogMap.get_full_path tlog_map name in
       let tmp = canon ^ ".tmp" in
       Logger.debug_f_ "save_tlog_file: %s" tmp >>= fun () ->
-      Lwt_io.with_file ~mode:Lwt_io.output tmp (fun oc -> Llio.copy_stream ~length ~ic ~oc) >>= fun () ->
+      Lwt_io.with_file ~mode:Lwt_io.output tmp
+                       (fun oc -> Llio.copy_stream ~length ~ic ~oc)
+      >>= fun () ->
       File_system.rename tmp canon
 
 
-    method remove_below i =
-      get_tlog_names tlog_dir tlf_dir >>= fun existing ->
-      let maybe_remove fn =
-        let n = get_number fn in
-        let fn_stop = Sn.of_int ((n+1) * !Tlogcommon.tlogEntriesPerFile) in
-        if fn_stop < i then
-          begin
-            let canonical = get_full_path tlog_dir tlf_dir fn in
-            Logger.debug_f_ "Unlinking %s" canonical >>= fun () ->
-            File_system.unlink canonical
-          end
-        else
-          Lwt.return ()
-      in
-      Lwt_list.iter_s maybe_remove existing
+    method remove_below i = TlogMap.remove_below tlog_map i
   end
 
-let get_last_tlog tlog_dir tlf_dir =
-  Logger.debug_ "get_last_tlog" >>= fun () ->
-  get_tlog_names tlog_dir tlf_dir >>= fun tlog_names ->
-  let new_c = get_count tlog_names in
-  Logger.debug_f_ "new_c:%i" new_c >>= fun () ->
-  Lwt.return (new_c, get_full_path tlog_dir tlf_dir (file_name new_c))
+
 
 let maybe_correct new_c last index =
   if new_c > 0 && last = None
@@ -847,10 +583,15 @@ let maybe_correct new_c last index =
   else
     Lwt.return (new_c, last, index)
 
-let make_tlc2 ~compressor tlog_dir tlf_dir head_dir ~fsync node_id ~fsync_tlog_dir =
+let make_tlc2 ~compressor
+              ?tlog_max_entries ?tlog_max_size
+              tlog_dir tlf_dir
+              head_dir ~fsync node_id ~fsync_tlog_dir
+  =
   Logger.debug_f_ "make_tlc2 %S" tlog_dir >>= fun () ->
-  get_last_tlog tlog_dir tlf_dir >>= fun (new_c, fn) ->
-  _validate_one fn node_id ~check_marker:true >>= fun (last, index) ->
+  TlogMap.make ?tlog_max_entries ?tlog_max_size tlog_dir tlf_dir node_id >>= fun tlog_map ->
+  TlogMap.get_last_tlog tlog_map >>= fun (new_c, fn) ->
+  _validate_one fn node_id ~check_marker:true >>= fun (last, index, pos) ->
   maybe_correct new_c last index >>= fun (new_c,last,new_index) ->
   Logger.debug_f_ "make_tlc2 after maybe_correct %s" (Index.to_string new_index) >>= fun () ->
   let msg =
@@ -859,7 +600,8 @@ let make_tlc2 ~compressor tlog_dir tlf_dir head_dir ~fsync node_id ~fsync_tlog_d
       | Some e -> let i = Entry.i_of e in "Some" ^ (Sn.string_of i)
   in
   Logger.debug_f_ "post_validation: last_i=%s" msg >>= fun () ->
-  let col = new tlc2 tlog_dir tlf_dir head_dir new_c last new_index ~compressor node_id ~fsync ~fsync_tlog_dir in
+  let col = new tlc2 tlog_dir tlf_dir head_dir new_c last new_index tlog_map
+                ~compressor node_id ~fsync ~fsync_tlog_dir in
   (* rewrite last entry with ANOTHER marker so we can see we got here *)
   begin
     match last with
@@ -871,7 +613,7 @@ let make_tlc2 ~compressor tlog_dir tlf_dir head_dir ~fsync node_id ~fsync_tlog_d
         let i = Entry.i_of e in
         let v = Entry.v_of e in
         let marker = Some (_make_open_marker node_id) in
-        _init_file fsync_tlog_dir tlog_dir tlf_dir new_c >>= fun file ->
+        _init_file fsync_tlog_dir tlog_map new_c >>= fun file ->
         let oc = F.oc_of file in
         Tlogcommon.write_marker oc i v marker >>= fun () ->
         F.close file >>= fun () ->
@@ -881,7 +623,6 @@ let make_tlc2 ~compressor tlog_dir tlf_dir head_dir ~fsync node_id ~fsync_tlog_d
   end
   >>= fun () ->
   Lwt.return col
-
 let _truncate_tlog filename =
   let skipper () _entry = Lwt.return () in
   let folder,extension,index = folder_for filename None in
diff --git a/src/tlog/tlc2_test.ml b/src/tlog/tlc2_test.ml
index 4936eeeb..555b8dac 100644
--- a/src/tlog/tlc2_test.ml
+++ b/src/tlog/tlc2_test.ml
@@ -18,21 +18,26 @@ open OUnit
 open Unix
 open Lwt
 open Update
-open Tlogcollection
 open Tlogcommon
+open Tlog_map
+open Tlogcollection
+open Tlogcollection_test
 
 let section = Logger.Section.main
+let node_id = "node_id"
 
-let create_test_tlc dn =
+let create_test_tlc ?tlog_max_entries ?tlog_max_size dn =
   let tlx_dir = (dn ^ "_tlx") in
   let compressor = Compression.Snappy in
-  Tlc2.make_tlc2 dn tlx_dir tlx_dir ~compressor ~fsync:false ~fsync_tlog_dir:false
+  Tlc2.make_tlc2
+    dn
+    ?tlog_max_entries ?tlog_max_size tlx_dir tlx_dir
+    ~compressor ~fsync:false ~fsync_tlog_dir:false
+
 let wrap_tlc = Tlogcollection_test.wrap create_test_tlc
 
-let prepare_tlog_scenarios (dn,factory) =
-  let old_tlog_entries_value = !Tlogcommon.tlogEntriesPerFile in
-  Tlogcommon.tlogEntriesPerFile := 5 ;
-  factory dn "node_name" >>= fun (tlog_coll:tlog_collection) ->
+let prepare_tlog_scenarios (dn,(factory:factory)) =
+  factory ~tlog_max_entries:5 dn node_id >>= fun (tlog_coll:tlog_collection) ->
   let value = Value.create_master_value ~lease_start:0. "me" in
   tlog_coll # log_value  0L value  >>= fun _ ->
   tlog_coll # log_value  1L value  >>= fun _ ->
@@ -40,26 +45,24 @@ let prepare_tlog_scenarios (dn,factory) =
   tlog_coll # log_value  3L value  >>= fun _ ->
   tlog_coll # log_value  4L value  >>= fun _ ->
   tlog_coll # close () >>= fun _ ->
-  Lwt.return old_tlog_entries_value
+  Lwt.return ()
 
 let test_interrupted_rollover (dn, tlx_dir, factory) =
-  prepare_tlog_scenarios (dn,factory) >>= fun old_tlog_entries_value ->
-  (*let fn = Tlc2.get_full_path dn tlx_dir "001.tlog" in
-    Unix.unlink fn; *)
-  factory dn "node_name" >>= fun tlog_coll ->
+  let open Tlog_map in
+  prepare_tlog_scenarios (dn,factory) >>= fun () ->
+  factory ~tlog_max_entries:5 dn node_id >>= fun tlog_coll ->
   let value = Value.create_master_value ~lease_start:0. "me" in
   tlog_coll # log_value 5L value >>= fun _ ->
-  tlog_coll # close () >>= fun _ ->
-  Tlc2.get_tlog_names dn tlx_dir >>= fun tlog_names ->
+  tlog_coll # close () >>= fun _ ->  
+  Tlog_map._get_tlog_names dn tlx_dir >>= fun tlog_names ->
   let n = List.length tlog_names in
-  Tlogcommon.tlogEntriesPerFile := old_tlog_entries_value;
   let msg = Printf.sprintf "Number of tlogs incorrect. Expected 2, got %d" n in
   Lwt.return (OUnit.assert_equal ~msg n 2)
 
 
-let test_validate_at_rollover_boundary (dn, tlx_dir, factory) =
-  prepare_tlog_scenarios (dn,factory) >>= fun old_tlog_entries_value ->
-  factory dn "node_name" >>= fun val_tlog_coll ->
+let test_validate_at_rollover_boundary (dn, tlx_dir, (factory:Tlogcollection_test.factory)) =
+  prepare_tlog_scenarios (dn,factory) >>= fun () ->
+  factory ~tlog_max_entries:5 dn node_id >>= fun val_tlog_coll ->
   Logger.debug_ "1" >>= fun () ->
   val_tlog_coll # validate_last_tlog () >>= fun (_validity, lasteo, _index) ->
   let lasti, lasti_str =
@@ -71,33 +74,31 @@ let test_validate_at_rollover_boundary (dn, tlx_dir, factory) =
   in
   val_tlog_coll # close () >>= fun () ->
   let msg = Printf.sprintf "Values of is are different 4 <> %s" lasti_str in
-  begin
-    if lasti <> 4L
-    then Tlogcommon.tlogEntriesPerFile := old_tlog_entries_value
-  end;
   OUnit.assert_equal ~msg lasti 4L;
-  factory dn "node_name" >>= fun (tlog_coll:tlog_collection) ->
+  Logger.debug_ "2" >>= fun () ->
+  factory ~tlog_max_entries:5 dn node_id >>= fun (tlog_coll:tlog_collection) ->
+  Logger.debug_ "3" >>= fun () ->
   let value = Value.create_master_value ~lease_start:0. "me" in
   tlog_coll # log_value 5L value >>= fun _ ->
   tlog_coll # log_value 6L value >>= fun _ ->
   tlog_coll # log_value 7L value >>= fun _ ->
   tlog_coll # log_value 8L value >>= fun _ ->
   tlog_coll # log_value 9L value >>= fun _ ->
-  Tlc2.get_tlog_names dn tlx_dir >>= fun tlog_names ->
+  tlog_coll # close () >>= fun () ->
+  Tlog_map._get_tlog_names dn tlx_dir >>= fun tlog_names ->
   let n = List.length tlog_names in
-  Tlogcommon.tlogEntriesPerFile := old_tlog_entries_value;
   let msg = Printf.sprintf "Number of tlogs incorrect. Expected 2, got %d" n in
   Lwt.return (OUnit.assert_equal ~msg n 2)
 
-let test_iterate4 (dn, tlx_dir, factory) =
+let test_iterate4 (dn, tlx_dir, (factory:Tlogcollection_test.factory)) =
   Logger.debug_ "test_iterate4" >>= fun () ->
-  let () = Tlogcommon.tlogEntriesPerFile := 100 in
-  factory dn "node_name" >>= fun (tlc:tlog_collection) ->
+  factory ~tlog_max_entries:100 dn node_id >>= fun (tlc:tlog_collection) ->
   let value = Value.create_client_value [Update.Set("test_iterate4","xxx")] false in
   Tlogcollection_test._log_repeat tlc value 120 >>= fun () ->
   Lwt_unix.sleep 3.0 >>= fun () -> (* TODO: compression should have callback *)
-  let extension = Tlc2.extension Compression.Snappy in
-  let fnc = Tlc2.get_full_path dn tlx_dir ("000" ^ extension) in
+  let extension = Tlog_map.extension Compression.Snappy in
+
+  let fnc = Tlog_map._get_full_path dn tlx_dir ("000" ^ extension) in
   File_system.unlink fnc >>= fun () ->
   (* remove 000.tlog & 000.tlx ; errors? *)
   tlc # get_infimum_i () >>= fun inf ->
@@ -108,9 +109,9 @@ let test_iterate4 (dn, tlx_dir, factory) =
   Lwt.return ()
 
 
-let test_iterate5 (dn, _tlx_dir, factory) =
-  let () = Tlogcommon.tlogEntriesPerFile := 10 in
-  factory dn "node_name" >>= fun (tlc:tlog_collection) ->
+let test_iterate5 (dn, _tlx_dir, (factory:factory)) =
+  TlogMap.make ~tlog_max_entries:10 dn _tlx_dir node_id >>= fun tlog_map ->
+  factory dn node_id >>= fun (tlc:tlog_collection) ->
   let rec loop (tlc:tlog_collection) i =
     if i = 33
     then Lwt.return ()
@@ -125,7 +126,7 @@ let test_iterate5 (dn, _tlx_dir, factory) =
           then
             begin
               tlc # close ~wait_for_compression:true () >>= fun () ->
-              factory dn "node_name"
+              factory dn node_id
             end
           else Lwt.return tlc
         end
@@ -145,10 +146,10 @@ let test_iterate5 (dn, _tlx_dir, factory) =
   tlc # iterate start_i too_far_i f >>= fun () ->
   Lwt.return ()
 
-let test_iterate6 (dn, _tlx_dir, factory) =
-  let () = Tlogcommon.tlogEntriesPerFile := 10 in
+let test_iterate6 (dn, _tlx_dir, (factory:factory)) =
+  TlogMap.make ~tlog_max_entries:10 dn _tlx_dir node_id >>= fun tlog_map ->
   let sync = false in
-  factory dn "node_name" >>= fun (tlc:tlog_collection) ->
+  factory dn node_id >>= fun (tlc:tlog_collection) ->
   let rec loop i =
     if i = 33
     then Lwt.return ()
@@ -190,12 +191,11 @@ let test_iterate6 (dn, _tlx_dir, factory) =
   Lwt.return ()
 
 
-let test_compression_bug (dn, tlx_dir, factory) =
+let test_compression_bug (dn, tlx_dir, (factory:factory)) =
   Logger.info_ "test_compression_bug" >>= fun () ->
-  let () = Tlogcommon.tlogEntriesPerFile := 10 in
+  let tlog_max_entries = 10 in
+  factory ~tlog_max_entries dn node_id >>= fun (tlc:tlog_collection) ->
   let v = Bytes.create (1024 * 1024) in
-  factory dn "node_name" >>= fun (tlc:tlog_collection) ->
-  Logger.info_ "have tls" >>= fun () ->
   let sync = false in
   let n = 12 in
   let rec loop i =
@@ -214,7 +214,7 @@ let test_compression_bug (dn, tlx_dir, factory) =
   File_system.stat (tlx_dir ^ "/000.tlx") >>= fun stat ->
   OUnit.assert_bool "file should have size >0" (stat.st_size > 0);
   let entries = ref [] in
-  factory dn "node_name" >>= fun tlc2 ->
+  factory ~tlog_max_entries dn node_id >>= fun tlc2 ->
   tlc2 # iterate 0L (Sn.of_int n)
     (fun entry ->
        let i = Entry.i_of entry in
@@ -223,21 +223,20 @@ let test_compression_bug (dn, tlx_dir, factory) =
   >>= fun () ->
   OUnit.assert_equal
     ~printer:string_of_int
-    ~msg:"tls has a hole" (n+2) (List.length !entries);
+    ~msg:"tlog_collection has a hole" (n+2) (List.length !entries);
   Lwt.return ()
 
-let test_compression_previous (dn, tlx_dir, factory) =
+let test_compression_previous (dn, tlx_dir, (factory:factory)) =
   Logger.info_ "test_compression_previous_tlogs" >>= fun () ->
-  let () = Tlogcommon.tlogEntriesPerFile := 10 in
   let v = Bytes.create (1024 * 1024) in
-  factory dn "node_name" >>= fun (tlc:tlog_collection) ->
+  factory ~tlog_max_entries:10 dn node_id >>= fun (tlc:tlog_collection) ->
   Logger.info_ "have tlc" >>= fun () ->
   let sync = false in
   let n = 42 in
   let rec loop i =
     if i = n then Lwt.return ()
     else
-      let key = Printf.sprintf "test_compression_bug_%i" i in
+      let key = Printf.sprintf "test_compression_previous_%i" i in
       let value = Value.create_client_value [Update.Set(key, v)] sync in
       let sni = Sn.of_int i in
       tlc # log_value sni value >>= fun _ ->
@@ -250,7 +249,7 @@ let test_compression_previous (dn, tlx_dir, factory) =
   tlc # close ~wait_for_compression:true () >>= fun () ->
 
   (* mess around : uncompress tlxs to tlogs again, put some temp files in the way *)
-  let ext = Tlc2.extension Compression.Snappy in
+  let ext = Tlog_map.extension Compression.Snappy in
   let uncompress tlx =
     let tlxpath =  (tlx_dir ^ "/" ^ tlx ^ ext) in
     Compression.uncompress_tlog tlxpath (dn ^ "/" ^ tlx ^ ".tlog") >>= fun () ->
@@ -266,7 +265,7 @@ let test_compression_previous (dn, tlx_dir, factory) =
   touch (tlx_dir ^ "/000" ^ ext ^ ".part");
 
   (* open tlog again *)
-  factory dn "node_name" >>= fun tlc2 ->
+  factory dn node_id >>= fun tlc2 ->
   (* give tlogcollection a little time to create the compression jobs *)
   Lwt_unix.sleep 1.0 >>= fun () ->
   tlc2 # close ~wait_for_compression:true () >>= fun () ->
@@ -280,7 +279,32 @@ let test_compression_previous (dn, tlx_dir, factory) =
   assert ((List.length tlog_entries) = 1);
   Lwt.return ()
 
+let test_size_based_roll_over1 (dn, tlx_dir, (factory:factory)) =
+  Logger.info_ "test_size_based_roll_over" >>= fun () ->
+  let v = Bytes.create 1024 in
+  factory ~tlog_max_size:32000 dn node_id >>= fun tlc ->
+  let n = 100 in
+  let rec loop i =
+    if i = n
+    then Lwt.return_unit
+    else
+      let key = Printf.sprintf "test_size_based_roll_over_%02i" i in
+      let value = Value.create_client_value [Update.Set(key,v)] false in
+      let sni = Sn.of_int i in
+      tlc # log_value sni value >>= fun _ ->
+      loop (i+1)
+  in
+  loop 0 >>= fun () ->
+  tlc # close ~wait_for_compression:true () >>= fun () ->
+  File_system.lwt_directory_list dn >>= fun tlog_entries ->
+  let printer = string_of_int in
+  OUnit.assert_equal 1 (List.length tlog_entries) ~printer;
+  File_system.lwt_directory_list tlx_dir >>= fun tlx_entries ->
+  OUnit.assert_equal 3 (List.length tlx_entries) ~printer;
 
+  TlogMap.make dn tlx_dir node_id >>= fun second_map ->
+  OUnit.assert_equal 3 (TlogMap.get_tlog_number second_map) ~printer;
+  Lwt.return_unit
 
 let make_test_tlc (x, y) = x >:: wrap_tlc y x
 
@@ -304,4 +328,5 @@ let suite = "tlc2" >:::
                   ("test_interrupted_rollover", test_interrupted_rollover);
                   ("test_compression_bug", test_compression_bug);
                   ("test_compression_previous", test_compression_previous);
+                  ("test_size_based_roll_over1",test_size_based_roll_over1)
                 ]
diff --git a/src/tlog/tlog_main.ml b/src/tlog/tlog_main.ml
index 0d9551b1..81e65fef 100644
--- a/src/tlog/tlog_main.ml
+++ b/src/tlog/tlog_main.ml
@@ -27,7 +27,7 @@ let dump_tlog filename ~values=
     Lwt_io.printlf "%s:%s%s" (Sn.string_of i)
       (Value.value2s v ~values) ms
   in
-  let folder,_,index = Tlc2.folder_for filename None in
+  let folder,_,index = Tlog_map.folder_for filename None in
 
   let t () =
     begin
@@ -53,7 +53,7 @@ let _last_entry filename =
     let () = last := Some entry in
     Lwt_io.printlf "%s:%Li" (Sn.string_of i) p
   in
-  let folder,_,index = Tlc2.folder_for filename None in
+  let folder,_,index = Tlog_map.folder_for filename None in
   let do_it ic =
     let lowerI = Sn.start in
     let higherI = None
diff --git a/src/tlog/tlog_map.ml b/src/tlog/tlog_map.ml
new file mode 100644
index 00000000..c21d3268
--- /dev/null
+++ b/src/tlog/tlog_map.ml
@@ -0,0 +1,466 @@
+open Lwt.Infix
+open Tlogcommon
+open Tlogreader2
+
+let section = Logger.Section.main
+
+let tlog_regexp = Str.regexp "^[0-9]+\\.tlog$"
+let file_regexp = Str.regexp "^[0-9]+\\.tl\\(og\\|f\\|c\\|x\\)$"
+
+let extension_of filename =
+  let len = String.length filename in
+  let dot_pos = String.rindex filename '.' in
+  String.sub filename dot_pos (len - dot_pos)
+
+let folder_for filename index =
+  let extension = extension_of filename in
+  match extension with
+  | ".tlog" -> AU.fold, extension, index
+  | ".tlx"  -> AS.fold, extension, None
+  | ".tlf"  -> AC.fold, extension, None
+  | ".tlc"  -> AO.fold, extension, None
+  | _       -> failwith (Printf.sprintf "no folder for '%s'" extension)
+
+let first_i_of filename =
+  let extension = extension_of filename in
+  let reader = 
+    match extension with
+    | ".tlog" -> (fun ic -> Sn.input_sn ic )                   
+    | ".tlx" ->
+       (fun ic ->
+         Compression.read_format ic Compression.Snappy >>= fun ()->
+         Sn.input_sn ic >>= fun _last_i ->
+         Llio.input_string ic >>= fun compressed ->
+         let uncompressed = Compression.uncompress_snappy compressed in
+         let buffer = Llio.make_buffer uncompressed 0 in
+         let i =  Sn.sn_from buffer in
+         Lwt.return i
+       )
+    | ".tlf"  ->
+       (fun ic ->
+         Compression.read_format ic Compression.Bz2 >>= fun ()->
+         Sn.input_sn ic >>= fun _last_i ->
+         Llio.input_string ic >>= fun compressed ->
+         let uncompressed = Compression.uncompress_bz2 compressed in
+         let buffer = Llio.make_buffer uncompressed 0 in
+         let i =  Sn.sn_from buffer in
+         Lwt.return i)
+    | ".tlc"  -> failwith "todo"
+    | x -> failwith (Printf.sprintf "not a tlog_file:%s" x)
+  in
+  Lwt_io.with_file ~mode:Lwt_io.input filename reader >>= fun i ->
+  Logger.debug_f_ "%s starts with i:%Li" filename i >>= fun () ->
+  Lwt.return i
+                   
+
+let get_number fn =
+  let dot_pos = String.index fn '.' in
+  let pre = String.sub fn 0 dot_pos in
+  int_of_string pre
+
+let extension comp =
+  let open Compression in
+  match comp with
+    | Snappy -> ".tlx"
+    | Bz2 -> ".tlf"
+    | No -> ".tlog"
+
+let archive_name comp c =
+  let x = extension comp in
+  Printf.sprintf "%03i%s" c x
+
+let file_name c = Printf.sprintf "%03i.tlog" c
+
+let _get_full_path tlog_dir tlx_dir name =
+    if Filename.check_suffix name ".tlx"
+       || Filename.check_suffix name ".tlx.part"
+       || Filename.check_suffix name ".tlf"
+       || Filename.check_suffix name ".tlf.part"
+    then
+      Filename.concat tlx_dir name
+    else
+      Filename.concat tlog_dir name
+
+let get_count tlog_names =
+  match List.length tlog_names with
+    | 0 -> 0
+    | n -> let last = List.nth tlog_names (n-1) in
+      let length = String.length last in
+      let postfix = String.sub last (length -4) 4 in
+      let number = get_number last in
+      if postfix = "tlog"
+      then number
+      else number + 1
+
+let _get_entries dir invalid_dir_exn =
+  Lwt.catch
+    (fun () ->
+     File_system.lwt_directory_list dir
+    )
+    (function
+      | Unix.Unix_error(Unix.EIO, fn, _) as exn ->
+         if fn = "readdir"
+         then Lwt.fail invalid_dir_exn
+         else Lwt.fail exn
+      | exn -> Lwt.fail exn
+    )
+
+let _get_tlog_names tlog_dir tlx_dir =
+  _get_entries tlog_dir (Node_cfg.InvalidTlogDir tlog_dir)
+  >>= fun tlog_entries ->
+  (if tlog_dir = tlx_dir
+   then
+     Lwt.return []
+   else
+     _get_entries tlx_dir (Node_cfg.InvalidTlxDir tlx_dir))
+  >>= fun tlf_entries ->
+  let entries = List.rev_append tlf_entries tlog_entries in
+  let filtered = List.filter
+                   (fun e -> Str.string_match file_regexp e 0)
+                   entries
+  in
+  let my_compare fn1 fn2 =
+    let n1 = get_number fn1
+    and n2 = get_number fn2 in
+    if n1 = n2
+              (* make .tlx < .tlf < .tlog , so we can filter later*)
+    then
+      let fn1x = extension_of fn1
+      and fn2x = extension_of fn2
+      in
+      let order x = match x with
+        | ".tlx"  -> 1
+        | ".tlf"  -> 2
+        | ".tlog" -> 3
+        | _       -> 4
+      in compare (order fn1x) (order fn2x)
+    else compare n1 n2
+  in
+  let sorted = List.sort my_compare filtered in
+  let filtered2 = List.fold_left
+                    (fun acc name ->
+                     match acc with
+                     | [] -> name :: acc
+                     | prev :: rest ->
+                        if get_number prev = get_number name
+                        then prev :: rest (* smaller is better *)
+                        else name :: acc
+                    ) [] sorted
+  in
+  let sorted2 = List.rev filtered2 in
+  Lwt.return sorted2
+
+let _make_close_marker node_id = "closed:" ^ node_id
+let _make_open_marker node_id =  "opened:" ^ node_id
+
+type validation_result = (Entry.t option * Index.index * int)
+exception TLCNotProperlyClosed of string
+
+let _validate_one tlog_name node_id ~check_marker : validation_result Lwt.t =
+  Logger.debug_f_ "Tlog_map._validate_one %s" tlog_name >>= fun () ->
+  let e2s e = let i = Entry.i_of e in Printf.sprintf "(%s,_)" (Sn.string_of i) in
+  let prev_entry = ref None in
+  let new_index = Index.make tlog_name in
+  Lwt.catch
+    (fun () ->
+       let first = Sn.of_int 0 in
+       let folder, _, index = folder_for tlog_name None in
+
+       let do_it ic = folder ic ~index Sn.start None ~first None
+                        (fun _a0 entry ->
+                           let () = Index.note entry new_index in
+                           let r = Some entry in
+                           let () = prev_entry := r in
+                           Lwt.return r)
+       in
+       Lwt_io.with_file tlog_name ~mode:Lwt_io.input do_it
+       >>= fun eo ->
+       begin
+         if not check_marker
+         then Lwt.return eo
+         else
+           let eo' =
+             match eo with
+               | None -> None
+               | Some e ->
+                 let s = Some (_make_close_marker node_id) in
+                 if Entry.check_marker e s
+                 then !prev_entry
+                 else raise (TLCNotProperlyClosed (Entry.entry2s e))
+           in
+           Lwt.return eo'
+       end
+       >>= fun eo' ->
+       Logger.debug_f_ "XX:a=%s" (Log_extra.option2s e2s eo') >>= fun () ->
+       Logger.debug_f_ "After validation index=%s" (Index.to_string new_index) >>= fun () ->
+       File_system.stat tlog_name >>= fun stat ->
+       let size = stat.Lwt_unix.st_size in
+       Logger.debug_f_ "%s size=%i" tlog_name size >>= fun () ->
+       Lwt.return (eo', new_index, size)
+    )
+    (function
+      | Unix.Unix_error(Unix.ENOENT,_,_) -> Lwt.return (None, new_index, 0)
+      | exn -> Lwt.fail exn
+    )
+
+module TlogMap = struct
+  type t = {
+      tlog_dir: string;
+      tlx_dir : string;
+      tlog_max_entries : int;
+      tlog_max_size : int;
+      mutable tlog_size: int;
+      mutable tlog_number: int;
+      mutable i_to_tlog_number : (Sn.t * int) list;
+    }
+
+  let new_entry t s = t.tlog_size <- t.tlog_size + s
+
+  let get_tlog_number t= t.tlog_number
+
+  let get_tlog_names t = _get_tlog_names t.tlog_dir t.tlx_dir
+
+  let get_full_path t x = _get_full_path t.tlog_dir t.tlx_dir x
+
+  let make ?tlog_max_entries
+           ?tlog_max_size
+           tlog_dir tlx_dir node_id
+    =
+    let with_default o d =
+      match o with
+      | None -> d
+      | Some x -> x
+    in
+    let tlog_max_entries = with_default tlog_max_entries 100_000 in
+    let tlog_max_size    = with_default tlog_max_size 32_000_000 in
+
+    _get_tlog_names tlog_dir tlx_dir >>= fun tlog_names ->
+    let tlog_number = get_count tlog_names in
+    Logger.debug_f_ "tlog_number:%i" tlog_number >>= fun () ->
+    let full_path = _get_full_path tlog_dir tlx_dir (file_name tlog_number) in
+    _validate_one full_path node_id ~check_marker:true
+    >>= fun (last, index, tlog_size) ->
+    Lwt_list.map_s
+      (fun tlog_name ->
+        let n = get_number tlog_name in
+        let canonical = _get_full_path tlog_dir tlx_dir tlog_name in
+       first_i_of canonical >>= fun i ->
+       Lwt.return (i,n)
+      ) tlog_names
+    >>= fun i_to_tlog_number ->
+    let t = {tlog_dir;tlx_dir;tlog_max_entries; tlog_max_size;
+             tlog_size; tlog_number;
+             i_to_tlog_number;
+            }
+    in
+    Lwt.return t
+
+
+  let get_last_tlog t =
+    Logger.debug_ "get_last_tlog" >>= fun () ->
+    get_tlog_names t >>= fun tlog_names ->
+    let new_c = get_count tlog_names in
+    Logger.debug_f_ "new_c:%i" new_c >>= fun () ->
+    let full_path = _get_full_path t.tlog_dir t.tlx_dir (file_name new_c) in
+    Lwt.return (new_c, full_path)
+
+
+  
+  let get_tlog_i t n = Sn.mul n (Sn.of_int t.tlog_max_entries)
+
+
+
+  let find_tlog_file t c =
+    let open Compression in
+    let f0 = archive_name Snappy c in
+    let f1 = archive_name Bz2 c in
+    let f2 = file_name c in
+    let rec _find = function
+      | [] -> Lwt.fail (Failure (Printf.sprintf "no tlog file for %3i" c))
+      | f :: fs ->
+         let full = _get_full_path t.tlog_dir t.tlx_dir f in
+         File_system.exists full >>= fun ok ->
+         if ok
+         then Lwt.return full
+         else _find fs
+    in
+    _find [f0;f1;f2]
+
+  let fold_read tlog_map file_name
+                ~index
+                (lowerI:Sn.t)
+                (too_far_i:Sn.t option)
+                ~first
+                (a0:'a)
+                (f:'a -> Entry.t -> 'a Lwt.t) =
+
+    let full_name = _get_full_path tlog_map.tlog_dir tlog_map.tlx_dir file_name in
+    let folder, extension, index' = folder_for file_name index in
+    Logger.debug_f_ "fold_read extension=%s => index':%s" extension (Tlogreader2.Index.to_string index') >>= fun () ->
+    let ic_f ic = folder ic ~index:index' lowerI too_far_i ~first a0 f in
+    Lwt.catch
+      (fun () -> Lwt_io.with_file ~mode:Lwt_io.input full_name ic_f)
+      (function
+        | Unix.Unix_error(_,"open",_) as exn ->
+           if extension = ".tlog"
+           then (* file might have moved meanwhile *)
+             begin
+               Logger.debug_f_ "%s became archive" file_name
+               >>= fun () ->
+               let c = get_number file_name in
+               find_tlog_file tlog_map c >>= fun full_an ->
+               let real_folder,_,_ = folder_for full_an None in
+               Logger.debug_f_ "folding compressed %s" full_an >>= fun () ->
+               Lwt_io.with_file ~mode:Lwt_io.input full_an
+                                (fun ic -> real_folder ic ~index:None lowerI too_far_i ~first a0 f)
+             end
+           else
+             Lwt.fail exn
+        | exn -> Lwt.fail exn
+      )
+  let iterate_tlog_dir t ~index start_i too_far_i f =
+    let open Tlogcommon in
+    let open Tlogreader2 in
+    let tfs = Sn.string_of too_far_i in
+    Logger.debug_f_ "Tlc2.iterate_tlog_dir start_i:%s too_far_i:%s ~index:%s"
+                    (Sn.string_of start_i) tfs (Index.to_string index)
+    >>= fun () ->
+    get_tlog_names t >>= fun tlog_names ->
+    let acc_entry (_i0:Sn.t) entry = f entry >>= fun () -> let i = Entry.i_of entry in Lwt.return i in
+    let num_tlogs = List.length tlog_names in
+    let maybe_fold (cnt,low) fn =
+      let factor = Sn.of_int (t.tlog_max_entries) in
+      let linf = Sn.of_int (get_number fn) in
+      let ( * ) = Sn.mul in
+      let test0 = linf * factor in
+      let test1 = (Sn.succ linf) * factor in
+      let low_s = Sn.string_of low in
+      let lowp = Sn.succ low in
+      let lowp_s = Sn.string_of lowp in
+      let test_result =
+        (test0 <= lowp) &&
+          (low   <  test1) &&
+            low <= too_far_i
+      in
+      Logger.debug_f_ "%s <?= lowp:%s &&  low:%s <? %s && (low:%s <?= %s) yields:%b"
+                      (Sn.string_of test0 ) lowp_s
+                      low_s (Sn.string_of test1)
+                      low_s tfs
+                      test_result
+      >>= fun () ->
+      if test_result
+      then
+        begin
+          Logger.debug_f_ "fold_read over: %s (test0=%s)" fn
+                          (Sn.string_of test0) >>= fun () ->
+          let first = test0 in
+          Logger.info_f_ "Replaying tlog file: %s (%d/%d)" fn cnt num_tlogs  >>= fun () ->
+          let t1 = Unix.gettimeofday () in
+          fold_read t fn ~index low (Some too_far_i) ~first low acc_entry >>= fun x ->
+          Logger.info_f_ "Completed replay of %s, took %f seconds, %i to go" fn (Unix.gettimeofday () -. t1) (num_tlogs - cnt) >>= fun () ->
+          Lwt.return (cnt+1,x)
+        end
+      else Lwt.return (cnt+1,low)
+    in
+    Lwt_list.fold_left_s maybe_fold (1,start_i) tlog_names
+    >>= fun _x ->
+    Lwt.return ()
+
+  let inner_of_i t = function
+    | None -> 0
+    | Some i ->
+       let step = (Sn.of_int t.tlog_max_entries) in
+       (Sn.to_int (Sn.rem i step)) + 1
+
+  let outer_of_i t i =
+    let rec find pn = function
+      | [] -> pn
+      | (i0,n0) :: rest ->
+         if i0 < i
+         then find n0 rest
+         else n0
+    in
+    find 0 t.i_to_tlog_number
+
+  let get_start_i t n =
+    let rec find = function
+      | [] -> failwith "todo"
+      | (i0,n0) :: rest ->
+         if n0 = n
+         then i0 else
+           find rest
+    in
+    find t.i_to_tlog_number
+         
+  let is_first_of_new t i = Sn.rem i (Sn.of_int t.tlog_max_entries) = Sn.zero
+
+  let should_roll t outer i =
+    let ( *: ) = Sn.mul in
+    let r = i >= (Sn.of_int t.tlog_max_entries) *: (Sn.of_int (outer + 1)) in
+    let r' = r || t.tlog_size >= t.tlog_max_size in
+    let () =
+      Logger.ign_debug_f_
+        "should_roll tlog_max_entries:%i tlog_max_size:%i outer:%i i:%Li tlog_size:%i => %b"
+        t.tlog_max_entries t.tlog_max_size outer i t.tlog_size r'
+    in
+    let () =
+      if r'
+      then
+        t.tlog_size <- 0
+    in
+    r'
+
+
+  let new_outer t i =
+    let r = t.tlog_number + 1 in
+    let () = t.tlog_number <- r in
+    r
+
+  let infimum t =
+    get_tlog_names t >>= fun names ->
+    let f = t.tlog_max_entries in
+    let v =
+      match names with
+      | [] -> Sn.of_int (-f)
+      | h :: _ -> let n = Sn.of_int (get_number h) in
+                  Sn.mul (Sn.of_int f) n
+    in
+    Logger.debug_f_ "Tlc2.infimum=%s" (Sn.string_of v) >>= fun () ->
+    Lwt.return v
+
+  let remove_below t i =
+    get_tlog_names t >>= fun existing ->
+    let maybe_remove fn =
+      let n = get_number fn in
+      let fn_stop = Sn.of_int ((n+1) * t.tlog_max_entries) in
+      if fn_stop < i then
+        begin
+          let canonical = _get_full_path t.tlog_dir t.tlx_dir fn in
+          Logger.debug_f_ "Unlinking %s" canonical >>= fun () ->
+          File_system.unlink canonical
+        end
+      else
+        Lwt.return ()
+    in
+    Lwt_list.iter_s maybe_remove existing
+
+  let is_rollover_point t i =
+    (* TODO: this gets called a zillion times in catchup...
+     *)
+    Logger.ign_debug_f_ "is_rollover_point %s" (Sn.string_of i); 
+    let rec find = function
+      | [] -> false
+      | (i0,_ ) :: _ when i0 = i -> true
+      | (i0,_ ) :: _ when i0 > i -> false
+      | _ :: rest -> find rest
+    in
+    find t.i_to_tlog_number
+
+  let tlogs_to_collapse t head_i last_i tlogs_to_keep =
+    let lag = Sn.to_int (Sn.sub last_i head_i) in
+    let npt = t.tlog_max_entries in
+    let tlog_lag = (lag +npt - 1)/ npt in
+    let tlogs_to_collapse = tlog_lag - tlogs_to_keep - 1 in
+    tlogs_to_collapse
+
+end
diff --git a/src/tlog/tlogcollection.ml b/src/tlog/tlogcollection.ml
index 02b59816..1476a1a2 100644
--- a/src/tlog/tlogcollection.ml
+++ b/src/tlog/tlogcollection.ml
@@ -17,7 +17,8 @@ limitations under the License.
 open Tlogcommon
 open Tlogreader2
 
-class type tlog_collection = object
+class type tlog_collection =
+  object
   method validate_last_tlog: unit -> (tlogValidity * Entry.t option * Index.index) Lwt.t
   method iterate: Sn.t -> Sn.t -> (Entry.t -> unit Lwt.t) -> unit Lwt.t
   method accept : Sn.t -> Value.t    -> int Lwt.t
@@ -25,6 +26,8 @@ class type tlog_collection = object
   method log_value_explicit : Sn.t -> Value.t ->
                               sync:bool -> string option -> int Lwt.t
   method get_last_i: unit -> Sn.t
+  method is_rollover_point: Sn.t -> bool
+  method tlogs_to_collapse: Sn.t -> Sn.t -> int -> int
   method get_last_value: Sn.t -> Value.t option (* Lwt.t *)
   method get_last: unit -> (Value.t * Sn.t) option
   method close : ?wait_for_compression : bool -> unit -> unit Lwt.t
@@ -34,8 +37,18 @@ class type tlog_collection = object
   method dump_tlog_file : Sn.t -> Lwt_io.output_channel -> Sn.t Lwt.t
   method save_tlog_file : string -> int64 -> Lwt_io.input_channel -> unit Lwt.t
   method get_head_name : unit -> string
-  method get_tlog_from_i : Sn.t -> Sn.t
+  method get_tlog_from_i : Sn.t -> int
   method get_tlog_count: unit -> int Lwt.t
   method remove_below : Sn.t -> unit Lwt.t
+  method get_start_i : int -> Sn.t
   method which_tlog_file : Sn.t -> (string option) Lwt.t
-end
+  end
+
+type tlc_factory =
+  compressor:Compression.compressor ->
+  ?tlog_max_entries:int ->
+  ?tlog_max_size:int ->
+  string ->
+  string ->
+  string ->
+  fsync:bool -> string -> fsync_tlog_dir:bool -> tlog_collection Lwt.t
diff --git a/src/tlog/tlogcollection_test.ml b/src/tlog/tlogcollection_test.ml
index 48f60d01..801f58dc 100644
--- a/src/tlog/tlogcollection_test.ml
+++ b/src/tlog/tlogcollection_test.ml
@@ -22,10 +22,19 @@ open Lwt
 open Update
 open Tlogcollection
 open Tlogcommon
+open Tlog_map
 
 let section = Logger.Section.main
+let node_id = "node_id"
 
-let setup factory test_name () =
+type factory =
+  ?tlog_max_entries:int ->
+  ?tlog_max_size:int ->
+  string ->
+  string ->
+  Tlogcollection.tlog_collection Lwt.t
+
+let setup (factory:factory) test_name () =
   let dn = Printf.sprintf "/tmp/%s" test_name in
   let tlf_dir = Printf.sprintf "%s_tlx" dn in
   Logger.info_f_ "setup %s" dn >>= fun () ->
@@ -71,39 +80,38 @@ let _log_repeat tlc (value:Value.t) n =
       end
   in loop Sn.start
 
-let test_rollover (dn, tlf_dir, factory) =
-  Logger.info_f_ "test_rollover %s, %s" dn tlf_dir >>= fun () ->
-  let () = Tlogcommon.tlogEntriesPerFile := 5 in
-  factory dn "node_name" >>= fun (c:tlog_collection) ->
+let test_rollover (dn, tlx_dir, (factory:factory)) =
+  Logger.info_f_ "test_rollover %s, %s" dn tlx_dir >>= fun () ->
+  TlogMap.make ~tlog_max_entries:5 dn tlx_dir node_id >>= fun tlog_map ->
+  factory dn node_id >>= fun (c:tlog_collection) ->
   let value = _make_set_v "x" "y" in
   _log_repeat c value 101 >>= fun () ->
   c # close () >>= fun ()->
   Lwt.return ()
 
 
-let test_rollover_1002 (dn, tlf_dir, factory) =
-  Logger.info_f_ "test_rollover_1002 %s, %s" dn tlf_dir >>= fun () ->
+let test_rollover_1002 (dn, tlx_dir, (factory:factory)) =
+  Logger.info_f_ "test_rollover_1002 %s, %s" dn tlx_dir >>= fun () ->
   let n = 5 in
-  let () = Tlogcommon.tlogEntriesPerFile := n in
-  factory dn "node_name" >>= fun (c:tlog_collection) ->
+  factory ~tlog_max_entries:n dn node_id >>= fun (c:tlog_collection) ->
   let value = _make_set_v "x" "y" in
   let n_updates = 1002 * n + 3 in
   _log_repeat c value n_updates >>= fun () ->
   c # close () >>= fun () ->
-  factory dn "node_name" >>= fun tlc_two ->
+  factory dn node_id >>= fun tlc_two ->
   let vo = tlc_two # get_last_value (Sn.of_int (n_updates-1)) in
   let vos = Log_extra.option2s (Value.value2s ~values:false) vo in
   Logger.info_f_ "last_value = %s" vos >>= fun () ->
   tlc_two # close() >>= fun () ->
   Lwt.return ()
 
-let test_get_value_bug (dn, _tlf_dir, factory) =
+let test_get_value_bug (dn, _tlf_dir, (factory:factory)) =
   Logger.info_ "test_get_value_bug" >>= fun () ->
-  factory dn "node_name" >>= fun (c0:tlog_collection) ->
+  factory dn node_id >>= fun (c0:tlog_collection) ->
   let v0 = Value.create_master_value ~lease_start:0. "XXXX" in
   c0 # log_value 0L v0 >>= fun _wr_result ->
   c0 # close () >>= fun () ->
-  factory dn "node_name" >>= fun c1 ->
+  factory dn node_id >>= fun c1 ->
   (* c1 # validate () >>= fun _ -> *)
   match c1 # get_last_value 0L with
     | None -> Llio.lwt_failfmt "get_last_update 0 yields None"
@@ -113,34 +121,34 @@ let test_regexp (_dn, _tlf_dir, _factory) =
   Logger.info_ "test_get_regexp_bug" >>= fun () ->
   let open Compression in
   let tests = ["001.tlog", true;
-             "000" ^ Tlc2.extension Snappy, true;
-             "000" ^ Tlc2.extension Snappy ^ ".part", false;
-             "000" ^ Tlc2.extension Bz2, true;
-             "000" ^ Tlc2.extension Bz2  ^ ".part", false;
+             "000" ^ Tlog_map.extension Snappy, true;
+             "000" ^ Tlog_map.extension Snappy ^ ".part", false;
+             "000" ^ Tlog_map.extension Bz2, true;
+             "000" ^ Tlog_map.extension Bz2  ^ ".part", false;
             ]
   in
   let test (fn,e) =
-    let r = Str.string_match Tlc2.file_regexp fn 0 in
+    let r = Str.string_match Tlog_map.file_regexp fn 0 in
     OUnit.assert_equal r e
   in
   List.iter test tests;
   Lwt.return ()
 
-let test_restart (dn, _tlf_dir, factory) =
-  factory dn "node_name" >>= fun (tlc_one:tlog_collection) ->
+let test_restart (dn, _tlf_dir, (factory:factory)) =
+  factory dn node_id >>= fun (tlc_one:tlog_collection) ->
   let value = _make_set_v "x" "y" in
   _log_repeat tlc_one value 100 >>= fun () ->
   tlc_one # close () >>= fun () ->
-  factory dn "node_name" >>= fun tlc_two ->
+  factory dn node_id >>= fun tlc_two ->
   let _ = tlc_two # get_last_value (Sn.of_int 99) in
   tlc_two # close () >>= fun () ->
   Lwt.return ()
 
 
-let test_iterate (dn, tlf_dir, factory) =
-  Logger.info_f_ "test_iterate  %s, %s" dn tlf_dir >>= fun () ->
-  let () = Tlogcommon.tlogEntriesPerFile := 100 in
-  factory dn "node_name" >>= fun  (tlc:tlog_collection) ->
+let test_iterate (dn, tlx_dir, (factory: factory)) =
+  Logger.info_f_ "test_iterate  %s, %s" dn tlx_dir >>= fun () ->
+  TlogMap.make ~tlog_max_entries:100 dn tlx_dir node_id >>= fun tlog_map ->
+  factory dn node_id >>= fun  (tlc:tlog_collection) ->
   let value = _make_set_v "xxx" "y" in
   _log_repeat tlc value 323 >>= fun () ->
   let sum = ref 0 in
@@ -157,10 +165,10 @@ let test_iterate (dn, tlf_dir, factory) =
   Lwt.return ()
 
 
-let test_iterate2 (dn, tlf_dir, factory) =
-  Logger.info_f_ "test_iterate2  %s, %s" dn tlf_dir >>= fun () ->
-  let () = Tlogcommon.tlogEntriesPerFile := 100 in
-  factory dn "node_name" >>= fun (tlc:tlog_collection) ->
+let test_iterate2 (dn, tlx_dir, (factory:factory)) =
+  Logger.info_f_ "test_iterate2  %s, %s" dn tlx_dir >>= fun () ->
+  TlogMap.make ~tlog_max_entries:100 dn tlx_dir node_id >>= fun tlog_map ->
+  factory dn node_id >>= fun (tlc:tlog_collection) ->
   let value = _make_set_v "test_iterate0" "xxx" in
   _log_repeat tlc value 3 >>= fun () ->
   let result = ref [] in
@@ -176,10 +184,10 @@ let test_iterate2 (dn, tlf_dir, factory) =
   Lwt.return ()
 
 
-let test_iterate3 (dn, tlf_dir, factory) =
-  Logger.info_f_ "test_iterate3  %s, %s" dn tlf_dir >>= fun () ->
-  let () = Tlogcommon.tlogEntriesPerFile := 100 in
-  factory dn "node_name" >>= fun (tlc:tlog_collection) ->
+let test_iterate3 (dn, tlx_dir, (factory:factory)) =
+  Logger.info_f_ "test_iterate3  %s, %s" dn tlx_dir >>= fun () ->
+  TlogMap.make ~tlog_max_entries:100 dn tlx_dir node_id >>= fun tlog_map ->
+  factory dn node_id  >>= fun (tlc:tlog_collection) ->
   let value = _make_set_v "test_iterate3" "xxx" in
   _log_repeat tlc value 120 >>= fun () ->
   let result = ref [] in
@@ -195,15 +203,15 @@ let test_iterate3 (dn, tlf_dir, factory) =
   tlc # close () >>= fun () ->
   Lwt.return ()
 
-let test_validate_normal (dn, tlf_dir, factory) =
-  Logger.info_f_ "test_validate_normal  %s, %s" dn tlf_dir >>= fun () ->
-  let () = Tlogcommon.tlogEntriesPerFile:= 100 in
-  factory dn "node_name" >>= fun (tlc:tlog_collection) ->
+let test_validate_normal (dn, tlx_dir, (factory:factory)) =
+  Logger.info_f_ "test_validate_normal  %s, %s" dn tlx_dir >>= fun () ->
+  TlogMap.make ~tlog_max_entries:100 dn tlx_dir node_id >>= fun tlog_map ->
+  factory dn node_id >>= fun (tlc:tlog_collection) ->
   let value = _make_set_v "XXX" "X" in
   _log_repeat tlc value 123 >>= fun () ->
   tlc # close () >>= fun () ->
   Logger.debug_f_ "reopening %s" dn >>= fun () ->
-  factory dn "node_name" >>= fun (tlc_two:tlog_collection) ->
+  factory dn node_id >>= fun (tlc_two:tlog_collection) ->
   tlc_two # validate_last_tlog () >>= fun result ->
   let _validity, eo, _ = result in
   let wsn = Sn.of_int 122 in
@@ -215,13 +223,15 @@ let test_validate_normal (dn, tlf_dir, factory) =
   OUnit.assert_equal io wanted ;
   Lwt.return ()
 
-let test_validate_corrupt_1 (dn, tlf_dir, factory) =
-  let () = Tlogcommon.tlogEntriesPerFile:= 100 in
-  factory dn "node_name" >>= fun (tlc:tlog_collection) ->
+let test_validate_corrupt_1 (dn, tlx_dir, (factory:factory)) =
+  let open Tlog_map in
+  TlogMap.make ~tlog_max_entries:100 dn tlx_dir node_id >>= fun tlog_map ->
+  factory dn node_id >>= fun (tlc:tlog_collection) ->
   let value = _make_set_v "Incompetent" "Politicians" in
   _log_repeat tlc value 42 >>= fun () ->
   tlc # close () >>= fun () ->
-  let fn = Tlc2.get_full_path dn tlf_dir "000.tlog" in
+  TlogMap.make dn tlx_dir node_id >>= fun tlog_map ->
+  let fn = TlogMap.get_full_path tlog_map "000.tlog" in
   Lwt_unix.openfile fn [Unix.O_RDWR] 0o640 >>= fun fd ->
   Lwt_unix.lseek fd 666 Unix.SEEK_SET >>= fun _ ->
   Lwt_unix.write fd "\x00\x00\x00\x00\x00\x00" 0 6 >>= fun _ ->
@@ -229,7 +239,7 @@ let test_validate_corrupt_1 (dn, tlf_dir, factory) =
   Logger.info_f_ "corrupted 6 bytes" >>= fun () ->
   Lwt.catch
     (fun () ->
-       factory dn "node_name" >>= fun (tlc_two:tlog_collection) ->
+       factory dn node_id >>= fun (tlc_two:tlog_collection) ->
        tlc_two # validate_last_tlog () >>= fun _ ->
        tlc_two # close () >>= fun () ->
        OUnit.assert_bool "this tlog should not be valid" false;
@@ -248,14 +258,17 @@ let test_validate_corrupt_1 (dn, tlf_dir, factory) =
   >>= fun () ->
   Lwt.return ()
 
-let wrap factory test (name:string) = lwt_bracket (setup factory name) test teardown
+let wrap (factory:factory) test (name:string) = lwt_bracket (setup factory name) test teardown
 
-let create_test_tlc dn = Mem_tlogcollection.make_mem_tlog_collection dn None None ~fsync:false ~fsync_tlog_dir:false
+let create_test_tlc ?tlog_max_entries ?tlog_max_size dn =
+  Mem_tlogcollection.make_mem_tlog_collection
+    ?tlog_max_entries ?tlog_max_size
+    dn None None ~fsync:false ~fsync_tlog_dir:false
 
 let wrap_memory name = wrap create_test_tlc name
 
 let suite_mem = "mem_tlogcollection" >::: [
-    "rollover" >:: wrap_memory test_rollover "rollover";
+      "rollover" >:: wrap_memory test_rollover "rollover";
     (* "get_value_bug" >:: wrap_memory test_get_value_bug;
         (* assumption that different tlog_collections with the same name have the same state *)
     *)
diff --git a/src/tlog/tlogcommon.ml b/src/tlog/tlogcommon.ml
index 83a7ecb4..d03b08cb 100644
--- a/src/tlog/tlogcommon.ml
+++ b/src/tlog/tlogcommon.ml
@@ -39,9 +39,6 @@ exception TLogCheckSumError of Int64.t
 exception TLogUnexpectedEndOfFile of Int64.t
 exception TLogSabotage
 
-let tlogEntriesPerFile =
-  ref (IFDEF SMALLTLOG THEN 1000 ELSE (100 * 1000) END)
-
 let tlogExtension = ".tlog"
 let tlogFileRegex =
   let qex = Str.quote tlogExtension in

From 67c2cc302e3c1b44e17a058703e8226830bb4e6d Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Mon, 1 Feb 2016 18:34:15 +0100
Subject: [PATCH 080/373] catchup

---
 src/client/remote_nodestream.ml |  26 +--
 src/node/catchup.ml             |  72 +++------
 src/tlog/mem_tlogcollection.ml  |  17 +-
 src/tlog/tlc2.ml                |  88 +++++-----
 src/tlog/tlog_map.ml            | 275 ++++++++++++++++++++------------
 src/tlog/tlogcollection.ml      |  20 ++-
 6 files changed, 289 insertions(+), 209 deletions(-)

diff --git a/src/client/remote_nodestream.ml b/src/client/remote_nodestream.ml
index e9ca31e4..6aa57de2 100644
--- a/src/client/remote_nodestream.ml
+++ b/src/client/remote_nodestream.ml
@@ -83,12 +83,19 @@ class remote_nodestream ((ic,oc) as conn) =
             end
         end
       in
+      let _should_reinit = ref false in
       let rec loop_parts () =
         Llio.input_int ic >>= function
         | (-2) -> Logger.info_f_ "loop_parts done"
         | 1 ->
           begin
             Logger.debug_f_ "loop_entries" >>= fun () ->
+            begin
+              if !_should_reinit
+              then tlog_coll # reinit()
+              else Lwt.return ()
+            end
+            >>= fun () ->
             loop_entries () >>= fun () ->
             loop_parts ()
           end
@@ -101,17 +108,18 @@ class remote_nodestream ((ic,oc) as conn) =
             loop_parts ()
           end
         | 3 ->
-          begin
-            Logger.debug_f_ "save_file" >>= fun () ->
-            Llio.input_string ic >>= fun name ->
-            Llio.input_int64 ic >>= fun length ->
-            Logger.info_f_ "got %s (%Li bytes)" name length >>= fun () ->
-            tlog_coll # save_tlog_file name length ic >>= fun () ->
-            loop_parts ()
-          end
+           begin
+             let () = _should_reinit := true in
+             Logger.debug_f_ "save_file" >>= fun () ->
+             Llio.input_string ic >>= fun name ->
+             Llio.input_int64 ic >>= fun length ->
+             Logger.info_f_ "got %s (%Li bytes)" name length >>= fun () ->
+             tlog_coll # save_tlog_file name length ic >>= fun () ->
+             loop_parts ()
+           end
         | x  -> Llio.lwt_failfmt "don't know what %i means" x
       in
-      loop_parts()
+      loop_parts() 
     in
     request  oc outgoing >>= fun () ->
     response ic incoming
diff --git a/src/node/catchup.ml b/src/node/catchup.ml
index 1a764a4e..6c04d9b3 100644
--- a/src/node/catchup.ml
+++ b/src/node/catchup.ml
@@ -90,8 +90,7 @@ let catchup_tlog (type s) ~tls_ctx ~tcp_keepalive ~stop other_configs ~cluster_i
       tlog_coll # log_value_explicit i value ~sync:false None >>= fun _ ->
       stop_fuse stop
     in
-
-    client # iterate current_i f tlog_coll ~head_saved_cb
+    client # iterate current_i f tlog_coll ~head_saved_cb 
   in
 
   Lwt.catch
@@ -210,21 +209,9 @@ let catchup_store ~stop (type s) me
               Printf.sprintf
                 "Catchup store failed. Store counter is too low: %s < %s"
                 (Sn.string_of si) (Sn.string_of pred_too_far_i)
-            and n_entries = failwith "todo" (*Sn.of_int !Tlogcommon.tlogEntriesPerFile *) in
-            if Sn.rem si n_entries = Sn.pred n_entries
-            then
-              let ssi = Sn.succ si in
-              tlog_coll # which_tlog_file ssi >>= function
-              | None ->
-                let n = tlog_coll # get_tlog_from_i ssi in
-                Lwt.return
-                  (Printf.sprintf
-                     "%s (found tlog nor archive for %3i)" msg n)
-              | Some _ -> Lwt.return msg
-            else
-              Lwt.return msg
+            in
+            Lwt.fail (StoreCounterTooLow msg)
           end
-          >>= fun msg -> Lwt.fail (StoreCounterTooLow msg)
         else
           Lwt.return ()
       end
@@ -283,9 +270,6 @@ let last_entries
       (tlog_collection:Tlogcollection.tlog_collection)
       (start_i:Sn.t) (oc:Lwt_io.output_channel)
   =
-  (* This one is kept (for how long?)
-     for x-version clusters during upgrades
-   *)
   Lwt.fail_with "no longer supported"
 
 let last_entries2
@@ -301,11 +285,14 @@ let last_entries2
     match consensus_i with
       | None -> Lwt.return ()
       | Some ci ->
-        let stream_entries start_i too_far_i =
+         let stream_entries start_i too_far_i =
+           Logger.debug_f_ "stream_entries :%s %s" (Sn.string_of start_i) (Sn.string_of too_far_i)
+           >>= fun () ->
           let _write_entry entry =
             let i = Entry.i_of entry
             and v = Entry.v_of entry
             in
+            Logger.debug_f_ "write_entry:%s" (Sn.string_of i) >>= fun () ->
             Tlogcommon.write_entry oc i v >>= fun _total_size ->
             Lwt.return_unit
           in
@@ -319,22 +306,17 @@ let last_entries2
           Sn.output_sn oc (-1L)
         in
         let too_far_i = Sn.succ ci in
-        let step = failwith "todo" (*Sn.of_int (!Tlogcommon.tlogEntriesPerFile) *) in
         let rec loop_files (start_i2:Sn.t) =
-          if (* there are complete log files *)
-            Sn.rem start_i2 step = Sn.start &&
-              Sn.add start_i2 step < too_far_i
-          then
-            begin
-              Logger.debug_f_ "start_i2=%Li < %Li" start_i2 too_far_i
+          match tlog_collection # complete_file_to_deliver start_i2 with
+          | None -> Lwt.return start_i2
+          | Some (tlog_number, start_i2') ->
+             begin
+              Logger.debug_f_ "start_i2=%Li < %Li (outputting %03i)" start_i2 too_far_i tlog_number
               >>= fun () ->
               Llio.output_int oc 3 >>= fun () ->
-              tlog_collection # dump_tlog_file start_i2 oc
-              >>= fun start_i2' ->
+              tlog_collection # dump_tlog_file tlog_number oc >>= fun () ->
               loop_files start_i2'
             end
-          else
-            Lwt.return start_i2
         in
         let maybe_dump_head () =
           tlog_collection # get_infimum_i () >>= fun inf_i ->
@@ -358,22 +340,18 @@ let last_entries2
 
           (* maybe stream a bit *)
           begin
-            let (-:) = Sn.sub
-            and (+:) = Sn.add
-            and (/:) = Sn.rem and ( *: ) = Sn.mul
-            in
-            let next_rotation =
-              ((start_i2 +: (step -: 1L)) /: step) *: step
-            in
-            Logger.debug_f_ "next_rotation = %Li" next_rotation >>= fun () ->
-            (if start_i2 < next_rotation && next_rotation < too_far_i
-             then
-               begin
-                 stream_entries start_i2 next_rotation >>= fun () ->
-                 Lwt.return next_rotation
-               end
-             else Lwt.return start_i2
-            )
+            match tlog_collection # next_rollover start_i2 with
+            | None -> Lwt.return start_i2
+            | Some next_rollover ->
+               Logger.debug_f_ "next_rollover = %Li" next_rollover >>= fun () ->
+               (if next_rollover < too_far_i
+                then
+                  begin
+                    stream_entries start_i2 next_rollover >>= fun () ->
+                    Lwt.return next_rollover
+                  end
+                else Lwt.return start_i2
+               )
           end
           >>= fun start_i3 ->
           (* push out files *)
diff --git a/src/tlog/mem_tlogcollection.ml b/src/tlog/mem_tlogcollection.ml
index b93954c1..e9d0df1e 100644
--- a/src/tlog/mem_tlogcollection.ml
+++ b/src/tlog/mem_tlogcollection.ml
@@ -68,16 +68,11 @@ class mem_tlog_collection _name =
 
     method save_tlog_file _name _length _ic = failwith "save_tlog_file not supported"
 
+    method reinit () = Lwt.return ()
     method which_tlog_file _start_i = failwith "which_tlog_file not supported"
 
     method tlogs_to_collapse _ _ _ = failwith "tlogs_to_collapse not supported"
-    method is_rollover_point _ = failwith "is_rollover_point not supported"
 
-    method log_value_explicit i (v:Value.t) ~sync marker =
-      let entry = Entry.make i v 0L marker in
-      let () = data <- entry::data in
-      let () = last_entry <- (Some entry) in
-      Lwt.return 0
 
     method log_value i v = self #log_value_explicit i v ~sync:false None
 
@@ -90,12 +85,22 @@ class mem_tlog_collection _name =
 
     method get_tlog_from_i _ = 0
     method get_start_i n = Sn.zero
+    method is_rollover_point _ = failwith "is_rollover_point not supported"
+    method next_rollover _ = Some Int64.max_int
+
+    method log_value_explicit i (v:Value.t) ~sync marker =
+      let entry = Entry.make i v 0L marker in
+      let () = data <- entry::data in
+      let () = last_entry <- (Some entry) in
+      Lwt.return 0
+
 
     method close ?(wait_for_compression = false) () =
         let () = ignore wait_for_compression in
         Lwt.return ()
 
     method remove_below _i = Lwt.return ()
+    method complete_file_to_deliver = failwith "complete_tlog_file_to_deliver not supported"
   end
 
 let make_mem_tlog_collection
diff --git a/src/tlog/tlc2.ml b/src/tlog/tlc2.ml
index 7acae737..26983e3a 100644
--- a/src/tlog/tlc2.ml
+++ b/src/tlog/tlc2.ml
@@ -125,7 +125,8 @@ module F = struct
 end
 
 
-let _init_file fsync_dir tlog_map c =
+let _init_file fsync_dir tlog_map =
+  let c = TlogMap.get_tlog_number tlog_map in
   let fn = file_name c in
   let full_name = TlogMap.get_full_path tlog_map fn in
   Lwt_unix.openfile full_name [Unix.O_CREAT;Unix.O_APPEND;Unix.O_WRONLY] 0o644 >>= fun fd ->
@@ -151,18 +152,10 @@ class tlc2
         (tlog_map: TlogMap.t)
         (node_id:string) ~(fsync:bool) ~(fsync_tlog_dir:bool)
   =
-  let inner =
-    let io = match last with
-      | None -> None
-      | Some e -> Some (Entry.i_of e)
-    in
-    TlogMap.inner_of_i tlog_map io in
 
   object(self: # tlog_collection)
     val mutable _file = (None:F.t option)
     val mutable _index = (None: Index.index)
-    val mutable _inner = inner (* ~ pos in file *)
-    val mutable _outer = new_c (* ~ pos in dir *)
     val mutable _previous_entry = last
     val mutable _compression_q = Lwt_buffer.create_fixed_capacity 5
     val mutable _compression_thread = None
@@ -177,7 +170,7 @@ class tlc2
     method validate_last_tlog () =
       validate_last tlog_map node_id ~check_marker:false >>= fun r ->
       let (_validity, _entry, new_index) = r in
-      let tlu = TlogMap.get_full_path tlog_map (file_name _outer) in
+      let tlu = TlogMap.get_full_path tlog_map (TlogMap.get_tlog_number tlog_map|> file_name) in
       let matches = Index.match_filename tlu new_index in
       Logger.debug_f_ "tlu=%S new_index=%s index=%s => matches=%b"
         tlu
@@ -271,11 +264,6 @@ class tlc2
                else Lwt.return ()
              end
              >>= fun () ->
-             let () = match _previous_entry with
-               | None -> _inner <- _inner +1
-               | Some pe->
-                 let pi = Entry.i_of pe in if pi < i then _inner <- _inner +1
-             in
              let entry = Entry.make i value p marker in
              _previous_entry <- Some entry;
              Index.note entry _index;
@@ -295,38 +283,40 @@ class tlc2
       match _file with
         | None ->
           begin
-
             let outer = TlogMap.outer_of_i tlog_map i in
-            _outer <- outer;
             Logger.debug_f_ "prelude i:%s: outer=%i" (Sn.string_of i) outer>>= fun () ->
             begin
-              if (_outer <> 0) && (TlogMap.is_first_of_new tlog_map i )
+              if (outer <> 0) && (TlogMap.is_first_of_new tlog_map i )
               then
                 (* the first thing logged falls exactly into a new tlog,
                    the compression job for the previous one thus is not yet picked up *)
-                self # _add_compression_job (pred _outer)
+                self # _add_compression_job (pred outer)
               else
                 Lwt.return ()
             end >>= fun () ->
             let () =
-              if TlogMap.should_roll tlog_map _outer i
+              if TlogMap.should_roll tlog_map
               then
-                let new_outer = TlogMap.new_outer tlog_map i in
-                _outer <- new_outer
+                let _new_outer = TlogMap.new_outer tlog_map i in
+                ()
             in
-            _init_file fsync_tlog_dir tlog_map _outer >>= fun file ->
+            _init_file fsync_tlog_dir tlog_map  >>= fun file ->
             _file <- Some file;
             _index <- Index.make (F.fn_of file);
             Lwt.return file
           end
         | Some (file:F.t) ->
-          if TlogMap.should_roll tlog_map _outer i
+          if TlogMap.should_roll tlog_map
           then
             let do_jump () =
+              let old_outer = TlogMap.get_tlog_number tlog_map  in
               let new_outer = TlogMap.new_outer tlog_map i in
-              Logger.info_f_ "i= %s & outer = %i => jump to %i" (Sn.string_of i) _outer new_outer
+              Logger.info_f_ "i= %s old_outer = %i => jump to %i"
+                             (Sn.string_of i)
+                             old_outer 
+                             new_outer
               >>= fun () ->
-              self # _jump_to_tlog file new_outer
+              self # _jump_to_tlog file old_outer new_outer
             in
             do_jump ()
           else
@@ -346,15 +336,12 @@ class tlc2
          end
       | No -> Lwt.return ()
 
-    method private _jump_to_tlog file new_outer =
+    method private _jump_to_tlog file old_outer new_outer =
       F.close file >>= fun () ->
-      let old_outer = _outer in
-      _inner <- 0;
-      _outer <- new_outer;
-      _init_file fsync_tlog_dir tlog_map new_outer >>= fun new_file ->
+      _init_file fsync_tlog_dir tlog_map >>= fun new_file ->
       _file <- Some new_file;
       let index =
-        let fn = file_name _outer in
+        let fn = file_name new_outer in
         let full_name = TlogMap.get_full_path tlog_map fn in
         Index.make full_name
       in
@@ -477,7 +464,7 @@ class tlc2
       Logger.debug_ "tlc2::closes () (part2)" >>= fun () ->
       let last_file () =
         match _file with
-          | None -> _init_file fsync_tlog_dir tlog_map _outer
+          | None -> _init_file fsync_tlog_dir tlog_map 
           | Some file -> Lwt.return file
       in
       last_file () >>= fun file ->
@@ -499,14 +486,14 @@ class tlc2
 
     method get_start_i (n:int) : Sn.t = TlogMap.get_start_i tlog_map n
     method is_rollover_point i = TlogMap.is_rollover_point tlog_map i
-
+    method next_rollover i     = TlogMap.next_rollover tlog_map i
+                                                       
     method get_tlog_count () =
       TlogMap.get_tlog_names tlog_map >>= fun tlogs ->
       Lwt.return (List.length tlogs)
 
-    method which_tlog_file (start_i : Sn.t) =
+    method which_tlog_file (n:int) =
       let open Compression in
-      let n = self # get_tlog_from_i start_i in
       let an0 = archive_name Snappy n
       and an1 = archive_name Bz2 n
       and fn = file_name n
@@ -525,15 +512,15 @@ class tlc2
       loop [an0_c;an1_c;fn_c]
 
 
-    method dump_tlog_file start_i oc =
-      self # which_tlog_file start_i >>= fun co ->
+    method dump_tlog_file n oc =
+      self # which_tlog_file n >>= fun co ->
       let canonical = match co with
         | Some fn -> fn
-        | None -> let n = self # get_tlog_from_i start_i in
+        | None -> 
                   let fn = file_name n in
                   TlogMap.get_full_path tlog_map fn
       in
-      Logger.debug_f_ "start_i = %Li => canonical=%s" start_i canonical >>= fun () ->
+      Logger.debug_f_ "n = %03i => canonical=%s" n canonical >>= fun () ->
       let name = Filename.basename canonical in
       Llio.output_string oc name >>= fun () ->
       File_system.stat canonical >>= fun stats ->
@@ -548,12 +535,15 @@ class tlc2
         )
       >>= fun () ->
       Logger.debug_f_ "dumped:%s (%Li) bytes" canonical length
-      >>= fun () ->
-      let next_i = failwith "Sn.add start_i (Sn.of_int !tlogEntriesPerFile)" in
-      Lwt.return next_i
 
     method save_tlog_file name length ic =
       (* what with rotation (jump to new tlog), open streams, ...*)
+      begin
+        match _file with
+        | None -> Lwt.return ()
+        | Some file -> F.close file >>= fun () -> _file <- None;Lwt.return ()
+      end 
+      >>= fun () ->                                                               
       let canon = TlogMap.get_full_path tlog_map name in
       let tmp = canon ^ ".tmp" in
       Logger.debug_f_ "save_tlog_file: %s" tmp >>= fun () ->
@@ -563,7 +553,17 @@ class tlc2
       File_system.rename tmp canon
 
 
+    method reinit () =
+      begin
+        match _file with
+        | None -> Lwt.return ()
+        | Some file -> F.close file >>= fun () -> _file <- None;Lwt.return ()
+      end 
+      >>= fun () ->                                                               
+      TlogMap.reinit tlog_map
+                     
     method remove_below i = TlogMap.remove_below tlog_map i
+    method complete_file_to_deliver i = TlogMap.complete_file_to_deliver tlog_map i 
   end
 
 
@@ -613,7 +613,7 @@ let make_tlc2 ~compressor
         let i = Entry.i_of e in
         let v = Entry.v_of e in
         let marker = Some (_make_open_marker node_id) in
-        _init_file fsync_tlog_dir tlog_map new_c >>= fun file ->
+        _init_file fsync_tlog_dir tlog_map >>= fun file ->
         let oc = F.oc_of file in
         Tlogcommon.write_marker oc i v marker >>= fun () ->
         F.close file >>= fun () ->
diff --git a/src/tlog/tlog_map.ml b/src/tlog/tlog_map.ml
index c21d3268..9b485686 100644
--- a/src/tlog/tlog_map.ml
+++ b/src/tlog/tlog_map.ml
@@ -21,6 +21,12 @@ let folder_for filename index =
   | ".tlc"  -> AO.fold, extension, None
   | _       -> failwith (Printf.sprintf "no folder for '%s'" extension)
 
+let is_archive filename =
+  let extension = extension_of filename in
+  match extension with
+  | ".tlx" | ".tlf" | ".tlc"-> true
+  | _ -> false
+  
 let first_i_of filename =
   let extension = extension_of filename in
   let reader = 
@@ -45,7 +51,7 @@ let first_i_of filename =
          let buffer = Llio.make_buffer uncompressed 0 in
          let i =  Sn.sn_from buffer in
          Lwt.return i)
-    | ".tlc"  -> failwith "todo"
+    | ".tlc"  -> failwith "todo: first_i_if .tlc"
     | x -> failwith (Printf.sprintf "not a tlog_file:%s" x)
   in
   Lwt_io.with_file ~mode:Lwt_io.input filename reader >>= fun i ->
@@ -204,17 +210,39 @@ let _validate_one tlog_name node_id ~check_marker : validation_result Lwt.t =
     )
 
 module TlogMap = struct
+  type tlog_number = int
+  type is_archive = bool
   type t = {
       tlog_dir: string;
       tlx_dir : string;
       tlog_max_entries : int;
       tlog_max_size : int;
+      node_id : string;
       mutable tlog_size: int;
-      mutable tlog_number: int;
-      mutable i_to_tlog_number : (Sn.t * int) list;
+      mutable tlog_entries : int;
+      mutable tlog_number: tlog_number;
+      mutable i_to_tlog_number : (Sn.t * tlog_number * is_archive) list;
+      mutable should_roll: bool;
     }
+             
+  let show_map t =
+    "[" ^ 
+    String.concat ";"
+                  (List.map
+                     (fun (i,n,a) ->
+                       Printf.sprintf "(%s,%03i,%b)" (Sn.string_of i) n a
+                     ) t.i_to_tlog_number
+                  ) ^ "]"
+                  
+  let new_entry t s =
+    t.tlog_size <- t.tlog_size + s;
+    t.tlog_entries <- t.tlog_entries + 1;
+    if t.tlog_size >= t.tlog_max_size
+       || t.tlog_entries + 1 = t.tlog_max_entries
+    then
+      let () = t.should_roll <- true in
+      Logger.ign_debug_f_ "should roll over"
 
-  let new_entry t s = t.tlog_size <- t.tlog_size + s
 
   let get_tlog_number t= t.tlog_number
 
@@ -222,39 +250,66 @@ module TlogMap = struct
 
   let get_full_path t x = _get_full_path t.tlog_dir t.tlx_dir x
 
-  let make ?tlog_max_entries
-           ?tlog_max_size
-           tlog_dir tlx_dir node_id
-    =
-    let with_default o d =
-      match o with
-      | None -> d
-      | Some x -> x
-    in
-    let tlog_max_entries = with_default tlog_max_entries 100_000 in
-    let tlog_max_size    = with_default tlog_max_size 32_000_000 in
 
+  let _init tlog_dir tlx_dir node_id ~check_marker =
+    Logger.debug_f_ "_init ~check_marker:%b" check_marker >>= fun () ->
     _get_tlog_names tlog_dir tlx_dir >>= fun tlog_names ->
     let tlog_number = get_count tlog_names in
     Logger.debug_f_ "tlog_number:%i" tlog_number >>= fun () ->
     let full_path = _get_full_path tlog_dir tlx_dir (file_name tlog_number) in
-    _validate_one full_path node_id ~check_marker:true
+    _validate_one full_path node_id ~check_marker
     >>= fun (last, index, tlog_size) ->
     Lwt_list.map_s
       (fun tlog_name ->
         let n = get_number tlog_name in
         let canonical = _get_full_path tlog_dir tlx_dir tlog_name in
-       first_i_of canonical >>= fun i ->
-       Lwt.return (i,n)
+        first_i_of canonical >>= fun i ->
+        let b = is_archive tlog_name in
+       Lwt.return (i,n, b)
       ) tlog_names
     >>= fun i_to_tlog_number ->
-    let t = {tlog_dir;tlx_dir;tlog_max_entries; tlog_max_size;
-             tlog_size; tlog_number;
-             i_to_tlog_number;
+    let should_roll,last_start =
+      let rec find = function
+          | [] -> false , Sn.zero
+          | [(i0,_,true)] -> true, i0
+          | _ :: rest -> find rest
+      in find i_to_tlog_number
+    in
+    let tlog_entries =
+      match last with
+      | None -> 0
+      | Some e -> let i = Entry.i_of e in (Sn.to_int i) - (Sn.to_int last_start)
+    in
+    Lwt.return (tlog_entries, tlog_size, tlog_number, i_to_tlog_number, should_roll)
+      
+  let make ?tlog_max_entries
+           ?tlog_max_size
+           tlog_dir tlx_dir node_id
+    =
+    let with_default o d =
+      match o with
+      | None -> d
+      | Some x -> x
+    in
+    let tlog_max_entries = with_default tlog_max_entries 100_000 in
+    let tlog_max_size    = with_default tlog_max_size 32_000_000 in
+    _init tlog_dir tlx_dir node_id ~check_marker:true
+    >>= fun (tlog_entries, tlog_size,tlog_number,i_to_tlog_number, should_roll) ->
+    
+    let t = {node_id; tlog_dir;tlx_dir;tlog_max_entries; tlog_max_size;
+             tlog_entries; tlog_size; tlog_number; i_to_tlog_number;should_roll;
             }
     in
     Lwt.return t
 
+  let reinit t =
+    _init t.tlog_dir t.tlx_dir t.node_id ~check_marker:false 
+    >>= fun (tlog_entries, tlog_size, tlog_number,i_to_tlog_number,should_roll) ->
+    t.tlog_size   <- tlog_size;
+    t.tlog_number <- tlog_number;
+    t.i_to_tlog_number <- i_to_tlog_number;
+    t.should_roll <- should_roll;
+    Lwt.return ()
 
   let get_last_tlog t =
     Logger.debug_ "get_last_tlog" >>= fun () ->
@@ -296,7 +351,9 @@ module TlogMap = struct
 
     let full_name = _get_full_path tlog_map.tlog_dir tlog_map.tlx_dir file_name in
     let folder, extension, index' = folder_for file_name index in
-    Logger.debug_f_ "fold_read extension=%s => index':%s" extension (Tlogreader2.Index.to_string index') >>= fun () ->
+    Logger.debug_f_ "fold_read extension=%s => index':%s" extension
+                    (Tlogreader2.Index.to_string index')
+    >>= fun () ->
     let ic_f ic = folder ic ~index:index' lowerI too_far_i ~first a0 f in
     Lwt.catch
       (fun () -> Lwt_io.with_file ~mode:Lwt_io.input full_name ic_f)
@@ -318,102 +375,51 @@ module TlogMap = struct
              Lwt.fail exn
         | exn -> Lwt.fail exn
       )
-  let iterate_tlog_dir t ~index start_i too_far_i f =
-    let open Tlogcommon in
-    let open Tlogreader2 in
-    let tfs = Sn.string_of too_far_i in
-    Logger.debug_f_ "Tlc2.iterate_tlog_dir start_i:%s too_far_i:%s ~index:%s"
-                    (Sn.string_of start_i) tfs (Index.to_string index)
-    >>= fun () ->
-    get_tlog_names t >>= fun tlog_names ->
-    let acc_entry (_i0:Sn.t) entry = f entry >>= fun () -> let i = Entry.i_of entry in Lwt.return i in
-    let num_tlogs = List.length tlog_names in
-    let maybe_fold (cnt,low) fn =
-      let factor = Sn.of_int (t.tlog_max_entries) in
-      let linf = Sn.of_int (get_number fn) in
-      let ( * ) = Sn.mul in
-      let test0 = linf * factor in
-      let test1 = (Sn.succ linf) * factor in
-      let low_s = Sn.string_of low in
-      let lowp = Sn.succ low in
-      let lowp_s = Sn.string_of lowp in
-      let test_result =
-        (test0 <= lowp) &&
-          (low   <  test1) &&
-            low <= too_far_i
-      in
-      Logger.debug_f_ "%s <?= lowp:%s &&  low:%s <? %s && (low:%s <?= %s) yields:%b"
-                      (Sn.string_of test0 ) lowp_s
-                      low_s (Sn.string_of test1)
-                      low_s tfs
-                      test_result
-      >>= fun () ->
-      if test_result
-      then
-        begin
-          Logger.debug_f_ "fold_read over: %s (test0=%s)" fn
-                          (Sn.string_of test0) >>= fun () ->
-          let first = test0 in
-          Logger.info_f_ "Replaying tlog file: %s (%d/%d)" fn cnt num_tlogs  >>= fun () ->
-          let t1 = Unix.gettimeofday () in
-          fold_read t fn ~index low (Some too_far_i) ~first low acc_entry >>= fun x ->
-          Logger.info_f_ "Completed replay of %s, took %f seconds, %i to go" fn (Unix.gettimeofday () -. t1) (num_tlogs - cnt) >>= fun () ->
-          Lwt.return (cnt+1,x)
-        end
-      else Lwt.return (cnt+1,low)
-    in
-    Lwt_list.fold_left_s maybe_fold (1,start_i) tlog_names
-    >>= fun _x ->
-    Lwt.return ()
-
-  let inner_of_i t = function
-    | None -> 0
-    | Some i ->
-       let step = (Sn.of_int t.tlog_max_entries) in
-       (Sn.to_int (Sn.rem i step)) + 1
-
+      
   let outer_of_i t i =
     let rec find pn = function
       | [] -> pn
-      | (i0,n0) :: rest ->
+      | (i0,n0,_) :: rest ->
          if i0 < i
          then find n0 rest
          else n0
     in
     find 0 t.i_to_tlog_number
+      
+  
+
+  let inner_of_i t = function
+    | None -> 0
+    | Some i ->
+       let step = (Sn.of_int t.tlog_max_entries) in
+       (Sn.to_int (Sn.rem i step)) + 1
+
+  
 
   let get_start_i t n =
     let rec find = function
-      | [] -> failwith "todo"
-      | (i0,n0) :: rest ->
+      | [] -> Sn.zero (* TODO: maybe assert *)
+      | (i0,n0,_) :: rest ->
          if n0 = n
          then i0 else
            find rest
     in
-    find t.i_to_tlog_number
+    let r = find t.i_to_tlog_number in
+    Logger.ign_debug_f_ "get_start_i %s %i => %s"  (show_map t) n (Sn.string_of r) ;
+    r
+
+  
          
   let is_first_of_new t i = Sn.rem i (Sn.of_int t.tlog_max_entries) = Sn.zero
 
-  let should_roll t outer i =
-    let ( *: ) = Sn.mul in
-    let r = i >= (Sn.of_int t.tlog_max_entries) *: (Sn.of_int (outer + 1)) in
-    let r' = r || t.tlog_size >= t.tlog_max_size in
-    let () =
-      Logger.ign_debug_f_
-        "should_roll tlog_max_entries:%i tlog_max_size:%i outer:%i i:%Li tlog_size:%i => %b"
-        t.tlog_max_entries t.tlog_max_size outer i t.tlog_size r'
-    in
-    let () =
-      if r'
-      then
-        t.tlog_size <- 0
-    in
-    r'
-
-
+  let should_roll t = t.should_roll
+ 
   let new_outer t i =
     let r = t.tlog_number + 1 in
     let () = t.tlog_number <- r in
+    let () = t.tlog_size <- 0 in
+    let () = t.tlog_entries <- 0 in
+    let () = t.should_roll <- false in
     r
 
   let infimum t =
@@ -450,12 +456,83 @@ module TlogMap = struct
     Logger.ign_debug_f_ "is_rollover_point %s" (Sn.string_of i); 
     let rec find = function
       | [] -> false
-      | (i0,_ ) :: _ when i0 = i -> true
-      | (i0,_ ) :: _ when i0 > i -> false
+      | (i0,_,_ ) :: _ when i0 = i -> true
+      | (i0,_,_ ) :: _ when i0 > i -> false
+      | _ :: rest -> find rest
+    in
+    find t.i_to_tlog_number
+
+  let complete_file_to_deliver t i =
+    let rec find = function
+      | [] -> None
+      | (i0,n,_)  :: (i1,_,_) :: _ when i0 = i -> Some (n,i1)
+      | (i0,_,_)  :: _ when i0 > i -> None
       | _ :: rest -> find rest
     in
     find t.i_to_tlog_number
 
+  let next_rollover t i =
+    let rec find = function
+      | [] -> None
+      | (i0,_,_) :: (i1,_,_) :: _ when i0 <= i && i < i1 -> Some i1
+      | _ :: rest -> find rest
+    in
+    let r = find t.i_to_tlog_number in
+    let () =
+      Logger.ign_debug_f_
+        "next_rollover %s : %s => %s" (Sn.string_of i)
+        (show_map t) (To_string.option Sn.string_of r)
+    in
+    r
+
+  let iterate_tlog_dir t ~index start_i too_far_i f =
+    let open Tlogcommon in
+    let open Tlogreader2 in
+    let tfs = Sn.string_of too_far_i in
+    Logger.debug_f_ "TlogMap.iterate_tlog_dir start_i:%s too_far_i:%s ~index:%s"
+                    (Sn.string_of start_i) tfs (Index.to_string index)
+    >>= fun () ->
+    get_tlog_names t >>= fun tlog_names ->
+    let acc_entry (_i0:Sn.t) entry =
+      f entry >>= fun () -> let i = Entry.i_of entry in Lwt.return i
+    in
+    let num_tlogs = List.length tlog_names in
+    let maybe_fold (cnt,low) fn =
+      let fn_start = get_start_i t (get_number fn) in
+      let fn_next  = next_rollover t fn_start in
+      let low_n = Sn.succ low in
+      (* 000.tlx starting from (1, 14001) *)
+      let should_fold = 
+        match fn_next with
+        | None -> fn_start <= low_n
+        | Some next -> fn_start <= low_n && low_n < next
+      in
+      Logger.debug_f_ "maybe_fold %s low:%s" fn (Sn.string_of low) >>= fun () ->
+      Logger.debug_f_ "fn: fn_start:%s <= low_n:%s fn_next:%s => %b"
+                      (Sn.string_of fn_start) (Sn.string_of low_n)
+                      (To_string.option Sn.string_of fn_next)
+                      should_fold
+      >>= fun () ->
+      
+      let fold () =
+        Logger.debug_f_ "fold over: %s: [%s,...) "
+                           fn (Sn.string_of fn_start) 
+           >>= fun () ->
+           let first = fn_start in
+           Logger.info_f_ "Replaying tlog file: %s (%d/%d)" fn cnt num_tlogs  >>= fun () ->
+           let t1 = Unix.gettimeofday () in
+           fold_read t fn ~index low (Some too_far_i) ~first low acc_entry >>= fun low' ->
+           Logger.info_f_ "Completed replay of %s, took %f seconds, %i to go"
+                          fn (Unix.gettimeofday () -. t1) (num_tlogs - cnt)
+           >>= fun () ->
+           Lwt.return (cnt+1, low')
+      in
+      if should_fold then fold () else Lwt.return (cnt+1,low)
+    in
+    Lwt_list.fold_left_s maybe_fold (1,start_i) tlog_names
+    >>= fun _x ->
+    Lwt.return ()
+      
   let tlogs_to_collapse t head_i last_i tlogs_to_keep =
     let lag = Sn.to_int (Sn.sub last_i head_i) in
     let npt = t.tlog_max_entries in
diff --git a/src/tlog/tlogcollection.ml b/src/tlog/tlogcollection.ml
index 1476a1a2..cec4b210 100644
--- a/src/tlog/tlogcollection.ml
+++ b/src/tlog/tlogcollection.ml
@@ -25,8 +25,8 @@ class type tlog_collection =
   method log_value : Sn.t -> Value.t -> int Lwt.t
   method log_value_explicit : Sn.t -> Value.t ->
                               sync:bool -> string option -> int Lwt.t
-  method get_last_i: unit -> Sn.t
-  method is_rollover_point: Sn.t -> bool
+  
+  
   method tlogs_to_collapse: Sn.t -> Sn.t -> int -> int
   method get_last_value: Sn.t -> Value.t option (* Lwt.t *)
   method get_last: unit -> (Value.t * Sn.t) option
@@ -34,14 +34,26 @@ class type tlog_collection =
   method get_infimum_i : unit -> Sn.t Lwt.t
   method dump_head : Lwt_io.output_channel -> Sn.t Lwt.t
   method save_head : Lwt_io.input_channel -> unit Lwt.t
-  method dump_tlog_file : Sn.t -> Lwt_io.output_channel -> Sn.t Lwt.t
+
+  (**
+     returns file number & starting i of next file  
+   *)
+  method complete_file_to_deliver: Sn.t -> (int * Sn.t) option
+                                               
+  method dump_tlog_file : int -> Lwt_io.output_channel -> unit Lwt.t
   method save_tlog_file : string -> int64 -> Lwt_io.input_channel -> unit Lwt.t
+  method reinit : unit -> unit Lwt.t 
   method get_head_name : unit -> string
   method get_tlog_from_i : Sn.t -> int
   method get_tlog_count: unit -> int Lwt.t
   method remove_below : Sn.t -> unit Lwt.t
+                                     
   method get_start_i : int -> Sn.t
-  method which_tlog_file : Sn.t -> (string option) Lwt.t
+  method get_last_i: unit -> Sn.t
+  method is_rollover_point: Sn.t -> bool
+  method next_rollover: Sn.t -> Sn.t option
+  method which_tlog_file : int -> (string option) Lwt.t
+  
   end
 
 type tlc_factory =

From e57df1f4129d19cad0b03ec5bebacf3edb77fc5a Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Tue, 2 Feb 2016 08:59:38 +0100
Subject: [PATCH 081/373] autofix

---
 src/node/node_main.ml | 20 +++++++-------------
 src/tlog/tlc2.ml      |  2 ++
 src/tlog/tlog_main.ml |  2 ++
 src/tlog/tlog_map.ml  |  4 ++--
 4 files changed, 13 insertions(+), 15 deletions(-)

diff --git a/src/node/node_main.ml b/src/node/node_main.ml
index 017e22c7..bd7658c0 100644
--- a/src/node/node_main.ml
+++ b/src/node/node_main.ml
@@ -570,22 +570,16 @@ let _main_2 (type s)
              in
              let open Tlog_map in
              match exn with
-             | Tlog_map.TLCNotProperlyClosed _ ->
+             | Tlog_map.TLCNotProperlyClosed (fn,_) ->
                 begin
-                  Logger.warning_ ~exn "AUTOFIX: improperly closed tlog"
-                  >>= fun ()->
-                  TlogMap.make me.tlog_dir me.tlx_dir me.node_name >>= fun tlog_map ->
-                  TlogMap.get_last_tlog tlog_map
-                  >>= fun (_, fn) ->
+                  Logger.warning_ ~exn "AUTOFIX: improperly closed tlog" >>= fun ()->
                   clean_up_retry fn
                 end
              | Tlogcommon.TLogUnexpectedEndOfFile _ ->
                 begin
-                  Logger.warning_ ~exn "AUTOFIX: unexpected end of tlog"
-                  >>= fun () ->
-                  TlogMap.make me.tlog_dir me.tlx_dir me.node_name >>= fun tlog_map ->
-                  TlogMap.get_last_tlog tlog_map
-                  >>= fun (_, fn) ->
+                  Logger.warning_ ~exn "AUTOFIX: unexpected end of tlog" >>= fun () ->
+                  Tlog_map._get_tlog_names me.tlog_dir me.tlx_dir >>= fun tlog_names ->
+                  let fn = List.hd (List.rev tlog_names) |> Filename.concat me.tlog_dir in
                   Tlc2._truncate_tlog fn >>= fun ()->
                   clean_up_retry fn
                 end
@@ -952,9 +946,9 @@ let _main_2 (type s)
             Logger.fatal_f_ "[rc=%i] Store counter too low: %s" rc msg >>= fun () ->
             maybe_dump_crash_log () >>= fun () ->
             Lwt.return rc
-          | Tlog_map.TLCNotProperlyClosed msg ->
+          | Tlog_map.TLCNotProperlyClosed (fn, msg) ->
             let rc = 42 in
-            Logger.fatal_f_ "[rc=%i] tlog not properly closed %s" rc msg >>= fun () ->
+            Logger.fatal_f_ "[rc=%i] tlog %s not properly closed %s" rc fn msg >>= fun () ->
             Lwt.return rc
           | Node_cfg.InvalidTlogDir dir ->
             let rc = 43 in
diff --git a/src/tlog/tlc2.ml b/src/tlog/tlc2.ml
index 26983e3a..21409084 100644
--- a/src/tlog/tlc2.ml
+++ b/src/tlog/tlc2.ml
@@ -623,7 +623,9 @@ let make_tlc2 ~compressor
   end
   >>= fun () ->
   Lwt.return col
+             
 let _truncate_tlog filename =
+  Logger.debug_f_ "_truncate %s" filename >>= fun () ->
   let skipper () _entry = Lwt.return () in
   let folder,extension,index = folder_for filename None in
   if extension <> ".tlog"
diff --git a/src/tlog/tlog_main.ml b/src/tlog/tlog_main.ml
index 81e65fef..7379ac4e 100644
--- a/src/tlog/tlog_main.ml
+++ b/src/tlog/tlog_main.ml
@@ -92,6 +92,8 @@ let strip_tlog filename =
   Lwt_extra.run t
 
 let _mark_tlog file_name node_name =
+  let section = Logger.Section.main in
+  Logger.debug_f_ "_mark_tlog %s %s" file_name node_name >>= fun () ->
   _last_entry file_name >>= fun last ->
   match last with
   | None -> Lwt.return 0
diff --git a/src/tlog/tlog_map.ml b/src/tlog/tlog_map.ml
index 9b485686..76e6c888 100644
--- a/src/tlog/tlog_map.ml
+++ b/src/tlog/tlog_map.ml
@@ -160,7 +160,7 @@ let _make_close_marker node_id = "closed:" ^ node_id
 let _make_open_marker node_id =  "opened:" ^ node_id
 
 type validation_result = (Entry.t option * Index.index * int)
-exception TLCNotProperlyClosed of string
+exception TLCNotProperlyClosed of (string * string)
 
 let _validate_one tlog_name node_id ~check_marker : validation_result Lwt.t =
   Logger.debug_f_ "Tlog_map._validate_one %s" tlog_name >>= fun () ->
@@ -192,7 +192,7 @@ let _validate_one tlog_name node_id ~check_marker : validation_result Lwt.t =
                  let s = Some (_make_close_marker node_id) in
                  if Entry.check_marker e s
                  then !prev_entry
-                 else raise (TLCNotProperlyClosed (Entry.entry2s e))
+                 else raise (TLCNotProperlyClosed (tlog_name, Entry.entry2s e))
            in
            Lwt.return eo'
        end

From 11538e545d2407d502b33f54568dbdf66ab14393 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Tue, 2 Feb 2016 09:08:29 +0100
Subject: [PATCH 082/373] cleanup

---
 src/tlog/tlog_map.ml | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/src/tlog/tlog_map.ml b/src/tlog/tlog_map.ml
index 76e6c888..2bf4db09 100644
--- a/src/tlog/tlog_map.ml
+++ b/src/tlog/tlog_map.ml
@@ -319,12 +319,7 @@ module TlogMap = struct
     let full_path = _get_full_path t.tlog_dir t.tlx_dir (file_name new_c) in
     Lwt.return (new_c, full_path)
 
-
   
-  let get_tlog_i t n = Sn.mul n (Sn.of_int t.tlog_max_entries)
-
-
-
   let find_tlog_file t c =
     let open Compression in
     let f0 = archive_name Snappy c in
@@ -386,16 +381,6 @@ module TlogMap = struct
     in
     find 0 t.i_to_tlog_number
       
-  
-
-  let inner_of_i t = function
-    | None -> 0
-    | Some i ->
-       let step = (Sn.of_int t.tlog_max_entries) in
-       (Sn.to_int (Sn.rem i step)) + 1
-
-  
-
   let get_start_i t n =
     let rec find = function
       | [] -> Sn.zero (* TODO: maybe assert *)

From 251dc149d124f1e5fbd11201a4b4bf677588c144 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Tue, 2 Feb 2016 14:37:41 +0100
Subject: [PATCH 083/373] reverse map

---
 pylabs/test/server/system_tests_common.py | 10 +--
 src/node/catchup.ml                       | 32 ++++----
 src/tlog/tlc2.ml                          | 13 ++--
 src/tlog/tlog_map.ml                      | 95 +++++++++++++----------
 4 files changed, 86 insertions(+), 64 deletions(-)

diff --git a/pylabs/test/server/system_tests_common.py b/pylabs/test/server/system_tests_common.py
index 0d525043..d93801a0 100644
--- a/pylabs/test/server/system_tests_common.py
+++ b/pylabs/test/server/system_tests_common.py
@@ -754,7 +754,7 @@ def setup_2_nodes_forced_master (home_dir):
     setup_n_nodes( 2, True, home_dir, witness_nodes = True)
 
 def setup_2_nodes_forced_master_mini (home_dir):
-    extra = {'__tainted_tlog_entries_per_file':'1000'}
+    extra = {'tlog_max_entries':'1000'}
     setup_n_nodes( 2, True, home_dir, extra)
 
 def setup_2_nodes_forced_master_normal_slaves (home_dir):
@@ -767,15 +767,15 @@ def setup_3_nodes_witness_slave (home_dir):
     setup_n_nodes( 3, False, home_dir, witness_nodes = True)
 
 def setup_3_nodes_mini(home_dir):
-    extra = {'__tainted_tlog_entries_per_file':'1000'}
+    extra = {'tlog_max_entries':'1000'}
     setup_n_nodes( 3, False, home_dir, extra)
 
 def setup_2_nodes_mini(home_dir):
-    extra = {'__tainted_tlog_entries_per_file':'1000'}
+    extra = {'tlog_max_entries':'1000'}
     setup_n_nodes(2, False, home_dir, extra)
 
 def setup_3_nodes_mini_forced_master(home_dir):
-    extra = {'__tainted_tlog_entries_per_file':'1000'}
+    extra = {'tlog_max_entries':'1000'}
     setup_n_nodes( 3, True, home_dir, extra)
 
 def setup_3_nodes (home_dir) :
@@ -788,7 +788,7 @@ def setup_1_node (home_dir):
     setup_n_nodes( 1, False, home_dir )
 
 def setup_1_node_mini (home_dir):
-    extra = {'__tainted_tlog_entries_per_file':'1000'}
+    extra = {'tlog_max_entries':'1000'}
     setup_n_nodes(1, False, home_dir, extra)
 
 default_setup = setup_3_nodes
diff --git a/src/node/catchup.ml b/src/node/catchup.ml
index 6c04d9b3..2ceb8514 100644
--- a/src/node/catchup.ml
+++ b/src/node/catchup.ml
@@ -307,11 +307,13 @@ let last_entries2
         in
         let too_far_i = Sn.succ ci in
         let rec loop_files (start_i2:Sn.t) =
+          Logger.debug_f_ "loop_files start_i2:%s" (start_i2 |> Sn.string_of) >>= fun () ->
           match tlog_collection # complete_file_to_deliver start_i2 with
           | None -> Lwt.return start_i2
           | Some (tlog_number, start_i2') ->
              begin
-              Logger.debug_f_ "start_i2=%Li < %Li (outputting %03i)" start_i2 too_far_i tlog_number
+               Logger.debug_f_ "start_i2=%Li < %Li (outputting %03i) start_i2'=%Li"
+                               start_i2 too_far_i tlog_number start_i2'
               >>= fun () ->
               Llio.output_int oc 3 >>= fun () ->
               tlog_collection # dump_tlog_file tlog_number oc >>= fun () ->
@@ -340,18 +342,22 @@ let last_entries2
 
           (* maybe stream a bit *)
           begin
-            match tlog_collection # next_rollover start_i2 with
-            | None -> Lwt.return start_i2
-            | Some next_rollover ->
-               Logger.debug_f_ "next_rollover = %Li" next_rollover >>= fun () ->
-               (if next_rollover < too_far_i
-                then
-                  begin
-                    stream_entries start_i2 next_rollover >>= fun () ->
-                    Lwt.return next_rollover
-                  end
-                else Lwt.return start_i2
-               )
+            if tlog_collection # is_rollover_point start_i2
+            then
+              Lwt.return start_i2
+            else
+              match tlog_collection # next_rollover start_i2 with
+              | None -> Lwt.return start_i2
+              | Some next_rollover ->
+                 Logger.debug_f_ "next_rollover = %Li" next_rollover >>= fun () ->
+                 (if next_rollover < too_far_i
+                  then
+                    begin
+                      stream_entries start_i2 next_rollover >>= fun () ->
+                      Lwt.return next_rollover
+                    end
+                  else Lwt.return start_i2
+                 )
           end
           >>= fun start_i3 ->
           (* push out files *)
diff --git a/src/tlog/tlc2.ml b/src/tlog/tlc2.ml
index 21409084..7d81bb76 100644
--- a/src/tlog/tlc2.ml
+++ b/src/tlog/tlc2.ml
@@ -140,7 +140,7 @@ let _init_file fsync_dir tlog_map =
   Lwt_unix.LargeFile.fstat fd >>= fun stats ->
   let pos0 = stats.st_size in
   let oc = Lwt_io.of_fd ~mode:Lwt_io.output fd in
-  Logger.debug_f_ "_init_file pos0 : %Li" pos0 >>= fun () ->
+  Logger.debug_f_ "_init_file %s pos0:%Li" fn pos0 >>= fun () ->
   let f = F.make oc fd fn c pos0 in
   Lwt.return f
 
@@ -284,13 +284,14 @@ class tlc2
         | None ->
           begin
             let outer = TlogMap.outer_of_i tlog_map i in
-            Logger.debug_f_ "prelude i:%s: outer=%i" (Sn.string_of i) outer>>= fun () ->
+            Logger.debug_f_ "prelude i:%s: outer=%i No File" (Sn.string_of i) outer>>= fun () ->
             begin
-              if (outer <> 0) && (TlogMap.is_first_of_new tlog_map i )
+              if outer <> 0 && (TlogMap.is_rollover_point tlog_map i)
               then
                 (* the first thing logged falls exactly into a new tlog,
-                   the compression job for the previous one thus is not yet picked up *)
-                self # _add_compression_job (pred outer)
+                   the compression job for the previous one(s) thus is not yet picked up *)
+                let need_compression = TlogMap.old_uncompressed tlog_map in
+                Lwt_list.iter_s (fun (_,n,_) -> self # _add_compression_job n) need_compression
               else
                 Lwt.return ()
             end >>= fun () ->
@@ -311,7 +312,7 @@ class tlc2
             let do_jump () =
               let old_outer = TlogMap.get_tlog_number tlog_map  in
               let new_outer = TlogMap.new_outer tlog_map i in
-              Logger.info_f_ "i= %s old_outer = %i => jump to %i"
+              Logger.info_f_ "should roll: i= %s old_outer = %i => jump to %i"
                              (Sn.string_of i)
                              old_outer 
                              new_outer
diff --git a/src/tlog/tlog_map.ml b/src/tlog/tlog_map.ml
index 2bf4db09..b2bcb2dc 100644
--- a/src/tlog/tlog_map.ml
+++ b/src/tlog/tlog_map.ml
@@ -221,24 +221,21 @@ module TlogMap = struct
       mutable tlog_size: int;
       mutable tlog_entries : int;
       mutable tlog_number: tlog_number;
-      mutable i_to_tlog_number : (Sn.t * tlog_number * is_archive) list;
+      
+      mutable i_to_tlog_number : (Sn.t * tlog_number * is_archive) list; (* sorted: most recent first *)
       mutable should_roll: bool;
     }
              
   let show_map t =
-    "[" ^ 
-    String.concat ";"
-                  (List.map
-                     (fun (i,n,a) ->
+    To_string.list (fun (i,n,a) ->
                        Printf.sprintf "(%s,%03i,%b)" (Sn.string_of i) n a
                      ) t.i_to_tlog_number
-                  ) ^ "]"
-                  
+                      
   let new_entry t s =
     t.tlog_size <- t.tlog_size + s;
     t.tlog_entries <- t.tlog_entries + 1;
     if t.tlog_size >= t.tlog_max_size
-       || t.tlog_entries + 1 = t.tlog_max_entries
+       || t.tlog_entries = t.tlog_max_entries
     then
       let () = t.should_roll <- true in
       Logger.ign_debug_f_ "should roll over"
@@ -251,7 +248,7 @@ module TlogMap = struct
   let get_full_path t x = _get_full_path t.tlog_dir t.tlx_dir x
 
 
-  let _init tlog_dir tlx_dir node_id ~check_marker =
+  let _init tlog_dir tlx_dir node_id tlog_max_entries tlog_max_size ~check_marker =
     Logger.debug_f_ "_init ~check_marker:%b" check_marker >>= fun () ->
     _get_tlog_names tlog_dir tlx_dir >>= fun tlog_names ->
     let tlog_number = get_count tlog_names in
@@ -259,7 +256,7 @@ module TlogMap = struct
     let full_path = _get_full_path tlog_dir tlx_dir (file_name tlog_number) in
     _validate_one full_path node_id ~check_marker
     >>= fun (last, index, tlog_size) ->
-    Lwt_list.map_s
+    Lwt_list.rev_map_s
       (fun tlog_name ->
         let n = get_number tlog_name in
         let canonical = _get_full_path tlog_dir tlx_dir tlog_name in
@@ -268,18 +265,22 @@ module TlogMap = struct
        Lwt.return (i,n, b)
       ) tlog_names
     >>= fun i_to_tlog_number ->
-    let should_roll,last_start =
-      let rec find = function
-          | [] -> false , Sn.zero
-          | [(i0,_,true)] -> true, i0
-          | _ :: rest -> find rest
-      in find i_to_tlog_number
-    in
-    let tlog_entries =
+    let last_i =
       match last with
-      | None -> 0
-      | Some e -> let i = Entry.i_of e in (Sn.to_int i) - (Sn.to_int last_start)
+      | None -> Sn.zero
+      | Some e -> Entry.i_of e 
     in
+    let tlog_entries, should_roll,last_start =
+      match i_to_tlog_number with
+      | []                -> 0, false, Sn.zero
+      | (i0,_,true)  :: _ -> 0, true , i0
+      | (i0,_,false) :: _ ->
+         let tlog_entries = Sn.sub last_i i0 |> Sn.to_int in
+         let should_roll = tlog_entries + 1 >= tlog_max_entries in
+         tlog_entries, should_roll, i0
+    in 
+    Logger.debug_f_ "_init: tlog_entries:%i should_roll:%b" tlog_entries should_roll
+    >>= fun () ->
     Lwt.return (tlog_entries, tlog_size, tlog_number, i_to_tlog_number, should_roll)
       
   let make ?tlog_max_entries
@@ -293,7 +294,9 @@ module TlogMap = struct
     in
     let tlog_max_entries = with_default tlog_max_entries 100_000 in
     let tlog_max_size    = with_default tlog_max_size 32_000_000 in
-    _init tlog_dir tlx_dir node_id ~check_marker:true
+    _init
+      tlog_dir tlx_dir node_id ~check_marker:true
+      tlog_max_entries tlog_max_size
     >>= fun (tlog_entries, tlog_size,tlog_number,i_to_tlog_number, should_roll) ->
     
     let t = {node_id; tlog_dir;tlx_dir;tlog_max_entries; tlog_max_size;
@@ -303,7 +306,9 @@ module TlogMap = struct
     Lwt.return t
 
   let reinit t =
-    _init t.tlog_dir t.tlx_dir t.node_id ~check_marker:false 
+    _init
+      t.tlog_dir t.tlx_dir t.node_id ~check_marker:false
+      t.tlog_max_entries t.tlog_max_size
     >>= fun (tlog_entries, tlog_size, tlog_number,i_to_tlog_number,should_roll) ->
     t.tlog_size   <- tlog_size;
     t.tlog_number <- tlog_number;
@@ -372,14 +377,14 @@ module TlogMap = struct
       )
       
   let outer_of_i t i =
-    let rec find pn = function
-      | [] -> pn
+    let rec find = function
+      | [] -> 0
       | (i0,n0,_) :: rest ->
-         if i0 < i
-         then find n0 rest
-         else n0
+         if  i >= i0
+         then n0
+         else find rest
     in
-    find 0 t.i_to_tlog_number
+    find t.i_to_tlog_number
       
   let get_start_i t n =
     let rec find = function
@@ -392,10 +397,8 @@ module TlogMap = struct
     let r = find t.i_to_tlog_number in
     Logger.ign_debug_f_ "get_start_i %s %i => %s"  (show_map t) n (Sn.string_of r) ;
     r
-
-  
          
-  let is_first_of_new t i = Sn.rem i (Sn.of_int t.tlog_max_entries) = Sn.zero
+  
 
   let should_roll t = t.should_roll
  
@@ -405,6 +408,7 @@ module TlogMap = struct
     let () = t.tlog_size <- 0 in
     let () = t.tlog_entries <- 0 in
     let () = t.should_roll <- false in
+    let () = t.i_to_tlog_number <- (i,r,false) :: t.i_to_tlog_number in
     r
 
   let infimum t =
@@ -438,28 +442,37 @@ module TlogMap = struct
   let is_rollover_point t i =
     (* TODO: this gets called a zillion times in catchup...
      *)
-    Logger.ign_debug_f_ "is_rollover_point %s" (Sn.string_of i); 
+    
     let rec find = function
       | [] -> false
       | (i0,_,_ ) :: _ when i0 = i -> true
-      | (i0,_,_ ) :: _ when i0 > i -> false
+      | (i0,_,_ ) :: _ when i0 < i -> false
       | _ :: rest -> find rest
     in
-    find t.i_to_tlog_number
+    let r = find t.i_to_tlog_number in
+    Logger.ign_debug_f_ "is_rollover_point %s => %b" (Sn.string_of i) r;
+    r
+    
 
   let complete_file_to_deliver t i =
+    
     let rec find = function
       | [] -> None
-      | (i0,n,_)  :: (i1,_,_) :: _ when i0 = i -> Some (n,i1)
-      | (i0,_,_)  :: _ when i0 > i -> None
+      | (i1,_,_)  :: (i0,n,_) :: _ when i0 = i -> Some (n,i1)
+      | (i0,_,_)  :: _ when i0 < i -> None
       | _ :: rest -> find rest
     in
-    find t.i_to_tlog_number
+    let r = find t.i_to_tlog_number in
+    Logger.ign_debug_f_
+      "complete_file_to_deliver %s : %s => %s"
+      (Sn.string_of i) (show_map t)
+      (To_string.option (fun (n,i) -> Printf.sprintf ("(%i,%Li)") n i) r);
+    r
 
   let next_rollover t i =
     let rec find = function
       | [] -> None
-      | (i0,_,_) :: (i1,_,_) :: _ when i0 <= i && i < i1 -> Some i1
+      | (i0,_,_) :: (i1,_,_) :: _ when i1 <=i && i < i0 -> Some i0
       | _ :: rest -> find rest
     in
     let r = find t.i_to_tlog_number in
@@ -486,7 +499,6 @@ module TlogMap = struct
       let fn_start = get_start_i t (get_number fn) in
       let fn_next  = next_rollover t fn_start in
       let low_n = Sn.succ low in
-      (* 000.tlx starting from (1, 14001) *)
       let should_fold = 
         match fn_next with
         | None -> fn_start <= low_n
@@ -524,5 +536,8 @@ module TlogMap = struct
     let tlog_lag = (lag +npt - 1)/ npt in
     let tlogs_to_collapse = tlog_lag - tlogs_to_keep - 1 in
     tlogs_to_collapse
-
+    
+  let old_uncompressed t =
+    List.filter (fun (_,_,is_archive) -> not is_archive) t.i_to_tlog_number
+                
 end

From 278b5fffe3691e9d79e18ccb4bc07b0986babb02 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Tue, 2 Feb 2016 19:56:27 +0100
Subject: [PATCH 084/373] test_sabotage

---
 .../server/right/system_tests_long_right.py   | 26 +++++++--
 src/main/replay_main.ml                       |  9 ++-
 src/node/catchup_test.ml                      |  2 +-
 src/node/collapser.ml                         | 26 +++------
 src/node/collapser_test.ml                    |  4 +-
 src/node/dump_store.ml                        |  2 +-
 src/tlog/mem_tlogcollection.ml                |  4 +-
 src/tlog/tlc2.ml                              | 36 ++++--------
 src/tlog/tlc2_test.ml                         |  6 +-
 src/tlog/tlog_map.ml                          | 58 +++++++++++++++----
 src/tlog/tlogcollection.ml                    |  7 ++-
 src/tlog/tlogcollection_test.ml               | 14 ++---
 12 files changed, 118 insertions(+), 76 deletions(-)

diff --git a/pylabs/test/server/right/system_tests_long_right.py b/pylabs/test/server/right/system_tests_long_right.py
index 613613a8..4e423382 100644
--- a/pylabs/test/server/right/system_tests_long_right.py
+++ b/pylabs/test/server/right/system_tests_long_right.py
@@ -21,6 +21,7 @@
 import arakoon
 import logging
 import time
+import threading
 import subprocess
 from nose.tools import *
 import os
@@ -490,7 +491,9 @@ def test_sabotage():
     test_size = num_tlogs * tlog_size
     Common.iterate_n_times(test_size, Common.simple_set)
     time.sleep(10)
+    print "stopping"
     clu.stop()
+    
     node_id = Common.node_names[0]
     node_cfg = clu.getNodeConfig(node_id)
     node_home_dir = node_cfg ['home']
@@ -502,12 +505,27 @@ def test_sabotage():
                   #"%s.db.wal" % (node_id,), # should not exist after a `proper` close
                   ])
     for f in files:
-        print f
+        print "removeing", f
         X.removeFile(f)
-
+    print "starting"
     cmd = clu._cmd('sturdy_0')
-    returncode = X.subprocess.call(cmd)
-    assert_equals(returncode, 50)
+    print cmd
+    
+    def start_node():
+        returncode = X.subprocess.call(cmd)
+        assert_equals(returncode, 50)
+        print "startup failure + correct returncode"
+    try:
+        
+        t = threading.Thread(target = start_node)
+        t.start()
+        t.join(10.0)
+        
+    except Exception,e:
+        print e
+        assert_equals(False,True)
+        
+    print "done"
 
 @Common.with_custom_setup( Common.setup_3_nodes_forced_master, Common.basic_teardown )
 def test_large_catchup_while_running():
diff --git a/src/main/replay_main.ml b/src/main/replay_main.ml
index e522183b..cfcff708 100644
--- a/src/main/replay_main.ml
+++ b/src/main/replay_main.ml
@@ -40,7 +40,7 @@ let replay_tlogs tlog_dir tlx_dir db_name end_i =
     let open Tlog_map in
     Lwt.catch
       (fun () ->
-         TlogMap.make tlog_dir tlx_dir "node_name???" >>= fun tlog_map ->
+         TlogMap.make tlog_dir tlx_dir "node_name???" ~check_marker:false >>= fun tlog_map ->
          S.make_store ~lcnum:1024 ~ncnum:512 db_name >>= fun store ->
          let cio = S.consensus_i store in
          let cios = Log_extra.option2s Sn.string_of cio in
@@ -50,9 +50,12 @@ let replay_tlogs tlog_dir tlx_dir db_name end_i =
              end_i with
              | None ->
                 begin
-
                  TlogMap.get_last_tlog tlog_map >>= fun (_new_c,fn) ->
-                 Tlog_map._validate_one fn "" ~check_marker:false >>= fun (last, _index, _pos) ->
+                 Tlog_map._validate_one
+                   fn ""
+                   ~check_marker:false
+                   ~check_sabotage:false
+                 >>= fun (last, _index, _pos) ->
                  let i =
                    match last with
                      | None -> Sn.start
diff --git a/src/node/catchup_test.ml b/src/node/catchup_test.ml
index c8c18055..bcd3e164 100644
--- a/src/node/catchup_test.ml
+++ b/src/node/catchup_test.ml
@@ -139,7 +139,7 @@ let teardown () =
 let _tic filler_function n name verify_store =
   let open Tlog_map in
   let node_id = "node_name" in
-  TlogMap.make ~tlog_max_entries:101 _dir_name _tlx_dir node_id >>= fun tlog_map ->
+  TlogMap.make ~tlog_max_entries:101 _dir_name _tlx_dir node_id ~check_marker:true >>= fun tlog_map ->
   Tlc2.make_tlc2 ~compressor:Compression.Snappy  _dir_name _tlx_dir _tlx_dir
                  ~fsync:false node_id ~fsync_tlog_dir:true >>= fun tlog_coll ->
   filler_function tlog_coll n >>= fun () ->
diff --git a/src/node/collapser.ml b/src/node/collapser.ml
index 6b9b50e1..6bdb226d 100644
--- a/src/node/collapser.ml
+++ b/src/node/collapser.ml
@@ -222,26 +222,18 @@ let collapse_many
     Lwt.return head_i
   in
   get_head_i () >>= fun head_i ->
-  let tlogs_to_collapse = tlog_coll # tlogs_to_collapse head_i last_i tlogs_to_keep in
-  Logger.debug_f_ "tlogs_to_collapse = %i" tlogs_to_collapse >>= fun () ->
-  if tlogs_to_collapse <= 0
-  then
-    Logger.info_f_ "Nothing to collapse..." >>= fun () ->
-    cb' 0
-  else
+  let todo = tlog_coll # tlogs_to_collapse ~head_i ~last_i ~tlogs_to_keep in
+  match todo with
+  | None -> Logger.info_f_ "Nothing to collapse..." >>= fun () -> cb' 0
+  | Some (n, too_far_i) ->
     begin
-      Logger.info_f_ "Going to collapse %d tlogs" tlogs_to_collapse >>= fun () ->
-      cb' (tlogs_to_collapse+1) >>= fun () ->
-      let g_too_far_i =
-        failwith "Sn.add (Sn.of_int 2) (Sn.add head_i (Sn.of_int (tlogs_to_collapse * npt)))"
-      in
-      (* +2 because before X goes to the store, you need to have seen X+1 and thus too_far = X+2 *)
-      Logger.debug_f_ "g_too_far_i = %s" (Sn.string_of g_too_far_i) >>= fun () ->
-      collapse_until tlog_coll (module S) store_fs g_too_far_i cb slowdown >>= fun () ->
+      Logger.info_f_ "Going to collapse %d tlogs" n >>= fun () ->
+      cb' (n + 1) >>= fun () ->
+      Logger.debug_f_ "too_far_i = %s" (Sn.string_of too_far_i) >>= fun () ->
+      collapse_until tlog_coll (module S) store_fs too_far_i cb slowdown >>= fun () ->
       get_head_i () >>= fun head_i ->
       tlog_coll # remove_below head_i >>= fun () ->
-      let x = failwith "(Sn.div g_too_far_i (Sn.of_int !Tlogcommon.tlogEntriesPerFile))" in
-      cb x
+      cb n
     end
 
 let collapse_out_of_band cfg name tlogs_to_keep =
diff --git a/src/node/collapser_test.ml b/src/node/collapser_test.ml
index 9b93dd26..0706c6a3 100644
--- a/src/node/collapser_test.ml
+++ b/src/node/collapser_test.ml
@@ -57,7 +57,7 @@ let _make_values tlc n =
 
 let test_collapse_until (dn, tlx_dir, head_dir) =
   let node_id = "node_name" in
-  TlogMap.make ~tlog_max_entries:1000 dn tlx_dir node_id >>= fun tlog_map ->
+  TlogMap.make ~tlog_max_entries:1000 dn tlx_dir node_id ~check_marker:true >>= fun tlog_map ->
   Logger.debug_f_ "dn=%s, tlf_dir=%s, head_dir=%s" dn tlx_dir head_dir >>= fun () ->
   Tlc2.make_tlc2 ~compressor dn tlx_dir head_dir
                  ~fsync:false node_id ~fsync_tlog_dir:false
@@ -92,7 +92,7 @@ let _head_dir = "/tmp/collapser_head"
 
 let test_collapse_many (dn, tlx_dir, head_dir) =
   let node_id = "node_name" in
-  TlogMap.make ~tlog_max_entries:100 dn tlx_dir node_id >>= fun tlog_map ->
+  TlogMap.make ~tlog_max_entries:100 dn tlx_dir node_id ~check_marker:true >>= fun tlog_map ->
 
   Logger.debug_f_ "test_collapse_many_regime dn=%s, tlx_dir=%s, head_dir=%s" dn tlx_dir head_dir
   >>= fun () ->
diff --git a/src/node/dump_store.ml b/src/node/dump_store.ml
index e111fa57..d7f66787 100644
--- a/src/node/dump_store.ml
+++ b/src/node/dump_store.ml
@@ -77,7 +77,7 @@ let inject_as_head fn node_id cfg_url ~force ~in_place =
       | [] -> failwith (Printf.sprintf "unknown node: %S" node_id)
       | x :: _ -> x
     in
-    TlogMap.make node_cfg.tlog_dir node_cfg.tlx_dir node_id >>= fun tlog_map ->
+    TlogMap.make node_cfg.tlog_dir node_cfg.tlx_dir node_id ~check_marker:false >>= fun tlog_map ->
     let head_dir = node_cfg.head_dir in
     let old_head_name = Filename.concat head_dir Tlc2.head_fname  in
 
diff --git a/src/tlog/mem_tlogcollection.ml b/src/tlog/mem_tlogcollection.ml
index e9d0df1e..e570e61e 100644
--- a/src/tlog/mem_tlogcollection.ml
+++ b/src/tlog/mem_tlogcollection.ml
@@ -71,8 +71,8 @@ class mem_tlog_collection _name =
     method reinit () = Lwt.return ()
     method which_tlog_file _start_i = failwith "which_tlog_file not supported"
 
-    method tlogs_to_collapse _ _ _ = failwith "tlogs_to_collapse not supported"
-
+    method tlogs_to_collapse ~head_i ~last_i ~tlogs_to_keep =
+      failwith "tlogs_to_collapse not supported"
 
     method log_value i v = self #log_value_explicit i v ~sync:false None
 
diff --git a/src/tlog/tlc2.ml b/src/tlog/tlc2.ml
index 7d81bb76..baedbecc 100644
--- a/src/tlog/tlc2.ml
+++ b/src/tlog/tlc2.ml
@@ -71,7 +71,8 @@ let folder_for filename index =
 
 let _validate_list tlog_names node_id ~check_marker=
   Lwt_list.fold_left_s
-    (fun _ tn -> _validate_one tn node_id ~check_marker)
+    (fun _ tlog_name ->
+      _validate_one tlog_name node_id ~check_marker ~check_sabotage:true)
     (None,None,0)
     tlog_names
   >>= fun (eo,index, pos) ->
@@ -147,7 +148,7 @@ let _init_file fsync_dir tlog_map =
 
 class tlc2
         ?(compressor=Compression.Snappy)
-        (tlog_dir:string) (tlf_dir:string) (head_dir:string) (new_c:int)
+        (head_dir:string) 
         (last:Entry.t option) (index:Index.index)
         (tlog_map: TlogMap.t)
         (node_id:string) ~(fsync:bool) ~(fsync_tlog_dir:bool)
@@ -223,7 +224,7 @@ class tlc2
       in
 
       let add_previous_compression_jobs () =
-        File_system.lwt_directory_list tlog_dir >>= fun entries ->
+        File_system.lwt_directory_list tlog_map.TlogMap.tlog_dir >>= fun entries ->
         let filtered = List.filter
                          (fun e -> Str.string_match tlog_regexp e 0)
                          entries
@@ -431,7 +432,7 @@ class tlc2
         | None -> None
         | Some pe ->  Some (Entry.v_of pe, Entry.i_of pe)
 
-    method tlogs_to_collapse head_i last_i tlogs_to_keep =
+    method tlogs_to_collapse ~head_i ~last_i ~tlogs_to_keep =
       TlogMap.tlogs_to_collapse tlog_map head_i last_i tlogs_to_keep
 
     method close ?(wait_for_compression=false) () =
@@ -569,39 +570,26 @@ class tlc2
 
 
 
-let maybe_correct new_c last index =
-  if new_c > 0 && last = None
-  then
-    begin
-      (* somebody sabotaged us:
-         (s)he deleted the .tlog file but we have .tlf's
-         meaning rotation happened correctly.
-         This means the marker can not be present.
-         Let the node die; they should fix this manually.
-      *)
-      Lwt.fail TLogSabotage
-    end
-  else
-    Lwt.return (new_c, last, index)
-
 let make_tlc2 ~compressor
               ?tlog_max_entries ?tlog_max_size
               tlog_dir tlf_dir
               head_dir ~fsync node_id ~fsync_tlog_dir
   =
   Logger.debug_f_ "make_tlc2 %S" tlog_dir >>= fun () ->
-  TlogMap.make ?tlog_max_entries ?tlog_max_size tlog_dir tlf_dir node_id >>= fun tlog_map ->
+  TlogMap.make ?tlog_max_entries ?tlog_max_size
+               tlog_dir tlf_dir node_id ~check_marker:true
+  >>= fun tlog_map ->
   TlogMap.get_last_tlog tlog_map >>= fun (new_c, fn) ->
-  _validate_one fn node_id ~check_marker:true >>= fun (last, index, pos) ->
-  maybe_correct new_c last index >>= fun (new_c,last,new_index) ->
-  Logger.debug_f_ "make_tlc2 after maybe_correct %s" (Index.to_string new_index) >>= fun () ->
+  (* TODO: move index to tlogmap *)
+  _validate_one fn node_id ~check_marker:true ~check_sabotage:true >>= fun (last, index, pos) ->
+  Logger.debug_f_ "make_tlc2 index = %s" (Index.to_string index) >>= fun () ->
   let msg =
     match last with
       | None -> "None"
       | Some e -> let i = Entry.i_of e in "Some" ^ (Sn.string_of i)
   in
   Logger.debug_f_ "post_validation: last_i=%s" msg >>= fun () ->
-  let col = new tlc2 tlog_dir tlf_dir head_dir new_c last new_index tlog_map
+  let col = new tlc2 head_dir last index tlog_map
                 ~compressor node_id ~fsync ~fsync_tlog_dir in
   (* rewrite last entry with ANOTHER marker so we can see we got here *)
   begin
diff --git a/src/tlog/tlc2_test.ml b/src/tlog/tlc2_test.ml
index 555b8dac..c54e47ff 100644
--- a/src/tlog/tlc2_test.ml
+++ b/src/tlog/tlc2_test.ml
@@ -110,7 +110,7 @@ let test_iterate4 (dn, tlx_dir, (factory:Tlogcollection_test.factory)) =
 
 
 let test_iterate5 (dn, _tlx_dir, (factory:factory)) =
-  TlogMap.make ~tlog_max_entries:10 dn _tlx_dir node_id >>= fun tlog_map ->
+  TlogMap.make ~tlog_max_entries:10 dn _tlx_dir node_id ~check_marker:true >>= fun tlog_map ->
   factory dn node_id >>= fun (tlc:tlog_collection) ->
   let rec loop (tlc:tlog_collection) i =
     if i = 33
@@ -147,7 +147,7 @@ let test_iterate5 (dn, _tlx_dir, (factory:factory)) =
   Lwt.return ()
 
 let test_iterate6 (dn, _tlx_dir, (factory:factory)) =
-  TlogMap.make ~tlog_max_entries:10 dn _tlx_dir node_id >>= fun tlog_map ->
+  TlogMap.make ~tlog_max_entries:10 dn _tlx_dir node_id ~check_marker:true >>= fun tlog_map ->
   let sync = false in
   factory dn node_id >>= fun (tlc:tlog_collection) ->
   let rec loop i =
@@ -302,7 +302,7 @@ let test_size_based_roll_over1 (dn, tlx_dir, (factory:factory)) =
   File_system.lwt_directory_list tlx_dir >>= fun tlx_entries ->
   OUnit.assert_equal 3 (List.length tlx_entries) ~printer;
 
-  TlogMap.make dn tlx_dir node_id >>= fun second_map ->
+  TlogMap.make dn tlx_dir node_id ~check_marker:true >>= fun second_map ->
   OUnit.assert_equal 3 (TlogMap.get_tlog_number second_map) ~printer;
   Lwt.return_unit
 
diff --git a/src/tlog/tlog_map.ml b/src/tlog/tlog_map.ml
index b2bcb2dc..b3824474 100644
--- a/src/tlog/tlog_map.ml
+++ b/src/tlog/tlog_map.ml
@@ -162,7 +162,12 @@ let _make_open_marker node_id =  "opened:" ^ node_id
 type validation_result = (Entry.t option * Index.index * int)
 exception TLCNotProperlyClosed of (string * string)
 
-let _validate_one tlog_name node_id ~check_marker : validation_result Lwt.t =
+let _validate_one
+      tlog_name node_id
+      ~(check_marker:bool)
+      ~(check_sabotage:bool)
+    : validation_result Lwt.t
+  =
   Logger.debug_f_ "Tlog_map._validate_one %s" tlog_name >>= fun () ->
   let e2s e = let i = Entry.i_of e in Printf.sprintf "(%s,_)" (Sn.string_of i) in
   let prev_entry = ref None in
@@ -205,7 +210,20 @@ let _validate_one tlog_name node_id ~check_marker : validation_result Lwt.t =
        Lwt.return (eo', new_index, size)
     )
     (function
-      | Unix.Unix_error(Unix.ENOENT,_,_) -> Lwt.return (None, new_index, 0)
+     | Unix.Unix_error(Unix.ENOENT,_,_) ->
+        let ext = extension_of tlog_name in
+        let base = Filename.basename tlog_name in
+        Logger.debug_f_ "not found: %s. %s was removed?!" tlog_name base >>= fun () ->
+        if (not check_sabotage) || (ext = ".tlog" && get_number base = 0)
+        then Lwt.return (None, new_index, 0)
+        else (* 
+         somebody sabotaged us:
+         (s)he deleted the .tlog file but we have .tlf's
+         meaning rotation happened correctly.
+         This means the marker can not be present.
+         Let the node die; they should fix this manually.
+         *)
+          Lwt.fail TLogSabotage
       | exn -> Lwt.fail exn
     )
 
@@ -248,13 +266,13 @@ module TlogMap = struct
   let get_full_path t x = _get_full_path t.tlog_dir t.tlx_dir x
 
 
-  let _init tlog_dir tlx_dir node_id tlog_max_entries tlog_max_size ~check_marker =
+  let _init tlog_dir tlx_dir node_id tlog_max_entries tlog_max_size ~check_marker ~check_sabotage =
     Logger.debug_f_ "_init ~check_marker:%b" check_marker >>= fun () ->
     _get_tlog_names tlog_dir tlx_dir >>= fun tlog_names ->
     let tlog_number = get_count tlog_names in
     Logger.debug_f_ "tlog_number:%i" tlog_number >>= fun () ->
     let full_path = _get_full_path tlog_dir tlx_dir (file_name tlog_number) in
-    _validate_one full_path node_id ~check_marker
+    _validate_one full_path node_id ~check_marker ~check_sabotage
     >>= fun (last, index, tlog_size) ->
     Lwt_list.rev_map_s
       (fun tlog_name ->
@@ -286,6 +304,7 @@ module TlogMap = struct
   let make ?tlog_max_entries
            ?tlog_max_size
            tlog_dir tlx_dir node_id
+           ~check_marker
     =
     let with_default o d =
       match o with
@@ -295,7 +314,7 @@ module TlogMap = struct
     let tlog_max_entries = with_default tlog_max_entries 100_000 in
     let tlog_max_size    = with_default tlog_max_size 32_000_000 in
     _init
-      tlog_dir tlx_dir node_id ~check_marker:true
+      tlog_dir tlx_dir node_id ~check_marker ~check_sabotage:false
       tlog_max_entries tlog_max_size
     >>= fun (tlog_entries, tlog_size,tlog_number,i_to_tlog_number, should_roll) ->
     
@@ -307,7 +326,7 @@ module TlogMap = struct
 
   let reinit t =
     _init
-      t.tlog_dir t.tlx_dir t.node_id ~check_marker:false
+      t.tlog_dir t.tlx_dir t.node_id ~check_marker:false ~check_sabotage:false
       t.tlog_max_entries t.tlog_max_size
     >>= fun (tlog_entries, tlog_size, tlog_number,i_to_tlog_number,should_roll) ->
     t.tlog_size   <- tlog_size;
@@ -531,11 +550,28 @@ module TlogMap = struct
     Lwt.return ()
       
   let tlogs_to_collapse t head_i last_i tlogs_to_keep =
-    let lag = Sn.to_int (Sn.sub last_i head_i) in
-    let npt = t.tlog_max_entries in
-    let tlog_lag = (lag +npt - 1)/ npt in
-    let tlogs_to_collapse = tlog_lag - tlogs_to_keep - 1 in
-    tlogs_to_collapse
+    
+    let collapsable = List.filter (fun (i,n,archive) -> i >= head_i) t.i_to_tlog_number in
+    let rec drop n = function
+      | [] -> []
+      | x :: rest ->
+         if n = 0
+         then rest
+         else drop (n-1) rest
+    in
+    let to_collapse_rev = drop tlogs_to_keep collapsable in
+    let r = match to_collapse_rev with
+    | [] -> None
+    | (i, n, archive) :: rest -> Some (List.length rest, i)
+    in
+    let () = Logger.ign_debug_f_
+               "tlogs_to_collapse %s %s %i => %s"
+               (Sn.string_of head_i)
+               (Sn.string_of last_i)
+               tlogs_to_keep
+               (To_string.option (fun (n,i) -> Printf.sprintf "(%i,%s)" n (Sn.string_of i)) r)
+    in            
+    r
     
   let old_uncompressed t =
     List.filter (fun (_,_,is_archive) -> not is_archive) t.i_to_tlog_number
diff --git a/src/tlog/tlogcollection.ml b/src/tlog/tlogcollection.ml
index cec4b210..bfb8d8e3 100644
--- a/src/tlog/tlogcollection.ml
+++ b/src/tlog/tlogcollection.ml
@@ -27,7 +27,12 @@ class type tlog_collection =
                               sync:bool -> string option -> int Lwt.t
   
   
-  method tlogs_to_collapse: Sn.t -> Sn.t -> int -> int
+  method tlogs_to_collapse:
+           head_i:Sn.t ->
+           last_i:Sn.t ->
+           tlogs_to_keep:int ->
+           (int * Sn.t) option
+               
   method get_last_value: Sn.t -> Value.t option (* Lwt.t *)
   method get_last: unit -> (Value.t * Sn.t) option
   method close : ?wait_for_compression : bool -> unit -> unit Lwt.t
diff --git a/src/tlog/tlogcollection_test.ml b/src/tlog/tlogcollection_test.ml
index 801f58dc..074b9202 100644
--- a/src/tlog/tlogcollection_test.ml
+++ b/src/tlog/tlogcollection_test.ml
@@ -82,7 +82,7 @@ let _log_repeat tlc (value:Value.t) n =
 
 let test_rollover (dn, tlx_dir, (factory:factory)) =
   Logger.info_f_ "test_rollover %s, %s" dn tlx_dir >>= fun () ->
-  TlogMap.make ~tlog_max_entries:5 dn tlx_dir node_id >>= fun tlog_map ->
+  TlogMap.make ~tlog_max_entries:5 dn tlx_dir node_id ~check_marker:true >>= fun tlog_map ->
   factory dn node_id >>= fun (c:tlog_collection) ->
   let value = _make_set_v "x" "y" in
   _log_repeat c value 101 >>= fun () ->
@@ -147,7 +147,7 @@ let test_restart (dn, _tlf_dir, (factory:factory)) =
 
 let test_iterate (dn, tlx_dir, (factory: factory)) =
   Logger.info_f_ "test_iterate  %s, %s" dn tlx_dir >>= fun () ->
-  TlogMap.make ~tlog_max_entries:100 dn tlx_dir node_id >>= fun tlog_map ->
+  TlogMap.make ~tlog_max_entries:100 dn tlx_dir node_id ~check_marker:true >>= fun tlog_map ->
   factory dn node_id >>= fun  (tlc:tlog_collection) ->
   let value = _make_set_v "xxx" "y" in
   _log_repeat tlc value 323 >>= fun () ->
@@ -167,7 +167,7 @@ let test_iterate (dn, tlx_dir, (factory: factory)) =
 
 let test_iterate2 (dn, tlx_dir, (factory:factory)) =
   Logger.info_f_ "test_iterate2  %s, %s" dn tlx_dir >>= fun () ->
-  TlogMap.make ~tlog_max_entries:100 dn tlx_dir node_id >>= fun tlog_map ->
+  TlogMap.make ~tlog_max_entries:100 dn tlx_dir node_id ~check_marker:true >>= fun tlog_map ->
   factory dn node_id >>= fun (tlc:tlog_collection) ->
   let value = _make_set_v "test_iterate0" "xxx" in
   _log_repeat tlc value 3 >>= fun () ->
@@ -186,7 +186,7 @@ let test_iterate2 (dn, tlx_dir, (factory:factory)) =
 
 let test_iterate3 (dn, tlx_dir, (factory:factory)) =
   Logger.info_f_ "test_iterate3  %s, %s" dn tlx_dir >>= fun () ->
-  TlogMap.make ~tlog_max_entries:100 dn tlx_dir node_id >>= fun tlog_map ->
+  TlogMap.make ~tlog_max_entries:100 dn tlx_dir node_id ~check_marker:true >>= fun tlog_map ->
   factory dn node_id  >>= fun (tlc:tlog_collection) ->
   let value = _make_set_v "test_iterate3" "xxx" in
   _log_repeat tlc value 120 >>= fun () ->
@@ -205,7 +205,7 @@ let test_iterate3 (dn, tlx_dir, (factory:factory)) =
 
 let test_validate_normal (dn, tlx_dir, (factory:factory)) =
   Logger.info_f_ "test_validate_normal  %s, %s" dn tlx_dir >>= fun () ->
-  TlogMap.make ~tlog_max_entries:100 dn tlx_dir node_id >>= fun tlog_map ->
+  TlogMap.make ~tlog_max_entries:100 dn tlx_dir node_id ~check_marker:true >>= fun tlog_map ->
   factory dn node_id >>= fun (tlc:tlog_collection) ->
   let value = _make_set_v "XXX" "X" in
   _log_repeat tlc value 123 >>= fun () ->
@@ -225,12 +225,12 @@ let test_validate_normal (dn, tlx_dir, (factory:factory)) =
 
 let test_validate_corrupt_1 (dn, tlx_dir, (factory:factory)) =
   let open Tlog_map in
-  TlogMap.make ~tlog_max_entries:100 dn tlx_dir node_id >>= fun tlog_map ->
+  TlogMap.make ~tlog_max_entries:100 dn tlx_dir node_id ~check_marker:true >>= fun tlog_map ->
   factory dn node_id >>= fun (tlc:tlog_collection) ->
   let value = _make_set_v "Incompetent" "Politicians" in
   _log_repeat tlc value 42 >>= fun () ->
   tlc # close () >>= fun () ->
-  TlogMap.make dn tlx_dir node_id >>= fun tlog_map ->
+  TlogMap.make dn tlx_dir node_id ~check_marker:true >>= fun tlog_map ->
   let fn = TlogMap.get_full_path tlog_map "000.tlog" in
   Lwt_unix.openfile fn [Unix.O_RDWR] 0o640 >>= fun fd ->
   Lwt_unix.lseek fd 666 Unix.SEEK_SET >>= fun _ ->

From 87cdf1d8afcc16e052e597349241786f989aebf6 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Wed, 3 Feb 2016 11:01:57 +0100
Subject: [PATCH 085/373] arakoon to redis logging (wip)

---
 Makefile                                      |  2 +-
 _tags                                         |  3 +
 docker/arakoon_centos7/Dockerfile             |  3 +-
 docker/arakoon_debian/Dockerfile              |  3 +-
 myocamlbuild.ml                               |  2 +
 src/all_test.ml                               |  4 +-
 src/arakoon_client.mllib                      |  2 +-
 src/client/client_main.ml                     |  2 +-
 src/main/arakoon.ml                           |  2 +-
 src/node/node_cfg.ml                          | 51 +++++++++-
 src/node/node_cfg_test.ml                     | 30 +++++-
 src/nursery/client_cfg_test.ml                |  2 +-
 src/nursery/nursery_main.ml                   |  4 +-
 src/system/startup.ml                         |  1 +
 .../{arakoon_url.ml => arakoon_config_url.ml} |  0
 src/tools/arakoon_etcd.ml                     |  4 +-
 src/tools/arakoon_redis.ml                    | 99 +++++++++++++++++++
 travis.sh                                     | 12 ++-
 18 files changed, 208 insertions(+), 18 deletions(-)
 rename src/tools/{arakoon_url.ml => arakoon_config_url.ml} (100%)
 create mode 100644 src/tools/arakoon_redis.ml

diff --git a/Makefile b/Makefile
index ddcbce2c..57e657ee 100644
--- a/Makefile
+++ b/Makefile
@@ -67,7 +67,7 @@ install_client:
           _build/src/tools/typed_ssl.cmi \
           _build/src/tools/unix_fd.cmi \
 	  _build/src/tools/arakoon_etcd.cmi \
-	  _build/src/tools/arakoon_url.cmi \
+	  _build/src/tools/arakoon_config_url.cmi \
           _build/src/tools/tcp_keepalive_sockopt_stubs.o \
           _build/src/tools/tcp_keepalive.cmi \
           _build/src/arakoon_client.a
diff --git a/_tags b/_tags
index 42545d8a..ecf5ecba 100644
--- a/_tags
+++ b/_tags
@@ -14,6 +14,9 @@ true: package(quickcheck)
 true: pp(camlp4of)
 true: package(snappy)
 true: package(ocplib-endian)
+true: package(uri)
+true: package(redis)
+true: package(redis.lwt)
 "src":include
 "src/lib":include
 "src/tools":include
diff --git a/docker/arakoon_centos7/Dockerfile b/docker/arakoon_centos7/Dockerfile
index 207621b1..336b0b9e 100644
--- a/docker/arakoon_centos7/Dockerfile
+++ b/docker/arakoon_centos7/Dockerfile
@@ -58,7 +58,8 @@ RUN eval `${opam_env}` && \
         quickcheck \
         conf-libev \
         ocplib-endian \
-        depext
+        depext \
+        redis
 
 
 RUN yum -y install autoconf help2man perl-Thread-Queue
diff --git a/docker/arakoon_debian/Dockerfile b/docker/arakoon_debian/Dockerfile
index 400fb41b..ee917585 100644
--- a/docker/arakoon_debian/Dockerfile
+++ b/docker/arakoon_debian/Dockerfile
@@ -64,7 +64,8 @@ RUN eval `${opam_env}` && \
         quickcheck \
         conf-libev \
         ocplib-endian \
-        depext
+        depext \
+        redis
 
 RUN echo "jenkins ALL=NOPASSWD: ALL" >/etc/sudoers.d/jenkins
 ENV LANG=C
diff --git a/myocamlbuild.ml b/myocamlbuild.ml
index 0a928aff..d4e94673 100644
--- a/myocamlbuild.ml
+++ b/myocamlbuild.ml
@@ -7,6 +7,8 @@ let dependencies = [ "lwt";
                      "bz2";
                      "quickcheck";
                      "ocplib-endian";
+                     "redis";
+                     "uri";
                    ]
 
 (* Enabled compiler warnings, argument for '-w', see `man ocamlc` *)
diff --git a/src/all_test.ml b/src/all_test.ml
index 16077424..d5250387 100644
--- a/src/all_test.ml
+++ b/src/all_test.ml
@@ -59,8 +59,6 @@ let crc32c_tests = "crc32c" >::: [Crc32c_test.suite]
 
 let nursery = "nursery" >::: [
     Routing_test.suite;
-    Client_cfg_test.suite;
-    Node_cfg_test.suite;
   ]
 
 let system = "system" >::: [
@@ -87,5 +85,7 @@ let suite = "universe" >::: [
     system;
     nursery;
     store;
+    Client_cfg_test.suite;
+    Node_cfg_test.suite;
     Std.List.Test.suite;
   ]
diff --git a/src/arakoon_client.mllib b/src/arakoon_client.mllib
index dde6c419..253866ed 100644
--- a/src/arakoon_client.mllib
+++ b/src/arakoon_client.mllib
@@ -16,7 +16,7 @@ Routing
 Interval
 Ncfg
 Client_cfg
-Arakoon_url
+Arakoon_config_url
 Arakoon_etcd
 Ini
 Inifiles
diff --git a/src/client/client_main.ml b/src/client/client_main.ml
index 07272d4a..5859f39c 100644
--- a/src/client/client_main.ml
+++ b/src/client/client_main.ml
@@ -213,7 +213,7 @@ let _cluster_and_node_cfg node_name' cfg_url =
   Node_cfg.retrieve_cfg cfg_url >>= fun cluster_cfg ->
   let _find cfgs =
     let rec loop = function
-      | [] -> failwith (node_name' ^ " is not known in config " ^ (Arakoon_url.to_string cfg_url))
+      | [] -> failwith (node_name' ^ " is not known in config " ^ (Arakoon_config_url.to_string cfg_url))
       | cfg :: rest ->
         if cfg.Node_cfg.node_name = node_name' then cfg
         else loop rest
diff --git a/src/main/arakoon.ml b/src/main/arakoon.ml
index d53b3634..d7dc6683 100644
--- a/src/main/arakoon.ml
+++ b/src/main/arakoon.ml
@@ -315,7 +315,7 @@ let main () =
      "run system tests (you need a running installation)");
     ("--version", set_laction ShowVersion, "shows version");
     (* ("-port", Arg.Set_int port, "specifies server port"); *)
-    ("-config", Arg.String (fun s -> config_url := Arakoon_url.make s),
+    ("-config", Arg.String (fun s -> config_url := Arakoon_config_url.make s),
      "specifies config url (if protocol is absent, 'file' is assumed; default = cfg/arakoon.ini )");
     ("-autofix", Arg.Set autofix, "attempt to autofix (option to -node)");
     ("-catchup-only", Arg.Set catchup_only,
diff --git a/src/node/node_cfg.ml b/src/node/node_cfg.ml
index dc694fcb..f82bedb5 100644
--- a/src/node/node_cfg.ml
+++ b/src/node/node_cfg.ml
@@ -18,7 +18,7 @@ limitations under the License.
 
 let section = Logger.Section.main
 
-let config_url = ref (Arakoon_url.make "cfg/arakoon.ini")
+let config_url = ref (Arakoon_config_url.make "cfg/arakoon.ini")
 
 let default_lease_period = 10
 let default_max_value_size = 8 * 1024 * 1024
@@ -129,6 +129,41 @@ module TLSConfig = struct
     end
 end
 
+module Log_sink = struct
+  type t =
+    | Redis of string * int * string
+    | File of string
+
+  let to_string = function
+    | Redis (host, port, key) -> Printf.sprintf "Redis(%s,%i,%s)" host port key
+    | File file -> Printf.sprintf "File(%s)" file
+
+  let make s =
+    let uri = Uri.of_string s in
+    let path = Uri.path uri in
+    match Uri.scheme uri with
+    | None
+    | Some "file" ->
+       let canonical =
+         if path.[0] = '/'
+         then path
+         else Filename.concat (Unix.getcwd()) path
+       in
+       File canonical
+    | Some "redis" ->
+       let host = match Uri.host uri with
+         | None -> failwith "host is required"
+         | Some h -> h
+       in
+       let port = match Uri.port uri with
+         | None -> 6379
+         | Some p -> p
+       in
+       Redis (host, port, path)
+    | Some protocol ->
+       failwith (Printf.sprintf "unknown protocol:%s" protocol)
+end
+
 module Node_cfg = struct
   open Lwt.Infix
   type t = {node_name:string;
@@ -140,6 +175,7 @@ module Node_cfg = struct
             tlx_dir:string;
             head_dir:string;
             log_dir:string;
+            log_sinks : Log_sink.t list;
             log_level:string;
             log_config:string option;
             batched_transaction_config:string option;
@@ -168,6 +204,7 @@ module Node_cfg = struct
            ; "home", string t.home
            ; "tlog_dir", string t.tlog_dir
            ; "log_dir", string t.log_dir
+           ; "log_sinks", list Log_sink.to_string t.log_sinks
            ; "tlx_dir", string t.tlx_dir
            ; "head_dir", string t.head_dir
            ; "log_level", string t.log_level
@@ -298,6 +335,7 @@ module Node_cfg = struct
         tlx_dir = home;
         head_dir = home;
         log_dir = ":None";
+        log_sinks = [];
         log_level = "DEBUG";
         log_config = Some "default_log_config";
         batched_transaction_config = Some "default_batched_transaction_config";
@@ -562,6 +600,16 @@ module Node_cfg = struct
       try get_string "log_dir"
       with _ -> home
     in
+    let log_sinks =
+      let default_sink = Printf.sprintf "%s/%s.log" log_dir node_name in
+      (try Ini.get
+             inifile node_name "log_sinks"
+             Ini.p_string_list
+             (Ini.default [ default_sink ])
+       with _ -> [ default_sink ])
+      |> List.map
+           (fun x -> Log_sink.File x)
+    in
     let reporting = Ini.get inifile node_name "reporting" Ini.p_int (Ini.default 300) in
     let get_string_option x = Ini.get inifile node_name x (Ini.p_option Ini.p_string) (Ini.default None) in
     let tls_cert = get_string_option "tls_cert"
@@ -592,6 +640,7 @@ module Node_cfg = struct
      tlx_dir;
      head_dir;
      log_dir;
+     log_sinks;
      log_level;
      log_config;
      batched_transaction_config;
diff --git a/src/node/node_cfg_test.ml b/src/node/node_cfg_test.ml
index 45331aa2..85c0ff87 100644
--- a/src/node/node_cfg_test.ml
+++ b/src/node/node_cfg_test.ml
@@ -27,6 +27,30 @@ let test_correctness () =
   in
   Lwt_main.run t
 
-
-
-let suite = "node_cfg" >::: ["correctness" >:: test_correctness]
+let test_log_sink () =
+  let sinks =
+    let open Log_sink in
+    [ File (Unix.getcwd () ^ "/x"), "x";
+      File "/tmp", "/tmp";
+      File "/tmp", "file:///tmp";
+      File (Unix.getcwd () ^ "/x/a"), "x/a";
+      File "/tmp/a", "/tmp/a";
+      File "/tmp/a", "file:///tmp/a";
+      Redis ("localhost", 628, "/key1"), "redis://localhost:628/key1";
+      Redis ("127.0.0.1", 6379, "//key2"), "redis://127.0.0.1//key2";
+    ]
+  in
+  List.iter
+    (fun (expected, url) ->
+     let actual = Log_sink.make url in
+     Printf.printf "url = %s, expected = %s, actual = %s\n"
+                   url
+                   (Log_sink.to_string expected)
+                   (Log_sink.to_string actual);
+     assert (actual = expected))
+    sinks
+
+let suite = "node_cfg" >::: [
+      "correctness" >:: test_correctness;
+      "test_log_sink" >:: test_log_sink;
+    ]
diff --git a/src/nursery/client_cfg_test.ml b/src/nursery/client_cfg_test.ml
index 41dd4fda..b2d0ed01 100644
--- a/src/nursery/client_cfg_test.ml
+++ b/src/nursery/client_cfg_test.ml
@@ -48,7 +48,7 @@ let test_parsing () =
       Lwt_io.Output fn
       (fun oc -> Lwt_io.write oc contents)
     >>= fun () ->
-    ClientCfg.from_url "global" (Arakoon_url.File fn) >>= fun cfg ->
+    ClientCfg.from_url "global" (Arakoon_config_url.File fn) >>= fun cfg ->
     let sa0 = ClientCfg.get cfg "sturdy_0" in
     let r = OUnit.assert_equal sa0 (["127.0.0.1"],7080) in
     Lwt.return r
diff --git a/src/nursery/nursery_main.ml b/src/nursery/nursery_main.ml
index 809c4680..8e22d177 100644
--- a/src/nursery/nursery_main.ml
+++ b/src/nursery/nursery_main.ml
@@ -91,8 +91,8 @@ let _get_keeper_config_generic fetcher url=
 let get_keeper_config cfg_url =
   let fetcher =
     match cfg_url with
-    | Arakoon_url.File f             -> (fun () -> Lwt_extra.read_file f)
-    | Arakoon_url.Etcd (peers, path) -> (fun () -> Arakoon_etcd.retrieve_value peers path)
+    | Arakoon_config_url.File f             -> (fun () -> Lwt_extra.read_file f)
+    | Arakoon_config_url.Etcd (peers, path) -> (fun () -> Arakoon_etcd.retrieve_value peers path)
   in
   _get_keeper_config_generic fetcher cfg_url
 
diff --git a/src/system/startup.ml b/src/system/startup.ml
index e5161d97..e84d67cf 100644
--- a/src/system/startup.ml
+++ b/src/system/startup.ml
@@ -54,6 +54,7 @@ let _make_cfg name n lease_period =
     tlx_dir = name;
     head_dir = name;
     log_dir = "none";
+    log_sinks = [];
     log_level = "DEBUG";
     log_config = Some "log_cfg";
     batched_transaction_config = Some "batched_transaction_config";
diff --git a/src/tools/arakoon_url.ml b/src/tools/arakoon_config_url.ml
similarity index 100%
rename from src/tools/arakoon_url.ml
rename to src/tools/arakoon_config_url.ml
diff --git a/src/tools/arakoon_etcd.ml b/src/tools/arakoon_etcd.ml
index 0e8910c2..85ca3b4e 100644
--- a/src/tools/arakoon_etcd.ml
+++ b/src/tools/arakoon_etcd.ml
@@ -50,7 +50,7 @@ let retrieve_value peers path =
     )
 
 let retrieve_cfg (p:string -> 'a Lwt.t) = function
-  | Arakoon_url.File cfg_file ->
+  | Arakoon_config_url.File cfg_file ->
      Lwt_extra.read_file cfg_file >>= p
-  | Arakoon_url.Etcd (peers,path)->
+  | Arakoon_config_url.Etcd (peers,path)->
      retrieve_value peers path >>= p
diff --git a/src/tools/arakoon_redis.ml b/src/tools/arakoon_redis.ml
new file mode 100644
index 00000000..c605b437
--- /dev/null
+++ b/src/tools/arakoon_redis.ml
@@ -0,0 +1,99 @@
+(*
+Copyright 2016 iNuron NV
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*)
+
+let make_redis_logger
+      ~host ~port
+      ~key ~hostname
+      ~component
+  =
+  let open Lwt_buffer in
+  let buffer = Lwt_buffer.create_fixed_capacity 100 in
+  let seqnum = ref 0 in
+  let stop = ref false in
+  let logger =
+    Lwt_log.make
+      ~output:(fun section level lines ->
+               let id = !seqnum in
+               let () = incr seqnum in
+               if Lwt_buffer.is_full buffer
+               then
+                 (* dropping a log message *)
+                 Lwt.return_unit
+               else
+                 Lwt_buffer.add
+                   (id, Unix.gettimeofday (),
+                    section, level,
+                    lines)
+                   buffer)
+      ~close:(fun () ->
+              stop := true;
+              Lwt.return_unit)
+  in
+  let redis_t () =
+    let module R = Redis_lwt.Client in
+    let pid = Unix.getpid () in
+    let rec outer () =
+      Lwt.catch
+        (fun () ->
+         R.connect R.({ host ;port; }) >>= fun client ->
+         let rec inner () =
+           (* TODO push multiple items in one go? *)
+           if !stop
+           then
+             Lwt.fail Canceled
+           else
+             begin
+               Lwt_buffer.take buffer >>= fun (seqnum, ts,
+                                               section, level,
+                                               lines) ->
+
+               let logline =
+                 Printf.sprintf
+                   "%s - %s - %i/0 - %s - %i - %s - %s"
+                   (let open Unix in
+                    let tm = gmtime ts in
+                    Printf.sprintf
+                      "%04d/%02d/%02d %02d:%02d:%02d %d"
+                      (tm.tm_year + 1900)
+                      (tm.tm_mon + 1)
+                      tm.tm_mday
+                      tm.tm_hour
+                      tm.tm_min
+                      tm.tm_sec
+                      (int_of_float ((ts -. floor ts) *. 10_000.))
+                   )
+                   hostname
+                   pid
+                   component
+                   seqnum
+                   (Lwt_log.string_of_level level)
+                   (String.concat "; " lines)
+               in
+
+               R.rpush client key logline >>= fun _list_length ->
+               inner ()
+             end
+         in
+         inner ())
+        (fun _ -> Lwt.return ()) >>= fun () ->
+      if !stop
+      then Lwt.return ()
+      else outer ()
+    in
+    outer ()
+  in
+  Lwt.ignore_result (redis_t ());
+  logger
diff --git a/travis.sh b/travis.sh
index c6430306..aa850937 100755
--- a/travis.sh
+++ b/travis.sh
@@ -2,7 +2,17 @@
 
 APT_DEPENDS="libev-dev libssl-dev libsnappy-dev libgmp3-dev"
 APT_OCAML_DEPENDS="ocaml ocaml-native-compilers camlp4-extra opam"
-OPAM_DEPENDS="ssl.0.5.2 conf-libev lwt.2.5.1 camlbz2 camltc.999 snappy quickcheck ocplib-endian"
+OPAM_DEPENDS="ssl.0.5.2 \
+              conf-libev \
+              lwt.2.5.1 \
+              camlbz2 \
+              camltc.999 \
+              snappy \
+              quickcheck \
+              ocplib-endian \
+              redis \
+              uri
+"
 
 export OPAMYES=1
 export OPAMVERBOSE=1

From 854340a749d4649a077cc14e651921f1504bad3c Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 3 Feb 2016 09:59:42 +0100
Subject: [PATCH 086/373] zooming in on testRestartOne (Jenkins)

---
 pylabs/test/server/quick/TestCmdTools.py | 34 +++++++++++++++---------
 src/node/node_main.ml                    |  4 +++
 2 files changed, 26 insertions(+), 12 deletions(-)

diff --git a/pylabs/test/server/quick/TestCmdTools.py b/pylabs/test/server/quick/TestCmdTools.py
index 2caa69ad..e3371712 100644
--- a/pylabs/test/server/quick/TestCmdTools.py
+++ b/pylabs/test/server/quick/TestCmdTools.py
@@ -61,20 +61,30 @@ def _assert_n_running(self,n):
             if status[key] == X.AppStatusType.RUNNING:
                 c = c + 1
         if c != n:
-            for key in status.keys():
-                if status[key] == X.AppStatusType.HALTED:
+            for node in status.keys():
+                if status[node] == X.AppStatusType.HALTED:
                     cfg = cluster._getConfigFile()
-                    logDir = cfg.get(key, "log_dir")
-                    fn = "%s/%s.log" % (logDir, key)
+                    logDir = cfg.get(node, "log_dir")
+                    fn = "%s/%s.log" % (logDir, node)
                     logging.info("fn=%s",fn)
-                    with open(fn,'r') as f:
-                        lines = f.readlines()
-                        logging.info("log file for %s", key)
-                        for l in lines:
-                            ls = l.strip()
-                            logging.info(ls)
-
-        assert_equals(c, n)
+                    def cat_log(fn):
+                        with open(fn,'r') as f:
+                            lines = f.readlines()
+                            logging.info("fn:%s for %s", fn, node)
+                            for l in lines:
+                                ls = l.strip()
+                                logging.info(ls)
+
+                    #crash log ?
+                    cat_log(fn)
+                    filter = '%s.debug.*' % node    
+                    crash = X.listFilesInDir(logDir,filter = filter)
+                    if len(crash):
+                        crash_fn = crash[0]
+                        cat_log(crash_fn)
+                                             
+        msg = "expected %i running nodes, but got %i" % (n,c)
+        assert_equals(c, n, msg = msg)
 
     def testStart(self):
         cluster = self._getCluster()
diff --git a/src/node/node_main.ml b/src/node/node_main.ml
index bd7658c0..7558c25b 100644
--- a/src/node/node_main.ml
+++ b/src/node/node_main.ml
@@ -524,9 +524,12 @@ let _main_2 (type s)
         let db_name = full_db_name me in
         let head_copied = ref false in
         let _open_tlc_and_store () =
+          Logger.debug_ "_open_tlc_and_store" >>= fun () ->
           Node_cfg.Node_cfg.validate_dirs me >>= fun () ->
+          Logger.debug_ "validated directories" >>= fun () ->
           let snapshot_name = get_snapshot_name() in
           let full_snapshot_path = Filename.concat me.head_dir snapshot_name in
+          Logger.debug_f_ "full_snapshot_path:%s" full_snapshot_path >>= fun () ->
           Lwt.catch
             (fun () ->
              S.copy_store2 full_snapshot_path db_name
@@ -589,6 +592,7 @@ let _main_2 (type s)
                 Lwt.fail exn
            end
          else
+           Logger.fatal_f_ ~exn "propagating" >>= fun () ->
            Lwt.fail exn
         )
       in

From 654d4c5f8e6fdd817a7259a933ca1cdb8b00841c Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 3 Feb 2016 11:43:43 +0100
Subject: [PATCH 087/373] testRestartOne

---
 src/main/replay_main.ml         | 51 ++++++++---------
 src/node/catchup.ml             |  7 ++-
 src/node/dump_store.ml          |  3 +-
 src/tlog/tlc2.ml                | 99 +++++++++++++--------------------
 src/tlog/tlc2_test.ml           |  2 +-
 src/tlog/tlog_map.ml            | 68 ++++++++++++++--------
 src/tlog/tlogcollection_test.ml |  2 +-
 7 files changed, 122 insertions(+), 110 deletions(-)

diff --git a/src/main/replay_main.ml b/src/main/replay_main.ml
index cfcff708..24c911d3 100644
--- a/src/main/replay_main.ml
+++ b/src/main/replay_main.ml
@@ -40,31 +40,32 @@ let replay_tlogs tlog_dir tlx_dir db_name end_i =
     let open Tlog_map in
     Lwt.catch
       (fun () ->
-         TlogMap.make tlog_dir tlx_dir "node_name???" ~check_marker:false >>= fun tlog_map ->
-         S.make_store ~lcnum:1024 ~ncnum:512 db_name >>= fun store ->
-         let cio = S.consensus_i store in
-         let cios = Log_extra.option2s Sn.string_of cio in
-         console "store @ %s" cios >>= fun () ->
-         begin
-           match
-             end_i with
-             | None ->
-                begin
-                 TlogMap.get_last_tlog tlog_map >>= fun (_new_c,fn) ->
-                 Tlog_map._validate_one
-                   fn ""
-                   ~check_marker:false
-                   ~check_sabotage:false
-                 >>= fun (last, _index, _pos) ->
-                 let i =
-                   match last with
-                     | None -> Sn.start
-                     | Some e -> Sn.succ (Tlogcommon.Entry.i_of e)
-                 in
-                 Lwt.return i
-               end
-             | Some i -> Lwt.return i
-         end
+        TlogMap.make tlog_dir tlx_dir "node_name???" ~check_marker:false
+        >>= fun (tlog_map,_,_) ->
+        S.make_store ~lcnum:1024 ~ncnum:512 db_name >>= fun store ->
+        let cio = S.consensus_i store in
+        let cios = Log_extra.option2s Sn.string_of cio in
+        console "store @ %s" cios >>= fun () ->
+        begin
+          match
+            end_i with
+          | None ->
+             begin
+               TlogMap.get_last_tlog tlog_map >>= fun (_new_c,fn) ->
+               Tlog_map._validate_one
+                 fn ""
+                 ~check_marker:false
+                 ~check_sabotage:false
+               >>= fun (last, _index, _pos) ->
+               let i =
+                 match last with
+                 | None -> Sn.start
+                 | Some e -> Sn.succ (Tlogcommon.Entry.i_of e)
+               in
+               Lwt.return i
+             end
+          | Some i -> Lwt.return i
+        end
          >>= fun too_far_i ->
          let start_i =
            match cio with
diff --git a/src/node/catchup.ml b/src/node/catchup.ml
index 2ceb8514..1d93e4ce 100644
--- a/src/node/catchup.ml
+++ b/src/node/catchup.ml
@@ -162,13 +162,18 @@ let catchup_store ~stop (type s) me
       | None -> Sn.start
       | Some i -> Sn.succ i
   in
-  if Sn.compare start_i too_far_i > 0
+  let comp = Sn.compare start_i too_far_i in
+  if comp > 0
   then
     let msg = Printf.sprintf
                 "Store counter (%s) is ahead of tlog counter (%s). Aborting."
                 (Sn.string_of start_i) (Sn.string_of too_far_i) in
     Logger.error_ msg >>= fun () ->
     Lwt.fail (StoreAheadOfTlogs(start_i, too_far_i))
+  else if comp = 0
+  then
+    Logger.debug_f_ "catchup_store: start_i:%s == too_far_i:%s nothing to do"
+                    (Sn.string_of start_i) (Sn.string_of too_far_i)
   else
     begin
       Logger.debug_f_ "will replay starting from %s into store, too_far_i:%s"
diff --git a/src/node/dump_store.ml b/src/node/dump_store.ml
index d7f66787..849812a5 100644
--- a/src/node/dump_store.ml
+++ b/src/node/dump_store.ml
@@ -77,7 +77,8 @@ let inject_as_head fn node_id cfg_url ~force ~in_place =
       | [] -> failwith (Printf.sprintf "unknown node: %S" node_id)
       | x :: _ -> x
     in
-    TlogMap.make node_cfg.tlog_dir node_cfg.tlx_dir node_id ~check_marker:false >>= fun tlog_map ->
+    TlogMap.make node_cfg.tlog_dir node_cfg.tlx_dir node_id ~check_marker:false
+    >>= fun (tlog_map,_,_) ->
     let head_dir = node_cfg.head_dir in
     let old_head_name = Filename.concat head_dir Tlc2.head_fname  in
 
diff --git a/src/tlog/tlc2.ml b/src/tlog/tlc2.ml
index baedbecc..bd3039f7 100644
--- a/src/tlog/tlc2.ml
+++ b/src/tlog/tlc2.ml
@@ -149,14 +149,15 @@ let _init_file fsync_dir tlog_map =
 class tlc2
         ?(compressor=Compression.Snappy)
         (head_dir:string) 
-        (last:Entry.t option) (index:Index.index)
+        (last:Entry.t option) (index:Index.index) (file:F.t option)
         (tlog_map: TlogMap.t)
         (node_id:string) ~(fsync:bool) ~(fsync_tlog_dir:bool)
+        
   =
 
   object(self: # tlog_collection)
-    val mutable _file = (None:F.t option)
-    val mutable _index = (None: Index.index)
+    val mutable _file = file
+    val mutable _index = index
     val mutable _previous_entry = last
     val mutable _compression_q = Lwt_buffer.create_fixed_capacity 5
     val mutable _compression_thread = None
@@ -464,25 +465,22 @@ class tlc2
               Logger.info_ ~exn "Exception while canceling compression thread")
       end >>= fun () ->
       Logger.debug_ "tlc2::closes () (part2)" >>= fun () ->
-      let last_file () =
-        match _file with
-          | None -> _init_file fsync_tlog_dir tlog_map 
-          | Some file -> Lwt.return file
-      in
-      last_file () >>= fun file ->
-      begin
-        match self # get_last() with
-          | None -> Lwt.return ()
-          | Some(v,i) ->
-            begin
+      match self # get_last () with
+      | None -> Logger.debug_f_ "... no last, we never logged anything=> no marker"
+      | Some(v,i) ->
+         begin
+           match _file with
+           | Some file ->
               let oc = F.oc_of file in
               let marker = Some (_make_close_marker node_id) in
               Tlogcommon.write_marker oc i v marker >>= fun () ->
               Logger.debug_f_ "wrote %S marker @i=%s for %S"
-                (Log_extra.string_option2s marker) (Sn.string_of i) node_id
-            end
-      end >>= fun () ->
-      F.close file
+                              (Log_extra.string_option2s marker) (Sn.string_of i) node_id
+              >>= fun () ->
+              F.close file 
+           | None -> Logger.fatal_f_ "how can this be?"
+         end
+      
 
     method get_tlog_from_i (i:Sn.t) : int = TlogMap.outer_of_i tlog_map i
 
@@ -495,23 +493,8 @@ class tlc2
       Lwt.return (List.length tlogs)
 
     method which_tlog_file (n:int) =
-      let open Compression in
-      let an0 = archive_name Snappy n
-      and an1 = archive_name Bz2 n
-      and fn = file_name n
-      in
-      let an0_c = TlogMap.get_full_path tlog_map an0
-      and an1_c = TlogMap.get_full_path tlog_map an1
-      and fn_c  = TlogMap.get_full_path tlog_map fn
-      in
-      let rec loop = function
-        | []     -> Lwt.return None
-        | x::xs  -> File_system.exists x >>=
-                      (function
-                        | true -> Lwt.return (Some x)
-                        | false -> loop xs)
-      in
-      loop [an0_c;an1_c;fn_c]
+      TlogMap.which_tlog_file tlog_map n
+
 
 
     method dump_tlog_file n oc =
@@ -578,39 +561,37 @@ let make_tlc2 ~compressor
   Logger.debug_f_ "make_tlc2 %S" tlog_dir >>= fun () ->
   TlogMap.make ?tlog_max_entries ?tlog_max_size
                tlog_dir tlf_dir node_id ~check_marker:true
-  >>= fun tlog_map ->
-  TlogMap.get_last_tlog tlog_map >>= fun (new_c, fn) ->
-  (* TODO: move index to tlogmap *)
-  _validate_one fn node_id ~check_marker:true ~check_sabotage:true >>= fun (last, index, pos) ->
-  Logger.debug_f_ "make_tlc2 index = %s" (Index.to_string index) >>= fun () ->
+  >>= fun (tlog_map , last, index )->
+  Logger.debug_f_ "make_tlc2 index = %s" (Index.to_string index)
+  >>= fun () ->
   let msg =
     match last with
       | None -> "None"
-      | Some e -> let i = Entry.i_of e in "Some" ^ (Sn.string_of i)
+      | Some e -> let i = Entry.i_of e in "Some " ^ (Sn.string_of i)
   in
   Logger.debug_f_ "post_validation: last_i=%s" msg >>= fun () ->
-  let col = new tlc2 head_dir last index tlog_map
-                ~compressor node_id ~fsync ~fsync_tlog_dir in
   (* rewrite last entry with ANOTHER marker so we can see we got here *)
   begin
     match last with
-      | None   -> Lwt.return ()
+      | None   -> Lwt.return None
       | Some e ->
-        Logger.debug_f_ "will write marker: new_c:%i \n%s" new_c (Tlogreader2.Index.to_string index)
-        >>= fun() ->
-        (* for some reason this will cause mayhem 'test_243' *)
-        let i = Entry.i_of e in
-        let v = Entry.v_of e in
-        let marker = Some (_make_open_marker node_id) in
-        _init_file fsync_tlog_dir tlog_map >>= fun file ->
-        let oc = F.oc_of file in
-        Tlogcommon.write_marker oc i v marker >>= fun () ->
-        F.close file >>= fun () ->
-        Logger.debug_f_ "wrote %S marker @i=%s for %S"
-          (Log_extra.string_option2s marker) (Sn.string_of i) node_id  >>= fun () ->
-        Lwt.return ()
-  end
-  >>= fun () ->
+         (* for some reason this will cause mayhem 'test_243' *)
+         let i = Entry.i_of e in
+         let v = Entry.v_of e in
+         let marker = Some (_make_open_marker node_id) in
+         _init_file fsync_tlog_dir tlog_map >>= fun file ->
+         let oc = F.oc_of file in
+         Tlogcommon.write_marker oc i v marker >>= fun () ->
+         Logger.debug_f_ "wrote %S marker @i=%s for %S"
+                         (Log_extra.string_option2s marker)
+                         (Sn.string_of i)
+                         node_id
+         >>= fun () ->
+         Lwt.return (Some file)
+  end >>= fun file ->
+  let col = new tlc2 head_dir last index
+                file tlog_map 
+                ~compressor node_id ~fsync ~fsync_tlog_dir in
   Lwt.return col
              
 let _truncate_tlog filename =
diff --git a/src/tlog/tlc2_test.ml b/src/tlog/tlc2_test.ml
index c54e47ff..5af1928a 100644
--- a/src/tlog/tlc2_test.ml
+++ b/src/tlog/tlc2_test.ml
@@ -302,7 +302,7 @@ let test_size_based_roll_over1 (dn, tlx_dir, (factory:factory)) =
   File_system.lwt_directory_list tlx_dir >>= fun tlx_entries ->
   OUnit.assert_equal 3 (List.length tlx_entries) ~printer;
 
-  TlogMap.make dn tlx_dir node_id ~check_marker:true >>= fun second_map ->
+  TlogMap.make dn tlx_dir node_id ~check_marker:true >>= fun (second_map,_,_) ->
   OUnit.assert_equal 3 (TlogMap.get_tlog_number second_map) ~printer;
   Lwt.return_unit
 
diff --git a/src/tlog/tlog_map.ml b/src/tlog/tlog_map.ml
index b3824474..bedb9a40 100644
--- a/src/tlog/tlog_map.ml
+++ b/src/tlog/tlog_map.ml
@@ -2,7 +2,10 @@ open Lwt.Infix
 open Tlogcommon
 open Tlogreader2
 
-let section = Logger.Section.main
+let section =
+  let s= Logger.Section.make "tlog_map" in
+  let () = Logger.Section.set_level s Logger.Debug in
+  s
 
 let tlog_regexp = Str.regexp "^[0-9]+\\.tlog$"
 let file_regexp = Str.regexp "^[0-9]+\\.tl\\(og\\|f\\|c\\|x\\)$"
@@ -168,7 +171,7 @@ let _validate_one
       ~(check_sabotage:bool)
     : validation_result Lwt.t
   =
-  Logger.debug_f_ "Tlog_map._validate_one %s" tlog_name >>= fun () ->
+  Logger.debug_f_ "_validate_one %s" tlog_name >>= fun () ->
   let e2s e = let i = Entry.i_of e in Printf.sprintf "(%s,_)" (Sn.string_of i) in
   let prev_entry = ref None in
   let new_index = Index.make tlog_name in
@@ -213,7 +216,6 @@ let _validate_one
      | Unix.Unix_error(Unix.ENOENT,_,_) ->
         let ext = extension_of tlog_name in
         let base = Filename.basename tlog_name in
-        Logger.debug_f_ "not found: %s. %s was removed?!" tlog_name base >>= fun () ->
         if (not check_sabotage) || (ext = ".tlog" && get_number base = 0)
         then Lwt.return (None, new_index, 0)
         else (* 
@@ -222,7 +224,8 @@ let _validate_one
          meaning rotation happened correctly.
          This means the marker can not be present.
          Let the node die; they should fix this manually.
-         *)
+              *)
+          Logger.fatal_f_ "not found: %s. %s was removed?!" tlog_name base >>= fun () ->
           Lwt.fail TLogSabotage
       | exn -> Lwt.fail exn
     )
@@ -299,7 +302,8 @@ module TlogMap = struct
     in 
     Logger.debug_f_ "_init: tlog_entries:%i should_roll:%b" tlog_entries should_roll
     >>= fun () ->
-    Lwt.return (tlog_entries, tlog_size, tlog_number, i_to_tlog_number, should_roll)
+    Lwt.return ((tlog_entries, tlog_size, tlog_number, i_to_tlog_number, should_roll),
+                last, index)
       
   let make ?tlog_max_entries
            ?tlog_max_size
@@ -316,19 +320,19 @@ module TlogMap = struct
     _init
       tlog_dir tlx_dir node_id ~check_marker ~check_sabotage:false
       tlog_max_entries tlog_max_size
-    >>= fun (tlog_entries, tlog_size,tlog_number,i_to_tlog_number, should_roll) ->
+    >>= fun ((tlog_entries, tlog_size,tlog_number,i_to_tlog_number, should_roll), last, index ) ->
     
     let t = {node_id; tlog_dir;tlx_dir;tlog_max_entries; tlog_max_size;
              tlog_entries; tlog_size; tlog_number; i_to_tlog_number;should_roll;
             }
     in
-    Lwt.return t
+    Lwt.return (t, last, index)
 
   let reinit t =
     _init
       t.tlog_dir t.tlx_dir t.node_id ~check_marker:false ~check_sabotage:false
       t.tlog_max_entries t.tlog_max_size
-    >>= fun (tlog_entries, tlog_size, tlog_number,i_to_tlog_number,should_roll) ->
+    >>= fun ((tlog_entries, tlog_size, tlog_number,i_to_tlog_number,should_roll),_,_) ->
     t.tlog_size   <- tlog_size;
     t.tlog_number <- tlog_number;
     t.i_to_tlog_number <- i_to_tlog_number;
@@ -439,24 +443,44 @@ module TlogMap = struct
       | h :: _ -> let n = Sn.of_int (get_number h) in
                   Sn.mul (Sn.of_int f) n
     in
-    Logger.debug_f_ "Tlc2.infimum=%s" (Sn.string_of v) >>= fun () ->
+    Logger.debug_f_ "infimum=%s" (Sn.string_of v) >>= fun () ->
     Lwt.return v
 
+  let which_tlog_file t n =
+    let open Compression in
+    let options = [archive_name Snappy n;
+                   archive_name Bz2 n;
+                   file_name n]
+    in
+    let canonicals = List.map (get_full_path t) options in
+    let rec loop = function
+      | []     -> Lwt.return None
+      | x::xs  -> File_system.exists x >>=
+                    (function
+                     | true -> Lwt.return (Some x)
+                     | false -> loop xs)
+    in
+    loop canonicals
+         
   let remove_below t i =
-    get_tlog_names t >>= fun existing ->
-    let maybe_remove fn =
-      let n = get_number fn in
-      let fn_stop = Sn.of_int ((n+1) * t.tlog_max_entries) in
-      if fn_stop < i then
-        begin
-          let canonical = _get_full_path t.tlog_dir t.tlx_dir fn in
-          Logger.debug_f_ "Unlinking %s" canonical >>= fun () ->
-          File_system.unlink canonical
-        end
-      else
-        Lwt.return ()
+    Logger.debug_f_ "remove_below %s" (Sn.string_of i) >>= fun () ->
+    let rec find_start = function
+      | [] -> []
+      | (i0,n0,_) :: rest when i0 >= i -> rest
+      | _ :: rest -> find_start rest
+    in
+    let to_remove = find_start t.i_to_tlog_number in
+    
+    let remove (_,n,_) =
+      which_tlog_file  t n >>= function
+      | None -> Lwt.return_unit
+      | Some canonical ->
+         begin
+           Logger.debug_f_ "Unlinking %s" canonical >>= fun () ->
+           File_system.unlink canonical
+         end
     in
-    Lwt_list.iter_s maybe_remove existing
+    Lwt_list.iter_s remove to_remove
 
   let is_rollover_point t i =
     (* TODO: this gets called a zillion times in catchup...
diff --git a/src/tlog/tlogcollection_test.ml b/src/tlog/tlogcollection_test.ml
index 074b9202..5e2aa607 100644
--- a/src/tlog/tlogcollection_test.ml
+++ b/src/tlog/tlogcollection_test.ml
@@ -230,7 +230,7 @@ let test_validate_corrupt_1 (dn, tlx_dir, (factory:factory)) =
   let value = _make_set_v "Incompetent" "Politicians" in
   _log_repeat tlc value 42 >>= fun () ->
   tlc # close () >>= fun () ->
-  TlogMap.make dn tlx_dir node_id ~check_marker:true >>= fun tlog_map ->
+  TlogMap.make dn tlx_dir node_id ~check_marker:true >>= fun (tlog_map,_,_) ->
   let fn = TlogMap.get_full_path tlog_map "000.tlog" in
   Lwt_unix.openfile fn [Unix.O_RDWR] 0o640 >>= fun fd ->
   Lwt_unix.lseek fd 666 Unix.SEEK_SET >>= fun _ ->

From 7ccd87d453b45924db3c88e19420b460e0394817 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 3 Feb 2016 15:10:33 +0100
Subject: [PATCH 088/373] mixed_tlog_formats

---
 pylabs/test/server/left/system_tests_long_left.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pylabs/test/server/left/system_tests_long_left.py b/pylabs/test/server/left/system_tests_long_left.py
index ba9e3901..b7b021e0 100644
--- a/pylabs/test/server/left/system_tests_long_left.py
+++ b/pylabs/test/server/left/system_tests_long_left.py
@@ -167,7 +167,7 @@ def test_mixed_tlog_formats():
     logging.info("going to do %i sets",s0)
     C.iterate_n_times(s0,C.simple_set)
     C.stop_all()
-    cluster.enableTlogCompression(compressor = 'snappy')
+    cluster.enableTlogCompression(compressor = 'bz2')
 
     C.start_all()
     logging.info("another %i sets", s0)

From 1c2eb23013b083167852e7acaa0f21411c1cc257 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 3 Feb 2016 15:11:22 +0100
Subject: [PATCH 089/373] assert_true;assert_false

---
 pylabs/test/server/quick/system_tests_basic.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/pylabs/test/server/quick/system_tests_basic.py b/pylabs/test/server/quick/system_tests_basic.py
index 20f0e32e..ef339134 100644
--- a/pylabs/test/server/quick/system_tests_basic.py
+++ b/pylabs/test/server/quick/system_tests_basic.py
@@ -281,7 +281,7 @@ def test_aSSert_scenario_1():
         client.aSSert('x','x')
     except X.arakoon_client.ArakoonException as ex:
         logging.error ( "Bad stuff happened: %s" % ex)
-        assert_equals(True,False)
+        assert_true(False)
 
 @C.with_custom_setup(C.default_setup, C.basic_teardown)
 def test_aSSert_scenario_2():
@@ -335,7 +335,7 @@ def test_aSSert_exists_scenario_1():
         client.aSSert_exists('x_e')
     except X.arakoon_client.ArakoonException as ex:
         logging.error ( "Bad stuff happened: %s" % ex)
-        assert_equals(True,False)
+        assert_true(False)
 
 @C.with_custom_setup(C.default_setup, C.basic_teardown)
 def test_aSSert_exists_scenario_2():
@@ -448,7 +448,7 @@ def test_replace():
     old3 = client.replace("xxx",None)
     assert_equals(old3,"yyy")
     e = client.exists("xxx")
-    assert_equals(e,False)
+    assert_false(e)
 
 
 @C.with_custom_setup( C.setup_3_nodes_forced_master , C.basic_teardown )

From f294b7d944c8ac12ce5185cdcafe6c99e312797a Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 3 Feb 2016 15:33:52 +0100
Subject: [PATCH 090/373] test_missing_tlog

---
 pylabs/test/server/right/system_tests_long_right.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/pylabs/test/server/right/system_tests_long_right.py b/pylabs/test/server/right/system_tests_long_right.py
index 4e423382..01569683 100644
--- a/pylabs/test/server/right/system_tests_long_right.py
+++ b/pylabs/test/server/right/system_tests_long_right.py
@@ -523,7 +523,7 @@ def start_node():
         
     except Exception,e:
         print e
-        assert_equals(False,True)
+        assert_true(False)
         
     print "done"
 
@@ -637,7 +637,8 @@ def test_missing_tlog():
     cluster = Common._getCluster()
     cfg = cluster.getNodeConfig(nn)
     node_tlf_dir  = cfg['tlf_dir']
-    tlx_full_path = '/'.join ([node_tlf_dir, "002.tlf"])
+    
+    tlx_full_path = '/'.join ([node_tlf_dir, "002.tlx"])
     logging.info("removing %s", tlx_full_path)
     os.remove(tlx_full_path)
     logging.info("cfg keys:%s", cfg.keys())

From ad354a830b59c9563703e5bcbc302bd775666867 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Thu, 4 Feb 2016 16:38:24 +0100
Subject: [PATCH 091/373] test for tlogs_to_collapse

---
 pylabs/test/server/left/test_collapse.py | 11 +--
 src/all_test.ml                          | 10 +--
 src/node/collapser.ml                    |  2 -
 src/node/dump_store.ml                   |  2 +-
 src/tlog/tlc2.ml                         |  2 +-
 src/tlog/tlog_map.ml                     | 92 ++++++++++++++----------
 src/tlog/tlog_map_test.ml                | 48 +++++++++++++
 7 files changed, 118 insertions(+), 49 deletions(-)
 create mode 100644 src/tlog/tlog_map_test.ml

diff --git a/pylabs/test/server/left/test_collapse.py b/pylabs/test/server/left/test_collapse.py
index 2a201a5e..a81379dd 100644
--- a/pylabs/test/server/left/test_collapse.py
+++ b/pylabs/test/server/left/test_collapse.py
@@ -48,11 +48,11 @@ def test_remote_collapse_witness_node():
         up2date = cli.expectProgressPossible()
     logging.info("catchup from collapsed node finished")
 
-@Common.with_custom_setup(Common.setup_2_nodes, Common.basic_teardown)
+@Common.with_custom_setup(Common.setup_2_nodes_mini, Common.basic_teardown)
 def test_remote_collapse():
     zero = Common.node_names[0]
     one = Common.node_names[1]
-    n = 298765
+    n = 29876
     Common.iterate_n_times(n, Common.simple_set)
     logging.info("did %i sets, now going into collapse scenario" % n)
     Common.collapse(zero,1)
@@ -69,6 +69,7 @@ def test_remote_collapse():
         counter = counter + 1
         up2date = cli.expectProgressPossible()
     logging.info("catchup from collapsed node finished")
+    
 
 @Common.with_custom_setup(Common.setup_2_nodes_forced_master, Common.basic_teardown)
 def test_local_collapse_witness_node():
@@ -92,11 +93,13 @@ def test_local_collapse_witness_node():
         up2date = cli.expectProgressPossible()
     logging.info("catchup from collapsed node finished")
 
-@Common.with_custom_setup(Common.setup_2_nodes, Common.basic_teardown)
+@Common.with_custom_setup(Common.setup_2_nodes_mini, Common.basic_teardown)
 def test_local_collapse():
+    print "starting test_local_collapse"
+    
     zero = Common.node_names[0]
     one = Common.node_names[1]
-    n = 298765
+    n = 29876
     Common.iterate_n_times(n, Common.simple_set)
     logging.info("did %i sets, now going into collapse scenario" % n)
     Common.local_collapse(zero,1)
diff --git a/src/all_test.ml b/src/all_test.ml
index 16077424..1797c3e4 100644
--- a/src/all_test.ml
+++ b/src/all_test.ml
@@ -47,10 +47,12 @@ let paxos_tests = "paxos" >::: [Multi_paxos_test.suite;]
 let update_tests = "updates" >::: [Update_test.suite]
 
 let tlog_tests = "tlogs" >::: [
-    Tlogcollection_test.suite_mem;
-    Tlc2_test.suite;
-    Tlogreader2_test.suite;
-  ]
+      Tlog_map_test.suite;
+      Tlogcollection_test.suite_mem;
+      Tlc2_test.suite;
+      Tlogreader2_test.suite;
+    ]
+                                
 let small_catchup = "small_catchup" >:::[Catchup_test.suite;]
 let store = "store" >:::[Store_test.suite;]
 let compression = "compression" >::: [Compression_test.suite;]
diff --git a/src/node/collapser.ml b/src/node/collapser.ml
index 6bdb226d..9a8f0763 100644
--- a/src/node/collapser.ml
+++ b/src/node/collapser.ml
@@ -210,8 +210,6 @@ let collapse_many
       tlogs_to_keep cb' (cb:int -> unit Lwt.t) slowdown =
 
   Logger.debug_f_ "collapse_many" >>= fun () ->
-  tlog_coll # get_tlog_count () >>= fun total_tlogs ->
-  Logger.debug_f_ "total_tlogs = %i; tlogs_to_keep=%i" total_tlogs tlogs_to_keep >>= fun () ->
   let (_,(head_location:string), _) = store_fs in
   let last_i = tlog_coll # get_last_i () in
   let get_head_i () =
diff --git a/src/node/dump_store.ml b/src/node/dump_store.ml
index 849812a5..203d61e4 100644
--- a/src/node/dump_store.ml
+++ b/src/node/dump_store.ml
@@ -119,7 +119,7 @@ let inject_as_head fn node_id cfg_url ~force ~in_place =
     if not ok then failwith "new head is not an improvement";
     let bottom_n = match new_head_i with
       | None -> failwith "can't happen"
-      | Some i -> failwith "todo" (*Sn.to_int (Tlc2.get_file_number i)*)
+      | Some i -> TlogMap.outer_of_i tlog_map i
     in
     begin
       if (not in_place)
diff --git a/src/tlog/tlc2.ml b/src/tlog/tlc2.ml
index bd3039f7..0d82a9e7 100644
--- a/src/tlog/tlc2.ml
+++ b/src/tlog/tlc2.ml
@@ -293,7 +293,7 @@ class tlc2
                 (* the first thing logged falls exactly into a new tlog,
                    the compression job for the previous one(s) thus is not yet picked up *)
                 let need_compression = TlogMap.old_uncompressed tlog_map in
-                Lwt_list.iter_s (fun (_,n,_) -> self # _add_compression_job n) need_compression
+                Lwt_list.iter_s (self # _add_compression_job) need_compression
               else
                 Lwt.return ()
             end >>= fun () ->
diff --git a/src/tlog/tlog_map.ml b/src/tlog/tlog_map.ml
index bedb9a40..07ed1258 100644
--- a/src/tlog/tlog_map.ml
+++ b/src/tlog/tlog_map.ml
@@ -233,6 +233,9 @@ let _validate_one
 module TlogMap = struct
   type tlog_number = int
   type is_archive = bool
+  type item = { i:Sn.t ;
+                n: tlog_number;
+                  is_archive: is_archive}
   type t = {
       tlog_dir: string;
       tlx_dir : string;
@@ -243,14 +246,17 @@ module TlogMap = struct
       mutable tlog_entries : int;
       mutable tlog_number: tlog_number;
       
-      mutable i_to_tlog_number : (Sn.t * tlog_number * is_archive) list; (* sorted: most recent first *)
+      mutable i_to_tlog_number : item list; (* sorted: most recent first *)
       mutable should_roll: bool;
     }
              
   let show_map t =
-    To_string.list (fun (i,n,a) ->
-                       Printf.sprintf "(%s,%03i,%b)" (Sn.string_of i) n a
-                     ) t.i_to_tlog_number
+    To_string.list
+      (fun item ->
+        Printf.sprintf "{ i:%s; n:%03i; is_archive: %b}"
+                       (Sn.string_of item.i) item.n item.is_archive
+      )
+      t.i_to_tlog_number
                       
   let new_entry t s =
     t.tlog_size <- t.tlog_size + s;
@@ -282,8 +288,8 @@ module TlogMap = struct
         let n = get_number tlog_name in
         let canonical = _get_full_path tlog_dir tlx_dir tlog_name in
         first_i_of canonical >>= fun i ->
-        let b = is_archive tlog_name in
-       Lwt.return (i,n, b)
+        let is_archive = is_archive tlog_name in
+       Lwt.return {i;n;is_archive}
       ) tlog_names
     >>= fun i_to_tlog_number ->
     let last_i =
@@ -294,11 +300,11 @@ module TlogMap = struct
     let tlog_entries, should_roll,last_start =
       match i_to_tlog_number with
       | []                -> 0, false, Sn.zero
-      | (i0,_,true)  :: _ -> 0, true , i0
-      | (i0,_,false) :: _ ->
-         let tlog_entries = Sn.sub last_i i0 |> Sn.to_int in
+      | item :: _ when item.is_archive -> 0, true , item.i
+      | item :: _ ->
+         let tlog_entries = Sn.sub last_i item.i |> Sn.to_int in
          let should_roll = tlog_entries + 1 >= tlog_max_entries in
-         tlog_entries, should_roll, i0
+         tlog_entries, should_roll, item.i
     in 
     Logger.debug_f_ "_init: tlog_entries:%i should_roll:%b" tlog_entries should_roll
     >>= fun () ->
@@ -402,9 +408,9 @@ module TlogMap = struct
   let outer_of_i t i =
     let rec find = function
       | [] -> 0
-      | (i0,n0,_) :: rest ->
-         if  i >= i0
-         then n0
+      | item :: rest ->
+         if  i >= item.i
+         then item.n
          else find rest
     in
     find t.i_to_tlog_number
@@ -412,9 +418,9 @@ module TlogMap = struct
   let get_start_i t n =
     let rec find = function
       | [] -> Sn.zero (* TODO: maybe assert *)
-      | (i0,n0,_) :: rest ->
-         if n0 = n
-         then i0 else
+      | item :: rest ->
+         if item.n = n
+         then item.i else
            find rest
     in
     let r = find t.i_to_tlog_number in
@@ -426,13 +432,13 @@ module TlogMap = struct
   let should_roll t = t.should_roll
  
   let new_outer t i =
-    let r = t.tlog_number + 1 in
-    let () = t.tlog_number <- r in
+    let n = t.tlog_number + 1 in
+    let () = t.tlog_number <- n in
     let () = t.tlog_size <- 0 in
     let () = t.tlog_entries <- 0 in
     let () = t.should_roll <- false in
-    let () = t.i_to_tlog_number <- (i,r,false) :: t.i_to_tlog_number in
-    r
+    let () = t.i_to_tlog_number <- {i; n;is_archive = false} :: t.i_to_tlog_number in
+    n
 
   let infimum t =
     get_tlog_names t >>= fun names ->
@@ -463,16 +469,17 @@ module TlogMap = struct
     loop canonicals
          
   let remove_below t i =
-    Logger.debug_f_ "remove_below %s" (Sn.string_of i) >>= fun () ->
-    let rec find_start = function
+    Logger.debug_f_ "remove_below %s tlog_map:%s" (Sn.string_of i) (show_map t)
+    >>= fun () ->
+    let rec find_start acc = function
       | [] -> []
-      | (i0,n0,_) :: rest when i0 >= i -> rest
-      | _ :: rest -> find_start rest
+      | item :: rest when item.i >= i -> (List.rev acc)
+      | x :: rest -> find_start (x::acc) rest
     in
-    let to_remove = find_start t.i_to_tlog_number in
+    let to_remove = find_start [] (List.rev t.i_to_tlog_number) in
     
-    let remove (_,n,_) =
-      which_tlog_file  t n >>= function
+    let remove item  =
+      which_tlog_file  t item.n >>= function
       | None -> Lwt.return_unit
       | Some canonical ->
          begin
@@ -488,21 +495,20 @@ module TlogMap = struct
     
     let rec find = function
       | [] -> false
-      | (i0,_,_ ) :: _ when i0 = i -> true
-      | (i0,_,_ ) :: _ when i0 < i -> false
+      | item :: _ when item.i = i -> true
+      | item :: _ when item.i < i -> false
       | _ :: rest -> find rest
     in
     let r = find t.i_to_tlog_number in
     Logger.ign_debug_f_ "is_rollover_point %s => %b" (Sn.string_of i) r;
     r
     
-
   let complete_file_to_deliver t i =
     
     let rec find = function
       | [] -> None
-      | (i1,_,_)  :: (i0,n,_) :: _ when i0 = i -> Some (n,i1)
-      | (i0,_,_)  :: _ when i0 < i -> None
+      | item1 :: item0 :: _ when item0.i = i -> Some (item0.n,item1.i)
+      | item  :: _ when item.i < i -> None
       | _ :: rest -> find rest
     in
     let r = find t.i_to_tlog_number in
@@ -515,7 +521,7 @@ module TlogMap = struct
   let next_rollover t i =
     let rec find = function
       | [] -> None
-      | (i0,_,_) :: (i1,_,_) :: _ when i1 <=i && i < i0 -> Some i0
+      | item0 :: item1 :: _ when item0.i > i && i >= item1.i -> Some item0.i
       | _ :: rest -> find rest
     in
     let r = find t.i_to_tlog_number in
@@ -574,19 +580,28 @@ module TlogMap = struct
     Lwt.return ()
       
   let tlogs_to_collapse t head_i last_i tlogs_to_keep =
+    let () = Logger.ign_debug_f_
+               "tlogs_to_collapse head_i:%s last_i:%s tlogs_to_keep:%i tlog_map:%s"
+               (Sn.string_of head_i) (Sn.string_of last_i)
+               tlogs_to_keep (show_map t)
+    in
     
-    let collapsable = List.filter (fun (i,n,archive) -> i >= head_i) t.i_to_tlog_number in
+    let collapsable =
+      List.filter (fun item -> item.i >= head_i) t.i_to_tlog_number
+    in
+    let len = List.length collapsable in
+    let n_to_collapse = len - tlogs_to_keep -1 in
     let rec drop n = function
       | [] -> []
       | x :: rest ->
-         if n = 0
+         if n = 1
          then rest
          else drop (n-1) rest
     in
     let to_collapse_rev = drop tlogs_to_keep collapsable in
     let r = match to_collapse_rev with
     | [] -> None
-    | (i, n, archive) :: rest -> Some (List.length rest, i)
+    | item :: _ -> Some (n_to_collapse, item.i)
     in
     let () = Logger.ign_debug_f_
                "tlogs_to_collapse %s %s %i => %s"
@@ -598,6 +613,9 @@ module TlogMap = struct
     r
     
   let old_uncompressed t =
-    List.filter (fun (_,_,is_archive) -> not is_archive) t.i_to_tlog_number
+    List.fold_left
+      (fun acc item -> if item.is_archive then acc else item.n :: acc)
+      []
+      t.i_to_tlog_number |> List.rev 
                 
 end
diff --git a/src/tlog/tlog_map_test.ml b/src/tlog/tlog_map_test.ml
new file mode 100644
index 00000000..6ce257e3
--- /dev/null
+++ b/src/tlog/tlog_map_test.ml
@@ -0,0 +1,48 @@
+open Tlog_map.TlogMap
+       
+let test_tlogs_to_collapse () =
+  let make_item (i,n,is_archive) = {i;n;is_archive} in
+  let tlog_map = {
+      i_to_tlog_number = List.map
+                           make_item [
+                             (12000L,12,false);
+                             (11000L,11,false);
+                             (10000L,10,false);
+                             ( 9000L, 9,false);
+                             ( 8000L, 8,false);
+                             ( 7000L, 7,false);
+                             ( 6000L, 6,false);
+                             ( 5000L, 5,false);
+                             ( 4000L, 4,false);
+                             ( 3000L, 3,false);
+                             ( 2000L, 2,false);
+                             ( 1000L, 1,false);
+                             (    0L, 0,false);  
+                           ];
+      tlog_dir = "tlog_dir";
+      tlx_dir = "tlx_dir";
+      tlog_max_entries = 1000;
+      tlog_max_size = 5_000_000;
+      node_id = "node_id";
+      tlog_size = 400;
+      tlog_entries = 393;
+      tlog_number = 12;
+      should_roll = false;
+    }
+  in
+  let head_i = 0L in
+  let last_i = 12393L in
+  let tlogs_to_keep = 10 in
+  
+  let r = tlogs_to_collapse tlog_map head_i last_i tlogs_to_keep in
+  let open To_string in
+  let printer = To_string.option (pair string_of_int Sn.string_of) in
+  let expected = Some (2,2_000L) in
+  OUnit.assert_equal expected r ~printer
+                     
+open OUnit
+       
+let suite =
+  "tlog_map">:::[
+      "tlogs_to_collapse">:: test_tlogs_to_collapse;
+    ]

From 4b2004557c9e60d586d9ffb966c3ed3588446d88 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Fri, 5 Feb 2016 09:45:51 +0100
Subject: [PATCH 092/373] add initial roll over point when there's no tlog

---
 src/tlog/tlog_map.ml | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/src/tlog/tlog_map.ml b/src/tlog/tlog_map.ml
index 07ed1258..57108780 100644
--- a/src/tlog/tlog_map.ml
+++ b/src/tlog/tlog_map.ml
@@ -291,7 +291,12 @@ module TlogMap = struct
         let is_archive = is_archive tlog_name in
        Lwt.return {i;n;is_archive}
       ) tlog_names
-    >>= fun i_to_tlog_number ->
+    >>= fun i_to_tlog_number0 ->
+    let i_to_tlog_number =
+      if i_to_tlog_number0 = []
+      then [{i = 0L;n = 0;is_archive=false}]
+      else i_to_tlog_number0
+    in
     let last_i =
       match last with
       | None -> Sn.zero
@@ -471,12 +476,12 @@ module TlogMap = struct
   let remove_below t i =
     Logger.debug_f_ "remove_below %s tlog_map:%s" (Sn.string_of i) (show_map t)
     >>= fun () ->
-    let rec find_start acc = function
-      | [] -> []
-      | item :: rest when item.i >= i -> (List.rev acc)
-      | x :: rest -> find_start (x::acc) rest
+    let rec find_start to_remove_rev = function
+      | [] -> ([],[])
+      | item :: rest when item.i >= i -> (List.rev to_remove_rev), (item :: rest)
+      | x :: rest -> find_start (x::to_remove_rev) rest
     in
-    let to_remove = find_start [] (List.rev t.i_to_tlog_number) in
+    let to_remove,to_keep_rev = find_start [] (List.rev t.i_to_tlog_number) in
     
     let remove item  =
       which_tlog_file  t item.n >>= function
@@ -487,7 +492,10 @@ module TlogMap = struct
            File_system.unlink canonical
          end
     in
-    Lwt_list.iter_s remove to_remove
+    Lwt_list.iter_s remove to_remove >>= fun () ->
+    t.i_to_tlog_number <- List.rev to_keep_rev;
+    Logger.debug_f_ "after remove_below %s tlog_map:%s" (Sn.string_of i) (show_map t) >>= fun () ->
+    Lwt.return_unit
 
   let is_rollover_point t i =
     (* TODO: this gets called a zillion times in catchup...

From c924706a254551acd1ce3d7f661a8b4f6e54349e Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Fri, 5 Feb 2016 14:03:13 +0100
Subject: [PATCH 093/373] use configured log_sinks, change format for file
 based logging

---
 Makefile                           |  1 +
 src/all_test.ml                    |  1 +
 src/arakoon_client.mllib           |  1 +
 src/node/node_cfg.ml               | 59 ++++++------------
 src/node/node_cfg_test.ml          | 24 --------
 src/node/node_main.ml              | 27 +++-----
 src/system/startup.ml              |  2 +-
 src/tools/arakoon_log_sink.ml      | 48 +++++++++++++++
 src/tools/arakoon_log_sink_test.ml | 43 +++++++++++++
 src/tools/arakoon_redis.ml         | 73 ++++++----------------
 src/tools/crash_logger.ml          | 98 ++++++++++++++++++++++++++----
 src/tools/lwt_buffer.ml            |  4 +-
 12 files changed, 229 insertions(+), 152 deletions(-)
 create mode 100644 src/tools/arakoon_log_sink.ml
 create mode 100644 src/tools/arakoon_log_sink_test.ml

diff --git a/Makefile b/Makefile
index 57e657ee..34929664 100644
--- a/Makefile
+++ b/Makefile
@@ -62,6 +62,7 @@ install_client:
           _build/src/tools/llio.mli \
           _build/src/tools/llio.cmi \
           _build/src/tools/network.cmi \
+          _build/src/tools/lwt_buffer.cmi \
           _build/src/tools/lwt_extra.cmi \
           _build/src/tools/typed_ssl.mli \
           _build/src/tools/typed_ssl.cmi \
diff --git a/src/all_test.ml b/src/all_test.ml
index d5250387..9ee1e2d5 100644
--- a/src/all_test.ml
+++ b/src/all_test.ml
@@ -88,4 +88,5 @@ let suite = "universe" >::: [
     Client_cfg_test.suite;
     Node_cfg_test.suite;
     Std.List.Test.suite;
+    Arakoon_log_sink_test.suite;
   ]
diff --git a/src/arakoon_client.mllib b/src/arakoon_client.mllib
index 253866ed..80503ecc 100644
--- a/src/arakoon_client.mllib
+++ b/src/arakoon_client.mllib
@@ -35,3 +35,4 @@ Network
 Typed_ssl
 Unix_fd
 Tcp_keepalive
+Lwt_buffer
diff --git a/src/node/node_cfg.ml b/src/node/node_cfg.ml
index f82bedb5..3a3804bc 100644
--- a/src/node/node_cfg.ml
+++ b/src/node/node_cfg.ml
@@ -129,41 +129,6 @@ module TLSConfig = struct
     end
 end
 
-module Log_sink = struct
-  type t =
-    | Redis of string * int * string
-    | File of string
-
-  let to_string = function
-    | Redis (host, port, key) -> Printf.sprintf "Redis(%s,%i,%s)" host port key
-    | File file -> Printf.sprintf "File(%s)" file
-
-  let make s =
-    let uri = Uri.of_string s in
-    let path = Uri.path uri in
-    match Uri.scheme uri with
-    | None
-    | Some "file" ->
-       let canonical =
-         if path.[0] = '/'
-         then path
-         else Filename.concat (Unix.getcwd()) path
-       in
-       File canonical
-    | Some "redis" ->
-       let host = match Uri.host uri with
-         | None -> failwith "host is required"
-         | Some h -> h
-       in
-       let port = match Uri.port uri with
-         | None -> 6379
-         | Some p -> p
-       in
-       Redis (host, port, path)
-    | Some protocol ->
-       failwith (Printf.sprintf "unknown protocol:%s" protocol)
-end
-
 module Node_cfg = struct
   open Lwt.Infix
   type t = {node_name:string;
@@ -174,8 +139,8 @@ module Node_cfg = struct
             tlog_dir:string;
             tlx_dir:string;
             head_dir:string;
-            log_dir:string;
-            log_sinks : Log_sink.t list;
+            log_sinks : Arakoon_log_sink.t list;
+            crash_log_sinks : Arakoon_log_sink.t list;
             log_level:string;
             log_config:string option;
             batched_transaction_config:string option;
@@ -203,8 +168,8 @@ module Node_cfg = struct
            ; "messaging_port", int t.messaging_port
            ; "home", string t.home
            ; "tlog_dir", string t.tlog_dir
-           ; "log_dir", string t.log_dir
-           ; "log_sinks", list Log_sink.to_string t.log_sinks
+           ; "log_sinks", list Arakoon_log_sink.to_string t.log_sinks
+           ; "crash_log_sinks", list Arakoon_log_sink.to_string t.crash_log_sinks
            ; "tlx_dir", string t.tlx_dir
            ; "head_dir", string t.head_dir
            ; "log_level", string t.log_level
@@ -334,8 +299,8 @@ module Node_cfg = struct
         tlog_dir = home;
         tlx_dir = home;
         head_dir = home;
-        log_dir = ":None";
         log_sinks = [];
+        crash_log_sinks = [];
         log_level = "DEBUG";
         log_config = Some "default_log_config";
         batched_transaction_config = Some "default_batched_transaction_config";
@@ -608,7 +573,17 @@ module Node_cfg = struct
              (Ini.default [ default_sink ])
        with _ -> [ default_sink ])
       |> List.map
-           (fun x -> Log_sink.File x)
+           (fun x -> Arakoon_log_sink.File x)
+    in
+    let crash_log_sinks =
+      let default = Printf.sprintf "%s/%s.debug" log_dir node_name in
+      (try Ini.get
+             inifile node_name "crash_log_sinks"
+             Ini.p_string_list
+             (Ini.default [ default ])
+       with _ -> [ default ])
+      |> List.map
+           (fun x -> Arakoon_log_sink.File x)
     in
     let reporting = Ini.get inifile node_name "reporting" Ini.p_int (Ini.default 300) in
     let get_string_option x = Ini.get inifile node_name x (Ini.p_option Ini.p_string) (Ini.default None) in
@@ -639,8 +614,8 @@ module Node_cfg = struct
      tlog_dir;
      tlx_dir;
      head_dir;
-     log_dir;
      log_sinks;
+     crash_log_sinks;
      log_level;
      log_config;
      batched_transaction_config;
diff --git a/src/node/node_cfg_test.ml b/src/node/node_cfg_test.ml
index 85c0ff87..f2e105ba 100644
--- a/src/node/node_cfg_test.ml
+++ b/src/node/node_cfg_test.ml
@@ -27,30 +27,6 @@ let test_correctness () =
   in
   Lwt_main.run t
 
-let test_log_sink () =
-  let sinks =
-    let open Log_sink in
-    [ File (Unix.getcwd () ^ "/x"), "x";
-      File "/tmp", "/tmp";
-      File "/tmp", "file:///tmp";
-      File (Unix.getcwd () ^ "/x/a"), "x/a";
-      File "/tmp/a", "/tmp/a";
-      File "/tmp/a", "file:///tmp/a";
-      Redis ("localhost", 628, "/key1"), "redis://localhost:628/key1";
-      Redis ("127.0.0.1", 6379, "//key2"), "redis://127.0.0.1//key2";
-    ]
-  in
-  List.iter
-    (fun (expected, url) ->
-     let actual = Log_sink.make url in
-     Printf.printf "url = %s, expected = %s, actual = %s\n"
-                   url
-                   (Log_sink.to_string expected)
-                   (Log_sink.to_string actual);
-     assert (actual = expected))
-    sinks
-
 let suite = "node_cfg" >::: [
       "correctness" >:: test_correctness;
-      "test_log_sink" >:: test_log_sink;
     ]
diff --git a/src/node/node_main.ml b/src/node/node_main.ml
index 1b546443..be699f50 100644
--- a/src/node/node_main.ml
+++ b/src/node/node_main.ml
@@ -60,22 +60,17 @@ let _config_logging me get_cfgs =
   let () = set_level Client_protocol.section log_config.client_protocol in
   let () = set_level Multi_paxos.section log_config.paxos in
   let () = set_level Tcp_messaging.section log_config.tcp_messaging in
-  let log_dir = cfg.log_dir in
-  let node_name = cfg.node_name in
-  let common_prefix = log_dir ^ "/" ^ node_name in
-  let log_file_name = common_prefix ^ ".log" in
-  let get_crash_file_name () =
-    Printf.sprintf "%s.debug.%f" common_prefix (Unix.time()) in
+
   if not cfg.is_test
   then
-    begin
-      Crash_logger.setup_default_logger log_file_name get_crash_file_name
-      >>= fun logger -> Lwt.return (Some logger)
-    end
+    Crash_logger.setup_log_sinks cfg.log_sinks cfg.crash_log_sinks >>= fun dumper ->
+    let dump_crash_log () =
+      dumper () >>= fun () ->
+      Logger.info_ "Crash log dumped"
+    in
+    Lwt.return dump_crash_log
   else
-    begin
-      Lwt.return None
-    end
+    Lwt.return (fun () -> Logger.info_ "No crash_log defined")
 
 let _config_batched_transactions node_cfg cluster_cfg =
   let get_optional o default = match o with
@@ -386,11 +381,7 @@ let _main_2 (type s)
   make_config () >>= fun cluster_cfg ->
   let cfgs = cluster_cfg.cfgs in
   let me, others = Node_cfg.Node_cfg.split name cfgs in
-  _config_logging me.node_name make_config >>= fun dump_crash_log ->
-  let maybe_dump_crash_log () =
-    match dump_crash_log with
-    | None -> Logger.info_ "No crash_log defined"
-    | Some f -> f () >>= fun () -> Logger.info_ "Crash log dumped" in
+  _config_logging me.node_name make_config >>= fun maybe_dump_crash_log ->
   let _ = Lwt_unix.on_signal Sys.sigusr2 (fun _ ->
       let handle () =
         Lwt_unix.sleep 0.001 >>= fun () ->
diff --git a/src/system/startup.ml b/src/system/startup.ml
index e84d67cf..dad4a8f4 100644
--- a/src/system/startup.ml
+++ b/src/system/startup.ml
@@ -53,8 +53,8 @@ let _make_cfg name n lease_period =
     tlog_dir = name;
     tlx_dir = name;
     head_dir = name;
-    log_dir = "none";
     log_sinks = [];
+    crash_log_sinks = [];
     log_level = "DEBUG";
     log_config = Some "log_cfg";
     batched_transaction_config = Some "batched_transaction_config";
diff --git a/src/tools/arakoon_log_sink.ml b/src/tools/arakoon_log_sink.ml
new file mode 100644
index 00000000..74f29c03
--- /dev/null
+++ b/src/tools/arakoon_log_sink.ml
@@ -0,0 +1,48 @@
+(*
+Copyright 2016 iNuron NV
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*)
+
+type t =
+  | Redis of string * int * string
+  | File of string
+
+let to_string = function
+  | Redis (host, port, key) -> Printf.sprintf "Redis(%s,%i,%s)" host port key
+  | File file -> Printf.sprintf "File(%s)" file
+
+let make s =
+  let uri = Uri.of_string s in
+  let path = Uri.path uri in
+  match Uri.scheme uri with
+  | None
+  | Some "file" ->
+     let canonical =
+       if path.[0] = '/'
+       then path
+       else Filename.concat (Unix.getcwd()) path
+     in
+     File canonical
+  | Some "redis" ->
+     let host = match Uri.host uri with
+       | None -> failwith "host is required"
+       | Some h -> h
+     in
+     let port = match Uri.port uri with
+       | None -> 6379
+       | Some p -> p
+     in
+     Redis (host, port, path)
+  | Some protocol ->
+     failwith (Printf.sprintf "unknown protocol:%s" protocol)
diff --git a/src/tools/arakoon_log_sink_test.ml b/src/tools/arakoon_log_sink_test.ml
new file mode 100644
index 00000000..d808624d
--- /dev/null
+++ b/src/tools/arakoon_log_sink_test.ml
@@ -0,0 +1,43 @@
+(*
+Copyright 2016 iNuron NV
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*)
+
+let test_log_sink () =
+  let sinks =
+    let open Arakoon_log_sink in
+    [ File (Unix.getcwd () ^ "/x"), "x";
+      File "/tmp", "/tmp";
+      File "/tmp", "file:///tmp";
+      File (Unix.getcwd () ^ "/x/a"), "x/a";
+      File "/tmp/a", "/tmp/a";
+      File "/tmp/a", "file:///tmp/a";
+      Redis ("localhost", 628, "/key1"), "redis://localhost:628/key1";
+      Redis ("127.0.0.1", 6379, "//key2"), "redis://127.0.0.1//key2";
+    ]
+  in
+  List.iter
+    (fun (expected, url) ->
+     let actual = Arakoon_log_sink.make url in
+     Printf.printf "url = %s, expected = %s, actual = %s\n"
+                   url
+                   (Arakoon_log_sink.to_string expected)
+                   (Arakoon_log_sink.to_string actual);
+     assert (actual = expected))
+    sinks
+
+open OUnit
+let suite = "arakoon_log_sink" >::: [
+      "test_log_sink" >:: test_log_sink;
+    ]
diff --git a/src/tools/arakoon_redis.ml b/src/tools/arakoon_redis.ml
index c605b437..2c1c6656 100644
--- a/src/tools/arakoon_redis.ml
+++ b/src/tools/arakoon_redis.ml
@@ -14,37 +14,28 @@ See the License for the specific language governing permissions and
 limitations under the License.
 *)
 
-let make_redis_logger
-      ~host ~port
-      ~key ~hostname
-      ~component
-  =
+open Lwt.Infix
+
+let make_redis_logger ~host ~port ~key =
   let open Lwt_buffer in
   let buffer = Lwt_buffer.create_fixed_capacity 100 in
-  let seqnum = ref 0 in
   let stop = ref false in
-  let logger =
-    Lwt_log.make
-      ~output:(fun section level lines ->
-               let id = !seqnum in
-               let () = incr seqnum in
-               if Lwt_buffer.is_full buffer
-               then
-                 (* dropping a log message *)
-                 Lwt.return_unit
-               else
-                 Lwt_buffer.add
-                   (id, Unix.gettimeofday (),
-                    section, level,
-                    lines)
-                   buffer)
-      ~close:(fun () ->
-              stop := true;
-              Lwt.return_unit)
+  let lg_output line =
+    if Lwt_buffer.is_full buffer
+    then
+      (* dropping a log message *)
+      Lwt.return_unit
+    else
+      Lwt_buffer.add
+        line
+        buffer
+  in
+  let close () =
+    stop := true;
+    Lwt.return_unit
   in
   let redis_t () =
     let module R = Redis_lwt.Client in
-    let pid = Unix.getpid () in
     let rec outer () =
       Lwt.catch
         (fun () ->
@@ -53,36 +44,10 @@ let make_redis_logger
            (* TODO push multiple items in one go? *)
            if !stop
            then
-             Lwt.fail Canceled
+             Lwt.fail Lwt.Canceled
            else
              begin
-               Lwt_buffer.take buffer >>= fun (seqnum, ts,
-                                               section, level,
-                                               lines) ->
-
-               let logline =
-                 Printf.sprintf
-                   "%s - %s - %i/0 - %s - %i - %s - %s"
-                   (let open Unix in
-                    let tm = gmtime ts in
-                    Printf.sprintf
-                      "%04d/%02d/%02d %02d:%02d:%02d %d"
-                      (tm.tm_year + 1900)
-                      (tm.tm_mon + 1)
-                      tm.tm_mday
-                      tm.tm_hour
-                      tm.tm_min
-                      tm.tm_sec
-                      (int_of_float ((ts -. floor ts) *. 10_000.))
-                   )
-                   hostname
-                   pid
-                   component
-                   seqnum
-                   (Lwt_log.string_of_level level)
-                   (String.concat "; " lines)
-               in
-
+               Lwt_buffer.take buffer >>= fun logline ->
                R.rpush client key logline >>= fun _list_length ->
                inner ()
              end
@@ -96,4 +61,4 @@ let make_redis_logger
     outer ()
   in
   Lwt.ignore_result (redis_t ());
-  logger
+  lg_output, close
diff --git a/src/tools/crash_logger.ml b/src/tools/crash_logger.ml
index 697ba308..0cd10d97 100644
--- a/src/tools/crash_logger.ml
+++ b/src/tools/crash_logger.ml
@@ -113,12 +113,91 @@ let setup_crash_log crash_file_gen =
       (fun oc -> log_buffer oc circ_buf)
   in
 
-  let fake_close () = Lwt.return () in
-
-  (add_to_crash_log, fake_close, dump_crash_log)
-
-
-let setup_default_logger file_log_path crash_log_prefix =
+  (add_to_crash_log, dump_crash_log)
+
+let file_logger file_name =
+  let flags = [Unix.O_WRONLY; Unix.O_CREAT; Unix.O_APPEND; Unix.O_NONBLOCK] in
+  Lwt_unix.openfile file_name flags 0o640 >>= fun fd ->
+  Lwt_unix.set_close_on_exec fd;
+  let mutex = Lwt_mutex.create () in
+  let lg_output line =
+    Lwt_mutex.with_lock
+      mutex
+      (fun () ->
+       let rec inner offset len =
+         Lwt_unix.write fd line offset len >>= fun written ->
+         if written = len
+         then Lwt.return_unit
+         else inner (offset + written) (len - written)
+       in
+       inner 0 (Bytes.length line) >>= fun () ->
+       Lwt_unix.write fd "\n" 0 1 >>= fun _ ->
+       Lwt.return_unit)
+  in
+  let close () = Lwt_unix.close fd in
+  Lwt.return (lg_output, close)
+
+
+let setup_log_sinks log_sinks crash_log_sinks =
+  Lwt_list.map_p
+    (let open Arakoon_log_sink in
+     function
+     | File file_name ->
+        file_logger file_name
+     | Redis (host, port, key) ->
+        Arakoon_redis.make_redis_logger
+          ~host ~port ~key
+        |> Lwt.return)
+    log_sinks >>= fun loggers ->
+  let hostname = Unix.gethostname () in
+  let component = "arakoon" in
+  let pid = Unix.getpid () in
+  let seqnum = ref 0 in
+  let logger = Lwt_log.make
+                 ~output:(fun section level lines ->
+                          let seqnum' = !seqnum in
+                          let () = incr seqnum in
+                          let ts = Unix.gettimeofday () in
+                          let logline =
+                            Printf.sprintf
+                              "%s - %s - %i/0 - %s - %i - %s - %s"
+                              (let open Unix in
+                               let tm = gmtime ts in
+                               Printf.sprintf
+                                 "%04d/%02d/%02d %02d:%02d:%02d %d"
+                                 (tm.tm_year + 1900)
+                                 (tm.tm_mon + 1)
+                                 tm.tm_mday
+                                 tm.tm_hour
+                                 tm.tm_min
+                                 tm.tm_sec
+                                 (int_of_float ((mod_float ts 1.) *. 1_000_000.))
+                              )
+                              hostname
+                              pid
+                              component
+                              seqnum'
+                              (Lwt_log.string_of_level level)
+                              (String.concat "\n" lines)
+                          in
+
+                          Lwt_list.iter_p
+                            (fun (lg_output, _) -> lg_output logline)
+                            loggers)
+                 ~close:(fun () ->
+                         Lwt_list.iter_p
+                           (fun (_, close) -> close ())
+                           loggers)
+  in
+  Lwt_log.default := logger;
+  Lwt.return (fun () ->
+              (* TODO this should dump the crash log *)
+              Lwt.return_unit)
+
+let setup_default_logger
+      file_log_path crash_log_prefix
+      (* log_sinks crash_log_sinks *)
+  =
   Lwt.catch
     (fun () ->
        Lwt_log.file
@@ -131,7 +210,7 @@ let setup_default_logger file_log_path crash_log_prefix =
        Lwt.fail (Failure text)
     )
   >>= fun file_logger ->
-  let (log_crash_msg, close_crash_log, dump_crash_log) =
+  let (log_crash_msg, dump_crash_log) =
     setup_crash_log crash_log_prefix in
 
   let add_log_msg section level msgs =
@@ -151,10 +230,7 @@ let setup_default_logger file_log_path crash_log_prefix =
       )
   in
 
-  let close_default_logger () =
-    Lwt_log.close file_logger >>= fun () ->
-    close_crash_log ()
-  in
+  let close_default_logger () = Lwt_log.close file_logger in
   let default_logger = Lwt_log.make ~output:add_log_msg ~close:close_default_logger in
   Lwt_log.default := default_logger;
   Lwt.return dump_crash_log
diff --git a/src/tools/lwt_buffer.ml b/src/tools/lwt_buffer.ml
index b9cfd09b..68a1752e 100644
--- a/src/tools/lwt_buffer.ml
+++ b/src/tools/lwt_buffer.ml
@@ -42,7 +42,7 @@ module Lwt_buffer = struct
     let capacity = Some i in
     create ~capacity ()
 
-  let _is_full t =
+  let is_full t =
     match t.capacity with
       | None -> false
       | Some c -> Queue.length t.q = c
@@ -55,7 +55,7 @@ module Lwt_buffer = struct
            let () = Lwt_condition.signal t.empty () in
            Lwt.return ()
          in
-         if _is_full t
+         if is_full t
          then
            if t.leaky
            then Lwt.return () (* Logger.debug "leaky buffer reached capacity: dropping" *)

From 6304bb575f2fede750cbf11cbd243b76b9ec6e7b Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Fri, 5 Feb 2016 16:12:27 +0100
Subject: [PATCH 094/373] collapse

---
 pylabs/test/server/left/test_collapse.py | 30 +++++++++++++++++++-----
 src/client/remote_nodestream.ml          | 11 +++++----
 src/node/collapser.ml                    |  2 +-
 src/node/collapser_test.ml               | 19 ++++++++-------
 src/tlog/tlog_map.ml                     | 15 ++++++++----
 src/tlog/tlog_map_test.ml                |  9 ++++---
 6 files changed, 56 insertions(+), 30 deletions(-)

diff --git a/pylabs/test/server/left/test_collapse.py b/pylabs/test/server/left/test_collapse.py
index a81379dd..472c08ad 100644
--- a/pylabs/test/server/left/test_collapse.py
+++ b/pylabs/test/server/left/test_collapse.py
@@ -14,8 +14,6 @@
 limitations under the License.
 """
 
-
-
 from .. import system_tests_common as Common
 from arakoon.ArakoonExceptions import *
 from Compat import X
@@ -25,16 +23,31 @@
 from threading import Thread
 from nose.tools import *
 
+def _check_tlog_dirs(node, n):
+    (home_dir, _, tlf_dir, head_dir) = Common.build_node_dir_names(node)
+    
+    tlogs = X.listFilesInDir( home_dir, filter="*.tlog" )
+    tlxs  = X.listFilesInDir( tlf_dir,  filter="*.tlx" )
+    logging.info("tlogs:%s", tlogs)
+    logging.info("tlxs:%s", tlxs)
+    print tlxs
+    
+    assert_equals(len(tlogs) + len(tlxs), n,
+                  msg = "(%s + %s) should have %i file(s)" % (tlogs,tlxs,n))
+    assert_true(X.fileExists(head_dir + "/head.db"))
+    logging.info("tlog_dirs are as expected")
 
-@Common.with_custom_setup(Common.setup_2_nodes_forced_master, Common.basic_teardown)
+@Common.with_custom_setup(Common.setup_2_nodes_forced_master_mini, Common.basic_teardown)
 def test_remote_collapse_witness_node():
     master = Common.node_names[0]
     witness = Common.node_names[1]
-    n = 298765
+    n = 29876
     Common.iterate_n_times(n, Common.simple_set)
     logging.info("did %i sets, now going into collapse scenario" % n)
     Common.collapse(witness,1)
     logging.info("collapsing done")
+    _check_tlog_dirs(witness,1)
+    
     Common.stopOne(master)
     Common.wipe(master)
     Common.startOne(master)
@@ -57,6 +70,8 @@ def test_remote_collapse():
     logging.info("did %i sets, now going into collapse scenario" % n)
     Common.collapse(zero,1)
     logging.info("collapsing done")
+    _check_tlog_dirs(zero,1)
+    
     Common.stopOne(one)
     Common.wipe(one)
     Common.startOne(one)
@@ -71,15 +86,17 @@ def test_remote_collapse():
     logging.info("catchup from collapsed node finished")
     
 
-@Common.with_custom_setup(Common.setup_2_nodes_forced_master, Common.basic_teardown)
+@Common.with_custom_setup(Common.setup_2_nodes_forced_master_mini, Common.basic_teardown)
 def test_local_collapse_witness_node():
     master = Common.node_names[0]
     witness = Common.node_names[1]
-    n = 298765
+    n = 29876
     Common.iterate_n_times(n, Common.simple_set)
     logging.info("did %i sets, now going into collapse scenario" % n)
     Common.local_collapse(witness,1)
     logging.info("collapsing done")
+    _check_tlog_firs(witness,1)
+    
     Common.stopOne(master)
     Common.wipe(master)
     Common.startOne(master)
@@ -103,6 +120,7 @@ def test_local_collapse():
     Common.iterate_n_times(n, Common.simple_set)
     logging.info("did %i sets, now going into collapse scenario" % n)
     Common.local_collapse(zero,1)
+    #
     logging.info("collapsing done")
     Common.stopOne(one)
     Common.wipe(one)
diff --git a/src/client/remote_nodestream.ml b/src/client/remote_nodestream.ml
index 6aa57de2..450b5d13 100644
--- a/src/client/remote_nodestream.ml
+++ b/src/client/remote_nodestream.ml
@@ -132,6 +132,7 @@ class remote_nodestream ((ic,oc) as conn) =
     in
     let incoming ic =
       Llio.input_int ic >>= fun collapse_count ->
+      Logger.debug_f_ "collapse_count:%i" collapse_count >>= fun () ->
       let rec loop i =
         if i = 0
         then Lwt.return ()
@@ -139,12 +140,12 @@ class remote_nodestream ((ic,oc) as conn) =
           begin
             Llio.input_int ic >>= function
             | 0 ->
-              Llio.input_int64 ic >>= fun took ->
-              Logger.debug_f_ "collapsing one file took %Li" took >>= fun () ->
-              loop (i-1)
+               Llio.input_int64 ic >>= fun total ->
+               Logger.debug_f_ "%i:collapsed one file. (total = %Li)" i total >>= fun () ->
+               loop (i-1)
             | e ->
-              Llio.input_string ic >>= fun msg ->
-              Llio.lwt_failfmt "%s (EC: %d)" msg e
+               Llio.input_string ic >>= fun msg ->
+               Llio.lwt_failfmt "%s (EC: %d)" msg e
           end
       in
       loop collapse_count
diff --git a/src/node/collapser.ml b/src/node/collapser.ml
index 9a8f0763..410dcf94 100644
--- a/src/node/collapser.ml
+++ b/src/node/collapser.ml
@@ -226,7 +226,7 @@ let collapse_many
   | Some (n, too_far_i) ->
     begin
       Logger.info_f_ "Going to collapse %d tlogs" n >>= fun () ->
-      cb' (n + 1) >>= fun () ->
+      cb' (n + 2) >>= fun () -> 
       Logger.debug_f_ "too_far_i = %s" (Sn.string_of too_far_i) >>= fun () ->
       collapse_until tlog_coll (module S) store_fs too_far_i cb slowdown >>= fun () ->
       get_head_i () >>= fun head_i ->
diff --git a/src/node/collapser_test.ml b/src/node/collapser_test.ml
index 0706c6a3..bf0a9b5f 100644
--- a/src/node/collapser_test.ml
+++ b/src/node/collapser_test.ml
@@ -92,23 +92,26 @@ let _head_dir = "/tmp/collapser_head"
 
 let test_collapse_many (dn, tlx_dir, head_dir) =
   let node_id = "node_name" in
-  TlogMap.make ~tlog_max_entries:100 dn tlx_dir node_id ~check_marker:true >>= fun tlog_map ->
-
   Logger.debug_f_ "test_collapse_many_regime dn=%s, tlx_dir=%s, head_dir=%s" dn tlx_dir head_dir
   >>= fun () ->
-  Tlc2.make_tlc2 ~compressor dn tlx_dir head_dir
-                 ~fsync:false node_id ~fsync_tlog_dir:false >>= fun tlc ->
+  let make_tlc () = 
+    Tlc2.make_tlc2 ~compressor ~tlog_max_entries:100
+                 dn tlx_dir head_dir
+                 ~fsync:false node_id ~fsync_tlog_dir:false
+  in
+  make_tlc () >>= fun tlc ->
   _make_values tlc 632 >>= fun () ->
-  tlc # close () >>= fun () ->
-  Lwt_unix.sleep 5.0 >>= fun () -> (* compression finished ? *)
+  tlc # close () ~wait_for_compression:true >>= fun () ->
+  
+  make_tlc () >>= fun tlc ->
   let storename = Filename.concat test_dn "head.db" in
   let cb n = Logger.debug_f_ "collapsed %03i" n in
   let cb' = fun _ -> Lwt.return () in
   File_system.unlink storename >>= fun () ->
   let store_methods = (Batched_store.Local_store.copy_store2, storename, 0.0) in
-  Collapser.collapse_many tlc (module S) store_methods 5 cb' cb None >>= fun () ->
+  Collapser.collapse_many tlc (module S) store_methods 6 cb' cb None >>= fun () ->
   Logger.debug_ "collapsed 000" >>= fun () ->
-  Collapser.collapse_many tlc (module S) store_methods 3 cb' cb None >>= fun () ->
+  Collapser.collapse_many tlc (module S) store_methods 4 cb' cb None >>= fun () ->
   Logger.debug_ "collapsed 001 & 002" >>= fun () ->
   Collapser.collapse_many tlc (module S) store_methods 1 cb' cb None >>= fun () ->
   Logger.debug_ "collapsed 003 & 004" >>= fun () -> (* ends @ 510 *)
diff --git a/src/tlog/tlog_map.ml b/src/tlog/tlog_map.ml
index 57108780..bdd224ae 100644
--- a/src/tlog/tlog_map.ml
+++ b/src/tlog/tlog_map.ml
@@ -265,7 +265,7 @@ module TlogMap = struct
        || t.tlog_entries = t.tlog_max_entries
     then
       let () = t.should_roll <- true in
-      Logger.ign_debug_f_ "should roll over"
+      Logger.ign_info_f_ "should roll over"
 
 
   let get_tlog_number t= t.tlog_number
@@ -598,18 +598,23 @@ module TlogMap = struct
       List.filter (fun item -> item.i >= head_i) t.i_to_tlog_number
     in
     let len = List.length collapsable in
-    let n_to_collapse = len - tlogs_to_keep -1 in
+    let n_to_collapse = len - tlogs_to_keep  in
     let rec drop n = function
       | [] -> []
-      | x :: rest ->
+      | (_ :: rest ) as list ->
          if n = 1
-         then rest
+         then list
          else drop (n-1) rest
     in
     let to_collapse_rev = drop tlogs_to_keep collapsable in
     let r = match to_collapse_rev with
     | [] -> None
-    | item :: _ -> Some (n_to_collapse, item.i)
+    | item :: _ ->
+       (* 
+          +2 because before X goes to the store, 
+          you need to have seen X+1 and thus too_far = X+2 
+       *)
+       Some (n_to_collapse, Int64.add item.i 2L)
     in
     let () = Logger.ign_debug_f_
                "tlogs_to_collapse %s %s %i => %s"
diff --git a/src/tlog/tlog_map_test.ml b/src/tlog/tlog_map_test.ml
index 6ce257e3..b04bc5e6 100644
--- a/src/tlog/tlog_map_test.ml
+++ b/src/tlog/tlog_map_test.ml
@@ -15,7 +15,7 @@ let test_tlogs_to_collapse () =
                              ( 5000L, 5,false);
                              ( 4000L, 4,false);
                              ( 3000L, 3,false);
-                             ( 2000L, 2,false);
+                             ( 2000L, 2,false); 
                              ( 1000L, 1,false);
                              (    0L, 0,false);  
                            ];
@@ -31,13 +31,12 @@ let test_tlogs_to_collapse () =
     }
   in
   let head_i = 0L in
-  let last_i = 12393L in
-  let tlogs_to_keep = 10 in
-  
+  let last_i = 12393L  in
+  let tlogs_to_keep = 1 in
   let r = tlogs_to_collapse tlog_map head_i last_i tlogs_to_keep in
   let open To_string in
   let printer = To_string.option (pair string_of_int Sn.string_of) in
-  let expected = Some (2,2_000L) in
+  let expected = Some (12,12_002L) in
   OUnit.assert_equal expected r ~printer
                      
 open OUnit

From 0465cb7670b68fe2480a08a55846b139f74817bd Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Fri, 5 Feb 2016 16:13:12 +0100
Subject: [PATCH 095/373] non-trivial cluster in arakoon.ini

---
 cfg/arakoon.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cfg/arakoon.ini b/cfg/arakoon.ini
index 1fa9cb19..ae76557e 100644
--- a/cfg/arakoon.ini
+++ b/cfg/arakoon.ini
@@ -1,6 +1,6 @@
 [global]
 #comma separated list of node names that are part of the cluster
-cluster = arakoon_0 
+cluster = arakoon_0 , arakoon_1, arakoon_2
 #optional:
 #master = arakoon_0
 # modifier for master, if preferred, the master is no longer forced

From d8b15b63e2801333776878c40f55f345d0788688 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Fri, 5 Feb 2016 17:28:37 +0100
Subject: [PATCH 096/373] reinstate crashlogger functionality

---
 src/node/node_cfg.ml        |   2 +-
 src/node/node_main.ml       |   6 +-
 src/tools/arakoon_logger.ml |  94 ++++++++++++++++++
 src/tools/crash_logger.ml   | 189 ++++++++----------------------------
 src/tools/logger.ml         |  16 ++-
 5 files changed, 147 insertions(+), 160 deletions(-)
 create mode 100644 src/tools/arakoon_logger.ml

diff --git a/src/node/node_cfg.ml b/src/node/node_cfg.ml
index 3a3804bc..612ae521 100644
--- a/src/node/node_cfg.ml
+++ b/src/node/node_cfg.ml
@@ -576,7 +576,7 @@ module Node_cfg = struct
            (fun x -> Arakoon_log_sink.File x)
     in
     let crash_log_sinks =
-      let default = Printf.sprintf "%s/%s.debug" log_dir node_name in
+      let default = Printf.sprintf "%s/%s" log_dir node_name in
       (try Ini.get
              inifile node_name "crash_log_sinks"
              Ini.p_string_list
diff --git a/src/node/node_main.ml b/src/node/node_main.ml
index be699f50..4b05098e 100644
--- a/src/node/node_main.ml
+++ b/src/node/node_main.ml
@@ -63,9 +63,11 @@ let _config_logging me get_cfgs =
 
   if not cfg.is_test
   then
-    Crash_logger.setup_log_sinks cfg.log_sinks cfg.crash_log_sinks >>= fun dumper ->
+    Arakoon_logger.setup_log_sinks cfg.log_sinks >>= fun () ->
     let dump_crash_log () =
-      dumper () >>= fun () ->
+      Lwt_list.iter_p
+        Crash_logger.dump_crash_log
+        cfg.crash_log_sinks >>= fun () ->
       Logger.info_ "Crash log dumped"
     in
     Lwt.return dump_crash_log
diff --git a/src/tools/arakoon_logger.ml b/src/tools/arakoon_logger.ml
new file mode 100644
index 00000000..9e20cf2c
--- /dev/null
+++ b/src/tools/arakoon_logger.ml
@@ -0,0 +1,94 @@
+(*
+Copyright 2016 iNuron NV
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*)
+
+open Lwt.Infix
+
+let file_logger file_name =
+  let flags = [Unix.O_WRONLY; Unix.O_CREAT; Unix.O_APPEND; Unix.O_NONBLOCK] in
+  Lwt_unix.openfile file_name flags 0o640 >>= fun fd ->
+  Lwt_unix.set_close_on_exec fd;
+  let mutex = Lwt_mutex.create () in
+  let lg_output line =
+    Lwt_mutex.with_lock
+      mutex
+      (fun () ->
+       let rec inner offset len =
+         Lwt_unix.write fd line offset len >>= fun written ->
+         if written = len
+         then Lwt.return_unit
+         else inner (offset + written) (len - written)
+       in
+       inner 0 (Bytes.length line) >>= fun () ->
+       Lwt_unix.write fd "\n" 0 1 >>= fun _ ->
+       Lwt.return_unit)
+  in
+  let close () = Lwt_unix.close fd in
+  Lwt.return (lg_output, close)
+
+
+let setup_log_sinks log_sinks =
+  Lwt_list.map_p
+    (let open Arakoon_log_sink in
+     function
+     | File file_name ->
+        file_logger file_name
+     | Redis (host, port, key) ->
+        Arakoon_redis.make_redis_logger
+          ~host ~port ~key
+        |> Lwt.return)
+    log_sinks >>= fun loggers ->
+  let hostname = Unix.gethostname () in
+  let component = "arakoon" in
+  let pid = Unix.getpid () in
+  let seqnum = ref 0 in
+  let logger = Lwt_log.make
+                 ~output:(fun section level lines ->
+                          let seqnum' = !seqnum in
+                          let () = incr seqnum in
+                          let ts = Unix.gettimeofday () in
+                          let logline =
+                            Printf.sprintf
+                              "%s - %s - %i/0 - %s - %i - %s - %s"
+                              (let open Unix in
+                               let tm = gmtime ts in
+                               Printf.sprintf
+                                 "%04d/%02d/%02d %02d:%02d:%02d %d"
+                                 (tm.tm_year + 1900)
+                                 (tm.tm_mon + 1)
+                                 tm.tm_mday
+                                 tm.tm_hour
+                                 tm.tm_min
+                                 tm.tm_sec
+                                 (int_of_float ((mod_float ts 1.) *. 1_000_000.))
+                              )
+                              hostname
+                              pid
+                              component
+                              seqnum'
+                              (Lwt_log.string_of_level level)
+                              (String.concat "\n" lines)
+                          in
+
+                          Lwt_list.iter_p
+                            (fun (lg_output, _) -> lg_output logline)
+                            loggers)
+                 ~close:(fun () ->
+                         Lwt_list.iter_p
+                           (fun (_, close) -> close ())
+                           loggers)
+  in
+  Lwt_log.default := logger;
+  Lwt.return ()
diff --git a/src/tools/crash_logger.ml b/src/tools/crash_logger.ml
index 0cd10d97..ab92b5ff 100644
--- a/src/tools/crash_logger.ml
+++ b/src/tools/crash_logger.ml
@@ -72,7 +72,7 @@ let add_to_crash_log section level msgs =
   }) in
   R.insert entry circ_buf
 
-let setup_crash_log crash_file_gen =
+let dump_crash_log crash_log_sink =
   let string_of_level = function
     | Lwt_log.Debug -> "debug"
     | Lwt_log.Info -> "info"
@@ -82,155 +82,50 @@ let setup_crash_log crash_file_gen =
     | Lwt_log.Fatal -> "fatal"
   in
 
-  let dump_crash_log () =
-    let log_entry oc e =
-      let format_message msg exn = Entry.(
-        Printf.sprintf "%Ld: %s %s: %s"
-          (Int64.of_float e.time)
-          (Lwt_log.Section.name e.section)
-          (string_of_level e.level)
-          (match exn with
-            | None -> msg
-            | Some e -> msg ^ Printexc.to_string e))
-      in
-      Message.traverse_ (fun m e ->
-        Lwt_io.write_line oc (format_message m e))
-        e.Entry.payload
-    in
-
-    let log_buffer oc buffer =
-      let go = R.fold ~f:(fun acc e -> fun () ->
-        acc () >>= fun () ->
-        log_entry oc e)
-        ~acc:(fun () -> Lwt.return_unit)
-        buffer
-      in
-      go ()
-    in
-
-    let crash_file_path = crash_file_gen () in
-    Lwt_io.with_file ~mode:Lwt_io.output crash_file_path
-      (fun oc -> log_buffer oc circ_buf)
+  let format_message e msg exn = Entry.(
+      Printf.sprintf "%Ld: %s %s: %s"
+                     (Int64.of_float e.time)
+                     (Lwt_log.Section.name e.section)
+                     (string_of_level e.level)
+                     (match exn with
+                      | None -> msg
+                      | Some e -> msg ^ Printexc.to_string e))
   in
 
-  (add_to_crash_log, dump_crash_log)
-
-let file_logger file_name =
-  let flags = [Unix.O_WRONLY; Unix.O_CREAT; Unix.O_APPEND; Unix.O_NONBLOCK] in
-  Lwt_unix.openfile file_name flags 0o640 >>= fun fd ->
-  Lwt_unix.set_close_on_exec fd;
-  let mutex = Lwt_mutex.create () in
-  let lg_output line =
-    Lwt_mutex.with_lock
-      mutex
-      (fun () ->
-       let rec inner offset len =
-         Lwt_unix.write fd line offset len >>= fun written ->
-         if written = len
-         then Lwt.return_unit
-         else inner (offset + written) (len - written)
-       in
-       inner 0 (Bytes.length line) >>= fun () ->
-       Lwt_unix.write fd "\n" 0 1 >>= fun _ ->
-       Lwt.return_unit)
-  in
-  let close () = Lwt_unix.close fd in
-  Lwt.return (lg_output, close)
-
-
-let setup_log_sinks log_sinks crash_log_sinks =
-  Lwt_list.map_p
-    (let open Arakoon_log_sink in
-     function
-     | File file_name ->
-        file_logger file_name
-     | Redis (host, port, key) ->
-        Arakoon_redis.make_redis_logger
-          ~host ~port ~key
-        |> Lwt.return)
-    log_sinks >>= fun loggers ->
-  let hostname = Unix.gethostname () in
-  let component = "arakoon" in
-  let pid = Unix.getpid () in
-  let seqnum = ref 0 in
-  let logger = Lwt_log.make
-                 ~output:(fun section level lines ->
-                          let seqnum' = !seqnum in
-                          let () = incr seqnum in
-                          let ts = Unix.gettimeofday () in
-                          let logline =
-                            Printf.sprintf
-                              "%s - %s - %i/0 - %s - %i - %s - %s"
-                              (let open Unix in
-                               let tm = gmtime ts in
-                               Printf.sprintf
-                                 "%04d/%02d/%02d %02d:%02d:%02d %d"
-                                 (tm.tm_year + 1900)
-                                 (tm.tm_mon + 1)
-                                 tm.tm_mday
-                                 tm.tm_hour
-                                 tm.tm_min
-                                 tm.tm_sec
-                                 (int_of_float ((mod_float ts 1.) *. 1_000_000.))
-                              )
-                              hostname
-                              pid
-                              component
-                              seqnum'
-                              (Lwt_log.string_of_level level)
-                              (String.concat "\n" lines)
-                          in
-
-                          Lwt_list.iter_p
-                            (fun (lg_output, _) -> lg_output logline)
-                            loggers)
-                 ~close:(fun () ->
-                         Lwt_list.iter_p
-                           (fun (_, close) -> close ())
-                           loggers)
+  let log_entry oc e =
+    Message.traverse_ (fun m exn ->
+                       Lwt_io.write_line oc (format_message e m exn))
+                      e.Entry.payload
   in
-  Lwt_log.default := logger;
-  Lwt.return (fun () ->
-              (* TODO this should dump the crash log *)
-              Lwt.return_unit)
-
-let setup_default_logger
-      file_log_path crash_log_prefix
-      (* log_sinks crash_log_sinks *)
-  =
-  Lwt.catch
-    (fun () ->
-       Lwt_log.file
-         ~template:"$(date) $(milliseconds): ($(section)|$(level)): $(message)"
-         ~mode:`Append ~file_name:file_log_path ()
-    )
-    (fun exn ->
-       let msg = Printexc.to_string exn in
-       let text = Printf.sprintf "could not create file logger %S : %s" file_log_path msg in
-       Lwt.fail (Failure text)
-    )
-  >>= fun file_logger ->
-  let (log_crash_msg, dump_crash_log) =
-    setup_crash_log crash_log_prefix in
-
-  let add_log_msg section level msgs =
-    let log_file_msg msg = Lwt_log.log
-                             ~section
-                             ~logger:file_logger
-                             ~level msg
+
+  let log_buffer oc buffer =
+    let go = R.fold ~f:(fun acc e -> fun () ->
+                                     acc () >>= fun () ->
+                                     log_entry oc e)
+                    ~acc:(fun () -> Lwt.return_unit)
+                    buffer
     in
-    Lwt.catch
-      (fun () ->
-         Lwt_list.iter_s log_file_msg msgs >>= fun () ->
-         log_crash_msg section level (Message.ImmediateN msgs);
-         Lwt.return_unit)
-      (function
-        | Lwt_log.Logger_closed -> Lwt.return ()
-        | e -> Lwt.fail e
-      )
+    go ()
   in
 
-  let close_default_logger () = Lwt_log.close file_logger in
-  let default_logger = Lwt_log.make ~output:add_log_msg ~close:close_default_logger in
-  Lwt_log.default := default_logger;
-  Lwt.return dump_crash_log
+  let open Arakoon_log_sink in
+  match crash_log_sink with
+  | File file_name ->
+     Lwt_io.with_file
+       ~mode:Lwt_io.output (Printf.sprintf "%s.debug.%f" file_name (Unix.time ()))
+       (fun oc -> log_buffer oc circ_buf)
+  | Redis (host, port, key) ->
+     let key = Printf.sprintf "%s.debug.%f" key (Unix.time ()) in
+     let module Re = Redis_lwt.Client in
+     Re.connect Re.({ host ;port; }) >>= fun client ->
+     R.fold
+       ~f:(fun acc e -> fun () ->
+                        acc () >>= fun () ->
+                        Message.traverse_
+                          (fun m exn ->
+                           Re.rpush client key (format_message e m exn) >>= fun _list_length ->
+                           Lwt.return_unit)
+                          e.Entry.payload)
+       ~acc:(fun () -> Lwt.return_unit)
+       circ_buf
+       ()
diff --git a/src/tools/logger.ml b/src/tools/logger.ml
index c3420c0e..842f0e9a 100644
--- a/src/tools/logger.ml
+++ b/src/tools/logger.ml
@@ -32,23 +32,19 @@ type level =
   | Fatal
 
 let log ?exn section level msg =
+  Crash_logger.add_to_crash_log section level (Crash_logger.Message.Immediate1_exn (msg, exn));
   if level < Lwt_log.Section.level section
-  then begin
-    Crash_logger.add_to_crash_log section level (Crash_logger.Message.Immediate1_exn (msg, exn));
-    Lwt.return_unit end
-  else
-    Lwt_log.log ?exn ~section ~level msg
+  then Lwt.return_unit
+  else Lwt_log.log ?exn ~section ~level msg
 
 let ign_log ?exn section level msg =
   Lwt.ignore_result (log ?exn section level msg)
 
 let log_ ?exn section level dmsg =
+  Crash_logger.add_to_crash_log section level (Crash_logger.Message.Delayed1_exn (dmsg, exn));
   if level < Lwt_log.Section.level section
-  then begin
-    Crash_logger.add_to_crash_log section level (Crash_logger.Message.Delayed1_exn (dmsg, exn));
-    Lwt.return_unit end
-  else
-    Lwt_log.log ?exn ~section ~level (dmsg ())
+  then Lwt.return_unit
+  else Lwt_log.log ?exn ~section ~level (dmsg ())
 
 let ign_log_ ?exn section level dmsg =
   Lwt.ignore_result (log_ ?exn section level dmsg)

From 5992ce796e514e72c758cdc4a187a3ad6b1cf222 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Fri, 5 Feb 2016 16:58:40 +0100
Subject: [PATCH 097/373] typos

---
 pylabs/test/server/left/test_collapse.py            | 2 +-
 pylabs/test/server/right/system_tests_long_right.py | 2 +-
 pylabs/test/server/system_tests_common.py           | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/pylabs/test/server/left/test_collapse.py b/pylabs/test/server/left/test_collapse.py
index 472c08ad..1f0fda98 100644
--- a/pylabs/test/server/left/test_collapse.py
+++ b/pylabs/test/server/left/test_collapse.py
@@ -95,7 +95,7 @@ def test_local_collapse_witness_node():
     logging.info("did %i sets, now going into collapse scenario" % n)
     Common.local_collapse(witness,1)
     logging.info("collapsing done")
-    _check_tlog_firs(witness,1)
+    _check_tlog_dirs(witness,1)
     
     Common.stopOne(master)
     Common.wipe(master)
diff --git a/pylabs/test/server/right/system_tests_long_right.py b/pylabs/test/server/right/system_tests_long_right.py
index 01569683..85b4c46c 100644
--- a/pylabs/test/server/right/system_tests_long_right.py
+++ b/pylabs/test/server/right/system_tests_long_right.py
@@ -505,7 +505,7 @@ def test_sabotage():
                   #"%s.db.wal" % (node_id,), # should not exist after a `proper` close
                   ])
     for f in files:
-        print "removeing", f
+        print "removing", f
         X.removeFile(f)
     print "starting"
     cmd = clu._cmd('sturdy_0')
diff --git a/pylabs/test/server/system_tests_common.py b/pylabs/test/server/system_tests_common.py
index d93801a0..d0bbb1c0 100644
--- a/pylabs/test/server/system_tests_common.py
+++ b/pylabs/test/server/system_tests_common.py
@@ -650,7 +650,7 @@ def setup_n_nodes_base(c_id, node_names, force_master,
                        extra = None, witness_nodes = False, useIPV6=False,
                        slowCollapser = False):
 
-    X.subprocess.check_call("sudo /sbin/iptables -F".split(' ') )
+    #X.subprocess.check_call("sudo /sbin/iptables -F".split(' ') )
 
     cluster = _getCluster( c_id )
     cluster.tearDown()

From e49fe457f42e5c93a114b5502f22c7ac469a8d65 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Sat, 6 Feb 2016 01:49:59 +0100
Subject: [PATCH 098/373] less calls to is_rollover_point

---
 src/node/collapser.ml | 24 +++++++++++++++++++-----
 src/tlog/tlog_map.ml  |  4 ++--
 2 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/src/node/collapser.ml b/src/node/collapser.ml
index 410dcf94..ce9e1521 100644
--- a/src/node/collapser.ml
+++ b/src/node/collapser.ml
@@ -102,6 +102,14 @@ let collapse_until (type s) (tlog_coll:Tlogcollection.tlog_collection)
                Lwt_unix.sleep period >>= fun () ->
                Lwt.return r
           in
+          let get_next_point i =
+              let next_r = tlog_coll # next_rollover i in
+              match next_r with
+                | None -> Sn.zero
+                | Some i -> i
+          in
+          let next_rollover_point = ref (get_next_point (Sn.pred start_i))
+          in
           let add_to_store entry =
             let i = Entry.i_of entry
             and value = Entry.v_of entry
@@ -120,13 +128,19 @@ let collapse_until (type s) (tlog_coll:Tlogcollection.tlog_collection)
                     begin
                       maybe_log pi >>= fun () ->
                       begin
-                        if tlog_coll # is_rollover_point pi
+                        if pi = !next_rollover_point
                         then
-
-                          cb (first_tlog + !processed) >>= fun () ->
-                          Lwt.return( processed := !processed + 1 )
+                          begin
+                            Logger.info_f_ "rollover_point pi:%Li next_rollover_point =%s"
+                                          pi (Sn.string_of !next_rollover_point)
+                            >>= fun () ->
+                            cb (first_tlog + !processed) >>= fun () ->
+                            let () = incr processed in
+                            let () = next_rollover_point := get_next_point (Sn.succ pi) in
+                            Lwt.return_unit
+                          end
                         else
-                          Lwt.return ()
+                          Lwt.return_unit
                       end
                       >>= fun () ->
                       slowdown (fun () -> S.safe_insert_value new_store pi pv) >>= fun _ ->
diff --git a/src/tlog/tlog_map.ml b/src/tlog/tlog_map.ml
index bdd224ae..44440ba9 100644
--- a/src/tlog/tlog_map.ml
+++ b/src/tlog/tlog_map.ml
@@ -265,7 +265,7 @@ module TlogMap = struct
        || t.tlog_entries = t.tlog_max_entries
     then
       let () = t.should_roll <- true in
-      Logger.ign_info_f_ "should roll over"
+      Logger.ign_info_f_ "should roll over: tlog_entries=%i" (t.tlog_entries)
 
 
   let get_tlog_number t= t.tlog_number
@@ -498,7 +498,7 @@ module TlogMap = struct
     Lwt.return_unit
 
   let is_rollover_point t i =
-    (* TODO: this gets called a zillion times in catchup...
+    (* TODO: this gets called a zillion times in collapse...
      *)
     
     let rec find = function

From 937724eec2cdb51393448967aca6b2ddf2e8eb14 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Sun, 7 Feb 2016 18:46:18 +0100
Subject: [PATCH 099/373] benchmark used to have less scenarios

---
 pylabs/test/server/shaky/test_272.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pylabs/test/server/shaky/test_272.py b/pylabs/test/server/shaky/test_272.py
index 3c25cd6b..5617f96f 100644
--- a/pylabs/test/server/shaky/test_272.py
+++ b/pylabs/test/server/shaky/test_272.py
@@ -34,7 +34,7 @@ def test_272():
     path = cluster._getConfigFileName() + ".cfg"
     logging.info('path=%s', path)
     bench = subprocess.Popen([Common.CONFIG.binary_full_path, '-config', path ,'--benchmark',
-                              '-max_n', '100000'])
+                              '-max_n', '20000'])
     time.sleep(10.0) # give it time to get up to speed
     rc = bench.returncode
     if rc <> None:

From 39fa3f4b9e717fd59fa719a0024de976ed119505 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Mon, 8 Feb 2016 09:01:13 +0100
Subject: [PATCH 100/373] first_i for .tlc

---
 pylabs/test/server/left/test_collapse.py |  3 ++-
 src/tlog/tlog_map.ml                     | 29 ++++++++++++------------
 2 files changed, 17 insertions(+), 15 deletions(-)

diff --git a/pylabs/test/server/left/test_collapse.py b/pylabs/test/server/left/test_collapse.py
index 1f0fda98..14648dfb 100644
--- a/pylabs/test/server/left/test_collapse.py
+++ b/pylabs/test/server/left/test_collapse.py
@@ -93,9 +93,10 @@ def test_local_collapse_witness_node():
     n = 29876
     Common.iterate_n_times(n, Common.simple_set)
     logging.info("did %i sets, now going into collapse scenario" % n)
+
     Common.local_collapse(witness,1)
     logging.info("collapsing done")
-    _check_tlog_dirs(witness,1)
+    
     
     Common.stopOne(master)
     Common.wipe(master)
diff --git a/src/tlog/tlog_map.ml b/src/tlog/tlog_map.ml
index 44440ba9..edab44c5 100644
--- a/src/tlog/tlog_map.ml
+++ b/src/tlog/tlog_map.ml
@@ -32,6 +32,13 @@ let is_archive filename =
   
 let first_i_of filename =
   let extension = extension_of filename in
+  let read_uncompress ic inflate =
+    Llio.input_string ic >>= fun compressed ->
+    let uncompressed = inflate compressed in
+    let buffer = Llio.make_buffer uncompressed 0 in
+    let i =  Sn.sn_from buffer in
+    Lwt.return i
+  in
   let reader = 
     match extension with
     | ".tlog" -> (fun ic -> Sn.input_sn ic )                   
@@ -39,22 +46,19 @@ let first_i_of filename =
        (fun ic ->
          Compression.read_format ic Compression.Snappy >>= fun ()->
          Sn.input_sn ic >>= fun _last_i ->
-         Llio.input_string ic >>= fun compressed ->
-         let uncompressed = Compression.uncompress_snappy compressed in
-         let buffer = Llio.make_buffer uncompressed 0 in
-         let i =  Sn.sn_from buffer in
-         Lwt.return i
+         read_uncompress ic Compression.uncompress_snappy 
        )
     | ".tlf"  ->
        (fun ic ->
          Compression.read_format ic Compression.Bz2 >>= fun ()->
          Sn.input_sn ic >>= fun _last_i ->
-         Llio.input_string ic >>= fun compressed ->
-         let uncompressed = Compression.uncompress_bz2 compressed in
-         let buffer = Llio.make_buffer uncompressed 0 in
-         let i =  Sn.sn_from buffer in
-         Lwt.return i)
-    | ".tlc"  -> failwith "todo: first_i_if .tlc"
+         read_uncompress ic Compression.uncompress_bz2
+       )
+    | ".tlc"  ->
+       (fun ic ->
+         Llio.input_int ic >>= fun _ ->
+         read_uncompress ic Compression.uncompress_bz2
+       )
     | x -> failwith (Printf.sprintf "not a tlog_file:%s" x)
   in
   Lwt_io.with_file ~mode:Lwt_io.input filename reader >>= fun i ->
@@ -498,9 +502,6 @@ module TlogMap = struct
     Lwt.return_unit
 
   let is_rollover_point t i =
-    (* TODO: this gets called a zillion times in collapse...
-     *)
-    
     let rec find = function
       | [] -> false
       | item :: _ when item.i = i -> true

From 1c9d528b4e37ff8432c9c28ca6c78074388ccefe Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Mon, 8 Feb 2016 09:06:05 +0100
Subject: [PATCH 101/373] kill the iptable rules

---
 pylabs/test/server/system_tests_common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pylabs/test/server/system_tests_common.py b/pylabs/test/server/system_tests_common.py
index d0bbb1c0..d93801a0 100644
--- a/pylabs/test/server/system_tests_common.py
+++ b/pylabs/test/server/system_tests_common.py
@@ -650,7 +650,7 @@ def setup_n_nodes_base(c_id, node_names, force_master,
                        extra = None, witness_nodes = False, useIPV6=False,
                        slowCollapser = False):
 
-    #X.subprocess.check_call("sudo /sbin/iptables -F".split(' ') )
+    X.subprocess.check_call("sudo /sbin/iptables -F".split(' ') )
 
     cluster = _getCluster( c_id )
     cluster.tearDown()

From 0e5454c9bb0cec273a0b70243cca08d292a051d0 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Mon, 8 Feb 2016 09:10:55 +0100
Subject: [PATCH 102/373] really test collapse_until

---
 src/node/collapser_test.ml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/node/collapser_test.ml b/src/node/collapser_test.ml
index bf0a9b5f..464bc255 100644
--- a/src/node/collapser_test.ml
+++ b/src/node/collapser_test.ml
@@ -57,9 +57,10 @@ let _make_values tlc n =
 
 let test_collapse_until (dn, tlx_dir, head_dir) =
   let node_id = "node_name" in
-  TlogMap.make ~tlog_max_entries:1000 dn tlx_dir node_id ~check_marker:true >>= fun tlog_map ->
   Logger.debug_f_ "dn=%s, tlf_dir=%s, head_dir=%s" dn tlx_dir head_dir >>= fun () ->
-  Tlc2.make_tlc2 ~compressor dn tlx_dir head_dir
+  Tlc2.make_tlc2 ~compressor
+                 ~tlog_max_entries:1000
+                 dn tlx_dir head_dir
                  ~fsync:false node_id ~fsync_tlog_dir:false
   >>= fun tlc ->
   _make_values tlc 1111 >>= fun () ->

From 9760ba11b169d58205517ee9502963699a8013f3 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Mon, 8 Feb 2016 14:57:53 +0100
Subject: [PATCH 103/373] fix based on manual redis logging test

---
 cfg/arakoon.ini      | 2 ++
 src/node/node_cfg.ml | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/cfg/arakoon.ini b/cfg/arakoon.ini
index e7e9ef66..0f57208e 100644
--- a/cfg/arakoon.ini
+++ b/cfg/arakoon.ini
@@ -88,6 +88,8 @@ home = /tmp/arakoon/arakoon_0
 #optional different directory for .tlog, .tlc and friends
 #tlog_dir = /tmp/arakoon/arakoon_0
 #log_dir = /tmp/arakoon/arakoon_0
+#log_sinks = redis://127.0.0.1:6379/logs/arakoon/arakoon_0
+#crash_log_sinks = redis://127.0.0.1:6379/logs/arakoon/arakoon_0/crash
 #tlog_compression = bz2 # default is bz2; valid choices are: none snappy bz2
 
 fsync=true
diff --git a/src/node/node_cfg.ml b/src/node/node_cfg.ml
index 612ae521..c5f396c2 100644
--- a/src/node/node_cfg.ml
+++ b/src/node/node_cfg.ml
@@ -573,7 +573,7 @@ module Node_cfg = struct
              (Ini.default [ default_sink ])
        with _ -> [ default_sink ])
       |> List.map
-           (fun x -> Arakoon_log_sink.File x)
+           Arakoon_log_sink.make
     in
     let crash_log_sinks =
       let default = Printf.sprintf "%s/%s" log_dir node_name in

From 1adcf02b67d62219a5e558684f5e3734ac4e8759 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Mon, 8 Feb 2016 16:18:54 +0100
Subject: [PATCH 104/373] test redis logging

---
 docker/arakoon_centos7/Dockerfile             |  3 ++-
 docker/arakoon_debian/Dockerfile              | 10 +++++++++-
 .../test/server/quick/system_tests_basic.py   | 20 +++++++++++++++++++
 pylabs/test/server/system_tests_common.py     | 14 +++++++++----
 4 files changed, 41 insertions(+), 6 deletions(-)

diff --git a/docker/arakoon_centos7/Dockerfile b/docker/arakoon_centos7/Dockerfile
index 336b0b9e..2f50e990 100644
--- a/docker/arakoon_centos7/Dockerfile
+++ b/docker/arakoon_centos7/Dockerfile
@@ -59,7 +59,8 @@ RUN eval `${opam_env}` && \
         conf-libev \
         ocplib-endian \
         depext \
-        redis
+        redis \
+        uri
 
 
 RUN yum -y install autoconf help2man perl-Thread-Queue
diff --git a/docker/arakoon_debian/Dockerfile b/docker/arakoon_debian/Dockerfile
index ee917585..9d24ac83 100644
--- a/docker/arakoon_debian/Dockerfile
+++ b/docker/arakoon_debian/Dockerfile
@@ -65,7 +65,8 @@ RUN eval `${opam_env}` && \
         conf-libev \
         ocplib-endian \
         depext \
-        redis
+        redis \
+        uri
 
 RUN echo "jenkins ALL=NOPASSWD: ALL" >/etc/sudoers.d/jenkins
 ENV LANG=C
@@ -76,3 +77,10 @@ RUN curl -L  https://github.com/coreos/etcd/releases/download/v2.2.4/etcd-v2.2.4
 RUN tar xzvf etcd-v2.2.4-linux-amd64.tar.gz
 RUN cp ./etcd-v2.2.4-linux-amd64/etcd /usr/bin \
     && cp ./etcd-v2.2.4-linux-amd64/etcdctl /usr/bin
+
+# Install redis:
+RUN wget http://download.redis.io/releases/redis-3.0.7.tar.gz
+RUN tar xzvf redis-3.0.7.tar.gz
+RUN cd redis-3.0.7 && make
+RUN cp ./redis-3.0.7/src/redis-server /usr/bin
+RUN cp ./redis-3.0.7/src/redis-cli /usr/bin
diff --git a/pylabs/test/server/quick/system_tests_basic.py b/pylabs/test/server/quick/system_tests_basic.py
index 20f0e32e..deb3535b 100644
--- a/pylabs/test/server/quick/system_tests_basic.py
+++ b/pylabs/test/server/quick/system_tests_basic.py
@@ -943,3 +943,23 @@ def test_statistics():
                     "Wrong value for var timing of %s == 0.0" % k)
                 assert_not_equals( timing["min"], 0.0,
                     "Wrong value for min timing of %s == 0.0" % k)
+
+def setup_redis_arakoon():
+    X.subprocess.call(["redis-cli"])
+    C.setup_n_nodes(C.cluster_id, node_names, False, C.data_base_dir,
+                    node_msg_base_port, node_client_base_port,
+                    nodes_extra =
+                    {"arakoon_0": {"log_sinks":"redis://127.0.0.1/logs/arakoon_0"},
+                     "arakoon_1": {"log_sinks":"redis://127.0.0.1/logs/arakoon_1"}})
+
+def redis_arakoon_teardown():
+    X.subprocess.call(["pkill", "redis-cli"])
+
+@C.with_custom_setup(setup_redis_arakoon, redis_arakoon_teardown)
+def test_redis_logging():
+    cli = C.get_client()
+    key = "my-very-unique-key-which-should-be-in-there"
+    cli.set(key,"X")
+    cmd = ["nohup", "redis-cli", "lrange", "/logs/arakoon/arakoon_0", "0", "100000", "&"]
+    stdout = subprocess.check_output(cmd)
+    assert(stdout.find(key) > 0)
diff --git a/pylabs/test/server/system_tests_common.py b/pylabs/test/server/system_tests_common.py
index 0d525043..485e5fcc 100644
--- a/pylabs/test/server/system_tests_common.py
+++ b/pylabs/test/server/system_tests_common.py
@@ -647,7 +647,8 @@ def build_node_dir_names ( nodeName, base_dir = None ):
 
 def setup_n_nodes_base(c_id, node_names, force_master,
                        base_dir, base_msg_port, base_client_port,
-                       extra = None, witness_nodes = False, useIPV6=False,
+                       extra = None, nodes_extra = None,
+                       witness_nodes = False, useIPV6=False,
                        slowCollapser = False):
 
     X.subprocess.check_call("sudo /sbin/iptables -F".split(' ') )
@@ -686,6 +687,10 @@ def setup_n_nodes_base(c_id, node_names, force_master,
         cluster.addLocalNode(nodeName)
         cluster.createDirs(nodeName)
 
+        if nodes_extra and nodes_extra[nodeName]:
+            for k,v in nodes_extra.items():
+                config.set(nodeName, k, v)
+
     cluster.disableFsync()
 
     if force_master:
@@ -724,14 +729,15 @@ def setup_n_nodes_base(c_id, node_names, force_master,
     cluster.setMasterLease( lease )
 
 
-def setup_n_nodes ( n, force_master, home_dir , extra = None,
+def setup_n_nodes ( n, force_master, home_dir,
+                    extra = None, nodes_extra = None,
                     witness_nodes = False, useIPV6 = False,
                     slowCollapser = False):
 
     setup_n_nodes_base(cluster_id, node_names[0:n], force_master, data_base_dir,
                        node_msg_base_port, node_client_base_port,
-                       extra = extra, witness_nodes = witness_nodes,
-                       useIPV6 = useIPV6,
+                       extra = extra, nodes_extra = nodes_extra,
+                       witness_nodes = witness_nodes, useIPV6 = useIPV6,
                        slowCollapser = slowCollapser)
 
     logging.info( "Starting cluster" )

From ba7dbda7939b80bdd47194209dfa0457604d0a0e Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Mon, 8 Feb 2016 16:40:15 +0100
Subject: [PATCH 105/373] fiddling so client package still works in alba

---
 META                     | 2 +-
 Makefile                 | 2 ++
 src/arakoon_client.mllib | 1 +
 3 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/META b/META
index fd732960..299fdb83 100644
--- a/META
+++ b/META
@@ -3,7 +3,7 @@
 description = "Arakoon client"
 version = "1.8.13"
 exists_if = "arakoon_client.cma,arakoon_client.cmxa,arakoon_client.cmxs"
-requires = "threads lwt lwt.unix str oUnit quickcheck lwt.ssl ocplib-endian"
+requires = "threads lwt lwt.unix str oUnit quickcheck lwt.ssl ocplib-endian redis.lwt uri"
 archive(byte) = "arakoon_client.cma tcp_keepalive_sockopt_stubs.o"
 archive(native) = "arakoon_client.cmxa tcp_keepalive_sockopt_stubs.o"
 
diff --git a/Makefile b/Makefile
index 34929664..600d0408 100644
--- a/Makefile
+++ b/Makefile
@@ -59,6 +59,8 @@ install_client:
           _build/src/tlog/interval.cmi \
           _build/src/tlog/update.cmi \
           _build/src/tools/ini.cmi \
+          _build/src/tools/arakoon_log_sink.cmi \
+          _build/src/tools/crash_logger.cmi \
           _build/src/tools/llio.mli \
           _build/src/tools/llio.cmi \
           _build/src/tools/network.cmi \
diff --git a/src/arakoon_client.mllib b/src/arakoon_client.mllib
index 80503ecc..59692091 100644
--- a/src/arakoon_client.mllib
+++ b/src/arakoon_client.mllib
@@ -25,6 +25,7 @@ Inilexer
 Client_log
 Simple_store
 Logger
+Arakoon_log_sink
 Crash_logger
 Ring_buffer
 Plugin_helper

From b68772d088d975b30798062831aa04f65bef2b86 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Mon, 8 Feb 2016 16:42:10 +0100
Subject: [PATCH 106/373] fiddle

---
 pylabs/test/server/quick/system_tests_basic.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/pylabs/test/server/quick/system_tests_basic.py b/pylabs/test/server/quick/system_tests_basic.py
index deb3535b..9bf08be6 100644
--- a/pylabs/test/server/quick/system_tests_basic.py
+++ b/pylabs/test/server/quick/system_tests_basic.py
@@ -944,16 +944,16 @@ def test_statistics():
                 assert_not_equals( timing["min"], 0.0,
                     "Wrong value for min timing of %s == 0.0" % k)
 
-def setup_redis_arakoon():
+def setup_redis_arakoon(home_dir):
     X.subprocess.call(["redis-cli"])
-    C.setup_n_nodes(C.cluster_id, node_names, False, C.data_base_dir,
-                    node_msg_base_port, node_client_base_port,
+    C.setup_n_nodes(C.cluster_id, node_names, home_dir,
                     nodes_extra =
                     {"arakoon_0": {"log_sinks":"redis://127.0.0.1/logs/arakoon_0"},
                      "arakoon_1": {"log_sinks":"redis://127.0.0.1/logs/arakoon_1"}})
 
-def redis_arakoon_teardown():
+def redis_arakoon_teardown(removeDirs):
     X.subprocess.call(["pkill", "redis-cli"])
+    C.basic_teardown(removeDirs)
 
 @C.with_custom_setup(setup_redis_arakoon, redis_arakoon_teardown)
 def test_redis_logging():

From dde2e88ce87efcaef52c7d2d5c5e3670001562fe Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Mon, 8 Feb 2016 16:59:21 +0100
Subject: [PATCH 107/373] don't leak redis connections

---
 src/tools/arakoon_redis.ml | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/tools/arakoon_redis.ml b/src/tools/arakoon_redis.ml
index 2c1c6656..6bf892c7 100644
--- a/src/tools/arakoon_redis.ml
+++ b/src/tools/arakoon_redis.ml
@@ -39,8 +39,7 @@ let make_redis_logger ~host ~port ~key =
     let rec outer () =
       Lwt.catch
         (fun () ->
-         R.connect R.({ host ;port; }) >>= fun client ->
-         let rec inner () =
+         let rec inner client =
            (* TODO push multiple items in one go? *)
            if !stop
            then
@@ -49,10 +48,12 @@ let make_redis_logger ~host ~port ~key =
              begin
                Lwt_buffer.take buffer >>= fun logline ->
                R.rpush client key logline >>= fun _list_length ->
-               inner ()
+               inner client
              end
          in
-         inner ())
+         R.with_connection
+           R.({ host ;port; })
+           inner)
         (fun _ -> Lwt.return ()) >>= fun () ->
       if !stop
       then Lwt.return ()

From 108f52125c69a6b1928014533d497b5e7d0487f5 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Mon, 8 Feb 2016 17:03:40 +0100
Subject: [PATCH 108/373] don't spin when redis is not available

---
 src/tools/arakoon_redis.ml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/tools/arakoon_redis.ml b/src/tools/arakoon_redis.ml
index 6bf892c7..df06b9a0 100644
--- a/src/tools/arakoon_redis.ml
+++ b/src/tools/arakoon_redis.ml
@@ -57,7 +57,11 @@ let make_redis_logger ~host ~port ~key =
         (fun _ -> Lwt.return ()) >>= fun () ->
       if !stop
       then Lwt.return ()
-      else outer ()
+      else
+        begin
+          Lwt_unix.sleep 1. >>=
+          outer
+        end
     in
     outer ()
   in

From 191c6ae5eeccd818a1679479e8a53fbf7dfe51de Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Mon, 8 Feb 2016 21:19:27 +0100
Subject: [PATCH 109/373] comments

---
 jenkins/D.sh                                  |  2 +-
 pylabs/test/node.key                          |  1 +
 .../test/server/quick/system_tests_basic.py   |  2 +-
 src/client/client_protocol.ml                 |  6 ++
 src/client/protocol_common.ml                 |  2 +
 src/client/remote_nodestream.ml               | 18 ++--
 src/main/replay_main.ml                       |  3 +-
 src/node/backend.ml                           |  1 +
 src/node/catchup.ml                           | 99 ++++++++++++++++---
 src/node/catchup_test.ml                      |  6 +-
 src/node/collapser.ml                         | 24 +----
 src/node/collapser_test.ml                    | 68 ++++++++++---
 src/node/sync_backend.ml                      |  5 +
 src/system/startup.ml                         |  3 +-
 src/tlog/mem_tlogcollection.ml                |  2 +-
 src/tlog/tlc2.ml                              | 31 ++++--
 src/tlog/tlc2_test.ml                         | 24 +++--
 src/tlog/tlog_map.ml                          | 25 +++--
 src/tlog/tlogcollection.ml                    |  5 +-
 src/tlog/tlogcollection_test.ml               | 34 ++++---
 20 files changed, 253 insertions(+), 108 deletions(-)
 create mode 100644 pylabs/test/node.key

diff --git a/jenkins/D.sh b/jenkins/D.sh
index bd1e733f..dc990117 100755
--- a/jenkins/D.sh
+++ b/jenkins/D.sh
@@ -9,4 +9,4 @@ export PATH=${PATH}:${TEST_HOME}/apps/arakoon/bin
 
 make clean
 make
-python test_it.py -v --with-xunit --xunit-file=${PWD}/testresults.xml ./server/shaky
+python test_it.py -v -s --with-xunit --xunit-file=${PWD}/testresults.xml ./server/shaky
diff --git a/pylabs/test/node.key b/pylabs/test/node.key
new file mode 100644
index 00000000..8773f398
--- /dev/null
+++ b/pylabs/test/node.key
@@ -0,0 +1 @@
+file content
\ No newline at end of file
diff --git a/pylabs/test/server/quick/system_tests_basic.py b/pylabs/test/server/quick/system_tests_basic.py
index ef339134..dd44a00e 100644
--- a/pylabs/test/server/quick/system_tests_basic.py
+++ b/pylabs/test/server/quick/system_tests_basic.py
@@ -725,7 +725,7 @@ def test_close_on_sigterm():
         tail = lines[-40:]
         found = False
         for l in tail:
-            print l
+            print l.strip()
             if l.find("fatal") > 0 and l.find ("OK") >0:
                 found = True
                 break
diff --git a/src/client/client_protocol.ml b/src/client/client_protocol.ml
index 6d5f6a8c..b7f0f806 100644
--- a/src/client/client_protocol.ml
+++ b/src/client/client_protocol.ml
@@ -327,6 +327,12 @@ let one_command stop (ic,oc,id as conn) (backend:Backend.backend) =
       backend # last_entries2 i oc >>= fun () ->
       Lwt.return false
     end
+  | LAST_ENTRIES3 ->
+     Sn.input_sn ic >>= fun i ->
+      Logger.debug_f_ "connection=%s LAST_ENTRIES2: i=%Li" id i >>= fun () ->
+      response_ok oc >>= fun () ->
+      backend # last_entries3 i oc >>= fun () ->
+      Lwt.return false
   | WHO_MASTER ->
     begin
       Logger.debug_f_ "connection=%s WHO_MASTER" id >>= fun () ->
diff --git a/src/client/protocol_common.ml b/src/client/protocol_common.ml
index 8e1da0d1..ae3b9b93 100644
--- a/src/client/protocol_common.ml
+++ b/src/client/protocol_common.ml
@@ -83,6 +83,7 @@ type client_command =
   | GET_TXID
   | COPY_DB_TO_HEAD
   | USER_HOOK
+  | LAST_ENTRIES3
 
 
 let code2int = [
@@ -133,6 +134,7 @@ let code2int = [
   GET_TXID                , 0x43l;
   COPY_DB_TO_HEAD         , 0x44l;
   USER_HOOK               , 0x45l;
+  LAST_ENTRIES3           , 0x46l;
 ]
 
 let int2code =
diff --git a/src/client/remote_nodestream.ml b/src/client/remote_nodestream.ml
index 450b5d13..95705c8a 100644
--- a/src/client/remote_nodestream.ml
+++ b/src/client/remote_nodestream.ml
@@ -57,7 +57,7 @@ class remote_nodestream ((ic,oc) as conn) =
     ~head_saved_cb
     =
     let outgoing buf =
-      command_to buf LAST_ENTRIES2;
+      command_to buf LAST_ENTRIES3;
       Sn.sn_to buf i
     in
     let incoming ic =
@@ -83,19 +83,12 @@ class remote_nodestream ((ic,oc) as conn) =
             end
         end
       in
-      let _should_reinit = ref false in
       let rec loop_parts () =
         Llio.input_int ic >>= function
         | (-2) -> Logger.info_f_ "loop_parts done"
         | 1 ->
           begin
             Logger.debug_f_ "loop_entries" >>= fun () ->
-            begin
-              if !_should_reinit
-              then tlog_coll # reinit()
-              else Lwt.return ()
-            end
-            >>= fun () ->
             loop_entries () >>= fun () ->
             loop_parts ()
           end
@@ -109,11 +102,11 @@ class remote_nodestream ((ic,oc) as conn) =
           end
         | 3 ->
            begin
-             let () = _should_reinit := true in
              Logger.debug_f_ "save_file" >>= fun () ->
              Llio.input_string ic >>= fun name ->
-             Llio.input_int64 ic >>= fun length ->
-             Logger.info_f_ "got %s (%Li bytes)" name length >>= fun () ->
+             Sn.input_sn ic       >>= fun start_i ->
+             Llio.input_int64 ic  >>= fun length ->
+             Logger.info_f_ "retrieving %s (%Li bytes)" name length >>= fun () ->
              tlog_coll # save_tlog_file name length ic >>= fun () ->
              loop_parts ()
            end
@@ -132,7 +125,7 @@ class remote_nodestream ((ic,oc) as conn) =
     in
     let incoming ic =
       Llio.input_int ic >>= fun collapse_count ->
-      Logger.debug_f_ "collapse_count:%i" collapse_count >>= fun () ->
+      Logger.info_f_ "collapse_count:%i" collapse_count >>= fun () ->
       let rec loop i =
         if i = 0
         then Lwt.return ()
@@ -142,6 +135,7 @@ class remote_nodestream ((ic,oc) as conn) =
             | 0 ->
                Llio.input_int64 ic >>= fun total ->
                Logger.debug_f_ "%i:collapsed one file. (total = %Li)" i total >>= fun () ->
+                                    
                loop (i-1)
             | e ->
                Llio.input_string ic >>= fun msg ->
diff --git a/src/main/replay_main.ml b/src/main/replay_main.ml
index 24c911d3..a46aee50 100644
--- a/src/main/replay_main.ml
+++ b/src/main/replay_main.ml
@@ -79,10 +79,11 @@ let replay_tlogs tlog_dir tlx_dir db_name end_i =
              let acc = ref None in
              let log_i pi = maybe_log_i start_i too_far_i pi in
              let f = Catchup.make_f ~stop:(ref false) (module S) db_name log_i acc store in
+             let cb _ = Lwt.return_unit in
              TlogMap.iterate_tlog_dir tlog_map ~index:None
                start_i
                too_far_i
-               f
+               f cb
              >>= fun () ->
              Catchup.epilogue (module S) db_name acc store >>= fun () ->
              S.close store
diff --git a/src/node/backend.ml b/src/node/backend.ml
index ed3b0454..7a17ad53 100644
--- a/src/node/backend.ml
+++ b/src/node/backend.ml
@@ -53,6 +53,7 @@ class type backend = object
     consistency:consistency -> string -> int -> (Key.t counted_list)
   method last_entries : Sn.t ->Lwt_io.output_channel -> unit Lwt.t
   method last_entries2: Sn.t ->Lwt_io.output_channel -> unit Lwt.t
+  method last_entries3: Sn.t ->Lwt_io.output_channel -> unit Lwt.t
 
   method multi_get:
     consistency:consistency ->
diff --git a/src/node/catchup.ml b/src/node/catchup.ml
index 1d93e4ce..3d0de525 100644
--- a/src/node/catchup.ml
+++ b/src/node/catchup.ml
@@ -195,7 +195,10 @@ let catchup_store ~stop (type s) me
           Lwt.return ()
       in
       let f = make_f ~stop (module S) me maybe_log_progress acc store in
-      tlog_coll # iterate start_i too_far_i f >>= fun () ->
+      let cb _ = Lwt.return_unit in
+      tlog_coll # iterate start_i too_far_i
+                f cb
+      >>= fun () ->
       epilogue (module S) me acc store >>= fun () ->
       let store_i' = S.consensus_i store in
       Logger.info_f_ "catchup_store completed, store is @ %s"
@@ -256,18 +259,18 @@ let verify_n_catchup_store
   >>= fun () ->
 
   match too_far_i, si_o with
-    | i, None when i <= 0L -> Lwt.return ()
-    | i, Some j when i = j -> Lwt.return ()
-    | i, Some j when i > j -> catchup_store ~stop me ((module S),store,tlog_coll) too_far_i
-    | _, None              -> catchup_store ~stop me ((module S),store,tlog_coll) too_far_i
-    | _,_ ->
-      let msg = Printf.sprintf
-        "too_far_i:%s, si_o:%s should not happen: tlogs have been removed?"
-        (Sn.string_of too_far_i) (io_s si_o)
-      in
-      Logger.fatal_ msg >>= fun () ->
-      let maybe a = function | None -> a | Some b -> b in
-      Lwt.fail (StoreAheadOfTlogs(maybe (-1L) si_o, too_far_i))
+  | i, None when i <= 0L -> Lwt.return ()
+  | i, Some j when i = j -> Lwt.return ()
+  | i, Some j when i > j -> catchup_store ~stop me ((module S),store,tlog_coll) too_far_i
+  | _, None              -> catchup_store ~stop me ((module S),store,tlog_coll) too_far_i
+  | _,_ ->
+     let msg = Printf.sprintf
+                 "too_far_i:%s, si_o:%s should not happen: tlogs have been removed?"
+                 (Sn.string_of too_far_i) (io_s si_o)
+     in
+     Logger.fatal_ msg >>= fun () ->
+     let maybe a = function | None -> a | Some b -> b in
+     Lwt.fail (StoreAheadOfTlogs(maybe (-1L) si_o, too_far_i))
 
 let last_entries
       (type s) (module S : Store.STORE with type t = s)
@@ -285,6 +288,71 @@ let last_entries2
   =
   Logger.debug_f_ "last_entries2 %s" (Sn.string_of start_i) >>= fun () ->
   let consensus_i = S.consensus_i store in
+  (* This version is kept for mixed setups were non-flexible nodes
+     are catching up from flexible ones 
+   *)
+  begin
+    match consensus_i with
+    | None -> Lwt.return ()
+    | Some ci ->
+       begin
+         let stream_entries start_i too_far_i =
+           Logger.debug_f_ "stream_entries :%s %s" (Sn.string_of start_i) (Sn.string_of too_far_i)
+           >>= fun () ->
+           let _write_entry entry =
+             let i = Entry.i_of entry
+             and v = Entry.v_of entry
+             in
+             Logger.debug_f_ "write_entry:%s" (Sn.string_of i) >>= fun () ->
+             Tlogcommon.write_entry oc i v >>= fun _total_size ->
+             Lwt.return_unit
+           in
+           let _cb _ = Lwt.return_unit in
+           Llio.output_int oc 1 >>= fun () ->
+           Lwt.catch
+             (fun () -> tlog_collection # iterate start_i too_far_i _write_entry _cb)
+             (function
+              | Tlogcommon.TLogUnexpectedEndOfFile _ -> Lwt.return ()
+              | ex -> Lwt.fail ex)
+           >>= fun () ->
+           Sn.output_sn oc (-1L)
+         in
+         let too_far_i = Sn.succ ci in
+         let maybe_dump_head () =
+           tlog_collection # get_infimum_i () >>= fun inf_i ->
+           Logger.debug_f_
+             "inf_i:%s too_far_i:%s" (Sn.string_of inf_i)
+             (Sn.string_of too_far_i)
+           >>= fun () ->
+           begin
+             if start_i < inf_i
+             then
+               begin
+                 Llio.output_int oc 2 >>= fun () ->
+                 tlog_collection # dump_head oc
+               end
+             else
+               Lwt.return start_i
+           end
+         in
+         begin
+           maybe_dump_head () >>= fun start_i2->
+           stream_entries start_i2 too_far_i
+         end
+         >>= fun () ->
+         Sn.output_sn oc (-2L) >>= fun () ->
+         Logger.info_f_ "done with_last_entries2"
+       end
+  end
+         
+let last_entries3
+      (type s) (module S : Store.STORE with type t = s)
+      store
+      (tlog_collection:Tlogcollection.tlog_collection)
+      (start_i:Sn.t) (oc:Lwt_io.output_channel)
+  =
+  Logger.debug_f_ "last_entries3 %s" (Sn.string_of start_i) >>= fun () ->
+  let consensus_i = S.consensus_i store in
 
   begin
     match consensus_i with
@@ -301,9 +369,12 @@ let last_entries2
             Tlogcommon.write_entry oc i v >>= fun _total_size ->
             Lwt.return_unit
           in
+          let _cb _ = Lwt.return_unit in
           Llio.output_int oc 1 >>= fun () ->
           Lwt.catch
-            (fun () -> tlog_collection # iterate start_i too_far_i _write_entry)
+            (fun () -> tlog_collection # iterate start_i too_far_i
+                                       _write_entry _cb
+            )
             (function
               | Tlogcommon.TLogUnexpectedEndOfFile _ -> Lwt.return ()
               | ex -> Lwt.fail ex)
diff --git a/src/node/catchup_test.ml b/src/node/catchup_test.ml
index bcd3e164..6c313e0a 100644
--- a/src/node/catchup_test.ml
+++ b/src/node/catchup_test.ml
@@ -137,10 +137,10 @@ let teardown () =
   Logger.debug_ "end of teardown"
 
 let _tic filler_function n name verify_store =
-  let open Tlog_map in
   let node_id = "node_name" in
-  TlogMap.make ~tlog_max_entries:101 _dir_name _tlx_dir node_id ~check_marker:true >>= fun tlog_map ->
-  Tlc2.make_tlc2 ~compressor:Compression.Snappy  _dir_name _tlx_dir _tlx_dir
+  Tlc2.make_tlc2 ~compressor:Compression.Snappy
+                 ~tlog_max_entries:101
+                 _dir_name _tlx_dir _tlx_dir
                  ~fsync:false node_id ~fsync_tlog_dir:true >>= fun tlog_coll ->
   filler_function tlog_coll n >>= fun () ->
   let db_name = _dir_name ^ "/" ^ name ^ ".db" in
diff --git a/src/node/collapser.ml b/src/node/collapser.ml
index ce9e1521..3f07def9 100644
--- a/src/node/collapser.ml
+++ b/src/node/collapser.ml
@@ -56,10 +56,8 @@ let collapse_until (type s) (tlog_coll:Tlogcollection.tlog_collection)
   Lwt.finalize (
     fun () ->
       tlog_coll # get_infimum_i () >>= fun min_i ->
-      let first_tlog = tlog_coll # get_tlog_from_i  min_i in
       let store_i = S.consensus_i new_store in
       let tfs = Sn.string_of too_far_i in
-      let processed = ref 0 in
       let start_i =
         begin
           match store_i with
@@ -102,7 +100,7 @@ let collapse_until (type s) (tlog_coll:Tlogcollection.tlog_collection)
                Lwt_unix.sleep period >>= fun () ->
                Lwt.return r
           in
-          let get_next_point i =
+          (* let get_next_point i =
               let next_r = tlog_coll # next_rollover i in
               match next_r with
                 | None -> Sn.zero
@@ -110,6 +108,7 @@ let collapse_until (type s) (tlog_coll:Tlogcollection.tlog_collection)
           in
           let next_rollover_point = ref (get_next_point (Sn.pred start_i))
           in
+           *)
           let add_to_store entry =
             let i = Entry.i_of entry
             and value = Entry.v_of entry
@@ -126,22 +125,7 @@ let collapse_until (type s) (tlog_coll:Tlogcollection.tlog_collection)
                 | Some (pi,pv) ->
                   if pi < i then
                     begin
-                      maybe_log pi >>= fun () ->
-                      begin
-                        if pi = !next_rollover_point
-                        then
-                          begin
-                            Logger.info_f_ "rollover_point pi:%Li next_rollover_point =%s"
-                                          pi (Sn.string_of !next_rollover_point)
-                            >>= fun () ->
-                            cb (first_tlog + !processed) >>= fun () ->
-                            let () = incr processed in
-                            let () = next_rollover_point := get_next_point (Sn.succ pi) in
-                            Lwt.return_unit
-                          end
-                        else
-                          Lwt.return_unit
-                      end
+                      maybe_log pi 
                       >>= fun () ->
                       slowdown (fun () -> S.safe_insert_value new_store pi pv) >>= fun _ ->
                       let () = acc := Some(i,value) in
@@ -157,7 +141,7 @@ let collapse_until (type s) (tlog_coll:Tlogcollection.tlog_collection)
           in
           Logger.debug_f_ "collapse_until: start_i=%s" (Sn.string_of start_i)
           >>= fun () ->
-          tlog_coll # iterate start_i too_far_i add_to_store
+          tlog_coll # iterate start_i too_far_i add_to_store cb
           >>= fun () ->
           let m_si = S.consensus_i new_store in
           let si =
diff --git a/src/node/collapser_test.ml b/src/node/collapser_test.ml
index 464bc255..a19c6d80 100644
--- a/src/node/collapser_test.ml
+++ b/src/node/collapser_test.ml
@@ -38,7 +38,7 @@ let _should_fail x error_msg success_msg =
   if bad then Lwt.fail (Failure error_msg)
   else Lwt.return ()
 
-let _make_values tlc n =
+let _make_values tlc start_i n =
   let sync = false in
   let rec loop i =
     if i = n
@@ -53,7 +53,7 @@ let _make_values tlc n =
       tlc # log_value sni value >>= fun _wr_result ->
       loop (i+1)
   in
-  loop 0
+  loop start_i
 
 let test_collapse_until (dn, tlx_dir, head_dir) =
   let node_id = "node_name" in
@@ -63,7 +63,7 @@ let test_collapse_until (dn, tlx_dir, head_dir) =
                  dn tlx_dir head_dir
                  ~fsync:false node_id ~fsync_tlog_dir:false
   >>= fun tlc ->
-  _make_values tlc 1111 >>= fun () ->
+  _make_values tlc 0 1111 >>= fun () ->
   tlc # close () >>= fun () ->
   Lwt_unix.sleep 5.0 >>= fun () -> (* give it time to generate the .tlc *)
   (* now collapse first file into a tc *)
@@ -87,9 +87,7 @@ let test_collapse_until (dn, tlx_dir, head_dir) =
   Lwt.return ()
 
 
-let test_dn = "/tmp/collapser"
-let _tlf_dir = "/tmp/collapser_tlf"
-let _head_dir = "/tmp/collapser_head"
+
 
 let test_collapse_many (dn, tlx_dir, head_dir) =
   let node_id = "node_name" in
@@ -101,11 +99,11 @@ let test_collapse_many (dn, tlx_dir, head_dir) =
                  ~fsync:false node_id ~fsync_tlog_dir:false
   in
   make_tlc () >>= fun tlc ->
-  _make_values tlc 632 >>= fun () ->
+  _make_values tlc 0 632 >>= fun () ->
   tlc # close () ~wait_for_compression:true >>= fun () ->
   
   make_tlc () >>= fun tlc ->
-  let storename = Filename.concat test_dn "head.db" in
+  let storename = Filename.concat dn "head.db" in
   let cb n = Logger.debug_f_ "collapsed %03i" n in
   let cb' = fun _ -> Lwt.return () in
   File_system.unlink storename >>= fun () ->
@@ -118,21 +116,60 @@ let test_collapse_many (dn, tlx_dir, head_dir) =
   Logger.debug_ "collapsed 003 & 004" >>= fun () -> (* ends @ 510 *)
   Lwt.return ()
 
-
+let test_repeated_collapse (dn,tlx_dir, head_dir) =
+  let node_id = "node_name" in
+  Logger.debug_f_ "test_collapse_many dn=%s, tlx_dir=%s, head_dir=%s" dn tlx_dir head_dir
+  >>= fun () ->
+  Tlc2.make_tlc2 ~compressor ~tlog_max_entries:100
+                 dn tlx_dir head_dir
+                 ~fsync:false node_id ~fsync_tlog_dir:false
+  >>= fun tlc ->
+  let head_location= Filename.concat dn "head.db" in
+  let count = ref 0 in
+  let cb' x =
+    Logger.debug_f_ "cb':%i" x >>= fun () ->
+    count := x;
+    Lwt.return_unit
+  in
+  let cb  n =
+    Logger.debug_f_ "cb:collapsed %03i" n >>= fun () ->
+    decr count ;
+    Lwt.return_unit
+  in
+  
+  let store_methods = (Batched_store.Local_store.copy_store2, head_location, 0.0) in
+  File_system.unlink head_location >>= fun () ->
+
+  _make_values tlc 0 520 >>= fun () ->
+  let collapse_slowdown = None in
+  Collapser.collapse_many tlc (module S) store_methods 1 cb' cb collapse_slowdown >>= fun () ->
+  Logger.debug_f_"post collapse: count=%i" !count >>= fun () ->
+  OUnit.assert_equal 0 !count ~printer:string_of_int ~msg:"count should be zero";
+  _make_values tlc 520 1040 >>= fun () ->
+  Lwt_unix.sleep 10.0 >>= fun () ->
+  Collapser.collapse_many tlc (module S) store_methods 1 cb' cb collapse_slowdown >>= fun () ->
+  Logger.debug_f_"post collapse: count=%i" !count >>= fun () ->
+  OUnit.assert_equal 0 !count ~printer:string_of_int ~msg:"count should be zero";
+  tlc # close ~wait_for_compression:true () >>= fun () ->
+  Lwt.return()
+  
 let setup () =
   Logger.info_ "Collapser_test.setup" >>= fun () ->
-
+  let test_dn = "/tmp/collapser" 
+  and _tlx_dir = "/tmp/collapser_tlx"
+  and _head_dir = "/tmp/collapser_head"
+  in
   let _ = Sys.command (Printf.sprintf "rm -rf '%s'" test_dn) in
-  let _ = Sys.command (Printf.sprintf "rm -rf '%s'" _tlf_dir) in
+  let _ = Sys.command (Printf.sprintf "rm -rf '%s'" _tlx_dir) in 
   let _ = Sys.command (Printf.sprintf "rm -rf '%s'" _head_dir) in
   File_system.mkdir test_dn 0o755 >>= fun () ->
-  File_system.mkdir _tlf_dir 0o755 >>= fun () ->
+  File_system.mkdir _tlx_dir 0o755 >>= fun () ->
   File_system.mkdir _head_dir 0o755 >>= fun () ->
-  Lwt.return (test_dn, _tlf_dir, _head_dir)
+  Lwt.return (test_dn, _tlx_dir, _head_dir)
 
 
-let teardown (dn, tlf_dir, head_dir) =
-  Logger.debug_f_ "teardown %s, %s, %s" dn tlf_dir head_dir
+let teardown (dn, tlx_dir, head_dir) =
+  Logger.debug_f_ "teardown %s, %s, %s" dn tlx_dir head_dir
 
 
 let suite =
@@ -141,4 +178,5 @@ let suite =
   "collapser_test" >:::[
     "collapse_until" >:: wrapTest test_collapse_until;
     "collapse_many" >:: wrapTest test_collapse_many;
+    "repeated_collapse" >:: wrapTest test_repeated_collapse;
   ]
diff --git a/src/node/sync_backend.ml b/src/node/sync_backend.ml
index db3a1bfa..2a75c3ba 100644
--- a/src/node/sync_backend.ml
+++ b/src/node/sync_backend.ml
@@ -200,6 +200,11 @@ struct
           (fun () ->
              Catchup.last_entries2 (module S) store tlog_collection start_i oc
           )
+      method last_entries3 (start_i:Sn.t) (oc:Lwt_io.output_channel) =
+        self # with_blocked_collapser start_i
+          (fun () ->
+             Catchup.last_entries3 (module S) store tlog_collection start_i oc
+          )
 
       method range_entries ~consistency
                (first:string option) finc (last:string option) linc max =
diff --git a/src/system/startup.ml b/src/system/startup.ml
index 0c75e447..57a8a2be 100644
--- a/src/system/startup.ml
+++ b/src/system/startup.ml
@@ -137,7 +137,8 @@ let _dump_tlc ~tlcs node =
     Logger.debug_f_ "%s:%s" (Sn.string_of i) (Value.value2s v)
   in
   Logger.debug_f_ "--- %s ---" node >>= fun () ->
-  tlc0 # iterate Sn.start 20L printer >>= fun () ->
+  let cb _ = Lwt.return_unit in
+  tlc0 # iterate Sn.start 20L printer cb >>= fun () ->
   Lwt.return ()
 
 
diff --git a/src/tlog/mem_tlogcollection.ml b/src/tlog/mem_tlogcollection.ml
index e570e61e..0599512b 100644
--- a/src/tlog/mem_tlogcollection.ml
+++ b/src/tlog/mem_tlogcollection.ml
@@ -53,7 +53,7 @@ class mem_tlog_collection _name =
         | Some e -> Some (Entry.v_of e, Entry.i_of e)
 
 
-    method iterate from_i too_far_i f =
+    method iterate from_i too_far_i f _cb =
       let data' =
         List.filter
           (fun entry ->
diff --git a/src/tlog/tlc2.ml b/src/tlog/tlc2.ml
index 0d82a9e7..1284d1f8 100644
--- a/src/tlog/tlc2.ml
+++ b/src/tlog/tlc2.ml
@@ -354,10 +354,14 @@ class tlc2
 
 
 
-    method iterate (start_i:Sn.t) (too_far_i:Sn.t) (f:Entry.t -> unit Lwt.t) =
+    method iterate (start_i:Sn.t) (too_far_i:Sn.t)
+                   (f:Entry.t -> unit Lwt.t)
+                   (cb : int -> unit Lwt.t)
+      =
       let index = _index in
       Logger.debug_f_ "tlc2.iterate : index=%s" (Index.to_string index) >>= fun () ->
-      TlogMap.iterate_tlog_dir tlog_map ~index start_i too_far_i f
+      TlogMap.iterate_tlog_dir tlog_map ~index start_i too_far_i 
+                               f cb
 
 
     method get_infimum_i () = TlogMap.infimum tlog_map
@@ -507,7 +511,9 @@ class tlc2
       in
       Logger.debug_f_ "n = %03i => canonical=%s" n canonical >>= fun () ->
       let name = Filename.basename canonical in
+      let start_i = TlogMap.get_start_i tlog_map n in
       Llio.output_string oc name >>= fun () ->
+      Sn.output_sn oc start_i    >>= fun () ->
       File_system.stat canonical >>= fun stats ->
       let length = Int64.of_int(stats.Unix.st_size)
       (* why don't they use largefile ? *)
@@ -521,21 +527,30 @@ class tlc2
       >>= fun () ->
       Logger.debug_f_ "dumped:%s (%Li) bytes" canonical length
 
-    method save_tlog_file name length ic =
+    method save_tlog_file remote_name length ic =
       (* what with rotation (jump to new tlog), open streams, ...*)
       begin
         match _file with
         | None -> Lwt.return ()
-        | Some file -> F.close file >>= fun () -> _file <- None;Lwt.return ()
+        | Some file ->
+           F.close file >>= fun () -> _file <- None;
+           Lwt.return ()
       end 
       >>= fun () ->                                                               
-      let canon = TlogMap.get_full_path tlog_map name in
-      let tmp = canon ^ ".tmp" in
-      Logger.debug_f_ "save_tlog_file: %s" tmp >>= fun () ->
+      let full_name = TlogMap.get_full_path tlog_map remote_name in
+      let extension = Tlog_map.extension_of remote_name in
+      let tmp = full_name ^ ".tmp" in
+      Logger.info_f_ "save_tlog_file: remote_name: %s" tmp >>= fun () ->
       Lwt_io.with_file ~mode:Lwt_io.output tmp
                        (fun oc -> Llio.copy_stream ~length ~ic ~oc)
       >>= fun () ->
-      File_system.rename tmp canon
+      Tlog_map.first_i_of ~extension_override:extension tmp >>= fun first_i ->
+      let new_outer = TlogMap.new_outer tlog_map first_i in
+      let new_name = Printf.sprintf "%03i%s" new_outer extension in
+      let canon = TlogMap.get_full_path tlog_map new_name in
+      File_system.rename tmp canon >>= fun () ->
+      let () = TlogMap.set_should_roll tlog_map in
+      Lwt.return_unit
 
 
     method reinit () =
diff --git a/src/tlog/tlc2_test.ml b/src/tlog/tlc2_test.ml
index 5af1928a..691c8012 100644
--- a/src/tlog/tlc2_test.ml
+++ b/src/tlog/tlc2_test.ml
@@ -143,7 +143,9 @@ let test_iterate5 (dn, _tlx_dir, (factory:factory)) =
     Logger.debug_f_ "test_iterate5: %s %s" (Sn.string_of i)
       (Value.value2s v)
   in
-  tlc # iterate start_i too_far_i f >>= fun () ->
+  let cb x = Logger.debug_f_ "rotation_point:%i" x in
+  tlc # iterate start_i too_far_i f cb
+  >>= fun () ->
   Lwt.return ()
 
 let test_iterate6 (dn, _tlx_dir, (factory:factory)) =
@@ -176,7 +178,7 @@ let test_iterate6 (dn, _tlx_dir, (factory:factory)) =
   let sum = ref 0 in
   let start_i = Sn.of_int 19 in
   let too_far_i = Sn.of_int 20 in
-  tlc # iterate start_i too_far_i
+  let f =
     (fun entry ->
        let i = Entry.i_of entry in
        let v = Entry.v_of entry in
@@ -184,6 +186,9 @@ let test_iterate6 (dn, _tlx_dir, (factory:factory)) =
        Logger.debug_f_ "i=%s : %s" (Sn.string_of i) (Value.value2s v)
        >>= fun () ->
        Lwt.return ())
+  in
+  let cb _ = Lwt.return_unit in
+  tlc # iterate start_i too_far_i f cb
   >>= fun () ->
   tlc # close () >>= fun () ->
   Logger.debug_f_ "sum =%i " !sum >>= fun () ->
@@ -215,12 +220,15 @@ let test_compression_bug (dn, tlx_dir, (factory:factory)) =
   OUnit.assert_bool "file should have size >0" (stat.st_size > 0);
   let entries = ref [] in
   factory ~tlog_max_entries dn node_id >>= fun tlc2 ->
-  tlc2 # iterate 0L (Sn.of_int n)
-    (fun entry ->
-       let i = Entry.i_of entry in
-       entries := i :: !entries;
-       Logger.debug_f_ "ENTRY: i=%Li" i)
-  >>= fun () ->
+  let f entry =
+    let i = Entry.i_of entry in
+    entries := i :: !entries;
+    Logger.debug_f_ "ENTRY: i=%Li" i
+  in
+  let cb _ = Lwt.return_unit in
+  
+  tlc2 # iterate 0L (Sn.of_int n) f cb >>= fun () ->
+
   OUnit.assert_equal
     ~printer:string_of_int
     ~msg:"tlog_collection has a hole" (n+2) (List.length !entries);
diff --git a/src/tlog/tlog_map.ml b/src/tlog/tlog_map.ml
index edab44c5..92504e1b 100644
--- a/src/tlog/tlog_map.ml
+++ b/src/tlog/tlog_map.ml
@@ -1,3 +1,4 @@
+
 open Lwt.Infix
 open Tlogcommon
 open Tlogreader2
@@ -30,8 +31,11 @@ let is_archive filename =
   | ".tlx" | ".tlf" | ".tlc"-> true
   | _ -> false
   
-let first_i_of filename =
-  let extension = extension_of filename in
+let first_i_of ?extension_override filename =
+  let extension = match extension_override with
+    | None -> extension_of filename
+    | Some ext ->  ext
+  in
   let read_uncompress ic inflate =
     Llio.input_string ic >>= fun compressed ->
     let uncompressed = inflate compressed in
@@ -433,13 +437,15 @@ module TlogMap = struct
            find rest
     in
     let r = find t.i_to_tlog_number in
-    Logger.ign_debug_f_ "get_start_i %s %i => %s"  (show_map t) n (Sn.string_of r) ;
+    (* Logger.ign_debug_f_ "get_start_i %s %i => %s"  (show_map t) n (Sn.string_of r) ;*)
     r
          
   
 
   let should_roll t = t.should_roll
- 
+
+  let set_should_roll t = t.should_roll <- true
+                                             
   let new_outer t i =
     let n = t.tlog_number + 1 in
     let () = t.tlog_number <- n in
@@ -534,14 +540,14 @@ module TlogMap = struct
       | _ :: rest -> find rest
     in
     let r = find t.i_to_tlog_number in
-    let () =
+    (*let () =
       Logger.ign_debug_f_
         "next_rollover %s : %s => %s" (Sn.string_of i)
         (show_map t) (To_string.option Sn.string_of r)
-    in
+    in *)
     r
 
-  let iterate_tlog_dir t ~index start_i too_far_i f =
+  let iterate_tlog_dir t ~index start_i too_far_i f (cb: int -> unit Lwt.t) =
     let open Tlogcommon in
     let open Tlogreader2 in
     let tfs = Sn.string_of too_far_i in
@@ -580,9 +586,12 @@ module TlogMap = struct
            Logger.info_f_ "Completed replay of %s, took %f seconds, %i to go"
                           fn (Unix.gettimeofday () -. t1) (num_tlogs - cnt)
            >>= fun () ->
+           cb cnt >>= fun () ->
            Lwt.return (cnt+1, low')
       in
-      if should_fold then fold () else Lwt.return (cnt+1,low)
+      if should_fold
+      then fold ()
+      else Lwt.return (cnt+1,low)
     in
     Lwt_list.fold_left_s maybe_fold (1,start_i) tlog_names
     >>= fun _x ->
diff --git a/src/tlog/tlogcollection.ml b/src/tlog/tlogcollection.ml
index bfb8d8e3..d117ba3c 100644
--- a/src/tlog/tlogcollection.ml
+++ b/src/tlog/tlogcollection.ml
@@ -20,7 +20,10 @@ open Tlogreader2
 class type tlog_collection =
   object
   method validate_last_tlog: unit -> (tlogValidity * Entry.t option * Index.index) Lwt.t
-  method iterate: Sn.t -> Sn.t -> (Entry.t -> unit Lwt.t) -> unit Lwt.t
+  method iterate: Sn.t -> Sn.t
+                  -> (Entry.t -> unit Lwt.t)
+                  -> (int -> unit Lwt.t)
+                  -> unit Lwt.t
   method accept : Sn.t -> Value.t    -> int Lwt.t
   method log_value : Sn.t -> Value.t -> int Lwt.t
   method log_value_explicit : Sn.t -> Value.t ->
diff --git a/src/tlog/tlogcollection_test.ml b/src/tlog/tlogcollection_test.ml
index 5e2aa607..8602eb32 100644
--- a/src/tlog/tlogcollection_test.ml
+++ b/src/tlog/tlogcollection_test.ml
@@ -152,12 +152,14 @@ let test_iterate (dn, tlx_dir, (factory: factory)) =
   let value = _make_set_v "xxx" "y" in
   _log_repeat tlc value 323 >>= fun () ->
   let sum = ref 0 in
-  tlc # iterate (Sn.of_int 125) (Sn.of_int 304)
-    (fun entry ->
+  let cb _ = Lwt.return_unit in
+  let f entry =
        let i = Entry.i_of entry in
        sum := !sum + (Int64.to_int i);
        Logger.debug_f_ "i=%s" (Sn.string_of i) >>= fun () ->
-       Lwt.return ())
+       Lwt.return ()
+  in
+  tlc # iterate (Sn.of_int 125) (Sn.of_int 304) f cb
   >>= fun () ->
   tlc # close () >>= fun () ->
   Logger.debug_f_ "sum =%i " !sum >>= fun () ->
@@ -172,12 +174,15 @@ let test_iterate2 (dn, tlx_dir, (factory:factory)) =
   let value = _make_set_v "test_iterate0" "xxx" in
   _log_repeat tlc value 3 >>= fun () ->
   let result = ref [] in
-  tlc # iterate (Sn.of_int 0) (Sn.of_int 1)
-    (fun entry ->
-       let i = Entry.i_of entry in
+  let f entry =
+    let i = Entry.i_of entry in
        result := i :: ! result;
        Logger.debug_f_ "i=%s" (Sn.string_of i) >>= fun () ->
-       Lwt.return ())
+       Lwt.return ()
+  in
+  let cb _ = Lwt.return_unit in
+  tlc # iterate (Sn.of_int 0) (Sn.of_int 1) f cb
+    
   >>= fun () ->
   OUnit.assert_equal ~printer:string_of_int 1 (List.length !result);
   tlc # close () >>= fun () ->
@@ -191,13 +196,14 @@ let test_iterate3 (dn, tlx_dir, (factory:factory)) =
   let value = _make_set_v "test_iterate3" "xxx" in
   _log_repeat tlc value 120 >>= fun () ->
   let result = ref [] in
-  tlc # iterate (Sn.of_int 99) (Sn.of_int 101)
-    (fun entry ->
-       let i = Entry.i_of entry in
-       Logger.debug_f_ "i=%s" (Sn.string_of i) >>= fun () ->
-       let () = result := i :: !result in
-       Lwt.return ()
-    )
+  let f entry =
+    let i = Entry.i_of entry in
+    Logger.debug_f_ "i=%s" (Sn.string_of i) >>= fun () ->
+    let () = result := i :: !result in
+    Lwt.return ()
+  in
+  let cb _ = Lwt.return_unit in
+  tlc # iterate (Sn.of_int 99) (Sn.of_int 101) f cb
   >>= fun () ->
   OUnit.assert_equal (List.mem (Sn.of_int 99) !result) true;
   tlc # close () >>= fun () ->

From 58346b536c27ee92ab21921ecedd4df124ab8d77 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Tue, 9 Feb 2016 09:42:24 +0100
Subject: [PATCH 110/373] fix redis test

---
 pylabs/test/server/quick/system_tests_basic.py | 14 ++++++++------
 pylabs/test/server/system_tests_common.py      |  7 +++----
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/pylabs/test/server/quick/system_tests_basic.py b/pylabs/test/server/quick/system_tests_basic.py
index 9bf08be6..b52dad23 100644
--- a/pylabs/test/server/quick/system_tests_basic.py
+++ b/pylabs/test/server/quick/system_tests_basic.py
@@ -945,14 +945,14 @@ def test_statistics():
                     "Wrong value for min timing of %s == 0.0" % k)
 
 def setup_redis_arakoon(home_dir):
-    X.subprocess.call(["redis-cli"])
-    C.setup_n_nodes(C.cluster_id, node_names, home_dir,
+    proc = subprocess.Popen(["redis-server"], stdout=subprocess.PIPE, stderr = subprocess.PIPE)
+    C.setup_n_nodes(3, False, home_dir,
                     nodes_extra =
-                    {"arakoon_0": {"log_sinks":"redis://127.0.0.1/logs/arakoon_0"},
-                     "arakoon_1": {"log_sinks":"redis://127.0.0.1/logs/arakoon_1"}})
+                    {"sturdy_0": {"log_sinks":"redis://127.0.0.1/logs/sturdy_0"},
+                     "sturdy_1": {"log_sinks":"redis://127.0.0.1/logs/sturdy_1"}})
 
 def redis_arakoon_teardown(removeDirs):
-    X.subprocess.call(["pkill", "redis-cli"])
+    X.subprocess.call(["pkill", "redis-server"])
     C.basic_teardown(removeDirs)
 
 @C.with_custom_setup(setup_redis_arakoon, redis_arakoon_teardown)
@@ -960,6 +960,8 @@ def test_redis_logging():
     cli = C.get_client()
     key = "my-very-unique-key-which-should-be-in-there"
     cli.set(key,"X")
-    cmd = ["nohup", "redis-cli", "lrange", "/logs/arakoon/arakoon_0", "0", "100000", "&"]
+    time.sleep(1)
+    cmd = ["redis-cli", "lrange", "/logs/sturdy_0", "0", "100000"]
     stdout = subprocess.check_output(cmd)
+    print "redis-cli output: ", stdout
     assert(stdout.find(key) > 0)
diff --git a/pylabs/test/server/system_tests_common.py b/pylabs/test/server/system_tests_common.py
index 485e5fcc..a7d35196 100644
--- a/pylabs/test/server/system_tests_common.py
+++ b/pylabs/test/server/system_tests_common.py
@@ -687,10 +687,6 @@ def setup_n_nodes_base(c_id, node_names, force_master,
         cluster.addLocalNode(nodeName)
         cluster.createDirs(nodeName)
 
-        if nodes_extra and nodes_extra[nodeName]:
-            for k,v in nodes_extra.items():
-                config.set(nodeName, k, v)
-
     cluster.disableFsync()
 
     if force_master:
@@ -704,6 +700,9 @@ def setup_n_nodes_base(c_id, node_names, force_master,
     for i in range (n):
         nodeName = node_names[ i ]
         config.set(nodeName, '__tainted_fsync_tlog_dir', 'false')
+        if nodes_extra and nodes_extra.get(nodeName):
+            for k,v in nodes_extra[nodeName].items():
+                config.set(nodeName, k, v)
 
     #
     #

From da219e8ce6b1305f87410b1fd7b82211334f4767 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Tue, 9 Feb 2016 09:39:35 +0100
Subject: [PATCH 111/373] give all collapser tests their own directory

---
 src/extra.ml               |  6 +++++-
 src/node/collapser_test.ml | 36 ++++++++++++++++++++++--------------
 2 files changed, 27 insertions(+), 15 deletions(-)

diff --git a/src/extra.ml b/src/extra.ml
index 39b56809..a217ae29 100644
--- a/src/extra.ml
+++ b/src/extra.ml
@@ -27,7 +27,11 @@ open Lwt
 
 let lwt_bracket setup testcase teardown () =
   let try_lwt_ f =
-    Lwt.catch f (fun exn -> Lwt.fail exn)
+    Lwt.catch
+      f
+      (fun exn ->
+        Lwt_log.fatal ~exn "test failed" >>= fun () ->
+        Lwt.fail exn)
   in
   Lwt_main.run
     begin
diff --git a/src/node/collapser_test.ml b/src/node/collapser_test.ml
index a19c6d80..59ce4653 100644
--- a/src/node/collapser_test.ml
+++ b/src/node/collapser_test.ml
@@ -146,26 +146,34 @@ let test_repeated_collapse (dn,tlx_dir, head_dir) =
   Logger.debug_f_"post collapse: count=%i" !count >>= fun () ->
   OUnit.assert_equal 0 !count ~printer:string_of_int ~msg:"count should be zero";
   _make_values tlc 520 1040 >>= fun () ->
-  Lwt_unix.sleep 10.0 >>= fun () ->
+  Lwt_unix.sleep 2.0 >>= fun () -> (* just to make debugging less messy *)
   Collapser.collapse_many tlc (module S) store_methods 1 cb' cb collapse_slowdown >>= fun () ->
   Logger.debug_f_"post collapse: count=%i" !count >>= fun () ->
   OUnit.assert_equal 0 !count ~printer:string_of_int ~msg:"count should be zero";
   tlc # close ~wait_for_compression:true () >>= fun () ->
   Lwt.return()
   
-let setup () =
-  Logger.info_ "Collapser_test.setup" >>= fun () ->
-  let test_dn = "/tmp/collapser" 
-  and _tlx_dir = "/tmp/collapser_tlx"
-  and _head_dir = "/tmp/collapser_head"
-  in
-  let _ = Sys.command (Printf.sprintf "rm -rf '%s'" test_dn) in
-  let _ = Sys.command (Printf.sprintf "rm -rf '%s'" _tlx_dir) in 
-  let _ = Sys.command (Printf.sprintf "rm -rf '%s'" _head_dir) in
-  File_system.mkdir test_dn 0o755 >>= fun () ->
-  File_system.mkdir _tlx_dir 0o755 >>= fun () ->
-  File_system.mkdir _head_dir 0o755 >>= fun () ->
-  Lwt.return (test_dn, _tlx_dir, _head_dir)
+let setup =
+  let c = ref 0 in
+  (fun () ->
+    Logger.info_f_ "Collapser_test.setup_%i" !c >>= fun () ->
+    let root =
+      let w = try Sys.getenv "WORKSPACE"  with _ -> "" in
+      w ^ "/tmp"
+    in
+    let test_dn  =  Printf.sprintf "%s/collapser_%i"      root !c 
+    and _tlx_dir =  Printf.sprintf "%s/collapser_tlx_%i"  root !c 
+    and _head_dir = Printf.sprintf "%s/collapser_head_%i" root !c
+    in
+    let () = incr c in
+    let _ = Sys.command (Printf.sprintf "rm -rf '%s'" test_dn) in
+    let _ = Sys.command (Printf.sprintf "rm -rf '%s'" _tlx_dir) in 
+    let _ = Sys.command (Printf.sprintf "rm -rf '%s'" _head_dir) in
+    let _ = Sys.command (Printf.sprintf "mkdir -p '%s'" root) in
+    File_system.mkdir test_dn 0o755 >>= fun () ->
+    File_system.mkdir _tlx_dir 0o755 >>= fun () ->
+    File_system.mkdir _head_dir 0o755 >>= fun () ->
+    Lwt.return (test_dn, _tlx_dir, _head_dir))
 
 
 let teardown (dn, tlx_dir, head_dir) =

From a2be14dd05d9461374aab26dc50300997759f9a0 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Tue, 9 Feb 2016 11:24:31 +0100
Subject: [PATCH 112/373] arakoon_redis in client lib

---
 Makefile                 | 1 +
 src/arakoon_client.mllib | 1 +
 2 files changed, 2 insertions(+)

diff --git a/Makefile b/Makefile
index 600d0408..3e1f477b 100644
--- a/Makefile
+++ b/Makefile
@@ -61,6 +61,7 @@ install_client:
           _build/src/tools/ini.cmi \
           _build/src/tools/arakoon_log_sink.cmi \
           _build/src/tools/crash_logger.cmi \
+          _build/src/tools/arakoon_redis.cmi \
           _build/src/tools/llio.mli \
           _build/src/tools/llio.cmi \
           _build/src/tools/network.cmi \
diff --git a/src/arakoon_client.mllib b/src/arakoon_client.mllib
index 59692091..84be2170 100644
--- a/src/arakoon_client.mllib
+++ b/src/arakoon_client.mllib
@@ -26,6 +26,7 @@ Client_log
 Simple_store
 Logger
 Arakoon_log_sink
+Arakoon_redis
 Crash_logger
 Ring_buffer
 Plugin_helper

From dcb55b27749025eee3063706777c4b8c2a478949 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Tue, 9 Feb 2016 11:39:06 +0100
Subject: [PATCH 113/373] more changes for alba

---
 Makefile                    | 25 ++++++++++---------
 src/arakoon_client.mllib    |  1 +
 src/tools/arakoon_logger.ml | 49 ++++++++++++++++++++++---------------
 src/tools/lwt_buffer.ml     |  8 ++++++
 4 files changed, 51 insertions(+), 32 deletions(-)

diff --git a/Makefile b/Makefile
index 3e1f477b..b9dd3855 100644
--- a/Makefile
+++ b/Makefile
@@ -32,17 +32,17 @@ install_server:
 install_client:
 	mkdir -p $(OCAML_LIBDIR)
 	$(OCAML_FIND) install arakoon_client -destdir $(OCAML_LIBDIR) META \
-	  _build/src/arakoon_client.cma \
-	  _build/src/arakoon_client.cmxa \
-	  _build/src/arakoon_client.cmxs \
-	  _build/src/client/stamp.cmi \
-	  _build/src/client/arakoon_exc.mli \
-	  _build/src/client/arakoon_exc.cmi \
+          _build/src/arakoon_client.cma \
+          _build/src/arakoon_client.cmxa \
+          _build/src/arakoon_client.cmxs \
+          _build/src/client/stamp.cmi \
+          _build/src/client/arakoon_exc.mli \
+          _build/src/client/arakoon_exc.cmi \
           _build/src/client/arakoon_client_config.cmi \
-	  _build/src/client/arakoon_client.mli \
-	  _build/src/client/arakoon_client.cmi \
-	  _build/src/client/arakoon_remote_client.mli \
-	  _build/src/client/arakoon_remote_client.cmi \
+          _build/src/client/arakoon_client.mli \
+          _build/src/client/arakoon_client.cmi \
+          _build/src/client/arakoon_remote_client.mli \
+          _build/src/client/arakoon_remote_client.cmi \
           _build/src/client/protocol_common.cmi \
           _build/src/client/client_helper.cmi \
           _build/src/inifiles/inifiles.mli \
@@ -62,6 +62,7 @@ install_client:
           _build/src/tools/arakoon_log_sink.cmi \
           _build/src/tools/crash_logger.cmi \
           _build/src/tools/arakoon_redis.cmi \
+          _build/src/tools/arakoon_logger.cmi \
           _build/src/tools/llio.mli \
           _build/src/tools/llio.cmi \
           _build/src/tools/network.cmi \
@@ -70,8 +71,8 @@ install_client:
           _build/src/tools/typed_ssl.mli \
           _build/src/tools/typed_ssl.cmi \
           _build/src/tools/unix_fd.cmi \
-	  _build/src/tools/arakoon_etcd.cmi \
-	  _build/src/tools/arakoon_config_url.cmi \
+          _build/src/tools/arakoon_etcd.cmi \
+          _build/src/tools/arakoon_config_url.cmi \
           _build/src/tools/tcp_keepalive_sockopt_stubs.o \
           _build/src/tools/tcp_keepalive.cmi \
           _build/src/arakoon_client.a
diff --git a/src/arakoon_client.mllib b/src/arakoon_client.mllib
index 84be2170..5a955fc5 100644
--- a/src/arakoon_client.mllib
+++ b/src/arakoon_client.mllib
@@ -27,6 +27,7 @@ Simple_store
 Logger
 Arakoon_log_sink
 Arakoon_redis
+Arakoon_logger
 Crash_logger
 Ring_buffer
 Plugin_helper
diff --git a/src/tools/arakoon_logger.ml b/src/tools/arakoon_logger.ml
index 9e20cf2c..b77e1502 100644
--- a/src/tools/arakoon_logger.ml
+++ b/src/tools/arakoon_logger.ml
@@ -38,6 +38,31 @@ let file_logger file_name =
   let close () = Lwt_unix.close fd in
   Lwt.return (lg_output, close)
 
+let format_message
+      ~hostname ~pid ~component
+      ~ts ~seqnum
+      ~section ~level ~lines
+  =
+  Printf.sprintf
+    "%s - %s - %i/0 - %s - %i - %s - %s"
+    (let open Unix in
+     let tm = gmtime ts in
+     Printf.sprintf
+       "%04d/%02d/%02d %02d:%02d:%02d %d"
+       (tm.tm_year + 1900)
+       (tm.tm_mon + 1)
+       tm.tm_mday
+       tm.tm_hour
+       tm.tm_min
+       tm.tm_sec
+       (int_of_float ((mod_float ts 1.) *. 1_000_000.))
+    )
+    hostname
+    pid
+    component
+    seqnum
+    (Lwt_log.string_of_level level)
+    (String.concat "\n" lines)
 
 let setup_log_sinks log_sinks =
   Lwt_list.map_p
@@ -60,26 +85,10 @@ let setup_log_sinks log_sinks =
                           let () = incr seqnum in
                           let ts = Unix.gettimeofday () in
                           let logline =
-                            Printf.sprintf
-                              "%s - %s - %i/0 - %s - %i - %s - %s"
-                              (let open Unix in
-                               let tm = gmtime ts in
-                               Printf.sprintf
-                                 "%04d/%02d/%02d %02d:%02d:%02d %d"
-                                 (tm.tm_year + 1900)
-                                 (tm.tm_mon + 1)
-                                 tm.tm_mday
-                                 tm.tm_hour
-                                 tm.tm_min
-                                 tm.tm_sec
-                                 (int_of_float ((mod_float ts 1.) *. 1_000_000.))
-                              )
-                              hostname
-                              pid
-                              component
-                              seqnum'
-                              (Lwt_log.string_of_level level)
-                              (String.concat "\n" lines)
+                            format_message
+                              ~hostname ~pid ~component
+                              ~ts ~seqnum:seqnum'
+                              ~section ~level ~lines
                           in
 
                           Lwt_list.iter_p
diff --git a/src/tools/lwt_buffer.ml b/src/tools/lwt_buffer.ml
index 68a1752e..1023930e 100644
--- a/src/tools/lwt_buffer.ml
+++ b/src/tools/lwt_buffer.ml
@@ -79,6 +79,14 @@ module Lwt_buffer = struct
     let () = Lwt_condition.signal t.full () in
     Lwt.return e
 
+  let take_no_wait t =
+    if Queue.is_empty t.q
+    then None
+    else
+      let item = Queue.take t.q in
+      let () = Lwt_condition.signal t.full () in
+      Some item
+
   let harvest t =
     Lwt_mutex.with_lock t.empty_m
       (fun () -> if Queue.is_empty t.q

From 94687e2737592c71d36a4b59b6a7315f5c99e86d Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Tue, 9 Feb 2016 13:48:18 +0100
Subject: [PATCH 114/373] add support for console logging

---
 cfg/arakoon.ini                    |  2 +-
 src/tools/arakoon_log_sink.ml      |  4 ++++
 src/tools/arakoon_log_sink_test.ml |  1 +
 src/tools/arakoon_logger.ml        | 17 ++++++++++++++++-
 src/tools/crash_logger.ml          |  8 +++++---
 5 files changed, 27 insertions(+), 5 deletions(-)

diff --git a/cfg/arakoon.ini b/cfg/arakoon.ini
index 0f57208e..5eba14e7 100644
--- a/cfg/arakoon.ini
+++ b/cfg/arakoon.ini
@@ -88,7 +88,7 @@ home = /tmp/arakoon/arakoon_0
 #optional different directory for .tlog, .tlc and friends
 #tlog_dir = /tmp/arakoon/arakoon_0
 #log_dir = /tmp/arakoon/arakoon_0
-#log_sinks = redis://127.0.0.1:6379/logs/arakoon/arakoon_0
+#log_sinks = redis://127.0.0.1:6379/logs/arakoon/arakoon_0, /path/to/file, console:
 #crash_log_sinks = redis://127.0.0.1:6379/logs/arakoon/arakoon_0/crash
 #tlog_compression = bz2 # default is bz2; valid choices are: none snappy bz2
 
diff --git a/src/tools/arakoon_log_sink.ml b/src/tools/arakoon_log_sink.ml
index 74f29c03..c590e3e9 100644
--- a/src/tools/arakoon_log_sink.ml
+++ b/src/tools/arakoon_log_sink.ml
@@ -17,10 +17,12 @@ limitations under the License.
 type t =
   | Redis of string * int * string
   | File of string
+  | Console
 
 let to_string = function
   | Redis (host, port, key) -> Printf.sprintf "Redis(%s,%i,%s)" host port key
   | File file -> Printf.sprintf "File(%s)" file
+  | Console -> "Console"
 
 let make s =
   let uri = Uri.of_string s in
@@ -44,5 +46,7 @@ let make s =
        | Some p -> p
      in
      Redis (host, port, path)
+  | Some "console" ->
+     Console
   | Some protocol ->
      failwith (Printf.sprintf "unknown protocol:%s" protocol)
diff --git a/src/tools/arakoon_log_sink_test.ml b/src/tools/arakoon_log_sink_test.ml
index d808624d..bf2e9166 100644
--- a/src/tools/arakoon_log_sink_test.ml
+++ b/src/tools/arakoon_log_sink_test.ml
@@ -25,6 +25,7 @@ let test_log_sink () =
       File "/tmp/a", "file:///tmp/a";
       Redis ("localhost", 628, "/key1"), "redis://localhost:628/key1";
       Redis ("127.0.0.1", 6379, "//key2"), "redis://127.0.0.1//key2";
+      Console, "console:";
     ]
   in
   List.iter
diff --git a/src/tools/arakoon_logger.ml b/src/tools/arakoon_logger.ml
index b77e1502..aea8abcc 100644
--- a/src/tools/arakoon_logger.ml
+++ b/src/tools/arakoon_logger.ml
@@ -38,6 +38,19 @@ let file_logger file_name =
   let close () = Lwt_unix.close fd in
   Lwt.return (lg_output, close)
 
+let console_logger =
+  let mutex = Lwt_mutex.create () in
+  let lg_output line =
+    Lwt_mutex.with_lock
+      mutex
+      (fun () ->
+       Lwt_io.write Lwt_io.stdout line >>= fun () ->
+       Lwt_io.write_char Lwt_io.stdout '\n')
+  in
+  let close () = Lwt.return_unit in
+  lg_output, close
+
+
 let format_message
       ~hostname ~pid ~component
       ~ts ~seqnum
@@ -73,7 +86,9 @@ let setup_log_sinks log_sinks =
      | Redis (host, port, key) ->
         Arakoon_redis.make_redis_logger
           ~host ~port ~key
-        |> Lwt.return)
+        |> Lwt.return
+     | Console ->
+        Lwt.return console_logger)
     log_sinks >>= fun loggers ->
   let hostname = Unix.gethostname () in
   let component = "arakoon" in
diff --git a/src/tools/crash_logger.ml b/src/tools/crash_logger.ml
index ab92b5ff..8f29ba1a 100644
--- a/src/tools/crash_logger.ml
+++ b/src/tools/crash_logger.ml
@@ -98,12 +98,12 @@ let dump_crash_log crash_log_sink =
                       e.Entry.payload
   in
 
-  let log_buffer oc buffer =
+  let log_buffer oc =
     let go = R.fold ~f:(fun acc e -> fun () ->
                                      acc () >>= fun () ->
                                      log_entry oc e)
                     ~acc:(fun () -> Lwt.return_unit)
-                    buffer
+                    circ_buf
     in
     go ()
   in
@@ -113,7 +113,7 @@ let dump_crash_log crash_log_sink =
   | File file_name ->
      Lwt_io.with_file
        ~mode:Lwt_io.output (Printf.sprintf "%s.debug.%f" file_name (Unix.time ()))
-       (fun oc -> log_buffer oc circ_buf)
+       (fun oc -> log_buffer oc)
   | Redis (host, port, key) ->
      let key = Printf.sprintf "%s.debug.%f" key (Unix.time ()) in
      let module Re = Redis_lwt.Client in
@@ -129,3 +129,5 @@ let dump_crash_log crash_log_sink =
        ~acc:(fun () -> Lwt.return_unit)
        circ_buf
        ()
+  | Console ->
+     log_buffer Lwt_io.stdout

From dbfd6455bf832c21942f4eab02b2ef89c5324b51 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Tue, 9 Feb 2016 11:17:36 +0100
Subject: [PATCH 115/373] comments(2)

---
 src/client/client_protocol.ml   |  2 +-
 src/client/remote_nodestream.ml |  6 ++++--
 src/node/collapser.ml           |  9 ---------
 src/node/collapser_test.ml      | 10 ++++++----
 src/tlog/mem_tlogcollection.ml  |  2 +-
 src/tlog/tlc2.ml                | 10 +++++++---
 src/tlog/tlog_map.ml            | 15 ++++++++++++++-
 src/tlog/tlogcollection.ml      |  5 ++++-
 8 files changed, 37 insertions(+), 22 deletions(-)

diff --git a/src/client/client_protocol.ml b/src/client/client_protocol.ml
index b7f0f806..0830ad22 100644
--- a/src/client/client_protocol.ml
+++ b/src/client/client_protocol.ml
@@ -329,7 +329,7 @@ let one_command stop (ic,oc,id as conn) (backend:Backend.backend) =
     end
   | LAST_ENTRIES3 ->
      Sn.input_sn ic >>= fun i ->
-      Logger.debug_f_ "connection=%s LAST_ENTRIES2: i=%Li" id i >>= fun () ->
+      Logger.debug_f_ "connection=%s LAST_ENTRIES3: i=%Li" id i >>= fun () ->
       response_ok oc >>= fun () ->
       backend # last_entries3 i oc >>= fun () ->
       Lwt.return false
diff --git a/src/client/remote_nodestream.ml b/src/client/remote_nodestream.ml
index 95705c8a..936a56e3 100644
--- a/src/client/remote_nodestream.ml
+++ b/src/client/remote_nodestream.ml
@@ -106,8 +106,10 @@ class remote_nodestream ((ic,oc) as conn) =
              Llio.input_string ic >>= fun name ->
              Sn.input_sn ic       >>= fun start_i ->
              Llio.input_int64 ic  >>= fun length ->
-             Logger.info_f_ "retrieving %s (%Li bytes)" name length >>= fun () ->
-             tlog_coll # save_tlog_file name length ic >>= fun () ->
+             Logger.info_f_ "retrieving start_i:%s (name:%s, %Li bytes)"
+                            (Sn.string_of start_i) name length
+             >>= fun () ->
+             tlog_coll # save_tlog_file start_i name length ic >>= fun () ->
              loop_parts ()
            end
         | x  -> Llio.lwt_failfmt "don't know what %i means" x
diff --git a/src/node/collapser.ml b/src/node/collapser.ml
index 3f07def9..635f85a2 100644
--- a/src/node/collapser.ml
+++ b/src/node/collapser.ml
@@ -100,15 +100,6 @@ let collapse_until (type s) (tlog_coll:Tlogcollection.tlog_collection)
                Lwt_unix.sleep period >>= fun () ->
                Lwt.return r
           in
-          (* let get_next_point i =
-              let next_r = tlog_coll # next_rollover i in
-              match next_r with
-                | None -> Sn.zero
-                | Some i -> i
-          in
-          let next_rollover_point = ref (get_next_point (Sn.pred start_i))
-          in
-           *)
           let add_to_store entry =
             let i = Entry.i_of entry
             and value = Entry.v_of entry
diff --git a/src/node/collapser_test.ml b/src/node/collapser_test.ml
index 59ce4653..aecbcc47 100644
--- a/src/node/collapser_test.ml
+++ b/src/node/collapser_test.ml
@@ -117,6 +117,7 @@ let test_collapse_many (dn, tlx_dir, head_dir) =
   Lwt.return ()
 
 let test_repeated_collapse (dn,tlx_dir, head_dir) =
+  Logger.info_f_ "---------- test_repeated_collapse ----------" >>= fun () ->
   let node_id = "node_name" in
   Logger.debug_f_ "test_collapse_many dn=%s, tlx_dir=%s, head_dir=%s" dn tlx_dir head_dir
   >>= fun () ->
@@ -139,13 +140,14 @@ let test_repeated_collapse (dn,tlx_dir, head_dir) =
   
   let store_methods = (Batched_store.Local_store.copy_store2, head_location, 0.0) in
   File_system.unlink head_location >>= fun () ->
-
-  _make_values tlc 0 520 >>= fun () ->
+  let n_entries = 520 in
+  _make_values tlc 0 n_entries >>= fun () ->
   let collapse_slowdown = None in
   Collapser.collapse_many tlc (module S) store_methods 1 cb' cb collapse_slowdown >>= fun () ->
   Logger.debug_f_"post collapse: count=%i" !count >>= fun () ->
   OUnit.assert_equal 0 !count ~printer:string_of_int ~msg:"count should be zero";
-  _make_values tlc 520 1040 >>= fun () ->
+  Logger.debug_f_ "adding %i more entries" n_entries >>= fun () ->
+  _make_values tlc n_entries (2 * n_entries) >>= fun () ->
   Lwt_unix.sleep 2.0 >>= fun () -> (* just to make debugging less messy *)
   Collapser.collapse_many tlc (module S) store_methods 1 cb' cb collapse_slowdown >>= fun () ->
   Logger.debug_f_"post collapse: count=%i" !count >>= fun () ->
@@ -156,6 +158,7 @@ let test_repeated_collapse (dn,tlx_dir, head_dir) =
 let setup =
   let c = ref 0 in
   (fun () ->
+    let () = incr c in
     Logger.info_f_ "Collapser_test.setup_%i" !c >>= fun () ->
     let root =
       let w = try Sys.getenv "WORKSPACE"  with _ -> "" in
@@ -165,7 +168,6 @@ let setup =
     and _tlx_dir =  Printf.sprintf "%s/collapser_tlx_%i"  root !c 
     and _head_dir = Printf.sprintf "%s/collapser_head_%i" root !c
     in
-    let () = incr c in
     let _ = Sys.command (Printf.sprintf "rm -rf '%s'" test_dn) in
     let _ = Sys.command (Printf.sprintf "rm -rf '%s'" _tlx_dir) in 
     let _ = Sys.command (Printf.sprintf "rm -rf '%s'" _head_dir) in
diff --git a/src/tlog/mem_tlogcollection.ml b/src/tlog/mem_tlogcollection.ml
index 0599512b..8c095687 100644
--- a/src/tlog/mem_tlogcollection.ml
+++ b/src/tlog/mem_tlogcollection.ml
@@ -66,7 +66,7 @@ class mem_tlog_collection _name =
 
     method dump_tlog_file _start_i _oc = failwith "dump_tlog_file not supported"
 
-    method save_tlog_file _name _length _ic = failwith "save_tlog_file not supported"
+    method save_tlog_file _start_i _name _length _ic = failwith "save_tlog_file not supported"
 
     method reinit () = Lwt.return ()
     method which_tlog_file _start_i = failwith "which_tlog_file not supported"
diff --git a/src/tlog/tlc2.ml b/src/tlog/tlc2.ml
index 1284d1f8..c43f5b8e 100644
--- a/src/tlog/tlc2.ml
+++ b/src/tlog/tlc2.ml
@@ -527,7 +527,7 @@ class tlc2
       >>= fun () ->
       Logger.debug_f_ "dumped:%s (%Li) bytes" canonical length
 
-    method save_tlog_file remote_name length ic =
+    method save_tlog_file first_i remote_name length ic =
       (* what with rotation (jump to new tlog), open streams, ...*)
       begin
         match _file with
@@ -540,11 +540,15 @@ class tlc2
       let full_name = TlogMap.get_full_path tlog_map remote_name in
       let extension = Tlog_map.extension_of remote_name in
       let tmp = full_name ^ ".tmp" in
-      Logger.info_f_ "save_tlog_file: remote_name: %s" tmp >>= fun () ->
+      
+      Logger.info_f_
+        "save_tlog_file: first_i:%s remote_name:%s in %s"
+        (Sn.string_of first_i) remote_name tmp
+      >>= fun () ->
+      
       Lwt_io.with_file ~mode:Lwt_io.output tmp
                        (fun oc -> Llio.copy_stream ~length ~ic ~oc)
       >>= fun () ->
-      Tlog_map.first_i_of ~extension_override:extension tmp >>= fun first_i ->
       let new_outer = TlogMap.new_outer tlog_map first_i in
       let new_name = Printf.sprintf "%03i%s" new_outer extension in
       let canon = TlogMap.get_full_path tlog_map new_name in
diff --git a/src/tlog/tlog_map.ml b/src/tlog/tlog_map.ml
index 92504e1b..7189596c 100644
--- a/src/tlog/tlog_map.ml
+++ b/src/tlog/tlog_map.ml
@@ -1,3 +1,15 @@
+(*
+Copyright 2016 iNuron NV
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+     http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*)
 
 open Lwt.Infix
 open Tlogcommon
@@ -415,7 +427,8 @@ module TlogMap = struct
              end
            else
              Lwt.fail exn
-        | exn -> Lwt.fail exn
+        | exn -> Logger.debug_f_ ~exn "filename:%s" full_name >>= fun () ->
+                 Lwt.fail exn
       )
       
   let outer_of_i t i =
diff --git a/src/tlog/tlogcollection.ml b/src/tlog/tlogcollection.ml
index d117ba3c..44970ddb 100644
--- a/src/tlog/tlogcollection.ml
+++ b/src/tlog/tlogcollection.ml
@@ -49,7 +49,10 @@ class type tlog_collection =
   method complete_file_to_deliver: Sn.t -> (int * Sn.t) option
                                                
   method dump_tlog_file : int -> Lwt_io.output_channel -> unit Lwt.t
-  method save_tlog_file : string -> int64 -> Lwt_io.input_channel -> unit Lwt.t
+  method save_tlog_file : Sn.t
+                          -> string -> int64
+                          -> Lwt_io.input_channel
+                          -> unit Lwt.t
   method reinit : unit -> unit Lwt.t 
   method get_head_name : unit -> string
   method get_tlog_from_i : Sn.t -> int

From 000a8d0e859bf3dc1f41c589f220cb1de80d3bfa Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Tue, 9 Feb 2016 16:14:14 +0100
Subject: [PATCH 116/373] cope with `rogue` tlog files

---
 src/node/sync_backend.ml       |  4 +-
 src/tlog/mem_tlogcollection.ml |  2 +-
 src/tlog/tlc2.ml               | 35 +++++++++-------
 src/tlog/tlog_map.ml           | 73 ++++++++++++++++++----------------
 src/tlog/tlogcollection.ml     |  2 +-
 5 files changed, 63 insertions(+), 53 deletions(-)

diff --git a/src/node/sync_backend.ml b/src/node/sync_backend.ml
index 2a75c3ba..5b71ea7d 100644
--- a/src/node/sync_backend.ml
+++ b/src/node/sync_backend.ml
@@ -694,7 +694,9 @@ struct
               begin
                 self # wait_for_tlog_release keep_bottom_n >>= fun () ->
                 let keep_i = tlog_collection # get_start_i keep_bottom_n in
-                tlog_collection # remove_below keep_i
+                match keep_i with
+                | None        -> Lwt.return ()
+                | Some keep_i -> tlog_collection # remove_below keep_i
               end
             else
               Lwt.return ()
diff --git a/src/tlog/mem_tlogcollection.ml b/src/tlog/mem_tlogcollection.ml
index 8c095687..fa3d617e 100644
--- a/src/tlog/mem_tlogcollection.ml
+++ b/src/tlog/mem_tlogcollection.ml
@@ -84,7 +84,7 @@ class mem_tlog_collection _name =
     method get_head_name () = failwith "get_head_name not implemented"
 
     method get_tlog_from_i _ = 0
-    method get_start_i n = Sn.zero
+    method get_start_i n = Some Sn.zero
     method is_rollover_point _ = failwith "is_rollover_point not supported"
     method next_rollover _ = Some Int64.max_int
 
diff --git a/src/tlog/tlc2.ml b/src/tlog/tlc2.ml
index c43f5b8e..0441b2c8 100644
--- a/src/tlog/tlc2.ml
+++ b/src/tlog/tlc2.ml
@@ -488,7 +488,7 @@ class tlc2
 
     method get_tlog_from_i (i:Sn.t) : int = TlogMap.outer_of_i tlog_map i
 
-    method get_start_i (n:int) : Sn.t = TlogMap.get_start_i tlog_map n
+    method get_start_i (n:int) : Sn.t option = TlogMap.get_start_i tlog_map n
     method is_rollover_point i = TlogMap.is_rollover_point tlog_map i
     method next_rollover i     = TlogMap.next_rollover tlog_map i
                                                        
@@ -512,20 +512,25 @@ class tlc2
       Logger.debug_f_ "n = %03i => canonical=%s" n canonical >>= fun () ->
       let name = Filename.basename canonical in
       let start_i = TlogMap.get_start_i tlog_map n in
-      Llio.output_string oc name >>= fun () ->
-      Sn.output_sn oc start_i    >>= fun () ->
-      File_system.stat canonical >>= fun stats ->
-      let length = Int64.of_int(stats.Unix.st_size)
-      (* why don't they use largefile ? *)
-      in
-      Logger.info_f_ "dumping %S (%Li bytes)" canonical length >>= fun () ->
-      Lwt_io.with_file ~mode:Lwt_io.input canonical
-        (fun ic ->
-           Llio.output_int64 oc length >>= fun () ->
-           Llio.copy_stream ~length ~ic ~oc
-        )
-      >>= fun () ->
-      Logger.debug_f_ "dumped:%s (%Li) bytes" canonical length
+      match start_i with
+      | None -> failwith (Printf.sprintf "tlogcollection doesn't know tlog:%i" n)
+      | Some start_i ->
+         begin
+           Llio.output_string oc name >>= fun () ->
+           Sn.output_sn oc start_i    >>= fun () ->
+           File_system.stat canonical >>= fun stats ->
+           let length = Int64.of_int(stats.Unix.st_size)
+                                    (* why don't they use largefile ? *)
+           in
+           Logger.info_f_ "dumping %S (%Li bytes)" canonical length >>= fun () ->
+           Lwt_io.with_file ~mode:Lwt_io.input canonical
+                            (fun ic ->
+                              Llio.output_int64 oc length >>= fun () ->
+                              Llio.copy_stream ~length ~ic ~oc
+                            )
+           >>= fun () ->
+           Logger.debug_f_ "dumped:%s (%Li) bytes" canonical length
+         end
 
     method save_tlog_file first_i remote_name length ic =
       (* what with rotation (jump to new tlog), open streams, ...*)
diff --git a/src/tlog/tlog_map.ml b/src/tlog/tlog_map.ml
index 7189596c..87aea62d 100644
--- a/src/tlog/tlog_map.ml
+++ b/src/tlog/tlog_map.ml
@@ -443,14 +443,16 @@ module TlogMap = struct
       
   let get_start_i t n =
     let rec find = function
-      | [] -> Sn.zero (* TODO: maybe assert *)
+      | [] -> None
       | item :: rest ->
          if item.n = n
-         then item.i else
-           find rest
+         then Some item.i
+         else find rest
     in
     let r = find t.i_to_tlog_number in
-    (* Logger.ign_debug_f_ "get_start_i %s %i => %s"  (show_map t) n (Sn.string_of r) ;*)
+    (* Logger.ign_debug_f_ "get_start_i %s %i => %s"  (show_map t) n 
+                                                      (To_string.option Sn.string_of r) ;
+     *)
     r
          
   
@@ -567,44 +569,45 @@ module TlogMap = struct
     Logger.debug_f_ "TlogMap.iterate_tlog_dir start_i:%s too_far_i:%s ~index:%s"
                     (Sn.string_of start_i) tfs (Index.to_string index)
     >>= fun () ->
-    get_tlog_names t >>= fun tlog_names ->
+    get_tlog_names t >>= fun tlog_names -> (* TODO: we probably shouldn't look at the fs *) 
     let acc_entry (_i0:Sn.t) entry =
       f entry >>= fun () -> let i = Entry.i_of entry in Lwt.return i
     in
     let num_tlogs = List.length tlog_names in
     let maybe_fold (cnt,low) fn =
       let fn_start = get_start_i t (get_number fn) in
-      let fn_next  = next_rollover t fn_start in
-      let low_n = Sn.succ low in
-      let should_fold = 
-        match fn_next with
-        | None -> fn_start <= low_n
-        | Some next -> fn_start <= low_n && low_n < next
-      in
-      Logger.debug_f_ "maybe_fold %s low:%s" fn (Sn.string_of low) >>= fun () ->
-      Logger.debug_f_ "fn: fn_start:%s <= low_n:%s fn_next:%s => %b"
-                      (Sn.string_of fn_start) (Sn.string_of low_n)
-                      (To_string.option Sn.string_of fn_next)
-                      should_fold
-      >>= fun () ->
       
-      let fold () =
-        Logger.debug_f_ "fold over: %s: [%s,...) "
-                           fn (Sn.string_of fn_start) 
-           >>= fun () ->
-           let first = fn_start in
-           Logger.info_f_ "Replaying tlog file: %s (%d/%d)" fn cnt num_tlogs  >>= fun () ->
-           let t1 = Unix.gettimeofday () in
-           fold_read t fn ~index low (Some too_far_i) ~first low acc_entry >>= fun low' ->
-           Logger.info_f_ "Completed replay of %s, took %f seconds, %i to go"
-                          fn (Unix.gettimeofday () -. t1) (num_tlogs - cnt)
-           >>= fun () ->
-           cb cnt >>= fun () ->
-           Lwt.return (cnt+1, low')
-      in
-      if should_fold
-      then fold ()
-      else Lwt.return (cnt+1,low)
+      let low_n = Sn.succ low in
+      match fn_start with
+      | Some fn_start ->
+         begin
+           let fn_next  = next_rollover t fn_start in
+           let should_fold =
+             match fn_next with
+             | None -> fn_start <= low_n
+             | Some next -> fn_start <= low_n && low_n < next
+           in
+           if should_fold
+           then
+             begin
+               Logger.debug_f_ "fold over: %s: [%s,...) "
+                               fn (Sn.string_of fn_start) 
+               >>= fun () ->
+               let first = fn_start in
+               Logger.info_f_ "Replaying tlog file: %s (%d/%d)" fn cnt num_tlogs  >>= fun () ->
+               let t1 = Unix.gettimeofday () in
+               fold_read t fn ~index low (Some too_far_i) ~first low acc_entry >>= fun low' ->
+               Logger.info_f_ "Completed replay of %s, took %f seconds, %i to go"
+                              fn (Unix.gettimeofday () -. t1) (num_tlogs - cnt)
+               >>= fun () ->
+               cb cnt >>= fun () ->
+               Lwt.return (cnt+1, low')
+             end
+           else
+             Lwt.return (cnt +1, low)
+         end
+      | None -> (* it's a file that's not in the map *)
+         Lwt.return (cnt +1, low)
     in
     Lwt_list.fold_left_s maybe_fold (1,start_i) tlog_names
     >>= fun _x ->
diff --git a/src/tlog/tlogcollection.ml b/src/tlog/tlogcollection.ml
index 44970ddb..ec0b047c 100644
--- a/src/tlog/tlogcollection.ml
+++ b/src/tlog/tlogcollection.ml
@@ -59,7 +59,7 @@ class type tlog_collection =
   method get_tlog_count: unit -> int Lwt.t
   method remove_below : Sn.t -> unit Lwt.t
                                      
-  method get_start_i : int -> Sn.t
+  method get_start_i : int -> Sn.t option
   method get_last_i: unit -> Sn.t
   method is_rollover_point: Sn.t -> bool
   method next_rollover: Sn.t -> Sn.t option

From 480d806a6fe1d0320c338d4963a623d3a81bd16f Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Tue, 9 Feb 2016 18:25:09 +0100
Subject: [PATCH 117/373] log time zone offset

---
 _tags                             |  1 +
 docker/arakoon_centos7/Dockerfile |  5 +++--
 docker/arakoon_debian/Dockerfile  |  5 +++--
 src/tools/arakoon_logger.ml       | 18 ++++++++++++++----
 travis.sh                         |  5 +++--
 5 files changed, 24 insertions(+), 10 deletions(-)

diff --git a/_tags b/_tags
index ecf5ecba..ab056408 100644
--- a/_tags
+++ b/_tags
@@ -17,6 +17,7 @@ true: package(ocplib-endian)
 true: package(uri)
 true: package(redis)
 true: package(redis.lwt)
+true: package(core)
 "src":include
 "src/lib":include
 "src/tools":include
diff --git a/docker/arakoon_centos7/Dockerfile b/docker/arakoon_centos7/Dockerfile
index 2f50e990..d392fc16 100644
--- a/docker/arakoon_centos7/Dockerfile
+++ b/docker/arakoon_centos7/Dockerfile
@@ -59,8 +59,9 @@ RUN eval `${opam_env}` && \
         conf-libev \
         ocplib-endian \
         depext \
-        redis \
-        uri
+        redis.0.2.3 \
+        uri.1.9.1 \
+        core.113.00.00
 
 
 RUN yum -y install autoconf help2man perl-Thread-Queue
diff --git a/docker/arakoon_debian/Dockerfile b/docker/arakoon_debian/Dockerfile
index 9d24ac83..99c67b1d 100644
--- a/docker/arakoon_debian/Dockerfile
+++ b/docker/arakoon_debian/Dockerfile
@@ -65,8 +65,9 @@ RUN eval `${opam_env}` && \
         conf-libev \
         ocplib-endian \
         depext \
-        redis \
-        uri
+        redis.0.2.3 \
+        uri.1.9.1 \
+        core.113.00.00
 
 RUN echo "jenkins ALL=NOPASSWD: ALL" >/etc/sudoers.d/jenkins
 ENV LANG=C
diff --git a/src/tools/arakoon_logger.ml b/src/tools/arakoon_logger.ml
index aea8abcc..d905611f 100644
--- a/src/tools/arakoon_logger.ml
+++ b/src/tools/arakoon_logger.ml
@@ -59,16 +59,26 @@ let format_message
   Printf.sprintf
     "%s - %s - %i/0 - %s - %i - %s - %s"
     (let open Unix in
-     let tm = gmtime ts in
+     let tm = Core.Time_internal.to_tm ts in
+     let offset =
+       Core.Time.utc_offset ~zone:Core.Time.Zone.local ts
+       |> Core.Span.to_parts
+     in
      Printf.sprintf
-       "%04d/%02d/%02d %02d:%02d:%02d %d"
+       "%04d/%02d/%02d %02d:%02d:%02d %d %c%02d%02d"
        (tm.tm_year + 1900)
        (tm.tm_mon + 1)
        tm.tm_mday
        tm.tm_hour
        tm.tm_min
        tm.tm_sec
-       (int_of_float ((mod_float ts 1.) *. 1_000_000.))
+       (int_of_float ((mod_float (Core.Time.to_float ts) 1.) *. 1_000_000.))
+       (let open Core.Std.Float in
+        match offset.Core.Span.Parts.sign with
+        | Sign.Neg -> '-'
+        | Sign.Pos | Sign.Zero -> '+')
+       offset.Core.Span.Parts.hr
+       offset.Core.Span.Parts.min
     )
     hostname
     pid
@@ -98,7 +108,7 @@ let setup_log_sinks log_sinks =
                  ~output:(fun section level lines ->
                           let seqnum' = !seqnum in
                           let () = incr seqnum in
-                          let ts = Unix.gettimeofday () in
+                          let ts = Core.Time.now () in
                           let logline =
                             format_message
                               ~hostname ~pid ~component
diff --git a/travis.sh b/travis.sh
index aa850937..9497e5cb 100755
--- a/travis.sh
+++ b/travis.sh
@@ -10,8 +10,9 @@ OPAM_DEPENDS="ssl.0.5.2 \
               snappy \
               quickcheck \
               ocplib-endian \
-              redis \
-              uri
+              redis.0.2.3 \
+              uri.1.9.1 \
+              core.113.00.00
 "
 
 export OPAMYES=1

From 84610a6809344537fe2a093c41e29515f93da765 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Tue, 9 Feb 2016 19:20:07 +0100
Subject: [PATCH 118/373] after receiving head, close file

---
 src/tlog/tlc2.ml | 23 ++++++++++++++---------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/src/tlog/tlc2.ml b/src/tlog/tlc2.ml
index 0441b2c8..1c43d5e3 100644
--- a/src/tlog/tlc2.ml
+++ b/src/tlog/tlc2.ml
@@ -406,8 +406,11 @@ class tlc2
         tmp
         hf_name
         (fun oc ->
-          Llio.copy_stream ~length ~ic ~oc) >>= fun () ->
-
+          Llio.copy_stream ~length ~ic ~oc) 
+      >>= fun () ->
+      self # _maybe_close_current_file ()
+      >>= fun () ->                                                               
+      let () = TlogMap.set_should_roll tlog_map in
       Logger.info_ "done: save_head"
 
 
@@ -532,15 +535,17 @@ class tlc2
            Logger.debug_f_ "dumped:%s (%Li) bytes" canonical length
          end
 
+    method private _maybe_close_current_file () =
+      match _file with
+      | None -> Lwt.return ()
+      | Some file ->
+         F.close file >>= fun () ->
+         _file <- None;
+         Lwt.return_unit
+
     method save_tlog_file first_i remote_name length ic =
       (* what with rotation (jump to new tlog), open streams, ...*)
-      begin
-        match _file with
-        | None -> Lwt.return ()
-        | Some file ->
-           F.close file >>= fun () -> _file <- None;
-           Lwt.return ()
-      end 
+      self # _maybe_close_current_file ()
       >>= fun () ->                                                               
       let full_name = TlogMap.get_full_path tlog_map remote_name in
       let extension = Tlog_map.extension_of remote_name in

From 137f5c89121f8d1e2d3d7d5a42cee95224041103 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 10 Feb 2016 10:53:40 +0100
Subject: [PATCH 119/373] add core as a dependency;cope with wide versions

---
 myocamlbuild.ml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/myocamlbuild.ml b/myocamlbuild.ml
index d4e94673..54d068d7 100644
--- a/myocamlbuild.ml
+++ b/myocamlbuild.ml
@@ -9,6 +9,7 @@ let dependencies = [ "lwt";
                      "ocplib-endian";
                      "redis";
                      "uri";
+                     "core";
                    ]
 
 (* Enabled compiler warnings, argument for '-w', see `man ocamlc` *)
@@ -57,7 +58,7 @@ let list_dependencies () =
   let pkgs = List.map query dependencies in
   let lines = List.map (fun (pkg, version, descr) ->
     let tabs = if String.length pkg < 8 then "\t\t" else "\t" in
-    Printf.sprintf "%s%s%s\t%s" pkg tabs version descr)
+    Printf.sprintf "%s%s%12s\t%s" pkg tabs version descr)
     pkgs
   in
   String.concat "\n" lines

From a27d01cb2ae026b77832f0ee5fc4b2856c6db521 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 10 Feb 2016 13:26:09 +0100
Subject: [PATCH 120/373] strengthen + shorten some tests; remove duplication

---
 .../test/server/right/system_tests_learner.py |  4 +-
 .../server/right/system_tests_long_right.py   | 44 ++++---------------
 2 files changed, 11 insertions(+), 37 deletions(-)

diff --git a/pylabs/test/server/right/system_tests_learner.py b/pylabs/test/server/right/system_tests_learner.py
index b824c121..e8bc19e0 100644
--- a/pylabs/test/server/right/system_tests_learner.py
+++ b/pylabs/test/server/right/system_tests_learner.py
@@ -22,9 +22,9 @@
 import logging
 from Compat import X
 
-@Common.with_custom_setup(Common.setup_2_nodes, Common.basic_teardown)
+@Common.with_custom_setup(Common.setup_2_nodes_mini, Common.basic_teardown)
 def test_learner():
-    op_count = 54321
+    op_count = 5432
     Common.iterate_n_times(op_count, Common.simple_set)
     cluster = Common._getCluster(Common.cluster_id)
     logging.info("adding learner")
diff --git a/pylabs/test/server/right/system_tests_long_right.py b/pylabs/test/server/right/system_tests_long_right.py
index 85b4c46c..27f45674 100644
--- a/pylabs/test/server/right/system_tests_long_right.py
+++ b/pylabs/test/server/right/system_tests_long_right.py
@@ -527,9 +527,13 @@ def start_node():
         
     print "done"
 
-@Common.with_custom_setup( Common.setup_3_nodes_forced_master, Common.basic_teardown )
+@Common.with_custom_setup( Common.setup_3_nodes_forced_master_mini, Common.basic_teardown )
 def test_large_catchup_while_running():
-    """ make sure catchup does not interphere with normal operation (eta : 720s) """
+    pass # workaround nose listing comment iso test name
+
+    #make sure catchup does not interfere with normal operation (duration ~ 100s)
+    
+    count = 20000
     cli = Common.get_client()
     cluster = Common._getCluster()
 
@@ -545,7 +549,7 @@ def test_large_catchup_while_running():
 
     time.sleep(0.1)
     X.subprocess.call(["kill","-STOP",str(node_pid)])
-    Common.iterate_n_times( 200000, Common.simple_set )
+    Common.iterate_n_times( count, Common.simple_set )
     for n in others:
         Common.collapse(n)
 
@@ -564,36 +568,6 @@ def test_log_rotation():
         time.sleep(0.2)
         Common.assert_running_nodes(1)
 
-@Common.with_custom_setup( Common.setup_3_nodes_forced_master, Common.basic_teardown )
-def test_large_catchup_while_running():
-    """
-    test_large_catchup_while_running : assert cluster is still ok after (catchup // collapse) (eta : 600s)
-    """
-    cli = Common.get_client()
-    cluster = Common._getCluster()
-
-    cli.set('k','v')
-    m = cli.whoMaster()
-
-    nod1 = Common.node_names[0]
-    nod2 = Common.node_names[1]
-    nod3 = Common.node_names[2]
-
-    n_name,others = (nod1, [nod2,nod3]) if nod1 != m else (nod2, [nod1, nod3])
-    node_pid = cluster._getPid(n_name)
-
-    time.sleep(0.1)
-    X.subprocess.call(["kill","-STOP", str(node_pid) ])
-    Common.iterate_n_times( 200000, Common.simple_set )
-    for n in others:
-        Common.collapse(n)
-
-    time.sleep(1.0)
-    X.subprocess.call(["kill", "-CONT", str(node_pid) ])
-    cli.delete('k')
-    time.sleep(10.0)
-    Common.assert_running_nodes(3)
-
 
 @Common.with_custom_setup( Common.setup_3_nodes_forced_master,
                            Common.basic_teardown)
@@ -621,10 +595,10 @@ def test_db_optimize():
     msg = template % (start_size, opt_size)
     assert_true( opt_size < 0.1*start_size, msg)
 
-@Common.with_custom_setup(Common.setup_1_node, Common.basic_teardown)
+@Common.with_custom_setup(Common.setup_1_node_mini, Common.basic_teardown)
 def test_missing_tlog():
     t0 = time.time()
-    n_sets = 450000
+    n_sets = 45000
     Common.iterate_n_times(n_sets,Common.simple_set)
     t1 = time.time()
     d0 = t1 - t0

From e41a394c881a48f63c477c85308bc1ad0d56278a Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 10 Feb 2016 16:34:10 +0100
Subject: [PATCH 121/373] add test for mixed configs

---
 .../server/right/test_catchup_exercises.py    | 79 ++++++++++++++++++-
 1 file changed, 78 insertions(+), 1 deletion(-)

diff --git a/pylabs/test/server/right/test_catchup_exercises.py b/pylabs/test/server/right/test_catchup_exercises.py
index 429d297b..7c16e9be 100644
--- a/pylabs/test/server/right/test_catchup_exercises.py
+++ b/pylabs/test/server/right/test_catchup_exercises.py
@@ -17,6 +17,7 @@
 
 
 from .. import system_tests_common as Common
+from ..quick import system_tests_basic
 from arakoon.ArakoonExceptions import *
 import arakoon
 import logging
@@ -24,7 +25,8 @@
 import subprocess
 from threading import Thread
 from nose.tools import *
-
+from Compat import X
+from arakoon_ext.client import ArakoonClient
 
 @Common.with_custom_setup(Common.setup_2_nodes_forced_master_mini, Common.basic_teardown)
 def test_catchup_exercises():
@@ -60,3 +62,78 @@ def do_one(n, max_wait):
         do_one(n,w)
         n = n * 2
         w = w * 2
+
+def test_catchup_mixed_config():
+    pass
+    """
+    catchup where 2 nodes have different rollover points. 
+    (It's a configuration error that's bound to happen)
+    """
+    Common.data_base_dir = '/'.join([X.tmpDir,'arakoon_system_tests' ,
+                                         "test_catchup_mixed_config"])
+    
+    def setup(c_id,tlog_max_entries):
+            c_nodes = ["%s_0" % c_id]
+            c_home = "/".join([Common.data_base_dir, c_id])
+            c = Common.setup_n_nodes_base(c_id, c_nodes,False, c_home,
+                                          Common.node_msg_base_port, Common.node_client_base_port,
+                                          extra = {'tlog_max_entries':str(tlog_max_entries)}
+            )
+            return c
+
+    def wait(c0):
+        system_tests_basic._wait_for_master(c0)
+        
+    def _inner(c0):
+        Common.regenerateClientConfig(id0)
+        c0.start()
+        wait(c0)
+        
+        c0_client = c0.getClient()
+
+        for x in range(100):
+            c0_client.set("key_%i" % x, "value_%i" % x)
+        
+        bad = 'nineteen'
+        c0.addNode(bad,
+                   clientPort = Common.node_client_base_port + 1,
+                   messagingPort = Common.node_msg_base_port + 1,
+                   home = '/'.join([Common.data_base_dir, bad]),
+                   logLevel = "debug", isLocal = True
+        )
+
+        bad
+        cfg = c0._getConfigFile()
+        cfg.set('global','tlog_max_entries', str(19))
+        c0._saveConfig(cfg)
+        c0.createDirs(bad)
+        c0.catchupOnly(bad)
+
+        c0.stop()
+        c0.start()
+
+        wait(c0)
+        Common.regenerateClientConfig(id0)
+        logging.info("status:%s", c0.getStatus())
+
+        c0_client = c0.getClient()
+        logging.info('who_master:%s', c0_client.whoMaster())
+        stats = c0_client.statistics()
+        node_is = stats['node_is']
+        logging.info("node_is:%s", node_is)
+        lo = min(node_is.values())
+        hi = max(node_is.values())
+        assert_true(hi-lo <= 1)
+        
+        for x in range(100):
+            c0_client.set("x","x")
+            
+    id0 = 'thirteen'
+    c0 = setup(id0, 13)
+    try:
+        _inner(c0)
+    finally:
+        pass
+        c0.stop()
+        c0.tearDown()
+        c0.remove()

From ac48e4a5139755d3ed90ec240c1b5b5f122059c9 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 10 Feb 2016 17:38:17 +0100
Subject: [PATCH 122/373] keep 5 logs

---
 pylabs/test/server/shaky/test_272.py      | 2 +-
 pylabs/test/server/system_tests_common.py | 5 +++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/pylabs/test/server/shaky/test_272.py b/pylabs/test/server/shaky/test_272.py
index 5617f96f..57268802 100644
--- a/pylabs/test/server/shaky/test_272.py
+++ b/pylabs/test/server/shaky/test_272.py
@@ -41,7 +41,7 @@ def test_272():
         raise Exception ("benchmark should not have finished yet.")
 
     for i in range(100):
-        Common.rotate_log(node, 1, False)
+        Common.rotate_log(node, 5, False)
         time.sleep(0.2)
         Common.assert_running_nodes(1)
         rc = bench.returncode
diff --git a/pylabs/test/server/system_tests_common.py b/pylabs/test/server/system_tests_common.py
index 4deda8c5..1ef0bab9 100644
--- a/pylabs/test/server/system_tests_common.py
+++ b/pylabs/test/server/system_tests_common.py
@@ -726,6 +726,7 @@ def setup_n_nodes_base(c_id, node_names, force_master,
     lease = int(lease_duration)
     logging.info( "Setting lease expiration to %d" % lease)
     cluster.setMasterLease( lease )
+    return cluster
 
 
 def setup_n_nodes ( n, force_master, home_dir,
@@ -762,6 +763,10 @@ def setup_2_nodes_forced_master_mini (home_dir):
     extra = {'tlog_max_entries':'1000'}
     setup_n_nodes( 2, True, home_dir, extra)
 
+def setup_3_nodes_forced_master_mini (home_dir):
+    extra = {'tlog_max_entries':'1000'}
+    setup_n_nodes( 3, True, home_dir, extra)
+    
 def setup_2_nodes_forced_master_normal_slaves (home_dir):
     setup_n_nodes( 2, True, home_dir)
 

From fab7eb513d96da7efa93b68df7660398620889d9 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Thu, 11 Feb 2016 14:19:58 +0100
Subject: [PATCH 123/373] shorter but sterner test_272

---
 pylabs/test/server/shaky/test_272.py | 47 ++++++++++++++++++++++++----
 1 file changed, 41 insertions(+), 6 deletions(-)

diff --git a/pylabs/test/server/shaky/test_272.py b/pylabs/test/server/shaky/test_272.py
index 3c25cd6b..961e2e5f 100644
--- a/pylabs/test/server/shaky/test_272.py
+++ b/pylabs/test/server/shaky/test_272.py
@@ -22,26 +22,42 @@
 import subprocess
 from threading import Thread
 from nose.tools import *
-
+import os
+import signal
 
 @Common.with_custom_setup(Common.setup_1_node, Common.basic_teardown)
 def test_272():
+    pass
     """
-    test_272 : arakoon can go down during log rotation, but you need to have load to reproduce it (eta: 1110s)
+    test_272 : arakoon can go down during log rotation, but you need to have load to reproduce it 
     """
     node = Common.node_names[0]
     cluster = Common._getCluster()
     path = cluster._getConfigFileName() + ".cfg"
     logging.info('path=%s', path)
-    bench = subprocess.Popen([Common.CONFIG.binary_full_path, '-config', path ,'--benchmark',
-                              '-max_n', '100000'])
+    f = open("./outputFile","wb")
+    bench = subprocess.Popen([Common.CONFIG.binary_full_path,
+                              '-config', path ,'--benchmark',
+                              '-scenario','master, set, set_tx, get',
+                              '-max_n', '20000'], stdout=f,stderr=f)
     time.sleep(10.0) # give it time to get up to speed
     rc = bench.returncode
     if rc <> None:
         raise Exception ("benchmark should not have finished yet.")
 
-    for i in range(100):
-        Common.rotate_log(node, 1, False)
+    cfg = Common.getConfig(node)
+    log_dir = cfg['log_dir']
+    log_file = '/'.join([log_dir, "%s.log" % node ])
+
+    def target(i):
+        fn = "%s.%03i" % (log_file,i)
+        return fn
+
+    for i in xrange(100):
+        new_file = target(i)
+        print "%s => %s" % (log_file, new_file)
+        os.rename (log_file, new_file)
+        Common.send_signal(node,signal.SIGUSR1)
         time.sleep(0.2)
         Common.assert_running_nodes(1)
         rc = bench.returncode
@@ -51,5 +67,24 @@ def test_272():
     Common.assert_running_nodes(1)
     logging.info("now wait for benchmark to finish")
     rc = bench.wait()
+    f.close()
+    Common.assert_running_nodes(1)
     if rc <> 0:
         raise Exception("benchmark exited with rc = %s" % rc)
+
+    # now check there are no holes.
+
+    seqn = -1
+    for i in xrange(100):
+        fn = target(i)
+        with open(fn,'r') as f:
+            lines = f.readlines()
+            for line in lines:
+                parts =  line.split('-')
+                if len(parts) > 5:
+                    s = parts[4]
+                    seqn_next = int(s)
+                    assert_equals(seqn_next, seqn + 1,
+                                  msg = "%s:sequence numbers do not follow: %s;%s" % (fn, seqn , seqn_next))
+                    seqn = seqn_next
+    logging.info("last seqn:%i" % seqn)

From f96fb9593c8a22d9620386c8a92a738baefc1a0e Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Thu, 11 Feb 2016 14:47:15 +0100
Subject: [PATCH 124/373] fix test_272

---
 src/node/node_main.ml       | 28 +++++++++++++++----------
 src/tools/arakoon_logger.ml | 42 ++++++++++++++++++++++++++++---------
 src/tools/arakoon_redis.ml  | 29 +++++++------------------
 3 files changed, 56 insertions(+), 43 deletions(-)

diff --git a/src/node/node_main.ml b/src/node/node_main.ml
index 4b05098e..8a804d84 100644
--- a/src/node/node_main.ml
+++ b/src/node/node_main.ml
@@ -26,12 +26,7 @@ open Statistics
 let section = Logger.Section.main
 
 
-let _config_logging me get_cfgs =
-  get_cfgs () >>= fun cluster_cfg ->
-  let cfg =
-    try List.find (fun c -> c.node_name = me) cluster_cfg.cfgs
-    with Not_found -> failwith ("Could not find config for node " ^ me )
-  in
+let _config_log_levels me cluster_cfg cfg =
   let log_config =
     match cfg.log_config with
       | None -> Node_cfg.Node_cfg.get_default_log_config ()
@@ -60,7 +55,15 @@ let _config_logging me get_cfgs =
   let () = set_level Client_protocol.section log_config.client_protocol in
   let () = set_level Multi_paxos.section log_config.paxos in
   let () = set_level Tcp_messaging.section log_config.tcp_messaging in
+  Lwt.return ()
 
+let _config_logging me get_cfgs =
+  get_cfgs () >>= fun cluster_cfg ->
+  let cfg =
+    try List.find (fun c -> c.node_name = me) cluster_cfg.cfgs
+    with Not_found -> failwith ("Could not find config for node " ^ me )
+  in
+  _config_log_levels me cluster_cfg cfg >>= fun () ->
   if not cfg.is_test
   then
     Arakoon_logger.setup_log_sinks cfg.log_sinks >>= fun () ->
@@ -148,13 +151,16 @@ let _config_service ?ssl_context ~tcp_keepalive cfg stop backend =
   let uber_service () = Lwt_list.iter_p (fun f -> f ()) services in
   uber_service
 
-let _log_rotate cfg i get_cfgs =
+let _log_rotate me i get_cfgs =
   Logger.warning_f_ "received USR1 (%i) going to close/reopen log file" i
   >>= fun () ->
-  let logger = !Lwt_log.default in
-  _config_logging cfg get_cfgs >>= fun _ ->
-  Lwt_log.close logger >>= fun () ->
-  Lwt.return ()
+  get_cfgs () >>= fun cluster_cfg ->
+  let cfg =
+    try List.find (fun c -> c.node_name = me) cluster_cfg.cfgs
+    with Not_found -> failwith ("Could not find config for node " ^ me )
+  in
+  _config_log_levels me cluster_cfg cfg >>= fun () ->
+  !Arakoon_logger.reopen_loggers ()
 
 let log_prelude cluster_cfg =
   Logger.info_ "--- NODE STARTED ---" >>= fun () ->
diff --git a/src/tools/arakoon_logger.ml b/src/tools/arakoon_logger.ml
index d905611f..b5f3e61d 100644
--- a/src/tools/arakoon_logger.ml
+++ b/src/tools/arakoon_logger.ml
@@ -18,13 +18,22 @@ open Lwt.Infix
 
 let file_logger file_name =
   let flags = [Unix.O_WRONLY; Unix.O_CREAT; Unix.O_APPEND; Unix.O_NONBLOCK] in
-  Lwt_unix.openfile file_name flags 0o640 >>= fun fd ->
-  Lwt_unix.set_close_on_exec fd;
+  let fd_ref = ref None in
+  let get_fd () =
+    match !fd_ref with
+    | None ->
+       Lwt_unix.openfile file_name flags 0o640 >>= fun fd ->
+       fd_ref := Some fd;
+       Lwt.return fd
+    | Some fd ->
+       Lwt.return fd
+  in
   let mutex = Lwt_mutex.create () in
   let lg_output line =
     Lwt_mutex.with_lock
       mutex
       (fun () ->
+       get_fd () >>= fun fd ->
        let rec inner offset len =
          Lwt_unix.write fd line offset len >>= fun written ->
          if written = len
@@ -35,8 +44,17 @@ let file_logger file_name =
        Lwt_unix.write fd "\n" 0 1 >>= fun _ ->
        Lwt.return_unit)
   in
-  let close () = Lwt_unix.close fd in
-  Lwt.return (lg_output, close)
+  let reopen () =
+    Lwt_mutex.with_lock
+      mutex
+      (fun () ->
+       match !fd_ref with
+       | None -> Lwt.return_unit
+       | Some fd ->
+          fd_ref := None;
+          Lwt_unix.close fd)
+  in
+  Lwt.return (lg_output, reopen)
 
 let console_logger =
   let mutex = Lwt_mutex.create () in
@@ -47,8 +65,8 @@ let console_logger =
        Lwt_io.write Lwt_io.stdout line >>= fun () ->
        Lwt_io.write_char Lwt_io.stdout '\n')
   in
-  let close () = Lwt.return_unit in
-  lg_output, close
+  let reopen () = Lwt.return_unit in
+  lg_output, reopen
 
 
 let format_message
@@ -87,6 +105,8 @@ let format_message
     (Lwt_log.string_of_level level)
     (String.concat "\n" lines)
 
+let reopen_loggers = ref (fun () -> Lwt.return ())
+
 let setup_log_sinks log_sinks =
   Lwt_list.map_p
     (let open Arakoon_log_sink in
@@ -119,10 +139,12 @@ let setup_log_sinks log_sinks =
                           Lwt_list.iter_p
                             (fun (lg_output, _) -> lg_output logline)
                             loggers)
-                 ~close:(fun () ->
-                         Lwt_list.iter_p
-                           (fun (_, close) -> close ())
-                           loggers)
+                 ~close:(fun () -> Lwt.return ())
   in
+  reopen_loggers :=
+    (fun () ->
+     Lwt_list.iter_p
+       (fun (_, reopen) -> reopen ())
+       loggers);
   Lwt_log.default := logger;
   Lwt.return ()
diff --git a/src/tools/arakoon_redis.ml b/src/tools/arakoon_redis.ml
index df06b9a0..9abb6a88 100644
--- a/src/tools/arakoon_redis.ml
+++ b/src/tools/arakoon_redis.ml
@@ -19,7 +19,6 @@ open Lwt.Infix
 let make_redis_logger ~host ~port ~key =
   let open Lwt_buffer in
   let buffer = Lwt_buffer.create_fixed_capacity 100 in
-  let stop = ref false in
   let lg_output line =
     if Lwt_buffer.is_full buffer
     then
@@ -30,10 +29,7 @@ let make_redis_logger ~host ~port ~key =
         line
         buffer
   in
-  let close () =
-    stop := true;
-    Lwt.return_unit
-  in
+  let reopen () = Lwt.return_unit in
   let redis_t () =
     let module R = Redis_lwt.Client in
     let rec outer () =
@@ -41,29 +37,18 @@ let make_redis_logger ~host ~port ~key =
         (fun () ->
          let rec inner client =
            (* TODO push multiple items in one go? *)
-           if !stop
-           then
-             Lwt.fail Lwt.Canceled
-           else
-             begin
-               Lwt_buffer.take buffer >>= fun logline ->
-               R.rpush client key logline >>= fun _list_length ->
-               inner client
-             end
+           Lwt_buffer.take buffer >>= fun logline ->
+           R.rpush client key logline >>= fun _list_length ->
+           inner client
          in
          R.with_connection
            R.({ host ;port; })
            inner)
         (fun _ -> Lwt.return ()) >>= fun () ->
-      if !stop
-      then Lwt.return ()
-      else
-        begin
-          Lwt_unix.sleep 1. >>=
-          outer
-        end
+      Lwt_unix.sleep 1. >>= fun () ->
+      outer ()
     in
     outer ()
   in
   Lwt.ignore_result (redis_t ());
-  lg_output, close
+  lg_output, reopen

From aa4f6d1e1f7b28500a8970fefeb8655a83e27b70 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Thu, 11 Feb 2016 15:18:05 +0100
Subject: [PATCH 125/373] really fix test_272

---
 pylabs/test/server/shaky/test_272.py | 9 ++++++---
 src/tools/arakoon_logger.ml          | 2 +-
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/pylabs/test/server/shaky/test_272.py b/pylabs/test/server/shaky/test_272.py
index 961e2e5f..97cb5085 100644
--- a/pylabs/test/server/shaky/test_272.py
+++ b/pylabs/test/server/shaky/test_272.py
@@ -39,7 +39,7 @@ def test_272():
     bench = subprocess.Popen([Common.CONFIG.binary_full_path,
                               '-config', path ,'--benchmark',
                               '-scenario','master, set, set_tx, get',
-                              '-max_n', '20000'], stdout=f,stderr=f)
+                              '-max_n', '60000'], stdout=f,stderr=f)
     time.sleep(10.0) # give it time to get up to speed
     rc = bench.returncode
     if rc <> None:
@@ -80,11 +80,14 @@ def target(i):
         with open(fn,'r') as f:
             lines = f.readlines()
             for line in lines:
-                parts =  line.split('-')
-                if len(parts) > 5:
+                try:
+                    parts =  line.split(" - ")
                     s = parts[4]
                     seqn_next = int(s)
                     assert_equals(seqn_next, seqn + 1,
                                   msg = "%s:sequence numbers do not follow: %s;%s" % (fn, seqn , seqn_next))
                     seqn = seqn_next
+                except Exception as ex:
+                    logging.info("Error while parsing line %s" % line)
+                    raise ex
     logging.info("last seqn:%i" % seqn)
diff --git a/src/tools/arakoon_logger.ml b/src/tools/arakoon_logger.ml
index b5f3e61d..32646f70 100644
--- a/src/tools/arakoon_logger.ml
+++ b/src/tools/arakoon_logger.ml
@@ -103,7 +103,7 @@ let format_message
     component
     seqnum
     (Lwt_log.string_of_level level)
-    (String.concat "\n" lines)
+    (String.concat "; " lines)
 
 let reopen_loggers = ref (fun () -> Lwt.return ())
 

From ca992a95e756cf6ac0b0c871a6b031ffd1ccd6de Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Thu, 11 Feb 2016 16:05:34 +0100
Subject: [PATCH 126/373] a better file name for the incoming files

---
 src/client/remote_nodestream.ml |  8 ++++----
 src/tlog/tlc2.ml                | 25 ++++++++++++-------------
 2 files changed, 16 insertions(+), 17 deletions(-)

diff --git a/src/client/remote_nodestream.ml b/src/client/remote_nodestream.ml
index 936a56e3..f460629f 100644
--- a/src/client/remote_nodestream.ml
+++ b/src/client/remote_nodestream.ml
@@ -103,13 +103,13 @@ class remote_nodestream ((ic,oc) as conn) =
         | 3 ->
            begin
              Logger.debug_f_ "save_file" >>= fun () ->
-             Llio.input_string ic >>= fun name ->
+             Llio.input_string ic >>= fun extension ->
              Sn.input_sn ic       >>= fun start_i ->
              Llio.input_int64 ic  >>= fun length ->
-             Logger.info_f_ "retrieving start_i:%s (name:%s, %Li bytes)"
-                            (Sn.string_of start_i) name length
+             Logger.info_f_ "retrieving start_i:%s (extension:%s, %Li bytes)"
+                            (Sn.string_of start_i) extension length
              >>= fun () ->
-             tlog_coll # save_tlog_file start_i name length ic >>= fun () ->
+             tlog_coll # save_tlog_file start_i extension length ic >>= fun () ->
              loop_parts ()
            end
         | x  -> Llio.lwt_failfmt "don't know what %i means" x
diff --git a/src/tlog/tlc2.ml b/src/tlog/tlc2.ml
index 1c43d5e3..f24e2329 100644
--- a/src/tlog/tlc2.ml
+++ b/src/tlog/tlc2.ml
@@ -514,14 +514,15 @@ class tlc2
       in
       Logger.debug_f_ "n = %03i => canonical=%s" n canonical >>= fun () ->
       let name = Filename.basename canonical in
+      let extension = extension_of name in
       let start_i = TlogMap.get_start_i tlog_map n in
       match start_i with
       | None -> failwith (Printf.sprintf "tlogcollection doesn't know tlog:%i" n)
       | Some start_i ->
          begin
-           Llio.output_string oc name >>= fun () ->
-           Sn.output_sn oc start_i    >>= fun () ->
-           File_system.stat canonical >>= fun stats ->
+           Llio.output_string oc extension >>= fun () ->
+           Sn.output_sn oc start_i         >>= fun () ->
+           File_system.stat canonical      >>= fun stats ->
            let length = Int64.of_int(stats.Unix.st_size)
                                     (* why don't they use largefile ? *)
            in
@@ -543,25 +544,23 @@ class tlc2
          _file <- None;
          Lwt.return_unit
 
-    method save_tlog_file first_i remote_name length ic =
+    method save_tlog_file first_i extension length ic =
       (* what with rotation (jump to new tlog), open streams, ...*)
       self # _maybe_close_current_file ()
-      >>= fun () ->                                                               
-      let full_name = TlogMap.get_full_path tlog_map remote_name in
-      let extension = Tlog_map.extension_of remote_name in
-      let tmp = full_name ^ ".tmp" in
+      >>= fun () ->
+      let new_outer = TlogMap.new_outer tlog_map first_i in
+      let new_name = Printf.sprintf "%03i%s" new_outer extension in
+      let canon = TlogMap.get_full_path tlog_map new_name in
+      let tmp = canon ^ ".tmp" in
       
       Logger.info_f_
-        "save_tlog_file: first_i:%s remote_name:%s in %s"
-        (Sn.string_of first_i) remote_name tmp
+        "save_tlog_file: first_i:%s extension:%s in %s"
+        (Sn.string_of first_i) extension tmp
       >>= fun () ->
       
       Lwt_io.with_file ~mode:Lwt_io.output tmp
                        (fun oc -> Llio.copy_stream ~length ~ic ~oc)
       >>= fun () ->
-      let new_outer = TlogMap.new_outer tlog_map first_i in
-      let new_name = Printf.sprintf "%03i%s" new_outer extension in
-      let canon = TlogMap.get_full_path tlog_map new_name in
       File_system.rename tmp canon >>= fun () ->
       let () = TlogMap.set_should_roll tlog_map in
       Lwt.return_unit

From 7aa65bd5316c1d2c112d1cacede629ea57131895 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Fri, 12 Feb 2016 09:33:16 +0100
Subject: [PATCH 127/373] ignore signals sigusr1 & sigusr2 for local cli
 actions

---
 src/main/arakoon.ml | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/main/arakoon.ml b/src/main/arakoon.ml
index d7dc6683..6ca1c27e 100644
--- a/src/main/arakoon.ml
+++ b/src/main/arakoon.ml
@@ -555,8 +555,12 @@ let main () =
 
   let exit_code =
     match !action with
-      | LocalAction la -> do_local ~tls la
-      | ServerAction sa -> do_server sa
+    | LocalAction la ->
+       List.iter
+         (fun s -> Sys.set_signal s Sys.Signal_ignore)
+         [ Sys.sigusr1; Sys.sigusr2 ];
+       do_local ~tls la
+    | ServerAction sa -> do_server sa
   in
   (* let () = Printf.printf "[rc=%i]\n" rc in *)
   let () =

From c34efb424cd43675f6d39a2e3976a021bbdd7437 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Fri, 12 Feb 2016 10:41:53 +0100
Subject: [PATCH 128/373] update travis icon for 1.9 branch

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index da8efd45..a8cd5c8f 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
 Arakoon
 =======
-[![](https://travis-ci.org/openvstorage/arakoon.svg?branch=1.8)](https://travis-ci.org/openvstorage/arakoon)
+[![](https://travis-ci.org/openvstorage/arakoon.svg?branch=1.9)](https://travis-ci.org/openvstorage/arakoon)
 
 Arakoon is a distributed key/value store with preference for consistency. More technically, it's a [Multi-Paxos](http://en.wikipedia.org/wiki/Paxos_%28computer_science%29#Multi-Paxos) implementation (written in [OCaml](http://ocaml.org/)) running on top of [TokyoCabinet](http://fallabs.com/tokyocabinet/).
 

From be6d6de96833941b33372997a1a3f130a03e96a1 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Fri, 12 Feb 2016 14:09:50 +0100
Subject: [PATCH 129/373] tlog_rollover: test_size/10; tlog_size/100

---
 pylabs/test/server/right/system_tests_long_right.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/pylabs/test/server/right/system_tests_long_right.py b/pylabs/test/server/right/system_tests_long_right.py
index 27f45674..9422eb6d 100644
--- a/pylabs/test/server/right/system_tests_long_right.py
+++ b/pylabs/test/server/right/system_tests_long_right.py
@@ -87,12 +87,12 @@ def test_20_clients_1000_sets() :
     arakoon.ArakoonProtocol.ARA_CFG_TIMEOUT = 60.0
     Common.create_and_wait_for_threads ( 20, 1000, Common.simple_set, 200.0 )
 
-@Common.with_custom_setup( Common.setup_3_nodes, Common.basic_teardown)
+@Common.with_custom_setup( Common.setup_3_nodes_mini, Common.basic_teardown)
 def test_tlog_rollover():
-    Common.iterate_n_times( 150000, Common.simple_set )
+    Common.iterate_n_times( 15000, Common.simple_set )
     Common.stop_all()
     Common.start_all()
-    Common.iterate_n_times( 150000, Common.simple_set )
+    Common.iterate_n_times( 15000, Common.simple_set )
 
 @Common.with_custom_setup( Common.setup_2_nodes, Common.basic_teardown)
 def test_catchup_while_collapsing():

From 9f22cc5a5906fa14a9801bf30affbf171970415e Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Fri, 12 Feb 2016 14:37:10 +0100
Subject: [PATCH 130/373] fix test; fix mem_tlog_collection

---
 src/system/startup.ml          | 17 ++++++++++++++---
 src/tlog/mem_tlogcollection.ml |  4 ++--
 2 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/src/system/startup.ml b/src/system/startup.ml
index c7e3d1a7..87cba5de 100644
--- a/src/system/startup.ml
+++ b/src/system/startup.ml
@@ -295,13 +295,15 @@ let ahead_master_loses_role () =
     }
   in
   let get_cfgs () = cluster_cfg |> Lwt.return in
+  
   let v0 = Value.create_master_value ~lease_start:0. node0 in
   let v1 = Value.create_client_value [Update.Set("xxx","xxx")] false in
   let v2 = Value.create_client_value [Update.Set("invalidkey", "shouldnotbepresent")] false in
   let tlcs = Hashtbl.create 5 in
   let stores = Hashtbl.create 5 in
   let now = Unix.gettimeofday () in
-
+  let dump_tlogs () = Lwt_list.iter_s (_dump_tlc ~tlcs) [node0;node1;node2]
+  in
   let t_node0 = _make_run ~stores ~tlcs ~now ~get_cfgs ~values:[v0;v0] node0 () in
   let t_node1 = _make_run ~stores ~tlcs ~now ~get_cfgs ~values:[v0;v0;v1] node1 () in
   let run_previous_master = _make_run ~stores ~tlcs ~now ~get_cfgs ~values:[v0;v0;v1;v2] node2 in
@@ -309,7 +311,16 @@ let ahead_master_loses_role () =
   Lwt.ignore_result t_node0;
   Lwt.ignore_result t_node1;
   (* sleep a bit so the previous 2 slaves can make progress *)
-  Lwt_unix.sleep ((float lease_period) *. 1.5) >>= fun () ->
+  Lwt_unix.sleep ((float lease_period) *. 3.0) >>= fun () ->
+  Lwt.catch
+    (fun () -> Client_main.find_master cluster_cfg ~tls:None >>= fun master ->
+               Lwt.return_unit)
+    (fun exn ->
+      Logger.fatal_f_ ~exn "NO MASTER" >>= fun () ->
+      dump_tlogs () >>= fun () ->
+      Lwt.fail exn
+    )
+  >>= fun () ->
   let t_previous_master = run_previous_master () in
   Lwt.ignore_result t_previous_master;
   (* allow previous master to catch up with the others *)
@@ -325,7 +336,7 @@ let ahead_master_loses_role () =
     OUnit.assert_bool (Printf.sprintf "value for '%s' should not be in store" key) (not exists);
     Lwt.return ()
   in
-  Lwt_list.iter_s (_dump_tlc ~tlcs)   [node0;node1;node2]>>= fun () ->
+  dump_tlogs () >>= fun () ->
   Lwt_list.iter_s check_store [node0;node1;node2]
 
 
diff --git a/src/tlog/mem_tlogcollection.ml b/src/tlog/mem_tlogcollection.ml
index fa3d617e..e428a135 100644
--- a/src/tlog/mem_tlogcollection.ml
+++ b/src/tlog/mem_tlogcollection.ml
@@ -85,7 +85,7 @@ class mem_tlog_collection _name =
 
     method get_tlog_from_i _ = 0
     method get_start_i n = Some Sn.zero
-    method is_rollover_point _ = failwith "is_rollover_point not supported"
+    method is_rollover_point _ = false
     method next_rollover _ = Some Int64.max_int
 
     method log_value_explicit i (v:Value.t) ~sync marker =
@@ -100,7 +100,7 @@ class mem_tlog_collection _name =
         Lwt.return ()
 
     method remove_below _i = Lwt.return ()
-    method complete_file_to_deliver = failwith "complete_tlog_file_to_deliver not supported"
+    method complete_file_to_deliver _ = None
   end
 
 let make_mem_tlog_collection

From c426f23e5b29ecd6052cd3a4c315d286329edc8b Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Fri, 12 Feb 2016 14:59:23 +0100
Subject: [PATCH 131/373] make_release: 1.9.0

---
 META                      | 2 +-
 debian/changelog          | 5 +++++
 redhat/SPECS/arakoon.spec | 4 +++-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/META b/META
index 299fdb83..6dddd571 100644
--- a/META
+++ b/META
@@ -1,7 +1,7 @@
 # -*- conf -*-
 
 description = "Arakoon client"
-version = "1.8.13"
+version = "1.9.0"
 exists_if = "arakoon_client.cma,arakoon_client.cmxa,arakoon_client.cmxs"
 requires = "threads lwt lwt.unix str oUnit quickcheck lwt.ssl ocplib-endian redis.lwt uri"
 archive(byte) = "arakoon_client.cma tcp_keepalive_sockopt_stubs.o"
diff --git a/debian/changelog b/debian/changelog
index ba1f7a0b..81adadd5 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,8 @@
+arakoon (1.9.0) unstable; urgency=low
+  * new upstream version
+
+ -- Romain Slootmaekers <romain.slootmaekers@openvstorage.com>  Fri, 12 Feb 2016 14:59:23 +0100
+
 arakoon (1.8.13) unstable; urgency=low
   * new upstream version
 
diff --git a/redhat/SPECS/arakoon.spec b/redhat/SPECS/arakoon.spec
index 756d4195..3d5f1daa 100644
--- a/redhat/SPECS/arakoon.spec
+++ b/redhat/SPECS/arakoon.spec
@@ -2,7 +2,7 @@
 
 Summary: Arakoon
 Name: arakoon
-Version: 1.8.13
+Version: 1.9.0
 Release: 3%{?dist}
 License: Apache 2.0
 Requires: libev >= 4
@@ -33,6 +33,8 @@ cp arakoon.native %{buildroot}%{_bindir}/arakoon
 %{_bindir}/arakoon
 
 %changelog
+* Fri Feb 12 2016 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.0
+- Create arakoon 1.9.0 RPM package
 * Mon Jan 25 2016 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.8.13
 - Create arakoon 1.8.13 RPM package
 * Wed Dec 23 2015 Jan Doms <jan.doms@gmail.com> - 1.8.12

From f60f63999f41ed0f6446a72c391b89cfffa90108 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Sat, 13 Feb 2016 02:25:48 +0100
Subject: [PATCH 132/373] install iproute

---
 docker/arakoon_centos7/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/arakoon_centos7/Dockerfile b/docker/arakoon_centos7/Dockerfile
index d392fc16..634ec6c6 100644
--- a/docker/arakoon_centos7/Dockerfile
+++ b/docker/arakoon_centos7/Dockerfile
@@ -66,7 +66,7 @@ RUN eval `${opam_env}` && \
 
 RUN yum -y install autoconf help2man perl-Thread-Queue
 
-RUN yum -y install sudo
+RUN yum -y install sudo iproute
 #
 # Disable "ssh hostname sudo <cmd>", because it will show the password in clear.
 #         You have to run "ssh -t hostname sudo <cmd>".

From 2fb4317a3f02dadc170d1442d6073fe0f7bf8a34 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Mon, 15 Feb 2016 18:54:58 +0100
Subject: [PATCH 133/373] let client deliver consistency class

---
 pylabs/pyrakoon                                |  2 +-
 pylabs/test/server/quick/system_tests_basic.py |  7 ++++---
 pylabs/test/server/quick/test_readonly.py      | 16 +++++++++++++---
 3 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/pylabs/pyrakoon b/pylabs/pyrakoon
index ac1386c2..2d5d99cf 160000
--- a/pylabs/pyrakoon
+++ b/pylabs/pyrakoon
@@ -1 +1 @@
-Subproject commit ac1386c2e1777b9e3222350c510581c9e0a39cbc
+Subproject commit 2d5d99cf7864d3e19b9dfaf882feffb2387703ae
diff --git a/pylabs/test/server/quick/system_tests_basic.py b/pylabs/test/server/quick/system_tests_basic.py
index 000afb17..c5629553 100644
--- a/pylabs/test/server/quick/system_tests_basic.py
+++ b/pylabs/test/server/quick/system_tests_basic.py
@@ -25,7 +25,6 @@
 from Compat import X
 
 CONFIG = C.CONFIG
-from arakoon.ArakoonProtocol import AtLeast
 
 try:
     assert_in
@@ -478,13 +477,15 @@ def test_consistency():
     client.set('x','X')
     client.set('z','Z')
     m = client.get_txid()
-    logging.debug("m = %s", m)
+    logging.debug("m = %s,class:%s", m, m.__class__)
     assert_equals(str(m).find("AtLeast"),0)
     time.sleep(5)
     client.setConsistency(m)
     v = client.get('x')
     assert_equals(v,'X')
-    m2 = AtLeast(1000)
+
+    
+    m2 = X.arakoon_client.AtLeast(1000)
     client.setConsistency(m2)
     try:
         v2 = client.get('z')
diff --git a/pylabs/test/server/quick/test_readonly.py b/pylabs/test/server/quick/test_readonly.py
index 0575f9f1..9710b236 100644
--- a/pylabs/test/server/quick/test_readonly.py
+++ b/pylabs/test/server/quick/test_readonly.py
@@ -17,9 +17,10 @@
 
 
 from .. import system_tests_common as C
-
+from Compat import X
 from nose.tools import *
-from arakoon.ArakoonExceptions import *
+import time
+import logging
 
 @C.with_custom_setup(C.setup_1_node, C.basic_teardown)
 def test_read_only():
@@ -30,7 +31,16 @@ def test_read_only():
    cluster.stop()
    cluster.setReadOnly()
    cluster.start()
-   assert_raises(ArakoonException, client.set, 'xxx','yyy')
+   time.sleep(10)
+   client = C.get_client()
+   try:
+      client.set('xxx','yyy')
+      logging.fatal("we should not get here")
+      raise Exception ("should fail")
+   except Exception, e:
+      logging.info("exception :%s class:%s", e, e.__class__)
+      assert_true(isinstance(e,X.arakoon_client.ArakoonException))
+                  
    xxx = client['xxx']
    assert_equals(xxx,v)
    

From b106b1d37f27c267edb9e8f29c5d71f1e20e3831 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Tue, 16 Feb 2016 13:59:40 +0100
Subject: [PATCH 134/373] fix etcd.retrieve_value + put it under test

---
 src/all_test.ml                 |  1 +
 src/node/node_cfg.ml            |  4 +--
 src/nursery/client_cfg.ml       |  4 +--
 src/tools/arakoon_config_url.ml |  6 ++++
 src/tools/arakoon_etcd.ml       | 26 +++++++-------
 src/tools/arakoon_etcd_test.ml  | 62 +++++++++++++++++++++++++++++++++
 6 files changed, 84 insertions(+), 19 deletions(-)
 create mode 100644 src/tools/arakoon_etcd_test.ml

diff --git a/src/all_test.ml b/src/all_test.ml
index d844df97..12b03db3 100644
--- a/src/all_test.ml
+++ b/src/all_test.ml
@@ -91,4 +91,5 @@ let suite = "universe" >::: [
     Node_cfg_test.suite;
     Std.List.Test.suite;
     Arakoon_log_sink_test.suite;
+    Arakoon_etcd_test.suite;
   ]
diff --git a/src/node/node_cfg.ml b/src/node/node_cfg.ml
index 24599b44..48f05735 100644
--- a/src/node/node_cfg.ml
+++ b/src/node/node_cfg.ml
@@ -771,7 +771,7 @@ module Node_cfg = struct
             };
         }
       in
-      cluster_cfg |> Lwt.return
+      cluster_cfg
 
   let node_name t = t.node_name
 
@@ -781,7 +781,7 @@ module Node_cfg = struct
 
   let get_master t = t.master
 
-  let retrieve_cfg url = Arakoon_etcd.retrieve_cfg _retrieve_cfg_from_txt url
+  let retrieve_cfg url = Arakoon_config_url.retrieve url >|= _retrieve_cfg_from_txt
 
   let retrieve_client_cfg url = retrieve_cfg url >>= fun cfg -> Lwt.return (to_client_cfg cfg)
 
diff --git a/src/nursery/client_cfg.ml b/src/nursery/client_cfg.ml
index 7723daf2..799cc590 100644
--- a/src/nursery/client_cfg.ml
+++ b/src/nursery/client_cfg.ml
@@ -87,8 +87,6 @@ module ClientCfg = struct
     let inifile = new Inifiles.inifile txt in
     from_ini section inifile
 
-  let _from_txt_lwt section txt =_from_txt section txt |> Lwt.return
-
-  let from_url section url = Arakoon_etcd.retrieve_cfg (_from_txt_lwt section) url
+  let from_url section url = Arakoon_config_url.retrieve url >|= (_from_txt section)
 
 end
diff --git a/src/tools/arakoon_config_url.ml b/src/tools/arakoon_config_url.ml
index e0bc2c9f..97f851a6 100644
--- a/src/tools/arakoon_config_url.ml
+++ b/src/tools/arakoon_config_url.ml
@@ -52,3 +52,9 @@ let make string =
      in
      Etcd (peers, path)
   | _ -> failwith (Printf.sprintf "unknown protocol:%s" protocol)
+
+let retrieve = function
+  | File f ->
+     Lwt_extra.read_file f
+  | Etcd (peers, path) ->
+     Arakoon_etcd.retrieve_value peers path
diff --git a/src/tools/arakoon_etcd.ml b/src/tools/arakoon_etcd.ml
index 85ca3b4e..9334d41e 100644
--- a/src/tools/arakoon_etcd.ml
+++ b/src/tools/arakoon_etcd.ml
@@ -37,20 +37,18 @@ let retrieve_value peers path =
             ]
   in
   let cmd_s = String.concat " " cmd in
-  Lwt_io.eprintlf "cmd_s: %s" cmd_s >>= fun () ->
   Lwt_log.info_f  "ETCD: %s" cmd_s >>= fun () ->
   let command = Lwt_process.shell cmd_s in
-  Lwt_process.with_process_in
-    command
-    (fun p ->
-     let stream = Lwt_io.read_lines (p # stdout) in
-     Lwt_stream.to_list stream >>= fun lines ->
-     let txt = String.concat "\n" lines in
-     Lwt.return txt
-    )
+  Lwt_process.pread command >>= fun value ->
+  (* etcdctl adds a '\n' after the value, cut it off *)
+  String.sub value 0 (String.length value - 1) |> Lwt.return
 
-let retrieve_cfg (p:string -> 'a Lwt.t) = function
-  | Arakoon_config_url.File cfg_file ->
-     Lwt_extra.read_file cfg_file >>= p
-  | Arakoon_config_url.Etcd (peers,path)->
-     retrieve_value peers path >>= p
+let store_value peers path value =
+  let peers_s = String.concat "," (List.map (fun (h,p) -> Printf.sprintf "%s:%i" h p) peers) in
+  let cmd = [| "etcdctl";
+               "--peers=" ^ peers_s;
+               "set";
+               path;
+              |]
+  in
+  Lwt_process.pwrite ("", cmd) value
diff --git a/src/tools/arakoon_etcd_test.ml b/src/tools/arakoon_etcd_test.ml
new file mode 100644
index 00000000..3cd64480
--- /dev/null
+++ b/src/tools/arakoon_etcd_test.ml
@@ -0,0 +1,62 @@
+(*
+Copyright 2016 iNuron NV
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+ *)
+
+open Lwt.Infix
+
+let test_get_set () =
+  let t () =
+    let etcd =
+      Lwt_process.open_process_none
+        ("", [| "etcd";
+                "--listen-client-urls"; "http://127.0.0.1:2379";
+                "--advertise-client-urls"; "http://127.0.0.1:2379";
+               |])
+    in
+    Lwt_unix.sleep 0.3 >>= fun () ->
+    Lwt.finalize
+      (fun () ->
+       let peers = [ "127.0.0.1", 2379 ] in
+       let key = "/test/etcd/get_set" in
+       let test value =
+         Arakoon_etcd.store_value peers key value >>= fun () ->
+         Arakoon_etcd.retrieve_value peers key >>= fun value' ->
+         Printf.printf "expected = %S, actual = %S\n"
+                       value value';
+         assert (value = value');
+         Lwt.return ()
+       in
+       let values =
+         [ "lala";
+           "lala\r\n";
+           "la la";
+           "la\r\n la";
+           "la ' \\ \' \" \n \" la";
+         ]
+       in
+       Lwt_list.iter_s
+         test
+         values)
+      (fun () ->
+       etcd # terminate;
+       Lwt.return ())
+  in
+  Lwt_main.run (t ())
+
+open OUnit
+
+let suite = "etcd_test" >::: [
+    "test_get_set" >:: test_get_set;
+  ]

From e392f1c61adc0a65da5508848968143c5ac672d3 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Tue, 16 Feb 2016 15:10:56 +0100
Subject: [PATCH 135/373] add test_etcd_down & test_get_non_existing_key + fix
 the behavior

---
 src/tools/arakoon_etcd.ml      |  26 ++++++--
 src/tools/arakoon_etcd_test.ml | 118 ++++++++++++++++++++++-----------
 2 files changed, 101 insertions(+), 43 deletions(-)

diff --git a/src/tools/arakoon_etcd.ml b/src/tools/arakoon_etcd.ml
index 9334d41e..374c0b3a 100644
--- a/src/tools/arakoon_etcd.ml
+++ b/src/tools/arakoon_etcd.ml
@@ -23,6 +23,13 @@ let parse_url url =
   let peers = [host,port] in
   peers, path
 
+exception ProcessFailure of Unix.process_status
+
+let verify_process_status p =
+  p # status >>= function
+  | Unix.WEXITED 0 -> Lwt.return_unit
+  | s -> Lwt.fail (ProcessFailure s)
+
 let retrieve_value peers path =
   (*
     curl -X GET "http://127.0.0.1.:5000/v2/keys/path_to/asd/055a61c0_0"
@@ -39,9 +46,14 @@ let retrieve_value peers path =
   let cmd_s = String.concat " " cmd in
   Lwt_log.info_f  "ETCD: %s" cmd_s >>= fun () ->
   let command = Lwt_process.shell cmd_s in
-  Lwt_process.pread command >>= fun value ->
-  (* etcdctl adds a '\n' after the value, cut it off *)
-  String.sub value 0 (String.length value - 1) |> Lwt.return
+
+  Lwt_process.with_process_in
+    command
+    (fun p ->
+     Lwt_io.read (p # stdout) >>= fun value ->
+     verify_process_status p >>= fun () ->
+     (* etcdctl adds a '\n' after the value, cut it off *)
+     String.sub value 0 (String.length value - 1) |> Lwt.return)
 
 let store_value peers path value =
   let peers_s = String.concat "," (List.map (fun (h,p) -> Printf.sprintf "%s:%i" h p) peers) in
@@ -51,4 +63,10 @@ let store_value peers path value =
                path;
               |]
   in
-  Lwt_process.pwrite ("", cmd) value
+  Lwt_log.info_f  "ETCD: %s" (Array.to_list cmd |> String.concat " ") >>= fun () ->
+  Lwt_process.with_process_out
+    ("", cmd)
+    (fun p ->
+     Lwt_io.write (p # stdin) value >>= fun () ->
+     Lwt_io.close (p # stdin) >>= fun () ->
+     verify_process_status p)
diff --git a/src/tools/arakoon_etcd_test.ml b/src/tools/arakoon_etcd_test.ml
index 3cd64480..d5aa12cc 100644
--- a/src/tools/arakoon_etcd_test.ml
+++ b/src/tools/arakoon_etcd_test.ml
@@ -16,47 +16,87 @@ limitations under the License.
 
 open Lwt.Infix
 
-let test_get_set () =
-  let t () =
-    let etcd =
-      Lwt_process.open_process_none
-        ("", [| "etcd";
-                "--listen-client-urls"; "http://127.0.0.1:2379";
-                "--advertise-client-urls"; "http://127.0.0.1:2379";
-               |])
-    in
-    Lwt_unix.sleep 0.3 >>= fun () ->
-    Lwt.finalize
-      (fun () ->
-       let peers = [ "127.0.0.1", 2379 ] in
-       let key = "/test/etcd/get_set" in
-       let test value =
-         Arakoon_etcd.store_value peers key value >>= fun () ->
-         Arakoon_etcd.retrieve_value peers key >>= fun value' ->
-         Printf.printf "expected = %S, actual = %S\n"
-                       value value';
-         assert (value = value');
+let fail_with_etcd_process_failure f =
+  Lwt.catch
+    (fun () ->
+     f () >>= fun _ ->
+     Lwt.fail_with "f did not throw!")
+    (function
+      | Arakoon_etcd.ProcessFailure _ ->
          Lwt.return ()
-       in
-       let values =
-         [ "lala";
-           "lala\r\n";
-           "la la";
-           "la\r\n la";
-           "la ' \\ \' \" \n \" la";
-         ]
-       in
-       Lwt_list.iter_s
-         test
-         values)
-      (fun () ->
-       etcd # terminate;
-       Lwt.return ())
+      | exn -> Lwt.fail exn)
+
+let test_etcd_down () =
+  let peers = [ "127.0.0.1", 2390 ] in
+  fail_with_etcd_process_failure
+    (fun () -> Arakoon_etcd.retrieve_value peers "fs") >>= fun () ->
+  fail_with_etcd_process_failure
+    (fun () -> Arakoon_etcd.store_value peers "ci" "sfdi") >>= fun () ->
+  Lwt.return ()
+
+let test_with_etcd f =
+  Lwt_process.with_process_none
+    ("", [| "pkill"; "etcd"; |])
+    (fun _ -> Lwt.return ()) >>= fun () ->
+  Lwt_process.with_process_none
+    ("", [| "rm"; "-rf"; "default.etcd"; |])
+    (fun _ -> Lwt.return ()) >>= fun () ->
+  let peers = [ "127.0.0.1", 2379 ] in
+  let etcd =
+    Lwt_process.open_process_none
+      ("", [| "etcd";
+              "--listen-client-urls"; "http://127.0.0.1:2379";
+              "--advertise-client-urls"; "http://127.0.0.1:2379";
+             |])
   in
-  Lwt_main.run (t ())
+  Lwt_unix.sleep 1. >>= fun () ->
+  Lwt.finalize
+    (fun () -> f peers)
+    (fun () ->
+     etcd # terminate;
+     Lwt.return ())
+
+let test_get_set () =
+  test_with_etcd
+    (fun peers ->
+     let key = "/test/etcd/get_set" in
+     let test value =
+       Arakoon_etcd.store_value peers key value >>= fun () ->
+       Arakoon_etcd.retrieve_value peers key >>= fun value' ->
+       Printf.printf "expected = %S, actual = %S\n"
+                     value value';
+       assert (value = value');
+       Lwt.return ()
+     in
+     let values =
+       [ "lala";
+         "lala\r\n";
+         "la la";
+         "la\r\n la";
+         "la ' \\ \' \" \n \" la";
+         (* can't set the empty string when using the cli? *)
+         (* ""; *)
+       ]
+     in
+     Lwt_list.iter_s
+       test
+       values)
+
+let test_get_non_existing_key () =
+  test_with_etcd
+    (fun peers ->
+     fail_with_etcd_process_failure
+       (fun () ->
+        Arakoon_etcd.retrieve_value
+          peers
+          (Random.int64 Int64.max_int |> Int64.to_string)))
 
 open OUnit
 
-let suite = "etcd_test" >::: [
-    "test_get_set" >:: test_get_set;
-  ]
+let suite =
+  "etcd_test" >:::
+    let t x () = Lwt_main.run (x ()) in
+    [ "test_get_set" >:: t test_get_set;
+      "test_etcd_down" >:: t test_etcd_down;
+      "test_get_non_existing_key" >:: t test_get_non_existing_key;
+    ]

From 0ab0f7d880525d4267e3efdc954e3e692307e29c Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Tue, 16 Feb 2016 15:47:00 +0100
Subject: [PATCH 136/373] fix travis (install etcd)

---
 travis.sh | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/travis.sh b/travis.sh
index 9497e5cb..4a6f5c53 100755
--- a/travis.sh
+++ b/travis.sh
@@ -51,6 +51,12 @@ install () {
     opam update
 
     opam install ${OPAM_DEPENDS}
+
+    # Install etcd:
+    curl -L  https://github.com/coreos/etcd/releases/download/v2.2.4/etcd-v2.2.4-linux-amd64.tar.gz -o etcd-v2.2.4-linux-amd64.tar.gz
+    tar xzvf etcd-v2.2.4-linux-amd64.tar.gz
+    sudo cp ./etcd-v2.2.4-linux-amd64/etcd /usr/bin
+    sudo cp ./etcd-v2.2.4-linux-amd64/etcdctl /usr/bin
 }
 
 script () {

From 83c70ae503abd4431cc556fff073342fcf93b3de Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Tue, 16 Feb 2016 17:09:37 +0100
Subject: [PATCH 137/373] add protocol for collapse

---
 pylabs/pyrakoon                      | 2 +-
 src/client/python/Arakoon.py         | 4 ++--
 src/client/python/ArakoonProtocol.py | 8 ++++++++
 3 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/pylabs/pyrakoon b/pylabs/pyrakoon
index 2d5d99cf..28868e44 160000
--- a/pylabs/pyrakoon
+++ b/pylabs/pyrakoon
@@ -1 +1 @@
-Subproject commit 2d5d99cf7864d3e19b9dfaf882feffb2387703ae
+Subproject commit 28868e4463b0b244109ebe5cb12759d89e94f603
diff --git a/src/client/python/Arakoon.py b/src/client/python/Arakoon.py
index e83ad824..97fc1e52 100644
--- a/src/client/python/Arakoon.py
+++ b/src/client/python/Arakoon.py
@@ -38,7 +38,7 @@
 #from arakoon import utils
 import utils
 
-FILTER = ''.join([(len(repr(chr(x)))==3) and chr(x) or '.' for x in range(256)])
+FILTER = ''.join([(len(repr(chr(x)))==3) and chr(x) or '.' for x in xrange(256)])
 
 def dump(src, length=8):
     N = 0
@@ -738,7 +738,7 @@ def _sendMessage(self, nodeId, msgBuffer, tryCount = -1):
         if tryCount == -1 :
             tryCount = self._config.getTryCount()
 
-        for i in range(tryCount) :
+        for i in xrange(tryCount) :
 
             if i > 0:
                 maxSleep = i * ArakoonClientConfig.getBackoffInterval()
diff --git a/src/client/python/ArakoonProtocol.py b/src/client/python/ArakoonProtocol.py
index 6f7a2179..f5ab1594 100644
--- a/src/client/python/ArakoonProtocol.py
+++ b/src/client/python/ArakoonProtocol.py
@@ -295,6 +295,7 @@ def dump(src, length=8):
 ARA_CMD_REPLACE                  = 0x00000033 | ARA_CMD_MAG
 ARA_CMD_NOP                      = 0x00000041 | ARA_CMD_MAG
 ARA_CMD_GET_TXID                 = 0x00000043 | ARA_CMD_MAG
+ARA_CMD_COLLAPSE                 = 0x00000014 | ARA_CMD_MAG
 
 # Arakoon error codes
 # Success
@@ -761,6 +762,12 @@ def encodeDeletePrefix(prefix):
         retVal += _packString(prefix)
         return retVal
 
+    @staticmethod
+    def encodeCollapse(n):
+        retVal = _packInt(ARA_CMD_COLLAPSE)
+        retVal += _packInt(n)
+        return retVal
+    
     @staticmethod
     def _evaluateErrorCode( con ):
         errorCode = _recvInt ( con )
@@ -792,6 +799,7 @@ def _evaluateErrorCode( con ):
         if errorCode != ARA_ERR_SUCCESS:
             raise ArakoonException( "EC=%d. %s" % (errorCode, errorMsg) )
 
+    
     @staticmethod
     def decodeInt64Result( con ) :
         ArakoonProtocol._evaluateErrorCode( con )

From 737b2d442aea2e149b200fb8472e0aec5957037a Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Tue, 16 Feb 2016 18:12:23 +0100
Subject: [PATCH 138/373] unify admin; add test for collapsing client

---
 pylabs/Compat.py                         | 17 ++++++++++++++++-
 pylabs/test/server/left/test_collapse.py | 22 ++++++++++++++++++++++
 2 files changed, 38 insertions(+), 1 deletion(-)

diff --git a/pylabs/Compat.py b/pylabs/Compat.py
index c4744255..216f2a6e 100644
--- a/pylabs/Compat.py
+++ b/pylabs/Compat.py
@@ -19,14 +19,28 @@ def __init__(self):
 if os.environ.has_key(VAR) and os.environ[VAR] == 'pyrakoon':
     logging.info("opting for pyrakoon")
     print "pyrakoon"
+    # TODO: This is ugly as hell...
     from pyrakoon import compat
+    from pyrakoon.protocol.admin import CollapseTlogs
     arakoon_client = compat
+    
+    class ArakoonAdmin(compat.ArakoonClient):
+
+        def collapse(self, node_id, n):
+            message = CollapseTlogs(n)
+            self._client._process(message, node_id = node_id)
+
+    class dummy:
+        pass            
+    arakoon_admin = dummy()
+    arakoon_admin.ArakoonAdmin = ArakoonAdmin
 else:
     logging.info("opting for normal client")
     print "arakoon"
     from arakoon import Arakoon
+    import arakoon.arakoon_admin
     arakoon_client = Arakoon
-
+    arakoon_admin = arakoon.arakoon_admin
 
 """
 cluster_id = 'sturdy'
@@ -227,3 +241,4 @@ def which_compat():
 
 X = which_compat()
 X.arakoon_client = arakoon_client
+X.arakoon_admin  = arakoon_admin
diff --git a/pylabs/test/server/left/test_collapse.py b/pylabs/test/server/left/test_collapse.py
index 14648dfb..f19bddbc 100644
--- a/pylabs/test/server/left/test_collapse.py
+++ b/pylabs/test/server/left/test_collapse.py
@@ -84,6 +84,27 @@ def test_remote_collapse():
         counter = counter + 1
         up2date = cli.expectProgressPossible()
     logging.info("catchup from collapsed node finished")
+
+    
+@Common.with_custom_setup(Common.setup_2_nodes_mini, Common.basic_teardown)
+def test_remote_collapse_client():
+    print 
+    cluster = Common._getCluster()
+    cluster_id = cluster._getClusterId()
+    config = cluster.getClientConfig()
+    ccfg = X.arakoon_client.ArakoonClientConfig(cluster_id,
+                                                config)
+    client = X.arakoon_admin.ArakoonAdmin(ccfg)
+    
+    zero = Common.node_names[0]
+    one = Common.node_names[1]
+    n = 29876
+    Common.iterate_n_times(n, Common.simple_set)
+    logging.info("did %i sets, now going into collapse scenario" % n)
+    
+    client.collapse(zero,1)
+    logging.info("collapsing done")
+    _check_tlog_dirs(zero,1)
     
 
 @Common.with_custom_setup(Common.setup_2_nodes_forced_master_mini, Common.basic_teardown)
@@ -127,6 +148,7 @@ def test_local_collapse():
     Common.wipe(one)
     Common.startOne(one)
     cli = Common.get_client()
+    print cli.__class__
     assert_false(cli.expectProgressPossible())
     up2date = False
     counter = 0

From b3d0bebc9b4f4a762b15fa0e2c10c684d94f87a2 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Tue, 16 Feb 2016 19:20:46 +0100
Subject: [PATCH 139/373] more robust test_272

---
 pylabs/test/server/shaky/test_272.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/pylabs/test/server/shaky/test_272.py b/pylabs/test/server/shaky/test_272.py
index 7d828d8f..0abf58ce 100644
--- a/pylabs/test/server/shaky/test_272.py
+++ b/pylabs/test/server/shaky/test_272.py
@@ -17,6 +17,7 @@
 
 
 from .. import system_tests_common as Common
+from Compat import X
 import logging
 import time
 import subprocess
@@ -59,10 +60,14 @@ def target(i):
     for i in xrange(100):
         new_file = target(i)
         print "%s => %s" % (log_file, new_file)
+        count = 0
+        while not X.fileExists(log_file) and count < 10:
+            print "%s does not exist" % log_file
+            time.sleep(0.2)
+            count += 1
+            
         os.rename (log_file, new_file)
         Common.send_signal(node,signal.SIGUSR1)
-
-        time.sleep(0.2)
         Common.assert_running_nodes(1)
         rc = bench.returncode
         if rc <> None:

From d844c4033c84657e5746030db85782bb1935f89d Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Tue, 16 Feb 2016 20:10:36 +0100
Subject: [PATCH 140/373] assert exception from correct hierarchy

---
 pylabs/test/server/right/system_tests_anomaly.py    | 4 ++--
 pylabs/test/server/right/system_tests_long_right.py | 4 +++-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/pylabs/test/server/right/system_tests_anomaly.py b/pylabs/test/server/right/system_tests_anomaly.py
index 53fe0704..93db1941 100644
--- a/pylabs/test/server/right/system_tests_anomaly.py
+++ b/pylabs/test/server/right/system_tests_anomaly.py
@@ -442,7 +442,7 @@ def validate_reelection( old_master_id) :
         new_cli = arakoon.Arakoon.ArakoonClient(
             arakoon.Arakoon.ArakoonClientConfig(CONFIG.cluster_id,
                                                 cli_cfg_only_master ) )
-        NT.assert_raises( ArakoonNoMaster, new_cli.whoMaster )
+        NT.assert_raises( X.arakoon_client.ArakoonNoMaster, new_cli.whoMaster )
         new_cli.dropConnections()
 
         return new_master_id
@@ -462,7 +462,7 @@ def validate_reelection( old_master_id) :
     rules = {"filter": block_rules}
     apply_iptables_rules(rules )
 
-    NT.assert_raises( ArakoonException, cli.set, "key", "value" )
+    NT.assert_raises( X.arakoon_client.ArakoonException, cli.set, "key", "value" )
 
     new_master_id = validate_reelection( old_master_id )
 
diff --git a/pylabs/test/server/right/system_tests_long_right.py b/pylabs/test/server/right/system_tests_long_right.py
index 9422eb6d..fa3a9929 100644
--- a/pylabs/test/server/right/system_tests_long_right.py
+++ b/pylabs/test/server/right/system_tests_long_right.py
@@ -421,6 +421,7 @@ def test_drop_master_witness_slave():
 
 @Common.with_custom_setup( Common.setup_3_nodes, Common.basic_teardown )
 def test_3_nodes_2_slaves_down ():
+    pass
     """ make sure the 'set' operation fails when 2 slaves are down, (eta: 63s) """
     cli = Common.get_client()
     cli.nop()
@@ -430,7 +431,8 @@ def test_3_nodes_2_slaves_down ():
     for slave in slaves:
         Common.stopOne( slave )
 
-    assert_raises( ArakoonSockNotReadable, cli.set, 'k', 'v' )
+    
+    assert_raises( X.arakoon_client.ArakoonSockNotReadable, cli.set, 'k', 'v' )
 
     cli.dropConnections()
 

From cf970ffb6a231cd42ff97318fb581bcc9cb0f776 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Wed, 17 Feb 2016 09:25:06 +0100
Subject: [PATCH 141/373] arakoon_etcd: allow setting the empty string

---
 src/tools/arakoon_etcd.ml      | 25 ++++++++++---------------
 src/tools/arakoon_etcd_test.ml |  3 +--
 2 files changed, 11 insertions(+), 17 deletions(-)

diff --git a/src/tools/arakoon_etcd.ml b/src/tools/arakoon_etcd.ml
index 374c0b3a..2a9be7ac 100644
--- a/src/tools/arakoon_etcd.ml
+++ b/src/tools/arakoon_etcd.ml
@@ -37,18 +37,15 @@ let retrieve_value peers path =
     etcdctl --peers=127.0.0.1:5000 get path_to/asd/055a61c0_0
    *)
   let peers_s = String.concat "," (List.map (fun (h,p) -> Printf.sprintf "%s:%i" h p) peers) in
-  let cmd = ["etcdctl";
-             "--peers=" ^ peers_s;
-             "get";
-             path
-            ]
+  let cmd = [| "etcdctl";
+               "--peers=" ^ peers_s;
+               "get";
+               path;
+              |]
   in
-  let cmd_s = String.concat " " cmd in
-  Lwt_log.info_f  "ETCD: %s" cmd_s >>= fun () ->
-  let command = Lwt_process.shell cmd_s in
-
+  Lwt_log.info_f  "ETCD: %s" (Array.to_list cmd |> String.concat " ") >>= fun () ->
   Lwt_process.with_process_in
-    command
+    ("", cmd)
     (fun p ->
      Lwt_io.read (p # stdout) >>= fun value ->
      verify_process_status p >>= fun () ->
@@ -61,12 +58,10 @@ let store_value peers path value =
                "--peers=" ^ peers_s;
                "set";
                path;
+               value;
               |]
   in
   Lwt_log.info_f  "ETCD: %s" (Array.to_list cmd |> String.concat " ") >>= fun () ->
-  Lwt_process.with_process_out
+  Lwt_process.with_process_none
     ("", cmd)
-    (fun p ->
-     Lwt_io.write (p # stdin) value >>= fun () ->
-     Lwt_io.close (p # stdin) >>= fun () ->
-     verify_process_status p)
+    (fun p -> verify_process_status p)
diff --git a/src/tools/arakoon_etcd_test.ml b/src/tools/arakoon_etcd_test.ml
index d5aa12cc..ca7f1226 100644
--- a/src/tools/arakoon_etcd_test.ml
+++ b/src/tools/arakoon_etcd_test.ml
@@ -74,8 +74,7 @@ let test_get_set () =
          "la la";
          "la\r\n la";
          "la ' \\ \' \" \n \" la";
-         (* can't set the empty string when using the cli? *)
-         (* ""; *)
+         "";
        ]
      in
      Lwt_list.iter_s

From b9e7bcfb1401a0d3ba231d651d3779e0fb52893d Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 17 Feb 2016 10:15:48 +0100
Subject: [PATCH 142/373] an unknown ahead node shouldn't kill this node

---
 src/node/catchup.ml | 24 ++++++++++++++++--------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/src/node/catchup.ml b/src/node/catchup.ml
index 3d0de525..e5df8763 100644
--- a/src/node/catchup.ml
+++ b/src/node/catchup.ml
@@ -66,12 +66,16 @@ let stop_fuse stop =
 let catchup_tlog (type s) ~tls_ctx ~tcp_keepalive ~stop other_configs ~cluster_id  mr_name ((module S : Store.STORE with type t = s),store,tlog_coll)
   =
   let current_i = tlog_coll # get_last_i () in
-  Logger.info_f_ "catchup_tlog %s" (Sn.string_of current_i) >>= fun () ->
-  let mr_cfg = List.find (fun cfg -> Node_cfg.node_name cfg = mr_name)
-                 other_configs in
-  let mr_addresses = Node_cfg.client_addresses mr_cfg
-  and mr_name = Node_cfg.node_name mr_cfg in
-  Logger.info_f_ "getting last_entries from %s" mr_name >>= fun () ->
+  Logger.info_f_ "catchup_tlog %s from %s " (Sn.string_of current_i) mr_name >>= fun () ->
+  let mr_addresses =
+    try
+      let mr_cfg =
+      List.find (fun cfg -> Node_cfg.node_name cfg = mr_name)
+                other_configs
+    in
+    Some (Node_cfg.client_addresses mr_cfg)
+    with Not_found -> None
+  in
   let head_saved_cb hfn =
     Logger.info_f_ "head_saved_cb %s" hfn >>= fun () ->
     head_saved_epilogue hfn tlog_coll >>= fun () ->
@@ -95,8 +99,12 @@ let catchup_tlog (type s) ~tls_ctx ~tcp_keepalive ~stop other_configs ~cluster_i
 
   Lwt.catch
     (fun () ->
-      _with_client_connection  ~tls_ctx ~tcp_keepalive mr_addresses copy_tlog >>= fun () ->
-      Logger.info_f_ "catchup_tlog completed"
+      match mr_addresses with
+      | Some mr_addresses ->
+         _with_client_connection  ~tls_ctx ~tcp_keepalive mr_addresses copy_tlog >>= fun () ->
+         Logger.info_f_ "catchup_tlog completed"
+      | None ->
+         Logger.warning_f_ "catchup tlog from unknown node:%S is config up to date?" mr_name
     )
     (fun exn -> Logger.warning_ ~exn "catchup_tlog failed")
   >>= fun () ->

From 0487d72c4305f1327d8b2e41415e8cc6ab313bfd Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 17 Feb 2016 10:35:00 +0100
Subject: [PATCH 143/373] log class

---
 pylabs/test/server/left/test_collapse.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pylabs/test/server/left/test_collapse.py b/pylabs/test/server/left/test_collapse.py
index f19bddbc..9080e132 100644
--- a/pylabs/test/server/left/test_collapse.py
+++ b/pylabs/test/server/left/test_collapse.py
@@ -134,7 +134,7 @@ def test_local_collapse_witness_node():
 
 @Common.with_custom_setup(Common.setup_2_nodes_mini, Common.basic_teardown)
 def test_local_collapse():
-    print "starting test_local_collapse"
+    logging.info("starting test_local_collapse")
     
     zero = Common.node_names[0]
     one = Common.node_names[1]
@@ -148,7 +148,7 @@ def test_local_collapse():
     Common.wipe(one)
     Common.startOne(one)
     cli = Common.get_client()
-    print cli.__class__
+    logging.info("cli class:%s", cli.__class__)
     assert_false(cli.expectProgressPossible())
     up2date = False
     counter = 0

From 96875951986ea463b0b576ef040814947cab4024 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 17 Feb 2016 11:01:22 +0100
Subject: [PATCH 144/373] move admin to pyrakoon compat

---
 pylabs/Compat.py | 12 +-----------
 pylabs/pyrakoon  |  2 +-
 2 files changed, 2 insertions(+), 12 deletions(-)

diff --git a/pylabs/Compat.py b/pylabs/Compat.py
index 216f2a6e..684e6c5a 100644
--- a/pylabs/Compat.py
+++ b/pylabs/Compat.py
@@ -23,17 +23,7 @@ def __init__(self):
     from pyrakoon import compat
     from pyrakoon.protocol.admin import CollapseTlogs
     arakoon_client = compat
-    
-    class ArakoonAdmin(compat.ArakoonClient):
-
-        def collapse(self, node_id, n):
-            message = CollapseTlogs(n)
-            self._client._process(message, node_id = node_id)
-
-    class dummy:
-        pass            
-    arakoon_admin = dummy()
-    arakoon_admin.ArakoonAdmin = ArakoonAdmin
+    arakoon_admin  = compat
 else:
     logging.info("opting for normal client")
     print "arakoon"
diff --git a/pylabs/pyrakoon b/pylabs/pyrakoon
index 28868e44..bc053f5b 160000
--- a/pylabs/pyrakoon
+++ b/pylabs/pyrakoon
@@ -1 +1 @@
-Subproject commit 28868e4463b0b244109ebe5cb12759d89e94f603
+Subproject commit bc053f5bf1da3cb0ba0dea48176bfb26173f357a

From 63f37a0e141189faa4643830b658b911342fa6f4 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 17 Feb 2016 14:59:04 +0100
Subject: [PATCH 145/373] valid exceptions from correct hierarchy

---
 pylabs/pyrakoon                               |  2 +-
 .../server/shaky/test_restart_master_long.py  | 30 +++++++++++--------
 pylabs/test/server/system_tests_common.py     | 19 ++++++------
 3 files changed, 28 insertions(+), 23 deletions(-)

diff --git a/pylabs/pyrakoon b/pylabs/pyrakoon
index bc053f5b..f38a7183 160000
--- a/pylabs/pyrakoon
+++ b/pylabs/pyrakoon
@@ -1 +1 @@
-Subproject commit bc053f5bf1da3cb0ba0dea48176bfb26173f357a
+Subproject commit f38a71835584584eff5ae668fd6179f0cfa21869
diff --git a/pylabs/test/server/shaky/test_restart_master_long.py b/pylabs/test/server/shaky/test_restart_master_long.py
index cec01192..95d8264a 100644
--- a/pylabs/test/server/shaky/test_restart_master_long.py
+++ b/pylabs/test/server/shaky/test_restart_master_long.py
@@ -24,40 +24,42 @@
 import subprocess
 from threading import Thread
 from nose.tools import *
+from Compat import X
 
 @Common.with_custom_setup( Common.default_setup, Common.basic_teardown )
 def test_restart_master_long ():
     restart_iter_cnt = 10
+    factor = 100
     def write_loop ():
-        Common.iterate_n_times( 100000,
+        Common.iterate_n_times( 100* factor,
                                 Common.retrying_set_get_and_delete,
                                 failure_max=2*restart_iter_cnt,
-                                valid_exceptions=[ArakoonSockNotReadable,
-                                                  ArakoonNotFound,
-                                                  ArakoonGoingDown
+                                valid_exceptions=[X.arakoon_client.ArakoonSockNotReadable,
+                                                  X.arakoon_client.ArakoonNotFound,
+                                                  X.arakoon_client.ArakoonGoingDown
                                                   ] )
 
-    valid_excs = [ArakoonNotFound,
-                  ArakoonSocketException,
-                  ArakoonGoingDown,
-                  ArakoonNoMaster]
+    valid_excs = [X.arakoon_client.ArakoonNotFound,
+                  X.arakoon_client.ArakoonSocketException,
+                  X.arakoon_client.ArakoonGoingDown,
+                  X.arakoon_client.ArakoonNoMaster]
     def range_query_loop1 ():
-        Common.heavy_range_entries_scenario (200000,
+        Common.heavy_range_entries_scenario (200 * factor,
                                              1500,
                                              1000,
                                              valid_exceptions=valid_excs)
     def range_query_loop2 ():
-        Common.heavy_range_entries_scenario (210000,
+        Common.heavy_range_entries_scenario (210 * factor ,
                                              1500,
                                              1000,
                                              valid_exceptions=valid_excs)
     def range_query_loop3 ():
-        Common.heavy_range_entries_scenario (220000,
+        Common.heavy_range_entries_scenario (220 * factor,
                                              1500,
                                              1000,
                                              valid_exceptions=valid_excs)
     def range_query_loop4 ():
-        Common.heavy_range_entries_scenario (230000,
+        Common.heavy_range_entries_scenario (230* factor,
                                              1500,
                                              1000,
                                              valid_exceptions=valid_excs)
@@ -67,12 +69,14 @@ def restart_loop ():
                                             1.5 * Common.lease_duration )
     global test_failed
     test_failed = False
+    logging.info("set_get_deletes done, now heavy scenario")
     Common.create_and_wait_for_thread_list( [restart_loop,
                                              write_loop,
                                              range_query_loop1,
                                              range_query_loop2,
                                              range_query_loop3,
-                                             range_query_loop4] )
+                                             range_query_loop4
+    ] )
 
     cli = Common.get_client()
     time.sleep(2.0)
diff --git a/pylabs/test/server/system_tests_common.py b/pylabs/test/server/system_tests_common.py
index 1ef0bab9..fe55c1a3 100644
--- a/pylabs/test/server/system_tests_common.py
+++ b/pylabs/test/server/system_tests_common.py
@@ -1032,17 +1032,18 @@ def generic_retrying_set_get_and_delete( client, key, value, is_valid_ex ):
         pass
 
 def retrying_set_get_and_delete( client, key, value ):
+    valids = (X.arakoon_client.ArakoonSockNotReadable,
+              X.arakoon_client.ArakoonSockReadNoBytes,
+              X.arakoon_client.ArakoonSockRecvError,
+              X.arakoon_client.ArakoonSockRecvClosed,
+              X.arakoon_client.ArakoonSockSendError,
+              X.arakoon_client.ArakoonNotConnected,
+              X.arakoon_client.ArakoonNodeNotMaster,
+              X.arakoon_client.ArakoonNodeNoLongerMaster,
+              )
     def validate_ex ( ex, tryCnt ):
         ex_msg = "%s" % ex
-        valids = (X.arakoon_client.ArakoonSockNotReadable,
-                  X.arakoon_client.ArakoonSockReadNoBytes,
-                  X.arakoon_client.ArakoonSockRecvError,
-                  X.arakoon_client.ArakoonSockRecvClosed,
-                  X.arakoon_client.ArakoonSockSendError,
-                  X.arakoon_client.ArakoonNotConnected,
-                  X.arakoon_client.ArakoonNodeNotMaster,
-                  X.arakoon_client.ArakoonNodeNoLongerMaster,
-              )
+        
         validEx = isinstance(ex, valids)
         if validEx:
             logging.debug( "Ignoring exception: %s", ex_msg )

From 416832b505ec516518dbeee28318d00fc79459d9 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 17 Feb 2016 15:36:16 +0100
Subject: [PATCH 146/373] pyrakoon master

---
 pylabs/pyrakoon | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pylabs/pyrakoon b/pylabs/pyrakoon
index f38a7183..b40bdb17 160000
--- a/pylabs/pyrakoon
+++ b/pylabs/pyrakoon
@@ -1 +1 @@
-Subproject commit f38a71835584584eff5ae668fd6179f0cfa21869
+Subproject commit b40bdb17179ca6b2eb2a05cb0b54e6c6f07d9aac

From b83ad6419eb29c2fcef491bba5cd3e7859491f8b Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 17 Feb 2016 18:18:16 +0100
Subject: [PATCH 147/373] move pyrakoon master version

---
 pylabs/pyrakoon | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pylabs/pyrakoon b/pylabs/pyrakoon
index b40bdb17..153f40cb 160000
--- a/pylabs/pyrakoon
+++ b/pylabs/pyrakoon
@@ -1 +1 @@
-Subproject commit b40bdb17179ca6b2eb2a05cb0b54e6c6f07d9aac
+Subproject commit 153f40cb5544429a6c63ff2a6ca69314c97f85ca

From 88ac832f30c0c9c6ef81e7cca63ec09728416473 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Thu, 18 Feb 2016 01:03:17 +0100
Subject: [PATCH 148/373] assert correct exception

---
 pylabs/test/server/right/system_tests_anomaly.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pylabs/test/server/right/system_tests_anomaly.py b/pylabs/test/server/right/system_tests_anomaly.py
index 93db1941..56d5fd49 100644
--- a/pylabs/test/server/right/system_tests_anomaly.py
+++ b/pylabs/test/server/right/system_tests_anomaly.py
@@ -442,7 +442,7 @@ def validate_reelection( old_master_id) :
         new_cli = arakoon.Arakoon.ArakoonClient(
             arakoon.Arakoon.ArakoonClientConfig(CONFIG.cluster_id,
                                                 cli_cfg_only_master ) )
-        NT.assert_raises( X.arakoon_client.ArakoonNoMaster, new_cli.whoMaster )
+        NT.assert_raises( arakoon.ArakoonExceptions.ArakoonNoMaster, new_cli.whoMaster )
         new_cli.dropConnections()
 
         return new_master_id

From 34f76c4dbf45a3d913b2e04f2f7ca4c9a2787602 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Thu, 18 Feb 2016 19:04:39 +0100
Subject: [PATCH 149/373] add arakoon_admin

---
 src/client/python/arakoon_admin.py | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)
 create mode 100644 src/client/python/arakoon_admin.py

diff --git a/src/client/python/arakoon_admin.py b/src/client/python/arakoon_admin.py
new file mode 100644
index 00000000..b05268cc
--- /dev/null
+++ b/src/client/python/arakoon_admin.py
@@ -0,0 +1,23 @@
+import Arakoon
+import ArakoonProtocol
+
+import logging
+class ArakoonAdmin(Arakoon.ArakoonClient):
+
+    def collapse(self, node_id, n):
+        """
+        @param node_id: which node to collapse
+        @param n : the number of tlogs that should remain after collapse
+        """
+        msg = ArakoonProtocol.ArakoonProtocol.encodeCollapse(n)
+        conn = None
+        conn = self._sendMessage(node_id, msg )
+        ArakoonProtocol.ArakoonProtocol._evaluateErrorCode(conn)
+        collapse_count = ArakoonProtocol._recvInt(conn)
+        logging.debug("collapse_count %i", collapse_count)
+        for i in xrange(collapse_count):
+            logging.debug("i=%i", i)
+            ArakoonProtocol.ArakoonProtocol._evaluateErrorCode(conn)
+            v = ArakoonProtocol._recvInt64(conn)
+            logging.info("v=%i",v)
+        

From ef8b452415f890d747eedc3431020f518a511e97 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Fri, 19 Feb 2016 08:28:56 +0100
Subject: [PATCH 150/373] make_release: 1.9.1

---
 META                      | 2 +-
 debian/changelog          | 5 +++++
 redhat/SPECS/arakoon.spec | 4 +++-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/META b/META
index 6dddd571..8f0eb133 100644
--- a/META
+++ b/META
@@ -1,7 +1,7 @@
 # -*- conf -*-
 
 description = "Arakoon client"
-version = "1.9.0"
+version = "1.9.1"
 exists_if = "arakoon_client.cma,arakoon_client.cmxa,arakoon_client.cmxs"
 requires = "threads lwt lwt.unix str oUnit quickcheck lwt.ssl ocplib-endian redis.lwt uri"
 archive(byte) = "arakoon_client.cma tcp_keepalive_sockopt_stubs.o"
diff --git a/debian/changelog b/debian/changelog
index 81adadd5..1b18460d 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,8 @@
+arakoon (1.9.1) unstable; urgency=low
+  * new upstream version
+
+ -- Romain Slootmaekers <romain.slootmaekers@openvstorage.com>  Fri, 19 Feb 2016 08:28:56 +0100
+
 arakoon (1.9.0) unstable; urgency=low
   * new upstream version
 
diff --git a/redhat/SPECS/arakoon.spec b/redhat/SPECS/arakoon.spec
index 3d5f1daa..d8e0b273 100644
--- a/redhat/SPECS/arakoon.spec
+++ b/redhat/SPECS/arakoon.spec
@@ -2,7 +2,7 @@
 
 Summary: Arakoon
 Name: arakoon
-Version: 1.9.0
+Version: 1.9.1
 Release: 3%{?dist}
 License: Apache 2.0
 Requires: libev >= 4
@@ -33,6 +33,8 @@ cp arakoon.native %{buildroot}%{_bindir}/arakoon
 %{_bindir}/arakoon
 
 %changelog
+* Fri Feb 19 2016 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.1
+- Create arakoon 1.9.1 RPM package
 * Fri Feb 12 2016 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.0
 - Create arakoon 1.9.0 RPM package
 * Mon Jan 25 2016 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.8.13

From 5f8d72ab3cd08c1986e3826c33b90a4fab190a4d Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Fri, 26 Feb 2016 12:37:51 +0100
Subject: [PATCH 151/373] fix for OVS-4268

- correct infimum
- test with size based rollover + collapse + catchup
- change default --compress-tlog
---
 .../server/right/test_catchup_exercises.py    | 37 +++++++++++++++++--
 pylabs/test/server/system_tests_common.py     |  7 ++++
 src/client/remote_nodestream.ml               |  2 +-
 src/main/arakoon.ml                           |  4 +-
 src/paxos/sn.ml                               |  4 +-
 src/tlog/tlog_main.ml                         | 10 +++--
 src/tlog/tlog_map.ml                          | 33 ++++++++++++-----
 7 files changed, 75 insertions(+), 22 deletions(-)

diff --git a/pylabs/test/server/right/test_catchup_exercises.py b/pylabs/test/server/right/test_catchup_exercises.py
index 7c16e9be..bdf795d0 100644
--- a/pylabs/test/server/right/test_catchup_exercises.py
+++ b/pylabs/test/server/right/test_catchup_exercises.py
@@ -28,9 +28,7 @@
 from Compat import X
 from arakoon_ext.client import ArakoonClient
 
-@Common.with_custom_setup(Common.setup_2_nodes_forced_master_mini, Common.basic_teardown)
-def test_catchup_exercises():
-
+def _scenario():
     def do_one(n, max_wait):
         logging.info("do_one(%i,%f)", n, max_wait)
         Common.iterate_n_times(n, Common.simple_set)
@@ -63,6 +61,14 @@ def do_one(n, max_wait):
         n = n * 2
         w = w * 2
 
+    
+    
+        
+@Common.with_custom_setup(Common.setup_2_nodes_forced_master_mini, Common.basic_teardown)
+def test_catchup_exercises():
+    _scenario()
+    
+
 def test_catchup_mixed_config():
     pass
     """
@@ -137,3 +143,28 @@ def _inner(c0):
         c0.stop()
         c0.tearDown()
         c0.remove()
+
+@Common.with_custom_setup(Common.setup_3_nodes_forced_master_mini_rollover_on_size,
+                          Common.basic_teardown)
+def test_catchup_rollover_on_size():
+    _scenario()
+    lagger = Common.node_names[2]
+    print ("now with collapsing as well")
+    Common.stopOne(lagger)
+    Common.iterate_n_times(1234, Common.simple_set)
+    Common.collapse(Common.node_names[0], 20)    
+    Common.stopOne(Common.node_names[1])
+    Common.startOne(lagger)
+    time.sleep(20)
+    cli = Common.get_client ()
+    ok = cli.expectProgressPossible()
+    assert_true(ok)
+    Common.stop_all()
+    head_dir = Common.build_node_dir_names(lagger)[3]
+    head_file = head_dir + "/head.db"
+    print head_file
+    exists = X.fileExists(head_file)
+    assert_false(exists)
+    Common.assert_last_i_in_sync(Common.node_names[0], lagger)
+    
+
diff --git a/pylabs/test/server/system_tests_common.py b/pylabs/test/server/system_tests_common.py
index fe55c1a3..e55c1c8f 100644
--- a/pylabs/test/server/system_tests_common.py
+++ b/pylabs/test/server/system_tests_common.py
@@ -763,6 +763,13 @@ def setup_2_nodes_forced_master_mini (home_dir):
     extra = {'tlog_max_entries':'1000'}
     setup_n_nodes( 2, True, home_dir, extra)
 
+def setup_3_nodes_forced_master_mini_rollover_on_size(home_dir):
+    extra = {'tlog_max_entries': '100_000',
+             'tlog_max_size'   : '64000'
+            }
+    
+    setup_n_nodes( 3, True, home_dir, extra)
+    
 def setup_3_nodes_forced_master_mini (home_dir):
     extra = {'tlog_max_entries':'1000'}
     setup_n_nodes( 3, True, home_dir, extra)
diff --git a/src/client/remote_nodestream.ml b/src/client/remote_nodestream.ml
index f460629f..2f19fc18 100644
--- a/src/client/remote_nodestream.ml
+++ b/src/client/remote_nodestream.ml
@@ -88,7 +88,7 @@ class remote_nodestream ((ic,oc) as conn) =
         | (-2) -> Logger.info_f_ "loop_parts done"
         | 1 ->
           begin
-            Logger.debug_f_ "loop_entries" >>= fun () ->
+            Logger.info_f_ "loop_entries" >>= fun () ->
             loop_entries () >>= fun () ->
             loop_parts ()
           end
diff --git a/src/main/arakoon.ml b/src/main/arakoon.ml
index 80f41f2f..a632a4fc 100644
--- a/src/main/arakoon.ml
+++ b/src/main/arakoon.ml
@@ -209,7 +209,7 @@ let main () =
   and tls_version = ref "1.0"
   and force = ref false
   and in_place = ref false
-  and archive_type = ref ".tlf"
+  and archive_type = ref ".tlx"
   and left = ref None
   and linc = ref true
   and right = ref None
@@ -279,7 +279,7 @@ let main () =
                                   Arg.Set_string filename],
      "<filename> : compress a tlog file");
     ("-archive", Arg.Set_string archive_type,
-     "either '.tlf' or '.tls'");
+     "either '.tlf' or '.tlf'");
     ("--uncompress-tlog", Arg.Tuple[set_laction UncompressTlog;
                                     Arg.Set_string filename],
      "<filename> : uncompress a tlog file");
diff --git a/src/paxos/sn.ml b/src/paxos/sn.ml
index e59e3451..300d7a0d 100644
--- a/src/paxos/sn.ml
+++ b/src/paxos/sn.ml
@@ -22,8 +22,8 @@ let zero = 0L
 let succ t = Int64.succ t
 let pred t = Int64.pred t
 let compare = Int64.compare
-let mul = Int64.mul
-let div = Int64.div
+
+
 let add = Int64.add
 let sub = Int64.sub
 let rem = Int64.rem
diff --git a/src/tlog/tlog_main.ml b/src/tlog/tlog_main.ml
index 7379ac4e..3a243520 100644
--- a/src/tlog/tlog_main.ml
+++ b/src/tlog/tlog_main.ml
@@ -128,13 +128,15 @@ let make_tlog tlog_name (i:int) =
 
 
 let compress_tlog tlu archive_type=
+  let failwith x =
+    Printf.ksprintf failwith x in
   let compressor =
     match archive_type with
-    | ".tls" | "tls" -> Compression.Snappy
-    | _ -> Compression.Bz2
+    | ".tlx"  -> Compression.Snappy
+    | ".tlf"  -> Compression.Bz2
+    | _ -> failwith "invalid archive_type:%s" archive_type
   in
-  let failwith x =
-    Printf.ksprintf failwith x in
+  
   let () = if not (Sys.file_exists tlu) then failwith "Input file %s does not exist" tlu in
   let tlx = Tlc2.to_archive_name compressor tlu in
   let () = if Sys.file_exists tlx then failwith "Can't compress %s as %s already exists" tlu tlx in
diff --git a/src/tlog/tlog_map.ml b/src/tlog/tlog_map.ml
index 87aea62d..6a711485 100644
--- a/src/tlog/tlog_map.ml
+++ b/src/tlog/tlog_map.ml
@@ -303,15 +303,25 @@ module TlogMap = struct
     let full_path = _get_full_path tlog_dir tlx_dir (file_name tlog_number) in
     _validate_one full_path node_id ~check_marker ~check_sabotage
     >>= fun (last, index, tlog_size) ->
-    Lwt_list.rev_map_s
-      (fun tlog_name ->
+    Lwt_list.fold_left_s
+      (fun (acc,prev)  tlog_name ->
         let n = get_number tlog_name in
         let canonical = _get_full_path tlog_dir tlx_dir tlog_name in
         first_i_of canonical >>= fun i ->
+        (if i > prev
+        then Lwt.return_unit
+        else Lwt.fail_with
+               (Printf.sprintf "%S starts with %s, which ain't higher than %s"
+                               canonical
+                               (Sn.string_of i) (Sn.string_of prev)
+               )
+        ) >>= fun () ->                  
         let is_archive = is_archive tlog_name in
-       Lwt.return {i;n;is_archive}
-      ) tlog_names
-    >>= fun i_to_tlog_number0 ->
+        let acc' = {i;n;is_archive}:: acc in
+        let prev' = i in
+       Lwt.return (acc',prev')
+      ) ([],-1L) tlog_names
+    >>= fun (i_to_tlog_number0,_) ->
     let i_to_tlog_number =
       if i_to_tlog_number0 = []
       then [{i = 0L;n = 0;is_archive=false}]
@@ -472,14 +482,17 @@ module TlogMap = struct
 
   let infimum t =
     get_tlog_names t >>= fun names ->
-    let f = t.tlog_max_entries in
     let v =
       match names with
-      | [] -> Sn.of_int (-f)
-      | h :: _ -> let n = Sn.of_int (get_number h) in
-                  Sn.mul (Sn.of_int f) n
+      | [] -> Sn.zero
+      | name :: _ ->
+         let n = get_number name in
+         match get_start_i t n with
+         | None -> (* somebody added older tlog files behind our back ?!*)
+            Sn.zero
+         | Some i -> i
     in
-    Logger.debug_f_ "infimum=%s" (Sn.string_of v) >>= fun () ->
+    Logger.info_f_ "infimum=%s" (Sn.string_of v) >>= fun () ->
     Lwt.return v
 
   let which_tlog_file t n =

From 747cf84a73ef041abb38f6af5a68737c5bbeb1cd Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Fri, 26 Feb 2016 14:50:34 +0100
Subject: [PATCH 152/373] fix typo

---
 src/main/arakoon.ml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/arakoon.ml b/src/main/arakoon.ml
index a632a4fc..61ada387 100644
--- a/src/main/arakoon.ml
+++ b/src/main/arakoon.ml
@@ -279,7 +279,7 @@ let main () =
                                   Arg.Set_string filename],
      "<filename> : compress a tlog file");
     ("-archive", Arg.Set_string archive_type,
-     "either '.tlf' or '.tlf'");
+     "either '.tlx' or the older '.tlf'");
     ("--uncompress-tlog", Arg.Tuple[set_laction UncompressTlog;
                                     Arg.Set_string filename],
      "<filename> : uncompress a tlog file");

From 5ad050c1da85f4d3ce49ac9e0d5566633d2295cb Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Fri, 26 Feb 2016 15:00:08 +0100
Subject: [PATCH 153/373] make_release: 1.9.2

---
 META                      | 2 +-
 debian/changelog          | 5 +++++
 redhat/SPECS/arakoon.spec | 4 +++-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/META b/META
index 8f0eb133..b9f455ba 100644
--- a/META
+++ b/META
@@ -1,7 +1,7 @@
 # -*- conf -*-
 
 description = "Arakoon client"
-version = "1.9.1"
+version = "1.9.2"
 exists_if = "arakoon_client.cma,arakoon_client.cmxa,arakoon_client.cmxs"
 requires = "threads lwt lwt.unix str oUnit quickcheck lwt.ssl ocplib-endian redis.lwt uri"
 archive(byte) = "arakoon_client.cma tcp_keepalive_sockopt_stubs.o"
diff --git a/debian/changelog b/debian/changelog
index 1b18460d..5ad5c4b2 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,8 @@
+arakoon (1.9.2) unstable; urgency=low
+  * new upstream version
+
+ -- Romain Slootmaekers <romain.slootmaekers@openvstorage.com>  Fri, 26 Feb 2016 15:00:08 +0100
+
 arakoon (1.9.1) unstable; urgency=low
   * new upstream version
 
diff --git a/redhat/SPECS/arakoon.spec b/redhat/SPECS/arakoon.spec
index d8e0b273..84ad93a4 100644
--- a/redhat/SPECS/arakoon.spec
+++ b/redhat/SPECS/arakoon.spec
@@ -2,7 +2,7 @@
 
 Summary: Arakoon
 Name: arakoon
-Version: 1.9.1
+Version: 1.9.2
 Release: 3%{?dist}
 License: Apache 2.0
 Requires: libev >= 4
@@ -33,6 +33,8 @@ cp arakoon.native %{buildroot}%{_bindir}/arakoon
 %{_bindir}/arakoon
 
 %changelog
+* Fri Feb 26 2016 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.2
+- Create arakoon 1.9.2 RPM package
 * Fri Feb 19 2016 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.1
 - Create arakoon 1.9.1 RPM package
 * Fri Feb 12 2016 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.0

From 2a84fbee95589d88bc73ed73eb5a91e46e23b6f9 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Fri, 26 Feb 2016 16:04:44 +0100
Subject: [PATCH 154/373] restore the api (for Alba)

---
 src/paxos/sn.ml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/paxos/sn.ml b/src/paxos/sn.ml
index 300d7a0d..9f1f90b1 100644
--- a/src/paxos/sn.ml
+++ b/src/paxos/sn.ml
@@ -23,6 +23,8 @@ let succ t = Int64.succ t
 let pred t = Int64.pred t
 let compare = Int64.compare
 
+let mul = Int64.mul
+let div = Int64.div
 
 let add = Int64.add
 let sub = Int64.sub

From e197e29b8a596f72c8fc0f144e48a93599dbb9b4 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Thu, 3 Mar 2016 15:26:39 +0100
Subject: [PATCH 155/373] upstream update

---
 pylabs/pyrakoon | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pylabs/pyrakoon b/pylabs/pyrakoon
index 153f40cb..0adf7aec 160000
--- a/pylabs/pyrakoon
+++ b/pylabs/pyrakoon
@@ -1 +1 @@
-Subproject commit 153f40cb5544429a6c63ff2a6ca69314c97f85ca
+Subproject commit 0adf7aecef833a0fd85e6529d1a8d6188d1fecaa

From b501316126a615a40b7bea6143fc66c4ea135dab Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Mon, 14 Mar 2016 10:19:22 +0100
Subject: [PATCH 156/373] fix for autofix behavior with corrupted db (it was
 erroneously not removed)

---
 src/node/catchup.ml      | 3 ++-
 src/node/collapser.ml    | 5 +++--
 src/node/dump_store.ml   | 3 ++-
 src/node/local_store.ml  | 2 +-
 src/node/mem_store.ml    | 2 +-
 src/node/node_main.ml    | 6 ++++--
 src/node/simple_store.ml | 2 +-
 src/node/store.ml        | 2 +-
 src/node/sync_backend.ml | 7 +++++--
 src/tools/file_system.ml | 6 ++++--
 10 files changed, 24 insertions(+), 14 deletions(-)

diff --git a/src/node/catchup.ml b/src/node/catchup.ml
index e5df8763..6cca95fc 100644
--- a/src/node/catchup.ml
+++ b/src/node/catchup.ml
@@ -82,7 +82,8 @@ let catchup_tlog (type s) ~tls_ctx ~tcp_keepalive ~stop other_configs ~cluster_i
     let when_closed () =
       Logger.debug_ "when_closed" >>= fun () ->
       let target_name = S.get_location store in
-      File_system.copy_file hfn target_name ~overwrite:true ~throttling:0.0
+      File_system.copy_file hfn target_name ~overwrite:true ~throttling:0.0 >>= fun _copied ->
+      Lwt.return ()
     in
     S.reopen store when_closed >>= fun () ->
     Lwt.return ()
diff --git a/src/node/collapser.ml b/src/node/collapser.ml
index 635f85a2..a81812a5 100644
--- a/src/node/collapser.ml
+++ b/src/node/collapser.ml
@@ -26,7 +26,7 @@ let section = Logger.Section.main
 let collapse_until (type s) (tlog_coll:Tlogcollection.tlog_collection)
     (module S : Store.STORE with type t = s)
     ((copy_store : string -> string ->
-                   overwrite:bool -> throttling:float -> unit Lwt.t),
+                   overwrite:bool -> throttling:float -> bool Lwt.t),
      (head_location:string),
      (throttling:float)
     )
@@ -42,7 +42,8 @@ let collapse_until (type s) (tlog_coll:Tlogcollection.tlog_collection)
     fun () ->
       copy_store head_location new_location
                  ~overwrite:true
-                 ~throttling
+                 ~throttling >>= fun _copied ->
+      Lwt.return ()
   ) (
     function
       | Not_found -> Logger.debug_f_ "head db at '%s' does not exist" head_location
diff --git a/src/node/dump_store.ml b/src/node/dump_store.ml
index 203d61e4..4af11193 100644
--- a/src/node/dump_store.ml
+++ b/src/node/dump_store.ml
@@ -125,7 +125,8 @@ let inject_as_head fn node_id cfg_url ~force ~in_place =
       if (not in_place)
       then begin
         Lwt_io.printf "cp %S %S" fn old_head_name >>=fun () ->
-        File_system.copy_file fn old_head_name ~overwrite:true ~throttling:0.0
+        File_system.copy_file fn old_head_name ~overwrite:true ~throttling:0.0 >>= fun _ ->
+        Lwt.return ()
       end
       else begin
         Lwt_io.printf "rename %S %S" fn old_head_name >>= fun () ->
diff --git a/src/node/local_store.ml b/src/node/local_store.ml
index c56da22d..031ef9d3 100644
--- a/src/node/local_store.ml
+++ b/src/node/local_store.ml
@@ -205,7 +205,7 @@ let copy_store ls networkClient (oc: Lwt_io.output_channel) =
 let relocate ls new_location =
   copy_store2 ls.location new_location ~overwrite:true
               ~throttling:0.0
-  >>= fun () ->
+  >>= fun _copied ->
   let old_location = ls.location in
   let () = ls.location <- new_location in
   Logger.info_f_ "Attempting to unlink file '%s'" old_location >>= fun () ->
diff --git a/src/node/mem_store.ml b/src/node/mem_store.ml
index 44272dbd..aa7dcee1 100644
--- a/src/node/mem_store.ml
+++ b/src/node/mem_store.ml
@@ -148,7 +148,7 @@ let get_key_count ms =
 let copy_store2 (_old_location:string) (_new_location:string)
                 ~(overwrite:bool) ~(throttling:float) =
   ignore (overwrite,throttling);
-  Lwt.return ()
+  Lwt.return_false
 
 let relocate _new_location = failwith "Memstore.relocation not implemented"
 
diff --git a/src/node/node_main.ml b/src/node/node_main.ml
index dd55c1cb..d334075d 100644
--- a/src/node/node_main.ml
+++ b/src/node/node_main.ml
@@ -196,6 +196,8 @@ let only_catchup
      S.copy_store2 full_snapshot_path db_name
                    ~overwrite:false
                    ~throttling:Node_cfg.default_head_copy_throttling
+     >>= fun _ ->
+     Lwt.return ()
     )
     (function
       | Not_found -> Lwt.return ()
@@ -534,8 +536,8 @@ let _main_2 (type s)
              S.copy_store2 full_snapshot_path db_name
                            ~overwrite:false
                            ~throttling:Node_cfg.default_head_copy_throttling
-             >>= fun () ->
-             head_copied := true;
+             >>= fun copied ->
+             head_copied := copied;
              Lwt.return_unit
             )
             (function
diff --git a/src/node/simple_store.ml b/src/node/simple_store.ml
index 46290e75..be341265 100644
--- a/src/node/simple_store.ml
+++ b/src/node/simple_store.ml
@@ -203,7 +203,7 @@ module type Simple_store = sig
   val copy_store2 : string -> string ->
                     overwrite:bool ->
                     throttling:float ->
-                    unit Lwt.t
+                    bool Lwt.t
 end
 
 let _f _pf = function
diff --git a/src/node/store.ml b/src/node/store.ml
index 65ae15cb..d01c8a43 100644
--- a/src/node/store.ml
+++ b/src/node/store.ml
@@ -59,7 +59,7 @@ sig
   val copy_store2 : string -> string ->
                     overwrite:bool ->
                     throttling:float ->
-                    unit Lwt.t
+                    bool Lwt.t
 
   val get_succ_store_i : t -> int64
   val get_catchup_start_i : t -> int64
diff --git a/src/node/sync_backend.ml b/src/node/sync_backend.ml
index 5b71ea7d..001bacc5 100644
--- a/src/node/sync_backend.ml
+++ b/src/node/sync_backend.ml
@@ -99,7 +99,7 @@ struct
     (store: 'a)
     (store_methods: (string -> string ->
                      overwrite:bool -> throttling:float ->
-                     unit Lwt.t) * string * float)
+                     bool Lwt.t) * string * float)
     (tlog_collection:Tlogcollection.tlog_collection)
     (lease_expiration:int)
     ~quorum_function n_nodes
@@ -679,7 +679,10 @@ struct
                      ~mode
                      (fun () ->
                       S.copy_store2 (S.get_location store) head_path
-                                    ~overwrite:true ~throttling:copy_head_throttling) >>= fun () ->
+                                    ~overwrite:true
+                                    ~throttling:copy_head_throttling
+                      >>= fun _ ->
+                      Lwt.return ()) >>= fun () ->
 
             (* remove all but tlogs_to_keep last tlogs *)
             Collapser._head_i (module S) head_path >>= fun head_io ->
diff --git a/src/tools/file_system.ml b/src/tools/file_system.ml
index d870f667..96cb4891 100644
--- a/src/tools/file_system.ml
+++ b/src/tools/file_system.ml
@@ -153,7 +153,8 @@ let copy_file source target ~overwrite ~throttling =
   exists target >>= fun target_exists ->
   if target_exists && not overwrite
   then
-    Logger.info_f_ "Not copying %s to %s because target already exists" source target
+    Logger.info_f_ "Not copying %s to %s because target already exists" source target >>= fun () ->
+    Lwt.return_false
   else
     begin
       let tmp_file = target ^ ".tmp" in
@@ -161,7 +162,8 @@ let copy_file source target ~overwrite ~throttling =
       Lwt_io.with_file ~mode:Lwt_io.input source
         (fun ic ->
          with_tmp_file tmp_file target
-                       (fun oc -> copy_all ic oc))
+                       (fun oc -> copy_all ic oc)) >>= fun () ->
+      Lwt.return_true
     end
 
 let safe_rename src dest =

From 36e91243d78ca01ed7fa17b3b4ae92979138c341 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Mon, 14 Mar 2016 10:54:18 +0100
Subject: [PATCH 157/373] big sigh

---
 docker/arakoon_debian/Dockerfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docker/arakoon_debian/Dockerfile b/docker/arakoon_debian/Dockerfile
index 99c67b1d..459130ed 100644
--- a/docker/arakoon_debian/Dockerfile
+++ b/docker/arakoon_debian/Dockerfile
@@ -33,7 +33,8 @@ RUN apt-get -y update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
         libtool \
         libboost-all-dev \
         iptables \
-        net-tools
+        net-tools \
+        ncurses-dev
 
 RUN useradd jenkins -u 1001 -g root
 

From 6e466c53bf50c7f8a197b23639f473e53044339c Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Mon, 14 Mar 2016 16:44:02 +0100
Subject: [PATCH 158/373] set TCP_NODELAY once

---
 src/client/client_helper.ml | 33 ++++++++++++++++--------
 src/tools/server.ml         | 51 +++++++++++++++++++------------------
 2 files changed, 48 insertions(+), 36 deletions(-)

diff --git a/src/client/client_helper.ml b/src/client/client_helper.ml
index a51c4cfc..c77dfdcf 100644
--- a/src/client/client_helper.ml
+++ b/src/client/client_helper.ml
@@ -36,18 +36,29 @@ let default_create_client_context ~ca_cert ~creds ~protocol =
 
 
 let with_connection ~tls ~tcp_keepalive sa do_it = match tls with
-  | None -> Lwt_io.with_connection sa do_it
+  | None -> 
+     let fd = Lwt_unix.socket (Unix.domain_of_sockaddr sa) Unix.SOCK_STREAM 0 in
+     Tcp_keepalive.apply fd tcp_keepalive;
+     Lwt_unix.setsockopt fd Lwt_unix.TCP_NODELAY true;
+     Lwt_unix.set_close_on_exec fd;
+     Lwt_unix.connect fd sa >>= fun () ->
+     let ic = Lwt_io.of_fd ~mode:Lwt_io.input fd 
+     and oc = Lwt_io.of_fd ~mode:Lwt_io.output fd in
+     Lwt.finalize
+       (fun () -> do_it (ic, oc))
+       (fun () -> Lwt_unix.close fd)
   | Some ctx ->
-    let fd = Lwt_unix.socket (Unix.domain_of_sockaddr sa) Unix.SOCK_STREAM 0 in
-    Tcp_keepalive.apply fd tcp_keepalive;
-    Lwt_unix.set_close_on_exec fd;
-    Lwt_unix.connect fd sa >>= fun () ->
-    Typed_ssl.Lwt.ssl_connect fd ctx >>= fun (_, sock) ->
-    let ic = Lwt_ssl.in_channel_of_descr sock
-    and oc = Lwt_ssl.out_channel_of_descr sock in
-    Lwt.finalize
-      (fun () -> do_it (ic, oc))
-      (fun () -> Lwt_ssl.close sock)
+     let fd = Lwt_unix.socket (Unix.domain_of_sockaddr sa) Unix.SOCK_STREAM 0 in
+     Tcp_keepalive.apply fd tcp_keepalive;
+     Lwt_unix.setsockopt fd Lwt_unix.TCP_NODELAY true;
+     Lwt_unix.set_close_on_exec fd;
+     Lwt_unix.connect fd sa >>= fun () ->
+     Typed_ssl.Lwt.ssl_connect fd ctx >>= fun (_, sock) ->
+     let ic = Lwt_ssl.in_channel_of_descr sock
+     and oc = Lwt_ssl.out_channel_of_descr sock in
+     Lwt.finalize
+       (fun () -> do_it (ic, oc))
+       (fun () -> Lwt_ssl.close sock)
 
 
 exception No_connection
diff --git a/src/tools/server.ml b/src/tools/server.ml
index c89fc90a..b62d451d 100644
--- a/src/tools/server.ml
+++ b/src/tools/server.ml
@@ -209,31 +209,32 @@ let make_server_thread
       let serve () =
         Lwt.catch
           (fun () ->
-           Lwt_unix.accept listening_socket >>= fun (plain_fd, cl_socket_address) ->
-           Tcp_keepalive.apply plain_fd tcp_keepalive;
-           let cid = name ^ "_" ^ Int64.to_string (connection_counter ()) in
-           let finalize () =
-             Hashtbl.remove client_threads cid;
-             Lwt_condition.signal _condition ();
-             Lwt.return ()
-           in
-           possible_denial cid plain_fd cl_socket_address >>= fun mt ->
-           begin
-             match mt with
-             | Some t ->
-                Lwt.ignore_result
-                  (Lwt.catch
-                     (fun () ->
-                      Hashtbl.add client_threads cid t;
-                      t >>= fun () ->
-                      finalize ()
-                     )
-                     (fun exn ->
-                      Logger.info_f_ ~exn "Exception in client thread %s" cid >>= fun () ->
-                      finalize ()))
-             | None -> ()
-           end;
-           Lwt.return ()
+            Lwt_unix.accept listening_socket >>= fun (plain_fd, cl_socket_address) ->
+            Lwt_unix.setsockopt plain_fd Lwt_unix.TCP_NODELAY true;
+            Tcp_keepalive.apply plain_fd tcp_keepalive;
+            let cid = name ^ "_" ^ Int64.to_string (connection_counter ()) in
+            let finalize () =
+              Hashtbl.remove client_threads cid;
+              Lwt_condition.signal _condition ();
+              Lwt.return ()
+            in
+            possible_denial cid plain_fd cl_socket_address >>= fun mt ->
+            begin
+              match mt with
+              | Some t ->
+                 Lwt.ignore_result
+                   (Lwt.catch
+                      (fun () ->
+                        Hashtbl.add client_threads cid t;
+                        t >>= fun () ->
+                        finalize ()
+                      )
+                      (fun exn ->
+                        Logger.info_f_ ~exn "Exception in client thread %s" cid >>= fun () ->
+                        finalize ()))
+              | None -> ()
+            end;
+            Lwt.return ()
           )
           (function
             | Unix.Unix_error (Unix.EMFILE,s0,s1) ->

From 4e30e3d77048750feeb00854b7f4b740e991c86b Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Tue, 15 Mar 2016 11:24:36 +0100
Subject: [PATCH 159/373] be more prudent

---
 src/client/client_helper.ml | 50 +++++++++++++++++++++++--------------
 1 file changed, 31 insertions(+), 19 deletions(-)

diff --git a/src/client/client_helper.ml b/src/client/client_helper.ml
index c77dfdcf..d0d242da 100644
--- a/src/client/client_helper.ml
+++ b/src/client/client_helper.ml
@@ -35,32 +35,44 @@ let default_create_client_context ~ca_cert ~creds ~protocol =
   ctx
 
 
-let with_connection ~tls ~tcp_keepalive sa do_it = match tls with
+let with_connection ~tls ~tcp_keepalive sa do_it =
+  let fd = Lwt_unix.socket (Unix.domain_of_sockaddr sa) Unix.SOCK_STREAM 0 in
+  Tcp_keepalive.apply fd tcp_keepalive;
+  Lwt_unix.setsockopt fd Lwt_unix.TCP_NODELAY true;
+  Lwt_unix.set_close_on_exec fd;
+  Lwt.catch
+    (fun () -> Lwt_unix.connect fd sa)
+    (fun exn ->
+      Lwt.catch
+        (fun () -> Lwt_unix.close fd)
+        (fun _ -> Lwt.return_unit)
+      >>= fun () ->
+      Lwt.fail exn
+    )
+  >>= fun () ->
+  begin
+  match tls with
   | None -> 
-     let fd = Lwt_unix.socket (Unix.domain_of_sockaddr sa) Unix.SOCK_STREAM 0 in
-     Tcp_keepalive.apply fd tcp_keepalive;
-     Lwt_unix.setsockopt fd Lwt_unix.TCP_NODELAY true;
-     Lwt_unix.set_close_on_exec fd;
-     Lwt_unix.connect fd sa >>= fun () ->
      let ic = Lwt_io.of_fd ~mode:Lwt_io.input fd 
      and oc = Lwt_io.of_fd ~mode:Lwt_io.output fd in
-     Lwt.finalize
-       (fun () -> do_it (ic, oc))
-       (fun () -> Lwt_unix.close fd)
+     let conn = (ic,oc)
+     and fin = fun () -> Lwt_unix.close fd
+     in
+     Lwt.return (conn,fin)
   | Some ctx ->
-     let fd = Lwt_unix.socket (Unix.domain_of_sockaddr sa) Unix.SOCK_STREAM 0 in
-     Tcp_keepalive.apply fd tcp_keepalive;
-     Lwt_unix.setsockopt fd Lwt_unix.TCP_NODELAY true;
-     Lwt_unix.set_close_on_exec fd;
-     Lwt_unix.connect fd sa >>= fun () ->
      Typed_ssl.Lwt.ssl_connect fd ctx >>= fun (_, sock) ->
      let ic = Lwt_ssl.in_channel_of_descr sock
      and oc = Lwt_ssl.out_channel_of_descr sock in
-     Lwt.finalize
-       (fun () -> do_it (ic, oc))
-       (fun () -> Lwt_ssl.close sock)
-
-
+     let conn = (ic,oc) 
+     and fin = fun () -> Lwt_ssl.close sock
+     in
+     Lwt.return (conn,fin)
+  end
+  >>= fun (conn,fin) ->
+  Lwt.finalize
+    (fun () -> do_it conn)
+    fin
+  
 exception No_connection
 
 let with_connection' ~tls ~tcp_keepalive addrs f =

From d583f3842b240c35ea9a710e84ef55f3c634d0f0 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Thu, 17 Mar 2016 14:10:47 +0100
Subject: [PATCH 160/373] be more gentle to the reader

---
 src/client/client_helper.ml | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/client/client_helper.ml b/src/client/client_helper.ml
index d0d242da..bda92523 100644
--- a/src/client/client_helper.ml
+++ b/src/client/client_helper.ml
@@ -37,11 +37,12 @@ let default_create_client_context ~ca_cert ~creds ~protocol =
 
 let with_connection ~tls ~tcp_keepalive sa do_it =
   let fd = Lwt_unix.socket (Unix.domain_of_sockaddr sa) Unix.SOCK_STREAM 0 in
-  Tcp_keepalive.apply fd tcp_keepalive;
-  Lwt_unix.setsockopt fd Lwt_unix.TCP_NODELAY true;
-  Lwt_unix.set_close_on_exec fd;
   Lwt.catch
-    (fun () -> Lwt_unix.connect fd sa)
+    (fun () ->
+      Tcp_keepalive.apply fd tcp_keepalive;
+      Lwt_unix.setsockopt fd Lwt_unix.TCP_NODELAY true;
+      Lwt_unix.set_close_on_exec fd;
+      Lwt_unix.connect fd sa)
     (fun exn ->
       Lwt.catch
         (fun () -> Lwt_unix.close fd)

From 83e9fd96140936a17396ef89a5be0efe782f549c Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Sun, 20 Mar 2016 19:33:57 +0100
Subject: [PATCH 161/373] make_release: 1.9.3

---
 META                      | 2 +-
 debian/changelog          | 5 +++++
 redhat/SPECS/arakoon.spec | 4 +++-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/META b/META
index b9f455ba..16a33cb5 100644
--- a/META
+++ b/META
@@ -1,7 +1,7 @@
 # -*- conf -*-
 
 description = "Arakoon client"
-version = "1.9.2"
+version = "1.9.3"
 exists_if = "arakoon_client.cma,arakoon_client.cmxa,arakoon_client.cmxs"
 requires = "threads lwt lwt.unix str oUnit quickcheck lwt.ssl ocplib-endian redis.lwt uri"
 archive(byte) = "arakoon_client.cma tcp_keepalive_sockopt_stubs.o"
diff --git a/debian/changelog b/debian/changelog
index 5ad5c4b2..11565472 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,8 @@
+arakoon (1.9.3) unstable; urgency=low
+  * new upstream version
+
+ -- Romain Slootmaekers <romain.slootmaekers@openvstorage.com>  Sun, 20 Mar 2016 19:33:57 +0100
+
 arakoon (1.9.2) unstable; urgency=low
   * new upstream version
 
diff --git a/redhat/SPECS/arakoon.spec b/redhat/SPECS/arakoon.spec
index 84ad93a4..81d52e19 100644
--- a/redhat/SPECS/arakoon.spec
+++ b/redhat/SPECS/arakoon.spec
@@ -2,7 +2,7 @@
 
 Summary: Arakoon
 Name: arakoon
-Version: 1.9.2
+Version: 1.9.3
 Release: 3%{?dist}
 License: Apache 2.0
 Requires: libev >= 4
@@ -33,6 +33,8 @@ cp arakoon.native %{buildroot}%{_bindir}/arakoon
 %{_bindir}/arakoon
 
 %changelog
+* Sun Mar 20 2016 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.3
+- Create arakoon 1.9.3 RPM package
 * Fri Feb 26 2016 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.2
 - Create arakoon 1.9.2 RPM package
 * Fri Feb 19 2016 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.1

From e9e391bd7106337afe62ea2141098ad07e9e9e1b Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Mon, 21 Mar 2016 01:30:01 +0100
Subject: [PATCH 162/373] and yet it moves

---
 docker/arakoon_centos7/Dockerfile | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/docker/arakoon_centos7/Dockerfile b/docker/arakoon_centos7/Dockerfile
index 634ec6c6..469de9a8 100644
--- a/docker/arakoon_centos7/Dockerfile
+++ b/docker/arakoon_centos7/Dockerfile
@@ -13,6 +13,11 @@ RUN yum -y install python-devel
 
 RUN useradd jenkins -u 1001 -g root
 
+RUN rpm -iUvh http://dl.fedoraproject.org/pub/epel/7/x86_64/e/epel-release-7-5.noarch.rpm
+RUN yum -y update
+RUN yum -y install python-pip
+RUN yum -y install openssl-devel gmp-devel which
+RUN yum -y install zlib-devel ncurses-devel
 
 RUN wget https://raw.github.com/ocaml/opam/master/shell/opam_installer.sh
 RUN sh ./opam_installer.sh /usr/local/bin 4.02.3
@@ -21,13 +26,6 @@ ENV opam_env='opam config env --root=/home/opam/OPAM'
 RUN opam init --root=/home/opam/OPAM --comp 4.02.3
 
 
-
-RUN rpm -iUvh http://dl.fedoraproject.org/pub/epel/7/x86_64/e/epel-release-7-5.noarch.rpm
-RUN yum -y update
-RUN yum -y install python-pip
-RUN yum -y install openssl-devel gmp-devel which
-RUN yum -y install zlib-devel
-
 # protobuf
 RUN yum -y install bzip2
 RUN wget https://github.com/google/protobuf/releases/download/v2.6.1/protobuf-2.6.1.tar.bz2 -O protobuf-2.6.1.tar.bz2 \

From 715be027ecb16a5bbcc9ba70c83e8836469514d2 Mon Sep 17 00:00:00 2001
From: Kenneth Henderick <kenneth@ketronic.be>
Date: Fri, 25 Mar 2016 15:35:28 +0100
Subject: [PATCH 163/373] Point pyrakoon to the its latest version on master

---
 pylabs/pyrakoon | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pylabs/pyrakoon b/pylabs/pyrakoon
index 0adf7aec..d4da5fb1 160000
--- a/pylabs/pyrakoon
+++ b/pylabs/pyrakoon
@@ -1 +1 @@
-Subproject commit 0adf7aecef833a0fd85e6529d1a8d6188d1fecaa
+Subproject commit d4da5fb1b5ad910a8c141e19ac32fd4eb96b6752

From 4c42c9f5c21940f1ec53ee2cb5a57b8597a7a5a6 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Fri, 25 Mar 2016 16:11:03 +0100
Subject: [PATCH 164/373] minor improvements

---
 pylabs/test/server/quick/system_tests_basic.py | 2 +-
 src/tools/arakoon_logger.ml                    | 9 +++------
 2 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/pylabs/test/server/quick/system_tests_basic.py b/pylabs/test/server/quick/system_tests_basic.py
index c5629553..6715bc92 100644
--- a/pylabs/test/server/quick/system_tests_basic.py
+++ b/pylabs/test/server/quick/system_tests_basic.py
@@ -503,7 +503,7 @@ def test_get_version():
     vt = client.getVersion()
     logging.debug("tuple = %s", str(vt))
     (major,minor,patch, info) = vt
-    majors = [1, 2 ** 32 - 1] # 2 ** 32 - 1 ~= -1
+    majors = [1, 2 ** 32 - 1, -1] # 2 ** 32 - 1 ~= -1
     assert_in(major, majors)
     #then on specific level:
     vt2 = client.getVersion(C.node_names[0])
diff --git a/src/tools/arakoon_logger.ml b/src/tools/arakoon_logger.ml
index 32646f70..b1b02bf3 100644
--- a/src/tools/arakoon_logger.ml
+++ b/src/tools/arakoon_logger.ml
@@ -57,13 +57,10 @@ let file_logger file_name =
   Lwt.return (lg_output, reopen)
 
 let console_logger =
-  let mutex = Lwt_mutex.create () in
   let lg_output line =
-    Lwt_mutex.with_lock
-      mutex
-      (fun () ->
-       Lwt_io.write Lwt_io.stdout line >>= fun () ->
-       Lwt_io.write_char Lwt_io.stdout '\n')
+    Lwt_io.atomic
+      (fun oc -> Lwt_io.write_line oc line)
+      Lwt_io.stdout
   in
   let reopen () = Lwt.return_unit in
   lg_output, reopen

From 0c964671612177f1da200a4036aa388c04fc2d5b Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 27 Apr 2016 09:03:51 +0200
Subject: [PATCH 165/373] tlog_map's log section in cfg

---
 cfg/arakoon.ini       | 2 +-
 pylabs/pyrakoon       | 2 +-
 src/node/node_cfg.ml  | 4 ++++
 src/node/node_main.ml | 1 +
 src/system/startup.ml | 1 +
 5 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/cfg/arakoon.ini b/cfg/arakoon.ini
index 75d676b3..87968a89 100644
--- a/cfg/arakoon.ini
+++ b/cfg/arakoon.ini
@@ -78,7 +78,7 @@ tlog_max_size    = 5_000_000 # in bytes
 client_protocol = debug
 paxos = debug
 tcp_messaging = debug
-
+tlog_map = debug
 
 #[default_batched_transaction_config]
 #max_entries = 100    # number of sets and deletes
diff --git a/pylabs/pyrakoon b/pylabs/pyrakoon
index d4da5fb1..0adf7aec 160000
--- a/pylabs/pyrakoon
+++ b/pylabs/pyrakoon
@@ -1 +1 @@
-Subproject commit d4da5fb1b5ad910a8c141e19ac32fd4eb96b6752
+Subproject commit 0adf7aecef833a0fd85e6529d1a8d6188d1fecaa
diff --git a/src/node/node_cfg.ml b/src/node/node_cfg.ml
index 48f05735..0345cee1 100644
--- a/src/node/node_cfg.ml
+++ b/src/node/node_cfg.ml
@@ -196,6 +196,7 @@ module Node_cfg = struct
       client_protocol: string option;
       paxos: string option;
       tcp_messaging: string option;
+      tlog_map: string option;
     }
 
   let string_of_log_cfg lcfg =
@@ -210,6 +211,7 @@ module Node_cfg = struct
       client_protocol = None;
       paxos = None;
       tcp_messaging = None;
+      tlog_map = None;
     }
 
   type batched_transaction_cfg =
@@ -471,10 +473,12 @@ module Node_cfg = struct
     let client_protocol = get_log_level "client_protocol" in
     let paxos = get_log_level "paxos" in
     let tcp_messaging = get_log_level "tcp_messaging" in
+    let tlog_map = get_log_level "tlog_map" in
     {
       client_protocol;
       paxos;
       tcp_messaging;
+      tlog_map;
     }
 
   let _batched_transaction_config inifile section_name =
diff --git a/src/node/node_main.ml b/src/node/node_main.ml
index d334075d..a96b1825 100644
--- a/src/node/node_main.ml
+++ b/src/node/node_main.ml
@@ -55,6 +55,7 @@ let _config_log_levels me cluster_cfg cfg =
   let () = set_level Client_protocol.section log_config.client_protocol in
   let () = set_level Multi_paxos.section log_config.paxos in
   let () = set_level Tcp_messaging.section log_config.tcp_messaging in
+  let () = set_level Tlog_map.section log_config.tlog_map in
   Lwt.return ()
 
 let _config_logging me get_cfgs =
diff --git a/src/system/startup.ml b/src/system/startup.ml
index 87cba5de..375fcff7 100644
--- a/src/system/startup.ml
+++ b/src/system/startup.ml
@@ -34,6 +34,7 @@ let _make_log_cfg () =
      client_protocol = None;
      paxos = None;
      tcp_messaging = None;
+     tlog_map = None;
    })
 
 let _make_batched_transaction_cfg () =

From c43d4029a90c0d83ceb186f54a2359256086edc6 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Tue, 3 May 2016 10:41:39 +0200
Subject: [PATCH 166/373] expose bug in range(None, False, None, False, 0)

---
 pylabs/test/server/quick/system_tests_basic.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/pylabs/test/server/quick/system_tests_basic.py b/pylabs/test/server/quick/system_tests_basic.py
index 6715bc92..7b51c6af 100644
--- a/pylabs/test/server/quick/system_tests_basic.py
+++ b/pylabs/test/server/quick/system_tests_basic.py
@@ -272,6 +272,16 @@ def test_large_value ():
 def test_range_entries ():
     C.range_entries_scenario( 1000 )
 
+@C.with_custom_setup(C.setup_1_node, C.basic_teardown)
+def test_range_0():
+    client = C.get_client()
+    k = 'xey001'
+    client.set(k,k)
+    results = client.range(None, False, None, False,-1)
+    assert_equals(results,[k], 'should have 1 element')
+    results = client.range(None, False, None, False, 0)
+    assert_equals(results,[])
+
 @C.with_custom_setup(C.default_setup, C.basic_teardown)
 def test_aSSert_scenario_1():
     client = C.get_client()

From e8d76a6fbf17aa1bf4cf7e88878aa14949eae446 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Thu, 5 May 2016 00:50:44 +0200
Subject: [PATCH 167/373] update camltc

---
 docker/arakoon_centos7/Dockerfile | 2 +-
 docker/arakoon_debian/Dockerfile  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/arakoon_centos7/Dockerfile b/docker/arakoon_centos7/Dockerfile
index 469de9a8..c49fc208 100644
--- a/docker/arakoon_centos7/Dockerfile
+++ b/docker/arakoon_centos7/Dockerfile
@@ -52,7 +52,7 @@ RUN eval `${opam_env}` && \
         snappy \
         bisect \
         lwt.2.5.1 \
-        camltc \
+        camltc.0.9.3 \
         quickcheck \
         conf-libev \
         ocplib-endian \
diff --git a/docker/arakoon_debian/Dockerfile b/docker/arakoon_debian/Dockerfile
index 459130ed..1868a1f6 100644
--- a/docker/arakoon_debian/Dockerfile
+++ b/docker/arakoon_debian/Dockerfile
@@ -61,7 +61,7 @@ RUN eval `${opam_env}` && \
         snappy \
         bisect \
         lwt.2.5.1 \
-        camltc \
+        camltc.0.9.3 \
         quickcheck \
         conf-libev \
         ocplib-endian \

From 65092983c464d9752a4e8690205db16fba74d635 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Mon, 23 May 2016 08:34:20 +0200
Subject: [PATCH 168/373] fixes race condition in collapse

changes log level of collapse epilogue
---
 src/node/collapser.ml |  6 ++--
 src/tlog/tlog_map.ml  | 76 +++++++++++++++++++++----------------------
 2 files changed, 41 insertions(+), 41 deletions(-)

diff --git a/src/node/collapser.ml b/src/node/collapser.ml
index a81812a5..b611dc1a 100644
--- a/src/node/collapser.ml
+++ b/src/node/collapser.ml
@@ -117,7 +117,7 @@ let collapse_until (type s) (tlog_coll:Tlogcollection.tlog_collection)
                 | Some (pi,pv) ->
                   if pi < i then
                     begin
-                      maybe_log pi 
+                      maybe_log pi
                       >>= fun () ->
                       slowdown (fun () -> S.safe_insert_value new_store pi pv) >>= fun _ ->
                       let () = acc := Some(i,value) in
@@ -144,7 +144,7 @@ let collapse_until (type s) (tlog_coll:Tlogcollection.tlog_collection)
             end
           in
 
-          Logger.debug_f_ "Done replaying to head (%s : %s)" (Sn.string_of si) (Sn.string_of too_far_i) >>= fun() ->
+          Logger.info_f_ "Done replaying to head (%s : %s)" (Sn.string_of si) (Sn.string_of too_far_i) >>= fun() ->
           begin
             if si = Sn.pred (Sn.pred too_far_i) then
               Lwt.return ()
@@ -216,7 +216,7 @@ let collapse_many
   | Some (n, too_far_i) ->
     begin
       Logger.info_f_ "Going to collapse %d tlogs" n >>= fun () ->
-      cb' (n + 2) >>= fun () -> 
+      cb' (n + 2) >>= fun () ->
       Logger.debug_f_ "too_far_i = %s" (Sn.string_of too_far_i) >>= fun () ->
       collapse_until tlog_coll (module S) store_fs too_far_i cb slowdown >>= fun () ->
       get_head_i () >>= fun head_i ->
diff --git a/src/tlog/tlog_map.ml b/src/tlog/tlog_map.ml
index 6a711485..f1b8069c 100644
--- a/src/tlog/tlog_map.ml
+++ b/src/tlog/tlog_map.ml
@@ -42,7 +42,7 @@ let is_archive filename =
   match extension with
   | ".tlx" | ".tlf" | ".tlc"-> true
   | _ -> false
-  
+
 let first_i_of ?extension_override filename =
   let extension = match extension_override with
     | None -> extension_of filename
@@ -55,14 +55,14 @@ let first_i_of ?extension_override filename =
     let i =  Sn.sn_from buffer in
     Lwt.return i
   in
-  let reader = 
+  let reader =
     match extension with
-    | ".tlog" -> (fun ic -> Sn.input_sn ic )                   
+    | ".tlog" -> (fun ic -> Sn.input_sn ic )
     | ".tlx" ->
        (fun ic ->
          Compression.read_format ic Compression.Snappy >>= fun ()->
          Sn.input_sn ic >>= fun _last_i ->
-         read_uncompress ic Compression.uncompress_snappy 
+         read_uncompress ic Compression.uncompress_snappy
        )
     | ".tlf"  ->
        (fun ic ->
@@ -80,7 +80,7 @@ let first_i_of ?extension_override filename =
   Lwt_io.with_file ~mode:Lwt_io.input filename reader >>= fun i ->
   Logger.debug_f_ "%s starts with i:%Li" filename i >>= fun () ->
   Lwt.return i
-                   
+
 
 let get_number fn =
   let dot_pos = String.index fn '.' in
@@ -238,7 +238,7 @@ let _validate_one
         let base = Filename.basename tlog_name in
         if (not check_sabotage) || (ext = ".tlog" && get_number base = 0)
         then Lwt.return (None, new_index, 0)
-        else (* 
+        else (*
          somebody sabotaged us:
          (s)he deleted the .tlog file but we have .tlf's
          meaning rotation happened correctly.
@@ -265,11 +265,11 @@ module TlogMap = struct
       mutable tlog_size: int;
       mutable tlog_entries : int;
       mutable tlog_number: tlog_number;
-      
+
       mutable i_to_tlog_number : item list; (* sorted: most recent first *)
       mutable should_roll: bool;
     }
-             
+
   let show_map t =
     To_string.list
       (fun item ->
@@ -277,7 +277,7 @@ module TlogMap = struct
                        (Sn.string_of item.i) item.n item.is_archive
       )
       t.i_to_tlog_number
-                      
+
   let new_entry t s =
     t.tlog_size <- t.tlog_size + s;
     t.tlog_entries <- t.tlog_entries + 1;
@@ -315,7 +315,7 @@ module TlogMap = struct
                                canonical
                                (Sn.string_of i) (Sn.string_of prev)
                )
-        ) >>= fun () ->                  
+        ) >>= fun () ->
         let is_archive = is_archive tlog_name in
         let acc' = {i;n;is_archive}:: acc in
         let prev' = i in
@@ -330,7 +330,7 @@ module TlogMap = struct
     let last_i =
       match last with
       | None -> Sn.zero
-      | Some e -> Entry.i_of e 
+      | Some e -> Entry.i_of e
     in
     let tlog_entries, should_roll,last_start =
       match i_to_tlog_number with
@@ -340,12 +340,12 @@ module TlogMap = struct
          let tlog_entries = Sn.sub last_i item.i |> Sn.to_int in
          let should_roll = tlog_entries + 1 >= tlog_max_entries in
          tlog_entries, should_roll, item.i
-    in 
+    in
     Logger.debug_f_ "_init: tlog_entries:%i should_roll:%b" tlog_entries should_roll
     >>= fun () ->
     Lwt.return ((tlog_entries, tlog_size, tlog_number, i_to_tlog_number, should_roll),
                 last, index)
-      
+
   let make ?tlog_max_entries
            ?tlog_max_size
            tlog_dir tlx_dir node_id
@@ -362,7 +362,7 @@ module TlogMap = struct
       tlog_dir tlx_dir node_id ~check_marker ~check_sabotage:false
       tlog_max_entries tlog_max_size
     >>= fun ((tlog_entries, tlog_size,tlog_number,i_to_tlog_number, should_roll), last, index ) ->
-    
+
     let t = {node_id; tlog_dir;tlx_dir;tlog_max_entries; tlog_max_size;
              tlog_entries; tlog_size; tlog_number; i_to_tlog_number;should_roll;
             }
@@ -388,7 +388,7 @@ module TlogMap = struct
     let full_path = _get_full_path t.tlog_dir t.tlx_dir (file_name new_c) in
     Lwt.return (new_c, full_path)
 
-  
+
   let find_tlog_file t c =
     let open Compression in
     let f0 = archive_name Snappy c in
@@ -440,7 +440,7 @@ module TlogMap = struct
         | exn -> Logger.debug_f_ ~exn "filename:%s" full_name >>= fun () ->
                  Lwt.fail exn
       )
-      
+
   let outer_of_i t i =
     let rec find = function
       | [] -> 0
@@ -450,7 +450,7 @@ module TlogMap = struct
          else find rest
     in
     find t.i_to_tlog_number
-      
+
   let get_start_i t n =
     let rec find = function
       | [] -> None
@@ -460,17 +460,17 @@ module TlogMap = struct
          else find rest
     in
     let r = find t.i_to_tlog_number in
-    (* Logger.ign_debug_f_ "get_start_i %s %i => %s"  (show_map t) n 
+    (* Logger.ign_debug_f_ "get_start_i %s %i => %s"  (show_map t) n
                                                       (To_string.option Sn.string_of r) ;
      *)
     r
-         
-  
+
+
 
   let should_roll t = t.should_roll
 
   let set_should_roll t = t.should_roll <- true
-                                             
+
   let new_outer t i =
     let n = t.tlog_number + 1 in
     let () = t.tlog_number <- n in
@@ -510,7 +510,7 @@ module TlogMap = struct
                      | false -> loop xs)
     in
     loop canonicals
-         
+
   let remove_below t i =
     Logger.debug_f_ "remove_below %s tlog_map:%s" (Sn.string_of i) (show_map t)
     >>= fun () ->
@@ -520,7 +520,8 @@ module TlogMap = struct
       | x :: rest -> find_start (x::to_remove_rev) rest
     in
     let to_remove,to_keep_rev = find_start [] (List.rev t.i_to_tlog_number) in
-    
+    t.i_to_tlog_number <- List.rev to_keep_rev;
+
     let remove item  =
       which_tlog_file  t item.n >>= function
       | None -> Lwt.return_unit
@@ -531,7 +532,6 @@ module TlogMap = struct
          end
     in
     Lwt_list.iter_s remove to_remove >>= fun () ->
-    t.i_to_tlog_number <- List.rev to_keep_rev;
     Logger.debug_f_ "after remove_below %s tlog_map:%s" (Sn.string_of i) (show_map t) >>= fun () ->
     Lwt.return_unit
 
@@ -545,9 +545,9 @@ module TlogMap = struct
     let r = find t.i_to_tlog_number in
     Logger.ign_debug_f_ "is_rollover_point %s => %b" (Sn.string_of i) r;
     r
-    
+
   let complete_file_to_deliver t i =
-    
+
     let rec find = function
       | [] -> None
       | item1 :: item0 :: _ when item0.i = i -> Some (item0.n,item1.i)
@@ -582,14 +582,14 @@ module TlogMap = struct
     Logger.debug_f_ "TlogMap.iterate_tlog_dir start_i:%s too_far_i:%s ~index:%s"
                     (Sn.string_of start_i) tfs (Index.to_string index)
     >>= fun () ->
-    get_tlog_names t >>= fun tlog_names -> (* TODO: we probably shouldn't look at the fs *) 
+    get_tlog_names t >>= fun tlog_names -> (* TODO: we probably shouldn't look at the fs *)
     let acc_entry (_i0:Sn.t) entry =
       f entry >>= fun () -> let i = Entry.i_of entry in Lwt.return i
     in
     let num_tlogs = List.length tlog_names in
     let maybe_fold (cnt,low) fn =
       let fn_start = get_start_i t (get_number fn) in
-      
+
       let low_n = Sn.succ low in
       match fn_start with
       | Some fn_start ->
@@ -604,7 +604,7 @@ module TlogMap = struct
            then
              begin
                Logger.debug_f_ "fold over: %s: [%s,...) "
-                               fn (Sn.string_of fn_start) 
+                               fn (Sn.string_of fn_start)
                >>= fun () ->
                let first = fn_start in
                Logger.info_f_ "Replaying tlog file: %s (%d/%d)" fn cnt num_tlogs  >>= fun () ->
@@ -625,14 +625,14 @@ module TlogMap = struct
     Lwt_list.fold_left_s maybe_fold (1,start_i) tlog_names
     >>= fun _x ->
     Lwt.return ()
-      
+
   let tlogs_to_collapse t head_i last_i tlogs_to_keep =
     let () = Logger.ign_debug_f_
                "tlogs_to_collapse head_i:%s last_i:%s tlogs_to_keep:%i tlog_map:%s"
                (Sn.string_of head_i) (Sn.string_of last_i)
                tlogs_to_keep (show_map t)
     in
-    
+
     let collapsable =
       List.filter (fun item -> item.i >= head_i) t.i_to_tlog_number
     in
@@ -649,9 +649,9 @@ module TlogMap = struct
     let r = match to_collapse_rev with
     | [] -> None
     | item :: _ ->
-       (* 
-          +2 because before X goes to the store, 
-          you need to have seen X+1 and thus too_far = X+2 
+       (*
+          +2 because before X goes to the store,
+          you need to have seen X+1 and thus too_far = X+2
        *)
        Some (n_to_collapse, Int64.add item.i 2L)
     in
@@ -661,13 +661,13 @@ module TlogMap = struct
                (Sn.string_of last_i)
                tlogs_to_keep
                (To_string.option (fun (n,i) -> Printf.sprintf "(%i,%s)" n (Sn.string_of i)) r)
-    in            
+    in
     r
-    
+
   let old_uncompressed t =
     List.fold_left
       (fun acc item -> if item.is_archive then acc else item.n :: acc)
       []
-      t.i_to_tlog_number |> List.rev 
-                
+      t.i_to_tlog_number |> List.rev
+
 end

From d07222e5f49ac6da5ca70898227cf586ea23030c Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Mon, 23 May 2016 08:43:08 +0200
Subject: [PATCH 169/373] sets tlog_max_entries default to 50_000

---
 src/node/node_cfg.ml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/node/node_cfg.ml b/src/node/node_cfg.ml
index 0345cee1..e91caaac 100644
--- a/src/node/node_cfg.ml
+++ b/src/node/node_cfg.ml
@@ -367,7 +367,7 @@ module Node_cfg = struct
   let _ips inifile node_name =
     Ini.get inifile node_name "ip" Ini.p_string_list Ini.required
 
-  let _tlog_max_entries inifile = _global_int inifile "tlog_max_entries"    10_000
+  let _tlog_max_entries inifile = _global_int inifile "tlog_max_entries"    50_000
   let _tlog_max_size    inifile = _global_int inifile "tlog_max_size"    5_000_000
 
   let _tlog_entries_overwrite inifile =

From db67b5cf4c51835980dddfd0c4a22c6bdd6705e6 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Mon, 23 May 2016 09:08:26 +0200
Subject: [PATCH 170/373] make_release: 1.9.4

---
 META                      | 2 +-
 debian/changelog          | 5 +++++
 redhat/SPECS/arakoon.spec | 4 +++-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/META b/META
index 16a33cb5..0380ecf7 100644
--- a/META
+++ b/META
@@ -1,7 +1,7 @@
 # -*- conf -*-
 
 description = "Arakoon client"
-version = "1.9.3"
+version = "1.9.4"
 exists_if = "arakoon_client.cma,arakoon_client.cmxa,arakoon_client.cmxs"
 requires = "threads lwt lwt.unix str oUnit quickcheck lwt.ssl ocplib-endian redis.lwt uri"
 archive(byte) = "arakoon_client.cma tcp_keepalive_sockopt_stubs.o"
diff --git a/debian/changelog b/debian/changelog
index 11565472..11587ac2 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,8 @@
+arakoon (1.9.4) unstable; urgency=low
+  * new upstream version
+
+ -- Romain Slootmaekers <romain.slootmaekers@openvstorage.com>  Mon, 23 May 2016 09:08:26 +0100
+
 arakoon (1.9.3) unstable; urgency=low
   * new upstream version
 
diff --git a/redhat/SPECS/arakoon.spec b/redhat/SPECS/arakoon.spec
index 81d52e19..d702163e 100644
--- a/redhat/SPECS/arakoon.spec
+++ b/redhat/SPECS/arakoon.spec
@@ -2,7 +2,7 @@
 
 Summary: Arakoon
 Name: arakoon
-Version: 1.9.3
+Version: 1.9.4
 Release: 3%{?dist}
 License: Apache 2.0
 Requires: libev >= 4
@@ -33,6 +33,8 @@ cp arakoon.native %{buildroot}%{_bindir}/arakoon
 %{_bindir}/arakoon
 
 %changelog
+* Mon May 23 2016 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.4
+- Create arakoon 1.9.4 RPM package
 * Sun Mar 20 2016 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.3
 - Create arakoon 1.9.3 RPM package
 * Fri Feb 26 2016 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.2

From d10489ccd7d50d903cc6dc25e6a0b9401dc431b5 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Wed, 25 May 2016 14:49:43 +0200
Subject: [PATCH 171/373] do etcd reads with --quorum

---
 docker/arakoon_debian/Dockerfile | 8 ++++----
 src/tools/arakoon_etcd.ml        | 1 +
 travis.sh                        | 8 ++++----
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/docker/arakoon_debian/Dockerfile b/docker/arakoon_debian/Dockerfile
index 1868a1f6..ba7e2d19 100644
--- a/docker/arakoon_debian/Dockerfile
+++ b/docker/arakoon_debian/Dockerfile
@@ -75,10 +75,10 @@ ENV LANG=C
 RUN chmod 777 /home/opam/OPAM/4.02.3/lib
 
 # Install etcd:
-RUN curl -L  https://github.com/coreos/etcd/releases/download/v2.2.4/etcd-v2.2.4-linux-amd64.tar.gz -o etcd-v2.2.4-linux-amd64.tar.gz
-RUN tar xzvf etcd-v2.2.4-linux-amd64.tar.gz
-RUN cp ./etcd-v2.2.4-linux-amd64/etcd /usr/bin \
-    && cp ./etcd-v2.2.4-linux-amd64/etcdctl /usr/bin
+RUN curl -L https://github.com/coreos/etcd/releases/download/v2.3.5/etcd-v2.3.5-linux-amd64.tar.gz -o etcd-v2.3.5-linux-amd64.tar.gz
+RUN tar xzvf etcd-v2.3.5-linux-amd64.tar.gz
+RUN cp ./etcd-v2.3.5-linux-amd64/etcd /usr/bin
+RUN cp ./etcd-v2.3.5-linux-amd64/etcdctl /usr/bin
 
 # Install redis:
 RUN wget http://download.redis.io/releases/redis-3.0.7.tar.gz
diff --git a/src/tools/arakoon_etcd.ml b/src/tools/arakoon_etcd.ml
index 2a9be7ac..53dd1b52 100644
--- a/src/tools/arakoon_etcd.ml
+++ b/src/tools/arakoon_etcd.ml
@@ -40,6 +40,7 @@ let retrieve_value peers path =
   let cmd = [| "etcdctl";
                "--peers=" ^ peers_s;
                "get";
+               "--quorum";
                path;
               |]
   in
diff --git a/travis.sh b/travis.sh
index 4a6f5c53..e41ffc46 100755
--- a/travis.sh
+++ b/travis.sh
@@ -53,10 +53,10 @@ install () {
     opam install ${OPAM_DEPENDS}
 
     # Install etcd:
-    curl -L  https://github.com/coreos/etcd/releases/download/v2.2.4/etcd-v2.2.4-linux-amd64.tar.gz -o etcd-v2.2.4-linux-amd64.tar.gz
-    tar xzvf etcd-v2.2.4-linux-amd64.tar.gz
-    sudo cp ./etcd-v2.2.4-linux-amd64/etcd /usr/bin
-    sudo cp ./etcd-v2.2.4-linux-amd64/etcdctl /usr/bin
+    curl -L https://github.com/coreos/etcd/releases/download/v2.3.5/etcd-v2.3.5-linux-amd64.tar.gz -o etcd-v2.3.5-linux-amd64.tar.gz
+    tar xzvf etcd-v2.3.5-linux-amd64.tar.gz
+    sudo cp ./etcd-v2.3.5-linux-amd64/etcd /usr/bin
+    sudo cp ./etcd-v2.3.5-linux-amd64/etcdctl /usr/bin
 }
 
 script () {

From 86cd731eff52623b234faaa86b308e20d6acab5c Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Wed, 1 Jun 2016 17:42:33 +0200
Subject: [PATCH 172/373] take a lock on a file before doing anything else so
 no 2 processes can be started for the same node

---
 src/main/arakoon.ml   |  6 +++++-
 src/node/node_main.ml | 25 +++++++++++++++++++++----
 src/system/startup.ml |  1 +
 3 files changed, 27 insertions(+), 5 deletions(-)

diff --git a/src/main/arakoon.ml b/src/main/arakoon.ml
index 61ada387..ec654c9a 100644
--- a/src/main/arakoon.ml
+++ b/src/main/arakoon.ml
@@ -514,7 +514,11 @@ let main () =
           let main_t =
             Node_main.main_t
               make_config
-              !node_id !daemonize !catchup_only !autofix
+              !node_id
+              ~daemonize:!daemonize
+              ~catchup_only:!catchup_only
+              ~autofix:!autofix
+              ~lock:true
           in
           (* Lwt_engine.set (new Lwt_engine.select :> Lwt_engine.t); *)
           try
diff --git a/src/node/node_main.ml b/src/node/node_main.ml
index a96b1825..d5859d5e 100644
--- a/src/node/node_main.ml
+++ b/src/node/node_main.ml
@@ -388,13 +388,28 @@ let _main_2 (type s)
       get_snapshot_name ~name
       ~daemonize ~catchup_only
       ~autofix
-      ~stop : int Lwt.t
+      ~stop
+      ~lock
+    : int Lwt.t
   =
   Lwt_io.set_default_buffer_size 32768;
   make_config () >>= fun cluster_cfg ->
   let cfgs = cluster_cfg.cfgs in
   let me, others = Node_cfg.Node_cfg.split name cfgs in
   _config_logging me.node_name make_config >>= fun maybe_dump_crash_log ->
+
+  (let lock_file = me.home ^ "/LOCK" in
+   if not lock
+      || Core.Lock_file.create
+           ~message:"Do not remove this file, it's meant to ensure an arakoon process is not running twice"
+           lock_file
+   then Lwt.return ()
+   else Lwt.fail_with
+          (Printf.sprintf
+             "Could not get the required lock at %s, this node must already be running in another process"
+             lock_file))
+  >>= fun () ->
+
   let _ = Lwt_unix.on_signal Sys.sigusr2 (fun _ ->
       let handle () =
         Lwt_unix.sleep 0.001 >>= fun () ->
@@ -1000,7 +1015,7 @@ let _main_2 (type s)
 
 
 let main_t (make_config: unit -> 'a Lwt.t)
-           name daemonize catchup_only autofix : int Lwt.t
+           name ~daemonize ~catchup_only ~autofix ~lock : int Lwt.t
   =
   let module S = (val (Store.make_store_module (module Batched_store.Local_store))) in
   let (make_tlog_coll :Tlogcollection.tlc_factory) = Tlc2.make_tlc2 in
@@ -1008,7 +1023,8 @@ let main_t (make_config: unit -> 'a Lwt.t)
   _main_2 (module S)
           make_tlog_coll
           make_config get_snapshot_name
-          ~name ~daemonize ~catchup_only ~autofix ~stop:(ref false)
+          ~name ~daemonize ~catchup_only ~autofix
+          ~stop:(ref false) ~lock:true
 
 let test_t make_config name ~stop =
   let module S = (val (Store.make_store_module (module Mem_store))) in
@@ -1021,4 +1037,5 @@ let test_t make_config name ~stop =
   and catchup_only = false
   and autofix = false in
   _main_2 (module S) make_tlog_coll make_config get_snapshot_name
-          ~name ~daemonize ~catchup_only ~stop ~autofix
+          ~name ~daemonize ~catchup_only
+          ~stop ~autofix ~lock:false
diff --git a/src/system/startup.ml b/src/system/startup.ml
index 375fcff7..af9b247b 100644
--- a/src/system/startup.ml
+++ b/src/system/startup.ml
@@ -126,6 +126,7 @@ let _make_run ~stores ~tlcs ~now ~values ~get_cfgs name () =
       ~catchup_only:false
       ~autofix:false
       ~stop:!stop
+      ~lock:false
     >>= fun _ -> Lwt.return ()
   in
   node_ts := t :: !node_ts;

From 56bda7de335731ce443d47e5b194af073ba68952 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Thu, 2 Jun 2016 13:31:15 +0200
Subject: [PATCH 173/373] make_release: 1.9.5

---
 META                      | 2 +-
 debian/changelog          | 5 +++++
 redhat/SPECS/arakoon.spec | 4 +++-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/META b/META
index 0380ecf7..862eab86 100644
--- a/META
+++ b/META
@@ -1,7 +1,7 @@
 # -*- conf -*-
 
 description = "Arakoon client"
-version = "1.9.4"
+version = "1.9.5"
 exists_if = "arakoon_client.cma,arakoon_client.cmxa,arakoon_client.cmxs"
 requires = "threads lwt lwt.unix str oUnit quickcheck lwt.ssl ocplib-endian redis.lwt uri"
 archive(byte) = "arakoon_client.cma tcp_keepalive_sockopt_stubs.o"
diff --git a/debian/changelog b/debian/changelog
index 11587ac2..fc68f2c1 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,8 @@
+arakoon (1.9.5) unstable; urgency=low
+  * new upstream version
+
+ -- Jan Doms <jan.doms@gmail.com>  Thu, 02 Jun 2016 13:31:15 +0100
+
 arakoon (1.9.4) unstable; urgency=low
   * new upstream version
 
diff --git a/redhat/SPECS/arakoon.spec b/redhat/SPECS/arakoon.spec
index d702163e..aeb420a5 100644
--- a/redhat/SPECS/arakoon.spec
+++ b/redhat/SPECS/arakoon.spec
@@ -2,7 +2,7 @@
 
 Summary: Arakoon
 Name: arakoon
-Version: 1.9.4
+Version: 1.9.5
 Release: 3%{?dist}
 License: Apache 2.0
 Requires: libev >= 4
@@ -33,6 +33,8 @@ cp arakoon.native %{buildroot}%{_bindir}/arakoon
 %{_bindir}/arakoon
 
 %changelog
+* Thu Jun 02 2016 Jan Doms <jan.doms@gmail.com> - 1.9.5
+- Create arakoon 1.9.5 RPM package
 * Mon May 23 2016 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.4
 - Create arakoon 1.9.4 RPM package
 * Sun Mar 20 2016 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.3

From 20a6c039e26392674184f534a3335ff795cbd2d7 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Thu, 2 Jun 2016 13:37:06 +0200
Subject: [PATCH 174/373] fix for issue #67

---
 src/tlog/tlog_map.ml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tlog/tlog_map.ml b/src/tlog/tlog_map.ml
index f1b8069c..d5481191 100644
--- a/src/tlog/tlog_map.ml
+++ b/src/tlog/tlog_map.ml
@@ -308,7 +308,7 @@ module TlogMap = struct
         let n = get_number tlog_name in
         let canonical = _get_full_path tlog_dir tlx_dir tlog_name in
         first_i_of canonical >>= fun i ->
-        (if i > prev
+        (if i >= prev
         then Lwt.return_unit
         else Lwt.fail_with
                (Printf.sprintf "%S starts with %s, which ain't higher than %s"

From a9231fbf6a49569a4ab595c0414edf2bb92c0111 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Mon, 6 Jun 2016 10:11:50 +0200
Subject: [PATCH 175/373] adapt to std log format

---
 src/tools/arakoon_logger.ml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tools/arakoon_logger.ml b/src/tools/arakoon_logger.ml
index b1b02bf3..4dac75ac 100644
--- a/src/tools/arakoon_logger.ml
+++ b/src/tools/arakoon_logger.ml
@@ -80,7 +80,7 @@ let format_message
        |> Core.Span.to_parts
      in
      Printf.sprintf
-       "%04d/%02d/%02d %02d:%02d:%02d %d %c%02d%02d"
+       "%04d-%02d-%02d %02d:%02d:%02d %d %c%02d%02d"
        (tm.tm_year + 1900)
        (tm.tm_mon + 1)
        tm.tm_mday

From 965511cbb3f7d88d12a9638641444928f8ea7a0a Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Mon, 6 Jun 2016 17:45:16 +0200
Subject: [PATCH 176/373] be less dramatic about disconnecting clients

---
 src/client/client_protocol.ml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/client/client_protocol.ml b/src/client/client_protocol.ml
index 0830ad22..7e62bb81 100644
--- a/src/client/client_protocol.ml
+++ b/src/client/client_protocol.ml
@@ -104,6 +104,9 @@ let handle_exception oc exn=
         Lwt.return (Arakoon_exc.E_UNKNOWN_FAILURE, "unkown failure", true, Logger.Error)
       | Canceled ->
         Lwt.fail Canceled
+      | End_of_file ->
+        Lwt.return (Arakoon_exc.E_UNKNOWN_FAILURE, "end of file because the client disconnected",
+                    true, Logger.Info)
       | _ ->
         Lwt.return (Arakoon_exc.E_UNKNOWN_FAILURE, "unknown failure", true, Logger.Error)
   end

From 4db7adc83486458870178639d0bc927fe23851cf Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Fri, 10 Jun 2016 15:00:28 +0200
Subject: [PATCH 177/373] make_release: 1.9.6

---
 META                      | 2 +-
 debian/changelog          | 5 +++++
 redhat/SPECS/arakoon.spec | 4 +++-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/META b/META
index 862eab86..eed2febe 100644
--- a/META
+++ b/META
@@ -1,7 +1,7 @@
 # -*- conf -*-
 
 description = "Arakoon client"
-version = "1.9.5"
+version = "1.9.6"
 exists_if = "arakoon_client.cma,arakoon_client.cmxa,arakoon_client.cmxs"
 requires = "threads lwt lwt.unix str oUnit quickcheck lwt.ssl ocplib-endian redis.lwt uri"
 archive(byte) = "arakoon_client.cma tcp_keepalive_sockopt_stubs.o"
diff --git a/debian/changelog b/debian/changelog
index fc68f2c1..05e4c308 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,8 @@
+arakoon (1.9.6) unstable; urgency=low
+  * new upstream version
+
+ -- Jan Doms <jan.doms@gmail.com>  Fri, 10 Jun 2016 15:00:28 +0100
+
 arakoon (1.9.5) unstable; urgency=low
   * new upstream version
 
diff --git a/redhat/SPECS/arakoon.spec b/redhat/SPECS/arakoon.spec
index aeb420a5..1837af14 100644
--- a/redhat/SPECS/arakoon.spec
+++ b/redhat/SPECS/arakoon.spec
@@ -2,7 +2,7 @@
 
 Summary: Arakoon
 Name: arakoon
-Version: 1.9.5
+Version: 1.9.6
 Release: 3%{?dist}
 License: Apache 2.0
 Requires: libev >= 4
@@ -33,6 +33,8 @@ cp arakoon.native %{buildroot}%{_bindir}/arakoon
 %{_bindir}/arakoon
 
 %changelog
+* Fri Jun 10 2016 Jan Doms <jan.doms@gmail.com> - 1.9.6
+- Create arakoon 1.9.6 RPM package
 * Thu Jun 02 2016 Jan Doms <jan.doms@gmail.com> - 1.9.5
 - Create arakoon 1.9.5 RPM package
 * Mon May 23 2016 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.4

From cd6c010f098376fcaae824884208e1165372b7fa Mon Sep 17 00:00:00 2001
From: matthias <matthias.deblock@openvstorage.com>
Date: Tue, 21 Jun 2016 14:01:48 +0200
Subject: [PATCH 178/373] Creating gitbook + troubleshooting

---
 GITBOOK.md                                    |  1 +
 SUMMARY.md                                    |  1 +
 book.json                                     |  5 ++
 docs/retrieveArakoonMasters.md                | 15 ++++++
 docs/troubleshooting.md                       |  5 ++
 ...DiscrepancyBetweenAlbaAndArakoonVersion.md | 53 +++++++++++++++++++
 ...eToStartArakoonServiceDueToPowerFailure.md |  0
 7 files changed, 80 insertions(+)
 create mode 100644 GITBOOK.md
 create mode 100644 SUMMARY.md
 create mode 100644 book.json
 create mode 100644 docs/retrieveArakoonMasters.md
 create mode 100644 docs/troubleshooting.md
 create mode 100644 docs/unableToStartArakoonServiceDueToDiscrepancyBetweenAlbaAndArakoonVersion.md
 create mode 100644 docs/unableToStartArakoonServiceDueToPowerFailure.md

diff --git a/GITBOOK.md b/GITBOOK.md
new file mode 100644
index 00000000..5a5001e6
--- /dev/null
+++ b/GITBOOK.md
@@ -0,0 +1 @@
+# Arakoon
\ No newline at end of file
diff --git a/SUMMARY.md b/SUMMARY.md
new file mode 100644
index 00000000..27870e5e
--- /dev/null
+++ b/SUMMARY.md
@@ -0,0 +1 @@
+# Summary
\ No newline at end of file
diff --git a/book.json b/book.json
new file mode 100644
index 00000000..82b5c6dc
--- /dev/null
+++ b/book.json
@@ -0,0 +1,5 @@
+{
+    "structure": {
+        "readme": "GITBOOK.md"
+    }
+}
\ No newline at end of file
diff --git a/docs/retrieveArakoonMasters.md b/docs/retrieveArakoonMasters.md
new file mode 100644
index 00000000..e6746750
--- /dev/null
+++ b/docs/retrieveArakoonMasters.md
@@ -0,0 +1,15 @@
+# Easy way to retrieve the Arakoon masters on a cluster
+
+## 1. Description
+
+Retrieve the Arakoon master on a cluster
+
+## 2. Steps
+### 2.1 Eugene-updates
+```
+echo -e "##########################\nWho is the Arakoon master?\n##########################" ; for i in $(pgrep -a arakoon | cut -d ' ' -f 6); do arakoon --who-master -config $i 2>/dev/null ; echo -ne "     => " ; echo $i | cut -d '/' -f 6 ; done ; echo -e "\nThere should be $(find /var/log/arakoon/ -mindepth 1 -maxdepth 1 -type d | wc -l) arakoons available!"
+```
+### 2.2 Fargo and unstable
+```
+echo -e "##########################\nWho is the Arakoon master?\n##########################" ; for i in $(pgrep -a arakoon | cut -d ' ' -f 6); do arakoon --who-master -config $i 2>/dev/null ; echo -ne "     => " ; echo $i | cut -d '/' -f 6 ; done ; echo -e "\nThere should be $(find /var/log/upstart/ovs-arakoon-* | grep -v gz | wc -l) arakoons available!"
+```
diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md
new file mode 100644
index 00000000..4759c5a4
--- /dev/null
+++ b/docs/troubleshooting.md
@@ -0,0 +1,5 @@
+# Troubleshooting
+
+* [Retrieve Arakoon masters](retrieveArakoonMasters.md)
+* [Unable to start Arakoon service due to discrepancy between Alba and Arakoon version](unableToStartArakoonServiceDueToDiscrepancyBetweenAlbaAndArakoonVersion.md) 
+* [Unable to start Arakoon service due to a power failure](unableToStartArakoonServiceDueToPowerFailure.md)
diff --git a/docs/unableToStartArakoonServiceDueToDiscrepancyBetweenAlbaAndArakoonVersion.md b/docs/unableToStartArakoonServiceDueToDiscrepancyBetweenAlbaAndArakoonVersion.md
new file mode 100644
index 00000000..4837f416
--- /dev/null
+++ b/docs/unableToStartArakoonServiceDueToDiscrepancyBetweenAlbaAndArakoonVersion.md
@@ -0,0 +1,53 @@
+# Unable to start Arakoon service due to discrepancy between Alba and Arakoon version
+
+## 1. Description
+Due to an upgrade of ALBA the Arakoon went down. It was impossible to restart the Arakoon.
+
+Possible error message:
+```
+less /var/log/upstart/ovs-arakoon-du-conv-1-nsm_2.log
+
+Jun 20 10:15:20 7743: (main|fatal): nsm_host_plugin: Dynlink.Error "error loading shared library: /mnt/ovs-db-write/arakoon/du-conv-1-nsm_2/db/nsm_host_plugin.cmxs: undefined symbol: camlArakoon_config_url"
+```
+In the error message above there was a missing config file. After checking the version of Alba and Arakoon this was due to a discrepancy of the version.
+ 
+## 2. Steps
+
+### 2.1 Check the version of Alba and Arakoon
+
+```
+root@du-conv-1-02:~# alba version
+0.9.7
+```
+
+```
+root@du-conv-1-02:~# arakoon --version
+version: 1.8.12
+```
+
+### 2.2 Check if it's the latest version of Alba and Arakoon
+
+```
+root@du-conv-1-02:~# apt-cache policy alba
+alba:
+  Installed: 0.9.7
+  Candidate: 0.9.7
+```
+
+```
+root@du-conv-1-02:~# apt-cache policy arakoon
+arakoon:
+  Installed: 1.8.12
+  Candidate: 1.9.5
+```
+
+As you can see, the Arakoon version isn't the latest one.
+
+<div style="color:black;background-color:#cccccc;font-size:150%">
+  <strong>Warning!</strong> When you upgrade Alba you need to upgrade Arakoon as well.
+</div>
+
+### 2.3 Upgrade arakoon
+```
+apt-get install --only-upgrade arakoon
+```
diff --git a/docs/unableToStartArakoonServiceDueToPowerFailure.md b/docs/unableToStartArakoonServiceDueToPowerFailure.md
new file mode 100644
index 00000000..e69de29b

From 89577274a1ec454a8bfb7643ebddf0e332298926 Mon Sep 17 00:00:00 2001
From: matthias <matthias.deblock@openvstorage.com>
Date: Tue, 21 Jun 2016 14:31:32 +0200
Subject: [PATCH 179/373] Creating gitbook + troubleshooting

---
 docs/retrieveArakoonMasters.md                                | 4 +++-
 docs/troubleshooting.md                                       | 1 -
 ...koonServiceDueToDiscrepancyBetweenAlbaAndArakoonVersion.md | 4 +++-
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/docs/retrieveArakoonMasters.md b/docs/retrieveArakoonMasters.md
index e6746750..6b464f36 100644
--- a/docs/retrieveArakoonMasters.md
+++ b/docs/retrieveArakoonMasters.md
@@ -1,8 +1,10 @@
 # Easy way to retrieve the Arakoon masters on a cluster
 
+Applies to: Eugene-updates & Fargo
+
 ## 1. Description
 
-Retrieve the Arakoon master on a cluster
+Retrieve the Arakoon master on a cluster.
 
 ## 2. Steps
 ### 2.1 Eugene-updates
diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md
index 4759c5a4..c73d9679 100644
--- a/docs/troubleshooting.md
+++ b/docs/troubleshooting.md
@@ -2,4 +2,3 @@
 
 * [Retrieve Arakoon masters](retrieveArakoonMasters.md)
 * [Unable to start Arakoon service due to discrepancy between Alba and Arakoon version](unableToStartArakoonServiceDueToDiscrepancyBetweenAlbaAndArakoonVersion.md) 
-* [Unable to start Arakoon service due to a power failure](unableToStartArakoonServiceDueToPowerFailure.md)
diff --git a/docs/unableToStartArakoonServiceDueToDiscrepancyBetweenAlbaAndArakoonVersion.md b/docs/unableToStartArakoonServiceDueToDiscrepancyBetweenAlbaAndArakoonVersion.md
index 4837f416..6ca2647f 100644
--- a/docs/unableToStartArakoonServiceDueToDiscrepancyBetweenAlbaAndArakoonVersion.md
+++ b/docs/unableToStartArakoonServiceDueToDiscrepancyBetweenAlbaAndArakoonVersion.md
@@ -1,7 +1,9 @@
 # Unable to start Arakoon service due to discrepancy between Alba and Arakoon version
 
+Applies to: Eugene-updates & Fargo
+
 ## 1. Description
-Due to an upgrade of ALBA the Arakoon went down. It was impossible to restart the Arakoon.
+Due to an upgrade of ALBA the Arakoon went down. It was impossible to restart the Arakoon. 
 
 Possible error message:
 ```

From e3a5e70b81323f68cba60e88dc0a1f937082e13f Mon Sep 17 00:00:00 2001
From: matthias <matthias.deblock@openvstorage.com>
Date: Tue, 21 Jun 2016 14:37:52 +0200
Subject: [PATCH 180/373] Creating gitbook + troubleshooting

---
 GITBOOK.md                                           | 6 +++++-
 SUMMARY.md                                           | 6 +++++-
 docs/unableToStartArakoonServiceDueToPowerFailure.md | 0
 3 files changed, 10 insertions(+), 2 deletions(-)
 delete mode 100644 docs/unableToStartArakoonServiceDueToPowerFailure.md

diff --git a/GITBOOK.md b/GITBOOK.md
index 5a5001e6..3f6379b6 100644
--- a/GITBOOK.md
+++ b/GITBOOK.md
@@ -1 +1,5 @@
-# Arakoon
\ No newline at end of file
+# Arakoon
+
+Arakoon is a distributed key-value store that guarantees consistency above anything else. It was created by the Open vStorage team due to a lack of existing solutions fitting our requirements, and is available as Open Source software.
+
+[www.arakoon.org](http://arakoon.org/)
diff --git a/SUMMARY.md b/SUMMARY.md
index 27870e5e..d3718ff8 100644
--- a/SUMMARY.md
+++ b/SUMMARY.md
@@ -1 +1,5 @@
-# Summary
\ No newline at end of file
+# Summary
+
+* [Arakoon Troubleshooting](docs/troubleshooting.md)
+    * [Retrieve Arakoon masters](docs/retrieveArakoonMasters.md)
+    * [Unable to start Arakoon service due to discrepancy between Alba and Arakoon version](docs/unableToStartArakoonServiceDueToDiscrepancyBetweenAlbaAndArakoonVersion.md)
\ No newline at end of file
diff --git a/docs/unableToStartArakoonServiceDueToPowerFailure.md b/docs/unableToStartArakoonServiceDueToPowerFailure.md
deleted file mode 100644
index e69de29b..00000000

From 82b478214065bd4c6e8e1a4c28519a3047ad9035 Mon Sep 17 00:00:00 2001
From: Wim Provoost <wim@openvstorage.com>
Date: Tue, 21 Jun 2016 14:44:58 +0200
Subject: [PATCH 181/373] Adding Gitbook link

---
 README.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/README.md b/README.md
index a8cd5c8f..466569a4 100644
--- a/README.md
+++ b/README.md
@@ -51,6 +51,11 @@ An overview of all command-line options is available by running `arakoon.native
 
 Of course you want to use Arakoon from within a program. We provide OCaml, C and Python clients.
 
+Documentation
+--------------
+For additional information please visit the [Arakoon GitBook](https://openvstorage.gitbooks.io/arakoon).
+
+
 Releases
 --------
 

From 9a44510f08f24e5393764df91d7797b06af5852e Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Wed, 22 Jun 2016 17:47:35 +0200
Subject: [PATCH 182/373] write a failing test for tlc # get_last_i

---
 src/tlog/tlc2_test.ml           | 111 +++++++++++++++++++++++++++++++-
 src/tlog/tlogcollection_test.ml |   6 --
 2 files changed, 108 insertions(+), 9 deletions(-)

diff --git a/src/tlog/tlc2_test.ml b/src/tlog/tlc2_test.ml
index 691c8012..eb5bf1b5 100644
--- a/src/tlog/tlc2_test.ml
+++ b/src/tlog/tlc2_test.ml
@@ -110,7 +110,6 @@ let test_iterate4 (dn, tlx_dir, (factory:Tlogcollection_test.factory)) =
 
 
 let test_iterate5 (dn, _tlx_dir, (factory:factory)) =
-  TlogMap.make ~tlog_max_entries:10 dn _tlx_dir node_id ~check_marker:true >>= fun tlog_map ->
   factory dn node_id >>= fun (tlc:tlog_collection) ->
   let rec loop (tlc:tlog_collection) i =
     if i = 33
@@ -149,7 +148,6 @@ let test_iterate5 (dn, _tlx_dir, (factory:factory)) =
   Lwt.return ()
 
 let test_iterate6 (dn, _tlx_dir, (factory:factory)) =
-  TlogMap.make ~tlog_max_entries:10 dn _tlx_dir node_id ~check_marker:true >>= fun tlog_map ->
   let sync = false in
   factory dn node_id >>= fun (tlc:tlog_collection) ->
   let rec loop i =
@@ -195,6 +193,48 @@ let test_iterate6 (dn, _tlx_dir, (factory:factory)) =
   (* OUnit.assert_equal ~printer:string_of_int 19 !sum; *)
   Lwt.return ()
 
+let test_iterate7 (dn, tlx_dir, (factory:factory)) =
+  Logger.info_f_ "test_iterate7  %s, %s" dn tlx_dir >>= fun () ->
+  let tlog_max_size = 32000 in
+  factory ~tlog_max_size dn node_id >>= fun tlc ->
+  let make_value size =
+    Value.create_client_value [ Update.Set ("", Bytes.create size); ] false
+  in
+  tlc # log_value 0L (make_value 5) >>= fun _ ->
+  tlc # log_value 1L (make_value 5) >>= fun _ ->
+  tlc # log_value 2L (make_value 5) >>= fun _ ->
+  tlc # log_value 2L (make_value tlog_max_size) >>= fun _ ->
+  tlc # log_value 3L (make_value tlog_max_size) >>= fun _ ->
+  tlc # log_value 4L (make_value 5) >>= fun _ ->
+  tlc # log_value 5L (make_value tlog_max_size) >>= fun _ ->
+  tlc # log_value 6L (make_value tlog_max_size) >>= fun _ ->
+  tlc # log_value 7L (make_value tlog_max_size) >>= fun _ ->
+
+  TlogMap.make dn tlx_dir node_id ~check_marker:false >>= fun (tlog_map,_,_) ->
+  Lwt_log.debug_f "%s" (TlogMap.show_map tlog_map) >>= fun () ->
+  assert ([ (7L, 4);
+            (6L, 3);
+            (4L, 2);
+            (3L, 1);
+            (0L, 0);
+          ] =
+            List.map
+              (fun ({ TlogMap.i; n; is_archive; }) ->
+               i, n)
+              tlog_map.TlogMap.i_to_tlog_number);
+
+  let next = ref 0L in
+  tlc # iterate
+      0L 6L
+      (fun entry ->
+       Lwt_log.debug_f "iter entry: %s" (Tlogcommon.Entry.entry2s entry) >>= fun () ->
+       assert (!next = Tlogcommon.Entry.i_of entry);
+       next := Int64.succ !next;
+       Lwt.return ())
+      (fun i -> Lwt.return ())
+  >>= fun () ->
+  assert (!next = 5L);
+  Lwt.return ()
 
 let test_compression_bug (dn, tlx_dir, (factory:factory)) =
   Logger.info_ "test_compression_bug" >>= fun () ->
@@ -314,6 +354,69 @@ let test_size_based_roll_over1 (dn, tlx_dir, (factory:factory)) =
   OUnit.assert_equal 3 (TlogMap.get_tlog_number second_map) ~printer;
   Lwt.return_unit
 
+let test_get_last_i (dn, tlx_dir, (factory:factory)) =
+  let make_value size =
+    Value.create_client_value [ Update.Set ("", Bytes.create size); ] false
+  in
+  let value = make_value 0 in
+
+  factory dn node_id >>= fun tlc ->
+
+  (* this one is a bit special ...
+   * IMO the correct return value should be None *)
+  assert (0L = tlc # get_last_i ());
+
+  tlc # log_value 0L value >>= fun _ ->
+  assert (0L = tlc # get_last_i ());
+
+  tlc # log_value 1L value >>= fun _ ->
+  assert (1L = tlc # get_last_i ());
+
+  let ic, oc = Lwt_io.pipe () in
+
+  let _ =
+    let tlog_max_size = 32000 in
+    Tlogcollection_test.setup factory "test_get_last_i_x2" () >>= fun (dn, tlx_dir, factory) ->
+    factory ~tlog_max_size dn node_id >>= fun tlc2 ->
+    tlc2 # log_value 0L value >>= fun _ ->
+    tlc2 # log_value 1L value >>= fun _ ->
+    tlc2 # log_value 2L value >>= fun _ ->
+    tlc2 # log_value 3L (make_value tlog_max_size) >>= fun _ ->
+    tlc2 # log_value 4L value >>= fun _ ->
+    tlc2 # log_value 5L value >>= fun _ ->
+
+    (* did we do what we expected to do? *)
+    TlogMap.make dn tlx_dir node_id ~check_marker:false >>= fun (tlog_map,_,_) ->
+    assert ([ (4L, 1);
+              (0L, 0);
+            ] =
+              List.map
+                (fun ({ TlogMap.i; n; is_archive; }) ->
+                 i, n)
+                tlog_map.TlogMap.i_to_tlog_number);
+
+    tlc2 # dump_tlog_file
+         1
+         oc
+  in
+
+  Llio.input_string ic >>= fun extension ->
+  Sn.input_sn ic >>= fun start_i ->
+  assert (start_i = 4L);
+  Llio.input_int64 ic >>= fun length ->
+
+  tlc # save_tlog_file
+      start_i
+      extension
+      length
+      ic >>= fun () ->
+  let last_i = tlc # get_last_i () in
+  Lwt_log.debug_f "got last_i = %Li" last_i >>= fun () ->
+  assert (5L = last_i);
+
+  Lwt.return_unit
+
+
 let make_test_tlc (x, y) = x >:: wrap_tlc y x
 
 let suite = "tlc2" >:::
@@ -329,6 +432,7 @@ let suite = "tlc2" >:::
                   ("test_iterate4", test_iterate4);
                   ("test_iterate5", test_iterate5);
                   ("test_iterate6", test_iterate6);
+                  ("test_iterate7", test_iterate7);
                   ("validate",  Tlogcollection_test.test_validate_normal);
                   ("validate_corrupt", Tlogcollection_test.test_validate_corrupt_1);
                   ("test_rollover_1002", Tlogcollection_test.test_rollover_1002);
@@ -336,5 +440,6 @@ let suite = "tlc2" >:::
                   ("test_interrupted_rollover", test_interrupted_rollover);
                   ("test_compression_bug", test_compression_bug);
                   ("test_compression_previous", test_compression_previous);
-                  ("test_size_based_roll_over1",test_size_based_roll_over1)
+                  ("test_size_based_roll_over1", test_size_based_roll_over1);
+                  ("test_get_last_i", test_get_last_i);
                 ]
diff --git a/src/tlog/tlogcollection_test.ml b/src/tlog/tlogcollection_test.ml
index 8602eb32..b5c202bf 100644
--- a/src/tlog/tlogcollection_test.ml
+++ b/src/tlog/tlogcollection_test.ml
@@ -82,7 +82,6 @@ let _log_repeat tlc (value:Value.t) n =
 
 let test_rollover (dn, tlx_dir, (factory:factory)) =
   Logger.info_f_ "test_rollover %s, %s" dn tlx_dir >>= fun () ->
-  TlogMap.make ~tlog_max_entries:5 dn tlx_dir node_id ~check_marker:true >>= fun tlog_map ->
   factory dn node_id >>= fun (c:tlog_collection) ->
   let value = _make_set_v "x" "y" in
   _log_repeat c value 101 >>= fun () ->
@@ -147,7 +146,6 @@ let test_restart (dn, _tlf_dir, (factory:factory)) =
 
 let test_iterate (dn, tlx_dir, (factory: factory)) =
   Logger.info_f_ "test_iterate  %s, %s" dn tlx_dir >>= fun () ->
-  TlogMap.make ~tlog_max_entries:100 dn tlx_dir node_id ~check_marker:true >>= fun tlog_map ->
   factory dn node_id >>= fun  (tlc:tlog_collection) ->
   let value = _make_set_v "xxx" "y" in
   _log_repeat tlc value 323 >>= fun () ->
@@ -169,7 +167,6 @@ let test_iterate (dn, tlx_dir, (factory: factory)) =
 
 let test_iterate2 (dn, tlx_dir, (factory:factory)) =
   Logger.info_f_ "test_iterate2  %s, %s" dn tlx_dir >>= fun () ->
-  TlogMap.make ~tlog_max_entries:100 dn tlx_dir node_id ~check_marker:true >>= fun tlog_map ->
   factory dn node_id >>= fun (tlc:tlog_collection) ->
   let value = _make_set_v "test_iterate0" "xxx" in
   _log_repeat tlc value 3 >>= fun () ->
@@ -191,7 +188,6 @@ let test_iterate2 (dn, tlx_dir, (factory:factory)) =
 
 let test_iterate3 (dn, tlx_dir, (factory:factory)) =
   Logger.info_f_ "test_iterate3  %s, %s" dn tlx_dir >>= fun () ->
-  TlogMap.make ~tlog_max_entries:100 dn tlx_dir node_id ~check_marker:true >>= fun tlog_map ->
   factory dn node_id  >>= fun (tlc:tlog_collection) ->
   let value = _make_set_v "test_iterate3" "xxx" in
   _log_repeat tlc value 120 >>= fun () ->
@@ -211,7 +207,6 @@ let test_iterate3 (dn, tlx_dir, (factory:factory)) =
 
 let test_validate_normal (dn, tlx_dir, (factory:factory)) =
   Logger.info_f_ "test_validate_normal  %s, %s" dn tlx_dir >>= fun () ->
-  TlogMap.make ~tlog_max_entries:100 dn tlx_dir node_id ~check_marker:true >>= fun tlog_map ->
   factory dn node_id >>= fun (tlc:tlog_collection) ->
   let value = _make_set_v "XXX" "X" in
   _log_repeat tlc value 123 >>= fun () ->
@@ -231,7 +226,6 @@ let test_validate_normal (dn, tlx_dir, (factory:factory)) =
 
 let test_validate_corrupt_1 (dn, tlx_dir, (factory:factory)) =
   let open Tlog_map in
-  TlogMap.make ~tlog_max_entries:100 dn tlx_dir node_id ~check_marker:true >>= fun tlog_map ->
   factory dn node_id >>= fun (tlc:tlog_collection) ->
   let value = _make_set_v "Incompetent" "Politicians" in
   _log_repeat tlc value 42 >>= fun () ->

From 8e567c8e9d6c10c1aa603ce704b7450a8eb17d54 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Thu, 23 Jun 2016 10:17:16 +0200
Subject: [PATCH 183/373] change tlogcollection # get_last_x signature

---
 src/node/catchup.ml             |  9 ++++----
 src/node/collapser.ml           |  2 +-
 src/node/node_main.ml           |  2 +-
 src/paxos/multi_paxos.ml        |  4 ++--
 src/paxos/multi_paxos_fsm.ml    |  2 +-
 src/tlog/mem_tlogcollection.ml  | 23 +++++++++++---------
 src/tlog/tlc2.ml                | 37 ++++++++++++++++++---------------
 src/tlog/tlc2_test.ml           | 11 ++++++----
 src/tlog/tlogcollection.ml      |  8 ++++---
 src/tlog/tlogcollection_test.ml |  4 ++--
 10 files changed, 57 insertions(+), 45 deletions(-)

diff --git a/src/node/catchup.ml b/src/node/catchup.ml
index 6cca95fc..c88870b4 100644
--- a/src/node/catchup.ml
+++ b/src/node/catchup.ml
@@ -65,7 +65,7 @@ let stop_fuse stop =
 
 let catchup_tlog (type s) ~tls_ctx ~tcp_keepalive ~stop other_configs ~cluster_id  mr_name ((module S : Store.STORE with type t = s),store,tlog_coll)
   =
-  let current_i = tlog_coll # get_last_i () in
+  tlog_coll # get_last_i () >>= fun current_i ->
   Logger.info_f_ "catchup_tlog %s from %s " (Sn.string_of current_i) mr_name >>= fun () ->
   let mr_addresses =
     try
@@ -109,7 +109,7 @@ let catchup_tlog (type s) ~tls_ctx ~tcp_keepalive ~stop other_configs ~cluster_i
     )
     (fun exn -> Logger.warning_ ~exn "catchup_tlog failed")
   >>= fun () ->
-  Lwt.return (tlog_coll # get_last_i ())
+  tlog_coll # get_last_i ()
 
 
 let prepare_and_insert_value (type s) (module S : Store.STORE with type t = s) me store i v =
@@ -237,8 +237,9 @@ let catchup_store ~stop (type s) me
 let catchup
       ~tls_ctx ~tcp_keepalive
       ~stop me other_configs ~cluster_id ((_,_,tlog_coll) as dbt) mr_name =
+  tlog_coll # get_last_i () >>= fun last_i ->
   Logger.info_f_ "CATCHUP start: I'm @ %s and %s is more recent"
-    (Sn.string_of (tlog_coll # get_last_i ())) mr_name
+    (Sn.string_of last_i) mr_name
   >>= fun () ->
   catchup_tlog
     ~tls_ctx ~tcp_keepalive
@@ -252,7 +253,7 @@ let verify_n_catchup_store
       (type s) ~stop me ?(apply_last_tlog_value=false)
       ((module S : Store.STORE with type t = s), store, tlog_coll)
   =
-  let current_i = tlog_coll # get_last_i () in
+  tlog_coll # get_last_i () >>= fun current_i ->
   let too_far_i =
     if apply_last_tlog_value
     then
diff --git a/src/node/collapser.ml b/src/node/collapser.ml
index b611dc1a..056e8950 100644
--- a/src/node/collapser.ml
+++ b/src/node/collapser.ml
@@ -201,7 +201,7 @@ let collapse_many
 
   Logger.debug_f_ "collapse_many" >>= fun () ->
   let (_,(head_location:string), _) = store_fs in
-  let last_i = tlog_coll # get_last_i () in
+  tlog_coll # get_last_i () >>= fun last_i ->
   let get_head_i () =
     _head_i (module S) head_location >>= fun head_io ->
     Logger.debug_f_ "head @ %s : last_i %s " (Log_extra.option2s Sn.string_of head_io) (Sn.string_of last_i)
diff --git a/src/node/node_main.ml b/src/node/node_main.ml
index d5859d5e..54922e0f 100644
--- a/src/node/node_main.ml
+++ b/src/node/node_main.ml
@@ -618,7 +618,7 @@ let _main_2 (type s)
 
           open_tlog_collection_and_store me ~autofix
           >>= fun (tlog_coll, store) ->
-          let last_i = tlog_coll # get_last_i () in
+          tlog_coll # get_last_i () >>= fun last_i ->
           begin
             if me.is_witness
             then
diff --git a/src/paxos/multi_paxos.ml b/src/paxos/multi_paxos.ml
index 11bfda8c..a7a26027 100644
--- a/src/paxos/multi_paxos.ml
+++ b/src/paxos/multi_paxos.ml
@@ -107,7 +107,7 @@ type 'a constants =
   {me:id;
    others: id list;
    send: MPMessage.t -> id -> id -> unit Lwt.t;
-   get_value: Sn.t -> Value.t option;
+   get_value: Sn.t -> Value.t option Lwt.t;
    on_accept: Value.t * Sn.t * Sn.t -> unit Lwt.t;
    on_consensus:
      Value.t * Mp_msg.MPMessage.n * Mp_msg.MPMessage.n ->
@@ -353,7 +353,7 @@ let handle_prepare (type s) constants dest n n' i' =
               (* Ok, we can make a Promise to the other node, if we want to *)
               let make_promise () =
                 constants.respect_run_master <- Some (dest, Unix.gettimeofday () +. (float constants.lease_expiration) /. 4.0);
-                let lv = constants.get_value nak_max in
+                constants.get_value nak_max >>= fun lv ->
                 let reply = Promise(n',nak_max,lv) in
                 Logger.info_f_ "%s: handle_prepare: starting election timer" me >>= fun () ->
                 start_election_timeout constants n' i' >>= fun () ->
diff --git a/src/paxos/multi_paxos_fsm.ml b/src/paxos/multi_paxos_fsm.ml
index 8f54b5be..1f28c115 100644
--- a/src/paxos/multi_paxos_fsm.ml
+++ b/src/paxos/multi_paxos_fsm.ml
@@ -28,7 +28,7 @@ open Master_type
 let election_suggest (type s) constants (n, counter) () =
   let module S = (val constants.store_module : Store.STORE with type t = s) in
   let i = S.get_succ_store_i constants.store in
-  let vo = constants.get_value i in
+  constants.get_value i >>= fun vo ->
   let me = constants.me in
   let v_lims, msg =
     match vo with
diff --git a/src/tlog/mem_tlogcollection.ml b/src/tlog/mem_tlogcollection.ml
index e428a135..11c0bacf 100644
--- a/src/tlog/mem_tlogcollection.ml
+++ b/src/tlog/mem_tlogcollection.ml
@@ -29,14 +29,15 @@ class mem_tlog_collection _name =
     method get_infimum_i () = Lwt.return Sn.start
 
     method get_last_i () =
-      match last_entry with
-        | None -> Sn.start
-        | Some entry -> Entry.i_of entry
+      (match last_entry with
+       | None -> Sn.start
+       | Some entry -> Entry.i_of entry)
+      |> Lwt.return
 
     method get_last_value i =
-      match last_entry with
-        | None -> None
-        | Some entry ->
+      (match last_entry with
+       | None -> None
+       | Some entry ->
           let i' = Entry.i_of entry in
           begin
             if i = i'
@@ -45,12 +46,14 @@ class mem_tlog_collection _name =
               Some v
             else
               None
-          end
+          end)
+      |> Lwt.return
 
     method get_last () =
-      match last_entry with
-        | None -> None
-        | Some e -> Some (Entry.v_of e, Entry.i_of e)
+      (match last_entry with
+       | None -> None
+       | Some e -> Some (Entry.v_of e, Entry.i_of e))
+      |> Lwt.return
 
 
     method iterate from_i too_far_i f _cb =
diff --git a/src/tlog/tlc2.ml b/src/tlog/tlc2.ml
index f24e2329..1072cc5c 100644
--- a/src/tlog/tlc2.ml
+++ b/src/tlog/tlc2.ml
@@ -415,30 +415,33 @@ class tlc2
 
 
     method get_last_i () =
-      match _previous_entry with
-        | None -> Sn.start
-        | Some pe -> let pi = Entry.i_of pe in pi
+      (match _previous_entry with
+       | None -> Sn.start
+       | Some pe -> let pi = Entry.i_of pe in pi)
+      |> Lwt.return
 
     method get_last_value i =
-      match _previous_entry with
-        | None -> None
-        | Some pe ->
+      (match _previous_entry with
+       | None -> None
+       | Some pe ->
           let pi = Entry.i_of pe in
           if pi = i
           then Some (Entry.v_of pe)
           else
-          if i > pi
-          then None
-          else (* pi > i *)
-            let msg = Printf.sprintf "get_last_value %s > %s can't look back so far"
-                        (Sn.string_of pi) (Sn.string_of i)
-            in
-            failwith msg
+            if i > pi
+            then None
+            else (* pi > i *)
+              let msg = Printf.sprintf "get_last_value %s > %s can't look back so far"
+                                       (Sn.string_of pi) (Sn.string_of i)
+              in
+              failwith msg)
+      |> Lwt.return
 
     method get_last () =
-      match _previous_entry with
-        | None -> None
-        | Some pe ->  Some (Entry.v_of pe, Entry.i_of pe)
+      (match _previous_entry with
+       | None -> None
+       | Some pe ->  Some (Entry.v_of pe, Entry.i_of pe))
+      |> Lwt.return
 
     method tlogs_to_collapse ~head_i ~last_i ~tlogs_to_keep =
       TlogMap.tlogs_to_collapse tlog_map head_i last_i tlogs_to_keep
@@ -472,7 +475,7 @@ class tlc2
               Logger.info_ ~exn "Exception while canceling compression thread")
       end >>= fun () ->
       Logger.debug_ "tlc2::closes () (part2)" >>= fun () ->
-      match self # get_last () with
+      self # get_last () >>= function
       | None -> Logger.debug_f_ "... no last, we never logged anything=> no marker"
       | Some(v,i) ->
          begin
diff --git a/src/tlog/tlc2_test.ml b/src/tlog/tlc2_test.ml
index eb5bf1b5..75fefed6 100644
--- a/src/tlog/tlc2_test.ml
+++ b/src/tlog/tlc2_test.ml
@@ -364,13 +364,16 @@ let test_get_last_i (dn, tlx_dir, (factory:factory)) =
 
   (* this one is a bit special ...
    * IMO the correct return value should be None *)
-  assert (0L = tlc # get_last_i ());
+  tlc # get_last_i () >>= fun last_i ->
+  assert (0L = last_i);
 
   tlc # log_value 0L value >>= fun _ ->
-  assert (0L = tlc # get_last_i ());
+  tlc # get_last_i () >>= fun last_i ->
+  assert (0L = last_i);
 
   tlc # log_value 1L value >>= fun _ ->
-  assert (1L = tlc # get_last_i ());
+  tlc # get_last_i () >>= fun last_i ->
+  assert (1L = last_i);
 
   let ic, oc = Lwt_io.pipe () in
 
@@ -410,7 +413,7 @@ let test_get_last_i (dn, tlx_dir, (factory:factory)) =
       extension
       length
       ic >>= fun () ->
-  let last_i = tlc # get_last_i () in
+  tlc # get_last_i () >>= fun last_i ->
   Lwt_log.debug_f "got last_i = %Li" last_i >>= fun () ->
   assert (5L = last_i);
 
diff --git a/src/tlog/tlogcollection.ml b/src/tlog/tlogcollection.ml
index ec0b047c..ea248e83 100644
--- a/src/tlog/tlogcollection.ml
+++ b/src/tlog/tlogcollection.ml
@@ -36,8 +36,6 @@ class type tlog_collection =
            tlogs_to_keep:int ->
            (int * Sn.t) option
                
-  method get_last_value: Sn.t -> Value.t option (* Lwt.t *)
-  method get_last: unit -> (Value.t * Sn.t) option
   method close : ?wait_for_compression : bool -> unit -> unit Lwt.t
   method get_infimum_i : unit -> Sn.t Lwt.t
   method dump_head : Lwt_io.output_channel -> Sn.t Lwt.t
@@ -60,7 +58,11 @@ class type tlog_collection =
   method remove_below : Sn.t -> unit Lwt.t
                                      
   method get_start_i : int -> Sn.t option
-  method get_last_i: unit -> Sn.t
+
+  method get_last_i: unit -> Sn.t Lwt.t
+  method get_last_value: Sn.t -> Value.t option Lwt.t
+  method get_last: unit -> (Value.t * Sn.t) option Lwt.t
+
   method is_rollover_point: Sn.t -> bool
   method next_rollover: Sn.t -> Sn.t option
   method which_tlog_file : int -> (string option) Lwt.t
diff --git a/src/tlog/tlogcollection_test.ml b/src/tlog/tlogcollection_test.ml
index b5c202bf..dabfefbe 100644
--- a/src/tlog/tlogcollection_test.ml
+++ b/src/tlog/tlogcollection_test.ml
@@ -98,7 +98,7 @@ let test_rollover_1002 (dn, tlx_dir, (factory:factory)) =
   _log_repeat c value n_updates >>= fun () ->
   c # close () >>= fun () ->
   factory dn node_id >>= fun tlc_two ->
-  let vo = tlc_two # get_last_value (Sn.of_int (n_updates-1)) in
+  tlc_two # get_last_value (Sn.of_int (n_updates-1)) >>= fun vo ->
   let vos = Log_extra.option2s (Value.value2s ~values:false) vo in
   Logger.info_f_ "last_value = %s" vos >>= fun () ->
   tlc_two # close() >>= fun () ->
@@ -112,7 +112,7 @@ let test_get_value_bug (dn, _tlf_dir, (factory:factory)) =
   c0 # close () >>= fun () ->
   factory dn node_id >>= fun c1 ->
   (* c1 # validate () >>= fun _ -> *)
-  match c1 # get_last_value 0L with
+  c1 # get_last_value 0L >>= function
     | None -> Llio.lwt_failfmt "get_last_update 0 yields None"
     | Some v -> let () = OUnit.assert_equal v v0 in Lwt.return ()
 

From 745258472996bd1fca2add65afe99681e106c1d6 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Thu, 23 Jun 2016 10:43:09 +0200
Subject: [PATCH 184/373] fix tlogcollection # get_last implementation

---
 src/tlog/tlc2.ml      | 54 ++++++++++++++++++++++++++++---------------
 src/tlog/tlc2_test.ml |  3 +--
 src/tlog/tlog_map.ml  |  4 ++--
 3 files changed, 39 insertions(+), 22 deletions(-)

diff --git a/src/tlog/tlc2.ml b/src/tlog/tlc2.ml
index 1072cc5c..a1d64032 100644
--- a/src/tlog/tlc2.ml
+++ b/src/tlog/tlc2.ml
@@ -154,11 +154,26 @@ class tlc2
         (node_id:string) ~(fsync:bool) ~(fsync_tlog_dir:bool)
         
   =
-
+  let _previous_entry = ref (Some last) in
+  let set_previous_entry e =
+    _previous_entry := Some e
+  in
+  let invalidate_previous_entry () =
+    _previous_entry := None
+  in
+  let get_previous_entry () =
+    match !_previous_entry with
+    | None ->
+       TlogMap.reinit tlog_map >>= fun last ->
+       set_previous_entry last;
+       Lwt.return last
+    | Some p ->
+       Lwt.return p
+  in
   object(self: # tlog_collection)
     val mutable _file = file
     val mutable _index = index
-    val mutable _previous_entry = last
+
     val mutable _compression_q = Lwt_buffer.create_fixed_capacity 5
     val mutable _compression_thread = None
     val mutable _compressing = false
@@ -267,7 +282,7 @@ class tlc2
              end
              >>= fun () ->
              let entry = Entry.make i value p marker in
-             _previous_entry <- Some entry;
+             set_previous_entry (Some entry);
              Index.note entry _index;
              let () = TlogMap.new_entry tlog_map total_size in
              Lwt.return total_size
@@ -395,6 +410,7 @@ class tlc2
       Lwt.return next_i
 
     method save_head ic =
+      invalidate_previous_entry ();
       Logger.info_ "save_head()" >>= fun () ->
       Llio.input_int64 ic >>= fun length ->
       let hf_name = self # get_head_name () in
@@ -415,33 +431,33 @@ class tlc2
 
 
     method get_last_i () =
-      (match _previous_entry with
-       | None -> Sn.start
-       | Some pe -> let pi = Entry.i_of pe in pi)
-      |> Lwt.return
+      get_previous_entry () >>= function
+      | None ->
+         Lwt.return Sn.start
+      | Some pe ->
+         let pi = Entry.i_of pe in
+         Lwt.return pi
 
     method get_last_value i =
-      (match _previous_entry with
-       | None -> None
+      get_previous_entry () >>= function
+       | None -> Lwt.return None
        | Some pe ->
           let pi = Entry.i_of pe in
           if pi = i
-          then Some (Entry.v_of pe)
+          then Lwt.return (Some (Entry.v_of pe))
           else
             if i > pi
-            then None
+            then Lwt.return None
             else (* pi > i *)
               let msg = Printf.sprintf "get_last_value %s > %s can't look back so far"
                                        (Sn.string_of pi) (Sn.string_of i)
               in
-              failwith msg)
-      |> Lwt.return
+              Lwt.fail_with msg
 
     method get_last () =
-      (match _previous_entry with
-       | None -> None
-       | Some pe ->  Some (Entry.v_of pe, Entry.i_of pe))
-      |> Lwt.return
+      get_previous_entry () >>= function
+      | None -> Lwt.return None
+      | Some pe -> Lwt.return (Some (Entry.v_of pe, Entry.i_of pe))
 
     method tlogs_to_collapse ~head_i ~last_i ~tlogs_to_keep =
       TlogMap.tlogs_to_collapse tlog_map head_i last_i tlogs_to_keep
@@ -548,6 +564,7 @@ class tlc2
          Lwt.return_unit
 
     method save_tlog_file first_i extension length ic =
+      invalidate_previous_entry ();
       (* what with rotation (jump to new tlog), open streams, ...*)
       self # _maybe_close_current_file ()
       >>= fun () ->
@@ -570,13 +587,14 @@ class tlc2
 
 
     method reinit () =
+      invalidate_previous_entry ();
       begin
         match _file with
         | None -> Lwt.return ()
         | Some file -> F.close file >>= fun () -> _file <- None;Lwt.return ()
       end 
       >>= fun () ->                                                               
-      TlogMap.reinit tlog_map
+      TlogMap.reinit tlog_map >|= ignore
                      
     method remove_below i = TlogMap.remove_below tlog_map i
     method complete_file_to_deliver i = TlogMap.complete_file_to_deliver tlog_map i 
diff --git a/src/tlog/tlc2_test.ml b/src/tlog/tlc2_test.ml
index 75fefed6..996714f6 100644
--- a/src/tlog/tlc2_test.ml
+++ b/src/tlog/tlc2_test.ml
@@ -202,7 +202,6 @@ let test_iterate7 (dn, tlx_dir, (factory:factory)) =
   in
   tlc # log_value 0L (make_value 5) >>= fun _ ->
   tlc # log_value 1L (make_value 5) >>= fun _ ->
-  tlc # log_value 2L (make_value 5) >>= fun _ ->
   tlc # log_value 2L (make_value tlog_max_size) >>= fun _ ->
   tlc # log_value 3L (make_value tlog_max_size) >>= fun _ ->
   tlc # log_value 4L (make_value 5) >>= fun _ ->
@@ -233,7 +232,7 @@ let test_iterate7 (dn, tlx_dir, (factory:factory)) =
        Lwt.return ())
       (fun i -> Lwt.return ())
   >>= fun () ->
-  assert (!next = 5L);
+  assert (!next = 6L);
   Lwt.return ()
 
 let test_compression_bug (dn, tlx_dir, (factory:factory)) =
diff --git a/src/tlog/tlog_map.ml b/src/tlog/tlog_map.ml
index d5481191..fdc28e57 100644
--- a/src/tlog/tlog_map.ml
+++ b/src/tlog/tlog_map.ml
@@ -373,12 +373,12 @@ module TlogMap = struct
     _init
       t.tlog_dir t.tlx_dir t.node_id ~check_marker:false ~check_sabotage:false
       t.tlog_max_entries t.tlog_max_size
-    >>= fun ((tlog_entries, tlog_size, tlog_number,i_to_tlog_number,should_roll),_,_) ->
+    >>= fun ((tlog_entries, tlog_size, tlog_number,i_to_tlog_number,should_roll), last,_) ->
     t.tlog_size   <- tlog_size;
     t.tlog_number <- tlog_number;
     t.i_to_tlog_number <- i_to_tlog_number;
     t.should_roll <- should_roll;
-    Lwt.return ()
+    Lwt.return last
 
   let get_last_tlog t =
     Logger.debug_ "get_last_tlog" >>= fun () ->

From f1f6569a1c1252197d283ec2b69ad60dd6775536 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Thu, 23 Jun 2016 10:58:58 +0200
Subject: [PATCH 185/373] change default tlog_max_size

---
 src/node/node_cfg.ml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/node/node_cfg.ml b/src/node/node_cfg.ml
index e91caaac..7df5a2e0 100644
--- a/src/node/node_cfg.ml
+++ b/src/node/node_cfg.ml
@@ -367,8 +367,8 @@ module Node_cfg = struct
   let _ips inifile node_name =
     Ini.get inifile node_name "ip" Ini.p_string_list Ini.required
 
-  let _tlog_max_entries inifile = _global_int inifile "tlog_max_entries"    50_000
-  let _tlog_max_size    inifile = _global_int inifile "tlog_max_size"    5_000_000
+  let _tlog_max_entries inifile = _global_int inifile "tlog_max_entries"       50_000
+  let _tlog_max_size    inifile = _global_int inifile "tlog_max_size"    32*1024*1024
 
   let _tlog_entries_overwrite inifile =
     Ini.get inifile "global" "__tainted_tlog_entries_per_file"

From 88bcf204e141ee29e81e0c68acceb04c142ccd97 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Fri, 24 Jun 2016 08:34:20 +0200
Subject: [PATCH 186/373] make_release: 1.9.7

---
 META                      | 2 +-
 debian/changelog          | 5 +++++
 redhat/SPECS/arakoon.spec | 4 +++-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/META b/META
index eed2febe..8547e077 100644
--- a/META
+++ b/META
@@ -1,7 +1,7 @@
 # -*- conf -*-
 
 description = "Arakoon client"
-version = "1.9.6"
+version = "1.9.7"
 exists_if = "arakoon_client.cma,arakoon_client.cmxa,arakoon_client.cmxs"
 requires = "threads lwt lwt.unix str oUnit quickcheck lwt.ssl ocplib-endian redis.lwt uri"
 archive(byte) = "arakoon_client.cma tcp_keepalive_sockopt_stubs.o"
diff --git a/debian/changelog b/debian/changelog
index 05e4c308..ced8e87d 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,8 @@
+arakoon (1.9.7) unstable; urgency=low
+  * new upstream version
+
+ -- Romain Slootmaekers <romain.slootmaekers@openvstorage.com>  Fri, 24 Jun 2016 08:34:20 +0100
+
 arakoon (1.9.6) unstable; urgency=low
   * new upstream version
 
diff --git a/redhat/SPECS/arakoon.spec b/redhat/SPECS/arakoon.spec
index 1837af14..ac0da5d8 100644
--- a/redhat/SPECS/arakoon.spec
+++ b/redhat/SPECS/arakoon.spec
@@ -2,7 +2,7 @@
 
 Summary: Arakoon
 Name: arakoon
-Version: 1.9.6
+Version: 1.9.7
 Release: 3%{?dist}
 License: Apache 2.0
 Requires: libev >= 4
@@ -33,6 +33,8 @@ cp arakoon.native %{buildroot}%{_bindir}/arakoon
 %{_bindir}/arakoon
 
 %changelog
+* Fri Jun 24 2016 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.7
+- Create arakoon 1.9.7 RPM package
 * Fri Jun 10 2016 Jan Doms <jan.doms@gmail.com> - 1.9.6
 - Create arakoon 1.9.6 RPM package
 * Thu Jun 02 2016 Jan Doms <jan.doms@gmail.com> - 1.9.5

From 876bad05637b47e5e214fce79bfa16b63c2c85fd Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 6 Jul 2016 16:37:44 +0200
Subject: [PATCH 187/373] fix for #97 & #98

---
 pylabs/test/server/left/test_collapse.py | 34 +++++++++++---------
 src/main/arakoon.ml                      | 10 ++----
 src/node/collapser.ml                    |  6 +---
 src/node/collapser_main.ml               | 18 +++++++++++
 src/node/node_main.ml                    |  4 ++-
 src/tlog/tlc2.ml                         | 40 +++++++++++++-----------
 src/tlog/tlc2_test.ml                    |  5 +--
 7 files changed, 68 insertions(+), 49 deletions(-)

diff --git a/pylabs/test/server/left/test_collapse.py b/pylabs/test/server/left/test_collapse.py
index 9080e132..8c247156 100644
--- a/pylabs/test/server/left/test_collapse.py
+++ b/pylabs/test/server/left/test_collapse.py
@@ -25,13 +25,13 @@
 
 def _check_tlog_dirs(node, n):
     (home_dir, _, tlf_dir, head_dir) = Common.build_node_dir_names(node)
-    
+
     tlogs = X.listFilesInDir( home_dir, filter="*.tlog" )
     tlxs  = X.listFilesInDir( tlf_dir,  filter="*.tlx" )
     logging.info("tlogs:%s", tlogs)
     logging.info("tlxs:%s", tlxs)
     print tlxs
-    
+
     assert_equals(len(tlogs) + len(tlxs), n,
                   msg = "(%s + %s) should have %i file(s)" % (tlogs,tlxs,n))
     assert_true(X.fileExists(head_dir + "/head.db"))
@@ -47,7 +47,7 @@ def test_remote_collapse_witness_node():
     Common.collapse(witness,1)
     logging.info("collapsing done")
     _check_tlog_dirs(witness,1)
-    
+
     Common.stopOne(master)
     Common.wipe(master)
     Common.startOne(master)
@@ -71,7 +71,7 @@ def test_remote_collapse():
     Common.collapse(zero,1)
     logging.info("collapsing done")
     _check_tlog_dirs(zero,1)
-    
+
     Common.stopOne(one)
     Common.wipe(one)
     Common.startOne(one)
@@ -85,27 +85,27 @@ def test_remote_collapse():
         up2date = cli.expectProgressPossible()
     logging.info("catchup from collapsed node finished")
 
-    
+
 @Common.with_custom_setup(Common.setup_2_nodes_mini, Common.basic_teardown)
 def test_remote_collapse_client():
-    print 
+    print
     cluster = Common._getCluster()
     cluster_id = cluster._getClusterId()
     config = cluster.getClientConfig()
     ccfg = X.arakoon_client.ArakoonClientConfig(cluster_id,
                                                 config)
     client = X.arakoon_admin.ArakoonAdmin(ccfg)
-    
+
     zero = Common.node_names[0]
     one = Common.node_names[1]
     n = 29876
     Common.iterate_n_times(n, Common.simple_set)
     logging.info("did %i sets, now going into collapse scenario" % n)
-    
+
     client.collapse(zero,1)
     logging.info("collapsing done")
     _check_tlog_dirs(zero,1)
-    
+
 
 @Common.with_custom_setup(Common.setup_2_nodes_forced_master_mini, Common.basic_teardown)
 def test_local_collapse_witness_node():
@@ -115,10 +115,12 @@ def test_local_collapse_witness_node():
     Common.iterate_n_times(n, Common.simple_set)
     logging.info("did %i sets, now going into collapse scenario" % n)
 
-    Common.local_collapse(witness,1)
+    rc = Common.local_collapse(witness,1)
+    assert_equal(rc,0)
+
     logging.info("collapsing done")
-    
-    
+
+
     Common.stopOne(master)
     Common.wipe(master)
     Common.startOne(master)
@@ -135,13 +137,17 @@ def test_local_collapse_witness_node():
 @Common.with_custom_setup(Common.setup_2_nodes_mini, Common.basic_teardown)
 def test_local_collapse():
     logging.info("starting test_local_collapse")
-    
+
     zero = Common.node_names[0]
     one = Common.node_names[1]
     n = 29876
     Common.iterate_n_times(n, Common.simple_set)
     logging.info("did %i sets, now going into collapse scenario" % n)
-    Common.local_collapse(zero,1)
+    rc= Common.local_collapse(zero,1)
+    assert_equal(rc,0)
+    head_name = '%s/%s/head/head.db' %(Common.data_base_dir, zero)
+    logging.info(head_name)
+    assert_true(X.fileExists(head_name))
     #
     logging.info("collapsing done")
     Common.stopOne(one)
diff --git a/src/main/arakoon.ml b/src/main/arakoon.ml
index ec654c9a..0a4fd0f5 100644
--- a/src/main/arakoon.ml
+++ b/src/main/arakoon.ml
@@ -461,14 +461,8 @@ let main () =
     | WHO_MASTER -> Client_main.who_master ~tls !config_url ()
     | EXPECT_PROGRESS_POSSIBLE -> Client_main.expect_progress_possible ~tls !config_url
     | STATISTICS -> Client_main.statistics ~tls !config_url
-    | Collapse_local ->
-       Lwt_main.run
-         begin
-           let open Lwt.Infix in
-           make_config () >>= fun cfg ->
-           Collapser.collapse_out_of_band cfg !node_id !n_tlogs
-         end;
-       0
+    | Collapse_local -> Collapser_main.collapse_local make_config !node_id !n_tlogs
+
     | Collapse_remote -> Collapser_main.collapse_remote
                            ~tls ~tcp_keepalive
                            !ip !port !cluster_id !n_tlogs
diff --git a/src/node/collapser.ml b/src/node/collapser.ml
index 056e8950..aebbb847 100644
--- a/src/node/collapser.ml
+++ b/src/node/collapser.ml
@@ -232,14 +232,10 @@ let collapse_out_of_band cfg name tlogs_to_keep =
   let head_location = Filename.concat me.head_dir Tlc2.head_fname in
 
   let module S = (val (Store.make_store_module (module Batched_store.Local_store))) in
-  S.make_store
-    ~lcnum:cfg.lcnum
-    ~ncnum:cfg.ncnum
-    ~read_only:true head_location >>= fun head ->
-
   Tlc2.make_tlc2 ~compressor:me.compressor
                  me.tlog_dir me.tlx_dir me.head_dir
                  ~fsync:true name ~fsync_tlog_dir:true
+                 ~check_marker:false
   >>= fun tlog_coll ->
 
   collapse_many
diff --git a/src/node/collapser_main.ml b/src/node/collapser_main.ml
index d9346bd3..1b454c8e 100644
--- a/src/node/collapser_main.ml
+++ b/src/node/collapser_main.ml
@@ -40,3 +40,21 @@ let collapse_remote ~tls ~tcp_keepalive ip port cluster_id n =
       )
   in
   Lwt_main.run (t () )
+
+let collapse_local make_config node_id n_tlogs =
+  let t =
+    Lwt.catch
+      (fun() ->
+        let open Lwt.Infix in
+        make_config () >>= fun cfg ->
+        Collapser.collapse_out_of_band cfg node_id n_tlogs >>= fun () ->
+        Lwt.return 0
+
+      )
+      (fun exn ->
+        Logger.fatal Logger.Section.main ~exn "collapse_local failed"
+        >>= fun () ->
+        Lwt.return 1
+      )
+  in
+  Lwt_main.run t
diff --git a/src/node/node_main.ml b/src/node/node_main.ml
index 54922e0f..9a7c60b4 100644
--- a/src/node/node_main.ml
+++ b/src/node/node_main.ml
@@ -1018,7 +1018,9 @@ let main_t (make_config: unit -> 'a Lwt.t)
            name ~daemonize ~catchup_only ~autofix ~lock : int Lwt.t
   =
   let module S = (val (Store.make_store_module (module Batched_store.Local_store))) in
-  let (make_tlog_coll :Tlogcollection.tlc_factory) = Tlc2.make_tlc2 in
+  let (make_tlog_coll :Tlogcollection.tlc_factory) =
+    Tlc2.make_tlc2 ~check_marker:true
+  in
   let get_snapshot_name = Tlc2.head_name in
   _main_2 (module S)
           make_tlog_coll
diff --git a/src/tlog/tlc2.ml b/src/tlog/tlc2.ml
index a1d64032..86ed5b06 100644
--- a/src/tlog/tlc2.ml
+++ b/src/tlog/tlc2.ml
@@ -148,11 +148,11 @@ let _init_file fsync_dir tlog_map =
 
 class tlc2
         ?(compressor=Compression.Snappy)
-        (head_dir:string) 
+        (head_dir:string)
         (last:Entry.t option) (index:Index.index) (file:F.t option)
         (tlog_map: TlogMap.t)
         (node_id:string) ~(fsync:bool) ~(fsync_tlog_dir:bool)
-        
+
   =
   let _previous_entry = ref (Some last) in
   let set_previous_entry e =
@@ -331,7 +331,7 @@ class tlc2
               let new_outer = TlogMap.new_outer tlog_map i in
               Logger.info_f_ "should roll: i= %s old_outer = %i => jump to %i"
                              (Sn.string_of i)
-                             old_outer 
+                             old_outer
                              new_outer
               >>= fun () ->
               self # _jump_to_tlog file old_outer new_outer
@@ -375,7 +375,7 @@ class tlc2
       =
       let index = _index in
       Logger.debug_f_ "tlc2.iterate : index=%s" (Index.to_string index) >>= fun () ->
-      TlogMap.iterate_tlog_dir tlog_map ~index start_i too_far_i 
+      TlogMap.iterate_tlog_dir tlog_map ~index start_i too_far_i
                                f cb
 
 
@@ -422,10 +422,10 @@ class tlc2
         tmp
         hf_name
         (fun oc ->
-          Llio.copy_stream ~length ~ic ~oc) 
+          Llio.copy_stream ~length ~ic ~oc)
       >>= fun () ->
       self # _maybe_close_current_file ()
-      >>= fun () ->                                                               
+      >>= fun () ->
       let () = TlogMap.set_should_roll tlog_map in
       Logger.info_ "done: save_head"
 
@@ -503,17 +503,17 @@ class tlc2
               Logger.debug_f_ "wrote %S marker @i=%s for %S"
                               (Log_extra.string_option2s marker) (Sn.string_of i) node_id
               >>= fun () ->
-              F.close file 
+              F.close file
            | None -> Logger.fatal_f_ "how can this be?"
          end
-      
+
 
     method get_tlog_from_i (i:Sn.t) : int = TlogMap.outer_of_i tlog_map i
 
     method get_start_i (n:int) : Sn.t option = TlogMap.get_start_i tlog_map n
     method is_rollover_point i = TlogMap.is_rollover_point tlog_map i
     method next_rollover i     = TlogMap.next_rollover tlog_map i
-                                                       
+
     method get_tlog_count () =
       TlogMap.get_tlog_names tlog_map >>= fun tlogs ->
       Lwt.return (List.length tlogs)
@@ -527,7 +527,7 @@ class tlc2
       self # which_tlog_file n >>= fun co ->
       let canonical = match co with
         | Some fn -> fn
-        | None -> 
+        | None ->
                   let fn = file_name n in
                   TlogMap.get_full_path tlog_map fn
       in
@@ -572,12 +572,12 @@ class tlc2
       let new_name = Printf.sprintf "%03i%s" new_outer extension in
       let canon = TlogMap.get_full_path tlog_map new_name in
       let tmp = canon ^ ".tmp" in
-      
+
       Logger.info_f_
         "save_tlog_file: first_i:%s extension:%s in %s"
         (Sn.string_of first_i) extension tmp
       >>= fun () ->
-      
+
       Lwt_io.with_file ~mode:Lwt_io.output tmp
                        (fun oc -> Llio.copy_stream ~length ~ic ~oc)
       >>= fun () ->
@@ -592,24 +592,26 @@ class tlc2
         match _file with
         | None -> Lwt.return ()
         | Some file -> F.close file >>= fun () -> _file <- None;Lwt.return ()
-      end 
-      >>= fun () ->                                                               
+      end
+      >>= fun () ->
       TlogMap.reinit tlog_map >|= ignore
-                     
+
     method remove_below i = TlogMap.remove_below tlog_map i
-    method complete_file_to_deliver i = TlogMap.complete_file_to_deliver tlog_map i 
+    method complete_file_to_deliver i = TlogMap.complete_file_to_deliver tlog_map i
   end
 
 
 
 let make_tlc2 ~compressor
               ?tlog_max_entries ?tlog_max_size
+              ?(check_marker=true)
               tlog_dir tlf_dir
               head_dir ~fsync node_id ~fsync_tlog_dir
+
   =
   Logger.debug_f_ "make_tlc2 %S" tlog_dir >>= fun () ->
   TlogMap.make ?tlog_max_entries ?tlog_max_size
-               tlog_dir tlf_dir node_id ~check_marker:true
+               tlog_dir tlf_dir node_id ~check_marker
   >>= fun (tlog_map , last, index )->
   Logger.debug_f_ "make_tlc2 index = %s" (Index.to_string index)
   >>= fun () ->
@@ -639,10 +641,10 @@ let make_tlc2 ~compressor
          Lwt.return (Some file)
   end >>= fun file ->
   let col = new tlc2 head_dir last index
-                file tlog_map 
+                file tlog_map
                 ~compressor node_id ~fsync ~fsync_tlog_dir in
   Lwt.return col
-             
+
 let _truncate_tlog filename =
   Logger.debug_f_ "_truncate %s" filename >>= fun () ->
   let skipper () _entry = Lwt.return () in
diff --git a/src/tlog/tlc2_test.ml b/src/tlog/tlc2_test.ml
index 996714f6..45327ccd 100644
--- a/src/tlog/tlc2_test.ml
+++ b/src/tlog/tlc2_test.ml
@@ -33,6 +33,7 @@ let create_test_tlc ?tlog_max_entries ?tlog_max_size dn =
     dn
     ?tlog_max_entries ?tlog_max_size tlx_dir tlx_dir
     ~compressor ~fsync:false ~fsync_tlog_dir:false
+    ~check_marker:true
 
 let wrap_tlc = Tlogcollection_test.wrap create_test_tlc
 
@@ -53,7 +54,7 @@ let test_interrupted_rollover (dn, tlx_dir, factory) =
   factory ~tlog_max_entries:5 dn node_id >>= fun tlog_coll ->
   let value = Value.create_master_value ~lease_start:0. "me" in
   tlog_coll # log_value 5L value >>= fun _ ->
-  tlog_coll # close () >>= fun _ ->  
+  tlog_coll # close () >>= fun _ ->
   Tlog_map._get_tlog_names dn tlx_dir >>= fun tlog_names ->
   let n = List.length tlog_names in
   let msg = Printf.sprintf "Number of tlogs incorrect. Expected 2, got %d" n in
@@ -265,7 +266,7 @@ let test_compression_bug (dn, tlx_dir, (factory:factory)) =
     Logger.debug_f_ "ENTRY: i=%Li" i
   in
   let cb _ = Lwt.return_unit in
-  
+
   tlc2 # iterate 0L (Sn.of_int n) f cb >>= fun () ->
 
   OUnit.assert_equal

From 722ff896966df6721dcddbb7ba94c091174c6948 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Thu, 7 Jul 2016 12:11:00 +0200
Subject: [PATCH 188/373] only rewrite marker when you check it.

---
 src/tlog/tlc2.ml | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/src/tlog/tlc2.ml b/src/tlog/tlc2.ml
index 86ed5b06..7983db1e 100644
--- a/src/tlog/tlc2.ml
+++ b/src/tlog/tlc2.ml
@@ -621,7 +621,7 @@ let make_tlc2 ~compressor
       | Some e -> let i = Entry.i_of e in "Some " ^ (Sn.string_of i)
   in
   Logger.debug_f_ "post_validation: last_i=%s" msg >>= fun () ->
-  (* rewrite last entry with ANOTHER marker so we can see we got here *)
+  (* maybe rewrite last entry with ANOTHER marker so we can see we got here *)
   begin
     match last with
       | None   -> Lwt.return None
@@ -632,11 +632,17 @@ let make_tlc2 ~compressor
          let marker = Some (_make_open_marker node_id) in
          _init_file fsync_tlog_dir tlog_map >>= fun file ->
          let oc = F.oc_of file in
-         Tlogcommon.write_marker oc i v marker >>= fun () ->
-         Logger.debug_f_ "wrote %S marker @i=%s for %S"
-                         (Log_extra.string_option2s marker)
-                         (Sn.string_of i)
-                         node_id
+         begin
+           if check_marker
+           then
+             Tlogcommon.write_marker oc i v marker
+             >>= fun () ->
+             Logger.info_f_ "wrote %S marker @i=%s for %S"
+                             (Log_extra.string_option2s marker)
+                             (Sn.string_of i)
+                             node_id
+           else Lwt.return_unit
+         end
          >>= fun () ->
          Lwt.return (Some file)
   end >>= fun file ->

From 981821b6f4b8931b485cc5cd12300b6a557d2be2 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Mon, 18 Jul 2016 11:07:51 +0200
Subject: [PATCH 189/373] update ocaml-redis version

---
 docker/arakoon_centos7/Dockerfile | 2 +-
 docker/arakoon_debian/Dockerfile  | 2 +-
 travis.sh                         | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/docker/arakoon_centos7/Dockerfile b/docker/arakoon_centos7/Dockerfile
index c49fc208..77071c96 100644
--- a/docker/arakoon_centos7/Dockerfile
+++ b/docker/arakoon_centos7/Dockerfile
@@ -57,7 +57,7 @@ RUN eval `${opam_env}` && \
         conf-libev \
         ocplib-endian \
         depext \
-        redis.0.2.3 \
+        redis.0.3.1 \
         uri.1.9.1 \
         core.113.00.00
 
diff --git a/docker/arakoon_debian/Dockerfile b/docker/arakoon_debian/Dockerfile
index ba7e2d19..139c80dc 100644
--- a/docker/arakoon_debian/Dockerfile
+++ b/docker/arakoon_debian/Dockerfile
@@ -66,7 +66,7 @@ RUN eval `${opam_env}` && \
         conf-libev \
         ocplib-endian \
         depext \
-        redis.0.2.3 \
+        redis.0.3.1 \
         uri.1.9.1 \
         core.113.00.00
 
diff --git a/travis.sh b/travis.sh
index e41ffc46..682dfbb9 100755
--- a/travis.sh
+++ b/travis.sh
@@ -10,7 +10,7 @@ OPAM_DEPENDS="ssl.0.5.2 \
               snappy \
               quickcheck \
               ocplib-endian \
-              redis.0.2.3 \
+              redis.0.3.1 \
               uri.1.9.1 \
               core.113.00.00
 "

From 5c3ae502361ebf3eb7578acaeab1122ec41fd0cc Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 20 Jul 2016 14:53:50 +0200
Subject: [PATCH 190/373] make_release: 1.9.8

---
 META                      | 2 +-
 debian/changelog          | 5 +++++
 redhat/SPECS/arakoon.spec | 4 +++-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/META b/META
index 8547e077..7c5c32f3 100644
--- a/META
+++ b/META
@@ -1,7 +1,7 @@
 # -*- conf -*-
 
 description = "Arakoon client"
-version = "1.9.7"
+version = "1.9.8"
 exists_if = "arakoon_client.cma,arakoon_client.cmxa,arakoon_client.cmxs"
 requires = "threads lwt lwt.unix str oUnit quickcheck lwt.ssl ocplib-endian redis.lwt uri"
 archive(byte) = "arakoon_client.cma tcp_keepalive_sockopt_stubs.o"
diff --git a/debian/changelog b/debian/changelog
index ced8e87d..a5c76a7d 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,8 @@
+arakoon (1.9.8) unstable; urgency=low
+  * new upstream version
+
+ -- Romain Slootmaekers <romain.slootmaekers@openvstorage.com>  Wed, 20 Jul 2016 14:53:50 +0100
+
 arakoon (1.9.7) unstable; urgency=low
   * new upstream version
 
diff --git a/redhat/SPECS/arakoon.spec b/redhat/SPECS/arakoon.spec
index ac0da5d8..e8515caf 100644
--- a/redhat/SPECS/arakoon.spec
+++ b/redhat/SPECS/arakoon.spec
@@ -2,7 +2,7 @@
 
 Summary: Arakoon
 Name: arakoon
-Version: 1.9.7
+Version: 1.9.8
 Release: 3%{?dist}
 License: Apache 2.0
 Requires: libev >= 4
@@ -33,6 +33,8 @@ cp arakoon.native %{buildroot}%{_bindir}/arakoon
 %{_bindir}/arakoon
 
 %changelog
+* Wed Jul 20 2016 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.8
+- Create arakoon 1.9.8 RPM package
 * Fri Jun 24 2016 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.7
 - Create arakoon 1.9.7 RPM package
 * Fri Jun 10 2016 Jan Doms <jan.doms@gmail.com> - 1.9.6

From 1c814455b6afddaece810fc6d7258b7e6e45964a Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 20 Jul 2016 17:49:49 +0200
Subject: [PATCH 191/373] 7.5 -> 7.7

---
 docker/arakoon_centos7/Dockerfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docker/arakoon_centos7/Dockerfile b/docker/arakoon_centos7/Dockerfile
index 77071c96..54878143 100644
--- a/docker/arakoon_centos7/Dockerfile
+++ b/docker/arakoon_centos7/Dockerfile
@@ -13,7 +13,8 @@ RUN yum -y install python-devel
 
 RUN useradd jenkins -u 1001 -g root
 
-RUN rpm -iUvh http://dl.fedoraproject.org/pub/epel/7/x86_64/e/epel-release-7-5.noarch.rpm
+RUN rpm -iUvh http://dl.fedoraproject.org/pub/epel/7/x86_64/e/epel-release-7-7.noarch.rpm
+
 RUN yum -y update
 RUN yum -y install python-pip
 RUN yum -y install openssl-devel gmp-devel which

From 956b94a4da1ee18b944b5eb5c861ffe8e38ed7ef Mon Sep 17 00:00:00 2001
From: jeroenmaelbrancke <jeroen.maelbrancke@gmail.com>
Date: Mon, 1 Aug 2016 13:47:37 +0200
Subject: [PATCH 192/373] added documentation how to collapse an Arakoon.

---
 SUMMARY.md              |  3 +-
 docs/collapseArakoon.md | 94 +++++++++++++++++++++++++++++++++++++++++
 docs/troubleshooting.md |  1 +
 3 files changed, 97 insertions(+), 1 deletion(-)
 create mode 100644 docs/collapseArakoon.md

diff --git a/SUMMARY.md b/SUMMARY.md
index d3718ff8..d7282fd7 100644
--- a/SUMMARY.md
+++ b/SUMMARY.md
@@ -2,4 +2,5 @@
 
 * [Arakoon Troubleshooting](docs/troubleshooting.md)
     * [Retrieve Arakoon masters](docs/retrieveArakoonMasters.md)
-    * [Unable to start Arakoon service due to discrepancy between Alba and Arakoon version](docs/unableToStartArakoonServiceDueToDiscrepancyBetweenAlbaAndArakoonVersion.md)
\ No newline at end of file
+    * [Unable to start Arakoon service due to discrepancy between Alba and Arakoon version](docs/unableToStartArakoonServiceDueToDiscrepancyBetweenAlbaAndArakoonVersion.md)
+    * [Collapse arakoon](docs/collapseArakoon.md)
\ No newline at end of file
diff --git a/docs/collapseArakoon.md b/docs/collapseArakoon.md
new file mode 100644
index 00000000..1c6d82bd
--- /dev/null
+++ b/docs/collapseArakoon.md
@@ -0,0 +1,94 @@
+# Easy way to collapse an Arakoon on a cluster
+
+Applies to: Eugene-updates & Fargo
+
+## 1. Description
+
+Collapse an Arakoon on a cluster.
+
+## 2. Possible error when collapsing
+
+```
+(client_protocol|info): ... Start collapsing ... (n=2)
+(main|info): Starting collapse
+(main|info): Creating local store at /opt/OpenvStorage/db/arakoon/ovsdb/tlogs/head.db (lcnum=16384) (ncnum=8192)
+(main|info): returning assuming no I "/opt/OpenvStorage/db/arakoon/ovsdb/tlogs/head.db": Failure("success")
+(main|info): Going to collapse 5917 tlogs
+...
+(client_protocol|error): Exception during client request (Failure("success")) => rc:ff msg:unknown failure
+```
+
+## 3. Multiple methods to collapse
+### 3.1 Arakoon catch up
+<div style="background-color:orange; color:black">
+  <strong>Warning!</strong><br />Use this method only when the 2 other Arakoons are properly collapsed.
+</div>
+
+Lets Arakoon do the catch up for you.
+
+#### 3.1.1 Steps
+Stop your Arakoon services. ( stop ovs-arakoon-..)
+
+Remove every file in the correct tlogs directory. (check the config from the Arakoon where tlogs directory is located)
+
+Remove the <node_id>.db file in the db directory. (check the config from the Arakoon where the home directory is located)
+
+Start your Arakoon services. (start ovs-arakoon-..)
+
+Verify the log file and check if he download the head.db from another node and reply the tlogs.
+```
+(main|info): CATCHUP start: I'm @ 0 and T3BpM2jJy5qloYS5 is more recent
+(main|info): catchup_tlog 0
+...
+(main|info): catchup_tlog completed
+(main|info): CATCHUP phase 1 done (until_i = 29630399); now the store
+(main|info): replaying log to store
+(main|info): Replaying tlog file: 5921.tlx (1/6)
+(main|info): Completed replay of 5921.tlx, took 0.080827 seconds, 5 to go
+(main|info): Replaying tlog file: 5922.tlx (2/6)
+(main|info): Completed replay of 5922.tlx, took 0.067129 seconds, 4 to go
+(main|info): Replaying tlog file: 5923.tlog (3/6)
+... 
+(main|info): T3BpM2jJy5qloYS5 is master
+
+```
+
+### 3.2 Remote collapse
+Collapse the Arakoon manually with collapse-remote
+
+#### 3.2.1 Steps
+Check the IP and PORT where your Arakoon is running. (check the Arakoon config)
+
+Command:
+```
+arakoon --collapse-remote <cluster_id> <ip> <port> <n> tells node to collapse all but <n> tlogs into its head database
+```
+
+Example:
+```
+arakoon --collapse-remote T3BpM2jJy5qloYS5 127.0.0.1 26409 2
+```
+
+### 3.3 Manually
+<div style="background-color:orange; color:black">
+  <strong>Warning!</strong><br />Use this method only when <strong>one</strong> Arakoon is properly collapsed.
+</div>
+
+Manually copy the files from one Arakoon to another.
+
+#### 3.3.1 Steps
+You can try this method if 3.1 is not working and you received errors in the log file.
+
+Stop your Arakoon services. ( stop ovs-arakoon-..)
+
+Remove every file in the correct tlogs directory. (check the config from the Arakoon where tlogs directory is located)
+
+Remove the <node_id>.db file in the db directory. (check the config from the Arakoon where the home directory is located)
+
+Copy the head.db, *.tlx, *.tlog files from the Arakoon that is properly collapsed to the tlogs directory. (use scp for this)
+
+Change the owner and group for all these files. (chown ovs:ovs *)
+
+Start your Arakoon services. ( start ovs-arakoon-..)
+
+Verify the log file if the Arakoon find a master.
\ No newline at end of file
diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md
index c73d9679..7f1cec70 100644
--- a/docs/troubleshooting.md
+++ b/docs/troubleshooting.md
@@ -2,3 +2,4 @@
 
 * [Retrieve Arakoon masters](retrieveArakoonMasters.md)
 * [Unable to start Arakoon service due to discrepancy between Alba and Arakoon version](unableToStartArakoonServiceDueToDiscrepancyBetweenAlbaAndArakoonVersion.md) 
+* [Collapse arakoon](collapseArakoon.md)

From 5bafcfcb5e94c7cc2c993df80b0b499424628373 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Thu, 4 Aug 2016 13:36:13 +0200
Subject: [PATCH 193/373] support building & running tests inside an ubuntu
 16.04 docker container

---
 .travis.yml                    |  4 +-
 COMPILING                      | 60 ++++++++--------------
 docker/centos-7/Dockerfile     | 91 ++++++++++++++++++++++++++++++++++
 docker/docker-entrypoint.sh    | 16 ++++++
 docker/run.sh                  | 12 +++++
 docker/suites.sh               | 44 ++++++++++++++++
 docker/ubuntu-14.04/Dockerfile | 83 +++++++++++++++++++++++++++++++
 docker/ubuntu-16.04/Dockerfile | 82 ++++++++++++++++++++++++++++++
 jenkins/B.sh                   |  7 ---
 jenkins/C.sh                   |  7 ---
 jenkins/D.sh                   |  7 ---
 jenkins/common.sh              | 10 ++--
 jenkins/kwik.sh                |  6 ---
 jenkins/package_deb.sh         |  1 -
 jenkins/package_rpm.sh         |  1 -
 jenkins/run.sh                 | 14 ------
 logger_macro.ml                |  8 +--
 src/inifiles/inifiles.ml       |  2 +-
 src/node/node_cfg.ml           |  6 +--
 src/tools/arakoon_logger.ml    |  2 +-
 travis.sh                      | 71 ++------------------------
 21 files changed, 371 insertions(+), 163 deletions(-)
 create mode 100644 docker/centos-7/Dockerfile
 create mode 100755 docker/docker-entrypoint.sh
 create mode 100755 docker/run.sh
 create mode 100755 docker/suites.sh
 create mode 100644 docker/ubuntu-14.04/Dockerfile
 create mode 100644 docker/ubuntu-16.04/Dockerfile
 delete mode 100755 jenkins/run.sh

diff --git a/.travis.yml b/.travis.yml
index 59d7c812..93be3f17 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -3,8 +3,10 @@ sudo: required
 env:
   - PPA=avsm/ocaml42+opam12
 
-before_install: ./travis.sh before_install
 install: ./travis.sh install
 script: ./travis.sh script
 git:
     submodules: false
+
+services:
+  - docker
diff --git a/COMPILING b/COMPILING
index cc426352..35c97a70 100644
--- a/COMPILING
+++ b/COMPILING
@@ -1,50 +1,32 @@
-We use opam
+We use docker to build arakoon, so the easiest way to get started is to use our docker images.
 
-Some of the packages we use are not in the main opam repository (yet). So it's easiest if you add our devel repo:
+Preliminary steps
 
-opam remote --kind git add devel git://github.com/Incubaid/opam-repository-devel.git
+Install docker for your system, see https://docs.docker.com/engine/installation for instructions.
+After that make sure your user belongs to the docker group: sudo usermod -a -G docker your_user
 
-Packages we use:
+Building
 
-ocamlfind.1.3.3
-snappy.0.1.0
-camlbz2.0.6.0
-ssl
-lwt with SSL support
-camltc
-quickcheck
+Run
+./docker/run.sh ubuntu-16.04 build
+from the root of the repository.
 
-When it's been a while you might want to run `opam update` first to get the latest versions.
+You can also build using ubuntu 14.04 as follows:
+./docker/run.sh ubuntu-14.04 build
 
+Running the tests
 
-COMPILING:
+./docker/run.sh ubuntu-16.04 unit
+./docker/run.sh ubuntu-16.04 b
+./docker/run.sh ubuntu-16.04 c
+./docker/run.sh ubuntu-16.04 d
 
-make
-(you can inspect the makefile for more fine grained targets)
-(if you get errors and you've built arakoon before you could try a `make clean` first)
+Enter the docker container
 
-RUNNING THE TESTS:
+./docker/run.sh ubuntu-16.04 bash
 
-	./arakoon.native --run-all-tests
-
-COVERAGE:
-Bisect needs to be installed. 
-
-	make coverage
-        ./arakoon.d.byte --run-all-tests
-	./report.sh
-
-DEBIAN packages:
-	fakeroot debian/rules clean
-	fakeroot debian/rules build
-	fakeroot debian/rules binary
-
-Python egg:
-	python setup.py bdist_egg
-
-
-
-have fun,
-
-The arakoon team
+Build packages
 
+./docker/run.sh ubuntu-16.04 package_deb
+./docker/run.sh ubuntu-14.04 package_deb
+./docker/run.sh centos package_rpm
diff --git a/docker/centos-7/Dockerfile b/docker/centos-7/Dockerfile
new file mode 100644
index 00000000..3d3a7271
--- /dev/null
+++ b/docker/centos-7/Dockerfile
@@ -0,0 +1,91 @@
+FROM centos:7
+
+RUN useradd jenkins -u 1500 -g root
+
+RUN rpm --import file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
+RUN yum -y update && yum -y install \
+        wget curl \
+        make m4 gcc patch unzip \
+        git rsync mercurial \
+        gcc-c++ \
+        bzip2-devel libffi-devel snappy-devel libev-devel \
+        python-devel
+
+# TODO: What about darcs ?
+
+RUN rpm -iUvh http://dl.fedoraproject.org/pub/epel/7/x86_64/e/epel-release-7-7.noarch.rpm
+
+RUN yum -y update && yum -y install \
+         python-pip \
+         openssl-devel gmp-devel which \
+         zlib-devel ncurses-devel
+
+
+RUN wget https://raw.github.com/ocaml/opam/master/shell/opam_installer.sh
+
+env ocaml_version=4.03.0
+RUN sh ./opam_installer.sh /usr/local/bin ${ocaml_version}
+
+ENV opam_root=/home/jenkins/OPAM
+ENV opam_env="opam config env --root=${opam_root}"
+RUN opam init --root=${opam_root} --comp ${ocaml_version}
+
+
+# protobuf
+RUN yum -y install bzip2
+RUN wget https://github.com/google/protobuf/releases/download/v2.6.1/protobuf-2.6.1.tar.bz2 -O protobuf-2.6.1.tar.bz2 \
+    && tar -jxvf protobuf-2.6.1.tar.bz2 \
+    && cd protobuf-2.6.1 \
+    && ./configure && make && make install
+
+#RUN rpm -ivh http://cbs.centos.org/kojifiles/packages/protobuf/2.5.0/10.el7.centos/src/protobuf-2.5.0-10.el7.centos.src.rpm
+
+RUN pip install fabric junit-xml
+
+RUN wget http://cudf-solvers.irill.org/cudf_remote_proxy
+RUN chmod u+x cudf_remote_proxy
+RUN mv cudf_remote_proxy /usr/local/bin/
+
+ENV OPAMEXTERNALSOLVER="cudf_remote_proxy %{input}% %{output}% %{criteria}%"
+
+RUN eval `${opam_env}` && \
+    opam update && \
+    opam install -y \
+        ocamlfind \
+        ssl.0.5.2 \
+        camlbz2 \
+        snappy \
+        bisect \
+        lwt.2.5.2 \
+        camltc.0.9.3 \
+        ocplib-endian.1.0 \
+        quickcheck.1.0.2 \
+        conf-libev \
+        core.113.33.02+4.03 \
+        redis.0.3.1 \
+        uri.1.9.2
+
+
+RUN yum -y update && yum -y install \
+        autoconf help2man perl-Thread-Queue \
+        sudo iproute
+
+#
+# Disable "ssh hostname sudo <cmd>", because it will show the password in clear.
+#         You have to run "ssh -t hostname sudo <cmd>".
+#
+# Defaults    requiretty          # is line 56
+RUN awk 'NR == 56 {next} {print}' /etc/sudoers >/tmp/__sudoers && mv /tmp/__sudoers /etc/sudoers
+
+RUN yum -y install rpm-build
+
+RUN chmod ugoa+rxw -R ${opam_root}
+RUN su - -c "echo 'eval `${opam_env}`' >> /home/jenkins/.profile"
+RUN echo "jenkins ALL=NOPASSWD: ALL" >/etc/sudoers.d/jenkins
+
+
+ENTRYPOINT ["/home/jenkins/arakoon/docker/docker-entrypoint.sh"]
+
+CMD eval `${opam_env}` \
+    && echo ${SUITE} \
+    && ./jenkins/run.sh ${SUITE}
diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh
new file mode 100755
index 00000000..0bb849e5
--- /dev/null
+++ b/docker/docker-entrypoint.sh
@@ -0,0 +1,16 @@
+#!/bin/bash -xue
+
+# this script is executed at each startup of the container
+
+# hack to make sure we have access to files in the jenkins home directory
+# the UID of jenkins in the container should match our UID on the host
+if [ ${UID} -ne 1500 ]
+then
+    cat /etc/passwd
+    sed -i "s/x:1500:/x:${UID}:/" /etc/passwd
+
+    chown ${UID} /home/jenkins
+fi
+
+# finally execute the command the user requested
+exec sudo -i -u jenkins "$@"
diff --git a/docker/run.sh b/docker/run.sh
new file mode 100755
index 00000000..ea38a9da
--- /dev/null
+++ b/docker/run.sh
@@ -0,0 +1,12 @@
+#!/bin/bash -xue
+
+IMAGE=$1
+shift
+
+if [ -t 1 ] ; then TTY="-t"; else TTY=""; fi
+
+docker build --rm=true --tag=arakoon_$IMAGE ./docker/$IMAGE
+docker run -i $TTY --privileged=true -e UID=${UID} \
+       -v ${PWD}:/home/jenkins/arakoon \
+       -w /home/jenkins/arakoon arakoon_$IMAGE \
+       bash -l -c "cd arakoon && ./docker/suites.sh $@"
diff --git a/docker/suites.sh b/docker/suites.sh
new file mode 100755
index 00000000..c32cd2fd
--- /dev/null
+++ b/docker/suites.sh
@@ -0,0 +1,44 @@
+#!/bin/bash -xue
+
+export TEST_HOME='/home/tests'
+
+case "${1-bash}" in
+    bash)
+        bash
+        ;;
+    clean)
+        make clean
+        ;;
+    build)
+        make build
+        ;;
+    unit)
+        make build
+        ./arakoon.native --run-all-tests-xml testresults.xml || true
+        cat testresults.xml
+        ;;
+    unit-travis)
+        ./arakoon.native --run-all-tests 2>&1 | tail -n256
+        exit $PIPESTATUS
+        ;;
+    kwik)
+        ./jenkins/kwik.sh
+        ;;
+    b)
+        ./jenkins/B.sh
+        ;;
+    c)
+        ./jenkins/C.sh
+        ;;
+    d)
+        ./jenkins/D.sh
+        ;;
+    package_deb)
+        ./jenkins/package_deb.sh
+        ;;
+    package_rpm)
+        ./jenkins/package_rpm.sh
+        ;;
+    *)
+        echo "invalid test suite specified"
+esac
diff --git a/docker/ubuntu-14.04/Dockerfile b/docker/ubuntu-14.04/Dockerfile
new file mode 100644
index 00000000..9fee40c2
--- /dev/null
+++ b/docker/ubuntu-14.04/Dockerfile
@@ -0,0 +1,83 @@
+FROM ubuntu:14.04
+
+RUN useradd jenkins -u 1500 -g root
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+        build-essential m4 apt-utils \
+        libssl-dev \
+        libev-dev \
+        libbz2-dev \
+        libsnappy-dev \
+        help2man \
+        pkg-config \
+        time \
+        aspcud \
+        wget \
+        curl \
+        rsync \
+        darcs \
+        git \
+        unzip \
+        automake \
+        python-dev \
+        python-pip \
+        debhelper \
+        sudo \
+        iptables \
+        net-tools \
+        ncurses-dev
+
+
+RUN wget https://raw.github.com/ocaml/opam/master/shell/opam_installer.sh
+
+env ocaml_version=4.03.0
+RUN sh ./opam_installer.sh /usr/local/bin ${ocaml_version}
+
+ENV opam_root=/home/jenkins/OPAM
+ENV opam_env="opam config env --root=${opam_root}"
+RUN opam init --root=${opam_root} --comp ${ocaml_version}
+
+RUN eval `${opam_env}` && \
+    opam update && \
+    opam install -y \
+        ocamlfind \
+        ssl.0.5.2 \
+        camlbz2 \
+        snappy \
+        bisect \
+        lwt.2.5.2 \
+        camltc.0.9.3 \
+        ocplib-endian.1.0 \
+        quickcheck.1.0.2 \
+        conf-libev \
+        core.113.33.02+4.03 \
+        redis.0.3.1 \
+        uri.1.9.2
+
+
+RUN pip install fabric junit-xml nose simplejson python-etcd
+
+RUN mkdir /home/tests
+RUN chmod 777 /home/tests
+
+# Install etcd:
+ENV etcd_version=2.3.5
+RUN curl -L  https://github.com/coreos/etcd/releases/download/v${etcd_version}/etcd-v${etcd_version}-linux-amd64.tar.gz -o etcd-v{etcd_version}-linux-amd64.tar.gz
+RUN tar xzvf etcd-v{etcd_version}-linux-amd64.tar.gz
+RUN cp ./etcd-v${etcd_version}-linux-amd64/etcd /usr/bin \
+    && cp ./etcd-v${etcd_version}-linux-amd64/etcdctl /usr/bin
+
+# Install redis:
+RUN wget http://download.redis.io/releases/redis-3.0.7.tar.gz
+RUN tar xzvf redis-3.0.7.tar.gz
+RUN cd redis-3.0.7 && make
+RUN cp ./redis-3.0.7/src/redis-server /usr/bin
+RUN cp ./redis-3.0.7/src/redis-cli /usr/bin
+
+
+RUN chmod ugoa+rxw -R ${opam_root}
+RUN su - -c "echo 'eval `${opam_env}`' >> /home/jenkins/.profile"
+RUN echo "jenkins ALL=NOPASSWD: ALL" >/etc/sudoers.d/jenkins
+
+
+ENTRYPOINT ["/home/jenkins/arakoon/docker/docker-entrypoint.sh"]
diff --git a/docker/ubuntu-16.04/Dockerfile b/docker/ubuntu-16.04/Dockerfile
new file mode 100644
index 00000000..f5800858
--- /dev/null
+++ b/docker/ubuntu-16.04/Dockerfile
@@ -0,0 +1,82 @@
+FROM ubuntu:16.04
+
+RUN useradd jenkins -u 1500 -g root
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+        build-essential m4 apt-utils \
+        libssl-dev \
+        libev-dev \
+        libbz2-dev \
+        libsnappy-dev \
+        help2man \
+        pkg-config \
+        time \
+        aspcud \
+        wget \
+        curl \
+        darcs \
+        git \
+        unzip \
+        automake \
+        python-dev \
+        python-pip \
+        debhelper \
+        sudo \
+        iptables \
+        net-tools \
+        ncurses-dev
+
+
+RUN wget https://raw.github.com/ocaml/opam/master/shell/opam_installer.sh
+
+env ocaml_version=4.03.0
+RUN sh ./opam_installer.sh /usr/local/bin ${ocaml_version}
+
+ENV opam_root=/home/jenkins/OPAM
+ENV opam_env="opam config env --root=${opam_root}"
+RUN opam init --root=${opam_root} --comp ${ocaml_version}
+
+RUN eval `${opam_env}` && \
+    opam update && \
+    opam install -y \
+        ocamlfind \
+        ssl.0.5.2 \
+        camlbz2 \
+        snappy \
+        bisect \
+        lwt.2.5.2 \
+        camltc.0.9.3 \
+        ocplib-endian.1.0 \
+        quickcheck.1.0.2 \
+        conf-libev \
+        core.113.33.02+4.03 \
+        redis.0.3.1 \
+        uri.1.9.2
+
+
+RUN pip install fabric junit-xml nose simplejson python-etcd
+
+RUN mkdir /home/tests
+RUN chmod 777 /home/tests
+
+# Install etcd:
+ENV etcd_version=2.3.5
+RUN curl -L  https://github.com/coreos/etcd/releases/download/v${etcd_version}/etcd-v${etcd_version}-linux-amd64.tar.gz -o etcd-v{etcd_version}-linux-amd64.tar.gz
+RUN tar xzvf etcd-v{etcd_version}-linux-amd64.tar.gz
+RUN cp ./etcd-v${etcd_version}-linux-amd64/etcd /usr/bin \
+    && cp ./etcd-v${etcd_version}-linux-amd64/etcdctl /usr/bin
+
+# Install redis:
+RUN wget http://download.redis.io/releases/redis-3.0.7.tar.gz
+RUN tar xzvf redis-3.0.7.tar.gz
+RUN cd redis-3.0.7 && make
+RUN cp ./redis-3.0.7/src/redis-server /usr/bin
+RUN cp ./redis-3.0.7/src/redis-cli /usr/bin
+
+
+RUN chmod ugoa+rxw -R ${opam_root}
+RUN su - -c "echo 'eval `${opam_env}`' >> /home/jenkins/.profile"
+RUN echo "jenkins ALL=NOPASSWD: ALL" >/etc/sudoers.d/jenkins
+
+
+ENTRYPOINT ["/home/jenkins/arakoon/docker/docker-entrypoint.sh"]
diff --git a/jenkins/B.sh b/jenkins/B.sh
index 793a45c8..4689b57b 100755
--- a/jenkins/B.sh
+++ b/jenkins/B.sh
@@ -2,11 +2,4 @@
 
 ./jenkins/common.sh
 
-echo $PWD
-eval `${opam_env}`
-echo TEST_HOME=${TEST_HOME}
-export PATH=${PATH}:${TEST_HOME}/apps/arakoon/bin
-
-make clean
-make
 python test_it.py -v --with-xunit --xunit-file=${PWD}/testresults.xml ./server/left
diff --git a/jenkins/C.sh b/jenkins/C.sh
index 7a609bda..7aa36fbf 100755
--- a/jenkins/C.sh
+++ b/jenkins/C.sh
@@ -2,11 +2,4 @@
 
 ./jenkins/common.sh
 
-echo $PWD
-eval `${opam_env}`
-echo TEST_HOME=${TEST_HOME}
-export PATH=${PATH}:${TEST_HOME}/apps/arakoon/bin
-
-make clean
-make
 python test_it.py -v --with-xunit --xunit-file=${PWD}/testresults.xml ./server/right
diff --git a/jenkins/D.sh b/jenkins/D.sh
index dc990117..31262df1 100755
--- a/jenkins/D.sh
+++ b/jenkins/D.sh
@@ -2,11 +2,4 @@
 
 ./jenkins/common.sh
 
-echo $PWD
-eval `${opam_env}`
-echo TEST_HOME=${TEST_HOME}
-export PATH=${PATH}:${TEST_HOME}/apps/arakoon/bin
-
-make clean
-make
 python test_it.py -v -s --with-xunit --xunit-file=${PWD}/testresults.xml ./server/shaky
diff --git a/jenkins/common.sh b/jenkins/common.sh
index 7f0214af..221aeee3 100755
--- a/jenkins/common.sh
+++ b/jenkins/common.sh
@@ -1,8 +1,12 @@
-#!/bin/bash
-
-set -e
+#!/bin/bash -xue
 
 env | sort
 
 git submodule init
 git submodule update
+
+echo $PWD
+echo TEST_HOME=${TEST_HOME}
+export PATH=${PATH}:${TEST_HOME}/apps/arakoon/bin
+
+make
diff --git a/jenkins/kwik.sh b/jenkins/kwik.sh
index 7c1cb7b0..5c922ced 100755
--- a/jenkins/kwik.sh
+++ b/jenkins/kwik.sh
@@ -2,10 +2,4 @@
 
 ./jenkins/common.sh
 
-echo $PWD
-eval `${opam_env}`
-echo TEST_HOME=${TEST_HOME}
-export PATH=${PATH}:${TEST_HOME}/apps/arakoon/bin
-
-make
 python test_it.py -v --with-xunit --xunit-file=$PWD/testresults.xml ./server/quick
diff --git a/jenkins/package_deb.sh b/jenkins/package_deb.sh
index 986f5a50..04e0fcea 100755
--- a/jenkins/package_deb.sh
+++ b/jenkins/package_deb.sh
@@ -1,5 +1,4 @@
 #!/bin/bash -xue
 
-eval `${opam_env}`
 DEB_BUILD_OPTIONS="nostrip nocheck" fakeroot debian/rules clean build binary
 cp ../*.deb .
diff --git a/jenkins/package_rpm.sh b/jenkins/package_rpm.sh
index a1ee1319..cf3c7d9d 100755
--- a/jenkins/package_rpm.sh
+++ b/jenkins/package_rpm.sh
@@ -1,6 +1,5 @@
 #!/bin/bash -xue
 
-eval `${opam_env}`
 export START=${PWD}
 echo START=${START}
 rm -rf ${START}/rpmbuild/
diff --git a/jenkins/run.sh b/jenkins/run.sh
deleted file mode 100755
index 7b77dd05..00000000
--- a/jenkins/run.sh
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/bash
-
-set -e
-
-./jenkins/common.sh
-
-for f in $(ls ./jenkins/$1);
-do
-    extension="${f##*.}"
-    if [ $extension = "sh" ]; then
-        echo $f
-        ./jenkins/$1/$f
-    fi
-done;
diff --git a/logger_macro.ml b/logger_macro.ml
index 32aaa86a..dd91be71 100644
--- a/logger_macro.ml
+++ b/logger_macro.ml
@@ -75,7 +75,7 @@ let split e =
         then
           `No_match
         else
-          let level' = String.capitalize level in
+          let level' = String.capitalize_ascii level in
           if implicit_section
           then
             `Log_e_l_ (arg::acc, level', ign)
@@ -83,7 +83,7 @@ let split e =
             `Log_e_l(acc, level', arg, ign)
       else
         let level = String.sub func 0 (pos - 1) in
-        let level' = String.capitalize level in
+        let level' = String.capitalize_ascii level in
         if implicit_section
         then
           `Log_e_f_l_ (arg::acc, level', ign)
@@ -108,7 +108,7 @@ let split e =
         then
           `No_match
         else
-          let level' = String.capitalize level in
+          let level' = String.capitalize_ascii level in
           if implicit_section
           then
             `Log_l_(arg::acc, level', ign)
@@ -116,7 +116,7 @@ let split e =
             `Log_l (acc, level', arg, ign)
       else
         let level = String.sub func 0 (pos - 1) in
-        let level' = String.capitalize level in
+        let level' = String.capitalize_ascii level in
         if implicit_section
         then
           `Log_f_l_ (arg::acc, level', ign)
diff --git a/src/inifiles/inifiles.ml b/src/inifiles/inifiles.ml
index c3586c77..6552b686 100644
--- a/src/inifiles/inifiles.ml
+++ b/src/inifiles/inifiles.ml
@@ -70,7 +70,7 @@ module Ordstr =
 struct
   type t = string
   let compare (x:t) (y:t) =
-    String.compare (String.lowercase x) (String.lowercase y)
+    String.compare (String.lowercase_ascii x) (String.lowercase_ascii y)
 end
 
 module Strset = Set.Make(Ordstr)
diff --git a/src/node/node_cfg.ml b/src/node/node_cfg.ml
index 7df5a2e0..2cd7c3c8 100644
--- a/src/node/node_cfg.ml
+++ b/src/node/node_cfg.ml
@@ -469,7 +469,7 @@ module Node_cfg = struct
       let v = Ini.get inifile log_name x (Ini.p_option Ini.p_string) (Ini.default None) in
       match v with
         | None -> None
-        | Some v -> Some (String.lowercase v) in
+        | Some v -> Some (String.lowercase_ascii v) in
     let client_protocol = get_log_level "client_protocol" in
     let paxos = get_log_level "paxos" in
     let tcp_messaging = get_log_level "tcp_messaging" in
@@ -539,7 +539,7 @@ module Node_cfg = struct
       try get_string "head_dir"
       with _ -> tlx_dir
     in
-    let log_level = String.lowercase (get_string "log_level")  in
+    let log_level = String.lowercase_ascii (get_string "log_level")  in
     let log_config = Ini.get inifile node_name "log_config" (Ini.p_option Ini.p_string) (Ini.default None) in
     let batched_transaction_config = Ini.get inifile node_name "batched_transaction_config" (Ini.p_option Ini.p_string) (Ini.default None) in
     let is_laggy = get_bool "laggy" in
@@ -553,7 +553,7 @@ module Node_cfg = struct
           let s = Ini.get inifile node_name "tlog_compression" Ini.p_string
                           (Ini.default "snappy")
           in
-          match String.lowercase s with
+          match String.lowercase_ascii s with
           | "snappy"  -> Compression.Snappy
           | "bz2"     -> Compression.Bz2
           | "none"    -> Compression.No
diff --git a/src/tools/arakoon_logger.ml b/src/tools/arakoon_logger.ml
index 4dac75ac..2643713d 100644
--- a/src/tools/arakoon_logger.ml
+++ b/src/tools/arakoon_logger.ml
@@ -88,7 +88,7 @@ let format_message
        tm.tm_min
        tm.tm_sec
        (int_of_float ((mod_float (Core.Time.to_float ts) 1.) *. 1_000_000.))
-       (let open Core.Std.Float in
+       (let open Core_kernel in
         match offset.Core.Span.Parts.sign with
         | Sign.Neg -> '-'
         | Sign.Pos | Sign.Zero -> '+')
diff --git a/travis.sh b/travis.sh
index 682dfbb9..3e19449e 100755
--- a/travis.sh
+++ b/travis.sh
@@ -1,83 +1,18 @@
 #!/bin/bash -xue
 
-APT_DEPENDS="libev-dev libssl-dev libsnappy-dev libgmp3-dev"
-APT_OCAML_DEPENDS="ocaml ocaml-native-compilers camlp4-extra opam"
-OPAM_DEPENDS="ssl.0.5.2 \
-              conf-libev \
-              lwt.2.5.1 \
-              camlbz2 \
-              camltc.999 \
-              snappy \
-              quickcheck \
-              ocplib-endian \
-              redis.0.3.1 \
-              uri.1.9.1 \
-              core.113.00.00
-"
-
-export OPAMYES=1
-export OPAMVERBOSE=1
-
-before_install () {
-    echo "Running 'before_install' phase"
-
-    echo "Adding PPA '$PPA'"
-    echo "yes" | sudo add-apt-repository ppa:$PPA
-    echo "Updating Apt cache"
-    sudo apt-get update -qq
-    echo "updating keys"
-    sudo apt-key update
-    echo "Installing general dependencies"
-    sudo apt-get install -qq ${APT_DEPENDS}
-    echo "Installing dependencies"
-    sudo apt-get install -qq ${APT_OCAML_DEPENDS}
-
-    echo "OCaml versions:"
-    ocaml -version
-    ocamlopt -version
-
-    echo "Opam versions:"
-    opam --version
-    opam --git-version
-}
-
 install () {
     echo "Running 'install' phase"
 
-    opam init
-    eval `opam config env`
-
-    opam remote add incubaid-devel -k git git://github.com/Incubaid/opam-repository-devel.git
-    opam update
-
-    opam install ${OPAM_DEPENDS}
-
-    # Install etcd:
-    curl -L https://github.com/coreos/etcd/releases/download/v2.3.5/etcd-v2.3.5-linux-amd64.tar.gz -o etcd-v2.3.5-linux-amd64.tar.gz
-    tar xzvf etcd-v2.3.5-linux-amd64.tar.gz
-    sudo cp ./etcd-v2.3.5-linux-amd64/etcd /usr/bin
-    sudo cp ./etcd-v2.3.5-linux-amd64/etcdctl /usr/bin
+    ./docker/run.sh ubuntu-16.04 build
 }
 
 script () {
     echo "Running 'script' phase"
-    eval `opam config env`
 
-    echo "Building 'arakoon.native'"
-    ocamlbuild -use-ocamlfind -classic-display arakoon.native
-
-    echo "Arakoon version:"
-    ./arakoon.native --version
-
-    echo "Executing tests"
-    ./arakoon.native --run-all-tests 2>&1 | tail -n256
-    exit ${PIPESTATUS[0]}
+    ./docker/run.sh ubuntu-16.04 unit-travis
 }
 
 case "$1" in
-    before_install)
-        before_install
-        ;;
     install)
         install
         ;;
@@ -85,6 +20,6 @@ case "$1" in
         script
         ;;
     *)
-        echo "Usage: $0 {before_install|install|script}"
+        echo "Usage: $0 {install|script}"
         exit 1
 esac

From 01410e21b2cb6c0e0e7feea2270d8157d50f9e27 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Thu, 4 Aug 2016 16:25:12 +0200
Subject: [PATCH 194/373] centos dockerfile improvements so it can run all
 tests too

---
 docker/centos-7/Dockerfile  | 22 +++++++++++++++++++++-
 docker/docker-entrypoint.sh |  2 ++
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/docker/centos-7/Dockerfile b/docker/centos-7/Dockerfile
index 3d3a7271..266f94cc 100644
--- a/docker/centos-7/Dockerfile
+++ b/docker/centos-7/Dockerfile
@@ -66,6 +66,26 @@ RUN eval `${opam_env}` && \
         uri.1.9.2
 
 
+RUN pip install fabric junit-xml nose simplejson python-etcd
+
+RUN mkdir /home/tests
+RUN chmod 777 /home/tests
+
+# Install etcd:
+ENV etcd_version=2.3.5
+RUN curl -L  https://github.com/coreos/etcd/releases/download/v${etcd_version}/etcd-v${etcd_version}-linux-amd64.tar.gz -o etcd-v{etcd_version}-linux-amd64.tar.gz
+RUN tar xzvf etcd-v{etcd_version}-linux-amd64.tar.gz
+RUN cp ./etcd-v${etcd_version}-linux-amd64/etcd /usr/bin \
+    && cp ./etcd-v${etcd_version}-linux-amd64/etcdctl /usr/bin
+
+# Install redis:
+RUN wget http://download.redis.io/releases/redis-3.0.7.tar.gz
+RUN tar xzvf redis-3.0.7.tar.gz
+RUN cd redis-3.0.7 && make
+RUN cp ./redis-3.0.7/src/redis-server /usr/bin
+RUN cp ./redis-3.0.7/src/redis-cli /usr/bin
+
+
 RUN yum -y update && yum -y install \
         autoconf help2man perl-Thread-Queue \
         sudo iproute
@@ -80,7 +100,7 @@ RUN awk 'NR == 56 {next} {print}' /etc/sudoers >/tmp/__sudoers && mv /tmp/__sudo
 RUN yum -y install rpm-build
 
 RUN chmod ugoa+rxw -R ${opam_root}
-RUN su - -c "echo 'eval `${opam_env}`' >> /home/jenkins/.profile"
+RUN su - -c "echo 'eval `${opam_env}`' >> /home/jenkins/.bash_profile"
 RUN echo "jenkins ALL=NOPASSWD: ALL" >/etc/sudoers.d/jenkins
 
 
diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh
index 0bb849e5..de73f473 100755
--- a/docker/docker-entrypoint.sh
+++ b/docker/docker-entrypoint.sh
@@ -10,6 +10,8 @@ then
     sed -i "s/x:1500:/x:${UID}:/" /etc/passwd
 
     chown ${UID} /home/jenkins
+    chown ${UID} /home/jenkins/OPAM
+    chown ${UID} /home/jenkins/.bash* || true
 fi
 
 # finally execute the command the user requested

From f85e0a197b78744c39a9fe925b63ae84de0ef420 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Thu, 4 Aug 2016 16:50:18 +0200
Subject: [PATCH 195/373] try speedup the travis build

---
 .travis.yml | 10 ++++++++--
 travis.sh   | 28 ++++++++++++++++++++++++++++
 2 files changed, 36 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 93be3f17..59191ae2 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,12 +1,18 @@
 language: c
 sudo: required
-env:
-  - PPA=avsm/ocaml42+opam12
 
 install: ./travis.sh install
 script: ./travis.sh script
+
 git:
     submodules: false
 
 services:
   - docker
+
+cache:
+  directories:
+    - ~/cache
+  timeout:
+    1000
+before_cache: ./travis.sh before_cache
diff --git a/travis.sh b/travis.sh
index 3e19449e..56a8105f 100755
--- a/travis.sh
+++ b/travis.sh
@@ -3,13 +3,38 @@
 install () {
     echo "Running 'install' phase"
 
+    date
+
+    if [[ -e ~/cache/image.tar.gz ]];
+    then docker load -i ~/cache/image.tar.gz; fi
+
+    date
+
     ./docker/run.sh ubuntu-16.04 build
+
+    date
 }
 
 script () {
     echo "Running 'script' phase"
 
+    date
+
     ./docker/run.sh ubuntu-16.04 unit-travis
+
+    date
+}
+
+before_cache () {
+    echo "Running 'before_cache' phase"
+
+    date
+
+    mkdir ~/cache || true
+    docker save -o ~/cache/image.tar.gz arakoon_ubuntu-16.04
+    ls -ahl ~/cache
+
+    date
 }
 
 case "$1" in
@@ -19,6 +44,9 @@ case "$1" in
     script)
         script
         ;;
+    before_cache)
+        before_cache
+        ;;
     *)
         echo "Usage: $0 {install|script}"
         exit 1

From 0c75538c52cb44bf4520563c953fa14eb95c8aef Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Fri, 5 Aug 2016 10:36:35 +0200
Subject: [PATCH 196/373] more CI fiddling

---
 docker/run.sh                  | 14 ++++++++++++--
 docker/suites.sh               |  4 ----
 docker/ubuntu-16.04/Dockerfile | 11 +++++++----
 travis.sh                      |  7 +++++--
 4 files changed, 24 insertions(+), 12 deletions(-)

diff --git a/docker/run.sh b/docker/run.sh
index ea38a9da..1ed3066a 100755
--- a/docker/run.sh
+++ b/docker/run.sh
@@ -3,9 +3,19 @@
 IMAGE=$1
 shift
 
-if [ -t 1 ] ; then TTY="-t"; else TTY=""; fi
-
 docker build --rm=true --tag=arakoon_$IMAGE ./docker/$IMAGE
+
+if [ -t 1 ];
+then TTY="-t";
+else
+    # this path is taken on jenkins, clean previous builds first
+    TTY="";
+    docker run -i $TTY --privileged=true -e UID=${UID} \
+           -v ${PWD}:/home/jenkins/arakoon \
+           -w /home/jenkins/arakoon arakoon_$IMAGE \
+           bash -l -c "cd arakoon && ./docker/suites.sh clean"
+fi
+
 docker run -i $TTY --privileged=true -e UID=${UID} \
        -v ${PWD}:/home/jenkins/arakoon \
        -w /home/jenkins/arakoon arakoon_$IMAGE \
diff --git a/docker/suites.sh b/docker/suites.sh
index c32cd2fd..c34def7c 100755
--- a/docker/suites.sh
+++ b/docker/suites.sh
@@ -17,10 +17,6 @@ case "${1-bash}" in
         ./arakoon.native --run-all-tests-xml testresults.xml || true
         cat testresults.xml
         ;;
-    unit-travis)
-        ./arakoon.native --run-all-tests 2>&1 | tail -n256
-        exit $PIPESTATUS
-        ;;
     kwik)
         ./jenkins/kwik.sh
         ;;
diff --git a/docker/ubuntu-16.04/Dockerfile b/docker/ubuntu-16.04/Dockerfile
index f5800858..c554cfe1 100644
--- a/docker/ubuntu-16.04/Dockerfile
+++ b/docker/ubuntu-16.04/Dockerfile
@@ -61,10 +61,13 @@ RUN chmod 777 /home/tests
 
 # Install etcd:
 ENV etcd_version=2.3.5
-RUN curl -L  https://github.com/coreos/etcd/releases/download/v${etcd_version}/etcd-v${etcd_version}-linux-amd64.tar.gz -o etcd-v{etcd_version}-linux-amd64.tar.gz
-RUN tar xzvf etcd-v{etcd_version}-linux-amd64.tar.gz
-RUN cp ./etcd-v${etcd_version}-linux-amd64/etcd /usr/bin \
-    && cp ./etcd-v${etcd_version}-linux-amd64/etcdctl /usr/bin
+RUN curl \
+      -L https://github.com/coreos/etcd/releases/download/v${etcd_version}/etcd-v${etcd_version}-linux-amd64.tar.gz \
+      -o etcd-v${etcd_version}-linux-amd64.tar.gz \
+    && tar xzvf etcd-v${etcd_version}-linux-amd64.tar.gz \
+    && cp ./etcd-v${etcd_version}-linux-amd64/etcd /usr/bin \
+    && cp ./etcd-v${etcd_version}-linux-amd64/etcdctl /usr/bin \
+    && rm -rf etcd-*
 
 # Install redis:
 RUN wget http://download.redis.io/releases/redis-3.0.7.tar.gz
diff --git a/travis.sh b/travis.sh
index 56a8105f..36b7a2b3 100755
--- a/travis.sh
+++ b/travis.sh
@@ -10,7 +10,7 @@ install () {
 
     date
 
-    ./docker/run.sh ubuntu-16.04 build
+    ./docker/run.sh ubuntu-16.04 clean | tail -n256
 
     date
 }
@@ -20,9 +20,12 @@ script () {
 
     date
 
-    ./docker/run.sh ubuntu-16.04 unit-travis
+    ./docker/run.sh ubuntu-16.04 unit 2&>1 | tail -n256
+    RESULT=$PIPESTATUS
 
     date
+
+    exit $RESULT
 }
 
 before_cache () {

From d0487cbaef2a3c7c2bdfbc75e31a57675d73795e Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Fri, 5 Aug 2016 15:00:43 +0200
Subject: [PATCH 197/373] more docker images + make the kwik suite succeed on
 ubuntu

---
 docker/arch/Dockerfile         | 88 ++++++++++++++++++++++++++++++++++
 docker/trisquel/Dockerfile     | 85 ++++++++++++++++++++++++++++++++
 docker/ubuntu-14.04/Dockerfile | 10 ++--
 docker/ubuntu-16.04/Dockerfile | 10 ++--
 4 files changed, 187 insertions(+), 6 deletions(-)
 create mode 100644 docker/arch/Dockerfile
 create mode 100644 docker/trisquel/Dockerfile

diff --git a/docker/arch/Dockerfile b/docker/arch/Dockerfile
new file mode 100644
index 00000000..bafb85f9
--- /dev/null
+++ b/docker/arch/Dockerfile
@@ -0,0 +1,88 @@
+FROM pritunl/archlinux:2016-04-16
+
+RUN useradd jenkins -u 1500 -g root
+
+RUN pacman -Sy --noconfirm gcc clang sudo autoconf base-devel \
+                           wget git unzip python2 \
+                           libev snappy help2man \
+                           python-pip python2
+
+## extra packages not available in the default repo
+## makepkg refuses running as root, so switch to jenkins first
+
+RUN echo 'jenkins ALL=NOPASSWD: ALL' >/etc/sudoers.d/jenkins
+RUN mkdir /home/jenkins
+RUN chown jenkins /home/jenkins
+
+WORKDIR /home/jenkins
+USER jenkins
+
+RUN git clone https://aur.archlinux.org/clasp.git && \
+    cd clasp && \
+    makepkg -sri --noconfirm && \
+    cd .. && \
+    rm -rf clasp
+
+RUN git clone https://aur.archlinux.org/gringo.git && \
+    cd gringo && \
+    makepkg -sri --noconfirm && \
+    cd .. && \
+    rm -rf gringo
+
+RUN git clone https://aur.archlinux.org/aspcud.git && \
+    cd aspcud && \
+    makepkg -sri --noconfirm && \
+    cd .. && \
+    rm -rf aspcud
+
+RUN git clone https://aur.archlinux.org/opam.git && \
+    cd opam && \
+    makepkg -sri --noconfirm && \
+    cd .. && \
+    rm -rf opam
+
+ENV ocaml_version=4.03.0
+ENV opam_root=/home/jenkins/OPAM
+ENV opam_env="opam config env --root=${opam_root}"
+RUN opam init --root ${opam_root} --comp ${ocaml_version}
+
+RUN eval `${opam_env}` && \
+    opam update && \
+    opam install -y \
+        ocamlfind \
+        ssl.0.5.2 \
+        camlbz2 \
+        snappy \
+        bisect \
+        lwt.2.5.2 \
+        camltc.0.9.3 \
+        ocplib-endian.1.0 \
+        quickcheck.1.0.2 \
+        conf-libev \
+        core.113.33.02+4.03 \
+        redis.0.3.1 \
+        uri.1.9.2
+
+# Install etcd:
+ENV etcd_version=2.3.5
+RUN curl \
+      -L https://github.com/coreos/etcd/releases/download/v${etcd_version}/etcd-v${etcd_version}-linux-amd64.tar.gz \
+      -o etcd-v${etcd_version}-linux-amd64.tar.gz \
+    && tar xzvf etcd-v${etcd_version}-linux-amd64.tar.gz \
+    && sudo cp ./etcd-v${etcd_version}-linux-amd64/etcd /usr/bin \
+    && sudo cp ./etcd-v${etcd_version}-linux-amd64/etcdctl /usr/bin \
+    && rm -rf etcd-*
+
+# Install redis:
+RUN wget http://download.redis.io/releases/redis-3.0.7.tar.gz
+RUN tar xzvf redis-3.0.7.tar.gz
+RUN cd redis-3.0.7 && make
+RUN sudo cp ./redis-3.0.7/src/redis-server /usr/bin
+RUN sudo cp ./redis-3.0.7/src/redis-cli /usr/bin
+
+RUN chmod ugoa+rxw -R ${opam_root}
+RUN eval ${opam_env} >> /home/jenkins/.profile
+
+USER root
+
+ENTRYPOINT ["/home/jenkins/arakoon/docker/docker-entrypoint.sh"]
diff --git a/docker/trisquel/Dockerfile b/docker/trisquel/Dockerfile
new file mode 100644
index 00000000..22bdf91a
--- /dev/null
+++ b/docker/trisquel/Dockerfile
@@ -0,0 +1,85 @@
+FROM kpengboy/trisquel:latest
+
+RUN useradd jenkins -u 1500 -g root
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+        build-essential m4 apt-utils \
+        libssl-dev \
+        libev-dev \
+        libbz2-dev \
+        libsnappy-dev \
+        help2man \
+        pkg-config \
+        time \
+        aspcud \
+        wget \
+        curl \
+        rsync \
+        darcs \
+        git \
+        unzip \
+        automake \
+        python-dev \
+        python-pip \
+        debhelper \
+        sudo \
+        iptables \
+        net-tools \
+        ncurses-dev
+
+RUN wget https://raw.github.com/ocaml/opam/master/shell/opam_installer.sh
+
+env ocaml_version=4.03.0
+RUN sh ./opam_installer.sh /usr/local/bin ${ocaml_version}
+
+ENV opam_root=/home/jenkins/OPAM
+ENV opam_env="opam config env --root=${opam_root}"
+RUN opam init --root=${opam_root} --comp ${ocaml_version}
+
+RUN eval `${opam_env}` && \
+    opam update && \
+    opam install -y \
+        ocamlfind \
+        ssl.0.5.2 \
+        camlbz2 \
+        snappy \
+        bisect \
+        lwt.2.5.2 \
+        camltc.0.9.3 \
+        ocplib-endian.1.0 \
+        quickcheck.1.0.2 \
+        conf-libev \
+        core.113.33.02+4.03 \
+        redis.0.3.1 \
+        uri.1.9.2
+
+
+RUN pip install fabric junit-xml nose simplejson python-etcd
+
+RUN mkdir /home/tests
+RUN chmod 777 /home/tests
+
+# Install etcd:
+ENV etcd_version=2.3.5
+RUN curl \
+      -L https://github.com/coreos/etcd/releases/download/v${etcd_version}/etcd-v${etcd_version}-linux-amd64.tar.gz \
+      -o etcd-v${etcd_version}-linux-amd64.tar.gz \
+    && tar xzvf etcd-v${etcd_version}-linux-amd64.tar.gz \
+    && cp ./etcd-v${etcd_version}-linux-amd64/etcd /usr/bin \
+    && cp ./etcd-v${etcd_version}-linux-amd64/etcdctl /usr/bin \
+    && rm -rf etcd-*
+
+# Install redis:
+RUN wget http://download.redis.io/releases/redis-3.0.7.tar.gz
+RUN tar xzvf redis-3.0.7.tar.gz
+RUN cd redis-3.0.7 && make
+RUN cp ./redis-3.0.7/src/redis-server /usr/bin
+RUN cp ./redis-3.0.7/src/redis-cli /usr/bin
+
+
+RUN chmod ugoa+rxw -R ${opam_root}
+RUN su - -c "echo 'eval `${opam_env}`' >> /home/jenkins/.profile"
+RUN echo "jenkins ALL=NOPASSWD: ALL" >/etc/sudoers.d/jenkins
+
+
+ENTRYPOINT ["/home/jenkins/arakoon/docker/docker-entrypoint.sh"]
diff --git a/docker/ubuntu-14.04/Dockerfile b/docker/ubuntu-14.04/Dockerfile
index 9fee40c2..80072fc4 100644
--- a/docker/ubuntu-14.04/Dockerfile
+++ b/docker/ubuntu-14.04/Dockerfile
@@ -2,12 +2,14 @@ FROM ubuntu:14.04
 
 RUN useradd jenkins -u 1500 -g root
 
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+RUN apt-get -y update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
         build-essential m4 apt-utils \
-        libssl-dev \
-        libev-dev \
+        libffi-dev libssl-dev \
         libbz2-dev \
+        libgmp3-dev \
+        libev-dev \
         libsnappy-dev \
+        libxen-dev \
         help2man \
         pkg-config \
         time \
@@ -22,7 +24,9 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
         python-dev \
         python-pip \
         debhelper \
+        psmisc \
         sudo \
+        libtool \
         iptables \
         net-tools \
         ncurses-dev
diff --git a/docker/ubuntu-16.04/Dockerfile b/docker/ubuntu-16.04/Dockerfile
index c554cfe1..3b704c55 100644
--- a/docker/ubuntu-16.04/Dockerfile
+++ b/docker/ubuntu-16.04/Dockerfile
@@ -2,12 +2,14 @@ FROM ubuntu:16.04
 
 RUN useradd jenkins -u 1500 -g root
 
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+RUN apt-get -y update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
         build-essential m4 apt-utils \
-        libssl-dev \
-        libev-dev \
+        libffi-dev libssl-dev \
         libbz2-dev \
+        libgmp3-dev \
+        libev-dev \
         libsnappy-dev \
+        libxen-dev \
         help2man \
         pkg-config \
         time \
@@ -21,7 +23,9 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
         python-dev \
         python-pip \
         debhelper \
+        psmisc \
         sudo \
+        libtool \
         iptables \
         net-tools \
         ncurses-dev

From 59b1a7d70315a60fa13d9898d82c2cdf4a16634a Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Mon, 8 Aug 2016 11:51:29 +0200
Subject: [PATCH 198/373] version of core should be compatible for alba

---
 docker/centos-7/Dockerfile     | 8 +++-----
 docker/ubuntu-14.04/Dockerfile | 5 ++++-
 docker/ubuntu-16.04/Dockerfile | 5 ++++-
 3 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/docker/centos-7/Dockerfile b/docker/centos-7/Dockerfile
index 266f94cc..464afc31 100644
--- a/docker/centos-7/Dockerfile
+++ b/docker/centos-7/Dockerfile
@@ -30,6 +30,8 @@ ENV opam_root=/home/jenkins/OPAM
 ENV opam_env="opam config env --root=${opam_root}"
 RUN opam init --root=${opam_root} --comp ${ocaml_version}
 
+RUN eval `${opam_env}` && \
+    opam repo --root=${opam_root} add janestreet git://github.com/janestreet/opam-repository.git
 
 # protobuf
 RUN yum -y install bzip2
@@ -61,7 +63,7 @@ RUN eval `${opam_env}` && \
         ocplib-endian.1.0 \
         quickcheck.1.0.2 \
         conf-libev \
-        core.113.33.02+4.03 \
+        core.114.04+57 \
         redis.0.3.1 \
         uri.1.9.2
 
@@ -105,7 +107,3 @@ RUN echo "jenkins ALL=NOPASSWD: ALL" >/etc/sudoers.d/jenkins
 
 
 ENTRYPOINT ["/home/jenkins/arakoon/docker/docker-entrypoint.sh"]
-
-CMD eval `${opam_env}` \
-    && echo ${SUITE} \
-    && ./jenkins/run.sh ${SUITE}
diff --git a/docker/ubuntu-14.04/Dockerfile b/docker/ubuntu-14.04/Dockerfile
index 80072fc4..696f47d9 100644
--- a/docker/ubuntu-14.04/Dockerfile
+++ b/docker/ubuntu-14.04/Dockerfile
@@ -41,6 +41,9 @@ ENV opam_root=/home/jenkins/OPAM
 ENV opam_env="opam config env --root=${opam_root}"
 RUN opam init --root=${opam_root} --comp ${ocaml_version}
 
+RUN eval `${opam_env}` && \
+    opam repo --root=${opam_root} add janestreet git://github.com/janestreet/opam-repository.git
+
 RUN eval `${opam_env}` && \
     opam update && \
     opam install -y \
@@ -54,7 +57,7 @@ RUN eval `${opam_env}` && \
         ocplib-endian.1.0 \
         quickcheck.1.0.2 \
         conf-libev \
-        core.113.33.02+4.03 \
+        core.114.04+57 \
         redis.0.3.1 \
         uri.1.9.2
 
diff --git a/docker/ubuntu-16.04/Dockerfile b/docker/ubuntu-16.04/Dockerfile
index 3b704c55..b4d0996f 100644
--- a/docker/ubuntu-16.04/Dockerfile
+++ b/docker/ubuntu-16.04/Dockerfile
@@ -40,6 +40,9 @@ ENV opam_root=/home/jenkins/OPAM
 ENV opam_env="opam config env --root=${opam_root}"
 RUN opam init --root=${opam_root} --comp ${ocaml_version}
 
+RUN eval `${opam_env}` && \
+    opam repo --root=${opam_root} add janestreet git://github.com/janestreet/opam-repository.git
+
 RUN eval `${opam_env}` && \
     opam update && \
     opam install -y \
@@ -53,7 +56,7 @@ RUN eval `${opam_env}` && \
         ocplib-endian.1.0 \
         quickcheck.1.0.2 \
         conf-libev \
-        core.113.33.02+4.03 \
+        core.114.04+57 \
         redis.0.3.1 \
         uri.1.9.2
 

From f4b995b7d7259682318b940be7142f7db17bc37f Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Tue, 9 Aug 2016 09:09:11 +0200
Subject: [PATCH 199/373] pass through ARAKOON_PYTHON_CLIENT env var

---
 docker/run.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docker/run.sh b/docker/run.sh
index 1ed3066a..174e07d4 100755
--- a/docker/run.sh
+++ b/docker/run.sh
@@ -17,6 +17,7 @@ else
 fi
 
 docker run -i $TTY --privileged=true -e UID=${UID} \
+       --env ARAKOON_PYTHON_CLIENT \
        -v ${PWD}:/home/jenkins/arakoon \
        -w /home/jenkins/arakoon arakoon_$IMAGE \
        bash -l -c "cd arakoon && ./docker/suites.sh $@"

From 4be05ceb7ddc545813fba3bba3a8b6c82cd31b1d Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Tue, 9 Aug 2016 09:18:25 +0200
Subject: [PATCH 200/373] try to fix the travis build

---
 run_with_timeout_and_progress.sh | 23 +++++++++++++++++++++++
 travis.sh                        |  7 ++-----
 2 files changed, 25 insertions(+), 5 deletions(-)
 create mode 100755 run_with_timeout_and_progress.sh

diff --git a/run_with_timeout_and_progress.sh b/run_with_timeout_and_progress.sh
new file mode 100755
index 00000000..47b844ad
--- /dev/null
+++ b/run_with_timeout_and_progress.sh
@@ -0,0 +1,23 @@
+#!/bin/bash -ue
+
+TIMEOUT=$1
+COMMAND=$2
+shift 2
+
+timeout $TIMEOUT $COMMAND $@ > output 2>&1 &
+PID=$!
+
+echo $PID
+
+while kill -0 $PID 2>/dev/null
+do
+    echo -ne .
+    sleep 1
+done
+
+wait $PID
+RESULT=$?
+
+tail -n256 output
+
+exit $RESULT
diff --git a/travis.sh b/travis.sh
index 36b7a2b3..dfd8d09e 100755
--- a/travis.sh
+++ b/travis.sh
@@ -10,7 +10,7 @@ install () {
 
     date
 
-    ./docker/run.sh ubuntu-16.04 clean | tail -n256
+    ./run_with_timeout_and_progress.sh 9000 ./docker/run.sh ubuntu-16.04 clean
 
     date
 }
@@ -20,12 +20,9 @@ script () {
 
     date
 
-    ./docker/run.sh ubuntu-16.04 unit 2&>1 | tail -n256
-    RESULT=$PIPESTATUS
+    ./run_with_timeout_and_progress.sh 9000 ./docker/run.sh ubuntu-16.04 unit
 
     date
-
-    exit $RESULT
 }
 
 before_cache () {

From 60e350a46ae7f0d1bfb2dfa5dc325a015979d2dd Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Tue, 9 Aug 2016 11:36:14 +0200
Subject: [PATCH 201/373] pass through ARAKOON_PYTHON_CLIENT env var & remove
 obsolete docker files

---
 docker/arakoon_centos7/Dockerfile | 78 ---------------------------
 docker/arakoon_debian/Dockerfile  | 88 -------------------------------
 docker/docker-entrypoint.sh       |  2 +-
 3 files changed, 1 insertion(+), 167 deletions(-)
 delete mode 100644 docker/arakoon_centos7/Dockerfile
 delete mode 100644 docker/arakoon_debian/Dockerfile

diff --git a/docker/arakoon_centos7/Dockerfile b/docker/arakoon_centos7/Dockerfile
deleted file mode 100644
index 54878143..00000000
--- a/docker/arakoon_centos7/Dockerfile
+++ /dev/null
@@ -1,78 +0,0 @@
-FROM centos:7
-
-RUN rpm --import file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
-RUN yum -y install wget
-RUN yum -y install curl
-RUN yum -y install make m4 gcc patch unzip
-RUN yum -y install git rsync mercurial
-RUN yum -y install gcc-c++
-RUN yum -y install bzip2-devel libffi-devel snappy-devel libev-devel
-RUN yum -y install python-devel
-
-# TODO: What about darcs ?
-
-RUN useradd jenkins -u 1001 -g root
-
-RUN rpm -iUvh http://dl.fedoraproject.org/pub/epel/7/x86_64/e/epel-release-7-7.noarch.rpm
-
-RUN yum -y update
-RUN yum -y install python-pip
-RUN yum -y install openssl-devel gmp-devel which
-RUN yum -y install zlib-devel ncurses-devel
-
-RUN wget https://raw.github.com/ocaml/opam/master/shell/opam_installer.sh
-RUN sh ./opam_installer.sh /usr/local/bin 4.02.3
-RUN mkdir /home/opam
-ENV opam_env='opam config env --root=/home/opam/OPAM'
-RUN opam init --root=/home/opam/OPAM --comp 4.02.3
-
-
-# protobuf
-RUN yum -y install bzip2
-RUN wget https://github.com/google/protobuf/releases/download/v2.6.1/protobuf-2.6.1.tar.bz2 -O protobuf-2.6.1.tar.bz2 \
-    && tar -jxvf protobuf-2.6.1.tar.bz2 \
-    && cd protobuf-2.6.1 \
-    && ./configure && make && make install
-
-#RUN rpm -ivh http://cbs.centos.org/kojifiles/packages/protobuf/2.5.0/10.el7.centos/src/protobuf-2.5.0-10.el7.centos.src.rpm
-
-RUN pip install fabric junit-xml
-
-RUN wget http://cudf-solvers.irill.org/cudf_remote_proxy
-RUN chmod u+x cudf_remote_proxy
-RUN mv cudf_remote_proxy /usr/local/bin/
-
-ENV OPAMEXTERNALSOLVER="cudf_remote_proxy %{input}% %{output}% %{criteria}%"
-
-RUN eval `${opam_env}` && \
-    opam update && \
-    opam install -y \
-        ocamlfind \
-        ssl.0.5.2 \
-        camlbz2 \
-        snappy \
-        bisect \
-        lwt.2.5.1 \
-        camltc.0.9.3 \
-        quickcheck \
-        conf-libev \
-        ocplib-endian \
-        depext \
-        redis.0.3.1 \
-        uri.1.9.1 \
-        core.113.00.00
-
-
-RUN yum -y install autoconf help2man perl-Thread-Queue
-
-RUN yum -y install sudo iproute
-#
-# Disable "ssh hostname sudo <cmd>", because it will show the password in clear.
-#         You have to run "ssh -t hostname sudo <cmd>".
-#
-# Defaults    requiretty          # is line 56
-RUN awk 'NR == 56 {next} {print}' /etc/sudoers >/tmp/__sudoers && mv /tmp/__sudoers /etc/sudoers
-
-RUN yum -y install rpm-build
-
-CMD eval `${opam_env}` && opam list
diff --git a/docker/arakoon_debian/Dockerfile b/docker/arakoon_debian/Dockerfile
deleted file mode 100644
index 139c80dc..00000000
--- a/docker/arakoon_debian/Dockerfile
+++ /dev/null
@@ -1,88 +0,0 @@
-FROM ubuntu:15.04
-RUN apt-get -y update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
-        build-essential m4 apt-utils \
-        libffi-dev libssl-dev \
-        libbz2-dev \
-        libgmp3-dev \
-        libev-dev \
-        libsnappy-dev \
-        libxen-dev \
-        help2man \
-        pkg-config \
-        time \
-        aspcud \
-        wget \
-        rsync \
-        darcs \
-        git \
-        unzip \
-        protobuf-compiler \
-        libgcrypt20-dev \
-        libjerasure-dev \
-        yasm \
-        automake \
-        python-dev \
-        python-pip \
-        debhelper \
-        psmisc \
-        strace \
-        curl \
-        g++ \
-        libgflags-dev \
-        sudo \
-        libtool \
-        libboost-all-dev \
-        iptables \
-        net-tools \
-        ncurses-dev
-
-RUN useradd jenkins -u 1001 -g root
-
-RUN pip install fabric junit-xml nose simplejson python-etcd
-
-RUN wget https://raw.github.com/ocaml/opam/master/shell/opam_installer.sh
-RUN sh ./opam_installer.sh /usr/local/bin 4.02.3
-
-RUN mkdir /home/tests
-RUN chmod 777 /home/tests
-ENV TEST_HOME='/home/tests'
-ENV PATH=${PATH}:/home/tests/apps/arakoon/bin
-
-RUN mkdir /home/opam
-ENV opam_env='opam config env --root=/home/opam/OPAM'
-RUN opam init --root=/home/opam/OPAM --comp 4.02.3
-
-RUN eval `${opam_env}` && \
-    opam update && \
-    opam install -y \
-        ocamlfind \
-        ssl.0.5.2 \
-        camlbz2 \
-        snappy \
-        bisect \
-        lwt.2.5.1 \
-        camltc.0.9.3 \
-        quickcheck \
-        conf-libev \
-        ocplib-endian \
-        depext \
-        redis.0.3.1 \
-        uri.1.9.1 \
-        core.113.00.00
-
-RUN echo "jenkins ALL=NOPASSWD: ALL" >/etc/sudoers.d/jenkins
-ENV LANG=C
-RUN chmod 777 /home/opam/OPAM/4.02.3/lib
-
-# Install etcd:
-RUN curl -L https://github.com/coreos/etcd/releases/download/v2.3.5/etcd-v2.3.5-linux-amd64.tar.gz -o etcd-v2.3.5-linux-amd64.tar.gz
-RUN tar xzvf etcd-v2.3.5-linux-amd64.tar.gz
-RUN cp ./etcd-v2.3.5-linux-amd64/etcd /usr/bin
-RUN cp ./etcd-v2.3.5-linux-amd64/etcdctl /usr/bin
-
-# Install redis:
-RUN wget http://download.redis.io/releases/redis-3.0.7.tar.gz
-RUN tar xzvf redis-3.0.7.tar.gz
-RUN cd redis-3.0.7 && make
-RUN cp ./redis-3.0.7/src/redis-server /usr/bin
-RUN cp ./redis-3.0.7/src/redis-cli /usr/bin
diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh
index de73f473..18876387 100755
--- a/docker/docker-entrypoint.sh
+++ b/docker/docker-entrypoint.sh
@@ -15,4 +15,4 @@ then
 fi
 
 # finally execute the command the user requested
-exec sudo -i -u jenkins "$@"
+exec sudo ARAKOON_PYTHON_CLIENT=$ARAKOON_PYTHON_CLIENT -i -u jenkins "$@"

From 8f9656e2966c067d31086899fb682c1673bdb78d Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Tue, 9 Aug 2016 12:02:10 +0200
Subject: [PATCH 202/373] lala

---
 docker/docker-entrypoint.sh      | 2 +-
 run_with_timeout_and_progress.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh
index 18876387..c5d15f8c 100755
--- a/docker/docker-entrypoint.sh
+++ b/docker/docker-entrypoint.sh
@@ -15,4 +15,4 @@ then
 fi
 
 # finally execute the command the user requested
-exec sudo ARAKOON_PYTHON_CLIENT=$ARAKOON_PYTHON_CLIENT -i -u jenkins "$@"
+exec sudo ARAKOON_PYTHON_CLIENT=${ARAKOON_PYTHON_CLIENT-} -i -u jenkins "$@"
diff --git a/run_with_timeout_and_progress.sh b/run_with_timeout_and_progress.sh
index 47b844ad..0f198ac2 100755
--- a/run_with_timeout_and_progress.sh
+++ b/run_with_timeout_and_progress.sh
@@ -1,4 +1,4 @@
-#!/bin/bash -ue
+#!/bin/bash -u
 
 TIMEOUT=$1
 COMMAND=$2

From e0c6ecd4f778b41446dc7b20504691a89275382b Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Tue, 9 Aug 2016 13:00:51 +0200
Subject: [PATCH 203/373] speed up travis build by only updating the cache when
 needed

---
 travis.sh | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/travis.sh b/travis.sh
index dfd8d09e..0a79bf69 100755
--- a/travis.sh
+++ b/travis.sh
@@ -8,11 +8,15 @@ install () {
     if [[ -e ~/cache/image.tar.gz ]];
     then docker load -i ~/cache/image.tar.gz; fi
 
-    date
+    START_BUILD=date
+    export START_BUILD
+    echo $START_BUILD
 
     ./run_with_timeout_and_progress.sh 9000 ./docker/run.sh ubuntu-16.04 clean
 
-    date
+    END_BUILD=date
+    export END_BUILD
+    echo $END_BUILD
 }
 
 script () {
@@ -30,9 +34,15 @@ before_cache () {
 
     date
 
-    mkdir ~/cache || true
-    docker save -o ~/cache/image.tar.gz arakoon_ubuntu-16.04
-    ls -ahl ~/cache
+    DIFF=$(echo "$END_BUILD - $START_BUILD" | bc)
+    if [ $DIFF \> 5 ]
+    then
+        mkdir ~/cache || true
+        docker save -o ~/cache/image.tar.gz arakoon_ubuntu-16.04
+        ls -ahl ~/cache;
+    else
+        echo Building of container took only $DIFF seconds, not updating cache this time;
+    fi
 
     date
 }

From 909cc8c5b7eca3f0bc0129b358409fba3baf41f5 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Tue, 9 Aug 2016 21:33:41 +0200
Subject: [PATCH 204/373] non fleeting core depency

---
 docker/centos-7/Dockerfile     | 7 +++++--
 docker/ubuntu-14.04/Dockerfile | 9 +++++++--
 docker/ubuntu-16.04/Dockerfile | 7 +++++--
 3 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/docker/centos-7/Dockerfile b/docker/centos-7/Dockerfile
index 464afc31..39c053ee 100644
--- a/docker/centos-7/Dockerfile
+++ b/docker/centos-7/Dockerfile
@@ -30,8 +30,11 @@ ENV opam_root=/home/jenkins/OPAM
 ENV opam_env="opam config env --root=${opam_root}"
 RUN opam init --root=${opam_root} --comp ${ocaml_version}
 
+RUN git clone git://github.com/janestreet/opam-repository.git /janestreet-opam-repo && \
+    cd /janestreet-opam-repo && \
+    git checkout b98fd1964856f0c0b022a42ec4e6fc6c7bad2e81
 RUN eval `${opam_env}` && \
-    opam repo --root=${opam_root} add janestreet git://github.com/janestreet/opam-repository.git
+    opam repo --root=${opam_root} -k local add janestreet /janestreet-opam-repo
 
 # protobuf
 RUN yum -y install bzip2
@@ -63,7 +66,7 @@ RUN eval `${opam_env}` && \
         ocplib-endian.1.0 \
         quickcheck.1.0.2 \
         conf-libev \
-        core.114.04+57 \
+        core.114.05+21 \
         redis.0.3.1 \
         uri.1.9.2
 
diff --git a/docker/ubuntu-14.04/Dockerfile b/docker/ubuntu-14.04/Dockerfile
index 696f47d9..9d03586b 100644
--- a/docker/ubuntu-14.04/Dockerfile
+++ b/docker/ubuntu-14.04/Dockerfile
@@ -41,8 +41,13 @@ ENV opam_root=/home/jenkins/OPAM
 ENV opam_env="opam config env --root=${opam_root}"
 RUN opam init --root=${opam_root} --comp ${ocaml_version}
 
+
+RUN git clone git://github.com/janestreet/opam-repository.git /janestreet-opam-repo && \
+    cd /janestreet-opam-repo && \
+    git checkout b98fd1964856f0c0b022a42ec4e6fc6c7bad2e81
 RUN eval `${opam_env}` && \
-    opam repo --root=${opam_root} add janestreet git://github.com/janestreet/opam-repository.git
+    opam repo --root=${opam_root} -k local add janestreet /janestreet-opam-repo
+
 
 RUN eval `${opam_env}` && \
     opam update && \
@@ -57,7 +62,7 @@ RUN eval `${opam_env}` && \
         ocplib-endian.1.0 \
         quickcheck.1.0.2 \
         conf-libev \
-        core.114.04+57 \
+        core.114.05+21 \
         redis.0.3.1 \
         uri.1.9.2
 
diff --git a/docker/ubuntu-16.04/Dockerfile b/docker/ubuntu-16.04/Dockerfile
index b4d0996f..613937f3 100644
--- a/docker/ubuntu-16.04/Dockerfile
+++ b/docker/ubuntu-16.04/Dockerfile
@@ -40,8 +40,11 @@ ENV opam_root=/home/jenkins/OPAM
 ENV opam_env="opam config env --root=${opam_root}"
 RUN opam init --root=${opam_root} --comp ${ocaml_version}
 
+RUN git clone git://github.com/janestreet/opam-repository.git /janestreet-opam-repo && \
+    cd /janestreet-opam-repo && \
+    git checkout b98fd1964856f0c0b022a42ec4e6fc6c7bad2e81
 RUN eval `${opam_env}` && \
-    opam repo --root=${opam_root} add janestreet git://github.com/janestreet/opam-repository.git
+    opam repo --root=${opam_root} -k local add janestreet /janestreet-opam-repo
 
 RUN eval `${opam_env}` && \
     opam update && \
@@ -56,7 +59,7 @@ RUN eval `${opam_env}` && \
         ocplib-endian.1.0 \
         quickcheck.1.0.2 \
         conf-libev \
-        core.114.04+57 \
+        core.114.05+21 \
         redis.0.3.1 \
         uri.1.9.2
 

From 8ba4f64686f457310788474d36a01591469881df Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Wed, 10 Aug 2016 08:40:37 +0200
Subject: [PATCH 205/373] one more change for travis

---
 .travis.yml |  1 -
 travis.sh   | 35 ++++++++++++-----------------------
 2 files changed, 12 insertions(+), 24 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 59191ae2..18d2a435 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -15,4 +15,3 @@ cache:
     - ~/cache
   timeout:
     1000
-before_cache: ./travis.sh before_cache
diff --git a/travis.sh b/travis.sh
index 0a79bf69..9226517d 100755
--- a/travis.sh
+++ b/travis.sh
@@ -8,41 +8,33 @@ install () {
     if [[ -e ~/cache/image.tar.gz ]];
     then docker load -i ~/cache/image.tar.gz; fi
 
-    START_BUILD=date
-    export START_BUILD
+    START_BUILD=$(date +%s.%N)
     echo $START_BUILD
 
     ./run_with_timeout_and_progress.sh 9000 ./docker/run.sh ubuntu-16.04 clean
 
-    END_BUILD=date
-    export END_BUILD
+    END_BUILD=$(date +%s.%N)
     echo $END_BUILD
-}
-
-script () {
-    echo "Running 'script' phase"
-
-    date
-
-    ./run_with_timeout_and_progress.sh 9000 ./docker/run.sh ubuntu-16.04 unit
-
-    date
-}
-
-before_cache () {
-    echo "Running 'before_cache' phase"
-
-    date
 
     DIFF=$(echo "$END_BUILD - $START_BUILD" | bc)
     if [ $DIFF \> 5 ]
     then
+        df -h
         mkdir ~/cache || true
+        rm -f ~/cache/image.tar.gz
         docker save -o ~/cache/image.tar.gz arakoon_ubuntu-16.04
         ls -ahl ~/cache;
     else
         echo Building of container took only $DIFF seconds, not updating cache this time;
     fi
+}
+
+script () {
+    echo "Running 'script' phase"
+
+    date
+
+    ./run_with_timeout_and_progress.sh 9000 ./docker/run.sh ubuntu-16.04 unit
 
     date
 }
@@ -54,9 +46,6 @@ case "$1" in
     script)
         script
         ;;
-    before_cache)
-        before_cache
-        ;;
     *)
         echo "Usage: $0 {install|script}"
         exit 1

From 1a0c810c8b9206464ed119c7f04410a8d27f2494 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Wed, 10 Aug 2016 13:19:17 +0200
Subject: [PATCH 206/373] fix for race between tlogmap reinit & tlog
 compression

---
 src/tlog/tlc2.ml           | 42 +++------------------
 src/tlog/tlog_map.ml       | 76 ++++++++++++++++++++++++++++----------
 src/tlog/tlogcollection.ml |  2 +-
 3 files changed, 63 insertions(+), 57 deletions(-)

diff --git a/src/tlog/tlc2.ml b/src/tlog/tlc2.ml
index 7983db1e..762ddbd1 100644
--- a/src/tlog/tlc2.ml
+++ b/src/tlog/tlc2.ml
@@ -27,46 +27,14 @@ open Tlog_map
 
 let section = Logger.Section.main
 
-
-let to_archive_name comp fn =
-  let length = String.length fn in
-  let ext = String.sub fn (length -5) 5 in
-  if ext = ".tlog"
-  then
-    let root = String.sub fn 0 (length -5) in
-    let e = extension comp in
-    root ^ e
-  else failwith (Printf.sprintf "extension is '%s' and should be '.tlog'" ext)
-
-
-
 let head_fname = "head.db"
 
 let head_name () = head_fname
 
+let to_tlog_name = Tlog_map.to_tlog_name
+let to_archive_name = Tlog_map.to_archive_name
 
-
-let to_tlog_name fn =
-  let length = String.length fn in
-  let extension = String.sub fn (length -4) 4 in
-  if extension = ".tlx"
-     || extension = ".tlf"
-     || extension = ".tlc"
-  then
-    let root = String.sub fn 0 (length -4) in
-    root ^ ".tlog"
-  else failwith (Printf.sprintf "to_tlog_name:extension is '%s' and should be one of .tlc .tlf .tlx" extension)
-
-
-let folder_for filename index =
-  let extension = extension_of filename in
-  match extension with
-  | ".tlog" -> Tlogreader2.AU.fold, extension, index
-  | ".tlx"  -> Tlogreader2.AS.fold, extension, None
-  | ".tlf"  -> Tlogreader2.AC.fold, extension, None
-  | ".tlc"  -> Tlogreader2.AO.fold, extension, None
-  | _       -> failwith (Printf.sprintf "no folder for '%s'" extension)
-
+let folder_for = Tlog_map.folder_for
 
 
 let _validate_list tlog_names node_id ~check_marker=
@@ -301,7 +269,7 @@ class tlc2
         | None ->
           begin
             let outer = TlogMap.outer_of_i tlog_map i in
-            Logger.debug_f_ "prelude i:%s: outer=%i No File" (Sn.string_of i) outer>>= fun () ->
+            Logger.info_f_ "prelude i:%s: outer=%i No File" (Sn.string_of i) outer>>= fun () ->
             begin
               if outer <> 0 && (TlogMap.is_rollover_point tlog_map i)
               then
@@ -559,8 +527,8 @@ class tlc2
       match _file with
       | None -> Lwt.return ()
       | Some file ->
-         F.close file >>= fun () ->
          _file <- None;
+         F.close file >>= fun () ->
          Lwt.return_unit
 
     method save_tlog_file first_i extension length ic =
diff --git a/src/tlog/tlog_map.ml b/src/tlog/tlog_map.ml
index fdc28e57..e219c054 100644
--- a/src/tlog/tlog_map.ml
+++ b/src/tlog/tlog_map.ml
@@ -28,6 +28,36 @@ let extension_of filename =
   let dot_pos = String.rindex filename '.' in
   String.sub filename dot_pos (len - dot_pos)
 
+let extension comp =
+  let open Compression in
+  match comp with
+    | Snappy -> ".tlx"
+    | Bz2 -> ".tlf"
+    | No -> ".tlog"
+
+let to_archive_name comp fn =
+  let length = String.length fn in
+  let ext = String.sub fn (length -5) 5 in
+  if ext = ".tlog"
+  then
+    let root = String.sub fn 0 (length -5) in
+    let e = extension comp in
+    root ^ e
+  else failwith (Printf.sprintf "extension is '%s' and should be '.tlog'" ext)
+
+
+let to_tlog_name fn =
+  let length = String.length fn in
+  let extension = String.sub fn (length -4) 4 in
+  if extension = ".tlx"
+     || extension = ".tlf"
+     || extension = ".tlc"
+  then
+    let root = String.sub fn 0 (length -4) in
+    root ^ ".tlog"
+  else failwith (Printf.sprintf "to_tlog_name:extension is '%s' and should be one of .tlc .tlf .tlx" extension)
+
+
 let folder_for filename index =
   let extension = extension_of filename in
   match extension with
@@ -43,11 +73,7 @@ let is_archive filename =
   | ".tlx" | ".tlf" | ".tlc"-> true
   | _ -> false
 
-let first_i_of ?extension_override filename =
-  let extension = match extension_override with
-    | None -> extension_of filename
-    | Some ext ->  ext
-  in
+let first_i_of filename tlx_dir =
   let read_uncompress ic inflate =
     Llio.input_string ic >>= fun compressed ->
     let uncompressed = inflate compressed in
@@ -55,8 +81,8 @@ let first_i_of ?extension_override filename =
     let i =  Sn.sn_from buffer in
     Lwt.return i
   in
-  let reader =
-    match extension with
+  let reader filename =
+    match extension_of filename with
     | ".tlog" -> (fun ic -> Sn.input_sn ic )
     | ".tlx" ->
        (fun ic ->
@@ -77,7 +103,26 @@ let first_i_of ?extension_override filename =
        )
     | x -> failwith (Printf.sprintf "not a tlog_file:%s" x)
   in
-  Lwt_io.with_file ~mode:Lwt_io.input filename reader >>= fun i ->
+  let get_i filename =
+    Lwt_io.with_file ~mode:Lwt_io.input filename (reader filename)
+  in
+  Lwt.catch
+    (fun () -> get_i filename)
+    (function
+      | Unix.Unix_error(Unix.ENOENT, "open", _) when extension_of filename = ".tlog" ->
+         (* the tlog might already be compressed in the mean time *)
+         Lwt_log.info_f
+           "Could not get first_i_of %s, trying compressed tlog now"
+           filename >>= fun () ->
+         Lwt.catch
+           (fun () -> get_i (to_archive_name Compression.Snappy filename))
+           (function
+             | Unix.Unix_error(Unix.ENOENT, _, _) ->
+                get_i (to_archive_name Compression.Bz2 filename))
+      | exn ->
+         Lwt_log.warning_f ~exn "Exception during first_i_of %s" filename >>= fun () ->
+         Lwt.fail exn)
+  >>= fun i ->
   Logger.debug_f_ "%s starts with i:%Li" filename i >>= fun () ->
   Lwt.return i
 
@@ -87,13 +132,6 @@ let get_number fn =
   let pre = String.sub fn 0 dot_pos in
   int_of_string pre
 
-let extension comp =
-  let open Compression in
-  match comp with
-    | Snappy -> ".tlx"
-    | Bz2 -> ".tlf"
-    | No -> ".tlog"
-
 let archive_name comp c =
   let x = extension comp in
   Printf.sprintf "%03i%s" c x
@@ -296,10 +334,10 @@ module TlogMap = struct
 
 
   let _init tlog_dir tlx_dir node_id tlog_max_entries tlog_max_size ~check_marker ~check_sabotage =
-    Logger.debug_f_ "_init ~check_marker:%b" check_marker >>= fun () ->
+    Logger.info_f_ "_init ~check_marker:%b" check_marker >>= fun () ->
     _get_tlog_names tlog_dir tlx_dir >>= fun tlog_names ->
     let tlog_number = get_count tlog_names in
-    Logger.debug_f_ "tlog_number:%i" tlog_number >>= fun () ->
+    Logger.info_f_ "tlog_number:%i" tlog_number >>= fun () ->
     let full_path = _get_full_path tlog_dir tlx_dir (file_name tlog_number) in
     _validate_one full_path node_id ~check_marker ~check_sabotage
     >>= fun (last, index, tlog_size) ->
@@ -307,7 +345,7 @@ module TlogMap = struct
       (fun (acc,prev)  tlog_name ->
         let n = get_number tlog_name in
         let canonical = _get_full_path tlog_dir tlx_dir tlog_name in
-        first_i_of canonical >>= fun i ->
+        first_i_of canonical tlx_dir >>= fun i ->
         (if i >= prev
         then Lwt.return_unit
         else Lwt.fail_with
@@ -341,7 +379,7 @@ module TlogMap = struct
          let should_roll = tlog_entries + 1 >= tlog_max_entries in
          tlog_entries, should_roll, item.i
     in
-    Logger.debug_f_ "_init: tlog_entries:%i should_roll:%b" tlog_entries should_roll
+    Logger.info_f_ "_init: tlog_entries:%i should_roll:%b" tlog_entries should_roll
     >>= fun () ->
     Lwt.return ((tlog_entries, tlog_size, tlog_number, i_to_tlog_number, should_roll),
                 last, index)
diff --git a/src/tlog/tlogcollection.ml b/src/tlog/tlogcollection.ml
index ea248e83..761b7733 100644
--- a/src/tlog/tlogcollection.ml
+++ b/src/tlog/tlogcollection.ml
@@ -45,7 +45,7 @@ class type tlog_collection =
      returns file number & starting i of next file  
    *)
   method complete_file_to_deliver: Sn.t -> (int * Sn.t) option
-                                               
+
   method dump_tlog_file : int -> Lwt_io.output_channel -> unit Lwt.t
   method save_tlog_file : Sn.t
                           -> string -> int64

From df3d7051d4ff35e34864594acc63abdfc75478af Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Wed, 10 Aug 2016 13:48:45 +0200
Subject: [PATCH 207/373] expand test_get_last_i + remove unused
 tlogcollection#reinit method

---
 src/tlog/mem_tlogcollection.ml |  1 -
 src/tlog/tlc2.ml               | 11 -----------
 src/tlog/tlc2_test.ml          |  5 +++++
 src/tlog/tlogcollection.ml     |  2 +-
 4 files changed, 6 insertions(+), 13 deletions(-)

diff --git a/src/tlog/mem_tlogcollection.ml b/src/tlog/mem_tlogcollection.ml
index 11c0bacf..bf748141 100644
--- a/src/tlog/mem_tlogcollection.ml
+++ b/src/tlog/mem_tlogcollection.ml
@@ -71,7 +71,6 @@ class mem_tlog_collection _name =
 
     method save_tlog_file _start_i _name _length _ic = failwith "save_tlog_file not supported"
 
-    method reinit () = Lwt.return ()
     method which_tlog_file _start_i = failwith "which_tlog_file not supported"
 
     method tlogs_to_collapse ~head_i ~last_i ~tlogs_to_keep =
diff --git a/src/tlog/tlc2.ml b/src/tlog/tlc2.ml
index 762ddbd1..a5557140 100644
--- a/src/tlog/tlc2.ml
+++ b/src/tlog/tlc2.ml
@@ -553,17 +553,6 @@ class tlc2
       let () = TlogMap.set_should_roll tlog_map in
       Lwt.return_unit
 
-
-    method reinit () =
-      invalidate_previous_entry ();
-      begin
-        match _file with
-        | None -> Lwt.return ()
-        | Some file -> F.close file >>= fun () -> _file <- None;Lwt.return ()
-      end
-      >>= fun () ->
-      TlogMap.reinit tlog_map >|= ignore
-
     method remove_below i = TlogMap.remove_below tlog_map i
     method complete_file_to_deliver i = TlogMap.complete_file_to_deliver tlog_map i
   end
diff --git a/src/tlog/tlc2_test.ml b/src/tlog/tlc2_test.ml
index 45327ccd..1d99fcb7 100644
--- a/src/tlog/tlc2_test.ml
+++ b/src/tlog/tlc2_test.ml
@@ -417,6 +417,11 @@ let test_get_last_i (dn, tlx_dir, (factory:factory)) =
   Lwt_log.debug_f "got last_i = %Li" last_i >>= fun () ->
   assert (5L = last_i);
 
+  tlc # log_value 6L value >>= fun _ ->
+  tlc # get_last_i () >>= fun last_i ->
+  Lwt_log.debug_f "got last_i = %Li" last_i >>= fun () ->
+  assert (6L = last_i);
+
   Lwt.return_unit
 
 
diff --git a/src/tlog/tlogcollection.ml b/src/tlog/tlogcollection.ml
index 761b7733..991aa363 100644
--- a/src/tlog/tlogcollection.ml
+++ b/src/tlog/tlogcollection.ml
@@ -51,7 +51,7 @@ class type tlog_collection =
                           -> string -> int64
                           -> Lwt_io.input_channel
                           -> unit Lwt.t
-  method reinit : unit -> unit Lwt.t 
+
   method get_head_name : unit -> string
   method get_tlog_from_i : Sn.t -> int
   method get_tlog_count: unit -> int Lwt.t

From 526c25f717c39cfeaed332f5fd38d48dce5da4d0 Mon Sep 17 00:00:00 2001
From: matthias <matthias.deblock@openvstorage.com>
Date: Wed, 10 Aug 2016 15:13:41 +0200
Subject: [PATCH 208/373] Edit collapse arakoon manual

---
 .idea/vcs.xml           |  6 ++++++
 docs/collapseArakoon.md | 27 +++++++++++++++++++++++----
 2 files changed, 29 insertions(+), 4 deletions(-)
 create mode 100644 .idea/vcs.xml

diff --git a/.idea/vcs.xml b/.idea/vcs.xml
new file mode 100644
index 00000000..94a25f7f
--- /dev/null
+++ b/.idea/vcs.xml
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>
\ No newline at end of file
diff --git a/docs/collapseArakoon.md b/docs/collapseArakoon.md
index 1c6d82bd..2e27debe 100644
--- a/docs/collapseArakoon.md
+++ b/docs/collapseArakoon.md
@@ -57,16 +57,35 @@ Verify the log file and check if he download the head.db from another node and r
 Collapse the Arakoon manually with collapse-remote
 
 #### 3.2.1 Steps
-Check the IP and PORT where your Arakoon is running. (check the Arakoon config)
+Check the IP and PORT where your Arakoon is running and collapse-remote the Arakoon database.
 
-Command:
+**Arakoon config (Fargo):**
+
+* Get the Arakoon config location:
+```
+etcdctl ls --recursive | grep arakoon | grep config
+
+/ovs/arakoon/mybackend-abm/config
+/ovs/arakoon/mybackend-nsm_0/config
+/ovs/arakoon/voldrv/config
+/ovs/arakoon/ovsdb/config
+```
+* Print the correct Arakoon config:
+```
+etcdctl get /ovs/arakoon/ovsdb/config
+```
+
+**Collapse remote**
+
+* Command:
 ```
 arakoon --collapse-remote <cluster_id> <ip> <port> <n> tells node to collapse all but <n> tlogs into its head database
 ```
+The cluster_id, ip and port can be found in the previous printed Arakoon config
 
-Example:
+* Example:
 ```
-arakoon --collapse-remote T3BpM2jJy5qloYS5 127.0.0.1 26409 2
+arakoon --collapse-remote ovsdb 10.100.1.2 26400 2
 ```
 
 ### 3.3 Manually

From 009ccc2867ffc0eec2e5c04c3da1914fc654b0cb Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Thu, 11 Aug 2016 09:37:18 +0200
Subject: [PATCH 209/373] test Tlog_map.first_i_of

---
 src/tlog/tlc2_test.ml | 24 ++++++++++++++++++++++++
 src/tlog/tlog_map.ml  |  3 +++
 2 files changed, 27 insertions(+)

diff --git a/src/tlog/tlc2_test.ml b/src/tlog/tlc2_test.ml
index 1d99fcb7..9e740400 100644
--- a/src/tlog/tlc2_test.ml
+++ b/src/tlog/tlc2_test.ml
@@ -424,6 +424,29 @@ let test_get_last_i (dn, tlx_dir, (factory:factory)) =
 
   Lwt.return_unit
 
+let first_i_of_compress_race (dn, tlx_dir, (factory : factory)) =
+  let tlog_max_size = 1000 in
+  factory ~tlog_max_size dn "node_id" >>= fun tlc ->
+  let make_value size =
+    let open Update in
+    Value.create_client_value [ Update.Set ("", Bytes.create size); ] false
+  in
+  let value = make_value 0 in
+  tlc # log_value 0L value >>= fun _ ->
+  tlc # log_value 1L value >>= fun _ ->
+  tlc # log_value 2L value >>= fun _ ->
+  tlc # log_value 3L (make_value tlog_max_size) >>= fun _ ->
+  tlc # log_value 4L value >>= fun _ ->
+  tlc # log_value 5L value >>= fun _ ->
+  (* wait for compress *)
+  tlc # close ~wait_for_compression:true () >>= fun () ->
+
+  let tlog = dn ^ "/000.tlog" in
+  Lwt_log.debug_f "tlog=%s, tlx_dir=%s" tlog tlx_dir >>= fun () ->
+  Tlog_map.first_i_of tlog tlx_dir >>= fun i ->
+  assert (i = 0L);
+  Lwt.return ()
+
 
 let make_test_tlc (x, y) = x >:: wrap_tlc y x
 
@@ -450,4 +473,5 @@ let suite = "tlc2" >:::
                   ("test_compression_previous", test_compression_previous);
                   ("test_size_based_roll_over1", test_size_based_roll_over1);
                   ("test_get_last_i", test_get_last_i);
+                  ("first_i_of_compress_race", first_i_of_compress_race);
                 ]
diff --git a/src/tlog/tlog_map.ml b/src/tlog/tlog_map.ml
index e219c054..726f2f26 100644
--- a/src/tlog/tlog_map.ml
+++ b/src/tlog/tlog_map.ml
@@ -114,6 +114,9 @@ let first_i_of filename tlx_dir =
          Lwt_log.info_f
            "Could not get first_i_of %s, trying compressed tlog now"
            filename >>= fun () ->
+         let to_archive_name c f =
+           Printf.sprintf "%s/%s" tlx_dir (to_archive_name c f |> Filename.basename)
+         in
          Lwt.catch
            (fun () -> get_i (to_archive_name Compression.Snappy filename))
            (function

From 3194525da74f9770f67507a481acf37561038a7a Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Thu, 11 Aug 2016 09:50:33 +0200
Subject: [PATCH 210/373] expose copy-db-to-head call via cli

---
 src/client/nodestream_main.ml    | 13 +++++++++++++
 src/client/protocol_common.ml    |  5 +++++
 src/client/remote_nodestream.ml  |  4 ++++
 src/client/remote_nodestream.mli |  1 +
 src/main/arakoon.ml              | 10 ++++++++++
 5 files changed, 33 insertions(+)

diff --git a/src/client/nodestream_main.ml b/src/client/nodestream_main.ml
index f513b3b6..9acf8ba8 100644
--- a/src/client/nodestream_main.ml
+++ b/src/client/nodestream_main.ml
@@ -45,6 +45,19 @@ let defrag_db ~tls ~tcp_keepalive ip port cluster_id =
   in
   Lwt_main.run (t ())
 
+
+let copy_db_to_head ~tls ~tcp_keepalive ip port cluster_id =
+  let do_it connection =
+    make_remote_nodestream cluster_id connection >>= fun client ->
+    client # copy_db_to_head ()
+  in
+  let address = make_address ip port in
+  let t () =
+    Client_main.with_connection ~tls ~tcp_keepalive address do_it >>= fun () ->
+    Lwt.return 0
+  in
+  Lwt_main.run (t ())
+
 let get_db ~tls ~tcp_keepalive ip port cluster_id location =
   let do_it connection =
     make_remote_nodestream cluster_id connection >>= fun client ->
diff --git a/src/client/protocol_common.ml b/src/client/protocol_common.ml
index ae3b9b93..35fc3863 100644
--- a/src/client/protocol_common.ml
+++ b/src/client/protocol_common.ml
@@ -481,6 +481,11 @@ let defrag_db (ic,oc) =
   request oc outgoing >>= fun () ->
   response ic nothing
 
+let copy_db_to_head (ic, oc) =
+  let outgoing buf = command_to buf COPY_DB_TO_HEAD in
+  request oc outgoing >>= fun () ->
+  response ic nothing
+
 let version (ic,oc) =
   let outgoing buf = command_to buf VERSION in
   request oc outgoing >>= fun () ->
diff --git a/src/client/remote_nodestream.ml b/src/client/remote_nodestream.ml
index 2f19fc18..4e2dfb4d 100644
--- a/src/client/remote_nodestream.ml
+++ b/src/client/remote_nodestream.ml
@@ -37,6 +37,7 @@ class type nodestream = object
 
   method optimize_db: unit -> unit Lwt.t
   method defrag_db: unit -> unit Lwt.t
+  method copy_db_to_head : unit -> unit Lwt.t
   method get_db: string -> unit Lwt.t
 
   method get_fringe: string option -> Routing.range_direction -> ((string * string) list) Lwt.t
@@ -162,6 +163,9 @@ class remote_nodestream ((ic,oc) as conn) =
   method optimize_db () = Protocol_common.optimize_db conn
 
   method defrag_db () = Protocol_common.defrag_db conn
+
+  method copy_db_to_head () = Protocol_common.copy_db_to_head conn
+
   method get_db db_location =
 
     let outgoing buf =
diff --git a/src/client/remote_nodestream.mli b/src/client/remote_nodestream.mli
index 555085f4..3ee0ea8b 100644
--- a/src/client/remote_nodestream.mli
+++ b/src/client/remote_nodestream.mli
@@ -33,6 +33,7 @@ class type nodestream = object
 
   method optimize_db: unit -> unit Lwt.t
   method defrag_db:unit -> unit Lwt.t
+  method copy_db_to_head : unit -> unit Lwt.t
   method get_db: string -> unit Lwt.t
 
   method get_fringe: string option -> Routing.range_direction -> ((string * string) list) Lwt.t
diff --git a/src/main/arakoon.ml b/src/main/arakoon.ml
index 0a4fd0f5..d53c4920 100644
--- a/src/main/arakoon.ml
+++ b/src/main/arakoon.ml
@@ -53,6 +53,7 @@ type local_action =
   | Backup_db
   | Optimize_db
   | Defrag_db
+  | Copy_db_to_head
   | NumberOfValues
   | InitNursery
   | MigrateNurseryRange
@@ -379,6 +380,12 @@ let main () =
                               Arg.Set_int port;
                              ],
      "<cluster_id> <ip> <port> requests the node to defragment its database");
+    ("--copy-db-to-head", Arg.Tuple[set_laction Copy_db_to_head;
+                              Arg.Set_string cluster_id;
+                              Arg.Set_string ip;
+                              Arg.Set_int port;
+                             ],
+     "<cluster_id> <ip> <port> requests the node to copy its database to the head location");
     ("--n-values", set_laction NumberOfValues,
      "returns the number of values in the store");
     ("--ping", Arg.Tuple[set_laction PING;
@@ -475,6 +482,9 @@ let main () =
     | Defrag_db   -> Nodestream_main.defrag_db
                        ~tls ~tcp_keepalive
                        !ip !port !cluster_id
+    | Copy_db_to_head -> Nodestream_main.copy_db_to_head
+                           ~tls ~tcp_keepalive
+                           !ip !port !cluster_id
     | NumberOfValues -> Client_main.get_key_count ~tls !config_url ()
     | InitNursery -> Nursery_main.init_nursery !config_url !cluster_id
     | MigrateNurseryRange -> Nursery_main.migrate_nursery_range

From ca8b73a31b243260559c450f17997640555fba14 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Thu, 11 Aug 2016 10:50:25 +0200
Subject: [PATCH 211/373] make_release: 1.9.9

---
 META                      | 2 +-
 debian/changelog          | 5 +++++
 redhat/SPECS/arakoon.spec | 4 +++-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/META b/META
index 7c5c32f3..d7ec0cd2 100644
--- a/META
+++ b/META
@@ -1,7 +1,7 @@
 # -*- conf -*-
 
 description = "Arakoon client"
-version = "1.9.8"
+version = "1.9.9"
 exists_if = "arakoon_client.cma,arakoon_client.cmxa,arakoon_client.cmxs"
 requires = "threads lwt lwt.unix str oUnit quickcheck lwt.ssl ocplib-endian redis.lwt uri"
 archive(byte) = "arakoon_client.cma tcp_keepalive_sockopt_stubs.o"
diff --git a/debian/changelog b/debian/changelog
index a5c76a7d..808039d8 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,8 @@
+arakoon (1.9.9) unstable; urgency=low
+  * new upstream version
+
+ -- Jan Doms <jan.doms@gmail.com>  Thu, 11 Aug 2016 10:50:25 +0100
+
 arakoon (1.9.8) unstable; urgency=low
   * new upstream version
 
diff --git a/redhat/SPECS/arakoon.spec b/redhat/SPECS/arakoon.spec
index e8515caf..6e3b5684 100644
--- a/redhat/SPECS/arakoon.spec
+++ b/redhat/SPECS/arakoon.spec
@@ -2,7 +2,7 @@
 
 Summary: Arakoon
 Name: arakoon
-Version: 1.9.8
+Version: 1.9.9
 Release: 3%{?dist}
 License: Apache 2.0
 Requires: libev >= 4
@@ -33,6 +33,8 @@ cp arakoon.native %{buildroot}%{_bindir}/arakoon
 %{_bindir}/arakoon
 
 %changelog
+* Thu Aug 11 2016 Jan Doms <jan.doms@gmail.com> - 1.9.9
+- Create arakoon 1.9.9 RPM package
 * Wed Jul 20 2016 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.8
 - Create arakoon 1.9.8 RPM package
 * Fri Jun 24 2016 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.7

From eb792a88ae741afd29536f0951bab4d39e4f6b97 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 17 Aug 2016 10:14:29 +0200
Subject: [PATCH 212/373] make it work with git

---
 setup.py | 25 +++++++------------------
 1 file changed, 7 insertions(+), 18 deletions(-)

diff --git a/setup.py b/setup.py
index d7c6f56c..a746ceb2 100644
--- a/setup.py
+++ b/setup.py
@@ -8,21 +8,12 @@ def shell(cmd):
     v2 = v.strip()
     return v2
 
-def get_info(option):
-    return shell(["hg","id",option])
-
-def get_tag():
-    return get_info('-t')
-
 def get_version():
-    return get_info('-i')
-
-def get_branch():
-    return shell(["hg","branch"])
+    return shell(["git", "describe", "--tags", "--exact-match", "--dirty"])
 
 def get_license():
     data = None
-    with open('COPYING','r') as f:
+    with open('LICENSE','r') as f:
         data = f.read()
     return data
 
@@ -30,18 +21,17 @@ def get_license():
 """Arakoon is a simple distributed key value store.
 This package provides a pure python client for Arakoon.
 
-Mercurial version: %s
+Git version: %s
 """ % (get_version(),)
 
 setup(name='arakoon',
-      version=get_branch(),
+      version=get_version(),
       package_dir={'arakoon':'src/client/python'},
       packages=['arakoon'],
-      data_files = [('license',['COPYING'])],
-      url='http://www.arakoon.org',
+      data_files = [('license',['LICENSE'])],
+      url='https://github.com/openvstorage/arakoon',
       description=description,
-      author='incubaid',
-      author_email='arakoon@incubaid.com',
+      author='openvstorage.com & incubaid',
       classifiers=['Development Status :: 5 - Production/Stable',
                    'Operating System :: OS Independent',
                    'Topic :: Database',
@@ -49,4 +39,3 @@ def get_license():
       zip_safe=True,
       license= get_license()
       )
-

From fb337d7e77a977acce01e905da100bcf64b1c3c1 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 17 Aug 2016 17:45:28 +0200
Subject: [PATCH 213/373] add cmx files in installed client

---
 Makefile | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/Makefile b/Makefile
index b9dd3855..99ebf53f 100644
--- a/Makefile
+++ b/Makefile
@@ -36,45 +36,73 @@ install_client:
           _build/src/arakoon_client.cmxa \
           _build/src/arakoon_client.cmxs \
           _build/src/client/stamp.cmi \
+	  _build/src/client/stamp.cmx \
           _build/src/client/arakoon_exc.mli \
           _build/src/client/arakoon_exc.cmi \
+	  _build/src/client/arakoon_exc.cmx \
           _build/src/client/arakoon_client_config.cmi \
+	  _build/src/client/arakoon_client_config.cmx \
           _build/src/client/arakoon_client.mli \
           _build/src/client/arakoon_client.cmi \
           _build/src/client/arakoon_remote_client.mli \
           _build/src/client/arakoon_remote_client.cmi \
+	  _build/src/client/arakoon_remote_client.cmx \
           _build/src/client/protocol_common.cmi \
+	  _build/src/client/protocol_common.cmx \
           _build/src/client/client_helper.cmi \
+	  _build/src/client/client_helper.cmx \
           _build/src/inifiles/inifiles.mli \
           _build/src/inifiles/inifiles.cmi \
+	  _build/src/inifiles/inifiles.cmx \
           _build/src/lib/std.cmi \
+          _build/src/lib/std.cmx \
           _build/src/plugins/registry.mli \
           _build/src/plugins/registry.cmi \
+	  _build/src/plugins/registry.cmx \
           _build/src/plugins/plugin_helper.mli \
           _build/src/plugins/plugin_helper.cmi \
+	  _build/src/plugins/plugin_helper.cmx \
           _build/src/node/key.mli \
           _build/src/node/key.cmi \
+	  _build/src/node/key.cmx \
           _build/src/node/simple_store.cmi \
+	  _build/src/node/simple_store.cmx \
           _build/src/nursery/routing.cmi \
           _build/src/tlog/interval.cmi \
+          _build/src/tlog/interval.cmx \
           _build/src/tlog/update.cmi \
+	  _build/src/tlog/update.cmx \
           _build/src/tools/ini.cmi \
+          _build/src/tools/ini.cmx \
           _build/src/tools/arakoon_log_sink.cmi \
+	  _build/src/tools/arakoon_log_sink.cmx \
           _build/src/tools/crash_logger.cmi \
+	  _build/src/tools/crash_logger.cmx \
           _build/src/tools/arakoon_redis.cmi \
+          _build/src/tools/arakoon_redis.cmx \
           _build/src/tools/arakoon_logger.cmi \
+	  _build/src/tools/arakoon_logger.cmx \
           _build/src/tools/llio.mli \
           _build/src/tools/llio.cmi \
+	  _build/src/tools/llio.cmx \
           _build/src/tools/network.cmi \
+	  _build/src/tools/network.cmx \
           _build/src/tools/lwt_buffer.cmi \
+	  _build/src/tools/lwt_buffer.cmx \
           _build/src/tools/lwt_extra.cmi \
+	  _build/src/tools/lwt_extra.cmx \
           _build/src/tools/typed_ssl.mli \
           _build/src/tools/typed_ssl.cmi \
+	  _build/src/tools/typed_ssl.cmx \
           _build/src/tools/unix_fd.cmi \
+          _build/src/tools/unix_fd.cmx \
           _build/src/tools/arakoon_etcd.cmi \
+	  _build/src/tools/arakoon_etcd.cmx \
           _build/src/tools/arakoon_config_url.cmi \
+	  _build/src/tools/arakoon_config_url.cmx \
           _build/src/tools/tcp_keepalive_sockopt_stubs.o \
           _build/src/tools/tcp_keepalive.cmi \
+	  _build/src/tools/tcp_keepalive.cmx \
           _build/src/arakoon_client.a
 
 uninstall_client:

From 7f4f6fae99c994e9aad2c4130f6e053c256f74ca Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Thu, 18 Aug 2016 16:56:02 +0200
Subject: [PATCH 214/373] sigh centos

---
 docker/centos-7/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/centos-7/Dockerfile b/docker/centos-7/Dockerfile
index 39c053ee..a64feb99 100644
--- a/docker/centos-7/Dockerfile
+++ b/docker/centos-7/Dockerfile
@@ -13,7 +13,7 @@ RUN yum -y update && yum -y install \
 
 # TODO: What about darcs ?
 
-RUN rpm -iUvh http://dl.fedoraproject.org/pub/epel/7/x86_64/e/epel-release-7-7.noarch.rpm
+RUN rpm -iUvh http://dl.fedoraproject.org/pub/epel/7/x86_64/e/epel-release-7-8.noarch.rpm
 
 RUN yum -y update && yum -y install \
          python-pip \

From 36b207096cb605e003c0a0b10e5e6adb87543a30 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Fri, 19 Aug 2016 10:13:03 +0200
Subject: [PATCH 215/373] disable travis cache as it currently makes the build
 fail due to a lack of space

---
 .travis.yml | 13 ++++++++-----
 travis.sh   | 28 +++++++++++++++++-----------
 2 files changed, 25 insertions(+), 16 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 18d2a435..cfdc5c14 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -10,8 +10,11 @@ git:
 services:
   - docker
 
-cache:
-  directories:
-    - ~/cache
-  timeout:
-    1000
+# cache:
+#   directories:
+#     - ~/cache
+#   timeout:
+#     1000
+
+env:
+  - USE_CACHE=0 # set to 1 to enable
diff --git a/travis.sh b/travis.sh
index 9226517d..9a1f6f5f 100755
--- a/travis.sh
+++ b/travis.sh
@@ -5,8 +5,11 @@ install () {
 
     date
 
-    if [[ -e ~/cache/image.tar.gz ]];
-    then docker load -i ~/cache/image.tar.gz; fi
+    if [ "$USE_CACHE" == "1" ];
+    then
+        if [[ -e ~/cache/image.tar.gz ]];
+        then docker load -i ~/cache/image.tar.gz; fi;
+    fi
 
     START_BUILD=$(date +%s.%N)
     echo $START_BUILD
@@ -16,16 +19,19 @@ install () {
     END_BUILD=$(date +%s.%N)
     echo $END_BUILD
 
-    DIFF=$(echo "$END_BUILD - $START_BUILD" | bc)
-    if [ $DIFF \> 5 ]
+    if [ "$USE_CACHE" == "1" ];
     then
-        df -h
-        mkdir ~/cache || true
-        rm -f ~/cache/image.tar.gz
-        docker save -o ~/cache/image.tar.gz arakoon_ubuntu-16.04
-        ls -ahl ~/cache;
-    else
-        echo Building of container took only $DIFF seconds, not updating cache this time;
+        DIFF=$(echo "$END_BUILD - $START_BUILD" | bc)
+        if [ $DIFF \> 5 ]
+        then
+            df -h
+            mkdir ~/cache || true
+            rm -f ~/cache/image.tar.gz
+            docker save -o ~/cache/image.tar.gz arakoon_ubuntu-16.04
+            ls -ahl ~/cache;
+        else
+            echo Building of container took only $DIFF seconds, not updating cache this time;
+        fi;
     fi
 }
 

From 4eccd9ca9aa69baff22b7e17107a599ee7f7f5da Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Mon, 22 Aug 2016 10:37:14 +0200
Subject: [PATCH 216/373] assert serialization of counted lists

---
 src/tools/llio.ml | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/src/tools/llio.ml b/src/tools/llio.ml
index 56409bc1..799c4e43 100644
--- a/src/tools/llio.ml
+++ b/src/tools/llio.ml
@@ -323,7 +323,14 @@ let input_kv_list ic = input_list input_string_pair ic
 let _output_list output_element oc count list =
   output_int oc count >>= fun () ->
   Client_log.debug_f "Outputting list with %d elements" count >>= fun () ->
-  Lwt_list.iter_s (output_element oc) list
+  let count2 = ref 0 in
+  Lwt_list.iter_s
+    (fun e -> incr count2;output_element oc e)
+    list
+  >>= fun () ->
+  assert (count = !count2);
+  Lwt.return_unit
+
 
 let output_counted_list output_element oc (count, list) =
   _output_list output_element oc count list
@@ -370,8 +377,13 @@ let output_string_array_reversed oc strings =
   output_array_reversed output_string oc strings
 
 let counted_list_to e_to buf list =
-  int_to buf (fst list);
-  List.iter (e_to buf) (List.rev (snd list))
+  let size = fst list in
+  int_to buf size;
+  let size2 = ref 0 in
+  List.iter
+    (fun e -> e_to buf e; incr size2)
+    (List.rev (snd list));
+  assert (size = !size2)
 
 let list_to e_to buf list =
   int_to buf (List.length list);

From c335df40261587b36a705d0caab4549fc63e0e05 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Mon, 22 Aug 2016 16:35:41 +0200
Subject: [PATCH 217/373] introduce client config ini format, which includes
 tls info

---
 src/client/arakoon_client_config.ml      | 141 +++++++++++++++++++++++
 src/client/arakoon_remote_client_test.ml |  15 +--
 src/client/client_helper.ml              |  42 +++++--
 src/client/client_main.ml                |  76 +++++-------
 src/client/remote_nodestream_test.ml     |  12 +-
 src/main/arakoon.ml                      |   2 +-
 src/msg/tcp_messaging_test.ml            |   2 +-
 src/node/node_cfg.ml                     |  48 +-------
 src/nursery/nursery.ml                   |   2 +-
 src/nursery/nursery_main.ml              |   2 +-
 src/paxos/multi_paxos_test.ml            |   8 +-
 src/system/drop_master.ml                |  13 ++-
 src/system/single.ml                     |  52 +++------
 src/system/startup.ml                    |  17 +--
 src/tools/server_test.ml                 |   4 +-
 src/tools/tcp_keepalive.ml               |   7 ++
 16 files changed, 266 insertions(+), 177 deletions(-)

diff --git a/src/client/arakoon_client_config.ml b/src/client/arakoon_client_config.ml
index d49d5c58..9eb6b591 100644
--- a/src/client/arakoon_client_config.ml
+++ b/src/client/arakoon_client_config.ml
@@ -1,5 +1,23 @@
 type cluster_id = string
 
+type tcp_keepalive_cfg = Tcp_keepalive.t = {
+      enable_tcp_keepalive : bool;
+      tcp_keepalive_time : int;
+      tcp_keepalive_intvl : int;
+      tcp_keepalive_probes : int;
+    }
+
+let show_tcp_keepalive_cfg { enable_tcp_keepalive;
+                             tcp_keepalive_time;
+                             tcp_keepalive_intvl;
+                             tcp_keepalive_probes; } =
+  let open To_string in
+  record
+    [ "enable_tcp_keepalive", bool enable_tcp_keepalive;
+      "tcp_keepalive_time", int tcp_keepalive_time;
+      "tcp_keepalive_intvl", int tcp_keepalive_intvl;
+      "tcp_keepalive_probes", int tcp_keepalive_probes; ]
+
 type node_cfg = {
   ips : string list;
   port : int;
@@ -14,8 +32,131 @@ type ssl_cfg = {
   protocol : Ssl.protocol;
 }
 
+let show_ssl_cfg { ca_cert; creds; protocol; } =
+  Printf.sprintf "{ ca_cert=%s; creds=%s; protocol=%s; }"
+                 ca_cert
+                 (To_string.option
+                    (To_string.pair Std.id Std.id)
+                    creds)
+                 (let open Ssl in
+                  match protocol with
+                  | SSLv23 -> "SSLv23"
+                  | SSLv3 -> "SSLv3"
+                  | TLSv1 -> "TLSv1"
+                  | TLSv1_1 -> "TLSv1_1"
+                  | TLSv1_2 -> "TLSv1_2"
+                 )
+
 type t = {
   cluster_id : cluster_id;
   node_cfgs : (string * node_cfg) list;
   ssl_cfg : ssl_cfg option;
+  tcp_keepalive : tcp_keepalive_cfg;
 }
+
+let show { cluster_id; node_cfgs; ssl_cfg; } =
+  Printf.sprintf
+    "(cluster_id=%s, node_cfgs=%s, ssl_cfg=%s)"
+    cluster_id
+    (String.concat
+       "; "
+       (List.map
+          (fun (node_name, node_cfg) ->
+           Printf.sprintf "%s=%s" node_name (show_node_cfg node_cfg))
+          node_cfgs))
+    (To_string.option show_ssl_cfg ssl_cfg)
+
+let get_node_cfg ({ node_cfgs; _ } as ccfg) node_name =
+  try
+    List.filter
+      (fun (node_name', _cfg) -> node_name' = node_name)
+      node_cfgs
+    |> List.hd
+    |> snd
+  with exn ->
+    failwith (node_name ^ " is not known in config " ^ (show ccfg))
+
+let tcp_keepalive_from_ini inifile =
+  let enable_tcp_keepalive =
+    Ini.get
+      inifile "global" "enable_tcp_keepalive"
+      Ini.p_bool (Ini.default true)
+  in
+  let tcp_keepalive_time =
+    Ini.get
+      inifile "global" "tcp_keepalive_time"
+      Ini.p_int (Ini.default 20)
+  in
+  let tcp_keepalive_intvl =
+    Ini.get
+      inifile "global" "tcp_keepalive_intvl"
+      Ini.p_int (Ini.default 20)
+  in
+  let tcp_keepalive_probes =
+    Ini.get
+      inifile "global" "tcp_keepalive_probes"
+      Ini.p_int (Ini.default 3)
+  in
+  { enable_tcp_keepalive;
+    tcp_keepalive_time;
+    tcp_keepalive_intvl;
+    tcp_keepalive_probes;
+  }
+
+let from_ini txt =
+  let inifile = new Inifiles.inifile txt in
+  let cluster_id = Ini.get inifile "global" "cluster_id" Ini.p_string Ini.required in
+  let nodes = Ini.get inifile "global" "cluster" Ini.p_string_list Ini.required in
+  let node_cfgs =
+    List.map
+      (fun node_name ->
+       let ips = Ini.get inifile node_name "ips" Ini.p_string_list Ini.required in
+       let port = Ini.get inifile node_name "client_port" Ini.p_int Ini.required in
+       (node_name, { ips; port; }))
+      nodes
+  in
+  let tls_ca_cert = Ini.get inifile
+                            "global" "tls_ca_cert"
+                            (Ini.p_option Ini.p_string)
+                            (Ini.default None)
+  in
+  let tls_cert = Ini.get inifile
+                         "global" "tls_cert"
+                         (Ini.p_option Ini.p_string)
+                         (Ini.default None)
+  in
+  let tls_version = Ini.get inifile
+                            "global" "tls_version"
+                            (Ini.p_option Ini.p_string)
+                            (Ini.default None)
+  in
+  let tls_key = Ini.get inifile
+                        "global" "tls_key"
+                        (Ini.p_option Ini.p_string)
+                        (Ini.default None)
+  in
+  let ssl_cfg =
+    match tls_ca_cert, tls_version with
+    | None, _
+    | _, None -> None
+    | Some ca_cert, Some tls_version ->
+       let protocol = match tls_version with
+         | "1.0" -> Ssl.TLSv1
+         | "1.1" -> Ssl.TLSv1_1
+         | "1.2" -> Ssl.TLSv1_2
+         | _ -> failwith "unsuppored tls protocol version"
+       in
+       let creds = match tls_cert, tls_key with
+         | None, _
+         | _, None -> None
+         | Some s, Some k -> Some (s, k)
+       in
+       Some { ca_cert;
+              creds;
+              protocol; }
+  in
+  { cluster_id;
+    node_cfgs;
+    ssl_cfg;
+    tcp_keepalive = tcp_keepalive_from_ini inifile;
+  }
diff --git a/src/client/arakoon_remote_client_test.ml b/src/client/arakoon_remote_client_test.ml
index 26d55324..0aa44244 100644
--- a/src/client/arakoon_remote_client_test.ml
+++ b/src/client/arakoon_remote_client_test.ml
@@ -61,22 +61,15 @@ let __client_server_wrapper__ (real_test:real_test) =
     let sp = float(lease_period) *. 0.5 in
     Lwt_unix.sleep sp >>= fun () -> (* let the cluster reach stability *)
     Logger.info_ "cluster should have reached stability" >>= fun () ->
-    Client_main.find_master ~tls:None cluster_cfg >>= fun master_name ->
+    Client_main.find_master cluster_cfg >>= fun master_name ->
     Logger.info_f_ "master=%S" master_name >>= fun () ->
-    let master_cfg =
-      List.hd
-        (List.filter (fun cfg -> cfg.Node_cfg.node_name = master_name) cluster_cfg.Node_cfg.cfgs)
-    in
-    Client_main.with_client
-      ~tls:None ~tcp_keepalive:default_tcp_keepalive
-      master_cfg
-      cluster_cfg.Node_cfg.cluster_id
-      real_test
+    Client_helper.with_client'' cluster_cfg master_name real_test
   in
   let main () =
     make_config () >>= fun cluster_cfg ->
     Lwt.pick [(t0 >>= fun () -> Lwt.return false);
-              (client_t cluster_cfg >>= fun () -> Lwt.return true);] >>= fun succeeded ->
+              (client_t (Node_cfg.to_client_cfg cluster_cfg) >>= fun () ->
+               Lwt.return true);] >>= fun succeeded ->
     stop := true;
     OUnit.assert_bool "client thread should finish before server" succeeded;
     Lwt.return ()
diff --git a/src/client/client_helper.ml b/src/client/client_helper.ml
index bda92523..0066bb48 100644
--- a/src/client/client_helper.ml
+++ b/src/client/client_helper.ml
@@ -35,6 +35,18 @@ let default_create_client_context ~ca_cert ~creds ~protocol =
   ctx
 
 
+let get_tls_from_ssl_cfg (ssl_cfg : Arakoon_client_config.ssl_cfg option) =
+  let open Arakoon_client_config in
+  match ssl_cfg with
+  | None -> None
+  | Some ssl ->
+     let ca_cert = ssl.ca_cert in
+     let protocol = ssl.protocol in
+     let creds = ssl.creds in
+     let ctx = default_create_client_context ~ca_cert ~creds ~protocol in
+     Some ctx
+
+
 let with_connection ~tls ~tcp_keepalive sa do_it =
   let fd = Lwt_unix.socket (Unix.domain_of_sockaddr sa) Unix.SOCK_STREAM 0 in
   Lwt.catch
@@ -172,9 +184,14 @@ end
     applicable) exceptions, except {! Lwt.Canceled } which is passed through
     (you most likely don't want to catch that anyway).
 *)
-let find_master' ~tls ~tcp_keepalive cluster_cfg =
+let find_master' ?tls cluster_cfg =
   let open Arakoon_client_config in
 
+  let tls = match tls with
+    | None -> get_tls_from_ssl_cfg cluster_cfg.ssl_cfg
+    | Some x -> Some x
+  in
+
   let lookup_cfg n =
     let rec loop = function
       | [] -> None
@@ -211,7 +228,7 @@ let find_master' ~tls ~tcp_keepalive cluster_cfg =
                 Lwt.return None);
                with_client'
                  ~tls
-                 ~tcp_keepalive
+                 ~tcp_keepalive:cluster_cfg.tcp_keepalive
                  cfg cluster_cfg.cluster_id
                  (fun client ->
                   client # who_master ()); ]
@@ -268,10 +285,10 @@ let find_master' ~tls ~tcp_keepalive cluster_cfg =
     In some circumstances, this action could loop forever, so it's mostly
     useful in combination with {! Lwt_unix.with_timeout } or something related.
 *)
-let find_master_loop ~tls ~tcp_keepalive cluster_cfg =
+let find_master_loop cluster_cfg =
   let open MasterLookupResult in
   let rec loop () =
-    find_master' ~tls ~tcp_keepalive cluster_cfg >>= fun r -> match r with
+    find_master' cluster_cfg >>= fun r -> match r with
       | No_master
       | Too_many_nodes_down -> begin
           Logger.debug_f_ "Client_main.find_master_loop: %s" (to_string r) >>=
@@ -283,10 +300,21 @@ let find_master_loop ~tls ~tcp_keepalive cluster_cfg =
   in
   loop ()
 
-let with_master_client' ~tls ~tcp_keepalive cluster_cfg f =
+let with_client'' ?tls ccfg node_name f =
+  let open Arakoon_client_config in
+  with_client'
+    ~tls:(match tls with
+          | None -> get_tls_from_ssl_cfg ccfg.ssl_cfg
+          | Some x -> Some x)
+    ~tcp_keepalive:ccfg.tcp_keepalive
+    (get_node_cfg ccfg node_name)
+    ccfg.cluster_id
+    f
+
+let with_master_client' cluster_cfg f =
   let open MasterLookupResult in
-  find_master' ~tls ~tcp_keepalive cluster_cfg >>= function
+  find_master' cluster_cfg >>= function
   | Found (master_name, master_cfg) ->
-    with_client' ~tls ~tcp_keepalive master_cfg cluster_cfg.Arakoon_client_config.cluster_id f
+     with_client'' cluster_cfg master_name f
   | master_result ->
     Lwt.fail (Error master_result)
diff --git a/src/client/client_main.ml b/src/client/client_main.ml
index 5859f39c..ed994d16 100644
--- a/src/client/client_main.ml
+++ b/src/client/client_main.ml
@@ -24,8 +24,6 @@ let default_create_client_context = default_create_client_context
 let with_connection = with_connection
 let with_connection' = with_connection'
 
-let with_client ~tls ~tcp_keepalive node_cfg cluster_id f =
-  with_client' ~tls ~tcp_keepalive (Node_cfg.node_cfg_to_node_client_cfg node_cfg) cluster_id f
 
 let with_remote_nodestream ~tls ~tcp_keepalive node_cfg cluster_id f =
   let open Node_cfg in
@@ -52,26 +50,22 @@ let ping ~tls ~tcp_keepalive ip port cluster_id =
   run t
 
 
-
-
-let find_master ~tls cluster_cfg =
-  let cluster_cfg' = Node_cfg.to_client_cfg cluster_cfg in
-  let tcp_keepalive = cluster_cfg.Node_cfg.tcp_keepalive in
+let find_master ?tls cluster_cfg =
   let open MasterLookupResult in
-  find_master' ~tls ~tcp_keepalive cluster_cfg' >>= function
+  find_master' ?tls cluster_cfg >>= function
     | Found (node_name, _) -> return node_name
     | No_master -> Lwt.fail (Failure "No Master")
     | Too_many_nodes_down -> Lwt.fail (Failure "too many nodes down")
     | Unknown_node (n, _) -> return n (* Keep original behaviour *)
     | Exception exn -> Lwt.fail exn
 
+let retrieve_cfg cfg_url =
+  Arakoon_config_url.retrieve cfg_url >|= Arakoon_client_config.from_ini
 
 let with_master_client ~tls cfg_url f =
-  let open Node_cfg in
   retrieve_cfg cfg_url >>= fun ccfg ->
-  find_master ~tls ccfg >>= fun master_name ->
-  let master_cfg = List.hd (List.filter (fun cfg -> cfg.node_name = master_name) ccfg.cfgs) in
-  with_client ~tls ~tcp_keepalive:ccfg.tcp_keepalive master_cfg ccfg.cluster_id f
+  find_master ?tls ccfg >>= fun master_name ->
+  with_client'' ?tls ccfg master_name f
 
 let set ~tls cfg_name key value =
   let t () = with_master_client ~tls cfg_name (fun client -> client # set key value)
@@ -203,54 +197,36 @@ let statistics ~tls cfg_name =
 let who_master ~tls cfg_url () =
 
   let t () =
-    Node_cfg.retrieve_cfg cfg_url >>= fun cluster_cfg ->
-    find_master ~tls cluster_cfg >>= fun master_name ->
+    retrieve_cfg cfg_url >>= fun cluster_cfg ->
+    find_master ?tls cluster_cfg >>= fun master_name ->
     Lwt_io.printl master_name
   in
   run t
 
-let _cluster_and_node_cfg node_name' cfg_url =
-  Node_cfg.retrieve_cfg cfg_url >>= fun cluster_cfg ->
-  let _find cfgs =
-    let rec loop = function
-      | [] -> failwith (node_name' ^ " is not known in config " ^ (Arakoon_config_url.to_string cfg_url))
-      | cfg :: rest ->
-        if cfg.Node_cfg.node_name = node_name' then cfg
-        else loop rest
-    in
-    loop cfgs
-  in
-  let node_cfg = _find cluster_cfg.Node_cfg.cfgs in
-  Lwt.return (cluster_cfg, node_cfg)
-
-let node_state ~tls node_name' cfg_name =
-  let f client =
-    client # current_state () >>= fun state ->
-    Lwt_io.printl state
-  in
-
+let node_state ~tls node_name cfg_url =
   let t () =
-    _cluster_and_node_cfg node_name' cfg_name
-    >>= fun (cluster_cfg, node_cfg) ->
-    let open Node_cfg in
-    let cluster = cluster_cfg.cluster_id in
-    let tcp_keepalive = cluster_cfg.tcp_keepalive in
-    with_client ~tls ~tcp_keepalive node_cfg cluster f in
+    retrieve_cfg cfg_url >>= fun ccfg ->
+    with_client''
+      ?tls
+      ccfg node_name
+      (fun client ->
+       client # current_state () >>= fun state ->
+       Lwt_io.printl state
+      )
+  in
   run t
 
 
-
-let node_version ~tls node_name' cfg_name =
-  let open Node_cfg in
+let node_version ~tls node_name cfg_url =
   let t () =
-    _cluster_and_node_cfg node_name' cfg_name >>= fun (cluster_cfg, node_cfg) ->
-    let tcp_keepalive = cluster_cfg.tcp_keepalive in
-    let cluster = cluster_cfg.cluster_id in
-    with_client ~tls ~tcp_keepalive node_cfg cluster
+    retrieve_cfg cfg_url >>= fun ccfg ->
+    with_client''
+      ?tls
+      ccfg node_name
       (fun client ->
-         client # version () >>= fun (major,minor,patch, info) ->
-         Lwt_io.printlf "%i.%i.%i" major minor patch >>= fun () ->
-         Lwt_io.printl info
+       client # version () >>= fun (major,minor,patch, info) ->
+       Lwt_io.printlf "%i.%i.%i" major minor patch >>= fun () ->
+       Lwt_io.printl info
       )
   in
   run t
diff --git a/src/client/remote_nodestream_test.ml b/src/client/remote_nodestream_test.ml
index 2f5b8e46..319bac9d 100644
--- a/src/client/remote_nodestream_test.ml
+++ b/src/client/remote_nodestream_test.ml
@@ -48,13 +48,11 @@ let __wrap__ port conversation =
     let sp = float(lease_period) *. 0.5 in
     Lwt_unix.sleep sp >>= fun () -> (* let the cluster reach stability *)
     Logger.info_ "cluster should have reached stability" >>= fun () ->
-    Client_main.find_master ~tls:None cluster_cfg >>= fun master_name ->
+    Client_main.find_master cluster_cfg >>= fun master_name ->
     Logger.info_f_ "master=%S" master_name >>= fun () ->
-    let master_cfg =
-      List.hd
-        (List.filter (fun cfg -> cfg.Node_cfg.node_name = master_name) cluster_cfg.Node_cfg.cfgs)
-    in
-    let address = Unix.ADDR_INET (Unix.inet_addr_loopback, master_cfg.Node_cfg.client_port) in
+    let master_cfg = Arakoon_client_config.get_node_cfg cluster_cfg master_name in
+    let address = Unix.ADDR_INET (Unix.inet_addr_loopback,
+                                  master_cfg.Arakoon_client_config.port) in
     Lwt_io.open_connection address >>= fun (ic,oc) ->
     conversation (ic, oc) >>= fun () ->
     Logger.debug section "end_of_senario" >>= fun () ->
@@ -63,7 +61,7 @@ let __wrap__ port conversation =
     Lwt.return ()
   in
   make_config () >>= fun cluster_cfg ->
-  Lwt.pick [client_t cluster_cfg;t0] >>= fun () ->
+  Lwt.pick [ client_t (Node_cfg.to_client_cfg cluster_cfg); t0; ] >>= fun () ->
   stop := true;
   Lwt.return ()
 
diff --git a/src/main/arakoon.ml b/src/main/arakoon.ml
index d53c4920..74356023 100644
--- a/src/main/arakoon.ml
+++ b/src/main/arakoon.ml
@@ -432,7 +432,7 @@ let main () =
   let interface = actions @ options in
 
   let make_config () = Node_cfg.retrieve_cfg !config_url in
-  let tcp_keepalive = default_tcp_keepalive in
+  let tcp_keepalive = Tcp_keepalive.default_tcp_keepalive in
 
   let do_local ~tls = function
     | ShowUsage -> print_endline usage;0
diff --git a/src/msg/tcp_messaging_test.ml b/src/msg/tcp_messaging_test.ml
index 06c9d3b9..62dbba65 100644
--- a/src/msg/tcp_messaging_test.ml
+++ b/src/msg/tcp_messaging_test.ml
@@ -97,7 +97,7 @@ let make_transport addresses =
   let tcp_transport =
     new tcp_messaging
         addresses "yummy" (fun _ _ _-> false) Node_cfg.default_max_buffer_size ~stop:(ref false)
-        ~tcp_keepalive:Node_cfg.default_tcp_keepalive
+        ~tcp_keepalive:Tcp_keepalive.default_tcp_keepalive
   in
   (tcp_transport :> messaging)
 
diff --git a/src/node/node_cfg.ml b/src/node/node_cfg.ml
index 2cd7c3c8..a039255d 100644
--- a/src/node/node_cfg.ml
+++ b/src/node/node_cfg.ml
@@ -28,20 +28,6 @@ let default_lcnum = 16384
 let default_ncnum = 8192
 let default_head_copy_throttling = 0.0
 
-type tcp_keepalive_cfg = Tcp_keepalive.t = {
-      enable_tcp_keepalive : bool;
-      tcp_keepalive_time : int;
-      tcp_keepalive_intvl : int;
-      tcp_keepalive_probes : int;
-    }
-
-let default_tcp_keepalive = {
-    enable_tcp_keepalive = true;
-    tcp_keepalive_time = 20;
-    tcp_keepalive_intvl = 20;
-    tcp_keepalive_probes = 3;
-  }
-
 open Master_type
 open Client_cfg
 
@@ -251,7 +237,7 @@ module Node_cfg = struct
       lcnum : int; (* tokyo cabinet: leaf nodes in cache *)
       ncnum : int; (* tokyo cabinet: internal nodes in cache *)
       tls : TLSConfig.Cluster.t option;
-      tcp_keepalive : tcp_keepalive_cfg;
+      tcp_keepalive : Arakoon_client_config.tcp_keepalive_cfg;
     }
 
   let node_cfg_to_node_client_cfg (cfg : t) =
@@ -266,6 +252,7 @@ module Node_cfg = struct
           (fun (cfg : t) -> cfg.node_name, node_cfg_to_node_client_cfg cfg)
           t.cfgs;
       Arakoon_client_config.ssl_cfg = None;
+      tcp_keepalive = t.tcp_keepalive;
     }
 
   let string_of_cluster_cfg c =
@@ -352,7 +339,7 @@ module Node_cfg = struct
       lcnum = default_lcnum;
       ncnum = default_ncnum;
       tls = None;
-      tcp_keepalive = default_tcp_keepalive;
+      tcp_keepalive = Tcp_keepalive.default_tcp_keepalive;
     }
     in
     cluster_cfg
@@ -729,26 +716,6 @@ module Node_cfg = struct
            in
            Some cfg
       in
-      let enable_tcp_keepalive =
-        Ini.get
-          inifile "global" "enable_tcp_keepalive"
-          Ini.p_bool (Ini.default true)
-      in
-      let tcp_keepalive_time =
-        Ini.get
-          inifile "global" "tcp_keepalive_time"
-          Ini.p_int (Ini.default 20)
-      in
-      let tcp_keepalive_intvl =
-        Ini.get
-          inifile "global" "tcp_keepalive_intvl"
-          Ini.p_int (Ini.default 20)
-      in
-      let tcp_keepalive_probes =
-        Ini.get
-          inifile "global" "tcp_keepalive_probes"
-          Ini.p_int (Ini.default 3)
-      in
       let cluster_cfg =
         { cfgs;
           log_cfgs;
@@ -767,12 +734,7 @@ module Node_cfg = struct
           lcnum;
           ncnum;
           tls;
-          tcp_keepalive = {
-              enable_tcp_keepalive;
-              tcp_keepalive_time;
-              tcp_keepalive_intvl;
-              tcp_keepalive_probes;
-            };
+          tcp_keepalive = Arakoon_client_config.tcp_keepalive_from_ini inifile;
         }
       in
       cluster_cfg
@@ -787,8 +749,6 @@ module Node_cfg = struct
 
   let retrieve_cfg url = Arakoon_config_url.retrieve url >|= _retrieve_cfg_from_txt
 
-  let retrieve_client_cfg url = retrieve_cfg url >>= fun cfg -> Lwt.return (to_client_cfg cfg)
-
   let test ccfg ~cluster_id = ccfg.cluster_id = cluster_id
 
   open Lwt
diff --git a/src/nursery/nursery.ml b/src/nursery/nursery.ml
index 36446e2e..1de42ee3 100644
--- a/src/nursery/nursery.ml
+++ b/src/nursery/nursery.ml
@@ -443,7 +443,7 @@ let nursery_test_main () =
   let () = NCFG.add_cluster nursery_cfg "left" left_cfg in
   let () = NCFG.add_cluster nursery_cfg "right" right_cfg in
   let keeper = "left" in
-  let nc = NC.make nursery_cfg keeper Node_cfg.default_tcp_keepalive in
+  let nc = NC.make nursery_cfg keeper Tcp_keepalive.default_tcp_keepalive in
   (*
   let test k v =
     NC.set client k v >>= fun () ->
diff --git a/src/nursery/nursery_main.ml b/src/nursery/nursery_main.ml
index 8e22d177..9132e680 100644
--- a/src/nursery/nursery_main.ml
+++ b/src/nursery/nursery_main.ml
@@ -99,7 +99,7 @@ let get_keeper_config cfg_url =
 let get_nursery_client keeper_id cli_cfg =
   let get_nc client =
     client # get_nursery_cfg () >>= fun ncfg ->
-    Lwt.return ( Nursery.NC.make ncfg keeper_id default_tcp_keepalive)
+    Lwt.return ( Nursery.NC.make ncfg keeper_id Tcp_keepalive.default_tcp_keepalive)
   in
   with_master_remote_stream keeper_id cli_cfg get_nc
 
diff --git a/src/paxos/multi_paxos_test.ml b/src/paxos/multi_paxos_test.ml
index 8f5d5e01..cd20e905 100644
--- a/src/paxos/multi_paxos_test.ml
+++ b/src/paxos/multi_paxos_test.ml
@@ -86,7 +86,7 @@ let test_generic network_factory n_nodes () =
               lease_expiration_id = 0;
               respect_run_master = None;
               catchup_tls_ctx = None;
-              tcp_keepalive = Node_cfg.default_tcp_keepalive;
+              tcp_keepalive = Tcp_keepalive.default_tcp_keepalive;
              }
   in
   let all_happy = build_names (n_nodes -1) in
@@ -284,7 +284,7 @@ let test_master_loop network_factory ()  =
                    lease_expiration_id = 0;
                    respect_run_master = None;
                    catchup_tls_ctx = None;
-                   tcp_keepalive = Node_cfg.default_tcp_keepalive;
+                   tcp_keepalive = Tcp_keepalive.default_tcp_keepalive;
                   } in
   let continue = ref 2 in
   let c0_t () =
@@ -352,7 +352,7 @@ let build_tcp () =
   let (m : messaging) =
     new Tcp_messaging.tcp_messaging (["127.0.0.1"], 7777) "yummie"
         (fun _ _ _ -> false) Node_cfg.default_max_buffer_size ~stop:(ref false)
-        ~tcp_keepalive:Node_cfg.default_tcp_keepalive
+        ~tcp_keepalive:Tcp_keepalive.default_tcp_keepalive
   in
   let network = network_of_messaging m in
   m # get_buffer, network
@@ -424,7 +424,7 @@ let test_simulation filters () =
     lease_expiration_id = 0;
     respect_run_master = None;
     catchup_tls_ctx = None;
-    tcp_keepalive = Node_cfg.default_tcp_keepalive;
+    tcp_keepalive = Tcp_keepalive.default_tcp_keepalive;
   } in
   let c0_t () =
     let expected prev_key key =
diff --git a/src/system/drop_master.ml b/src/system/drop_master.ml
index d5e35787..45ef313c 100644
--- a/src/system/drop_master.ml
+++ b/src/system/drop_master.ml
@@ -39,7 +39,7 @@ let setup tn master base () =
 let find_master cluster_cfg =
   Lwt.catch
     (fun () ->
-      Client_main.find_master ~tls:None cluster_cfg >>= fun m ->
+      Client_main.find_master cluster_cfg >>= fun m ->
       Lwt.return (Some m))
     (function
     | Failure "too many nodes down"
@@ -71,10 +71,11 @@ let _drop_master do_maintenance
                  (_tn, (make_cluster_cfg: unit -> cluster_cfg Lwt.t), _)
   =
   make_cluster_cfg () >>= fun cluster_cfg ->
+  let client_cluster_cfg = to_client_cfg cluster_cfg in
   let lease_period = cluster_cfg._lease_period in
   let sp = (float lease_period) *. 1.2 in
   Lwt_unix.sleep sp >>= fun () -> (* let the cluster reach stability *)
-  Client_main.find_master ~tls:None cluster_cfg >>= fun master_name ->
+  Client_main.find_master client_cluster_cfg >>= fun master_name ->
   if do_maintenance
   then
     begin
@@ -108,13 +109,13 @@ let _drop_master do_maintenance
          if not do_maintenance
          then
            begin
-             Client_main.find_master ~tls:None cluster_cfg >>= fun new_master ->
+             Client_main.find_master client_cluster_cfg >>= fun new_master ->
              Logger.info_f_ "new? master = %s" new_master >>= fun () ->
              OUnit.assert_bool "master should have been changed" (new_master <> master_name);
              Lwt.return ()
            end
          else
-           wait_until_master cluster_cfg >>= fun new_master ->
+           wait_until_master client_cluster_cfg >>= fun new_master ->
            Logger.info_f_ "new? master = %s" new_master >>= fun () ->
            OUnit.assert_bool "master should be the same" (new_master = master_name);
            Lwt.return ()));
@@ -126,9 +127,9 @@ let _drop_master do_maintenance
   then
     Lwt_condition.signal Mem_store.defrag_condition ();
   Lwt_unix.sleep (float lease_period) >>= fun () ->
-  find_master cluster_cfg >>= fun _ ->
+  find_master client_cluster_cfg >>= fun _ ->
   Lwt_unix.sleep (float lease_period) >>= fun () ->
-  find_master cluster_cfg >>= fun _ ->
+  find_master client_cluster_cfg >>= fun _ ->
   Lwt.return ()
 
 let drop_master                   tpl = _drop_master false tpl
diff --git a/src/system/single.ml b/src/system/single.ml
index 832f8986..682e3e5a 100644
--- a/src/system/single.ml
+++ b/src/system/single.ml
@@ -34,14 +34,9 @@ let all_same_master (_tn, make_cluster_cfg, all_t) =
     Lwt_unix.sleep q >>= fun () ->
     Logger.debug_ "start of scenario" >>= fun () ->
     let set_one client = client # set "key" "value" in
-    Client_main.find_master ~tls:None cluster_cfg >>= fun master_name ->
-    let master_cfg = List.hd (List.filter (fun cfg -> cfg.node_name = master_name)
-                                cluster_cfg.cfgs)
-    in
-
-    Client_main.with_client
-      ~tls:None ~tcp_keepalive:Node_cfg.default_tcp_keepalive
-      master_cfg cluster_cfg.cluster_id set_one >>= fun () ->
+    Client_helper.with_master_client'
+      (to_client_cfg cluster_cfg)
+      set_one >>= fun () ->
     let masters = ref [] in
     let do_one cfg =
       let nn = node_name cfg in
@@ -51,9 +46,7 @@ let all_same_master (_tn, make_cluster_cfg, all_t) =
           masters := master :: !masters;
           Logger.info_f_ "Client:%s got: %s" nn (Log_extra.string_option2s master)
       in
-      Client_main.with_client
-        ~tls:None ~tcp_keepalive:Node_cfg.default_tcp_keepalive
-        cfg cluster_cfg.cluster_id f
+      Client_helper.with_client'' (to_client_cfg cluster_cfg) nn f
     in
     let cfgs = cluster_cfg.cfgs in
     Lwt_list.iter_s do_one cfgs >>= fun () ->
@@ -84,7 +77,7 @@ let nothing_on_slave (_tn, make_cluster_cfg, all_t) =
   make_cluster_cfg () >>= fun cluster_cfg ->
   let cfgs = cluster_cfg.cfgs in
   let find_slaves cfgs =
-    Client_main.find_master ~tls:None cluster_cfg >>= fun m ->
+    Client_main.find_master (to_client_cfg cluster_cfg) >>= fun m ->
     let slave_cfgs = List.filter (fun cfg -> cfg.node_name <> m) cfgs in
     Lwt.return slave_cfgs
   in
@@ -117,9 +110,9 @@ let nothing_on_slave (_tn, make_cluster_cfg, all_t) =
       with_client delete_on_slave >>= fun () ->
       with_client test_and_set_on_slave
     in
-    f (fun client_action -> Client_main.with_client
-                              ~tls:None ~tcp_keepalive:Node_cfg.default_tcp_keepalive
-                              cfg cluster_id client_action)
+    f (fun client_action -> Client_helper.with_client''
+                              (to_client_cfg cluster_cfg) cfg.node_name
+                              client_action)
   in
   let test_slaves ccfg =
     find_slaves cfgs >>= fun slave_cfgs ->
@@ -134,16 +127,13 @@ let nothing_on_slave (_tn, make_cluster_cfg, all_t) =
 
 let dirty_on_slave (_tn, make_cluster_cfg,_) =
   make_cluster_cfg () >>= fun cluster_cfg ->
+  let client_cluster_cfg = to_client_cfg cluster_cfg in
   Lwt_unix.sleep (float (cluster_cfg._lease_period)) >>= fun () ->
   Logger.debug_ "dirty_on_slave" >>= fun () ->
   let cfgs = cluster_cfg.cfgs in
-  Client_main.find_master ~tls:None cluster_cfg >>= fun master_name ->
-  let master_cfg = List.hd (List.filter (fun cfg -> cfg.node_name = master_name)
-                              cluster_cfg.cfgs)
-  in
-  Client_main.with_client
-    ~tls:None ~tcp_keepalive:Node_cfg.default_tcp_keepalive
-    master_cfg cluster_cfg.cluster_id
+  Client_main.find_master client_cluster_cfg >>= fun master_name ->
+  Client_helper.with_master_client'
+    client_cluster_cfg
     (fun client -> client # set "xxx" "xxx")
   >>= fun () ->
 
@@ -162,8 +152,9 @@ let dirty_on_slave (_tn, make_cluster_cfg,_) =
         "dirty get should fail with not found"
         "dirty get failed with not found as intended"
     in
-    Client_main.with_client ~tls:None ~tcp_keepalive:Node_cfg.default_tcp_keepalive
-                            cfg cluster_id dirty_get
+    Client_helper.with_client''
+      client_cluster_cfg cfg.node_name
+      dirty_get
   in
   let do_slaves ccfg =
     find_slaves cfgs >>= fun slave_cfgs ->
@@ -567,8 +558,6 @@ let find_master ~tls cluster_cfg =
   let go () =
     let open Client_helper.MasterLookupResult in
     Client_helper.find_master_loop
-      ~tls
-      ~tcp_keepalive:Node_cfg.default_tcp_keepalive
       (Node_cfg.Node_cfg.to_client_cfg cluster_cfg) >>= function
       | Found (name, cfg) -> return name
       | No_master -> Lwt.fail (Failure "No Master")
@@ -585,14 +574,9 @@ let _with_master ((_tn:string), make_cluster_cfg, _) f =
   Logger.info_ "cluster should have reached stability" >>= fun () ->
   find_master ~tls:None cluster_cfg >>= fun master_name ->
   Logger.info_f_ "master=%S" master_name >>= fun () ->
-  let master_cfg =
-    List.hd
-      (List.filter (fun cfg -> cfg.node_name = master_name) cluster_cfg.cfgs)
-  in
-  Client_main.with_client
-    ~tls:None
-    ~tcp_keepalive:Node_cfg.default_tcp_keepalive
-    master_cfg cluster_cfg.cluster_id f
+  Client_helper.with_master_client'
+    (to_client_cfg cluster_cfg)
+    f
 
 
 let trivial_master tpl =
diff --git a/src/system/startup.ml b/src/system/startup.ml
index af9b247b..3d232fe7 100644
--- a/src/system/startup.ml
+++ b/src/system/startup.ml
@@ -171,7 +171,7 @@ let post_failure () =
     lcnum = 8192;
     ncnum = 4096;
     tls = None;
-    tcp_keepalive = Node_cfg.default_tcp_keepalive;
+    tcp_keepalive = Tcp_keepalive.default_tcp_keepalive;
   }
   in
   let get_cfgs () = cluster_cfg |> Lwt.return in
@@ -233,7 +233,7 @@ let restart_slaves () =
      lcnum = Node_cfg.default_lcnum;
      ncnum = Node_cfg.default_ncnum;
      tls = None;
-     tcp_keepalive = Node_cfg.default_tcp_keepalive;
+     tcp_keepalive = Tcp_keepalive.default_tcp_keepalive;
     }
   in
   let get_cfgs () = cluster_cfg |> Lwt.return in
@@ -293,7 +293,7 @@ let ahead_master_loses_role () =
      lcnum = 8192;
      ncnum = 4096;
      tls = None;
-     tcp_keepalive = Node_cfg.default_tcp_keepalive;
+     tcp_keepalive = Tcp_keepalive.default_tcp_keepalive;
     }
   in
   let get_cfgs () = cluster_cfg |> Lwt.return in
@@ -315,7 +315,7 @@ let ahead_master_loses_role () =
   (* sleep a bit so the previous 2 slaves can make progress *)
   Lwt_unix.sleep ((float lease_period) *. 3.0) >>= fun () ->
   Lwt.catch
-    (fun () -> Client_main.find_master cluster_cfg ~tls:None >>= fun master ->
+    (fun () -> Client_main.find_master (to_client_cfg cluster_cfg) >>= fun master ->
                Lwt.return_unit)
     (fun exn ->
       Logger.fatal_f_ ~exn "NO MASTER" >>= fun () ->
@@ -377,7 +377,7 @@ let interrupted_election () =
      lcnum = 8192;
      ncnum = 4096;
      tls = None;
-     tcp_keepalive = Node_cfg.default_tcp_keepalive;
+     tcp_keepalive = Tcp_keepalive.default_tcp_keepalive;
     }
   in
   let get_cfgs () = cluster_cfg |> Lwt.return in
@@ -394,8 +394,9 @@ let interrupted_election () =
      and a timeout on all this
    *)
   let get_master_from cfg =
-    Client_main.with_client
-      cfg ~tls:None ~tcp_keepalive:Node_cfg.default_tcp_keepalive
+    Client_helper.with_client'
+      (node_cfg_to_node_client_cfg cfg)
+      ~tls:None ~tcp_keepalive:Tcp_keepalive.default_tcp_keepalive
       cluster_id
       (fun client -> client # who_master ())
   in
@@ -432,7 +433,7 @@ let interrupted_election () =
   let rec phase2 () =
     Lwt.catch
       (fun () ->
-       Client_main.find_master ~tls:None cluster_cfg >>= fun m ->
+       Client_main.find_master (to_client_cfg cluster_cfg) >>= fun m ->
        if m <> wannabe_master
        then Lwt.return true
        else Lwt.return false)
diff --git a/src/tools/server_test.ml b/src/tools/server_test.ml
index aa7cccaa..ddb5b611 100644
--- a/src/tools/server_test.ml
+++ b/src/tools/server_test.ml
@@ -46,7 +46,7 @@ let test_echo () =
                  ~setup_callback host port echo_protocol
                  ~teardown_callback
                  ~scheme ~stop
-                 ~tcp_keepalive:Node_cfg.default_tcp_keepalive
+                 ~tcp_keepalive:Tcp_keepalive.default_tcp_keepalive
   in
   let client () =
     Logger.debug_ "sleeping until server socket started" >>= fun () ->
@@ -100,7 +100,7 @@ let test_max_connections () =
   let server = Server.make_server_thread ~scheme
                  ~setup_callback ~teardown_callback
                  host port echo_protocol2 ~stop
-                 ~tcp_keepalive:Node_cfg.default_tcp_keepalive
+                 ~tcp_keepalive:Tcp_keepalive.default_tcp_keepalive
   in
   let n_problems = ref 0 in
   let rc_x = ref None in
diff --git a/src/tools/tcp_keepalive.ml b/src/tools/tcp_keepalive.ml
index de35616e..d608aa97 100644
--- a/src/tools/tcp_keepalive.ml
+++ b/src/tools/tcp_keepalive.ml
@@ -26,6 +26,13 @@ type t = {
     tcp_keepalive_probes : int;
   }
 
+let default_tcp_keepalive = {
+    enable_tcp_keepalive = true;
+    tcp_keepalive_time = 20;
+    tcp_keepalive_intvl = 20;
+    tcp_keepalive_probes = 3;
+  }
+
 external set_tcp_keepalive_time : socket -> int -> unit = "arakoon_tcp_set_tcp_keepalive_time"
 external set_tcp_keepalive_intvl : socket -> int -> unit = "arakoon_tcp_set_tcp_keepalive_intvl"
 external set_tcp_keepalive_probes : socket -> int -> unit = "arakoon_tcp_set_tcp_keepalive_probes"

From 2000bc881dfe678ec15350a05834164372633e9d Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Mon, 22 Aug 2016 18:02:45 +0200
Subject: [PATCH 218/373] allow getting arakoon config from another arakoon

---
 src/client/arakoon_client_config.ml |  2 +-
 src/client/client_helper.ml         |  1 -
 src/nursery/client_cfg.ml           |  2 -
 src/nursery/client_cfg_test.ml      |  2 +-
 src/nursery/nursery_main.ml         |  6 +--
 src/tools/arakoon_config_url.ml     | 75 ++++++++++++++++++++++-------
 6 files changed, 60 insertions(+), 28 deletions(-)

diff --git a/src/client/arakoon_client_config.ml b/src/client/arakoon_client_config.ml
index 9eb6b591..bdbf586d 100644
--- a/src/client/arakoon_client_config.ml
+++ b/src/client/arakoon_client_config.ml
@@ -110,7 +110,7 @@ let from_ini txt =
   let node_cfgs =
     List.map
       (fun node_name ->
-       let ips = Ini.get inifile node_name "ips" Ini.p_string_list Ini.required in
+       let ips = Ini.get inifile node_name "ip" Ini.p_string_list Ini.required in
        let port = Ini.get inifile node_name "client_port" Ini.p_int Ini.required in
        (node_name, { ips; port; }))
       nodes
diff --git a/src/client/client_helper.ml b/src/client/client_helper.ml
index 0066bb48..dc7d229a 100644
--- a/src/client/client_helper.ml
+++ b/src/client/client_helper.ml
@@ -153,7 +153,6 @@ let with_client'
 
 (** Result type and utilities for {! find_master' } *)
 module MasterLookupResult = struct
-  open Node_cfg
 
   (** Result type for a master lookup using {! find_master' } *)
   type t =
diff --git a/src/nursery/client_cfg.ml b/src/nursery/client_cfg.ml
index 799cc590..3ef6aa7e 100644
--- a/src/nursery/client_cfg.ml
+++ b/src/nursery/client_cfg.ml
@@ -87,6 +87,4 @@ module ClientCfg = struct
     let inifile = new Inifiles.inifile txt in
     from_ini section inifile
 
-  let from_url section url = Arakoon_config_url.retrieve url >|= (_from_txt section)
-
 end
diff --git a/src/nursery/client_cfg_test.ml b/src/nursery/client_cfg_test.ml
index b2d0ed01..3532525f 100644
--- a/src/nursery/client_cfg_test.ml
+++ b/src/nursery/client_cfg_test.ml
@@ -48,7 +48,7 @@ let test_parsing () =
       Lwt_io.Output fn
       (fun oc -> Lwt_io.write oc contents)
     >>= fun () ->
-    ClientCfg.from_url "global" (Arakoon_config_url.File fn) >>= fun cfg ->
+    Arakoon_config_url.(retrieve (File fn)) >|= (ClientCfg._from_txt "global") >>= fun cfg ->
     let sa0 = ClientCfg.get cfg "sturdy_0" in
     let r = OUnit.assert_equal sa0 (["127.0.0.1"],7080) in
     Lwt.return r
diff --git a/src/nursery/nursery_main.ml b/src/nursery/nursery_main.ml
index 9132e680..7b71e0b6 100644
--- a/src/nursery/nursery_main.ml
+++ b/src/nursery/nursery_main.ml
@@ -89,11 +89,7 @@ let _get_keeper_config_generic fetcher url=
   end
 
 let get_keeper_config cfg_url =
-  let fetcher =
-    match cfg_url with
-    | Arakoon_config_url.File f             -> (fun () -> Lwt_extra.read_file f)
-    | Arakoon_config_url.Etcd (peers, path) -> (fun () -> Arakoon_etcd.retrieve_value peers path)
-  in
+  let fetcher () = Arakoon_config_url.retrieve cfg_url in
   _get_keeper_config_generic fetcher cfg_url
 
 let get_nursery_client keeper_id cli_cfg =
diff --git a/src/tools/arakoon_config_url.ml b/src/tools/arakoon_config_url.ml
index 97f851a6..c3bb2692 100644
--- a/src/tools/arakoon_config_url.ml
+++ b/src/tools/arakoon_config_url.ml
@@ -14,9 +14,16 @@ See the License for the specific language governing permissions and
 limitations under the License.
 *)
 
+
+type arakoon_cfg =
+  { cluster_id : string;
+    key : string;
+    ini_location : string }
+
 type url =
-  File of string
+  | File of string
   | Etcd of (((string * int) list) * string)
+  | Arakoon of arakoon_cfg
 
 let to_string = function
   | File f -> f
@@ -28,33 +35,65 @@ let to_string = function
        |> String.concat ","
      in
      Printf.sprintf "etcd://%s/%s" peers_s path
+  | Arakoon { cluster_id; key; ini_location; } ->
+     Uri.make
+       ~scheme:"arakoon"
+       ~host:cluster_id
+       ~path:key
+       ~query:[ ("ini", [ ini_location; ]); ]
+       ()
+     |> Uri.to_string
 
-let make string =
-  let protocol, rest =
-    try
-      Scanf.sscanf string "%s@://%s" (fun p r -> p,r)
-    with End_of_file ->
-      "file", string
-  in
-  match protocol with
-  | "file" ->
+
+let make s =
+  let uri = Uri.of_string s in
+  let path = Uri.path uri in
+  match Uri.scheme uri with
+  | None
+  | Some "file" ->
       let canonical =
-        if rest.[0] = '/'
-        then rest
-        else Filename.concat (Unix.getcwd()) rest
+        if path.[0] = '/'
+        then path
+        else Filename.concat (Unix.getcwd()) path
       in
       File canonical
-  | "etcd" ->
-     let host,port,path =
-       Scanf.sscanf rest "%s@:%i/%s" (fun h i p -> h,i,p)
+  | Some "etcd" ->
+     let host = match Uri.host uri with
+       | None -> failwith "host is required"
+       | Some h -> h
      in
-     let peers = [host,port]
+     let port = match Uri.port uri with
+       | None -> 6379
+       | Some p -> p
      in
+     let peers = [ host, port ] in
      Etcd (peers, path)
-  | _ -> failwith (Printf.sprintf "unknown protocol:%s" protocol)
+  | Some "arakoon" ->
+     let cluster_id = match Uri.host uri with
+       | None -> failwith "host/cluster_id is required"
+       | Some h -> h
+     in
+     let qs = Uri.query uri in
+     let ini_location =
+       try List.assoc "ini" qs |> List.hd
+       with Not_found ->
+         failwith "ini location was not specified"
+     in
+     Arakoon { cluster_id; key = path; ini_location; }
+  | Some protocol -> failwith (Printf.sprintf "unknown protocol:%s" protocol)
+
+open Lwt.Infix
 
 let retrieve = function
   | File f ->
      Lwt_extra.read_file f
   | Etcd (peers, path) ->
      Arakoon_etcd.retrieve_value peers path
+  | Arakoon { cluster_id; key; ini_location; } ->
+     Lwt_extra.read_file ini_location >>= fun txt ->
+     let cfg = Arakoon_client_config.from_ini txt in
+     Client_helper.with_master_client'
+       cfg
+       (fun client -> client # get key)
+
+let default_url = ref (make "cfg/arakoon.ini")

From e1a08383029563e4364eb9155babe4e17905a7ac Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Mon, 22 Aug 2016 20:59:26 +0200
Subject: [PATCH 219/373] update arakoon_client.mllib

---
 src/arakoon_client.mllib | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/arakoon_client.mllib b/src/arakoon_client.mllib
index 5a955fc5..0160851f 100644
--- a/src/arakoon_client.mllib
+++ b/src/arakoon_client.mllib
@@ -39,3 +39,4 @@ Typed_ssl
 Unix_fd
 Tcp_keepalive
 Lwt_buffer
+To_string

From 4a5e35c94e9682d8f96c1b1f29d06ee160680dc7 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Wed, 24 Aug 2016 10:35:33 +0200
Subject: [PATCH 220/373] implement to_ini for arakoon_client_config

---
 src/all_test.ml                          |  1 +
 src/client/arakoon_client_config.ml      | 69 ++++++++++++++++++++++++
 src/client/arakoon_client_config_test.ml | 56 +++++++++++++++++++
 src/inifiles/inifiles.ml                 | 20 +++++++
 src/inifiles/inifiles.mli                |  3 ++
 src/tools/arakoon_config_url.ml          |  1 +
 6 files changed, 150 insertions(+)
 create mode 100644 src/client/arakoon_client_config_test.ml

diff --git a/src/all_test.ml b/src/all_test.ml
index 12b03db3..ece557fb 100644
--- a/src/all_test.ml
+++ b/src/all_test.ml
@@ -92,4 +92,5 @@ let suite = "universe" >::: [
     Std.List.Test.suite;
     Arakoon_log_sink_test.suite;
     Arakoon_etcd_test.suite;
+    Arakoon_client_config_test.suite;
   ]
diff --git a/src/client/arakoon_client_config.ml b/src/client/arakoon_client_config.ml
index bdbf586d..e3d87c72 100644
--- a/src/client/arakoon_client_config.ml
+++ b/src/client/arakoon_client_config.ml
@@ -1,3 +1,19 @@
+(*
+Copyright 2016 iNuron NV
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+ *)
+
 type cluster_id = string
 
 type tcp_keepalive_cfg = Tcp_keepalive.t = {
@@ -160,3 +176,56 @@ let from_ini txt =
     ssl_cfg;
     tcp_keepalive = tcp_keepalive_from_ini inifile;
   }
+
+
+let to_ini { cluster_id;
+             node_cfgs;
+             ssl_cfg;
+             tcp_keepalive; } =
+  let inifile = new Inifiles.inifile "[global]\ncluster_id = x" in
+
+  let node_names =
+    List.fold_left
+      (fun acc (node_name, { ips; port; }) ->
+       inifile # setval node_name "ip" (String.concat ", " ips);
+       inifile # setval node_name "client_port" (string_of_int port);
+       node_name :: acc)
+      []
+      node_cfgs
+  in
+
+  inifile # setval "global" "cluster_id" cluster_id;
+  inifile # setval "global" "cluster" (String.concat ", " node_names);
+
+  let { enable_tcp_keepalive;
+        tcp_keepalive_time;
+        tcp_keepalive_intvl;
+        tcp_keepalive_probes; } = tcp_keepalive in
+  inifile # setval "global" "enable_tcp_keepalive" (string_of_bool enable_tcp_keepalive);
+  inifile # setval "global" "tcp_keepalive_time"   (string_of_int  tcp_keepalive_time);
+  inifile # setval "global" "tcp_keepalive_intvl"  (string_of_int  tcp_keepalive_intvl);
+  inifile # setval "global" "tcp_keepalive_probes" (string_of_int  tcp_keepalive_probes);
+
+  let () = match ssl_cfg with
+    | None -> ()
+    | Some { ca_cert;
+             creds;
+             protocol; } ->
+       inifile # setval "global" "tls_ca_cert" ca_cert;
+       let () = match creds with
+         | None -> ()
+         | Some (tls_cert, tls_key) ->
+            inifile # setval "global" "tls_cert" tls_cert;
+            inifile # setval "global" "tls_key"  tls_key
+       in
+       inifile # setval
+               "global" "tls_version"
+               (match protocol with
+                | Ssl.TLSv1   -> "1.0"
+                | Ssl.TLSv1_1 -> "1.1"
+                | Ssl.TLSv1_2 -> "1.2"
+                | Ssl.SSLv23
+                | Ssl.SSLv3 -> failwith "unsupported ssl version")
+  in
+
+  inifile # to_string
diff --git a/src/client/arakoon_client_config_test.ml b/src/client/arakoon_client_config_test.ml
new file mode 100644
index 00000000..2db3f31b
--- /dev/null
+++ b/src/client/arakoon_client_config_test.ml
@@ -0,0 +1,56 @@
+(*
+Copyright 2016 iNuron NV
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+ *)
+
+open Arakoon_client_config
+
+(* TODO
+- test
+met quickcheck een config aanmaken
+to string
+parsen
+assert equal
+
+ *)
+
+let test_to_from_ini () =
+  let cluster_id = "x" in
+  let tcp_keepalive =
+    { enable_tcp_keepalive = true;
+      tcp_keepalive_time = 9;
+      tcp_keepalive_intvl = 90;
+      tcp_keepalive_probes = 21; }
+  in
+  let node_cfgs = [ ("node1", { ips = [ "23"; "99fsdjv"; ]; port = 423; });
+                    ("node2", { ips = []; port = 23; }); ]
+  in
+  let ssl_cfg = Some {
+                    ca_cert = "ca_cert";
+                    creds = Some ("x", "y");
+                    protocol = Ssl.TLSv1;
+                  } in
+  let t = { cluster_id; tcp_keepalive; node_cfgs; ssl_cfg; } in
+  let t' =
+    let x = to_ini t |> from_ini in
+    { x with node_cfgs = List.rev x.node_cfgs }
+  in
+  assert (t = t')
+
+open OUnit
+
+let suite =
+  "arakoon_client_config" >::: [
+      "test_to_from_ini" >:: test_to_from_ini;
+    ]
diff --git a/src/inifiles/inifiles.ml b/src/inifiles/inifiles.ml
index 6552b686..6d7cbd75 100644
--- a/src/inifiles/inifiles.ml
+++ b/src/inifiles/inifiles.ml
@@ -135,6 +135,23 @@ let write_inifile oc tbl =
      Lwt_io.write oc "\n"
     ) sections
 
+let ini_to_string tbl =
+  let buf = Buffer.create 20 in
+  Hashtbl.iter
+    (fun k v ->
+       Buffer.add_string buf "[";
+       Buffer.add_string buf k;
+       Buffer.add_string buf "]\n";
+       (Hashtbl.iter
+          (fun k v ->
+             Buffer.add_string buf k;
+             Buffer.add_string buf " = ";
+             Buffer.add_string buf v;
+             Buffer.add_string buf "\n") v);
+       Buffer.add_string buf "\n")
+    tbl;
+  Buffer.contents buf
+
 class inifile ?(spec=[]) txt =
   object (self)
     val data = Hashtbl.create 50
@@ -254,6 +271,9 @@ class inifile ?(spec=[]) txt =
       Lwt_io.with_file
         Lwt_io.Output file
         (fun oc -> write_inifile oc data)
+
+    method to_string =
+      ini_to_string data
   end
 
 let readdir path =
diff --git a/src/inifiles/inifiles.mli b/src/inifiles/inifiles.mli
index 2a507baf..52904a89 100644
--- a/src/inifiles/inifiles.mli
+++ b/src/inifiles/inifiles.mli
@@ -79,6 +79,9 @@ class inifile : ?spec:specification -> string ->
         optionally save to a different file *)
     method save : Lwt_io.file_name -> unit Lwt.t
 
+    (** get a string representation of the inifile *)
+    method to_string : string
+
     (** iterates across a section. passes all key valu pairs to f
         exactly once.*)
     method iter : (string -> string -> unit) -> string -> unit
diff --git a/src/tools/arakoon_config_url.ml b/src/tools/arakoon_config_url.ml
index c3bb2692..79b788a9 100644
--- a/src/tools/arakoon_config_url.ml
+++ b/src/tools/arakoon_config_url.ml
@@ -92,6 +92,7 @@ let retrieve = function
   | Arakoon { cluster_id; key; ini_location; } ->
      Lwt_extra.read_file ini_location >>= fun txt ->
      let cfg = Arakoon_client_config.from_ini txt in
+     assert (cluster_id = cfg.Arakoon_client_config.cluster_id);
      Client_helper.with_master_client'
        cfg
        (fun client -> client # get key)

From f4895c31b1b3cd6dc617bbde78a0b8134762717c Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Fri, 26 Aug 2016 14:43:33 +0200
Subject: [PATCH 221/373] fix config bug tlog_max_size

---
 src/node/node_cfg.ml      | 8 +++++---
 src/node/node_cfg_test.ml | 7 +++++--
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/src/node/node_cfg.ml b/src/node/node_cfg.ml
index 2cd7c3c8..283ff8c8 100644
--- a/src/node/node_cfg.ml
+++ b/src/node/node_cfg.ml
@@ -368,7 +368,7 @@ module Node_cfg = struct
     Ini.get inifile node_name "ip" Ini.p_string_list Ini.required
 
   let _tlog_max_entries inifile = _global_int inifile "tlog_max_entries"       50_000
-  let _tlog_max_size    inifile = _global_int inifile "tlog_max_size"    32*1024*1024
+  let _tlog_max_size    inifile = _global_int inifile "tlog_max_size"    (32*1024*1024)
 
   let _tlog_entries_overwrite inifile =
     Ini.get inifile "global" "__tainted_tlog_entries_per_file"
@@ -697,8 +697,6 @@ module Node_cfg = struct
       let cluster_id = _get_cluster_id inifile in
       let m_n_cfg = get_nursery_cfg inifile in
       let overwrite_tlog_entries = _tlog_entries_overwrite inifile in
-      let tlog_max_entries = _tlog_max_entries inifile in
-      let tlog_max_size    = _tlog_max_size inifile in
       let () = match overwrite_tlog_entries with
         | None -> ()
         | Some _ ->
@@ -707,6 +705,10 @@ module Node_cfg = struct
            in
            Logger.ign_warning_ msg
       in
+      let tlog_max_entries = _tlog_max_entries inifile in
+      let () = Printf.printf "before\n" in
+      let tlog_max_size    = _tlog_max_size inifile in
+      let () = Printf.printf "tlog_max_size:%i\n" tlog_max_size in
       let max_value_size = _max_value_size inifile in
       let max_buffer_size = _max_buffer_size inifile in
       let client_buffer_capacity = _client_buffer_capacity inifile in
diff --git a/src/node/node_cfg_test.ml b/src/node/node_cfg_test.ml
index f2e105ba..4fc3bb69 100644
--- a/src/node/node_cfg_test.ml
+++ b/src/node/node_cfg_test.ml
@@ -22,8 +22,11 @@ let test_correctness () =
 
   let t =
     Node_cfg.retrieve_cfg !config_url >>= fun cfg ->
-    let r = OUnit.assert_equal cfg.cluster_id "ricky" in
-    Lwt.return r
+    let () = OUnit.assert_equal cfg.cluster_id "ricky" in
+    let printer = string_of_int in
+    let () = OUnit.assert_equal ~printer 5_000_000 cfg.tlog_max_size in
+    let () = OUnit.assert_equal ~printer 64 cfg.client_buffer_capacity in
+    Lwt.return_unit
   in
   Lwt_main.run t
 

From d44b3b4726e2e24a2f732c223505171f405883ee Mon Sep 17 00:00:00 2001
From: root <root@vultr.guest>
Date: Mon, 29 Aug 2016 08:39:45 +0000
Subject: [PATCH 222/373] fix build

---
 docker/centos-7/Dockerfile     | 1 +
 docker/ubuntu-14.04/Dockerfile | 1 +
 docker/ubuntu-16.04/Dockerfile | 1 +
 3 files changed, 3 insertions(+)

diff --git a/docker/centos-7/Dockerfile b/docker/centos-7/Dockerfile
index a64feb99..f2c13526 100644
--- a/docker/centos-7/Dockerfile
+++ b/docker/centos-7/Dockerfile
@@ -56,6 +56,7 @@ ENV OPAMEXTERNALSOLVER="cudf_remote_proxy %{input}% %{output}% %{criteria}%"
 RUN eval `${opam_env}` && \
     opam update && \
     opam install -y \
+        oasis.0.4.6 \
         ocamlfind \
         ssl.0.5.2 \
         camlbz2 \
diff --git a/docker/ubuntu-14.04/Dockerfile b/docker/ubuntu-14.04/Dockerfile
index 9d03586b..97c3c44c 100644
--- a/docker/ubuntu-14.04/Dockerfile
+++ b/docker/ubuntu-14.04/Dockerfile
@@ -52,6 +52,7 @@ RUN eval `${opam_env}` && \
 RUN eval `${opam_env}` && \
     opam update && \
     opam install -y \
+        oasis.0.4.6 \
         ocamlfind \
         ssl.0.5.2 \
         camlbz2 \
diff --git a/docker/ubuntu-16.04/Dockerfile b/docker/ubuntu-16.04/Dockerfile
index 613937f3..ee6d845f 100644
--- a/docker/ubuntu-16.04/Dockerfile
+++ b/docker/ubuntu-16.04/Dockerfile
@@ -49,6 +49,7 @@ RUN eval `${opam_env}` && \
 RUN eval `${opam_env}` && \
     opam update && \
     opam install -y \
+        oasis.0.4.6 \
         ocamlfind \
         ssl.0.5.2 \
         camlbz2 \

From 5fcc0da26ca18747f603731fefe6c455041622a9 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Mon, 29 Aug 2016 10:58:11 +0200
Subject: [PATCH 223/373] autofix for missing .tlog file and (failing) test

---
 .../server/right/test_catchup_exercises.py    | 56 ++++++++++++++-----
 pylabs/test/server/system_tests_common.py     | 14 ++---
 src/node/node_main.ml                         | 25 ++++++++-
 src/tlog/tlog_map.ml                          | 20 ++++---
 4 files changed, 84 insertions(+), 31 deletions(-)

diff --git a/pylabs/test/server/right/test_catchup_exercises.py b/pylabs/test/server/right/test_catchup_exercises.py
index bdf795d0..55ed0daa 100644
--- a/pylabs/test/server/right/test_catchup_exercises.py
+++ b/pylabs/test/server/right/test_catchup_exercises.py
@@ -61,23 +61,23 @@ def do_one(n, max_wait):
         n = n * 2
         w = w * 2
 
-    
-    
-        
+
+
+
 @Common.with_custom_setup(Common.setup_2_nodes_forced_master_mini, Common.basic_teardown)
 def test_catchup_exercises():
     _scenario()
-    
+
 
 def test_catchup_mixed_config():
     pass
     """
-    catchup where 2 nodes have different rollover points. 
+    catchup where 2 nodes have different rollover points.
     (It's a configuration error that's bound to happen)
     """
     Common.data_base_dir = '/'.join([X.tmpDir,'arakoon_system_tests' ,
                                          "test_catchup_mixed_config"])
-    
+
     def setup(c_id,tlog_max_entries):
             c_nodes = ["%s_0" % c_id]
             c_home = "/".join([Common.data_base_dir, c_id])
@@ -89,17 +89,17 @@ def setup(c_id,tlog_max_entries):
 
     def wait(c0):
         system_tests_basic._wait_for_master(c0)
-        
+
     def _inner(c0):
         Common.regenerateClientConfig(id0)
         c0.start()
         wait(c0)
-        
+
         c0_client = c0.getClient()
 
         for x in range(100):
             c0_client.set("key_%i" % x, "value_%i" % x)
-        
+
         bad = 'nineteen'
         c0.addNode(bad,
                    clientPort = Common.node_client_base_port + 1,
@@ -110,7 +110,7 @@ def _inner(c0):
 
         bad
         cfg = c0._getConfigFile()
-        cfg.set('global','tlog_max_entries', str(19))
+        cfg.set('global','tlog_max_entries', str(13))
         c0._saveConfig(cfg)
         c0.createDirs(bad)
         c0.catchupOnly(bad)
@@ -130,10 +130,10 @@ def _inner(c0):
         lo = min(node_is.values())
         hi = max(node_is.values())
         assert_true(hi-lo <= 1)
-        
+
         for x in range(100):
             c0_client.set("x","x")
-            
+
     id0 = 'thirteen'
     c0 = setup(id0, 13)
     try:
@@ -152,7 +152,7 @@ def test_catchup_rollover_on_size():
     print ("now with collapsing as well")
     Common.stopOne(lagger)
     Common.iterate_n_times(1234, Common.simple_set)
-    Common.collapse(Common.node_names[0], 20)    
+    Common.collapse(Common.node_names[0], 20)
     Common.stopOne(Common.node_names[1])
     Common.startOne(lagger)
     time.sleep(20)
@@ -166,5 +166,33 @@ def test_catchup_rollover_on_size():
     exists = X.fileExists(head_file)
     assert_false(exists)
     Common.assert_last_i_in_sync(Common.node_names[0], lagger)
-    
 
+@Common.with_custom_setup(Common.setup_3_nodes_forced_master_mini_rollover_on_size,
+                          Common.basic_teardown)
+def test_issue125():
+    n = 31234
+    Common.iterate_n_times(n, Common.simple_set)
+    slave = Common.CONFIG.node_names[1]
+
+    Common.collapse(slave,10)
+    Common.stopOne(slave)
+
+    Common.iterate_n_times(n, Common.simple_set) # others move forward
+
+    dir_names = Common.build_node_dir_names(slave)
+    db_dir = dir_names[0]
+    log_dir = dir_names[1]
+    tlx_dir =dir_names[2]
+
+
+    X.removeFile('%s/035.tlog' % db_dir)
+    X.removeFile('%s/034.tlx' % tlx_dir)
+    X.removeFile('%s/%s.log' % (log_dir,slave))
+    X.removeFile('%s/%s.db' % (db_dir, slave))
+    # we have a cluster with head.db, no db and no tlog file.
+
+    # can we launch ?
+
+    Common.startOne(slave, ['-autofix'])
+    time.sleep(20)
+    Common.assert_running_nodes(3)
diff --git a/pylabs/test/server/system_tests_common.py b/pylabs/test/server/system_tests_common.py
index e55c1c8f..55d2178d 100644
--- a/pylabs/test/server/system_tests_common.py
+++ b/pylabs/test/server/system_tests_common.py
@@ -392,9 +392,9 @@ def stopOne(name):
     rc = cluster.stopOne(name)
     assert (rc == 0)
 
-def startOne(name):
+def startOne(name, extraParams = None):
     cluster = _getCluster()
-    cluster.startOne(name)
+    cluster.startOne(name, extraParams)
 
 def dropMaster(name):
     cluster = _getCluster()
@@ -760,20 +760,20 @@ def setup_2_nodes_forced_master (home_dir):
     setup_n_nodes( 2, True, home_dir, witness_nodes = True)
 
 def setup_2_nodes_forced_master_mini (home_dir):
-    extra = {'tlog_max_entries':'1000'}
+    extra = {'tlog_max_entries':'500'}
     setup_n_nodes( 2, True, home_dir, extra)
 
 def setup_3_nodes_forced_master_mini_rollover_on_size(home_dir):
     extra = {'tlog_max_entries': '100_000',
              'tlog_max_size'   : '64000'
             }
-    
+
     setup_n_nodes( 3, True, home_dir, extra)
-    
+
 def setup_3_nodes_forced_master_mini (home_dir):
     extra = {'tlog_max_entries':'1000'}
     setup_n_nodes( 3, True, home_dir, extra)
-    
+
 def setup_2_nodes_forced_master_normal_slaves (home_dir):
     setup_n_nodes( 2, True, home_dir)
 
@@ -1050,7 +1050,7 @@ def retrying_set_get_and_delete( client, key, value ):
               )
     def validate_ex ( ex, tryCnt ):
         ex_msg = "%s" % ex
-        
+
         validEx = isinstance(ex, valids)
         if validEx:
             logging.debug( "Ignoring exception: %s", ex_msg )
diff --git a/src/node/node_main.ml b/src/node/node_main.ml
index 9a7c60b4..1c7c0a8c 100644
--- a/src/node/node_main.ml
+++ b/src/node/node_main.ml
@@ -603,7 +603,30 @@ let _main_2 (type s)
                   Tlc2._truncate_tlog fn >>= fun ()->
                   clean_up_retry fn
                 end
-
+             | Tlogcommon.TLogSabotage ->
+                begin
+                  Logger.warning_ ~exn "AUTOFIX: no .tlog file" >>= fun () ->
+                  Tlog_map._get_tlog_names me.tlog_dir me.tlx_dir >>= fun tlog_names ->
+                  let tlx = List.hd (List.rev tlog_names) |> Filename.concat me.tlx_dir in
+                  let tlx_base = Filename.basename tlx in
+                  let tlx_number = Tlog_map.get_number tlx_base in
+                  let tlog_base = Tlog_map.file_name (tlx_number+1) in
+                  let tlog = Filename.concat me.tlog_dir tlog_base in
+                  Tlog_main._last_entry tlx >>= fun e_o ->
+                  let e = match e_o with
+                    | None -> failwith (Printf.sprintf "empty tlx:%s" tlx)
+                    | Some e -> e
+                  in
+                  Lwt_io.with_file
+                    ~mode:Lwt_io.output tlog
+                    (fun oc ->
+                      let marker = Tlog_map._make_close_marker me.node_name in
+                      let open Tlogcommon.Entry in
+                      Tlogcommon.write_marker oc e.i e.v (Some marker)
+                    )
+                  >>= fun () ->
+                  _open_tlc_and_store()
+                end
              | _ ->
                 Logger.fatal_f_ ~exn "AUTOFIX: can't autofix this" >>= fun () ->
                 Lwt.fail exn
diff --git a/src/tlog/tlog_map.ml b/src/tlog/tlog_map.ml
index 726f2f26..d1ef3085 100644
--- a/src/tlog/tlog_map.ml
+++ b/src/tlog/tlog_map.ml
@@ -280,13 +280,12 @@ let _validate_one
         if (not check_sabotage) || (ext = ".tlog" && get_number base = 0)
         then Lwt.return (None, new_index, 0)
         else (*
-         somebody sabotaged us:
-         (s)he deleted the .tlog file but we have .tlf's
-         meaning rotation happened correctly.
-         This means the marker can not be present.
-         Let the node die; they should fix this manually.
-              *)
-          Logger.fatal_f_ "not found: %s. %s was removed?!" tlog_name base >>= fun () ->
+         somebody sabotaged us (deleting the tlog file),
+         or (more likely?) the previous incarnation died
+         in catchup while it was downloading .tlx files.
+
+         *)
+          Logger.warning_f_ "not found: %s. %s was removed?!" tlog_name base >>= fun () ->
           Lwt.fail TLogSabotage
       | exn -> Lwt.fail exn
     )
@@ -337,7 +336,9 @@ module TlogMap = struct
 
 
   let _init tlog_dir tlx_dir node_id tlog_max_entries tlog_max_size ~check_marker ~check_sabotage =
-    Logger.info_f_ "_init ~check_marker:%b" check_marker >>= fun () ->
+    Logger.info_f_ "_init ~check_marker:%b ~check_sabotage:%b"
+                   check_marker check_sabotage
+    >>= fun () ->
     _get_tlog_names tlog_dir tlx_dir >>= fun tlog_names ->
     let tlog_number = get_count tlog_names in
     Logger.info_f_ "tlog_number:%i" tlog_number >>= fun () ->
@@ -400,7 +401,7 @@ module TlogMap = struct
     let tlog_max_entries = with_default tlog_max_entries 100_000 in
     let tlog_max_size    = with_default tlog_max_size 32_000_000 in
     _init
-      tlog_dir tlx_dir node_id ~check_marker ~check_sabotage:false
+      tlog_dir tlx_dir node_id ~check_marker ~check_sabotage:true
       tlog_max_entries tlog_max_size
     >>= fun ((tlog_entries, tlog_size,tlog_number,i_to_tlog_number, should_roll), last, index ) ->
 
@@ -411,6 +412,7 @@ module TlogMap = struct
     Lwt.return (t, last, index)
 
   let reinit t =
+    Logger.debug_f_ "tlog_map.reinit" >>= fun () ->
     _init
       t.tlog_dir t.tlx_dir t.node_id ~check_marker:false ~check_sabotage:false
       t.tlog_max_entries t.tlog_max_size

From a4e7dffc08cb999e1b1de45eb7f6efe8c978eb81 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Mon, 29 Aug 2016 12:16:12 +0200
Subject: [PATCH 224/373] bugfix

---
 src/client/arakoon_client_config.ml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/client/arakoon_client_config.ml b/src/client/arakoon_client_config.ml
index e3d87c72..f48f21db 100644
--- a/src/client/arakoon_client_config.ml
+++ b/src/client/arakoon_client_config.ml
@@ -184,6 +184,8 @@ let to_ini { cluster_id;
              tcp_keepalive; } =
   let inifile = new Inifiles.inifile "[global]\ncluster_id = x" in
 
+  inifile # delval "global" "cluster_id";
+
   let node_names =
     List.fold_left
       (fun acc (node_name, { ips; port; }) ->

From ff458c9720bdc1c3449a3c74bff6eccdf1a1c67b Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Mon, 29 Aug 2016 13:42:16 +0200
Subject: [PATCH 225/373] bug fix: correct iterate for tiny tlogs

---
 src/tlog/tlog_map.ml | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/src/tlog/tlog_map.ml b/src/tlog/tlog_map.ml
index d1ef3085..46ff810d 100644
--- a/src/tlog/tlog_map.ml
+++ b/src/tlog/tlog_map.ml
@@ -632,7 +632,6 @@ module TlogMap = struct
     let num_tlogs = List.length tlog_names in
     let maybe_fold (cnt,low) fn =
       let fn_start = get_start_i t (get_number fn) in
-
       let low_n = Sn.succ low in
       match fn_start with
       | Some fn_start ->
@@ -641,16 +640,15 @@ module TlogMap = struct
            let should_fold =
              match fn_next with
              | None -> fn_start <= low_n
-             | Some next -> fn_start <= low_n && low_n < next
+             | Some next -> fn_start <= low_n && low_n <= next
            in
            if should_fold
            then
              begin
-               Logger.debug_f_ "fold over: %s: [%s,...) "
-                               fn (Sn.string_of fn_start)
-               >>= fun () ->
                let first = fn_start in
-               Logger.info_f_ "Replaying tlog file: %s (%d/%d)" fn cnt num_tlogs  >>= fun () ->
+               Logger.info_f_ "Replaying tlog file: %s [%s,...] (%d/%d)"
+                              fn (Sn.string_of fn_start) cnt num_tlogs
+               >>= fun () ->
                let t1 = Unix.gettimeofday () in
                fold_read t fn ~index low (Some too_far_i) ~first low acc_entry >>= fun low' ->
                Logger.info_f_ "Completed replay of %s, took %f seconds, %i to go"

From 63ccbc49f1a82b09cc8a91ffe9656fcd6800c303 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Mon, 29 Aug 2016 13:55:31 +0200
Subject: [PATCH 226/373] remove cucu

---
 src/node/node_cfg.ml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/node/node_cfg.ml b/src/node/node_cfg.ml
index 283ff8c8..237bab36 100644
--- a/src/node/node_cfg.ml
+++ b/src/node/node_cfg.ml
@@ -706,9 +706,7 @@ module Node_cfg = struct
            Logger.ign_warning_ msg
       in
       let tlog_max_entries = _tlog_max_entries inifile in
-      let () = Printf.printf "before\n" in
       let tlog_max_size    = _tlog_max_size inifile in
-      let () = Printf.printf "tlog_max_size:%i\n" tlog_max_size in
       let max_value_size = _max_value_size inifile in
       let max_buffer_size = _max_buffer_size inifile in
       let client_buffer_capacity = _client_buffer_capacity inifile in

From e23bcd13f524003562bc8a949994e97e515af299 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Mon, 29 Aug 2016 16:51:59 +0200
Subject: [PATCH 227/373] conditionally remove database

---
 src/node/node_main.ml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/node/node_main.ml b/src/node/node_main.ml
index 1c7c0a8c..b40a83bb 100644
--- a/src/node/node_main.ml
+++ b/src/node/node_main.ml
@@ -617,6 +617,10 @@ let _main_2 (type s)
                     | None -> failwith (Printf.sprintf "empty tlx:%s" tlx)
                     | Some e -> e
                   in
+                  (if not !head_copied
+                  then File_system.unlink db_name
+                   else Lwt.return_unit)
+                  >>= fun () ->
                   Lwt_io.with_file
                     ~mode:Lwt_io.output tlog
                     (fun oc ->

From f49df62943b31632dabba6123a636d2793fb4174 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Tue, 30 Aug 2016 14:28:54 +0200
Subject: [PATCH 228/373] skip leading slash in arakoon config path

---
 src/all_test.ml                      |  1 +
 src/tools/arakoon_config_url.ml      |  1 +
 src/tools/arakoon_config_url_test.ml | 45 ++++++++++++++++++++++++++++
 3 files changed, 47 insertions(+)
 create mode 100644 src/tools/arakoon_config_url_test.ml

diff --git a/src/all_test.ml b/src/all_test.ml
index ece557fb..a96d7373 100644
--- a/src/all_test.ml
+++ b/src/all_test.ml
@@ -93,4 +93,5 @@ let suite = "universe" >::: [
     Arakoon_log_sink_test.suite;
     Arakoon_etcd_test.suite;
     Arakoon_client_config_test.suite;
+    Arakoon_config_url_test.suite;
   ]
diff --git a/src/tools/arakoon_config_url.ml b/src/tools/arakoon_config_url.ml
index 79b788a9..426251ad 100644
--- a/src/tools/arakoon_config_url.ml
+++ b/src/tools/arakoon_config_url.ml
@@ -69,6 +69,7 @@ let make s =
      let peers = [ host, port ] in
      Etcd (peers, path)
   | Some "arakoon" ->
+     let path = Str.string_after path 1 in
      let cluster_id = match Uri.host uri with
        | None -> failwith "host/cluster_id is required"
        | Some h -> h
diff --git a/src/tools/arakoon_config_url_test.ml b/src/tools/arakoon_config_url_test.ml
new file mode 100644
index 00000000..0276a4e0
--- /dev/null
+++ b/src/tools/arakoon_config_url_test.ml
@@ -0,0 +1,45 @@
+(*
+Copyright 2016 iNuron NV
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*)
+
+open Arakoon_config_url
+
+let test_parse_url () =
+  let urls = [
+      Etcd (["127.0.0.1", 5000], "/albas/xxxx//arakoons/abm"), "etcd://127.0.0.1:5000/albas/xxxx//arakoons/abm";
+      File "/tmp", "/tmp";
+      File "/tmp", "file:///tmp";
+      File (Unix.getcwd () ^ "/x/a"), "x/a";
+      Arakoon ({ cluster_id = "id";
+                 key = "some/key";
+                 ini_location = "/tmp/x";
+               }),
+      "arakoon://id/some/key?ini=/tmp/x";
+    ]
+  in
+  List.iter
+    (fun (expected, url) ->
+     let actual = make url in
+     Printf.printf "url = %s, expected = %s, actual = %s\n"
+                   url
+                   (to_string expected)
+                   (to_string actual);
+     assert (expected = actual))
+    urls
+
+open OUnit
+let suite = "arakoon_config_url" >::: [
+      "test_parse_url" >:: test_parse_url;
+    ]

From b9ae34e05fa0e090b5441723a988b583f6feae9d Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Tue, 30 Aug 2016 14:32:10 +0200
Subject: [PATCH 229/373] verify_store;get-raw; set < file

---
 src/client/client_main.ml | 15 ++++++++++----
 src/main/arakoon.ml       | 25 +++++++++++++++++++++--
 src/node/dump_store.ml    | 42 +++++++++++++++++++++++++++++++++++++++
 3 files changed, 76 insertions(+), 6 deletions(-)

diff --git a/src/client/client_main.ml b/src/client/client_main.ml
index ed994d16..3d474754 100644
--- a/src/client/client_main.ml
+++ b/src/client/client_main.ml
@@ -67,14 +67,21 @@ let with_master_client ~tls cfg_url f =
   find_master ?tls ccfg >>= fun master_name ->
   with_client'' ?tls ccfg master_name f
 
-let set ~tls cfg_name key value =
-  let t () = with_master_client ~tls cfg_name (fun client -> client # set key value)
+let set ~tls cfg_name key value_option =
+  let t () =
+    (match value_option with
+    | Some value -> Lwt.return value
+    | None -> Lwt_io.read Lwt_io.stdin
+    ) >>= fun value ->
+    with_master_client ~tls cfg_name (fun client -> client # set key value)
   in run t
 
-let get ~tls cfg_name key =
+let get ?(raw=false)~tls cfg_name key =
   let f (client:Arakoon_client.client) =
     client # get key >>= fun value ->
-    Lwt_io.printlf "%S%!" value
+    if raw
+    then Lwt_io.print value
+    else Lwt_io.printlf "%S%!" value
   in
   let t () = with_master_client ~tls cfg_name f in
   run t
diff --git a/src/main/arakoon.ml b/src/main/arakoon.ml
index 74356023..2a073ee2 100644
--- a/src/main/arakoon.ml
+++ b/src/main/arakoon.ml
@@ -39,7 +39,9 @@ type local_action =
   | BENCHMARK
   | LOAD
   | SET
+  | SET_RAW
   | GET
+  | GET_RAW
   | DELETE
   | DELETE_PREFIX
   | NOP
@@ -65,6 +67,7 @@ type local_action =
   | NODE_STATE
   | RANGE_ENTRIES
   | REV_RANGE_ENTRIES
+  | VerifyStore
 
 type server_action =
   | Node
@@ -287,10 +290,20 @@ let main () =
     ("--set", Arg.Tuple [set_laction SET;
                          Arg.Set_string key;
                          Arg.Set_string value;
-                        ], "<key> <value> : arakoon[<key>] = <value>");
+                        ], "<key> : arakoon[<key>] = <value> (explicit)");
+    ("--set-raw", Arg.Tuple [set_laction SET_RAW;
+                         Arg.Set_string key;
+                        ], "<key> : arakoon[<key>] = <stdin>"
+
+    );
     ("--get", Arg.Tuple [set_laction GET;
                          Arg.Set_string key
                         ],"<key> : arakoon[<key>]");
+    ("--get-raw", Arg.Tuple [set_laction GET_RAW;
+                             Arg.Set_string key;
+                            ],
+     "same as get, but doesn't escape the value"
+    );
     ("--delete", Arg.Tuple[set_laction DELETE;
                            Arg.Set_string key;
                           ], "<key> : delete arakoon[<key>]");
@@ -368,6 +381,11 @@ let main () =
                               Arg.Set_string location;
                              ],
      "<cluster_id> <ip> <port> <location> requests the node to stream over its database (only works on slaves)");
+    ("--verify-db", Arg.Tuple [set_laction VerifyStore;
+                               Arg.Set_string filename;
+                              ],
+     "<filename>"
+    );
     ("--optimize-db", Arg.Tuple[set_laction Optimize_db;
                                 Arg.Set_string cluster_id;
                                 Arg.Set_string ip;
@@ -449,11 +467,14 @@ let main () =
     | CloseTlog -> Tlog_main.mark_tlog !filename (Tlog_map._make_close_marker !node_id)
     | ReplayTlogs -> Replay_main.replay_tlogs !tlog_dir !tlf_dir !filename !end_i
     | DumpStore -> Dump_store.dump_store !filename
+    | VerifyStore -> Dump_store.verify_store !filename None
     | TruncateTlog -> Tlc2.truncate_tlog !filename
     | CompressTlog -> Tlog_main.compress_tlog !filename !archive_type
     | UncompressTlog -> Tlog_main.uncompress_tlog !filename
-    | SET -> Client_main.set ~tls !config_url !key !value
+    | SET -> Client_main.set ~tls !config_url !key (Some !value)
+    | SET_RAW -> Client_main.set ~tls !config_url !key None
     | GET -> Client_main.get ~tls !config_url !key
+    | GET_RAW -> Client_main.get ~tls !config_url !key ~raw:true
     | PREFIX -> Client_main.prefix ~tls !config_url !key !max_results
     | DELETE_PREFIX -> Client_main.delete_prefix ~tls !config_url !key
     | NOP -> Client_main.nop ~tls !config_url
diff --git a/src/node/dump_store.ml b/src/node/dump_store.ml
index 4af11193..7ae83be6 100644
--- a/src/node/dump_store.ml
+++ b/src/node/dump_store.ml
@@ -153,3 +153,45 @@ let inject_as_head fn node_id cfg_url ~force ~in_place =
   with
     | ExitWithCode i -> i
     | exn -> raise exn
+
+
+let verify_store fn start_key =
+  let open Camltc in
+  let open Local_store in
+  let read_only = true in
+  let t () =
+    Local_store.make_store ~lcnum:1024 ~ncnum:512 read_only fn >>= fun ls ->
+    let go () =
+      Bdb.with_cursor
+        (Hotc.get_bdb ls.db)
+        (fun bdb cursor ->
+          let () = Bdb.first bdb cursor in
+          let () = match start_key with
+            | None -> ()
+            | Some key -> Bdb.jump bdb cursor key
+          in
+          let rec loop i =
+            let key = Bdb.key bdb cursor in
+            let _value = Bdb.value bdb cursor in
+            begin
+              if (i < 10 || i mod 20000 = 0)
+              then Lwt_io.eprintlf "%10i: %S\tok%!" i key
+              else Lwt.return_unit
+            end
+            >>= fun ()->
+            let cont =
+              try
+                Bdb.next bdb cursor;
+                true
+              with Not_found -> false
+            in
+            if cont
+            then loop (i+1)
+            else Lwt.return 0
+          in
+          loop 0
+        )
+    in
+    go ()
+  in
+  Lwt_extra.run t

From 4b93dac623833d4ff1458d1428ba3f1b2c51b473 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Tue, 30 Aug 2016 17:21:54 +0200
Subject: [PATCH 230/373] removed 'down', loop when not canceled

---
 src/client/client_helper.ml | 35 +++++++++++++++--------------------
 1 file changed, 15 insertions(+), 20 deletions(-)

diff --git a/src/client/client_helper.ml b/src/client/client_helper.ml
index dc7d229a..c8433e91 100644
--- a/src/client/client_helper.ml
+++ b/src/client/client_helper.ml
@@ -65,8 +65,8 @@ let with_connection ~tls ~tcp_keepalive sa do_it =
   >>= fun () ->
   begin
   match tls with
-  | None -> 
-     let ic = Lwt_io.of_fd ~mode:Lwt_io.input fd 
+  | None ->
+     let ic = Lwt_io.of_fd ~mode:Lwt_io.input fd
      and oc = Lwt_io.of_fd ~mode:Lwt_io.output fd in
      let conn = (ic,oc)
      and fin = fun () -> Lwt_unix.close fd
@@ -76,7 +76,7 @@ let with_connection ~tls ~tcp_keepalive sa do_it =
      Typed_ssl.Lwt.ssl_connect fd ctx >>= fun (_, sock) ->
      let ic = Lwt_ssl.in_channel_of_descr sock
      and oc = Lwt_ssl.out_channel_of_descr sock in
-     let conn = (ic,oc) 
+     let conn = (ic,oc)
      and fin = fun () -> Lwt_ssl.close sock
      in
      Lwt.return (conn,fin)
@@ -85,7 +85,7 @@ let with_connection ~tls ~tcp_keepalive sa do_it =
   Lwt.finalize
     (fun () -> do_it conn)
     fin
-  
+
 exception No_connection
 
 let with_connection' ~tls ~tcp_keepalive addrs f =
@@ -205,7 +205,7 @@ let find_master' ?tls cluster_cfg =
 
   let open MasterLookupResult in
 
-  let rec loop down unknown = function
+  let rec loop unknown = function
     | [] -> begin
         let res =
           if unknown = []
@@ -235,7 +235,7 @@ let find_master' ?tls cluster_cfg =
                 | None -> begin
                     Logger.debug_f_
                       "Client_main.find_master': %S doesn't know" node_name >>= fun () ->
-                    loop down ((node_name, cfg) :: unknown) rest
+                    loop ((node_name, cfg) :: unknown) rest
                 end
                 | Some n -> begin
                     Logger.debug_f_ "Client_main.find_master': %S thinks %S" node_name n >>= fun () ->
@@ -248,7 +248,7 @@ let find_master' ?tls cluster_cfg =
                       else
                         match lookup_cfg n with
                           | Some ncfg ->
-                              loop down unknown ((n, ncfg) :: rest)
+                              loop unknown ((n, ncfg) :: rest)
                           | None -> begin
                               let res = Unknown_node (n, (node_name, cfg)) in
                               Logger.warning_f_
@@ -257,21 +257,16 @@ let find_master' ?tls cluster_cfg =
                           end
                 end)
           (function
-            | Unix.Unix_error(Unix.ECONNREFUSED, _, _) ->
-                Logger.debug_f_
-                  "Client_main.find_master': Connection to %S refused" node_name >>= fun () ->
-                loop (cfg :: down) unknown rest
-            | Lwt.Canceled as exn ->
-                Lwt.fail exn
-            | exn -> begin
-                let res = Exception exn in
-                Logger.debug_f_
-                  "Client_main.find_master': %s" (to_string res) >>= fun () ->
-                return res
-            end)
+           | Lwt.Canceled as exn ->
+              Lwt.fail exn
+           | exn ->
+              Logger.debug_f_ ~exn
+                "Client_main.find_master': problem with %S" node_name >>= fun () ->
+              loop unknown rest
+          )
     end
   in
-  loop [] [] (cluster_cfg.node_cfgs)
+  loop [] (cluster_cfg.node_cfgs)
 
 
 (** Lookup the master of a cluster in a loop

From c83108f944aee60e7f03483694012598996379b5 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Tue, 30 Aug 2016 17:31:26 +0200
Subject: [PATCH 231/373] make_release: 1.9.10

---
 META                      | 2 +-
 debian/changelog          | 5 +++++
 redhat/SPECS/arakoon.spec | 4 +++-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/META b/META
index d7ec0cd2..2d4553df 100644
--- a/META
+++ b/META
@@ -1,7 +1,7 @@
 # -*- conf -*-
 
 description = "Arakoon client"
-version = "1.9.9"
+version = "1.9.10"
 exists_if = "arakoon_client.cma,arakoon_client.cmxa,arakoon_client.cmxs"
 requires = "threads lwt lwt.unix str oUnit quickcheck lwt.ssl ocplib-endian redis.lwt uri"
 archive(byte) = "arakoon_client.cma tcp_keepalive_sockopt_stubs.o"
diff --git a/debian/changelog b/debian/changelog
index 808039d8..a5fad3a9 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,8 @@
+arakoon (1.9.10) unstable; urgency=low
+  * new upstream version
+
+ -- Romain Slootmaekers <romain.slootmaekers@openvstorage.com>  Tue, 30 Aug 2016 17:31:26 +0100
+
 arakoon (1.9.9) unstable; urgency=low
   * new upstream version
 
diff --git a/redhat/SPECS/arakoon.spec b/redhat/SPECS/arakoon.spec
index 6e3b5684..a27426ea 100644
--- a/redhat/SPECS/arakoon.spec
+++ b/redhat/SPECS/arakoon.spec
@@ -2,7 +2,7 @@
 
 Summary: Arakoon
 Name: arakoon
-Version: 1.9.9
+Version: 1.9.10
 Release: 3%{?dist}
 License: Apache 2.0
 Requires: libev >= 4
@@ -33,6 +33,8 @@ cp arakoon.native %{buildroot}%{_bindir}/arakoon
 %{_bindir}/arakoon
 
 %changelog
+* Tue Aug 30 2016 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.10
+- Create arakoon 1.9.10 RPM package
 * Thu Aug 11 2016 Jan Doms <jan.doms@gmail.com> - 1.9.9
 - Create arakoon 1.9.9 RPM package
 * Wed Jul 20 2016 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.8

From eed92f215ecdde2b1466b577baa92db163845dcd Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 31 Aug 2016 15:14:13 +0200
Subject: [PATCH 232/373] fix race

make sure reinit and remove_below keep out of each other's way.
---
 src/tlog/tlog_map.ml      | 75 ++++++++++++++++++++++-----------------
 src/tlog/tlog_map_test.ml | 11 +++---
 2 files changed, 48 insertions(+), 38 deletions(-)

diff --git a/src/tlog/tlog_map.ml b/src/tlog/tlog_map.ml
index 46ff810d..d4fbbe3b 100644
--- a/src/tlog/tlog_map.ml
+++ b/src/tlog/tlog_map.ml
@@ -308,6 +308,7 @@ module TlogMap = struct
 
       mutable i_to_tlog_number : item list; (* sorted: most recent first *)
       mutable should_roll: bool;
+      lock: Lwt_mutex.t;
     }
 
   let show_map t =
@@ -404,24 +405,28 @@ module TlogMap = struct
       tlog_dir tlx_dir node_id ~check_marker ~check_sabotage:true
       tlog_max_entries tlog_max_size
     >>= fun ((tlog_entries, tlog_size,tlog_number,i_to_tlog_number, should_roll), last, index ) ->
-
+    let lock = Lwt_mutex.create () in
     let t = {node_id; tlog_dir;tlx_dir;tlog_max_entries; tlog_max_size;
              tlog_entries; tlog_size; tlog_number; i_to_tlog_number;should_roll;
+             lock;
             }
     in
     Lwt.return (t, last, index)
 
   let reinit t =
-    Logger.debug_f_ "tlog_map.reinit" >>= fun () ->
-    _init
-      t.tlog_dir t.tlx_dir t.node_id ~check_marker:false ~check_sabotage:false
-      t.tlog_max_entries t.tlog_max_size
-    >>= fun ((tlog_entries, tlog_size, tlog_number,i_to_tlog_number,should_roll), last,_) ->
-    t.tlog_size   <- tlog_size;
-    t.tlog_number <- tlog_number;
-    t.i_to_tlog_number <- i_to_tlog_number;
-    t.should_roll <- should_roll;
-    Lwt.return last
+    Lwt_mutex.with_lock
+      t.lock
+      (fun () ->
+        Logger.info_f_ "tlog_map.reinit" >>= fun () ->
+        _init
+          t.tlog_dir t.tlx_dir t.node_id ~check_marker:false ~check_sabotage:false
+          t.tlog_max_entries t.tlog_max_size
+        >>= fun ((tlog_entries, tlog_size, tlog_number,i_to_tlog_number,should_roll), last,_) ->
+        t.tlog_size   <- tlog_size;
+        t.tlog_number <- tlog_number;
+        t.i_to_tlog_number <- i_to_tlog_number;
+        t.should_roll <- should_roll;
+        Lwt.return last)
 
   let get_last_tlog t =
     Logger.debug_ "get_last_tlog" >>= fun () ->
@@ -555,28 +560,32 @@ module TlogMap = struct
     loop canonicals
 
   let remove_below t i =
-    Logger.debug_f_ "remove_below %s tlog_map:%s" (Sn.string_of i) (show_map t)
-    >>= fun () ->
-    let rec find_start to_remove_rev = function
-      | [] -> ([],[])
-      | item :: rest when item.i >= i -> (List.rev to_remove_rev), (item :: rest)
-      | x :: rest -> find_start (x::to_remove_rev) rest
-    in
-    let to_remove,to_keep_rev = find_start [] (List.rev t.i_to_tlog_number) in
-    t.i_to_tlog_number <- List.rev to_keep_rev;
-
-    let remove item  =
-      which_tlog_file  t item.n >>= function
-      | None -> Lwt.return_unit
-      | Some canonical ->
-         begin
-           Logger.debug_f_ "Unlinking %s" canonical >>= fun () ->
-           File_system.unlink canonical
-         end
-    in
-    Lwt_list.iter_s remove to_remove >>= fun () ->
-    Logger.debug_f_ "after remove_below %s tlog_map:%s" (Sn.string_of i) (show_map t) >>= fun () ->
-    Lwt.return_unit
+    Lwt_mutex.with_lock
+      t.lock
+      (fun () ->
+        Logger.debug_f_ "remove_below %s tlog_map:%s" (Sn.string_of i) (show_map t)
+        >>= fun () ->
+        let rec find_start to_remove_rev = function
+          | [] -> ([],[])
+          | item :: rest when item.i >= i -> (List.rev to_remove_rev), (item :: rest)
+          | x :: rest -> find_start (x::to_remove_rev) rest
+        in
+        let to_remove,to_keep_rev = find_start [] (List.rev t.i_to_tlog_number) in
+        t.i_to_tlog_number <- List.rev to_keep_rev;
+
+        let remove item  =
+          which_tlog_file  t item.n >>= function
+          | None -> Lwt.return_unit
+          | Some canonical ->
+             begin
+               Logger.debug_f_ "Unlinking %s" canonical >>= fun () ->
+               File_system.unlink canonical
+             end
+        in
+        Lwt_list.iter_s remove to_remove >>= fun () ->
+        Logger.debug_f_ "after remove_below %s tlog_map:%s" (Sn.string_of i) (show_map t) >>= fun () ->
+        Lwt.return_unit
+      )
 
   let is_rollover_point t i =
     let rec find = function
diff --git a/src/tlog/tlog_map_test.ml b/src/tlog/tlog_map_test.ml
index b04bc5e6..39d5a80a 100644
--- a/src/tlog/tlog_map_test.ml
+++ b/src/tlog/tlog_map_test.ml
@@ -1,5 +1,5 @@
 open Tlog_map.TlogMap
-       
+
 let test_tlogs_to_collapse () =
   let make_item (i,n,is_archive) = {i;n;is_archive} in
   let tlog_map = {
@@ -15,9 +15,9 @@ let test_tlogs_to_collapse () =
                              ( 5000L, 5,false);
                              ( 4000L, 4,false);
                              ( 3000L, 3,false);
-                             ( 2000L, 2,false); 
+                             ( 2000L, 2,false);
                              ( 1000L, 1,false);
-                             (    0L, 0,false);  
+                             (    0L, 0,false);
                            ];
       tlog_dir = "tlog_dir";
       tlx_dir = "tlx_dir";
@@ -28,6 +28,7 @@ let test_tlogs_to_collapse () =
       tlog_entries = 393;
       tlog_number = 12;
       should_roll = false;
+      lock = Lwt_mutex.create ();
     }
   in
   let head_i = 0L in
@@ -38,9 +39,9 @@ let test_tlogs_to_collapse () =
   let printer = To_string.option (pair string_of_int Sn.string_of) in
   let expected = Some (12,12_002L) in
   OUnit.assert_equal expected r ~printer
-                     
+
 open OUnit
-       
+
 let suite =
   "tlog_map">:::[
       "tlogs_to_collapse">:: test_tlogs_to_collapse;

From 84e12675bf1031ac89eb99cd142e969b5985f8d7 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 31 Aug 2016 16:16:58 +0200
Subject: [PATCH 233/373] 2 more tests for collapse

---
 pylabs/test/server/left/test_collapse.py | 60 ++++++++++++++++++++++++
 1 file changed, 60 insertions(+)

diff --git a/pylabs/test/server/left/test_collapse.py b/pylabs/test/server/left/test_collapse.py
index 8c247156..c6186f08 100644
--- a/pylabs/test/server/left/test_collapse.py
+++ b/pylabs/test/server/left/test_collapse.py
@@ -20,6 +20,7 @@
 import arakoon
 import time
 import logging
+import subprocess
 from threading import Thread
 from nose.tools import *
 
@@ -85,6 +86,65 @@ def test_remote_collapse():
         up2date = cli.expectProgressPossible()
     logging.info("catchup from collapsed node finished")
 
+@Common.with_custom_setup(Common.setup_3_nodes, Common.basic_teardown)
+def test_remote_collapse_under_load():
+    n = 54321
+    Common.iterate_n_times(n, Common.set_get_and_delete)
+    logging.info("did %i sets, now do bench", n)
+    cluster = Common._getCluster()
+    path = cluster._getConfigFileName() + ".cfg"
+    f = open("./outputFile", 'wb')
+    bench = subprocess.Popen([Common.CONFIG.binary_full_path,
+                              '-config', path ,'--benchmark',
+                              '-scenario','master, set, set_tx, get',
+                              '-max_n', '60000'],
+                             stdout=f,
+                             stderr=f)
+    time.sleep(10.0)
+
+    # bench is up to speed. collapse
+
+    zero = Common.node_names[0]
+    config = Common.getConfig(zero)
+    cluster_id = Common.CONFIG.cluster_id
+    ip = config['ip']
+    port = config['client_port']
+    f2 = open('collapse_output','wb')
+    args = [Common.CONFIG.binary_full_path,
+                                 '--collapse-remote', cluster_id,
+                                 ip,port,str(5)]
+    collapse = subprocess.Popen(args,
+                                stdout = f2,
+                                stderr = f2)
+
+    time.sleep(5)
+    #but stop while collapsing.
+    Common.stopOne(zero)
+
+    #cluster moves on
+    Common.iterate_n_times(n, Common.set_get_and_delete)
+
+
+    Common.startOne(zero)
+    time.sleep(20)
+
+    Common.assert_running_nodes(3)
+    client = Common.get_client()
+    m = client.whoMaster()
+    logging.info("master is:%s", m)
+    assert_true(m in Common.node_names)
+
+@Common.with_custom_setup(Common.setup_3_nodes_mini, Common.basic_teardown)
+def test_collapse_during_catchup():
+    n = 54321
+    Common.iterate_n_times(n, Common.set_get_and_delete)
+    zero = Common.node_names[0]
+    Common.stopOne(zero)
+    Common.iterate_n_times(n, Common.set_get_and_delete)
+    Common.startOne(zero)
+    time.sleep(5)
+    Common.collapse(zero, n=5)
+    Common.assert_running_nodes(3)
 
 @Common.with_custom_setup(Common.setup_2_nodes_mini, Common.basic_teardown)
 def test_remote_collapse_client():

From 1b4925c90d6123770c9391e464737cdc0584c126 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Thu, 1 Sep 2016 08:29:21 +0200
Subject: [PATCH 234/373] align ubuntu starting image

CI will eat less
---
 docker/ubuntu-14.04/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/ubuntu-14.04/Dockerfile b/docker/ubuntu-14.04/Dockerfile
index 97c3c44c..5696064e 100644
--- a/docker/ubuntu-14.04/Dockerfile
+++ b/docker/ubuntu-14.04/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:14.04
+FROM ubuntu:14.04.3
 
 RUN useradd jenkins -u 1500 -g root
 

From 7d59ebebfe2b8d1e24c24af3a375762ccc46823b Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Thu, 1 Sep 2016 11:12:40 +0200
Subject: [PATCH 235/373] make_release: 1.9.11

---
 META                      | 2 +-
 debian/changelog          | 5 +++++
 redhat/SPECS/arakoon.spec | 4 +++-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/META b/META
index 2d4553df..93f7f30c 100644
--- a/META
+++ b/META
@@ -1,7 +1,7 @@
 # -*- conf -*-
 
 description = "Arakoon client"
-version = "1.9.10"
+version = "1.9.11"
 exists_if = "arakoon_client.cma,arakoon_client.cmxa,arakoon_client.cmxs"
 requires = "threads lwt lwt.unix str oUnit quickcheck lwt.ssl ocplib-endian redis.lwt uri"
 archive(byte) = "arakoon_client.cma tcp_keepalive_sockopt_stubs.o"
diff --git a/debian/changelog b/debian/changelog
index a5fad3a9..dd287aae 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,8 @@
+arakoon (1.9.11) unstable; urgency=low
+  * new upstream version
+
+ -- Romain Slootmaekers <romain.slootmaekers@openvstorage.com>  Thu, 01 Sep 2016 11:12:40 +0100
+
 arakoon (1.9.10) unstable; urgency=low
   * new upstream version
 
diff --git a/redhat/SPECS/arakoon.spec b/redhat/SPECS/arakoon.spec
index a27426ea..757abaa5 100644
--- a/redhat/SPECS/arakoon.spec
+++ b/redhat/SPECS/arakoon.spec
@@ -2,7 +2,7 @@
 
 Summary: Arakoon
 Name: arakoon
-Version: 1.9.10
+Version: 1.9.11
 Release: 3%{?dist}
 License: Apache 2.0
 Requires: libev >= 4
@@ -33,6 +33,8 @@ cp arakoon.native %{buildroot}%{_bindir}/arakoon
 %{_bindir}/arakoon
 
 %changelog
+* Thu Sep 01 2016 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.11
+- Create arakoon 1.9.11 RPM package
 * Tue Aug 30 2016 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.10
 - Create arakoon 1.9.10 RPM package
 * Thu Aug 11 2016 Jan Doms <jan.doms@gmail.com> - 1.9.9

From 3aad4b7ca7e833f7ce661135afc4cb2d1f24d419 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Thu, 1 Sep 2016 13:46:13 +0200
Subject: [PATCH 236/373] master can have more valid states

---
 pylabs/test/server/left/test_collapse.py | 13 ++++++++++++-
 pylabs/test/server/quick/TestEtcd.py     |  6 +++---
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/pylabs/test/server/left/test_collapse.py b/pylabs/test/server/left/test_collapse.py
index c6186f08..378dfe4f 100644
--- a/pylabs/test/server/left/test_collapse.py
+++ b/pylabs/test/server/left/test_collapse.py
@@ -144,7 +144,18 @@ def test_collapse_during_catchup():
     Common.startOne(zero)
     time.sleep(5)
     Common.collapse(zero, n=5)
-    Common.assert_running_nodes(3)
+    cluster = Common._getCluster()
+    status = cluster.getStatusOne(zero)
+    ok = status == X.AppStatusType.RUNNING
+    if not ok:
+        cfg = Common.getConfig(zero)
+        log_dir = cfg['log_dir']
+        log_file = '/'.join([log_dir, "%s.log" % (zero) ])
+        tail = subprocess.check_output("tail -50 %s" % log_file, shell = True)
+        for line in tail.split('\n'):
+            logging.info("%s:%s" % (zero, line))
+
+    assert_true(ok)
 
 @Common.with_custom_setup(Common.setup_2_nodes_mini, Common.basic_teardown)
 def test_remote_collapse_client():
diff --git a/pylabs/test/server/quick/TestEtcd.py b/pylabs/test/server/quick/TestEtcd.py
index 25a55c15..27c7c431 100644
--- a/pylabs/test/server/quick/TestEtcd.py
+++ b/pylabs/test/server/quick/TestEtcd.py
@@ -148,7 +148,7 @@ def setUp(self):
         self._server.start()
         cfg = self._config()
         cfg_s = X.cfg2str(cfg)
-        logging.debug("cfg_s=%s",cfg_s)
+        logging.debug("cfg_s='%s'",cfg_s)
         self._etcdClient.set(self._cluster_id,cfg_s)
 
 
@@ -191,7 +191,7 @@ def _arakoon_cli(x, tail = None):
         logging.debug('calling: %s', ' '.join(cmd))
         r = subprocess.check_output(cmd).strip()
         logging.debug("node state:%s", r)
-        self.assertEquals(r, 'Stable_master')
+        assert_true(r in ['Stable_master','Master_dictate'])
         # cli crud
 
         key = "some_key"
@@ -215,7 +215,7 @@ def get() :
         node = 'etcd_ara0'
         cfg.set(node, 'log_level','debug')
         cfg_s = X.cfg2str(cfg)
-        logging.debug("cfg_s= %s", cfg_s)
+        logging.debug("cfg_s= '%s'", cfg_s)
         self._etcdClient.set(self._cluster_id, cfg_s)
         cmd = ["fuser","-n","tcp", "4000","-k", "-%s" % 'USR1' ]
         subprocess.call(cmd)

From f4e8ec3ce5e85d0c31237179dbdf1157f3be9672 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Fri, 2 Sep 2016 08:00:52 +0200
Subject: [PATCH 237/373] test for scenario

---
 pylabs/test/server/shaky/test_issue_93.py | 76 +++++++++++++++++++++++
 1 file changed, 76 insertions(+)
 create mode 100644 pylabs/test/server/shaky/test_issue_93.py

diff --git a/pylabs/test/server/shaky/test_issue_93.py b/pylabs/test/server/shaky/test_issue_93.py
new file mode 100644
index 00000000..1d80bab0
--- /dev/null
+++ b/pylabs/test/server/shaky/test_issue_93.py
@@ -0,0 +1,76 @@
+"""
+"""
+
+from .. import system_tests_common as Common
+from arakoon.ArakoonExceptions import *
+from Compat import X
+from nose.tools import *
+import logging
+import time
+import subprocess
+
+@Common.with_custom_setup(Common.setup_3_nodes, Common.basic_teardown)
+def test_issue_93():
+    pass
+    """
+1. Startup 3 node arakoons
+2. Put alot of data in it
+3. Stop 1 node
+4. Check master
+5. wipe node 1
+6. start node 1 (rebuild starts)
+7. Check master (lost master here)
+"""
+    def launch_bench(n, output, ks = 20):
+        cluster = Common._getCluster()
+        path = cluster._getConfigFileName() + ".cfg"
+        f = open(output, 'wb')
+        bench = subprocess.Popen([Common.CONFIG.binary_full_path,
+                                  '-config', path ,'--benchmark',
+                                  '-scenario','master, set_tx',
+                                  '-key_size', str(ks),
+                                  '-tx_size', '200',
+                                  '-value_size', '400',
+                                  '-max_n', str(n)],
+                                 stdout=f,
+                                 stderr=f)
+        return (bench, f)
+    ## 2
+    n = 1180 * 1000
+
+
+    t0 = time.time()
+    bench, f = launch_bench(n, './bench_output.txt')
+    bench.wait()
+    t1 = time.time()
+    ## 3
+    zero = Common.node_names[0]
+    Common.stopOne(zero)
+
+    ## 4
+    cli = Common.get_client()
+    cli.nop()
+
+    master0 = cli.whoMaster()
+
+    ## 5
+    Common.wipe(zero)
+
+    ## 6
+    Common.startOne(zero)
+
+    ## 7
+
+
+    #### (while rebuilding, put new load.)
+    bench2, f2 = launch_bench(n, './bench_output2.txt', ks = 21 ) # make it grow
+    d = t1 - t0
+    d2 = int(2 * d)
+    for i in xrange(d2):
+        master = cli.whoMaster()
+        assert_true(master == master0)
+        logging.debug("master:%s", master)
+        time.sleep(1.0)
+        cli.nop()
+    bench2.wait()
+    Common.assert_running_nodes(3)

From afd7051b2fc0392427ece24a6bb87895be923b0b Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Mon, 12 Sep 2016 15:03:24 +0200
Subject: [PATCH 238/373] load plugins for local collapse

---
 src/node/collapser.ml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/node/collapser.ml b/src/node/collapser.ml
index aebbb847..b9323531 100644
--- a/src/node/collapser.ml
+++ b/src/node/collapser.ml
@@ -231,6 +231,8 @@ let collapse_out_of_band cfg name tlogs_to_keep =
   let me, _ = split name cfg.cfgs in
   let head_location = Filename.concat me.head_dir Tlc2.head_fname in
 
+  Plugin_loader.load me.home cfg.plugins >>= fun () ->
+
   let module S = (val (Store.make_store_module (module Batched_store.Local_store))) in
   Tlc2.make_tlc2 ~compressor:me.compressor
                  me.tlog_dir me.tlx_dir me.head_dir

From 70ebaa4ac2a081a76a60edf2db17e442f453d6cf Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Tue, 20 Sep 2016 10:34:43 +0200
Subject: [PATCH 239/373] make_release: 1.9.12

---
 META                      | 2 +-
 debian/changelog          | 5 +++++
 redhat/SPECS/arakoon.spec | 4 +++-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/META b/META
index 93f7f30c..72a73e10 100644
--- a/META
+++ b/META
@@ -1,7 +1,7 @@
 # -*- conf -*-
 
 description = "Arakoon client"
-version = "1.9.11"
+version = "1.9.12"
 exists_if = "arakoon_client.cma,arakoon_client.cmxa,arakoon_client.cmxs"
 requires = "threads lwt lwt.unix str oUnit quickcheck lwt.ssl ocplib-endian redis.lwt uri"
 archive(byte) = "arakoon_client.cma tcp_keepalive_sockopt_stubs.o"
diff --git a/debian/changelog b/debian/changelog
index dd287aae..180d2b90 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,8 @@
+arakoon (1.9.12) unstable; urgency=low
+  * new upstream version
+
+ -- Jan Doms <jan.doms@gmail.com>  Tue, 20 Sep 2016 10:34:43 +0100
+
 arakoon (1.9.11) unstable; urgency=low
   * new upstream version
 
diff --git a/redhat/SPECS/arakoon.spec b/redhat/SPECS/arakoon.spec
index 757abaa5..fcbf40ef 100644
--- a/redhat/SPECS/arakoon.spec
+++ b/redhat/SPECS/arakoon.spec
@@ -2,7 +2,7 @@
 
 Summary: Arakoon
 Name: arakoon
-Version: 1.9.11
+Version: 1.9.12
 Release: 3%{?dist}
 License: Apache 2.0
 Requires: libev >= 4
@@ -33,6 +33,8 @@ cp arakoon.native %{buildroot}%{_bindir}/arakoon
 %{_bindir}/arakoon
 
 %changelog
+* Tue Sep 20 2016 Jan Doms <jan.doms@gmail.com> - 1.9.12
+- Create arakoon 1.9.12 RPM package
 * Thu Sep 01 2016 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.11
 - Create arakoon 1.9.11 RPM package
 * Tue Aug 30 2016 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.10

From b0097de9b4cd53d64800de339406b25ddecd6754 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Thu, 22 Sep 2016 16:38:54 +0200
Subject: [PATCH 240/373] update ocaml-redis

---
 docker/arch/Dockerfile         | 2 +-
 docker/centos-7/Dockerfile     | 2 +-
 docker/trisquel/Dockerfile     | 2 +-
 docker/ubuntu-14.04/Dockerfile | 2 +-
 docker/ubuntu-16.04/Dockerfile | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/docker/arch/Dockerfile b/docker/arch/Dockerfile
index bafb85f9..28787cc3 100644
--- a/docker/arch/Dockerfile
+++ b/docker/arch/Dockerfile
@@ -60,7 +60,7 @@ RUN eval `${opam_env}` && \
         quickcheck.1.0.2 \
         conf-libev \
         core.113.33.02+4.03 \
-        redis.0.3.1 \
+        redis.0.3.3 \
         uri.1.9.2
 
 # Install etcd:
diff --git a/docker/centos-7/Dockerfile b/docker/centos-7/Dockerfile
index f2c13526..3e98db57 100644
--- a/docker/centos-7/Dockerfile
+++ b/docker/centos-7/Dockerfile
@@ -68,7 +68,7 @@ RUN eval `${opam_env}` && \
         quickcheck.1.0.2 \
         conf-libev \
         core.114.05+21 \
-        redis.0.3.1 \
+        redis.0.3.3 \
         uri.1.9.2
 
 
diff --git a/docker/trisquel/Dockerfile b/docker/trisquel/Dockerfile
index 22bdf91a..9849f06f 100644
--- a/docker/trisquel/Dockerfile
+++ b/docker/trisquel/Dockerfile
@@ -50,7 +50,7 @@ RUN eval `${opam_env}` && \
         quickcheck.1.0.2 \
         conf-libev \
         core.113.33.02+4.03 \
-        redis.0.3.1 \
+        redis.0.3.3 \
         uri.1.9.2
 
 
diff --git a/docker/ubuntu-14.04/Dockerfile b/docker/ubuntu-14.04/Dockerfile
index 5696064e..07289c32 100644
--- a/docker/ubuntu-14.04/Dockerfile
+++ b/docker/ubuntu-14.04/Dockerfile
@@ -64,7 +64,7 @@ RUN eval `${opam_env}` && \
         quickcheck.1.0.2 \
         conf-libev \
         core.114.05+21 \
-        redis.0.3.1 \
+        redis.0.3.3 \
         uri.1.9.2
 
 
diff --git a/docker/ubuntu-16.04/Dockerfile b/docker/ubuntu-16.04/Dockerfile
index ee6d845f..000b0671 100644
--- a/docker/ubuntu-16.04/Dockerfile
+++ b/docker/ubuntu-16.04/Dockerfile
@@ -61,7 +61,7 @@ RUN eval `${opam_env}` && \
         quickcheck.1.0.2 \
         conf-libev \
         core.114.05+21 \
-        redis.0.3.1 \
+        redis.0.3.3 \
         uri.1.9.2
 
 

From f9048a5f60b0181378bab76a9850dbf6dbb9f9ba Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Fri, 30 Sep 2016 10:31:44 +0200
Subject: [PATCH 241/373] make_release: 1.9.13

---
 META                      | 2 +-
 debian/changelog          | 5 +++++
 redhat/SPECS/arakoon.spec | 4 +++-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/META b/META
index 72a73e10..c0a5e1f5 100644
--- a/META
+++ b/META
@@ -1,7 +1,7 @@
 # -*- conf -*-
 
 description = "Arakoon client"
-version = "1.9.12"
+version = "1.9.13"
 exists_if = "arakoon_client.cma,arakoon_client.cmxa,arakoon_client.cmxs"
 requires = "threads lwt lwt.unix str oUnit quickcheck lwt.ssl ocplib-endian redis.lwt uri"
 archive(byte) = "arakoon_client.cma tcp_keepalive_sockopt_stubs.o"
diff --git a/debian/changelog b/debian/changelog
index 180d2b90..86f20ccc 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,8 @@
+arakoon (1.9.13) unstable; urgency=low
+  * new upstream version
+
+ -- Romain Slootmaekers <romain.slootmaekers@openvstorage.com>  Fri, 30 Sep 2016 10:31:44 +0100
+
 arakoon (1.9.12) unstable; urgency=low
   * new upstream version
 
diff --git a/redhat/SPECS/arakoon.spec b/redhat/SPECS/arakoon.spec
index fcbf40ef..ecaf3ad8 100644
--- a/redhat/SPECS/arakoon.spec
+++ b/redhat/SPECS/arakoon.spec
@@ -2,7 +2,7 @@
 
 Summary: Arakoon
 Name: arakoon
-Version: 1.9.12
+Version: 1.9.13
 Release: 3%{?dist}
 License: Apache 2.0
 Requires: libev >= 4
@@ -33,6 +33,8 @@ cp arakoon.native %{buildroot}%{_bindir}/arakoon
 %{_bindir}/arakoon
 
 %changelog
+* Fri Sep 30 2016 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.13
+- Create arakoon 1.9.13 RPM package
 * Tue Sep 20 2016 Jan Doms <jan.doms@gmail.com> - 1.9.12
 - Create arakoon 1.9.12 RPM package
 * Thu Sep 01 2016 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.11

From 6389c41da671adba3fe9f8497b3f9a48ff185a19 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Mon, 3 Oct 2016 09:57:58 +0200
Subject: [PATCH 242/373] failing test for learner case

---
 pylabs/test/server/right/system_tests_learner.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/pylabs/test/server/right/system_tests_learner.py b/pylabs/test/server/right/system_tests_learner.py
index e8bc19e0..221479ea 100644
--- a/pylabs/test/server/right/system_tests_learner.py
+++ b/pylabs/test/server/right/system_tests_learner.py
@@ -24,7 +24,7 @@
 
 @Common.with_custom_setup(Common.setup_2_nodes_mini, Common.basic_teardown)
 def test_learner():
-    op_count = 5432
+    op_count = 54321
     Common.iterate_n_times(op_count, Common.simple_set)
     cluster = Common._getCluster(Common.cluster_id)
     logging.info("adding learner")
@@ -54,5 +54,15 @@ def test_learner():
     time.sleep(op_count / 1000 + 1 ) # 1000/s in catchup should be no problem
     #use a client ??"
     Common.stop_all()
+    i1 = int(Common.get_last_i_tlog(name))
+    assert_true(i1 >= op_count - 1)
+
+    Common.start_all()
+    time.sleep(1.0)
+    Common.iterate_n_times(1000, Common.simple_set)
+    Common.stop_all()
+
     i2 = int(Common.get_last_i_tlog(name))
-    assert_true(i2 >= op_count - 1)
+    assert_true(i2 >= i1 + 900)
+    i_normal_node = int(Common.get_last_i_tlog(Common.node_names[0]))
+    assert_true (i2 >= i_normal_node - 100) # it should approximately follow the normal node

From c4324381973b3d3db3e276f8ae88fce7cde062d8 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Tue, 4 Oct 2016 11:45:16 +0200
Subject: [PATCH 243/373] control where tests live

---
 docker/run.sh    | 1 +
 docker/suites.sh | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docker/run.sh b/docker/run.sh
index 174e07d4..8fc89da1 100755
--- a/docker/run.sh
+++ b/docker/run.sh
@@ -16,6 +16,7 @@ else
            bash -l -c "cd arakoon && ./docker/suites.sh clean"
 fi
 
+
 docker run -i $TTY --privileged=true -e UID=${UID} \
        --env ARAKOON_PYTHON_CLIENT \
        -v ${PWD}:/home/jenkins/arakoon \
diff --git a/docker/suites.sh b/docker/suites.sh
index c34def7c..da60e61d 100755
--- a/docker/suites.sh
+++ b/docker/suites.sh
@@ -1,6 +1,6 @@
 #!/bin/bash -xue
 
-export TEST_HOME='/home/tests'
+export TEST_HOME='/home/jenkins/arakoon/TESTS'
 
 case "${1-bash}" in
     bash)

From d17c6481b390599e2095dde17e3dbaf4db515b7b Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Tue, 4 Oct 2016 14:03:37 +0200
Subject: [PATCH 244/373] add a 'nose' suite

---
 docker/docker-entrypoint.sh | 1 +
 docker/run.sh               | 3 ++-
 docker/suites.sh            | 5 +++++
 jenkins/nose.sh             | 5 +++++
 4 files changed, 13 insertions(+), 1 deletion(-)
 create mode 100755 jenkins/nose.sh

diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh
index c5d15f8c..f2d874db 100755
--- a/docker/docker-entrypoint.sh
+++ b/docker/docker-entrypoint.sh
@@ -15,4 +15,5 @@ then
 fi
 
 # finally execute the command the user requested
+echo "PARAMETERS=$@"
 exec sudo ARAKOON_PYTHON_CLIENT=${ARAKOON_PYTHON_CLIENT-} -i -u jenkins "$@"
diff --git a/docker/run.sh b/docker/run.sh
index 8fc89da1..dfa8cd5d 100755
--- a/docker/run.sh
+++ b/docker/run.sh
@@ -16,9 +16,10 @@ else
            bash -l -c "cd arakoon && ./docker/suites.sh clean"
 fi
 
+command="cd arakoon && ./docker/suites.sh ${@}"
 
 docker run -i $TTY --privileged=true -e UID=${UID} \
        --env ARAKOON_PYTHON_CLIENT \
        -v ${PWD}:/home/jenkins/arakoon \
        -w /home/jenkins/arakoon arakoon_$IMAGE \
-       bash -l -c "cd arakoon && ./docker/suites.sh $@"
+       bash -l -c "${command}"
diff --git a/docker/suites.sh b/docker/suites.sh
index da60e61d..6d80df66 100755
--- a/docker/suites.sh
+++ b/docker/suites.sh
@@ -35,6 +35,11 @@ case "${1-bash}" in
     package_rpm)
         ./jenkins/package_rpm.sh
         ;;
+    nose)
+        shift
+        echo "parameters=$@"
+        ./jenkins/nose.sh $@
+        ;;
     *)
         echo "invalid test suite specified"
 esac
diff --git a/jenkins/nose.sh b/jenkins/nose.sh
new file mode 100755
index 00000000..7660a6ca
--- /dev/null
+++ b/jenkins/nose.sh
@@ -0,0 +1,5 @@
+#!/bin/bash -xue
+
+./jenkins/common.sh
+echo "nose.sh:PARAMETERS=$@"
+python test_it.py -v --with-xunit --xunit-file=${PWD}/testresults.xml $@

From c6934cf555d3508891a27bfc5f65cc75559b0476 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Tue, 4 Oct 2016 15:57:25 +0200
Subject: [PATCH 245/373] which pgrep?

---
 Makefile                             | 1 +
 jenkins/nose.sh                      | 2 +-
 pylabs/test/server/quick/TestEtcd.py | 4 +++-
 3 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 99ebf53f..53b98578 100644
--- a/Makefile
+++ b/Makefile
@@ -13,6 +13,7 @@ clean:
 	rm -rf ./debian/tmp/*
 	rm -rf ./debian/libarakoon-ocaml-dev/*
 	ocamlbuild -clean
+	rm -f  ./arakoon.byte ./arakoon.native
 
 build:
 	ocamlbuild -j 4 -use-ocamlfind arakoon.byte arakoon.native arakoon_client.cma arakoon_client.cmxa arakoon_client.a arakoon_client.cmxs plugin_helper.cmi
diff --git a/jenkins/nose.sh b/jenkins/nose.sh
index 7660a6ca..4c34697f 100755
--- a/jenkins/nose.sh
+++ b/jenkins/nose.sh
@@ -2,4 +2,4 @@
 
 ./jenkins/common.sh
 echo "nose.sh:PARAMETERS=$@"
-python test_it.py -v --with-xunit --xunit-file=${PWD}/testresults.xml $@
+python test_it.py -v -s --with-xunit --xunit-file=${PWD}/testresults.xml $@
diff --git a/pylabs/test/server/quick/TestEtcd.py b/pylabs/test/server/quick/TestEtcd.py
index 27c7c431..2a566370 100644
--- a/pylabs/test/server/quick/TestEtcd.py
+++ b/pylabs/test/server/quick/TestEtcd.py
@@ -168,7 +168,9 @@ def tearDown(self):
     def testEtcdUsage(self):
         logging.debug("testUsage")
         url = self._server.url(self._cluster_id)
-        subprocess.call("pgrep -a arakoon", shell = True)
+
+        subprocess.call("for x in $(pgrep arakoon); do cat /proc/$x/cmdline; echo; done",
+                        shell = True)
 
         # startup
         def _arakoon_cli(x, tail = None):

From ea16607a93a9385bca81d86f37ad90eaa41375a5 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 5 Oct 2016 13:46:47 +0200
Subject: [PATCH 246/373] is fuser a better option?

---
 .../arakoon_ext/server/ArakoonManagement.py         | 13 ++++++++++++-
 pylabs/test/server/system_tests_common.py           |  5 +++++
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/pylabs/extensions/arakoon_ext/server/ArakoonManagement.py b/pylabs/extensions/arakoon_ext/server/ArakoonManagement.py
index 1a18290c..f980afa4 100644
--- a/pylabs/extensions/arakoon_ext/server/ArakoonManagement.py
+++ b/pylabs/extensions/arakoon_ext/server/ArakoonManagement.py
@@ -1303,9 +1303,20 @@ def _getIp(self,ip_mess):
 
     def _stopOne(self, name):
         line = self._cmdLine(name)
-        cmd = ['pkill', '-f',  line]
+
+        use_fuser = True
+        cmd = None
+        if use_fuser:
+            config = self._getConfigFile()
+            port = int(config.get(name, "client_port"))
+            cmd = ("fuser -s -n tcp %i -k -SIGTERM" % port ).split(' ')
+        else:
+            #This flavour gave problems on jenkins 14.04.3 docker containers
+            cmd = ['pkill', '-f',  line]
+
         logging.debug("stopping '%s' with: %s",name, string.join(cmd, ' '))
         rc = subprocess.call(cmd, close_fds = True)
+
         logging.debug("%s=>rc=%i" % (cmd,rc))
         i = 0
         while(self._getStatusOne(name) == X.AppStatusType.RUNNING):
diff --git a/pylabs/test/server/system_tests_common.py b/pylabs/test/server/system_tests_common.py
index 55d2178d..5d44a8c0 100644
--- a/pylabs/test/server/system_tests_common.py
+++ b/pylabs/test/server/system_tests_common.py
@@ -651,6 +651,11 @@ def setup_n_nodes_base(c_id, node_names, force_master,
                        witness_nodes = False, useIPV6=False,
                        slowCollapser = False):
 
+    running_arakoons = X.subprocess.check_output("pgrep -a arakoon || true",
+                                                 shell = True)
+    print "running_arakoons\n", running_arakoons
+    logging.info("running_arakoons=%s", running_arakoons)
+
     X.subprocess.check_call("sudo /sbin/iptables -F".split(' ') )
 
     cluster = _getCluster( c_id )

From 2fd32697a837ac9186b0ba87ba16ef452646633e Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Thu, 6 Oct 2016 17:15:37 +0200
Subject: [PATCH 247/373] Popen + wait

& no naked subprocess usage
---
 .../arakoon_ext/server/ArakoonManagement.py   | 21 +++++++++---------
 pylabs/test/server/left/test_collapse.py      | 14 +++++++-----
 pylabs/test/server/quick/system_tests_ssl.py  | 22 +++++++++----------
 pylabs/test/server/system_tests_common.py     |  8 +++----
 4 files changed, 35 insertions(+), 30 deletions(-)

diff --git a/pylabs/extensions/arakoon_ext/server/ArakoonManagement.py b/pylabs/extensions/arakoon_ext/server/ArakoonManagement.py
index f980afa4..55a1f2bb 100644
--- a/pylabs/extensions/arakoon_ext/server/ArakoonManagement.py
+++ b/pylabs/extensions/arakoon_ext/server/ArakoonManagement.py
@@ -19,7 +19,7 @@
 import ArakoonRemoteControl
 import os.path
 import itertools
-import subprocess
+
 import time
 import types
 import signal
@@ -1108,7 +1108,7 @@ def catchupOnly(self, nodeName):
                '--node',
                nodeName,
                '-catchup-only']
-        return subprocess.call(cmd)
+        return X.subprocess.call(cmd)
 
     def stopOne(self, nodeName):
         """
@@ -1179,7 +1179,7 @@ def injectAsHead(self, nodeName, newHead, force=False, inPlace=False):
         if inPlace:
             cmd.append('--inplace')
 
-        p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+        p = X.subprocess.check_call(cmd, stdout = X. subprocess.PIPE)
         output = p.communicate()[0]
         rc = p.returncode
         logging.debug("injectAsHead returned [%d] %s", rc, output)
@@ -1288,7 +1288,7 @@ def _startOne(self, name, extraParams = None):
             cmd.extend(extraParams)
         cmd.append('-daemonize')
         logging.debug('calling: %s', ' '.join(cmd))
-        return subprocess.call(cmd, close_fds = True)
+        return X.subprocess.call(cmd, close_fds = True)
 
     def _getIp(self,ip_mess):
         t_mess = type(ip_mess)
@@ -1312,15 +1312,15 @@ def _stopOne(self, name):
             cmd = ("fuser -s -n tcp %i -k -SIGTERM" % port ).split(' ')
         else:
             #This flavour gave problems on jenkins 14.04.3 docker containers
-            cmd = ['pkill', '-f',  line]
+            cmd = ['pkill', '-fn',  line]
 
         logging.debug("stopping '%s' with: %s",name, string.join(cmd, ' '))
-        rc = subprocess.call(cmd, close_fds = True)
+        rc = X.subprocess.call(cmd, close_fds = True)
 
         logging.debug("%s=>rc=%i" % (cmd,rc))
         i = 0
         while(self._getStatusOne(name) == X.AppStatusType.RUNNING):
-            rc = subprocess.call(cmd, close_fds = True)
+            rc = X.subprocess.call(cmd, close_fds = True)
             logging.debug("%s=>rc=%i" % (cmd,rc))
             time.sleep(1)
             i += 1
@@ -1369,10 +1369,11 @@ def _getPid(self, name):
     def _getStatusOne(self,name):
         line = self._cmdLine(name)
         cmd = ['pgrep','-fn', line]
-        proc = subprocess.Popen(cmd,
-                                close_fds = True,
-                                stdout=subprocess.PIPE)
+        proc = X.subprocess.Popen(cmd,
+                                  close_fds = True,
+                                  stdout = X.subprocess.PIPE)
         pids = proc.communicate()[0]
+        proc.wait()
         pid_list = pids.split()
         lenp = len(pid_list)
         result = None
diff --git a/pylabs/test/server/left/test_collapse.py b/pylabs/test/server/left/test_collapse.py
index 378dfe4f..90614c07 100644
--- a/pylabs/test/server/left/test_collapse.py
+++ b/pylabs/test/server/left/test_collapse.py
@@ -20,7 +20,7 @@
 import arakoon
 import time
 import logging
-import subprocess
+
 from threading import Thread
 from nose.tools import *
 
@@ -94,7 +94,7 @@ def test_remote_collapse_under_load():
     cluster = Common._getCluster()
     path = cluster._getConfigFileName() + ".cfg"
     f = open("./outputFile", 'wb')
-    bench = subprocess.Popen([Common.CONFIG.binary_full_path,
+    bench = X.subprocess.Popen([Common.CONFIG.binary_full_path,
                               '-config', path ,'--benchmark',
                               '-scenario','master, set, set_tx, get',
                               '-max_n', '60000'],
@@ -113,9 +113,9 @@ def test_remote_collapse_under_load():
     args = [Common.CONFIG.binary_full_path,
                                  '--collapse-remote', cluster_id,
                                  ip,port,str(5)]
-    collapse = subprocess.Popen(args,
-                                stdout = f2,
-                                stderr = f2)
+    collapse = X.subprocess.Popen(args,
+                                  stdout = f2,
+                                  stderr = f2)
 
     time.sleep(5)
     #but stop while collapsing.
@@ -134,6 +134,10 @@ def test_remote_collapse_under_load():
     logging.info("master is:%s", m)
     assert_true(m in Common.node_names)
 
+    #
+    bench.wait()
+    collapse.wait()
+
 @Common.with_custom_setup(Common.setup_3_nodes_mini, Common.basic_teardown)
 def test_collapse_during_catchup():
     n = 54321
diff --git a/pylabs/test/server/quick/system_tests_ssl.py b/pylabs/test/server/quick/system_tests_ssl.py
index 071dd59c..c40be0e3 100644
--- a/pylabs/test/server/quick/system_tests_ssl.py
+++ b/pylabs/test/server/quick/system_tests_ssl.py
@@ -24,7 +24,7 @@
 import tempfile
 import unittest
 import contextlib
-import subprocess
+
 
 from .. import system_tests_common as C
 from Compat import X
@@ -79,8 +79,8 @@ def wait_for_master(cfg, nodes, settings, ca_path, key, timeout, interval):
     while time.time() < end:
         LOGGER.info('Retrieving master')
 
-        proc = subprocess.Popen(args, close_fds=True,
-            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        proc = X.subprocess.Popen(args, close_fds=True,
+                                  stdout = X.subprocess.PIPE, stderr = X.subprocess.PIPE)
 
         rc = proc.wait()
 
@@ -124,7 +124,7 @@ def start_nodes(self, config_path, nodes):
         return map(lambda n: self.start_node(config_path, n), nodes)
 
     def run_async(self, *a, **k):
-        proc = subprocess.Popen(*a, **k)
+        proc = X.subprocess.Popen(*a, **k)
         self._processes.add(proc)
         return proc
 
@@ -167,12 +167,12 @@ def make_ca(path):
         'openssl req -new -nodes -out cacert-req.pem -keyout cacert.key -subj'.split()
     args.append(subject)
 
-    subprocess.check_call(args, close_fds=True, cwd=path)
+    X.subprocess.check_call(args, close_fds=True, cwd=path)
 
     # Self-sign CA CSR
     args = \
         'openssl x509 -signkey cacert.key -req -in cacert-req.pem -out cacert.pem'.split()
-    subprocess.check_call(args, close_fds=True, cwd=path)
+    X.subprocess.check_call(args, close_fds=True, cwd=path)
 
     os.unlink(os.path.join(path, 'cacert-req.pem'))
 
@@ -185,7 +185,7 @@ def make_node_cert(path, name, serial):
     args = args.split()
     args.append(subject)
 
-    subprocess.check_call(args, close_fds=True, cwd=path)
+    X.subprocess.check_call(args, close_fds=True, cwd=path)
 
     # Sign
     args = \
@@ -193,7 +193,7 @@ def make_node_cert(path, name, serial):
         {'name': name, 'serial': serial}
     args = args.split()
 
-    subprocess.check_call(args, close_fds=True, cwd=path)
+    X.subprocess.check_call(args, close_fds=True, cwd=path)
 
     os.unlink(os.path.join(path, '%s-req.pem' % name))
 
@@ -202,7 +202,7 @@ def make_node_cert(path, name, serial):
         'openssl verify -CAfile cacert.pem %(name)s.pem' % {'name': name}
     args = args.split()
 
-    subprocess.check_call(args, close_fds=True, cwd=path)
+    X.subprocess.check_call(args, close_fds=True, cwd=path)
 
 @contextlib.contextmanager
 def config(template, nodes):
@@ -345,8 +345,8 @@ def test_cert(self):
 
                 args = args.split()
 
-                proc = subprocess.Popen(args, stdin=subprocess.PIPE,
-                    stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                proc = X.subprocess.Popen(args, stdin = X.subprocess.PIPE,
+                    stdout=subprocess.PIPE, stderr= X.subprocess.PIPE,
                     close_fds=True)
                 proc.stdin.close()
 
diff --git a/pylabs/test/server/system_tests_common.py b/pylabs/test/server/system_tests_common.py
index 5d44a8c0..1a74e560 100644
--- a/pylabs/test/server/system_tests_common.py
+++ b/pylabs/test/server/system_tests_common.py
@@ -651,10 +651,10 @@ def setup_n_nodes_base(c_id, node_names, force_master,
                        witness_nodes = False, useIPV6=False,
                        slowCollapser = False):
 
-    running_arakoons = X.subprocess.check_output("pgrep -a arakoon || true",
-                                                 shell = True)
-    print "running_arakoons\n", running_arakoons
-    logging.info("running_arakoons=%s", running_arakoons)
+    running_arakoons = X.subprocess.check_output("pgrep -c arakoon || true",
+                                                 shell = True).strip()
+    print "\n\tpgrep -c arakoon =>%s" % running_arakoons
+    logging.info("pgrep -c arakoon =>%s", running_arakoons)
 
     X.subprocess.check_call("sudo /sbin/iptables -F".split(' ') )
 

From 2fe4fc9d9c16351448be6d3792d9d23fa7f5e29d Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Fri, 7 Oct 2016 01:11:54 +0200
Subject: [PATCH 248/373] does python matter?

---
 docker/ubuntu-14.04/Dockerfile | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/docker/ubuntu-14.04/Dockerfile b/docker/ubuntu-14.04/Dockerfile
index 07289c32..aad649be 100644
--- a/docker/ubuntu-14.04/Dockerfile
+++ b/docker/ubuntu-14.04/Dockerfile
@@ -67,7 +67,11 @@ RUN eval `${opam_env}` && \
         redis.0.3.3 \
         uri.1.9.2
 
-
+#
+RUN wget https://www.python.org/ftp/python/2.7.12/Python-2.7.12.tar.xz
+RUN tar -Jxvf ./Python-2.7.12.tar.xz
+RUN cd /Python-2.7.12 && ./configure && make && sudo make install
+RUN curl https://bootstrap.pypa.io/get-pip.py | python
 RUN pip install fabric junit-xml nose simplejson python-etcd
 
 RUN mkdir /home/tests
@@ -93,4 +97,5 @@ RUN su - -c "echo 'eval `${opam_env}`' >> /home/jenkins/.profile"
 RUN echo "jenkins ALL=NOPASSWD: ALL" >/etc/sudoers.d/jenkins
 
 
+
 ENTRYPOINT ["/home/jenkins/arakoon/docker/docker-entrypoint.sh"]

From 59c51cd27837c91b04cbf50b64c2b8ae94e80db8 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Fri, 7 Oct 2016 11:50:03 +0200
Subject: [PATCH 249/373] avoid -daemonize

---
 .../arakoon_ext/server/ArakoonManagement.py   |  5 +-
 .../test/server/quick/system_tests_basic.py   | 65 +++++++++++--------
 pylabs/test/server/system_tests_common.py     |  2 +-
 3 files changed, 41 insertions(+), 31 deletions(-)

diff --git a/pylabs/extensions/arakoon_ext/server/ArakoonManagement.py b/pylabs/extensions/arakoon_ext/server/ArakoonManagement.py
index 55a1f2bb..b26127d2 100644
--- a/pylabs/extensions/arakoon_ext/server/ArakoonManagement.py
+++ b/pylabs/extensions/arakoon_ext/server/ArakoonManagement.py
@@ -1286,9 +1286,9 @@ def _startOne(self, name, extraParams = None):
         cmd.extend(command)
         if extraParams:
             cmd.extend(extraParams)
-        cmd.append('-daemonize')
+        #cmd.append('-daemonize') # daemonize & docker don't play nice
         logging.debug('calling: %s', ' '.join(cmd))
-        return X.subprocess.call(cmd, close_fds = True)
+        return X.subprocess.Popen(cmd, close_fds = True)
 
     def _getIp(self,ip_mess):
         t_mess = type(ip_mess)
@@ -1333,6 +1333,7 @@ def _stopOne(self, name):
                 time.sleep(1)
 
                 logging.debug("stopping '%s' with kill -9" % name)
+                rc = X.subprocess.call(['pkill', '--signal', 'SIGCHLD' '-f', line], close_fds = True)
                 rc = X.subprocess.call(['pkill', '-9', '-f', line], close_fds = True)
                 if rc == 0:
                     rc = 9
diff --git a/pylabs/test/server/quick/system_tests_basic.py b/pylabs/test/server/quick/system_tests_basic.py
index 7b51c6af..ae7d3dd5 100644
--- a/pylabs/test/server/quick/system_tests_basic.py
+++ b/pylabs/test/server/quick/system_tests_basic.py
@@ -55,35 +55,44 @@ def test_start_stop_three_nodes_forced () :
 
 def test_start_stop_wrapper():
     cluster = C._getCluster()
-    fn = "/tmp/my_wrapper.sh"
-    with open(fn,'w') as f:
-        f.write('#!/bin/bash\n')
-        f.write("logger wrapper called with '$@'\n")
-        f.write('$@\n')
-    subprocess.call(['chmod','+x',fn])
-    nn = "wrapper"
-    cluster.addNode(nn, "127.0.0.1", 8000, wrapper = fn)
-    cluster.addLocalNode(nn)
-    cluster.createDirs(nn)
-    C.assert_running_nodes(0)
-    cluster.start()
-    C.assert_running_nodes(1)
-    cluster.stop()
-    C.assert_running_nodes(0)
-    cluster.tearDown()
+    fn = "%s/my_wrapper.sh" % X.tmpDir
+    X.createDir(X.tmpDir)
+    try:
+        with open(fn,'w') as f:
+            f.write('#!/bin/bash -xue\n')
+            f.write("logger wrapper called with '$@'\n")
+            f.write('$@\n')
+        subprocess.call(['chmod','+x',fn])
+        nn = "wrapper"
+        cluster.addNode(nn, "127.0.0.1", 8000, wrapper = fn)
+        cluster.addLocalNode(nn)
+        cluster.createDirs(nn)
+        C.assert_running_nodes(0)
+        cluster.start()
+        C.assert_running_nodes(1)
+        cluster.stop()
+        C.assert_running_nodes(0)
+    finally:
+        cluster.tearDown()
 
 def test_start_arakoon383():
     cluster = C._getCluster()
-    wrapper = 'false'
-    name = 'wrapper'
-    cluster.addNode(name, '127.0.0.1', 8000, wrapper=wrapper)
-    cluster.addLocalNode(name)
-    C.assert_running_nodes(0)
-    assert_equals(cluster.startOne(name), 1)
-    C.assert_running_nodes(0)
-    assert_equals(cluster.restartOne(name), 1)
-    C.assert_running_nodes(0)
-    cluster.tearDown()
+    try:
+        wrapper = 'false'
+        name = 'wrapper'
+        cluster.addNode(name, '127.0.0.1', 8000, wrapper=wrapper)
+        cluster.addLocalNode(name)
+        C.assert_running_nodes(0)
+        p = cluster.startOne(name)
+        rc = p.wait()
+        assert_equals(rc, 1)
+        C.assert_running_nodes(0)
+        p = cluster.restartOne(name)
+        rc = p.wait()
+        assert_equals(rc, 1)
+        C.assert_running_nodes(0)
+    finally:
+        cluster.tearDown()
 
 @C.with_custom_setup(C.setup_1_node, C.basic_teardown)
 def test_max_value_size_tinkering ():
@@ -494,7 +503,7 @@ def test_consistency():
     v = client.get('x')
     assert_equals(v,'X')
 
-    
+
     m2 = X.arakoon_client.AtLeast(1000)
     client.setConsistency(m2)
     try:
@@ -755,7 +764,7 @@ def test_download_db():
     C.flush_store(n1)
     C.stop_all()
 
-    C.startOne(n0)
+    C.startOne
     time.sleep(1.0)
     db_file = C.get_node_db_file (n1)
 
diff --git a/pylabs/test/server/system_tests_common.py b/pylabs/test/server/system_tests_common.py
index 1a74e560..74fa955a 100644
--- a/pylabs/test/server/system_tests_common.py
+++ b/pylabs/test/server/system_tests_common.py
@@ -1112,7 +1112,7 @@ def assert_running_nodes(n):
                                            stderr = X.subprocess.STDOUT
         )
         lines = output.strip().split('\n')
-        logging.info(lines)
+        logging.info("assert_running_nodes:%s", lines)
         count = len(lines)
     except subprocess.CalledProcessError,ex:
         logging.info("ex:%s => count = 0" % ex)

From 255287c8863eb99f9ff2f93c6394bb517b316c2e Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Fri, 7 Oct 2016 14:47:21 +0200
Subject: [PATCH 250/373] bash as PID 1 in container

---
 docker/docker-entrypoint.sh    | 4 ++--
 docker/run.sh                  | 7 +++----
 docker/ubuntu-14.04/Dockerfile | 2 +-
 3 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh
index f2d874db..f221357d 100755
--- a/docker/docker-entrypoint.sh
+++ b/docker/docker-entrypoint.sh
@@ -15,5 +15,5 @@ then
 fi
 
 # finally execute the command the user requested
-echo "PARAMETERS=$@"
-exec sudo ARAKOON_PYTHON_CLIENT=${ARAKOON_PYTHON_CLIENT-} -i -u jenkins "$@"
+command="cd arakoon && ./docker/suites.sh $@"
+sudo -E -i -u jenkins bash -l -c "$command"
diff --git a/docker/run.sh b/docker/run.sh
index dfa8cd5d..cd2bae71 100755
--- a/docker/run.sh
+++ b/docker/run.sh
@@ -13,13 +13,12 @@ else
     docker run -i $TTY --privileged=true -e UID=${UID} \
            -v ${PWD}:/home/jenkins/arakoon \
            -w /home/jenkins/arakoon arakoon_$IMAGE \
-           bash -l -c "cd arakoon && ./docker/suites.sh clean"
+           bash clean
 fi
 
-command="cd arakoon && ./docker/suites.sh ${@}"
 
 docker run -i $TTY --privileged=true -e UID=${UID} \
        --env ARAKOON_PYTHON_CLIENT \
        -v ${PWD}:/home/jenkins/arakoon \
-       -w /home/jenkins/arakoon arakoon_$IMAGE \
-       bash -l -c "${command}"
+       arakoon_$IMAGE \
+       bash $@
diff --git a/docker/ubuntu-14.04/Dockerfile b/docker/ubuntu-14.04/Dockerfile
index aad649be..cd54b5bb 100644
--- a/docker/ubuntu-14.04/Dockerfile
+++ b/docker/ubuntu-14.04/Dockerfile
@@ -98,4 +98,4 @@ RUN echo "jenkins ALL=NOPASSWD: ALL" >/etc/sudoers.d/jenkins
 
 
 
-ENTRYPOINT ["/home/jenkins/arakoon/docker/docker-entrypoint.sh"]
+ENTRYPOINT ["/bin/bash", "-c", "set -e && /home/jenkins/arakoon/docker/docker-entrypoint.sh $@"]

From 5c78b694b6d26a003486dbbc071609fc41818d08 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Fri, 7 Oct 2016 11:50:03 +0200
Subject: [PATCH 251/373] avoid -daemonize

---
 .../arakoon_ext/server/ArakoonManagement.py   |  5 +-
 .../test/server/quick/system_tests_basic.py   | 66 +++++++++++--------
 pylabs/test/server/system_tests_common.py     |  6 +-
 3 files changed, 46 insertions(+), 31 deletions(-)

diff --git a/pylabs/extensions/arakoon_ext/server/ArakoonManagement.py b/pylabs/extensions/arakoon_ext/server/ArakoonManagement.py
index 55a1f2bb..b26127d2 100644
--- a/pylabs/extensions/arakoon_ext/server/ArakoonManagement.py
+++ b/pylabs/extensions/arakoon_ext/server/ArakoonManagement.py
@@ -1286,9 +1286,9 @@ def _startOne(self, name, extraParams = None):
         cmd.extend(command)
         if extraParams:
             cmd.extend(extraParams)
-        cmd.append('-daemonize')
+        #cmd.append('-daemonize') # daemonize & docker don't play nice
         logging.debug('calling: %s', ' '.join(cmd))
-        return X.subprocess.call(cmd, close_fds = True)
+        return X.subprocess.Popen(cmd, close_fds = True)
 
     def _getIp(self,ip_mess):
         t_mess = type(ip_mess)
@@ -1333,6 +1333,7 @@ def _stopOne(self, name):
                 time.sleep(1)
 
                 logging.debug("stopping '%s' with kill -9" % name)
+                rc = X.subprocess.call(['pkill', '--signal', 'SIGCHLD' '-f', line], close_fds = True)
                 rc = X.subprocess.call(['pkill', '-9', '-f', line], close_fds = True)
                 if rc == 0:
                     rc = 9
diff --git a/pylabs/test/server/quick/system_tests_basic.py b/pylabs/test/server/quick/system_tests_basic.py
index 7b51c6af..f7a54774 100644
--- a/pylabs/test/server/quick/system_tests_basic.py
+++ b/pylabs/test/server/quick/system_tests_basic.py
@@ -55,35 +55,47 @@ def test_start_stop_three_nodes_forced () :
 
 def test_start_stop_wrapper():
     cluster = C._getCluster()
-    fn = "/tmp/my_wrapper.sh"
-    with open(fn,'w') as f:
-        f.write('#!/bin/bash\n')
-        f.write("logger wrapper called with '$@'\n")
-        f.write('$@\n')
-    subprocess.call(['chmod','+x',fn])
-    nn = "wrapper"
-    cluster.addNode(nn, "127.0.0.1", 8000, wrapper = fn)
-    cluster.addLocalNode(nn)
-    cluster.createDirs(nn)
-    C.assert_running_nodes(0)
-    cluster.start()
-    C.assert_running_nodes(1)
-    cluster.stop()
-    C.assert_running_nodes(0)
-    cluster.tearDown()
+    fn = "%s/my_wrapper.sh" % X.tmpDir
+    X.createDir(X.tmpDir)
+    try:
+        with open(fn,'w') as f:
+            f.write('#!/bin/bash -xue\n')
+            f.write("logger wrapper called with '$@'\n")
+            f.write('$@\n')
+        subprocess.call(['chmod','+x',fn])
+        nn = "wrapper"
+        cluster.addNode(nn, "127.0.0.1", 8000, wrapper = fn)
+        cluster.addLocalNode(nn)
+        cluster.createDirs(nn)
+        C.assert_running_nodes(0)
+        cluster.start()
+        time.sleep(1)
+        C.assert_running_nodes(1)
+        cluster.stop()
+        C.assert_running_nodes(0)
+    finally:
+        cluster.remove()
+
 
 def test_start_arakoon383():
     cluster = C._getCluster()
-    wrapper = 'false'
-    name = 'wrapper'
-    cluster.addNode(name, '127.0.0.1', 8000, wrapper=wrapper)
-    cluster.addLocalNode(name)
-    C.assert_running_nodes(0)
-    assert_equals(cluster.startOne(name), 1)
-    C.assert_running_nodes(0)
-    assert_equals(cluster.restartOne(name), 1)
-    C.assert_running_nodes(0)
-    cluster.tearDown()
+    try:
+        wrapper = 'false'
+        name = 'wrapper'
+        cluster.addNode(name, '127.0.0.1', 8000, wrapper=wrapper)
+        cluster.addLocalNode(name)
+        C.assert_running_nodes(0)
+        p = cluster.startOne(name)
+        rc = p.wait()
+        assert_equals(rc, 1)
+        C.assert_running_nodes(0)
+        p = cluster.restartOne(name)
+        rc = p.wait()
+        assert_equals(rc, 1)
+        C.assert_running_nodes(0)
+    finally:
+        cluster.remove()
+
 
 @C.with_custom_setup(C.setup_1_node, C.basic_teardown)
 def test_max_value_size_tinkering ():
@@ -494,7 +506,7 @@ def test_consistency():
     v = client.get('x')
     assert_equals(v,'X')
 
-    
+
     m2 = X.arakoon_client.AtLeast(1000)
     client.setConsistency(m2)
     try:
diff --git a/pylabs/test/server/system_tests_common.py b/pylabs/test/server/system_tests_common.py
index 1a74e560..f2fdb0c5 100644
--- a/pylabs/test/server/system_tests_common.py
+++ b/pylabs/test/server/system_tests_common.py
@@ -1112,8 +1112,10 @@ def assert_running_nodes(n):
                                            stderr = X.subprocess.STDOUT
         )
         lines = output.strip().split('\n')
-        logging.info(lines)
-        count = len(lines)
+        logging.info("assert_running_nodes:%s", lines)
+        lines2=filter(lambda x : x.find('/bin/bash') == -1, lines)
+        logging.info("lines2:%s", lines2)
+        count = len(lines2)
     except subprocess.CalledProcessError,ex:
         logging.info("ex:%s => count = 0" % ex)
         count = 0

From e0f6765f29dbbf884e681e25a3c402ac636590bb Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Fri, 7 Oct 2016 17:24:46 +0200
Subject: [PATCH 252/373] zombie reaper for 1604

---
 docker/ubuntu-16.04/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/ubuntu-16.04/Dockerfile b/docker/ubuntu-16.04/Dockerfile
index 000b0671..e22fb403 100644
--- a/docker/ubuntu-16.04/Dockerfile
+++ b/docker/ubuntu-16.04/Dockerfile
@@ -93,4 +93,4 @@ RUN su - -c "echo 'eval `${opam_env}`' >> /home/jenkins/.profile"
 RUN echo "jenkins ALL=NOPASSWD: ALL" >/etc/sudoers.d/jenkins
 
 
-ENTRYPOINT ["/home/jenkins/arakoon/docker/docker-entrypoint.sh"]
+ENTRYPOINT ["/bin/bash", "-c", "set -e && /home/jenkins/arakoon/docker/docker-entrypoint.sh $@"]

From d8e8f09614f0db76c0ff485e4ac613ebd2372ccd Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Fri, 7 Oct 2016 17:54:35 +0200
Subject: [PATCH 253/373] inject via Popen

---
 pylabs/extensions/arakoon_ext/server/ArakoonManagement.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/pylabs/extensions/arakoon_ext/server/ArakoonManagement.py b/pylabs/extensions/arakoon_ext/server/ArakoonManagement.py
index b26127d2..557cbe27 100644
--- a/pylabs/extensions/arakoon_ext/server/ArakoonManagement.py
+++ b/pylabs/extensions/arakoon_ext/server/ArakoonManagement.py
@@ -1179,8 +1179,10 @@ def injectAsHead(self, nodeName, newHead, force=False, inPlace=False):
         if inPlace:
             cmd.append('--inplace')
 
-        p = X.subprocess.check_call(cmd, stdout = X. subprocess.PIPE)
+
+        p = X.subprocess.Popen(cmd, stdout = X. subprocess.PIPE)
         output = p.communicate()[0]
+        p.wait()
         rc = p.returncode
         logging.debug("injectAsHead returned [%d] %s", rc, output)
         return rc

From 64aaf3fe094051cac20443cb252c16083317ae19 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Sat, 8 Oct 2016 14:58:54 +0200
Subject: [PATCH 254/373] more robust etcd test

---
 pylabs/test/server/quick/TestEtcd.py | 36 +++++++++++++++++++++-------
 1 file changed, 27 insertions(+), 9 deletions(-)

diff --git a/pylabs/test/server/quick/TestEtcd.py b/pylabs/test/server/quick/TestEtcd.py
index 2a566370..927ceae1 100644
--- a/pylabs/test/server/quick/TestEtcd.py
+++ b/pylabs/test/server/quick/TestEtcd.py
@@ -185,16 +185,34 @@ def _arakoon_cli(x, tail = None):
         logging.debug('calling: %s', ' '.join(cmd))
         r = subprocess.call(cmd, close_fds = True)
 
-        time.sleep(1.)
-
         # cli node-state, & verify server's running
-
-        cmd = _arakoon_cli(['--node-state','etcd_ara0'])
-        logging.debug('calling: %s', ' '.join(cmd))
-        r = subprocess.check_output(cmd).strip()
-        logging.debug("node state:%s", r)
-        assert_true(r in ['Stable_master','Master_dictate'])
-        # cli crud
+        def wait_for_master():
+            time.sleep(1.0)
+            have_master = False
+            count = 0
+            master_states = [
+                'Stable_master',
+                'Master_dictate',
+                'Accepteds_check_done'
+            ]
+
+            while (not have_master) and (count < 10) :
+                cmd = _arakoon_cli(['--node-state','etcd_ara0'])
+                logging.debug('calling: %s', ' '.join(cmd))
+                r = subprocess.check_output(cmd).strip()
+                logging.debug("node state:%s", r)
+
+                if r in master_states:
+                    have_master = True
+                else:
+                    count = count + 1
+
+            if have_master:
+                return
+            else:
+                raise Exception("no master");
+
+        wait_for_master()
 
         key = "some_key"
         value = "some_value"

From 69080fbb095f6cbdf2a6d89ac4f7257940595147 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Sat, 8 Oct 2016 19:36:34 +0200
Subject: [PATCH 255/373] stability

---
 pylabs/test/server/left/test_collapse.py       | 2 +-
 pylabs/test/server/quick/system_tests_basic.py | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/pylabs/test/server/left/test_collapse.py b/pylabs/test/server/left/test_collapse.py
index 90614c07..eb2b75f3 100644
--- a/pylabs/test/server/left/test_collapse.py
+++ b/pylabs/test/server/left/test_collapse.py
@@ -155,7 +155,7 @@ def test_collapse_during_catchup():
         cfg = Common.getConfig(zero)
         log_dir = cfg['log_dir']
         log_file = '/'.join([log_dir, "%s.log" % (zero) ])
-        tail = subprocess.check_output("tail -50 %s" % log_file, shell = True)
+        tail = X.subprocess.check_output("tail -50 %s" % log_file, shell = True)
         for line in tail.split('\n'):
             logging.info("%s:%s" % (zero, line))
 
diff --git a/pylabs/test/server/quick/system_tests_basic.py b/pylabs/test/server/quick/system_tests_basic.py
index f8fde5f0..91bcf2cf 100644
--- a/pylabs/test/server/quick/system_tests_basic.py
+++ b/pylabs/test/server/quick/system_tests_basic.py
@@ -887,6 +887,9 @@ def test_rev_range_entries_arakoon368():
 
 @C.with_custom_setup( C.setup_3_nodes, C.basic_teardown )
 def test_statistics():
+    cluster = C._getCluster()
+    _wait_for_master(cluster)
+
     cli = C.get_client()
     stat_dict = cli.statistics()
 

From 78828ef231eec65e332f0524e640aafc84bfe1e0 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Mon, 10 Oct 2016 17:04:39 +0200
Subject: [PATCH 256/373] invalidate tlog map after catchup_tlog

---
 pylabs/test/server/left/test_collapse.py | 33 +++++++++++++++++++++++-
 src/client/remote_nodestream.ml          | 20 +++++++-------
 src/client/remote_nodestream.mli         |  2 +-
 src/node/catchup.ml                      | 12 ++++++---
 src/tlog/mem_tlogcollection.ml           |  1 +
 src/tlog/tlc2.ml                         |  2 ++
 src/tlog/tlogcollection.ml               | 13 +++++-----
 7 files changed, 63 insertions(+), 20 deletions(-)

diff --git a/pylabs/test/server/left/test_collapse.py b/pylabs/test/server/left/test_collapse.py
index eb2b75f3..626ef181 100644
--- a/pylabs/test/server/left/test_collapse.py
+++ b/pylabs/test/server/left/test_collapse.py
@@ -141,14 +141,44 @@ def test_remote_collapse_under_load():
 @Common.with_custom_setup(Common.setup_3_nodes_mini, Common.basic_teardown)
 def test_collapse_during_catchup():
     n = 54321
+    t0 = time.time()
     Common.iterate_n_times(n, Common.set_get_and_delete)
+    t1 = time.time()
+    duration = t1 - t0
     zero = Common.node_names[0]
     Common.stopOne(zero)
     Common.iterate_n_times(n, Common.set_get_and_delete)
     Common.startOne(zero)
-    time.sleep(5)
+    logging.info("duration = %d", duration)
+
+    time.sleep(duration / 3)
+
     Common.collapse(zero, n=5)
     cluster = Common._getCluster()
+
+
+    client = cluster.getClient()
+
+    def zero_state():
+        state = client.getCurrentState(zero)
+        return state
+
+    state = None
+    count = 0
+
+    # collapsing and catchup will be racing.
+    # the most annoying scenario is when catchup wins,
+    # but whatever happens, the node needs to become a slave
+    #
+
+    try:
+        while count < 100 and state != 'Slave_steady_state' :
+            state = zero_state()
+            logging.info("%i zero state:%s", count, state)
+            count = count + 1
+    except Exception,e :
+        logging.info("Exception:", e)
+
     status = cluster.getStatusOne(zero)
     ok = status == X.AppStatusType.RUNNING
     if not ok:
@@ -159,6 +189,7 @@ def test_collapse_during_catchup():
         for line in tail.split('\n'):
             logging.info("%s:%s" % (zero, line))
 
+
     assert_true(ok)
 
 @Common.with_custom_setup(Common.setup_2_nodes_mini, Common.basic_teardown)
diff --git a/src/client/remote_nodestream.ml b/src/client/remote_nodestream.ml
index 4e2dfb4d..ac6edaca 100644
--- a/src/client/remote_nodestream.ml
+++ b/src/client/remote_nodestream.ml
@@ -27,7 +27,7 @@ class type nodestream = object
   method iterate:
     Sn.t -> (Sn.t * Value.t -> unit Lwt.t) ->
     Tlogcollection.tlog_collection ->
-    head_saved_cb:(string -> unit Lwt.t) -> unit Lwt.t
+    head_saved_cb:(string -> unit Lwt.t) -> bool Lwt.t
 
   method collapse: int -> unit Lwt.t
 
@@ -55,7 +55,7 @@ class remote_nodestream ((ic,oc) as conn) =
         (object
   method iterate (i:Sn.t) (f: Sn.t * Value.t -> unit Lwt.t)
     (tlog_coll: Tlogcollection.tlog_collection)
-    ~head_saved_cb
+    ~(head_saved_cb:(string -> unit Lwt.t)) : bool Lwt.t
     =
     let outgoing buf =
       command_to buf LAST_ENTRIES3;
@@ -84,14 +84,16 @@ class remote_nodestream ((ic,oc) as conn) =
             end
         end
       in
-      let rec loop_parts () =
+      let rec loop_parts messed_with_fs =
         Llio.input_int ic >>= function
-        | (-2) -> Logger.info_f_ "loop_parts done"
+        | (-2) ->
+           Logger.info_f_ "loop_parts done" >>= fun () ->
+           Lwt.return messed_with_fs
         | 1 ->
           begin
             Logger.info_f_ "loop_entries" >>= fun () ->
             loop_entries () >>= fun () ->
-            loop_parts ()
+            loop_parts messed_with_fs
           end
         | 2 ->
           begin
@@ -99,7 +101,7 @@ class remote_nodestream ((ic,oc) as conn) =
             save_head () >>= fun () ->
             let hf_name = tlog_coll # get_head_name () in
             head_saved_cb hf_name >>= fun () ->
-            loop_parts ()
+            loop_parts true
           end
         | 3 ->
            begin
@@ -111,11 +113,11 @@ class remote_nodestream ((ic,oc) as conn) =
                             (Sn.string_of start_i) extension length
              >>= fun () ->
              tlog_coll # save_tlog_file start_i extension length ic >>= fun () ->
-             loop_parts ()
+             loop_parts true
            end
         | x  -> Llio.lwt_failfmt "don't know what %i means" x
       in
-      loop_parts() 
+      loop_parts false
     in
     request  oc outgoing >>= fun () ->
     response ic incoming
@@ -138,7 +140,7 @@ class remote_nodestream ((ic,oc) as conn) =
             | 0 ->
                Llio.input_int64 ic >>= fun total ->
                Logger.debug_f_ "%i:collapsed one file. (total = %Li)" i total >>= fun () ->
-                                    
+
                loop (i-1)
             | e ->
                Llio.input_string ic >>= fun msg ->
diff --git a/src/client/remote_nodestream.mli b/src/client/remote_nodestream.mli
index 3ee0ea8b..0a55c5bc 100644
--- a/src/client/remote_nodestream.mli
+++ b/src/client/remote_nodestream.mli
@@ -23,7 +23,7 @@ class type nodestream = object
   method iterate:
     Sn.t -> (Sn.t * Value.t -> unit Lwt.t) ->
     Tlogcollection.tlog_collection ->
-    head_saved_cb:(string -> unit Lwt.t) -> unit Lwt.t
+    head_saved_cb:(string -> unit Lwt.t) -> bool Lwt.t
 
   method collapse: int -> unit Lwt.t
 
diff --git a/src/node/catchup.ml b/src/node/catchup.ml
index c88870b4..c7b57d14 100644
--- a/src/node/catchup.ml
+++ b/src/node/catchup.ml
@@ -95,7 +95,13 @@ let catchup_tlog (type s) ~tls_ctx ~tcp_keepalive ~stop other_configs ~cluster_i
       tlog_coll # log_value_explicit i value ~sync:false None >>= fun _ ->
       stop_fuse stop
     in
-    client # iterate current_i f tlog_coll ~head_saved_cb 
+    client # iterate current_i
+           f
+           tlog_coll
+           ~head_saved_cb
+    >>= fun fs_changed ->
+    if fs_changed then tlog_coll # invalidate();
+    Lwt.return_unit
   in
 
   Lwt.catch
@@ -299,7 +305,7 @@ let last_entries2
   Logger.debug_f_ "last_entries2 %s" (Sn.string_of start_i) >>= fun () ->
   let consensus_i = S.consensus_i store in
   (* This version is kept for mixed setups were non-flexible nodes
-     are catching up from flexible ones 
+     are catching up from flexible ones
    *)
   begin
     match consensus_i with
@@ -354,7 +360,7 @@ let last_entries2
          Logger.info_f_ "done with_last_entries2"
        end
   end
-         
+
 let last_entries3
       (type s) (module S : Store.STORE with type t = s)
       store
diff --git a/src/tlog/mem_tlogcollection.ml b/src/tlog/mem_tlogcollection.ml
index bf748141..45ba5cc7 100644
--- a/src/tlog/mem_tlogcollection.ml
+++ b/src/tlog/mem_tlogcollection.ml
@@ -90,6 +90,7 @@ class mem_tlog_collection _name =
     method is_rollover_point _ = false
     method next_rollover _ = Some Int64.max_int
 
+    method invalidate () = ()
     method log_value_explicit i (v:Value.t) ~sync marker =
       let entry = Entry.make i v 0L marker in
       let () = data <- entry::data in
diff --git a/src/tlog/tlc2.ml b/src/tlog/tlc2.ml
index a5557140..770835d9 100644
--- a/src/tlog/tlc2.ml
+++ b/src/tlog/tlc2.ml
@@ -398,6 +398,8 @@ class tlc2
       Logger.info_ "done: save_head"
 
 
+    method invalidate () = invalidate_previous_entry ()
+
     method get_last_i () =
       get_previous_entry () >>= function
       | None ->
diff --git a/src/tlog/tlogcollection.ml b/src/tlog/tlogcollection.ml
index 991aa363..ebe57a21 100644
--- a/src/tlog/tlogcollection.ml
+++ b/src/tlog/tlogcollection.ml
@@ -28,21 +28,21 @@ class type tlog_collection =
   method log_value : Sn.t -> Value.t -> int Lwt.t
   method log_value_explicit : Sn.t -> Value.t ->
                               sync:bool -> string option -> int Lwt.t
-  
-  
+
+
   method tlogs_to_collapse:
            head_i:Sn.t ->
            last_i:Sn.t ->
            tlogs_to_keep:int ->
            (int * Sn.t) option
-               
+
   method close : ?wait_for_compression : bool -> unit -> unit Lwt.t
   method get_infimum_i : unit -> Sn.t Lwt.t
   method dump_head : Lwt_io.output_channel -> Sn.t Lwt.t
   method save_head : Lwt_io.input_channel -> unit Lwt.t
 
   (**
-     returns file number & starting i of next file  
+     returns file number & starting i of next file
    *)
   method complete_file_to_deliver: Sn.t -> (int * Sn.t) option
 
@@ -56,7 +56,7 @@ class type tlog_collection =
   method get_tlog_from_i : Sn.t -> int
   method get_tlog_count: unit -> int Lwt.t
   method remove_below : Sn.t -> unit Lwt.t
-                                     
+
   method get_start_i : int -> Sn.t option
 
   method get_last_i: unit -> Sn.t Lwt.t
@@ -66,7 +66,8 @@ class type tlog_collection =
   method is_rollover_point: Sn.t -> bool
   method next_rollover: Sn.t -> Sn.t option
   method which_tlog_file : int -> (string option) Lwt.t
-  
+
+  method invalidate : unit -> unit
   end
 
 type tlc_factory =

From bbf7e3bf9c36c0d272724149bbba4eba5f5c4974 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Tue, 11 Oct 2016 07:51:17 +0200
Subject: [PATCH 257/373] only add to compression after entry is written

---
 src/tlog/tlc2.ml | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/src/tlog/tlc2.ml b/src/tlog/tlc2.ml
index a5557140..41298acb 100644
--- a/src/tlog/tlc2.ml
+++ b/src/tlog/tlc2.ml
@@ -238,7 +238,7 @@ class tlc2
       Lwt_mutex.with_lock _write_lock
         (fun () ->
            begin
-             self # _prelude i >>= fun file ->
+             self # _prelude i >>= fun (file, maybe_add_to_compression) ->
              let p = F.file_pos file in
              let oc = F.oc_of file in
              Tlogcommon.write_entry oc i value >>= fun total_size ->
@@ -249,6 +249,12 @@ class tlc2
                else Lwt.return ()
              end
              >>= fun () ->
+             begin
+               match maybe_add_to_compression with
+               | None -> Lwt.return_unit
+               | Some old_outer -> self # _add_compression_job old_outer
+             end
+             >>= fun ()->
              let entry = Entry.make i value p marker in
              set_previous_entry (Some entry);
              Index.note entry _index;
@@ -289,7 +295,7 @@ class tlc2
             _init_file fsync_tlog_dir tlog_map  >>= fun file ->
             _file <- Some file;
             _index <- Index.make (F.fn_of file);
-            Lwt.return file
+            Lwt.return (file, None)
           end
         | Some (file:F.t) ->
           if TlogMap.should_roll tlog_map
@@ -302,11 +308,12 @@ class tlc2
                              old_outer
                              new_outer
               >>= fun () ->
-              self # _jump_to_tlog file old_outer new_outer
+              self # _jump_to_tlog file new_outer >>= fun file ->
+              Lwt.return (file, Some old_outer)
             in
             do_jump ()
           else
-            Lwt.return file
+            Lwt.return (file, None)
 
     method private _add_compression_job old_outer =
       let open Compression in
@@ -322,7 +329,7 @@ class tlc2
          end
       | No -> Lwt.return ()
 
-    method private _jump_to_tlog file old_outer new_outer =
+    method private _jump_to_tlog file new_outer =
       F.close file >>= fun () ->
       _init_file fsync_tlog_dir tlog_map >>= fun new_file ->
       _file <- Some new_file;
@@ -332,7 +339,6 @@ class tlc2
         Index.make full_name
       in
       _index <- index;
-      self # _add_compression_job old_outer >>= fun () ->
       Lwt.return new_file
 
 

From e0cbada97a49a2b7e348821333f0bc06b9bf6354 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Fri, 21 Oct 2016 10:24:58 +0200
Subject: [PATCH 258/373] move to camltc-0.9.4

---
 docker/centos-7/Dockerfile     | 2 +-
 docker/ubuntu-14.04/Dockerfile | 2 +-
 docker/ubuntu-16.04/Dockerfile | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/docker/centos-7/Dockerfile b/docker/centos-7/Dockerfile
index 3e98db57..6b0a147e 100644
--- a/docker/centos-7/Dockerfile
+++ b/docker/centos-7/Dockerfile
@@ -63,7 +63,7 @@ RUN eval `${opam_env}` && \
         snappy \
         bisect \
         lwt.2.5.2 \
-        camltc.0.9.3 \
+        camltc.0.9.4 \
         ocplib-endian.1.0 \
         quickcheck.1.0.2 \
         conf-libev \
diff --git a/docker/ubuntu-14.04/Dockerfile b/docker/ubuntu-14.04/Dockerfile
index cd54b5bb..8150ae9c 100644
--- a/docker/ubuntu-14.04/Dockerfile
+++ b/docker/ubuntu-14.04/Dockerfile
@@ -59,7 +59,7 @@ RUN eval `${opam_env}` && \
         snappy \
         bisect \
         lwt.2.5.2 \
-        camltc.0.9.3 \
+        camltc.0.9.4 \
         ocplib-endian.1.0 \
         quickcheck.1.0.2 \
         conf-libev \
diff --git a/docker/ubuntu-16.04/Dockerfile b/docker/ubuntu-16.04/Dockerfile
index e22fb403..aa2ff09d 100644
--- a/docker/ubuntu-16.04/Dockerfile
+++ b/docker/ubuntu-16.04/Dockerfile
@@ -56,7 +56,7 @@ RUN eval `${opam_env}` && \
         snappy \
         bisect \
         lwt.2.5.2 \
-        camltc.0.9.3 \
+        camltc.0.9.4 \
         ocplib-endian.1.0 \
         quickcheck.1.0.2 \
         conf-libev \

From 3a1fac0b99a22c8b1d5ca45d27aea0a68413c07b Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Mon, 24 Oct 2016 09:32:13 +0200
Subject: [PATCH 259/373] make_release: 1.9.14

---
 META                      | 2 +-
 debian/changelog          | 5 +++++
 redhat/SPECS/arakoon.spec | 4 +++-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/META b/META
index c0a5e1f5..8ab82f02 100644
--- a/META
+++ b/META
@@ -1,7 +1,7 @@
 # -*- conf -*-
 
 description = "Arakoon client"
-version = "1.9.13"
+version = "1.9.14"
 exists_if = "arakoon_client.cma,arakoon_client.cmxa,arakoon_client.cmxs"
 requires = "threads lwt lwt.unix str oUnit quickcheck lwt.ssl ocplib-endian redis.lwt uri"
 archive(byte) = "arakoon_client.cma tcp_keepalive_sockopt_stubs.o"
diff --git a/debian/changelog b/debian/changelog
index 86f20ccc..dd03c307 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,8 @@
+arakoon (1.9.14) unstable; urgency=low
+  * new upstream version
+
+ -- Romain Slootmaekers <romain.slootmaekers@openvstorage.com>  Mon, 24 Oct 2016 09:32:13 +0100
+
 arakoon (1.9.13) unstable; urgency=low
   * new upstream version
 
diff --git a/redhat/SPECS/arakoon.spec b/redhat/SPECS/arakoon.spec
index ecaf3ad8..70a0f8d7 100644
--- a/redhat/SPECS/arakoon.spec
+++ b/redhat/SPECS/arakoon.spec
@@ -2,7 +2,7 @@
 
 Summary: Arakoon
 Name: arakoon
-Version: 1.9.13
+Version: 1.9.14
 Release: 3%{?dist}
 License: Apache 2.0
 Requires: libev >= 4
@@ -33,6 +33,8 @@ cp arakoon.native %{buildroot}%{_bindir}/arakoon
 %{_bindir}/arakoon
 
 %changelog
+* Mon Oct 24 2016 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.14
+- Create arakoon 1.9.14 RPM package
 * Fri Sep 30 2016 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.13
 - Create arakoon 1.9.13 RPM package
 * Tue Sep 20 2016 Jan Doms <jan.doms@gmail.com> - 1.9.12

From e20fa95dc5f0422aa4b624b7c2e50ddcbf3ed533 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Mon, 24 Oct 2016 12:10:23 +0200
Subject: [PATCH 260/373] route through "$@"

---
 docker/centos-7/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/centos-7/Dockerfile b/docker/centos-7/Dockerfile
index 3e98db57..d32e7667 100644
--- a/docker/centos-7/Dockerfile
+++ b/docker/centos-7/Dockerfile
@@ -110,4 +110,4 @@ RUN su - -c "echo 'eval `${opam_env}`' >> /home/jenkins/.bash_profile"
 RUN echo "jenkins ALL=NOPASSWD: ALL" >/etc/sudoers.d/jenkins
 
 
-ENTRYPOINT ["/home/jenkins/arakoon/docker/docker-entrypoint.sh"]
+ENTRYPOINT ["/bin/bash", "-c", "set -e && /home/jenkins/arakoon/docker/docker-entrypoint.sh $@"]

From a1e7d21e89d08eaffe8562fea0fd85151c187548 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Thu, 3 Nov 2016 21:42:26 +0100
Subject: [PATCH 261/373] make_release: 1.9.15

---
 META                      | 2 +-
 debian/changelog          | 5 +++++
 redhat/SPECS/arakoon.spec | 4 +++-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/META b/META
index 8ab82f02..7f2ae286 100644
--- a/META
+++ b/META
@@ -1,7 +1,7 @@
 # -*- conf -*-
 
 description = "Arakoon client"
-version = "1.9.14"
+version = "1.9.15"
 exists_if = "arakoon_client.cma,arakoon_client.cmxa,arakoon_client.cmxs"
 requires = "threads lwt lwt.unix str oUnit quickcheck lwt.ssl ocplib-endian redis.lwt uri"
 archive(byte) = "arakoon_client.cma tcp_keepalive_sockopt_stubs.o"
diff --git a/debian/changelog b/debian/changelog
index dd03c307..d4424873 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,8 @@
+arakoon (1.9.15) unstable; urgency=low
+  * new upstream version
+
+ -- Jan Doms <jan.doms@gmail.com>  Thu, 03 Nov 2016 21:42:26 +0100
+
 arakoon (1.9.14) unstable; urgency=low
   * new upstream version
 
diff --git a/redhat/SPECS/arakoon.spec b/redhat/SPECS/arakoon.spec
index 70a0f8d7..bc247a8f 100644
--- a/redhat/SPECS/arakoon.spec
+++ b/redhat/SPECS/arakoon.spec
@@ -2,7 +2,7 @@
 
 Summary: Arakoon
 Name: arakoon
-Version: 1.9.14
+Version: 1.9.15
 Release: 3%{?dist}
 License: Apache 2.0
 Requires: libev >= 4
@@ -33,6 +33,8 @@ cp arakoon.native %{buildroot}%{_bindir}/arakoon
 %{_bindir}/arakoon
 
 %changelog
+* Thu Nov 03 2016 Jan Doms <jan.doms@gmail.com> - 1.9.15
+- Create arakoon 1.9.15 RPM package
 * Mon Oct 24 2016 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.14
 - Create arakoon 1.9.14 RPM package
 * Fri Sep 30 2016 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.13

From 1a204d98f0d62d5244257936ad1947e952c50b01 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Wed, 11 Jan 2017 13:50:06 +0100
Subject: [PATCH 262/373] fix centos build (new epel release)

---
 docker/centos-7/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/centos-7/Dockerfile b/docker/centos-7/Dockerfile
index ef71f5a3..855776cf 100644
--- a/docker/centos-7/Dockerfile
+++ b/docker/centos-7/Dockerfile
@@ -13,7 +13,7 @@ RUN yum -y update && yum -y install \
 
 # TODO: What about darcs ?
 
-RUN rpm -iUvh http://dl.fedoraproject.org/pub/epel/7/x86_64/e/epel-release-7-8.noarch.rpm
+RUN rpm -iUvh http://dl.fedoraproject.org/pub/epel/7/x86_64/e/epel-release-7-9.noarch.rpm
 
 RUN yum -y update && yum -y install \
          python-pip \

From 4cf9424d19a135047db4d9df90a722756f003541 Mon Sep 17 00:00:00 2001
From: Wim Provoost <wim@openvstorage.com>
Date: Fri, 20 Jan 2017 10:15:15 +0100
Subject: [PATCH 263/373] =?UTF-8?q?Update=20Index=20page=20of=20Arakoon=20?=
 =?UTF-8?q?Gitbook=20https://github.com/openvstorage/=E2=80=A6=20(#156)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Update Index page of Arakoon Gitbook https://github.com/openvstorage/ovs-documentation/issues/54

* Remove "Where is Arakoon used in Open vStorage" section

* Review comments on PR
---
 GITBOOK.md                        |  57 ++++++++++++++++++++++++++++--
 docs/Images/ArakoonDeployment.png | Bin 0 -> 22180 bytes
 2 files changed, 55 insertions(+), 2 deletions(-)
 create mode 100644 docs/Images/ArakoonDeployment.png

diff --git a/GITBOOK.md b/GITBOOK.md
index 3f6379b6..a941807a 100644
--- a/GITBOOK.md
+++ b/GITBOOK.md
@@ -1,5 +1,58 @@
 # Arakoon
 
-Arakoon is a distributed key-value store that guarantees consistency above anything else. It was created by the Open vStorage team due to a lack of existing solutions fitting our requirements, and is available as Open Source software.
+[Arakoon](http://arakoon.org/) is a distributed key-value store that guarantees consistency above anything else. It was created by the Open vStorage team due to a lack of existing solutions fitting our requirements, and is available as Open Source software.
+
+Arakoon aims to be easy to understand and use, whilst at the same time taking the following features into consideration:
+* Consistency: The system as a whole needs to provide a consistent view on the distributed state. This stems from the experience that eventual consistency is too heavy a burden for a user application to manage. A simple example is the retrieval of the value for a key where you might receive none, one or multiple values depending on the weather conditions. The next question is always: Why don’t a get a result? Is it because there is no value, or merely because I currently cannot retrieve it?
+* Conditional and Atomic Updates: We don’t need full blown transactions (would be nice to have though), but we do need updates that abort if the state is not what we expect it to be. So at least an atomic conditional update and an atomic multi-update are needed.
+* Robustness: The system must be able to cope with failure of individual components, without concessions to consistency. However, whenever consistency can no longer be guaranteed, updates must simply fail.
+* Locality Control: When we deploy a system over 2 datacenters, we want guarantees that the entire state is indeed present in both datacenters. This is something we could not get from distributed hash tables using consistent hashing.
+* Healing & Recovery: Whenever a component dies and is subsequently revived or replaced, the system must be able to guide that component towards a situation where that node again fully participates. If this cannot be done fully automatically, then human intervention should be trivial.
+* Explicit Failure: Whenever there is something wrong, failure should propagate quite quickly.
+
+## An Arakoon deployment
+An Arakoon cluster consist of a small collection of nodes (typically 1,2,3 or 5 nodes) that contains the full range of key-value pairs, and clients that manipulate the key/value space. In principle, all nodes have the entire key/value space. There is one distinguished node called the master with which all clients communicate to perform updates. A client contacts any node to find out the master, and then just conversates with the master. If a master dies, a new one is elected automatically, and clients fail over to that master. A slave node is a node that is not master. A node that is not up-to-date cannot become master.
+
+The diagram below shows 3 clients connected to an Arakoon cluster that consists of 3 Arakoon nodes.
+![](docs/Images/ArakoonDeployment.png)
+
+Updates to the Arakoon database are consistent. An Arakoon client always looks up the master of a cluster and then sends a request to the master. The master node of a cluster has a queue of all client requests. The moment that a request is queued, the master node sends the request to all his slaves and writes the request in the Transaction Log (TLog). When the slaves receive a request, they store this also in their proper TLog and send an acknowledgement to the master.
+
+A master awaits for the acknowledgements of the slaves. When he receives an acknowledgement of half the nodes plus one, the master pushes the key/value pair in its database. In a five node setup (one master and four slaves), the master must receive an acknowledgement of two slaves before he writes his data to the database, since he is also taken into account as node.
+
+After having written his data in his database, the master starts the following request in his queue. When a slave receives this new request, the slaves first write the previous request in their proper database before handling the new request. This way a slave is always certain that the master has successfully written the data in his proper database.
+
+If the master node dies, a new master election happens using [paxos](https://en.wikipedia.org/wiki/Paxos_(computer_science)). If a slave dies, the master is not affected. When a slave comes up, it tries to catchup. The slave first downloads the missing parts of the log. Then it iterates over the tlog while adding the missing updates to its local store. When finished the client again compares its log state with that of the other nodes to make sure it is up to date.
+
+The directory structure of a node typically looks like this:
+```
+root@cmp02:/opt/OpenvStorage/db/arakoon/ovsdb# ls -als
+total 0
+0 drwxr-xr-x 1 ovs ovs 14 Dec  1 17:29 .
+0 drwxr-xr-x 1 ovs ovs 10 Dec  1 17:29 ..
+0 drwxr-xr-x 1 ovs ovs 84 Dec  2 13:22 db
+0 drwxr-xr-x 1 ovs ovs 58 Dec  3 12:07 tlogs
+root@cmp02:/opt/OpenvStorage/db/arakoon/ovsdb/db# ls -als
+total 680
+  0 drwxr-xr-x 1 ovs ovs     84 Dec  2 13:22 .
+  0 drwxr-xr-x 1 ovs ovs     14 Dec  1 17:29 ..
+676 -rwxr-xr-x 1 ovs ovs 690176 Dec  3 14:03 tGLpPNHEDxNOOqOZ.db
+  4 -rw-r--r-- 1 ovs ovs    276 Dec  3 14:03 tGLpPNHEDxNOOqOZ.db.wal
+root@cmp02:/opt/OpenvStorage/db/arakoon/ovsdb/tlogs# ls -als
+total 13744
+   0 drwxr-xr-x 1 ovs ovs      58 Dec  3 12:07 .
+   0 drwxr-xr-x 1 ovs ovs      14 Dec  1 17:29 ..
+3732 -rw-r--r-- 1 ovs ovs 3819824 Dec  2 08:32 000.tlx
+3776 -rw-r--r-- 1 ovs ovs 3863492 Dec  2 21:09 001.tlx
+3920 -rw-r--r-- 1 ovs ovs 4013559 Dec  3 12:07 002.tlx
+2316 -rw-r--r-- 1 ovs ovs 2368293 Dec  3 14:03 003.tlog
+```
+
+nodename.db: the actual key/value store.
+nodename.db.wal: security database to keep the key/value store consistent, for example in case of a failure.
+XXX.tlog: The transaction log files, contains the logs for all transactions executed on the node.
+XXX.tlx: A compressed transaction log file.
+
+## Arakoon Log files
+The log files for the different Arakoon databases can be found under `/var/log/syslog` or through `journalctl -u <DB name>....service -n 10000 --no-pager | less`.
 
-[www.arakoon.org](http://arakoon.org/)
diff --git a/docs/Images/ArakoonDeployment.png b/docs/Images/ArakoonDeployment.png
new file mode 100644
index 0000000000000000000000000000000000000000..76b9b816be501ee19424b5aaa6ffa88c3b9959d7
GIT binary patch
literal 22180
zcmd42^<NgvA2)i@-Q8R?NJ@8ubW3+P(#=J;l!T;&v>@G`(%m7AbR!^emiv32*LnVg
z1HZxS?(EF$eBL$Ds>-rx$i&DX5C~1~gOoZ51kDZtL75@K1AjR*-bM$3KHAGkiEDbV
zoOU3&<65rVli}6=w9&y;RmGi(gUF<vp<7YLVB&r0iK6`0UxgeDJ7&@kv5BILP!fZd
zibAYq3<;5vmXgA^#3>BH2xQN=G!+^fX5j1-V!ICTPq^1#H1zT)7hIPsSuC%5J<mQ)
ztU6Q9EI(>}_4x7W6VQqOzy5GY&2B6^;*c$q;gGq?geVbzu}K-^I?G8pI&HYg^v-f?
zwTk8`k*Z!CTYUQO9sU*gOxFC^d4HAZ<Hx!IR#Et=aDZpN5!WBRc+a>R&{0x#&p3%k
z!&hXE<mppFmCST_?lxVBNQJG!rw7~sG7zOgl7;Q;WmUG_+aiaDJOjtdfSdn25|h6~
zr@^cZ?^BKN2J4TQrcufEj>%b=!p_FQ<dX^aYUn|5`Xa4AuBk>8H4<<m9qD$yq5^@i
zo(HKmuGwP}jk=VUNIHqB&4LT{5Egn3EV-6wPwO%&?**p3PkCn6xcS`e=vx{&zg<4g
zyoQHq(TMWCy$|t+WJ4zv6u%Hdqr*{2asGGQDD70pUly;0aHYTX_ljUb4`uNQN<%Yr
zrq7{Szb!CKY47(xl!&Q?aIA&!Pnass2)L!_Fu=iseFjuQgOV1Qz{&%Oxf9qrgDI~n
zFT%7Vvx7S{Ly12py9Nm7*`$9?#H11bkjs3+0koqe^w)&L!o|<0S%GcKRtbR@d{<=l
zGe+C!yIf99B+vjP`mIRgD7{x9>U4(I8<vXJY}fzF<d39s!L}{^9VXD=cl_LmMTgBa
z%eVB>T_Qn-5DN=>b#$~;p9<QYD~S_aph-8(M*TRe;an^}_Inx|RcKO24vm>%?Qnb~
zg<NeMYCHp=jaVOSoZiZqB;F<ZL22ui5#-aieq%5n8<S|4oUncKZz!OR8B!i?`$$2g
z5CIOkinK#aVxGuDQ|A5eU~S+#gOvRmwjWH9EHnGiqA}Eq?-vChGTZMBr!6PAH)C=R
zT+WWi<)W(Fzwd)8R{0TM5aCYwq_;0vbHyoz>F3*IWWXciTtY>krAj^h{VBOCtAju>
zl-WwZBLw`<gs(KkUg@E(zOiO}8O0CICQ5{}<WML$5$Kmmsi1Ra$k=&3l#3uQ14Yb%
zL<_uPxdeR6d*DJU*lLDeGA=}^-{nRE9f|xf38+9{e@f)y$xjtwVi~)G5LJg&jc~^e
zT#<+(4L*~>0JGAMqU>qbKF)8*OMjr=F3HsU8LFq?E=4??7`PUb+Tg|7o%NL^LmgG(
zT_TnUbj=4l&-aXO>VJS0$B0I_BRN>Q2r*nyl9@+8`pw}3<{p<nVY!vHRsn?=YkoPO
zhf-=*J^m{NEDSOjD!B39?VJFx3xwz!p<lZsIV)ZMN5A?qr_!7QuUOBrZfg=Z5Jcs$
zIJh&?Uj_?=C9|tv0~_hi{Oe3{h#YE=np^NI-6V2COctW;w6l-`*#k7mzd7|Z(IA|x
z1}lQ5D@WO-kyEX;ddHM>C-c7CA0~|)bvkWhavQMYj2l<PXU?F1q4EZ5`^W{f-~1Gw
z*O}6A#Y2(yx(Sc;u&{e)UDX7|vq}=opIr~!IBGvl-d7%VvB5e}kUuA(WJSiL)!D-f
z{Q+C;=G^RctAWHk=7%2lWNM6Y&9^th*q_PhP{UNEVqsk!bNEzv+m|>fncg-RCJDhR
z&z(P42CO_Hq7MqH1q0~;y3Z;fFMHH@EL{2Tj}H44@tobzAzIEfbEdHb<_1#<Q_(23
zTbS(oZCiLP3IA4<S`wDtQMwjc(QEwZqBE#cq!^MBdKIV;G9bgB-X6Wn{rgpkY4r%p
zD%p!l1zrb9-o50ylTL0?m$>hBJt6$34$I3r6qLC#gRDBg@#NMZj?_ibPQ%N=p~P~L
z3Ob2lZRY9YT}c!&fr_B}qc)G@fM<aBA8f7n;C&3{tAIH=>7hq6mkzVnilxaIZmpAS
z?=>js!6J)Y?mlG@7XLK%=}gv&Avx`m(?-zy^4@_g*$5$Vno0vxT5oeP)Pl6{SGLnt
zxvcoZL0?&=2B0th;$1d3qeIQGDUYW<Xx^{&ZC{GwKE2{Tsr{?6x<$-*h<RpunNO))
zq8fP7zkfR9G{WmXuJ2kktjURaPB077t~7|^0f|Xswy>7e+fj{t;z)M6<dP>o<0|@a
z>~rZ{FOR@+;G>qZ|A(~8B<cF;{XqG^c3{?08WZ^|W$rzl$!Dg!EHk~_fib>&QsRyu
zWs=?P%;XeTm;y8+o!6I7owPreyBBR&U(WVC-c`B=YD^M}B!i6HG3TN!Mt{*@i4lyR
zQO8i{9$=TMVRLA)=+dx(bmm}E4u{Wdl3rbdYNE7#9ICNb&+ot{ADRyddQi-sya$rE
z=a`C7YfjrSpz6f_`5V-vkq23)d7ssr*%q<jwQ0~dT5(@^tZY6T)Ua-Ib}}^P)b%G7
zla6Mp5op<@LqWHVYU~^fsil_(GA#C^Zc97wvdl=xyD!ElCxSAPZ_TjSE|x^p9E{H;
zyVuV8vSt-HbZZ7AGT67uF8yC6F4@9ASKVH<y(kC;B@EpKn>|ItcPc-%3A<O@(l$O<
zYoRx89IKeoZJdquYJ>is`=}|rf9TEAXzg+Dm3E$EW@|Zb-kf_)aKn*81r>+5%CZhV
zpg$)*=kcAl(By)nJYxcMht4nkMQ5Y4G}-c$o($y3S5x}$Ub)`hDfS-IU_pWDZn#Yo
zh12B}{?^9`x1qwvsJQ*Hvuq&N>R5!|NaMRj3X{v1V16f>BIu*9rk5yQM#9_Db(co4
zQS?~@cN(AU-a9p#H7$TmhGDKnw+|eR0yl&F5mSoEd;2#RA0F4ze#Bt&&ZgJ%8y)^e
z@t2U?s80#^t?Uo+o_4q$+B_8Em=Pj%G+Ap?5rKS4t-5`rL^($rH+QX0I5@|fq_{Ni
zS#&k2T5vor_X{v*I}eh%Z*Ceo7X&wATrA}>HU@2WFN$}2=L36k?BrFg0z%>vihor4
z$!9&4Xb#SW!7tv^ktqiJ9OyoLp9R>7xe&GZMeo5clPyZjYhNT?za}gi(7{4FC4Mc2
zyt36i*YI(-OSRA$9~%Y5r{AaCAM0QKrB2w?CO*i`R)8*5F;H?n`v>2N4n0Uvw*2d6
zy&DhOyJLs?p45;FLD>K|1sEBk3QxOvIfBG3^f)R1l_gxmt8F5AtNOpJ)4m;X6$c)A
z8MbP|C@(9`t$nKZkKUrupEvs(1wB?7MGsytFWZ!v77Bu^Lm^P#@>N4{4Y>r-bH_+}
z2WlbU!m=g3ujB^F-I$x=`pU&0MdW#LnpbzDE9Wr6iKn^CCO_eoq>1O@oL@KmPD%7u
zs0kV>i*A@(+)HZCr1PPe0}~qIt%}?(DQihs#ym^cD8%GoNt9cHfQQ*$jC&bzy5BUg
z%Xu#6$HOoEd}_-eVAIPZG^}g;%7-F0r)}?{&X$wlyd8oH5#9b?q!!Yri`3)%2S-dm
zhG62-jaN^i{qS8%{j%I{>(DPy0jpVJIviv(%AW(v)g$+(s=2J#McVjC550335y?E9
zJx~T*lxhZ-f%;gC%b~Sxb;Il=^CM)X_Nz9Tn#bq)ew1bFq1W<r3{FIPIG`}c#kE_t
zj&!)md0Gp2%E#fHCzeb@sko)-skht6j6e2Acad1bMGX%u&}mUxVThGV$Jt9fh4*i$
z2c3M$5MEl2-gPPr?Qr!={SJppd(+tI7m_)3oXF|o3PD_FlwgO!oH-|O2DRXi%bSz#
zp|hu>*E)_dE5s6vUmZBBcd(;d(@vmcLS2UF2!l+a`zXu}yQ5HS|2(c^vQrVrxF2?e
z>5_o92Q*Q&;+%wD@JsGI9}H@G`5F#l>5~piToKTbKwH>sw(8cciH(Q68_B7%opAz~
zv2ev8Yb1y_htN3QD)~WM1uJ*=lVlj&*a9C{HVmb|>JCq9hB^{Qm_qz`fPJ+9bu@eg
z6lOUId|N%qHZ8Lza*bGyW8KaizYJH?;UJ&K-PG$~8JRN~7B2f?hP^6-ZEj|xnMww!
z-nSnH0;i+HB+^EVE4-zcQ29@t{B<@}Sp5ngh-ku~!{b+;yvOUdx!WjBn9-eywuQv!
z<rr#sILif}u=o5_Z_&bH+ipj566EG7Q8xw0R*^Zf!c@ImKQ9`TExE<48%K6uc->9y
z#f(R3AiRI+iZN7;6KbeUzh`GVFPX-y!+R2Mf1QjOkI|qv^Xu#~&L)0&#Es;4x4C}#
z&~Ngg-1Z!P4}AE%y-Jtrh&TVim45;sggQ!8v$RQE6xnYK#mPJ2RW+`0i%5vgDC;lO
z1px5pr-Z?>l<LK8_DJ7nw^W_Hfh8!MWE-zEy-#Oqp$F9bMdp0^Aa>^TezQDWFw`~1
zB_F49j;9iN>ZO^!hj5p3=8~r>N9(mSP1vGHjp3d5xgN$ODk2`&(`0JOzEE(C#%#-U
z5sUi;etQQ+$bhVdVEz=_wgVJeS6@mb<sdoM3pM^zF%~v^5LSX0V#26W;7i~(NzJKD
z!f8(b-K5ZlpCGz3pR}S8rz3<z3&HHF?JH~?p9{&oKSs}X--@NOy0l{E9FaBZ1pzke
z7vsUzj6Pv9rxzOhrz(&3{r3Y^$Vm{Q{H51v$JabQZn@@Y3TT;E5#zm$$NSzVY^#3I
zFz3Ig7$WcDtL`Z{p2HWmEK&Zkna4ZpOQh(NBc4+o(;7czv;eG*XVSO;*BccNk=KFy
zAalp+W6YaPx%7jWt<U^l`OG24x#{&aj5G%Q7T4iR)TY+)IwvT{-tof9<FxrxN`=0#
zW0sGjMHt{2i5hy`)oQ;N?Sa90YLgd@1jxhdF_s3U;->CZ2C&FrTi+F-I7B|7gzEin
zq7drQGoDQwH>!;x6Y%)l(R5&+HgoIA(_*1KEFHU%tlxNL2>o2rRoD4^j`nLapwy;C
zfZ!Khe2*^7ku({Vmqc?>lWI1ZAnl1o%)m<tU&CuG{TISE$#%MG^x(@Z@5=kRn=47h
zo6i~*F&P&>=V(nj309p-4_;xRV8jBTi+6ti5@)$c5aGyG^0sPSZM;1{H*Iq7V5x*I
zuG;*%GTOs)DukQdCD>}gEpP=bI-np&otd{M|4&9pIN6^L-=9gT7!xBN{Tr-)mKlKN
zMJ_I?yko+;j)eHXs7Qj#sEv%g^n{B0*Y~&2g_s(j`(i0ub(?-@Iuv3d2#4`TJt~gl
ztcr#MdpLRNT@Gvgx-2e2L-(eOhIOOmy>>%m(Ctpw5gPcE-!tO&j~7O4g351jX~)p(
z_FZcM(X|g@saoOS$sQibTJG%Fs<qzLpdc7WdBtQ1Aj1L^#Ht5M!H0G~miTu&q_TNp
z$YQU_GJSZfUPx25JKCMw(iis@F1<EpJI<V2PlNDRhuhqGhx{1R_t6mvK}WaU!$kRq
zP$`KS{pfiw1J2ITF9(%LDnFY<MTR^7Z3(^0tqT*r^8UPfEHjGZ&59CDfX(sZYa;RI
zAs7PnB1W4r;^UfoHG&P`^cvX$=u|Zg&Nj)IXQ0>&mMNW`U0(XCB#9ZN0|iD{nG9Ag
zH_JXC3duTqrEss{r+IH%0R=I?gtvX^*i)sG;NB!NZb3wDVgEWyj7RlOUGQOL5+V$V
zZuipQ7_wckilSXFdhS1dN~qbRBOZhHsmZ*LT6H|9Rc?M{J~sZ*Nl_?-Z8zvSWBy!%
z0%g!_s^5uQd0;SMN`=kJ=b_QdpauI~py{~y*?B$WOwR)!Eq~DH$shet={5))z;j~N
z^gXFIiT(n~rwclgV}6wjL-2mGm7c2H_F7Rr07$l56jv;Y9af}9FL|^$)DBCu*YrsI
ziP!phsdl$2?Q{}F;H`qcltQ<S)(5$cJnv*|ADq(}teiwsyH}8psd-i%!2}Zs6h*d8
zHlbFP^%2#C8}o=#R^$_Z23)7>X46SIK1<6`xP;s>lc@P(V^{9D?ww2rw3W-khsMi7
zHg7`P$F=V#jk!Esa40_Z3@5lwP;SOOW*k$~{PL?FLYgLS%G!;2SD2h7e?7T6U_g*1
zOb$=m)aFiYa6q@1Y6C1;2AP(1h~Dlyp1|S2MV3_hmBx-n;jFtEsxDWP50ItV^X>EF
z<e607{%h^-8xj}MY9r((NInzGifpG4bG@M=k8lM6=ZIVoM2Kf@56fB)HP8z;shsrQ
z{{#ud{HZPoIi4||-sEmtoN?0=)*QxP=lbT8L1O=REBeKN<aDlrY{L8nw!{80+P$L9
zU&4HcDA?2l^(RuTB$7dtpvqw1!V$s6yuK;fLPWwy17(6O+jotUmEJWN2`{D5iFIg(
zzf1MxP)FNKaX1Npk<Dti<Tgm6!+%-Z8I+8`{0`EAX?5x=bgD%8Bw`p9smK~>a%twa
zxl)0Q5n1<sDnQXL=lzoXU}M_}Y3hNgUQNr?3dYnrA$AiKh*B>`olIbeo>lVaTqD+G
z7b~{+_X5?EbDFj_BdqxF){D@JWX9<;=I^z_*{Sd{@!{HqHDC~kuotC}F3(BWo48=0
zue86t3U=dLzd2i3mW{CSDV-JROpSM>P<%i7gmP$rHmSO<5;yeYD<ErV=4Yd}GN}06
z21MZp`25|nQ_{@FxaauBo;)loaIp1tIo%}1HHiy909X8vOZeyCj4+@nrFbRcuntUV
z1`bACNiy?sJAn(CpYdD`)gL=CFFkD?f5_8C)~WxnkhpTzRDGAQpX7xq2m|_NP|;Mu
z!kMkRL^9pMel7SK@7EdWjkU1#BEd?|V9O~mL|aXC4qX-~$LbTup_64Y`b&xsP#X$D
z8752^etd3B#;tjtrmX-Q;TCrCZTDC5^j@mblU>Ku1T-*x{bC*|PY0~%-WA4jg+ImE
ze$?_&`C2C<H=i69n?!Yux{LS~q;Hchd*QhHVnnn<)_r>2;-F~Z%#9F0AgMFo%G<}@
zcnkMb8g4`LhihJ4z4ppABdwHnedBvWbHquXBQZH%4oS-uxKayf5afbte>-1b8qp|Y
z!Ar%&PvxJ3F4U~(xuw;h2t|8OrcycO)*dTB`u%a+dj>MLRQ)<niZ3yjh(MstMdVyr
zJ(d^|*BZ%!{qQ8O8F9iphj7|~(Rb&h?D6;CtJqTXbobYG`3E8G4us~vODO9F6&muh
zmq<Vm0RZdy@Zsr)mH%Nx+@&mvS5iCMeU&6Q^T;D@`r`fYZ<noV(1U~1l8?Gtw*U~~
z!pih%aeVBXy`WfCsyAV3y4YC(MT`J(nPht+#VgXX3El5AnZR>}C2XAAf^UPv3M*Ok
zfV3f*IVIlI>m)EhX!KHUjl}i->~^eC9e|cp++ttiovXJcQ3qlE$m19-coxQ0R)<+$
zBJ}R7cX-NGyCt0e_|L)X!&3k|(@E0YT>V4J93~yzwv?SG@*IcIOA6FaTjUZU_?Xmx
z)yY&lL-28^Y>aAq6p#{()*rR3mBZWC(e#T_D%9tGT|;jX(#w|@@G2vsiC36D?Xy{O
zqRyzr_(;^_-G;FDgvVIOE|3%246d08U!&Wb2``oIx^%dHN#V%4pV_G)S{a6%F5I=J
zauu+(EONjDdaJ-7WwmZx-gRg_e6X6rcJU`{yV>44IvnQPWk$UZR8$sU%;{H94B~16
z5K$%=svV7sCM0(!rHN`XYCONDm^ta)rq91x0K&}T0N&Bs&YAIe`R2CnucD1{by|&+
zJtVcmxHHUO*SPUroAq9v`T?KSM}9Lh`^_*ln)I;VJzl|_JoCPe;@)jMp8q|_;dDkI
z5v~dGAkx&Vi<*j;ARGWw#FgU&bwrMvJZSLzd|*4@&0{2%dFnyD8@usZnw@lxX`SYO
z*6Fdw;t+T}3orw>1=w)dZ_qFP(8l-4>+A58al}P)Sx38?IZ~F4F)LPJi%x$+#<!=x
z(si%?v5LlK6*xz3Zj!<@74Tk)K&as=nysA-UT7f3xRNX0<WhEOs&{_>jDDbHM@Za^
z<t8?QQ=y{qI7zE_1phqxD=;V<Jk<{oN|}vG>xsIDt>B&9Js1BxpC<Pg-n*?Md@Za#
z65rEVskmif4qTp@=ya6Jjv^<%;g^l|kyA#M5AKg16&3}}JWO9h9zK^vU%Qw5-9MH!
zcZKpP+-ARLYI!^kn~Ie^^BvdE;C3P+P|~S4-8fnI*sMix#b4Zg;@{ocx-hfnrCBpa
zD?nIhv89UWm9sj<k|c|8G`DIILcAb{{S=vxD5~~L(>@MrJ>VO?MCjUhMSUXHI_==q
zcFkD>^;#dzM5+(ifZ02llSNUd$n<4t>6d-|0?{k_&^SSOfgv?K1nLZ%Y(idu@>jsn
zGmrf^t44G>uvZ<B(~O%#eyTADb56LI^K6;YdU4bx^r{TAmG`+(%6O~#zcgI2G1s^-
z>)n)%hEv3-CSM@CLLp9~RDyYiTRsR9cty^lqyXDm|2*pvRdG?0XVZr_IS=C(p<z32
z^;%D6NrIaXVlP$@!qJ0xst{H1X@TtBD}F=rFCBB#;S^jYjtbf*wWU6J6xr9IgXlJ6
zYul-VY|M%8ZX&$}sP>7D28IKAcjFq91MFkM3O`4~R}=-9G2w$3=U2Z6%?9V+J<TCb
zMD>2Za^XHs7wj6O7R%A%KwvN<7sO`P^w<V=9DK*ZI1ZQFfEvNN0;<3L7fKc0#$1Kv
zls}-^ZO-$q{1`ZJP~J0R7M3$4r8n^GZhmSg$_pCWy5*h8snFxzlHVug`-Js$ahMQ+
z$;t(7`fn}Xw|exUlvx!b38ulqs51Ozkxn4<jzEqkP{z@_WJOBVjCWmd*)<kE+_zqE
z(?qCXt~4%ih}vgQC(aa#XkDoJQ8fHdglIPHDrK|aa0pH0KMe9AZP-3Z>Cu)0YCXAD
zK(*b89&GKz!5i*jM{i$5X5-MTa9ShIQB!w;9U`>hOb9_Yhb0x+w(L5&r&eHTaZw$>
zdwvbY`l<IJYoxx0un(I1N5ANF(mIy5ZLGML>b69$B%IRY?#~_-7E3$|(mi4AUKWGW
zdchKR82f;G*u{_!Xg%M2<Kr{CM}`RJ$}P5g`QlO)-Sd$xsccQ4W4zWH9QG6k50{yj
zE^-!1js}(~%)k+F4u%xjbp2clsP>zN6%vD~-OE7T6Zzw_cUPXM+(bA1esBI>_a18?
z_q4z(l9~u5j6lF@mt}T)fSc~TNWNG#fdli$KI2&jlbxv!*2}lg`5z6qN+p@PmcK2W
z1yji8_DSVf)nEcIW9J_=(eiDOy3mu#J<C%OOlP;52Z-JJ7k7b1D<>*=IMJ!J`WT;j
zBy!1`Fq4K&Roi+cgtJ7M86yaNiOjp-ef>)qN{Lf5e<fqof<5j-i^-13&W=cEVj@+r
zEyT92Zzz*bQEo-0;FeaK6{)-@>W9VTD_AkH_sJ^1mf_<7z$rcApfGR@u5bYqwGt+_
zZ8~W4a!+YOX=1W~&2D$7H!(D@9`~k8uUg4x?s_eu|H&BY-5s5{a5ae3l+mbF`)nl#
zUNh2sKTm8f=EH?{S|k_$y@1Rv3=#|p*8R|Z=!<Wv?h%JXqF!A7i>O_x<BD@-8m2n-
zW9FQ090wmzceCle58sERD*mtQ8SvTF*9OAB0FU#2OqH6`=)cXu$05!@0s{&L3Z^a%
zg7rM>Ejg)5004+Fc^NrsWTfbYe*|lTgiA%IM}#ouhV1|AzryzXgsyyKcQ^^?H`uBc
z=YbHWAz3cZ+#?D0v)(V<XAb!P9(jyJ_{nkG$jqP;3M7(`nXLC~*D@|xk~Lw05C)MD
zkuXDgw3PEI)7Hpr*9RK(ugzwu=#_yW{uOWD*(p`=3+~#KJl$J6>XjSok*~G(f4eqv
zH`^Z>+FCsXPbwJhf~Sk83X6zAQzlI;(RlxDcH)2w6VsoA1&a!+$TzmI!^(3}#cXx%
zqy6KcvGPIBM<f)5^rYetiBROjLRyM#m?{sb`?pOfNyqx-g*=@{oH&5lkVV5Ql98a{
zq3Pxteg8Nuv|5tfon<n;oRf^0KVLU(%!IP!Jdwkq3S2^i#z07zNn?Ix${Yo2XlfzV
zF6Rs6=D!moBWsU2-}+GMXz@{G)K_d<RP^7sF&UBU^E-dJY1bDv;Gd|VZrCa4GPH=B
zCnX}E@wuNGKRDnCKK#C5?G$ef0s|{W!gh6m!u*GRgXJheX-dl<V_(HpVPllH*#fak
z{~c~&%;-ev4b}LDm=2SB-CP=N^i5Lge-{p<q}Nt43a>X#o3Eh%?*fvee@Q4u1B`9v
zQ)UcGM<<Rm0>b-pJ!pjHAfN$`{BeM+D667Se6lCl1eMuDy)I?^{R=Dz(-cZFpOTo8
z_{dl(RnaUi7=ThrEp!P=0smWB`L$Y>Z8{Zcj^3$Bfr7NxGk5a}BH<8@ZxFeka-sE3
zTg^H!paQC726`FI_bmw=KSu@2l7FrKw*)d&m!l#F>*!@fPH$T{sOf5C>#itGP#H`z
z?<Slo@Q}l5#Q^*Sjmfzjt@(>)q*0jZYj)Ng2mDbu^4t7q{%TunIv?Zd2<$(;CebEW
z&pqEw8$*FI&&Eu?SbD!f01pD8irabC3JgND-~YL`oJeV+18n1v|2`q3?y=oPxJb>2
zSUOO*i>e6f-hgU}F$RaZ4-^h^C?xo+R`-QWW&tKhTOI_-j|Tkcf(;M;u22Y4JjCb6
zKNRMEcDIk<vee`QNi9YsP!!bPDVdQom2qXBqC|0YMFxov^EZM<#(^Ldj<NaxlX^A!
z_Ela`_sUdGT$;0)jtZ~>U{O(_Bl3n^MWP8ewn#uGpP+V#DJUI*4;;c#n2Osm;=7SM
z5fu1z+vjl6om6RgHD3v+4wZ(h$IFMZxp6j@Y6+_|57r?NsI>5-jrsQ)E$ltu^jM}4
zJJSP?zU#j{OdJ{~O07#Ne(_fbKMBO2x2OZ1rdu@MCfr=6?N0>mM;<?vSN`>+U$qng
zG3in%Ha>ORD0h+K%MMG4*-=P@kIXk;DNP_A-#Q6nE>sC{527(?PjhJ1B<mbMV4Bj0
zuApT3R>29BAim8<CQsX6G*5l)w&PF!{#2{&M+!}gl`Na1Rjsv_Qhsey)|rOwba(yR
z9VrhjQ6v`MS}y>FLOKs6z*>EwVADH|b<)z@s?u`yqk}b2aV%3KE0!Z=gDmpMUf)$b
zo}-*h{V{$puLVH9b=|^=8UJ^Zr2x2CUBINwHH+{P{O)<V(V7E&431#C`Ze9792Wr$
zV|0XYB7e-D!mMFJnHrhL>Mf7%N*2V&XJ5baI^NKyIl5Olh?F<hxnxRXfU03P`{n!4
z<L2dpJ%UeZ>8BlU&ml)I*Mx+Zhs%kB2kY1dx0*gk31K;e`P`Awmc|uh4Ss!jaLg=a
zq|ue*%JNpU<pa`SBe~FcjP{uc3_Yuvc5#B6J*G@7bo+bE$!-57>)lBtVC;8BKln4}
zK35%I7Nqmk@GRCJxW#`E6Vle%M`rWEVJE9<i&o01kD8QV8V*R}w)g$6K)A;r`!Gq9
zC^RtK@l0W=u<vAf{zGt(?sHBLJev31>4Ub>%9tn#ENT?vmc$n6vufMtQ2M%m4_VD#
zv^qWN=}C)LgXK8}G%Zbz=>)_L{`g_K-4|y~gXa$a>rW<+?F5e3?J5&5sr9vy&Lla-
zi>#;Yy-4LWUc8R!OSZ4%2mQr>W8brI#&;Qo+CeXF6sr(DxfYgWeYZ;q(Q4z)yoQgk
zDF-^xN~q%9VbnL{6-oPBOyppOp>FO`@fyJO^BJbc%}t9~IZEi-G9t=J<R3!#1BDWE
z)NX|<|Mo?46dHOi?-~4@yCp`(X>gifsDOYkIgYlFPLx;07Va|9s}6>#C8@%TKWf<g
zS?BF^eacBU4ulK(VRf_Qc9wj=gP5U;OxTbJrxu3DjM*xWarQ~~;_f9X7T>uMFr=Bs
zb&s&$UwZx#lK?bb1T?X3vF?nX<mAj&ACpX81*`MQm-000a3SsuSjBYZ^`$<HkxK6?
z^+pGy@2hSd>Q7h64kMo~|FvoJIQIlqS5HdkmC31dB4%75Wvm~W$l797C7Kp|I?vYJ
z*wb8|9zP#@b_8&r(B%RmU^oJr$Ed)XaVgSQltWj<w<<cOE}`^5x3c5+h_)AC%!D7<
z4>uTrm&fUI!|BA&<eQY7k=TE2#jxwPDcrZ7OLE`u_V3fgj)vXydfOBFRtq<-jIZm7
zV%iLC!$)+*5)$Wd2$C_PZGNlKukdDx2&g1uu>ifLAAPyzS@(Hd*)G$o2lPRvp?P}c
z_k)%1T`iFzZ+jqwD4RbfKES{H_%icKn7P4^_`X@yoAIIiV><?Fcvw491|uAb?vuUq
zE;-&q4OP>@$IdH`pnpfB=ZlYRxV{Blk`LTuf@@=G<E#63LIK4H=YFWrbnVCQ3VFal
zFaCrm@N7RUw(zmndQX$kKGNNj-Dmy%*QVx*_BI4@c5Vs_HTuqfOq+qxjCOUVH0i}%
z|H<zc&0UVHJ*+COB}29OT`rH*sMi61Yx^rn;W4#FxK!CvD1}CfW}Y}OfpeC;8!jk*
z-*3qJ{K+4gYY0W&YZ7tqrNz8hMd5}WNmTF@?U17u(j39t&5@q(=WB#VeaeMIMb}``
zd)Vx280HlIHx~|?S8MNX?;{?BZgi~_4WXcLtm@r}!|B1xU+vVfq6wHZc<KNv6i`qu
z|B-3eafYRs$#s)*JR(B9;S)mGxJmnrtAO8|NH-iPdPw*-riDPB!VECmq!mlR(h~i{
zf?HLwU)}Dqbsrht)?GKHi5Q>ex0TP&*kqb@k?}{wa^Lp)X;yN+^BvKZ&;|NW<L2xB
z@7Ab@EbjO6IC*~{Gkg#oT(-xvj$KprEQCFa?ZYgBjl2(JU>8(i^M!{Gh13+07;VTQ
zq!i1@n$w6C#ltJv6(S^`xcX&7?bB?qVyMBz)nWgiwkuS>T|O7U_yA&rPwt0Gz(sr9
zo3OT_wD8>evF|_>5Je5EcnCLUzh`U$ju7wG$prF4EJ(HP&rYLJ1hs%{sS_KxGoRAj
zG)F-n$|xMk^{YW(1B?f>l<B@$=5X6T?Uq1AS*nt!7@CwW)GUfwpw)NLavmNK<EkhW
zkNSws)*vZ2f%A!IIhv0e1RGZa7w4L9JnD8^?C^HsFi%yL>-Z>VBJTrpu{y6sBoK8f
zJBq^AD-#%sfsVKesHgMdN`bb9gFrZ%Az0w^jXo<AjtDr5NyVVZ7MU?Z+3su!&bEk_
zO`pD2Ab_I(=0Vc#Y+HzgZ?IFQ!Hxl#@&QxyQM>E;$0B;|hS$vwPMocdNh@{A50*`P
zWT@HpgQbOkgYT5AMn~?<#3Q(u0!3>z+x7_vKXjjbbaUNBHDEjz|Dt3}bFQ$rWv75b
zc-3ozb~ZNknOv;<uYDuAS52^&C5PW++gf9Sw~dhB#m2P5^--b2?6IuQ0t_~Z2-hMm
zv`o<Tp0@=ZzJ*Sx!*qcP_oeIfhb`ifpe_4EL-LS!mK{zyKE$V$!8inf6le3s!nL}W
zzQgbdiZLT*%qw`+EwHTq^jg%s4BAvBKc+`z`&u_}>%I4mqHeW~Uy<6RA8pC!UF41l
zTGP0Fr>P-4p6z@ICz^*N<~2g-vy%>uf6?G7lL#L&RN+$oY*#B&GG-`O66GPS2d4&+
z1r6NTaHap5Zhjs`psE!$#YA;kcXKW6eY;J<rtUM|rWHO^U$tptw&D*rJ}J~Oic8mn
z!a8?plEKVK*%0}7=;2ux%j!^?HqIPRO~7;l8@cnGEF()Yxv7?5YAb-jCTk9>kLHyg
z6^#xLAs7$lkLl#Ip1x~`F*22|eV-PO{~V06b@ggO%iwXSSJ9a<WNS8hW8mBCuoUKs
zGlrHlXQmTh(rLBvOm#KaF5Z48_&y6@0!jn-Mpr*ArT1*?Lc2a#Llwl(gX=N$Y?g)^
zzO9ahL-jd<o!iJ>HR>%}f(xZ;=CeqIIrI^cDGW7DvLBI11bxun<y>RwcRGlkFOkUW
z->+arL8mN*z_l&G043Nb-*9zjcCeW6az)Hs?a)u$$kJf_xZU0(_giSoS;`J3EBV6X
z8PeedB9yr=lq8<aNM40yBRNqOCidQ-P4#B&zI~cQEoL%FJD6VOq@&bl&ul&O{@32R
zMWG|F1-|RF4Q%0cEK^+T>lXilfO$w@kYNESj)02G9v27IL;QiCj_JOdq~nE#!a$-d
zBY`yvY#b5XSb42srPe5q!I#6P6qEeZigf>x4vu#O=kCHyP7i?#r(PXn?@YLWFy^HQ
z2Tz#<i^3kS{XDp_?jx2QhsnIb5>+*a4C~`acqqEVYkN9ar(NfAge)Tg7RuZXN|JU)
zCC-24*v(u<(JaAguSN|+#iTD{_u{=BHqTCN$z_r8EDGVfq@<h=n^4Zhh)v;Cx*qw}
zi(w-lNLA0VB?-e#P9~T~uKN==GEgbXo^j~d*5P}$2YAj!qlF0TqJjZy(@OB;7C!rz
zBENvA2`4-mCziSYz<mFP&LiV}b3d>0@9`7g?{|CSr<5jUyKTW?s+cLzgLIB1I8Uiw
z=wd(r+SnbGJM=ivg!-B6q82v5@P;7JEyyB)0K%?|5hoC{QC^qukHe7%ySC?zAHDsG
zNqyD@k{|M1gB;h=0NFLNTcW4`t2#;_sjIj7!Yl$#GF&evd&lgVXZ7PBceN;jc^>I=
z%HpYWkbmOx<BfT=oRW@u3Y|muw|s}m8cwKJvJngk1?ySFjBBeA?f)nS1ggeW5`T-V
zh)&p(REVDeT=V=JY2h)m(cvyx>tw3T?W9B4xb90;Q{69GHwjj~o3yps)WuUWK@U?l
z7NcJxqL<oE344kb_iL|_kXLlHrhaMsxAGy@4T&P-Y~xg*F>>zRKrDp#+(chG^m5Lw
zH~t6;!f5+`>}ZO|{_-*ru=(7NjhzDhA~>OYJYXVQhG>oUzrqh?YMdt>&VH^9W0zu)
zPF5ePF58LuC9!#{c*0@ytlG};^=K$)o<yQxN;xm^{^#Gnu;d{te*f5<s^4XD<TZ3u
z!JH#MUd5m}P1=6kJXn3c&1;Edn0hsS`KN?x9Ej-%rJ;VqbnN3N@Cu|nH)~3+4}1d*
zO<zb~{C{qnIX-MaHdl8V-Cum(W&;GPm?6Y&7soQLOZ)`_J|Eng?fQGlGX0v(eT9@;
zceRIcpnI83dy+Kdwr0eum)qU>_r=+uJuwQYALjbs$@?@7RlrhKWHE7S1&61Of25b0
zZQ0x_c4|k}dnINHI9uAX0v-#!KeEpP9mp1-{Mh~`Pk^8Yr^pHA(}caR4j{0K6&0&4
z0FVvR(;8o?Z<{#vc-+-j>Df+zENk7uj1uyd8U^nqK!SUEEpL%BmIs`FuC}fiCbTqD
z@FekO&vvnD`v{pGHYXxxm;@mNY%CAhzpXLg&G>}a^M3`mlqn$l-OsYEH<CThZu+P@
z2SD#%y{mFTL6|lVf>Q{9j*uD+2=Wd){dc9Q`YOtT01qFV2JxYE!yQd8Zi`0AWC++n
zYijPO+PQ7|`Q2^P4Tpkl(m&Rtv*z8sNuy$C=JTgD=B9f^KRb>*86_C9KlkHbI~Ps`
z%kIxqZT=6DY4r&@^tdvYZfDGGbZo!&d1-kYeKF9`7+6K4ty@r`hPRi&q@DJXRj(!K
zx$24R7MZY|M&PQiI*|4ck5sqjndsoaXxh4Ls4~6`Ub2r1H(6W_CN@&yK;7n#J+gfE
z@m_+>kMyUV&C|4$H@$^@9=tVWDfpLIP;f15wN^^i?oaD9MpxMb?e^z$qO4}8D@Nz@
zFmBGTvP;P=`pbUDpKGGct$&Kckb67{(Zt;p?5C#lTu-9C$rsSgY11Fz_x!{)%4&Gq
z^N(S0#wdqI0yb+a^y+uX36N=jRQ!uGTz&E7v0f&*Mb7BarCuIP==^P3^v5l|cGY*~
zA3pQ$kNlr02M4;1PVWd*G&jg%>`F34*e95?xgrSI)90#w(Z&O?j>~Z-yW{zBOAk82
zipX+9IPO*Pbw;tUwO14%K>Eimg01(a9qK`oR?0t20M0OCjaVk_YR_tPcyK#<|5lEh
zI~F)8BFX(?Xx|VVNW!NFLymqUCZ3{p(?em;>8NqB(R$WMZ&w=~6$#Vt(ePh@p?d38
zg!c<_Mj*`D!Ms9lUO3a|ZFklSKGBNQsc2r$3F|A?!DL9wN$sfaUc^pA|A2k<iHV$?
zQp>k*LB;_Op?ce)xzwg^C61K#eoB#gy%rrALQ0$BnW6QE-<6Eu#`e<v!u(Q)JoZKt
zWo2K9|JN_So*`zMF^L!!C=KbMav+=cE)5nn8poNOu$DEal|#1r#p!`Pck)fRdMlr!
z`&y4T6RJ_s5|X>Nmvr)_OwmGoFLeKA4jhx`{zIS;7sCsVc;eYob9n$E3A4M(Fe4F;
zc;sheJBNr93x5?Q=iE*(%3Ew_!kuY3&)sV1hu<5=#bf@x1!l?DzHUzRlJQ$#4~V?D
zvhiI{F}JS$!dm8mO!(NI<J3OS4qa?}e$u!r{oSpQwUCHww<or7z;c)SErf?&ug)@Z
z4KXQm9C+tdN}z)?zF2dUQ*5YbUo7Yw1UL43OwHkBN23Ed0+6OG(+xd|Ds<}mj;}S4
z@oVoEH-?27G#PgpdG;rlx^V&+6{WYsrBEH4?%QLvN&iSHm95YIwJ%*9tMyqu!ezhl
zp=dv^IYi2cOP09=T*v^g)UHoZ^G;B!K3JLYRY0U^8=e;A=A^!1EOVfHoolYYBTqy<
zWr&nh#PS%A!IZ@N@vVB#v(+e#3@n>LW5l^8sk61Z1~MaMw9p<e{b;&jpU(h2wrlp1
zp+UXbB)BNK_F$(Rj$ptb>-V)towEhJg0CKg7n*aY<u{)&Rkmsr45o_Ya?r@hmgRUY
ztQbE;ZGU#n{RY$a21r=Q-~7h;l#So?f-X3)*0}GYXrUx!_*tvJbebbyO61jbUZ^$0
z#x<(^avEl~*vZo!0z|9s<ho)BQxa|M%Vz>d{f~|5<6oPoDYJOo-ic!PVr2DX7X4xc
zo9`xOF6&81T~=wamXM`tv_cg~TUUs_$OukXd`g4ZpkxpfSN`6cu+q6XbeqU%rP_w3
z>!H)SSil^iN)@)n^K*Xo!UVJPzJV;j8XBgx@b-oDw3F&3@U#;3Y1NXy9qCxnB|Br2
zB9ya!3#A{1A;wXeu*bI=T_!7Z9hVEb1-+p)02AiiZJ%J+_qvZLwdcbsI+t}}vOl?`
z-b@;L#CYlqAjpD^Y;b%382cDuPObq`UI-_wST~_|H_h_i-N<>%hpywVf6h&UBOWYK
z@K4OfpLWX^eIkX{*r5mazpx{efeF+(yi9-hOJlOfjO*1YxNoMS|2_d>vJc|3yXPOw
ze@U3^>jkVQ;fE7_?lNfjKewqi=~e|N9Z*qrG=T^jKFupv<HrJMj1r+NASiQqsFb8d
z#Wv55k1ufemO7y<mKV5^T3InOLHOq`pL?}t&e@Y!<~~Od=;?Mdd;0|`T?>5q`4z)b
zyfDJj>hK-}h0_iJFN!{YUX8>ze<MA(p17w3p2XX&Ch6!?2l@Xr4W^#80_^s$X^GDW
zPaJ4C{fwQ+?nv9&tnjxBum?~({3uOA2-ruS4yMn)l~Xeh{uDbIUreymk|U5@hT{Ky
zECE2Hy+`h0#Jm9mf*>{ZoPFkoB1nD`@<nHc5Hs~vi;XG3Hz5cz?#41#tjvUk`@vSk
zuqf?GnK_lD{ci4*3XA8BU%Z#qJO;IVq;iBlz0c6>UfKiO*%kp5bP$`IP~<g+IL#EW
zgzV<RmlfUSAd`YR2Mhv<bK#zENd=ki8z9TrB3tuYS}{cc1!L@gd>9pa&@L8Y2(VmE
z!(995eNRAov&%1Zo6~@{hAdASdwAzf<NSvzOfcCg`=e@)>OK;3h*85Zd?Ly(%)n-U
z`a4hxP}cv$f(hLz=YQp3v$1@ZzLc2%c>12+)IzKhDI>FNF8n_eBSRN9j!>ozLz*TZ
ziFWUL%O=28=b@3BgVu<lOV(az==k+hZ$Bzkf_>h!!q!_o1)f`V43(;49%Vj^2R^S$
zT=QCU#&wFuyL7#tDjVZwN4dQN{r7k%93?1oFRQQLLuL%9WVjHhvCQvWu;b-RvDhYX
zzMO|rqrPDX;C|D%?0UTZV`9kMM6~hs?a;{dc-3pB@|SPtEUK~z2FfXoK>vx!R<KYh
zz|kSfB)-ZPxGL5Xf2$cl_Th;422V?q7iDG>gvI~A4M9+a=(OJpzAETs^_F2fD$A+G
zycUNlxrG9-+}37Gg32D)q`0e5Vw)5y5`Ut+b;NABSX5);%X;7Id?U-(w$$~F`3Y=n
zY+W-6Yy~fi+9o5xsAo$Mlq8E#|D!udd(g$$+fQyCn{<X_gZX4)ZF6vPeHcCGW!`1m
z@=V`P7a)WH+YP2$+&<JrAdVUw-Qldoc^kk=i~ox|8?3Gv`%(M(_8K($%%`uturp+>
z+fw}RL7Py-bhZ&hPBl<1?fn<BwSeMDQf!R6*Q!cjnzOhEYRDK8ugiecC}_!$E4+DJ
zIOzHv?!5NvwP=1`N_emLXnQqHi;f09Q3FHtW_d*4F;LfSfMnFp2$^{DNv))U$V{(1
zpftyfAVM=;lk<H_QfQvC0U`TrG3aV`BL<V5C&s)DOr;rh-IXpt>0ckV5G;AFw*0HB
zSATB1ExdoQJK=QSZfNB#2C~>nBU;K3yAIh12?fnYQswE|`~FD>K$NW<GUvSM-J?n^
z5+BzSn){O&*_Pb+HW17v|DXNiKTI*(b(7KP*Xe0Cf#T3Fxl^+V=<;+HQq1RHloee*
ziNoM52Dl2*<Ps4>iD^miI<~wI$Tn)uE|G@Xn}!r3>QGzXNdn9vY~}yv!iHUr>sQZX
z)5lJY9;duZ&Y0}oIK)Nt#J>%vV!z@)4_^QK0z{3L>+x0}qM)&9-j4Hl4bQhT-DJe|
zO5i3J<R{bUxCa<;wMls`&54ljS$8i&%Nnx2_1<jVcXK*Ho5Cmahit?#H;vx=Gt#kz
zFXufXK*{#-v#iDOiM7a0_${;r7J+W4i}w7Zgse{sY{yH#y1fYHm|RmQ-}>AC^9R%F
zfL?<`A7J3EIQ}1l$+EhmHZ(R}+c=P-cK`$(vA`DS7M|toCbmV!<osQ(r@S=DD<%<V
zZ!&vUK6hZpPUMRU*1I{W1Xp64-*|Ua7y2}+zq&9x-ahdfhcA;?{lg_4BD_}pG@jaz
zFmKjCOJcAMM+9UVo^F;nvVt4by-KAmCIRSd&~#}jUe(T<Umbm)qb8I8ysK*Nwf9DX
zri*!Fq}uOja|&I)!-F!01OZY4*s4_!wIKw9T{Kg_&yMl$s6gGMs^MQUuADpomr4Jp
z2?4%YMj0&XB%0RbW<WC~G$;=3JpD_`FPy1V!D{bFe<&aw3B7;2hwu}#Pm$K-OIMS0
zb{AYTAJCA9NuDH2g7fPalzF22tDpvBY>5xz;fdv76O&>bcJ8f0h0-E8&|i8ws?@QH
zh$z<wQdTeL9!93q{k@do;5Qdpy?ZK(gxU)U0Yr$ZR!gz4^?9ya4U5ITVOL07AC#rJ
z_2Bh3l}B~S1=_lktFsT2&Ru~SXykm!lWcZ#r5yH)cfLH2N%ZT3u>IjR<)uT8fBs#I
zb;DxNo?c<wWpN>DTY(^=&{p})$GU_wn^|-o%JmAI!V*%I#ggux1=`M-Zzid*=PVjy
zrNbTmiyNT6S&Ba5vPIKPUs(+hoM433F8>7cV_~u1@E>0HZwGb)K_AYmU0>jYsI7zB
z9rpG%D(xKn?I@saAn{&^Bw(s>a~s~!SG|!2jr4KWz|i%D`7>Wqio9W->c^Q=Zu`;u
z+KMU%A}jy{<ym`PW@fA|0?{8@A@*gmPv2odgmWm=gSxPXt+WK3@w6AHSksLWt$F{A
zLM_63*&2V_)?*It@<OM!4LSus9QHRF=C$$N1t&b`h0h6pQ#rpRM%r$Z6wtDcgNF30
z$#%C22IF^@$(q)G35%?_?c_{ePr^X~fxaXRYCPBJEtijvY<jhV$Te!GTaL@)1e>6P
zj5mruXGtT1KN+}q>piWIrITg9a@`_BLzVlE*)2RB5a!Vr*ZYf=_4=*Bc)Oj$ncN*S
zv~d8n`e%*(Ava`q<0auIty@#={Q{Xe&d`KOD7?^-@7aHJ>c3}W$tdM&j1n9JLJwvV
zm(20a5k8fAm@EUU``Z`cp*u@A?z3GLs1L9WhYghD<(<CQ;j_byNbc4GtaoF18qSEe
z$F0tWlO61qaBeq8lR;<ZB&zQ?n*MJl6k{@zKr<DF39Hzb&+a+3We<XQV8!ne-dbJy
ze@^}@6?GV*TyIb`LRHR=ym#;>OO%bWww<`L>Kyr^g-7c(*&S<jC;lb8y@2)nAG1MN
zblXh=CrM~jFxKCkkr=l;PmHp8<$aY(xaCa0NdTJ=gBm`nI_FOOl|MlMVuEtDqv)Ao
z8_izI<=$(#*v<`ut^%vF(XCs=9&hD9Bb}C(+vB>M%Q<aqL-$#WWfIxP4j5X1{nSKK
ze9^t!{qC^bF5h(UCOy$Ef7#i7@Tzv$2zfPf&WjdNNhz|zWDm}%fTMbK60GV4LAuq>
zAf^4f$7qFhob5CMN=Kw;32#oThA|TNUbaob`5)dd_W@-(f3XM%Nj~~d>`qTM@0A`(
z%L5g3GAsY*WWYVDm!H!b&gA6~e}YQct=IXPVA#QeD&e1BHJrNg>6%TX8OppH$WS;_
zE#>%_{i}Q6W)KVr`l6F>b^uel2@@+kRf3G_O?yHbgtz`|W=@fJ<lA2CkD+jyJ1ESG
zYb4)Uj&}|t2?jeuTgmrGEWE6cw8%;aRt}%Gq*nWnQFJ|CJvF?rhUXrhs|G3!q4U;R
zUN^Sn7EWIO4oDF1=E7`k466)>0d=c`3-9JSH?<^K;+h_s<XUjEJ`WzeJ-BRiKdsUn
zNJ;dm0)vFHw=UMrbs^-z_wIpv2&~eFM+EvU8})%)`O=DmPmO@eHEjkmU=t(-P|}RY
zlR*+E(>GcIIG=gc_<gc(qM&_=k8hzz9W01FvMHSYkclAnIu~333mmTEy=Ou5jvJKH
z`S0R5`CPy6FB_P7;UYtEW$!3Wh#!TuD~KgUdVvfB4A*JS`bS6A19B>LsZlKA44DpN
ze>R9~=gN{YYb4O^VBOhXMw7!wsc@+2W;tjOA-9x;msRvqt%#KS#~PnqtxJ0-0EgNe
z5w}kLzYY%QLgnj>15u>gmz7)1iIP|WT7wz$&hfWci9ehP>R&OLNX=pchlxLuQb?W$
zbRwG!5Y(W(XMKO}Mj%}P2^Q}LMWr?hmCb<DE2;vT-hT`_!45~9Gk|9#C>RtdJ$>Di
zkZvZu=A$n!<<~R>Q*}uvJw8pyzW;^(KlBKLicAqQB$(ZLKxgwt@pKc0tbT%1Mwfw5
z*z#fWJaf<s-F|jMt|%Rla?A5*?wU$3<yVMSn_GqP6I_sxqKz-zO1b!aC@FOS97Hm7
zmkclc7-LgF_P<yHJfY*(7Mb$*0N<Kvz?)~40gnAx0^Ba*RQQtf&5aLS1q_4Q@h6qq
z?Nq^~-z90aKr~cnV2lx2IA3vm7ffa3%D{Gmh?N9c?iJLDC`0<vIme;i!mek3DZ0yA
zvs;kF;md>5!4ZMbi7}s5XldyO<hD?$O&G{RW~g1HdE;GvDHpYHJo>z+(Hm}4?>t3(
zs(s8u!)AV28Wi4ibB>RUj2N*#kQM;rz>-jH$NHpZF=$E3h(|^d|0Hz6Hp=LY@-|^8
zD*q>}H~gd7W=*7ChnYCo_^`ot>n$kw=1tYxD>FDHN||<l_od%}F{d<mo?nrhvi*RS
zRbxm01f+<;1T39#QtFv?nrs%V7gdoNkG480WVGwZ)4$27Tt8U)pZJGu5s)9@;eh|@
z!0f%QS*j+{62vbM;r!*mC~nvEK3+PE#?y%cGJN3_4fZNF3z7}zD9^Dd9l>gBfA!8F
z{jCWMRwx@A6!^{xlI`z-$bc&fi?pj6^sRpj0G@{>=|*zwC8U6pZK=}w@-9rT+ALq6
z<74{L^tkYum-de0NXJ;`tj2V-V0%NFE+;EzWnwGQEE$M52w?g~vrlKb{+Kj$`aw@B
zMHjnV*2iFcuGNneE<uqU-n$r-;*_E}h&|#=z)Ui**>lOmZ>-g$ItNS+-4DsdT3Eva
zE|;;XI*F!eKotoZQioT;YSEcc6A#K87^8My=RPqsH{w(;U!juXX9W@(6{caa8y{4a
zj9TNlRVWUa+HRYAh!CZt80a;arwG<vzu<DnvAou#l&Y&te0z3Hb^WY2fO4((Ku{@{
zS?RkmQ#2vYM$E&Qeg~#Cb22at4=37h5`O+imtG>WL}GC3TVk}67-}mvmv(Bm5ahb~
zsSBXtqlvu(`a029u2OyvK`E<5vYd+G0>Dm~!~84(;{>M4s?X8R${0t%51y>xo_Wjf
zo+Fz3^!z`?ocTMHZ{Nqqnv7)#gGw`&WGCw|Neq!jgcinNkgY|s4cV7wj3vsFtuTbL
zWoa<7CwpVbKK5P6QVHLl%Y7X8b07Ed`~lDWe4f`a*ALg{{G9Li=k*?;E}j<IIJxqh
zmA6Zjf%>aVNU4F_K3k~>Q@0A-Q^MpHhVhJ5jLD4enOQNZmHI1yZ#?=f{4E-eRmqLO
zZ40=9-7ZW->O4=&>`n)d<uPsKeeXR51Qc`%!8&sccZ@F}E=}a^uf~S{naqh6x0-i&
z+yvVs+`^-lSn*Hvs<>Cq4Cor|s(-8ZVJ4|`zfF>(w~S33Iezi9y>PuZ{r&ciCbC#W
zXBv9q*|(GywR^Y111N$W1?yHHzvxPxSh;A~Q27v^1CCBDBt$|IN>XT4kH2y>&Q!_W
zo7b}vF%GUxZ!RvKz6f>~HjRRLS-77oI{6aAR=u9yPHrAzlWKBXpw8bIo!+K?^+l4g
zw|3#m<fX`|(TUq-8aa8inGaH>pKO)OHSP>&51~jOw%Gbwg?t~8s4oJla2L$12<-Ge
zIB%}Z;HDLNcvb6MW}CzAOJ&^WNZkI1qx=1pv6`hMO*k?`1hLWMDRJ`}ayY8_+Mw)n
z-aOY};U>ec$3oJoK|7(7ko#b><UsBIk|=&|IV$AZcW+TExhGK{#eH~69>`-0R7MRg
z4d8rADSppxn_ktBD1GH}lTzyQ%K>jC8&7<T<V3&Gmp&tDz=63x{a!J598Z_h(q+7j
zj!X9S-xUmN^4k+D*%!D|S=p+=e!4Aescitbfi7QL=iIxWc_$<>Q@`z%HG!~!kyQ?k
z-VQ+hR!O95Y8Pgra>twt$H*j>zzw;l4f-8bCN(DqZ3)e7Py6dVH-wuaC}U>>=R_LZ
znC@t>93bd`qm=Z6k;~6U7NW^J$z13!DF9&EfLQf4i54ke__Z=8lGUR~1k|VhMBSDk
zONn3pTa4$yjgB*eTta-4*`cBkD^><<S)oN5>xIunezYjm%@g)BMEpPzUF~tVZvg$u
zi_|MuJJMJ!EkxsFm_gqR_^M6)wi$VSW4-Oh+=n3dR(5#bTZ)t{hD+)V#nO;;0fe64
zn7m&ULD6Vv?N<(H!x<#YwYi^jeta(Kvx{$9P8GuP%-t)1A&x`K{D^VVM}y1bAvEpI
z8XZAS5NOp(Y+qKL*2OmG8uP>sCF8hluwJ0)AO&>H9i2*ucHRBN1ZN|0u2uamAuR9N
zsH`&pi50FMVzs=?`#D?CEn)6E1ux?$tyG=n2hV-_`h1|Lu3E0dA6qqNP1U3wHXa+G
z>5DN3fkL9*Cr~f`vLL5fhq1!0apH%}WcLDG3u5K(+6>Hp$?#?SUZ_smAmtwgbD4fa
z>Y}nIhqtf{cATv}uygD&UlV9tam)L-q}It=UVrpb<P2OCPgxf7?6`!Qm(?${UFLbP
zh7Uf8=?0ZGqJtC)Pd{2DrVF%4bzP~PuIkEgbloJ~<K~we)>KOtnqZoSWa9eP^iWT=
z6XYaM@hi{p#m<`uJou}pY6$=<I(qdAZVR;eym46t>=&2*>Ti^87vEtwf}wPOUpMVu
z_ZEc5pBFU-eqzQ+LsXPq1@b|{Oy51IWiuaXjRsI+S?_spw<7pdB;mqUqX+r<+&r|5
zBK-VV>#H+zKNObSaZpYGM1rwHr-N#C;20`*zyxoV*3-18_2-?Hcac^ELO}QZepmz{
zX8UYn^`Q7KxRG$k9Z*g9J<9JQ;_LUoU<MmBGtlR-RK5w)p_r)2fRUeXX-DdzqDREz
zg?g>Rv%8*^uWPV_aBQ^Ag`B(Z%;Y?07V2&N;-A}0{lh#nD;C4|GfS%T@fMG$=2i3_
zMD$|&q;OJ6ZVf+Z(V(6<KZ9?9dfve}X%4KwCXrs!%p_=+)MR5bVpA8*m<08YAQ(2@
zCjyZ?z|JX71|LSy;y+wFS$9{f{M0`U7<_V{ACl!gt?VENY?PVuX&qvgw>ZFvqoFQ=
zzFr3LV2CJy6kRALu6kJ4irju7$r{kwxD^ENY)S);V`k$?yy8;$?Y(VC!ura;PTzTA
z(<mDVL<?UMDH>h?Q_iRBHJ-_4sy<e|Wwqb$O`f8Ru%Ti}y|Fu(9~N?`d5>G$+xNXZ
zVvk6or2=|RydVPu<F`6DB+aEvR6|pcoDAGLhx3ZYYvZrmB*j0x8?SVTY|E>{<{nD@
z$NnAXK7_8RPX~P?yP7{_V&&Z1w+cR1aDE%1B`KE=EI4^O_R}=Hgi56o!86H~Upd2i
zqzc>VcZqj2L97Nnfp?AVpRRM9f_W;9s4``7_&uHq+G2JOL|dx`bV&Ow4<dfjvr6w?
z8y+8>Pef?5TrZHVagr$=ucOmnkP43dTy)x&O2y<Wo~<dNi!~VZe3g;Is^<RCV-9NW
z5B*FJM4ui?%~>Ji@i#q-^M!Qvbc@hSj37`pMa#?4mkf4qZ+gLmoj4bWL#A(adwqPc
zHZ|4qaK+zxge9h-Vdv}Xe*jJI*f2g;+i?t?d(}E-kvEGW3FtAC7JeR4#%9&PE4zeO
z#D6`sM$0iXijRd#`EP9Yko~`~S^NKBvv<eWG|LQPg)35S-}eYGsi@VS`sc4^S5|s`
zs+0C}{;j`CK+~!!@AkxGNlPAxtZ9uY88k`VVYc+9y%)`}(TNvKE|PfEq7PK5COlyE
z*S^U|7z6LqQStcqt^((F@^Wym@IG_E!Cd%y*SX$`S%mS3So{*D{$V9DH1NmK6vYqD
zGXUYf{`Ww{v8R+8EpcAeUPe8Tj~cQfSIlo|Edm#<In&dnQ$M9HcP^?|;@K0vpKKsn
z=r#oZ%tn0J+Pd+%hQ_gD%<_L7<N53#5_e_~+2Ci7b}o<3)v&zZk^9IhXE%YS6Cpe<
zWs(Np|F)c}O&A30@BwKzx9mcd7|x&PgOk&1$z{Hmk3QV$Jt?vGN6b#XVV3=)2jtSv
z+hwi$ubA}g$$)|o4pmfJsT8%!hohLoc3KS`Oo^;IUacfgxv962hd(Z<RGm1FG@_Dj
zJ#=hX3wmbD7$r@NJKTC_Yg8M`o;qp!_f?h4kmj;DmYrc57{~_^pb#frGXtBNO=}-Z
zQu4%{ZZ_FF>2<mLxNYB}Y(R?wgR+LZ`1o%c6d9uuxN+0R`bPRxSR_@I&u9Day1LJ&
z$ev*DvszY1+&}k;4Hun*-~qTp@>a4LAZ>*BF(iob#jZ>L8*3f?U##`Xzp>Wu$5?9{
zfVG-dXcl3&53Tw7KkTivzpj(6VF|du{-Ok#ZYMly)(hzsgk*tb_jo!xTH`>l!KvbP
zlug_6T{5T@A}9f&n=yiZdyCCH?=r)2>Wn-b1b^8{e16{hGo2vE5N;orAh^h;QxCmU
z)f_7<hKmP5aKK=D`6(QS8`S{X1Ei1Wmid(f(|WBQ3M-&q0SJ9F2RSjc-V0C4?gnC<
zR&~-F@qlg*k;Tda<jY3%m3dT*jY!Ty9t|V6E5!IF^bI02iOa`G?0Z-ZP&i;#PsO)H
zn|M*b^K}V=uIDgQpx@7yeFBsHGwdJTVHQRjLc99+>flz?r3OrErXrz0GCySyjo9op
zjT8RK3JRJe^bL3$U%f~LSVWuNx8ZaJU+oA|Z+`JeiTCtmTm>^q7`w+h!=ZC&R-*JS
z5bz-t(Iv6F_CYWwd$J8UX-DFUlpzWtGh*61Ky3`ko}QdP+ph%2_GaHsajIABMSJmH
z$JTPr>X#RvxH0_l#+<yMARC<kUJ@@m{1vhkojFvymeWp3Vu?Wuqa<~K;p+9S)nF@Z
zBhsQ0!>`CybH@3EB8QvtuJl8S9TRu}5Lcx2xp>#NO)Nx0H6GhL0)qL6O22&oeu`;!
zrAnR<IB5rK1${Gb*76JOEGfSo;-nt~WOE%Qf+T^&oE)$NFd>=zv)R=AGL8C>zn9(a
zx<ZWbwj~-%AZdUaXQ2kz=+B!Q$8k4KPi&_H8Sd%CoTHHUNKMZ*A@9V0CjW6z2>GKB
z4NRfc!u}Q8!EXdaS3N&qyv|hfwa_*OqU1bHWGH7QdKROEhu$>yX=x2z3Ir%^V<NQk
zh-OfddmyC<&f#npSY3*!XTY%}^m|}s67qFVt@BU6Uga<DE>8Svji;mqWLo5Ho{6jg
zh=iOF&mF_F1vUsQa5HPpeRIWBT~8CAco28+;267<!va1>lJ_|?*(kLfZ!Ln-S7p{z
zr^yw$z2Qidg7^p-v(68JSVZPpN(2UNr=;>L9=D6SSKofU0cxF&{UY?zno0Q$TQa*D
zgMWw>t6h8uY-0k0uJH48myJ#ZhPqYyx*7_Vc)2tQ<t+Af4;v>1D6^s2Mm;8r;erV<
zT)GULF9*g{xicsi^8d7$7E;teF}$BWDzB_hMwq%Y14kpZOYTt$xE!D>WUn0O-=JGf
zoo>qd;k;%e?fs<)-d}cRlLcK<ie5h8S{H)g%`!6!GDymJs0~Qei@|H_rW%jQSir3F
zKy*q*Kj(E{xI!b1<<^y5e35|tz^f54m}heyIpIr(rc7^x*oYLYKT9wR9t3<wRQKXn
z5ny~=)OGpNlc%#W!gqGLG=O+Oy;3oV6w2EyscWs60QCc1X*GL-%aE+(b+6bZj#8Ea
z<ZCM<Y-&%n(U=<#GB5ftvW=okm&r2kh1KI=olpw(9spby``k>tJy$UotV;ICCOdTc
zx@Kwj@*l)+N_e+iVQyT?+?L9Mvcj3dP~RavrFBblXKw;68^auC{UzzluX`6HUDoO(
z%K$Zl!!ob(&)lQq@#%LRzS;W*zZn^(`@&jajyf(##hXI0b&Yh;=elWNBz3R;d_nH_
zu#dm4T7EI55UzdNgW@7$KwIHys4jEJ#YS70X-G{{HBd1u)m<h7oZpWL(Gu5ydUK~?
zV&8wpGeJ_Od5uwrEh1)z`IB+uK#QnmwJkaQ?mCa%ED7s_%juC?QSeuRC3rd-Vj@~(
z^ymW8qz#YYY{1&NQPu^Pnh`v~67rQz$f}@B57gB4W3mouo8x}@CkKH@5^+kCk=)as
zJlWgEfj0yMoP~WU&hHX>dQjIgal8-f<*JuJ8%&1EO$^0un}V8mGJo#Z>$-GYR4-7$
zn7z{-qE5V=W84H2j9g$)B76J`y3;U+U*ewMeb!(7Kt0F+8ugx~QwZncrTC8ig_`$3
zq(}Go0p48v879loH{anA6C|?kaorR1FdKSD`(vH%){V#2M&Vw=lgdxUSsn_6TE2N6
zCwedH#q=8kgH?<GQMFONP}i`=PTH4|FSrfM0!#B<csjj?nDl)al=bWui;};dTMC`*
zt`Q*((j64noabOi6qfrSF4Gq?OfE6`y;6HnvEb-g9;Uzv`W$`(lDu;Y@E?yjPc#^r
zJya<R?`AqDlsjQ38Vp1thUC3r2t-OYs~%a=iS@WGhi_VvNsG7@l7im=0{F-qJqvb{
zqs*OcAKlo|Is~GPM$BlTi4QO7lakF-fe;%}DJZaNt2$(i%#LlsrBs`yYIMkdY{wGM
z<UG2KrYCwLDI=&<5nxe)ULz$Q;6=6lxO$Lu)zh3iVN>lu&=9XQ_BA=tMLIvk1vrsG
z!mKx@-+oS6t;BtdM!7YFTpCnFb+Ka?3fH28PG)NYD_|JYcH3te*^5pZ>ihnj-Xr-z
z%4~=sSo33NYGtOzRj0m@+rTm%z8QX#4e+i|mY#!ea$*5-=##0+g#_J8pRT2Q%N_N-
zP$x+f3eE^S10jbXJLVfq3tl4`18re}bW*$h$6@t*^Qy|>&@f#cw~y9{*GRNQSji(^
z!}<?Y_BuME&}H>FSu>S+^`~(b6#LUXrTX`(jp*VBq6ytX?YaXu*%HGsnip0U3!kws
zUyq~qu|N7I#l6&nNP1n*t|D(m^?_Y3eM)E2KIM7*){uZLlzrN(MRycYPkKlGDOvQ#
zC%5{9RM;bKB2Ve2d1OLg!i8Pxjck3oL{@|hG*O+kqZ`;4GL*gO@9JP)a6KIrE3I-t
zYNXCt%nm;L&nGuCkx%J~FFJPgnwd`k?Fa~DAMWY=1)QUP?RlOeq;?2si2uKTkf47Q
YMh8E?<S(1HIQ}n2`X;CnUAwUV0eZH7%K!iX

literal 0
HcmV?d00001


From c9ff60275308de0e2f3e160984e7de41985a03db Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Thu, 23 Mar 2017 10:43:17 +0100
Subject: [PATCH 264/373] fix docker (by moving to core version that still
 works)

- remove use of Core.Time_internal & move to 114.35+03
- update versions of other libs too
---
 docker/centos-7/Dockerfile     | 10 +++++-----
 docker/ubuntu-14.04/Dockerfile | 10 +++++-----
 docker/ubuntu-16.04/Dockerfile | 10 +++++-----
 src/tools/arakoon_logger.ml    | 34 +++++++++++++---------------------
 4 files changed, 28 insertions(+), 36 deletions(-)

diff --git a/docker/centos-7/Dockerfile b/docker/centos-7/Dockerfile
index 855776cf..5ee175d0 100644
--- a/docker/centos-7/Dockerfile
+++ b/docker/centos-7/Dockerfile
@@ -32,7 +32,7 @@ RUN opam init --root=${opam_root} --comp ${ocaml_version}
 
 RUN git clone git://github.com/janestreet/opam-repository.git /janestreet-opam-repo && \
     cd /janestreet-opam-repo && \
-    git checkout b98fd1964856f0c0b022a42ec4e6fc6c7bad2e81
+    git checkout 49a87b492c6d5e8a2c76bb8ac1956aac484c2f40
 RUN eval `${opam_env}` && \
     opam repo --root=${opam_root} -k local add janestreet /janestreet-opam-repo
 
@@ -56,18 +56,18 @@ ENV OPAMEXTERNALSOLVER="cudf_remote_proxy %{input}% %{output}% %{criteria}%"
 RUN eval `${opam_env}` && \
     opam update && \
     opam install -y \
-        oasis.0.4.6 \
+        oasis.0.4.8 \
         ocamlfind \
-        ssl.0.5.2 \
+        ssl.0.5.3 \
         camlbz2 \
         snappy \
         bisect \
-        lwt.2.5.2 \
+        lwt.2.7.0 \
         camltc.0.9.4 \
         ocplib-endian.1.0 \
         quickcheck.1.0.2 \
         conf-libev \
-        core.114.05+21 \
+        core.114.35+03 \
         redis.0.3.3 \
         uri.1.9.2
 
diff --git a/docker/ubuntu-14.04/Dockerfile b/docker/ubuntu-14.04/Dockerfile
index 8150ae9c..a7b90ff3 100644
--- a/docker/ubuntu-14.04/Dockerfile
+++ b/docker/ubuntu-14.04/Dockerfile
@@ -44,7 +44,7 @@ RUN opam init --root=${opam_root} --comp ${ocaml_version}
 
 RUN git clone git://github.com/janestreet/opam-repository.git /janestreet-opam-repo && \
     cd /janestreet-opam-repo && \
-    git checkout b98fd1964856f0c0b022a42ec4e6fc6c7bad2e81
+    git checkout 49a87b492c6d5e8a2c76bb8ac1956aac484c2f40
 RUN eval `${opam_env}` && \
     opam repo --root=${opam_root} -k local add janestreet /janestreet-opam-repo
 
@@ -52,18 +52,18 @@ RUN eval `${opam_env}` && \
 RUN eval `${opam_env}` && \
     opam update && \
     opam install -y \
-        oasis.0.4.6 \
+        oasis.0.4.8 \
         ocamlfind \
-        ssl.0.5.2 \
+        ssl.0.5.3 \
         camlbz2 \
         snappy \
         bisect \
-        lwt.2.5.2 \
+        lwt.2.7.0 \
         camltc.0.9.4 \
         ocplib-endian.1.0 \
         quickcheck.1.0.2 \
         conf-libev \
-        core.114.05+21 \
+        core.114.35+03 \
         redis.0.3.3 \
         uri.1.9.2
 
diff --git a/docker/ubuntu-16.04/Dockerfile b/docker/ubuntu-16.04/Dockerfile
index aa2ff09d..3e07662c 100644
--- a/docker/ubuntu-16.04/Dockerfile
+++ b/docker/ubuntu-16.04/Dockerfile
@@ -42,25 +42,25 @@ RUN opam init --root=${opam_root} --comp ${ocaml_version}
 
 RUN git clone git://github.com/janestreet/opam-repository.git /janestreet-opam-repo && \
     cd /janestreet-opam-repo && \
-    git checkout b98fd1964856f0c0b022a42ec4e6fc6c7bad2e81
+    git checkout 49a87b492c6d5e8a2c76bb8ac1956aac484c2f40
 RUN eval `${opam_env}` && \
     opam repo --root=${opam_root} -k local add janestreet /janestreet-opam-repo
 
 RUN eval `${opam_env}` && \
     opam update && \
     opam install -y \
-        oasis.0.4.6 \
+        oasis.0.4.8 \
         ocamlfind \
-        ssl.0.5.2 \
+        ssl.0.5.3 \
         camlbz2 \
         snappy \
         bisect \
-        lwt.2.5.2 \
+        lwt.2.7.0 \
         camltc.0.9.4 \
         ocplib-endian.1.0 \
         quickcheck.1.0.2 \
         conf-libev \
-        core.114.05+21 \
+        core.114.35+03 \
         redis.0.3.3 \
         uri.1.9.2
 
diff --git a/src/tools/arakoon_logger.ml b/src/tools/arakoon_logger.ml
index 2643713d..f15c6f6e 100644
--- a/src/tools/arakoon_logger.ml
+++ b/src/tools/arakoon_logger.ml
@@ -71,30 +71,22 @@ let format_message
       ~ts ~seqnum
       ~section ~level ~lines
   =
+
+  let d =
+    let (f : float) = Obj.magic ts in
+    let df = mod_float f 1.0 in
+    let udf = df *. 1_000_000. in
+    int_of_float udf
+  in
+  let tz_diff = "+0000" in
   Printf.sprintf
-    "%s - %s - %i/0 - %s - %i - %s - %s"
+    "%s %6d %s %s - %i/0 - %s - %i - %s - %s"
     (let open Unix in
-     let tm = Core.Time_internal.to_tm ts in
-     let offset =
-       Core.Time.utc_offset ~zone:Core.Time.Zone.local ts
-       |> Core.Span.to_parts
-     in
-     Printf.sprintf
-       "%04d-%02d-%02d %02d:%02d:%02d %d %c%02d%02d"
-       (tm.tm_year + 1900)
-       (tm.tm_mon + 1)
-       tm.tm_mday
-       tm.tm_hour
-       tm.tm_min
-       tm.tm_sec
-       (int_of_float ((mod_float (Core.Time.to_float ts) 1.) *. 1_000_000.))
-       (let open Core_kernel in
-        match offset.Core.Span.Parts.sign with
-        | Sign.Neg -> '-'
-        | Sign.Pos | Sign.Zero -> '+')
-       offset.Core.Span.Parts.hr
-       offset.Core.Span.Parts.min
+     let open Core in
+     Time.format ts "%Y/%m/%d %T" Time.Zone.utc
     )
+    d
+    tz_diff
     hostname
     pid
     component

From c300e2cd78fb84c164eb34641a96b7b33b775cd4 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Thu, 23 Mar 2017 13:16:09 +0100
Subject: [PATCH 265/373] closer to orig

---
 src/tools/arakoon_logger.ml | 49 ++++++++++++++++++++++++-------------
 1 file changed, 32 insertions(+), 17 deletions(-)

diff --git a/src/tools/arakoon_logger.ml b/src/tools/arakoon_logger.ml
index f15c6f6e..2db8b978 100644
--- a/src/tools/arakoon_logger.ml
+++ b/src/tools/arakoon_logger.ml
@@ -67,28 +67,42 @@ let console_logger =
 
 
 let format_message
-      ~hostname ~pid ~component
-      ~ts ~seqnum
+      ~hostname ~pid ~thread_id ~component
+      ~(ts:Core.Time.t) ~seqnum
       ~section ~level ~lines
   =
-
-  let d =
-    let (f : float) = Obj.magic ts in
-    let df = mod_float f 1.0 in
-    let udf = df *. 1_000_000. in
-    int_of_float udf
-  in
-  let tz_diff = "+0000" in
+  let thread_id = 0 in
   Printf.sprintf
-    "%s %6d %s %s - %i/0 - %s - %i - %s - %s"
+    "%s - %s - %i/%i - %s - %i - %s - %s"
     (let open Unix in
-     let open Core in
-     Time.format ts "%Y/%m/%d %T" Time.Zone.utc
+
+     let (fts:float) = Core.Time.to_span_since_epoch ts
+                       |> Core.Time.Span.to_sec
+     in
+     let tm = Unix.localtime fts in
+     let offset =
+       let zone = Lazy.force Core.Time.Zone.local in
+       Core.Time.utc_offset ~zone ts
+       |> Core.Time.Span.to_parts
+     in
+     Printf.sprintf
+       "%04d-%02d-%02d %02d:%02d:%02d %d %c%02d%02d"
+       (tm.tm_year + 1900)
+       (tm.tm_mon + 1)
+       tm.tm_mday
+       tm.tm_hour
+       tm.tm_min
+       tm.tm_sec
+       (int_of_float ((mod_float fts 1.) *. 1_000_000.))
+       (let open Core in
+        match offset.Time.Span.Parts.sign with
+        | Sign.Neg -> '-'
+        | Sign.Pos | Sign.Zero -> '+')
+       offset.Core.Time.Span.Parts.hr
+       offset.Core.Time.Span.Parts.min
     )
-    d
-    tz_diff
     hostname
-    pid
+    pid thread_id
     component
     seqnum
     (Lwt_log.string_of_level level)
@@ -118,9 +132,10 @@ let setup_log_sinks log_sinks =
                           let seqnum' = !seqnum in
                           let () = incr seqnum in
                           let ts = Core.Time.now () in
+                          let thread_id = 0 in
                           let logline =
                             format_message
-                              ~hostname ~pid ~component
+                              ~hostname ~pid ~thread_id ~component
                               ~ts ~seqnum:seqnum'
                               ~section ~level ~lines
                           in

From d38c4f3d1f5f849de1961b6acdc087652c442c64 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Thu, 23 Mar 2017 15:12:43 +0100
Subject: [PATCH 266/373] comments

---
 src/tools/arakoon_logger.ml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/tools/arakoon_logger.ml b/src/tools/arakoon_logger.ml
index 2db8b978..7c0a115b 100644
--- a/src/tools/arakoon_logger.ml
+++ b/src/tools/arakoon_logger.ml
@@ -71,9 +71,8 @@ let format_message
       ~(ts:Core.Time.t) ~seqnum
       ~section ~level ~lines
   =
-  let thread_id = 0 in
   Printf.sprintf
-    "%s - %s - %i/%i - %s - %i - %s - %s"
+    "%s - %s - %i/%04i - %s - %i - %s - %s"
     (let open Unix in
 
      let (fts:float) = Core.Time.to_span_since_epoch ts
@@ -86,7 +85,7 @@ let format_message
        |> Core.Time.Span.to_parts
      in
      Printf.sprintf
-       "%04d-%02d-%02d %02d:%02d:%02d %d %c%02d%02d"
+       "%04d-%02d-%02d %02d:%02d:%02d %06d %c%02d%02d"
        (tm.tm_year + 1900)
        (tm.tm_mon + 1)
        tm.tm_mday

From b10341e559dcd4f072e63f9cd058ec7deac77959 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Tue, 11 Apr 2017 11:00:08 +0200
Subject: [PATCH 267/373] dump synced sequence in full splendor too

---
 src/tlog/update.ml | 34 ++++++++++++++++++++--------------
 1 file changed, 20 insertions(+), 14 deletions(-)

diff --git a/src/tlog/update.ml b/src/tlog/update.ml
index 9e008b4d..a397ac20 100644
--- a/src/tlog/update.ml
+++ b/src/tlog/update.ml
@@ -74,25 +74,30 @@ module Update = struct
       | None -> "None"
       | Some s -> if values then s else "..."
     in
-    let rec _inner = function
+    let rec _sequence2s typ updates =
+      let buf = Buffer.create (64 * List.length updates) in
+      let add s = Buffer.add_string buf s in
+      let () = add typ in
+      let () = add "([" in
+      let rec loop = function
+        | [] -> add "])"
+        | u::us ->
+           let () = add (_inner u) in
+           let () = if (us <> []) then add "; "
+           in
+           loop us
+      in
+      let () = loop updates in
+      Buffer.contents buf
+    and
+      _inner = function
       | Set (k,v)                 -> Printf.sprintf "Set            ;%S;%i;%S" k (String.length v) (maybe v)
       | Delete k                  -> Printf.sprintf "Delete         ;%S" k
       | MasterSet (m,i)           -> Printf.sprintf "MasterSet      ;%S;%f" m i
       | TestAndSet (k, _, wo) ->
         let ws = _size_of wo in      Printf.sprintf "TestAndSet     ;%S;%i;%S" k ws (maybe_o wo)
       | Sequence updates ->
-        let buf = Buffer.create (64 * List.length updates) in
-        let () = Buffer.add_string buf "Sequence([" in
-        let rec loop = function
-          | [] -> Buffer.add_string buf "])"
-          | u::us ->
-            let () = Buffer.add_string buf (_inner u) in
-            let () = if (us <> []) then Buffer.add_string buf "; "
-            in
-            loop us
-        in
-        let () = loop updates in
-        Buffer.contents buf
+         _sequence2s "Sequence" updates
       | SetInterval range ->      Printf.sprintf "SetInterval     ;%s" (Interval.to_string range)
       | SetRouting routing ->     Printf.sprintf "SetRouting      ;%s" (Routing.to_s routing)
       | SetRoutingDelta (left,sep,right) -> Printf.sprintf "SetRoutingDelta %s < '%s' <= %s" left sep right
@@ -108,7 +113,8 @@ module Update = struct
         let ps = _size_of param in
         Printf.sprintf "UserFunction;%s;%i" name ps
       | AdminSet (key,vo)      -> Printf.sprintf "AdminSet        ;%S;%i;%S" key (_size_of vo) (maybe_o vo)
-      | SyncedSequence _us     -> Printf.sprintf "SyncedSequence  ;..."
+      | SyncedSequence updates     ->
+         _sequence2s "SyncedSequence" updates
       | DeletePrefix prefix    -> Printf.sprintf "DeletePrefix    ;%S" prefix
       | Replace(k,vo) ->
          Printf.sprintf "Replace            ;%S;%i" k (_size_of vo)

From 5d1727856bd716adb803ecc7ac2ed6c8d872c8e8 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Tue, 11 Apr 2017 15:59:32 +0200
Subject: [PATCH 268/373] add inspect-store command

---
 src/main/arakoon.ml    |  5 +++++
 src/node/dump_store.ml | 50 +++++++++++++++++++++++++++++++++++++-----
 2 files changed, 50 insertions(+), 5 deletions(-)

diff --git a/src/main/arakoon.ml b/src/main/arakoon.ml
index 2a073ee2..68c91de3 100644
--- a/src/main/arakoon.ml
+++ b/src/main/arakoon.ml
@@ -30,6 +30,7 @@ type local_action =
   | StripTlog
   | ReplayTlogs
   | DumpStore
+  | InspectStore
   | MakeTlog
   | MarkTlog
   | CloseTlog
@@ -279,6 +280,9 @@ let main () =
     ("--dump-store", Arg.Tuple [ set_laction DumpStore;
                                  Arg.Set_string filename],
      "<filename> : dump a store");
+    ("--inspect-store", Arg.Tuple [ set_laction InspectStore;
+                                    Arg.Set_string filename],
+     "<filename> : inspect a (tokyo cabinet) store (also use -left and -max_results)");
     ("--compress-tlog", Arg.Tuple[set_laction CompressTlog;
                                   Arg.Set_string filename],
      "<filename> : compress a tlog file");
@@ -467,6 +471,7 @@ let main () =
     | CloseTlog -> Tlog_main.mark_tlog !filename (Tlog_map._make_close_marker !node_id)
     | ReplayTlogs -> Replay_main.replay_tlogs !tlog_dir !tlf_dir !filename !end_i
     | DumpStore -> Dump_store.dump_store !filename
+    | InspectStore -> Dump_store.inspect_store !filename !left !max_results
     | VerifyStore -> Dump_store.verify_store !filename None
     | TruncateTlog -> Tlc2.truncate_tlog !filename
     | CompressTlog -> Tlog_main.compress_tlog !filename !archive_type
diff --git a/src/node/dump_store.ml b/src/node/dump_store.ml
index 7ae83be6..4cd3b061 100644
--- a/src/node/dump_store.ml
+++ b/src/node/dump_store.ml
@@ -175,10 +175,8 @@ let verify_store fn start_key =
             let _value = Bdb.value bdb cursor in
             begin
               if (i < 10 || i mod 20000 = 0)
-              then Lwt_io.eprintlf "%10i: %S\tok%!" i key
-              else Lwt.return_unit
-            end
-            >>= fun ()->
+              then Lwt_io.eprintlf "%10i: %S\tok%!" i key |> Lwt.ignore_result
+            end;
             let cont =
               try
                 Bdb.next bdb cursor;
@@ -187,11 +185,53 @@ let verify_store fn start_key =
             in
             if cont
             then loop (i+1)
-            else Lwt.return 0
+            else 0
           in
           loop 0
         )
+      |> Lwt.return
     in
     go ()
   in
   Lwt_extra.run t
+
+let inspect_store fn left max_results =
+  let t () =
+    let open Camltc in
+    let open Local_store in
+    make_store ~lcnum:1024 ~ncnum:512 true fn >>= fun ls ->
+    Bdb.with_cursor
+      (Hotc.get_bdb ls.db)
+      (fun bdb cursor ->
+        let () = match left with
+          | None -> Bdb.first bdb cursor
+          | Some left -> Bdb.jump bdb cursor left
+        in
+        let rec loop i =
+          if i < max_results
+          then
+            begin
+              let key = Bdb.key bdb cursor in
+              let value = Bdb.value bdb cursor in
+              Lwt_io.printlf "%i: key=%S value=%S" i key value |> Lwt.ignore_result;
+              let continue =
+                try Bdb.next bdb cursor; true
+                with Not_found -> false
+              in
+              if continue
+              then loop (i + 1)
+              else Lwt_io.printlf "Finished iterating over db" |> Lwt.ignore_result
+            end
+        in
+        loop 0
+      );
+    Lwt.return 0
+  in
+  Lwt_extra.run
+    (fun () ->
+      Lwt.catch
+        t
+        (fun exn ->
+          Lwt_io.printlf "Got exception during inspect store: %s" (Printexc.to_string exn) >>= fun () ->
+          Lwt.return 1
+    ))

From c1f27f91ec35c7815221e75c68cc8b740955ec2a Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 12 Apr 2017 13:31:39 +0200
Subject: [PATCH 269/373] 64KB Lwt_io.default_buffer_size

---
 src/client/client_main.ml | 1 -
 src/main/arakoon.ml       | 4 +++-
 src/node/local_store.ml   | 2 +-
 src/node/node_main.ml     | 1 -
 src/tools/file_system.ml  | 5 +++--
 5 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/src/client/client_main.ml b/src/client/client_main.ml
index 3d474754..6502a0b2 100644
--- a/src/client/client_main.ml
+++ b/src/client/client_main.ml
@@ -171,7 +171,6 @@ let benchmark
       ~tls
       cfg_name key_size value_size tx_size max_n n_clients
       scenario_s =
-  Lwt_io.set_default_buffer_size 32768;
   let scenario = Ini.p_string_list scenario_s in
   let t () =
     let with_c = with_master_client ~tls cfg_name in
diff --git a/src/main/arakoon.ml b/src/main/arakoon.ml
index 68c91de3..1292e375 100644
--- a/src/main/arakoon.ml
+++ b/src/main/arakoon.ml
@@ -163,10 +163,12 @@ let main () =
   Ssl_threads.init ();
   Ssl.init ~thread_safe:true ();
 
-  let () =Lwt.async_exception_hook :=
+  let () =
+    Lwt.async_exception_hook :=
     (fun exn -> Logger.ign_info_f_ ~exn "Caught async exception")
   in
   let _ = Bz2.version in
+  let () = Lwt_io.set_default_buffer_size 65536 in
   let () = Sys.set_signal Sys.sigpipe Sys.Signal_ignore in
   let () = Random.self_init () in
   let () = Client_log.enable_lwt_logging_for_client_lib_code () in
diff --git a/src/node/local_store.ml b/src/node/local_store.ml
index 031ef9d3..9b122dc7 100644
--- a/src/node/local_store.ml
+++ b/src/node/local_store.ml
@@ -41,7 +41,7 @@ let copy_store2 old_location new_location
   if not src_exists
   then
     Logger.info_f_ "File at %s does not exist" old_location >>= fun () ->
-    raise Not_found
+    Lwt.fail Not_found
   else
     File_system.copy_file old_location new_location ~overwrite ~throttling
 
diff --git a/src/node/node_main.ml b/src/node/node_main.ml
index b40a83bb..38d68f25 100644
--- a/src/node/node_main.ml
+++ b/src/node/node_main.ml
@@ -392,7 +392,6 @@ let _main_2 (type s)
       ~lock
     : int Lwt.t
   =
-  Lwt_io.set_default_buffer_size 32768;
   make_config () >>= fun cluster_cfg ->
   let cfgs = cluster_cfg.cfgs in
   let me, others = Node_cfg.Node_cfg.split name cfgs in
diff --git a/src/tools/file_system.ml b/src/tools/file_system.ml
index 96cb4891..81878654 100644
--- a/src/tools/file_system.ml
+++ b/src/tools/file_system.ml
@@ -114,9 +114,10 @@ let with_tmp_file tmp dest f =
 
 let copy_file source target ~overwrite ~throttling =
   (* LOOKS LIKE Clone.copy_stream ... *)
-  Logger.info_f_ "copy_file %s %s (overwrite=%b,throttling=%f)"
-                 source target overwrite throttling >>= fun () ->
   let bs = Lwt_io.default_buffer_size () in
+  Logger.info_f_ "copy_file %s %s (overwrite=%b,throttling=%f) buffer_size:%i"
+                 source target overwrite throttling bs >>= fun () ->
+
   let buffer = String.make bs 'a' in
   let throttle =
     match throttling with

From 5e2014505620f9a77bc9e20fc64191dc11286555 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 12 Apr 2017 14:00:10 +0200
Subject: [PATCH 270/373] 1MB buffer for file copy

---
 src/tools/file_system.ml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/tools/file_system.ml b/src/tools/file_system.ml
index 81878654..ba35f3cd 100644
--- a/src/tools/file_system.ml
+++ b/src/tools/file_system.ml
@@ -114,11 +114,11 @@ let with_tmp_file tmp dest f =
 
 let copy_file source target ~overwrite ~throttling =
   (* LOOKS LIKE Clone.copy_stream ... *)
-  let bs = Lwt_io.default_buffer_size () in
+  let bs = 1024 * 1024 in
   Logger.info_f_ "copy_file %s %s (overwrite=%b,throttling=%f) buffer_size:%i"
                  source target overwrite throttling bs >>= fun () ->
 
-  let buffer = String.make bs 'a' in
+  let buffer = Bytes.create bs in
   let throttle =
     match throttling with
     | 0.0   -> fun f  -> f ()

From 1626da79afaeb588e7fa945c1be4bf1f2d601005 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 12 Apr 2017 14:52:19 +0200
Subject: [PATCH 271/373] fix bug in copy_file

---
 src/tools/file_system.ml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/tools/file_system.ml b/src/tools/file_system.ml
index ba35f3cd..93ba15b8 100644
--- a/src/tools/file_system.ml
+++ b/src/tools/file_system.ml
@@ -139,7 +139,8 @@ let copy_file source target ~overwrite ~throttling =
          if bytes_read > 0
          then
            begin
-             Lwt_io.write oc buffer >>= fun () ->
+             Lwt_io.write_from_exactly oc buffer 0 bytes_read
+             >>= fun () ->
              Lwt.return `Continue
            end
          else

From 8b97ecd4d4cdef725337aeacadfbf39fafef6f81 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 12 Apr 2017 15:25:57 +0200
Subject: [PATCH 272/373] show problems in travis

---
 docker/suites.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docker/suites.sh b/docker/suites.sh
index 6d80df66..08a777e0 100755
--- a/docker/suites.sh
+++ b/docker/suites.sh
@@ -14,8 +14,10 @@ case "${1-bash}" in
         ;;
     unit)
         make build
-        ./arakoon.native --run-all-tests-xml testresults.xml || true
+        ./arakoon.native --run-all-tests-xml testresults.xml
+        x=$?
         cat testresults.xml
+        exit ${x}
         ;;
     kwik)
         ./jenkins/kwik.sh

From af1a06536ec7477d7567317b4147a0dbb526e120 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 12 Apr 2017 16:16:11 +0200
Subject: [PATCH 273/373] make_release: 1.9.16

---
 META                      | 2 +-
 debian/changelog          | 5 +++++
 redhat/SPECS/arakoon.spec | 4 +++-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/META b/META
index 7f2ae286..68db81d4 100644
--- a/META
+++ b/META
@@ -1,7 +1,7 @@
 # -*- conf -*-
 
 description = "Arakoon client"
-version = "1.9.15"
+version = "1.9.16"
 exists_if = "arakoon_client.cma,arakoon_client.cmxa,arakoon_client.cmxs"
 requires = "threads lwt lwt.unix str oUnit quickcheck lwt.ssl ocplib-endian redis.lwt uri"
 archive(byte) = "arakoon_client.cma tcp_keepalive_sockopt_stubs.o"
diff --git a/debian/changelog b/debian/changelog
index d4424873..db0e32a9 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,8 @@
+arakoon (1.9.16) unstable; urgency=low
+  * new upstream version
+
+ -- Romain Slootmaekers <romain.slootmaekers@openvstorage.com>  Wed, 12 Apr 2017 16:16:11 +0100
+
 arakoon (1.9.15) unstable; urgency=low
   * new upstream version
 
diff --git a/redhat/SPECS/arakoon.spec b/redhat/SPECS/arakoon.spec
index bc247a8f..d079fcc0 100644
--- a/redhat/SPECS/arakoon.spec
+++ b/redhat/SPECS/arakoon.spec
@@ -2,7 +2,7 @@
 
 Summary: Arakoon
 Name: arakoon
-Version: 1.9.15
+Version: 1.9.16
 Release: 3%{?dist}
 License: Apache 2.0
 Requires: libev >= 4
@@ -33,6 +33,8 @@ cp arakoon.native %{buildroot}%{_bindir}/arakoon
 %{_bindir}/arakoon
 
 %changelog
+* Wed Apr 12 2017 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.16
+- Create arakoon 1.9.16 RPM package
 * Thu Nov 03 2016 Jan Doms <jan.doms@gmail.com> - 1.9.15
 - Create arakoon 1.9.15 RPM package
 * Mon Oct 24 2016 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.14

From 143e990a66e03ff72980b6202fd12b751b385e0f Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 12 Apr 2017 21:37:04 +0200
Subject: [PATCH 274/373] test can be run twice in a row now

---
 pylabs/test/server/right/test_catchup_exercises.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/pylabs/test/server/right/test_catchup_exercises.py b/pylabs/test/server/right/test_catchup_exercises.py
index 55ed0daa..4ee35c58 100644
--- a/pylabs/test/server/right/test_catchup_exercises.py
+++ b/pylabs/test/server/right/test_catchup_exercises.py
@@ -79,10 +79,14 @@ def test_catchup_mixed_config():
                                          "test_catchup_mixed_config"])
 
     def setup(c_id,tlog_max_entries):
-            c_nodes = ["%s_0" % c_id]
+            node_0 = "%s_0" % c_id
+            c_nodes = [node_0]
             c_home = "/".join([Common.data_base_dir, c_id])
+            node_dir = "/".join([Common.data_base_dir, node_0])
+            X.removeDirTree(node_dir)
             c = Common.setup_n_nodes_base(c_id, c_nodes,False, c_home,
-                                          Common.node_msg_base_port, Common.node_client_base_port,
+                                          Common.node_msg_base_port,
+                                          Common.node_client_base_port,
                                           extra = {'tlog_max_entries':str(tlog_max_entries)}
             )
             return c

From 9cabff44879be41d3b040667b7263c3d5f56738c Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Fri, 14 Apr 2017 14:47:11 +0200
Subject: [PATCH 275/373] cleanup tlogs after local collapse, even when there
 was nothing to collapse

---
 src/node/collapser.ml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/node/collapser.ml b/src/node/collapser.ml
index b9323531..9c1fb332 100644
--- a/src/node/collapser.ml
+++ b/src/node/collapser.ml
@@ -212,7 +212,11 @@ let collapse_many
   get_head_i () >>= fun head_i ->
   let todo = tlog_coll # tlogs_to_collapse ~head_i ~last_i ~tlogs_to_keep in
   match todo with
-  | None -> Logger.info_f_ "Nothing to collapse..." >>= fun () -> cb' 0
+  | None ->
+     Logger.info_f_ "Nothing to collapse..." >>= fun () ->
+     get_head_i () >>= fun head_i ->
+     tlog_coll # remove_below head_i >>= fun () ->
+     cb' 0
   | Some (n, too_far_i) ->
     begin
       Logger.info_f_ "Going to collapse %d tlogs" n >>= fun () ->

From 06005d544237daa312b11a363efcbda8ab48b5da Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Tue, 18 Apr 2017 14:11:47 +0200
Subject: [PATCH 276/373] check cluster id in store, to prevent ops errors

---
 src/node/catchup.ml             | 11 +++++++----
 src/node/catchup_test.ml        | 10 +++++++---
 src/node/collapser.ml           | 20 +++++++++++++------
 src/node/collapser_main.ml      |  2 +-
 src/node/collapser_test.ml      | 18 ++++++++++-------
 src/node/dump_store.ml          |  5 ++++-
 src/node/node_main.ml           | 13 +++++++++----
 src/node/simple_store.ml        |  1 +
 src/node/store.ml               | 34 ++++++++++++++++++++++++++++++---
 src/node/store_test.ml          |  3 ++-
 src/node/sync_backend.ml        |  8 +++++---
 src/paxos/multi_paxos_test.ml   | 26 +++++++++++++++++++------
 src/system/startup.ml           |  8 ++++++--
 src/tlog/mem_tlogcollection.ml  |  1 +
 src/tlog/tlc2.ml                | 17 +++++++++++------
 src/tlog/tlc2_test.ml           |  2 ++
 src/tlog/tlogcollection.ml      |  4 +++-
 src/tlog/tlogcollection_test.ml |  1 +
 18 files changed, 136 insertions(+), 48 deletions(-)

diff --git a/src/node/catchup.ml b/src/node/catchup.ml
index c7b57d14..4b2ba369 100644
--- a/src/node/catchup.ml
+++ b/src/node/catchup.ml
@@ -34,15 +34,18 @@ let _with_client_connection ~tls_ctx ~tcp_keepalive (ips,port) f =
     (fun _ -> f)
 
 
-let head_saved_epilogue hfn tlog_coll =
+let head_saved_epilogue ~cluster_id hfn tlog_coll =
   (* maybe some tlogs must be thrown away because
      they were already collapsed into
      the received head
   *)
   Logger.info_f_ "head_saved_epilogue %s" hfn >>= fun () ->
   let module S = (val (Store.make_store_module (module Batched_store.Local_store))) in
-  S.make_store ~lcnum:default_lcnum
-    ~ncnum:default_ncnum ~read_only:true hfn >>= fun store ->
+  S.make_store
+    ~lcnum:default_lcnum
+    ~ncnum:default_ncnum ~read_only:true
+    ~cluster_id
+    hfn >>= fun store ->
   let hio = S.consensus_i store in
   S.close store ~flush:false ~sync:false >>= fun () ->
   Logger.info_ "closed head" >>= fun () ->
@@ -78,7 +81,7 @@ let catchup_tlog (type s) ~tls_ctx ~tcp_keepalive ~stop other_configs ~cluster_i
   in
   let head_saved_cb hfn =
     Logger.info_f_ "head_saved_cb %s" hfn >>= fun () ->
-    head_saved_epilogue hfn tlog_coll >>= fun () ->
+    head_saved_epilogue ~cluster_id hfn tlog_coll >>= fun () ->
     let when_closed () =
       Logger.debug_ "when_closed" >>= fun () ->
       let target_name = S.get_location store in
diff --git a/src/node/catchup_test.ml b/src/node/catchup_test.ml
index 6c313e0a..6c26125f 100644
--- a/src/node/catchup_test.ml
+++ b/src/node/catchup_test.ml
@@ -27,6 +27,7 @@ module S = (val (Store.make_store_module (module Batched_store.Local_store)))
 
 let _dir_name = "/tmp/catchup_test"
 let _tlx_dir = "/tmp/catchup_test/tlx"
+let cluster_id = "cluster_id"
 
 let _fill tlog_coll n =
   let sync = false in
@@ -110,11 +111,13 @@ let test_common () =
   Logger.info_ "test_common" >>= fun () ->
   Tlc2.make_tlc2 ~compressor:Compression.Snappy
                  _dir_name _tlx_dir _tlx_dir
-                 ~fsync:false "node_name" ~fsync_tlog_dir:true >>= fun tlog_coll ->
+                 ~fsync:false "node_name" ~fsync_tlog_dir:true
+                 ~cluster_id:""
+  >>= fun tlog_coll ->
   _fill tlog_coll 1000 >>= fun () ->
   let me = "" in
   let db_name = _dir_name ^ "/my_store1.db" in
-  S.make_store ~lcnum:1024 ~ncnum:512 db_name >>= fun store ->
+  S.make_store ~lcnum:1024 ~ncnum:512 db_name ~cluster_id >>= fun store ->
   Catchup.catchup_store ~stop:(ref false) me ((module S),store,tlog_coll) 500L >>= fun() ->
   Logger.info_ "TODO: validate store after this" >>= fun ()->
   S.close store >>= fun () ->
@@ -141,10 +144,11 @@ let _tic filler_function n name verify_store =
   Tlc2.make_tlc2 ~compressor:Compression.Snappy
                  ~tlog_max_entries:101
                  _dir_name _tlx_dir _tlx_dir
+                 ~cluster_id
                  ~fsync:false node_id ~fsync_tlog_dir:true >>= fun tlog_coll ->
   filler_function tlog_coll n >>= fun () ->
   let db_name = _dir_name ^ "/" ^ name ^ ".db" in
-  S.make_store ~lcnum:1024 ~ncnum:512 db_name >>= fun store ->
+  S.make_store ~lcnum:1024 ~ncnum:512 db_name ~cluster_id >>= fun store ->
   let me = "??me??" in
   Catchup.verify_n_catchup_store ~stop:(ref false) me ((module S), store, tlog_coll)
   >>= fun () ->
diff --git a/src/node/collapser.ml b/src/node/collapser.ml
index b9323531..abdb98f1 100644
--- a/src/node/collapser.ml
+++ b/src/node/collapser.ml
@@ -32,7 +32,9 @@ let collapse_until (type s) (tlog_coll:Tlogcollection.tlog_collection)
     )
     (too_far_i:Sn.t)
     (cb: int -> unit Lwt.t)
-    (slowdown : float option)=
+    (slowdown : float option)
+    ~cluster_id
+  =
 
   let new_location = head_location ^ ".clone" in
   Logger.info_f_ "Creating db clone at %s (throttling:%f)"
@@ -51,8 +53,10 @@ let collapse_until (type s) (tlog_coll:Tlogcollection.tlog_collection)
   )
   >>= fun () ->
   Logger.debug_f_ "Creating store at %s" new_location >>= fun () ->
-  S.make_store ~lcnum:Node_cfg.default_lcnum
+  S.make_store
+    ~lcnum:Node_cfg.default_lcnum
     ~ncnum:Node_cfg.default_ncnum
+    ?cluster_id
     new_location >>= fun new_store ->
   Lwt.finalize (
     fun () ->
@@ -176,13 +180,14 @@ let collapse_until (type s) (tlog_coll:Tlogcollection.tlog_collection)
       (fun exn -> Logger.warning_f_ ~exn "ignoring failure in cleanup")
     )
 
-let _head_i (type s) (module S : Store.STORE with type t = s) head_location =
+let _head_i (type s) (module S : Store.STORE with type t = s) ?cluster_id head_location =
   Lwt.catch
     (fun () ->
        let read_only=true in
        S.make_store
          ~lcnum:Node_cfg.default_lcnum
          ~ncnum:Node_cfg.default_ncnum
+         ?cluster_id
          ~read_only head_location >>= fun head ->
        let head_io = S.consensus_i head in
        S.close head ~sync:false ~flush:false >>= fun () ->
@@ -194,6 +199,7 @@ let _head_i (type s) (module S : Store.STORE with type t = s) head_location =
     )
 
 let collapse_many
+      ?cluster_id
       (type s) tlog_coll
       (module S : Store.STORE with type t = s)
       (store_fs: 'b * string * float)
@@ -203,7 +209,7 @@ let collapse_many
   let (_,(head_location:string), _) = store_fs in
   tlog_coll # get_last_i () >>= fun last_i ->
   let get_head_i () =
-    _head_i (module S) head_location >>= fun head_io ->
+    _head_i (module S) head_location ?cluster_id >>= fun head_io ->
     Logger.debug_f_ "head @ %s : last_i %s " (Log_extra.option2s Sn.string_of head_io) (Sn.string_of last_i)
     >>= fun () ->
     let head_i = match head_io with None -> Sn.start | Some i -> i in
@@ -218,13 +224,13 @@ let collapse_many
       Logger.info_f_ "Going to collapse %d tlogs" n >>= fun () ->
       cb' (n + 2) >>= fun () ->
       Logger.debug_f_ "too_far_i = %s" (Sn.string_of too_far_i) >>= fun () ->
-      collapse_until tlog_coll (module S) store_fs too_far_i cb slowdown >>= fun () ->
+      collapse_until tlog_coll (module S) store_fs too_far_i cb slowdown ~cluster_id >>= fun () ->
       get_head_i () >>= fun head_i ->
       tlog_coll # remove_below head_i >>= fun () ->
       cb n
     end
 
-let collapse_out_of_band cfg name tlogs_to_keep =
+let collapse_out_of_band ?cluster_id cfg name tlogs_to_keep =
   let open Lwt.Infix in
   let open Node_cfg.Node_cfg in
 
@@ -238,6 +244,7 @@ let collapse_out_of_band cfg name tlogs_to_keep =
                  me.tlog_dir me.tlx_dir me.head_dir
                  ~fsync:true name ~fsync_tlog_dir:true
                  ~check_marker:false
+                 ?cluster_id
   >>= fun tlog_coll ->
 
   collapse_many
@@ -250,5 +257,6 @@ let collapse_out_of_band cfg name tlogs_to_keep =
     (fun sn -> Lwt_log.debug_f "sn' = %i" sn)
     (fun n -> Lwt_log.debug_f "n = %i" n)
     me.collapse_slowdown
+    ?cluster_id
   >>= fun () ->
   Lwt.return ()
diff --git a/src/node/collapser_main.ml b/src/node/collapser_main.ml
index 1b454c8e..5be1b631 100644
--- a/src/node/collapser_main.ml
+++ b/src/node/collapser_main.ml
@@ -47,7 +47,7 @@ let collapse_local make_config node_id n_tlogs =
       (fun() ->
         let open Lwt.Infix in
         make_config () >>= fun cfg ->
-        Collapser.collapse_out_of_band cfg node_id n_tlogs >>= fun () ->
+        Collapser.collapse_out_of_band cfg node_id n_tlogs ?cluster_id:None >>= fun () ->
         Lwt.return 0
 
       )
diff --git a/src/node/collapser_test.ml b/src/node/collapser_test.ml
index aecbcc47..e8a072bf 100644
--- a/src/node/collapser_test.ml
+++ b/src/node/collapser_test.ml
@@ -23,6 +23,7 @@ open Tlog_map
 
 let section = Logger.Section.main
 let compressor = Compression.Snappy
+let cluster_id = Some "cluster_id"
 
 module S = (val (Store.make_store_module (module Batched_store.Local_store)))
 
@@ -62,6 +63,7 @@ let test_collapse_until (dn, tlx_dir, head_dir) =
                  ~tlog_max_entries:1000
                  dn tlx_dir head_dir
                  ~fsync:false node_id ~fsync_tlog_dir:false
+                 ~cluster_id:""
   >>= fun tlc ->
   _make_values tlc 0 1111 >>= fun () ->
   tlc # close () >>= fun () ->
@@ -73,14 +75,14 @@ let test_collapse_until (dn, tlx_dir, head_dir) =
   in
   let future_i = Sn.of_int 1001 in
   let cb = fun _s -> Lwt.return () in
-  Collapser.collapse_until tlc (module S) store_methods future_i cb None >>= fun () ->
+  Collapser.collapse_until tlc (module S) store_methods future_i cb None ~cluster_id >>= fun () ->
   (* some verification ? *)
 
   (* try to do it a second time, it should *)
   let future_i2 = Sn.of_int 1000 in
   _should_fail
     (fun () ->
-       Collapser.collapse_until tlc (module S) store_methods future_i2 cb None)
+       Collapser.collapse_until tlc (module S) store_methods future_i2 cb None ~cluster_id)
     "this should fail"
     "great, it indeed refuses to do this"
   >>= fun ()->
@@ -97,6 +99,7 @@ let test_collapse_many (dn, tlx_dir, head_dir) =
     Tlc2.make_tlc2 ~compressor ~tlog_max_entries:100
                  dn tlx_dir head_dir
                  ~fsync:false node_id ~fsync_tlog_dir:false
+                 ?cluster_id
   in
   make_tlc () >>= fun tlc ->
   _make_values tlc 0 632 >>= fun () ->
@@ -108,11 +111,11 @@ let test_collapse_many (dn, tlx_dir, head_dir) =
   let cb' = fun _ -> Lwt.return () in
   File_system.unlink storename >>= fun () ->
   let store_methods = (Batched_store.Local_store.copy_store2, storename, 0.0) in
-  Collapser.collapse_many tlc (module S) store_methods 6 cb' cb None >>= fun () ->
+  Collapser.collapse_many tlc (module S) store_methods 6 cb' cb None ?cluster_id >>= fun () ->
   Logger.debug_ "collapsed 000" >>= fun () ->
-  Collapser.collapse_many tlc (module S) store_methods 4 cb' cb None >>= fun () ->
+  Collapser.collapse_many tlc (module S) store_methods 4 cb' cb None ?cluster_id >>= fun () ->
   Logger.debug_ "collapsed 001 & 002" >>= fun () ->
-  Collapser.collapse_many tlc (module S) store_methods 1 cb' cb None >>= fun () ->
+  Collapser.collapse_many tlc (module S) store_methods 1 cb' cb None ?cluster_id >>= fun () ->
   Logger.debug_ "collapsed 003 & 004" >>= fun () -> (* ends @ 510 *)
   Lwt.return ()
 
@@ -124,6 +127,7 @@ let test_repeated_collapse (dn,tlx_dir, head_dir) =
   Tlc2.make_tlc2 ~compressor ~tlog_max_entries:100
                  dn tlx_dir head_dir
                  ~fsync:false node_id ~fsync_tlog_dir:false
+                 ?cluster_id
   >>= fun tlc ->
   let head_location= Filename.concat dn "head.db" in
   let count = ref 0 in
@@ -143,13 +147,13 @@ let test_repeated_collapse (dn,tlx_dir, head_dir) =
   let n_entries = 520 in
   _make_values tlc 0 n_entries >>= fun () ->
   let collapse_slowdown = None in
-  Collapser.collapse_many tlc (module S) store_methods 1 cb' cb collapse_slowdown >>= fun () ->
+  Collapser.collapse_many tlc (module S) store_methods 1 cb' cb collapse_slowdown ?cluster_id >>= fun () ->
   Logger.debug_f_"post collapse: count=%i" !count >>= fun () ->
   OUnit.assert_equal 0 !count ~printer:string_of_int ~msg:"count should be zero";
   Logger.debug_f_ "adding %i more entries" n_entries >>= fun () ->
   _make_values tlc n_entries (2 * n_entries) >>= fun () ->
   Lwt_unix.sleep 2.0 >>= fun () -> (* just to make debugging less messy *)
-  Collapser.collapse_many tlc (module S) store_methods 1 cb' cb collapse_slowdown >>= fun () ->
+  Collapser.collapse_many tlc (module S) store_methods 1 cb' cb collapse_slowdown ?cluster_id >>= fun () ->
   Logger.debug_f_"post collapse: count=%i" !count >>= fun () ->
   OUnit.assert_equal 0 !count ~printer:string_of_int ~msg:"count should be zero";
   tlc # close ~wait_for_compression:true () >>= fun () ->
diff --git a/src/node/dump_store.ml b/src/node/dump_store.ml
index 4cd3b061..1dc88aa9 100644
--- a/src/node/dump_store.ml
+++ b/src/node/dump_store.ml
@@ -39,6 +39,8 @@ let _dump_routing store =  try_fetch "routing" (fun () -> S.get_routing store) R
 
 let _dump_interval store = try_fetch "interval" (fun () -> S.get_interval store) Interval.to_string
 
+let _dump_cluster_id store = try_fetch "cluster_id" (fun () -> S.get_cluster_id store) Log_extra.string_option2s
+
 let summary store =
   let consensus_i = S.consensus_i store
   and mdo = S.who_master store
@@ -52,7 +54,8 @@ let summary store =
   Lwt_io.printlf "master: %s" s
   >>= fun () ->
   _dump_routing store >>= fun () ->
-  _dump_interval store
+  _dump_interval store >>= fun () ->
+  _dump_cluster_id store
 
 let dump_store filename =
   let t () =
diff --git a/src/node/node_main.ml b/src/node/node_main.ml
index 38d68f25..5b6235f1 100644
--- a/src/node/node_main.ml
+++ b/src/node/node_main.ml
@@ -208,13 +208,16 @@ let only_catchup
 
   S.make_store ~lcnum:cluster_cfg.lcnum
                ~ncnum:cluster_cfg.ncnum
-               db_name >>= fun store ->
+               db_name
+               ~cluster_id
+  >>= fun store ->
   let compressor = me.compressor in
   make_tlog_coll ~compressor
                  ~tlog_max_entries:cluster_cfg.tlog_max_entries
                  ~tlog_max_size:cluster_cfg.tlog_max_size
                  me.tlog_dir me.tlx_dir me.head_dir
                  ~fsync:false name ~fsync_tlog_dir:false
+                 ~cluster_id
   >>= fun tlc ->
   let try_catchup mr_name =
     Lwt.catch
@@ -566,11 +569,12 @@ let _main_2 (type s)
                          ~tlog_max_size:cluster_cfg.tlog_max_size
                          me.tlog_dir me.tlx_dir me.head_dir
                          ~fsync:me.fsync name ~fsync_tlog_dir:me._fsync_tlog_dir
+                         ~cluster_id
           >>= fun (tlog_coll:Tlogcollection.tlog_collection) ->
           let lcnum = cluster_cfg.lcnum
           and ncnum = cluster_cfg.ncnum
           in
-          S.make_store ~lcnum ~ncnum db_name >>= fun (store:S.t) ->
+          S.make_store ~lcnum ~ncnum db_name ~cluster_id >>= fun (store:S.t) ->
           Lwt.return (tlog_coll, store)
         in
         Lwt.catch
@@ -692,6 +696,7 @@ let _main_2 (type s)
               ~max_value_size:cluster_cfg.max_value_size
               ~collapse_slowdown:me.collapse_slowdown
               ~act_not_preferred
+              ~cluster_id
           in
           let backend = (sb :> Backend.backend) in
 
@@ -1056,9 +1061,9 @@ let main_t (make_config: unit -> 'a Lwt.t)
 
 let test_t make_config name ~stop =
   let module S = (val (Store.make_store_module (module Mem_store))) in
-  let make_tlog_coll ~compressor =
+  let make_tlog_coll ?cluster_id ~compressor =
     ignore compressor;
-    Mem_tlogcollection.make_mem_tlog_collection
+    Mem_tlogcollection.make_mem_tlog_collection ?cluster_id
   in
   let get_snapshot_name = fun () -> "DUMMY" in
   let daemonize = false
diff --git a/src/node/simple_store.ml b/src/node/simple_store.ml
index be341265..8c5b130c 100644
--- a/src/node/simple_store.ml
+++ b/src/node/simple_store.ml
@@ -45,6 +45,7 @@ let __j_key = "*j"
 let __interval_key = "*interval"
 let __routing_key = "*routing"
 let __master_key  = "*master"
+let __cluster_id_key = "*cluster_id"
 (* let __lease_key = "*lease" *)
 (* let __lease_key2 = "*lease2" *)
 let __prefix = "@"
diff --git a/src/node/store.ml b/src/node/store.ml
index d01c8a43..adf4a23f 100644
--- a/src/node/store.ml
+++ b/src/node/store.ml
@@ -40,7 +40,7 @@ module type STORE =
 sig
   type ss
   type t
-  val make_store : lcnum:int -> ncnum:int -> ?read_only:bool -> string -> t Lwt.t
+  val make_store : lcnum:int -> ncnum:int -> ?read_only:bool -> ?cluster_id:string -> string -> t Lwt.t
   val consensus_i : t -> Sn.t option
   val flush : t -> unit Lwt.t
   val close : ?flush : bool -> ?sync:bool -> t -> unit Lwt.t
@@ -89,6 +89,7 @@ sig
 
   val get_fringe :  t -> string option -> Routing.range_direction -> (Key.t * string) counted_list
 
+  val get_cluster_id : t -> string option
   val get_interval : t -> Interval.t
   val get_routing : t -> Routing.t
 
@@ -113,7 +114,7 @@ struct
              mutable quiesced : Quiesce.Mode.t;
              mutable closed : bool;
              mutable _tx_lock : transaction_lock option;
-             _tx_lock_mutex : Lwt_mutex.t
+             _tx_lock_mutex : Lwt_mutex.t;
            }
 
   let _get_interval store =
@@ -152,11 +153,38 @@ struct
     store.interval <- _get_interval store.s;
     store.routing <- _get_routing store.s
 
+  let _get_cluster_id simple_store =
+    try Some (S.get simple_store __cluster_id_key)
+    with Not_found -> None
+
+  let get_cluster_id store =
+    _get_cluster_id store.s
+
   let make_store
       ~lcnum
       ~ncnum
-      ?(read_only=false) db_name =
+      ?(read_only=false)
+      ?cluster_id
+      db_name =
     S.make_store ~lcnum ~ncnum read_only db_name >>= fun simple_store ->
+
+    begin
+      match cluster_id with
+      | None -> Lwt.return_unit
+      | Some cluster_id ->
+         try
+           let cluster_id' = S.get simple_store __cluster_id_key in
+           if cluster_id' <> cluster_id
+           then failwith (Printf.sprintf "Cluster_id mismatch: expected %s, but got %s instead" cluster_id cluster_id');
+           Lwt.return_unit
+         with Not_found ->
+           S.with_transaction
+             simple_store
+             (fun tx ->
+               S.set simple_store tx __cluster_id_key cluster_id;
+               Lwt.return_unit)
+    end >>= fun () ->
+
     let store =
       { s = simple_store;
         store_i = None;
diff --git a/src/node/store_test.ml b/src/node/store_test.ml
index 403f0e4f..f1e3b2f5 100644
--- a/src/node/store_test.ml
+++ b/src/node/store_test.ml
@@ -37,6 +37,7 @@ end
 module S = (val (module Make(Batched_store.Local_store) : Test_STORE with type ss = Batched_store.Local_store.t))
 
 let _dir_name = "/tmp/store_test"
+let cluster_id = "cluster_id"
 
 let setup () =
   Logger.info_ "Store_test.setup" >>= fun () ->
@@ -63,7 +64,7 @@ let teardown () =
 
 let with_store name f =
   let db_name = _dir_name ^ "/" ^ name ^ ".db" in
-  S.make_store ~lcnum:1024 ~ncnum:512 db_name >>= fun store ->
+  S.make_store ~lcnum:1024 ~ncnum:512 db_name ~cluster_id >>= fun store ->
   f store >>= fun () ->
   S.close store
 
diff --git a/src/node/sync_backend.ml b/src/node/sync_backend.ml
index 001bacc5..e78c6341 100644
--- a/src/node/sync_backend.ml
+++ b/src/node/sync_backend.ml
@@ -108,7 +108,8 @@ struct
     ~(read_only:bool)
     ~max_value_size
     ~collapse_slowdown
-    ~act_not_preferred ->
+    ~act_not_preferred
+    ~cluster_id ->
     let my_name =  Node_cfg.node_name cfg in
     let locked_tlogs = Hashtbl.create 8 in
     let blockers_cond = Lwt_condition.create() in
@@ -581,7 +582,8 @@ struct
                Logger.info_ "Starting collapse" >>= fun () ->
                Collapser.collapse_many tlog_collection (module S)
                                        store_methods n cb' new_cb
-                                       collapse_slowdown >>= fun () ->
+                                       collapse_slowdown
+                                       ~cluster_id >>= fun () ->
                Logger.info_ "Collapse completed")
 
       method get_routing () =
@@ -685,7 +687,7 @@ struct
                       Lwt.return ()) >>= fun () ->
 
             (* remove all but tlogs_to_keep last tlogs *)
-            Collapser._head_i (module S) head_path >>= fun head_io ->
+            Collapser._head_i (module S) head_path ~cluster_id >>= fun head_io ->
 
             let head_i, head_n = match head_io with
               | None -> failwith "impossible i for copied head"
diff --git a/src/paxos/multi_paxos_test.ml b/src/paxos/multi_paxos_test.ml
index cd20e905..d14fd29d 100644
--- a/src/paxos/multi_paxos_test.ml
+++ b/src/paxos/multi_paxos_test.ml
@@ -28,6 +28,8 @@ open Master_type
 
 module S = (val (Store.make_store_module (module Mem_store)))
 
+let cluster_id = "cluster_id"
+
 let sn2s = Sn.string_of
 
 let test_take () = Lwt.return (None, (fun _s -> Lwt.return ()))
@@ -60,8 +62,11 @@ let test_generic network_factory n_nodes () =
   let last_witnessed _who = Sn.of_int (-1000) in
   let inject_buffer = Lwt_buffer.create_fixed_capacity 1 in
   let inject_ev q e = Lwt_buffer.add e q in
-  S.make_store ~lcnum:1024 ~ncnum:512 "MEM#store" >>= fun store ->
-  Mem_tlogcollection.make_mem_tlog_collection "MEM#tlog" None None ~fsync:false "???" ~fsync_tlog_dir:false >>= fun tlog_coll ->
+  S.make_store ~lcnum:1024 ~ncnum:512 "MEM#store" ~cluster_id >>= fun store ->
+  Mem_tlogcollection.make_mem_tlog_collection
+    "MEM#tlog" None None ~fsync:false "???" 
+    ~fsync_tlog_dir:false ~cluster_id
+  >>= fun tlog_coll ->
   let base = {me = "???";
               others = [] ;
               is_learner = false;
@@ -258,8 +263,12 @@ let test_master_loop network_factory ()  =
   let election_timeout_buffer = Lwt_buffer.create() in
   let inject_event e = Lwt_buffer.add e inject_buffer in
 
-  S.make_store ~lcnum:1024 ~ncnum:512 "MEM#store" >>= fun store ->
-  Mem_tlogcollection.make_mem_tlog_collection "MEM#tlog" None None ~fsync:false me ~fsync_tlog_dir:false >>= fun tlog_coll ->
+  S.make_store ~lcnum:1024 ~ncnum:512 "MEM#store" ~cluster_id >>= fun store ->
+  Mem_tlogcollection.make_mem_tlog_collection
+    "MEM#tlog" None None ~cluster_id
+    ~fsync:false me ~fsync_tlog_dir:false
+  >>= fun tlog_coll ->
+
   let constants = {me = me;
                    is_learner = false;
                    others = others;
@@ -397,8 +406,13 @@ let test_simulation filters () =
       Logger.debug_f_ "got (%s,%s,%s) => dropping" msg_s source target
   in
 
-  S.make_store ~lcnum:1024 ~ncnum:512 "MEM#store"  >>= fun store ->
-  Mem_tlogcollection.make_mem_tlog_collection "MEM#tlog" None None ~fsync:false me ~fsync_tlog_dir:false >>= fun tlog_coll ->
+  S.make_store ~lcnum:1024 ~ncnum:512 "MEM#store" ~cluster_id >>= fun store ->
+  Mem_tlogcollection.make_mem_tlog_collection
+    "MEM#tlog" None None
+    ~fsync:false me ~fsync_tlog_dir:false
+    ~cluster_id
+  >>= fun tlog_coll ->
+
   let constants = {
     me = me;
     is_learner = false;
diff --git a/src/system/startup.ml b/src/system/startup.ml
index 3d232fe7..ddd2e84a 100644
--- a/src/system/startup.ml
+++ b/src/system/startup.ml
@@ -28,6 +28,8 @@ let section = Logger.Section.main
 
 module LS = (val (Store.make_store_module (module Mem_store)))
 
+let cluster_id = "cluster_id"
+
 let _make_log_cfg () =
   ("log_cfg",
    {
@@ -76,6 +78,7 @@ let _make_cfg name n lease_period =
   }
 
 let _make_tlog_coll
+      ?cluster_id
       ~tlcs ~values
       ~compressor ?(tlog_max_entries:int option) ?(tlog_max_size:int option)
       tlc_name tlf_dir head_dir
@@ -85,6 +88,7 @@ let _make_tlog_coll
   Mem_tlogcollection.make_mem_tlog_collection
     ?tlog_max_entries ?tlog_max_size
     tlc_name tlf_dir head_dir ~fsync node_id ~fsync_tlog_dir
+    ?cluster_id
   >>= fun tlc ->
   let rec loop i = function
     | [] -> Lwt.return ()
@@ -107,8 +111,8 @@ let _make_run ~stores ~tlcs ~now ~values ~get_cfgs name () =
       include LS
       let make_store
             ~lcnum ~ncnum
-            ?(read_only=false) (db_name:string) =
-        LS.make_store ~lcnum ~ncnum ~read_only db_name >>= fun store ->
+            ?(read_only=false) ?cluster_id (db_name:string) =
+        LS.make_store ~lcnum ~ncnum ~read_only db_name ?cluster_id >>= fun store ->
         LS.with_transaction store (fun tx -> LS.set_master store tx name now) >>= fun () ->
         Hashtbl.add stores db_name store;
         Lwt.return store
diff --git a/src/tlog/mem_tlogcollection.ml b/src/tlog/mem_tlogcollection.ml
index 45ba5cc7..bf1bbab4 100644
--- a/src/tlog/mem_tlogcollection.ml
+++ b/src/tlog/mem_tlogcollection.ml
@@ -107,6 +107,7 @@ class mem_tlog_collection _name =
   end
 
 let make_mem_tlog_collection
+      ?cluster_id
       ?tlog_max_entries ?tlog_max_size _tlog_dir _tlf_dir _head_dir ~fsync name ~fsync_tlog_dir =
   let () = ignore fsync in
   let () = ignore fsync_tlog_dir in
diff --git a/src/tlog/tlc2.ml b/src/tlog/tlc2.ml
index 6a5a56c0..76b01e0d 100644
--- a/src/tlog/tlc2.ml
+++ b/src/tlog/tlc2.ml
@@ -115,12 +115,12 @@ let _init_file fsync_dir tlog_map =
 
 
 class tlc2
+        ?cluster_id
         ?(compressor=Compression.Snappy)
         (head_dir:string)
         (last:Entry.t option) (index:Index.index) (file:F.t option)
         (tlog_map: TlogMap.t)
         (node_id:string) ~(fsync:bool) ~(fsync_tlog_dir:bool)
-
   =
   let _previous_entry = ref (Some last) in
   let set_previous_entry e =
@@ -371,8 +371,11 @@ class tlc2
       >>= fun () ->
       Lwt_io.flush oc >>= fun () ->
       let module S = (val (Store.make_store_module (module Batched_store.Local_store))) in
-      S.make_store ~lcnum:Node_cfg.default_lcnum
-        ~ncnum:Node_cfg.default_ncnum head_name >>= fun store ->
+      S.make_store
+        ~lcnum:Node_cfg.default_lcnum
+        ~ncnum:Node_cfg.default_ncnum
+        ?cluster_id
+        head_name >>= fun store ->
       let io = S.consensus_i store in
       S.close ~flush:false ~sync:false store >>= fun () ->
       Logger.debug_f_ "head has consensus_i=%s" (Log_extra.option2s Sn.string_of io)
@@ -567,12 +570,12 @@ class tlc2
 
 
 
-let make_tlc2 ~compressor
+let make_tlc2 ?cluster_id
+              ~compressor
               ?tlog_max_entries ?tlog_max_size
               ?(check_marker=true)
               tlog_dir tlf_dir
               head_dir ~fsync node_id ~fsync_tlog_dir
-
   =
   Logger.debug_f_ "make_tlc2 %S" tlog_dir >>= fun () ->
   TlogMap.make ?tlog_max_entries ?tlog_max_size
@@ -613,7 +616,9 @@ let make_tlc2 ~compressor
   end >>= fun file ->
   let col = new tlc2 head_dir last index
                 file tlog_map
-                ~compressor node_id ~fsync ~fsync_tlog_dir in
+                ~compressor node_id ~fsync ~fsync_tlog_dir
+                ?cluster_id
+  in
   Lwt.return col
 
 let _truncate_tlog filename =
diff --git a/src/tlog/tlc2_test.ml b/src/tlog/tlc2_test.ml
index 9e740400..79da0ae9 100644
--- a/src/tlog/tlc2_test.ml
+++ b/src/tlog/tlc2_test.ml
@@ -25,6 +25,7 @@ open Tlogcollection_test
 
 let section = Logger.Section.main
 let node_id = "node_id"
+let cluster_id = "cluster_id"
 
 let create_test_tlc ?tlog_max_entries ?tlog_max_size dn =
   let tlx_dir = (dn ^ "_tlx") in
@@ -34,6 +35,7 @@ let create_test_tlc ?tlog_max_entries ?tlog_max_size dn =
     ?tlog_max_entries ?tlog_max_size tlx_dir tlx_dir
     ~compressor ~fsync:false ~fsync_tlog_dir:false
     ~check_marker:true
+    ~cluster_id
 
 let wrap_tlc = Tlogcollection_test.wrap create_test_tlc
 
diff --git a/src/tlog/tlogcollection.ml b/src/tlog/tlogcollection.ml
index ebe57a21..62f1c211 100644
--- a/src/tlog/tlogcollection.ml
+++ b/src/tlog/tlogcollection.ml
@@ -71,10 +71,12 @@ class type tlog_collection =
   end
 
 type tlc_factory =
+  ?cluster_id:string ->
   compressor:Compression.compressor ->
   ?tlog_max_entries:int ->
   ?tlog_max_size:int ->
   string ->
   string ->
   string ->
-  fsync:bool -> string -> fsync_tlog_dir:bool -> tlog_collection Lwt.t
+  fsync:bool -> string -> fsync_tlog_dir:bool ->
+  tlog_collection Lwt.t
diff --git a/src/tlog/tlogcollection_test.ml b/src/tlog/tlogcollection_test.ml
index dabfefbe..681660c7 100644
--- a/src/tlog/tlogcollection_test.ml
+++ b/src/tlog/tlogcollection_test.ml
@@ -264,6 +264,7 @@ let create_test_tlc ?tlog_max_entries ?tlog_max_size dn =
   Mem_tlogcollection.make_mem_tlog_collection
     ?tlog_max_entries ?tlog_max_size
     dn None None ~fsync:false ~fsync_tlog_dir:false
+    ~cluster_id:""
 
 let wrap_memory name = wrap create_test_tlc name
 

From 6c8713f925cf4687dbdf8fb4e4d5913fafe322e4 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Wed, 19 Apr 2017 09:43:45 +0200
Subject: [PATCH 277/373] PR comment (use cluster_id from config for local
 collapse)

---
 src/node/collapser_main.ml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/node/collapser_main.ml b/src/node/collapser_main.ml
index 5be1b631..2d05307c 100644
--- a/src/node/collapser_main.ml
+++ b/src/node/collapser_main.ml
@@ -47,7 +47,7 @@ let collapse_local make_config node_id n_tlogs =
       (fun() ->
         let open Lwt.Infix in
         make_config () >>= fun cfg ->
-        Collapser.collapse_out_of_band cfg node_id n_tlogs ?cluster_id:None >>= fun () ->
+        Collapser.collapse_out_of_band cfg node_id n_tlogs ~cluster_id:cfg.Node_cfg.Node_cfg.cluster_id >>= fun () ->
         Lwt.return 0
 
       )

From 296047414cda40e695eed20d713731c9307c3e34 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Thu, 20 Apr 2017 08:39:27 +0200
Subject: [PATCH 278/373] add tlogs_to_keep parameter to cli

remove warnings
---
 src/client/nodestream_main.ml    | 4 ++--
 src/client/protocol_common.ml    | 7 +++++--
 src/client/remote_nodestream.ml  | 5 +++--
 src/client/remote_nodestream.mli | 2 +-
 src/main/arakoon.ml              | 4 +++-
 src/node/sync_backend.ml         | 4 +++-
 src/system/drop_master.ml        | 4 ++--
 src/system/startup.ml            | 4 ++--
 8 files changed, 21 insertions(+), 13 deletions(-)

diff --git a/src/client/nodestream_main.ml b/src/client/nodestream_main.ml
index 9acf8ba8..9cb5af7d 100644
--- a/src/client/nodestream_main.ml
+++ b/src/client/nodestream_main.ml
@@ -46,10 +46,10 @@ let defrag_db ~tls ~tcp_keepalive ip port cluster_id =
   Lwt_main.run (t ())
 
 
-let copy_db_to_head ~tls ~tcp_keepalive ip port cluster_id =
+let copy_db_to_head ~tls ~tcp_keepalive ip port cluster_id ~tlogs_to_keep =
   let do_it connection =
     make_remote_nodestream cluster_id connection >>= fun client ->
-    client # copy_db_to_head ()
+    client # copy_db_to_head tlogs_to_keep
   in
   let address = make_address ip port in
   let t () =
diff --git a/src/client/protocol_common.ml b/src/client/protocol_common.ml
index 35fc3863..4074f4ed 100644
--- a/src/client/protocol_common.ml
+++ b/src/client/protocol_common.ml
@@ -481,8 +481,11 @@ let defrag_db (ic,oc) =
   request oc outgoing >>= fun () ->
   response ic nothing
 
-let copy_db_to_head (ic, oc) =
-  let outgoing buf = command_to buf COPY_DB_TO_HEAD in
+let copy_db_to_head (ic, oc) tlogs_to_keep =
+  let outgoing buf =
+    command_to buf COPY_DB_TO_HEAD;
+    Llio.int_to buf tlogs_to_keep
+  in
   request oc outgoing >>= fun () ->
   response ic nothing
 
diff --git a/src/client/remote_nodestream.ml b/src/client/remote_nodestream.ml
index ac6edaca..0725f984 100644
--- a/src/client/remote_nodestream.ml
+++ b/src/client/remote_nodestream.ml
@@ -37,7 +37,7 @@ class type nodestream = object
 
   method optimize_db: unit -> unit Lwt.t
   method defrag_db: unit -> unit Lwt.t
-  method copy_db_to_head : unit -> unit Lwt.t
+  method copy_db_to_head : int -> unit Lwt.t
   method get_db: string -> unit Lwt.t
 
   method get_fringe: string option -> Routing.range_direction -> ((string * string) list) Lwt.t
@@ -166,7 +166,8 @@ class remote_nodestream ((ic,oc) as conn) =
 
   method defrag_db () = Protocol_common.defrag_db conn
 
-  method copy_db_to_head () = Protocol_common.copy_db_to_head conn
+  method copy_db_to_head tlogs_to_keep =
+    Protocol_common.copy_db_to_head conn tlogs_to_keep
 
   method get_db db_location =
 
diff --git a/src/client/remote_nodestream.mli b/src/client/remote_nodestream.mli
index 0a55c5bc..53e75a4a 100644
--- a/src/client/remote_nodestream.mli
+++ b/src/client/remote_nodestream.mli
@@ -33,7 +33,7 @@ class type nodestream = object
 
   method optimize_db: unit -> unit Lwt.t
   method defrag_db:unit -> unit Lwt.t
-  method copy_db_to_head : unit -> unit Lwt.t
+  method copy_db_to_head : int -> unit Lwt.t
   method get_db: string -> unit Lwt.t
 
   method get_fringe: string option -> Routing.range_direction -> ((string * string) list) Lwt.t
diff --git a/src/main/arakoon.ml b/src/main/arakoon.ml
index 1292e375..c4a27b71 100644
--- a/src/main/arakoon.ml
+++ b/src/main/arakoon.ml
@@ -408,8 +408,9 @@ let main () =
                               Arg.Set_string cluster_id;
                               Arg.Set_string ip;
                               Arg.Set_int port;
+                              Arg.Set_int n_tlogs;
                              ],
-     "<cluster_id> <ip> <port> requests the node to copy its database to the head location");
+     "<cluster_id> <ip> <port> <tlogs_to_keep> requests the node to copy its database to the head location, after which it will remove old tlogs");
     ("--n-values", set_laction NumberOfValues,
      "returns the number of values in the store");
     ("--ping", Arg.Tuple[set_laction PING;
@@ -513,6 +514,7 @@ let main () =
     | Copy_db_to_head -> Nodestream_main.copy_db_to_head
                            ~tls ~tcp_keepalive
                            !ip !port !cluster_id
+                           ~tlogs_to_keep:!n_tlogs
     | NumberOfValues -> Client_main.get_key_count ~tls !config_url ()
     | InitNursery -> Nursery_main.init_nursery !config_url !cluster_id
     | MigrateNurseryRange -> Nursery_main.migrate_nursery_range
diff --git a/src/node/sync_backend.ml b/src/node/sync_backend.ml
index e78c6341..06dc87ec 100644
--- a/src/node/sync_backend.ml
+++ b/src/node/sync_backend.ml
@@ -667,6 +667,8 @@ struct
         Logger.info_ "get_db: All done"
 
       method copy_db_to_head tlogs_to_keep =
+        Logger.info_f_ "copy_db_to_head tlogs_to_keep:%i" tlogs_to_keep
+        >>= fun () ->
         if tlogs_to_keep < 1 then
           let rc = Arakoon_exc.E_UNKNOWN_FAILURE
           and msg = Printf.sprintf "tlogs_to_keep=%i is not acceptable" tlogs_to_keep
@@ -694,7 +696,7 @@ struct
               | Some i -> i, tlog_collection # get_tlog_from_i i
             in
             let keep_bottom_n = head_n -  tlogs_to_keep + 1 in
-            if keep_bottom_n > 0 
+            if keep_bottom_n > 0
             then
               begin
                 self # wait_for_tlog_release keep_bottom_n >>= fun () ->
diff --git a/src/system/drop_master.ml b/src/system/drop_master.ml
index 45ef313c..2a570ad5 100644
--- a/src/system/drop_master.ml
+++ b/src/system/drop_master.ml
@@ -42,8 +42,8 @@ let find_master cluster_cfg =
       Client_main.find_master cluster_cfg >>= fun m ->
       Lwt.return (Some m))
     (function
-    | Failure "too many nodes down"
-    | Failure "No Master" ->
+    | Failure x when (x = "too many nodes down" || x = "No Master")
+      ->
       Lwt.return None
     | exn ->
       Lwt.fail exn)
diff --git a/src/system/startup.ml b/src/system/startup.ml
index ddd2e84a..b21f1448 100644
--- a/src/system/startup.ml
+++ b/src/system/startup.ml
@@ -301,7 +301,7 @@ let ahead_master_loses_role () =
     }
   in
   let get_cfgs () = cluster_cfg |> Lwt.return in
-  
+
   let v0 = Value.create_master_value ~lease_start:0. node0 in
   let v1 = Value.create_client_value [Update.Set("xxx","xxx")] false in
   let v2 = Value.create_client_value [Update.Set("invalidkey", "shouldnotbepresent")] false in
@@ -442,7 +442,7 @@ let interrupted_election () =
        then Lwt.return true
        else Lwt.return false)
       (function
-        | Failure "No Master" ->
+        | Failure x when x = "No Master" ->
            Lwt.return false
         | exn -> Lwt.fail exn) >>= fun continue ->
     if continue

From 42e81bc26fa7c4b7d6de94c7625a95ee2d2755a4 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Thu, 20 Apr 2017 09:10:03 +0200
Subject: [PATCH 279/373] make_release: 1.9.17

---
 META                      | 2 +-
 debian/changelog          | 5 +++++
 redhat/SPECS/arakoon.spec | 4 +++-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/META b/META
index 68db81d4..ad31deb7 100644
--- a/META
+++ b/META
@@ -1,7 +1,7 @@
 # -*- conf -*-
 
 description = "Arakoon client"
-version = "1.9.16"
+version = "1.9.17"
 exists_if = "arakoon_client.cma,arakoon_client.cmxa,arakoon_client.cmxs"
 requires = "threads lwt lwt.unix str oUnit quickcheck lwt.ssl ocplib-endian redis.lwt uri"
 archive(byte) = "arakoon_client.cma tcp_keepalive_sockopt_stubs.o"
diff --git a/debian/changelog b/debian/changelog
index db0e32a9..0eadb632 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,8 @@
+arakoon (1.9.17) unstable; urgency=low
+  * new upstream version
+
+ -- Romain Slootmaekers <romain.slootmaekers@openvstorage.com>  Thu, 20 Apr 2017 09:10:03 +0100
+
 arakoon (1.9.16) unstable; urgency=low
   * new upstream version
 
diff --git a/redhat/SPECS/arakoon.spec b/redhat/SPECS/arakoon.spec
index d079fcc0..6ad752d7 100644
--- a/redhat/SPECS/arakoon.spec
+++ b/redhat/SPECS/arakoon.spec
@@ -2,7 +2,7 @@
 
 Summary: Arakoon
 Name: arakoon
-Version: 1.9.16
+Version: 1.9.17
 Release: 3%{?dist}
 License: Apache 2.0
 Requires: libev >= 4
@@ -33,6 +33,8 @@ cp arakoon.native %{buildroot}%{_bindir}/arakoon
 %{_bindir}/arakoon
 
 %changelog
+* Thu Apr 20 2017 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.17
+- Create arakoon 1.9.17 RPM package
 * Wed Apr 12 2017 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.16
 - Create arakoon 1.9.16 RPM package
 * Thu Nov 03 2016 Jan Doms <jan.doms@gmail.com> - 1.9.15

From 2faccd29bc5fa09f6401c5e9acdd866550bb4df6 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Thu, 20 Apr 2017 10:51:26 +0200
Subject: [PATCH 280/373] move to v.0.9.114.41+37

because they broke the one we were using.
---
 docker/centos-7/Dockerfile     | 4 ++--
 docker/ubuntu-14.04/Dockerfile | 4 ++--
 docker/ubuntu-16.04/Dockerfile | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/docker/centos-7/Dockerfile b/docker/centos-7/Dockerfile
index 5ee175d0..a8e48152 100644
--- a/docker/centos-7/Dockerfile
+++ b/docker/centos-7/Dockerfile
@@ -32,7 +32,7 @@ RUN opam init --root=${opam_root} --comp ${ocaml_version}
 
 RUN git clone git://github.com/janestreet/opam-repository.git /janestreet-opam-repo && \
     cd /janestreet-opam-repo && \
-    git checkout 49a87b492c6d5e8a2c76bb8ac1956aac484c2f40
+    git checkout 18e10bbe6fd876523fe38216ae827f26bf46e677
 RUN eval `${opam_env}` && \
     opam repo --root=${opam_root} -k local add janestreet /janestreet-opam-repo
 
@@ -67,7 +67,7 @@ RUN eval `${opam_env}` && \
         ocplib-endian.1.0 \
         quickcheck.1.0.2 \
         conf-libev \
-        core.114.35+03 \
+        core.v0.9.114.41+37 \
         redis.0.3.3 \
         uri.1.9.2
 
diff --git a/docker/ubuntu-14.04/Dockerfile b/docker/ubuntu-14.04/Dockerfile
index a7b90ff3..279e6f29 100644
--- a/docker/ubuntu-14.04/Dockerfile
+++ b/docker/ubuntu-14.04/Dockerfile
@@ -44,7 +44,7 @@ RUN opam init --root=${opam_root} --comp ${ocaml_version}
 
 RUN git clone git://github.com/janestreet/opam-repository.git /janestreet-opam-repo && \
     cd /janestreet-opam-repo && \
-    git checkout 49a87b492c6d5e8a2c76bb8ac1956aac484c2f40
+    git checkout 18e10bbe6fd876523fe38216ae827f26bf46e677
 RUN eval `${opam_env}` && \
     opam repo --root=${opam_root} -k local add janestreet /janestreet-opam-repo
 
@@ -63,7 +63,7 @@ RUN eval `${opam_env}` && \
         ocplib-endian.1.0 \
         quickcheck.1.0.2 \
         conf-libev \
-        core.114.35+03 \
+        core.v0.9.114.41+37 \
         redis.0.3.3 \
         uri.1.9.2
 
diff --git a/docker/ubuntu-16.04/Dockerfile b/docker/ubuntu-16.04/Dockerfile
index 3e07662c..66038995 100644
--- a/docker/ubuntu-16.04/Dockerfile
+++ b/docker/ubuntu-16.04/Dockerfile
@@ -42,7 +42,7 @@ RUN opam init --root=${opam_root} --comp ${ocaml_version}
 
 RUN git clone git://github.com/janestreet/opam-repository.git /janestreet-opam-repo && \
     cd /janestreet-opam-repo && \
-    git checkout 49a87b492c6d5e8a2c76bb8ac1956aac484c2f40
+    git checkout 18e10bbe6fd876523fe38216ae827f26bf46e677
 RUN eval `${opam_env}` && \
     opam repo --root=${opam_root} -k local add janestreet /janestreet-opam-repo
 
@@ -60,7 +60,7 @@ RUN eval `${opam_env}` && \
         ocplib-endian.1.0 \
         quickcheck.1.0.2 \
         conf-libev \
-        core.114.35+03 \
+        core.v0.9.114.41+37 \
         redis.0.3.3 \
         uri.1.9.2
 

From cd521a71ee61357ef73202c2171953e5fd76dedd Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Thu, 20 Apr 2017 12:44:06 +0200
Subject: [PATCH 281/373] changes for Lwt.3.0.0

---
 _tags                          | 2 +-
 docker/centos-7/Dockerfile     | 3 ++-
 docker/ubuntu-14.04/Dockerfile | 3 ++-
 docker/ubuntu-16.04/Dockerfile | 3 ++-
 4 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/_tags b/_tags
index ab056408..e00b0c4f 100644
--- a/_tags
+++ b/_tags
@@ -4,7 +4,7 @@ true: thread
 true: package(lwt)
 true: package(lwt.preemptive)
 true: package(lwt.unix)
-true: package(lwt.ssl)
+true: package(lwt_ssl)
 true: package(oUnit)
 true: package(bz2)
 true: package(str)
diff --git a/docker/centos-7/Dockerfile b/docker/centos-7/Dockerfile
index a8e48152..a8766a84 100644
--- a/docker/centos-7/Dockerfile
+++ b/docker/centos-7/Dockerfile
@@ -62,7 +62,8 @@ RUN eval `${opam_env}` && \
         camlbz2 \
         snappy \
         bisect \
-        lwt.2.7.0 \
+        lwt_ssl \
+        lwt.3.0.0 \
         camltc.0.9.4 \
         ocplib-endian.1.0 \
         quickcheck.1.0.2 \
diff --git a/docker/ubuntu-14.04/Dockerfile b/docker/ubuntu-14.04/Dockerfile
index 279e6f29..07b9bcad 100644
--- a/docker/ubuntu-14.04/Dockerfile
+++ b/docker/ubuntu-14.04/Dockerfile
@@ -58,7 +58,8 @@ RUN eval `${opam_env}` && \
         camlbz2 \
         snappy \
         bisect \
-        lwt.2.7.0 \
+        lwt_ssl \
+        lwt.3.0.0 \
         camltc.0.9.4 \
         ocplib-endian.1.0 \
         quickcheck.1.0.2 \
diff --git a/docker/ubuntu-16.04/Dockerfile b/docker/ubuntu-16.04/Dockerfile
index 66038995..3a67b8ef 100644
--- a/docker/ubuntu-16.04/Dockerfile
+++ b/docker/ubuntu-16.04/Dockerfile
@@ -55,7 +55,8 @@ RUN eval `${opam_env}` && \
         camlbz2 \
         snappy \
         bisect \
-        lwt.2.7.0 \
+        lwt_ssl \
+        lwt.3.0.0 \
         camltc.0.9.4 \
         ocplib-endian.1.0 \
         quickcheck.1.0.2 \

From 91536145500281a4bb4e850e49d3fc581fceefef Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Thu, 20 Apr 2017 13:07:31 +0200
Subject: [PATCH 282/373] Lwt_unix.bind signature change

---
 META                         |  2 +-
 src/node/store.ml            |  2 +-
 src/tools/lwt_socket_test.ml |  5 +++--
 src/tools/server.ml          | 43 +++++++++++++++++++++---------------
 4 files changed, 30 insertions(+), 22 deletions(-)

diff --git a/META b/META
index ad31deb7..bcac62e2 100644
--- a/META
+++ b/META
@@ -3,7 +3,7 @@
 description = "Arakoon client"
 version = "1.9.17"
 exists_if = "arakoon_client.cma,arakoon_client.cmxa,arakoon_client.cmxs"
-requires = "threads lwt lwt.unix str oUnit quickcheck lwt.ssl ocplib-endian redis.lwt uri"
+requires = "threads lwt lwt.unix str oUnit quickcheck lwt_ssl ocplib-endian redis.lwt uri"
 archive(byte) = "arakoon_client.cma tcp_keepalive_sockopt_stubs.o"
 archive(native) = "arakoon_client.cmxa tcp_keepalive_sockopt_stubs.o"
 
diff --git a/src/node/store.ml b/src/node/store.ml
index adf4a23f..dfaa5074 100644
--- a/src/node/store.ml
+++ b/src/node/store.ml
@@ -655,7 +655,7 @@ struct
         let valid =
           try
             k.[0] = __prefix.[0]
-          with Invalid_argument "index out of bounds" ->
+          with Invalid_argument x when x = "index out of bounds" ->
             false in
         if valid
         then
diff --git a/src/tools/lwt_socket_test.ml b/src/tools/lwt_socket_test.ml
index 001c272a..017b8c30 100644
--- a/src/tools/lwt_socket_test.ml
+++ b/src/tools/lwt_socket_test.ml
@@ -22,8 +22,9 @@ open OUnit
 let test_leak () =
   let listening_socket = Lwt_unix.socket Unix.PF_INET Unix.SOCK_STREAM 0 in
   let () = Lwt_unix.setsockopt listening_socket Unix.SO_REUSEADDR true in
-  let () = Lwt_unix.bind listening_socket (Unix.ADDR_INET (Unix.inet_addr_any,
-                                                           6688)) in
+  Lwt_unix.bind listening_socket (Unix.ADDR_INET (Unix.inet_addr_any,
+                                                  6688))
+  >>= fun () ->
   let () = Lwt_unix.listen listening_socket 10 in
   Lwt.join
     [
diff --git a/src/tools/server.ml b/src/tools/server.ml
index b62d451d..6bb3a04f 100644
--- a/src/tools/server.ml
+++ b/src/tools/server.ml
@@ -155,22 +155,27 @@ let make_server_thread
       host port protocol =
 
   let socket_address = Network.make_address host port in
+  let maybe_take,release = scheme in
+  let connection_counter = make_counter () in
+  let client_threads = Hashtbl.create 10 in
+  let _condition = Lwt_condition.create () in
+  let bind () =
+      let listening_socket = new_socket socket_address in
+      Lwt_unix.setsockopt listening_socket Unix.SO_REUSEADDR true;
+      Lwt_unix.bind listening_socket socket_address >>= fun () ->
+      Lwt_unix.listen listening_socket 1024;
+      let () =
+        match ssl_context with
+        | None -> ()
+        | Some _ctx ->
+           let _ = Typed_ssl.embed_socket
+                     (Lwt_unix.unix_file_descr listening_socket)
+           in
+           ()
+      in
+      Lwt.return listening_socket
+  in
   begin
-    let listening_socket = new_socket socket_address in
-    Lwt_unix.setsockopt listening_socket Unix.SO_REUSEADDR true;
-    Lwt_unix.bind listening_socket socket_address;
-    Lwt_unix.listen listening_socket 1024;
-    let () = match ssl_context with
-      | None -> ()
-      | Some _ctx ->
-          let _ = Typed_ssl.embed_socket (Lwt_unix.unix_file_descr listening_socket) in
-          ()
-    in
-    let maybe_take,release = scheme in
-    let connection_counter = make_counter () in
-    let client_threads = Hashtbl.create 10 in
-    let _condition = Lwt_condition.create () in
-
     let possible_denial cid plain_fd cl_socket_address =
       make_socket plain_fd ssl_context >>= fun sock ->
       match !stop, maybe_take () with
@@ -205,7 +210,7 @@ let make_server_thread
            Lwt.return (Some t)
          end
     in
-    let rec server_loop () =
+    let rec server_loop listening_socket =
       let serve () =
         Lwt.catch
           (fun () ->
@@ -257,13 +262,15 @@ let make_server_thread
            )
       in
       serve () >>= fun () ->
-      server_loop ()
+      server_loop listening_socket
     in
     let r  = fun () ->
+      bind () >>= fun listening_socket ->
+
       Lwt.finalize
         (fun ()  ->
            setup_callback () >>= fun () ->
-           server_loop ())
+           server_loop listening_socket)
         (fun () ->
            Lwt.catch
              (fun () ->

From 7162afe7f16ece3bc65778a7228c0b54f257f785 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Tue, 23 May 2017 10:45:37 +0200
Subject: [PATCH 283/373] no longer depend on janestreet's opam repo

---
 docker/centos-7/Dockerfile     |  8 +-------
 docker/ubuntu-14.04/Dockerfile | 10 +---------
 docker/ubuntu-16.04/Dockerfile |  8 +-------
 3 files changed, 3 insertions(+), 23 deletions(-)

diff --git a/docker/centos-7/Dockerfile b/docker/centos-7/Dockerfile
index a8766a84..44291809 100644
--- a/docker/centos-7/Dockerfile
+++ b/docker/centos-7/Dockerfile
@@ -30,12 +30,6 @@ ENV opam_root=/home/jenkins/OPAM
 ENV opam_env="opam config env --root=${opam_root}"
 RUN opam init --root=${opam_root} --comp ${ocaml_version}
 
-RUN git clone git://github.com/janestreet/opam-repository.git /janestreet-opam-repo && \
-    cd /janestreet-opam-repo && \
-    git checkout 18e10bbe6fd876523fe38216ae827f26bf46e677
-RUN eval `${opam_env}` && \
-    opam repo --root=${opam_root} -k local add janestreet /janestreet-opam-repo
-
 # protobuf
 RUN yum -y install bzip2
 RUN wget https://github.com/google/protobuf/releases/download/v2.6.1/protobuf-2.6.1.tar.bz2 -O protobuf-2.6.1.tar.bz2 \
@@ -68,7 +62,7 @@ RUN eval `${opam_env}` && \
         ocplib-endian.1.0 \
         quickcheck.1.0.2 \
         conf-libev \
-        core.v0.9.114.41+37 \
+        core.v0.9.0 \
         redis.0.3.3 \
         uri.1.9.2
 
diff --git a/docker/ubuntu-14.04/Dockerfile b/docker/ubuntu-14.04/Dockerfile
index 07b9bcad..6ef0142d 100644
--- a/docker/ubuntu-14.04/Dockerfile
+++ b/docker/ubuntu-14.04/Dockerfile
@@ -41,14 +41,6 @@ ENV opam_root=/home/jenkins/OPAM
 ENV opam_env="opam config env --root=${opam_root}"
 RUN opam init --root=${opam_root} --comp ${ocaml_version}
 
-
-RUN git clone git://github.com/janestreet/opam-repository.git /janestreet-opam-repo && \
-    cd /janestreet-opam-repo && \
-    git checkout 18e10bbe6fd876523fe38216ae827f26bf46e677
-RUN eval `${opam_env}` && \
-    opam repo --root=${opam_root} -k local add janestreet /janestreet-opam-repo
-
-
 RUN eval `${opam_env}` && \
     opam update && \
     opam install -y \
@@ -64,7 +56,7 @@ RUN eval `${opam_env}` && \
         ocplib-endian.1.0 \
         quickcheck.1.0.2 \
         conf-libev \
-        core.v0.9.114.41+37 \
+        core.v0.9.0 \
         redis.0.3.3 \
         uri.1.9.2
 
diff --git a/docker/ubuntu-16.04/Dockerfile b/docker/ubuntu-16.04/Dockerfile
index 3a67b8ef..b586709a 100644
--- a/docker/ubuntu-16.04/Dockerfile
+++ b/docker/ubuntu-16.04/Dockerfile
@@ -40,12 +40,6 @@ ENV opam_root=/home/jenkins/OPAM
 ENV opam_env="opam config env --root=${opam_root}"
 RUN opam init --root=${opam_root} --comp ${ocaml_version}
 
-RUN git clone git://github.com/janestreet/opam-repository.git /janestreet-opam-repo && \
-    cd /janestreet-opam-repo && \
-    git checkout 18e10bbe6fd876523fe38216ae827f26bf46e677
-RUN eval `${opam_env}` && \
-    opam repo --root=${opam_root} -k local add janestreet /janestreet-opam-repo
-
 RUN eval `${opam_env}` && \
     opam update && \
     opam install -y \
@@ -61,7 +55,7 @@ RUN eval `${opam_env}` && \
         ocplib-endian.1.0 \
         quickcheck.1.0.2 \
         conf-libev \
-        core.v0.9.114.41+37 \
+        core.v0.9.0 \
         redis.0.3.3 \
         uri.1.9.2
 

From e3051dcc6dedab6312f886fce5d80b50296f8a28 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Thu, 1 Jun 2017 10:32:28 +0200
Subject: [PATCH 284/373] make_release: 1.9.18

---
 META                      | 2 +-
 debian/changelog          | 5 +++++
 redhat/SPECS/arakoon.spec | 4 +++-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/META b/META
index bcac62e2..9c1709ce 100644
--- a/META
+++ b/META
@@ -1,7 +1,7 @@
 # -*- conf -*-
 
 description = "Arakoon client"
-version = "1.9.17"
+version = "1.9.18"
 exists_if = "arakoon_client.cma,arakoon_client.cmxa,arakoon_client.cmxs"
 requires = "threads lwt lwt.unix str oUnit quickcheck lwt_ssl ocplib-endian redis.lwt uri"
 archive(byte) = "arakoon_client.cma tcp_keepalive_sockopt_stubs.o"
diff --git a/debian/changelog b/debian/changelog
index 0eadb632..87bed87a 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,8 @@
+arakoon (1.9.18) unstable; urgency=low
+  * new upstream version
+
+ -- Romain Slootmaekers <romain.slootmaekers@openvstorage.com>  Thu, 01 Jun 2017 10:32:28 +0100
+
 arakoon (1.9.17) unstable; urgency=low
   * new upstream version
 
diff --git a/redhat/SPECS/arakoon.spec b/redhat/SPECS/arakoon.spec
index 6ad752d7..b317965c 100644
--- a/redhat/SPECS/arakoon.spec
+++ b/redhat/SPECS/arakoon.spec
@@ -2,7 +2,7 @@
 
 Summary: Arakoon
 Name: arakoon
-Version: 1.9.17
+Version: 1.9.18
 Release: 3%{?dist}
 License: Apache 2.0
 Requires: libev >= 4
@@ -33,6 +33,8 @@ cp arakoon.native %{buildroot}%{_bindir}/arakoon
 %{_bindir}/arakoon
 
 %changelog
+* Thu Jun 01 2017 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.18
+- Create arakoon 1.9.18 RPM package
 * Thu Apr 20 2017 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.17
 - Create arakoon 1.9.17 RPM package
 * Wed Apr 12 2017 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.16

From fc8a0c4993d0d20c8f36b0c6e1f84d782f9ba365 Mon Sep 17 00:00:00 2001
From: Jeffrey Devloo <jeffreydevloo@hotmail.com>
Date: Tue, 13 Jun 2017 14:22:57 +0200
Subject: [PATCH 285/373] Updated pyrakoon

---
 pylabs/pyrakoon | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pylabs/pyrakoon b/pylabs/pyrakoon
index 0adf7aec..6bf01f5f 160000
--- a/pylabs/pyrakoon
+++ b/pylabs/pyrakoon
@@ -1 +1 @@
-Subproject commit 0adf7aecef833a0fd85e6529d1a8d6188d1fecaa
+Subproject commit 6bf01f5f6654aebfa1e10355376417a5d2fa8010

From 881e1d96ac6ff647261afb462e52c86ca72aa791 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Thu, 22 Jun 2017 21:59:08 +0200
Subject: [PATCH 286/373] failing tests

---
 src/tlog/tlog_map.ml      | 16 ++++++++------
 src/tlog/tlog_map_test.ml | 44 ++++++++++++++++++++++++++++++++++++++-
 2 files changed, 53 insertions(+), 7 deletions(-)

diff --git a/src/tlog/tlog_map.ml b/src/tlog/tlog_map.ml
index d4fbbe3b..6caa3f37 100644
--- a/src/tlog/tlog_map.ml
+++ b/src/tlog/tlog_map.ml
@@ -559,18 +559,22 @@ module TlogMap = struct
     in
     loop canonicals
 
+  let  _find_start i items =
+    let rec aux to_remove_rev = function
+      | [] -> ([],[])
+      | item :: rest when item.i >= i ->
+         (List.rev to_remove_rev), (item :: rest)
+      | x :: rest -> aux (x::to_remove_rev) rest
+    in
+    aux [] (List.rev items)
+
   let remove_below t i =
     Lwt_mutex.with_lock
       t.lock
       (fun () ->
         Logger.debug_f_ "remove_below %s tlog_map:%s" (Sn.string_of i) (show_map t)
         >>= fun () ->
-        let rec find_start to_remove_rev = function
-          | [] -> ([],[])
-          | item :: rest when item.i >= i -> (List.rev to_remove_rev), (item :: rest)
-          | x :: rest -> find_start (x::to_remove_rev) rest
-        in
-        let to_remove,to_keep_rev = find_start [] (List.rev t.i_to_tlog_number) in
+        let to_remove,to_keep_rev = _find_start i t.i_to_tlog_number in
         t.i_to_tlog_number <- List.rev to_keep_rev;
 
         let remove item  =
diff --git a/src/tlog/tlog_map_test.ml b/src/tlog/tlog_map_test.ml
index 39d5a80a..9a1493f3 100644
--- a/src/tlog/tlog_map_test.ml
+++ b/src/tlog/tlog_map_test.ml
@@ -1,7 +1,8 @@
 open Tlog_map.TlogMap
 
+let make_item (i,n,is_archive) = {i;n;is_archive}
+
 let test_tlogs_to_collapse () =
-  let make_item (i,n,is_archive) = {i;n;is_archive} in
   let tlog_map = {
       i_to_tlog_number = List.map
                            make_item [
@@ -40,9 +41,50 @@ let test_tlogs_to_collapse () =
   let expected = Some (12,12_002L) in
   OUnit.assert_equal expected r ~printer
 
+let test_find_start () =
+  let i_to_tlog_number =
+    List.map
+      make_item [
+        (12000L,12,false);
+        (11000L,11,false);
+        (10000L,10,false);
+        ( 9000L, 9,false);
+        ( 8000L, 8,false);
+        ( 7000L, 7,false);
+        ( 6000L, 6,false);
+        ( 5000L, 5,false);
+        ( 4000L, 4,false);
+        ( 3000L, 3,false);
+        ( 2000L, 2,false);
+        ( 1000L, 1,false);
+        (    0L, 0,false);
+      ]
+  in
+  let printer items =
+    let itemss = List.map (fun item -> Printf.sprintf "{i:%Li;...}" item.i) items
+    in Printf.sprintf "[%s]" (String.concat "; " itemss)
+  in
+  let to_remove, to_keep =
+    _find_start 12000L i_to_tlog_number
+  in
+  let expected = List.tl i_to_tlog_number |> List.rev in
+  OUnit.assert_equal expected to_remove ~printer ~msg:"test1";
+  let to_remove2, to_keep2 =
+    _find_start 11002L i_to_tlog_number
+  in
+  let expected2 = i_to_tlog_number |> List.tl |> List.tl |> List.rev in
+  OUnit.assert_equal expected2 to_remove2 ~printer ~msg:"test2";
+
+  let to_remove3, to_keep3 =
+    _find_start 12002L i_to_tlog_number
+  in
+  OUnit.assert_equal expected to_remove3 ~printer ~msg:"test3";
+  ()
+
 open OUnit
 
 let suite =
   "tlog_map">:::[
       "tlogs_to_collapse">:: test_tlogs_to_collapse;
+      "find_start" >:: test_find_start;
     ]

From ed882b7adba895c5030642b602b2cdea0555702e Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Fri, 23 Jun 2017 00:21:59 +0200
Subject: [PATCH 287/373] fix

---
 src/tlog/tlog_map.ml | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/tlog/tlog_map.ml b/src/tlog/tlog_map.ml
index 6caa3f37..68afe4ab 100644
--- a/src/tlog/tlog_map.ml
+++ b/src/tlog/tlog_map.ml
@@ -560,9 +560,17 @@ module TlogMap = struct
     loop canonicals
 
   let  _find_start i items =
+    let i' =
+      let rec _start_of_tlog = function
+        | [] -> Sn.zero
+        | item :: _ when item.i <= i-> item.i
+        | _ :: items -> _start_of_tlog items
+      in
+      _start_of_tlog items
+    in
     let rec aux to_remove_rev = function
       | [] -> ([],[])
-      | item :: rest when item.i >= i ->
+      | item :: rest when item.i >= i' ->
          (List.rev to_remove_rev), (item :: rest)
       | x :: rest -> aux (x::to_remove_rev) rest
     in

From eef445772c6cc64aea467a68a284a7d084bbc563 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Fri, 23 Jun 2017 08:46:47 +0200
Subject: [PATCH 288/373] extra test (and fix the one we had)

---
 .../test/server/left/test_copy_db_to_head.py  | 32 +++++++++++++++++--
 1 file changed, 30 insertions(+), 2 deletions(-)

diff --git a/pylabs/test/server/left/test_copy_db_to_head.py b/pylabs/test/server/left/test_copy_db_to_head.py
index a2a55f92..86e296cd 100644
--- a/pylabs/test/server/left/test_copy_db_to_head.py
+++ b/pylabs/test/server/left/test_copy_db_to_head.py
@@ -19,7 +19,7 @@
 from Compat import X
 from .. import system_tests_common as C
 from nose.tools import *
-import logging
+import logging, time
 
 @C.with_custom_setup(C.setup_2_nodes_forced_master_mini, C.basic_teardown)
 def test_copy_db_to_head():
@@ -30,7 +30,7 @@ def test_copy_db_to_head():
     # n < 1 fails
     assert_raises( Exception, lambda: C.copyDbToHead(slave, 0))
     # fails on master
-    assert_raises( Exception, lambda: C.copyDbToHead(Common.node_names[0], 2))
+    assert_raises( Exception, lambda: C.copyDbToHead(C.node_names[0], 2))
 
     C.copyDbToHead(slave, 1)
 
@@ -44,3 +44,31 @@ def test_copy_db_to_head():
     a = C.get_i(slave, True)
     logging.info("slave_head_i='%s'", a)
     assert(a >= 5000)
+
+@C.with_custom_setup(C.setup_2_nodes_mini, C.basic_teardown)
+def test_copy_db_to_head2():
+    logging.info("test_copy_db_to_head")
+    zero = C.node_names[0]
+    one = C.node_names[1]
+    n = 29876
+    C.iterate_n_times(n, C.simple_set)
+    logging.info("did %i sets, now copying db to head" % n)
+    C.copyDbToHead(one,1)
+
+    head_name = '%s/%s/head/head.db' %(C.data_base_dir, one)
+
+    logging.info(head_name)
+    assert_true(X.fileExists(head_name))
+    C.stopOne(zero)
+    C.wipe(zero)
+    C.startOne(zero)
+    cli = C.get_client()
+    logging.info("cli class:%s", cli.__class__)
+    assert_false(cli.expectProgressPossible())
+    up2date = False
+    counter = 0
+    while not up2date and counter < 100:
+        time.sleep(1.0)
+        counter = counter + 1
+        up2date = cli.expectProgressPossible()
+    logging.info("catchup from 'collapsed' node finished")

From c30139306aa8c7736d426e1e21a194f030327938 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Fri, 23 Jun 2017 10:22:47 +0200
Subject: [PATCH 289/373] allow fiddling with store i

---
 src/main/arakoon.ml    |  7 +++++++
 src/node/dump_store.ml | 27 +++++++++++++++++++++++++++
 2 files changed, 34 insertions(+)

diff --git a/src/main/arakoon.ml b/src/main/arakoon.ml
index c4a27b71..f35ceb3a 100644
--- a/src/main/arakoon.ml
+++ b/src/main/arakoon.ml
@@ -31,6 +31,7 @@ type local_action =
   | ReplayTlogs
   | DumpStore
   | InspectStore
+  | FiddleStoreI
   | MakeTlog
   | MarkTlog
   | CloseTlog
@@ -223,6 +224,7 @@ let main () =
   and rinc = ref false
   and scenario = ref (String.concat ", " Benchmark.default_scenario)
   and autofix = ref false
+  and i = ref 0
   in
   let set_action a = Arg.Unit (fun () -> action := a) in
   let set_laction a = set_action (LocalAction a) in
@@ -285,6 +287,10 @@ let main () =
     ("--inspect-store", Arg.Tuple [ set_laction InspectStore;
                                     Arg.Set_string filename],
      "<filename> : inspect a (tokyo cabinet) store (also use -left and -max_results)");
+    ("--dev-fiddle-store-i", Arg.Tuple [ set_laction FiddleStoreI;
+                                     Arg.Set_string filename;
+                                     Arg.Set_int i; ],
+     "<filename> <i> : fiddle with the i value stored in a (tokyo cabinet) store. Be careful, this is dangerous!");
     ("--compress-tlog", Arg.Tuple[set_laction CompressTlog;
                                   Arg.Set_string filename],
      "<filename> : compress a tlog file");
@@ -475,6 +481,7 @@ let main () =
     | ReplayTlogs -> Replay_main.replay_tlogs !tlog_dir !tlf_dir !filename !end_i
     | DumpStore -> Dump_store.dump_store !filename
     | InspectStore -> Dump_store.inspect_store !filename !left !max_results
+    | FiddleStoreI -> Dump_store.set_store_i !filename !i
     | VerifyStore -> Dump_store.verify_store !filename None
     | TruncateTlog -> Tlc2.truncate_tlog !filename
     | CompressTlog -> Tlog_main.compress_tlog !filename !archive_type
diff --git a/src/node/dump_store.ml b/src/node/dump_store.ml
index 1dc88aa9..9867f269 100644
--- a/src/node/dump_store.ml
+++ b/src/node/dump_store.ml
@@ -238,3 +238,30 @@ let inspect_store fn left max_results =
           Lwt_io.printlf "Got exception during inspect store: %s" (Printexc.to_string exn) >>= fun () ->
           Lwt.return 1
     ))
+
+let set_store_i fn new_i =
+  let t () =
+    Local_store.make_store ~lcnum:1024 ~ncnum:512 false fn >>= fun store ->
+    let current_i =
+      try
+        let s = Local_store.get store Simple_store.__i_key in
+        Some (Sn.sn_from (Llio.make_buffer s 0))
+      with Not_found ->
+        None
+    in
+    Lwt_io.printlf
+      "Current store i:%s, replacing with %i"
+      (Log_extra.option2s Int64.to_string current_i)
+      new_i >>= fun () ->
+
+    let buf = Buffer.create 8 in
+    Sn.sn_to buf (Int64.of_int new_i);
+    Local_store.with_transaction
+      store
+      (fun tx ->
+        Local_store.set store tx Simple_store.__i_key (Buffer.contents buf);
+        Lwt.return ()) >>= fun () ->
+
+    Lwt.return 0
+  in
+  Lwt_extra.run t

From c3029fac031db4653f10d66264c7f8fefb9ff3a7 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Fri, 23 Jun 2017 10:43:11 +0200
Subject: [PATCH 290/373] make_release: 1.9.19

---
 META                      | 2 +-
 debian/changelog          | 5 +++++
 redhat/SPECS/arakoon.spec | 4 +++-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/META b/META
index 9c1709ce..a4c34472 100644
--- a/META
+++ b/META
@@ -1,7 +1,7 @@
 # -*- conf -*-
 
 description = "Arakoon client"
-version = "1.9.18"
+version = "1.9.19"
 exists_if = "arakoon_client.cma,arakoon_client.cmxa,arakoon_client.cmxs"
 requires = "threads lwt lwt.unix str oUnit quickcheck lwt_ssl ocplib-endian redis.lwt uri"
 archive(byte) = "arakoon_client.cma tcp_keepalive_sockopt_stubs.o"
diff --git a/debian/changelog b/debian/changelog
index 87bed87a..b12872d1 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,8 @@
+arakoon (1.9.19) unstable; urgency=low
+  * new upstream version
+
+ -- Romain Slootmaekers <romain.slootmaekers@openvstorage.com>  Fri, 23 Jun 2017 10:43:11 +0100
+
 arakoon (1.9.18) unstable; urgency=low
   * new upstream version
 
diff --git a/redhat/SPECS/arakoon.spec b/redhat/SPECS/arakoon.spec
index b317965c..53351b92 100644
--- a/redhat/SPECS/arakoon.spec
+++ b/redhat/SPECS/arakoon.spec
@@ -2,7 +2,7 @@
 
 Summary: Arakoon
 Name: arakoon
-Version: 1.9.18
+Version: 1.9.19
 Release: 3%{?dist}
 License: Apache 2.0
 Requires: libev >= 4
@@ -33,6 +33,8 @@ cp arakoon.native %{buildroot}%{_bindir}/arakoon
 %{_bindir}/arakoon
 
 %changelog
+* Fri Jun 23 2017 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.19
+- Create arakoon 1.9.19 RPM package
 * Thu Jun 01 2017 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.18
 - Create arakoon 1.9.18 RPM package
 * Thu Apr 20 2017 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.17

From c180e8a70d3715db0b816f1122a24a6eda267f1d Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Fri, 23 Jun 2017 13:17:01 +0200
Subject: [PATCH 291/373] logging tweaks

- more fine grained configuration of connection logging
- replace ~exn with ": %S" (Printexc.to_string exn)
---
 cfg/arakoon.ini               |  1 +
 src/client/client_protocol.ml |  8 ++++++--
 src/main/arakoon.ml           |  4 +++-
 src/msg/tcp_messaging.ml      |  5 ++++-
 src/node/catchup.ml           |  3 ++-
 src/node/collapser.ml         |  6 +++++-
 src/node/node_cfg.ml          |  5 +++++
 src/node/node_main.ml         |  7 ++++++-
 src/system/startup.ml         |  1 +
 src/tools/network.ml          |  4 +++-
 src/tools/server.ml           | 31 +++++++++++++++++++++++--------
 11 files changed, 59 insertions(+), 16 deletions(-)

diff --git a/cfg/arakoon.ini b/cfg/arakoon.ini
index 87968a89..e636ed9f 100644
--- a/cfg/arakoon.ini
+++ b/cfg/arakoon.ini
@@ -79,6 +79,7 @@ client_protocol = debug
 paxos = debug
 tcp_messaging = debug
 tlog_map = debug
+server = debug
 
 #[default_batched_transaction_config]
 #max_entries = 100    # number of sets and deletes
diff --git a/src/client/client_protocol.ml b/src/client/client_protocol.ml
index 7e62bb81..e7b2f81d 100644
--- a/src/client/client_protocol.ml
+++ b/src/client/client_protocol.ml
@@ -103,10 +103,14 @@ let handle_exception oc exn=
       | Server.FOOBAR ->
         Lwt.return (Arakoon_exc.E_UNKNOWN_FAILURE, "unkown failure", true, Logger.Error)
       | Canceled ->
-        Lwt.fail Canceled
+         Lwt.fail Canceled
+      | Unix.Unix_error(Unix.ECONNRESET, _, _) ->
+         Lwt.return (Arakoon_exc.E_UNKNOWN_FAILURE,
+                     "ECONNRESET because the client disconnected",
+                    true, Logger.Debug)
       | End_of_file ->
         Lwt.return (Arakoon_exc.E_UNKNOWN_FAILURE, "end of file because the client disconnected",
-                    true, Logger.Info)
+                    true, Logger.Debug)
       | _ ->
         Lwt.return (Arakoon_exc.E_UNKNOWN_FAILURE, "unknown failure", true, Logger.Error)
   end
diff --git a/src/main/arakoon.ml b/src/main/arakoon.ml
index c4a27b71..7d38ca53 100644
--- a/src/main/arakoon.ml
+++ b/src/main/arakoon.ml
@@ -165,7 +165,9 @@ let main () =
 
   let () =
     Lwt.async_exception_hook :=
-    (fun exn -> Logger.ign_info_f_ ~exn "Caught async exception")
+      (fun exn -> Logger.ign_info_f_
+                    "Caught async exception: %S"
+                    (Printexc.to_string exn))
   in
   let _ = Bz2.version in
   let () = Lwt_io.set_default_buffer_size 65536 in
diff --git a/src/msg/tcp_messaging.ml b/src/msg/tcp_messaging.ml
index ba65d99c..3bd5231c 100644
--- a/src/msg/tcp_messaging.ml
+++ b/src/msg/tcp_messaging.ml
@@ -396,7 +396,10 @@ class tcp_messaging
         Lwt.catch
           (fun () -> loop (Bytes.create 1024))
           (fun exn ->
-             Logger.info_f_ ~exn "going to drop outgoing connection as well" >>= fun () ->
+            Logger.info_f_
+              "going to drop outgoing connection as well: %S"
+              (Printexc.to_string exn)
+            >>= fun () ->
              self # _drop_connection address >>= fun () ->
              Lwt.fail exn)
       in
diff --git a/src/node/catchup.ml b/src/node/catchup.ml
index 4b2ba369..005a9da2 100644
--- a/src/node/catchup.ml
+++ b/src/node/catchup.ml
@@ -116,7 +116,8 @@ let catchup_tlog (type s) ~tls_ctx ~tcp_keepalive ~stop other_configs ~cluster_i
       | None ->
          Logger.warning_f_ "catchup tlog from unknown node:%S is config up to date?" mr_name
     )
-    (fun exn -> Logger.warning_ ~exn "catchup_tlog failed")
+    (fun exn -> Logger.warning_f_ "catchup_tlog failed: %S"
+                                  (Printexc.to_string exn))
   >>= fun () ->
   tlog_coll # get_last_i ()
 
diff --git a/src/node/collapser.ml b/src/node/collapser.ml
index f652c9fa..0327a699 100644
--- a/src/node/collapser.ml
+++ b/src/node/collapser.ml
@@ -194,7 +194,11 @@ let _head_i (type s) (module S : Store.STORE with type t = s) ?cluster_id head_l
        Lwt.return head_io
     )
     (fun exn ->
-       Logger.info_f_ ~exn "returning assuming no I %S" head_location >>= fun () ->
+      Logger.info_f_
+        "returning assuming no I %S: %S"
+        head_location
+        (Printexc.to_string exn)
+      >>= fun () ->
        Lwt.return None
     )
 
diff --git a/src/node/node_cfg.ml b/src/node/node_cfg.ml
index 368f4b37..1805a7cc 100644
--- a/src/node/node_cfg.ml
+++ b/src/node/node_cfg.ml
@@ -183,6 +183,7 @@ module Node_cfg = struct
       paxos: string option;
       tcp_messaging: string option;
       tlog_map: string option;
+      server : string option;
     }
 
   let string_of_log_cfg lcfg =
@@ -190,6 +191,7 @@ module Node_cfg = struct
     record [ "client_protocol", option string lcfg.client_protocol
            ; "paxos", option string lcfg.paxos
            ; "tcp_messaging", option string lcfg.tcp_messaging
+           ; "server", option string lcfg.server
            ]
 
   let get_default_log_config () =
@@ -198,6 +200,7 @@ module Node_cfg = struct
       paxos = None;
       tcp_messaging = None;
       tlog_map = None;
+      server = None;
     }
 
   type batched_transaction_cfg =
@@ -461,11 +464,13 @@ module Node_cfg = struct
     let paxos = get_log_level "paxos" in
     let tcp_messaging = get_log_level "tcp_messaging" in
     let tlog_map = get_log_level "tlog_map" in
+    let server = get_log_level "server" in
     {
       client_protocol;
       paxos;
       tcp_messaging;
       tlog_map;
+      server;
     }
 
   let _batched_transaction_config inifile section_name =
diff --git a/src/node/node_main.ml b/src/node/node_main.ml
index 5b6235f1..c3bb9ed3 100644
--- a/src/node/node_main.ml
+++ b/src/node/node_main.ml
@@ -56,6 +56,7 @@ let _config_log_levels me cluster_cfg cfg =
   let () = set_level Multi_paxos.section log_config.paxos in
   let () = set_level Tcp_messaging.section log_config.tcp_messaging in
   let () = set_level Tlog_map.section log_config.tlog_map in
+  let () = set_level Server.section log_config.server in
   Lwt.return ()
 
 let _config_logging me get_cfgs =
@@ -229,7 +230,11 @@ let only_catchup
        tlc # close () >>= fun () ->
        Lwt.return true)
       (fun exn ->
-       Logger.warning_f_ ~exn "Catchup from %s failed" mr_name >>= fun () ->
+        Logger.warning_f_
+          "Catchup from %s failed: %S"
+          mr_name
+          (Printexc.to_string exn)
+        >>= fun () ->
        Lwt.return false)
   in
   let rec try_nodes = function
diff --git a/src/system/startup.ml b/src/system/startup.ml
index b21f1448..94b98195 100644
--- a/src/system/startup.ml
+++ b/src/system/startup.ml
@@ -37,6 +37,7 @@ let _make_log_cfg () =
      paxos = None;
      tcp_messaging = None;
      tlog_map = None;
+     server = None;
    })
 
 let _make_batched_transaction_cfg () =
diff --git a/src/tools/network.ml b/src/tools/network.ml
index 16acdf6e..b7e77988 100644
--- a/src/tools/network.ml
+++ b/src/tools/network.ml
@@ -72,7 +72,9 @@ let __open_connection ?(ssl_context : [> `Client ] Typed_ssl.t option)
              Lwt.return (socket, ic, oc)
        )
     (fun exn ->
-       Logger.info_f_ ~exn "__open_connection to %s failed" (a2s socket_address)
+      Logger.info_f_ "__open_connection to %s failed: %S"
+                     (a2s socket_address)
+                     (Printexc.to_string exn)
        >>= fun () ->
        Lwt_unix.close socket >>= fun () ->
        Lwt.fail exn)
diff --git a/src/tools/server.ml b/src/tools/server.ml
index 6bb3a04f..0c6e3fed 100644
--- a/src/tools/server.ml
+++ b/src/tools/server.ml
@@ -18,7 +18,10 @@ limitations under the License.
 
 open Lwt
 
-let section = Logger.Section.main
+let section =
+  let s = Logger.Section.make "server" in
+  let () = Logger.Section.set_level s Logger.Debug in
+  s
 
 let no_callback = Lwt.return
 
@@ -67,7 +70,9 @@ let session_thread (sid:string) cid protocol fd =
       | Canceled ->
         Lwt.fail Canceled
       | exn ->
-        Logger.info_f_ ~exn "exiting session (%s) connection=%s" sid cid)
+         Logger.info_f_
+           "exiting session (%s) connection=%s: %S"
+           sid cid (Printexc.to_string exn))
 
 let create_connection_allocation_scheme max =
   let counter = ref 0 in
@@ -142,7 +147,11 @@ let _socket_closer cid sock f =
             let level = match exn with
               | Unix.Unix_error(Unix.EBADF, _, _) -> Logger.Debug
               | _ -> Logger.Info in
-            Logger.log_ ~exn section level (fun () -> Printf.sprintf "Exception while closing client fd %s" cid))
+            Logger.log_
+              section level
+              (fun () -> Printf.sprintf "Exception while closing client fd %s: %S"
+                                        cid (Printexc.to_string exn))
+       )
     )
 
 let make_server_thread
@@ -235,7 +244,11 @@ let make_server_thread
                         finalize ()
                       )
                       (fun exn ->
-                        Logger.info_f_ ~exn "Exception in client thread %s" cid >>= fun () ->
+                        Logger.info_f_
+                          "Exception in client thread %s: %S"
+                          cid
+                          (Printexc.to_string exn)
+                        >>= fun () ->
                         finalize ()))
               | None -> ()
             end;
@@ -253,9 +266,8 @@ let make_server_thread
                  "OUT OF FDS during accept (%s,%s) on port %i => sleeping %.1fs"
                  s0 s1 port timeout >>= fun () ->
                Lwt_unix.sleep timeout
-            | Ssl.Accept_error _ as exn ->
-               Logger.warning_f_ ~exn
-                                 "Ssl.Accept_error in server_loop: %s"
+            | Ssl.Accept_error _ ->
+               Logger.warning_f_ "Ssl.Accept_error in server_loop: %s"
                                  (Ssl.get_error_string ())
 
             | e -> Lwt.fail e
@@ -278,7 +290,10 @@ let make_server_thread
                 Lwt_unix.close listening_socket >>= fun () ->
                 Logger.debug_ "closed listening socket")
              (fun exn ->
-                Logger.info_f_ ~exn "exception while closing listening socket") >>= fun () ->
+               Logger.info_f_ "exception while closing listening socket: %S"
+                              (Printexc.to_string exn)
+             )
+           >>= fun () ->
 
            let cancel _ t =
              try

From a3c9d016d14d1d4eb1f57d552a7470e3b9839c6f Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Mon, 31 Jul 2017 17:11:49 +0200
Subject: [PATCH 292/373] throttling for optimize db

---
 cfg/arakoon.ini          | 14 +++++++++++---
 src/node/local_store.ml  | 10 +++++++++-
 src/node/mem_store.ml    |  2 +-
 src/node/node_cfg.ml     | 11 ++++++++++-
 src/node/node_main.ml    |  1 +
 src/node/simple_store.ml |  2 +-
 src/node/store.ml        |  6 +++---
 src/node/sync_backend.ml |  8 +++++++-
 src/system/startup.ml    |  1 +
 9 files changed, 44 insertions(+), 11 deletions(-)

diff --git a/cfg/arakoon.ini b/cfg/arakoon.ini
index e636ed9f..9c180cd3 100644
--- a/cfg/arakoon.ini
+++ b/cfg/arakoon.ini
@@ -108,7 +108,7 @@ log_level = debug
 log_config = default_log_config
 #batched_transaction_config = default_batched_transaction_config
 
-# Throttle transaction processing during collapsing
+# Throttle transaction processing during collapsing (local & remote)
 # On some hardware and in some deployments, the collapsing process can be a
 # very IO and CPU-hungry process. This can have a negative impact on the
 # overall responsiveness of an Arakoon node.
@@ -137,12 +137,20 @@ log_config = default_log_config
 # tls_cert = /path/to/arakoon0.pem
 # tls_key = /path/to/arakoon0.key
 
-# slow down copying of head.db:
+# slow down copying of head.db (relevant for collapse (local & remote) & copy-db-to-head):
 # for a factor f, the scheme is:
 #     do_a step; // takes x
 #     wait (f * x)
 #     loop
-#head_copy_throttling = 1.0 # default = 0
+#head_copy_throttling = 1.0 # default = 0.0
+
+# slow down optimize of db
+# for a factor f, the scheme is:
+#     do_a step; // takes x
+#     wait (f * x)
+#     loop
+#optimize_db_slowdown = 1.3 # default = 0.0
+
 
 [arakoon_1]
 ip = 127.0.0.1
diff --git a/src/node/local_store.ml b/src/node/local_store.ml
index 9b122dc7..3664215d 100644
--- a/src/node/local_store.ml
+++ b/src/node/local_store.ml
@@ -283,7 +283,7 @@ let make_store ~lcnum ~ncnum read_only db_name  =
                lcnum;}
 
 
-let optimize ls ~quiesced ~stop =
+let optimize ls ~quiesced ~stop ~slowdown_factor =
   let open Camltc in
   let batch_size = 10_000 in
   let log_size = 1_000_000 in
@@ -302,17 +302,25 @@ let optimize ls ~quiesced ~stop =
           Bdb.first source cursor;
           let max = Some batch_size in
           let rec loop i =
+            let t0 = Unix.gettimeofday () in
             let count =
               Bdb.copy_from_cursor
                 ~source
                 ~cursor
                 ~target
                 ~max in
+            let t1 = Unix.gettimeofday () in
+            let dt = t1 -. t0 in
             if !stop
             then
               false
             else if count = batch_size
             then
+              let () =
+                match slowdown_factor with
+                | None -> ()
+                | Some factor -> Unix.sleepf (factor *. dt)
+              in
               begin
                 if i = log_batch_count
                 then
diff --git a/src/node/mem_store.ml b/src/node/mem_store.ml
index aa7dcee1..07e28dde 100644
--- a/src/node/mem_store.ml
+++ b/src/node/mem_store.ml
@@ -122,7 +122,7 @@ let range ms first finc last linc max =
                      []) in
   Array.of_list keys
 
-let optimize _ms ~quiesced ~stop =
+let optimize _ms ~quiesced ~stop ~slowdown_factor =
     let () = ignore quiesced in
     let () = ignore stop in
     Lwt.return true
diff --git a/src/node/node_cfg.ml b/src/node/node_cfg.ml
index 1805a7cc..70fff0d0 100644
--- a/src/node/node_cfg.ml
+++ b/src/node/node_cfg.ml
@@ -27,6 +27,7 @@ let default_client_buffer_capacity = 32
 let default_lcnum = 16384
 let default_ncnum = 8192
 let default_head_copy_throttling = 0.0
+let default_optimize_db_slowdown = 0.0
 
 open Master_type
 open Client_cfg
@@ -143,7 +144,8 @@ module Node_cfg = struct
             reporting: int;
             node_tls : TLSConfig.Node.t option;
             collapse_slowdown : float option;
-            head_copy_throttling: float;
+            head_copy_throttling : float;
+            optimize_db_slowdown : float;
            }
 
   let string_of t =
@@ -175,6 +177,7 @@ module Node_cfg = struct
            ; "node_tls", option TLSConfig.Node.to_string t.node_tls
            ; "collapse_slowdown", option To_string.float t.collapse_slowdown
            ; "head_copy_throttling", To_string.float t.head_copy_throttling
+           ; "optimize_db_slowdown", To_string.float t.optimize_db_slowdown;
            ]
 
   type log_cfg =
@@ -312,6 +315,7 @@ module Node_cfg = struct
         node_tls = None;
         collapse_slowdown = None;
         head_copy_throttling = default_head_copy_throttling;
+        optimize_db_slowdown = default_optimize_db_slowdown;
       }
     in
     let rec loop acc = function
@@ -610,6 +614,10 @@ module Node_cfg = struct
       Ini.get inifile node_name "head_copy_throttling"
               Ini.p_float (Ini.default default_head_copy_throttling)
     in
+    let optimize_db_slowdown =
+      Ini.get inifile node_name "optimize_db_slowdown"
+              Ini.p_float (Ini.default default_optimize_db_slowdown)
+    in
     {node_name;
      ips;
      client_port;
@@ -637,6 +645,7 @@ module Node_cfg = struct
      node_tls;
      collapse_slowdown;
      head_copy_throttling;
+     optimize_db_slowdown;
     }
 
 
diff --git a/src/node/node_main.ml b/src/node/node_main.ml
index c3bb9ed3..e85a789e 100644
--- a/src/node/node_main.ml
+++ b/src/node/node_main.ml
@@ -700,6 +700,7 @@ let _main_2 (type s)
               ~read_only
               ~max_value_size:cluster_cfg.max_value_size
               ~collapse_slowdown:me.collapse_slowdown
+              ~optimize_db_slowdown:me.optimize_db_slowdown
               ~act_not_preferred
               ~cluster_id
           in
diff --git a/src/node/simple_store.ml b/src/node/simple_store.ml
index 8c5b130c..f14eaf60 100644
--- a/src/node/simple_store.ml
+++ b/src/node/simple_store.ml
@@ -198,7 +198,7 @@ module type Simple_store = sig
 
   val get_key_count : t -> int64
 
-  val optimize : t -> quiesced:bool -> stop:bool ref -> bool Lwt.t
+  val optimize : t -> quiesced:bool -> stop:bool ref -> slowdown_factor:float option -> bool Lwt.t
   val defrag : t -> unit Lwt.t
   val copy_store : t -> bool -> Lwt_io.output_channel -> unit Lwt.t
   val copy_store2 : string -> string ->
diff --git a/src/node/store.ml b/src/node/store.ml
index dfaa5074..d50bd6be 100644
--- a/src/node/store.ml
+++ b/src/node/store.ml
@@ -53,7 +53,7 @@ sig
   val quiesced : t -> bool
   val quiesce : Quiesce.Mode.t -> t -> unit Lwt.t
   val unquiesce : t -> unit Lwt.t
-  val optimize : t -> bool Lwt.t
+  val optimize : t -> slowdown_factor:float option -> bool Lwt.t
   val defrag : t -> unit Lwt.t
   val copy_store : t -> Lwt_io.output_channel -> unit Lwt.t
   val copy_store2 : string -> string ->
@@ -292,11 +292,11 @@ struct
     store.quiesced <- Quiesce.Mode.NotQuiesced;
     reopen store (fun () -> Lwt.return ())
 
-  let optimize store =
+  let optimize store ~slowdown_factor =
     let m = match store.quiesced with
       | Quiesce.Mode.NotQuiesced | Quiesce.Mode.Writable -> false
       | Quiesce.Mode.ReadOnly -> true in
-    S.optimize store.s ~quiesced:m ~stop:(ref false)
+    S.optimize store.s ~quiesced:m ~stop:(ref false) ~slowdown_factor
 
   let set_master store tx master lease_start =
     _wrap_exception store "SET_MASTER" Server.FOOBAR (fun () ->
diff --git a/src/node/sync_backend.ml b/src/node/sync_backend.ml
index 06dc87ec..c11f7952 100644
--- a/src/node/sync_backend.ml
+++ b/src/node/sync_backend.ml
@@ -108,6 +108,7 @@ struct
     ~(read_only:bool)
     ~max_value_size
     ~collapse_slowdown
+    ~optimize_db_slowdown
     ~act_not_preferred
     ~cluster_id ->
     let my_name =  Node_cfg.node_name cfg in
@@ -642,7 +643,12 @@ struct
       method optimize_db () =
         Logger.info_ "optimize_db: enter" >>= fun () ->
         let mode = Quiesce.Mode.ReadOnly in
-        self # try_quiesced ~mode (fun () -> Lwt.map ignore (S.optimize store)) >>= fun () ->
+        self # try_quiesced
+             ~mode
+             (fun () ->
+               S.optimize store ~slowdown_factor:(Some optimize_db_slowdown)
+               >|= ignore)
+        >>= fun () ->
         Logger.info_ "optimize_db: All done"
 
       method defrag_db () =
diff --git a/src/system/startup.ml b/src/system/startup.ml
index 94b98195..ece2cc82 100644
--- a/src/system/startup.ml
+++ b/src/system/startup.ml
@@ -76,6 +76,7 @@ let _make_cfg name n lease_period =
     node_tls = None;
     collapse_slowdown = None;
     head_copy_throttling = 0.0;
+    optimize_db_slowdown = 0.0;
   }
 
 let _make_tlog_coll

From 2e81a1c0db235cb783f6b0c558d55ce3b677fa53 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Mon, 31 Jul 2017 18:04:51 +0200
Subject: [PATCH 293/373] optimize File_system.copy_file to avoid 2 userspace
 copies

---
 src/tools/file_system.ml | 48 +++++++++++++++++++++++++++++++---------
 1 file changed, 38 insertions(+), 10 deletions(-)

diff --git a/src/tools/file_system.ml b/src/tools/file_system.ml
index 93ba15b8..0330a672 100644
--- a/src/tools/file_system.ml
+++ b/src/tools/file_system.ml
@@ -112,13 +112,18 @@ let with_tmp_file tmp dest f =
      rename tmp dest)
     (fun () -> Lwt_unix.close fd)
 
+let with_fd filename ~flags ~perm f =
+  Lwt_unix.openfile filename flags perm >>= fun fd ->
+  Lwt.finalize
+    (fun () -> f fd)
+    (fun () -> Lwt_unix.close fd)
+
 let copy_file source target ~overwrite ~throttling =
   (* LOOKS LIKE Clone.copy_stream ... *)
   let bs = 1024 * 1024 in
   Logger.info_f_ "copy_file %s %s (overwrite=%b,throttling=%f) buffer_size:%i"
                  source target overwrite throttling bs >>= fun () ->
 
-  let buffer = Bytes.create bs in
   let throttle =
     match throttling with
     | 0.0   -> fun f  -> f ()
@@ -131,23 +136,29 @@ let copy_file source target ~overwrite ~throttling =
        Lwt_unix.sleep (factor *. dt) >>= fun () ->
        Lwt.return r
   in
-  let copy_all ic oc =
+  let copy_all buffer ~fd_in ~fd_out =
     let rec loop () =
       throttle
         (fun () ->
-         Lwt_io.read_into ic buffer 0 bs >>= fun bytes_read ->
+         Lwt_bytes.read fd_in buffer 0 bs >>= fun bytes_read ->
          if bytes_read > 0
          then
            begin
-             Lwt_io.write_from_exactly oc buffer 0 bytes_read
-             >>= fun () ->
+             let rec inner offset todo =
+               Lwt_bytes.write fd_out buffer offset todo >>= fun bytes_written ->
+               let todo = todo - bytes_written in
+               if todo > 0
+               then inner (offset + bytes_written) todo
+               else Lwt.return_unit
+             in
+             inner 0 bytes_read >>= fun () ->
              Lwt.return `Continue
            end
          else
            Lwt.return `Stop)
       >>= function
         | `Continue -> loop ()
-        | `Stop -> Lwt.return ()
+        | `Stop -> Lwt.return_unit
     in
     loop () >>= fun () ->
     Logger.info_ "done: copy_file"
@@ -161,10 +172,27 @@ let copy_file source target ~overwrite ~throttling =
     begin
       let tmp_file = target ^ ".tmp" in
       unlink tmp_file >>= fun () ->
-      Lwt_io.with_file ~mode:Lwt_io.input source
-        (fun ic ->
-         with_tmp_file tmp_file target
-                       (fun oc -> copy_all ic oc)) >>= fun () ->
+      with_fd
+        source
+        ~flags:[ Unix.O_RDONLY; Unix.O_NONBLOCK; ]
+        ~perm:0
+        (fun fd_in ->
+          with_fd
+            tmp_file
+            ~flags:[ Unix.O_WRONLY; Unix.O_CREAT; Unix.O_TRUNC; Unix.O_NONBLOCK; Unix.O_EXCL; ]
+            ~perm:0o644
+            (fun fd_out ->
+              let buffer = Lwt_bytes.create bs in
+              Lwt.finalize
+                (fun () -> copy_all buffer ~fd_in ~fd_out)
+                (fun () -> Core_kernel.Bigstring.unsafe_destroy buffer;
+                           Lwt.return_unit)
+              >>= fun () ->
+              Lwt_unix.fsync fd_out
+            )
+        )
+      >>= fun () ->
+      rename tmp_file target >>= fun () ->
       Lwt.return_true
     end
 

From 65ea372a888cd02dc7debbf75dbb8944215e095e Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Thu, 20 Jul 2017 16:10:39 +0200
Subject: [PATCH 294/373] harvest_limited

---
 .../test/server/quick/system_tests_basic.py   | 17 ++++++++++
 src/lib/std.ml                                |  6 ++++
 src/node/node_main.ml                         |  7 ++--
 src/node/sync_backend.ml                      | 20 +++++++++--
 src/nursery/routing.ml                        | 11 +++++++
 src/paxos/master.ml                           |  6 ++--
 src/paxos/multi_paxos.ml                      |  5 ++-
 src/paxos/multi_paxos_fsm.ml                  | 14 +++++---
 src/paxos/multi_paxos_test.ml                 |  7 ++--
 src/paxos/slave.ml                            |  2 +-
 src/tlog/interval.ml                          |  4 +++
 src/tlog/update.ml                            | 32 +++++++++++++++++-
 src/tools/llio.ml                             |  5 +++
 src/tools/llio.mli                            |  2 ++
 src/tools/lwt_buffer.ml                       | 33 +++++++++++++++----
 15 files changed, 148 insertions(+), 23 deletions(-)

diff --git a/pylabs/test/server/quick/system_tests_basic.py b/pylabs/test/server/quick/system_tests_basic.py
index 91bcf2cf..ef9ff8c5 100644
--- a/pylabs/test/server/quick/system_tests_basic.py
+++ b/pylabs/test/server/quick/system_tests_basic.py
@@ -885,6 +885,23 @@ def test_rev_range_entries_arakoon368():
     correct = [("key2","value2"), ("key1","value1"), ("key0","value0") ]
     assert_equals(l1, correct)
 
+@C.with_custom_setup(C.setup_3_nodes, C.basic_teardown)
+def test_harvest_limited():
+    logging.info("test_harvest_limited")
+    c = C.get_client()
+    value = 'xxxxxxxxxx' * (1000 * 1000)
+    seq = c.makeSequence()
+    for i in range(4):
+        seq.addSet("key_%02i" % i, value)
+    try:
+        logging.info("very fat sequence")
+        c.sequence(seq)
+        raise Exception("should not get here")
+    except X.arakoon_client.ArakoonBadInput as ex:
+        print "Exception:"
+        logging.info("ok:%s ", ex)
+
+
 @C.with_custom_setup( C.setup_3_nodes, C.basic_teardown )
 def test_statistics():
     cluster = C._getCluster()
diff --git a/src/lib/std.ml b/src/lib/std.ml
index 4f5cf605..8103778f 100644
--- a/src/lib/std.ml
+++ b/src/lib/std.ml
@@ -209,6 +209,12 @@ end
 module List = struct
   include List
 
+  let split3 xs =
+    let x0s_r, x1s_r, x2s_r =
+      fold_left
+        (fun (x0s,x1s,x2s) (x0,x1,x2) -> (x0::x0s, x1::x1s, x2::x2s))
+        ([], [], []) xs
+    in rev x0s_r, rev x1s_r, rev x2s_r
   let is_empty = function
     | [] -> true
     | _ -> false
diff --git a/src/node/node_main.ml b/src/node/node_main.ml
index e85a789e..eb7cb797 100644
--- a/src/node/node_main.ml
+++ b/src/node/node_main.ml
@@ -690,7 +690,7 @@ let _main_2 (type s)
             let snapshot_name = get_snapshot_name() in
             let full_snapshot_path = Filename.concat me.head_dir snapshot_name in
             new SB.sync_backend me
-              (client_push: (Update.t * (Store.update_result -> unit Lwt.t)) -> unit Lwt.t)
+              (client_push: (Update.t * int * (Store.update_result -> unit Lwt.t)) -> unit Lwt.t)
               inject_push
               store (S.copy_store2, full_snapshot_path, me.head_copy_throttling)
               tlog_coll lease_period
@@ -699,6 +699,7 @@ let _main_2 (type s)
               ~test
               ~read_only
               ~max_value_size:cluster_cfg.max_value_size
+              ~max_buffer_size:cluster_cfg.max_buffer_size
               ~collapse_slowdown:me.collapse_slowdown
               ~optimize_db_slowdown:me.optimize_db_slowdown
               ~act_not_preferred
@@ -833,7 +834,8 @@ let _main_2 (type s)
                 | Multi_paxos.FromClient fc ->
                   (fun () ->
                      Lwt_list.iter_s
-                       (fun u -> Lwt_buffer.add u client_buffer)
+                       (fun u ->
+                         Lwt_buffer.add u client_buffer)
                        fc),
                   "client_buffer"
                 | Multi_paxos.FromNode (m,source)  ->
@@ -886,6 +888,7 @@ let _main_2 (type s)
               inject_event
               is_alive
               ~cluster_id
+              ~max_buffer_size:cluster_cfg.max_buffer_size
               stop
           in
           let reporting_period = me.reporting in
diff --git a/src/node/sync_backend.ml b/src/node/sync_backend.ml
index c11f7952..6ba2e751 100644
--- a/src/node/sync_backend.ml
+++ b/src/node/sync_backend.ml
@@ -75,9 +75,10 @@ struct
 
   let _update_rendezvous self ~so_post update update_stats push =
     self # _write_allowed ();
+    self # _check_update_size update >>= fun update_size ->
     let sleep, awake = Lwt.wait () in
     let went_well = make_went_well update_stats awake sleep in
-    push (update, went_well) >>= fun () ->
+    push (update, update_size, went_well) >>= fun () ->
     let t0 = Unix.gettimeofday() in
     sleep >>= fun r ->
     let t1 = Unix.gettimeofday() in
@@ -94,7 +95,7 @@ struct
 
 
   class sync_backend = fun cfg
-    (push_update:Update.t * (Store.update_result -> unit Lwt.t) -> unit Lwt.t)
+    (push_update:Update.t * int * (Store.update_result -> unit Lwt.t) -> unit Lwt.t)
     (push_node_msg:Multi_paxos.paxos_event -> unit Lwt.t)
     (store: 'a)
     (store_methods: (string -> string ->
@@ -107,6 +108,7 @@ struct
     ~test
     ~(read_only:bool)
     ~max_value_size
+    ~max_buffer_size
     ~collapse_slowdown
     ~optimize_db_slowdown
     ~act_not_preferred
@@ -459,6 +461,20 @@ struct
               then raise (XException(Arakoon_exc.E_NOT_MASTER, m))
           end
 
+      method _check_update_size update =
+        let size = Update.serialized_size update in
+        if size > max_buffer_size
+        then
+          let msg =
+            Printf.sprintf "update %s has size %i > %i"
+                           (Update.update2s update)
+                           size
+                           max_buffer_size
+          in
+          Lwt.fail (XException(Arakoon_exc.E_BAD_INPUT, msg))
+        else
+          Lwt.return size
+
       method private _read_allowed (consistency:consistency) =
         if not read_only
         then
diff --git a/src/nursery/routing.ml b/src/nursery/routing.ml
index 3a08ac62..021a0851 100644
--- a/src/nursery/routing.ml
+++ b/src/nursery/routing.ml
@@ -199,6 +199,17 @@ module Routing = struct
     in
     _build ()
 
+  let serialized_size routing =
+    let rec walk = function
+      | Cluster x->
+         1 + Llio.string_ssize x
+      | Branch (left, sep, right) ->
+         1 + Llio.string_ssize sep
+         + walk left
+         + walk right
+    in
+    walk routing
+
   let output_routing oc routing =
     let buf = Buffer.create 97 in
     let () = routing_to buf routing in
diff --git a/src/paxos/master.ml b/src/paxos/master.ml
index e58af51d..38be336f 100644
--- a/src/paxos/master.ml
+++ b/src/paxos/master.ml
@@ -41,7 +41,9 @@ let master_consensus (type s) constants {mo;v;n;i; lew} () =
   let inject_e = EGen (fun () ->
       match v with
         | Value.Vm _ ->
-          let event = Multi_paxos.FromClient [(Update.Nop, fun _ -> Lwt.return ())] in
+           let open Update in
+           let event = Multi_paxos.FromClient
+                         [Nop, serialized_size Nop, fun _ -> Lwt.return_unit] in
           Lwt.ignore_result (constants.inject_event event);
           Lwt.return ()
         | Value.Vc _ ->
@@ -135,7 +137,7 @@ let stable_master (type s) constants ((n,new_i, lease_expire_waiters) as current
       end
     | FromClient ufs ->
       begin
-        let updates, finished_funs = List.split ufs in
+        let updates, _, finished_funs = Std.List.split3 ufs in
         let synced = List.fold_left (fun acc u -> acc || Update.is_synced u) false updates in
         let v = Value.create_client_value updates synced in
         let ms = {mo = Some finished_funs;v;n;i = new_i;
diff --git a/src/paxos/multi_paxos.ml b/src/paxos/multi_paxos.ml
index a7a26027..d60a4b60 100644
--- a/src/paxos/multi_paxos.ml
+++ b/src/paxos/multi_paxos.ml
@@ -86,7 +86,7 @@ let update_votes (nones, somes) = function
     |> fun s -> (nones, s)
 
 type paxos_event =
-  | FromClient of ((Update.Update.t) * (Store.update_result -> unit Lwt.t)) list
+  | FromClient of ((Update.Update.t) * int * (Store.update_result -> unit Lwt.t)) list
   | FromNode of (MPMessage.t * Messaging.id)
   | LeaseExpired of (float)
   | Quiesce of (Quiesce.Mode.t * Quiesce.Result.t Lwt.t * Quiesce.Result.t Lwt.u)
@@ -131,6 +131,7 @@ type 'a constants =
    mutable election_timeout : (Sn.t * Sn.t * float) option;
    mutable lease_expiration_id : int;
    mutable respect_run_master : (string * float) option;
+   max_buffer_size: int;
   }
 
 let am_forced_master constants me =
@@ -147,6 +148,7 @@ let make (type s) ~catchup_tls_ctx ~tcp_keepalive me is_learner others send get_
       on_accept on_consensus on_witness
       last_witnessed quorum_function (master:master) (module S : Store.STORE with type t = s) store tlog_coll
       other_cfgs lease_expiration inject_event is_alive ~cluster_id
+      ~max_buffer_size
       stop =
   {
     me=me;
@@ -174,6 +176,7 @@ let make (type s) ~catchup_tls_ctx ~tcp_keepalive me is_learner others send get_
     election_timeout = None;
     lease_expiration_id = 0;
     respect_run_master = None;
+    max_buffer_size;
   }
 
 let mcast constants msg =
diff --git a/src/paxos/multi_paxos_fsm.ml b/src/paxos/multi_paxos_fsm.ml
index 1f28c115..d45862a5 100644
--- a/src/paxos/multi_paxos_fsm.ml
+++ b/src/paxos/multi_paxos_fsm.ml
@@ -661,11 +661,15 @@ let paxos_produce buffers constants product_wanted =
                Lwt_buffer.take buffers.inject_buffer
              end
            | Some Client ->
-             begin
-               Lwt_buffer.harvest buffers.client_buffer >>= fun reqs ->
-               let event = FromClient reqs in
-               Lwt.return event
-             end
+              begin
+               let weight (_, size, _)= size in
+                let max = constants.max_buffer_size in
+                Lwt_buffer.harvest_limited
+                  buffers.client_buffer weight max
+                >>= fun reqs ->
+                let event = FromClient reqs in
+                Lwt.return event
+              end
            | Some Node ->
              begin
                Lwt_buffer.take buffers.node_buffer >>= fun (msg,source) ->
diff --git a/src/paxos/multi_paxos_test.ml b/src/paxos/multi_paxos_test.ml
index d14fd29d..ac3d1842 100644
--- a/src/paxos/multi_paxos_test.ml
+++ b/src/paxos/multi_paxos_test.ml
@@ -64,7 +64,7 @@ let test_generic network_factory n_nodes () =
   let inject_ev q e = Lwt_buffer.add e q in
   S.make_store ~lcnum:1024 ~ncnum:512 "MEM#store" ~cluster_id >>= fun store ->
   Mem_tlogcollection.make_mem_tlog_collection
-    "MEM#tlog" None None ~fsync:false "???" 
+    "MEM#tlog" None None ~fsync:false "???"
     ~fsync_tlog_dir:false ~cluster_id
   >>= fun tlog_coll ->
   let base = {me = "???";
@@ -92,6 +92,7 @@ let test_generic network_factory n_nodes () =
               respect_run_master = None;
               catchup_tls_ctx = None;
               tcp_keepalive = Tcp_keepalive.default_tcp_keepalive;
+              max_buffer_size = Node_cfg.default_max_buffer_size;
              }
   in
   let all_happy = build_names (n_nodes -1) in
@@ -248,7 +249,7 @@ let test_master_loop network_factory ()  =
   let () = Lwt.ignore_result (
       Lwt_list.iter_s
         (fun x ->
-           Lwt_buffer.add (x, finished) client_buffer >>= fun () ->
+           Lwt_buffer.add (x, 0, finished) client_buffer >>= fun () ->
            Lwt_unix.sleep 2.0
         ) updates
     ) in
@@ -294,6 +295,7 @@ let test_master_loop network_factory ()  =
                    respect_run_master = None;
                    catchup_tls_ctx = None;
                    tcp_keepalive = Tcp_keepalive.default_tcp_keepalive;
+                   max_buffer_size = Node_cfg.default_max_buffer_size;
                   } in
   let continue = ref 2 in
   let c0_t () =
@@ -439,6 +441,7 @@ let test_simulation filters () =
     respect_run_master = None;
     catchup_tls_ctx = None;
     tcp_keepalive = Tcp_keepalive.default_tcp_keepalive;
+    max_buffer_size = Node_cfg.default_max_buffer_size;
   } in
   let c0_t () =
     let expected prev_key key =
diff --git a/src/paxos/slave.ml b/src/paxos/slave.ml
index 370508e7..09b89beb 100644
--- a/src/paxos/slave.ml
+++ b/src/paxos/slave.ml
@@ -246,7 +246,7 @@ let slave_steady_state (type s) constants state event =
            that allows clients to get through before the node became a slave
            but I know I'm a slave now, so I let the update fail.
         *)
-        let finished_funs = List.map snd ufs in
+        let finished_funs = List.map (fun (_,_,f) -> f) ufs in
         let result = Store.Update_fail (Arakoon_exc.E_NOT_MASTER, "Not_Master") in
         let rec loop = function
           | []       -> Lwt.return ()
diff --git a/src/tlog/interval.ml b/src/tlog/interval.ml
index 45724592..7a5f65cf 100644
--- a/src/tlog/interval.ml
+++ b/src/tlog/interval.ml
@@ -53,6 +53,10 @@ module Interval = struct
     so2 buf t.pr_b;
     so2 buf t.pr_e
 
+  let serialized_size t =
+    let sos = Llio.string_option_ssize in
+    sos t.pu_b + sos t.pu_e + sos t.pr_b + sos t.pr_e
+
   let interval_from b =
     let sof () = Llio.string_option_from b in
     let pu_b = sof () in
diff --git a/src/tlog/update.ml b/src/tlog/update.ml
index a397ac20..d087035d 100644
--- a/src/tlog/update.ml
+++ b/src/tlog/update.ml
@@ -36,6 +36,9 @@ module Range_assertion = struct
         let ss = Llio.list_from Llio.string_from buf in
         ContainsExactly ss
       | k -> failwith (Printf.sprintf "%i: invalid Range_assertion" k)
+  let serialized_size = function
+    | ContainsExactly keys ->
+       8 + List.fold_left (fun acc x -> acc + Llio.string_ssize x) 0 keys
 end
 
 module Update = struct
@@ -178,7 +181,7 @@ module Update = struct
       | DeletePrefix prefix ->
         Llio.int_to b 14;
         Llio.string_to b prefix
-      | Assert_exists (k) ->
+      | Assert_exists k ->
         Llio.int_to b 15;
         Llio.string_to b k
       | Replace (k,vo) ->
@@ -285,4 +288,31 @@ module Update = struct
     | Replace _
     | Assert_range _ -> false
 
+  let rec serialized_size update =
+    let () = Lwt_log.ign_info_f "serialize_size %s " (update2s update) in
+    let sos = Llio.string_option_ssize in
+    let ss = Llio.string_ssize in
+    let seq_ssize seq =
+      4 + List.fold_left (fun acc u -> acc + serialized_size u) 0 seq
+    in
+    let inner = function
+    | Set(k,v)                -> ss k + ss v
+    | Delete k                -> ss k
+    | MasterSet(m, f)         -> ss m + 8
+    | TestAndSet(k,e,w)       -> ss k + sos e + sos w
+    | Sequence us             -> seq_ssize us
+    | Nop                     -> 0
+    | UserFunction (n, p)     -> ss n + sos p
+    | Assert (k, vo)          -> ss k + sos vo
+    | AdminSet(k, vo)         -> ss k + sos vo
+    | SetInterval i           -> Interval.serialized_size i
+    | SetRouting r            -> Routing.serialized_size r
+    | SetRoutingDelta (l,s,r) -> ss l + ss s + ss r
+    | SyncedSequence us       -> seq_ssize us
+    | DeletePrefix p          -> ss p
+    | Assert_exists k         -> ss k
+    | Replace (k,vo)          -> ss k  + sos vo
+    | Assert_range (p,a)      -> ss p  + Range_assertion.serialized_size a
+    in
+    4 + inner update
 end
diff --git a/src/tools/llio.ml b/src/tools/llio.ml
index 799c4e43..ae1683ce 100644
--- a/src/tools/llio.ml
+++ b/src/tools/llio.ml
@@ -197,6 +197,11 @@ let string_to buffer s =
   int_to buffer size;
   Buffer.add_string buffer s
 
+let string_ssize s = 4 + String.length s
+
+let string_option_ssize = function
+  | None -> 1
+  | Some s -> 1 + string_ssize s
 
 
 let unit_to _ () = ()
diff --git a/src/tools/llio.mli b/src/tools/llio.mli
index 5d51082f..218ec278 100644
--- a/src/tools/llio.mli
+++ b/src/tools/llio.mli
@@ -154,4 +154,6 @@ val input_string_list: string list lwt_deserializer
 val input_kv_list: (string * string) list lwt_deserializer
 val input_hashtbl: 'a lwt_deserializer -> 'b lwt_deserializer -> ('a,'b) Hashtbl.t lwt_deserializer
 
+val string_ssize : string -> int
+val string_option_ssize : string option -> int
 val copy_stream:  length:int64 -> ic:lwtic -> oc:lwtoc -> unit Lwt.t
diff --git a/src/tools/lwt_buffer.ml b/src/tools/lwt_buffer.ml
index 1023930e..195026b1 100644
--- a/src/tools/lwt_buffer.ml
+++ b/src/tools/lwt_buffer.ml
@@ -87,24 +87,43 @@ module Lwt_buffer = struct
       let () = Lwt_condition.signal t.full () in
       Some item
 
-  let harvest t =
+
+  let harvest_limited (t:'a t)
+                      (item_weight:'a -> int)
+                      (max_weight:int) =
     Lwt_mutex.with_lock t.empty_m
       (fun () -> if Queue.is_empty t.q
         then Lwt_condition.wait (* ~mutex:t.empty_m *) t.empty
         else Lwt.return ()
       ) >>= fun () ->
     let size = Queue.length t.q in
-    let rec loop es = function
-      | 0 -> List.rev es
+    let rec loop es tw = function
+      | 0 -> tw, es
       | i ->
-        let es' = Queue.take t.q :: es
-        and i' = i - 1 in
-        loop es' i'
+         let pot_item = Queue.peek t.q in
+         let w = item_weight pot_item in
+         let () = Lwt_log.ign_info_f "pot weight =%i" w in
+         if tw + w > max_weight
+         then tw, es
+         else
+           let es' = Queue.take t.q :: es
+           and i' = i - 1
+           and tw' = tw + w in
+           loop es' tw' i'
     in
-    let es = loop [] size in
+    let tw, es0 = loop [] 0 size in
+    let () = Lwt_log.ign_info_f "total weight=%i (max=%i)" tw max_weight in
+    let es = List.rev es0 in
+    assert (es <> []); (* input validation should have rejected this earlier *)
     let () = Lwt_condition.signal t.full() in
     Lwt.return es
 
+  let harvest t =
+    let item_weight _ = 0
+    and max = 1 in
+    harvest_limited t item_weight max
+
+
   let wait_for_item t =
     Lwt_mutex.with_lock t.empty_m
       (fun () ->

From 6346fd22b42bff667ff35c691365b29c5743d4a6 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Mon, 24 Jul 2017 11:38:12 +0200
Subject: [PATCH 295/373] change a few log levels from info to debug

---
 src/tlog/update.ml      | 2 +-
 src/tools/lwt_buffer.ml | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/tlog/update.ml b/src/tlog/update.ml
index d087035d..075da77b 100644
--- a/src/tlog/update.ml
+++ b/src/tlog/update.ml
@@ -289,7 +289,7 @@ module Update = struct
     | Assert_range _ -> false
 
   let rec serialized_size update =
-    let () = Lwt_log.ign_info_f "serialize_size %s " (update2s update) in
+    let () = Lwt_log.ign_debug_f "serialize_size %s " (update2s update) in
     let sos = Llio.string_option_ssize in
     let ss = Llio.string_ssize in
     let seq_ssize seq =
diff --git a/src/tools/lwt_buffer.ml b/src/tools/lwt_buffer.ml
index 195026b1..bc1501fe 100644
--- a/src/tools/lwt_buffer.ml
+++ b/src/tools/lwt_buffer.ml
@@ -102,7 +102,7 @@ module Lwt_buffer = struct
       | i ->
          let pot_item = Queue.peek t.q in
          let w = item_weight pot_item in
-         let () = Lwt_log.ign_info_f "pot weight =%i" w in
+         let () = Lwt_log.ign_debug_f "pot weight =%i" w in
          if tw + w > max_weight
          then tw, es
          else
@@ -112,7 +112,7 @@ module Lwt_buffer = struct
            loop es' tw' i'
     in
     let tw, es0 = loop [] 0 size in
-    let () = Lwt_log.ign_info_f "total weight=%i (max=%i)" tw max_weight in
+    let () = Lwt_log.ign_debug_f "total weight=%i (max=%i)" tw max_weight in
     let es = List.rev es0 in
     assert (es <> []); (* input validation should have rejected this earlier *)
     let () = Lwt_condition.signal t.full() in

From d0fb0859660af19c22d960e4b49bd01b1b6530b7 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 9 Aug 2017 09:38:05 +0200
Subject: [PATCH 296/373] cater for value & message when harvesting

---
 src/all_test.ml              |  3 ++-
 src/paxos/mp_msg_test.ml     | 47 ++++++++++++++++++++++++++++++++++++
 src/paxos/multi_paxos_fsm.ml |  5 ++--
 src/tlog/update.ml           |  3 ++-
 src/tools/lwt_buffer.ml      |  8 ++++--
 5 files changed, 60 insertions(+), 6 deletions(-)
 create mode 100644 src/paxos/mp_msg_test.ml

diff --git a/src/all_test.ml b/src/all_test.ml
index a96d7373..77ef1e21 100644
--- a/src/all_test.ml
+++ b/src/all_test.ml
@@ -52,7 +52,7 @@ let tlog_tests = "tlogs" >::: [
       Tlc2_test.suite;
       Tlogreader2_test.suite;
     ]
-                                
+
 let small_catchup = "small_catchup" >:::[Catchup_test.suite;]
 let store = "store" >:::[Store_test.suite;]
 let compression = "compression" >::: [Compression_test.suite;]
@@ -94,4 +94,5 @@ let suite = "universe" >::: [
     Arakoon_etcd_test.suite;
     Arakoon_client_config_test.suite;
     Arakoon_config_url_test.suite;
+    Mp_msg_test.suite;
   ]
diff --git a/src/paxos/mp_msg_test.ml b/src/paxos/mp_msg_test.ml
new file mode 100644
index 00000000..bb349734
--- /dev/null
+++ b/src/paxos/mp_msg_test.ml
@@ -0,0 +1,47 @@
+(*
+  Copyright (C) iNuron - info@openvstorage.com
+  For license information, see <LICENSE.txt>
+*)
+
+
+open OUnit
+open Value
+open Mp_msg
+open Message
+open Update
+
+let size_test_0 () =
+  let v = create_client_value [] false in
+  let p = MPMessage.Accept (0L, 0L, v) in
+  let generic = MPMessage.generic_of p in
+  let buff = Buffer.create 20 in
+  let () = Message.to_buffer generic buff in
+  let serialized = Buffer.contents buff in
+  let size = String.length serialized in
+  (* If ever, this test starts failing,
+     re-evaluate the heuristic in
+   *)
+  let msg = "please re-evaluate the heuristic in" ^
+              "multi_paxos_fsm's use of harvest_limited"
+  in
+  OUnit.assert_equal 40 size ~printer:string_of_int ~msg
+
+let size_test_1 () =
+  let u0 = Update.Set ("____00000012", "0123456789") in
+  let size_u0 = Update.serialized_size u0 in
+  let () = OUnit.assert_equal 34 size_u0 ~printer:string_of_int in
+  let us = [u0] in
+  let v = create_client_value us false in
+  let p = MPMessage.Accept (0L, 0L, v) in
+  let generic = MPMessage.generic_of p in
+  let buff = Buffer.create 20 in
+  let () = Message.to_buffer generic buff in
+  let serialized = Buffer.contents buff in
+  let size = String.length serialized in
+  OUnit.assert_equal (40+size_u0) size ~printer:string_of_int
+
+let suite =
+  "mp_msg" >::: [
+      "size_0" >:: size_test_0;
+      "size_1" >:: size_test_1;
+    ]
diff --git a/src/paxos/multi_paxos_fsm.ml b/src/paxos/multi_paxos_fsm.ml
index d45862a5..6a20a9f7 100644
--- a/src/paxos/multi_paxos_fsm.ml
+++ b/src/paxos/multi_paxos_fsm.ml
@@ -662,8 +662,9 @@ let paxos_produce buffers constants product_wanted =
              end
            | Some Client ->
               begin
-               let weight (_, size, _)= size in
-                let max = constants.max_buffer_size in
+                let weight (_, size, _)= size in
+                let safety = 100 (* value & message will add ~ 40 bytes *) in
+                let max = constants.max_buffer_size - safety in
                 Lwt_buffer.harvest_limited
                   buffers.client_buffer weight max
                 >>= fun reqs ->
diff --git a/src/tlog/update.ml b/src/tlog/update.ml
index 075da77b..bfe92d97 100644
--- a/src/tlog/update.ml
+++ b/src/tlog/update.ml
@@ -16,6 +16,7 @@ limitations under the License.
 
 open Interval
 open Routing
+let section = Logger.Section.main
 
 module Range_assertion = struct
   type t =
@@ -289,7 +290,7 @@ module Update = struct
     | Assert_range _ -> false
 
   let rec serialized_size update =
-    let () = Lwt_log.ign_debug_f "serialize_size %s " (update2s update) in
+    let () = Logger.ign_debug_f_ "serialize_size %s " (update2s update) in
     let sos = Llio.string_option_ssize in
     let ss = Llio.string_ssize in
     let seq_ssize seq =
diff --git a/src/tools/lwt_buffer.ml b/src/tools/lwt_buffer.ml
index bc1501fe..f7a763a0 100644
--- a/src/tools/lwt_buffer.ml
+++ b/src/tools/lwt_buffer.ml
@@ -17,6 +17,7 @@ limitations under the License.
 
 
 open Lwt
+let section = Logger.Section.main
 
 module Lwt_buffer = struct
   type 'a t = {
@@ -102,7 +103,7 @@ module Lwt_buffer = struct
       | i ->
          let pot_item = Queue.peek t.q in
          let w = item_weight pot_item in
-         let () = Lwt_log.ign_debug_f "pot weight =%i" w in
+         let () = Logger.ign_debug_f_ "pot weight =%i" w in
          if tw + w > max_weight
          then tw, es
          else
@@ -112,7 +113,10 @@ module Lwt_buffer = struct
            loop es' tw' i'
     in
     let tw, es0 = loop [] 0 size in
-    let () = Lwt_log.ign_debug_f "total weight=%i (max=%i)" tw max_weight in
+    let () =
+      Logger.ign_debug_f_
+        "total weight=%i (max=%i)" tw max_weight
+    in
     let es = List.rev es0 in
     assert (es <> []); (* input validation should have rejected this earlier *)
     let () = Lwt_condition.signal t.full() in

From fbcdc1390876232d5e5dc09777e2c36a1e4469c7 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 9 Aug 2017 09:58:03 +0200
Subject: [PATCH 297/373] recycle send buffer for msg serialization

---
 src/paxos/mp_msg.ml      | 14 +++++++-------
 src/paxos/multi_paxos.ml | 12 ++++++++----
 2 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/src/paxos/mp_msg.ml b/src/paxos/mp_msg.ml
index 985009e0..d71930b3 100644
--- a/src/paxos/mp_msg.ml
+++ b/src/paxos/mp_msg.ml
@@ -51,36 +51,36 @@ module MPMessage = struct
       Printf.sprintf "Nak (%s,(%s,%s))"
         (Sn.string_of n) (Sn.string_of n') (Sn.string_of i')
 
-  let generic_of = function
+  let generic_of' buf = function
     | Prepare (n,i) ->
-      let buf = Buffer.create 20 in
       Sn.sn_to buf n;
       Sn.sn_to buf i;
       Message.create "prepare" (Buffer.contents buf)
     | Promise (n,i, vo) ->
-      let buf = Buffer.create 20 in
       Sn.sn_to buf n;
       Sn.sn_to buf i;
       Llio.option_to Value.value_to buf vo;
       Message.create "promise" (Buffer.contents buf)
     | Accept(n,i , v) ->
-      let buf = Buffer.create 20 in
       Sn.sn_to buf n;
       Sn.sn_to buf i;
       Value.value_to buf v;
       Message.create "accept" (Buffer.contents buf)
     | Accepted(n,i) ->
-      let buf = Buffer.create 20 in
       Sn.sn_to buf n;
       Sn.sn_to buf i;
       Message.create "accepted" (Buffer.contents buf)
     | Nak (n,(n',i')) ->
-      let buf = Buffer.create 20 in
       Sn.sn_to buf n;
       Sn.sn_to buf n';
       Sn.sn_to buf i';
       Message.create "nak" (Buffer.contents buf)
 
+  let generic_of m =
+    let buf = Buffer.create 20 in
+    generic_of' buf m
+
+
   let of_generic m =
     let kind, payload = Message.kind_of m, Message.payload_of m in
     let buf = Llio.make_buffer payload 0 in
@@ -92,7 +92,7 @@ module MPMessage = struct
       | "nak"     ->
         let n = Sn.sn_from buf in
         let n' = Sn.sn_from buf in
-        let i' = Sn.sn_from buf in
+       let i' = Sn.sn_from buf in
         Nak (n,(n',i'))
       | "promise" ->
         let n = Sn.sn_from buf in
diff --git a/src/paxos/multi_paxos.ml b/src/paxos/multi_paxos.ml
index d60a4b60..85bb6027 100644
--- a/src/paxos/multi_paxos.ml
+++ b/src/paxos/multi_paxos.ml
@@ -58,10 +58,14 @@ type ballot = int * string list (* still needed, & who voted *)
 
 let network_of_messaging (m:messaging) =
   (* conversion since not all code is 'networked' *)
-  let send msg source target =
-    Logger.debug_f_ "%s: sending msg to %s: %s" source target (Mp_msg.MPMessage.string_of msg) >>= fun () ->
-    let g = MPMessage.generic_of msg in
-    m # send_message g ~source ~target
+  let send =
+    let buf = Buffer.create 100 in
+    (fun msg source target ->
+      Logger.debug_f_ "%s: sending msg to %s: %s" source target (Mp_msg.MPMessage.string_of msg) >>= fun () ->
+      let () = Buffer.clear buf in
+      let g = MPMessage.generic_of' buf msg in
+      m # send_message g ~source ~target
+    )
   in
   let register = m # register_receivers in
   let run () = m # run () in

From 9e24e299fc0fad3c28a3195f93852a9e3225e5b4 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 9 Aug 2017 10:51:10 +0200
Subject: [PATCH 298/373] make_release: 1.9.20

---
 META                      | 2 +-
 debian/changelog          | 5 +++++
 redhat/SPECS/arakoon.spec | 4 +++-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/META b/META
index a4c34472..042123f6 100644
--- a/META
+++ b/META
@@ -1,7 +1,7 @@
 # -*- conf -*-
 
 description = "Arakoon client"
-version = "1.9.19"
+version = "1.9.20"
 exists_if = "arakoon_client.cma,arakoon_client.cmxa,arakoon_client.cmxs"
 requires = "threads lwt lwt.unix str oUnit quickcheck lwt_ssl ocplib-endian redis.lwt uri"
 archive(byte) = "arakoon_client.cma tcp_keepalive_sockopt_stubs.o"
diff --git a/debian/changelog b/debian/changelog
index b12872d1..1226c1e5 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,8 @@
+arakoon (1.9.20) unstable; urgency=low
+  * new upstream version
+
+ -- Romain Slootmaekers <romain.slootmaekers@openvstorage.com>  Wed, 09 Aug 2017 10:51:10 +0100
+
 arakoon (1.9.19) unstable; urgency=low
   * new upstream version
 
diff --git a/redhat/SPECS/arakoon.spec b/redhat/SPECS/arakoon.spec
index 53351b92..83662d5d 100644
--- a/redhat/SPECS/arakoon.spec
+++ b/redhat/SPECS/arakoon.spec
@@ -2,7 +2,7 @@
 
 Summary: Arakoon
 Name: arakoon
-Version: 1.9.19
+Version: 1.9.20
 Release: 3%{?dist}
 License: Apache 2.0
 Requires: libev >= 4
@@ -33,6 +33,8 @@ cp arakoon.native %{buildroot}%{_bindir}/arakoon
 %{_bindir}/arakoon
 
 %changelog
+* Wed Aug 09 2017 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.20
+- Create arakoon 1.9.20 RPM package
 * Fri Jun 23 2017 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.19
 - Create arakoon 1.9.19 RPM package
 * Thu Jun 01 2017 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.18

From f7464f23f864e99c8866247498cb7d69a2588c69 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Tue, 22 Aug 2017 12:08:48 +0200
Subject: [PATCH 299/373] big sequences can flood logs: no more.

---
 src/tlog/update.ml | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/tlog/update.ml b/src/tlog/update.ml
index a397ac20..d9b17389 100644
--- a/src/tlog/update.ml
+++ b/src/tlog/update.ml
@@ -75,19 +75,24 @@ module Update = struct
       | Some s -> if values then s else "..."
     in
     let rec _sequence2s typ updates =
-      let buf = Buffer.create (64 * List.length updates) in
+      let len_seq = List.length updates in
+      let len' = min 100 len_seq in
+      let buf = Buffer.create (64 * len') in
       let add s = Buffer.add_string buf s in
       let () = add typ in
       let () = add "([" in
-      let rec loop = function
+      let rec loop c = function
         | [] -> add "])"
         | u::us ->
            let () = add (_inner u) in
            let () = if (us <> []) then add "; "
            in
-           loop us
+           let c' = c + 1 in
+           if c' = len'
+           then add (Printf.sprintf "... (%i more) ])"  (len_seq -len'))
+           else loop c' us
       in
-      let () = loop updates in
+      let () = loop 0 updates in
       Buffer.contents buf
     and
       _inner = function

From 849473c1e9687557e21756c4391d80f30e66dc6a Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Tue, 22 Aug 2017 15:13:24 +0200
Subject: [PATCH 300/373] add update.mli

---
 Makefile            |  1 +
 src/tlog/update.mli | 44 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 45 insertions(+)
 create mode 100644 src/tlog/update.mli

diff --git a/Makefile b/Makefile
index 53b98578..643ca3ed 100644
--- a/Makefile
+++ b/Makefile
@@ -71,6 +71,7 @@ install_client:
           _build/src/nursery/routing.cmi \
           _build/src/tlog/interval.cmi \
           _build/src/tlog/interval.cmx \
+          _build/src/tlog/update.mli \
           _build/src/tlog/update.cmi \
 	  _build/src/tlog/update.cmx \
           _build/src/tools/ini.cmi \
diff --git a/src/tlog/update.mli b/src/tlog/update.mli
new file mode 100644
index 00000000..bac746e9
--- /dev/null
+++ b/src/tlog/update.mli
@@ -0,0 +1,44 @@
+(*
+  Copyright (C) iNuron - info@openvstorage.com
+  This file is part of Open vStorage. For license information, see <LICENSE>
+*)
+
+val section : Lwt_log_core.section
+
+module Range_assertion :
+  sig
+    type t = ContainsExactly of string list
+    val to_string : t -> string
+    val to_buffer : Buffer.t -> t -> unit
+    val from_buffer : Llio.buffer -> t
+    val serialized_size : t -> int
+  end
+
+module Update :
+  sig
+    type t =
+        Set of string * string
+      | Delete of string
+      | MasterSet of string * float
+      | TestAndSet of string * string option * string option
+      | Sequence of t list
+      | SetInterval of Interval.Interval.t
+      | SetRouting of Routing.Routing.t
+      | SetRoutingDelta of (string * string * string)
+      | Nop
+      | Assert of string * string option
+      | Assert_exists of string
+      | Assert_range of string * Range_assertion.t
+      | UserFunction of string * string option
+      | AdminSet of string * string option
+      | SyncedSequence of t list
+      | DeletePrefix of string
+      | Replace of string * string option
+    val make_master_set : string -> float option -> t
+
+    val update2s : ?values:bool -> t -> string
+    val to_buffer : Buffer.t -> t -> unit
+    val from_buffer : Llio.buffer -> t
+    val is_synced : t -> bool
+    val serialized_size : t -> int
+  end

From 1021be41de1e790535ff5cfb43292f8b131e472e Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Tue, 22 Aug 2017 15:39:04 +0200
Subject: [PATCH 301/373] make_release: 1.9.21

---
 META                      | 2 +-
 debian/changelog          | 5 +++++
 redhat/SPECS/arakoon.spec | 4 +++-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/META b/META
index 042123f6..d3aa566b 100644
--- a/META
+++ b/META
@@ -1,7 +1,7 @@
 # -*- conf -*-
 
 description = "Arakoon client"
-version = "1.9.20"
+version = "1.9.21"
 exists_if = "arakoon_client.cma,arakoon_client.cmxa,arakoon_client.cmxs"
 requires = "threads lwt lwt.unix str oUnit quickcheck lwt_ssl ocplib-endian redis.lwt uri"
 archive(byte) = "arakoon_client.cma tcp_keepalive_sockopt_stubs.o"
diff --git a/debian/changelog b/debian/changelog
index 1226c1e5..b1c7a2a5 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,8 @@
+arakoon (1.9.21) unstable; urgency=low
+  * new upstream version
+
+ -- Romain Slootmaekers <romain.slootmaekers@openvstorage.com>  Tue, 22 Aug 2017 15:39:04 +0100
+
 arakoon (1.9.20) unstable; urgency=low
   * new upstream version
 
diff --git a/redhat/SPECS/arakoon.spec b/redhat/SPECS/arakoon.spec
index 83662d5d..45cb719b 100644
--- a/redhat/SPECS/arakoon.spec
+++ b/redhat/SPECS/arakoon.spec
@@ -2,7 +2,7 @@
 
 Summary: Arakoon
 Name: arakoon
-Version: 1.9.20
+Version: 1.9.21
 Release: 3%{?dist}
 License: Apache 2.0
 Requires: libev >= 4
@@ -33,6 +33,8 @@ cp arakoon.native %{buildroot}%{_bindir}/arakoon
 %{_bindir}/arakoon
 
 %changelog
+* Tue Aug 22 2017 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.21
+- Create arakoon 1.9.21 RPM package
 * Wed Aug 09 2017 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.20
 - Create arakoon 1.9.20 RPM package
 * Fri Jun 23 2017 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.19

From 6d1132410ca4222ec2deeb9a034070d9a7672e3b Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 9 Aug 2017 14:05:58 +0200
Subject: [PATCH 302/373] upgrade dependencies

---
 docker/centos-7/Dockerfile     | 10 +++++-----
 docker/ubuntu-14.04/Dockerfile | 10 +++++-----
 docker/ubuntu-16.04/Dockerfile | 10 +++++-----
 3 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/docker/centos-7/Dockerfile b/docker/centos-7/Dockerfile
index 44291809..a83d15bd 100644
--- a/docker/centos-7/Dockerfile
+++ b/docker/centos-7/Dockerfile
@@ -23,7 +23,7 @@ RUN yum -y update && yum -y install \
 
 RUN wget https://raw.github.com/ocaml/opam/master/shell/opam_installer.sh
 
-env ocaml_version=4.03.0
+env ocaml_version=4.04.2
 RUN sh ./opam_installer.sh /usr/local/bin ${ocaml_version}
 
 ENV opam_root=/home/jenkins/OPAM
@@ -50,21 +50,21 @@ ENV OPAMEXTERNALSOLVER="cudf_remote_proxy %{input}% %{output}% %{criteria}%"
 RUN eval `${opam_env}` && \
     opam update && \
     opam install -y \
-        oasis.0.4.8 \
+        oasis.0.4.10 \
         ocamlfind \
         ssl.0.5.3 \
         camlbz2 \
         snappy \
         bisect \
         lwt_ssl \
-        lwt.3.0.0 \
+        lwt.3.1.0 \
         camltc.0.9.4 \
         ocplib-endian.1.0 \
         quickcheck.1.0.2 \
         conf-libev \
-        core.v0.9.0 \
+        core.v0.9. \1
         redis.0.3.3 \
-        uri.1.9.2
+        uri.1.9.4
 
 
 RUN pip install fabric junit-xml nose simplejson python-etcd
diff --git a/docker/ubuntu-14.04/Dockerfile b/docker/ubuntu-14.04/Dockerfile
index 6ef0142d..fc536158 100644
--- a/docker/ubuntu-14.04/Dockerfile
+++ b/docker/ubuntu-14.04/Dockerfile
@@ -34,7 +34,7 @@ RUN apt-get -y update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
 
 RUN wget https://raw.github.com/ocaml/opam/master/shell/opam_installer.sh
 
-env ocaml_version=4.03.0
+env ocaml_version=4.04.2
 RUN sh ./opam_installer.sh /usr/local/bin ${ocaml_version}
 
 ENV opam_root=/home/jenkins/OPAM
@@ -44,21 +44,21 @@ RUN opam init --root=${opam_root} --comp ${ocaml_version}
 RUN eval `${opam_env}` && \
     opam update && \
     opam install -y \
-        oasis.0.4.8 \
+        oasis.0.4.10 \
         ocamlfind \
         ssl.0.5.3 \
         camlbz2 \
         snappy \
         bisect \
         lwt_ssl \
-        lwt.3.0.0 \
+        lwt.3.1.0 \
         camltc.0.9.4 \
         ocplib-endian.1.0 \
         quickcheck.1.0.2 \
         conf-libev \
-        core.v0.9.0 \
+        core.v0.9.1 \
         redis.0.3.3 \
-        uri.1.9.2
+        uri.1.9.4
 
 #
 RUN wget https://www.python.org/ftp/python/2.7.12/Python-2.7.12.tar.xz
diff --git a/docker/ubuntu-16.04/Dockerfile b/docker/ubuntu-16.04/Dockerfile
index b586709a..8372fb42 100644
--- a/docker/ubuntu-16.04/Dockerfile
+++ b/docker/ubuntu-16.04/Dockerfile
@@ -33,7 +33,7 @@ RUN apt-get -y update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
 
 RUN wget https://raw.github.com/ocaml/opam/master/shell/opam_installer.sh
 
-env ocaml_version=4.03.0
+env ocaml_version=4.04.2
 RUN sh ./opam_installer.sh /usr/local/bin ${ocaml_version}
 
 ENV opam_root=/home/jenkins/OPAM
@@ -43,21 +43,21 @@ RUN opam init --root=${opam_root} --comp ${ocaml_version}
 RUN eval `${opam_env}` && \
     opam update && \
     opam install -y \
-        oasis.0.4.8 \
+        oasis.0.4.10 \
         ocamlfind \
         ssl.0.5.3 \
         camlbz2 \
         snappy \
         bisect \
         lwt_ssl \
-        lwt.3.0.0 \
+        lwt.3.1.0 \
         camltc.0.9.4 \
         ocplib-endian.1.0 \
         quickcheck.1.0.2 \
         conf-libev \
-        core.v0.9.0 \
+        core.v0.9.1 \
         redis.0.3.3 \
-        uri.1.9.2
+        uri.1.9.4
 
 
 RUN pip install fabric junit-xml nose simplejson python-etcd

From f0d500222fb83dd8d62d2a8869035dd652b6ba7e Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 16 Aug 2017 19:09:52 +0200
Subject: [PATCH 303/373] typo

---
 docker/centos-7/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/centos-7/Dockerfile b/docker/centos-7/Dockerfile
index a83d15bd..74280d6c 100644
--- a/docker/centos-7/Dockerfile
+++ b/docker/centos-7/Dockerfile
@@ -62,7 +62,7 @@ RUN eval `${opam_env}` && \
         ocplib-endian.1.0 \
         quickcheck.1.0.2 \
         conf-libev \
-        core.v0.9. \1
+        core.v0.9.1 \
         redis.0.3.3 \
         uri.1.9.4
 

From 899ed39f10e83ac9b07437ffe076025cdb0e8936 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Thu, 24 Aug 2017 11:38:41 +0200
Subject: [PATCH 304/373] explicit option for ppx_deriving.4.1 & base.v0.9.3

---
 docker/centos-7/Dockerfile     | 4 +++-
 docker/ubuntu-14.04/Dockerfile | 4 +++-
 docker/ubuntu-16.04/Dockerfile | 4 +++-
 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/docker/centos-7/Dockerfile b/docker/centos-7/Dockerfile
index 74280d6c..e0f0ff2c 100644
--- a/docker/centos-7/Dockerfile
+++ b/docker/centos-7/Dockerfile
@@ -48,7 +48,7 @@ RUN mv cudf_remote_proxy /usr/local/bin/
 ENV OPAMEXTERNALSOLVER="cudf_remote_proxy %{input}% %{output}% %{criteria}%"
 
 RUN eval `${opam_env}` && \
-    opam update && \
+    opam update -v && \
     opam install -y \
         oasis.0.4.10 \
         ocamlfind \
@@ -62,6 +62,8 @@ RUN eval `${opam_env}` && \
         ocplib-endian.1.0 \
         quickcheck.1.0.2 \
         conf-libev \
+        ppx_deriving.4.1 \
+        base.v0.9.3 \
         core.v0.9.1 \
         redis.0.3.3 \
         uri.1.9.4
diff --git a/docker/ubuntu-14.04/Dockerfile b/docker/ubuntu-14.04/Dockerfile
index fc536158..e8b9af0c 100644
--- a/docker/ubuntu-14.04/Dockerfile
+++ b/docker/ubuntu-14.04/Dockerfile
@@ -42,7 +42,7 @@ ENV opam_env="opam config env --root=${opam_root}"
 RUN opam init --root=${opam_root} --comp ${ocaml_version}
 
 RUN eval `${opam_env}` && \
-    opam update && \
+    opam update -v && \
     opam install -y \
         oasis.0.4.10 \
         ocamlfind \
@@ -56,6 +56,8 @@ RUN eval `${opam_env}` && \
         ocplib-endian.1.0 \
         quickcheck.1.0.2 \
         conf-libev \
+        ppx_deriving.4.1 \
+        base.v0.9.3 \
         core.v0.9.1 \
         redis.0.3.3 \
         uri.1.9.4
diff --git a/docker/ubuntu-16.04/Dockerfile b/docker/ubuntu-16.04/Dockerfile
index 8372fb42..9351d8bb 100644
--- a/docker/ubuntu-16.04/Dockerfile
+++ b/docker/ubuntu-16.04/Dockerfile
@@ -41,7 +41,7 @@ ENV opam_env="opam config env --root=${opam_root}"
 RUN opam init --root=${opam_root} --comp ${ocaml_version}
 
 RUN eval `${opam_env}` && \
-    opam update && \
+    opam update -v && \
     opam install -y \
         oasis.0.4.10 \
         ocamlfind \
@@ -55,6 +55,8 @@ RUN eval `${opam_env}` && \
         ocplib-endian.1.0 \
         quickcheck.1.0.2 \
         conf-libev \
+        ppx_deriving.4.1 \
+        base.v0.9.3 \
         core.v0.9.1 \
         redis.0.3.3 \
         uri.1.9.4

From bbc30cb804ffc0f93505152ad113ae170eb21940 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Thu, 24 Aug 2017 14:41:08 +0200
Subject: [PATCH 305/373] back to Lwt.3.0.0

TLS tests fail with 3.1.0: fe
server.quick.system_tests_ssl.TestTLS.test_inter_node_communication
---
 docker/centos-7/Dockerfile     | 2 +-
 docker/ubuntu-14.04/Dockerfile | 2 +-
 docker/ubuntu-16.04/Dockerfile | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/docker/centos-7/Dockerfile b/docker/centos-7/Dockerfile
index e0f0ff2c..b6193f95 100644
--- a/docker/centos-7/Dockerfile
+++ b/docker/centos-7/Dockerfile
@@ -57,7 +57,7 @@ RUN eval `${opam_env}` && \
         snappy \
         bisect \
         lwt_ssl \
-        lwt.3.1.0 \
+        lwt.3.0.0 \
         camltc.0.9.4 \
         ocplib-endian.1.0 \
         quickcheck.1.0.2 \
diff --git a/docker/ubuntu-14.04/Dockerfile b/docker/ubuntu-14.04/Dockerfile
index e8b9af0c..da373476 100644
--- a/docker/ubuntu-14.04/Dockerfile
+++ b/docker/ubuntu-14.04/Dockerfile
@@ -51,7 +51,7 @@ RUN eval `${opam_env}` && \
         snappy \
         bisect \
         lwt_ssl \
-        lwt.3.1.0 \
+        lwt.3.0.0 \
         camltc.0.9.4 \
         ocplib-endian.1.0 \
         quickcheck.1.0.2 \
diff --git a/docker/ubuntu-16.04/Dockerfile b/docker/ubuntu-16.04/Dockerfile
index 9351d8bb..4f979d8c 100644
--- a/docker/ubuntu-16.04/Dockerfile
+++ b/docker/ubuntu-16.04/Dockerfile
@@ -50,7 +50,7 @@ RUN eval `${opam_env}` && \
         snappy \
         bisect \
         lwt_ssl \
-        lwt.3.1.0 \
+        lwt.3.0.0 \
         camltc.0.9.4 \
         ocplib-endian.1.0 \
         quickcheck.1.0.2 \

From 37c95422e3923aefc97e297918edec0363f99365 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Thu, 24 Aug 2017 15:34:37 +0200
Subject: [PATCH 306/373] make_release: 1.9.22

---
 META                      | 2 +-
 debian/changelog          | 5 +++++
 redhat/SPECS/arakoon.spec | 4 +++-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/META b/META
index d3aa566b..c7f6740c 100644
--- a/META
+++ b/META
@@ -1,7 +1,7 @@
 # -*- conf -*-
 
 description = "Arakoon client"
-version = "1.9.21"
+version = "1.9.22"
 exists_if = "arakoon_client.cma,arakoon_client.cmxa,arakoon_client.cmxs"
 requires = "threads lwt lwt.unix str oUnit quickcheck lwt_ssl ocplib-endian redis.lwt uri"
 archive(byte) = "arakoon_client.cma tcp_keepalive_sockopt_stubs.o"
diff --git a/debian/changelog b/debian/changelog
index b1c7a2a5..e07a37f7 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,8 @@
+arakoon (1.9.22) unstable; urgency=low
+  * new upstream version
+
+ -- Romain Slootmaekers <romain.slootmaekers@openvstorage.com>  Thu, 24 Aug 2017 15:34:37 +0100
+
 arakoon (1.9.21) unstable; urgency=low
   * new upstream version
 
diff --git a/redhat/SPECS/arakoon.spec b/redhat/SPECS/arakoon.spec
index 45cb719b..35591737 100644
--- a/redhat/SPECS/arakoon.spec
+++ b/redhat/SPECS/arakoon.spec
@@ -2,7 +2,7 @@
 
 Summary: Arakoon
 Name: arakoon
-Version: 1.9.21
+Version: 1.9.22
 Release: 3%{?dist}
 License: Apache 2.0
 Requires: libev >= 4
@@ -33,6 +33,8 @@ cp arakoon.native %{buildroot}%{_bindir}/arakoon
 %{_bindir}/arakoon
 
 %changelog
+* Thu Aug 24 2017 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.22
+- Create arakoon 1.9.22 RPM package
 * Tue Aug 22 2017 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.21
 - Create arakoon 1.9.21 RPM package
 * Wed Aug 09 2017 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.20

From 5f080ac315a73c1424c49003e7891e863398827e Mon Sep 17 00:00:00 2001
From: Yuriy Vostrikov <yuriy.vostrikov@openvstorage.com>
Date: Fri, 8 Dec 2017 13:56:12 +0300
Subject: [PATCH 307/373] Upgrade toolchain, opamify build

---
 META                           |  2 +-
 _tags                          |  2 +-
 docker/ubuntu-14.04/Dockerfile | 25 +++++---------------
 docker/ubuntu-16.04/Dockerfile | 26 +++++----------------
 opam                           | 42 ++++++++++++++++++++++++++++++++++
 src/tools/arakoon_redis.ml     |  2 +-
 src/tools/crash_logger.ml      |  2 +-
 7 files changed, 58 insertions(+), 43 deletions(-)
 create mode 100644 opam

diff --git a/META b/META
index c7f6740c..d8e81705 100644
--- a/META
+++ b/META
@@ -3,7 +3,7 @@
 description = "Arakoon client"
 version = "1.9.22"
 exists_if = "arakoon_client.cma,arakoon_client.cmxa,arakoon_client.cmxs"
-requires = "threads lwt lwt.unix str oUnit quickcheck lwt_ssl ocplib-endian redis.lwt uri"
+requires = "threads lwt lwt.unix str oUnit quickcheck lwt_ssl ocplib-endian redis-lwt uri"
 archive(byte) = "arakoon_client.cma tcp_keepalive_sockopt_stubs.o"
 archive(native) = "arakoon_client.cmxa tcp_keepalive_sockopt_stubs.o"
 
diff --git a/_tags b/_tags
index e00b0c4f..e5d23177 100644
--- a/_tags
+++ b/_tags
@@ -16,7 +16,7 @@ true: package(snappy)
 true: package(ocplib-endian)
 true: package(uri)
 true: package(redis)
-true: package(redis.lwt)
+true: package(redis-lwt)
 true: package(core)
 "src":include
 "src/lib":include
diff --git a/docker/ubuntu-14.04/Dockerfile b/docker/ubuntu-14.04/Dockerfile
index da373476..c66e31a2 100644
--- a/docker/ubuntu-14.04/Dockerfile
+++ b/docker/ubuntu-14.04/Dockerfile
@@ -34,33 +34,20 @@ RUN apt-get -y update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
 
 RUN wget https://raw.github.com/ocaml/opam/master/shell/opam_installer.sh
 
-env ocaml_version=4.04.2
+env ocaml_version=4.06.0+default-unsafe-string
 RUN sh ./opam_installer.sh /usr/local/bin ${ocaml_version}
 
 ENV opam_root=/home/jenkins/OPAM
 ENV opam_env="opam config env --root=${opam_root}"
 RUN opam init --root=${opam_root} --comp ${ocaml_version}
 
+ADD https://raw.githubusercontent.com/delamonpansie/ovs_opam/master/switch opam.switch
+
 RUN eval `${opam_env}` && \
+    opam repo add ovs git://github.com/delamonpansie/ovs_opam && \
     opam update -v && \
-    opam install -y \
-        oasis.0.4.10 \
-        ocamlfind \
-        ssl.0.5.3 \
-        camlbz2 \
-        snappy \
-        bisect \
-        lwt_ssl \
-        lwt.3.0.0 \
-        camltc.0.9.4 \
-        ocplib-endian.1.0 \
-        quickcheck.1.0.2 \
-        conf-libev \
-        ppx_deriving.4.1 \
-        base.v0.9.3 \
-        core.v0.9.1 \
-        redis.0.3.3 \
-        uri.1.9.4
+    grep -Ev '^(arakoon|kinetic-client|orocksdb|ordma)' opam.switch | \
+    (eval `${opam_env}` && opam switch import - -y)
 
 #
 RUN wget https://www.python.org/ftp/python/2.7.12/Python-2.7.12.tar.xz
diff --git a/docker/ubuntu-16.04/Dockerfile b/docker/ubuntu-16.04/Dockerfile
index 4f979d8c..49a26569 100644
--- a/docker/ubuntu-16.04/Dockerfile
+++ b/docker/ubuntu-16.04/Dockerfile
@@ -33,34 +33,20 @@ RUN apt-get -y update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
 
 RUN wget https://raw.github.com/ocaml/opam/master/shell/opam_installer.sh
 
-env ocaml_version=4.04.2
+env ocaml_version=4.06.0+default-unsafe-string
 RUN sh ./opam_installer.sh /usr/local/bin ${ocaml_version}
 
 ENV opam_root=/home/jenkins/OPAM
 ENV opam_env="opam config env --root=${opam_root}"
 RUN opam init --root=${opam_root} --comp ${ocaml_version}
 
+ADD https://raw.githubusercontent.com/delamonpansie/ovs_opam/master/switch opam.switch
+
 RUN eval `${opam_env}` && \
+    opam repo add ovs git://github.com/delamonpansie/ovs_opam && \
     opam update -v && \
-    opam install -y \
-        oasis.0.4.10 \
-        ocamlfind \
-        ssl.0.5.3 \
-        camlbz2 \
-        snappy \
-        bisect \
-        lwt_ssl \
-        lwt.3.0.0 \
-        camltc.0.9.4 \
-        ocplib-endian.1.0 \
-        quickcheck.1.0.2 \
-        conf-libev \
-        ppx_deriving.4.1 \
-        base.v0.9.3 \
-        core.v0.9.1 \
-        redis.0.3.3 \
-        uri.1.9.4
-
+    grep -Ev '^(arakoon|kinetic-client|orocksdb|ordma)' opam.switch | \
+    (eval `${opam_env}` && opam switch import - -y)
 
 RUN pip install fabric junit-xml nose simplejson python-etcd
 
diff --git a/opam b/opam
new file mode 100644
index 00000000..03544afc
--- /dev/null
+++ b/opam
@@ -0,0 +1,42 @@
+opam-version: "1.2"
+name: "arakoon"
+maintainer: "Romain Slootmaekers <romain.slootmaekers@openvstorage.com>"
+authors: ["Romain Slootmaekers <romain.slootmaekers@openvstorage.com>"
+          "Jan Doms <jan.doms@openvstorage.com>"]
+homepage: "https://github.com/openvstorage/arakoon"
+bug-reports: "https://github.com/openvstorage/arakoon/issues"
+license: "Apache 2.0"
+dev-repo: "https://github.com/openvstorage/arakoon.git"
+build: [ [make] ]
+
+install: [make "install" "OCAML_LIBDIR=%{prefix}%/lib" "PREFIX=%{prefix}%"]
+
+remove:  [
+  [make "uninstall_client" "OCAML_LIBDIR=%{prefix}%/lib" "PREFIX=%{prefix}%"]
+  ["rm" "-rf" "%{bin}%/arakoon"]
+]
+
+depends: [
+  "ocamlfind" {build}
+  "bisect" {test & >= "1.3"}
+  "camltc" {>= "0.9.4"}
+  "camlbz2" {>= "0.6.0"}
+  "conf-libev" {>= "4-11"}
+  "lwt" {= "3.0.0"}
+  "lwt_ssl" {>= "1.0.1"}
+  "ocplib-endian" {>= "1.0"}
+  "ssl" {>= "0.5.3"}
+  "snappy" {>= "0.1.0"}
+  "quickcheck" {>= "1.0.2"}
+  "core" {>="0.9.0"}
+  "ocamlbuild" {build}
+  "redis-lwt"
+  "uri" {>= "1.9.2"}
+]
+available: [ocaml-version >= "4.03.0" & opam-version >= "1.2" & os != "darwin" ]
+
+
+depexts:[
+  [[ "ubuntu"] ["help2man"] ]
+  [[ "debian"] ["help2man"] ]
+]
diff --git a/src/tools/arakoon_redis.ml b/src/tools/arakoon_redis.ml
index 9abb6a88..7551f77c 100644
--- a/src/tools/arakoon_redis.ml
+++ b/src/tools/arakoon_redis.ml
@@ -38,7 +38,7 @@ let make_redis_logger ~host ~port ~key =
          let rec inner client =
            (* TODO push multiple items in one go? *)
            Lwt_buffer.take buffer >>= fun logline ->
-           R.rpush client key logline >>= fun _list_length ->
+           R.rpush client key [logline] >>= fun _list_length ->
            inner client
          in
          R.with_connection
diff --git a/src/tools/crash_logger.ml b/src/tools/crash_logger.ml
index 8f29ba1a..1f652d46 100644
--- a/src/tools/crash_logger.ml
+++ b/src/tools/crash_logger.ml
@@ -123,7 +123,7 @@ let dump_crash_log crash_log_sink =
                         acc () >>= fun () ->
                         Message.traverse_
                           (fun m exn ->
-                           Re.rpush client key (format_message e m exn) >>= fun _list_length ->
+                           Re.rpush client key [format_message e m exn] >>= fun _list_length ->
                            Lwt.return_unit)
                           e.Entry.payload)
        ~acc:(fun () -> Lwt.return_unit)

From 8c5ec20b9290460562841b74a631aee067e6f02c Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Fri, 29 Dec 2017 13:25:49 +0100
Subject: [PATCH 308/373] make_release: 1.9.23

---
 META                      | 2 +-
 debian/changelog          | 5 +++++
 redhat/SPECS/arakoon.spec | 4 +++-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/META b/META
index d8e81705..78c3bfde 100644
--- a/META
+++ b/META
@@ -1,7 +1,7 @@
 # -*- conf -*-
 
 description = "Arakoon client"
-version = "1.9.22"
+version = "1.9.23"
 exists_if = "arakoon_client.cma,arakoon_client.cmxa,arakoon_client.cmxs"
 requires = "threads lwt lwt.unix str oUnit quickcheck lwt_ssl ocplib-endian redis-lwt uri"
 archive(byte) = "arakoon_client.cma tcp_keepalive_sockopt_stubs.o"
diff --git a/debian/changelog b/debian/changelog
index e07a37f7..63b042b8 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,8 @@
+arakoon (1.9.23) unstable; urgency=low
+  * new upstream version
+
+ -- Romain Slootmaekers <romain.slootmaekers@openvstorage.com>  Fri, 29 Dec 2017 13:25:49 +0100
+
 arakoon (1.9.22) unstable; urgency=low
   * new upstream version
 
diff --git a/redhat/SPECS/arakoon.spec b/redhat/SPECS/arakoon.spec
index 35591737..c5f90b87 100644
--- a/redhat/SPECS/arakoon.spec
+++ b/redhat/SPECS/arakoon.spec
@@ -2,7 +2,7 @@
 
 Summary: Arakoon
 Name: arakoon
-Version: 1.9.22
+Version: 1.9.23
 Release: 3%{?dist}
 License: Apache 2.0
 Requires: libev >= 4
@@ -33,6 +33,8 @@ cp arakoon.native %{buildroot}%{_bindir}/arakoon
 %{_bindir}/arakoon
 
 %changelog
+* Fri Dec 29 2017 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.23
+- Create arakoon 1.9.23 RPM package
 * Thu Aug 24 2017 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.22
 - Create arakoon 1.9.22 RPM package
 * Tue Aug 22 2017 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.21

From f2d36d93851c54d83961413e6276252c3761b598 Mon Sep 17 00:00:00 2001
From: Yuriy Vostrikov <yuriy.vostrikov@openvstorage.com>
Date: Tue, 16 Jan 2018 12:23:08 +0300
Subject: [PATCH 309/373] build arakoon_version.ml with -opaque

---
 _tags | 1 +
 1 file changed, 1 insertion(+)

diff --git a/_tags b/_tags
index e5d23177..0faef7da 100644
--- a/_tags
+++ b/_tags
@@ -42,3 +42,4 @@ true: package(core)
 <src/**/*.ml> : camlp4of, use_log_macro, maybe_use_bisect, unsafe_string
 "rpmbuild": -traverse
 "arakoon_docker_rpm": -traverse
+"arakoon_version.ml": opaque

From cee0fed3af45ab15f1aa51603b455ccd39847403 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Tue, 16 Jan 2018 15:15:03 +0100
Subject: [PATCH 310/373] needed opam switch inside the repo

---
 .../packages/bisect/bisect.1.3+ovs1/descr     |   6 +
 .../bisect.1.3+ovs1/files/bisect.install      |   4 +
 .../files/install-thread.patch                |  28 +++++
 .../bisect/bisect.1.3+ovs1/files/opam.patch   |  49 ++++++++
 .../packages/bisect/bisect.1.3+ovs1/opam      |  31 +++++
 .../packages/bisect/bisect.1.3+ovs1/url       |   2 +
 .../packages/optcomp/optcomp.1.6+ovs1/descr   |   1 +
 .../packages/optcomp/optcomp.1.6+ovs1/opam    |  23 ++++
 .../packages/optcomp/optcomp.1.6+ovs1/url     |   2 +
 .../redis-lwt/redis-lwt.0.3.5+ovs1/descr      |   1 +
 .../redis-lwt/redis-lwt.0.3.5+ovs1/opam       |  16 +++
 .../redis-lwt/redis-lwt.0.3.5+ovs1/url        |   2 +
 .../packages/redis/redis.0.3.5+ovs1/descr     |   1 +
 .../packages/redis/redis.0.3.5+ovs1/opam      |  15 +++
 .../packages/redis/redis.0.3.5+ovs1/url       |   2 +
 docker/opam.switch                            | 112 ++++++++++++++++++
 docker/run.sh                                 |  15 +--
 docker/ubuntu-16.04/Dockerfile                |  39 +++---
 18 files changed, 321 insertions(+), 28 deletions(-)
 create mode 100644 docker/opam-repository/packages/bisect/bisect.1.3+ovs1/descr
 create mode 100644 docker/opam-repository/packages/bisect/bisect.1.3+ovs1/files/bisect.install
 create mode 100644 docker/opam-repository/packages/bisect/bisect.1.3+ovs1/files/install-thread.patch
 create mode 100644 docker/opam-repository/packages/bisect/bisect.1.3+ovs1/files/opam.patch
 create mode 100644 docker/opam-repository/packages/bisect/bisect.1.3+ovs1/opam
 create mode 100644 docker/opam-repository/packages/bisect/bisect.1.3+ovs1/url
 create mode 100644 docker/opam-repository/packages/optcomp/optcomp.1.6+ovs1/descr
 create mode 100644 docker/opam-repository/packages/optcomp/optcomp.1.6+ovs1/opam
 create mode 100644 docker/opam-repository/packages/optcomp/optcomp.1.6+ovs1/url
 create mode 100644 docker/opam-repository/packages/redis-lwt/redis-lwt.0.3.5+ovs1/descr
 create mode 100644 docker/opam-repository/packages/redis-lwt/redis-lwt.0.3.5+ovs1/opam
 create mode 100644 docker/opam-repository/packages/redis-lwt/redis-lwt.0.3.5+ovs1/url
 create mode 100644 docker/opam-repository/packages/redis/redis.0.3.5+ovs1/descr
 create mode 100644 docker/opam-repository/packages/redis/redis.0.3.5+ovs1/opam
 create mode 100644 docker/opam-repository/packages/redis/redis.0.3.5+ovs1/url
 create mode 100644 docker/opam.switch

diff --git a/docker/opam-repository/packages/bisect/bisect.1.3+ovs1/descr b/docker/opam-repository/packages/bisect/bisect.1.3+ovs1/descr
new file mode 100644
index 00000000..410f1ba3
--- /dev/null
+++ b/docker/opam-repository/packages/bisect/bisect.1.3+ovs1/descr
@@ -0,0 +1,6 @@
+Code coverage tool for the OCaml language
+Bisect is a code coverage tool for the OCaml language. It is a
+camlp4-based tool that allows to instrument your application before
+running tests. After application execution, it is possible to generate
+a report in HTML format that is the replica of the application source
+code annotated with code coverage information.
diff --git a/docker/opam-repository/packages/bisect/bisect.1.3+ovs1/files/bisect.install b/docker/opam-repository/packages/bisect/bisect.1.3+ovs1/files/bisect.install
new file mode 100644
index 00000000..a47e8c04
--- /dev/null
+++ b/docker/opam-repository/packages/bisect/bisect.1.3+ovs1/files/bisect.install
@@ -0,0 +1,4 @@
+bin: [
+  "_build/src/report/report.native" {"bisect-report.opt"}
+  "_build/src/report/report.byte" {"bisect-report"}
+]
diff --git a/docker/opam-repository/packages/bisect/bisect.1.3+ovs1/files/install-thread.patch b/docker/opam-repository/packages/bisect/bisect.1.3+ovs1/files/install-thread.patch
new file mode 100644
index 00000000..97fe9473
--- /dev/null
+++ b/docker/opam-repository/packages/bisect/bisect.1.3+ovs1/files/install-thread.patch
@@ -0,0 +1,28 @@
+diff --git a/Makefile b/Makefile
+index 4a8ce17..3a184a1 100644
+--- a/Makefile
++++ b/Makefile
+@@ -89,10 +89,11 @@ install: FORCE
+ 	  $(PATH_OCAMLFIND) query $(PROJECT_NAME) && $(PATH_OCAMLFIND) remove $(PROJECT_NAME) || true; \
+ 	  $(PATH_OCAMLFIND) install $(PROJECT_NAME) META -optional \
+ 	    $(PATH_BUILD)/$(PROJECT_NAME)_pp.cmo \
+-	    $(PATH_BUILD)/src/$(PROJECT_NAME)Thread.cm* \
+-	    $(PATH_BUILD)/src/$(PROJECT_NAME)Thread.o \
+-	    $(PATH_BUILD)/src/$(PROJECT_NAME)Thread.jo \
++	    $(PATH_BUILD)/src/threads/$(PROJECT_NAME)Thread.cm* \
++	    $(PATH_BUILD)/src/threads/$(PROJECT_NAME)Thread.o \
++	    $(PATH_BUILD)/src/threads/$(PROJECT_NAME)Thread.jo \
+ 	    $(PATH_BUILD)/$(PROJECT_NAME).a \
++	    $(PATH_BUILD)/$(PROJECT_NAME).o \
+ 	    $(PATH_BUILD)/$(PROJECT_NAME).cma \
+ 	    $(PATH_BUILD)/$(PROJECT_NAME).cmi \
+ 	    $(PATH_BUILD)/$(PROJECT_NAME).cmo \
+@@ -106,7 +107,7 @@ install: FORCE
+ 	    cp $(PATH_BUILD)/$(PROJECT_NAME)_pp.cmo $(PATH_INSTALL); \
+ 	  fi; \
+ 	  for ext in cmi cmo cmx o cmj jo; do \
+-	    test -f $(PATH_BUILD)/src/$(PROJECT_NAME)Thread.$$ext && cp $(PATH_BUILD)/src/$(PROJECT_NAME)Thread.$$ext $(PATH_INSTALL) || true; \
++	    test -f $(PATH_BUILD)/src/threads/$(PROJECT_NAME)Thread.$$ext && cp $(PATH_BUILD)/src/threads/$(PROJECT_NAME)Thread.$$ext $(PATH_INSTALL) || true; \
+ 	  done; \
+ 	  for ext in a cmi cmo cmx cma cmxa cmja ja; do \
+ 	    test -f $(PATH_BUILD)/$(PROJECT_NAME).$$ext && cp $(PATH_BUILD)/$(PROJECT_NAME).$$ext $(PATH_INSTALL) || true; \
diff --git a/docker/opam-repository/packages/bisect/bisect.1.3+ovs1/files/opam.patch b/docker/opam-repository/packages/bisect/bisect.1.3+ovs1/files/opam.patch
new file mode 100644
index 00000000..c2894fbe
--- /dev/null
+++ b/docker/opam-repository/packages/bisect/bisect.1.3+ovs1/files/opam.patch
@@ -0,0 +1,49 @@
+diff --git a/Makefile b/Makefile
+index b86448a..81e29a0 100644
+--- a/Makefile
++++ b/Makefile
+@@ -31,7 +31,7 @@ PATH_INSTALL=$(PATH_OCAML_PREFIX)/lib/ocaml/bisect
+ # DEFINITIONS
+ 
+ PROJECT_NAME=bisect
+-OCAMLBUILD=$(PATH_OCAML_PREFIX)/bin/ocamlbuild
++OCAMLBUILD=ocamlbuild
+ OCAMLBUILD_ENV=WARNINGS=$(WARNINGS) PATH_OCAML_PREFIX=$(PATH_OCAML_PREFIX)
+ OCAMLBUILD_FLAGS=-classic-display -no-links
+ MODULES_ODOCL=$(PROJECT_NAME).odocl
+@@ -84,7 +84,7 @@ install: FORCE
+ 	if [ "$(PPX)" = "TRUE" ]; then \
+ 	  cp $(PATH_BUILD)/src/syntax/bisect_ppx.byte $(PATH_OCAML_PREFIX)/bin; \
+ 	fi; \
+-	(test -x $(PATH_OCAML_PREFIX)/bin/ocamlopt && cp $(PATH_BUILD)/src/report/report.native $(PATH_OCAML_PREFIX)/bin/bisect-report.opt || true); \
++	(which ocamlopt && cp $(PATH_BUILD)/src/report/report.native $(PATH_OCAML_PREFIX)/bin/bisect-report.opt || true); \
+ 	if [ -x "$(PATH_OCAMLFIND)" ]; then \
+ 	  $(PATH_OCAMLFIND) query $(PROJECT_NAME) && $(PATH_OCAMLFIND) remove $(PROJECT_NAME) || true; \
+ 	  $(PATH_OCAMLFIND) install $(PROJECT_NAME) META -optional \
+@@ -115,7 +115,7 @@ install: FORCE
+ 
+ generate: FORCE
+ 	echo '$(PROJECT_NAME).cma' > $(PROJECT_NAME).itarget
+-	(test -x $(PATH_OCAML_PREFIX)/bin/ocamlopt && echo '$(PROJECT_NAME).cmxa' >> $(PROJECT_NAME).itarget) || true
+-	(test -x $(PATH_OCAML_PREFIX)/bin/ocamljava && echo '$(PROJECT_NAME).cmja' >> $(PROJECT_NAME).itarget) || true
++	(which ocamlopt && echo '$(PROJECT_NAME).cmxa' >> $(PROJECT_NAME).itarget) || true
++	(which ocamljava && echo '$(PROJECT_NAME).cmja' >> $(PROJECT_NAME).itarget) || true
+ 
+ FORCE:
+diff --git a/myocamlbuild.ml b/myocamlbuild.ml
+index 8aa25fd..2ef7428 100644
+--- a/myocamlbuild.ml
++++ b/myocamlbuild.ml
+@@ -68,11 +68,7 @@ let () =
+     cp src dst in
+   dispatch begin function
+     | After_rules ->
+-        let camlp4of =
+-          try
+-            let path_bin = Filename.concat (Sys.getenv "PATH_OCAML_PREFIX") "bin" in
+-            Filename.concat path_bin "camlp4of"
+-          with _ -> "camlp4of" in
++        let camlp4of = "camlp4of" in
+         flag ["ocaml"; "compile"; "pp_camlp4of"] (S[A"-pp"; A camlp4of]);
+         flag ["ocaml"; "pp:dep"; "pp_camlp4of"] (S[A camlp4of]);
+         flag ["ocaml"; "compile"; "use_compiler_libs"] (S[A"-I"; A"+compiler-libs"]);
diff --git a/docker/opam-repository/packages/bisect/bisect.1.3+ovs1/opam b/docker/opam-repository/packages/bisect/bisect.1.3+ovs1/opam
new file mode 100644
index 00000000..2a204f94
--- /dev/null
+++ b/docker/opam-repository/packages/bisect/bisect.1.3+ovs1/opam
@@ -0,0 +1,31 @@
+opam-version: "1.2"
+maintainer: "https://github.com/ocaml/opam-repository/issues"
+authors: ["Xavier Clerc"]
+homepage: "http://bisect.x9c.fr/"
+license: "GPL v3"
+build: [
+  ["sh" "configure" "-ocaml-prefix" prefix "-ocamlfind" "%{bin}%/ocamlfind"]
+    {ocaml-version != "4.00.1"}
+  [
+    "sh"
+    "configure"
+    "-ocaml-prefix"
+    prefix
+    "-ocamlfind"
+    "%{bin}%/ocamlfind"
+    "-ppx"
+  ] {ocaml-version = "4.00.1"}
+  [make "all"]
+]
+remove: [["ocamlfind" "remove" "bisect"]]
+depends: [
+  "ocamlfind"
+  "camlp4"
+  "ocamlbuild" {build}
+]
+patches: [
+  "install-thread.patch"
+  "opam.patch"
+]
+install: [make "install"]
+available: [ocaml-version <= "4.06.0"]
diff --git a/docker/opam-repository/packages/bisect/bisect.1.3+ovs1/url b/docker/opam-repository/packages/bisect/bisect.1.3+ovs1/url
new file mode 100644
index 00000000..b9702a82
--- /dev/null
+++ b/docker/opam-repository/packages/bisect/bisect.1.3+ovs1/url
@@ -0,0 +1,2 @@
+archive: "http://bisect.x9c.fr/distrib/bisect-1.3.tar.gz"
+checksum: "8481342a3d9f17d5502bc84c04c206e3"
diff --git a/docker/opam-repository/packages/optcomp/optcomp.1.6+ovs1/descr b/docker/opam-repository/packages/optcomp/optcomp.1.6+ovs1/descr
new file mode 100644
index 00000000..8d192872
--- /dev/null
+++ b/docker/opam-repository/packages/optcomp/optcomp.1.6+ovs1/descr
@@ -0,0 +1 @@
+Optional compilation with cpp-like directives
diff --git a/docker/opam-repository/packages/optcomp/optcomp.1.6+ovs1/opam b/docker/opam-repository/packages/optcomp/optcomp.1.6+ovs1/opam
new file mode 100644
index 00000000..a68c5acd
--- /dev/null
+++ b/docker/opam-repository/packages/optcomp/optcomp.1.6+ovs1/opam
@@ -0,0 +1,23 @@
+opam-version: "1.2"
+maintainer: "jeremie@dimino.org"
+authors: ["Jérémie Dimino"]
+homepage: "https://github.com/diml/optcomp"
+license: "BSD3"
+build: [
+  ["./configure" "--prefix" prefix]
+  [make]
+]
+build-doc: [["ocaml" "setup.ml" "-doc"]]
+remove: [
+  ["ocamlfind" "remove" "optcomp"]
+  ["rm" "%{bin}%/optcomp-o"]
+  ["rm" "%{bin}%/optcomp-r"]
+]
+depends: [
+  "ocamlfind"
+  "camlp4"
+  "ocamlbuild" {build}
+]
+dev-repo: "git://github.com/diml/optcomp"
+available: [ocaml-version >= "3.12" & ocaml-version <= "4.06.0"]
+install: [make "install"]
diff --git a/docker/opam-repository/packages/optcomp/optcomp.1.6+ovs1/url b/docker/opam-repository/packages/optcomp/optcomp.1.6+ovs1/url
new file mode 100644
index 00000000..98256ace
--- /dev/null
+++ b/docker/opam-repository/packages/optcomp/optcomp.1.6+ovs1/url
@@ -0,0 +1,2 @@
+archive: "https://github.com/diml/optcomp/archive/1.6.tar.gz"
+checksum: "d3587244dba1b8b10f24d0b60a8c700d"
diff --git a/docker/opam-repository/packages/redis-lwt/redis-lwt.0.3.5+ovs1/descr b/docker/opam-repository/packages/redis-lwt/redis-lwt.0.3.5+ovs1/descr
new file mode 100644
index 00000000..0d0f883d
--- /dev/null
+++ b/docker/opam-repository/packages/redis-lwt/redis-lwt.0.3.5+ovs1/descr
@@ -0,0 +1 @@
+Lwt-based client for Redis
\ No newline at end of file
diff --git a/docker/opam-repository/packages/redis-lwt/redis-lwt.0.3.5+ovs1/opam b/docker/opam-repository/packages/redis-lwt/redis-lwt.0.3.5+ovs1/opam
new file mode 100644
index 00000000..0e80c8c1
--- /dev/null
+++ b/docker/opam-repository/packages/redis-lwt/redis-lwt.0.3.5+ovs1/opam
@@ -0,0 +1,16 @@
+opam-version: "1.2"
+maintainer: "aluuu@husa.su"
+authors: ["Mike Wells" "David Höppner" "Aleksandr Dinu"]
+homepage: "https://github.com/0xffea/ocaml-redis"
+bug-reports: "https://github.com/0xffea/ocaml-redis/issues"
+license: "BSD3"
+dev-repo: "https://github.com/0xffea/ocaml-redis.git"
+build: ["jbuilder" "build" "-p" name "-j" jobs]
+build-test: ["jbuilder" "runtest" "-j" "1"]
+depends: [
+  "jbuilder" {build}
+  "redis" {>= "0.3.5"}
+  "base-unix"
+  "lwt"
+]
+available: [ocaml-version >= "4.02.3"]
diff --git a/docker/opam-repository/packages/redis-lwt/redis-lwt.0.3.5+ovs1/url b/docker/opam-repository/packages/redis-lwt/redis-lwt.0.3.5+ovs1/url
new file mode 100644
index 00000000..987b2999
--- /dev/null
+++ b/docker/opam-repository/packages/redis-lwt/redis-lwt.0.3.5+ovs1/url
@@ -0,0 +1,2 @@
+http: "https://github.com/0xffea/ocaml-redis/archive/0.3.5.tar.gz"
+checksum: "462cbbba5e8d5743457b7c345b2d97a2"
diff --git a/docker/opam-repository/packages/redis/redis.0.3.5+ovs1/descr b/docker/opam-repository/packages/redis/redis.0.3.5+ovs1/descr
new file mode 100644
index 00000000..3c55f7f6
--- /dev/null
+++ b/docker/opam-repository/packages/redis/redis.0.3.5+ovs1/descr
@@ -0,0 +1 @@
+Bindings for the key-value cache and store redis.
diff --git a/docker/opam-repository/packages/redis/redis.0.3.5+ovs1/opam b/docker/opam-repository/packages/redis/redis.0.3.5+ovs1/opam
new file mode 100644
index 00000000..7c9538b0
--- /dev/null
+++ b/docker/opam-repository/packages/redis/redis.0.3.5+ovs1/opam
@@ -0,0 +1,15 @@
+opam-version: "1.2"
+maintainer: "aluuu@husa.su"
+authors: ["Mike Wells" "David Höppner" "Aleksandr Dinu"]
+homepage: "https://github.com/0xffea/ocaml-redis"
+bug-reports: "https://github.com/0xffea/ocaml-redis/issues"
+license: "BSD3"
+dev-repo: "https://github.com/0xffea/ocaml-redis.git"
+build: ["jbuilder" "build" "-p" name "-j" jobs]
+depends: [
+  "jbuilder" {build}
+  "base-bytes"
+  "uuidm"
+  "re"
+]
+available: [ocaml-version >= "4.02.3"]
diff --git a/docker/opam-repository/packages/redis/redis.0.3.5+ovs1/url b/docker/opam-repository/packages/redis/redis.0.3.5+ovs1/url
new file mode 100644
index 00000000..987b2999
--- /dev/null
+++ b/docker/opam-repository/packages/redis/redis.0.3.5+ovs1/url
@@ -0,0 +1,2 @@
+http: "https://github.com/0xffea/ocaml-redis/archive/0.3.5.tar.gz"
+checksum: "462cbbba5e8d5743457b7c345b2d97a2"
diff --git a/docker/opam.switch b/docker/opam.switch
new file mode 100644
index 00000000..5bfdfb6c
--- /dev/null
+++ b/docker/opam.switch
@@ -0,0 +1,112 @@
+base	v0.9.4	installed
+base-bigarray	base	root
+base-bytes	base	installed
+base-threads	base	root
+base-unix	base	root
+base64	2.2.0	installed
+bin_prot	v0.9.2	installed
+biniou	1.2.0	installed
+bisect	1.3+ovs1	installed
+camlbz2	0.6.0	installed
+camlp4	4.06+1	installed
+camltc	0.9.4	installed
+cmdliner	1.0.2	installed
+conf-gmp	1	installed
+conf-gmp-powm-sec	1	installed
+conf-libev	4-11	installed
+conf-m4	1	installed
+conf-openssl	1	installed
+conf-perl	1	installed
+conf-pkg-config	1.0	installed
+conf-which	1	installed
+conf-zlib	1	installed
+configurator	v0.9.1	installed
+core	v0.9.2	installed
+core_kernel	v0.9.1	installed
+cppo	1.6.0	installed
+cppo_ocamlbuild	1.6.0	installed
+cryptokit	1.12	installed
+ctypes	0.13.1	installed
+ctypes-foreign	0.4.0	installed
+depext	1.0.5	installed
+easy-format	1.3.0	installed
+ezxmlm	1.0.1	installed
+fieldslib	v0.9.0	installed
+integers	0.2.2	installed
+jane-street-headers	v0.9.0	installed
+jbuilder	1.0+beta16	installed
+kinetic-client	0.0.6+ovs1	installed
+lwt	3.0.0	root
+lwt_ssl	1.1.1	installed
+mirage-no-xen	1	installed
+num	1.1	installed
+oasis	0.4.10	installed
+ocaml-compiler-libs	v0.9.0	installed
+ocaml-migrate-parsetree	1.0.7	installed
+ocamlbuild	0.12.0	installed
+ocamlfind	1.7.3	installed
+ocamlify	0.0.1	installed
+ocamlmod	0.0.9	installed
+ocplib-endian	1.0	installed
+octavius	1.2.0	installed
+omake	0.10.3	installed
+optcomp	1.6+ovs1	installed
+ounit	2.0.6	installed
+piqi	0.7.5	installed
+piqilib	0.6.13	installed
+ppx_assert	v0.9.0	installed
+ppx_ast	v0.9.2	installed
+ppx_base	v0.9.0	installed
+ppx_bench	v0.9.1	installed
+ppx_bin_prot	v0.9.0	installed
+ppx_compare	v0.9.0	installed
+ppx_core	v0.9.3	installed
+ppx_custom_printf	v0.9.0	installed
+ppx_derivers	1.0	installed
+ppx_deriving	4.2.1	installed
+ppx_deriving_yojson	3.1	installed
+ppx_driver	v0.9.2	installed
+ppx_enumerate	v0.9.0	installed
+ppx_expect	v0.9.0	installed
+ppx_fail	v0.9.0	installed
+ppx_fields_conv	v0.9.0	installed
+ppx_hash	v0.9.0	installed
+ppx_here	v0.9.1	installed
+ppx_inline_test	v0.9.2	installed
+ppx_jane	v0.9.0	installed
+ppx_js_style	v0.9.0	installed
+ppx_let	v0.9.0	installed
+ppx_metaquot	v0.9.0	installed
+ppx_optcomp	v0.9.0	installed
+ppx_optional	v0.9.0	installed
+ppx_pipebang	v0.9.0	installed
+ppx_sexp_conv	v0.9.0	installed
+ppx_sexp_message	v0.9.0	installed
+ppx_sexp_value	v0.9.0	installed
+ppx_tools	5.1+4.06.0	installed
+ppx_tools_versioned	5.0.1	installed
+ppx_traverse	v0.9.0	installed
+ppx_traverse_builtins	v0.9.0	installed
+ppx_type_conv	v0.9.1	installed
+ppx_typerep_conv	v0.9.0	installed
+ppx_variants_conv	v0.9.0	installed
+quickcheck	1.0.2	installed
+re	1.7.1	installed
+redis	0.3.5+ovs1	installed
+redis-lwt	0.3.5+ovs1	root
+result	1.2	installed
+sexplib	v0.9.3	installed
+snappy	0.1.0	installed
+spawn	v0.9.0	installed
+ssl	0.5.5	installed
+stdio	v0.9.1	installed
+stringext	1.5.0	installed
+topkg	0.9.1	installed
+typerep	v0.9.0	installed
+ulex	1.2	installed
+uri	1.9.5	installed
+uuidm	0.9.6	installed
+variantslib	v0.9.0	installed
+xmlm	1.3.0	installed
+yojson	1.4.0	installed
+zarith	1.7	installed
diff --git a/docker/run.sh b/docker/run.sh
index cd2bae71..6a57417e 100755
--- a/docker/run.sh
+++ b/docker/run.sh
@@ -3,22 +3,19 @@
 IMAGE=$1
 shift
 
-docker build --rm=true --tag=arakoon_$IMAGE ./docker/$IMAGE
+docker build --rm=true \
+       --tag=arakoon_$IMAGE \
+       -f ./docker/$IMAGE/Dockerfile ./docker/ \
+
 
 if [ -t 1 ];
 then TTY="-t";
-else
-    # this path is taken on jenkins, clean previous builds first
-    TTY="";
-    docker run -i $TTY --privileged=true -e UID=${UID} \
-           -v ${PWD}:/home/jenkins/arakoon \
-           -w /home/jenkins/arakoon arakoon_$IMAGE \
-           bash clean
+else TTY="";
 fi
 
-
 docker run -i $TTY --privileged=true -e UID=${UID} \
        --env ARAKOON_PYTHON_CLIENT \
+       --env TRAVIS \
        -v ${PWD}:/home/jenkins/arakoon \
        arakoon_$IMAGE \
        bash $@
diff --git a/docker/ubuntu-16.04/Dockerfile b/docker/ubuntu-16.04/Dockerfile
index 49a26569..c6da88b6 100644
--- a/docker/ubuntu-16.04/Dockerfile
+++ b/docker/ubuntu-16.04/Dockerfile
@@ -30,26 +30,7 @@ RUN apt-get -y update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
         net-tools \
         ncurses-dev
 
-
-RUN wget https://raw.github.com/ocaml/opam/master/shell/opam_installer.sh
-
-env ocaml_version=4.06.0+default-unsafe-string
-RUN sh ./opam_installer.sh /usr/local/bin ${ocaml_version}
-
-ENV opam_root=/home/jenkins/OPAM
-ENV opam_env="opam config env --root=${opam_root}"
-RUN opam init --root=${opam_root} --comp ${ocaml_version}
-
-ADD https://raw.githubusercontent.com/delamonpansie/ovs_opam/master/switch opam.switch
-
-RUN eval `${opam_env}` && \
-    opam repo add ovs git://github.com/delamonpansie/ovs_opam && \
-    opam update -v && \
-    grep -Ev '^(arakoon|kinetic-client|orocksdb|ordma)' opam.switch | \
-    (eval `${opam_env}` && opam switch import - -y)
-
 RUN pip install fabric junit-xml nose simplejson python-etcd
-
 RUN mkdir /home/tests
 RUN chmod 777 /home/tests
 
@@ -71,6 +52,26 @@ RUN cp ./redis-3.0.7/src/redis-server /usr/bin
 RUN cp ./redis-3.0.7/src/redis-cli /usr/bin
 
 
+# most recent one (28/11/2017) is not compatible with our setup
+RUN wget https://raw.githubusercontent.com/ocaml/opam/afa822ec24c0f7afa278179cc59796a384129ce1/shell/opam_installer.sh
+
+env ocaml_version=4.06.0+default-unsafe-string
+RUN sh ./opam_installer.sh /usr/local/bin ${ocaml_version}
+
+ENV opam_root=/home/jenkins/OPAM
+ENV opam_env="opam config env --root=${opam_root}"
+RUN opam init --root=${opam_root} --comp ${ocaml_version}
+
+ADD opam.switch opam.switch
+ADD opam-repository ovs-opam-repository
+RUN chmod ugoa+rxw -R ovs-opam-repository
+
+RUN eval `${opam_env}` && \
+    opam repo add ovs ovs-opam-repository && \
+    opam update -v && \
+    (eval `${opam_env}` && opam switch import opam.switch -y)
+
+
 RUN chmod ugoa+rxw -R ${opam_root}
 RUN su - -c "echo 'eval `${opam_env}`' >> /home/jenkins/.profile"
 RUN echo "jenkins ALL=NOPASSWD: ALL" >/etc/sudoers.d/jenkins

From d61cab34e33f4113a19b53eb62b1e6a7924f5e66 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Tue, 16 Jan 2018 15:32:55 +0100
Subject: [PATCH 311/373] remove ubuntu.14.04 & centos-7

---
 COMPILING                      |   6 +-
 docker/centos-7/Dockerfile     | 110 ---------------------------------
 docker/ubuntu-14.04/Dockerfile |  83 -------------------------
 3 files changed, 1 insertion(+), 198 deletions(-)
 delete mode 100644 docker/centos-7/Dockerfile
 delete mode 100644 docker/ubuntu-14.04/Dockerfile

diff --git a/COMPILING b/COMPILING
index 35c97a70..107a11b0 100644
--- a/COMPILING
+++ b/COMPILING
@@ -11,9 +11,6 @@ Run
 ./docker/run.sh ubuntu-16.04 build
 from the root of the repository.
 
-You can also build using ubuntu 14.04 as follows:
-./docker/run.sh ubuntu-14.04 build
-
 Running the tests
 
 ./docker/run.sh ubuntu-16.04 unit
@@ -28,5 +25,4 @@ Enter the docker container
 Build packages
 
 ./docker/run.sh ubuntu-16.04 package_deb
-./docker/run.sh ubuntu-14.04 package_deb
-./docker/run.sh centos package_rpm
+
diff --git a/docker/centos-7/Dockerfile b/docker/centos-7/Dockerfile
deleted file mode 100644
index b6193f95..00000000
--- a/docker/centos-7/Dockerfile
+++ /dev/null
@@ -1,110 +0,0 @@
-FROM centos:7
-
-RUN useradd jenkins -u 1500 -g root
-
-RUN rpm --import file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
-RUN yum -y update && yum -y install \
-        wget curl \
-        make m4 gcc patch unzip \
-        git rsync mercurial \
-        gcc-c++ \
-        bzip2-devel libffi-devel snappy-devel libev-devel \
-        python-devel
-
-# TODO: What about darcs ?
-
-RUN rpm -iUvh http://dl.fedoraproject.org/pub/epel/7/x86_64/e/epel-release-7-9.noarch.rpm
-
-RUN yum -y update && yum -y install \
-         python-pip \
-         openssl-devel gmp-devel which \
-         zlib-devel ncurses-devel
-
-
-RUN wget https://raw.github.com/ocaml/opam/master/shell/opam_installer.sh
-
-env ocaml_version=4.04.2
-RUN sh ./opam_installer.sh /usr/local/bin ${ocaml_version}
-
-ENV opam_root=/home/jenkins/OPAM
-ENV opam_env="opam config env --root=${opam_root}"
-RUN opam init --root=${opam_root} --comp ${ocaml_version}
-
-# protobuf
-RUN yum -y install bzip2
-RUN wget https://github.com/google/protobuf/releases/download/v2.6.1/protobuf-2.6.1.tar.bz2 -O protobuf-2.6.1.tar.bz2 \
-    && tar -jxvf protobuf-2.6.1.tar.bz2 \
-    && cd protobuf-2.6.1 \
-    && ./configure && make && make install
-
-#RUN rpm -ivh http://cbs.centos.org/kojifiles/packages/protobuf/2.5.0/10.el7.centos/src/protobuf-2.5.0-10.el7.centos.src.rpm
-
-RUN pip install fabric junit-xml
-
-RUN wget http://cudf-solvers.irill.org/cudf_remote_proxy
-RUN chmod u+x cudf_remote_proxy
-RUN mv cudf_remote_proxy /usr/local/bin/
-
-ENV OPAMEXTERNALSOLVER="cudf_remote_proxy %{input}% %{output}% %{criteria}%"
-
-RUN eval `${opam_env}` && \
-    opam update -v && \
-    opam install -y \
-        oasis.0.4.10 \
-        ocamlfind \
-        ssl.0.5.3 \
-        camlbz2 \
-        snappy \
-        bisect \
-        lwt_ssl \
-        lwt.3.0.0 \
-        camltc.0.9.4 \
-        ocplib-endian.1.0 \
-        quickcheck.1.0.2 \
-        conf-libev \
-        ppx_deriving.4.1 \
-        base.v0.9.3 \
-        core.v0.9.1 \
-        redis.0.3.3 \
-        uri.1.9.4
-
-
-RUN pip install fabric junit-xml nose simplejson python-etcd
-
-RUN mkdir /home/tests
-RUN chmod 777 /home/tests
-
-# Install etcd:
-ENV etcd_version=2.3.5
-RUN curl -L  https://github.com/coreos/etcd/releases/download/v${etcd_version}/etcd-v${etcd_version}-linux-amd64.tar.gz -o etcd-v{etcd_version}-linux-amd64.tar.gz
-RUN tar xzvf etcd-v{etcd_version}-linux-amd64.tar.gz
-RUN cp ./etcd-v${etcd_version}-linux-amd64/etcd /usr/bin \
-    && cp ./etcd-v${etcd_version}-linux-amd64/etcdctl /usr/bin
-
-# Install redis:
-RUN wget http://download.redis.io/releases/redis-3.0.7.tar.gz
-RUN tar xzvf redis-3.0.7.tar.gz
-RUN cd redis-3.0.7 && make
-RUN cp ./redis-3.0.7/src/redis-server /usr/bin
-RUN cp ./redis-3.0.7/src/redis-cli /usr/bin
-
-
-RUN yum -y update && yum -y install \
-        autoconf help2man perl-Thread-Queue \
-        sudo iproute
-
-#
-# Disable "ssh hostname sudo <cmd>", because it will show the password in clear.
-#         You have to run "ssh -t hostname sudo <cmd>".
-#
-# Defaults    requiretty          # is line 56
-RUN awk 'NR == 56 {next} {print}' /etc/sudoers >/tmp/__sudoers && mv /tmp/__sudoers /etc/sudoers
-
-RUN yum -y install rpm-build
-
-RUN chmod ugoa+rxw -R ${opam_root}
-RUN su - -c "echo 'eval `${opam_env}`' >> /home/jenkins/.bash_profile"
-RUN echo "jenkins ALL=NOPASSWD: ALL" >/etc/sudoers.d/jenkins
-
-
-ENTRYPOINT ["/bin/bash", "-c", "set -e && /home/jenkins/arakoon/docker/docker-entrypoint.sh $@"]
diff --git a/docker/ubuntu-14.04/Dockerfile b/docker/ubuntu-14.04/Dockerfile
deleted file mode 100644
index c66e31a2..00000000
--- a/docker/ubuntu-14.04/Dockerfile
+++ /dev/null
@@ -1,83 +0,0 @@
-FROM ubuntu:14.04.3
-
-RUN useradd jenkins -u 1500 -g root
-
-RUN apt-get -y update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
-        build-essential m4 apt-utils \
-        libffi-dev libssl-dev \
-        libbz2-dev \
-        libgmp3-dev \
-        libev-dev \
-        libsnappy-dev \
-        libxen-dev \
-        help2man \
-        pkg-config \
-        time \
-        aspcud \
-        wget \
-        curl \
-        rsync \
-        darcs \
-        git \
-        unzip \
-        automake \
-        python-dev \
-        python-pip \
-        debhelper \
-        psmisc \
-        sudo \
-        libtool \
-        iptables \
-        net-tools \
-        ncurses-dev
-
-
-RUN wget https://raw.github.com/ocaml/opam/master/shell/opam_installer.sh
-
-env ocaml_version=4.06.0+default-unsafe-string
-RUN sh ./opam_installer.sh /usr/local/bin ${ocaml_version}
-
-ENV opam_root=/home/jenkins/OPAM
-ENV opam_env="opam config env --root=${opam_root}"
-RUN opam init --root=${opam_root} --comp ${ocaml_version}
-
-ADD https://raw.githubusercontent.com/delamonpansie/ovs_opam/master/switch opam.switch
-
-RUN eval `${opam_env}` && \
-    opam repo add ovs git://github.com/delamonpansie/ovs_opam && \
-    opam update -v && \
-    grep -Ev '^(arakoon|kinetic-client|orocksdb|ordma)' opam.switch | \
-    (eval `${opam_env}` && opam switch import - -y)
-
-#
-RUN wget https://www.python.org/ftp/python/2.7.12/Python-2.7.12.tar.xz
-RUN tar -Jxvf ./Python-2.7.12.tar.xz
-RUN cd /Python-2.7.12 && ./configure && make && sudo make install
-RUN curl https://bootstrap.pypa.io/get-pip.py | python
-RUN pip install fabric junit-xml nose simplejson python-etcd
-
-RUN mkdir /home/tests
-RUN chmod 777 /home/tests
-
-# Install etcd:
-ENV etcd_version=2.3.5
-RUN curl -L  https://github.com/coreos/etcd/releases/download/v${etcd_version}/etcd-v${etcd_version}-linux-amd64.tar.gz -o etcd-v{etcd_version}-linux-amd64.tar.gz
-RUN tar xzvf etcd-v{etcd_version}-linux-amd64.tar.gz
-RUN cp ./etcd-v${etcd_version}-linux-amd64/etcd /usr/bin \
-    && cp ./etcd-v${etcd_version}-linux-amd64/etcdctl /usr/bin
-
-# Install redis:
-RUN wget http://download.redis.io/releases/redis-3.0.7.tar.gz
-RUN tar xzvf redis-3.0.7.tar.gz
-RUN cd redis-3.0.7 && make
-RUN cp ./redis-3.0.7/src/redis-server /usr/bin
-RUN cp ./redis-3.0.7/src/redis-cli /usr/bin
-
-
-RUN chmod ugoa+rxw -R ${opam_root}
-RUN su - -c "echo 'eval `${opam_env}`' >> /home/jenkins/.profile"
-RUN echo "jenkins ALL=NOPASSWD: ALL" >/etc/sudoers.d/jenkins
-
-
-
-ENTRYPOINT ["/bin/bash", "-c", "set -e && /home/jenkins/arakoon/docker/docker-entrypoint.sh $@"]

From cc237f8a6556b07fe065f5471d84a92bc98ec7e1 Mon Sep 17 00:00:00 2001
From: Yuriy Vostrikov <yuriy.vostrikov@openvstorage.com>
Date: Wed, 24 Jan 2018 13:39:59 +0300
Subject: [PATCH 312/373] Simplify drop_master

---
 src/node/sync_backend.ml | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/src/node/sync_backend.ml b/src/node/sync_backend.ml
index 6ba2e751..8579d674 100644
--- a/src/node/sync_backend.ml
+++ b/src/node/sync_backend.ml
@@ -785,12 +785,7 @@ struct
         else
           begin
             match self # who_master () with
-            | None -> Lwt.return ()
-            | Some m ->
-              if m <> my_name
-              then Lwt.return ()
-              else
-                begin
+            | Some m when m = my_name ->
                   act_not_preferred := true;
                   let (sleep, awake) = Lwt.wait () in
                   let update = Multi_paxos.DropMaster (sleep, awake) in
@@ -802,7 +797,7 @@ struct
                   Logger.debug_f_ "drop_master: waiting another %1.1fs" delay >>= fun () ->
                   Lwt_unix.sleep delay >>= fun () ->
                   Logger.debug_ "drop_master: completed"
-                end
+            | _ -> Lwt.return ()
           end
 
       method get_current_state () =

From 95e556190ba6168c5c3ff4a4685db48ec7872ce5 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Wed, 24 Jan 2018 17:38:59 +0100
Subject: [PATCH 313/373] allow loading a plugin when using --replay-tlogs so
 userfunctions can be available

---
 src/main/arakoon.ml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/main/arakoon.ml b/src/main/arakoon.ml
index a9f53160..99cc57f5 100644
--- a/src/main/arakoon.ml
+++ b/src/main/arakoon.ml
@@ -278,6 +278,8 @@ let main () =
                                       then end_i := (Some (Scanf.sscanf is "%Li" (fun i -> i))))
                                 ],
      "<tlog_dir> <tlf_dir> <path-to-db> [<end-i>]");
+    ("-load-plugin", Arg.String (fun plugin -> Dynlink.loadfile_private plugin),
+     "<plugin_location>: loads the specified plugin (so user functions can be available for e.g. --replay-tlogs command)");
     ("-dump-values", Arg.Set dump_values, "also dumps values (in --dump-tlog)");
     ("--make-tlog", Arg.Tuple[ set_laction MakeTlog;
                                Arg.Set_string filename;

From df338a2fbbae214e965216a8f83087340e88b48f Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Thu, 25 Jan 2018 09:47:59 +0100
Subject: [PATCH 314/373] pr comment

---
 src/main/arakoon.ml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/arakoon.ml b/src/main/arakoon.ml
index 99cc57f5..ff208e58 100644
--- a/src/main/arakoon.ml
+++ b/src/main/arakoon.ml
@@ -278,7 +278,7 @@ let main () =
                                       then end_i := (Some (Scanf.sscanf is "%Li" (fun i -> i))))
                                 ],
      "<tlog_dir> <tlf_dir> <path-to-db> [<end-i>]");
-    ("-load-plugin", Arg.String (fun plugin -> Dynlink.loadfile_private plugin),
+    ("-load-plugin", Arg.String Dynlink.loadfile_private,
      "<plugin_location>: loads the specified plugin (so user functions can be available for e.g. --replay-tlogs command)");
     ("-dump-values", Arg.Set dump_values, "also dumps values (in --dump-tlog)");
     ("--make-tlog", Arg.Tuple[ set_laction MakeTlog;

From 9e006aa75ccfd23dbb3d202bec99ddb04d4fd456 Mon Sep 17 00:00:00 2001
From: Yuriy Vostrikov <yuriy.vostrikov@openvstorage.com>
Date: Thu, 25 Jan 2018 13:41:02 +0300
Subject: [PATCH 315/373] Minor cleanup

---
 src/paxos/slave.ml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/paxos/slave.ml b/src/paxos/slave.ml
index 09b89beb..4967a4d5 100644
--- a/src/paxos/slave.ml
+++ b/src/paxos/slave.ml
@@ -133,6 +133,11 @@ let slave_steady_state (type s) constants state event =
           let sides = log_e0::accept_e::send_e::log_e::oconsensus in
           Fsm.return ~sides (Slave_steady_state (n', Sn.succ i', Some v))
         in
+        let get_store_i () =
+          match S.consensus_i store with
+          | None -> Sn.pred Sn.start
+          | Some store_i -> store_i
+        in
         match msg with
           | Accept (n',i',v) when (n',i') = (n,i) ->
             begin
@@ -141,10 +146,8 @@ let slave_steady_state (type s) constants state event =
                 (* we should have either a previous value received for this n
                      (from same master) in that case store_i == i - 2
                    or we should have no previous value and store_i = i - 1 *)
-                let store_i = match S.consensus_i store with
-                  | None -> Sn.pred Sn.start
-                  | Some store_i -> store_i in
                 begin
+                  let store_i = get_store_i () in
                   match maybe_previous with
                     | None ->
                       if Sn.sub i store_i = 1L
@@ -168,10 +171,7 @@ let slave_steady_state (type s) constants state event =
               accept_value oconsensus n' i' v "steady_state :: replying with %S"
             end
           | Accept (n',i',v) when (Sn.succ i' = i) && (n' >= n) ->
-            let store_i = match S.consensus_i store with
-              | None -> Sn.pred Sn.start
-              | Some store_i -> store_i in
-            if Sn.sub i' store_i = 0L
+            if i' = get_store_i ()
             then
               begin
                 (* already learned this value, so no longer writing it to the tlog,

From 3ea6931565613775466ff7b7c1c34ad14abd50ae Mon Sep 17 00:00:00 2001
From: Yuriy Vostrikov <yuriy.vostrikov@openvstorage.com>
Date: Fri, 26 Jan 2018 17:42:29 +0300
Subject: [PATCH 316/373] Quote binary strings

---
 src/client/remote_nodestream_test.ml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/client/remote_nodestream_test.ml b/src/client/remote_nodestream_test.ml
index 319bac9d..8569266e 100644
--- a/src/client/remote_nodestream_test.ml
+++ b/src/client/remote_nodestream_test.ml
@@ -131,8 +131,8 @@ let set_route_delta port () =
     Routing.routing_to new_ser new_r;
     let old_str = Buffer.contents old_ser in
     let new_str = Buffer.contents new_ser in
-    Logger.debug_f_ "old_str: %s " old_str >>= fun () ->
-    Logger.debug_f_ "new_str: %s" new_str  >>= fun () ->
+    Logger.debug_f_ "old_str: %S " old_str >>= fun () ->
+    Logger.debug_f_ "new_str: %S" new_str  >>= fun () ->
     OUnit.assert_equal old_str new_str;
     Lwt.return ()
   in

From 9554085bf51a683b8902729ac1d4c732b4f5ecfc Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Fri, 26 Jan 2018 18:30:40 +0100
Subject: [PATCH 317/373] add tzdata package

---
 docker/ubuntu-16.04/Dockerfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docker/ubuntu-16.04/Dockerfile b/docker/ubuntu-16.04/Dockerfile
index c6da88b6..555016a3 100644
--- a/docker/ubuntu-16.04/Dockerfile
+++ b/docker/ubuntu-16.04/Dockerfile
@@ -28,7 +28,8 @@ RUN apt-get -y update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
         libtool \
         iptables \
         net-tools \
-        ncurses-dev
+        ncurses-dev \
+        tzdata
 
 RUN pip install fabric junit-xml nose simplejson python-etcd
 RUN mkdir /home/tests

From 467d378a328bf4fd687d110b0d596103d02e3684 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Fri, 26 Jan 2018 18:42:47 +0100
Subject: [PATCH 318/373] clean builds

---
 jenkins/common.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jenkins/common.sh b/jenkins/common.sh
index 221aeee3..dd9ed98c 100755
--- a/jenkins/common.sh
+++ b/jenkins/common.sh
@@ -9,4 +9,4 @@ echo $PWD
 echo TEST_HOME=${TEST_HOME}
 export PATH=${PATH}:${TEST_HOME}/apps/arakoon/bin
 
-make
+make clean build

From e857ff16d4133a961341600ca6cc791ecdda5f90 Mon Sep 17 00:00:00 2001
From: Yuriy Vostrikov <yuriy.vostrikov@openvstorage.com>
Date: Mon, 29 Jan 2018 16:45:32 +0300
Subject: [PATCH 319/373] Remove unused script

---
 run_docker_debian.sh | 6 ------
 1 file changed, 6 deletions(-)
 delete mode 100755 run_docker_debian.sh

diff --git a/run_docker_debian.sh b/run_docker_debian.sh
deleted file mode 100755
index 048a355f..00000000
--- a/run_docker_debian.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-PROJECT_HOME=/home/romain/workspace/ARAKOON/arakoon
-INSIDE=/home/jenkins/workspace/the_job/
-docker run --rm=true --interactive=true --privileged -t -v \
-       ${PROJECT_HOME}:${INSIDE} \
-       --env "SUITE=$1" \
-       -w ${INSIDE} arakoon_debian $@

From 9aa0c3e2289602cf5d7fd66b900cf2f867f21a59 Mon Sep 17 00:00:00 2001
From: Yuriy Vostrikov <yuriy.vostrikov@openvstorage.com>
Date: Mon, 29 Jan 2018 16:46:35 +0300
Subject: [PATCH 320/373] Fix: under "set -u" variable access must be guarded
 against undefined

---
 travis.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/travis.sh b/travis.sh
index 9a1f6f5f..dc101374 100755
--- a/travis.sh
+++ b/travis.sh
@@ -45,7 +45,7 @@ script () {
     date
 }
 
-case "$1" in
+case "${1-undefined}" in
     install)
         install
         ;;

From f46623fb6d77d0f05e62c95da5caeaa8bdc27f75 Mon Sep 17 00:00:00 2001
From: Yuriy Vostrikov <yuriy.vostrikov@openvstorage.com>
Date: Mon, 29 Jan 2018 17:43:21 +0300
Subject: [PATCH 321/373] Move ./run_with_timeout_and_progress.sh into
 travis.sh

---
 run_with_timeout_and_progress.sh | 23 -----------------------
 travis.sh                        | 25 +++++++++++++++++++++++--
 2 files changed, 23 insertions(+), 25 deletions(-)
 delete mode 100755 run_with_timeout_and_progress.sh

diff --git a/run_with_timeout_and_progress.sh b/run_with_timeout_and_progress.sh
deleted file mode 100755
index 0f198ac2..00000000
--- a/run_with_timeout_and_progress.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/bin/bash -u
-
-TIMEOUT=$1
-COMMAND=$2
-shift 2
-
-timeout $TIMEOUT $COMMAND $@ > output 2>&1 &
-PID=$!
-
-echo $PID
-
-while kill -0 $PID 2>/dev/null
-do
-    echo -ne .
-    sleep 1
-done
-
-wait $PID
-RESULT=$?
-
-tail -n256 output
-
-exit $RESULT
diff --git a/travis.sh b/travis.sh
index dc101374..96dd73f3 100755
--- a/travis.sh
+++ b/travis.sh
@@ -1,5 +1,26 @@
 #!/bin/bash -xue
 
+timeout_with_progress () {
+    OUTPUT=`mktemp`
+    timeout "$@" > $OUTPUT 2>&1 &
+    PID=$!
+
+    echo $PID
+
+    while kill -0 $PID 2>/dev/null
+    do
+	echo -ne .
+	sleep 1
+    done
+
+    wait $PID
+    RESULT=$?
+
+    tail -n256 $OUTPUT
+
+    return $RESULT
+}
+
 install () {
     echo "Running 'install' phase"
 
@@ -14,7 +35,7 @@ install () {
     START_BUILD=$(date +%s.%N)
     echo $START_BUILD
 
-    ./run_with_timeout_and_progress.sh 9000 ./docker/run.sh ubuntu-16.04 clean
+    timeout_with_progress 9000 ./docker/run.sh ubuntu-16.04 clean
 
     END_BUILD=$(date +%s.%N)
     echo $END_BUILD
@@ -40,7 +61,7 @@ script () {
 
     date
 
-    ./run_with_timeout_and_progress.sh 9000 ./docker/run.sh ubuntu-16.04 unit
+    timeout_with_progress 9000 ./docker/run.sh ubuntu-16.04 unit
 
     date
 }

From 4ae0ad034aaa2f5a2e433991034295c1e3bd387a Mon Sep 17 00:00:00 2001
From: Yuriy Vostrikov <yuriy.vostrikov@openvstorage.com>
Date: Mon, 29 Jan 2018 17:44:19 +0300
Subject: [PATCH 322/373] Remove unused depends

---
 docker/opam.switch | 24 ------------------------
 1 file changed, 24 deletions(-)

diff --git a/docker/opam.switch b/docker/opam.switch
index 5bfdfb6c..13d7a00d 100644
--- a/docker/opam.switch
+++ b/docker/opam.switch
@@ -5,51 +5,32 @@ base-threads	base	root
 base-unix	base	root
 base64	2.2.0	installed
 bin_prot	v0.9.2	installed
-biniou	1.2.0	installed
 bisect	1.3+ovs1	installed
 camlbz2	0.6.0	installed
 camlp4	4.06+1	installed
 camltc	0.9.4	installed
-cmdliner	1.0.2	installed
-conf-gmp	1	installed
-conf-gmp-powm-sec	1	installed
-conf-libev	4-11	installed
 conf-m4	1	installed
 conf-openssl	1	installed
-conf-perl	1	installed
 conf-pkg-config	1.0	installed
 conf-which	1	installed
-conf-zlib	1	installed
 configurator	v0.9.1	installed
 core	v0.9.2	installed
 core_kernel	v0.9.1	installed
 cppo	1.6.0	installed
 cppo_ocamlbuild	1.6.0	installed
-cryptokit	1.12	installed
-ctypes	0.13.1	installed
-ctypes-foreign	0.4.0	installed
-depext	1.0.5	installed
 easy-format	1.3.0	installed
-ezxmlm	1.0.1	installed
 fieldslib	v0.9.0	installed
-integers	0.2.2	installed
 jane-street-headers	v0.9.0	installed
 jbuilder	1.0+beta16	installed
-kinetic-client	0.0.6+ovs1	installed
 lwt	3.0.0	root
 lwt_ssl	1.1.1	installed
-mirage-no-xen	1	installed
 num	1.1	installed
-oasis	0.4.10	installed
 ocaml-compiler-libs	v0.9.0	installed
 ocaml-migrate-parsetree	1.0.7	installed
 ocamlbuild	0.12.0	installed
 ocamlfind	1.7.3	installed
-ocamlify	0.0.1	installed
-ocamlmod	0.0.9	installed
 ocplib-endian	1.0	installed
 octavius	1.2.0	installed
-omake	0.10.3	installed
 optcomp	1.6+ovs1	installed
 ounit	2.0.6	installed
 piqi	0.7.5	installed
@@ -62,9 +43,6 @@ ppx_bin_prot	v0.9.0	installed
 ppx_compare	v0.9.0	installed
 ppx_core	v0.9.3	installed
 ppx_custom_printf	v0.9.0	installed
-ppx_derivers	1.0	installed
-ppx_deriving	4.2.1	installed
-ppx_deriving_yojson	3.1	installed
 ppx_driver	v0.9.2	installed
 ppx_enumerate	v0.9.0	installed
 ppx_expect	v0.9.0	installed
@@ -108,5 +86,3 @@ uri	1.9.5	installed
 uuidm	0.9.6	installed
 variantslib	v0.9.0	installed
 xmlm	1.3.0	installed
-yojson	1.4.0	installed
-zarith	1.7	installed

From 3f41d5a99f6522f8463f4a1772638105df6ecaef Mon Sep 17 00:00:00 2001
From: Yuriy Vostrikov <yuriy.vostrikov@openvstorage.com>
Date: Fri, 26 Jan 2018 16:38:18 +0300
Subject: [PATCH 323/373] Remove dead code.

Condition current_i <= future_i is always true.
---
 src/paxos/master.ml           |  6 +--
 src/paxos/multi_paxos_fsm.ml  | 14 ++----
 src/paxos/multi_paxos_type.ml |  3 +-
 src/paxos/slave.ml            | 89 ++++++++++++-----------------------
 4 files changed, 38 insertions(+), 74 deletions(-)

diff --git a/src/paxos/master.ml b/src/paxos/master.ml
index 38be336f..bf9b64ba 100644
--- a/src/paxos/master.ml
+++ b/src/paxos/master.ml
@@ -176,9 +176,8 @@ let stable_master (type s) constants ((n,new_i, lease_expire_waiters) as current
                       Fsm.return (Slave_steady_state (n', new_i, None))
                     end
                   | Promise_sent_needs_catchup ->
-                    let i = S.get_catchup_start_i constants.store in
                     Multi_paxos.safe_wakeup_all () lease_expire_waiters >>= fun () ->
-                    Fsm.return (Slave_discovered_other_master (source, i, n', i'))
+                    Fsm.return (Slave_discovered_other_master (source, n', i'))
                 end
             end
           | Accepted(_n,i) ->
@@ -206,8 +205,7 @@ let stable_master (type s) constants ((n,new_i, lease_expire_waiters) as current
                 begin
                   (* Become slave, goto catchup *)
                   Logger.debug_f_ "%s: stable_master: received Accept from new master %S" me (string_of msg) >>= fun () ->
-                  let cu_pred = S.get_catchup_start_i constants.store in
-                  let new_state = (source,cu_pred,n',i') in
+                  let new_state = (source, n', i') in
                   Multi_paxos.safe_wakeup_all () lease_expire_waiters >>= fun () ->
                   Fsm.return (Slave_discovered_other_master new_state)
                 end
diff --git a/src/paxos/multi_paxos_fsm.ml b/src/paxos/multi_paxos_fsm.ml
index 6a20a9f7..34bcdcc1 100644
--- a/src/paxos/multi_paxos_fsm.ml
+++ b/src/paxos/multi_paxos_fsm.ml
@@ -203,8 +203,7 @@ let wait_for_promises (type s) constants state event =
                     >>= fun () ->
                     if i' > i
                     then
-                      let cu_pred = S.get_catchup_start_i constants.store in
-                      Fsm.return (Slave_discovered_other_master (source,cu_pred,n'',i'))
+                      Fsm.return (Slave_discovered_other_master (source, n'', i'))
                     else
                       let new_n = update_n constants (max n n'') in
                       Fsm.return (Election_suggest (new_n, counter + 1))
@@ -235,8 +234,7 @@ let wait_for_promises (type s) constants state event =
                     start_lease_expiration_thread constants >>= fun () ->
                     Fsm.return (Slave_steady_state (n', i, None))
                   | Promise_sent_needs_catchup ->
-                    let i = S.get_catchup_start_i constants.store in
-                    Fsm.return (Slave_discovered_other_master (source, i, n', i'))
+                    Fsm.return (Slave_discovered_other_master (source, n', i'))
               end
             | Accept (n',_i,_v) when n' < n ->
               begin
@@ -244,7 +242,7 @@ let wait_for_promises (type s) constants state event =
                 then
                   begin
                     Logger.debug_f_ "%s: wait_for_promises: still have an active master (received %s) -> catching up from master" me  (string_of msg) >>= fun () ->
-                    Fsm.return (Slave_discovered_other_master (source, i, n', _i))
+                    Fsm.return (Slave_discovered_other_master (source, n', _i))
                   end
                 else
                   Logger.debug_f_ "%s: wait_for_promises: ignoring old Accept %s" me (Sn.string_of n')
@@ -444,10 +442,9 @@ let wait_for_accepteds
                       end
                     | Promise_sent_needs_catchup ->
                       begin
-                        let i = S.get_catchup_start_i constants.store in
                         lost_master_role mo >>= fun () ->
                         Multi_paxos.safe_wakeup_all () lew >>= fun () ->
-                        Fsm.return (Slave_discovered_other_master (source, i, n', i'))
+                        Fsm.return (Slave_discovered_other_master (source, n', i'))
                       end
                   end
               end
@@ -480,8 +477,7 @@ let wait_for_accepteds
                   begin
                     (* Become slave, goto catchup *)
                     Logger.debug_f_ "%s: wait_for_accepteds: received Accept from new master %S" me (string_of msg) >>= fun () ->
-                    let cu_pred = S.get_catchup_start_i constants.store in
-                    let new_state = (source,cu_pred,n',i') in
+                    let new_state = (source, n', i') in
                     Logger.debug_f_ "%s: wait_for_accepteds: drop %S (it's still me)" me (string_of msg) >>= fun () ->
                     Fsm.return (Slave_discovered_other_master new_state)
                   end
diff --git a/src/paxos/multi_paxos_type.ml b/src/paxos/multi_paxos_type.ml
index 5ff754c8..ebc3d471 100644
--- a/src/paxos/multi_paxos_type.ml
+++ b/src/paxos/multi_paxos_type.ml
@@ -44,8 +44,7 @@ type transitions =
   (* slave or pending slave *)
   | Slave_fake_prepare of (n * i)
   | Slave_steady_state of (n * i * Value.t option) (* value received for this n and previous i *)
-  | Slave_discovered_other_master of (Messaging.id * Mp_msg.MPMessage.n *
-                                        Mp_msg.MPMessage.n * Mp_msg.MPMessage.n )
+  | Slave_discovered_other_master of (Messaging.id * Mp_msg.MPMessage.n * Mp_msg.MPMessage.n )
 
   | Promises_check_done of (n * i *
                               Messaging.id list *
diff --git a/src/paxos/slave.ml b/src/paxos/slave.ml
index 4967a4d5..fec835fc 100644
--- a/src/paxos/slave.ml
+++ b/src/paxos/slave.ml
@@ -193,8 +193,7 @@ let slave_steady_state (type s) constants state event =
                                    (Sn.string_of n') (Sn.string_of i') source (Sn.string_of  n) (Sn.string_of  i)
                                 )
                in
-               let cu_pred = S.get_catchup_start_i constants.store in
-               let new_state = (source,cu_pred,n',i') in
+               let new_state = (source, n', i') in
                Fsm.return ~sides:[log_e0;log_e] (Slave_discovered_other_master(new_state) )
              end
           | Accept (_,_,_) ->
@@ -217,15 +216,13 @@ let slave_steady_state (type s) constants state event =
                 start_lease_expiration_thread constants >>= fun () ->
                 Fsm.return (Slave_steady_state (n', next_i, None))
               | Promise_sent_needs_catchup ->
-                let i = S.get_catchup_start_i constants.store in
-                let new_state = (source, i, n', i') in
+                let new_state = (source, n', i') in
                 Fsm.return ~sides:[log_e0] (Slave_discovered_other_master(new_state) )
             end
           | Nak(_,(n2, i2)) when i2 > i ->
              begin
                Logger.debug_f_ "%s: got %s => go to catchup" constants.me (string_of msg) >>= fun () ->
-               let cu_pred =  S.get_catchup_start_i constants.store in
-               Fsm.return (Slave_discovered_other_master (source, cu_pred, n2, i2))
+               Fsm.return (Slave_discovered_other_master (source, n2, i2))
              end
           | Nak _
           | Promise _
@@ -270,64 +267,38 @@ let slave_steady_state (type s) constants state event =
 (* a pending slave that discovered another master has to do
    catchup and then go to steady state *)
 let slave_discovered_other_master (type s) constants state () =
-  let (master, current_i, future_n, future_i) = state in
+  let (master, future_n, future_i) = state in
   let me = constants.me
   and other_cfgs = constants.other_cfgs
   and store = constants.store
   and tlog_coll = constants.tlog_coll
   in
   let module S = (val constants.store_module : Store.STORE with type t = s) in
-  if current_i <= future_i then
-    begin
-      Logger.debug_f_
-        "%s: slave_discovered_other_master: catching up from %s @ %s"
-        me master (Sn.string_of future_i) >>= fun() ->
-      let cluster_id = constants.cluster_id in
-      let tls_ctx = constants.catchup_tls_ctx in
-      let master_before = S.who_master store in
-      let lease_expired = match master_before with
-        | None -> true
-        | Some (_, ls) -> ls +. (float_of_int constants.lease_expiration) <= Unix.gettimeofday () in
-      Catchup.catchup
-        ~tls_ctx
-        ~tcp_keepalive:constants.tcp_keepalive
-        ~stop:constants.stop
-        me other_cfgs ~cluster_id ((module S), store, tlog_coll) master >>= fun () ->
+  Logger.debug_f_
+    "%s: slave_discovered_other_master: catching up from %s @ %s"
+    me master (Sn.string_of future_i) >>= fun() ->
+  let cluster_id = constants.cluster_id in
+  let tls_ctx = constants.catchup_tls_ctx in
+  let master_before = S.who_master store in
+  let lease_expired = match master_before with
+    | None -> true
+    | Some (_, ls) -> ls +. (float_of_int constants.lease_expiration) <= Unix.gettimeofday () in
+  Catchup.catchup
+    ~tls_ctx
+    ~tcp_keepalive:constants.tcp_keepalive
+    ~stop:constants.stop
+    me other_cfgs ~cluster_id ((module S), store, tlog_coll) master >>= fun () ->
 
-      let master_after = S.who_master store in
-      let immediate_lease_expiration =
-        (* lease was previously expired and learned no new master leases *)
-        lease_expired && (master_after = master_before)
-      in
-      let current_i' = S.get_succ_store_i store in
-      let fake = Prepare( Sn.of_int (-2), (* make it completely harmless *)
-                          Sn.pred current_i') (* pred =  consensus_i *)
-      in
-      Multi_paxos.mcast constants fake >>= fun () ->
+  let master_after = S.who_master store in
+  let immediate_lease_expiration =
+    (* lease was previously expired and learned no new master leases *)
+    lease_expired && (master_after = master_before)
+  in
+  let current_i' = S.get_succ_store_i store in
+  let fake = Prepare( Sn.of_int (-2), (* make it completely harmless *)
+                      Sn.pred current_i') (* pred =  consensus_i *)
+  in
+  Multi_paxos.mcast constants fake >>= fun () ->
 
-      start_lease_expiration_thread ~immediate_lease_expiration constants >>= fun () ->
-      Fsm.return (Slave_steady_state (future_n, current_i', None));
-    end
-  else
-    begin
-      let next_i = S.get_succ_store_i constants.store in
-      begin
-        if is_election constants
-        then
-          (* we have to go to election here or we can get in a situation where
-             everybody just waits for each other *)
-          let new_n = update_n constants future_n in
-          Lwt.return
-            (Election_suggest (new_n, 0),
-             "slave_discovered_other_master: my i is bigger then theirs ; back to election")
-        else
-          begin
-            start_lease_expiration_thread constants >>= fun () ->
-            Lwt.return
-              (Slave_steady_state( future_n, next_i, None ),
-               "slave_discovered_other_master: forced slave, back to slave mode")
-          end
-      end >>= fun (s, m) ->
-      let log_e = ELog (fun () -> m) in
-      Fsm.return ~sides:[log_e] s
-    end
+  start_lease_expiration_thread ~immediate_lease_expiration constants >>= fun () ->
+  Fsm.return (Slave_steady_state (future_n, current_i', None));

From c11a9066d4ae30f8d5502c101547fe123c33c84f Mon Sep 17 00:00:00 2001
From: Yuriy Vostrikov <yuriy.vostrikov@openvstorage.com>
Date: Fri, 26 Jan 2018 16:57:03 +0300
Subject: [PATCH 324/373] Indent

---
 src/paxos/slave.ml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/paxos/slave.ml b/src/paxos/slave.ml
index fec835fc..27f75d03 100644
--- a/src/paxos/slave.ml
+++ b/src/paxos/slave.ml
@@ -194,7 +194,7 @@ let slave_steady_state (type s) constants state event =
                                 )
                in
                let new_state = (source, n', i') in
-               Fsm.return ~sides:[log_e0;log_e] (Slave_discovered_other_master(new_state) )
+               Fsm.return ~sides:[log_e0;log_e] (Slave_discovered_other_master new_state)
              end
           | Accept (_,_,_) ->
             begin
@@ -217,7 +217,7 @@ let slave_steady_state (type s) constants state event =
                 Fsm.return (Slave_steady_state (n', next_i, None))
               | Promise_sent_needs_catchup ->
                 let new_state = (source, n', i') in
-                Fsm.return ~sides:[log_e0] (Slave_discovered_other_master(new_state) )
+                Fsm.return ~sides:[log_e0] (Slave_discovered_other_master new_state)
             end
           | Nak(_,(n2, i2)) when i2 > i ->
              begin

From a4686ef48dbdf26089dfd28cd0dda3af7dff22e1 Mon Sep 17 00:00:00 2001
From: Yuriy Vostrikov <yuriy.vostrikov@openvstorage.com>
Date: Mon, 29 Jan 2018 19:16:42 +0300
Subject: [PATCH 325/373] Fix progress printing broken in f46623f

---
 travis.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/travis.sh b/travis.sh
index 96dd73f3..bb786a30 100755
--- a/travis.sh
+++ b/travis.sh
@@ -1,6 +1,7 @@
 #!/bin/bash -xue
 
-timeout_with_progress () {
+timeout_with_progress () (
+    set +x
     OUTPUT=`mktemp`
     timeout "$@" > $OUTPUT 2>&1 &
     PID=$!
@@ -19,7 +20,7 @@ timeout_with_progress () {
     tail -n256 $OUTPUT
 
     return $RESULT
-}
+)
 
 install () {
     echo "Running 'install' phase"

From d07ec3fa0d6edef212282aa0908a84d2d4fcf396 Mon Sep 17 00:00:00 2001
From: Yuriy Vostrikov <yuriy.vostrikov@openvstorage.com>
Date: Tue, 30 Jan 2018 13:00:38 +0300
Subject: [PATCH 326/373] Lock TZ at startup

This will save one stat(/etc/localtime) syscall per log print
---
 src/main/arakoon.ml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/main/arakoon.ml b/src/main/arakoon.ml
index ff208e58..b38f693d 100644
--- a/src/main/arakoon.ml
+++ b/src/main/arakoon.ml
@@ -159,11 +159,21 @@ let run_some_tests repeat_count filter =
 
     | None -> failwith (Printf.sprintf "no test matches '%s'" filter);;
 
+let try_lock_tz () =
+  match Unix.getenv "TZ" with
+    _ -> ()
+  | exception Not_found -> begin
+      ignore(Lazy.force Core.Time.Zone.local);
+      Unix.putenv "TZ" ":/etc/localtime";
+    end
+
 
 let main () =
   Ssl_threads.init ();
   Ssl.init ~thread_safe:true ();
 
+  try_lock_tz ();
+
   let () =
     Lwt.async_exception_hook :=
       (fun exn -> Logger.ign_info_f_

From 3cc12a6615733d1fea9e495a7a54589a62a6d832 Mon Sep 17 00:00:00 2001
From: Yuriy Vostrikov <yuriy.vostrikov@openvstorage.com>
Date: Thu, 18 Jan 2018 16:18:34 +0300
Subject: [PATCH 327/373] Dynamic JOBS calculation

---
 Makefile | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 643ca3ed..bd61f624 100644
--- a/Makefile
+++ b/Makefile
@@ -7,6 +7,8 @@ OCAML_LIBDIR ?= $(START)/lib/ocaml/
 #OCAML_LIBDIR ?= `ocamlfind printconf destdir`
 OCAML_FIND ?= ocamlfind
 
+JOBS := $(shell getconf _NPROCESSORS_ONLN 2>/dev/null || grep -c ^processor /proc/cpuinfo 2>/dev/null || echo 1)
+
 all: build
 
 clean:
@@ -16,7 +18,7 @@ clean:
 	rm -f  ./arakoon.byte ./arakoon.native
 
 build:
-	ocamlbuild -j 4 -use-ocamlfind arakoon.byte arakoon.native arakoon_client.cma arakoon_client.cmxa arakoon_client.a arakoon_client.cmxs plugin_helper.cmi
+	ocamlbuild -j $(JOBS) -use-ocamlfind arakoon.byte arakoon.native arakoon_client.cma arakoon_client.cmxa arakoon_client.a arakoon_client.cmxs plugin_helper.cmi
 
 bench:
 	ocamlbuild -use-ocamlfind bs_bench.native

From f4060fb41d69b1292e4ad3d8b0a0e1ddea33cc57 Mon Sep 17 00:00:00 2001
From: Yuriy Vostrikov <yuriy.vostrikov@openvstorage.com>
Date: Thu, 18 Jan 2018 16:37:44 +0300
Subject: [PATCH 328/373] Silence nonrelevant error

---
 myocamlbuild.ml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/myocamlbuild.ml b/myocamlbuild.ml
index 54d068d7..1191aff4 100644
--- a/myocamlbuild.ml
+++ b/myocamlbuild.ml
@@ -70,7 +70,7 @@ let time () =
     tm.tm_hour tm.tm_min tm.tm_sec
 
 let major_minor_patch () =
-  let tag_version = run_cmd "git describe --tags --exact-match --dirty"
+  let tag_version = run_cmd "git describe --tags --exact-match --dirty 2>/dev/null"
   and branch_version = run_cmd "git describe --all" in
 
   let (major, minor, patch) =

From 176a0e7637a0704e5e1c2e528d9ac7d399dbb68f Mon Sep 17 00:00:00 2001
From: Yuriy Vostrikov <yuriy.vostrikov@openvstorage.com>
Date: Thu, 18 Jan 2018 16:45:42 +0300
Subject: [PATCH 329/373] Pass -use-ocamlfind implicitly

---
 Makefile        | 8 +++-----
 myocamlbuild.ml | 2 ++
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/Makefile b/Makefile
index bd61f624..b7a637b5 100644
--- a/Makefile
+++ b/Makefile
@@ -18,10 +18,10 @@ clean:
 	rm -f  ./arakoon.byte ./arakoon.native
 
 build:
-	ocamlbuild -j $(JOBS) -use-ocamlfind arakoon.byte arakoon.native arakoon_client.cma arakoon_client.cmxa arakoon_client.a arakoon_client.cmxs plugin_helper.cmi
+	ocamlbuild -j $(JOBS) arakoon.byte arakoon.native arakoon_client.cma arakoon_client.cmxa arakoon_client.a arakoon_client.cmxs plugin_helper.cmi
 
 bench:
-	ocamlbuild -use-ocamlfind bs_bench.native
+	ocamlbuild bs_bench.native
 
 test:
 	./arakoon.native --run-all-tests
@@ -113,9 +113,7 @@ uninstall_client:
 	$(OCAML_FIND) remove arakoon_client -destdir $(OCAML_LIBDIR)
 
 coverage:
-	ocamlbuild -use-ocamlfind \
-	-tag use_bisect \
-	arakoon.d.byte
+	ocamlbuild -tag use_bisect arakoon.d.byte
 
 man:
 	ln -s ./arakoon.native arakoon
diff --git a/myocamlbuild.ml b/myocamlbuild.ml
index 1191aff4..d96a086a 100644
--- a/myocamlbuild.ml
+++ b/myocamlbuild.ml
@@ -114,6 +114,8 @@ let path_to_bisect () =
 
 
 let _ = dispatch & function
+     | Before_options ->
+        Options.use_ocamlfind := true
     | After_rules ->
       rule "arakoon_version.ml" ~prod:"arakoon_version.ml" make_version;
 

From a3daf4bbc237cb88961ff458a33a1ff7a3137f52 Mon Sep 17 00:00:00 2001
From: Yuriy Vostrikov <yuriy.vostrikov@openvstorage.com>
Date: Thu, 18 Jan 2018 16:52:53 +0300
Subject: [PATCH 330/373] Build arakoon.native before testing

---
 Makefile | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index b7a637b5..807d6ec7 100644
--- a/Makefile
+++ b/Makefile
@@ -23,7 +23,11 @@ build:
 bench:
 	ocamlbuild bs_bench.native
 
-test:
+arakoon.byte arakoon.native:
+	ocamlbuild -j $(JOBS) $@
+
+test:	arakoon.native
+	ocamlbuild -j $(JOBS) arakoon.native
 	./arakoon.native --run-all-tests
 
 install: install_client install_server man
@@ -120,7 +124,7 @@ man:
 	help2man --name='Arakoon, a consistent key value store' ./arakoon > debian/arakoon.man
 	rm arakoon
 
-.PHONY: install test build install_client bench
+.PHONY: install test build install_client bench arakoon.byte arakoon.native
 
 
 indent-tabs-to-spaces:

From 2e602c20103cc36519bd4cd4fa04e5fd0bb56048 Mon Sep 17 00:00:00 2001
From: Yuriy Vostrikov <yuriy.vostrikov@openvstorage.com>
Date: Fri, 19 Jan 2018 13:16:54 +0300
Subject: [PATCH 331/373] Remove unused read_only

---
 src/paxos/multi_paxos_fsm.ml  | 26 --------------------------
 src/paxos/multi_paxos_test.ml | 15 +++++----------
 src/paxos/multi_paxos_type.ml |  3 ---
 3 files changed, 5 insertions(+), 39 deletions(-)

diff --git a/src/paxos/multi_paxos_fsm.ml b/src/paxos/multi_paxos_fsm.ml
index 34bcdcc1..e82443ba 100644
--- a/src/paxos/multi_paxos_fsm.ml
+++ b/src/paxos/multi_paxos_fsm.ml
@@ -53,12 +53,6 @@ let election_suggest (type s) constants (n, counter) () =
   let state = (n, i, who_voted, v_lims, [], counter) in
   Fsm.return (Promises_check_done state)
 
-let read_only constants (_state : unit) () =
-  Lwt_unix.sleep 60.0 >>= fun () ->
-  Logger.debug_f_ "%s: read_only ..." constants.me >>= fun () ->
-  Fsm.return Read_only
-
-
 (* a potential master is waiting for promises and if enough
    promises are received the value is accepted and Accept is broadcasted *)
 let promises_check_done constants state () =
@@ -583,8 +577,6 @@ let machine constants =
 
     | Election_suggest state ->
       (Unit_arg (election_suggest constants state), nop)
-    | Read_only ->
-      (Unit_arg (read_only constants ()), nop)
     | Start_transition -> failwith "Start_transition?"
 
 (* Warning: This currently means we have only 1 fsm / executable. *)
@@ -814,24 +806,6 @@ let enter_simple_paxos (type s) ?(stop = ref false) constants buffers current_i
       run (election_suggest constants (current_n, 0))
     end
 
-let enter_read_only ?(stop = ref false) constants buffers _current_i =
-  let me = constants.me in
-  Logger.debug_f_ "%s: +starting FSM for read_only." me >>= fun () ->
-  let trace = trace_transition in
-  let produce = paxos_produce buffers constants in
-  Lwt.catch
-    (fun () ->
-       Fsm.loop ~trace ~stop
-         (_execute_effects constants)
-         produce
-         (machine constants)
-         (read_only constants ())
-    )
-    (fun exn ->
-       Logger.warning_ ~exn "READ ONLY BAILS OUT" >>= fun () ->
-       Lwt.fail exn
-    )
-
 let expect_run_forced_slave constants buffers expected step_count new_i =
   Logger.debug_f_ "%s: +starting forced_slave FSM with expect" constants.me >>= fun () ->
   let produce = paxos_produce buffers constants in
diff --git a/src/paxos/multi_paxos_test.ml b/src/paxos/multi_paxos_test.ml
index ac3d1842..bfee6468 100644
--- a/src/paxos/multi_paxos_test.ml
+++ b/src/paxos/multi_paxos_test.ml
@@ -129,8 +129,7 @@ let test_generic network_factory n_nodes () =
               | Multi_paxos_type.Accepteds_check_done _
               | Multi_paxos_type.Master_consensus _
               | Multi_paxos_type.Stable_master _
-              | Multi_paxos_type.Master_dictate _
-              | Multi_paxos_type.Read_only -> Lwt.return None
+              | Multi_paxos_type.Master_dictate _ -> Lwt.return None
           in
           let client_buffer = Lwt_buffer.create () in
           let inject_buffer = Lwt_buffer.create_fixed_capacity 1 in
@@ -173,8 +172,7 @@ let test_generic network_factory n_nodes () =
         | Multi_paxos_type.Wait_for_accepteds _
         | Multi_paxos_type.Accepteds_check_done _
         | Multi_paxos_type.Master_consensus _
-        | Multi_paxos_type.Master_dictate _
-        | Multi_paxos_type.Read_only  -> Lwt.return None
+        | Multi_paxos_type.Master_dictate _ -> Lwt.return None
     in
     let inject_buffer = Lwt_buffer.create () in
     let election_timeout_buffer = Lwt_buffer.create () in
@@ -319,8 +317,7 @@ let test_master_loop network_factory ()  =
         | Multi_paxos_type.Wait_for_accepteds _
         | Multi_paxos_type.Accepteds_check_done _
         | Multi_paxos_type.Master_consensus _
-        | Multi_paxos_type.Master_dictate _
-        | Multi_paxos_type.Read_only -> Lwt.return None
+        | Multi_paxos_type.Master_dictate _ -> Lwt.return None
     in
     let current_n = Sn.start in
     let buffers =
@@ -459,8 +456,7 @@ let test_simulation filters () =
         | Multi_paxos_type.Wait_for_accepteds _
         | Multi_paxos_type.Accepteds_check_done _
         | Multi_paxos_type.Master_consensus _
-        | Multi_paxos_type.Master_dictate _
-        | Multi_paxos_type.Read_only -> Lwt.return None
+        | Multi_paxos_type.Master_dictate _ -> Lwt.return None
     in
     let buffers = Multi_paxos_fsm.make_buffers
                     (client_buffer,
@@ -500,8 +496,7 @@ let test_simulation filters () =
         | Multi_paxos_type.Wait_for_accepteds _
         | Multi_paxos_type.Accepteds_check_done _
         | Multi_paxos_type.Master_consensus _
-        | Multi_paxos_type.Master_dictate _
-        | Multi_paxos_type.Read_only -> Lwt.return None
+        | Multi_paxos_type.Master_dictate _ -> Lwt.return None
     in
     let buffers = Multi_paxos_fsm.make_buffers
                     (client_buffer,
diff --git a/src/paxos/multi_paxos_type.ml b/src/paxos/multi_paxos_type.ml
index ebc3d471..81003bfd 100644
--- a/src/paxos/multi_paxos_type.ml
+++ b/src/paxos/multi_paxos_type.ml
@@ -60,8 +60,6 @@ type transitions =
   | Master_consensus of master_state
   | Stable_master    of (n * i * slave_awaiters)
   | Master_dictate   of master_state
-  (* read only *)
-  | Read_only
 
 (* utility functions *)
 let show_transition = function
@@ -77,7 +75,6 @@ let show_transition = function
   | Master_consensus _ -> "Master_consensus"
   | Stable_master _ -> "Stable_master"
   | Master_dictate _ -> "Master_dictate"
-  | Read_only -> "Read_only"
 
 type effect =
   | ELog of (unit -> string)

From d56ecceb7303107346ac180b91ee507980ebb37c Mon Sep 17 00:00:00 2001
From: Yuriy Vostrikov <yuriy.vostrikov@openvstorage.com>
Date: Tue, 23 Jan 2018 12:46:30 +0300
Subject: [PATCH 332/373] Do not polute $HOME

---
 pylabs/Compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pylabs/Compat.py b/pylabs/Compat.py
index 684e6c5a..ad570041 100644
--- a/pylabs/Compat.py
+++ b/pylabs/Compat.py
@@ -144,7 +144,7 @@ def copyDirTree(self,src,dst):
 
     root = os.environ.get('TEST_HOME')
     if root is None:
-        root = os.environ['HOME']
+        raise Exception("TEST_HOME is undefined")
 
     _base = root
     tmpDir = _base + '/tmp'

From fb4d17757d5b39314ea969bdaa8b1f79b481c2a7 Mon Sep 17 00:00:00 2001
From: Yuriy Vostrikov <yuriy.vostrikov@openvstorage.com>
Date: Tue, 23 Jan 2018 12:46:53 +0300
Subject: [PATCH 333/373] Decrease op_count to sane value

---
 pylabs/test/server/right/system_tests_learner.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pylabs/test/server/right/system_tests_learner.py b/pylabs/test/server/right/system_tests_learner.py
index 221479ea..c0ad800e 100644
--- a/pylabs/test/server/right/system_tests_learner.py
+++ b/pylabs/test/server/right/system_tests_learner.py
@@ -24,7 +24,7 @@
 
 @Common.with_custom_setup(Common.setup_2_nodes_mini, Common.basic_teardown)
 def test_learner():
-    op_count = 54321
+    op_count = 5432
     Common.iterate_n_times(op_count, Common.simple_set)
     cluster = Common._getCluster(Common.cluster_id)
     logging.info("adding learner")

From a31ae73804060f018c670cdd00e0adde29cf0060 Mon Sep 17 00:00:00 2001
From: Yuriy Vostrikov <yuriy.vostrikov@openvstorage.com>
Date: Tue, 23 Jan 2018 12:47:43 +0300
Subject: [PATCH 334/373] Indent

---
 src/tools/fsm.ml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/tools/fsm.ml b/src/tools/fsm.ml
index 8fe0c7d7..789e908e 100644
--- a/src/tools/fsm.ml
+++ b/src/tools/fsm.ml
@@ -52,7 +52,8 @@ let loop ?(trace=nop_trace)
         let arg, product_type = lookup key in
         match arg with
           | Unit_arg next -> _step_unit next
-          | Msg_arg next -> produce product_type >>= fun msg -> _step_msg next msg
+          | Msg_arg next -> produce product_type >>= fun msg ->
+                            _step_msg next msg
       end
   and _step_unit transition =
     transition () >>= fun (key, es) ->

From e32591391bd4a47dcbaf43c78ad5ba89c930df1d Mon Sep 17 00:00:00 2001
From: Yuriy Vostrikov <yuriy.vostrikov@openvstorage.com>
Date: Wed, 24 Jan 2018 13:12:58 +0300
Subject: [PATCH 335/373] Add comments

---
 src/paxos/master_type.ml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/paxos/master_type.ml b/src/paxos/master_type.ml
index c73fc69b..5795ac23 100644
--- a/src/paxos/master_type.ml
+++ b/src/paxos/master_type.ml
@@ -18,9 +18,10 @@ limitations under the License.
 
 type master =
   | Elected
-  | ReadOnly
-  | Forced of string
-  | Preferred of string list
+  | ReadOnly (* it's possible to start a 1 node cluster in read-only mode *)
+  | Forced of string (* master needs to be that node or nothing can be updated. *)
+  | Preferred of string list (* non-preferred nodes will always back off if
+                                an up to date preferred node wants to be master. *)
 
 let master2s = function
   | Elected -> "Elected"

From 82b4f1da765e9538fdcbb70b36a67f596bfb96cd Mon Sep 17 00:00:00 2001
From: Yuriy Vostrikov <yuriy.vostrikov@openvstorage.com>
Date: Wed, 24 Jan 2018 15:06:02 +0300
Subject: [PATCH 336/373] Factor out decline_client_update

---
 src/paxos/slave.ml | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/src/paxos/slave.ml b/src/paxos/slave.ml
index 27f75d03..7c3d0403 100644
--- a/src/paxos/slave.ml
+++ b/src/paxos/slave.ml
@@ -54,6 +54,16 @@ let time_for_elections ?invalidate_lease_start_until (type s) constants =
       end
   end
 
+let decline_client_update ufs =
+  let finished_funs = List.map (fun (_ ,_, f) -> f) ufs in
+  let result = Store.Update_fail (Arakoon_exc.E_NOT_MASTER, "Not_Master") in
+  let rec loop = function
+    | []       -> Lwt.return ()
+    | f :: ffs -> f result >>= fun () ->
+                  loop ffs
+  in
+  loop finished_funs
+
 (* a forced slave or someone who is outbidded sends a fake prepare
    in order to receive detailed info from the others in a Nak msg *)
 let slave_fake_prepare constants (current_i,current_n) () =
@@ -243,15 +253,8 @@ let slave_steady_state (type s) constants state event =
            that allows clients to get through before the node became a slave
            but I know I'm a slave now, so I let the update fail.
         *)
-        let finished_funs = List.map (fun (_,_,f) -> f) ufs in
-        let result = Store.Update_fail (Arakoon_exc.E_NOT_MASTER, "Not_Master") in
-        let rec loop = function
-          | []       -> Lwt.return ()
-          | f :: ffs -> f result >>= fun () ->
-            loop ffs
-        in
-        loop finished_funs >>= fun () ->
-        Fsm.return  (Slave_steady_state state)
+        decline_client_update ufs >>= fun () ->
+        Fsm.return (Slave_steady_state state)
       end
     | Quiesce (mode, sleep,awake) ->
       handle_quiesce_request (module S) constants.store mode sleep awake >>= fun () ->

From 3c041b7a6e026718e0305d43b261e53d6ecd87be Mon Sep 17 00:00:00 2001
From: Yuriy Vostrikov <yuriy.vostrikov@openvstorage.com>
Date: Wed, 24 Jan 2018 15:06:27 +0300
Subject: [PATCH 337/373] Simplify decline_client_update

---
 src/paxos/slave.ml | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/src/paxos/slave.ml b/src/paxos/slave.ml
index 7c3d0403..5aa7846d 100644
--- a/src/paxos/slave.ml
+++ b/src/paxos/slave.ml
@@ -55,14 +55,8 @@ let time_for_elections ?invalidate_lease_start_until (type s) constants =
   end
 
 let decline_client_update ufs =
-  let finished_funs = List.map (fun (_ ,_, f) -> f) ufs in
-  let result = Store.Update_fail (Arakoon_exc.E_NOT_MASTER, "Not_Master") in
-  let rec loop = function
-    | []       -> Lwt.return ()
-    | f :: ffs -> f result >>= fun () ->
-                  loop ffs
-  in
-  loop finished_funs
+  let fail = Store.Update_fail (Arakoon_exc.E_NOT_MASTER, "Not_Master") in
+  join (List.map (fun (_, _, f) -> f fail) ufs)
 
 (* a forced slave or someone who is outbidded sends a fake prepare
    in order to receive detailed info from the others in a Nak msg *)

From 926c8ea61c75f06f65445c1ffb80b5feba66e583 Mon Sep 17 00:00:00 2001
From: Yuriy Vostrikov <yuriy.vostrikov@openvstorage.com>
Date: Wed, 24 Jan 2018 19:16:23 +0300
Subject: [PATCH 338/373] Split slave_steady_state

---
 src/paxos/slave.ml | 316 +++++++++++++++++++++++----------------------
 1 file changed, 159 insertions(+), 157 deletions(-)

diff --git a/src/paxos/slave.ml b/src/paxos/slave.ml
index 5aa7846d..9152cc1e 100644
--- a/src/paxos/slave.ml
+++ b/src/paxos/slave.ml
@@ -72,175 +72,172 @@ let slave_fake_prepare constants (current_i,current_n) () =
   in
   Fsm.return ~sides:sides (Slave_steady_state (current_n, current_i, None))
 
-(* a pending slave that is in sync on i and is ready
-   to receive accepts *)
-let slave_steady_state (type s) constants state event =
-  let (n,i,maybe_previous) = state in
-  let store = constants.store in
-  let module S = (val constants.store_module : Store.STORE with type t = s) in
-  let handle_timeout ?invalidate_lease_start_until n' i' =
-    if not (is_election constants)
-    then
+let handle_timeout ((n,i,maybe_previous) as state) constants invalidate_lease_start_until n' i' =
+  if not (is_election constants)
+  then
+    begin
+      let log_e = ELog (fun () ->
+          Printf.sprintf "slave_steady_state: Ignoring old timeout because the master is forced or readonly")
+      in
+      Fsm.return ~sides:[log_e] (Slave_steady_state state)
+    end
+  else if n' < n || i' < i
+  then
+    begin
+      let log_e = ELog (fun () ->
+          Printf.sprintf "slave_steady_state: Ingoring old timeout (n'=%s n=%s i'=%s i=%s)" (Sn.string_of n') (Sn.string_of n) (Sn.string_of i') (Sn.string_of i))
+      in
+      Fsm.return ~sides:[log_e] (Slave_steady_state state)
+    end
+  else if constants.is_learner
+  then
+    let ns = (Sn.string_of n) in
+    let log_e = ELog (fun () ->
+        Printf.sprintf "steady_state: ignoring lease expiration because I am a learner (n=%s)" ns)
+    in
+    Fsm.return ~sides:[log_e] (Slave_steady_state state)
+  else
+    let elections_needed,why = time_for_elections ?invalidate_lease_start_until constants in
+    if elections_needed then
       begin
         let log_e = ELog (fun () ->
-            Printf.sprintf "slave_steady_state: Ignoring old timeout because the master is forced or readonly")
-        in
-        Fsm.return ~sides:[log_e] (Slave_steady_state state)
+            Printf.sprintf "slave_steady_state: Elections needed because %s" why) in
+        let new_n = update_n constants n in
+        Fsm.return ~sides:[log_e] (Election_suggest (new_n, 0))
       end
-    else if n' < n || i' < i
-    then
+    else
       begin
         let log_e = ELog (fun () ->
-            Printf.sprintf "slave_steady_state: Ingoring old timeout (n'=%s n=%s i'=%s i=%s)" (Sn.string_of n') (Sn.string_of n) (Sn.string_of i') (Sn.string_of i))
-        in
+            Printf.sprintf "slave_steady_state: Elections not needed because %s" why) in
         Fsm.return ~sides:[log_e] (Slave_steady_state state)
       end
-    else if constants.is_learner
-    then
-      let ns = (Sn.string_of n) in
-      let log_e = ELog (fun () ->
-          Printf.sprintf "steady_state: ignoring lease expiration because I am a learner (n=%s)" ns)
-      in
-      Fsm.return ~sides:[log_e] (Slave_steady_state state)
-    else
-      let elections_needed,why = time_for_elections ?invalidate_lease_start_until constants in
-      if elections_needed then
+
+let handle_from_node ((n,i,maybe_previous) as state) (type s) (module S : Store.STORE with type t = s) constants msg source  =
+  let store = constants.store in
+  let log_e0 = ELog (fun () ->
+      Printf.sprintf "slave_steady_state n:%s i:%s got %S from %s"
+        (Sn.string_of n) (Sn.string_of i) (string_of msg) source
+    )
+  in
+  let accept_value oconsensus n' i' v msg =
+    let reply = Accepted(n',i') in
+    let accept_e = EAccept (v,n',i') in
+    let send_e = ESend(reply, source) in
+    let log_e = ELog (fun () ->
+        Printf.sprintf msg (string_of reply)
+      )
+    in
+    let sides = log_e0::accept_e::send_e::log_e::oconsensus in
+    Fsm.return ~sides (Slave_steady_state (n', Sn.succ i', Some v))
+  in
+  let get_store_i () = match S.consensus_i store with
+    | None -> Sn.pred Sn.start
+    | Some store_i -> store_i in
+  match msg with
+    | Accept (n',i',v) when (n',i') = (n,i) ->
+      begin
+        let () = constants.on_witness source i' in
         begin
-          let log_e = ELog (fun () ->
-              Printf.sprintf "slave_steady_state: Elections needed because %s" why) in
-          let new_n = update_n constants n in
-          Fsm.return ~sides:[log_e] (Election_suggest (new_n, 0))
+          (* we should have either a previous value received for this n
+               (from same master) in that case store_i == i - 2
+             or we should have no previous value and store_i = i - 1 *)
+          let store_i = get_store_i () in
+          begin
+            match maybe_previous with
+              | None ->
+                if Sn.sub i store_i = 1L
+                then
+                  Logger.debug_f_ "%s: slave: no previous, so not pushing anything" constants.me >>= fun () ->
+                  Lwt.return []
+                else
+                  paxos_fatal constants.me "slave: no previous, mismatch store_i = %Li, i = %Li" store_i i
+              | Some previous ->
+                if Sn.sub i store_i = 2L
+                then
+                  begin
+                    Logger.debug_f_ "%s: slave: have previous, so that implies consensus" constants.me >>= fun () ->
+                    Lwt.return [EConsensus (None, previous,n,Sn.pred i)]
+                  end
+                else
+                  paxos_fatal constants.me "slave: with previous, mismatch store_i = %Li, i = %Li" store_i i
+          end
         end
-      else
+        >>= fun oconsensus ->
+        accept_value oconsensus n' i' v "steady_state :: replying with %S"
+      end
+    | Accept (n',i',v) when (Sn.succ i' = i) && (n' >= n) ->
+      if i' = get_store_i ()
+      then
         begin
-          let log_e = ELog (fun () ->
-              Printf.sprintf "slave_steady_state: Elections not needed because %s" why) in
-          Fsm.return ~sides:[log_e] (Slave_steady_state state)
-        end
-  in
-  match event with
-    | FromNode (msg,source) ->
-      begin
-        let log_e0 = ELog (fun () ->
-            Printf.sprintf "slave_steady_state n:%s i:%s got %S from %s"
-              (Sn.string_of n) (Sn.string_of i) (string_of msg) source
-          )
-        in
-        let accept_value oconsensus n' i' v msg =
+          (* already learned this value, so no longer writing it to the tlog,
+             but we can be polite and answer with accepted again, as it should
+             be the same paxos-value as before *)
+          Logger.debug_f_ "%s: slave, received old accepted (latest applied to store), ignoring" constants.me >>= fun () ->
           let reply = Accepted(n',i') in
-          let accept_e = EAccept (v,n',i') in
           let send_e = ESend(reply, source) in
-          let log_e = ELog (fun () ->
-              Printf.sprintf msg (string_of reply)
-            )
-          in
-          let sides = log_e0::accept_e::send_e::log_e::oconsensus in
-          Fsm.return ~sides (Slave_steady_state (n', Sn.succ i', Some v))
-        in
-        let get_store_i () =
-          match S.consensus_i store with
-          | None -> Sn.pred Sn.start
-          | Some store_i -> store_i
+          Fsm.return ~sides:[ send_e ] (Slave_steady_state (n', i, maybe_previous))
+        end
+      else
+        accept_value [] n' i' v "steady_state :: replying again to previous with %S"
+    | Accept (n',i',_) when i' > i || (i'=i && n' > n) ->
+                            (* TODO make helper function to check this! *)
+       begin
+         let log_e = ELog (fun () ->
+                           Printf.sprintf
+                             "slave_steady_state foreign (%s,%s) from %s <> local (%s,%s) discovered other master"
+                             (Sn.string_of n') (Sn.string_of i') source (Sn.string_of  n) (Sn.string_of  i)
+                          )
+         in
+         let new_state = (source, n', i') in
+         Fsm.return ~sides:[log_e0;log_e] (Slave_discovered_other_master new_state)
+       end
+    | Accept (_,_,_) ->
+      begin
+        let log_e = ELog (
+            fun () ->
+              Printf.sprintf "slave_steady_state received old %S for my i, ignoring"
+                (string_of msg) )
         in
-        match msg with
-          | Accept (n',i',v) when (n',i') = (n,i) ->
-            begin
-              let () = constants.on_witness source i' in
-              begin
-                (* we should have either a previous value received for this n
-                     (from same master) in that case store_i == i - 2
-                   or we should have no previous value and store_i = i - 1 *)
-                begin
-                  let store_i = get_store_i () in
-                  match maybe_previous with
-                    | None ->
-                      if Sn.sub i store_i = 1L
-                      then
-                        Logger.debug_f_ "%s: slave: no previous, so not pushing anything" constants.me >>= fun () ->
-                        Lwt.return []
-                      else
-                        paxos_fatal constants.me "slave: no previous, mismatch store_i = %Li, i = %Li" store_i i
-                    | Some previous ->
-                      if Sn.sub i store_i = 2L
-                      then
-                        begin
-                          Logger.debug_f_ "%s: slave: have previous, so that implies consensus" constants.me >>= fun () ->
-                          Lwt.return [EConsensus (None, previous,n,Sn.pred i)]
-                        end
-                      else
-                        paxos_fatal constants.me "slave: with previous, mismatch store_i = %Li, i = %Li" store_i i
-                end
-              end
-              >>= fun oconsensus ->
-              accept_value oconsensus n' i' v "steady_state :: replying with %S"
-            end
-          | Accept (n',i',v) when (Sn.succ i' = i) && (n' >= n) ->
-            if i' = get_store_i ()
-            then
-              begin
-                (* already learned this value, so no longer writing it to the tlog,
-                   but we can be polite and answer with accepted again, as it should
-                   be the same paxos-value as before *)
-                Logger.debug_f_ "%s: slave, received old accepted (latest applied to store), ignoring" constants.me >>= fun () ->
-                let reply = Accepted(n',i') in
-                let send_e = ESend(reply, source) in
-                Fsm.return ~sides:[ send_e ] (Slave_steady_state (n', i, maybe_previous))
-              end
-            else
-              accept_value [] n' i' v "steady_state :: replying again to previous with %S"
-          | Accept (n',i',_) when i' > i || (i'=i && n' > n) ->
-                                  (* TODO make helper function to check this! *)
-             begin
-               let log_e = ELog (fun () ->
-                                 Printf.sprintf
-                                   "slave_steady_state foreign (%s,%s) from %s <> local (%s,%s) discovered other master"
-                                   (Sn.string_of n') (Sn.string_of i') source (Sn.string_of  n) (Sn.string_of  i)
-                                )
-               in
-               let new_state = (source, n', i') in
-               Fsm.return ~sides:[log_e0;log_e] (Slave_discovered_other_master new_state)
-             end
-          | Accept (_,_,_) ->
-            begin
-              let log_e = ELog (
-                  fun () ->
-                    Printf.sprintf "slave_steady_state received old %S for my i, ignoring"
-                      (string_of msg) )
-              in
-              Fsm.return ~sides:[log_e0;log_e] (Slave_steady_state state)
-            end
-          | Prepare(n',i') ->
-            begin
-              handle_prepare constants source n n' i' >>= function
-              | Prepare_dropped
-              | Nak_sent ->
-                Fsm.return ~sides:[log_e0] (Slave_steady_state state)
-              | Promise_sent_up2date ->
-                let next_i = S.get_succ_store_i constants.store in
-                start_lease_expiration_thread constants >>= fun () ->
-                Fsm.return (Slave_steady_state (n', next_i, None))
-              | Promise_sent_needs_catchup ->
-                let new_state = (source, n', i') in
-                Fsm.return ~sides:[log_e0] (Slave_discovered_other_master new_state)
-            end
-          | Nak(_,(n2, i2)) when i2 > i ->
-             begin
-               Logger.debug_f_ "%s: got %s => go to catchup" constants.me (string_of msg) >>= fun () ->
-               Fsm.return (Slave_discovered_other_master (source, n2, i2))
-             end
-          | Nak _
-          | Promise _
-          | Accepted _ ->
-            let log_e = ELog (fun () ->
-                Printf.sprintf "steady state :: dropping %s" (string_of msg)
-              )
-            in
-            Fsm.return ~sides:[log_e0;log_e] (Slave_steady_state state)
+        Fsm.return ~sides:[log_e0;log_e] (Slave_steady_state state)
       end
+    | Prepare(n',i') ->
+      begin
+        handle_prepare constants source n n' i' >>= function
+        | Prepare_dropped
+        | Nak_sent ->
+          Fsm.return ~sides:[log_e0] (Slave_steady_state state)
+        | Promise_sent_up2date ->
+          let next_i = S.get_succ_store_i constants.store in
+          start_lease_expiration_thread constants >>= fun () ->
+          Fsm.return (Slave_steady_state (n', next_i, None))
+        | Promise_sent_needs_catchup ->
+          let new_state = (source, n', i') in
+          Fsm.return ~sides:[log_e0] (Slave_discovered_other_master new_state)
+      end
+    | Nak(_,(n2, i2)) when i2 > i ->
+       begin
+         Logger.debug_f_ "%s: got %s => go to catchup" constants.me (string_of msg) >>= fun () ->
+         Fsm.return (Slave_discovered_other_master (source, n2, i2))
+       end
+    | Nak _
+    | Promise _
+    | Accepted _ ->
+      let log_e = ELog (fun () ->
+          Printf.sprintf "steady state :: dropping %s" (string_of msg)
+        )
+      in
+      Fsm.return ~sides:[log_e0;log_e] (Slave_steady_state state)
+
+let common_steady_state (type s) handle_from_node handle_timeout next constants state event =
+  let (n,i,maybe_previous) = state in
+  let module S = (val constants.store_module : Store.STORE with type t = s) in
+  match event with
+    | FromNode (msg,source) ->
+      handle_from_node state (module S : Store.STORE with type t = s) constants msg source
     | ElectionTimeout (n', i') ->
-      handle_timeout n' i'
+      handle_timeout state constants None n' i'
     | LeaseExpired (lease_start) ->
-      handle_timeout ~invalidate_lease_start_until:lease_start n i
+      handle_timeout state constants (Some lease_start) n i
     | FromClient ufs ->
       begin
         (* there is a window in election
@@ -248,18 +245,23 @@ let slave_steady_state (type s) constants state event =
            but I know I'm a slave now, so I let the update fail.
         *)
         decline_client_update ufs >>= fun () ->
-        Fsm.return (Slave_steady_state state)
+        Fsm.return (next state)
       end
     | Quiesce (mode, sleep,awake) ->
       handle_quiesce_request (module S) constants.store mode sleep awake >>= fun () ->
-      Fsm.return (Slave_steady_state state)
+      Fsm.return (next state)
     | Unquiesce ->
       handle_unquiesce_request constants >>= fun () ->
-      Fsm.return  (Slave_steady_state state)
+      Fsm.return  (next state)
     | DropMaster (sleep, awake) ->
       Multi_paxos.safe_wakeup sleep awake () >>= fun () ->
-      Fsm.return (Slave_steady_state state)
+      Fsm.return (next state)
 
+(* a pending slave that is in sync on i and is ready
+   to receive accepts *)
+let slave_steady_state (type s) constants state event =
+  common_steady_state handle_from_node handle_timeout (fun x -> Slave_steady_state x)
+  constants state event
 
 (* a pending slave that discovered another master has to do
    catchup and then go to steady state *)

From 64a56904a186c49a4481ed634681c77dde89beb4 Mon Sep 17 00:00:00 2001
From: Yuriy Vostrikov <yuriy.vostrikov@openvstorage.com>
Date: Thu, 25 Jan 2018 11:52:44 +0300
Subject: [PATCH 339/373] Learner node

---
 .../arakoon_ext/server/ArakoonManagement.py   |   8 +-
 .../test/server/right/system_tests_learner.py |   3 +-
 src/node/node_cfg.ml                          |  10 +-
 src/node/node_main.ml                         |  39 +++---
 src/paxos/learner.ml                          | 122 ++++++++++++++++++
 src/paxos/multi_paxos.ml                      |  14 +-
 src/paxos/multi_paxos_fsm.ml                  |  29 +++++
 src/paxos/multi_paxos_test.ml                 |   5 +-
 src/paxos/multi_paxos_type.ml                 |   7 +
 src/paxos/slave.ml                            |  19 +--
 src/system/startup.ml                         |   1 -
 11 files changed, 197 insertions(+), 60 deletions(-)
 create mode 100644 src/paxos/learner.ml

diff --git a/pylabs/extensions/arakoon_ext/server/ArakoonManagement.py b/pylabs/extensions/arakoon_ext/server/ArakoonManagement.py
index 557cbe27..8f87e6c3 100644
--- a/pylabs/extensions/arakoon_ext/server/ArakoonManagement.py
+++ b/pylabs/extensions/arakoon_ext/server/ArakoonManagement.py
@@ -194,7 +194,6 @@ def addNode(self,
                 tlogDir = None,
                 wrapper = None,
                 isLearner = False,
-                targets = None,
                 isLocal = False,
                 logConfig = None,
                 batchedTransactionConfig = None,
@@ -215,7 +214,6 @@ def addNode(self,
         @param tlogDir : the directory used for tlogs (if none, home will be used)
         @param wrapper : wrapper line for the executable (for example 'softlimit -o 8192')
         @param isLearner : whether this node is a learner node or not
-        @param targets : for a learner node the targets (string list) it learns from
         @param isLocal : whether this node is a local node and should be added to the local nodes list
         @param logConfig : specifies the log config to be used for this node
         @param batchedTransactionConfig : specifies the batched transaction config to be used for this node
@@ -235,8 +233,7 @@ def addNode(self,
 
         if name in nodes:
             raise Exception("node %s already present" % name )
-        if not isLearner:
-            nodes.append(name)
+        nodes.append(name)
 
         config.add_section(name)
 
@@ -276,9 +273,6 @@ def addNode(self,
 
         if isLearner:
             config.set(name, "learner", "true")
-            if targets is None:
-                targets = self.listNodes()
-            config.set(name, "targets", string.join(targets,","))
 
         if isWitness:
             config.set(name, "witness", "true")
diff --git a/pylabs/test/server/right/system_tests_learner.py b/pylabs/test/server/right/system_tests_learner.py
index c0ad800e..e22a2f92 100644
--- a/pylabs/test/server/right/system_tests_learner.py
+++ b/pylabs/test/server/right/system_tests_learner.py
@@ -39,8 +39,7 @@ def test_learner():
                     headDir = head_dir,
                     logLevel = 'debug',
                     home = db_dir,
-                    isLearner = True,
-                    targets = [Common.node_names[0]])
+                    isLearner = True)
     cfg = cluster._getConfigFile()
     logging.info("cfg=%s", X.cfg2str(cfg))
     cluster.disableFsync([name])
diff --git a/src/node/node_cfg.ml b/src/node/node_cfg.ml
index 70fff0d0..00ebba38 100644
--- a/src/node/node_cfg.ml
+++ b/src/node/node_cfg.ml
@@ -136,7 +136,6 @@ module Node_cfg = struct
             is_laggy : bool;
             is_learner : bool;
             is_witness : bool;
-            targets : string list;
             compressor : Compression.compressor;
             fsync : bool;
             _fsync_tlog_dir : bool;
@@ -168,7 +167,6 @@ module Node_cfg = struct
            ; "is_laggy", bool t.is_laggy
            ; "is_learner", bool t.is_learner
            ; "is_witness", bool t.is_witness
-           ; "targets", list id t.targets
            ; "compressor", Compression.compressor2s t.compressor
            ; "fsync", bool t.fsync
            ; "is_test", bool t.is_test
@@ -306,7 +304,6 @@ module Node_cfg = struct
         is_laggy = false;
         is_learner = false;
         is_witness = false;
-        targets = [];
         compressor = Compression.Snappy;
         fsync = false;
         _fsync_tlog_dir = false;
@@ -563,11 +560,7 @@ module Node_cfg = struct
                             "__tainted_fsync_tlog_dir"
                             Ini.p_bool
                             (Ini.default true) in
-    let targets =
-      if is_learner
-      then Ini.get inifile node_name "targets" Ini.p_string_list Ini.required
-      else []
-    in
+
     let lease_period = _get_lease_period inifile in
     let log_dir =
       try get_string "log_dir"
@@ -636,7 +629,6 @@ module Node_cfg = struct
      is_laggy;
      is_learner;
      is_witness;
-     targets;
      compressor;
      fsync;
      _fsync_tlog_dir;
diff --git a/src/node/node_main.ml b/src/node/node_main.ml
index eb7cb797..b1f41f33 100644
--- a/src/node/node_main.ml
+++ b/src/node/node_main.ml
@@ -450,9 +450,8 @@ let _main_2 (type s)
       let master = cluster_cfg._master in
       let lease_period = cluster_cfg._lease_period in
       let quorum_function = cluster_cfg.quorum_function in
-      let in_cluster_cfgs = List.filter (fun cfg -> not cfg.is_learner ) cfgs in
-      let in_cluster_names = List.map (fun cfg -> cfg.node_name) in_cluster_cfgs in
-      let n_nodes = List.length in_cluster_names in
+      let in_cluster_cfgs = List.filter (fun cfg -> not cfg.is_learner) cfgs in
+      let n_nodes = List.length in_cluster_cfgs in
 
       let my_clicfg =
         begin
@@ -465,11 +464,13 @@ let _main_2 (type s)
           ccfg
         end
       in
-      let other_names =
-        if me.is_learner
-        then me.targets
-        else List.filter ((<>) name) in_cluster_names
-      in
+
+      let others, learners = List.filter (fun cfg -> cfg.node_name <> name) cfgs
+                             |> List.partition (fun cfg -> not cfg.is_learner) in
+      let names l = List.map (fun cfg -> cfg.node_name) l in
+      let other_names = names others in
+      let learner_names = names learners in
+
       let _ = Lwt_unix.on_signal Sys.sigusr1
                 (fun i -> Lwt.ignore_result (_log_rotate me.node_name i make_config ))
       in
@@ -872,7 +873,9 @@ let _main_2 (type s)
               ~catchup_tls_ctx:catchup_tls_ctx my_name
               ~tcp_keepalive
               me.is_learner
-              other_names send
+              other_names
+              learner_names
+              send
               get_last_value
               on_accept
               on_consensus
@@ -905,21 +908,15 @@ let _main_2 (type s)
 
       let start_backend stop (master, constants, buffers, new_i) =
         let to_run =
-          if me.is_witness
-          then
+          if me.is_witness then
             Multi_paxos_fsm.enter_forced_slave
+          else if me.is_learner then
+            Multi_paxos_fsm.enter_learner
           else
             match master with
-              | Forced master  ->
-                if master = my_name
-                then Multi_paxos_fsm.enter_forced_master
-                else
-                  begin
-                    if me.is_learner
-                    then Multi_paxos_fsm.enter_simple_paxos
-                    else Multi_paxos_fsm.enter_forced_slave
-                  end
-              | Elected | ReadOnly |Preferred _  -> Multi_paxos_fsm.enter_simple_paxos
+              | Forced master when master = my_name -> Multi_paxos_fsm.enter_forced_master
+              | Forced _ -> Multi_paxos_fsm.enter_forced_slave
+              | Elected | ReadOnly | Preferred _  -> Multi_paxos_fsm.enter_simple_paxos
         in
         to_run ~stop constants buffers new_i
       in
diff --git a/src/paxos/learner.ml b/src/paxos/learner.ml
new file mode 100644
index 00000000..df3e9ac8
--- /dev/null
+++ b/src/paxos/learner.ml
@@ -0,0 +1,122 @@
+(*
+  Copyright (C) iNuron - info@openvstorage.com
+  This file is part of Open vStorage. For license information, see <LICENSE>
+*)
+
+open Multi_paxos_type
+open Multi_paxos
+open Lwt
+open Mp_msg.MPMessage
+
+
+(* a forced slave or someone who is outbidded sends a fake prepare
+   in order to receive detailed info from the others in a Nak msg *)
+let fake_prepare constants (current_i, current_n) () =
+  (* fake a prepare, and hopefully wake up a dormant master *)
+  Logger.ign_debug_f_ "Learner.fake_prepare: sending Prepare(-1, %Li)" current_i;
+  let fake = Prepare (Sn.of_int (-1), current_i) in
+  let sides = [EMCast fake] in
+  Fsm.return ~sides (Learner_steady_state (current_n, current_i, None))
+
+let handle_timeout ((n,i,maybe_previous) as state) constants invalidate_lease_start_until n' i' =
+  Logger.ign_debug_f_ "steady_state: ignoring lease expiration because I am a learner";
+  Fsm.return (Learner_steady_state state)
+
+let handle_from_node ((n,i,previous) as state) (type s) (module S : Store.STORE with type t = s)
+      {store; me;on_witness} msg source =
+  Logger.ign_debug_f_ "Learner.steady_state n:%s i:%s v:%s store_i:%Li got %S from %s"
+    (Sn.string_of n) (Sn.string_of i) (To_string.option Value.value2s previous)
+    (match S.consensus_i store with None -> -1L | Some v -> v)
+    (string_of msg) source;
+
+  let accept_value ?consensus n' i' v =
+    let tail = match consensus with
+        None -> []
+      | Some (n, i, prev) -> [EConsensus (None, prev, n, i)] in
+    let sides = EAccept (v, n', i') :: tail in
+    Fsm.return ~sides (Learner_steady_state (n', Sn.succ i', Some v))
+  in
+  let get_store_i () =
+    match S.consensus_i store with
+    | None -> Sn.pred Sn.start
+    | Some store_i -> store_i
+  in
+  match msg with
+  | Accept (n',i',v) when (n', i') = (n, i) ->
+     begin
+       on_witness source i';
+       (* we should have either a previous value received for this n
+          from same master) in that case store_i == i - 2
+          or we should have no previous value and store_i = i - 1 *)
+       let store_i = get_store_i () in
+       match previous, Sn.sub i store_i with
+         None, 1L ->
+          Logger.ign_debug_f_ "%s: learner: no previous, so not pushing anything" me;
+          accept_value n' i' v
+       | Some previous, 2L ->
+          Logger.ign_debug_f_ "%s: learner: have previous, so that implies consensus (%s, n:%Li, i:%Li)" me (Value.value2s previous) n (Sn.pred i);
+        accept_value ~consensus:(n, Sn.pred i, previous) n' i' v
+       | None, _ ->
+          paxos_fatal me "learner: no previous, mismatch store_i = %Li, i = %Li" store_i i
+       | Some _, _ ->
+          paxos_fatal me "learner: with previous, mismatch store_i = %Li, i = %Li" store_i i
+     end
+
+  | Accept (n',i',v) when (Sn.succ i' = i) && (n' >= n) ->
+     if i' = get_store_i () then
+       begin
+         Logger.ign_debug_f_ "%s: learner, received old accepted (latest applied to store), ignoring" me;
+         Fsm.return (Learner_steady_state (n', i, previous))
+       end
+     else
+       accept_value n' i' v
+
+  | Accept (n',i',_) when i' > i || (i' = i && n' > n) ->
+     Logger.ign_debug_f_ "Learner.steady_state foreign (%s,%s) from %s <> local (%s,%s) discovered other master"
+         (Sn.string_of n') (Sn.string_of i') source (Sn.string_of  n) (Sn.string_of  i);
+     Fsm.return (Learner_discovered_other_master (source, n',i'))
+
+  | Accept (_,_,_) ->
+     Logger.ign_debug_f_ "Learner.steady_state received old %S for my i, ignoring"
+       (string_of msg);
+     Fsm.return (Learner_steady_state state)
+
+  | Nak (_,(n2, i2)) when i2 > i ->
+     Logger.ign_debug_f_ "%s: got %s => go to catchup" me (string_of msg);
+     Fsm.return (Learner_discovered_other_master (source, n2, i2))
+
+  | Nak _
+  | Promise _
+  | Prepare _
+  | Accepted _ ->
+     Logger.ign_debug_f_ "Learner.steady state :: ignoring %S" (string_of msg);
+     Fsm.return (Learner_steady_state state)
+
+let steady_state (type s) constants state event =
+  Slave.common_steady_state
+    handle_from_node handle_timeout (fun x -> Learner_steady_state x)
+    constants state event
+
+(* a pending slave that discovered another master has to do
+   catchup and then go to steady state *)
+let discovered_other_master (type s) constants state () =
+  let (master, future_n, future_i) = state in
+  let me = constants.me
+  and other_cfgs = constants.other_cfgs
+  and store = constants.store
+  and tlog_coll = constants.tlog_coll
+  in
+  let module S = (val constants.store_module : Store.STORE with type t = s) in
+
+  Logger.ign_debug_f_ "%s: Learner.discovered_other_master: catching up from %s @ %s"
+                           me master (Sn.string_of future_i);
+  let cluster_id = constants.cluster_id in
+  let tls_ctx = constants.catchup_tls_ctx in
+  Catchup.catchup
+    ~tls_ctx
+    ~tcp_keepalive:constants.tcp_keepalive
+    ~stop:constants.stop
+    me other_cfgs ~cluster_id ((module S), store, tlog_coll) master >>= fun () ->
+
+  let i = S.get_succ_store_i store in
+  Fsm.return (Learner_fake_prepare (i, future_n))
diff --git a/src/paxos/multi_paxos.ml b/src/paxos/multi_paxos.ml
index 85bb6027..ddd2c745 100644
--- a/src/paxos/multi_paxos.ml
+++ b/src/paxos/multi_paxos.ml
@@ -110,6 +110,7 @@ let paxos_event2s = function
 type 'a constants =
   {me:id;
    others: id list;
+   learners: id list;
    send: MPMessage.t -> id -> id -> unit Lwt.t;
    get_value: Sn.t -> Value.t option Lwt.t;
    on_accept: Value.t * Sn.t * Sn.t -> unit Lwt.t;
@@ -148,7 +149,7 @@ let is_election constants =
     | Elected | Preferred _ -> true
     | ReadOnly | Forced _ -> false
 
-let make (type s) ~catchup_tls_ctx ~tcp_keepalive me is_learner others send get_value
+let make (type s) ~catchup_tls_ctx ~tcp_keepalive me is_learner others learners send get_value
       on_accept on_consensus on_witness
       last_witnessed quorum_function (master:master) (module S : Store.STORE with type t = s) store tlog_coll
       other_cfgs lease_expiration inject_event is_alive ~cluster_id
@@ -158,6 +159,7 @@ let make (type s) ~catchup_tls_ctx ~tcp_keepalive me is_learner others send get_
     me=me;
     is_learner;
     others;
+    learners;
     send;
     get_value;
     on_accept;
@@ -183,11 +185,11 @@ let make (type s) ~catchup_tls_ctx ~tcp_keepalive me is_learner others send get_
     max_buffer_size;
   }
 
-let mcast constants msg =
-  let send = constants.send in
-  let me = constants.me in
-  let others = constants.others in
-  Lwt_list.iter_p (fun o -> send msg me o) others
+let mcast {send; me; others; learners} msg =
+  let dst = match msg with
+      Accept _ | Nak _ -> learners @ others
+    | _ -> others in
+  Lwt_list.iter_p (send msg me) dst
 
 
 let update_n constants n =
diff --git a/src/paxos/multi_paxos_fsm.ml b/src/paxos/multi_paxos_fsm.ml
index e82443ba..e1d581a3 100644
--- a/src/paxos/multi_paxos_fsm.ml
+++ b/src/paxos/multi_paxos_fsm.ml
@@ -579,6 +579,16 @@ let machine constants =
       (Unit_arg (election_suggest constants state), nop)
     | Start_transition -> failwith "Start_transition?"
 
+
+let learner_machine constants =
+  let nop = [] in
+  let full = [Node; Inject; Election_timeout; Client] in
+  function
+    | Learner_fake_prepare state ->		(Unit_arg (Learner.fake_prepare constants state), nop)
+    | Learner_steady_state state ->		(Msg_arg (Learner.steady_state constants state), full)
+    | Learner_discovered_other_master state ->	(Unit_arg (Learner.discovered_other_master constants state), nop)
+
+
 (* Warning: This currently means we have only 1 fsm / executable. *)
 let __state = ref Start_transition
 
@@ -729,6 +739,25 @@ let _execute_effects constants e =
 
 (* the entry methods *)
 
+let enter_learner ?(stop = ref false) constants buffers new_i =
+  let me = constants.me in
+  Logger.debug_f_ "%s: +starting FSM for learner." me >>= fun () ->
+  let produce = paxos_produce buffers constants in
+  let new_n = update_n constants Sn.start in
+
+  Lwt.catch
+    (fun () ->
+       Fsm.loop ~stop
+         (_execute_effects constants)
+         produce
+         (learner_machine constants)
+         (Learner.fake_prepare constants (new_i,new_n))
+    )
+    (fun exn ->
+       Logger.warning_ ~exn "FSM BAILED due to uncaught exception"
+       >>= fun () -> Lwt.fail exn
+    )
+
 let enter_forced_slave ?(stop = ref false) constants buffers new_i =
   let me = constants.me in
   Logger.debug_f_ "%s: +starting FSM for forced_slave." me >>= fun () ->
diff --git a/src/paxos/multi_paxos_test.ml b/src/paxos/multi_paxos_test.ml
index bfee6468..98a2f3b9 100644
--- a/src/paxos/multi_paxos_test.ml
+++ b/src/paxos/multi_paxos_test.ml
@@ -68,7 +68,8 @@ let test_generic network_factory n_nodes () =
     ~fsync_tlog_dir:false ~cluster_id
   >>= fun tlog_coll ->
   let base = {me = "???";
-              others = [] ;
+              others = [];
+              learners = [];
               is_learner = false;
               send = send;
               get_value = get_value tlog_coll;
@@ -271,6 +272,7 @@ let test_master_loop network_factory ()  =
   let constants = {me = me;
                    is_learner = false;
                    others = others;
+                   learners = [];
                    send = send;
                    get_value = get_value tlog_coll;
                    on_accept = on_accept;
@@ -416,6 +418,7 @@ let test_simulation filters () =
     me = me;
     is_learner = false;
     others = ["c1";"c2"];
+    learners = [];
     send = send;
     get_value = get_value tlog_coll;
     on_accept = on_accept me;
diff --git a/src/paxos/multi_paxos_type.ml b/src/paxos/multi_paxos_type.ml
index 81003bfd..7ad65c01 100644
--- a/src/paxos/multi_paxos_type.ml
+++ b/src/paxos/multi_paxos_type.ml
@@ -61,6 +61,13 @@ type transitions =
   | Stable_master    of (n * i * slave_awaiters)
   | Master_dictate   of master_state
 
+
+type learner_transitions =
+  | Learner_fake_prepare of (n * i)
+  | Learner_steady_state of (n * i * Value.t option) (* value received for this n and previous i *)
+  | Learner_discovered_other_master of (Messaging.id *
+                                        Mp_msg.MPMessage.n * Mp_msg.MPMessage.n )
+
 (* utility functions *)
 let show_transition = function
   | Start_transition -> "Start_transition"
diff --git a/src/paxos/slave.ml b/src/paxos/slave.ml
index 9152cc1e..7a13f720 100644
--- a/src/paxos/slave.ml
+++ b/src/paxos/slave.ml
@@ -86,19 +86,12 @@ let handle_timeout ((n,i,maybe_previous) as state) constants invalidate_lease_st
     begin
       let log_e = ELog (fun () ->
           Printf.sprintf "slave_steady_state: Ingoring old timeout (n'=%s n=%s i'=%s i=%s)" (Sn.string_of n') (Sn.string_of n) (Sn.string_of i') (Sn.string_of i))
-      in
-      Fsm.return ~sides:[log_e] (Slave_steady_state state)
-    end
-  else if constants.is_learner
-  then
-    let ns = (Sn.string_of n) in
-    let log_e = ELog (fun () ->
-        Printf.sprintf "steady_state: ignoring lease expiration because I am a learner (n=%s)" ns)
-    in
-    Fsm.return ~sides:[log_e] (Slave_steady_state state)
-  else
-    let elections_needed,why = time_for_elections ?invalidate_lease_start_until constants in
-    if elections_needed then
+        in
+        Fsm.return ~sides:[log_e] (Slave_steady_state state)
+      end
+    else
+      let elections_needed,why = time_for_elections ?invalidate_lease_start_until constants in
+      if elections_needed then
       begin
         let log_e = ELog (fun () ->
             Printf.sprintf "slave_steady_state: Elections needed because %s" why) in
diff --git a/src/system/startup.ml b/src/system/startup.ml
index ece2cc82..1f36f31e 100644
--- a/src/system/startup.ml
+++ b/src/system/startup.ml
@@ -67,7 +67,6 @@ let _make_cfg name n lease_period =
     is_laggy = false;
     is_learner = false;
     is_witness = false;
-    targets = [];
     compressor = Compression.Snappy;
     fsync = false;
     _fsync_tlog_dir = false;

From 08abd982a2ec7b5a44277d05bc1bd2c9b57e92a8 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Thu, 1 Feb 2018 14:42:25 +0100
Subject: [PATCH 340/373] add inspect-cluster command

---
 src/main/arakoon.ml  | 15 +++++++-
 src/main/inspect.ml  | 90 ++++++++++++++++++++++++++++++++++++++++++++
 src/tlog/tlog_map.ml |  6 +--
 3 files changed, 105 insertions(+), 6 deletions(-)
 create mode 100644 src/main/inspect.ml

diff --git a/src/main/arakoon.ml b/src/main/arakoon.ml
index b38f693d..bedd457f 100644
--- a/src/main/arakoon.ml
+++ b/src/main/arakoon.ml
@@ -70,6 +70,7 @@ type local_action =
   | RANGE_ENTRIES
   | REV_RANGE_ENTRIES
   | VerifyStore
+  | InspectCluster
 
 type server_action =
   | Node
@@ -470,7 +471,8 @@ let main () =
      "reverse list entries within range");
     ("-left", Arg.String  (fun s -> left  := Some s), "left boundary (range query)");
     ("-right", Arg.String (fun s -> right := Some s), "right boundary (range query)");
-
+    ("--inspect-cluster", Arg.Tuple [ set_laction InspectCluster; ],
+     "inspect/compare all nodes of a cluster for divergence");
   ] in
 
   let options = [] in
@@ -559,7 +561,16 @@ let main () =
        Client_main.rev_range_entries
          ~tls !config_url
          !left !linc !right !rinc !max_results
-
+    | InspectCluster ->
+       let open Lwt.Infix in
+       let t () =
+         make_config () >>= fun cfg ->
+         Inspect.inspect_cluster
+           ~tls
+           cfg
+       in
+       Lwt_main.run (t ());
+       0
   in
   let do_server node =
     match node with
diff --git a/src/main/inspect.ml b/src/main/inspect.ml
new file mode 100644
index 00000000..9dad9cb5
--- /dev/null
+++ b/src/main/inspect.ml
@@ -0,0 +1,90 @@
+(*
+Copyright (2018) iNuron BVBA
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*)
+
+open Lwt.Infix
+open Node_cfg
+
+module S = (val (Store.make_store_module (module Batched_store.Local_store)))
+
+let ensure_dir d p =
+  Lwt.catch
+    (fun () -> File_system.mkdir d p)
+    (function | Unix.Unix_error (Unix.EEXIST, _, _) -> Lwt.return ())
+
+let inspect_cluster ~tls (cfg : Node_cfg.cluster_cfg) =
+
+  let cluster_id = cfg.cluster_id in
+  ensure_dir cluster_id 0o755 >>= fun () ->
+  Lwt_list.map_p
+    (fun node_cfg ->
+      let open Node_cfg in
+      let node_name = node_cfg.node_name in
+      let path = cluster_id ^ "/" ^ node_name in
+      ensure_dir path 0o755 >>= fun () ->
+
+      (* TODO auto handle closing of tlog... *)
+      Tlc2.make_tlc2
+        ~cluster_id:cfg.cluster_id
+        ~compressor:Compression.Snappy
+        path path path
+        ~fsync:false node_name ~fsync_tlog_dir:false
+      >>= fun tlc ->
+
+      S.make_store ~lcnum:1024 ~ncnum:512 (path ^ "/store.db") ~cluster_id
+      >>= fun store ->
+
+      Catchup.catchup_tlog
+        ~tls_ctx:tls
+        ~tcp_keepalive:Tcp_keepalive.default_tcp_keepalive
+        ~stop:(ref false)
+        cfg.cfgs
+        ~cluster_id
+        node_name
+        ((module S), store, tlc) >>= fun tlog_i ->
+
+      Lwt_log.info_f "Got head & tlogs for %s, i=%Li" node_name tlog_i >>= fun () ->
+      Lwt.return (node_name, store, tlc, tlog_i)
+    )
+    cfg.cfgs
+  >>= fun nodes ->
+
+  let minimum_last_i =
+    List.fold_left
+      (fun minimum (_, _, _, current) -> min minimum current)
+      Int64.max_int
+      nodes
+  in
+
+  Lwt_log.info_f "Got all heads & tlogs, minimal i=%Li" minimum_last_i >>= fun () ->
+
+  Lwt_list.iter_s
+    (fun (node_name, store, tlc, _) ->
+      Catchup.catchup_store
+        ~stop:(ref false)
+        node_name
+        ((module S), store, tlc)
+        minimum_last_i >>= fun () ->
+      S.close store >>= fun () ->
+
+      tlc # close ()
+    )
+    nodes >>= fun () ->
+
+  (* - diff already, tell which are same
+   * - show diff cmds
+   *)
+
+  Lwt.return ()
diff --git a/src/tlog/tlog_map.ml b/src/tlog/tlog_map.ml
index 68afe4ab..b6cfc8a7 100644
--- a/src/tlog/tlog_map.ml
+++ b/src/tlog/tlog_map.ml
@@ -158,9 +158,7 @@ let get_count tlog_names =
       let length = String.length last in
       let postfix = String.sub last (length -4) 4 in
       let number = get_number last in
-      if postfix = "tlog"
-      then number
-      else number + 1
+      number
 
 let _get_entries dir invalid_dir_exn =
   Lwt.catch
@@ -232,7 +230,7 @@ let _validate_one
       ~(check_sabotage:bool)
     : validation_result Lwt.t
   =
-  Logger.debug_f_ "_validate_one %s" tlog_name >>= fun () ->
+  Logger.info_f_ "_validate_one %s" tlog_name >>= fun () ->
   let e2s e = let i = Entry.i_of e in Printf.sprintf "(%s,_)" (Sn.string_of i) in
   let prev_entry = ref None in
   let new_index = Index.make tlog_name in

From 342634a6e1c1a2e6b49eea4304f5b7415251ae9e Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Tue, 6 Feb 2018 17:53:49 +0100
Subject: [PATCH 341/373] fix issue in tlog_map when last part of a catchup is
 a .tlx

---
 src/main/replay_main.ml |  2 +-
 src/tlog/tlog_map.ml    | 38 +++++++++++++++++++++-----------------
 2 files changed, 22 insertions(+), 18 deletions(-)

diff --git a/src/main/replay_main.ml b/src/main/replay_main.ml
index a46aee50..ef9dc09e 100644
--- a/src/main/replay_main.ml
+++ b/src/main/replay_main.ml
@@ -51,7 +51,7 @@ let replay_tlogs tlog_dir tlx_dir db_name end_i =
             end_i with
           | None ->
              begin
-               TlogMap.get_last_tlog tlog_map >>= fun (_new_c,fn) ->
+               TlogMap.get_last_tlog tlog_map >>= fun fn ->
                Tlog_map._validate_one
                  fn ""
                  ~check_marker:false
diff --git a/src/tlog/tlog_map.ml b/src/tlog/tlog_map.ml
index b6cfc8a7..99b9cfa2 100644
--- a/src/tlog/tlog_map.ml
+++ b/src/tlog/tlog_map.ml
@@ -151,14 +151,11 @@ let _get_full_path tlog_dir tlx_dir name =
     else
       Filename.concat tlog_dir name
 
-let get_count tlog_names =
+let get_last_tlog tlog_names =
   match List.length tlog_names with
-    | 0 -> 0
+    | 0 -> None
     | n -> let last = List.nth tlog_names (n-1) in
-      let length = String.length last in
-      let postfix = String.sub last (length -4) 4 in
-      let number = get_number last in
-      number
+           Some last
 
 let _get_entries dir invalid_dir_exn =
   Lwt.catch
@@ -339,9 +336,12 @@ module TlogMap = struct
                    check_marker check_sabotage
     >>= fun () ->
     _get_tlog_names tlog_dir tlx_dir >>= fun tlog_names ->
-    let tlog_number = get_count tlog_names in
-    Logger.info_f_ "tlog_number:%i" tlog_number >>= fun () ->
-    let full_path = _get_full_path tlog_dir tlx_dir (file_name tlog_number) in
+    let tlog_name = match get_last_tlog tlog_names with
+      | None -> file_name 0
+      | Some n -> n
+    in
+    Logger.info_f_ "tlog_name:%s" tlog_name >>= fun () ->
+    let full_path = _get_full_path tlog_dir tlx_dir tlog_name in
     _validate_one full_path node_id ~check_marker ~check_sabotage
     >>= fun (last, index, tlog_size) ->
     Lwt_list.fold_left_s
@@ -373,14 +373,15 @@ module TlogMap = struct
       | None -> Sn.zero
       | Some e -> Entry.i_of e
     in
-    let tlog_entries, should_roll,last_start =
+
+    let tlog_number, tlog_entries, should_roll =
       match i_to_tlog_number with
-      | []                -> 0, false, Sn.zero
-      | item :: _ when item.is_archive -> 0, true , item.i
+      | []                -> 0, 0, false
+      | item :: _ when item.is_archive -> item.n, 0, true
       | item :: _ ->
          let tlog_entries = Sn.sub last_i item.i |> Sn.to_int in
          let should_roll = tlog_entries + 1 >= tlog_max_entries in
-         tlog_entries, should_roll, item.i
+         item.n, tlog_entries, should_roll
     in
     Logger.info_f_ "_init: tlog_entries:%i should_roll:%b" tlog_entries should_roll
     >>= fun () ->
@@ -429,10 +430,13 @@ module TlogMap = struct
   let get_last_tlog t =
     Logger.debug_ "get_last_tlog" >>= fun () ->
     get_tlog_names t >>= fun tlog_names ->
-    let new_c = get_count tlog_names in
-    Logger.debug_f_ "new_c:%i" new_c >>= fun () ->
-    let full_path = _get_full_path t.tlog_dir t.tlx_dir (file_name new_c) in
-    Lwt.return (new_c, full_path)
+    let new_c = match get_last_tlog tlog_names with
+      | None -> file_name 0
+      | Some n -> n
+    in
+    Logger.debug_f_ "new_c:%s" new_c >>= fun () ->
+    let full_path = _get_full_path t.tlog_dir t.tlx_dir new_c in
+    Lwt.return full_path
 
 
   let find_tlog_file t c =

From 5b62b2a1c45dbbea57e497cf6f46870f7b8bf6e6 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Tue, 6 Feb 2018 18:06:14 +0100
Subject: [PATCH 342/373] 'unit' (ocaml) test suite should make clean first too

---
 docker/suites.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/suites.sh b/docker/suites.sh
index 08a777e0..e86beece 100755
--- a/docker/suites.sh
+++ b/docker/suites.sh
@@ -13,7 +13,7 @@ case "${1-bash}" in
         make build
         ;;
     unit)
-        make build
+        ./jenkins/common.sh
         ./arakoon.native --run-all-tests-xml testresults.xml
         x=$?
         cat testresults.xml

From 2dd4a7e064dd6107ecaaf3ba0deb7dff8fb4aaa6 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Wed, 7 Feb 2018 13:39:47 +0100
Subject: [PATCH 343/373] fix failing tests

---
 pylabs/test/server/right/test_catchup_exercises.py | 1 -
 pylabs/test/server/system_tests_common.py          | 5 -----
 src/tlog/tlog_main.ml                              | 1 +
 src/tlog/tlog_map.ml                               | 4 +++-
 src/tools/llio.ml                                  | 2 ++
 5 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/pylabs/test/server/right/test_catchup_exercises.py b/pylabs/test/server/right/test_catchup_exercises.py
index 4ee35c58..76962904 100644
--- a/pylabs/test/server/right/test_catchup_exercises.py
+++ b/pylabs/test/server/right/test_catchup_exercises.py
@@ -191,7 +191,6 @@ def test_issue125():
 
     X.removeFile('%s/035.tlog' % db_dir)
     X.removeFile('%s/034.tlx' % tlx_dir)
-    X.removeFile('%s/%s.log' % (log_dir,slave))
     X.removeFile('%s/%s.db' % (db_dir, slave))
     # we have a cluster with head.db, no db and no tlog file.
 
diff --git a/pylabs/test/server/system_tests_common.py b/pylabs/test/server/system_tests_common.py
index f2fdb0c5..88be9eee 100644
--- a/pylabs/test/server/system_tests_common.py
+++ b/pylabs/test/server/system_tests_common.py
@@ -378,11 +378,6 @@ def last_entry_code(node_id):
 def get_last_i_tlog ( node_id ):
     tlog_dump = dump_tlog ( node_id, get_last_tlog_id(node_id) )
     tlog_dump_list = tlog_dump.split("\n")
-    tlog_first_entry = tlog_dump_list[0]
-    tlog_first_i = int(tlog_first_entry.split(":") [0].lstrip(" "))
-    if tlog_first_i % tlog_entries_per_tlog != 0 :
-        test_failed = True
-        raise Exception( "Problem with tlog rollover, first entry (%d) incorrect" % tlog_first_i )
     tlog_last_entry = tlog_dump_list [-2]
     tlog_last_i = tlog_last_entry.split(":") [0].lstrip( " 0" )
     return tlog_last_i
diff --git a/src/tlog/tlog_main.ml b/src/tlog/tlog_main.ml
index 3a243520..0d9b06a7 100644
--- a/src/tlog/tlog_main.ml
+++ b/src/tlog/tlog_main.ml
@@ -92,6 +92,7 @@ let strip_tlog filename =
   Lwt_extra.run t
 
 let _mark_tlog file_name node_name =
+  assert (Tlog_map.extension_of file_name = ".tlog");
   let section = Logger.Section.main in
   Logger.debug_f_ "_mark_tlog %s %s" file_name node_name >>= fun () ->
   _last_entry file_name >>= fun last ->
diff --git a/src/tlog/tlog_map.ml b/src/tlog/tlog_map.ml
index 99b9cfa2..5e26146d 100644
--- a/src/tlog/tlog_map.ml
+++ b/src/tlog/tlog_map.ml
@@ -256,7 +256,9 @@ let _validate_one
                  let s = Some (_make_close_marker node_id) in
                  if Entry.check_marker e s
                  then !prev_entry
-                 else raise (TLCNotProperlyClosed (tlog_name, Entry.entry2s e))
+                 else if extension_of tlog_name = ".tlog"
+                 then raise (TLCNotProperlyClosed (tlog_name, Entry.entry2s e))
+                 else raise TLogSabotage
            in
            Lwt.return eo'
        end
diff --git a/src/tools/llio.ml b/src/tools/llio.ml
index ae1683ce..f3efa8c1 100644
--- a/src/tools/llio.ml
+++ b/src/tools/llio.ml
@@ -288,6 +288,8 @@ let input_string ic =
   input_int ic >>= fun size ->
   if size > (4 * 1024 * 1024 * 1024) then
     lwt_failfmt "Unexpectedly large string size requested:%d" size
+  else if size < 0
+  then lwt_failfmt "Unexpectedly reading string with negative size:%d" size
   else Lwt.return size  >>= fun size2 ->
     let result = Bytes.create size2 in
     Lwt_io.read_into_exactly ic result 0 size2 >>= fun () ->

From aea5e5ce55421393d694ecc247f39df57301469f Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Wed, 7 Feb 2018 16:40:59 +0100
Subject: [PATCH 344/373] expand inspect-cluster command a bit more

---
 src/main/inspect.ml | 54 ++++++++++++++++++++++++++++++++++++---------
 1 file changed, 44 insertions(+), 10 deletions(-)

diff --git a/src/main/inspect.ml b/src/main/inspect.ml
index 9dad9cb5..b02b320c 100644
--- a/src/main/inspect.ml
+++ b/src/main/inspect.ml
@@ -43,7 +43,8 @@ let inspect_cluster ~tls (cfg : Node_cfg.cluster_cfg) =
         ~fsync:false node_name ~fsync_tlog_dir:false
       >>= fun tlc ->
 
-      S.make_store ~lcnum:1024 ~ncnum:512 (path ^ "/store.db") ~cluster_id
+      let db_path = path ^ "/store.db" in
+      S.make_store ~lcnum:1024 ~ncnum:512 db_path ~cluster_id
       >>= fun store ->
 
       Catchup.catchup_tlog
@@ -56,14 +57,14 @@ let inspect_cluster ~tls (cfg : Node_cfg.cluster_cfg) =
         ((module S), store, tlc) >>= fun tlog_i ->
 
       Lwt_log.info_f "Got head & tlogs for %s, i=%Li" node_name tlog_i >>= fun () ->
-      Lwt.return (node_name, store, tlc, tlog_i)
+      Lwt.return (node_name, db_path, store, tlc, tlog_i)
     )
     cfg.cfgs
   >>= fun nodes ->
 
   let minimum_last_i =
     List.fold_left
-      (fun minimum (_, _, _, current) -> min minimum current)
+      (fun minimum (_, _, _, _, current) -> min minimum current)
       Int64.max_int
       nodes
   in
@@ -71,7 +72,7 @@ let inspect_cluster ~tls (cfg : Node_cfg.cluster_cfg) =
   Lwt_log.info_f "Got all heads & tlogs, minimal i=%Li" minimum_last_i >>= fun () ->
 
   Lwt_list.iter_s
-    (fun (node_name, store, tlc, _) ->
+    (fun (node_name, db_path, store, tlc, _) ->
       Catchup.catchup_store
         ~stop:(ref false)
         node_name
@@ -79,12 +80,45 @@ let inspect_cluster ~tls (cfg : Node_cfg.cluster_cfg) =
         minimum_last_i >>= fun () ->
       S.close store >>= fun () ->
 
-      tlc # close ()
+      tlc # close () >>= fun () ->
+
+      let read_only = true in
+      Local_store.make_store ~lcnum:1024 ~ncnum:512 read_only db_path
+      >>= fun store ->
+
+      File_system.with_fd
+        (db_path ^ ".dump")
+        ~flags:[ Unix.O_WRONLY; Unix.O_CREAT; Unix.O_TRUNC; Unix.O_NONBLOCK; Unix.O_EXCL; ]
+        ~perm:0o644
+        (fun fd_out ->
+          let ch_out = Lwt_io.of_fd ~mode:Lwt_io.output fd_out in
+          let open Camltc in
+          Hotc.with_cursor
+            (Hotc.get_bdb store.Local_store.db)
+            (fun bdb cursor ->
+              let () = Bdb.first bdb cursor in
+              let rec loop () =
+                let key = Bdb.key bdb cursor in
+                let value = Bdb.value bdb cursor in
+                Lwt_io.fprintlf ch_out "key=%S value=%S" key value >>= fun () ->
+                let continue =
+                  try Bdb.next bdb cursor; true
+                  with Not_found -> false
+                in
+                if continue
+                then loop ()
+                else Lwt_io.fprintlf ch_out "Finished iterating over db"
+              in
+              loop ()
+            ) >>= fun () ->
+          Lwt_io.flush ch_out
+        )
     )
     nodes >>= fun () ->
 
-  (* - diff already, tell which are same
-   * - show diff cmds
-   *)
-
-  Lwt.return ()
+  Lwt_io.printlf "\nnow run the following commands to inspect the differences (if any)" >>= fun () ->
+  let _, first, _, _, _ = List.hd nodes in
+  Lwt_list.iter_s
+    (fun (_, db_path, _, _, _) ->
+      Lwt_io.printlf "diff %s.dump %s.dump" first db_path)
+    (List.tl nodes)

From ff786ba07f300863b5281ee1b86f0cf662105a95 Mon Sep 17 00:00:00 2001
From: Jan Doms <jan.doms@gmail.com>
Date: Mon, 12 Feb 2018 11:31:40 +0100
Subject: [PATCH 345/373] bugfix: failed catchup might apply one entry too far
 to the store

---
 src/lib/std.ml       | 13 +++++++++++++
 src/main/inspect.ml  |  4 ++++
 src/node/catchup.ml  | 35 ++++++++++++++++++++++++-----------
 src/tlog/tlog_map.ml |  7 ++++---
 4 files changed, 45 insertions(+), 14 deletions(-)

diff --git a/src/lib/std.ml b/src/lib/std.ml
index 8103778f..5c07b0a3 100644
--- a/src/lib/std.ml
+++ b/src/lib/std.ml
@@ -200,6 +200,19 @@ end
 external id : 'a -> 'a = "%identity"
 let const v = fun _ -> v
 
+module Option = struct
+  type 'a t = 'a option =
+            | None
+            | Some of 'a
+
+  let some x = Some x
+
+  let get_some = function
+    | None -> assert false
+    | Some x -> x
+end
+
+
 module String = struct
   include String
   module C = CompareLib.Default(String)
diff --git a/src/main/inspect.ml b/src/main/inspect.ml
index b02b320c..92107b73 100644
--- a/src/main/inspect.ml
+++ b/src/main/inspect.ml
@@ -14,6 +14,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 *)
 
+
+open Std
 open Lwt.Infix
 open Node_cfg
 
@@ -56,6 +58,8 @@ let inspect_cluster ~tls (cfg : Node_cfg.cluster_cfg) =
         node_name
         ((module S), store, tlc) >>= fun tlog_i ->
 
+      let tlog_i = Option.get_some tlog_i in
+
       Lwt_log.info_f "Got head & tlogs for %s, i=%Li" node_name tlog_i >>= fun () ->
       Lwt.return (node_name, db_path, store, tlc, tlog_i)
     )
diff --git a/src/node/catchup.ml b/src/node/catchup.ml
index 005a9da2..c6284cd1 100644
--- a/src/node/catchup.ml
+++ b/src/node/catchup.ml
@@ -15,7 +15,7 @@ limitations under the License.
 *)
 
 
-
+open Std
 open Lwt
 open Node_cfg
 open Remote_nodestream
@@ -112,14 +112,23 @@ let catchup_tlog (type s) ~tls_ctx ~tcp_keepalive ~stop other_configs ~cluster_i
       match mr_addresses with
       | Some mr_addresses ->
          _with_client_connection  ~tls_ctx ~tcp_keepalive mr_addresses copy_tlog >>= fun () ->
-         Logger.info_f_ "catchup_tlog completed"
+         Logger.info_f_ "catchup_tlog completed" >>= fun () ->
+         tlog_coll # get_last_i () >|= Option.some
       | None ->
-         Logger.warning_f_ "catchup tlog from unknown node:%S is config up to date?" mr_name
+         Logger.warning_f_
+           "catchup tlog from unknown node:%S is config up to date?" mr_name
+         >>= fun () ->
+         (* return None when no catchup was done, so no entries are
+          * applied for which there is possibly no consensus (yet or ever) *)
+         Lwt.return_none
+    )
+    (fun exn ->
+      Logger.warning_f_ "catchup_tlog failed: %S"
+                        (Printexc.to_string exn) >>= fun () ->
+      (* return None when no catchup was done, so no entries are
+       * applied for which there is possibly no consensus (yet or ever) *)
+      Lwt.return_none
     )
-    (fun exn -> Logger.warning_f_ "catchup_tlog failed: %S"
-                                  (Printexc.to_string exn))
-  >>= fun () ->
-  tlog_coll # get_last_i ()
 
 
 let prepare_and_insert_value (type s) (module S : Store.STORE with type t = s) me store i v =
@@ -254,10 +263,14 @@ let catchup
   catchup_tlog
     ~tls_ctx ~tcp_keepalive
     ~stop other_configs ~cluster_id mr_name dbt >>= fun until_i ->
-  Logger.info_f_ "CATCHUP phase 1 done (until_i = %s); now the store"
-    (Sn.string_of until_i) >>= fun () ->
-  catchup_store ~stop me dbt (Sn.succ until_i) >>= fun () ->
-  Logger.info_ "CATCHUP end"
+  match until_i with
+  | None ->
+     Logger.info_ "CATCHUP phase 1 failed, not going to the store"
+  | Some until_i ->
+     Logger.info_f_ "CATCHUP phase 1 done (until_i = %s); now the store"
+                    (Sn.string_of until_i) >>= fun () ->
+     catchup_store ~stop me dbt (Sn.succ until_i) >>= fun () ->
+     Logger.info_ "CATCHUP end"
 
 let verify_n_catchup_store
       (type s) ~stop me ?(apply_last_tlog_value=false)
diff --git a/src/tlog/tlog_map.ml b/src/tlog/tlog_map.ml
index 5e26146d..ca37ea24 100644
--- a/src/tlog/tlog_map.ml
+++ b/src/tlog/tlog_map.ml
@@ -290,9 +290,10 @@ let _validate_one
 module TlogMap = struct
   type tlog_number = int
   type is_archive = bool
-  type item = { i:Sn.t ;
-                n: tlog_number;
-                  is_archive: is_archive}
+  type item = { i : Sn.t ;
+                n : tlog_number;
+                is_archive: is_archive
+              }
   type t = {
       tlog_dir: string;
       tlx_dir : string;

From a8ec2b899d9b65571c7ed7dc7c4d7564d3008d30 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Tue, 13 Feb 2018 11:48:44 +0100
Subject: [PATCH 346/373] make_release: 1.9.24 (#221)

---
 META                      | 2 +-
 debian/changelog          | 5 +++++
 redhat/SPECS/arakoon.spec | 4 +++-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/META b/META
index 78c3bfde..5d048cde 100644
--- a/META
+++ b/META
@@ -1,7 +1,7 @@
 # -*- conf -*-
 
 description = "Arakoon client"
-version = "1.9.23"
+version = "1.9.24"
 exists_if = "arakoon_client.cma,arakoon_client.cmxa,arakoon_client.cmxs"
 requires = "threads lwt lwt.unix str oUnit quickcheck lwt_ssl ocplib-endian redis-lwt uri"
 archive(byte) = "arakoon_client.cma tcp_keepalive_sockopt_stubs.o"
diff --git a/debian/changelog b/debian/changelog
index 63b042b8..5a9bc645 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,8 @@
+arakoon (1.9.24) unstable; urgency=low
+  * new upstream version
+
+ -- Romain Slootmaekers <romain.slootmaekers@openvstorage.com>  Tue, 13 Feb 2018 11:32:42 +0100
+
 arakoon (1.9.23) unstable; urgency=low
   * new upstream version
 
diff --git a/redhat/SPECS/arakoon.spec b/redhat/SPECS/arakoon.spec
index c5f90b87..ba8afd53 100644
--- a/redhat/SPECS/arakoon.spec
+++ b/redhat/SPECS/arakoon.spec
@@ -2,7 +2,7 @@
 
 Summary: Arakoon
 Name: arakoon
-Version: 1.9.23
+Version: 1.9.24
 Release: 3%{?dist}
 License: Apache 2.0
 Requires: libev >= 4
@@ -33,6 +33,8 @@ cp arakoon.native %{buildroot}%{_bindir}/arakoon
 %{_bindir}/arakoon
 
 %changelog
+* Tue Feb 13 2018 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.24
+- Create arakoon 1.9.24 RPM package
 * Fri Dec 29 2017 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.23
 - Create arakoon 1.9.23 RPM package
 * Thu Aug 24 2017 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.22

From 4c1a509fff012e9fc742387689170efc94d76f0a Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Thu, 15 Feb 2018 08:52:18 +0100
Subject: [PATCH 347/373] large dependency upgrade + fix more packages (#222)

---
 docker/opam.switch | 121 +++++++++++++++++++++++++++------------------
 1 file changed, 73 insertions(+), 48 deletions(-)

diff --git a/docker/opam.switch b/docker/opam.switch
index 13d7a00d..f1cfc175 100644
--- a/docker/opam.switch
+++ b/docker/opam.switch
@@ -1,88 +1,113 @@
-base	v0.9.4	installed
+base	v0.10.0	installed
 base-bigarray	base	root
 base-bytes	base	installed
 base-threads	base	root
 base-unix	base	root
 base64	2.2.0	installed
-bin_prot	v0.9.2	installed
+bin_prot	v0.10.0	installed
+biniou	1.2.0	installed
 bisect	1.3+ovs1	installed
+bisect_ppx	1.3.2	installed
 camlbz2	0.6.0	installed
 camlp4	4.06+1	installed
-camltc	0.9.4	installed
+camltc	0.9.4	root
+cmdliner	1.0.2	installed
+conf-gmp	1	installed
+conf-gmp-powm-sec	1	installed
+conf-libev	4-11	installed
 conf-m4	1	installed
 conf-openssl	1	installed
+conf-perl	1	installed
 conf-pkg-config	1.0	installed
 conf-which	1	installed
-configurator	v0.9.1	installed
-core	v0.9.2	installed
-core_kernel	v0.9.1	installed
+conf-zlib	1	installed
+configurator	v0.10.0	installed
+core	v0.10.0	installed
+core_kernel	v0.10.0	installed
 cppo	1.6.0	installed
 cppo_ocamlbuild	1.6.0	installed
+cryptokit	1.12	installed
+ctypes	0.13.1	installed
+ctypes-foreign	0.4.0	installed
+depext	1.0.5	installed
 easy-format	1.3.0	installed
-fieldslib	v0.9.0	installed
-jane-street-headers	v0.9.0	installed
+ezxmlm	1.0.1	installed
+fieldslib	v0.10.0	installed
+integers	0.2.2	installed
+jane-street-headers	v0.10.0	installed
 jbuilder	1.0+beta16	installed
-lwt	3.0.0	root
-lwt_ssl	1.1.1	installed
+lwt	3.2.0	root
+lwt_ssl	1.1.2	root
+mirage-no-xen	1	installed
 num	1.1	installed
-ocaml-compiler-libs	v0.9.0	installed
+oasis	0.4.10	installed
+ocaml-compiler-libs	v0.10.0	installed
 ocaml-migrate-parsetree	1.0.7	installed
 ocamlbuild	0.12.0	installed
-ocamlfind	1.7.3	installed
+oocamlfind	1.7.3-1	installed
+ocamlify	0.0.1	installed
+ocamlmod	0.0.9	installed
 ocplib-endian	1.0	installed
 octavius	1.2.0	installed
+omake	0.10.3	installed
 optcomp	1.6+ovs1	installed
 ounit	2.0.6	installed
 piqi	0.7.5	installed
 piqilib	0.6.13	installed
-ppx_assert	v0.9.0	installed
-ppx_ast	v0.9.2	installed
-ppx_base	v0.9.0	installed
-ppx_bench	v0.9.1	installed
-ppx_bin_prot	v0.9.0	installed
-ppx_compare	v0.9.0	installed
-ppx_core	v0.9.3	installed
-ppx_custom_printf	v0.9.0	installed
-ppx_driver	v0.9.2	installed
-ppx_enumerate	v0.9.0	installed
-ppx_expect	v0.9.0	installed
-ppx_fail	v0.9.0	installed
-ppx_fields_conv	v0.9.0	installed
-ppx_hash	v0.9.0	installed
-ppx_here	v0.9.1	installed
-ppx_inline_test	v0.9.2	installed
-ppx_jane	v0.9.0	installed
-ppx_js_style	v0.9.0	installed
-ppx_let	v0.9.0	installed
-ppx_metaquot	v0.9.0	installed
-ppx_optcomp	v0.9.0	installed
-ppx_optional	v0.9.0	installed
-ppx_pipebang	v0.9.0	installed
-ppx_sexp_conv	v0.9.0	installed
-ppx_sexp_message	v0.9.0	installed
-ppx_sexp_value	v0.9.0	installed
+ppx_assert	v0.10.0	installed
+ppx_ast	v0.10.0	installed
+ppx_base	v0.10.0	installed
+ppx_bench	v0.10.0	installed
+ppx_bin_prot	v0.10.0	installed
+ppx_compare	v0.10.0	installed
+ppx_core	v0.10.0	installed
+ppx_custom_printf	v0.10.0	installed
+ppx_derivers	1.0	installed
+ppx_deriving	4.2.1	installed
+ppx_deriving_yojson	3.1	installed
+ppx_driver	v0.10.1	installed
+ppx_enumerate	v0.10.0	installed
+ppx_expect	v0.10.0	installed
+ppx_fail	v0.10.0	installed
+ppx_fields_conv	v0.10.0	installed
+ppx_hash	v0.10.0	installed
+ppx_here	v0.10.0	installed
+ppx_inline_test	v0.10.0	installed
+ppx_jane	v0.10.0	installed
+ppx_js_style	v0.10.0	installed
+ppx_let	v0.10.0	installed
+ppx_metaquot	v0.10.0	installed
+ppx_optcomp	v0.10.0	installed
+ppx_optional	v0.10.0	installed
+ppx_pipebang	v0.10.0	installed
+ppx_sexp_conv	v0.10.0	installed
+ppx_sexp_message	v0.10.0	installed
+ppx_sexp_value	v0.10.0	installed
 ppx_tools	5.1+4.06.0	installed
 ppx_tools_versioned	5.0.1	installed
-ppx_traverse	v0.9.0	installed
-ppx_traverse_builtins	v0.9.0	installed
-ppx_type_conv	v0.9.1	installed
-ppx_typerep_conv	v0.9.0	installed
-ppx_variants_conv	v0.9.0	installed
+ppx_traverse	v0.10.0	installed
+ppx_traverse_builtins	v0.10.0	installed
+ppx_type_conv	v0.10.0	installed
+ppx_typerep_conv	v0.10.0	installed
+ppx_variants_conv	v0.10.0	installed
 quickcheck	1.0.2	installed
 re	1.7.1	installed
 redis	0.3.5+ovs1	installed
 redis-lwt	0.3.5+ovs1	root
 result	1.2	installed
-sexplib	v0.9.3	installed
+sexplib	v0.10.0	installed
 snappy	0.1.0	installed
-spawn	v0.9.0	installed
+spawn	v0.10.1	installed
 ssl	0.5.5	installed
-stdio	v0.9.1	installed
+stdio	v0.10.0	installed
 stringext	1.5.0	installed
+tiny_json	1.1.5	root
 topkg	0.9.1	installed
-typerep	v0.9.0	installed
+typerep	v0.10.0	installed
 ulex	1.2	installed
 uri	1.9.5	installed
 uuidm	0.9.6	installed
-variantslib	v0.9.0	installed
+variantslib	v0.10.0	installed
 xmlm	1.3.0	installed
+yojson	1.4.0	installed
+zarith	1.7	installed

From 1b544cbf37a6e2e346b10374cb8f5e602cecfb1b Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Thu, 15 Feb 2018 10:46:00 +0100
Subject: [PATCH 348/373] move to ezxmlm.1.0.2 (#224)

---
 docker/opam.switch | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/opam.switch b/docker/opam.switch
index f1cfc175..1049e547 100644
--- a/docker/opam.switch
+++ b/docker/opam.switch
@@ -31,7 +31,7 @@ ctypes	0.13.1	installed
 ctypes-foreign	0.4.0	installed
 depext	1.0.5	installed
 easy-format	1.3.0	installed
-ezxmlm	1.0.1	installed
+ezxmlm	1.0.2	installed
 fieldslib	v0.10.0	installed
 integers	0.2.2	installed
 jane-street-headers	v0.10.0	installed

From a49e38dfb3ac00d2ce109a5a6c4340f5d5ca5b8d Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Thu, 15 Feb 2018 10:46:14 +0100
Subject: [PATCH 349/373] make_release: 1.9.25 (#223)

---
 META                      | 2 +-
 debian/changelog          | 5 +++++
 redhat/SPECS/arakoon.spec | 4 +++-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/META b/META
index 5d048cde..44aa92a6 100644
--- a/META
+++ b/META
@@ -1,7 +1,7 @@
 # -*- conf -*-
 
 description = "Arakoon client"
-version = "1.9.24"
+version = "1.9.25"
 exists_if = "arakoon_client.cma,arakoon_client.cmxa,arakoon_client.cmxs"
 requires = "threads lwt lwt.unix str oUnit quickcheck lwt_ssl ocplib-endian redis-lwt uri"
 archive(byte) = "arakoon_client.cma tcp_keepalive_sockopt_stubs.o"
diff --git a/debian/changelog b/debian/changelog
index 5a9bc645..9e131fc0 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,8 @@
+arakoon (1.9.25) unstable; urgency=low
+  * new upstream version
+
+ -- Romain Slootmaekers <romain.slootmaekers@openvstorage.com>  Thu, 15 Feb 2018 09:12:47 +0100
+
 arakoon (1.9.24) unstable; urgency=low
   * new upstream version
 
diff --git a/redhat/SPECS/arakoon.spec b/redhat/SPECS/arakoon.spec
index ba8afd53..feeb1224 100644
--- a/redhat/SPECS/arakoon.spec
+++ b/redhat/SPECS/arakoon.spec
@@ -2,7 +2,7 @@
 
 Summary: Arakoon
 Name: arakoon
-Version: 1.9.24
+Version: 1.9.25
 Release: 3%{?dist}
 License: Apache 2.0
 Requires: libev >= 4
@@ -33,6 +33,8 @@ cp arakoon.native %{buildroot}%{_bindir}/arakoon
 %{_bindir}/arakoon
 
 %changelog
+* Thu Feb 15 2018 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.25
+- Create arakoon 1.9.25 RPM package
 * Tue Feb 13 2018 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.24
 - Create arakoon 1.9.24 RPM package
 * Fri Dec 29 2017 Romain Slootmaekers <romain.slootmaekers@openvstorage.com> - 1.9.23

From 556f01eb033ab874ac3283a3d6758be560a882f4 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Thu, 15 Feb 2018 16:07:24 +0100
Subject: [PATCH 350/373] typo

---
 docker/opam.switch | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/opam.switch b/docker/opam.switch
index 1049e547..767f989d 100644
--- a/docker/opam.switch
+++ b/docker/opam.switch
@@ -44,7 +44,7 @@ oasis	0.4.10	installed
 ocaml-compiler-libs	v0.10.0	installed
 ocaml-migrate-parsetree	1.0.7	installed
 ocamlbuild	0.12.0	installed
-oocamlfind	1.7.3-1	installed
+ocamlfind	1.7.3-1	installed
 ocamlify	0.0.1	installed
 ocamlmod	0.0.9	installed
 ocplib-endian	1.0	installed

From a96b6ecee2fe20ed067166896ab9a717ac7ee013 Mon Sep 17 00:00:00 2001
From: Anders Fugmann <anders@fugmann.net>
Date: Mon, 5 Mar 2018 10:31:35 +0100
Subject: [PATCH 351/373] Prefix inifiles library with arakoon to avoid
 conflict with ocaml-inifiles

---
 Makefile                                      |  6 ++--
 arakoon.odocl                                 |  2 +-
 src/all_test.ml                               |  2 +-
 src/arakoon_client.mllib                      |  6 ++--
 src/client/arakoon_client_config.ml           |  4 +--
 .../{inifiles.ml => arakoon_inifiles.ml}      |  2 +-
 .../{inifiles.mli => arakoon_inifiles.mli}    |  0
 ...files_test.ml => arakoon_inifiles_test.ml} |  2 +-
 .../{inilexer.mll => arakoon_inilexer.mll}    | 28 +++++++++----------
 .../{parseini.mly => arakoon_parseini.mly}    |  0
 src/node/node_cfg.ml                          |  8 +++---
 src/nursery/client_cfg.ml                     |  2 +-
 src/nursery/nursery_main.ml                   |  2 +-
 src/tools/ini.ml                              |  2 +-
 14 files changed, 33 insertions(+), 33 deletions(-)
 rename src/inifiles/{inifiles.ml => arakoon_inifiles.ml} (99%)
 rename src/inifiles/{inifiles.mli => arakoon_inifiles.mli} (100%)
 rename src/inifiles/{inifiles_test.ml => arakoon_inifiles_test.ml} (96%)
 rename src/inifiles/{inilexer.mll => arakoon_inilexer.mll} (72%)
 rename src/inifiles/{parseini.mly => arakoon_parseini.mly} (100%)

diff --git a/Makefile b/Makefile
index 807d6ec7..8c5ca27c 100644
--- a/Makefile
+++ b/Makefile
@@ -58,9 +58,9 @@ install_client:
 	  _build/src/client/protocol_common.cmx \
           _build/src/client/client_helper.cmi \
 	  _build/src/client/client_helper.cmx \
-          _build/src/inifiles/inifiles.mli \
-          _build/src/inifiles/inifiles.cmi \
-	  _build/src/inifiles/inifiles.cmx \
+          _build/src/inifiles/arakoon_inifiles.mli \
+          _build/src/inifiles/arakoon_inifiles.cmi \
+	  _build/src/inifiles/arakoon_inifiles.cmx \
           _build/src/lib/std.cmi \
           _build/src/lib/std.cmx \
           _build/src/plugins/registry.mli \
diff --git a/arakoon.odocl b/arakoon.odocl
index 0d472066..87709265 100644
--- a/arakoon.odocl
+++ b/arakoon.odocl
@@ -23,7 +23,7 @@ Extra
 File_system
 Fsm
 Ini
-Inifiles
+Arakoon_inifiles
 Interval
 Key
 Limits
diff --git a/src/all_test.ml b/src/all_test.ml
index 77ef1e21..47d5f192 100644
--- a/src/all_test.ml
+++ b/src/all_test.ml
@@ -31,7 +31,7 @@ let configure_logging () =
 let tools_tests = "tools" >::: [
     Server_test.suite;
     Backoff_test.suite;
-    Inifiles_test.suite;
+    Arakoon_inifiles_test.suite;
   ]
 
 let client_tests = "client" >::: [
diff --git a/src/arakoon_client.mllib b/src/arakoon_client.mllib
index 0160851f..33afa14e 100644
--- a/src/arakoon_client.mllib
+++ b/src/arakoon_client.mllib
@@ -19,9 +19,9 @@ Client_cfg
 Arakoon_config_url
 Arakoon_etcd
 Ini
-Inifiles
-Parseini
-Inilexer
+Arakoon_inifiles
+Arakoon_parseini
+Arakoon_inilexer
 Client_log
 Simple_store
 Logger
diff --git a/src/client/arakoon_client_config.ml b/src/client/arakoon_client_config.ml
index f48f21db..0c6f8351 100644
--- a/src/client/arakoon_client_config.ml
+++ b/src/client/arakoon_client_config.ml
@@ -120,7 +120,7 @@ let tcp_keepalive_from_ini inifile =
   }
 
 let from_ini txt =
-  let inifile = new Inifiles.inifile txt in
+  let inifile = new Arakoon_inifiles.inifile txt in
   let cluster_id = Ini.get inifile "global" "cluster_id" Ini.p_string Ini.required in
   let nodes = Ini.get inifile "global" "cluster" Ini.p_string_list Ini.required in
   let node_cfgs =
@@ -182,7 +182,7 @@ let to_ini { cluster_id;
              node_cfgs;
              ssl_cfg;
              tcp_keepalive; } =
-  let inifile = new Inifiles.inifile "[global]\ncluster_id = x" in
+  let inifile = new Arakoon_inifiles.inifile "[global]\ncluster_id = x" in
 
   inifile # delval "global" "cluster_id";
 
diff --git a/src/inifiles/inifiles.ml b/src/inifiles/arakoon_inifiles.ml
similarity index 99%
rename from src/inifiles/inifiles.ml
rename to src/inifiles/arakoon_inifiles.ml
index 6d7cbd75..d9d02251 100644
--- a/src/inifiles/inifiles.ml
+++ b/src/inifiles/arakoon_inifiles.ml
@@ -107,7 +107,7 @@ let parse_inifile txt tbl =
     Lexing.from_string txt'
   in
   try
-    let parsed_txt = Parseini.inifile Inilexer.lexini lxbuf in
+    let parsed_txt = Arakoon_parseini.inifile Arakoon_inilexer.lexini lxbuf in
     List.iter
       (fun (section, values) ->
          Hashtbl.add tbl section
diff --git a/src/inifiles/inifiles.mli b/src/inifiles/arakoon_inifiles.mli
similarity index 100%
rename from src/inifiles/inifiles.mli
rename to src/inifiles/arakoon_inifiles.mli
diff --git a/src/inifiles/inifiles_test.ml b/src/inifiles/arakoon_inifiles_test.ml
similarity index 96%
rename from src/inifiles/inifiles_test.ml
rename to src/inifiles/arakoon_inifiles_test.ml
index cf4cd791..46b60702 100644
--- a/src/inifiles/inifiles_test.ml
+++ b/src/inifiles/arakoon_inifiles_test.ml
@@ -17,7 +17,7 @@ limitations under the License.
 let _parse txt =
   try
     let _cfg = Node_cfg.Node_cfg._retrieve_cfg_from_txt txt in ()
-  with (Inifiles.Ini_parse_error(lnum,txt) as exn)->
+  with (Arakoon_inifiles.Ini_parse_error(lnum,txt) as exn)->
     let () =
       Printf.printf "lnum:%i\n%s" lnum txt in
     raise exn
diff --git a/src/inifiles/inilexer.mll b/src/inifiles/arakoon_inilexer.mll
similarity index 72%
rename from src/inifiles/inilexer.mll
rename to src/inifiles/arakoon_inilexer.mll
index 352e4810..8750ad6a 100644
--- a/src/inifiles/inilexer.mll
+++ b/src/inifiles/arakoon_inilexer.mll
@@ -3,24 +3,24 @@
    Copyright (C) 2004 Eric Stokes, and The California State University
    at Northridge
 
-   This library is free software; you can redistribute it and/or               
-   modify it under the terms of the GNU Lesser General Public                  
-   License as published by the Free Software Foundation; either                
-   version 2.1 of the License, or (at your option) any later version.          
-   
-   This library is distributed in the hope that it will be useful,             
-   but WITHOUT ANY WARRANTY; without even the implied warranty of              
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU           
-   Lesser General Public License for more details.                             
-   
-   You should have received a copy of the GNU Lesser General Public            
-   License along with this library; if not, write to the Free Software         
-   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA   
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, write to the Free Software
+   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 *)
 
 
 {
-  open Parseini
+  open Arakoon_parseini
 }
 
 let newline = '\n' +
diff --git a/src/inifiles/parseini.mly b/src/inifiles/arakoon_parseini.mly
similarity index 100%
rename from src/inifiles/parseini.mly
rename to src/inifiles/arakoon_parseini.mly
diff --git a/src/node/node_cfg.ml b/src/node/node_cfg.ml
index 00ebba38..121fcec2 100644
--- a/src/node/node_cfg.ml
+++ b/src/node/node_cfg.ml
@@ -413,7 +413,7 @@ module Node_cfg = struct
           if _get_bool inifile "global" "preferred_master"
           then (Preferred [m])
           else (Forced m)
-      with (Inifiles.Invalid_element _) ->
+      with (Arakoon_inifiles.Invalid_element _) ->
         let pms = Ini.get inifile "global" "preferred_masters" Ini.p_string_list (fun _ _ -> []) in
         if pms <> []
         then
@@ -439,7 +439,7 @@ module Node_cfg = struct
     try
       let cids = inifile # getval "global" "cluster_id" in
       Scanf.sscanf cids "%s" (fun s -> s)
-    with (Inifiles.Invalid_element _ ) -> failwith "config has no cluster_id"
+    with (Arakoon_inifiles.Invalid_element _ ) -> failwith "config has no cluster_id"
 
   let _get_quorum_function inifile =
     let nodes = _node_names inifile in
@@ -453,7 +453,7 @@ module Node_cfg = struct
         let msg = Printf.sprintf "fixed quorum should be 1 <= %i <= %i"
                     qi n_nodes in
         failwith msg
-    with (Inifiles.Invalid_element _) -> Quorum.quorum_function
+    with (Arakoon_inifiles.Invalid_element _) -> Quorum.quorum_function
 
   let _log_config inifile log_name =
     let get_log_level x =
@@ -643,7 +643,7 @@ module Node_cfg = struct
 
 
     let _retrieve_cfg_from_txt txt =
-      let inifile = new Inifiles.inifile txt in
+      let inifile = new Arakoon_inifiles.inifile txt in
       let fm = _startup_mode inifile in
       let nodes = _node_names inifile in
       let plugin_names = _plugins inifile in
diff --git a/src/nursery/client_cfg.ml b/src/nursery/client_cfg.ml
index 3ef6aa7e..8a288f37 100644
--- a/src/nursery/client_cfg.ml
+++ b/src/nursery/client_cfg.ml
@@ -84,7 +84,7 @@ module ClientCfg = struct
     cfg
 
   let _from_txt section txt =
-    let inifile = new Inifiles.inifile txt in
+    let inifile = new Arakoon_inifiles.inifile txt in
     from_ini section inifile
 
 end
diff --git a/src/nursery/nursery_main.ml b/src/nursery/nursery_main.ml
index 7b71e0b6..088df4d8 100644
--- a/src/nursery/nursery_main.ml
+++ b/src/nursery/nursery_main.ml
@@ -79,7 +79,7 @@ let setup_logger file_name =
 
 let _get_keeper_config_generic fetcher url=
   fetcher() >>= fun txt ->
-  let inifile = new Inifiles.inifile txt in
+  let inifile = new Arakoon_inifiles.inifile txt in
   let m_cfg = Node_cfg.get_nursery_cfg inifile in
   begin
     match m_cfg with
diff --git a/src/tools/ini.ml b/src/tools/ini.ml
index cd93b675..2cb43537 100644
--- a/src/tools/ini.ml
+++ b/src/tools/ini.ml
@@ -38,4 +38,4 @@ let get inifile section name s2a not_found =
   try
     let v_s = inifile # getval section name in
     s2a v_s
-  with (Inifiles.Invalid_element _) -> not_found section name
+  with (Arakoon_inifiles.Invalid_element _) -> not_found section name

From f0726ec5e069ba55381eb9a536c35054770bb82e Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Tue, 3 Apr 2018 15:06:24 +0200
Subject: [PATCH 352/373] opam's snappy no longer likes 4.06

---
 .../packages/snappy/snappy.0.1.0+ovs1/descr   |  1 +
 .../files/myocamlbuild.ml.osx.patch           | 21 ++++++++++++++++++
 .../packages/snappy/snappy.0.1.0+ovs1/opam    | 22 +++++++++++++++++++
 .../packages/snappy/snappy.0.1.0+ovs1/url     |  2 ++
 docker/opam.switch                            |  2 +-
 docker/ubuntu-16.04/Dockerfile                |  5 ++++-
 6 files changed, 51 insertions(+), 2 deletions(-)
 create mode 100644 docker/opam-repository/packages/snappy/snappy.0.1.0+ovs1/descr
 create mode 100644 docker/opam-repository/packages/snappy/snappy.0.1.0+ovs1/files/myocamlbuild.ml.osx.patch
 create mode 100644 docker/opam-repository/packages/snappy/snappy.0.1.0+ovs1/opam
 create mode 100644 docker/opam-repository/packages/snappy/snappy.0.1.0+ovs1/url

diff --git a/docker/opam-repository/packages/snappy/snappy.0.1.0+ovs1/descr b/docker/opam-repository/packages/snappy/snappy.0.1.0+ovs1/descr
new file mode 100644
index 00000000..7b3712e1
--- /dev/null
+++ b/docker/opam-repository/packages/snappy/snappy.0.1.0+ovs1/descr
@@ -0,0 +1 @@
+Bindings to snappy - fast compression/decompression library
diff --git a/docker/opam-repository/packages/snappy/snappy.0.1.0+ovs1/files/myocamlbuild.ml.osx.patch b/docker/opam-repository/packages/snappy/snappy.0.1.0+ovs1/files/myocamlbuild.ml.osx.patch
new file mode 100644
index 00000000..ef48da76
--- /dev/null
+++ b/docker/opam-repository/packages/snappy/snappy.0.1.0+ovs1/files/myocamlbuild.ml.osx.patch
@@ -0,0 +1,21 @@
+--- myocamlbuild.ml	2016-02-14 13:30:11.000000000 +0000
++++ myocamlbuild.ml	2016-02-14 13:33:23.000000000 +0000
+@@ -484,15 +484,15 @@
+           (["oasis_library_snappy_ccopt"; "compile"],
+             [
+                (OASISExpr.EBool true,
+-                 S [A "-ccopt"; A "-x"; A "-ccopt"; A "c++"])
++                 S [A "-ccopt"; A "-x"; A "-ccopt"; A "c++"; A "-ccopt"; A "-I/usr/local/include"])
+             ]);
+           (["oasis_library_snappy_cclib"; "link"],
+             [
+                (OASISExpr.EBool true,
+-                 S [A "-cclib"; A "-lstdc++"; A "-cclib"; A "-lsnappy"])
++                 S [A "-cclib"; A "-lstdc++"; A "-cclib"; A "-lsnappy"; A "-cclib"; A "-L/usr/local/lib"])
+             ]);
+           (["oasis_library_snappy_cclib"; "ocamlmklib"; "c"],
+-            [(OASISExpr.EBool true, S [A "-lstdc++"; A "-lsnappy"])])
++            [(OASISExpr.EBool true, S [A "-lstdc++"; A "-L/usr/local/lib"; A "-lsnappy"])])
+        ];
+      includes = [("", ["src"])];
+      }
diff --git a/docker/opam-repository/packages/snappy/snappy.0.1.0+ovs1/opam b/docker/opam-repository/packages/snappy/snappy.0.1.0+ovs1/opam
new file mode 100644
index 00000000..9d62e19b
--- /dev/null
+++ b/docker/opam-repository/packages/snappy/snappy.0.1.0+ovs1/opam
@@ -0,0 +1,22 @@
+opam-version: "1.2"
+maintainer: "ygrek@autistici.org"
+homepage: "http://snappy.forge.ocamlcore.org"
+doc: ["http://snappy.forge.ocamlcore.org/api/index.html"]
+build: [
+  ["ocaml" "setup.ml" "-configure" "--prefix" prefix]
+  ["ocaml" "setup.ml" "-build"]
+]
+build-test: [["ocaml" "setup.ml" "-test"]]
+build-doc: [["ocaml" "setup.ml" "-doc"]]
+remove: [["ocamlfind" "remove" "snappy"]]
+depends: [
+  "ocamlfind"
+  "ocamlbuild" {build}
+]
+depexts: [
+  [["debian"] ["libsnappy-dev"]]
+  [["ubuntu"] ["libsnappy-dev"]]
+  [["osx" "homebrew"] ["snappy"]]
+]
+patches: [ "myocamlbuild.ml.osx.patch" { os = "darwin" } ]
+install: ["ocaml" "setup.ml" "-install"]
diff --git a/docker/opam-repository/packages/snappy/snappy.0.1.0+ovs1/url b/docker/opam-repository/packages/snappy/snappy.0.1.0+ovs1/url
new file mode 100644
index 00000000..c626d94d
--- /dev/null
+++ b/docker/opam-repository/packages/snappy/snappy.0.1.0+ovs1/url
@@ -0,0 +1,2 @@
+archive: "https://forge.ocamlcore.org/frs/download.php/1280/ocaml-snappy-0.1.0.tar.gz"
+checksum: "8067b7d19b37db22a512748640844ce3"
diff --git a/docker/opam.switch b/docker/opam.switch
index 767f989d..aa37c92e 100644
--- a/docker/opam.switch
+++ b/docker/opam.switch
@@ -96,7 +96,7 @@ redis	0.3.5+ovs1	installed
 redis-lwt	0.3.5+ovs1	root
 result	1.2	installed
 sexplib	v0.10.0	installed
-snappy	0.1.0	installed
+snappy	0.1.0+ovs1	root
 spawn	v0.10.1	installed
 ssl	0.5.5	installed
 stdio	v0.10.0	installed
diff --git a/docker/ubuntu-16.04/Dockerfile b/docker/ubuntu-16.04/Dockerfile
index 555016a3..002fd27c 100644
--- a/docker/ubuntu-16.04/Dockerfile
+++ b/docker/ubuntu-16.04/Dockerfile
@@ -70,8 +70,11 @@ RUN chmod ugoa+rxw -R ovs-opam-repository
 RUN eval `${opam_env}` && \
     opam repo add ovs ovs-opam-repository && \
     opam update -v && \
-    (eval `${opam_env}` && opam switch import opam.switch -y)
+    (eval `${opam_env}` && opam switch import opam.switch -y --strict) 
 
+RUN eval `${opam_env}` && opam list && \
+    opam switch export opam.switch.out && \
+    diff opam.switch opam.switch.out
 
 RUN chmod ugoa+rxw -R ${opam_root}
 RUN su - -c "echo 'eval `${opam_env}`' >> /home/jenkins/.profile"

From 5feff824b4a79d423caeb98081d87754bdc74125 Mon Sep 17 00:00:00 2001
From: Anders Fugmann <anders@fugmann.net>
Date: Fri, 6 Apr 2018 23:23:01 +0200
Subject: [PATCH 353/373] Rename interval.ml to arakoon_interval.ml

---
 Makefile                                      | 4 ++--
 arakoon.odocl                                 | 2 +-
 opam                                          | 2 +-
 src/arakoon_client.mllib                      | 2 +-
 src/client/client_protocol.ml                 | 2 +-
 src/client/protocol_common.ml                 | 2 +-
 src/client/remote_nodestream.ml               | 2 +-
 src/client/remote_nodestream.mli              | 2 +-
 src/client/remote_nodestream_test.ml          | 2 +-
 src/node/backend.ml                           | 2 +-
 src/node/dump_store.ml                        | 2 +-
 src/node/store.ml                             | 2 +-
 src/node/sync_backend.ml                      | 2 +-
 src/nursery/nursery.ml                        | 8 ++++----
 src/plugins/registry.ml                       | 2 +-
 src/plugins/registry.mli                      | 2 +-
 src/tlog/{interval.ml => arakoon_interval.ml} | 0
 src/tlog/update.ml                            | 2 +-
 src/tlog/update.mli                           | 2 +-
 src/tlog/update_test.ml                       | 2 +-
 20 files changed, 23 insertions(+), 23 deletions(-)
 rename src/tlog/{interval.ml => arakoon_interval.ml} (100%)

diff --git a/Makefile b/Makefile
index 8c5ca27c..4e7d339d 100644
--- a/Makefile
+++ b/Makefile
@@ -75,8 +75,8 @@ install_client:
           _build/src/node/simple_store.cmi \
 	  _build/src/node/simple_store.cmx \
           _build/src/nursery/routing.cmi \
-          _build/src/tlog/interval.cmi \
-          _build/src/tlog/interval.cmx \
+          _build/src/tlog/arakoon_interval.cmi \
+          _build/src/tlog/arakoon_interval.cmx \
           _build/src/tlog/update.mli \
           _build/src/tlog/update.cmi \
 	  _build/src/tlog/update.cmx \
diff --git a/arakoon.odocl b/arakoon.odocl
index 87709265..bdcb1f8c 100644
--- a/arakoon.odocl
+++ b/arakoon.odocl
@@ -24,7 +24,7 @@ File_system
 Fsm
 Ini
 Arakoon_inifiles
-Interval
+Arakoon_interval
 Key
 Limits
 Llio
diff --git a/opam b/opam
index 03544afc..13b28ff0 100644
--- a/opam
+++ b/opam
@@ -22,7 +22,7 @@ depends: [
   "camltc" {>= "0.9.4"}
   "camlbz2" {>= "0.6.0"}
   "conf-libev" {>= "4-11"}
-  "lwt" {= "3.0.0"}
+  "lwt" {>= "3.0.0"}
   "lwt_ssl" {>= "1.0.1"}
   "ocplib-endian" {>= "1.0"}
   "ssl" {>= "0.5.3"}
diff --git a/src/arakoon_client.mllib b/src/arakoon_client.mllib
index 33afa14e..90037e99 100644
--- a/src/arakoon_client.mllib
+++ b/src/arakoon_client.mllib
@@ -13,7 +13,7 @@ Value
 Statistics
 Registry
 Routing
-Interval
+Arakoon_interval
 Ncfg
 Client_cfg
 Arakoon_config_url
diff --git a/src/client/client_protocol.ml b/src/client/client_protocol.ml
index e7b2f81d..fd440ac7 100644
--- a/src/client/client_protocol.ml
+++ b/src/client/client_protocol.ml
@@ -19,7 +19,7 @@ open Protocol_common
 open Lwt
 open Log_extra
 open Update
-open Interval
+open Arakoon_interval
 open Routing
 open Statistics
 open Ncfg
diff --git a/src/client/protocol_common.ml b/src/client/protocol_common.ml
index 4074f4ed..b5e96055 100644
--- a/src/client/protocol_common.ml
+++ b/src/client/protocol_common.ml
@@ -17,7 +17,7 @@ limitations under the License.
 
 
 open Lwt
-open Interval
+open Arakoon_interval
 open Update
 open Routing
 open Client_cfg
diff --git a/src/client/remote_nodestream.ml b/src/client/remote_nodestream.ml
index 0725f984..924cbb5d 100644
--- a/src/client/remote_nodestream.ml
+++ b/src/client/remote_nodestream.ml
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 *)
 
-open Interval
+open Arakoon_interval
 open Routing
 open Protocol_common
 open Lwt
diff --git a/src/client/remote_nodestream.mli b/src/client/remote_nodestream.mli
index 53e75a4a..8ad3e040 100644
--- a/src/client/remote_nodestream.mli
+++ b/src/client/remote_nodestream.mli
@@ -15,7 +15,7 @@ limitations under the License.
 *)
 
 open Routing
-open Interval
+open Arakoon_interval
 open Client_cfg
 open Ncfg
 
diff --git a/src/client/remote_nodestream_test.ml b/src/client/remote_nodestream_test.ml
index 8569266e..7ceed7dc 100644
--- a/src/client/remote_nodestream_test.ml
+++ b/src/client/remote_nodestream_test.ml
@@ -20,7 +20,7 @@ open Remote_nodestream
 open Node_cfg
 open Arakoon_remote_client
 open Routing
-open Interval
+open Arakoon_interval
 
 let setup port = Lwt.return port
 
diff --git a/src/node/backend.ml b/src/node/backend.ml
index 7a17ad53..efa74c7f 100644
--- a/src/node/backend.ml
+++ b/src/node/backend.ml
@@ -18,7 +18,7 @@ limitations under the License.
 
 open Std
 open Update
-open Interval
+open Arakoon_interval
 open Routing
 open Statistics
 open Client_cfg
diff --git a/src/node/dump_store.ml b/src/node/dump_store.ml
index 9867f269..2d22f914 100644
--- a/src/node/dump_store.ml
+++ b/src/node/dump_store.ml
@@ -17,7 +17,7 @@ limitations under the License.
 open Lwt
 open Routing
 open Node_cfg.Node_cfg
-open Interval
+open Arakoon_interval
 open Tlog_map
 
 module S = (val (Store.make_store_module (module Batched_store.Local_store)))
diff --git a/src/node/store.ml b/src/node/store.ml
index d50bd6be..d76a0537 100644
--- a/src/node/store.ml
+++ b/src/node/store.ml
@@ -19,7 +19,7 @@ limitations under the License.
 
 open Std
 open Update
-open Interval
+open Arakoon_interval
 open Routing
 open Lwt
 open Simple_store
diff --git a/src/node/sync_backend.ml b/src/node/sync_backend.ml
index 8579d674..446956a0 100644
--- a/src/node/sync_backend.ml
+++ b/src/node/sync_backend.ml
@@ -25,7 +25,7 @@ struct
   open Lwt
   open Log_extra
   open Update
-  open Interval
+  open Arakoon_interval
   open Protocol_common
   open Master_type
   open Arakoon_client
diff --git a/src/nursery/nursery.ml b/src/nursery/nursery.ml
index 1de42ee3..dd1cef67 100644
--- a/src/nursery/nursery.ml
+++ b/src/nursery/nursery.ml
@@ -20,7 +20,7 @@ open Lwt
 open Routing
 open Client_cfg
 open Ncfg
-open Interval
+open Arakoon_interval
 open Arakoon_client
 
 let section = Logger.Section.main
@@ -182,7 +182,7 @@ module NC = struct
     get_interval from_cn >>= fun from_i ->
     Logger.debug_f_ "Getting initial interval from %s" to_cn >>= fun () ->
     get_interval to_cn >>= fun to_i ->
-    let open Interval in
+    let open Arakoon_interval in
     let rec loop (from_i:Interval.t) (to_i:Interval.t) =
       pull () >>= fun fringe ->
       match fringe with
@@ -275,7 +275,7 @@ module NC = struct
       end
     in
     let set_interval cn i = force_interval t (cn: string) (i: Interval.t) in
-    let open Interval in
+    let open Arakoon_interval in
     let finalize (from_i: Interval.t) (to_i: Interval.t) =
       Logger.debug_ "Setting final intervals en routing" >>= fun () ->
 
@@ -337,7 +337,7 @@ module NC = struct
       end
     in
     let set_interval cn i = force_interval t (cn: string) (i: Interval.t) in
-    let open Interval in
+    let open Arakoon_interval in
     let finalize from_cn to_cn direction (from_i: Interval.t) (to_i: Interval.t) =
       Logger.debug_ "Setting final intervals en routing" >>= fun () ->
       let fpu_b = from_i.pu_b
diff --git a/src/plugins/registry.ml b/src/plugins/registry.ml
index 0d827afe..da71b33a 100644
--- a/src/plugins/registry.ml
+++ b/src/plugins/registry.ml
@@ -82,7 +82,7 @@ class type read_user_db =
     method get_exn : string -> string
     method with_cursor : (cursor_db -> 'a) -> 'a
 
-    method get_interval : unit -> Interval.Interval.t
+    method get_interval : unit -> Arakoon_interval.Interval.t
   end
 
 class type user_db =
diff --git a/src/plugins/registry.mli b/src/plugins/registry.mli
index ba5960db..313b6bb8 100644
--- a/src/plugins/registry.mli
+++ b/src/plugins/registry.mli
@@ -44,7 +44,7 @@ class type read_user_db =
     method get_exn : string -> string
     method with_cursor : (cursor_db -> 'a) -> 'a
 
-    method get_interval : unit -> Interval.Interval.t
+    method get_interval : unit -> Arakoon_interval.Interval.t
   end
 
 class type user_db =
diff --git a/src/tlog/interval.ml b/src/tlog/arakoon_interval.ml
similarity index 100%
rename from src/tlog/interval.ml
rename to src/tlog/arakoon_interval.ml
diff --git a/src/tlog/update.ml b/src/tlog/update.ml
index 43b21c23..df34cef0 100644
--- a/src/tlog/update.ml
+++ b/src/tlog/update.ml
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 *)
 
-open Interval
+open Arakoon_interval
 open Routing
 let section = Logger.Section.main
 
diff --git a/src/tlog/update.mli b/src/tlog/update.mli
index bac746e9..0c4b79f2 100644
--- a/src/tlog/update.mli
+++ b/src/tlog/update.mli
@@ -22,7 +22,7 @@ module Update :
       | MasterSet of string * float
       | TestAndSet of string * string option * string option
       | Sequence of t list
-      | SetInterval of Interval.Interval.t
+      | SetInterval of Arakoon_interval.Interval.t
       | SetRouting of Routing.Routing.t
       | SetRoutingDelta of (string * string * string)
       | Nop
diff --git a/src/tlog/update_test.ml b/src/tlog/update_test.ml
index 56abd841..ce5de2df 100644
--- a/src/tlog/update_test.ml
+++ b/src/tlog/update_test.ml
@@ -18,7 +18,7 @@ limitations under the License.
 
 open OUnit
 open Update
-open Interval
+open Arakoon_interval
 
 open Lwt
 

From c31f0e85c5e817f16f6b52ce8b9d31b117128785 Mon Sep 17 00:00:00 2001
From: Anders Fugmann <anders@fugmann.net>
Date: Wed, 2 May 2018 10:23:40 +0200
Subject: [PATCH 354/373] Add core to the list of dependencies for arakoon
 client

---
 META | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/META b/META
index 44aa92a6..c648b1d7 100644
--- a/META
+++ b/META
@@ -3,7 +3,6 @@
 description = "Arakoon client"
 version = "1.9.25"
 exists_if = "arakoon_client.cma,arakoon_client.cmxa,arakoon_client.cmxs"
-requires = "threads lwt lwt.unix str oUnit quickcheck lwt_ssl ocplib-endian redis-lwt uri"
+requires = "threads lwt lwt.unix str oUnit quickcheck lwt_ssl ocplib-endian redis-lwt uri core"
 archive(byte) = "arakoon_client.cma tcp_keepalive_sockopt_stubs.o"
 archive(native) = "arakoon_client.cmxa tcp_keepalive_sockopt_stubs.o"
-

From 04c2ed345e2d9494a9cb52c2af703d5ba1ead57f Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 18 Jul 2018 14:07:53 +0200
Subject: [PATCH 355/373] allow DeletePrefix in a Sequence

---
 pylabs/pyrakoon                               |  2 +-
 .../test/server/quick/system_tests_basic.py   | 35 ++++++++++++++++++-
 src/client/arakoon_client.mli                 |  1 +
 src/client/protocol_common.ml                 |  3 +-
 src/client/python/ArakoonProtocol.py          | 13 ++++++-
 5 files changed, 50 insertions(+), 4 deletions(-)

diff --git a/pylabs/pyrakoon b/pylabs/pyrakoon
index 6bf01f5f..e69b8d2e 160000
--- a/pylabs/pyrakoon
+++ b/pylabs/pyrakoon
@@ -1 +1 @@
-Subproject commit 6bf01f5f6654aebfa1e10355376417a5d2fa8010
+Subproject commit e69b8d2e9852709b866bea25f594124385e9a364
diff --git a/pylabs/test/server/quick/system_tests_basic.py b/pylabs/test/server/quick/system_tests_basic.py
index ef9ff8c5..3f109ca0 100644
--- a/pylabs/test/server/quick/system_tests_basic.py
+++ b/pylabs/test/server/quick/system_tests_basic.py
@@ -787,7 +787,9 @@ def test_download_db():
 
 @C.with_custom_setup(C.setup_3_nodes, C.basic_teardown)
 def test_delete_prefix():
-    """ deletion of all key value pairs with same prefix """
+
+    # deletion of all key value pairs with same prefix
+
     cli = C.get_client()
     for i in xrange(100):
         cli.set("key_%04i" % i, "whatever")
@@ -800,6 +802,37 @@ def test_delete_prefix():
     assert_equals(l3,0)
     logging.debug("done")
 
+@C.with_custom_setup(C.setup_3_nodes, C.basic_teardown)
+def test_delete_prefix_sequence():
+
+    # deletion of all key value pairs with same prefix in a sequence
+
+    cli = C.get_client()
+    for i in xrange(100):
+        cli.set("key_%04i" % i, "whatever")
+
+    cli .set("count", "100")
+
+    prefix = "key_"
+    keys = cli.prefix(prefix)
+    l = len(keys)
+    assert_equals(l, 100, "wrong number of keys")
+    sequence = cli.makeSequence()
+    sequence.addDeletePrefix(prefix)
+    sequence.addSet("count","0")
+    cli.sequence(sequence)
+
+    keys2 = cli.prefix(prefix)
+    l2 = len(keys2)
+    assert_equals(l2,0, "wrong number of keys deleted")
+
+    count2 = cli["count"]
+    assert_equals(count2, "0", "wrong count")
+    l3 = cli.deletePrefix(prefix)
+    assert_equals(l3,0)
+    logging.debug("done")
+
+
 @C.with_custom_setup(C.setup_3_nodes, C.basic_teardown)
 def test_multi_get():
     cli = C.get_client()
diff --git a/src/client/arakoon_client.mli b/src/client/arakoon_client.mli
index 1364f5f4..fd986de6 100644
--- a/src/client/arakoon_client.mli
+++ b/src/client/arakoon_client.mli
@@ -27,6 +27,7 @@ type change =
   | Assert of (key * value option)
   | Assert_exists of (key)
   | Assert_range of key * Update.Range_assertion.t
+  | Delete_prefix of key
   | TestAndSet of key * value option * value option
   | UserFunction of string * string option
   | Sequence of change list
diff --git a/src/client/protocol_common.ml b/src/client/protocol_common.ml
index b5e96055..beb3775b 100644
--- a/src/client/protocol_common.ml
+++ b/src/client/protocol_common.ml
@@ -417,8 +417,9 @@ let rec change_to_update c =
   | TestAndSet (k,vo,v) -> Update.TestAndSet (k,vo,v)
   | Sequence cs -> Update.Sequence (List.map change_to_update cs)
   | Assert(k,vo) -> Update.Assert(k,vo)
-  | Assert_exists(k) -> Update.Assert_exists(k)
+  | Assert_exists k -> Update.Assert_exists k
   | Assert_range(p,a) -> Update.Assert_range(p,a)
+  | Delete_prefix k  -> Update.DeletePrefix k
   | UserFunction(name,vo) -> Update.UserFunction(name, vo)
 
 let _build_sequence_request buf changes =
diff --git a/src/client/python/ArakoonProtocol.py b/src/client/python/ArakoonProtocol.py
index f5ab1594..57c0e5a1 100644
--- a/src/client/python/ArakoonProtocol.py
+++ b/src/client/python/ArakoonProtocol.py
@@ -551,6 +551,13 @@ def write(self, fob):
         fob.write(_packInt(2))
         fob.write(_packString(self._key))
 
+class DeletePrefix(Update):
+    def __init__(self, key):
+        self._key = key
+    def write(self, fob):
+        fob.write(_packInt(14))
+        fob.write(_packString(self._key))
+
 class Assert(Update):
     def __init__(self, key, vo):
         self._key = key
@@ -584,7 +591,11 @@ def addSet(self, key,value):
     def addDelete(self, key):
         self._updates.append(Delete(key))
 
-    def addAssert(self, key,vo):
+    @SignatureValidator( 'string' )
+    def addDeletePrefix(self, prefix):
+        self._updates.append(DeletePrefix(prefix))
+
+    def addAssert(self, key, vo):
         self._updates.append(Assert(key,vo))
 
     def addAssertExists(self, key):

From 929a5b68bd3de5cb178a1df339fa4bd6e10a3a64 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Thu, 8 Nov 2018 17:05:34 +0100
Subject: [PATCH 356/373] allows catchup-only to pick a specific node

---
 .../server/left/system_tests_long_left.py     | 18 +++++++++++++
 src/client/arakoon_remote_client_test.ml      |  1 +
 src/main/arakoon.ml                           |  4 +++
 src/node/node_cfg.ml                          |  1 +
 src/node/node_main.ml                         | 27 ++++++++++++++-----
 src/system/startup.ml                         |  1 +
 6 files changed, 45 insertions(+), 7 deletions(-)

diff --git a/pylabs/test/server/left/system_tests_long_left.py b/pylabs/test/server/left/system_tests_long_left.py
index b7b021e0..4fc45d43 100644
--- a/pylabs/test/server/left/system_tests_long_left.py
+++ b/pylabs/test/server/left/system_tests_long_left.py
@@ -20,6 +20,7 @@
 
 from ..system_tests_common import setup_1_node, setup_2_nodes
 from ..system_tests_common import setup_2_nodes_forced_master
+from ..system_tests_common import setup_3_nodes_forced_master
 from ..system_tests_common import basic_teardown
 
 from .. import system_tests_common as C
@@ -51,6 +52,23 @@ def test_catchup_only():
     C.start_all()
     C.assert_running_nodes(2)
 
+@with_custom_setup(setup_3_nodes_forced_master, basic_teardown)
+def test_catchup_only2():
+    C.iterate_n_times(123000, C.simple_set)
+    n0 = C.node_names[0]
+    n1 = C.node_names[1]
+    n2 = C.node_names[2]
+    logging.info("stopping %s", n1)
+    C.stopOne(n1)
+    C.wipe(n1)
+    logging.info("catchup-only")
+    C.catchupOnly(n1, source_node = n0)
+    logging.info("done with catchup-only from %s", n0)
+    C.flush_store(n0)
+    C.stopOne(n0)
+    C.compare_stores(n1, n0)
+    C.start_all()
+    C.assert_running_nodes(2)
 
 
 
diff --git a/src/client/arakoon_remote_client_test.ml b/src/client/arakoon_remote_client_test.ml
index 0aa44244..2c65c265 100644
--- a/src/client/arakoon_remote_client_test.ml
+++ b/src/client/arakoon_remote_client_test.ml
@@ -49,6 +49,7 @@ let __client_server_wrapper__ (real_test:real_test) =
       Node_cfg.make_test_config
       ~cluster_id:_CLUSTER
       ~node_name:(Printf.sprintf "sweety_%i")
+      ~source_node:None
       1 (Master_type.Elected) lease_period
     in
     Lwt.return r
diff --git a/src/main/arakoon.ml b/src/main/arakoon.ml
index bedd457f..2490c65d 100644
--- a/src/main/arakoon.ml
+++ b/src/main/arakoon.ml
@@ -199,6 +199,7 @@ let main () =
   and xml_filename = ref "output.xml"
   and filter = ref ""
   and node_id = ref ""
+  and source_node = ref None
   and key = ref ""
   and value = ref ""
   and value_option = ref None
@@ -360,6 +361,8 @@ let main () =
     ("-autofix", Arg.Set autofix, "attempt to autofix (option to -node)");
     ("-catchup-only", Arg.Set catchup_only,
      "will only do a catchup of the node, without actually starting it (option to --node)");
+    ("-source-node", Arg.String (fun s -> source_node := Some s),
+     "do a catchup from that specific node (in combination with -catchup-only)");
     ("-daemonize", Arg.Set daemonize,
      "add if you want the process to daemonize (only for --node)");
     ("-start", Arg.Unit (fun () -> ()),
@@ -583,6 +586,7 @@ let main () =
               !node_id
               ~daemonize:!daemonize
               ~catchup_only:!catchup_only
+              ~source_node:!source_node
               ~autofix:!autofix
               ~lock:true
           in
diff --git a/src/node/node_cfg.ml b/src/node/node_cfg.ml
index 121fcec2..7d313b40 100644
--- a/src/node/node_cfg.ml
+++ b/src/node/node_cfg.ml
@@ -281,6 +281,7 @@ module Node_cfg = struct
 
   let make_test_config
         ?(base=4000) ?(cluster_id="ricky") ?(node_name = Printf.sprintf "t_arakoon_%i")
+        ?source_node
         n_nodes master lease_period =
     let make_one n =
       let ns = (string_of_int n) in
diff --git a/src/node/node_main.ml b/src/node/node_main.ml
index b1f41f33..12413a9a 100644
--- a/src/node/node_main.ml
+++ b/src/node/node_main.ml
@@ -183,11 +183,13 @@ let only_catchup
       (type s) (module S : Store.STORE with type t = s)
       ~tls_ctx
       ~name
+      ~source_node
       ~cluster_cfg
       ~(make_tlog_coll:Tlogcollection.tlc_factory)
       get_snapshot_name
   =
-  Logger.info_ "ONLY CATCHUP" >>= fun () ->
+  Logger.info_f_  "ONLY CATCHUP (from %s)" (Log_extra.string_option2s source_node)
+  >>= fun () ->
   let me, other_configs = Node_cfg.Node_cfg.split name cluster_cfg.cfgs in
   let cluster_id = cluster_cfg.cluster_id in
   let db_name = full_db_name me in
@@ -220,11 +222,18 @@ let only_catchup
                  ~fsync:false name ~fsync_tlog_dir:false
                  ~cluster_id
   >>= fun tlc ->
+  let other_configs' = match source_node with
+    | None -> other_configs
+    | Some n ->
+       List.filter
+         (fun ncfg -> ncfg.node_name = n)
+         other_configs
+  in
   let try_catchup mr_name =
     Lwt.catch
       (fun () ->
        Catchup.catchup ~tls_ctx ~tcp_keepalive:cluster_cfg.tcp_keepalive ~stop:(ref false)
-                       me.Node_cfg.Node_cfg.node_name other_configs ~cluster_id
+                       me.Node_cfg.Node_cfg.node_name other_configs' ~cluster_id
                        ((module S),store,tlc) mr_name >>= fun _ ->
        S.close store ~flush:true ~sync:true >>= fun () ->
        tlc # close () >>= fun () ->
@@ -246,7 +255,7 @@ let only_catchup
          Lwt.return ()
        else
          try_nodes others in
-  try_nodes other_configs
+  try_nodes other_configs'
 
 let build_ssl_context_helper f cluster = match cluster.tls with
   | None -> failwith "Node_main: No TLS configuration found"
@@ -394,7 +403,8 @@ let _main_2 (type s)
       (make_tlog_coll:Tlogcollection.tlc_factory)
       (make_config: unit -> 'a Lwt.t)
       get_snapshot_name ~name
-      ~daemonize ~catchup_only
+      ~daemonize
+      ~catchup_only ~source_node
       ~autofix
       ~stop
       ~lock
@@ -440,6 +450,7 @@ let _main_2 (type s)
         (module S)
         ~tls_ctx:ssl_context
         ~name ~cluster_cfg ~make_tlog_coll
+        ~source_node
         get_snapshot_name
       >>= fun () ->
       Lwt.return 0
@@ -1052,7 +1063,7 @@ let _main_2 (type s)
 
 
 let main_t (make_config: unit -> 'a Lwt.t)
-           name ~daemonize ~catchup_only ~autofix ~lock : int Lwt.t
+           name ~daemonize ~catchup_only ~source_node ~autofix ~lock : int Lwt.t
   =
   let module S = (val (Store.make_store_module (module Batched_store.Local_store))) in
   let (make_tlog_coll :Tlogcollection.tlc_factory) =
@@ -1062,7 +1073,8 @@ let main_t (make_config: unit -> 'a Lwt.t)
   _main_2 (module S)
           make_tlog_coll
           make_config get_snapshot_name
-          ~name ~daemonize ~catchup_only ~autofix
+          ~name ~daemonize ~catchup_only ~source_node
+          ~autofix
           ~stop:(ref false) ~lock:true
 
 let test_t make_config name ~stop =
@@ -1074,7 +1086,8 @@ let test_t make_config name ~stop =
   let get_snapshot_name = fun () -> "DUMMY" in
   let daemonize = false
   and catchup_only = false
+  and source_node = None
   and autofix = false in
   _main_2 (module S) make_tlog_coll make_config get_snapshot_name
-          ~name ~daemonize ~catchup_only
+          ~name ~daemonize ~catchup_only ~source_node
           ~stop ~autofix ~lock:false
diff --git a/src/system/startup.ml b/src/system/startup.ml
index 1f36f31e..3f2a42da 100644
--- a/src/system/startup.ml
+++ b/src/system/startup.ml
@@ -129,6 +129,7 @@ let _make_run ~stores ~tlcs ~now ~values ~get_cfgs name () =
       ~name
       ~daemonize:false
       ~catchup_only:false
+      ~source_node:None
       ~autofix:false
       ~stop:!stop
       ~lock:false

From 7e49a30b517ce2271d5476d28dfd31d72629c5ac Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Thu, 8 Nov 2018 21:18:50 +0100
Subject: [PATCH 357/373] vendoring camltc package

---
 .../packages/camltc/camltc.0.9.8+ovs1/desrc   |  1 +
 .../packages/camltc/camltc.0.9.8+ovs1/opam    | 22 +++++++++++++++++++
 .../packages/camltc/camltc.0.9.8+ovs1/url     |  2 ++
 docker/opam.switch                            |  8 ++++---
 4 files changed, 30 insertions(+), 3 deletions(-)
 create mode 100644 docker/opam-repository/packages/camltc/camltc.0.9.8+ovs1/desrc
 create mode 100644 docker/opam-repository/packages/camltc/camltc.0.9.8+ovs1/opam
 create mode 100644 docker/opam-repository/packages/camltc/camltc.0.9.8+ovs1/url

diff --git a/docker/opam-repository/packages/camltc/camltc.0.9.8+ovs1/desrc b/docker/opam-repository/packages/camltc/camltc.0.9.8+ovs1/desrc
new file mode 100644
index 00000000..16bc9936
--- /dev/null
+++ b/docker/opam-repository/packages/camltc/camltc.0.9.8+ovs1/desrc
@@ -0,0 +1 @@
+OCaml bindings for tokyo cabinet
diff --git a/docker/opam-repository/packages/camltc/camltc.0.9.8+ovs1/opam b/docker/opam-repository/packages/camltc/camltc.0.9.8+ovs1/opam
new file mode 100644
index 00000000..3cf0ea2a
--- /dev/null
+++ b/docker/opam-repository/packages/camltc/camltc.0.9.8+ovs1/opam
@@ -0,0 +1,22 @@
+opam-version: "2.0"
+maintainer: "romain.slootmaekers@openvstorage.com"
+authors: ["Jan Doms" "Joost Damad" "Romain Slootmaekers" "Nicolas Trangez"]
+homepage: "http://github.com/toolslive/camltc"
+bug-reports: "http://github.com/toolslive/camltc/issues"
+dev-repo: "git+https://github.com/toolslive/camltc.git"
+license: "Apache-2.0"
+build: [
+  ["dune" "build" "-p" name "-j" jobs]
+  ["dune" "runtest" "-p" name "-j" jobs] {with-test}
+]
+depends: [
+  "dune" {build & >= "1.1.0"}
+  "lwt" {>= "3.2.0"}
+  "logs"
+  "ounit" {with-test}
+]
+patches: "osx.patch" {os = "macos"}
+synopsis: "OCaml bindings for tokyo cabinet"
+
+
+
diff --git a/docker/opam-repository/packages/camltc/camltc.0.9.8+ovs1/url b/docker/opam-repository/packages/camltc/camltc.0.9.8+ovs1/url
new file mode 100644
index 00000000..64fa1a88
--- /dev/null
+++ b/docker/opam-repository/packages/camltc/camltc.0.9.8+ovs1/url
@@ -0,0 +1,2 @@
+archive: "https://github.com/toolslive/camltc/releases/download/camltc-0.9.8/camltc-0.9.8.tar.gz"
+checksum: "0fee185ce2ab744a938d6dd380ac07b8"
diff --git a/docker/opam.switch b/docker/opam.switch
index aa37c92e..7bf004b7 100644
--- a/docker/opam.switch
+++ b/docker/opam.switch
@@ -10,7 +10,7 @@ bisect	1.3+ovs1	installed
 bisect_ppx	1.3.2	installed
 camlbz2	0.6.0	installed
 camlp4	4.06+1	installed
-camltc	0.9.4	root
+camltc	0.9.8+ovs1	root
 cmdliner	1.0.2	installed
 conf-gmp	1	installed
 conf-gmp-powm-sec	1	installed
@@ -30,19 +30,21 @@ cryptokit	1.12	installed
 ctypes	0.13.1	installed
 ctypes-foreign	0.4.0	installed
 depext	1.0.5	installed
+dune	1.2.1	installed
 easy-format	1.3.0	installed
 ezxmlm	1.0.2	installed
 fieldslib	v0.10.0	installed
 integers	0.2.2	installed
 jane-street-headers	v0.10.0	installed
-jbuilder	1.0+beta16	installed
+jbuilder	transition	installed
+logs	0.6.2	installed
 lwt	3.2.0	root
 lwt_ssl	1.1.2	root
 mirage-no-xen	1	installed
 num	1.1	installed
 oasis	0.4.10	installed
 ocaml-compiler-libs	v0.10.0	installed
-ocaml-migrate-parsetree	1.0.7	installed
+ocaml-migrate-parsetree	1.0.11	root
 ocamlbuild	0.12.0	installed
 ocamlfind	1.7.3-1	installed
 ocamlify	0.0.1	installed

From e91c78d8ad6775782ffd55e756a68df5f38300fe Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Fri, 9 Nov 2018 10:47:31 +0100
Subject: [PATCH 358/373] sourceNode parameter in cluster mgmt

---
 pylabs/extensions/arakoon_ext/server/ArakoonManagement.py | 4 +++-
 pylabs/test/server/left/system_tests_long_left.py         | 4 ++--
 pylabs/test/server/system_tests_common.py                 | 4 ++--
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/pylabs/extensions/arakoon_ext/server/ArakoonManagement.py b/pylabs/extensions/arakoon_ext/server/ArakoonManagement.py
index 8f87e6c3..f66f7aa9 100644
--- a/pylabs/extensions/arakoon_ext/server/ArakoonManagement.py
+++ b/pylabs/extensions/arakoon_ext/server/ArakoonManagement.py
@@ -1089,7 +1089,7 @@ def startOne(self, nodeName, extraParams = None):
         return self._startOne(nodeName, extraParams)
 
 
-    def catchupOnly(self, nodeName):
+    def catchupOnly(self, nodeName, sourceNode = None):
         """
         make the node catchup, but don't start it.
         (This is handy if you want to minimize downtime before you,
@@ -1102,6 +1102,8 @@ def catchupOnly(self, nodeName):
                '--node',
                nodeName,
                '-catchup-only']
+        if sourceNode:
+            cmd.extend(['-source-node', sourceNode])
         return X.subprocess.call(cmd)
 
     def stopOne(self, nodeName):
diff --git a/pylabs/test/server/left/system_tests_long_left.py b/pylabs/test/server/left/system_tests_long_left.py
index 4fc45d43..f5a17b23 100644
--- a/pylabs/test/server/left/system_tests_long_left.py
+++ b/pylabs/test/server/left/system_tests_long_left.py
@@ -62,13 +62,13 @@ def test_catchup_only2():
     C.stopOne(n1)
     C.wipe(n1)
     logging.info("catchup-only")
-    C.catchupOnly(n1, source_node = n0)
+    C.catchupOnly(n1, sourceNode = n0)
     logging.info("done with catchup-only from %s", n0)
     C.flush_store(n0)
     C.stopOne(n0)
     C.compare_stores(n1, n0)
     C.start_all()
-    C.assert_running_nodes(2)
+    C.assert_running_nodes(3)
 
 
 
diff --git a/pylabs/test/server/system_tests_common.py b/pylabs/test/server/system_tests_common.py
index 88be9eee..ee5b85cc 100644
--- a/pylabs/test/server/system_tests_common.py
+++ b/pylabs/test/server/system_tests_common.py
@@ -407,9 +407,9 @@ def defragDb(name):
     cluster = _getCluster()
     cluster.defragDb(name)
 
-def catchupOnly(name):
+def catchupOnly(name, sourceNode = None):
     cluster = _getCluster()
-    cluster.catchupOnly(name)
+    cluster.catchupOnly(name, sourceNode)
 
 def restart_all():
     cluster = _getCluster()

From 7a5a24d26690ef1c5dede3c99c0a90a9fe5b7ce4 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 16 Jan 2019 11:06:41 +0100
Subject: [PATCH 359/373] deprecate fsync, replace with fsync_interval and
 fsync_entries

---
 cfg/arakoon.ini                 |  6 ++++-
 myocamlbuild.ml                 |  1 +
 src/main/inspect.ml             |  3 ++-
 src/node/catchup.ml             |  2 +-
 src/node/catchup_test.ml        |  6 +++--
 src/node/collapser.ml           |  3 ++-
 src/node/collapser_test.ml      | 10 +++++---
 src/node/node_cfg.ml            | 22 +++++++++++++---
 src/node/node_main.ml           | 45 ++++++++++++++++++++++++++-------
 src/paxos/multi_paxos_test.ml   |  6 ++---
 src/system/startup.ml           | 10 +++++---
 src/tlog/mem_tlogcollection.ml  | 10 +++++---
 src/tlog/tlc2.ml                | 40 +++++++++++++++++++++--------
 src/tlog/tlc2_test.ml           |  3 ++-
 src/tlog/tlogcollection.ml      | 10 +++++---
 src/tlog/tlogcollection_test.ml |  2 +-
 16 files changed, 131 insertions(+), 48 deletions(-)

diff --git a/cfg/arakoon.ini b/cfg/arakoon.ini
index 9c180cd3..ff0389e1 100644
--- a/cfg/arakoon.ini
+++ b/cfg/arakoon.ini
@@ -101,7 +101,11 @@ home = /tmp/arakoon/arakoon_0
 #log_sinks = redis://127.0.0.1:6379/logs/arakoon/arakoon_0, /path/to/file, console:
 #crash_log_sinks = redis://127.0.0.1:6379/logs/arakoon/arakoon_0/crash
 
-fsync=true
+#fsync=true  # deprecated: you can use fsync_interval and fsync_entries
+
+#fsync_interval = 1.0 # every second
+#fsync_entries = 1 # every entry, ( you can have both fsync_interval, fsync_entries then it syncs on whatever happens first )
+
 
 # available levels are: debug info notice warning error fatal
 log_level = debug
diff --git a/myocamlbuild.ml b/myocamlbuild.ml
index d96a086a..36c3bdf7 100644
--- a/myocamlbuild.ml
+++ b/myocamlbuild.ml
@@ -128,6 +128,7 @@ let _ = dispatch & function
             A"-ccopt"; A"-Wextra";
             A"-ccopt"; A"-Werror";
             A"-ccopt"; A"-ggdb3";
+            A"-ccopt"; A"-Wcast-align";
             A"-ccopt"; A"-O2";
             (* Optionally add something like -march=core2 -mtune=corei7-avx for
              * a minor extra performance gain in CRC32 calculations *)
diff --git a/src/main/inspect.ml b/src/main/inspect.ml
index 92107b73..5a0bc0a0 100644
--- a/src/main/inspect.ml
+++ b/src/main/inspect.ml
@@ -42,7 +42,8 @@ let inspect_cluster ~tls (cfg : Node_cfg.cluster_cfg) =
         ~cluster_id:cfg.cluster_id
         ~compressor:Compression.Snappy
         path path path
-        ~fsync:false node_name ~fsync_tlog_dir:false
+        ~should_fsync:(fun _ _ -> false)
+        ~fsync_tlog_dir:false node_name
       >>= fun tlc ->
 
       let db_path = path ^ "/store.db" in
diff --git a/src/node/catchup.ml b/src/node/catchup.ml
index c6284cd1..29b158f2 100644
--- a/src/node/catchup.ml
+++ b/src/node/catchup.ml
@@ -95,7 +95,7 @@ let catchup_tlog (type s) ~tls_ctx ~tcp_keepalive ~stop other_configs ~cluster_i
   let copy_tlog connection =
     make_remote_nodestream cluster_id connection >>= fun (client:nodestream) ->
     let f (i,value) =
-      tlog_coll # log_value_explicit i value ~sync:false None >>= fun _ ->
+      tlog_coll # log_value_explicit i value ~sync_override:false None >>= fun _ ->
       stop_fuse stop
     in
     client # iterate current_i
diff --git a/src/node/catchup_test.ml b/src/node/catchup_test.ml
index 6c26125f..23ee234a 100644
--- a/src/node/catchup_test.ml
+++ b/src/node/catchup_test.ml
@@ -111,7 +111,7 @@ let test_common () =
   Logger.info_ "test_common" >>= fun () ->
   Tlc2.make_tlc2 ~compressor:Compression.Snappy
                  _dir_name _tlx_dir _tlx_dir
-                 ~fsync:false "node_name" ~fsync_tlog_dir:true
+                 ~should_fsync:(fun _ _ -> false) "node_name" ~fsync_tlog_dir:true
                  ~cluster_id:""
   >>= fun tlog_coll ->
   _fill tlog_coll 1000 >>= fun () ->
@@ -145,7 +145,9 @@ let _tic filler_function n name verify_store =
                  ~tlog_max_entries:101
                  _dir_name _tlx_dir _tlx_dir
                  ~cluster_id
-                 ~fsync:false node_id ~fsync_tlog_dir:true >>= fun tlog_coll ->
+                 ~should_fsync:(fun _ _ -> false) node_id
+                 ~fsync_tlog_dir:true
+  >>= fun tlog_coll ->
   filler_function tlog_coll n >>= fun () ->
   let db_name = _dir_name ^ "/" ^ name ^ ".db" in
   S.make_store ~lcnum:1024 ~ncnum:512 db_name ~cluster_id >>= fun store ->
diff --git a/src/node/collapser.ml b/src/node/collapser.ml
index 0327a699..cfe6fa5f 100644
--- a/src/node/collapser.ml
+++ b/src/node/collapser.ml
@@ -250,7 +250,8 @@ let collapse_out_of_band ?cluster_id cfg name tlogs_to_keep =
   let module S = (val (Store.make_store_module (module Batched_store.Local_store))) in
   Tlc2.make_tlc2 ~compressor:me.compressor
                  me.tlog_dir me.tlx_dir me.head_dir
-                 ~fsync:true name ~fsync_tlog_dir:true
+                 ~should_fsync:(fun _ _ -> false) name
+                 ~fsync_tlog_dir:true
                  ~check_marker:false
                  ?cluster_id
   >>= fun tlog_coll ->
diff --git a/src/node/collapser_test.ml b/src/node/collapser_test.ml
index e8a072bf..1e9b5067 100644
--- a/src/node/collapser_test.ml
+++ b/src/node/collapser_test.ml
@@ -62,7 +62,9 @@ let test_collapse_until (dn, tlx_dir, head_dir) =
   Tlc2.make_tlc2 ~compressor
                  ~tlog_max_entries:1000
                  dn tlx_dir head_dir
-                 ~fsync:false node_id ~fsync_tlog_dir:false
+                 ~should_fsync:(fun _ _ -> false)
+                 node_id
+                 ~fsync_tlog_dir:false
                  ~cluster_id:""
   >>= fun tlc ->
   _make_values tlc 0 1111 >>= fun () ->
@@ -98,7 +100,7 @@ let test_collapse_many (dn, tlx_dir, head_dir) =
   let make_tlc () = 
     Tlc2.make_tlc2 ~compressor ~tlog_max_entries:100
                  dn tlx_dir head_dir
-                 ~fsync:false node_id ~fsync_tlog_dir:false
+                 ~should_fsync:(fun _ _ -> false) node_id ~fsync_tlog_dir:false
                  ?cluster_id
   in
   make_tlc () >>= fun tlc ->
@@ -126,7 +128,9 @@ let test_repeated_collapse (dn,tlx_dir, head_dir) =
   >>= fun () ->
   Tlc2.make_tlc2 ~compressor ~tlog_max_entries:100
                  dn tlx_dir head_dir
-                 ~fsync:false node_id ~fsync_tlog_dir:false
+                 ~should_fsync:(fun _ _ -> false)
+                 ~fsync_tlog_dir:false
+                 node_id
                  ?cluster_id
   >>= fun tlc ->
   let head_location= Filename.concat dn "head.db" in
diff --git a/src/node/node_cfg.ml b/src/node/node_cfg.ml
index 7d313b40..66530612 100644
--- a/src/node/node_cfg.ml
+++ b/src/node/node_cfg.ml
@@ -137,7 +137,9 @@ module Node_cfg = struct
             is_learner : bool;
             is_witness : bool;
             compressor : Compression.compressor;
-            fsync : bool;
+            fsync : bool option;
+            fsync_interval : float option;
+            fsync_entries : int option;
             _fsync_tlog_dir : bool;
             is_test : bool;
             reporting: int;
@@ -168,7 +170,9 @@ module Node_cfg = struct
            ; "is_learner", bool t.is_learner
            ; "is_witness", bool t.is_witness
            ; "compressor", Compression.compressor2s t.compressor
-           ; "fsync", bool t.fsync
+           ; "fsync", option bool t.fsync
+           ; "fsync_interval", option float t.fsync_interval
+           ; "fsync_entries", option int t.fsync_entries
            ; "is_test", bool t.is_test
            ; "reporting", int t.reporting
            ; "_fsync_tlog_dir", bool t._fsync_tlog_dir
@@ -306,7 +310,9 @@ module Node_cfg = struct
         is_learner = false;
         is_witness = false;
         compressor = Compression.Snappy;
-        fsync = false;
+        fsync = Some false;
+        fsync_entries = None;
+        fsync_interval = None;
         _fsync_tlog_dir = false;
         is_test = true;
         reporting = 300;
@@ -554,7 +560,11 @@ module Node_cfg = struct
           | _         -> failwith (Printf.sprintf "no compressor named %s" s)
         end
     in
-    let fsync = Ini.get inifile node_name "fsync" Ini.p_bool (Ini.default true) in
+    let fsync = Ini.get inifile node_name "fsync" (Ini.p_option Ini.p_bool) (Ini.default None) in
+
+    let fsync_interval = Ini.get inifile node_name "fsync_interval" (Ini.p_option Ini.p_float) (Ini.default None) in
+    let fsync_entries  = Ini.get inifile node_name "fsync_entries"  (Ini.p_option Ini.p_int)   (Ini.default None) in
+
     let _fsync_tlog_dir = Ini.get
                             inifile
                             node_name
@@ -631,7 +641,11 @@ module Node_cfg = struct
      is_learner;
      is_witness;
      compressor;
+
      fsync;
+     fsync_interval;
+     fsync_entries;
+
      _fsync_tlog_dir;
      is_test = false;
      reporting;
diff --git a/src/node/node_main.ml b/src/node/node_main.ml
index 12413a9a..28c62d0c 100644
--- a/src/node/node_main.ml
+++ b/src/node/node_main.ml
@@ -219,7 +219,9 @@ let only_catchup
                  ~tlog_max_entries:cluster_cfg.tlog_max_entries
                  ~tlog_max_size:cluster_cfg.tlog_max_size
                  me.tlog_dir me.tlx_dir me.head_dir
-                 ~fsync:false name ~fsync_tlog_dir:false
+                 ~should_fsync:(fun _  _  -> false)
+                 ~fsync_tlog_dir:false
+                 name
                  ~cluster_id
   >>= fun tlc ->
   let other_configs' = match source_node with
@@ -542,13 +544,36 @@ let _main_2 (type s)
                          cluster_cfg.max_buffer_size
                          ~stop in
       Logger.info_f_ "cfg = %s" (string_of me) >>= fun () ->
-      begin
-        if not me.fsync
-        then
-          Logger.warning_ "Be careful, fsync is set to false! For durability and consistency reasons - in the case of powerloss - it's recommended to set fsync=true; please make sure you understand the risks of this configuration."
-        else
-          Lwt.return ()
-      end >>= fun () ->
+      let should_fsync =
+        match me.fsync, me.fsync_interval, me.fsync_entries with
+        | Some _, Some _, None
+          | Some _, None  , Some _
+          | Some _, Some _, Some _ ->
+           failwith ("fsync is deprecated, You should remove this entry as cannot combine this with 'fsync_entries'" ^
+                       " or 'fsync_interval'")
+        | Some fsync, None, None ->
+           let () =
+             if fsync then
+                Logger.ign_warning_ "Be careful, fsync is set to false! (fsync is deprecated: if you really want to live dangerously, use fsync_entries or/and fsync_interval)"
+           in
+           (fun i time_since_last_fsync -> fsync)
+
+        | None, None, None       -> (fun _ _ -> true) (* always *)
+        | None, Some dt, None    -> (fun _ time_since_last_fsync -> time_since_last_fsync >= dt)
+        | None, None   , Some di ->
+           let di64 = Sn.of_int di in
+           (fun n_entries_since_last_fsync _ -> n_entries_since_last_fsync >= di64)
+        | None, Some dt, Some di ->
+           let di64 = Sn.of_int di in
+           fun n_entries_since_last_fsync time_since_last_fsync ->
+           let () =
+             Lwt_log.ign_debug_f "fsync n_entries_since_last_fsync:%s time_since_last_fsync:%f"
+               (Sn.string_of n_entries_since_last_fsync) time_since_last_fsync
+           in
+           n_entries_since_last_fsync >= di64
+           || time_since_last_fsync >= dt
+      in
+
       Lwt_list.iter_s (fun m -> Logger.info_f_ "other: %s" m)
         other_names >>= fun () ->
       Logger.info_f_ "quorum_function gives %i for %i"
@@ -581,11 +606,13 @@ let _main_2 (type s)
             )
           >>= fun () ->
           let compressor = me.compressor in
+
           make_tlog_coll ~compressor
                          ~tlog_max_entries:cluster_cfg.tlog_max_entries
                          ~tlog_max_size:cluster_cfg.tlog_max_size
                          me.tlog_dir me.tlx_dir me.head_dir
-                         ~fsync:me.fsync name ~fsync_tlog_dir:me._fsync_tlog_dir
+                         ~should_fsync name
+                         ~fsync_tlog_dir:me._fsync_tlog_dir
                          ~cluster_id
           >>= fun (tlog_coll:Tlogcollection.tlog_collection) ->
           let lcnum = cluster_cfg.lcnum
diff --git a/src/paxos/multi_paxos_test.ml b/src/paxos/multi_paxos_test.ml
index 98a2f3b9..15a1c4ca 100644
--- a/src/paxos/multi_paxos_test.ml
+++ b/src/paxos/multi_paxos_test.ml
@@ -64,7 +64,7 @@ let test_generic network_factory n_nodes () =
   let inject_ev q e = Lwt_buffer.add e q in
   S.make_store ~lcnum:1024 ~ncnum:512 "MEM#store" ~cluster_id >>= fun store ->
   Mem_tlogcollection.make_mem_tlog_collection
-    "MEM#tlog" None None ~fsync:false "???"
+    "MEM#tlog" None None ~should_fsync:false "???"
     ~fsync_tlog_dir:false ~cluster_id
   >>= fun tlog_coll ->
   let base = {me = "???";
@@ -266,7 +266,7 @@ let test_master_loop network_factory ()  =
   S.make_store ~lcnum:1024 ~ncnum:512 "MEM#store" ~cluster_id >>= fun store ->
   Mem_tlogcollection.make_mem_tlog_collection
     "MEM#tlog" None None ~cluster_id
-    ~fsync:false me ~fsync_tlog_dir:false
+    ~should_fsync:false me ~fsync_tlog_dir:false
   >>= fun tlog_coll ->
 
   let constants = {me = me;
@@ -410,7 +410,7 @@ let test_simulation filters () =
   S.make_store ~lcnum:1024 ~ncnum:512 "MEM#store" ~cluster_id >>= fun store ->
   Mem_tlogcollection.make_mem_tlog_collection
     "MEM#tlog" None None
-    ~fsync:false me ~fsync_tlog_dir:false
+    ~should_fsync:false me ~fsync_tlog_dir:false
     ~cluster_id
   >>= fun tlog_coll ->
 
diff --git a/src/system/startup.ml b/src/system/startup.ml
index 3f2a42da..097b71ed 100644
--- a/src/system/startup.ml
+++ b/src/system/startup.ml
@@ -68,7 +68,9 @@ let _make_cfg name n lease_period =
     is_learner = false;
     is_witness = false;
     compressor = Compression.Snappy;
-    fsync = false;
+    fsync = Some false;
+    fsync_entries = None;
+    fsync_interval = None;
     _fsync_tlog_dir = false;
     is_test = true;
     reporting = 300;
@@ -83,12 +85,14 @@ let _make_tlog_coll
       ~tlcs ~values
       ~compressor ?(tlog_max_entries:int option) ?(tlog_max_size:int option)
       tlc_name tlf_dir head_dir
-      ~(fsync:bool) (node_id:string) ~(fsync_tlog_dir:bool)
+      ~(should_fsync:Sn.t -> float -> bool)
+      (node_id:string)
+      ~(fsync_tlog_dir:bool)
   =
   let () = ignore compressor in
   Mem_tlogcollection.make_mem_tlog_collection
     ?tlog_max_entries ?tlog_max_size
-    tlc_name tlf_dir head_dir ~fsync node_id ~fsync_tlog_dir
+    tlc_name tlf_dir head_dir ~should_fsync node_id ~fsync_tlog_dir
     ?cluster_id
   >>= fun tlc ->
   let rec loop i = function
diff --git a/src/tlog/mem_tlogcollection.ml b/src/tlog/mem_tlogcollection.ml
index bf1bbab4..91a209bc 100644
--- a/src/tlog/mem_tlogcollection.ml
+++ b/src/tlog/mem_tlogcollection.ml
@@ -76,7 +76,9 @@ class mem_tlog_collection _name =
     method tlogs_to_collapse ~head_i ~last_i ~tlogs_to_keep =
       failwith "tlogs_to_collapse not supported"
 
-    method log_value i v = self #log_value_explicit i v ~sync:false None
+    method should_fsync _ = false
+         
+    method log_value i v = self #log_value_explicit i v ~sync_override:false None
 
     method accept i v = self # log_value i v
     method dump_head _oc = Llio.lwt_failfmt "dump_head not implemented"
@@ -91,7 +93,7 @@ class mem_tlog_collection _name =
     method next_rollover _ = Some Int64.max_int
 
     method invalidate () = ()
-    method log_value_explicit i (v:Value.t) ~sync marker =
+    method log_value_explicit i (v:Value.t) ~sync_override marker =
       let entry = Entry.make i v 0L marker in
       let () = data <- entry::data in
       let () = last_entry <- (Some entry) in
@@ -108,8 +110,8 @@ class mem_tlog_collection _name =
 
 let make_mem_tlog_collection
       ?cluster_id
-      ?tlog_max_entries ?tlog_max_size _tlog_dir _tlf_dir _head_dir ~fsync name ~fsync_tlog_dir =
-  let () = ignore fsync in
+      ?tlog_max_entries ?tlog_max_size _tlog_dir _tlf_dir _head_dir ~should_fsync name ~fsync_tlog_dir =
+  let () = ignore should_fsync in
   let () = ignore fsync_tlog_dir in
   let x = new mem_tlog_collection name in
   let x2 = (x :> tlog_collection) in
diff --git a/src/tlog/tlc2.ml b/src/tlog/tlc2.ml
index 76b01e0d..731ae129 100644
--- a/src/tlog/tlc2.ml
+++ b/src/tlog/tlc2.ml
@@ -120,7 +120,9 @@ class tlc2
         (head_dir:string)
         (last:Entry.t option) (index:Index.index) (file:F.t option)
         (tlog_map: TlogMap.t)
-        (node_id:string) ~(fsync:bool) ~(fsync_tlog_dir:bool)
+        (node_id:string)
+        ~(should_fsync:Sn.t -> float -> bool)
+        ~(fsync_tlog_dir:bool)
   =
   let _previous_entry = ref (Some last) in
   let set_previous_entry e =
@@ -146,6 +148,9 @@ class tlc2
     val mutable _compression_thread = None
     val mutable _compressing = false
 
+    val mutable last_sync_time  = 0.0
+    val mutable entries_since_last_sync = (Sn.pred Sn.start)
+
     val _closing = ref false
     val _write_lock = Lwt_mutex.create ()
     val _jc = (Lwt_condition.create () : unit Lwt_condition.t)
@@ -233,8 +238,9 @@ class tlc2
       _compression_thread <- Some t;
       Lwt.ignore_result t
 
+    method should_fsync now = should_fsync entries_since_last_sync (now -. last_sync_time)
 
-    method log_value_explicit i value ~sync marker =
+    method log_value_explicit i value ~sync_override marker =
       Lwt_mutex.with_lock _write_lock
         (fun () ->
            begin
@@ -244,9 +250,19 @@ class tlc2
              Tlogcommon.write_entry oc i value >>= fun total_size ->
              Lwt_io.flush oc >>= fun () ->
              begin
-               if sync
-               then F.fsync file
-               else Lwt.return ()
+               let () = entries_since_last_sync <- Sn.succ entries_since_last_sync in
+               let now = Unix.gettimeofday() in
+               let time_since_last_sync = now -. last_sync_time in
+               let should_fsync = sync_override || self # should_fsync time_since_last_sync in
+               if should_fsync
+               then
+                 let () =
+                   last_sync_time <- now;
+                   entries_since_last_sync  <- Sn.zero
+                 in
+                 F.fsync file
+               else
+                 Lwt.return_unit
              end
              >>= fun () ->
              begin
@@ -262,13 +278,12 @@ class tlc2
              Lwt.return total_size
            end)
 
-    method log_value i value =
-      self # log_value_explicit i value fsync None
+    method log_value i value = self # log_value_explicit i value ~sync_override:false None
 
     method accept i v =
-      let sync = fsync || Value.is_synced v in
+      let sync_override = Value.is_synced v in
       let marker = None in
-      self # log_value_explicit i v ~sync marker
+      self # log_value_explicit i v ~sync_override marker
 
     method private _prelude i =
       match _file with
@@ -482,6 +497,7 @@ class tlc2
               Logger.debug_f_ "wrote %S marker @i=%s for %S"
                               (Log_extra.string_option2s marker) (Sn.string_of i) node_id
               >>= fun () ->
+              F.fsync file >>= fun () ->
               F.close file
            | None -> Logger.fatal_f_ "how can this be?"
          end
@@ -575,7 +591,9 @@ let make_tlc2 ?cluster_id
               ?tlog_max_entries ?tlog_max_size
               ?(check_marker=true)
               tlog_dir tlf_dir
-              head_dir ~fsync node_id ~fsync_tlog_dir
+              head_dir
+              ~(should_fsync: Sn.t -> float -> bool)
+              node_id ~fsync_tlog_dir
   =
   Logger.debug_f_ "make_tlc2 %S" tlog_dir >>= fun () ->
   TlogMap.make ?tlog_max_entries ?tlog_max_size
@@ -616,7 +634,7 @@ let make_tlc2 ?cluster_id
   end >>= fun file ->
   let col = new tlc2 head_dir last index
                 file tlog_map
-                ~compressor node_id ~fsync ~fsync_tlog_dir
+                ~compressor node_id ~should_fsync ~fsync_tlog_dir
                 ?cluster_id
   in
   Lwt.return col
diff --git a/src/tlog/tlc2_test.ml b/src/tlog/tlc2_test.ml
index 79da0ae9..a1156fe1 100644
--- a/src/tlog/tlc2_test.ml
+++ b/src/tlog/tlc2_test.ml
@@ -33,7 +33,8 @@ let create_test_tlc ?tlog_max_entries ?tlog_max_size dn =
   Tlc2.make_tlc2
     dn
     ?tlog_max_entries ?tlog_max_size tlx_dir tlx_dir
-    ~compressor ~fsync:false ~fsync_tlog_dir:false
+    ~compressor ~should_fsync:(fun _ _ -> false)
+    ~fsync_tlog_dir:false
     ~check_marker:true
     ~cluster_id
 
diff --git a/src/tlog/tlogcollection.ml b/src/tlog/tlogcollection.ml
index 62f1c211..3c76e1fe 100644
--- a/src/tlog/tlogcollection.ml
+++ b/src/tlog/tlogcollection.ml
@@ -26,8 +26,10 @@ class type tlog_collection =
                   -> unit Lwt.t
   method accept : Sn.t -> Value.t    -> int Lwt.t
   method log_value : Sn.t -> Value.t -> int Lwt.t
-  method log_value_explicit : Sn.t -> Value.t ->
-                              sync:bool -> string option -> int Lwt.t
+
+  method should_fsync : float -> bool
+
+  method log_value_explicit : Sn.t -> Value.t -> sync_override:bool -> string option -> int Lwt.t
 
 
   method tlogs_to_collapse:
@@ -78,5 +80,7 @@ type tlc_factory =
   string ->
   string ->
   string ->
-  fsync:bool -> string -> fsync_tlog_dir:bool ->
+  should_fsync:(Sn.t -> float -> bool) ->
+  string
+  -> fsync_tlog_dir:bool ->
   tlog_collection Lwt.t
diff --git a/src/tlog/tlogcollection_test.ml b/src/tlog/tlogcollection_test.ml
index 681660c7..24164ec3 100644
--- a/src/tlog/tlogcollection_test.ml
+++ b/src/tlog/tlogcollection_test.ml
@@ -263,7 +263,7 @@ let wrap (factory:factory) test (name:string) = lwt_bracket (setup factory name)
 let create_test_tlc ?tlog_max_entries ?tlog_max_size dn =
   Mem_tlogcollection.make_mem_tlog_collection
     ?tlog_max_entries ?tlog_max_size
-    dn None None ~fsync:false ~fsync_tlog_dir:false
+    dn None None ~should_fsync:false ~fsync_tlog_dir:false
     ~cluster_id:""
 
 let wrap_memory name = wrap create_test_tlc name

From d99fa3894c4d5b8597393bb80d0a0d03c2316a6d Mon Sep 17 00:00:00 2001
From: Jeffrey Devloo <jeffreydevloo@hotmail.com>
Date: Tue, 26 Feb 2019 15:26:19 +0100
Subject: [PATCH 360/373] Update pyrakoon to the refactored branch

---
 pylabs/pyrakoon | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pylabs/pyrakoon b/pylabs/pyrakoon
index e69b8d2e..22cd6160 160000
--- a/pylabs/pyrakoon
+++ b/pylabs/pyrakoon
@@ -1 +1 @@
-Subproject commit e69b8d2e9852709b866bea25f594124385e9a364
+Subproject commit 22cd6160d5b8bc028072d64ffe5b50ca253cb8b3

From f205068cb5ad44f0f9736cc1418cb1a00a19dbe1 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Tue, 5 Mar 2019 16:44:35 +0100
Subject: [PATCH 361/373] add test for inspect

---
 pylabs/test/server/quick/TestInspect.py   | 106 ++++++++++++++++++++++
 pylabs/test/server/system_tests_common.py |   5 +
 src/main/arakoon.ml                       |  10 ++
 src/main/inspect.ml                       |   5 +-
 src/node/dump_store.ml                    |  56 ++++++++----
 5 files changed, 163 insertions(+), 19 deletions(-)
 create mode 100644 pylabs/test/server/quick/TestInspect.py

diff --git a/pylabs/test/server/quick/TestInspect.py b/pylabs/test/server/quick/TestInspect.py
new file mode 100644
index 00000000..93a0bf5d
--- /dev/null
+++ b/pylabs/test/server/quick/TestInspect.py
@@ -0,0 +1,106 @@
+'''
+  Copyright (C) iNuron - info@openvstorage.com
+  This file is part of Open vStorage. For license information, see <LICENSE.txt>
+'''
+
+import os
+import logging
+from .. import system_tests_common as C
+import time
+from Compat import X
+
+def compare_files(fn0, fn1):
+    with open(fn0, 'r') as f0:
+        with open(fn1, 'r') as f1:
+            go_on = True
+            count = 0
+            while go_on:
+                f0_line = f0.readline()
+                f1_line = f1.readline()
+                logging.debug("l0:%s\nl1:%s\n", f0_line.strip(), f1_line.strip())
+                if f0_line <> f1_line :
+                    raise Exception("%s and %s diff on line %i" % (fn0,fn1, count))
+                if len(f0_line) == 0:
+                    assert(len(f1_line) == 0)
+                    go_on = False
+                count = count + 1
+
+
+def inspect_cluster(cluster):
+
+    # cleanup inspect dumps
+    cluster_id = cluster._getClusterId()
+    node_names = cluster.listLocalNodes()
+    for node_name in node_names:
+        dn = '%s/%s/' % (cluster_id, node_name,)
+        logging.info( "deleting %s", dn)
+        X.removeDirTree(dn)
+
+    cfg_fn = cluster._getConfigFileName()
+    r = C.call_arakoon("--inspect-cluster", "-config", "%s.cfg" % cfg_fn)
+    
+
+    dump_paths = []
+    for node_name in node_names:
+        dump_path = './%s/%s/store.db.dump' % (cluster_id, node_name)
+        dump_paths.append(dump_path)
+
+    size = len(node_names)
+    for i in xrange(size -1):
+        for j in xrange(i+1, size):
+            fn0 = dump_paths[i]
+            fn1 = dump_paths[j]
+            compare_files(fn0,fn1)
+            logging.info("%s == %s",fn0,fn1)
+    logging.info("inspect_cluster(%s): ok", cluster_id)
+
+
+
+@C.with_custom_setup(C.setup_3_nodes_mini, C.basic_teardown)
+def test_inspect():
+    client = C.get_client()
+    for i in xrange(1500):
+        k = "key_%04i" % i
+        client.set(k,k)
+    cluster = C._getCluster()
+    cfg_fn = cluster._getConfigFileName()
+
+    
+    node_id = C.node_names[0]
+    C.dropMaster(node_id)
+    C.copyDbToHead(node_id, 1)
+    C.stop_all()
+
+    # tamper with node_0's store.
+    
+    node_cfg = cluster.getNodeConfig(node_id )
+    print node_cfg.keys()
+    head_dir = node_cfg['head_dir']
+    
+    head_file_name= '%s/head.db' % (head_dir)
+    logging.info("head_file_name=%s", head_file_name)
+    
+    r = C.call_arakoon("--dev-set-store", head_file_name, "@AAAAA", "AAA" * 300)
+    r = C.call_arakoon("--dev-set-store", head_file_name, "@zzzzz", "ZZZ" * 300)
+
+    
+
+    
+    C.start_all()
+
+    logging.info("%s's head has been updated", node_id)
+    
+    cli = C.get_client()
+    for i in xrange(1500,3001):
+        k = "key_%04i" % i
+        cli.set(k,k)
+
+    #time.sleep(100)
+    logging.info("inspect .... which should fail!")
+    ok = False
+    try:
+        inspect_cluster(cluster)
+    except Exception:
+        ok = True
+    assert(ok)
+
diff --git a/pylabs/test/server/system_tests_common.py b/pylabs/test/server/system_tests_common.py
index ee5b85cc..839b0dc7 100644
--- a/pylabs/test/server/system_tests_common.py
+++ b/pylabs/test/server/system_tests_common.py
@@ -125,6 +125,11 @@ def _getCluster( c_id = None):
     mgmt = ArakoonManagement.ArakoonManagement()
     return mgmt.getCluster(c_id)
 
+def call_arakoon(*params):
+    cmd = [CONFIG.binary_full_path] + list(params)
+    r = X.subprocess.check_output(cmd)
+    return r
+
 def dump_tlog (node_id, tlog_number) :
     cluster = _getCluster()
     node_home_dir = cluster.getNodeConfig(node_id ) ['home']
diff --git a/src/main/arakoon.ml b/src/main/arakoon.ml
index 2490c65d..38d1d988 100644
--- a/src/main/arakoon.ml
+++ b/src/main/arakoon.ml
@@ -71,6 +71,8 @@ type local_action =
   | REV_RANGE_ENTRIES
   | VerifyStore
   | InspectCluster
+  | StoreSet
+
 
 type server_action =
   | Node
@@ -307,6 +309,12 @@ let main () =
                                      Arg.Set_string filename;
                                      Arg.Set_int i; ],
      "<filename> <i> : fiddle with the i value stored in a (tokyo cabinet) store. Be careful, this is dangerous!");
+    ("--dev-set-store", Arg.Tuple [ set_laction StoreSet;
+                                    Arg.Set_string filename;
+                                    Arg.Set_string key;
+                                    Arg.Set_string value;
+                          ],
+     "<filename> <key> <value> : <filename>[<key>] := <value>. Be careful, this is dangerous");
     ("--compress-tlog", Arg.Tuple[set_laction CompressTlog;
                                   Arg.Set_string filename],
      "<filename> : compress a tlog file");
@@ -574,6 +582,8 @@ let main () =
        in
        Lwt_main.run (t ());
        0
+    | StoreSet -> Dump_store.store_set !filename !key !value
+
   in
   let do_server node =
     match node with
diff --git a/src/main/inspect.ml b/src/main/inspect.ml
index 5a0bc0a0..de22d692 100644
--- a/src/main/inspect.ml
+++ b/src/main/inspect.ml
@@ -24,7 +24,10 @@ module S = (val (Store.make_store_module (module Batched_store.Local_store)))
 let ensure_dir d p =
   Lwt.catch
     (fun () -> File_system.mkdir d p)
-    (function | Unix.Unix_error (Unix.EEXIST, _, _) -> Lwt.return ())
+    (function
+     | Unix.Unix_error (Unix.EEXIST, _, _) -> Lwt.return ()
+     | exn -> Lwt.fail exn
+    )
 
 let inspect_cluster ~tls (cfg : Node_cfg.cluster_cfg) =
 
diff --git a/src/node/dump_store.ml b/src/node/dump_store.ml
index 2d22f914..d82a37f4 100644
--- a/src/node/dump_store.ml
+++ b/src/node/dump_store.ml
@@ -173,26 +173,33 @@ let verify_store fn start_key =
             | None -> ()
             | Some key -> Bdb.jump bdb cursor key
           in
-          let rec loop i =
-            let key = Bdb.key bdb cursor in
-            let _value = Bdb.value bdb cursor in
-            begin
-              if (i < 10 || i mod 20000 = 0)
-              then Lwt_io.eprintlf "%10i: %S\tok%!" i key |> Lwt.ignore_result
-            end;
-            let cont =
-              try
-                Bdb.next bdb cursor;
-                true
-              with Not_found -> false
-            in
-            if cont
-            then loop (i+1)
-            else 0
+          let rec loop checksum cont i =
+            if not cont
+            then
+              Lwt_io.eprintlf "     ...\n%10i: checksum =  %04lx" i checksum >>= fun () ->
+              Lwt.return 0
+            else
+              begin
+                let key = Bdb.key bdb cursor in
+                let value = Bdb.value bdb cursor in
+                let checksum' =
+                  let tmp = Crc32c.update_crc32c checksum key    0 (String.length key)   in
+                  let tmp = Crc32c.update_crc32c tmp      value  0 (String.length value) in
+                  tmp
+                in
+                begin
+                  if (i < 10 || i mod 10000 = 0)
+                  then Lwt_io.eprintlf "%10i: %S\tok%!" i key
+                  else Lwt.return_unit
+                end
+                >>= fun () ->
+                let cont' = try Bdb.next bdb cursor; true with Not_found -> false in
+                loop checksum' cont' (i+1)
+              end;
           in
-          loop 0
+          let checksum0 = 0xffffl in
+          loop checksum0 true 0
         )
-      |> Lwt.return
     in
     go ()
   in
@@ -265,3 +272,16 @@ let set_store_i fn new_i =
     Lwt.return 0
   in
   Lwt_extra.run t
+
+let store_set fn key value =
+  let t () =
+    Local_store.make_store ~lcnum:1024 ~ncnum:512 false fn >>= fun store ->
+    Local_store.with_transaction
+      store
+      (fun tx ->
+        Local_store.set store tx key value;
+        Lwt.return_unit)
+    >>= fun () ->
+    Lwt.return 0
+  in
+  Lwt_extra.run t

From c09c0e2a935346c64675bf941b3b93637cfe7f61 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 6 Mar 2019 09:39:38 +0100
Subject: [PATCH 362/373] move inspect to common to make it usable outside of
 test

---
 extension/server/NurseryManagement.py         | 14 ++---
 extension/test/monkey/arakoon_monkey.py       | 21 ++++----
 .../test/monkey/arakoon_monkey_config.py      |  2 +-
 pylabs/test/server/quick/TestInspect.py       | 53 -------------------
 pylabs/test/server/system_tests_common.py     | 52 ++++++++++++++++--
 5 files changed, 65 insertions(+), 77 deletions(-)

diff --git a/extension/server/NurseryManagement.py b/extension/server/NurseryManagement.py
index 2312c1c4..dfd96645 100644
--- a/extension/server/NurseryManagement.py
+++ b/extension/server/NurseryManagement.py
@@ -14,11 +14,7 @@
 limitations under the License.
 """
 
-try:
-    from pymonkey import q
-except ImportError:
-    from pylabs import q
-
+from Compat import X
 import ArakoonManagement
 
 class NurseryManagement:
@@ -35,7 +31,7 @@ def getNursery(self, clusterId ):
 
     @staticmethod
     def getConfigLocation(clusterId):
-        cfg = q.config.getConfig( "arakoonclusters" )
+        cfg = X.getConfig( "arakoonclusters" )
         if cfg.has_key( clusterId ):
             return "%s/%s.cfg" % (cfg[ clusterId ]["path"], clusterId)
         else:
@@ -103,9 +99,9 @@ def delete(self, clusterId, separator = None):
         self.__runCmd( cmd )
 
     def __runCmd(self, cmd):
-        (exit, stdout, stderr) = q.system.process.run( commandline = cmd, stopOnError=False)
-        if exit :
-            raise RuntimeError( stderr )
+        rc  = X.subprocess.check_call(cmd)
+        if rc :
+            raise RuntimeError("rc=%i", rc)
 
     def __which(self):
         return ArakoonManagement.which_arakoon()
diff --git a/extension/test/monkey/arakoon_monkey.py b/extension/test/monkey/arakoon_monkey.py
index 864c7921..97a91fca 100644
--- a/extension/test/monkey/arakoon_monkey.py
+++ b/extension/test/monkey/arakoon_monkey.py
@@ -16,8 +16,6 @@
 
 import sys
 
-from pymonkey import InitBase
-
 import copy
 import re
 import logging
@@ -28,9 +26,10 @@
 import time
 import os
 import nose.tools as NT
-from pymonkey import q
+
+from Compat import X
+import server.system_tests_common as C
 import arakoon_monkey_config as MConfig
-from arakoon_system_tests.server import system_tests_common as C
 from arakoon.ArakoonProtocol import ArakoonProtocol
 from arakoon.ArakoonExceptions import *
 MEM_MAX_KB = 1024 * 256
@@ -40,7 +39,7 @@
 random.seed(42)
 
 def get_monkey_work_dir() :
-    return q.system.fs.joinPaths( q.dirs.tmpDir, "arakoon_monkey" )
+    return '/'.join([X.tmpDir,"arakoon_monkey"])
 
 def get_work_list_log_read ( iter_cnt ):
     return get_work_list_log ( iter_cnt, "r")
@@ -49,7 +48,7 @@ def get_work_list_log_write ( iter_cnt ):
     return get_work_list_log ( iter_cnt, "w")
 
 def get_work_list_log ( iter_cnt, flag ):
-    log_file_name = q.system.fs.joinPaths( get_monkey_work_dir() , "arakoon_monkey_%012d.wlist" % iter_cnt )
+    log_file_name = '/'.join([get_monkey_work_dir() , "arakoon_monkey_%012d.wlist" % iter_cnt] )
     file = open( log_file_name , flag )
     return file
 
@@ -320,20 +319,20 @@ def make_monkey_run() :
 
     global monkey_dies
 
-    C.data_base_dir = '/opt/qbase3/var/tmp/arakoon-monkey'
+    C.data_base_dir = '/'.join([X.tmpDir,"/arakoon_monkey/"])
 
     t = threading.Thread( target=memory_monitor)
     t.start()
 
     C.stop_all()
-    cluster = q.manage.arakoon.getCluster(C.cluster_id)
+    cluster = C._getCluster(C.cluster_id)
     cluster.tearDown()
     #setup_3_nodes_forced_master()
     C.setup_3_nodes( C.data_base_dir )
     monkey_dir = get_monkey_work_dir()
-    if q.system.fs.exists( monkey_dir ) :
-        q.system.fs.removeDirTree( monkey_dir )
-    q.system.fs.createDir( monkey_dir )
+    if X.fileExists( monkey_dir ) :
+        X.removeDirTree( monkey_dir )
+    X.createDir( monkey_dir )
     iteration = 0
     C.start_all()
     time.sleep( 1.0 )
diff --git a/extension/test/monkey/arakoon_monkey_config.py b/extension/test/monkey/arakoon_monkey_config.py
index 14878515..4caa3947 100644
--- a/extension/test/monkey/arakoon_monkey_config.py
+++ b/extension/test/monkey/arakoon_monkey_config.py
@@ -15,7 +15,7 @@
 """
 
 from arakoon.ArakoonExceptions import *
-from arakoon_system_tests.server import system_tests_common as C
+import server.system_tests_common as C
 #from right.system_tests_anomaly import *
 #from quick.system_tests_basic import *
 
diff --git a/pylabs/test/server/quick/TestInspect.py b/pylabs/test/server/quick/TestInspect.py
index 93a0bf5d..ff45d76b 100644
--- a/pylabs/test/server/quick/TestInspect.py
+++ b/pylabs/test/server/quick/TestInspect.py
@@ -6,55 +6,6 @@
 import os
 import logging
 from .. import system_tests_common as C
-import time
-from Compat import X
-
-def compare_files(fn0, fn1):
-    with open(fn0, 'r') as f0:
-        with open(fn1, 'r') as f1:
-            go_on = True
-            count = 0
-            while go_on:
-                f0_line = f0.readline()
-                f1_line = f1.readline()
-                logging.debug("l0:%s\nl1:%s\n", f0_line.strip(), f1_line.strip())
-                if f0_line <> f1_line :
-                    raise Exception("%s and %s diff on line %i" % (fn0,fn1, count))
-                if len(f0_line) == 0:
-                    assert(len(f1_line) == 0)
-                    go_on = False
-                count = count + 1
-
-
-def inspect_cluster(cluster):
-
-    # cleanup inspect dumps
-    cluster_id = cluster._getClusterId()
-    node_names = cluster.listLocalNodes()
-    for node_name in node_names:
-        dn = '%s/%s/' % (cluster_id, node_name,)
-        logging.info( "deleting %s", dn)
-        X.removeDirTree(dn)
-
-    cfg_fn = cluster._getConfigFileName()
-    r = C.call_arakoon("--inspect-cluster", "-config", "%s.cfg" % cfg_fn)
-    
-
-    dump_paths = []
-    for node_name in node_names:
-        dump_path = './%s/%s/store.db.dump' % (cluster_id, node_name)
-        dump_paths.append(dump_path)
-
-    size = len(node_names)
-    for i in xrange(size -1):
-        for j in xrange(i+1, size):
-            fn0 = dump_paths[i]
-            fn1 = dump_paths[j]
-            compare_files(fn0,fn1)
-            logging.info("%s == %s",fn0,fn1)
-    logging.info("inspect_cluster(%s): ok", cluster_id)
-
-
 
 @C.with_custom_setup(C.setup_3_nodes_mini, C.basic_teardown)
 def test_inspect():
@@ -83,9 +34,6 @@ def test_inspect():
     r = C.call_arakoon("--dev-set-store", head_file_name, "@AAAAA", "AAA" * 300)
     r = C.call_arakoon("--dev-set-store", head_file_name, "@zzzzz", "ZZZ" * 300)
 
-    
-
-    
     C.start_all()
 
     logging.info("%s's head has been updated", node_id)
@@ -95,7 +43,6 @@ def test_inspect():
         k = "key_%04i" % i
         cli.set(k,k)
 
-    #time.sleep(100)
     logging.info("inspect .... which should fail!")
     ok = False
     try:
diff --git a/pylabs/test/server/system_tests_common.py b/pylabs/test/server/system_tests_common.py
index 839b0dc7..1fd90f92 100644
--- a/pylabs/test/server/system_tests_common.py
+++ b/pylabs/test/server/system_tests_common.py
@@ -508,11 +508,11 @@ def get_memory_usage(node_name):
     pid = cluster._getPid(node_name )
     if pid is None:
         return 0
-    cmd = ["ps", "-p", pid, "-o", "vsz"]
+    cmd = ["ps", "-p", str(pid), "-o", "vsz"]
     try:
         stdout = X.subprocess.check_output( cmd )
-    except:
-        logging.error( "Coud not determine memory usage: %s" % stderr )
+    except Exception,e:
+        logging.error( "Coud not determine memory usage: %s" % e)
         return 0
 
     try:
@@ -1343,3 +1343,49 @@ def reverse_range_entries_scenario(start_suffix):
         assert_equals(kv_list[0][0], 'key_000000001099')
     except Exception, ex:
         raise ex
+
+
+def compare_files(fn0, fn1):
+    with open(fn0, 'r') as f0:
+        with open(fn1, 'r') as f1:
+            go_on = True
+            count = 0
+            while go_on:
+                f0_line = f0.readline()
+                f1_line = f1.readline()
+                logging.debug("l0:%s\nl1:%s\n", f0_line.strip(), f1_line.strip())
+                if f0_line <> f1_line :
+                    raise Exception("%s and %s diff on line %i" % (fn0,fn1, count))
+                if len(f0_line) == 0:
+                    assert(len(f1_line) == 0)
+                    go_on = False
+                count = count + 1
+
+
+def inspect_cluster(cluster):
+
+    # cleanup inspect dumps
+    cluster_id = cluster._getClusterId()
+    node_names = cluster.listLocalNodes()
+    for node_name in node_names:
+        dn = '%s/%s/' % (cluster_id, node_name,)
+        logging.info( "deleting %s", dn)
+        X.removeDirTree(dn)
+
+    cfg_fn = cluster._getConfigFileName()
+    r = call_arakoon("--inspect-cluster", "-config", "%s.cfg" % cfg_fn)
+
+
+    dump_paths = []
+    for node_name in node_names:
+        dump_path = './%s/%s/store.db.dump' % (cluster_id, node_name)
+        dump_paths.append(dump_path)
+
+    size = len(node_names)
+    for i in xrange(size -1):
+        for j in xrange(i+1, size):
+            fn0 = dump_paths[i]
+            fn1 = dump_paths[j]
+            compare_files(fn0,fn1)
+            logging.info("%s == %s",fn0,fn1)
+    logging.info("inspect_cluster(%s): ok", cluster_id)

From b207eb8dfa7152f8de8cae67b0a06abccaa90583 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 6 Mar 2019 10:05:07 +0100
Subject: [PATCH 363/373] remove compiler warning

---
 src/tlog/tlog_map.ml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/tlog/tlog_map.ml b/src/tlog/tlog_map.ml
index ca37ea24..0059ca3c 100644
--- a/src/tlog/tlog_map.ml
+++ b/src/tlog/tlog_map.ml
@@ -118,10 +118,11 @@ let first_i_of filename tlx_dir =
            Printf.sprintf "%s/%s" tlx_dir (to_archive_name c f |> Filename.basename)
          in
          Lwt.catch
-           (fun () -> get_i (to_archive_name Compression.Snappy filename))
+           (fun () -> to_archive_name Compression.Snappy filename |> get_i)
            (function
-             | Unix.Unix_error(Unix.ENOENT, _, _) ->
-                get_i (to_archive_name Compression.Bz2 filename))
+             | Unix.Unix_error(Unix.ENOENT, _, _) -> to_archive_name Compression.Bz2 filename |> get_i
+             | e -> Lwt.fail e
+           )
       | exn ->
          Lwt_log.warning_f ~exn "Exception during first_i_of %s" filename >>= fun () ->
          Lwt.fail exn)

From 74296a4f365cfacb9400c1d189f92fc57aa5dd36 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 6 Mar 2019 15:16:44 +0100
Subject: [PATCH 364/373] restoring dump_store in all its optional grandor

---
 .../server/shaky/test_restart_master_long.py  |  8 ++++++-
 pylabs/test/server/system_tests_common.py     | 20 ++++++------------
 src/main/arakoon.ml                           |  2 +-
 src/node/dump_store.ml                        | 21 +++++++++++++++++--
 4 files changed, 33 insertions(+), 18 deletions(-)

diff --git a/pylabs/test/server/shaky/test_restart_master_long.py b/pylabs/test/server/shaky/test_restart_master_long.py
index 95d8264a..4466c855 100644
--- a/pylabs/test/server/shaky/test_restart_master_long.py
+++ b/pylabs/test/server/shaky/test_restart_master_long.py
@@ -26,7 +26,7 @@
 from nose.tools import *
 from Compat import X
 
-@Common.with_custom_setup( Common.default_setup, Common.basic_teardown )
+@Common.with_custom_setup( Common.setup_3_nodes_mini, Common.basic_teardown )
 def test_restart_master_long ():
     restart_iter_cnt = 10
     factor = 100
@@ -90,6 +90,12 @@ def restart_loop ():
     Common.stop_all()
     Common.start_all()
     Common.flush_stores()
+
+    logging.info("going to inspect the running cluster")
+    cluster = Common._getCluster()
+    Common.inspect_cluster(cluster)
+    logging.info("inspection succesful")
+
     Common.stop_all()
     node_names = Common.node_names
 
diff --git a/pylabs/test/server/system_tests_common.py b/pylabs/test/server/system_tests_common.py
index 1fd90f92..c0c6dc90 100644
--- a/pylabs/test/server/system_tests_common.py
+++ b/pylabs/test/server/system_tests_common.py
@@ -166,21 +166,11 @@ def dump_store( node_id ):
 
     db_file = get_node_db_file ( node_id )
     dump_file = '/'.join([X.tmpDir,"%s.dump" % node_id])
-    cmd = get_tcbmgr_path() + " list -pv " + db_file
-    try:
-        dump_fd = open( dump_file, 'w' )
+    cmd = [CONFIG.binary_full_path, "--dump-store", db_file, "-dump-values"]
+
+    with open( dump_file, 'w' ) as dump_fd:
         logging.info( "Dumping store of %s to %s" % (node_id, dump_file) )
-        stdout= X.subprocess.check_output(cmd)
-        dump_fd.write(stdout)
-        dump_fd.close()
-    except OSError as ose:
-        if ose.errno == 2:
-            logging.info("store :%s is empty" % db_file)
-        else:
-            raise ose
-    except Exception as ex:
-        logging.info("Unexpected error: %s" % sys.exc_info()[0])
-        raise ex
+        rc = X.subprocess.check_call(cmd, stdout = dump_fd)
     return dump_file
 
 def flush_store(node_name):
@@ -1159,8 +1149,10 @@ def delayed_master_restart_loop ( iter_cnt, delay ) :
             master_id = cli.whoMaster()
             cli.dropConnections()
             stopOne( master_id )
+            logging.info("stopped master %s", master_id)
             cli.set('delayed_master_restart_loop', 'slaves elect new master and can make progress')
             startOne( master_id )
+            logging.info("started ex-master %s", master_id)
         except:
             logging.critical("!!!! Failing test. Exception in restart loop.")
             test_failed = True
diff --git a/src/main/arakoon.ml b/src/main/arakoon.ml
index 38d1d988..39ea6b91 100644
--- a/src/main/arakoon.ml
+++ b/src/main/arakoon.ml
@@ -506,7 +506,7 @@ let main () =
     | MarkTlog -> Tlog_main.mark_tlog !filename !key
     | CloseTlog -> Tlog_main.mark_tlog !filename (Tlog_map._make_close_marker !node_id)
     | ReplayTlogs -> Replay_main.replay_tlogs !tlog_dir !tlf_dir !filename !end_i
-    | DumpStore -> Dump_store.dump_store !filename
+    | DumpStore -> Dump_store.dump_store !filename ~dump_values:!dump_values
     | InspectStore -> Dump_store.inspect_store !filename !left !max_results
     | FiddleStoreI -> Dump_store.set_store_i !filename !i
     | VerifyStore -> Dump_store.verify_store !filename None
diff --git a/src/node/dump_store.ml b/src/node/dump_store.ml
index d82a37f4..56b6d750 100644
--- a/src/node/dump_store.ml
+++ b/src/node/dump_store.ml
@@ -57,10 +57,27 @@ let summary store =
   _dump_interval store >>= fun () ->
   _dump_cluster_id store
 
-let dump_store filename =
+let dump_store ~dump_values filename =
   let t () =
-    S.make_store ~lcnum:1024 ~ncnum:512 filename >>= fun store ->
+    let open Simple_store in
+    S.make_store ~lcnum:1024 ~ncnum:512 ~read_only:true filename >>= fun store ->
     summary store >>= fun () ->
+    let rec loop (first:string option) =
+      if first = None then Lwt.return_unit
+      else
+        let n,kvs = S.range_entries store first false None true 100 in
+        Lwt_list.fold_left_s
+          (fun last (k,v)  ->
+            let (kx:string) = Key.get k in
+            Lwt_io.printlf "%S\t%S" kx v >>= fun () ->
+            Lwt.return (Some kx)
+          )
+          None kvs
+        >>= fun lo ->
+        loop lo
+    in
+    loop (Some "")
+    >>= fun () ->
     S.close store ~sync:false ~flush:false
   in
   Lwt_extra.run t;

From 1c3fd64b41704c73aab5b83c4356fd986fcca9e7 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 6 Mar 2019 16:38:09 +0100
Subject: [PATCH 365/373] restart_master_long times 3

---
 pylabs/test/server/shaky/test_restart_master_long.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/pylabs/test/server/shaky/test_restart_master_long.py b/pylabs/test/server/shaky/test_restart_master_long.py
index 4466c855..32865d8c 100644
--- a/pylabs/test/server/shaky/test_restart_master_long.py
+++ b/pylabs/test/server/shaky/test_restart_master_long.py
@@ -28,8 +28,9 @@
 
 @Common.with_custom_setup( Common.setup_3_nodes_mini, Common.basic_teardown )
 def test_restart_master_long ():
-    restart_iter_cnt = 10
-    factor = 100
+    scale = 3
+    restart_iter_cnt = 10 * scale
+    factor = 100 * scale
     def write_loop ():
         Common.iterate_n_times( 100* factor,
                                 Common.retrying_set_get_and_delete,

From 020ddaa089db6f6523738c432c28df570857a3fd Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Thu, 7 Mar 2019 09:20:35 +0100
Subject: [PATCH 366/373] jenkins can't handle it

---
 pylabs/test/server/shaky/test_restart_master_long.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pylabs/test/server/shaky/test_restart_master_long.py b/pylabs/test/server/shaky/test_restart_master_long.py
index 32865d8c..e3e80568 100644
--- a/pylabs/test/server/shaky/test_restart_master_long.py
+++ b/pylabs/test/server/shaky/test_restart_master_long.py
@@ -28,7 +28,7 @@
 
 @Common.with_custom_setup( Common.setup_3_nodes_mini, Common.basic_teardown )
 def test_restart_master_long ():
-    scale = 3
+    scale = 1
     restart_iter_cnt = 10 * scale
     factor = 100 * scale
     def write_loop ():

From aab3fcc274a67c3b41a8e67ec3dffbc445482da5 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Mon, 11 Mar 2019 11:38:49 +0100
Subject: [PATCH 367/373] instead of competing, let the other racing slave win

---
 src/paxos/multi_paxos_fsm.ml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/paxos/multi_paxos_fsm.ml b/src/paxos/multi_paxos_fsm.ml
index e1d581a3..8afa370a 100644
--- a/src/paxos/multi_paxos_fsm.ml
+++ b/src/paxos/multi_paxos_fsm.ml
@@ -199,6 +199,16 @@ let wait_for_promises (type s) constants state event =
                     then
                       Fsm.return (Slave_discovered_other_master (source, n'', i'))
                     else
+                      if i' = i && n'' > n
+                      then
+                        drop msg
+                          (Printf.sprintf "%s has n:%s (> my n:%s, and i'=i=%s). stand down"
+                             source
+                             (Sn.string_of n'')
+                             (Sn.string_of n)
+                             (Sn.string_of i)
+                          )
+                      else
                       let new_n = update_n constants (max n n'') in
                       Fsm.return (Election_suggest (new_n, counter + 1))
                   end

From 80b3ac6785cccdb1eb6003ac69e3113ca4e3185c Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Mon, 11 Mar 2019 13:43:16 +0100
Subject: [PATCH 368/373] latest pyrakoon

---
 pylabs/pyrakoon | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pylabs/pyrakoon b/pylabs/pyrakoon
index 22cd6160..e114535f 160000
--- a/pylabs/pyrakoon
+++ b/pylabs/pyrakoon
@@ -1 +1 @@
-Subproject commit 22cd6160d5b8bc028072d64ffe5b50ca253cb8b3
+Subproject commit e114535fcc6cc97680bfdd1499868a9d28272e22

From 9ca79862de7485f0cdd3f560879d1c843930b231 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Fri, 26 Apr 2019 13:08:20 +0200
Subject: [PATCH 369/373] adds AssertRange to python client

---
 .../test/server/quick/system_tests_basic.py   | 18 ++++++++++
 src/client/python/ArakoonProtocol.py          | 33 +++++++++++++++++++
 2 files changed, 51 insertions(+)

diff --git a/pylabs/test/server/quick/system_tests_basic.py b/pylabs/test/server/quick/system_tests_basic.py
index 3f109ca0..817f9725 100644
--- a/pylabs/test/server/quick/system_tests_basic.py
+++ b/pylabs/test/server/quick/system_tests_basic.py
@@ -400,6 +400,24 @@ def test_aSSert_exists_sequences():
     v = client.get('test_assert')
     assert_equals(v, 'changed', 'second_sequence: %s <> %s' % (v,'changed'))
 
+@C.with_custom_setup(C.setup_1_node_forced_master, C.basic_teardown)
+def test_assert_range():
+    client = C.get_client()
+    client.set("test_assert_range_sequence", "bla bla bla")
+
+    seq = client.makeSequence()
+    seq.addSet("ok","BAD")
+    seq.addAssertPrefixContainsExactly("test_assert_range", ["test_assert_range_sequence_XXXX"])
+    assert_raises(X.arakoon_client.ArakoonAssertionFailed, client.sequence, seq)
+
+    
+    seq2 = client.makeSequence()
+    seq2.addAssertPrefixContainsExactly("test_assert_range", ["test_assert_range_sequence"])
+    seq2.addSet("ok","OK")
+    client.sequence(seq2)
+    v = client.get("ok")
+    assert_equals(v,"OK")
+    
 @C.with_custom_setup( C.default_setup, C.basic_teardown )
 def test_prefix ():
     C.prefix_scenario(1000)
diff --git a/src/client/python/ArakoonProtocol.py b/src/client/python/ArakoonProtocol.py
index 57c0e5a1..3057c084 100644
--- a/src/client/python/ArakoonProtocol.py
+++ b/src/client/python/ArakoonProtocol.py
@@ -31,6 +31,7 @@
 import operator
 import cStringIO
 import types
+from abc import ABCMeta, abstractmethod
 
 FILTER = ''.join([(len(repr(chr(x)))==3) and chr(x) or '.' for x in range(256)])
 
@@ -533,6 +534,7 @@ def isDirty(self):
 
 class Update(object):
     pass
+
 class Set(Update):
     def __init__(self,key,value):
         self._key = key
@@ -576,6 +578,33 @@ def write(self, fob):
         fob.write(_packInt(15))
         fob.write(_packString(self._key))
 
+class RangeAssertion:
+    __metaclass__ = ABCMeta
+
+    def write(self, fob):
+        raise NotImplementedError()
+
+class ContainsExactly(RangeAssertion):
+    def __init__(self, keys):
+        self._keys = keys
+
+    def write(self, fob):
+        fob.write(_packInt(1))
+        fob.write(_packInt(len(self._keys)))
+        for s in self._keys:
+            fob.write(_packString(s))
+
+class AssertRange(Update):
+    def __init__(self, prefix, rangeAssertion):
+        self._prefix = prefix
+        self._rangeAssertion = rangeAssertion
+
+    def write(self, fob):
+      fob.write(_packInt(17))
+      fob.write(_packString(self._prefix))
+      self._rangeAssertion.write(fob)
+
+
 class Sequence(Update):
     def __init__(self):
         self._updates = []
@@ -601,6 +630,10 @@ def addAssert(self, key, vo):
     def addAssertExists(self, key):
         self._updates.append(AssertExists(key))
 
+    def addAssertPrefixContainsExactly(self, prefix, keys):
+        ar = AssertRange(prefix, ContainsExactly(keys))
+        self.addUpdate(ar)
+
     def write(self, fob):
         fob.write( _packInt(5))
         fob.write( _packInt(len(self._updates)))

From f03adeece92c115639927acfcd3a4f8382373545 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Thu, 28 Mar 2019 12:54:35 +0100
Subject: [PATCH 370/373] move to 4.07.1

---
 .travis.yml                                   |   2 +
 Makefile                                      | 165 +++++--------
 _tags                                         |  13 +-
 arakoon.opam                                  |  38 +++
 arakoon_client.opam                           |  36 +++
 docker/docker-entrypoint.sh                   |  44 ++--
 docker/opam.switch                            | 220 +++++++++---------
 docker/run.sh                                 |  17 +-
 docker/ubuntu-16.04/Dockerfile                |  94 +++++---
 dune-project                                  |   1 +
 examples/ocaml/dune                           |   4 +
 jenkins/common.sh                             |  18 +-
 myocamlbuild.ml                               |   1 -
 pylabs/pyrakoon                               |   2 +-
 pylabs/test/server/quick/test_client_lib.py   |  10 +-
 src/all_test.ml                               |   1 -
 src/client/arakoon_client.mli                 |   3 +
 src/client/arakoon_client_config.ml           |   4 +-
 src/client/arakoon_remote_client_test.ml      |  71 +++---
 src/client/client_helper.ml                   |   3 +-
 src/client/client_protocol.ml                 |   2 +-
 src/client/remote_nodestream.ml               |   1 -
 src/client/remote_nodestream_test.ml          |   2 +-
 src/dune                                      | 143 ++++++++++++
 src/inifiles/arakoon_inifiles.ml              |  18 +-
 src/inifiles/arakoon_inifiles_test.ml         |   1 -
 src/inifiles/dune                             |  11 +
 src/lib/std.ml                                |  81 +------
 src/lib/test_utils.ml                         |  68 ------
 src/main/arakoon.ml                           |   6 +-
 src/{client => main}/bs_bench.ml              |   0
 src/main/dune                                 |  47 ++++
 src/{tools => main}/lwt_buffer_test.ml        |   0
 src/node/backend.ml                           |   4 +-
 src/node/catchup.ml                           |   1 +
 src/node/collapser.ml                         |   3 +-
 src/node/collapser_main.ml                    |   4 +-
 src/node/collapser_test.ml                    |   1 -
 src/node/dump_store.ml                        |   6 +-
 src/node/mem_store.ml                         |   1 +
 src/node/node_cfg.ml                          |   1 +
 src/node/node_main.ml                         |  10 +-
 src/node/store.ml                             |  10 +-
 src/node/sync_backend.ml                      |   4 +-
 src/nursery/client_cfg_test.ml                |   2 +-
 src/nursery/nursery_main.ml                   |   2 +-
 src/paxos/learner.ml                          |   8 +-
 src/paxos/multi_paxos.ml                      |   2 +-
 src/paxos/slave.ml                            |   6 +-
 .../{registry.ml => arakoon_registry.ml}      |   0
 .../{registry.mli => arakoon_registry.mli}    |   0
 src/plugins/plugin_update_max.ml              |   2 +-
 src/system/single.ml                          |   7 +-
 src/system/startup.ml                         |   2 +-
 src/tlog/compression.ml                       |   7 +-
 src/tlog/compression_test.ml                  |   2 +-
 src/tlog/mem_tlogcollection.ml                |   8 +-
 src/tlog/tlc2.ml                              |   2 +-
 src/tlog/tlc2_test.ml                         |  10 +-
 src/tlog/tlog_main.ml                         |   1 -
 src/tlog/tlog_map.ml                          |   3 +-
 src/tlog/tlog_map_test.ml                     |   6 +-
 src/tlog/tlogcollection_test.ml               |   1 -
 src/tlog/update.ml                            |   2 +-
 src/tools/arakoon_logger.ml                   |   1 +
 src/tools/fsm.ml                              |   2 +-
 src/tools/logger.ml                           |  43 ++--
 src/tools/lwt_extra.ml                        |   2 +-
 src/version/dune                              |  23 ++
 src/version/mk_version.ml                     |  89 +++++++
 travis.sh                                     |  39 ++--
 71 files changed, 833 insertions(+), 611 deletions(-)
 create mode 100644 arakoon.opam
 create mode 100644 arakoon_client.opam
 create mode 100644 dune-project
 create mode 100644 examples/ocaml/dune
 create mode 100644 src/dune
 create mode 100644 src/inifiles/dune
 delete mode 100644 src/lib/test_utils.ml
 rename src/{client => main}/bs_bench.ml (100%)
 create mode 100644 src/main/dune
 rename src/{tools => main}/lwt_buffer_test.ml (100%)
 rename src/plugins/{registry.ml => arakoon_registry.ml} (100%)
 rename src/plugins/{registry.mli => arakoon_registry.mli} (100%)
 create mode 100644 src/version/dune
 create mode 100644 src/version/mk_version.ml

diff --git a/.travis.yml b/.travis.yml
index cfdc5c14..5bfc4d4d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,4 +1,6 @@
+dist: xenial
 language: c
+
 sudo: required
 
 install: ./travis.sh install
diff --git a/Makefile b/Makefile
index 4e7d339d..19bf278f 100644
--- a/Makefile
+++ b/Makefile
@@ -1,130 +1,69 @@
 # This makefile wrapper makes DEBIAN packaging easier.
-PREFIX ?=/usr
 
-START = $(DESTDIR)$(PREFIX)
+# The debian installer defined DESTDIR. If set, we install where debian expects it,
+# othervice we install where opam expects it
+ifdef DESTDIR
+  LIBDIR=$(DESTDIR)/usr/lib/ocaml
+  PREFIX=$(DESTDIR)/usr
+else
+  PREFIX=$(shell opam config var prefix)
+  LIBDIR=$(shell opam config var lib)
+endif
 
-OCAML_LIBDIR ?= $(START)/lib/ocaml/
-#OCAML_LIBDIR ?= `ocamlfind printconf destdir`
-OCAML_FIND ?= ocamlfind
+all: arakoon.native bench.native lwt_buffer_test.native
 
-JOBS := $(shell getconf _NPROCESSORS_ONLN 2>/dev/null || grep -c ^processor /proc/cpuinfo 2>/dev/null || echo 1)
+build:
+	@dune build
+
+arakoon.native: build
+	@ln -sf _build/default/src/main/arakoon.exe arakoon.native
+
+bench.native:
+	@dune build src/main/bs_bench.exe
+	@ln -sf _build/default/src/main/bs_bench.exe bs_bench.native
 
-all: build
+lwt_buffer_test.native:
+	@dune build src/main/lwt_buffer_test.exe
+	@ln -sf _build/default/src/main/lwt_buffer_test.exe lwt_buffer_test.native
 
 clean:
-	rm -rf ./debian/tmp/*
-	rm -rf ./debian/libarakoon-ocaml-dev/*
-	ocamlbuild -clean
-	rm -f  ./arakoon.byte ./arakoon.native
+	dune clean
+	rm -f arakoon.native bs_bench.native lwt_buffer_test.native
 
-build:
-	ocamlbuild -j $(JOBS) arakoon.byte arakoon.native arakoon_client.cma arakoon_client.cmxa arakoon_client.a arakoon_client.cmxs plugin_helper.cmi
-
-bench:
-	ocamlbuild bs_bench.native
-
-arakoon.byte arakoon.native:
-	ocamlbuild -j $(JOBS) $@
-
-test:	arakoon.native
-	ocamlbuild -j $(JOBS) arakoon.native
-	./arakoon.native --run-all-tests
-
-install: install_client install_server man
-
-install_server:
-	mkdir -p $(START)/bin/
-	cp ./arakoon.native $(START)/bin/arakoon
-
-install_client:
-	mkdir -p $(OCAML_LIBDIR)
-	$(OCAML_FIND) install arakoon_client -destdir $(OCAML_LIBDIR) META \
-          _build/src/arakoon_client.cma \
-          _build/src/arakoon_client.cmxa \
-          _build/src/arakoon_client.cmxs \
-          _build/src/client/stamp.cmi \
-	  _build/src/client/stamp.cmx \
-          _build/src/client/arakoon_exc.mli \
-          _build/src/client/arakoon_exc.cmi \
-	  _build/src/client/arakoon_exc.cmx \
-          _build/src/client/arakoon_client_config.cmi \
-	  _build/src/client/arakoon_client_config.cmx \
-          _build/src/client/arakoon_client.mli \
-          _build/src/client/arakoon_client.cmi \
-          _build/src/client/arakoon_remote_client.mli \
-          _build/src/client/arakoon_remote_client.cmi \
-	  _build/src/client/arakoon_remote_client.cmx \
-          _build/src/client/protocol_common.cmi \
-	  _build/src/client/protocol_common.cmx \
-          _build/src/client/client_helper.cmi \
-	  _build/src/client/client_helper.cmx \
-          _build/src/inifiles/arakoon_inifiles.mli \
-          _build/src/inifiles/arakoon_inifiles.cmi \
-	  _build/src/inifiles/arakoon_inifiles.cmx \
-          _build/src/lib/std.cmi \
-          _build/src/lib/std.cmx \
-          _build/src/plugins/registry.mli \
-          _build/src/plugins/registry.cmi \
-	  _build/src/plugins/registry.cmx \
-          _build/src/plugins/plugin_helper.mli \
-          _build/src/plugins/plugin_helper.cmi \
-	  _build/src/plugins/plugin_helper.cmx \
-          _build/src/node/key.mli \
-          _build/src/node/key.cmi \
-	  _build/src/node/key.cmx \
-          _build/src/node/simple_store.cmi \
-	  _build/src/node/simple_store.cmx \
-          _build/src/nursery/routing.cmi \
-          _build/src/tlog/arakoon_interval.cmi \
-          _build/src/tlog/arakoon_interval.cmx \
-          _build/src/tlog/update.mli \
-          _build/src/tlog/update.cmi \
-	  _build/src/tlog/update.cmx \
-          _build/src/tools/ini.cmi \
-          _build/src/tools/ini.cmx \
-          _build/src/tools/arakoon_log_sink.cmi \
-	  _build/src/tools/arakoon_log_sink.cmx \
-          _build/src/tools/crash_logger.cmi \
-	  _build/src/tools/crash_logger.cmx \
-          _build/src/tools/arakoon_redis.cmi \
-          _build/src/tools/arakoon_redis.cmx \
-          _build/src/tools/arakoon_logger.cmi \
-	  _build/src/tools/arakoon_logger.cmx \
-          _build/src/tools/llio.mli \
-          _build/src/tools/llio.cmi \
-	  _build/src/tools/llio.cmx \
-          _build/src/tools/network.cmi \
-	  _build/src/tools/network.cmx \
-          _build/src/tools/lwt_buffer.cmi \
-	  _build/src/tools/lwt_buffer.cmx \
-          _build/src/tools/lwt_extra.cmi \
-	  _build/src/tools/lwt_extra.cmx \
-          _build/src/tools/typed_ssl.mli \
-          _build/src/tools/typed_ssl.cmi \
-	  _build/src/tools/typed_ssl.cmx \
-          _build/src/tools/unix_fd.cmi \
-          _build/src/tools/unix_fd.cmx \
-          _build/src/tools/arakoon_etcd.cmi \
-	  _build/src/tools/arakoon_etcd.cmx \
-          _build/src/tools/arakoon_config_url.cmi \
-	  _build/src/tools/arakoon_config_url.cmx \
-          _build/src/tools/tcp_keepalive_sockopt_stubs.o \
-          _build/src/tools/tcp_keepalive.cmi \
-	  _build/src/tools/tcp_keepalive.cmx \
-          _build/src/arakoon_client.a
-
-uninstall_client:
-	$(OCAML_FIND) remove arakoon_client -destdir $(OCAML_LIBDIR)
-
-coverage:
-	ocamlbuild -tag use_bisect arakoon.d.byte
+test:	build
+	dune exec -- arakoon test
+
+install: build
+	@mkdir -p $(LIBDIR) $(PREFIX)
+ifdef DESTDIR
+	mkdir -p $(DESTDIR)/usr/lib
+	ldd arakoon.native | \
+          grep librocksdb | \
+          awk '/=> /{print $$3}' | \
+          xargs cp --target-directory $(DESTDIR)/usr/lib
+endif
+	dune install --libdir=$(LIBDIR) --prefix=$(PREFIX)
+
+uninstall: build
+	dune uninstall --libdir=$(LIBDIR) --prefix=$(PREFIX)
+
+uninstall_client: build
+	dune uninstall -p arakoon_client --libdir=$(LIBDIR) --prefix=$(PREFIX)
 
 man:
 	ln -s ./arakoon.native arakoon
 	help2man --name='Arakoon, a consistent key value store' ./arakoon > debian/arakoon.man
 	rm arakoon
 
-.PHONY: install test build install_client bench arakoon.byte arakoon.native
+examples:
+	dune build examples/ocaml/plugin_demo.exe examples/ocaml/demo.exe
+
+doc:
+	dune build @doc
+
+
+.PHONY: install uninstall test build bench examples arakoon.native doc man
+
 
 
 indent-tabs-to-spaces:
diff --git a/_tags b/_tags
index 0faef7da..2e8665f9 100644
--- a/_tags
+++ b/_tags
@@ -2,7 +2,6 @@ true: debug
 true: annot
 true: thread
 true: package(lwt)
-true: package(lwt.preemptive)
 true: package(lwt.unix)
 true: package(lwt_ssl)
 true: package(oUnit)
@@ -10,14 +9,14 @@ true: package(bz2)
 true: package(str)
 true: package(camltc)
 true: package(dynlink)
-true: package(quickcheck)
 true: pp(camlp4of)
-true: package(snappy)
-true: package(ocplib-endian)
+true: package(lwt_log)
 true: package(uri)
-true: package(redis)
 true: package(redis-lwt)
 true: package(core)
+true: package(snappy)
+true: package(ocplib-endian)
+
 "src":include
 "src/lib":include
 "src/tools":include
@@ -31,13 +30,13 @@ true: package(core)
 "src/inifiles": include
 "src/nursery": include
 "src/plugins": include
-"examples/ocaml": include
+#"examples/ocaml": include
 "src/main" : include
 <**/arakoon.*{byte,native}>: is_main, linkall
 <**/nursery.*{byte,native}>: is_main
 <**/profile.*{byte,native}>: is_main, package(ocamlviz)
 <**/bs_bench.*{byte,native}>: is_main
-<examples/ocaml/*.{byte,native}>: is_main
+#<examples/ocaml/*.{byte,native}>: is_main
 "logger_macro.cmo" : use_camlp4, camlp4orf
 <src/**/*.ml> : camlp4of, use_log_macro, maybe_use_bisect, unsafe_string
 "rpmbuild": -traverse
diff --git a/arakoon.opam b/arakoon.opam
new file mode 100644
index 00000000..8e0c16b7
--- /dev/null
+++ b/arakoon.opam
@@ -0,0 +1,38 @@
+opam-version: "1.2"
+name: "arakoon"
+maintainer: "Romain Slootmaekers <romain.slootmaekers@openvstorage.com>"
+authors: ["Romain Slootmaekers <romain.slootmaekers@openvstorage.com>"
+          "Jan Doms <jan.doms@openvstorage.com>"]
+homepage: "https://github.com/openvstorage/arakoon"
+bug-reports: "https://github.com/openvstorage/arakoon/issues"
+license: "Apache 2.0"
+dev-repo: "https://github.com/openvstorage/arakoon.git"
+build: [
+  ["dune" "build" "-p" name "-j" jobs]
+]
+
+depends: [
+  "ocamlfind" {build}
+  "bisect" {test & >= "1.3"}
+  "camltc" {>= "0.9.4"}
+  "camlbz2" {>= "0.6.0"}
+  "conf-libev" {>= "4-11"}
+  "lwt" {>= "3.0.0"}
+  "lwt_log"
+  "lwt_ssl" {>= "1.0.1"}
+  "ocplib-endian" {>= "1.0"}
+  "ssl" {>= "0.5.3"}
+  "snappy" {>= "0.1.0"}
+  "core" {>="0.9.0"}
+  "dune" { build }
+  "redis-lwt"
+  "uri" {>= "1.9.2"}
+  "arakoon_client"
+]
+available: [ocaml-version >= "4.03.0" & opam-version >= "1.2" & os != "darwin" ]
+
+
+depexts:[
+  [[ "ubuntu"] ["help2man"] ]
+  [[ "debian"] ["help2man"] ]
+]
diff --git a/arakoon_client.opam b/arakoon_client.opam
new file mode 100644
index 00000000..2054764e
--- /dev/null
+++ b/arakoon_client.opam
@@ -0,0 +1,36 @@
+opam-version: "1.2"
+name: "arakoon"
+maintainer: "Romain Slootmaekers <romain.slootmaekers@openvstorage.com>"
+authors: ["Romain Slootmaekers <romain.slootmaekers@openvstorage.com>"
+          "Jan Doms <jan.doms@openvstorage.com>"]
+homepage: "https://github.com/openvstorage/arakoon"
+bug-reports: "https://github.com/openvstorage/arakoon/issues"
+license: "Apache 2.0"
+dev-repo: "https://github.com/openvstorage/arakoon.git"
+build: [
+  ["dune" "build" "-p" name "-j" jobs]
+]
+depends: [
+  "ocamlfind" {build}
+  "bisect" {test & >= "1.3"}
+  "camltc" {>= "0.9.4"}
+  "camlbz2" {>= "0.6.0"}
+  "conf-libev" {>= "4-11"}
+  "lwt" {>= "3.0.0"}
+  "lwt_log"
+  "lwt_ssl" {>= "1.0.1"}
+  "ocplib-endian" {>= "1.0"}
+  "ssl" {>= "0.5.3"}
+  "snappy" {>= "0.1.0"}
+  "core" {>="0.9.0"}
+  "dune" { build }
+  "redis-lwt"
+  "uri" {>= "1.9.2"}
+]
+available: [ocaml-version >= "4.03.0" & opam-version >= "1.2" & os != "darwin" ]
+
+
+depexts:[
+  [[ "ubuntu"] ["help2man"] ]
+  [[ "debian"] ["help2man"] ]
+]
diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh
index f221357d..945f74da 100755
--- a/docker/docker-entrypoint.sh
+++ b/docker/docker-entrypoint.sh
@@ -1,19 +1,37 @@
-#!/bin/bash -xue
-
+#!/bin/bash -l
 # this script is executed at each startup of the container
 
-# hack to make sure we have access to files in the jenkins home directory
-# the UID of jenkins in the container should match our UID on the host
-if [ ${UID} -ne 1500 ]
-then
-    cat /etc/passwd
-    sed -i "s/x:1500:/x:${UID}:/" /etc/passwd
+trap 'rc=$?; echo "ERR at line ${LINENO} (rc: $rc)"; exit $rc' ERR
+set -e
 
-    chown ${UID} /home/jenkins
-    chown ${UID} /home/jenkins/OPAM
-    chown ${UID} /home/jenkins/.bash* || true
+if [ $HOST_UID -ne $UID ]; then
+    echo "UID mismatch; please build and run container under same UID" 2>&1
+    exit 1
 fi
 
+cd $HOME/arakoon
+
+export TEST_HOME='/home/jenkins/arakoon/TESTS'
+export OPAMROOT=/home/jenkins/OPAM
+
+eval $(opam config env --root=${OPAMROOT})
+
+echo $PATH
+
 # finally execute the command the user requested
-command="cd arakoon && ./docker/suites.sh $@"
-sudo -E -i -u jenkins bash -l -c "$command"
+cmd=${1-bash}
+case "$cmd" in
+  bash|sh)
+	shift || true
+	exec $cmd "$@"
+	;;
+  clean)
+	make clean
+	;;
+  build)
+	make build
+	;;
+  *)    # everything else is considered as a jenkins suite
+	docker/suites.sh "$@"
+	;;
+esac
diff --git a/docker/opam.switch b/docker/opam.switch
index 7bf004b7..0f37252e 100644
--- a/docker/opam.switch
+++ b/docker/opam.switch
@@ -1,115 +1,105 @@
-base	v0.10.0	installed
-base-bigarray	base	root
-base-bytes	base	installed
-base-threads	base	root
-base-unix	base	root
-base64	2.2.0	installed
-bin_prot	v0.10.0	installed
-biniou	1.2.0	installed
-bisect	1.3+ovs1	installed
-bisect_ppx	1.3.2	installed
-camlbz2	0.6.0	installed
-camlp4	4.06+1	installed
-camltc	0.9.8+ovs1	root
-cmdliner	1.0.2	installed
-conf-gmp	1	installed
-conf-gmp-powm-sec	1	installed
-conf-libev	4-11	installed
-conf-m4	1	installed
-conf-openssl	1	installed
-conf-perl	1	installed
-conf-pkg-config	1.0	installed
-conf-which	1	installed
-conf-zlib	1	installed
-configurator	v0.10.0	installed
-core	v0.10.0	installed
-core_kernel	v0.10.0	installed
-cppo	1.6.0	installed
-cppo_ocamlbuild	1.6.0	installed
-cryptokit	1.12	installed
-ctypes	0.13.1	installed
-ctypes-foreign	0.4.0	installed
-depext	1.0.5	installed
-dune	1.2.1	installed
-easy-format	1.3.0	installed
-ezxmlm	1.0.2	installed
-fieldslib	v0.10.0	installed
-integers	0.2.2	installed
-jane-street-headers	v0.10.0	installed
-jbuilder	transition	installed
-logs	0.6.2	installed
-lwt	3.2.0	root
-lwt_ssl	1.1.2	root
-mirage-no-xen	1	installed
-num	1.1	installed
-oasis	0.4.10	installed
-ocaml-compiler-libs	v0.10.0	installed
-ocaml-migrate-parsetree	1.0.11	root
-ocamlbuild	0.12.0	installed
-ocamlfind	1.7.3-1	installed
-ocamlify	0.0.1	installed
-ocamlmod	0.0.9	installed
-ocplib-endian	1.0	installed
-octavius	1.2.0	installed
-omake	0.10.3	installed
-optcomp	1.6+ovs1	installed
-ounit	2.0.6	installed
-piqi	0.7.5	installed
-piqilib	0.6.13	installed
-ppx_assert	v0.10.0	installed
-ppx_ast	v0.10.0	installed
-ppx_base	v0.10.0	installed
-ppx_bench	v0.10.0	installed
-ppx_bin_prot	v0.10.0	installed
-ppx_compare	v0.10.0	installed
-ppx_core	v0.10.0	installed
-ppx_custom_printf	v0.10.0	installed
-ppx_derivers	1.0	installed
-ppx_deriving	4.2.1	installed
-ppx_deriving_yojson	3.1	installed
-ppx_driver	v0.10.1	installed
-ppx_enumerate	v0.10.0	installed
-ppx_expect	v0.10.0	installed
-ppx_fail	v0.10.0	installed
-ppx_fields_conv	v0.10.0	installed
-ppx_hash	v0.10.0	installed
-ppx_here	v0.10.0	installed
-ppx_inline_test	v0.10.0	installed
-ppx_jane	v0.10.0	installed
-ppx_js_style	v0.10.0	installed
-ppx_let	v0.10.0	installed
-ppx_metaquot	v0.10.0	installed
-ppx_optcomp	v0.10.0	installed
-ppx_optional	v0.10.0	installed
-ppx_pipebang	v0.10.0	installed
-ppx_sexp_conv	v0.10.0	installed
-ppx_sexp_message	v0.10.0	installed
-ppx_sexp_value	v0.10.0	installed
-ppx_tools	5.1+4.06.0	installed
-ppx_tools_versioned	5.0.1	installed
-ppx_traverse	v0.10.0	installed
-ppx_traverse_builtins	v0.10.0	installed
-ppx_type_conv	v0.10.0	installed
-ppx_typerep_conv	v0.10.0	installed
-ppx_variants_conv	v0.10.0	installed
-quickcheck	1.0.2	installed
-re	1.7.1	installed
-redis	0.3.5+ovs1	installed
-redis-lwt	0.3.5+ovs1	root
-result	1.2	installed
-sexplib	v0.10.0	installed
-snappy	0.1.0+ovs1	root
-spawn	v0.10.1	installed
-ssl	0.5.5	installed
-stdio	v0.10.0	installed
-stringext	1.5.0	installed
-tiny_json	1.1.5	root
-topkg	0.9.1	installed
-typerep	v0.10.0	installed
-ulex	1.2	installed
-uri	1.9.5	installed
-uuidm	0.9.6	installed
-variantslib	v0.10.0	installed
-xmlm	1.3.0	installed
-yojson	1.4.0	installed
-zarith	1.7	installed
+opam-version: "2.0"
+compiler: [
+  "base-bigarray.base"
+  "base-threads.base"
+  "base-unix.base"
+  "ocaml.4.07.1"
+  "ocaml-config.1"
+  "ocaml-variants.4.07.1+default-unsafe-string"
+]
+roots: [
+  "lwt.4.2.1"
+  "lwt_log.1.1.0"
+  "lwt_ppx.1.2.1"
+  "lwt_ssl.1.1.2"
+  "ocaml-variants.4.07.1+default-unsafe-string"
+]
+installed: [
+  "base.v0.11.1"
+  "base-bigarray.base"
+  "base-bytes.base"
+  "base-threads.base"
+  "base-unix.base"
+  "bin_prot.v0.11.0"
+  "camlbz2.0.6.0"
+  "camltc.0.9.8"
+  "cmdliner.1.0.3"
+  "conf-m4.1"
+  "conf-openssl.1"
+  "conf-pkg-config.1.1"
+  "conf-snappy.1"
+  "configurator.v0.11.0"
+  "core.v0.11.3"
+  "core_kernel.v0.11.1"
+  "cppo.1.6.5"
+  "ctypes.0.14.0"
+  "ctypes-foreign.0.4.0"
+  "dune.1.7.2"
+  "fieldslib.v0.11.0"
+  "integers.0.2.2"
+  "jane-street-headers.v0.11.0"
+  "jbuilder.transition"
+  "logs.0.6.2"
+  "lwt.4.2.1"
+  "lwt_log.1.1.0"
+  "lwt_ppx.1.2.1"
+  "lwt_ssl.1.1.2"
+  "mmap.1.1.0"
+  "num.1.1"
+  "ocaml.4.07.1"
+  "ocaml-compiler-libs.v0.11.0"
+  "ocaml-config.1"
+  "ocaml-migrate-parsetree.1.2.0"
+  "ocaml-variants.4.07.1+default-unsafe-string"
+  "ocamlbuild.0.14.0"
+  "ocamlfind.1.8.0"
+  "ocplib-endian.1.0"
+  "octavius.1.2.0"
+  "ounit.2.0.8"
+  "parsexp.v0.11.0"
+  "ppx_assert.v0.11.0"
+  "ppx_base.v0.11.0"
+  "ppx_bench.v0.11.0"
+  "ppx_bin_prot.v0.11.1"
+  "ppx_compare.v0.11.1"
+  "ppx_custom_printf.v0.11.0"
+  "ppx_derivers.1.0"
+  "ppx_enumerate.v0.11.1"
+  "ppx_expect.v0.11.1"
+  "ppx_fail.v0.11.0"
+  "ppx_fields_conv.v0.11.0"
+  "ppx_hash.v0.11.1"
+  "ppx_here.v0.11.0"
+  "ppx_inline_test.v0.11.0"
+  "ppx_jane.v0.11.0"
+  "ppx_js_style.v0.11.0"
+  "ppx_let.v0.11.0"
+  "ppx_optcomp.v0.11.0"
+  "ppx_optional.v0.11.0"
+  "ppx_pipebang.v0.11.0"
+  "ppx_sexp_conv.v0.11.2"
+  "ppx_sexp_message.v0.11.0"
+  "ppx_sexp_value.v0.11.0"
+  "ppx_tools_versioned.5.2.1"
+  "ppx_typerep_conv.v0.11.1"
+  "ppx_variants_conv.v0.11.1"
+  "ppxlib.0.5.0"
+  "re.1.8.0"
+  "redis.0.3.6"
+  "redis-lwt.0.3.6"
+  "result.1.3"
+  "seq.base"
+  "sexplib.v0.11.0"
+  "sexplib0.v0.11.0"
+  "snappy.0.1.1"
+  "spawn.v0.12.0"
+  "splittable_random.v0.11.0"
+  "ssl.0.5.7"
+  "stdio.v0.11.0"
+  "stringext.1.5.0"
+  "topkg.1.0.0"
+  "typerep.v0.11.0"
+  "uri.2.2.0"
+  "uuidm.0.9.6"
+  "variantslib.v0.11.0"
+]
diff --git a/docker/run.sh b/docker/run.sh
index 6a57417e..0c1b750b 100755
--- a/docker/run.sh
+++ b/docker/run.sh
@@ -1,21 +1,28 @@
-#!/bin/bash -xue
+#!/bin/bash
+set -ue
+[[ ${JENKINS_PARAM-} =~ shell_trace ]] && set -x
 
 IMAGE=$1
 shift
 
 docker build --rm=true \
        --tag=arakoon_$IMAGE \
-       -f ./docker/$IMAGE/Dockerfile ./docker/ \
+       --build-arg HOST_UID=$UID \
+       -f docker/$IMAGE/Dockerfile ./docker/ \
 
 
-if [ -t 1 ];
+       if [ -t 1 ];
 then TTY="-t";
 else TTY="";
 fi
 
-docker run -i $TTY --privileged=true -e UID=${UID} \
+docker run -i $TTY --privileged=true \
+       --env HOST_UID=$UID \
        --env ARAKOON_PYTHON_CLIENT \
+       --env ARAKOON_DB_TYPE \
+       --env JENKINS_PARAM="${JENKINS_PARAM-}" \
        --env TRAVIS \
+       --ulimit core=-1 \
        -v ${PWD}:/home/jenkins/arakoon \
        arakoon_$IMAGE \
-       bash $@
+       "$@"
diff --git a/docker/ubuntu-16.04/Dockerfile b/docker/ubuntu-16.04/Dockerfile
index 002fd27c..704ad8fe 100644
--- a/docker/ubuntu-16.04/Dockerfile
+++ b/docker/ubuntu-16.04/Dockerfile
@@ -1,7 +1,15 @@
 FROM ubuntu:16.04
 
-RUN useradd jenkins -u 1500 -g root
+ENV release_name=xenial
+ENV etcd_version=2.3.5
+ENV redis_version=3.0.7
+ENV ocaml_version=4.07.1+default-unsafe-string
+
 
+# force our apt to use look at mirrors (by prepending a mirrors line)
+# RUN sed 's@archive.ubuntu.com@ubuntu.mirror.atratoip.net@' -i /etc/apt/sources.list
+RUN sed -i "1s;^;deb mirror://mirrors.ubuntu.com/mirrors.txt ${release_name}-updates main restricted universe multiverse\n;" /etc/apt/sources.list
+RUN sed -i "1s;^;deb mirror://mirrors.ubuntu.com/mirrors.txt ${release_name}         main restricted universe multiverse\n;" /etc/apt/sources.list
 RUN apt-get -y update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
         build-essential m4 apt-utils \
         libffi-dev libssl-dev \
@@ -29,56 +37,72 @@ RUN apt-get -y update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
         iptables \
         net-tools \
         ncurses-dev \
-        tzdata
+        tzdata \
+        librdmacm-dev \
+        protobuf-compiler \
+        socat
 
-RUN pip install fabric junit-xml nose simplejson python-etcd
-RUN mkdir /home/tests
-RUN chmod 777 /home/tests
+ARG HOST_UID
+RUN useradd jenkins -u $HOST_UID -g root --create-home
+RUN echo "jenkins ALL=NOPASSWD: ALL" >/etc/sudoers.d/jenkins
+
+RUN pip install --upgrade pip
+RUN pip --disable-pip-version-check install fabric junit-xml nose simplejson python-etcd
 
 # Install etcd:
-ENV etcd_version=2.3.5
-RUN curl \
-      -L https://github.com/coreos/etcd/releases/download/v${etcd_version}/etcd-v${etcd_version}-linux-amd64.tar.gz \
-      -o etcd-v${etcd_version}-linux-amd64.tar.gz \
-    && tar xzvf etcd-v${etcd_version}-linux-amd64.tar.gz \
-    && cp ./etcd-v${etcd_version}-linux-amd64/etcd /usr/bin \
-    && cp ./etcd-v${etcd_version}-linux-amd64/etcdctl /usr/bin \
+
+RUN wget -q \
+    https://github.com/coreos/etcd/releases/download/v${etcd_version}/etcd-v${etcd_version}-linux-amd64.tar.gz -O - \
+    | tar zxf - \
+    && cp etcd-v${etcd_version}-linux-amd64/etcd /usr/bin \
+    && cp etcd-v${etcd_version}-linux-amd64/etcdctl /usr/bin \
     && rm -rf etcd-*
 
 # Install redis:
-RUN wget http://download.redis.io/releases/redis-3.0.7.tar.gz
-RUN tar xzvf redis-3.0.7.tar.gz
-RUN cd redis-3.0.7 && make
-RUN cp ./redis-3.0.7/src/redis-server /usr/bin
-RUN cp ./redis-3.0.7/src/redis-cli /usr/bin
 
+RUN wget -q \
+    http://download.redis.io/releases/redis-${redis_version}.tar.gz -O - \
+    | tar zxf - \
+    && make -j$(nproc 2>/dev/null || echo 1) -C redis-${redis_version} \
+    && cp redis-${redis_version}/src/redis-server /usr/bin \
+    && cp redis-${redis_version}/src/redis-cli /usr/bin \
+    && rm -rf redis-${redis_version}
+
+RUN wget https://github.com/ocaml/opam/releases/download/2.0.3/opam-2.0.3-x86_64-linux \
+    && mv opam-2.0.3-x86_64-linux /usr/bin/opam \
+    && chmod a+x /usr/bin/opam
+
+USER jenkins
 
-# most recent one (28/11/2017) is not compatible with our setup
-RUN wget https://raw.githubusercontent.com/ocaml/opam/afa822ec24c0f7afa278179cc59796a384129ce1/shell/opam_installer.sh
 
-env ocaml_version=4.06.0+default-unsafe-string
-RUN sh ./opam_installer.sh /usr/local/bin ${ocaml_version}
+ENV OPAMROOT=/home/jenkins/OPAM
 
-ENV opam_root=/home/jenkins/OPAM
-ENV opam_env="opam config env --root=${opam_root}"
-RUN opam init --root=${opam_root} --comp ${ocaml_version}
+# The special variable NO_PROXY contains host and port for accessing
+# the hosts SSH_AUTH_SOCK
+ENV SSH_AUTH_SOCK /tmp/auth.sock
+RUN mkdir /home/jenkins/.ssh
+RUN echo "\nHost *\n    StrictHostKeyChecking=no" >> /home/jenkins/.ssh/config
 
+env opam_env="opam config env --root=${OPAMROOT}"
+
+RUN opam init --root ${OPAMROOT} --compiler=${ocaml_version} --disable-sandboxing
 ADD opam.switch opam.switch
 ADD opam-repository ovs-opam-repository
-RUN chmod ugoa+rxw -R ovs-opam-repository
 
-RUN eval `${opam_env}` && \
+RUN socat UNIX-LISTEN:${SSH_AUTH_SOCK},unlink-early,mode=777,fork TCP:${NO_PROXY} & \
+    eval `${opam_env}` && \
     opam repo add ovs ovs-opam-repository && \
-    opam update -v && \
-    (eval `${opam_env}` && opam switch import opam.switch -y --strict) 
+    opam update -v
 
-RUN eval `${opam_env}` && opam list && \
-    opam switch export opam.switch.out && \
-    diff opam.switch opam.switch.out
+RUN socat UNIX-LISTEN:${SSH_AUTH_SOCK},unlink-early,mode=777,fork TCP:${NO_PROXY} & \
+    eval `${opam_env}` && export OPAMROOT=${OPAMROOT} && \
+    opam switch import opam.switch -y --strict
 
-RUN chmod ugoa+rxw -R ${opam_root}
-RUN su - -c "echo 'eval `${opam_env}`' >> /home/jenkins/.profile"
-RUN echo "jenkins ALL=NOPASSWD: ALL" >/etc/sudoers.d/jenkins
+RUN eval ${opam_env} && export OPAMROOT=${OPAMROOT} && \
+    opam list && \
+    opam switch export /home/jenkins/opam.switch.out && ls && \
+    cat /home/jenkins/opam.switch.out
 
+RUN opam list && bash -c 'diff opam.switch <(opam switch export -)'
 
-ENTRYPOINT ["/bin/bash", "-c", "set -e && /home/jenkins/arakoon/docker/docker-entrypoint.sh $@"]
+ENTRYPOINT ["/home/jenkins/arakoon/docker/docker-entrypoint.sh"]
diff --git a/dune-project b/dune-project
new file mode 100644
index 00000000..7655de07
--- /dev/null
+++ b/dune-project
@@ -0,0 +1 @@
+(lang dune 1.1)
diff --git a/examples/ocaml/dune b/examples/ocaml/dune
new file mode 100644
index 00000000..e6959a81
--- /dev/null
+++ b/examples/ocaml/dune
@@ -0,0 +1,4 @@
+(executables
+ (names        demo plugin_demo)
+ (libraries    arakoon_client lwt)
+)
diff --git a/jenkins/common.sh b/jenkins/common.sh
index dd9ed98c..e4f952f6 100755
--- a/jenkins/common.sh
+++ b/jenkins/common.sh
@@ -1,12 +1,14 @@
-#!/bin/bash -xue
-
-env | sort
+#!/bin/bash
+set -ue
+[[ ${JENKINS_PARAM-} =~ shell_trace ]] && set -x
+[[ ${JENKINS_PARAM-} =~ env_trace ]] && env | sort
 
 git submodule init
 git submodule update
 
-echo $PWD
-echo TEST_HOME=${TEST_HOME}
-export PATH=${PATH}:${TEST_HOME}/apps/arakoon/bin
-
-make clean build
+if [[ ! ${JENKINS_PARAM-} =~ no_make ]]; then
+    if [[ ! ${JENKINS_PARAM-} =~ no_clean ]]; then
+	make clean
+    fi
+    make arakoon.native
+fi
diff --git a/myocamlbuild.ml b/myocamlbuild.ml
index 36c3bdf7..d54b44d9 100644
--- a/myocamlbuild.ml
+++ b/myocamlbuild.ml
@@ -5,7 +5,6 @@ let dependencies = [ "lwt";
                      "snappy";
                      "ssl";
                      "bz2";
-                     "quickcheck";
                      "ocplib-endian";
                      "redis";
                      "uri";
diff --git a/pylabs/pyrakoon b/pylabs/pyrakoon
index e114535f..387a46ed 160000
--- a/pylabs/pyrakoon
+++ b/pylabs/pyrakoon
@@ -1 +1 @@
-Subproject commit e114535fcc6cc97680bfdd1499868a9d28272e22
+Subproject commit 387a46ed02dd54d1a5b400a1da0bbe85c468e097
diff --git a/pylabs/test/server/quick/test_client_lib.py b/pylabs/test/server/quick/test_client_lib.py
index 6cc7396f..98c8db42 100644
--- a/pylabs/test/server/quick/test_client_lib.py
+++ b/pylabs/test/server/quick/test_client_lib.py
@@ -28,11 +28,13 @@ def test_client_lib():
     env['OCAML_LIBDIR'] = OCAML_LIBDIR
     cmds = [
         (['make', 'uninstall_client'], None),
-        (['make', 'install_client'], None),
+        (['make', 'install'], None),
         (['mkdir', '-p',  my_temp], None),
-        (['cp', './examples/ocaml/demo.ml',
-          './examples/ocaml/_tags', my_temp], None),
-        (['ocamlbuild', '-use-ocamlfind', 'demo.native'], my_temp),
+        (['cp', './examples/ocaml/demo.ml', my_temp], None),
+        (['ocamlbuild', '-use-ocamlfind', '-package','lwt' ,
+          '-package','arakoon_client',
+          '-tags', 'annot,debug,thread',
+          'demo.native'], my_temp),
         (['make', 'uninstall_client'], None),
     ]
     for cmd, cwd in cmds:
diff --git a/src/all_test.ml b/src/all_test.ml
index 47d5f192..91027106 100644
--- a/src/all_test.ml
+++ b/src/all_test.ml
@@ -89,7 +89,6 @@ let suite = "universe" >::: [
     store;
     Client_cfg_test.suite;
     Node_cfg_test.suite;
-    Std.List.Test.suite;
     Arakoon_log_sink_test.suite;
     Arakoon_etcd_test.suite;
     Arakoon_client_config_test.suite;
diff --git a/src/client/arakoon_client.mli b/src/client/arakoon_client.mli
index fd986de6..5cc90f4e 100644
--- a/src/client/arakoon_client.mli
+++ b/src/client/arakoon_client.mli
@@ -88,15 +88,18 @@ class type client = object
   method nop : unit -> unit Lwt.t
   (** [nop ()] is a paxos no-operation.
    *)
+
   method confirm: key -> value -> unit Lwt.t
   (** [confirm key value] does nothing if this value was already
       associated to the key, otherwise, it behaves as [set key value]
   *)
+
   method aSSert: ?consistency:consistency -> key -> value option -> unit Lwt.t
   (**
      [aSSert key vo] throws Arakoon_exc.Exception (E_ASSERTION_FAILED,_) if
      the value associated with the key is not what was expected.
   *)
+
   method aSSert_exists: ?consistency:consistency -> key -> unit Lwt.t
 
   method delete: key -> unit Lwt.t
diff --git a/src/client/arakoon_client_config.ml b/src/client/arakoon_client_config.ml
index 0c6f8351..690e0a51 100644
--- a/src/client/arakoon_client_config.ml
+++ b/src/client/arakoon_client_config.ml
@@ -70,7 +70,7 @@ type t = {
   tcp_keepalive : tcp_keepalive_cfg;
 }
 
-let show { cluster_id; node_cfgs; ssl_cfg; } =
+let show { cluster_id; node_cfgs; ssl_cfg; _ } =
   Printf.sprintf
     "(cluster_id=%s, node_cfgs=%s, ssl_cfg=%s)"
     cluster_id
@@ -89,7 +89,7 @@ let get_node_cfg ({ node_cfgs; _ } as ccfg) node_name =
       node_cfgs
     |> List.hd
     |> snd
-  with exn ->
+  with _exn ->
     failwith (node_name ^ " is not known in config " ^ (show ccfg))
 
 let tcp_keepalive_from_ini inifile =
diff --git a/src/client/arakoon_remote_client_test.ml b/src/client/arakoon_remote_client_test.ml
index 2c65c265..4719c5fd 100644
--- a/src/client/arakoon_remote_client_test.ml
+++ b/src/client/arakoon_remote_client_test.ml
@@ -18,6 +18,7 @@ open OUnit
 open Lwt
 open Log_extra
 open Node_cfg
+open Arakoon_registry
 
 let section = Logger.Section.main
 
@@ -183,7 +184,7 @@ let _test_user_function (client:Arakoon_client.client) =
        String.iter (fun c -> r := (String.make 1 c ^ !r)) s;
        Some !r
   in
-  Registry.Registry.register "reverse" uf;
+  Registry.register "reverse" uf;
 
   client # user_function "reverse" (Some "echo") >>= fun ro ->
   Logger.debug_f_ "we got %s" (string_option2s ro) >>= fun () ->
@@ -195,8 +196,8 @@ let _test_user_function (client:Arakoon_client.client) =
     "non existing user function should throw exception"
     "ok!" >>= fun () ->
 
-  Registry.Registry.register "notfound" (fun _ _ -> raise Not_found);
-  Registry.Registry.register "failure" (fun _ _ -> raise (Failure ""));
+  Registry.register "notfound" (fun _ _ -> raise Not_found);
+  Registry.register "failure" (fun _ _ -> raise (Failure ""));
 
   should_fail
     (fun () -> Lwt.map ignore (client # user_function "notfound" None))
@@ -215,39 +216,39 @@ let test_user_function () =
   __client_server_wrapper__ _test_user_function
 
 let test_user_hook () =
-  Registry.HookRegistry.register "t3k"
-                                 (fun (ic,oc,_) _ _ ->
-                                  Llio.input_string ic >>= fun arg ->
-                                  Llio.output_string oc ("cucu " ^ arg));
-  Registry.HookRegistry.register "rev_range"
-                                 (fun (_,oc,_) user_db _ ->
-                                  let _count, keys = user_db # with_cursor
-                                                         (fun cur ->
-                                                          Registry.Cursor_store.fold_rev_range
-                                                            cur
-                                                            (Some "t")
-                                                            false
-                                                            "a"
-                                                            false
-                                                            (-1)
-                                                            (fun _ k _ acc ->
-                                                             k :: acc)
-                                                            []) in
-                                  Llio.output_list Llio.output_key oc keys
-                                 );
+  HookRegistry.register "t3k"
+    (fun (ic,oc,_) _ _ ->
+      Llio.input_string ic >>= fun arg ->
+      Llio.output_string oc ("cucu " ^ arg));
+  HookRegistry.register "rev_range"
+    (fun (_,oc,_) user_db _ ->
+      let _count, keys = user_db # with_cursor
+                           (fun cur ->
+                             Arakoon_registry.Cursor_store.fold_rev_range
+                               cur
+                               (Some "t")
+                               false
+                               "a"
+                               false
+                               (-1)
+                               (fun _ k _ acc ->
+                                 k :: acc)
+                               []) in
+      Llio.output_list Llio.output_key oc keys
+    );
   __client_server_wrapper__ (fun client ->
-                             client # user_hook "t3k" >>= fun (ic,oc) ->
-                             Llio.output_string oc "cthulhu" >>= fun () ->
-                             Llio.input_string ic >>= fun response ->
-                             OUnit.assert_equal response "cucu cthulhu";
-
-                             let keys = ["a"; "b"; "c"; "f"; "t"] in
-                             Lwt_list.iter_p (fun k -> client # set k k) keys >>= fun () ->
-
-                             client # user_hook "rev_range" >>= fun (ic,_) ->
-                             Llio.input_list Llio.input_string ic >>= fun response ->
-                             OUnit.assert_equal response ["f"; "c"; "b"];
-                             Lwt.return ())
+      client # user_hook "t3k" >>= fun (ic,oc) ->
+      Llio.output_string oc "cthulhu" >>= fun () ->
+      Llio.input_string ic >>= fun response ->
+      OUnit.assert_equal response "cucu cthulhu";
+
+      let keys = ["a"; "b"; "c"; "f"; "t"] in
+      Lwt_list.iter_p (fun k -> client # set k k) keys >>= fun () ->
+
+      client # user_hook "rev_range" >>= fun (ic,_) ->
+      Llio.input_list Llio.input_string ic >>= fun response ->
+      OUnit.assert_equal response ["f"; "c"; "b"];
+      Lwt.return ())
 
 let _clear (client:Arakoon_client.client)  () =
   client # range None true None true 1000 >>= fun xn ->
diff --git a/src/client/client_helper.ml b/src/client/client_helper.ml
index c8433e91..6dc9b586 100644
--- a/src/client/client_helper.ml
+++ b/src/client/client_helper.ml
@@ -165,7 +165,6 @@ module MasterLookupResult = struct
   (** Create a {! string } representation of a {! MasterLookupResult.t } *)
   let to_string t =
     let open Arakoon_client_config in
-    let open To_string in
     match t with
       | Found (name, cfg) -> Printf.sprintf "Found %s (%s)" name (show_node_cfg cfg)
       | No_master -> "No_master"
@@ -308,7 +307,7 @@ let with_client'' ?tls ccfg node_name f =
 let with_master_client' cluster_cfg f =
   let open MasterLookupResult in
   find_master' cluster_cfg >>= function
-  | Found (master_name, master_cfg) ->
+  | Found (master_name, _master_cfg) ->
      with_client'' cluster_cfg master_name f
   | master_result ->
     Lwt.fail (Error master_result)
diff --git a/src/client/client_protocol.ml b/src/client/client_protocol.ml
index fd440ac7..3c6b512b 100644
--- a/src/client/client_protocol.ml
+++ b/src/client/client_protocol.ml
@@ -406,7 +406,7 @@ let one_command stop (ic,oc,id as conn) (backend:Backend.backend) =
         (fun () ->
            backend # read_allowed consistency;
 
-           let open Registry in
+           let open Arakoon_registry in
            (match HookRegistry.lookup name with
              | h ->
                h conn (backend # get_read_user_db ()) (backend :> backend)
diff --git a/src/client/remote_nodestream.ml b/src/client/remote_nodestream.ml
index 924cbb5d..9f9e24b0 100644
--- a/src/client/remote_nodestream.ml
+++ b/src/client/remote_nodestream.ml
@@ -21,7 +21,6 @@ open Lwt
 open Client_cfg
 open Ncfg
 
-let section = Logger.Section.main
 
 class type nodestream = object
   method iterate:
diff --git a/src/client/remote_nodestream_test.ml b/src/client/remote_nodestream_test.ml
index 7ceed7dc..56932e65 100644
--- a/src/client/remote_nodestream_test.ml
+++ b/src/client/remote_nodestream_test.ml
@@ -55,7 +55,7 @@ let __wrap__ port conversation =
                                   master_cfg.Arakoon_client_config.port) in
     Lwt_io.open_connection address >>= fun (ic,oc) ->
     conversation (ic, oc) >>= fun () ->
-    Logger.debug section "end_of_senario" >>= fun () ->
+    Logger.debug ~section "end_of_senario" >>= fun () ->
     Lwt_io.close ic >>= fun () ->
     Lwt_io.close oc >>= fun () ->
     Lwt.return ()
diff --git a/src/dune b/src/dune
new file mode 100644
index 00000000..3d646017
--- /dev/null
+++ b/src/dune
@@ -0,0 +1,143 @@
+(copy_files# msg/*.ml*)
+(copy_files# nursery/*.ml*)
+(copy_files# plugins/*.ml*)
+(copy_files# tlog/*.ml*)
+(copy_files# client/*.ml*)
+(copy_files# lib/*.ml*)
+(copy_files# node/*.ml*)
+(copy_files# paxos/*.ml*)
+(copy_files# system/*.ml*)
+(copy_files# tools/*.ml*)
+(copy_files# inifiles/arakoon_inifiles_test.ml)
+(copy_files# tools/*.c)
+(copy_files# tools/*.h)
+; We could consider moving these modules to a alba_client subdir. that would make things easier.
+
+(library
+ (name        arakoon_client)
+ (public_name arakoon_client)
+ (modules     Statistics
+              Client_log
+              Sn
+              Ncfg
+              Log_extra Client_cfg
+              Stamp
+              Arakoon_exc
+              Arakoon_client_config
+              Arakoon_client
+              Arakoon_remote_client
+              Protocol_common
+              Client_helper
+              Std
+              Arakoon_registry
+              Plugin_helper
+              Key
+              Simple_store
+              Routing
+              Arakoon_interval
+              Update
+              Ini
+              Arakoon_redis
+              Arakoon_logger
+              Llio
+              Crc32c
+              Network
+              Lwt_extra
+              Typed_ssl
+              Unix_fd
+              Arakoon_etcd
+              Arakoon_config_url
+              Tcp_keepalive
+              To_string)
+  (libraries  arakoon_version parse_ini logger
+              lwt lwt.unix lwt_ssl lwt_log oUnit
+              bz2 str camltc dynlink snappy
+              ocplib-endian uri redis redis-lwt core )
+  (c_names    tcp_keepalive_sockopt_stubs
+              arakoon_crc32c bsd_crc32c)
+  (c_flags    :standard -msse4.2 -Wall -Wextra -Werror -ggdb3 -O2)
+  (modules_without_implementation Arakoon_client)
+  (flags      :standard -unsafe-string -annot)
+  (library_flags   -linkall)
+  (wrapped    false)
+)
+
+(library
+ (name        logger)
+ (public_name arakoon_client.logger)
+ (modules     Lwt_buffer Logger Crash_logger Ring_buffer Arakoon_log_sink)
+ (libraries   lwt lwt_log uri redis-lwt logs logs.lwt)
+ (flags       :standard -annot)
+ (wrapped     false)
+)
+
+
+(library
+ (name        arakoon_lib)
+ (modules     :standard \
+               ; exclude modules in lwt_buffer
+               Lwt_buffer
+               Logger
+               Crash_logger
+               Ring_buffer
+               Arakoon_log_sink
+               ; exclude modules in arakoon_client
+               Statistics
+               Client_log
+               Sn
+               Ncfg
+               Log_extra Client_cfg
+               Stamp
+               Arakoon_exc
+               Arakoon_client_config
+               Arakoon_client
+               Arakoon_remote_client
+               Protocol_common
+               Client_helper
+               Std
+               Arakoon_registry
+               Plugin_helper
+               Key
+               Simple_store
+               Routing
+               Arakoon_interval
+               Update
+               Ini
+               Arakoon_redis
+               Arakoon_logger
+               Llio
+               Crc32c
+               Network
+               Lwt_extra
+               Typed_ssl
+               Unix_fd
+               Arakoon_etcd
+               Arakoon_config_url
+               Tcp_keepalive
+               To_string
+  )
+  (libraries   arakoon_client.version arakoon_client arakoon_client.parse_ini
+               lwt lwt.unix lwt_ssl lwt_log oUnit
+               bz2 str camltc dynlink snappy
+               ocplib-endian uri redis redis-lwt core cmdliner )
+  (c_names     arakoon_limits
+               tcp_info_stubs)
+  (c_flags     :standard -msse4.2 -Wall -Wextra -Werror -ggdb3 -O2)
+  (flags       :standard -unsafe-string -annot)
+  (wrapped false)
+)
+
+
+;; Bisect would be implemented usign the :include stanza in both the pps and lib.
+;; The outside makefile could add the needed lines there.
+;; If jbuilder should create the include files, then we need to call promote.
+(rule
+  (targets bisect)
+  (deps force)
+  (action (with-stdout-to %{targets} (system "if [ -n $USE_BISECT ]; then echo \"${lib:str:str.cma} -unsafe ${lib:bisect:bisect_pp.cmo}\"; fi; date > %{depends}")))
+)
+
+(rule
+ (targets force)
+ (action (system "date > %{targets}"))
+)
diff --git a/src/inifiles/arakoon_inifiles.ml b/src/inifiles/arakoon_inifiles.ml
index d9d02251..a8c2802c 100644
--- a/src/inifiles/arakoon_inifiles.ml
+++ b/src/inifiles/arakoon_inifiles.ml
@@ -83,17 +83,6 @@ let setOfList list =
   in
   dosetOfList list Strset.empty
 
-let rec filterfile ?(buf=Buffer.create 500) f fd =
-  try
-    let theline = input_line fd in
-    if f theline then begin
-      Buffer.add_string buf (theline ^ "\n");
-      filterfile ~buf f fd
-    end
-    else
-      filterfile ~buf f fd
-  with End_of_file -> Buffer.contents buf
-
 let parse_inifile txt tbl =
   let lxbuf =
     let lines = Str.split (Str.regexp "\n") txt in
@@ -116,8 +105,9 @@ let parse_inifile txt tbl =
               (Hashtbl.create 10)
               values))
       parsed_txt
-  with Parsing.Parse_error | Failure "lexing: empty token" ->
-    raise (Ini_parse_error (lxbuf.Lexing.lex_curr_p.Lexing.pos_lnum, txt))
+  with
+  | Parsing.Parse_error                     -> raise (Ini_parse_error (lxbuf.Lexing.lex_curr_p.Lexing.pos_lnum, txt))
+  | Failure s when "lexing: empty token"= s -> raise (Ini_parse_error (lxbuf.Lexing.lex_curr_p.Lexing.pos_lnum, txt))
 
 let write_inifile oc tbl =
   let open Lwt.Infix in
@@ -269,7 +259,7 @@ class inifile ?(spec=[]) txt =
 
     method save file =
       Lwt_io.with_file
-        Lwt_io.Output file
+        ~mode:Lwt_io.Output file
         (fun oc -> write_inifile oc data)
 
     method to_string =
diff --git a/src/inifiles/arakoon_inifiles_test.ml b/src/inifiles/arakoon_inifiles_test.ml
index 46b60702..c4a84d0c 100644
--- a/src/inifiles/arakoon_inifiles_test.ml
+++ b/src/inifiles/arakoon_inifiles_test.ml
@@ -63,7 +63,6 @@ let test_parsing1 () =
 
 let test_parsing2 () =
   let txt =
-    let open Lwt.Infix in
     Lwt_extra.run
       (fun () ->
        Lwt_extra.read_file "./cfg/arakoon.ini"
diff --git a/src/inifiles/dune b/src/inifiles/dune
new file mode 100644
index 00000000..c37aed63
--- /dev/null
+++ b/src/inifiles/dune
@@ -0,0 +1,11 @@
+(ocamllex arakoon_inilexer)
+(ocamlyacc arakoon_parseini)
+
+(library
+ (name        parse_ini)
+ (public_name arakoon_client.parse_ini)
+ (modules     :standard \ arakoon_inifiles_test)
+ (libraries   lwt lwt.unix str unix)
+ (flags       :standard -w -3-8-52 -annot)
+ (wrapped     false)
+)
diff --git a/src/lib/std.ml b/src/lib/std.ml
index 5c07b0a3..818a5fb8 100644
--- a/src/lib/std.ml
+++ b/src/lib/std.ml
@@ -16,8 +16,6 @@ limitations under the License.
 
 
 
-open Test_utils
-
 module type Show = sig
   type t
   val show : t -> string
@@ -39,16 +37,6 @@ module EqLib : sig
   module Make : functor(S : S) -> Equable with type t := S.s
   module Default : functor(S : sig type s end) -> Equable with type t := S.s
 
-  module Test : sig
-    module type S = sig
-      type t
-      include QuickCheckUtils.Testable with type t := t
-      include Equable with type t := t
-    end
-
-    val quickCheck :
-      (module S with type t = 'a) -> 'a QuickCheckUtils.test
-  end
 end = struct
   module type S = sig
     type s
@@ -65,19 +53,7 @@ end = struct
     let (<>:) = (<>)
   end
 
-  module Test = struct
-    module type S = sig
-      type t
-      include QuickCheckUtils.Testable with type t := t
-      include Equable with type t := t
-    end
-
-    let quickCheck (type t)
-      (module M : S with type t = t) =
-      let open M in
-
-      QuickCheckUtils.prop_test (module M) (fun v -> v =: v)
-  end
+  
 end
 
 module Compare : sig
@@ -266,60 +242,7 @@ module List = struct
         | None -> l
         | Some v -> (key, v) :: l
 
-  module Test = struct
-    open OUnit
-
-    let assert_set_equal ?printer ?msg l1 l2 =
-      let l1' = List.sort compare l1
-      and l2' = List.sort compare l2 in
-      assert_equal ?printer ?msg l1' l2'
-
-    let printer l =
-      List.map (fun (k, v) -> Printf.sprintf "(%S, %d)" k v) l
-      |> String.concat "; "
-      |> Printf.sprintf "[%s]"
-
-    let suite = "List" >::: [
-      "alter" >::: [
-        "empty" >::
-          (fun () -> assert_set_equal ~printer
-            (alter ~list:[] ~key:"k" (const (Some 1)))
-            []);
-        "empty_add_default" >::
-          (fun () -> assert_set_equal ~printer
-            (alter ~list:[] ~key:"k" ~default:1 (const None))
-            [("k", 1)]);
-        "delete" >::
-          (fun () -> assert_set_equal ~printer
-            (alter ~list:[("k", 1)] ~key:"k" (const None))
-            []);
-        "delete_multi" >::
-          (fun () -> assert_set_equal ~printer
-            (alter ~list:[("k", 1); ("k", 2)] ~key:"k" (const None))
-            []);
-        "delete_multi_keep_others" >::
-          (fun () -> assert_set_equal ~printer
-            (alter ~list:[("k", 1); ("l", 2); ("k", 3); ("m", 4)] ~key:"k" (const None))
-            [("l", 2); ("m", 4)]);
-        "update" >::
-          (fun () -> assert_set_equal ~printer
-            (alter ~list:[("k", 1); ("l", 2); ("m", 3)] ~key:"l" (fun v -> Some (v + 1)))
-            [("k", 1); ("l", 3); ("m", 3)]);
-        "update_all" >::
-          (fun () -> assert_set_equal ~printer
-            (alter ~list:[("k", 1); ("l", 2); ("m", 3); ("l", 4)] ~key:"l" (fun v -> Some (v + 1)))
-            [("k", 1); ("l", 3); ("m", 3); ("l", 5)]);
-        "update_no_default" >::
-          (fun () -> assert_set_equal ~printer
-            (alter ~list:[("k", 1); ("l", 2)] ~key:"l" (fun v -> Some (v + 1)))
-            [("k", 1); ("l", 3)]);
-        "remove_no_default" >::
-          (fun () -> assert_set_equal ~printer
-            (alter ~list:[("k", 1); ("l", 2); ("m", 3)] ~key:"l" ~default:4 (const None))
-            [("k", 1); ("m", 3)]);
-      ]
-    ]
-  end
+
 end
 
 type direction =
diff --git a/src/lib/test_utils.ml b/src/lib/test_utils.ml
deleted file mode 100644
index 25f102fa..00000000
--- a/src/lib/test_utils.ml
+++ /dev/null
@@ -1,68 +0,0 @@
-(*
-Copyright (2010-2014) INCUBAID BVBA
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*)
-
-
-
-module type QuickCheckable = sig
-  type t
-  val arbitrary : t QuickCheck.arbitrary
-end
-
-module QuickCheckUtils : sig
-  module type Testable = sig
-    type t
-    val show : t -> string
-    include QuickCheckable with type t := t
-  end
-
-  type 'a test
-
-  val prop_test :
-    (module Testable with type t = 'a) -> ('a -> bool) -> 'a test
-
-  val quickCheck : 'a test -> QuickCheck.testresult
-  val verboseCheck : 'a test -> QuickCheck.testresult
-
-  val (>::~) : string -> 'a test -> OUnit.test
-end = struct
-  type 'a test = ('a -> bool) QuickCheck.testable * ('a -> bool)
-
-  module type Testable = sig
-    type t
-    val show : t -> string
-    include QuickCheckable with type t := t
-  end
-
-  let prop_test (type t) (module M : Testable with type t = t) prop =
-    let testable = QuickCheck.testable_fun
-      M.arbitrary
-      M.show
-      QuickCheck.testable_bool
-    in
-    testable, prop
-
-  let quickCheck (t, p) = QuickCheck.quickCheck t p
-  let verboseCheck (t, p) = QuickCheck.verboseCheck t p
-
-  let (>::~) name t =
-    let run () =
-      match quickCheck t with
-        | QuickCheck.Success -> ()
-        | QuickCheck.Failure _ -> OUnit.assert_failure "QuickCheck: Failure"
-        | QuickCheck.Exhausted _ -> OUnit.assert_failure "QuickCheck: Exhausted"
-    in
-    OUnit.(name >:: run)
-end
diff --git a/src/main/arakoon.ml b/src/main/arakoon.ml
index 39ea6b91..f53cdf4d 100644
--- a/src/main/arakoon.ml
+++ b/src/main/arakoon.ml
@@ -652,11 +652,7 @@ let main () =
   (* let () = Printf.printf "[rc=%i]\n" rc in *)
   let () =
     (* no exit hook processing needed: we've done everything we needed to do *)
-    let rec loop () =
-      let _ = Lwt_sequence.take_l Lwt_main.exit_hooks in
-      loop ()
-    in
-    try loop () with Lwt_sequence.Empty -> ()
+    Lwt_main.Exit_hooks.remove_all ()
   in
   exit exit_code
 
diff --git a/src/client/bs_bench.ml b/src/main/bs_bench.ml
similarity index 100%
rename from src/client/bs_bench.ml
rename to src/main/bs_bench.ml
diff --git a/src/main/dune b/src/main/dune
new file mode 100644
index 00000000..ede11cdc
--- /dev/null
+++ b/src/main/dune
@@ -0,0 +1,47 @@
+(executable
+ (name        arakoon)
+ (public_name arakoon)
+ (modules     arakoon replay_main inspect)
+ (libraries   arakoon_lib)
+ (flags       :standard -unsafe-string -annot)
+ (package     arakoon)
+)
+
+(executable
+ (name        bs_bench)
+ (libraries   camltc)
+ (modules     bs_bench)
+ (flags       :standard -unsafe-string -annot)
+)
+
+(executable
+ (name        lwt_buffer_test)
+ (libraries   arakoon_client.logger)
+ (modules     lwt_buffer_test)
+ (flags       :standard -unsafe-string -annot)
+)
+
+
+;;
+; Below is a crude attempt of getting bisect to work.
+; The major problem is that we need all compilation units to depend on bisect.
+; but since we linkall, bisect main will be there multiple times, and thats no good.
+; Bisect is only added to the main lib, but it is not added to any other compilation units.
+
+; (rule
+;  (targets arakoon_bisect.ml)
+;   (deps    arakoon.ml)
+;   (action  (copy# ${<} ${@}))
+; ))
+
+;(executable
+; (name        arakoon_bisect)
+; (public_name arakoon_bisect)
+; (libraries  arakoon_lib)
+; (modules arakoon_bisect replay_main inspect)
+; (preprocess (action (run camlp4orf.opt src/camlp4/arakoon_logger.cmxs ${<})))
+; (preprocessor_deps ../camlp4/arakoon_logger.cmxs)
+; (flags       :standard -unsafe-string -w -3-8-52)
+; (package arakoon)
+; (modes (native))
+;)
diff --git a/src/tools/lwt_buffer_test.ml b/src/main/lwt_buffer_test.ml
similarity index 100%
rename from src/tools/lwt_buffer_test.ml
rename to src/main/lwt_buffer_test.ml
diff --git a/src/node/backend.ml b/src/node/backend.ml
index efa74c7f..c2264f5e 100644
--- a/src/node/backend.ml
+++ b/src/node/backend.ml
@@ -25,7 +25,7 @@ open Client_cfg
 open Arakoon_client
 
 class type backend = object
-  inherit Registry.backend
+  inherit Arakoon_registry.backend
 
   method exists: consistency:consistency -> string -> bool
   method get: consistency:consistency -> string -> string
@@ -84,7 +84,7 @@ class type backend = object
   method user_function: string -> string option -> (string option) Lwt.t
 
   method read_allowed : consistency -> unit
-  method get_read_user_db : unit -> Registry.read_user_db
+  method get_read_user_db : unit -> Arakoon_registry.read_user_db
 
   method set_interval : Interval.t -> unit Lwt.t
   method get_interval : unit -> Interval.t
diff --git a/src/node/catchup.ml b/src/node/catchup.ml
index 29b158f2..2d5adb9b 100644
--- a/src/node/catchup.ml
+++ b/src/node/catchup.ml
@@ -311,6 +311,7 @@ let last_entries
       (tlog_collection:Tlogcollection.tlog_collection)
       (start_i:Sn.t) (oc:Lwt_io.output_channel)
   =
+  let () = ignore (store, tlog_collection, start_i, oc) in
   Lwt.fail_with "no longer supported"
 
 let last_entries2
diff --git a/src/node/collapser.ml b/src/node/collapser.ml
index cfe6fa5f..9327d250 100644
--- a/src/node/collapser.ml
+++ b/src/node/collapser.ml
@@ -19,7 +19,6 @@ limitations under the License.
 open Lwt
 type 'a store_maker= ?read_only:bool -> string -> 'a Lwt.t
 open Tlogcommon
-open Tlog_map
 
 let section = Logger.Section.main
 
@@ -60,7 +59,7 @@ let collapse_until (type s) (tlog_coll:Tlogcollection.tlog_collection)
     new_location >>= fun new_store ->
   Lwt.finalize (
     fun () ->
-      tlog_coll # get_infimum_i () >>= fun min_i ->
+      tlog_coll # get_infimum_i () >>= fun _min_i ->
       let store_i = S.consensus_i new_store in
       let tfs = Sn.string_of too_far_i in
       let start_i =
diff --git a/src/node/collapser_main.ml b/src/node/collapser_main.ml
index 2d05307c..44e44e3d 100644
--- a/src/node/collapser_main.ml
+++ b/src/node/collapser_main.ml
@@ -35,7 +35,7 @@ let collapse_remote ~tls ~tcp_keepalive ip port cluster_id n =
     in
     Lwt.catch
       (fun () -> Client_main.with_connection ~tls ~tcp_keepalive address collapse)
-      (fun exn -> Logger.fatal Logger.Section.main ~exn "remote_collapsing_failed"
+      (fun exn -> Logger.fatal ~exn "remote_collapsing_failed"
         >>= fun () -> Lwt.return (-1)
       )
   in
@@ -52,7 +52,7 @@ let collapse_local make_config node_id n_tlogs =
 
       )
       (fun exn ->
-        Logger.fatal Logger.Section.main ~exn "collapse_local failed"
+        Logger.fatal ~exn "collapse_local failed"
         >>= fun () ->
         Lwt.return 1
       )
diff --git a/src/node/collapser_test.ml b/src/node/collapser_test.ml
index 1e9b5067..f7dacf87 100644
--- a/src/node/collapser_test.ml
+++ b/src/node/collapser_test.ml
@@ -19,7 +19,6 @@ limitations under the License.
 open Lwt
 open OUnit
 open Update
-open Tlog_map
 
 let section = Logger.Section.main
 let compressor = Compression.Snappy
diff --git a/src/node/dump_store.ml b/src/node/dump_store.ml
index 56b6d750..96ab8e20 100644
--- a/src/node/dump_store.ml
+++ b/src/node/dump_store.ml
@@ -58,16 +58,16 @@ let summary store =
   _dump_cluster_id store
 
 let dump_store ~dump_values filename =
+  let () = ignore dump_values in
   let t () =
-    let open Simple_store in
     S.make_store ~lcnum:1024 ~ncnum:512 ~read_only:true filename >>= fun store ->
     summary store >>= fun () ->
     let rec loop (first:string option) =
       if first = None then Lwt.return_unit
       else
-        let n,kvs = S.range_entries store first false None true 100 in
+        let _n,kvs = S.range_entries store first false None true 100 in
         Lwt_list.fold_left_s
-          (fun last (k,v)  ->
+          (fun _last (k,v)  ->
             let (kx:string) = Key.get k in
             Lwt_io.printlf "%S\t%S" kx v >>= fun () ->
             Lwt.return (Some kx)
diff --git a/src/node/mem_store.ml b/src/node/mem_store.ml
index 07e28dde..c23a3a36 100644
--- a/src/node/mem_store.ml
+++ b/src/node/mem_store.ml
@@ -123,6 +123,7 @@ let range ms first finc last linc max =
   Array.of_list keys
 
 let optimize _ms ~quiesced ~stop ~slowdown_factor =
+    let () = ignore slowdown_factor in
     let () = ignore quiesced in
     let () = ignore stop in
     Lwt.return true
diff --git a/src/node/node_cfg.ml b/src/node/node_cfg.ml
index 66530612..57a17c63 100644
--- a/src/node/node_cfg.ml
+++ b/src/node/node_cfg.ml
@@ -287,6 +287,7 @@ module Node_cfg = struct
         ?(base=4000) ?(cluster_id="ricky") ?(node_name = Printf.sprintf "t_arakoon_%i")
         ?source_node
         n_nodes master lease_period =
+    let () = ignore source_node in
     let make_one n =
       let ns = (string_of_int n) in
       let home = ":MEM#t_arakoon_" ^ ns in
diff --git a/src/node/node_main.ml b/src/node/node_main.ml
index 28c62d0c..0692d7a7 100644
--- a/src/node/node_main.ml
+++ b/src/node/node_main.ml
@@ -414,7 +414,7 @@ let _main_2 (type s)
   =
   make_config () >>= fun cluster_cfg ->
   let cfgs = cluster_cfg.cfgs in
-  let me, others = Node_cfg.Node_cfg.split name cfgs in
+  let me, _others = Node_cfg.Node_cfg.split name cfgs in
   _config_logging me.node_name make_config >>= fun maybe_dump_crash_log ->
 
   (let lock_file = me.home ^ "/LOCK" in
@@ -556,7 +556,7 @@ let _main_2 (type s)
              if fsync then
                 Logger.ign_warning_ "Be careful, fsync is set to false! (fsync is deprecated: if you really want to live dangerously, use fsync_entries or/and fsync_interval)"
            in
-           (fun i time_since_last_fsync -> fsync)
+           (fun _i _time_since_last_fsync -> fsync)
 
         | None, None, None       -> (fun _ _ -> true) (* always *)
         | None, Some dt, None    -> (fun _ time_since_last_fsync -> time_since_last_fsync >= dt)
@@ -635,7 +635,6 @@ let _main_2 (type s)
                Tlog_main._mark_tlog fn (Tlog_map._make_close_marker me.node_name) >>= fun _ ->
                _open_tlc_and_store()
              in
-             let open Tlog_map in
              match exn with
              | Tlog_map.TLCNotProperlyClosed (fn,_) ->
                 begin
@@ -885,12 +884,12 @@ let _main_2 (type s)
                 | Multi_paxos.Unquiesce
                 | Multi_paxos.DropMaster _ -> (fun () -> Lwt_buffer.add e inject_buffer), "inject"
             in
-            Logger.debug_f Multi_paxos.section "XXX injecting event %s into '%s'"
+            Logger.debug_f ~section:Multi_paxos.section "XXX injecting event %s into '%s'"
               (Multi_paxos.paxos_event2s e)
               name
             >>= fun () ->
             add_to_buffer () >>= fun () ->
-            Logger.debug_f Multi_paxos.section "XXX injected event into '%s'" name
+            Logger.debug_f ~section:Multi_paxos.section "XXX injected event into '%s'" name
           in
           let buffers = Multi_paxos_fsm.make_buffers
                           (client_buffer,
@@ -1092,6 +1091,7 @@ let _main_2 (type s)
 let main_t (make_config: unit -> 'a Lwt.t)
            name ~daemonize ~catchup_only ~source_node ~autofix ~lock : int Lwt.t
   =
+  let () = ignore lock in
   let module S = (val (Store.make_store_module (module Batched_store.Local_store))) in
   let (make_tlog_coll :Tlogcollection.tlc_factory) =
     Tlc2.make_tlc2 ~check_marker:true
diff --git a/src/node/store.ml b/src/node/store.ml
index d76a0537..4bf58ff7 100644
--- a/src/node/store.ml
+++ b/src/node/store.ml
@@ -95,7 +95,7 @@ sig
 
   val on_consensus : t -> Value.t * int64 * Int64.t -> update_result list Lwt.t
 
-  val get_read_user_db : t -> Registry.read_user_db
+  val get_read_user_db : t -> Arakoon_registry.read_user_db
 
   val _set_i : t -> Sn.t -> unit
 end
@@ -640,7 +640,7 @@ struct
     S.set store.s tx __interval_key range_s
 
   class store_cursor_db cur =
-    object(self: #Registry.cursor_db)
+    object(self: #Arakoon_registry.cursor_db)
       val mutable is_valid = None
 
       method assert_valid () =
@@ -696,7 +696,7 @@ struct
   end
 
   class store_read_user_db store =
-    object(_ : #Registry.read_user_db)
+    object(_ : #Arakoon_registry.read_user_db)
       method get k =
         _get_option store k
 
@@ -709,7 +709,7 @@ struct
       method with_cursor f =
         S.with_cursor
           store.s
-          (fun cur -> f (new store_cursor_db cur :> Registry.cursor_db))
+          (fun cur -> f (new store_cursor_db cur :> Arakoon_registry.cursor_db))
 
       method get_interval () =
         store.interval
@@ -729,7 +729,7 @@ struct
   let _user_function store (name:string) (po:string option) tx =
     Lwt.catch
       (fun () ->
-        let f = Registry.Registry.lookup name in
+        let f = Arakoon_registry.Registry.lookup name in
         let user_db = new store_user_db store tx in
         let ro = f user_db po in
         Lwt.return (Ok ro))
diff --git a/src/node/sync_backend.ml b/src/node/sync_backend.ml
index 446956a0..cd9457aa 100644
--- a/src/node/sync_backend.ml
+++ b/src/node/sync_backend.ml
@@ -285,7 +285,7 @@ struct
 
       method user_function name po =
         log_o self "user_function %s" name >>= fun () ->
-        if Registry.Registry.exists name
+        if Arakoon_registry.Registry.exists name
         then
           begin
             let update = Update.UserFunction(name,po) in
@@ -713,7 +713,7 @@ struct
             (* remove all but tlogs_to_keep last tlogs *)
             Collapser._head_i (module S) head_path ~cluster_id >>= fun head_io ->
 
-            let head_i, head_n = match head_io with
+            let _head_i, head_n = match head_io with
               | None -> failwith "impossible i for copied head"
               | Some i -> i, tlog_collection # get_tlog_from_i i
             in
diff --git a/src/nursery/client_cfg_test.ml b/src/nursery/client_cfg_test.ml
index 3532525f..4540d00a 100644
--- a/src/nursery/client_cfg_test.ml
+++ b/src/nursery/client_cfg_test.ml
@@ -45,7 +45,7 @@ let test_parsing () =
     let contents = List.fold_left (fun acc v -> acc ^ "\n" ^ v) "" lines in
     let fn = "/tmp/client_cfg_test.ml" in
     Lwt_io.with_file
-      Lwt_io.Output fn
+      ~mode:Lwt_io.Output fn
       (fun oc -> Lwt_io.write oc contents)
     >>= fun () ->
     Arakoon_config_url.(retrieve (File fn)) >|= (ClientCfg._from_txt "global") >>= fun cfg ->
diff --git a/src/nursery/nursery_main.ml b/src/nursery/nursery_main.ml
index 088df4d8..7c5cd75f 100644
--- a/src/nursery/nursery_main.ml
+++ b/src/nursery/nursery_main.ml
@@ -77,7 +77,7 @@ let setup_logger file_name =
   Lwt_log.default := file_logger;
   Lwt.return ()
 
-let _get_keeper_config_generic fetcher url=
+let _get_keeper_config_generic fetcher _url=
   fetcher() >>= fun txt ->
   let inifile = new Arakoon_inifiles.inifile txt in
   let m_cfg = Node_cfg.get_nursery_cfg inifile in
diff --git a/src/paxos/learner.ml b/src/paxos/learner.ml
index df3e9ac8..2b501663 100644
--- a/src/paxos/learner.ml
+++ b/src/paxos/learner.ml
@@ -11,19 +11,19 @@ open Mp_msg.MPMessage
 
 (* a forced slave or someone who is outbidded sends a fake prepare
    in order to receive detailed info from the others in a Nak msg *)
-let fake_prepare constants (current_i, current_n) () =
+let fake_prepare _constants (current_i, current_n) () =
   (* fake a prepare, and hopefully wake up a dormant master *)
   Logger.ign_debug_f_ "Learner.fake_prepare: sending Prepare(-1, %Li)" current_i;
   let fake = Prepare (Sn.of_int (-1), current_i) in
   let sides = [EMCast fake] in
   Fsm.return ~sides (Learner_steady_state (current_n, current_i, None))
 
-let handle_timeout ((n,i,maybe_previous) as state) constants invalidate_lease_start_until n' i' =
+let handle_timeout state _constants _invalidate_lease_start_until _n' _i' =
   Logger.ign_debug_f_ "steady_state: ignoring lease expiration because I am a learner";
   Fsm.return (Learner_steady_state state)
 
 let handle_from_node ((n,i,previous) as state) (type s) (module S : Store.STORE with type t = s)
-      {store; me;on_witness} msg source =
+      {store; me;on_witness; _ } msg source =
   Logger.ign_debug_f_ "Learner.steady_state n:%s i:%s v:%s store_i:%Li got %S from %s"
     (Sn.string_of n) (Sn.string_of i) (To_string.option Value.value2s previous)
     (match S.consensus_i store with None -> -1L | Some v -> v)
@@ -92,7 +92,7 @@ let handle_from_node ((n,i,previous) as state) (type s) (module S : Store.STORE
      Logger.ign_debug_f_ "Learner.steady state :: ignoring %S" (string_of msg);
      Fsm.return (Learner_steady_state state)
 
-let steady_state (type s) constants state event =
+let steady_state (type _s) constants state event =
   Slave.common_steady_state
     handle_from_node handle_timeout (fun x -> Learner_steady_state x)
     constants state event
diff --git a/src/paxos/multi_paxos.ml b/src/paxos/multi_paxos.ml
index ddd2c745..b7b204fd 100644
--- a/src/paxos/multi_paxos.ml
+++ b/src/paxos/multi_paxos.ml
@@ -185,7 +185,7 @@ let make (type s) ~catchup_tls_ctx ~tcp_keepalive me is_learner others learners
     max_buffer_size;
   }
 
-let mcast {send; me; others; learners} msg =
+let mcast {send; me; others; learners ;_ } msg =
   let dst = match msg with
       Accept _ | Nak _ -> learners @ others
     | _ -> others in
diff --git a/src/paxos/slave.ml b/src/paxos/slave.ml
index 7a13f720..3b4da71f 100644
--- a/src/paxos/slave.ml
+++ b/src/paxos/slave.ml
@@ -72,7 +72,7 @@ let slave_fake_prepare constants (current_i,current_n) () =
   in
   Fsm.return ~sides:sides (Slave_steady_state (current_n, current_i, None))
 
-let handle_timeout ((n,i,maybe_previous) as state) constants invalidate_lease_start_until n' i' =
+let handle_timeout ((n,i,_maybe_previous) as state) constants invalidate_lease_start_until n' i' =
   if not (is_election constants)
   then
     begin
@@ -222,7 +222,7 @@ let handle_from_node ((n,i,maybe_previous) as state) (type s) (module S : Store.
       Fsm.return ~sides:[log_e0;log_e] (Slave_steady_state state)
 
 let common_steady_state (type s) handle_from_node handle_timeout next constants state event =
-  let (n,i,maybe_previous) = state in
+  let (n,i,_maybe_previous) = state in
   let module S = (val constants.store_module : Store.STORE with type t = s) in
   match event with
     | FromNode (msg,source) ->
@@ -252,7 +252,7 @@ let common_steady_state (type s) handle_from_node handle_timeout next constants
 
 (* a pending slave that is in sync on i and is ready
    to receive accepts *)
-let slave_steady_state (type s) constants state event =
+let slave_steady_state (type _s) constants state event =
   common_steady_state handle_from_node handle_timeout (fun x -> Slave_steady_state x)
   constants state event
 
diff --git a/src/plugins/registry.ml b/src/plugins/arakoon_registry.ml
similarity index 100%
rename from src/plugins/registry.ml
rename to src/plugins/arakoon_registry.ml
diff --git a/src/plugins/registry.mli b/src/plugins/arakoon_registry.mli
similarity index 100%
rename from src/plugins/registry.mli
rename to src/plugins/arakoon_registry.mli
diff --git a/src/plugins/plugin_update_max.ml b/src/plugins/plugin_update_max.ml
index 88f9bff6..916c81dc 100644
--- a/src/plugins/plugin_update_max.ml
+++ b/src/plugins/plugin_update_max.ml
@@ -15,7 +15,7 @@ limitations under the License.
 *)
 
 
-open Registry
+open Arakoon_registry
 
 let s2i = int_of_string
 let i2s = string_of_int
diff --git a/src/system/single.ml b/src/system/single.ml
index 682e3e5a..1b85d212 100644
--- a/src/system/single.ml
+++ b/src/system/single.ml
@@ -103,7 +103,7 @@ let nothing_on_slave (_tn, make_cluster_cfg, all_t) =
       )
   in
 
-  let test_slave cluster_id cfg =
+  let test_slave _cluster_id cfg =
     Logger.info_f_ "slave=%s" cfg.node_name  >>= fun () ->
     let f with_client =
       with_client set_on_slave >>= fun () ->
@@ -141,7 +141,7 @@ let dirty_on_slave (_tn, make_cluster_cfg,_) =
     let slave_cfgs = List.filter (fun cfg -> cfg.node_name <> master_name) cfgs in
     Lwt.return slave_cfgs
   in
-  let do_slave cluster_id cfg =
+  let do_slave _cluster_id cfg =
     let dirty_get (client:Arakoon_client.client) =
       Logger.debug_ "dirty_get" >>= fun () ->
       should_fail
@@ -553,13 +553,14 @@ let _multi_get_option (client:client) =
 
 
 let find_master ~tls cluster_cfg =
+  let () = ignore tls in
   let lp = cluster_cfg._lease_period in
   let timeout = 3 * lp in
   let go () =
     let open Client_helper.MasterLookupResult in
     Client_helper.find_master_loop
       (Node_cfg.Node_cfg.to_client_cfg cluster_cfg) >>= function
-      | Found (name, cfg) -> return name
+      | Found (name, _cfg) -> return name
       | No_master -> Lwt.fail (Failure "No Master")
       | Too_many_nodes_down -> Lwt.fail (Failure "too many nodes down")
       | Unknown_node (n, _) -> return n
diff --git a/src/system/startup.ml b/src/system/startup.ml
index 097b71ed..2857729e 100644
--- a/src/system/startup.ml
+++ b/src/system/startup.ml
@@ -325,7 +325,7 @@ let ahead_master_loses_role () =
   (* sleep a bit so the previous 2 slaves can make progress *)
   Lwt_unix.sleep ((float lease_period) *. 3.0) >>= fun () ->
   Lwt.catch
-    (fun () -> Client_main.find_master (to_client_cfg cluster_cfg) >>= fun master ->
+    (fun () -> Client_main.find_master (to_client_cfg cluster_cfg) >>= fun _master ->
                Lwt.return_unit)
     (fun exn ->
       Logger.fatal_f_ ~exn "NO MASTER" >>= fun () ->
diff --git a/src/tlog/compression.ml b/src/tlog/compression.ml
index 5b28a718..c0866926 100644
--- a/src/tlog/compression.ml
+++ b/src/tlog/compression.ml
@@ -70,7 +70,7 @@ let _compress_tlog
     ~mode:Lwt_io.input tlog_name
     (fun ic ->
        let tmp_file = archive_name ^ ".part" in
-       Logger.info_f Logger.Section.main
+       Logger.info_f ~section:Logger.Section.main
                      "Compressing %S to %S via %S"
                      tlog_name archive_name tmp_file >>= fun () ->
        File_system.unlink ~verbose:false tmp_file >>= fun () ->
@@ -114,12 +114,13 @@ let _compress_tlog
             let cl = String.length contents in
             let ol = String.length output in
             let factor = (float cl) /. (float ol) in
-            Logger.debug_f Logger.Section.main "compression: %i bytes into %i (in %f s) (factor=%2f)" cl ol d factor
+            let section = Logger.Section.main in 
+            Logger.debug_f ~section "compression: %i bytes into %i (in %f s) (factor=%2f)" cl ol d factor
             >>= fun () ->
             Llio.output_int64 oc last_i >>= fun () ->
             Llio.output_string oc output >>= fun () ->
             let sleep = 2.0 *. d in
-            Logger.debug_f Logger.Section.main "compression: sleeping %f" sleep >>= fun () ->
+            Logger.debug_f ~section "compression: sleeping %f" sleep >>= fun () ->
             Lwt_unix.sleep sleep
           in
           let buffer = Buffer.create buffer_size in
diff --git a/src/tlog/compression_test.ml b/src/tlog/compression_test.ml
index 1a68d123..2c4166e8 100644
--- a/src/tlog/compression_test.ml
+++ b/src/tlog/compression_test.ml
@@ -26,7 +26,7 @@ let section = Logger.Section.main
 
 let test_compress_file which () =
   let archive_name, compressor, _tlog_name = which in
-  Logger.info Logger.Section.main "test_compress_file" >>= fun () ->
+  Logger.info ~section "test_compress_file" >>= fun () ->
   let tlog_name = "/tmp/test_compress_file.tlog" in
   Lwt_io.with_file tlog_name ~mode:Lwt_io.output
     (fun oc ->
diff --git a/src/tlog/mem_tlogcollection.ml b/src/tlog/mem_tlogcollection.ml
index 91a209bc..b6ffb3ca 100644
--- a/src/tlog/mem_tlogcollection.ml
+++ b/src/tlog/mem_tlogcollection.ml
@@ -74,6 +74,7 @@ class mem_tlog_collection _name =
     method which_tlog_file _start_i = failwith "which_tlog_file not supported"
 
     method tlogs_to_collapse ~head_i ~last_i ~tlogs_to_keep =
+      let () = ignore (head_i,last_i,tlogs_to_keep) in
       failwith "tlogs_to_collapse not supported"
 
     method should_fsync _ = false
@@ -88,12 +89,14 @@ class mem_tlog_collection _name =
     method get_head_name () = failwith "get_head_name not implemented"
 
     method get_tlog_from_i _ = 0
-    method get_start_i n = Some Sn.zero
+    method get_start_i _n = Some Sn.zero
     method is_rollover_point _ = false
     method next_rollover _ = Some Int64.max_int
 
     method invalidate () = ()
+
     method log_value_explicit i (v:Value.t) ~sync_override marker =
+      let () = ignore sync_override in
       let entry = Entry.make i v 0L marker in
       let () = data <- entry::data in
       let () = last_entry <- (Some entry) in
@@ -111,8 +114,7 @@ class mem_tlog_collection _name =
 let make_mem_tlog_collection
       ?cluster_id
       ?tlog_max_entries ?tlog_max_size _tlog_dir _tlf_dir _head_dir ~should_fsync name ~fsync_tlog_dir =
-  let () = ignore should_fsync in
-  let () = ignore fsync_tlog_dir in
+  let () = ignore (cluster_id, should_fsync,fsync_tlog_dir, tlog_max_entries, tlog_max_size) in
   let x = new mem_tlog_collection name in
   let x2 = (x :> tlog_collection) in
   Lwt.return x2
diff --git a/src/tlog/tlc2.ml b/src/tlog/tlc2.ml
index 731ae129..4b020d9c 100644
--- a/src/tlog/tlc2.ml
+++ b/src/tlog/tlc2.ml
@@ -43,7 +43,7 @@ let _validate_list tlog_names node_id ~check_marker=
       _validate_one tlog_name node_id ~check_marker ~check_sabotage:true)
     (None,None,0)
     tlog_names
-  >>= fun (eo,index, pos) ->
+  >>= fun (eo,index, _pos) ->
   Logger.info_f_ "_validate_list %s => %s" (String.concat ";" tlog_names) (Index.to_string index) >>= fun () ->
   Lwt.return (TlogValidIncomplete, eo, index)
 
diff --git a/src/tlog/tlc2_test.ml b/src/tlog/tlc2_test.ml
index a1156fe1..ebf10feb 100644
--- a/src/tlog/tlc2_test.ml
+++ b/src/tlog/tlc2_test.ml
@@ -52,7 +52,6 @@ let prepare_tlog_scenarios (dn,(factory:factory)) =
   Lwt.return ()
 
 let test_interrupted_rollover (dn, tlx_dir, factory) =
-  let open Tlog_map in
   prepare_tlog_scenarios (dn,factory) >>= fun () ->
   factory ~tlog_max_entries:5 dn node_id >>= fun tlog_coll ->
   let value = Value.create_master_value ~lease_start:0. "me" in
@@ -222,7 +221,7 @@ let test_iterate7 (dn, tlx_dir, (factory:factory)) =
             (0L, 0);
           ] =
             List.map
-              (fun ({ TlogMap.i; n; is_archive; }) ->
+              (fun ({ TlogMap.i; n; _ }) ->
                i, n)
               tlog_map.TlogMap.i_to_tlog_number);
 
@@ -234,7 +233,7 @@ let test_iterate7 (dn, tlx_dir, (factory:factory)) =
        assert (!next = Tlogcommon.Entry.i_of entry);
        next := Int64.succ !next;
        Lwt.return ())
-      (fun i -> Lwt.return ())
+      (fun _i -> Lwt.return ())
   >>= fun () ->
   assert (!next = 6L);
   Lwt.return ()
@@ -357,7 +356,7 @@ let test_size_based_roll_over1 (dn, tlx_dir, (factory:factory)) =
   OUnit.assert_equal 3 (TlogMap.get_tlog_number second_map) ~printer;
   Lwt.return_unit
 
-let test_get_last_i (dn, tlx_dir, (factory:factory)) =
+let test_get_last_i (dn, _tlx_dir, (factory:factory)) =
   let make_value size =
     Value.create_client_value [ Update.Set ("", Bytes.create size); ] false
   in
@@ -397,7 +396,7 @@ let test_get_last_i (dn, tlx_dir, (factory:factory)) =
               (0L, 0);
             ] =
               List.map
-                (fun ({ TlogMap.i; n; is_archive; }) ->
+                (fun ({ TlogMap.i; n; _ }) ->
                  i, n)
                 tlog_map.TlogMap.i_to_tlog_number);
 
@@ -431,7 +430,6 @@ let first_i_of_compress_race (dn, tlx_dir, (factory : factory)) =
   let tlog_max_size = 1000 in
   factory ~tlog_max_size dn "node_id" >>= fun tlc ->
   let make_value size =
-    let open Update in
     Value.create_client_value [ Update.Set ("", Bytes.create size); ] false
   in
   let value = make_value 0 in
diff --git a/src/tlog/tlog_main.ml b/src/tlog/tlog_main.ml
index 0d9b06a7..ee3c47fc 100644
--- a/src/tlog/tlog_main.ml
+++ b/src/tlog/tlog_main.ml
@@ -93,7 +93,6 @@ let strip_tlog filename =
 
 let _mark_tlog file_name node_name =
   assert (Tlog_map.extension_of file_name = ".tlog");
-  let section = Logger.Section.main in
   Logger.debug_f_ "_mark_tlog %s %s" file_name node_name >>= fun () ->
   _last_entry file_name >>= fun last ->
   match last with
diff --git a/src/tlog/tlog_map.ml b/src/tlog/tlog_map.ml
index 0059ca3c..9a39152c 100644
--- a/src/tlog/tlog_map.ml
+++ b/src/tlog/tlog_map.ml
@@ -336,6 +336,7 @@ module TlogMap = struct
 
 
   let _init tlog_dir tlx_dir node_id tlog_max_entries tlog_max_size ~check_marker ~check_sabotage =
+    let () = ignore (tlog_max_size, tlog_max_entries) in
     Logger.info_f_ "_init ~check_marker:%b ~check_sabotage:%b"
                    check_marker check_sabotage
     >>= fun () ->
@@ -424,7 +425,7 @@ module TlogMap = struct
         _init
           t.tlog_dir t.tlx_dir t.node_id ~check_marker:false ~check_sabotage:false
           t.tlog_max_entries t.tlog_max_size
-        >>= fun ((tlog_entries, tlog_size, tlog_number,i_to_tlog_number,should_roll), last,_) ->
+        >>= fun ((_tlog_entries, tlog_size, tlog_number,i_to_tlog_number,should_roll), last,_) ->
         t.tlog_size   <- tlog_size;
         t.tlog_number <- tlog_number;
         t.i_to_tlog_number <- i_to_tlog_number;
diff --git a/src/tlog/tlog_map_test.ml b/src/tlog/tlog_map_test.ml
index 9a1493f3..693efb6d 100644
--- a/src/tlog/tlog_map_test.ml
+++ b/src/tlog/tlog_map_test.ml
@@ -64,18 +64,18 @@ let test_find_start () =
     let itemss = List.map (fun item -> Printf.sprintf "{i:%Li;...}" item.i) items
     in Printf.sprintf "[%s]" (String.concat "; " itemss)
   in
-  let to_remove, to_keep =
+  let to_remove, _to_keep =
     _find_start 12000L i_to_tlog_number
   in
   let expected = List.tl i_to_tlog_number |> List.rev in
   OUnit.assert_equal expected to_remove ~printer ~msg:"test1";
-  let to_remove2, to_keep2 =
+  let to_remove2, _to_keep2 =
     _find_start 11002L i_to_tlog_number
   in
   let expected2 = i_to_tlog_number |> List.tl |> List.tl |> List.rev in
   OUnit.assert_equal expected2 to_remove2 ~printer ~msg:"test2";
 
-  let to_remove3, to_keep3 =
+  let to_remove3, _to_keep3 =
     _find_start 12002L i_to_tlog_number
   in
   OUnit.assert_equal expected to_remove3 ~printer ~msg:"test3";
diff --git a/src/tlog/tlogcollection_test.ml b/src/tlog/tlogcollection_test.ml
index 24164ec3..47f41b00 100644
--- a/src/tlog/tlogcollection_test.ml
+++ b/src/tlog/tlogcollection_test.ml
@@ -22,7 +22,6 @@ open Lwt
 open Update
 open Tlogcollection
 open Tlogcommon
-open Tlog_map
 
 let section = Logger.Section.main
 let node_id = "node_id"
diff --git a/src/tlog/update.ml b/src/tlog/update.ml
index df34cef0..95255e7a 100644
--- a/src/tlog/update.ml
+++ b/src/tlog/update.ml
@@ -304,7 +304,7 @@ module Update = struct
     let inner = function
     | Set(k,v)                -> ss k + ss v
     | Delete k                -> ss k
-    | MasterSet(m, f)         -> ss m + 8
+    | MasterSet(m, _f)        -> ss m + 8
     | TestAndSet(k,e,w)       -> ss k + sos e + sos w
     | Sequence us             -> seq_ssize us
     | Nop                     -> 0
diff --git a/src/tools/arakoon_logger.ml b/src/tools/arakoon_logger.ml
index 7c0a115b..af9383a1 100644
--- a/src/tools/arakoon_logger.ml
+++ b/src/tools/arakoon_logger.ml
@@ -71,6 +71,7 @@ let format_message
       ~(ts:Core.Time.t) ~seqnum
       ~section ~level ~lines
   =
+  let _s = section in
   Printf.sprintf
     "%s - %s - %i/%04i - %s - %i - %s - %s"
     (let open Unix in
diff --git a/src/tools/fsm.ml b/src/tools/fsm.ml
index 789e908e..b8d92885 100644
--- a/src/tools/fsm.ml
+++ b/src/tools/fsm.ml
@@ -46,7 +46,7 @@ let loop ?(trace=nop_trace)
   let rec _interprete key =
     if !stop
     then
-      Logger.debug Logger.Section.main "Stopping fsm"
+      Logger.debug ~section:Logger.Section.main "Stopping fsm"
     else
       begin
         let arg, product_type = lookup key in
diff --git a/src/tools/logger.ml b/src/tools/logger.ml
index 842f0e9a..941d205c 100644
--- a/src/tools/logger.ml
+++ b/src/tools/logger.ml
@@ -14,14 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 *)
 
-module Section =
-struct
-  let make = Lwt_log.Section.make
-  let set_level = Lwt_log.Section.set_level
-  let level = Lwt_log.Section.level
-  let main = Lwt_log.Section.main
-end
-
+module Section = Lwt_log.Section
 type level =
     Lwt_log.level =
   | Debug
@@ -31,20 +24,40 @@ type level =
   | Error
   | Fatal
 
+let debug        = Lwt_log.debug 
+let debug_       = Lwt_log.debug
+let debug_f      = Lwt_log.debug_f
+let debug_f_     = Lwt_log.debug_f
+let ign_debug_f_ = Lwt_log.ign_debug_f
+let ign_debug_   = Lwt_log.ign_debug_f
+
+let info         = Lwt_log.info
+let info_        = Lwt_log.info
+let info_f       = Lwt_log.info_f
+let info_f_      = Lwt_log.info_f
+let ign_info_f_  = Lwt_log.ign_info_f
+
+let warning_f_   = Lwt_log.warning_f
+let warning_     = Lwt_log.warning
+let ign_warning_ = Lwt_log.ign_warning
+
+let error_       = Lwt_log.error
+let error_f_     = Lwt_log.error_f
+
+let fatal_       = Lwt_log.fatal
+let fatal        = Lwt_log.fatal
+let fatal_f_     = Lwt_log.fatal_f
+let ign_fatal_   = Lwt_log.ign_fatal
+let ign_fatal_f  = Lwt_log.ign_fatal_f
+
 let log ?exn section level msg =
   Crash_logger.add_to_crash_log section level (Crash_logger.Message.Immediate1_exn (msg, exn));
   if level < Lwt_log.Section.level section
   then Lwt.return_unit
   else Lwt_log.log ?exn ~section ~level msg
 
-let ign_log ?exn section level msg =
-  Lwt.ignore_result (log ?exn section level msg)
-
 let log_ ?exn section level dmsg =
   Crash_logger.add_to_crash_log section level (Crash_logger.Message.Delayed1_exn (dmsg, exn));
   if level < Lwt_log.Section.level section
   then Lwt.return_unit
-  else Lwt_log.log ?exn ~section ~level (dmsg ())
-
-let ign_log_ ?exn section level dmsg =
-  Lwt.ignore_result (log_ ?exn section level dmsg)
+else Lwt_log.log ?exn ~section ~level (dmsg ())
diff --git a/src/tools/lwt_extra.ml b/src/tools/lwt_extra.ml
index 71e1a019..adb6e1d8 100644
--- a/src/tools/lwt_extra.ml
+++ b/src/tools/lwt_extra.ml
@@ -83,7 +83,7 @@ let read_txt ic =
     stream >>= fun () ->
   Lwt.return (Buffer.contents buf)
 
-let read_file fn = Lwt_io.with_file Lwt_io.Input fn read_txt
+let read_file fn = Lwt_io.with_file ~mode:Lwt_io.Input fn read_txt
 
 let run f =
   Lwt_main.run (
diff --git a/src/version/dune b/src/version/dune
new file mode 100644
index 00000000..025c06f8
--- /dev/null
+++ b/src/version/dune
@@ -0,0 +1,23 @@
+(executable
+ (name        mk_version)
+ (libraries   unix)
+ (modules     Mk_version)
+)
+
+(library
+ (name        arakoon_version)
+ (public_name arakoon_client.version)
+ (modules     arakoon_version)
+ (flags       :standard -opaque -annot)
+)
+
+(rule
+ (targets arakoon_version.ml)
+ (deps    version.force mk_version.exe)
+ (action  (progn (with-stdout-to %{targets} (run ./mk_version.exe)) (system "date >> version.force")))
+)
+
+(rule
+ (targets version.force)
+ (action  (write-file version.force "force"))
+)
diff --git a/src/version/mk_version.ml b/src/version/mk_version.ml
new file mode 100644
index 00000000..9d5d60d8
--- /dev/null
+++ b/src/version/mk_version.ml
@@ -0,0 +1,89 @@
+let run_cmd cmd () =
+  try
+    let ch = Unix.open_process_in cmd in
+    let line = input_line ch in
+    let () = close_in ch in
+    line
+  with | End_of_file -> "Not available"
+
+let run_cmd_multiline cmd () =
+  let rec input_lines acc ch =
+    match input_line ch with
+    | line -> input_lines (line :: acc) ch
+    | exception End_of_file -> List.rev acc
+  in
+
+  let ch = Unix.open_process_in cmd in
+  let lines = input_lines [] ch in
+  let () = close_in ch in
+  lines
+
+
+let list_dependencies () =
+  let versions =
+    run_cmd_multiline {| dune installed-libraries | sed 's/[ ]*[(]version: \(.*\)[)]/ \1/' |} ()
+    |> List.fold_left (fun acc line ->
+        let fields = String.split_on_char ' ' line in
+        let pkg = List.hd fields in
+        let version = String.concat " " (List.tl fields) in
+        Hashtbl.add acc pkg version;
+        acc
+      ) (Hashtbl.create 0)
+  in
+
+  run_cmd_multiline "cd ../../../..; dune external-lib-deps @install" ()
+  |> List.tl
+  |> List.map (fun line ->
+      let pkg = List.tl (String.split_on_char ' ' line) |> List.hd in
+      let version = match Hashtbl.find_opt versions pkg with
+        | Some v -> v
+        | None -> "<unknown>"
+      in
+      Printf.sprintf "%-20s\t%16s" pkg version
+    )
+  |> String.concat "\n"
+
+let time () =
+  let tm = Unix.gmtime (Unix.time()) in
+  Printf.sprintf "%02d/%02d/%04d %02d:%02d:%02d UTC"
+    (tm.tm_mday) (tm.tm_mon + 1) (tm.tm_year + 1900)
+    tm.tm_hour tm.tm_min tm.tm_sec
+
+let major_minor_patch_modifier () =
+  let tag_version = run_cmd "git describe --tags --dirty" in
+  let tv =
+    try tag_version ()
+    with _ -> ""
+  in
+  let (major, minor, patch, modif) =
+    try
+      Scanf.sscanf tv "%i.%i.%i-%s" (fun ma mi p dev -> (ma,mi,p, Some dev))
+    with _ ->
+      try
+        Scanf.sscanf tv "%i.%i.%i" (fun ma mi p -> (ma,mi,p,None))
+      with _ ->
+        (-1,-1,-1,None)
+  in
+  let modif_s =
+    match modif with
+    | None -> "None"
+    | Some s -> Printf.sprintf "Some %S" s
+  in
+  Printf.sprintf "(%d, %d, %d, %s)" major minor patch modif_s
+
+let stringify v = Printf.sprintf "%S" v
+let id = fun x -> x
+
+let fields = [ "git_revision", run_cmd "git describe --all --long --always --dirty", stringify
+             ; "compile_time", time, stringify
+             ; "machine", run_cmd "uname -mnrpio", stringify
+             ; "compiler_version", run_cmd "ocamlopt -version", stringify
+             ; "(major, minor, patch, modifier)", major_minor_patch_modifier, id
+             ; "dependencies", list_dependencies, stringify
+             ]
+
+let vals = List.map (fun (n, f, r) -> (n, f (), r)) fields
+let lines = List.map (fun (n, v, r) -> Printf.sprintf "let %s = %s" n (r v)) vals
+
+
+let () = Printf.printf "%s\n" (String.concat "\n" lines)
diff --git a/travis.sh b/travis.sh
index bb786a30..a70575f4 100755
--- a/travis.sh
+++ b/travis.sh
@@ -1,8 +1,9 @@
 #!/bin/bash -xue
+OUTPUT=temp_file.txt
 
 timeout_with_progress () (
     set +x
-    OUTPUT=`mktemp`
+    
     timeout "$@" > $OUTPUT 2>&1 &
     PID=$!
 
@@ -17,7 +18,7 @@ timeout_with_progress () (
     wait $PID
     RESULT=$?
 
-    tail -n256 $OUTPUT
+    tail -n512 $OUTPUT
 
     return $RESULT
 )
@@ -27,34 +28,14 @@ install () {
 
     date
 
-    if [ "$USE_CACHE" == "1" ];
-    then
-        if [[ -e ~/cache/image.tar.gz ]];
-        then docker load -i ~/cache/image.tar.gz; fi;
-    fi
-
     START_BUILD=$(date +%s.%N)
     echo $START_BUILD
 
     timeout_with_progress 9000 ./docker/run.sh ubuntu-16.04 clean
 
     END_BUILD=$(date +%s.%N)
-    echo $END_BUILD
-
-    if [ "$USE_CACHE" == "1" ];
-    then
-        DIFF=$(echo "$END_BUILD - $START_BUILD" | bc)
-        if [ $DIFF \> 5 ]
-        then
-            df -h
-            mkdir ~/cache || true
-            rm -f ~/cache/image.tar.gz
-            docker save -o ~/cache/image.tar.gz arakoon_ubuntu-16.04
-            ls -ahl ~/cache;
-        else
-            echo Building of container took only $DIFF seconds, not updating cache this time;
-        fi;
-    fi
+    echo "build stopped after $END_BUILD"
+
 }
 
 script () {
@@ -67,6 +48,13 @@ script () {
     date
 }
 
+
+after_failure () {
+    echo "Something went wrong"
+    url= `cat temp_file.txt | nc termbin.com 9999`
+    echo $url
+}
+
 case "${1-undefined}" in
     install)
         install
@@ -74,6 +62,9 @@ case "${1-undefined}" in
     script)
         script
         ;;
+    after_failure)
+        after_failure
+        ;;
     *)
         echo "Usage: $0 {install|script}"
         exit 1

From c197304f09bc038b6933b34d6839034799d6fb93 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Tue, 7 May 2019 17:16:16 +0200
Subject: [PATCH 371/373] clean $TEST_HOME/tmp

---
 test_it.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/test_it.py b/test_it.py
index 56fbfabb..1980e185 100644
--- a/test_it.py
+++ b/test_it.py
@@ -5,7 +5,8 @@
 
 root = os.environ.get('TEST_HOME')
 if root is None:
-    root = os.environ['HOME']
+    raise Exception("TEST_HOME in undefined")
+
 
 bin_dir = '%s/apps/arakoon/bin' % root
 bin = bin_dir +'/arakoon'
@@ -26,6 +27,8 @@ def prologue():
         else:
             print "not copying arakoon executable"
 
+    subprocess.call(['rm','-rf', '%s/tmp/' % root])
+
 
 prologue()
 

From 87319ce36e3cb5fd1f1a864c27486251ea669919 Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Wed, 8 May 2019 09:23:55 +0200
Subject: [PATCH 372/373] removed Jan Doms as author (upon his request)

---
 arakoon.opam | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/arakoon.opam b/arakoon.opam
index 8e0c16b7..3fad2d2b 100644
--- a/arakoon.opam
+++ b/arakoon.opam
@@ -1,8 +1,7 @@
 opam-version: "1.2"
 name: "arakoon"
 maintainer: "Romain Slootmaekers <romain.slootmaekers@openvstorage.com>"
-authors: ["Romain Slootmaekers <romain.slootmaekers@openvstorage.com>"
-          "Jan Doms <jan.doms@openvstorage.com>"]
+authors: ["Romain Slootmaekers <romain.slootmaekers@openvstorage.com>"]
 homepage: "https://github.com/openvstorage/arakoon"
 bug-reports: "https://github.com/openvstorage/arakoon/issues"
 license: "Apache 2.0"

From 44efb7dd51174b1bba2fda452542457c82df9c7d Mon Sep 17 00:00:00 2001
From: Romain Slootmaekers <romain.slootmaekers@openvstorage.com>
Date: Tue, 7 May 2019 17:16:16 +0200
Subject: [PATCH 373/373] smoother jenkins experience (no more out of memory)

---
 pylabs/test/server/shaky/test_db_defrag.py |  2 +-
 pylabs/test/server/system_tests_common.py  |  2 +-
 src/main/arakoon.ml                        |  2 +-
 src/node/dump_store.ml                     | 33 +++++++++++-----------
 test_it.py                                 | 22 +++++++++------
 5 files changed, 34 insertions(+), 27 deletions(-)

diff --git a/pylabs/test/server/shaky/test_db_defrag.py b/pylabs/test/server/shaky/test_db_defrag.py
index ec38e5d0..e21a1ee3 100644
--- a/pylabs/test/server/shaky/test_db_defrag.py
+++ b/pylabs/test/server/shaky/test_db_defrag.py
@@ -57,7 +57,7 @@ def test_db_defrag():
         v = "xxxxxxxxxx" * vs
         client.set(key_s,key_s)
         client.set(key_s,v)
-        print i, key_s, vs
+        #print i, key_s, vs
         if i == 65537:
             seed = 1
 
diff --git a/pylabs/test/server/system_tests_common.py b/pylabs/test/server/system_tests_common.py
index c0c6dc90..2b73b98a 100644
--- a/pylabs/test/server/system_tests_common.py
+++ b/pylabs/test/server/system_tests_common.py
@@ -1345,7 +1345,7 @@ def compare_files(fn0, fn1):
             while go_on:
                 f0_line = f0.readline()
                 f1_line = f1.readline()
-                logging.debug("l0:%s\nl1:%s\n", f0_line.strip(), f1_line.strip())
+                #logging.debug("l0:%s\nl1:%s\n", f0_line.strip(), f1_line.strip())
                 if f0_line <> f1_line :
                     raise Exception("%s and %s diff on line %i" % (fn0,fn1, count))
                 if len(f0_line) == 0:
diff --git a/src/main/arakoon.ml b/src/main/arakoon.ml
index f53cdf4d..9fe2bae0 100644
--- a/src/main/arakoon.ml
+++ b/src/main/arakoon.ml
@@ -219,7 +219,7 @@ let main () =
   and n_clients = ref 1
   and catchup_only = ref false
   and dump_values = ref false
-  and max_results = ref 1000
+  and max_results = ref 100
   and location = ref ""
   and left_cluster = ref ""
   and separator = ref ""
diff --git a/src/node/dump_store.ml b/src/node/dump_store.ml
index 96ab8e20..838c4869 100644
--- a/src/node/dump_store.ml
+++ b/src/node/dump_store.ml
@@ -58,25 +58,26 @@ let summary store =
   _dump_cluster_id store
 
 let dump_store ~dump_values filename =
-  let () = ignore dump_values in
   let t () =
     S.make_store ~lcnum:1024 ~ncnum:512 ~read_only:true filename >>= fun store ->
     summary store >>= fun () ->
-    let rec loop (first:string option) =
-      if first = None then Lwt.return_unit
-      else
-        let _n,kvs = S.range_entries store first false None true 100 in
-        Lwt_list.fold_left_s
-          (fun _last (k,v)  ->
-            let (kx:string) = Key.get k in
-            Lwt_io.printlf "%S\t%S" kx v >>= fun () ->
-            Lwt.return (Some kx)
-          )
-          None kvs
-        >>= fun lo ->
-        loop lo
-    in
-    loop (Some "")
+    if dump_values then
+      let rec loop (first:string option) =
+        if first = None then Lwt.return_unit
+        else
+          let _n,kvs = S.range_entries store first false None true 100 in
+          Lwt_list.fold_left_s
+            (fun _last (k,v)  ->
+              let (kx:string) = Key.get k in
+              Lwt_io.printlf "%S\t%S" kx v >>= fun () ->
+              Lwt.return (Some kx)
+            )
+            None kvs
+          >>= fun lo ->
+          loop lo
+      in
+      loop (Some "")
+    else Lwt.return ()
     >>= fun () ->
     S.close store ~sync:false ~flush:false
   in
diff --git a/test_it.py b/test_it.py
index 56fbfabb..c2efc9e5 100644
--- a/test_it.py
+++ b/test_it.py
@@ -5,7 +5,8 @@
 
 root = os.environ.get('TEST_HOME')
 if root is None:
-    root = os.environ['HOME']
+    raise Exception("TEST_HOME in undefined")
+
 
 bin_dir = '%s/apps/arakoon/bin' % root
 bin = bin_dir +'/arakoon'
@@ -17,14 +18,19 @@ def prologue():
         subprocess.call(['cp','./arakoon.native', bin])
         print "=> copying exec to: %s" % bin
     else:
-        version = subprocess.check_output(['./arakoon.native','--version'])
-        version2 = subprocess.check_output([bin,'--version'])
-        if version <> version2:
-            print version,version2
-            print "=> copying exec"
+        try:
+            version = subprocess.check_output(['./arakoon.native','--version'])
+            version2 = subprocess.check_output([bin,'--version'])
+            if version <> version2:
+                print version,version2
+                print "=> copying exec"
+                subprocess.call(['cp','./arakoon.native', bin])
+            else:
+                print "not copying arakoon executable"
+        except:
             subprocess.call(['cp','./arakoon.native', bin])
-        else:
-            print "not copying arakoon executable"
+
+    subprocess.call(['rm','-rf', '%s/tmp/' % root])
 
 
 prologue()