Tests and Interator Implementations for Rados

.github/workflows/ci.yml

@@ -273,6 +273,47 @@ jobs:
           sleep 10
           ./scripts/benchmark.sh --duration=0
           ./scripts/setup.sh stop
+  rados:
+    name: Rados
+    needs: dependencies
+    runs-on: ubuntu-24.04
+    timeout-minutes: 60
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+          show-progress: false
+
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt --yes --no-install-recommends install meson ninja-build pkgconf libglib2.0-dev libbson-dev libfabric-dev libgdbm-dev liblmdb-dev libsqlite3-dev libleveldb-dev libmongoc-dev libmariadb-dev librocksdb-dev libfuse3-dev libopen-trace-format-dev librados-dev
+
+      - name: Configure and build
+        env:
+          CC: clang
+        run: |
+          ./scripts/ci/build.sh debug-no-sanitize system
+
+      - name: Create configuration
+        run: |
+          . scripts/environment.sh
+          julea-config --user --object-servers="$(hostname)" --kv-servers="$(hostname)" --db-servers="$(hostname)" --object-backend=rados --object-path=/var/snap/microceph/current/conf/ceph.conf:julea --kv-backend=lmdb --kv-path=/tmp/julea/kv/lmdb --db-backend=sqlite --db-path=/tmp/julea/db/sqlite
+
+      - name: Microceph setup
+        run: sudo ./scripts/rados-util.sh bootstrap
+
+      - name: Tests
+        run: |
+          . scripts/environment.sh
+          ./scripts/setup.sh start
+          ./scripts/test.sh
+          # sudo apt --yes --no-install-recommends install valgrind
+          # valgrind --leak-check=full --track-origins=yes julea-test 
+          sleep 10
+          ./scripts/test.sh
+          ./scripts/setup.sh stop
   doxygen:
     name: Doxygen
     runs-on: ubuntu-24.04

backend/object/gio.c

@@ -43,14 +43,14 @@ typedef struct JBackendIterator JBackendIterator;
 
 struct JBackendObject
 {
-	gchar* path;
+	gchar* name;
 	GFileIOStream* stream;
 };
 
 typedef struct JBackendObject JBackendObject;
 
 static gboolean
-backend_create(gpointer backend_data, gchar const* namespace, gchar const* path, gpointer* backend_object)
+backend_create(gpointer backend_data, gchar const* namespace, gchar const* name, gpointer* backend_object)
 {
 	JBackendData* bd = backend_data;
 	JBackendObject* bo;
@@ -59,7 +59,7 @@ backend_create(gpointer backend_data, gchar const* namespace, gchar const* path,
 	GFileIOStream* stream;
 	gchar* full_path;
 
-	full_path = g_build_filename(bd->path, namespace, path, NULL);
+	full_path = g_build_filename(bd->path, namespace, name, NULL);
 	file = g_file_new_for_path(full_path);
 
 	j_trace_file_begin(full_path, J_TRACE_FILE_CREATE);
@@ -73,7 +73,7 @@ backend_create(gpointer backend_data, gchar const* namespace, gchar const* path,
 	j_trace_file_end(full_path, J_TRACE_FILE_CREATE, 0, 0);
 
 	bo = g_new(JBackendObject, 1);
-	bo->path = full_path;
+	bo->name = full_path;
 	bo->stream = stream;
 
 	*backend_object = bo;
@@ -84,23 +84,23 @@ backend_create(gpointer backend_data, gchar const* namespace, gchar const* path,
 }
 
 static gboolean
-backend_open(gpointer backend_data, gchar const* namespace, gchar const* path, gpointer* backend_object)
+backend_open(gpointer backend_data, gchar const* namespace, gchar const* name, gpointer* backend_object)
 {
 	JBackendData* bd = backend_data;
 	JBackendObject* bo;
 	GFile* file;
 	GFileIOStream* stream;
 	gchar* full_path;
 
-	full_path = g_build_filename(bd->path, namespace, path, NULL);
+	full_path = g_build_filename(bd->path, namespace, name, NULL);
 	file = g_file_new_for_path(full_path);
 
 	j_trace_file_begin(full_path, J_TRACE_FILE_OPEN);
 	stream = g_file_open_readwrite(file, NULL, NULL);
 	j_trace_file_end(full_path, J_TRACE_FILE_OPEN, 0, 0);
 
 	bo = g_new(JBackendObject, 1);
-	bo->path = full_path;
+	bo->name = full_path;
 	bo->stream = stream;
 
 	*backend_object = bo;
@@ -119,16 +119,16 @@ backend_delete(gpointer backend_data, gpointer backend_object)
 
 	(void)backend_data;
 
-	file = g_file_new_for_path(bo->path);
+	file = g_file_new_for_path(bo->name);
 
-	j_trace_file_begin(bo->path, J_TRACE_FILE_DELETE);
+	j_trace_file_begin(bo->name, J_TRACE_FILE_DELETE);
 	ret = g_file_delete(file, NULL, NULL);
-	j_trace_file_end(bo->path, J_TRACE_FILE_DELETE, 0, 0);
+	j_trace_file_end(bo->name, J_TRACE_FILE_DELETE, 0, 0);
 
 	g_object_unref(file);
 
 	g_object_unref(bo->stream);
-	g_free(bo->path);
+	g_free(bo->name);
 	g_free(bo);
 
 	return ret;
@@ -142,12 +142,12 @@ backend_close(gpointer backend_data, gpointer backend_object)
 
 	(void)backend_data;
 
-	j_trace_file_begin(bo->path, J_TRACE_FILE_CLOSE);
+	j_trace_file_begin(bo->name, J_TRACE_FILE_CLOSE);
 	ret = g_io_stream_close(G_IO_STREAM(bo->stream), NULL, NULL);
-	j_trace_file_end(bo->path, J_TRACE_FILE_CLOSE, 0, 0);
+	j_trace_file_end(bo->name, J_TRACE_FILE_CLOSE, 0, 0);
 
 	g_object_unref(bo->stream);
-	g_free(bo->path);
+	g_free(bo->name);
 	g_free(bo);
 
 	return ret;
@@ -166,11 +166,11 @@ backend_status(gpointer backend_data, gpointer backend_object, gint64* modificat
 		GFile* file;
 		GFileInfo* file_info;
 
-		file = g_file_new_for_path(bo->path);
+		file = g_file_new_for_path(bo->name);
 
-		j_trace_file_begin(bo->path, J_TRACE_FILE_STATUS);
+		j_trace_file_begin(bo->name, J_TRACE_FILE_STATUS);
 		file_info = g_file_query_info(file, G_FILE_ATTRIBUTE_STANDARD_SIZE "," G_FILE_ATTRIBUTE_TIME_MODIFIED "," G_FILE_ATTRIBUTE_TIME_MODIFIED_USEC, G_FILE_QUERY_INFO_NONE, NULL, NULL);
-		j_trace_file_end(bo->path, J_TRACE_FILE_STATUS, 0, 0);
+		j_trace_file_end(bo->name, J_TRACE_FILE_STATUS, 0, 0);
 
 		ret = (file_info != NULL);
 
@@ -217,9 +217,9 @@ backend_sync(gpointer backend_data, gpointer backend_object)
 
 	output = g_io_stream_get_output_stream(G_IO_STREAM(bo->stream));
 
-	j_trace_file_begin(bo->path, J_TRACE_FILE_SYNC);
+	j_trace_file_begin(bo->name, J_TRACE_FILE_SYNC);
 	ret = g_output_stream_flush(output, NULL, NULL);
-	j_trace_file_end(bo->path, J_TRACE_FILE_SYNC, 0, 0);
+	j_trace_file_end(bo->name, J_TRACE_FILE_SYNC, 0, 0);
 
 	return ret;
 }
@@ -237,13 +237,13 @@ backend_read(gpointer backend_data, gpointer backend_object, gpointer buffer, gu
 
 	input = g_io_stream_get_input_stream(G_IO_STREAM(bo->stream));
 
-	j_trace_file_begin(bo->path, J_TRACE_FILE_SEEK);
+	j_trace_file_begin(bo->name, J_TRACE_FILE_SEEK);
 	g_seekable_seek(G_SEEKABLE(bo->stream), offset, G_SEEK_SET, NULL, NULL);
-	j_trace_file_end(bo->path, J_TRACE_FILE_SEEK, 0, offset);
+	j_trace_file_end(bo->name, J_TRACE_FILE_SEEK, 0, offset);
 
-	j_trace_file_begin(bo->path, J_TRACE_FILE_READ);
+	j_trace_file_begin(bo->name, J_TRACE_FILE_READ);
 	ret = g_input_stream_read_all(input, buffer, length, &nbytes, NULL, NULL);
-	j_trace_file_end(bo->path, J_TRACE_FILE_READ, nbytes, offset);
+	j_trace_file_end(bo->name, J_TRACE_FILE_READ, nbytes, offset);
 
 	if (bytes_read != NULL)
 	{
@@ -266,13 +266,13 @@ backend_write(gpointer backend_data, gpointer backend_object, gconstpointer buff
 
 	output = g_io_stream_get_output_stream(G_IO_STREAM(bo->stream));
 
-	j_trace_file_begin(bo->path, J_TRACE_FILE_SEEK);
+	j_trace_file_begin(bo->name, J_TRACE_FILE_SEEK);
 	g_seekable_seek(G_SEEKABLE(bo->stream), offset, G_SEEK_SET, NULL, NULL);
-	j_trace_file_end(bo->path, J_TRACE_FILE_SEEK, 0, offset);
+	j_trace_file_end(bo->name, J_TRACE_FILE_SEEK, 0, offset);
 
-	j_trace_file_begin(bo->path, J_TRACE_FILE_WRITE);
+	j_trace_file_begin(bo->name, J_TRACE_FILE_WRITE);
 	ret = g_output_stream_write_all(output, buffer, length, &nbytes, NULL, NULL);
-	j_trace_file_end(bo->path, J_TRACE_FILE_WRITE, nbytes, offset);
+	j_trace_file_end(bo->name, J_TRACE_FILE_WRITE, nbytes, offset);
 
 	if (bytes_written != NULL)
 	{

backend/object/null.c

@@ -24,13 +24,13 @@
 #include <julea.h>
 
 static gboolean
-backend_create(gpointer backend_data, gchar const* namespace, gchar const* path, gpointer* backend_object)
+backend_create(gpointer backend_data, gchar const* namespace, gchar const* name, gpointer* backend_object)
 {
 	gchar* full_path;
 
 	(void)backend_data;
 
-	full_path = g_build_filename(namespace, path, NULL);
+	full_path = g_build_filename(namespace, name, NULL);
 
 	j_trace_file_begin(full_path, J_TRACE_FILE_CREATE);
 	j_trace_file_end(full_path, J_TRACE_FILE_CREATE, 0, 0);
@@ -41,13 +41,13 @@ backend_create(gpointer backend_data, gchar const* namespace, gchar const* path,
 }
 
 static gboolean
-backend_open(gpointer backend_data, gchar const* namespace, gchar const* path, gpointer* backend_object)
+backend_open(gpointer backend_data, gchar const* namespace, gchar const* name, gpointer* backend_object)
 {
 	gchar* full_path;
 
 	(void)backend_data;
 
-	full_path = g_build_filename(namespace, path, NULL);
+	full_path = g_build_filename(namespace, name, NULL);
 
 	j_trace_file_begin(full_path, J_TRACE_FILE_OPEN);
 	j_trace_file_end(full_path, J_TRACE_FILE_OPEN, 0, 0);