From bd232e3088fd561f6c7f7d5f39e6abce7aa5e79f Mon Sep 17 00:00:00 2001 From: jan Date: Fri, 7 Nov 2025 16:40:46 +0100 Subject: [PATCH 01/25] Feat: Added basic flake.nix setup. - It builds but does not run yet. -> Dependency versions are incorrect. -> Some feature flags are missing. -> Some headers/runtime deps/programs are missing. --- .envrc | 1 + flake.lock | 25 +++++++++++++++ flake.nix | 92 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 118 insertions(+) create mode 100644 .envrc create mode 100644 flake.lock create mode 100644 flake.nix diff --git a/.envrc b/.envrc new file mode 100644 index 000000000..3550a30f2 --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +use flake diff --git a/flake.lock b/flake.lock new file mode 100644 index 000000000..54f23e134 --- /dev/null +++ b/flake.lock @@ -0,0 +1,25 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1762233356, + "narHash": "sha256-cGS3lLTYusbEP/IJIWGgnkzIl+FA5xDvtiHyjalGr4k=", + "rev": "ca534a76c4afb2bdc07b681dbc11b453bab21af8", + "revCount": 812378, + "type": "tarball", + "url": "https://api.flakehub.com/f/pinned/NixOS/nixpkgs/0.2505.812378%2Brev-ca534a76c4afb2bdc07b681dbc11b453bab21af8/019a53af-7da6-765a-ae7b-594fa3a7097b/source.tar.gz" + }, + "original": { + "type": "tarball", + "url": "https://flakehub.com/f/NixOS/nixpkgs/0" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 000000000..f21e589b6 --- /dev/null +++ b/flake.nix @@ -0,0 +1,92 @@ +{ + description = "A Flexible Storage Framework for HPC"; + + # Flake inputs + inputs.nixpkgs.url = "https://flakehub.com/f/NixOS/nixpkgs/0"; # Stable Nixpkgs (use 0.1 for unstable) + + # Flake outputs + outputs = + { self, ... }@inputs: + let + # The systems supported for this flake's outputs + supportedSystems = [ + "x86_64-linux" # 64-bit Intel/AMD Linux + "aarch64-linux" # 64-bit ARM Linux + # "x86_64-darwin" # 64-bit Intel macOS - lets not support mac for now + # "aarch64-darwin" # 64-bit ARM macOS - same as above + ]; + + # Helper for providing system-specific attributes + forEachSupportedSystem = + f: + inputs.nixpkgs.lib.genAttrs supportedSystems ( + system: + f { + inherit system; + # Provides a system-specific, configured Nixpkgs + pkgs = import inputs.nixpkgs { + inherit system; + # Enable using unfree packages + config.allowUnfree = true; + }; + } + ); + in + { + # Development environments output by this flake + + # To activate the default environment: + # nix develop + # Or if you use direnv: + # direnv allow + devShells = forEachSupportedSystem ( + { pkgs, system }: + { + # Run `nix develop` to activate this environment or `direnv allow` if you have direnv installed + default = pkgs.mkShell { + # The Nix packages provided in the environment + packages = with pkgs; [ + # Add the flake's formatter to your project's environment + self.formatter.${system} + + # Build basics + gcc + ninja + meson + pkg-config + + # Other packages + glib + libbson + libfabric + + hdf5 + fuse3 + lmdb + sqlite + leveldb + mariadb + rocksdb + ]; + + # Set any environment variables for your development environment + env = { }; + + # Add any shell logic you want executed when the environment is activated + shellHook = "echo \"Hello where is ponsay\""; + }; + } + ); + + # Nix formatter + + # This applies the formatter that follows RFC 166, which defines a standard format: + # https://github.com/NixOS/rfcs/pull/166 + + # To format all Nix files: + # git ls-files -z '*.nix' | xargs -0 -r nix fmt + # To check formatting: + # git ls-files -z '*.nix' | xargs -0 -r nix develop --command nixfmt --check + formatter = forEachSupportedSystem ({ pkgs, ... }: pkgs.nixfmt-rfc-style); + }; +} From b2651f877b49d6f8b21d468407a5ddea7f12aa48 Mon Sep 17 00:00:00 2001 From: jan Date: Fri, 7 Nov 2025 18:23:47 +0100 Subject: [PATCH 02/25] Feat: Added .idea/ to ignored files. --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 3a3f02351..e087d0b43 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,6 @@ bld*/ # Visual Studio Code .vscode/ + +# Clion +.idea/ \ No newline at end of file From ee42d65d449d9a66b8b8e4287247f8898c53460b Mon Sep 17 00:00:00 2001 From: jan Date: Fri, 14 Nov 2025 09:36:05 +0100 Subject: [PATCH 03/25] Feat: Added overlay for libfabric. --- .gitignore | 3 +++ flake.nix | 2 ++ libfabric_overlay.nix | 28 ++++++++++++++++++++++++++++ 3 files changed, 33 insertions(+) create mode 100644 libfabric_overlay.nix diff --git a/.gitignore b/.gitignore index e087d0b43..c010f1d26 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,9 @@ # Dependencies dependencies/ +# Nix outputs +result + # Example example/hello-world diff --git a/flake.nix b/flake.nix index f21e589b6..7024f758e 100644 --- a/flake.nix +++ b/flake.nix @@ -28,6 +28,8 @@ inherit system; # Enable using unfree packages config.allowUnfree = true; + # Load the overlay. + overlays = [ (import ./libfabric_overlay.nix) ]; }; } ); diff --git a/libfabric_overlay.nix b/libfabric_overlay.nix new file mode 100644 index 000000000..ffd6e8fc1 --- /dev/null +++ b/libfabric_overlay.nix @@ -0,0 +1,28 @@ +# We currently install libfabric like this: dependencies="${dependencies} libfabric#fabrics=sockets,tcp,udp,verbs,rxd,rxm" +# All these fabric flags gets parsed in the libfabric python file linked below. +# They define a list of fabrics and, if it was passed as an argument, append the --enable-{fabric} flag in line 249. +# https://github.com/spack/spack-packages/blob/develop/repos/spack_repo/builtin/packages/libfabric/package.py +# Sadly the current libfabric nix file: +# https://github.com/NixOS/nixpkgs/blob/nixpkgs-25.05-darwin/pkgs/by-name/li/libfabric/package.nix +# does not support these flags. It only exposes flags for psm2 and opx. +# In the future it would probably be best to submit a merge request to nixpkgs. +# For now i will attempt to add them myself here. + +final : prev : { + libfabric = prev.libfabric.overrideAttrs (old: { + # We append our own flags to the existing ones. These should then get passed to the ./configure script. + configureFlags = old.configureFlags + ++ [ + "-enable-sockets" + "-enable-tcp" + "-enable-udp" + "-enable-verbs" + "-enable-rxd" + "-enable-rxm" + ]; + + # verbs also requires rdma so we add that dependency + buildInputs = old.buildInputs + ++ [ prev.rdma-core ]; + }); +} From fdd04506b2e7470650462ee53316f9075d29a8c2 Mon Sep 17 00:00:00 2001 From: jan Date: Fri, 14 Nov 2025 14:21:03 +0100 Subject: [PATCH 04/25] Feat: Added path exports. All tests pass! --- flake.nix | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 7024f758e..9b424930f 100644 --- a/flake.nix +++ b/flake.nix @@ -60,6 +60,7 @@ # Other packages glib libbson + # This uses the libfabric-overlay! libfabric hdf5 @@ -72,10 +73,21 @@ ]; # Set any environment variables for your development environment - env = { }; + env = { + }; # Add any shell logic you want executed when the environment is activated - shellHook = "echo \"Hello where is ponsay\""; + # TODO: Is LD_LIBRARY_PATH ok like this? The spack version had a lot more stuff in it. + # TODO: Shouldn't it be possible to have this in the env section? + shellHook = let buildDir = "${self}/bld"; in '' + echo Welcome to the JULEA nix shell: + + export PATH="$PATH:${buildDir}" + export LD_LIBRARY_PATH="${buildDir}" + export PKG_CONFIG_PATH="$PKG_CONFIG_PATH:${buildDir}/meson-uninstalled/" + export JULEA_BACKEND_PATH="${buildDir}" + export HDF5_PLUGIN_PATH="${buildDir}" + ''; }; } ); From def9b391195b8f21a03ce88dc1a2f7d86b79eac5 Mon Sep 17 00:00:00 2001 From: jan Date: Fri, 14 Nov 2025 14:21:23 +0100 Subject: [PATCH 05/25] Fix: Removed old comment. --- flake.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/flake.nix b/flake.nix index 9b424930f..8be995126 100644 --- a/flake.nix +++ b/flake.nix @@ -78,7 +78,6 @@ # Add any shell logic you want executed when the environment is activated # TODO: Is LD_LIBRARY_PATH ok like this? The spack version had a lot more stuff in it. - # TODO: Shouldn't it be possible to have this in the env section? shellHook = let buildDir = "${self}/bld"; in '' echo Welcome to the JULEA nix shell: From 394f8c9b29c26f127d6b2c5ed3aae59e996086dc Mon Sep 17 00:00:00 2001 From: jan Date: Fri, 14 Nov 2025 15:04:04 +0100 Subject: [PATCH 06/25] Fix: Removed hardended flag from meson as nix hardenes by default. --- flake.nix | 10 ++++++++-- meson.build | 1 - 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index 8be995126..2710aee67 100644 --- a/flake.nix +++ b/flake.nix @@ -12,8 +12,8 @@ supportedSystems = [ "x86_64-linux" # 64-bit Intel/AMD Linux "aarch64-linux" # 64-bit ARM Linux - # "x86_64-darwin" # 64-bit Intel macOS - lets not support mac for now - # "aarch64-darwin" # 64-bit ARM macOS - same as above + "x86_64-darwin" # 64-bit Intel macOS - lets not support mac for now + "aarch64-darwin" # 64-bit ARM macOS - same as above ]; # Helper for providing system-specific attributes @@ -72,6 +72,12 @@ rocksdb ]; + # Most hardening flags are enabled by default in nix. + # The only one usually disabled is pie. + # For more details check here: + # https://ryantm.github.io/nixpkgs/stdenv/stdenv/#sec-hardening-in-nixpkgs + hardeningEnable = ["pie"]; + # Set any environment variables for your development environment env = { }; diff --git a/meson.build b/meson.build index fc03b1a2d..db34f0a57 100644 --- a/meson.build +++ b/meson.build @@ -30,7 +30,6 @@ pkg_config = import('pkgconfig') cc_args = cc.get_supported_arguments([ '-fno-omit-frame-pointer', - '-fhardened', ]) if get_option('debug') From a49cebdc40cc387a3fde8b3cc050966818f40cd6 Mon Sep 17 00:00:00 2001 From: jan Date: Fri, 14 Nov 2025 15:27:27 +0100 Subject: [PATCH 07/25] Fix: Has header "gdbm.h" : NO. --- flake.nix | 3 +++ scripts/environment.sh | 2 ++ 2 files changed, 5 insertions(+) diff --git a/flake.nix b/flake.nix index 2710aee67..77ebe406b 100644 --- a/flake.nix +++ b/flake.nix @@ -67,9 +67,12 @@ fuse3 lmdb sqlite + leveldb mariadb rocksdb + + gdbm ]; # Most hardening flags are enabled by default in nix. diff --git a/scripts/environment.sh b/scripts/environment.sh index 10e3b75ce..f78e8b292 100755 --- a/scripts/environment.sh +++ b/scripts/environment.sh @@ -14,6 +14,8 @@ # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see . +# TODO: Remove all the unneeded .sh scripts like enviornment.sh, setup.sh, spack etc + SELF_ZERO="$0" # shellcheck disable=SC2169,SC3028,SC3054 test -n "${BASH_VERSION}" && SELF_ZERO="${BASH_SOURCE[0]}" From 1d53f1e27e6c26262a351d27c3087358e65429e7 Mon Sep 17 00:00:00 2001 From: jan Date: Fri, 14 Nov 2025 15:46:56 +0100 Subject: [PATCH 08/25] Fix: Added rados by including ceph. --- flake.nix | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 77ebe406b..8bd82ea4e 100644 --- a/flake.nix +++ b/flake.nix @@ -68,11 +68,18 @@ lmdb sqlite + gdbm leveldb - mariadb + # TODO: This is not the correct dependency. + # We need a connector from here: https://downloads.mariadb.com/Connectors/c + # As far as i can tell no nixpkgs exists as of now. + # mariadb + ceph + + # TODO: I think we need otf1 instead - but not nixpkgs exists + # otf2 rocksdb - gdbm ]; # Most hardening flags are enabled by default in nix. From f221e591ac3c1b0d371835da93a1f95b6b9fa168 Mon Sep 17 00:00:00 2001 From: jan Date: Fri, 14 Nov 2025 16:56:33 +0100 Subject: [PATCH 09/25] Refactor: Removed now unneeded .sh scripts. --- flake.nix | 2 + scripts/benchmark.sh | 12 -- scripts/environment.sh | 80 ----------- scripts/install-dependencies.sh | 41 ------ scripts/setup.sh | 8 -- scripts/spack | 240 -------------------------------- scripts/test.sh | 13 -- 7 files changed, 2 insertions(+), 394 deletions(-) delete mode 100755 scripts/environment.sh delete mode 100755 scripts/install-dependencies.sh delete mode 100644 scripts/spack diff --git a/flake.nix b/flake.nix index 8bd82ea4e..d523d6b46 100644 --- a/flake.nix +++ b/flake.nix @@ -74,6 +74,8 @@ # We need a connector from here: https://downloads.mariadb.com/Connectors/c # As far as i can tell no nixpkgs exists as of now. # mariadb + + # TODO: Is this the correct way to include rados? ceph # TODO: I think we need otf1 instead - but not nixpkgs exists diff --git a/scripts/benchmark.sh b/scripts/benchmark.sh index c35de58ae..77daa05ec 100755 --- a/scripts/benchmark.sh +++ b/scripts/benchmark.sh @@ -33,11 +33,6 @@ usage () exit 1 } -set_path -set_library_path -set_backend_path -set_hdf_path - run_benchmark () { local ret @@ -49,11 +44,4 @@ run_benchmark () return ${ret} } -if test -z "${JULEA_SPACK_DIR}" -then - JULEA_SPACK_DIR="$(get_directory "${SELF_DIR}/..")/dependencies" -fi - -spack_load_dependencies - run_benchmark "$@" diff --git a/scripts/environment.sh b/scripts/environment.sh deleted file mode 100755 index f78e8b292..000000000 --- a/scripts/environment.sh +++ /dev/null @@ -1,80 +0,0 @@ -# JULEA - Flexible storage framework -# Copyright (C) 2017-2024 Michael Kuhn -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program. If not, see . - -# TODO: Remove all the unneeded .sh scripts like enviornment.sh, setup.sh, spack etc - -SELF_ZERO="$0" -# shellcheck disable=SC2169,SC3028,SC3054 -test -n "${BASH_VERSION}" && SELF_ZERO="${BASH_SOURCE[0]}" - -SELF_PATH="$(readlink --canonicalize-existing -- "${SELF_ZERO}")" -SELF_DIR="${SELF_PATH%/*}" -# shellcheck disable=SC2034 -SELF_BASE="${SELF_PATH##*/}" - -# shellcheck source=scripts/common -. "${SELF_DIR}/common" -# shellcheck source=scripts/spack -. "${SELF_DIR}/spack" - -JULEA_ENVIRONMENT_SOURCED=1 - -# See https://stackoverflow.com/a/28776166 -if test -n "${BASH_VERSION}" -then - (return 0 2>/dev/null) || JULEA_ENVIRONMENT_SOURCED=0 -elif test -n "${ZSH_EVAL_CONTEXT}" -then - case "${ZSH_EVAL_CONTEXT}" in - *:file) - ;; - *) - JULEA_ENVIRONMENT_SOURCED=0 - ;; - esac -fi - -if test "${JULEA_ENVIRONMENT_SOURCED}" -eq 0 -then - printf 'Warning: This script should be sourced using ". %s", otherwise changes to the environment will not persist.\n' "${SELF_PATH}" >&2 -fi - -JULEA_ENVIRONMENT=1 - -set_path -set_library_path -set_pkg_config_path -set_backend_path -set_hdf_path - -if test -z "${JULEA_SPACK_DIR}" -then - JULEA_SPACK_DIR="$(get_directory "${SELF_DIR}/..")/dependencies" -fi - -spack_load_dependencies - -# Do not filter out paths contained in CPATH and LIBRARY_PATH. -PKG_CONFIG_ALLOW_SYSTEM_CFLAGS=1 -PKG_CONFIG_ALLOW_SYSTEM_LIBS=1 - -export PKG_CONFIG_ALLOW_SYSTEM_CFLAGS -export PKG_CONFIG_ALLOW_SYSTEM_LIBS - -# FIXME The Spack pkg-config does not search in global directories and Meson does not provide a way to override this -PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:/usr/lib64/pkgconfig:/usr/lib/x86_64-linux-gnu/pkgconfig:/usr/lib/pkgconfig:/usr/share/pkgconfig" - -export PKG_CONFIG_PATH diff --git a/scripts/install-dependencies.sh b/scripts/install-dependencies.sh deleted file mode 100755 index 46a4ef24a..000000000 --- a/scripts/install-dependencies.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -# JULEA - Flexible storage framework -# Copyright (C) 2017-2024 Michael Kuhn -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program. If not, see . - -set -e - -SELF_PATH="$(readlink --canonicalize-existing -- "$0")" -SELF_DIR="${SELF_PATH%/*}" -SELF_BASE="${SELF_PATH##*/}" - -# shellcheck source=scripts/common -. "${SELF_DIR}/common" -# shellcheck source=scripts/spack -. "${SELF_DIR}/spack" - -usage () -{ - echo "Usage: ${SELF_BASE}" - exit 1 -} - -if test -z "${JULEA_SPACK_DIR}" -then - JULEA_SPACK_DIR="$(get_directory "${SELF_DIR}/..")/dependencies" -fi - -spack_install_dependencies diff --git a/scripts/setup.sh b/scripts/setup.sh index a041a458a..1d2617c82 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -27,8 +27,6 @@ SELF_BASE="${SELF_PATH##*/}" . "${SELF_DIR}/common" # shellcheck source=scripts/setup . "${SELF_DIR}/setup" -# shellcheck source=scripts/spack -. "${SELF_DIR}/spack" usage () { @@ -67,17 +65,11 @@ MODE="$1" #export G_MESSAGES_DEBUG=JULEA -set_path -set_library_path -set_backend_path - if test -z "${JULEA_SPACK_DIR}" then JULEA_SPACK_DIR="$(get_directory "${SELF_DIR}/..")/dependencies" fi -spack_load_dependencies - setup_init case "${MODE}" in diff --git a/scripts/spack b/scripts/spack deleted file mode 100644 index 56e375b79..000000000 --- a/scripts/spack +++ /dev/null @@ -1,240 +0,0 @@ -# JULEA - Flexible storage framework -# Copyright (C) 2017-2024 Michael Kuhn -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program. If not, see . - -spack_set_env () -{ - test -n "${JULEA_SPACK_DIR}" || return 1 - - export SPACK_DISABLE_LOCAL_CONFIG=true - # FIXME maybe put into /tmp in CI - export SPACK_USER_CACHE_PATH="${JULEA_SPACK_DIR}/cache" -} - -spack_clone () -{ - local spack_packages_commit - - spack_packages_commit='9ca712a716ef98dcc39e3a76b5e5b6b1aa89746e' - - test -n "${JULEA_SPACK_DIR}" || return 1 - - if test ! -d "${JULEA_SPACK_DIR}" - then - git clone --config feature.manyFiles=true https://github.com/spack/spack.git "${JULEA_SPACK_DIR}" - fi - - test -d "${JULEA_SPACK_DIR}" || return 1 - - if test ! -d "${JULEA_SPACK_DIR}/packages" - then - git clone --config feature.manyFiles=true https://github.com/spack/spack-packages.git "${JULEA_SPACK_DIR}/packages" - fi - - test -d "${JULEA_SPACK_DIR}/packages" || return 1 - - spack_set_env - - ( - cd "${JULEA_SPACK_DIR}" || return 1 - - git fetch - git checkout releases/v1.0 - git reset --hard origin/releases/v1.0 - ) - - ( - cd "${JULEA_SPACK_DIR}/packages" || return 1 - - git fetch - git checkout develop - git reset --hard "${spack_packages_commit}" - ) - - spack_cmd repo set --destination "${JULEA_SPACK_DIR}/packages" builtin - - return 0 -} - -spack_init () -{ - local spack_env - - test -n "${JULEA_SPACK_DIR}" || return 1 - test -d "${JULEA_SPACK_DIR}" || return 1 - - spack_set_env - - spack_env="${JULEA_SPACK_DIR}/share/spack/setup-env.sh" - - test -f "${spack_env}" || return 1 - - # shellcheck source=/dev/null - . "${spack_env}" - - return 0 -} - -spack_cmd () -{ - test -n "${JULEA_SPACK_DIR}" || return 1 - test -d "${JULEA_SPACK_DIR}" || return 1 - - ( - cd "${JULEA_SPACK_DIR}" || return 1 - - ./bin/spack "$@" - ) -} - -spack_load () -{ - local pkg_loaded - local spack_pkg - - pkg_loaded='YES' - spack_pkg="$1" - - test -n "${spack_pkg}" || return 1 - - spack_pkg="$(printf '%s' "${spack_pkg}" | sed 's/#/ /g')" - - spack load "${spack_pkg}" || pkg_loaded='NO' - - printf 'Dependency "%s" loaded: %s\n' "${spack_pkg}" "${pkg_loaded}" -} - -spack_install () -{ - local spack_pkg - - spack_pkg="$1" - - test -n "${spack_pkg}" || return 1 - - spack_pkg="$(printf '%s' "${spack_pkg}" | sed 's/#/ /g')" - - printf 'Installing "%s"\n' "${spack_pkg}" - # FIXME ignore errors? - spack_cmd install "${spack_pkg}" -} - -spack_get_dependencies () -{ - local dependencies - - dependencies='' - - # Required for building JULEA - dependencies="${dependencies} meson" - dependencies="${dependencies} pkgconf" - - # Mandatory dependencies - dependencies="${dependencies} glib" - dependencies="${dependencies} libbson" - dependencies="${dependencies} libfabric#fabrics=sockets,tcp,udp,verbs,rxd,rxm" - - # Recommended dependencies - dependencies="${dependencies} hdf5~mpi@1.14" - dependencies="${dependencies} libfuse~utils" - dependencies="${dependencies} lmdb" - dependencies="${dependencies} sqlite" - - # Optional dependencies - # FIXME https://github.com/spack/spack-packages/pull/1109 - dependencies="${dependencies} gdbm@1.23" - dependencies="${dependencies} leveldb" - dependencies="${dependencies} mariadb-c-client" - dependencies="${dependencies} mongo-c-driver" - dependencies="${dependencies} otf" - dependencies="${dependencies} rocksdb~static" - - if test -n "${CI}" - then - dependencies="${dependencies} py-gcovr" - fi - - #dependencies="${dependencies} enzo@main" - #dependencies="${dependencies} mpi" - - printf '%s' "${dependencies}" -} - -spack_load_dependencies () -{ - if test -z "${JULEA_SPACK_DEPENDENCIES_LOADED}" - then - if spack_init - then - for dependency in $(spack_get_dependencies) - do - spack_load "${dependency}" - done - - # FIXME this could be more clever by storing the actual dependencies loaded - JULEA_SPACK_DEPENDENCIES_LOADED=1 - - export JULEA_SPACK_DEPENDENCIES_LOADED - fi - else - printf 'Dependencies have already been loaded, skipping.\n' >&2 - fi -} - -spack_install_dependencies () -{ - # FIXME does not fail if Git is not installed - if spack_clone - then - spack_cmd config --scope site add 'concretizer:reuse:false' - spack_cmd config --scope site add 'config:connect_timeout:60' - # FIXME We currently need this for JULEA to find all dependencies - spack_cmd config --scope site add 'modules:prefix_inspections:lib:[LD_LIBRARY_PATH]' - spack_cmd config --scope site add 'modules:prefix_inspections:lib64:[LD_LIBRARY_PATH]' - - if test -n "${CI}" - then - spack_cmd config --scope site add 'packages:all:target:[x86_64]' - spack_cmd compiler find - spack_cmd compiler list - - if test -n "${JULEA_SPACK_COMPILER}" - then - # This should fail if Spack does not recognize the compiler - spack_cmd compiler info "${JULEA_SPACK_COMPILER}" - fi - fi - - if test -n "${JULEA_SPACK_COMPILER}" - then - spack_cmd config --scope site add "packages:all:require:\"%${JULEA_SPACK_COMPILER}\"" - fi - - spack_cmd mark --implicit --all - - for dependency in $(spack_get_dependencies) - do - spack_install "${dependency}" - done - - spack_cmd gc --yes-to-all - - if test -n "${CI}" - then - spack_cmd clean --downloads - spack_cmd find - fi - fi -} diff --git a/scripts/test.sh b/scripts/test.sh index d300dac68..b3ad09eac 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -24,8 +24,6 @@ SELF_BASE="${SELF_PATH##*/}" # shellcheck source=scripts/common . "${SELF_DIR}/common" -# shellcheck source=scripts/spack -. "${SELF_DIR}/spack" usage () { @@ -33,11 +31,6 @@ usage () exit 1 } -set_path -set_library_path -set_backend_path -set_hdf_path - run_test () { local build_dir @@ -58,11 +51,5 @@ run_test () return ${ret} } -if test -z "${JULEA_SPACK_DIR}" -then - JULEA_SPACK_DIR="$(get_directory "${SELF_DIR}/..")/dependencies" -fi - -spack_load_dependencies run_test "$@" From 8f053a314a5cefb382ef8dbbbdda6c5634ec4864 Mon Sep 17 00:00:00 2001 From: jan Date: Fri, 14 Nov 2025 17:17:08 +0100 Subject: [PATCH 10/25] Fix: Using pwd/bld instead of self/bld. --- flake.nix | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/flake.nix b/flake.nix index d523d6b46..76fa25812 100644 --- a/flake.nix +++ b/flake.nix @@ -96,14 +96,15 @@ # Add any shell logic you want executed when the environment is activated # TODO: Is LD_LIBRARY_PATH ok like this? The spack version had a lot more stuff in it. - shellHook = let buildDir = "${self}/bld"; in '' + shellHook = '' echo Welcome to the JULEA nix shell: - export PATH="$PATH:${buildDir}" - export LD_LIBRARY_PATH="${buildDir}" - export PKG_CONFIG_PATH="$PKG_CONFIG_PATH:${buildDir}/meson-uninstalled/" - export JULEA_BACKEND_PATH="${buildDir}" - export HDF5_PLUGIN_PATH="${buildDir}" + export BUILD_DIR="$PWD/bld" + export PATH="$PATH:"$BUILD_DIR + export LD_LIBRARY_PATH=""$BUILD_DIR + export PKG_CONFIG_PATH="$PKG_CONFIG_PATH:/$BUILD_DIRmeson-uninstalled/" + export JULEA_BACKEND_PATH=""$BUILD_DIR + export HDF5_PLUGIN_PATH=""$BUILD_DIR ''; }; } From 0eb702b43fa0f914b5657a7177c4f609b443d580 Mon Sep 17 00:00:00 2001 From: jan Date: Fri, 14 Nov 2025 18:12:38 +0100 Subject: [PATCH 11/25] Feat: Switched to flake-utils. --- flake.lock | 34 +++++++++ flake.nix | 198 ++++++++++++++++++++++------------------------------- 2 files changed, 114 insertions(+), 118 deletions(-) diff --git a/flake.lock b/flake.lock index 54f23e134..c1655e4ba 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,23 @@ { "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, "nixpkgs": { "locked": { "lastModified": 1762233356, @@ -16,8 +34,24 @@ }, "root": { "inputs": { + "flake-utils": "flake-utils", "nixpkgs": "nixpkgs" } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } } }, "root": "root", diff --git a/flake.nix b/flake.nix index 76fa25812..8112c45c0 100644 --- a/flake.nix +++ b/flake.nix @@ -1,124 +1,86 @@ { - description = "A Flexible Storage Framework for HPC"; - - # Flake inputs - inputs.nixpkgs.url = "https://flakehub.com/f/NixOS/nixpkgs/0"; # Stable Nixpkgs (use 0.1 for unstable) - - # Flake outputs - outputs = - { self, ... }@inputs: - let - # The systems supported for this flake's outputs - supportedSystems = [ - "x86_64-linux" # 64-bit Intel/AMD Linux - "aarch64-linux" # 64-bit ARM Linux - "x86_64-darwin" # 64-bit Intel macOS - lets not support mac for now - "aarch64-darwin" # 64-bit ARM macOS - same as above - ]; - - # Helper for providing system-specific attributes - forEachSupportedSystem = - f: - inputs.nixpkgs.lib.genAttrs supportedSystems ( - system: - f { - inherit system; - # Provides a system-specific, configured Nixpkgs - pkgs = import inputs.nixpkgs { - inherit system; - # Enable using unfree packages - config.allowUnfree = true; - # Load the overlay. - overlays = [ (import ./libfabric_overlay.nix) ]; - }; - } - ); - in - { - # Development environments output by this flake - - # To activate the default environment: - # nix develop - # Or if you use direnv: - # direnv allow - devShells = forEachSupportedSystem ( - { pkgs, system }: - { - # Run `nix develop` to activate this environment or `direnv allow` if you have direnv installed - default = pkgs.mkShell { - # The Nix packages provided in the environment - packages = with pkgs; [ - # Add the flake's formatter to your project's environment - self.formatter.${system} - - # Build basics - gcc - ninja - meson - pkg-config - - # Other packages - glib - libbson - # This uses the libfabric-overlay! - libfabric - - hdf5 - fuse3 - lmdb - sqlite - - gdbm - leveldb - # TODO: This is not the correct dependency. - # We need a connector from here: https://downloads.mariadb.com/Connectors/c - # As far as i can tell no nixpkgs exists as of now. - # mariadb - - # TODO: Is this the correct way to include rados? - ceph - - # TODO: I think we need otf1 instead - but not nixpkgs exists - # otf2 - rocksdb - - ]; - - # Most hardening flags are enabled by default in nix. - # The only one usually disabled is pie. - # For more details check here: - # https://ryantm.github.io/nixpkgs/stdenv/stdenv/#sec-hardening-in-nixpkgs - hardeningEnable = ["pie"]; + description = "A Flexible Storage Framework for HPC"; + + # Flake inputs + inputs = { + nixpkgs.url = "https://flakehub.com/f/NixOS/nixpkgs/0"; + flake-utils.url = "github:numtide/flake-utils"; + }; + + # Flake outputs + outputs = { self, nixpkgs, flake-utils }: + flake-utils.lib.eachDefaultSystem (system: + let + # Import nixpkgs for the specific system with our config + pkgs = import nixpkgs { + inherit system; + config.allowUnfree = true; + overlays = [ (import ./libfabric_overlay.nix) ]; + }; + in + { + # Development environments output by this flake + devShells.default = pkgs.mkShell { + # The Nix packages provided in the environment + packages = with pkgs; [ + # Add the flake's formatter to your project's environment + self.formatter.${system} + + # Build basics + gcc + ninja + meson + pkg-config + + # Other packages + glib + libbson + # This uses the libfabric-overlay! + libfabric + + hdf5 + fuse3 + lmdb + sqlite + + gdbm + leveldb + # TODO: This is not the correct dependency. + # We need a connector from here: https://downloads.mariadb.com/Connectors/c + # As far as i can tell no nixpkgs exists as of now. + # mariadb + + # TODO: Is this the correct way to include rados? + ceph + + # TODO: I think we need otf1 instead - but not nixpkgs exists + # otf2 + rocksdb + ]; + + # Most hardening flags are enabled by default in nix. + # The only one usually disabled is pie. + # For more details check here: + # https://ryantm.github.io/nixpkgs/stdenv/stdenv/#sec-hardening-in-nixpkgs + hardeningEnable = [ "pie" ]; # Set any environment variables for your development environment env = { }; - # Add any shell logic you want executed when the environment is activated - # TODO: Is LD_LIBRARY_PATH ok like this? The spack version had a lot more stuff in it. - shellHook = '' - echo Welcome to the JULEA nix shell: - - export BUILD_DIR="$PWD/bld" - export PATH="$PATH:"$BUILD_DIR - export LD_LIBRARY_PATH=""$BUILD_DIR - export PKG_CONFIG_PATH="$PKG_CONFIG_PATH:/$BUILD_DIRmeson-uninstalled/" - export JULEA_BACKEND_PATH=""$BUILD_DIR - export HDF5_PLUGIN_PATH=""$BUILD_DIR - ''; - }; - } - ); - - # Nix formatter - - # This applies the formatter that follows RFC 166, which defines a standard format: - # https://github.com/NixOS/rfcs/pull/166 - - # To format all Nix files: - # git ls-files -z '*.nix' | xargs -0 -r nix fmt - # To check formatting: - # git ls-files -z '*.nix' | xargs -0 -r nix develop --command nixfmt --check - formatter = forEachSupportedSystem ({ pkgs, ... }: pkgs.nixfmt-rfc-style); - }; -} + # Add any shell logic you want executed when the environment is activated + # TODO: Is LD_LIBRARY_PATH ok like this? The spack version had a lot more stuff in it. + shellHook = '' + echo "Welcome to the JULEA nix shell:" + + export BUILD_DIR="$PWD/bld" + export PATH="$PATH:$BUILD_DIR" + export LD_LIBRARY_PATH="$BUILD_DIR" + export PKG_CONFIG_PATH="$PKG_CONFIG_PATH:$BUILD_DIR/meson-uninstalled/" + export JULEA_BACKEND_PATH="$BUILD_DIR" + export HDF5_PLUGIN_PATH="$BUILD_DIR" + ''; + }; + } + ); + } From 6d938688c33d65b4a1b1905a76428676d8d048b0 Mon Sep 17 00:00:00 2001 From: jan Date: Fri, 14 Nov 2025 18:27:47 +0100 Subject: [PATCH 12/25] Refactoring: Moved dependencies to a separate list. --- flake.nix | 71 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 36 insertions(+), 35 deletions(-) diff --git a/flake.nix b/flake.nix index 8112c45c0..fc08b7a40 100644 --- a/flake.nix +++ b/flake.nix @@ -9,6 +9,41 @@ # Flake outputs outputs = { self, nixpkgs, flake-utils }: + let + # This is a function the returns all the needed dependencies given the system pkgs. + dependencies = pkgs: with pkgs; [ + # Build basics + gcc + ninja + meson + pkg-config + + # Other packages + glib + libbson + # This uses the libfabric-overlay! + libfabric + + hdf5 + fuse3 + lmdb + sqlite + + gdbm + leveldb + # TODO: This is not the correct dependency. + # We need a connector from here: https://downloads.mariadb.com/Connectors/c + # As far as i can tell no nixpkgs exists as of now. + # mariadb + + # TODO: Is this the correct way to include rados? + ceph + + # TODO: I think we need otf1 instead - but no nixpkgs exists + # otf2 + rocksdb + ]; + in flake-utils.lib.eachDefaultSystem (system: let # Import nixpkgs for the specific system with our config @@ -22,41 +57,7 @@ # Development environments output by this flake devShells.default = pkgs.mkShell { # The Nix packages provided in the environment - packages = with pkgs; [ - # Add the flake's formatter to your project's environment - self.formatter.${system} - - # Build basics - gcc - ninja - meson - pkg-config - - # Other packages - glib - libbson - # This uses the libfabric-overlay! - libfabric - - hdf5 - fuse3 - lmdb - sqlite - - gdbm - leveldb - # TODO: This is not the correct dependency. - # We need a connector from here: https://downloads.mariadb.com/Connectors/c - # As far as i can tell no nixpkgs exists as of now. - # mariadb - - # TODO: Is this the correct way to include rados? - ceph - - # TODO: I think we need otf1 instead - but not nixpkgs exists - # otf2 - rocksdb - ]; + packages = dependencies pkgs; # Most hardening flags are enabled by default in nix. # The only one usually disabled is pie. From 0d321982f7378a72c9d1ff5c8f199f41ab1a61eb Mon Sep 17 00:00:00 2001 From: jan Date: Thu, 20 Nov 2025 13:15:16 +0100 Subject: [PATCH 13/25] Feat: Added `nix build` support. --- flake.nix | 170 ++++++++++++++++++++++++------------------- scripts/benchmark.sh | 2 - 2 files changed, 95 insertions(+), 77 deletions(-) diff --git a/flake.nix b/flake.nix index fc08b7a40..ef32bc690 100644 --- a/flake.nix +++ b/flake.nix @@ -1,87 +1,107 @@ { - description = "A Flexible Storage Framework for HPC"; + description = "A Flexible Storage Framework for HPC"; - # Flake inputs - inputs = { - nixpkgs.url = "https://flakehub.com/f/NixOS/nixpkgs/0"; - flake-utils.url = "github:numtide/flake-utils"; - }; + # Flake inputs + inputs = { + nixpkgs.url = "https://flakehub.com/f/NixOS/nixpkgs/0"; + flake-utils.url = "github:numtide/flake-utils"; + }; - # Flake outputs - outputs = { self, nixpkgs, flake-utils }: - let - # This is a function the returns all the needed dependencies given the system pkgs. - dependencies = pkgs: with pkgs; [ - # Build basics - gcc - ninja - meson - pkg-config + # Flake outputs + # Useful documentation: https://saylesss88.github.io/flakes/flake_outputs_4.2.html + outputs = { + self, + nixpkgs, + flake-utils, + }: let + # This is a function the returns all the needed dependencies given the system pkgs. + dependencies = pkgs: + with pkgs; [ + # Build basics + gcc + ninja + meson + pkg-config - # Other packages - glib - libbson - # This uses the libfabric-overlay! - libfabric + # Other packages + glib + libbson + # This uses the libfabric-overlay! + libfabric - hdf5 - fuse3 - lmdb - sqlite + hdf5 + fuse3 + lmdb + sqlite - gdbm - leveldb - # TODO: This is not the correct dependency. - # We need a connector from here: https://downloads.mariadb.com/Connectors/c - # As far as i can tell no nixpkgs exists as of now. - # mariadb + gdbm + leveldb + # TODO: This is not the correct dependency. + # We need a connector from here: https://downloads.mariadb.com/Connectors/c + # As far as i can tell no nixpkgs exists as of now. + # mariadb - # TODO: Is this the correct way to include rados? - ceph + # TODO: Is this the correct way to include rados? + ceph - # TODO: I think we need otf1 instead - but no nixpkgs exists - # otf2 - rocksdb - ]; - in - flake-utils.lib.eachDefaultSystem (system: - let - # Import nixpkgs for the specific system with our config - pkgs = import nixpkgs { - inherit system; - config.allowUnfree = true; - overlays = [ (import ./libfabric_overlay.nix) ]; - }; - in - { - # Development environments output by this flake - devShells.default = pkgs.mkShell { - # The Nix packages provided in the environment - packages = dependencies pkgs; + # TODO: I think we need otf1 instead - but no nixpkgs exists + # otf2 + rocksdb + ]; - # Most hardening flags are enabled by default in nix. - # The only one usually disabled is pie. - # For more details check here: - # https://ryantm.github.io/nixpkgs/stdenv/stdenv/#sec-hardening-in-nixpkgs - hardeningEnable = [ "pie" ]; + # Function that gets pkgs and results in julea being build + mkJulea = pkgs: + pkgs.stdenv.mkDerivation { + pname = "julea"; + # TODO: Do we have a versioning scheme? + version = "1.0.0"; + src = self; - # Set any environment variables for your development environment - env = { - }; + mesonBuildType = release; - # Add any shell logic you want executed when the environment is activated - # TODO: Is LD_LIBRARY_PATH ok like this? The spack version had a lot more stuff in it. - shellHook = '' - echo "Welcome to the JULEA nix shell:" + buildInputs = dependencies pkgs; + }; + in + flake-utils.lib.eachDefaultSystem ( + system: let + # Import nixpkgs for the specific system with our config + pkgs = import nixpkgs { + inherit system; + config.allowUnfree = true; + overlays = [(import ./libfabric_overlay.nix)]; + }; + in { + # Development environments output by this flake + devShells.default = pkgs.mkShell { + # The Nix packages provided in the environment + packages = dependencies pkgs; - export BUILD_DIR="$PWD/bld" - export PATH="$PATH:$BUILD_DIR" - export LD_LIBRARY_PATH="$BUILD_DIR" - export PKG_CONFIG_PATH="$PKG_CONFIG_PATH:$BUILD_DIR/meson-uninstalled/" - export JULEA_BACKEND_PATH="$BUILD_DIR" - export HDF5_PLUGIN_PATH="$BUILD_DIR" - ''; - }; - } - ); - } + # Most hardening flags are enabled by default in nix. + # The only one usually disabled is pie. + # For more details check here: + # https://ryantm.github.io/nixpkgs/stdenv/stdenv/#sec-hardening-in-nixpkgs + hardeningEnable = ["pie"]; + + # Set any environment variables for your development environment + env = { + }; + + # Add any shell logic you want executed when the environment is activated + # TODO: Is LD_LIBRARY_PATH ok like this? The spack version had a lot more stuff in it. + shellHook = '' + echo "Welcome to the JULEA nix shell:" + + export BUILD_DIR="$PWD/bld" + export PATH="$PATH:$BUILD_DIR" + export LD_LIBRARY_PATH="$BUILD_DIR" + export PKG_CONFIG_PATH="$PKG_CONFIG_PATH:$BUILD_DIR/meson-uninstalled/" + export JULEA_BACKEND_PATH="$BUILD_DIR" + export HDF5_PLUGIN_PATH="$BUILD_DIR" + ''; + }; + + # Can be called with `nix build`. + packages.default = mkJulea pkgs; + } + ); +} diff --git a/scripts/benchmark.sh b/scripts/benchmark.sh index 77daa05ec..662438ad6 100755 --- a/scripts/benchmark.sh +++ b/scripts/benchmark.sh @@ -24,8 +24,6 @@ SELF_BASE="${SELF_PATH##*/}" # shellcheck source=scripts/common . "${SELF_DIR}/common" -# shellcheck source=scripts/spack -. "${SELF_DIR}/spack" usage () { From aae45297a59905143f74a7980bef576c8c01373e Mon Sep 17 00:00:00 2001 From: jan Date: Thu, 20 Nov 2025 14:20:47 +0100 Subject: [PATCH 14/25] Feat: Adjusted build portion of ci. --- .github/workflows/ci.yml | 167 +---------------------------- .github/workflows/dependencies.yml | 62 ----------- flake.nix | 3 +- scripts/ci/build.sh | 23 +--- 4 files changed, 9 insertions(+), 246 deletions(-) delete mode 100644 .github/workflows/dependencies.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cbcfc66c3..3fde35e11 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,114 +33,8 @@ jobs: - name: Check shell scripts run: | ./scripts/check.sh - dependencies: - name: Dependencies - runs-on: ${{ matrix.os.dist }} - timeout-minutes: 180 - strategy: - fail-fast: false - matrix: - os: - - dist: ubuntu-24.04 - compiler: gcc - compiler_spack: gcc - compiler_version: 13.3.0 - - dist: ubuntu-24.04 - compiler: clang - compiler_spack: llvm - compiler_version: 18.1.3 - - dist: ubuntu-22.04 - compiler: gcc - compiler_spack: gcc - compiler_version: 11.4.0 - - dist: ubuntu-22.04 - compiler: clang - compiler_spack: llvm - compiler_version: 14.0.0 - steps: - - name: Checkout - uses: actions/checkout@v5 - with: - persist-credentials: false - show-progress: false - - name: Cache dependencies - id: cache - uses: actions/cache@v4 - with: - path: dependencies - key: ${{ matrix.os.dist }}-${{ matrix.os.compiler }}-${{ matrix.os.compiler_version }}-${{ hashFiles('scripts/spack', 'scripts/install-dependencies.sh') }} - - name: Install dependencies - if: ${{ steps.cache.outputs.cache-hit != 'true' }} - env: - JULEA_SPACK_COMPILER: ${{ matrix.os.compiler_spack }}@${{ matrix.os.compiler_version }} - run: | - ./scripts/install-dependencies.sh - dependencies-containers: - name: Dependencies (Containers) - runs-on: ubuntu-24.04 - container: ${{ matrix.os.registry }}/${{ matrix.os.image }} - timeout-minutes: 180 - strategy: - fail-fast: false - matrix: - os: - - image: ubuntu:24.04 - registry: docker.io/library - compiler: gcc - compiler_spack: gcc - compiler_version: 13.3.0 - - image: ubuntu:24.04 - registry: docker.io/library - compiler: clang - compiler_spack: llvm - compiler_version: 18.1.3 - - image: ubuntu:22.04 - registry: docker.io/library - compiler: gcc - compiler_spack: gcc - compiler_version: 11.4.0 - - image: ubuntu:22.04 - registry: docker.io/library - compiler: clang - compiler_spack: llvm - compiler_version: 14.0.0 - steps: - - name: Checkout - uses: actions/checkout@v5 - with: - persist-credentials: false - show-progress: false - - name: Install packages - if: ${{ matrix.os.image == 'ubuntu:24.04' || matrix.os.image == 'ubuntu:22.04' }} - run: | - apt update - apt --yes --no-install-recommends install build-essential zstd - apt --yes --no-install-recommends install ca-certificates git gnupg patchelf python3 unzip - if test "${{ matrix.os.compiler }}" = 'clang' - then - apt --yes --no-install-recommends install clang - - if test "${{ matrix.os.image }}" = 'ubuntu:24.04' - then - apt --yes --no-install-recommends install libclang-rt-dev - fi - fi - - name: Cache dependencies - id: cache - uses: actions/cache@v4 - with: - path: /dependencies - key: ${{ matrix.os.image }}-${{ matrix.os.compiler }}-${{ matrix.os.compiler_version }}-${{ hashFiles('scripts/spack', 'scripts/install-dependencies.sh') }} - - name: Install dependencies - if: ${{ steps.cache.outputs.cache-hit != 'true' }} - env: - JULEA_SPACK_COMPILER: ${{ matrix.os.compiler_spack }}@${{ matrix.os.compiler_version }} - JULEA_SPACK_DIR: /dependencies - run: | - ./scripts/install-dependencies.sh build: name: Build - needs: dependencies runs-on: ${{ matrix.os.dist }} timeout-minutes: 60 strategy: @@ -148,58 +42,18 @@ jobs: matrix: os: - dist: ubuntu-24.04 - compiler: gcc - compiler_spack: gcc - compiler_version: 13.3.0 - - dist: ubuntu-24.04 - compiler: clang - compiler_spack: llvm - compiler_version: 18.1.3 - dist: ubuntu-22.04 - compiler: gcc - compiler_spack: gcc - compiler_version: 11.4.0 - - dist: ubuntu-22.04 - compiler: clang - compiler_spack: llvm - compiler_version: 14.0.0 - dependencies: [system, spack] steps: - name: Checkout uses: actions/checkout@v5 with: persist-credentials: false show-progress: false - - name: Clean up system packages - # We need to remove glib-network, otherwise libproxy might cause crashes. - run: | - sudo apt --yes purge glib-networking - sudo apt --yes --purge autoremove - sudo aa-remove-unknown || true - - name: Install dependencies - if: ${{ matrix.dependencies == 'system' }} - run: | - sudo apt update || true - sudo apt --yes --no-install-recommends install meson ninja-build pkgconf libglib2.0-dev libbson-dev libfabric-dev libgdbm-dev liblmdb-dev libsqlite3-dev libleveldb-dev libmongoc-dev libmariadb-dev librocksdb-dev libfuse3-dev libopen-trace-format-dev librados-dev - - name: Cache dependencies - id: cache - if: ${{ matrix.dependencies == 'spack' }} - uses: actions/cache/restore@v4 - with: - path: dependencies - key: ${{ matrix.os.dist }}-${{ matrix.os.compiler }}-${{ matrix.os.compiler_version }}-${{ hashFiles('scripts/spack', 'scripts/install-dependencies.sh') }} - - name: Check dependencies - if: ${{ matrix.dependencies == 'spack' && steps.cache.outputs.cache-hit != 'true' }} - run: | - exit 1 - name: Configure, build and install - env: - CC: ${{ matrix.os.compiler }} run: | - ./scripts/ci/build.sh release "${{ matrix.dependencies }}" + nix build tests: name: Tests - needs: dependencies runs-on: ${{ matrix.os.dist }} timeout-minutes: 60 strategy: @@ -207,22 +61,8 @@ jobs: matrix: os: - dist: ubuntu-24.04 - compiler: gcc - compiler_spack: gcc - compiler_version: 13.3.0 - - dist: ubuntu-24.04 - compiler: clang - compiler_spack: llvm - compiler_version: 18.1.3 - dist: ubuntu-22.04 - compiler: gcc - compiler_spack: gcc - compiler_version: 11.4.0 - - dist: ubuntu-22.04 - compiler: clang - compiler_spack: llvm - compiler_version: 14.0.0 - dependencies: [system, spack] + julea: # Default - object: posix @@ -322,7 +162,7 @@ jobs: env: CC: ${{ matrix.os.compiler }} run: | - ./scripts/ci/build.sh debug "${{ matrix.dependencies }}" + ./scripts/ci/build.sh debug - name: Create configuration # FIXME We need to use 127.0.0.1 because localhost tries to use the socket (/tmp/mysql.sock), which does not exist. run: | @@ -391,7 +231,6 @@ jobs: # FIXME Reduce redundancies # FIXME Increase coverage (currently, only one configuration runs) name: Coverage - needs: dependencies runs-on: ubuntu-24.04 timeout-minutes: 60 steps: diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml deleted file mode 100644 index 6228e434b..000000000 --- a/.github/workflows/dependencies.yml +++ /dev/null @@ -1,62 +0,0 @@ -# JULEA - Flexible storage framework -# Copyright (C) 2021-2024 Michael Kuhn -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program. If not, see . - -name: Dependencies -on: - schedule: - - cron: '0 0 * * 0' -jobs: - dependencies: - name: Dependencies - runs-on: ${{ matrix.os.dist }} - timeout-minutes: 180 - strategy: - fail-fast: false - matrix: - os: - - dist: ubuntu-24.04 - compiler: gcc - compiler_spack: gcc - compiler_version: 13.3.0 - - dist: ubuntu-24.04 - compiler: clang - compiler_spack: llvm - compiler_version: 18.1.3 - - dist: ubuntu-22.04 - compiler: gcc - compiler_spack: gcc - compiler_version: 11.4.0 - - dist: ubuntu-22.04 - compiler: clang - compiler_spack: llvm - compiler_version: 14.0.0 - steps: - - name: Checkout - uses: actions/checkout@v5 - with: - persist-credentials: false - show-progress: false - - name: Cache dependencies - uses: actions/cache@v4 - with: - path: dependencies - key: ${{ matrix.os.dist }}-${{ matrix.os.compiler }}-${{ matrix.os.compiler_version }}-${{ hashFiles('scripts/spack', 'scripts/install-dependencies.sh') }} - - name: Install dependencies - env: - JULEA_SPACK_COMPILER: ${{ matrix.os.compiler_spack }}@${{ matrix.os.compiler_version }} - run: | - rm --recursive --force dependencies - ./scripts/install-dependencies.sh diff --git a/flake.nix b/flake.nix index ef32bc690..eb57b5d84 100644 --- a/flake.nix +++ b/flake.nix @@ -18,6 +18,7 @@ dependencies = pkgs: with pkgs; [ # Build basics + # TODO: Currently the ci/cd pipelines tests for clang and gcc. Is gcc only ok? gcc ninja meson @@ -57,7 +58,7 @@ version = "1.0.0"; src = self; - mesonBuildType = release; + mesonBuildType = "release"; buildInputs = dependencies pkgs; }; diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh index 0639f2b1a..1f87116c3 100755 --- a/scripts/ci/build.sh +++ b/scripts/ci/build.sh @@ -19,38 +19,23 @@ set -e MODE="$1" -DEPS="$2" GDBM_PREFIX='' -. scripts/environment.sh - -if test "${DEPS}" = 'spack' -then - GDBM_PREFIX="-Dgdbm_prefix=$(spack location --install-dir gdbm)" -fi - case "${MODE}" in - release) - # shellcheck disable=SC2086 - meson setup --prefix="${GITHUB_WORKSPACE}/julea-install" --buildtype=release --werror ${GDBM_PREFIX} bld - ninja -C bld - ninja -C bld install - ;; debug) - CLANG_LUNDEF='' - if test "${CC}" = 'clang' - then - CLANG_LUNDEF='-Db_lundef=false' - fi # shellcheck disable=SC2086 + nix develop meson setup -Db_sanitize=address,undefined ${CLANG_LUNDEF} ${GDBM_PREFIX} bld ninja -C bld + exit ;; coverage) # shellcheck disable=SC2086 + nix develop meson setup -Db_coverage=true -Db_sanitize=address,undefined ${GDBM_PREFIX} bld ninja -C bld + exit ;; *) exit 1 From dc498004a714313fff564e380624355877e527e6 Mon Sep 17 00:00:00 2001 From: jan Date: Thu, 20 Nov 2025 14:26:36 +0100 Subject: [PATCH 15/25] Fix: Pipeline? --- .github/workflows/ci.yml | 34 ++++------------------------------ scripts/ci/build.sh | 6 ++---- 2 files changed, 6 insertions(+), 34 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3fde35e11..85b0cacaf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -101,7 +101,6 @@ jobs: # FIXME Ubuntu 24.04's RocksDB triggers asan - os: dist: ubuntu-24.04 - dependencies: system julea: kv: rocksdb services: @@ -133,53 +132,28 @@ jobs: with: persist-credentials: false show-progress: false - - name: Clean up system packages - # We need to remove glib-network, otherwise libproxy might cause crashes. - # FIXME https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 - # Ubuntu 22.04 and 24.04 have MySQL 8.0 - run: | - sudo apt --yes purge glib-networking mysql-client mysql-client-8.0 mysql-server mysql-server-8.0 - sudo apt --yes --purge autoremove - sudo aa-remove-unknown || true - sudo rm --recursive --force /var/lib/mysql - - name: Install dependencies - if: ${{ matrix.dependencies == 'system' }} - run: | - sudo apt update || true - sudo apt --yes --no-install-recommends install meson ninja-build pkgconf libglib2.0-dev libbson-dev libfabric-dev libgdbm-dev liblmdb-dev libsqlite3-dev libleveldb-dev libmongoc-dev libmariadb-dev librocksdb-dev libfuse3-dev libopen-trace-format-dev librados-dev - - name: Cache dependencies - id: cache - if: ${{ matrix.dependencies == 'spack' }} - uses: actions/cache/restore@v4 - with: - path: dependencies - key: ${{ matrix.os.dist }}-${{ matrix.os.compiler }}-${{ matrix.os.compiler_version }}-${{ hashFiles('scripts/spack', 'scripts/install-dependencies.sh') }} - - name: Check dependencies - if: ${{ matrix.dependencies == 'spack' && steps.cache.outputs.cache-hit != 'true' }} - run: | - exit 1 - name: Configure and build - env: - CC: ${{ matrix.os.compiler }} run: | ./scripts/ci/build.sh debug - name: Create configuration # FIXME We need to use 127.0.0.1 because localhost tries to use the socket (/tmp/mysql.sock), which does not exist. run: | - . scripts/environment.sh + nix develop JULEA_KV_PATH="/tmp/julea/kv/${{ matrix.julea.kv }}" if test "${{ matrix.julea.kv }}" = 'mongodb'; then JULEA_KV_PATH='127.0.0.1:juleadb'; fi JULEA_DB_PATH="/tmp/julea/db/${{ matrix.julea.db }}" if test "${{ matrix.julea.db }}" = 'mysql'; then JULEA_DB_PATH='127.0.0.1:juleadb:julea:aeluj'; fi julea-config --user --object-servers="$(hostname)" --kv-servers="$(hostname)" --db-servers="$(hostname)" --object-backend="${{ matrix.julea.object }}" --object-path="/tmp/julea/object/${{ matrix.julea.object }}" --kv-backend="${{ matrix.julea.kv }}" --kv-path="${JULEA_KV_PATH}" --db-backend="${{ matrix.julea.db }}" --db-path="${JULEA_DB_PATH}" + exit - name: Tests run: | - . scripts/environment.sh + nix develop ./scripts/setup.sh start ./scripts/test.sh sleep 10 ./scripts/test.sh ./scripts/setup.sh stop + exit - name: HDF5 Tests env: LSAN_OPTIONS: exitcode=0 diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh index 1f87116c3..570706a55 100755 --- a/scripts/ci/build.sh +++ b/scripts/ci/build.sh @@ -20,20 +20,18 @@ set -e MODE="$1" -GDBM_PREFIX='' - case "${MODE}" in debug) # shellcheck disable=SC2086 nix develop - meson setup -Db_sanitize=address,undefined ${CLANG_LUNDEF} ${GDBM_PREFIX} bld + meson setup -Db_sanitize=address,undefined bld ninja -C bld exit ;; coverage) # shellcheck disable=SC2086 nix develop - meson setup -Db_coverage=true -Db_sanitize=address,undefined ${GDBM_PREFIX} bld + meson setup -Db_coverage=true -Db_sanitize=address,undefined bld ninja -C bld exit ;; From f12f5129451a2b5301caa8152e50b7cfa4d1f8d8 Mon Sep 17 00:00:00 2001 From: jan Date: Thu, 20 Nov 2025 14:41:48 +0100 Subject: [PATCH 16/25] Fix: Pipeline? --- .github/workflows/ci.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 85b0cacaf..d141deba5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,21 +35,26 @@ jobs: ./scripts/check.sh build: name: Build - runs-on: ${{ matrix.os.dist }} + # Todo: Remove matrix entirely? + runs-on: ${{ matrix.dist }} timeout-minutes: 60 strategy: fail-fast: false matrix: - os: - - dist: ubuntu-24.04 - - dist: ubuntu-22.04 + dist: + - ubuntu-24.04 + - ubuntu-22.04 steps: - name: Checkout uses: actions/checkout@v5 with: persist-credentials: false show-progress: false - - name: Configure, build and install + - name: Install nix + uses: cachix/install-nix-action@v31 + with: + github_access_token: ${{ secrets.GITHUB_TOKEN }} + - name: Build run: | nix build tests: @@ -158,6 +163,7 @@ jobs: env: LSAN_OPTIONS: exitcode=0 run: | + // . scripts/environment.sh ./scripts/setup.sh start export HDF5_VOL_CONNECTOR=julea-kv From 02f3d65905233f7702ed165261e840f709316a78 Mon Sep 17 00:00:00 2001 From: jan Date: Thu, 20 Nov 2025 14:46:42 +0100 Subject: [PATCH 17/25] Fix: Pipeline? --- .github/workflows/ci.yml | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d141deba5..150d87e43 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -136,7 +136,11 @@ jobs: uses: actions/checkout@v5 with: persist-credentials: false - show-progress: false + show-progress: fals + - name: Install nix + uses: cachix/install-nix-action@v31 + with: + github_access_token: ${{ secrets.GITHUB_TOKEN }}e - name: Configure and build run: | ./scripts/ci/build.sh debug @@ -218,39 +222,36 @@ jobs: uses: actions/checkout@v5 with: persist-credentials: false - show-progress: false - - name: Cache dependencies - id: cache - uses: actions/cache/restore@v4 + show-progress: fals + - name: Install nix + uses: cachix/install-nix-action@v31 with: - path: dependencies - key: ubuntu-24.04-gcc-13.3.0-${{ hashFiles('scripts/spack', 'scripts/install-dependencies.sh') }} - - name: Check dependencies - if: ${{ steps.cache.outputs.cache-hit != 'true' }} - run: | - exit 1 + github_access_token: ${{ secrets.GITHUB_TOKEN }}e - name: Configure and build env: CC: gcc run: | - ./scripts/ci/build.sh coverage spack + ./scripts/ci/build.sh coverage - name: Create configuration run: | - . scripts/environment.sh + nix develop julea-config --user --object-servers="$(hostname)" --kv-servers="$(hostname)" --db-servers="$(hostname)" --object-backend=posix --object-path=/tmp/julea/object/posix --kv-backend=lmdb --kv-path=/tmp/julea/kv/lmdb --db-backend=sqlite --db-path=/tmp/julea/db/sqlite + exit - name: Tests env: HDF5_VOL_CONNECTOR: julea-db LSAN_OPTIONS: exitcode=0 run: | - . scripts/environment.sh + nix develop ./scripts/setup.sh start ./scripts/test.sh ./scripts/setup.sh stop + exit - name: Generate coverage run: | - . scripts/environment.sh + nix develop ninja -C bld coverage + exit - name: Upload coverage uses: actions/upload-artifact@v4 with: From 1514d51e5a826836cbb7c876994d1eb5d0cde125 Mon Sep 17 00:00:00 2001 From: jan Date: Thu, 20 Nov 2025 14:47:27 +0100 Subject: [PATCH 18/25] Fix: Pipeline? --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 150d87e43..cab32321d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -136,11 +136,11 @@ jobs: uses: actions/checkout@v5 with: persist-credentials: false - show-progress: fals + show-progress: false - name: Install nix uses: cachix/install-nix-action@v31 with: - github_access_token: ${{ secrets.GITHUB_TOKEN }}e + github_access_token: ${{ secrets.GITHUB_TOKEN }} - name: Configure and build run: | ./scripts/ci/build.sh debug @@ -222,11 +222,11 @@ jobs: uses: actions/checkout@v5 with: persist-credentials: false - show-progress: fals + show-progress: false - name: Install nix uses: cachix/install-nix-action@v31 with: - github_access_token: ${{ secrets.GITHUB_TOKEN }}e + github_access_token: ${{ secrets.GITHUB_TOKEN }} - name: Configure and build env: CC: gcc From 7a3abdccf97e97ee4b0b1f44e36393b5b1bef017 Mon Sep 17 00:00:00 2001 From: jan Date: Thu, 20 Nov 2025 14:59:32 +0100 Subject: [PATCH 19/25] Fix: Pipeline? --- .github/workflows/ci.yml | 16 ++++++++-------- scripts/ci/build.sh | 41 ---------------------------------------- 2 files changed, 8 insertions(+), 49 deletions(-) delete mode 100755 scripts/ci/build.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cab32321d..b824e1a7d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -141,34 +141,32 @@ jobs: uses: cachix/install-nix-action@v31 with: github_access_token: ${{ secrets.GITHUB_TOKEN }} + - name: Enter nix shell + uses: nicknovitski/nix-develop@v1 - name: Configure and build run: | - ./scripts/ci/build.sh debug + meson setup -Db_sanitize=address,undefined bld + ninja -C bld - name: Create configuration # FIXME We need to use 127.0.0.1 because localhost tries to use the socket (/tmp/mysql.sock), which does not exist. run: | - nix develop JULEA_KV_PATH="/tmp/julea/kv/${{ matrix.julea.kv }}" if test "${{ matrix.julea.kv }}" = 'mongodb'; then JULEA_KV_PATH='127.0.0.1:juleadb'; fi JULEA_DB_PATH="/tmp/julea/db/${{ matrix.julea.db }}" if test "${{ matrix.julea.db }}" = 'mysql'; then JULEA_DB_PATH='127.0.0.1:juleadb:julea:aeluj'; fi julea-config --user --object-servers="$(hostname)" --kv-servers="$(hostname)" --db-servers="$(hostname)" --object-backend="${{ matrix.julea.object }}" --object-path="/tmp/julea/object/${{ matrix.julea.object }}" --kv-backend="${{ matrix.julea.kv }}" --kv-path="${JULEA_KV_PATH}" --db-backend="${{ matrix.julea.db }}" --db-path="${JULEA_DB_PATH}" - exit - name: Tests run: | - nix develop ./scripts/setup.sh start ./scripts/test.sh sleep 10 ./scripts/test.sh ./scripts/setup.sh stop - exit - name: HDF5 Tests env: LSAN_OPTIONS: exitcode=0 run: | // - . scripts/environment.sh ./scripts/setup.sh start export HDF5_VOL_CONNECTOR=julea-kv ./scripts/test.sh -r /hdf5 @@ -183,7 +181,6 @@ jobs: - name: Benchmarks if: ${{ matrix.julea.object == 'posix' && matrix.julea.kv == 'lmdb' && matrix.julea.db == 'sqlite' }} run: | - . scripts/environment.sh ./scripts/setup.sh start ./scripts/benchmark.sh --duration=0 sleep 10 @@ -231,7 +228,10 @@ jobs: env: CC: gcc run: | - ./scripts/ci/build.sh coverage + nix develop + meson setup -Db_coverage=true -Db_sanitize=address,undefined bld + ninja -C bld + exit - name: Create configuration run: | nix develop diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh deleted file mode 100755 index 570706a55..000000000 --- a/scripts/ci/build.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -# JULEA - Flexible storage framework -# Copyright (C) 2024 Michael Kuhn -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program. If not, see . - -set -e - -MODE="$1" - -case "${MODE}" in - debug) - # shellcheck disable=SC2086 - nix develop - meson setup -Db_sanitize=address,undefined bld - ninja -C bld - exit - ;; - coverage) - # shellcheck disable=SC2086 - nix develop - meson setup -Db_coverage=true -Db_sanitize=address,undefined bld - ninja -C bld - exit - ;; - *) - exit 1 - ;; -esac From 1ff065094bd8889405f42cc06510c0ea7774da5e Mon Sep 17 00:00:00 2001 From: jan Date: Thu, 20 Nov 2025 15:13:41 +0100 Subject: [PATCH 20/25] Fix: Pipeline? --- .github/workflows/ci.yml | 64 ++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 38 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b824e1a7d..ccec41533 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -141,12 +141,10 @@ jobs: uses: cachix/install-nix-action@v31 with: github_access_token: ${{ secrets.GITHUB_TOKEN }} - - name: Enter nix shell - uses: nicknovitski/nix-develop@v1 - name: Configure and build run: | - meson setup -Db_sanitize=address,undefined bld - ninja -C bld + nix develop --command meson setup -Db_sanitize=address,undefined bld + nix develop --command ninja -C bld - name: Create configuration # FIXME We need to use 127.0.0.1 because localhost tries to use the socket (/tmp/mysql.sock), which does not exist. run: | @@ -154,38 +152,38 @@ jobs: if test "${{ matrix.julea.kv }}" = 'mongodb'; then JULEA_KV_PATH='127.0.0.1:juleadb'; fi JULEA_DB_PATH="/tmp/julea/db/${{ matrix.julea.db }}" if test "${{ matrix.julea.db }}" = 'mysql'; then JULEA_DB_PATH='127.0.0.1:juleadb:julea:aeluj'; fi - julea-config --user --object-servers="$(hostname)" --kv-servers="$(hostname)" --db-servers="$(hostname)" --object-backend="${{ matrix.julea.object }}" --object-path="/tmp/julea/object/${{ matrix.julea.object }}" --kv-backend="${{ matrix.julea.kv }}" --kv-path="${JULEA_KV_PATH}" --db-backend="${{ matrix.julea.db }}" --db-path="${JULEA_DB_PATH}" + nix develop --command julea-config --user --object-servers="$(hostname)" --kv-servers="$(hostname)" --db-servers="$(hostname)" --object-backend="${{ matrix.julea.object }}" --object-path="/tmp/julea/object/${{ matrix.julea.object }}" --kv-backend="${{ matrix.julea.kv }}" --kv-path="${JULEA_KV_PATH}" --db-backend="${{ matrix.julea.db }}" --db-path="${JULEA_DB_PATH}" - name: Tests run: | - ./scripts/setup.sh start - ./scripts/test.sh + nix develop --command ./scripts/setup.sh start + nix develop --command ./scripts/test.sh sleep 10 - ./scripts/test.sh - ./scripts/setup.sh stop + nix develop --command ./scripts/test.sh + nix develop --command ./scripts/setup.sh stop - name: HDF5 Tests env: LSAN_OPTIONS: exitcode=0 run: | // - ./scripts/setup.sh start - export HDF5_VOL_CONNECTOR=julea-kv - ./scripts/test.sh -r /hdf5 + nix develop --command ./scripts/setup.sh start + nix develop --command export HDF5_VOL_CONNECTOR=julea-kv + nix develop --command ./scripts/test.sh -r /hdf5 sleep 10 - ./scripts/test.sh -r /hdf5 + nix develop --command ./scripts/test.sh -r /hdf5 sleep 10 - export HDF5_VOL_CONNECTOR=julea-db - ./scripts/test.sh -r /hdf5 + nix develop --command export HDF5_VOL_CONNECTOR=julea-db + nix develop --command ./scripts/test.sh -r /hdf5 sleep 10 - ./scripts/test.sh -r /hdf5 - ./scripts/setup.sh stop + nix develop --command ./scripts/test.sh -r /hdf5 + nix develop --command ./scripts/setup.sh stop - name: Benchmarks if: ${{ matrix.julea.object == 'posix' && matrix.julea.kv == 'lmdb' && matrix.julea.db == 'sqlite' }} run: | - ./scripts/setup.sh start - ./scripts/benchmark.sh --duration=0 + nix develop --command ./scripts/setup.sh start + nix develop --command ./scripts/benchmark.sh --duration=0 sleep 10 - ./scripts/benchmark.sh --duration=0 - ./scripts/setup.sh stop + nix develop --command ./scripts/benchmark.sh --duration=0 + nix develop --command ./scripts/setup.sh stop doxygen: name: Doxygen runs-on: ubuntu-24.04 @@ -225,33 +223,23 @@ jobs: with: github_access_token: ${{ secrets.GITHUB_TOKEN }} - name: Configure and build - env: - CC: gcc run: | - nix develop - meson setup -Db_coverage=true -Db_sanitize=address,undefined bld - ninja -C bld - exit + nix develop --command meson setup -Db_coverage=true -Db_sanitize=address,undefined bld + nix develop --command ninja -C bld - name: Create configuration run: | - nix develop - julea-config --user --object-servers="$(hostname)" --kv-servers="$(hostname)" --db-servers="$(hostname)" --object-backend=posix --object-path=/tmp/julea/object/posix --kv-backend=lmdb --kv-path=/tmp/julea/kv/lmdb --db-backend=sqlite --db-path=/tmp/julea/db/sqlite - exit + nix develop --command julea-config --user --object-servers="$(hostname)" --kv-servers="$(hostname)" --db-servers="$(hostname)" --object-backend=posix --object-path=/tmp/julea/object/posix --kv-backend=lmdb --kv-path=/tmp/julea/kv/lmdb --db-backend=sqlite --db-path=/tmp/julea/db/sqlite - name: Tests env: HDF5_VOL_CONNECTOR: julea-db LSAN_OPTIONS: exitcode=0 run: | - nix develop - ./scripts/setup.sh start - ./scripts/test.sh - ./scripts/setup.sh stop - exit + nix develop --command ./scripts/setup.sh start + nix develop --command ./scripts/test.sh + nix develop --command ./scripts/setup.sh stop - name: Generate coverage run: | - nix develop - ninja -C bld coverage - exit + nix develop --command ninja -C bld coverage - name: Upload coverage uses: actions/upload-artifact@v4 with: From dc76da6f79674ed2c78204b175015abd63150680 Mon Sep 17 00:00:00 2001 From: jan Date: Thu, 20 Nov 2025 15:20:13 +0100 Subject: [PATCH 21/25] Fix: Pipeline? --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ccec41533..dfffde4b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -164,7 +164,6 @@ jobs: env: LSAN_OPTIONS: exitcode=0 run: | - // nix develop --command ./scripts/setup.sh start nix develop --command export HDF5_VOL_CONNECTOR=julea-kv nix develop --command ./scripts/test.sh -r /hdf5 From 7330a47da99b66735a066da50d1d00d44febefd2 Mon Sep 17 00:00:00 2001 From: Jan Frase Date: Fri, 21 Nov 2025 12:57:01 +0100 Subject: [PATCH 22/25] Fix: Pipeline? --- .github/workflows/ci.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dfffde4b9..9976bebca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -165,15 +165,13 @@ jobs: LSAN_OPTIONS: exitcode=0 run: | nix develop --command ./scripts/setup.sh start - nix develop --command export HDF5_VOL_CONNECTOR=julea-kv - nix develop --command ./scripts/test.sh -r /hdf5 + HDF5_VOL_CONNECTOR=julea-kv nix develop --command ./scripts/test.sh -r /hdf5 sleep 10 - nix develop --command ./scripts/test.sh -r /hdf5 - sleep 10 - nix develop --command export HDF5_VOL_CONNECTOR=julea-db - nix develop --command ./scripts/test.sh -r /hdf5 + HDF5_VOL_CONNECTOR=julea-kv nix develop --command ./scripts/test.sh -r /hdf5 + + HDF5_VOL_CONNECTOR=julea-db nix develop --command ./scripts/test.sh -r /hdf5 sleep 10 - nix develop --command ./scripts/test.sh -r /hdf5 + HDF5_VOL_CONNECTOR=julea-db nix develop --command ./scripts/test.sh -r /hdf5 nix develop --command ./scripts/setup.sh stop - name: Benchmarks if: ${{ matrix.julea.object == 'posix' && matrix.julea.kv == 'lmdb' && matrix.julea.db == 'sqlite' }} From 1a5adabddf766c5fd5001beab722ac6dfbb4530e Mon Sep 17 00:00:00 2001 From: Jan Frase Date: Fri, 21 Nov 2025 13:41:04 +0100 Subject: [PATCH 23/25] Feat: Added coverage dependency. --- flake.nix | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index eb57b5d84..fcaf5626d 100644 --- a/flake.nix +++ b/flake.nix @@ -75,7 +75,13 @@ # Development environments output by this flake devShells.default = pkgs.mkShell { # The Nix packages provided in the environment - packages = dependencies pkgs; + # Consists of the usual build dependencies and some dev dependencies. + packages = (dependencies pkgs) ++ [ + # Needed to run coverage tests. + pkgs.gcovr + ]; + + # Most hardening flags are enabled by default in nix. # The only one usually disabled is pie. From 01da78d81870c169be139b43eb5dbe431481413f Mon Sep 17 00:00:00 2001 From: Jan Frase Date: Tue, 10 Feb 2026 16:16:05 +0100 Subject: [PATCH 24/25] Updated nix. Removed hardening flag as it is enabled by default now. --- flake.lock | 18 ++++++++++-------- flake.nix | 12 ++---------- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/flake.lock b/flake.lock index c1655e4ba..c86067072 100644 --- a/flake.lock +++ b/flake.lock @@ -20,16 +20,18 @@ }, "nixpkgs": { "locked": { - "lastModified": 1762233356, - "narHash": "sha256-cGS3lLTYusbEP/IJIWGgnkzIl+FA5xDvtiHyjalGr4k=", - "rev": "ca534a76c4afb2bdc07b681dbc11b453bab21af8", - "revCount": 812378, - "type": "tarball", - "url": "https://api.flakehub.com/f/pinned/NixOS/nixpkgs/0.2505.812378%2Brev-ca534a76c4afb2bdc07b681dbc11b453bab21af8/019a53af-7da6-765a-ae7b-594fa3a7097b/source.tar.gz" + "lastModified": 1769900590, + "narHash": "sha256-I7Lmgj3owOTBGuauy9FL6qdpeK2umDoe07lM4V+PnyA=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "41e216c0ca66c83b12ab7a98cc326b5db01db646", + "type": "github" }, "original": { - "type": "tarball", - "url": "https://flakehub.com/f/NixOS/nixpkgs/0" + "owner": "NixOS", + "ref": "nixos-25.11", + "repo": "nixpkgs", + "type": "github" } }, "root": { diff --git a/flake.nix b/flake.nix index fcaf5626d..55da4eb68 100644 --- a/flake.nix +++ b/flake.nix @@ -3,7 +3,7 @@ # Flake inputs inputs = { - nixpkgs.url = "https://flakehub.com/f/NixOS/nixpkgs/0"; + nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.11"; flake-utils.url = "github:numtide/flake-utils"; }; @@ -26,7 +26,7 @@ # Other packages glib - libbson + mongoc # libbson has been moved to mongoc # This uses the libfabric-overlay! libfabric @@ -81,14 +81,6 @@ pkgs.gcovr ]; - - - # Most hardening flags are enabled by default in nix. - # The only one usually disabled is pie. - # For more details check here: - # https://ryantm.github.io/nixpkgs/stdenv/stdenv/#sec-hardening-in-nixpkgs - hardeningEnable = ["pie"]; - # Set any environment variables for your development environment env = { }; From b59a411352a03be36a302c5c5a3438789659c4c8 Mon Sep 17 00:00:00 2001 From: Jan Frase Date: Fri, 13 Feb 2026 17:38:34 +0100 Subject: [PATCH 25/25] Add interactive bash shell. --- flake.lock | 6 +++--- flake.nix | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index c86067072..8f8b8f309 100644 --- a/flake.lock +++ b/flake.lock @@ -20,11 +20,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1769900590, - "narHash": "sha256-I7Lmgj3owOTBGuauy9FL6qdpeK2umDoe07lM4V+PnyA=", + "lastModified": 1770770419, + "narHash": "sha256-iKZMkr6Cm9JzWlRYW/VPoL0A9jVKtZYiU4zSrVeetIs=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "41e216c0ca66c83b12ab7a98cc326b5db01db646", + "rev": "6c5e707c6b5339359a9a9e215c5e66d6d802fd7a", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 55da4eb68..5c24156fe 100644 --- a/flake.nix +++ b/flake.nix @@ -79,6 +79,8 @@ packages = (dependencies pkgs) ++ [ # Needed to run coverage tests. pkgs.gcovr + # Add interactive bash support. + pkgs.bashInteractive ]; # Set any environment variables for your development environment