From 22f616f1b973cb33e77c768220a01312ce67cd7d Mon Sep 17 00:00:00 2001 From: Razze Date: Sat, 13 Dec 2025 17:37:15 +0100 Subject: [PATCH 1/4] fix: Replace in-memory bspatch output buffer with a memory-mapped temporary file to prevent OOM for large files. --- .../ostree-repo-static-delta-processing.c | 63 ++++++++++++++++--- 1 file changed, 56 insertions(+), 7 deletions(-) diff --git a/src/libostree/ostree-repo-static-delta-processing.c b/src/libostree/ostree-repo-static-delta-processing.c index 94876fa448..9ed78f829d 100644 --- a/src/libostree/ostree-repo-static-delta-processing.c +++ b/src/libostree/ostree-repo-static-delta-processing.c @@ -414,7 +414,25 @@ dispatch_bspatch (OstreeRepo *repo, StaticDeltaExecutionState *state, GCancellab if (!input_mfile) return FALSE; - g_autofree guchar *buf = g_malloc0 (state->content_size); + /* Instead of allocating the entire content_size in memory, use a tmpfile + * that can be paged to disk by the OS if needed. This prevents OOM on + * systems with limited RAM when processing large files. + * See: https://github.com/flatpak/flatpak/issues/6255 + */ + g_auto (GLnxTmpfile) tmpf = { + 0, + }; + if (!glnx_open_anonymous_tmpfile (O_RDWR | O_CLOEXEC, &tmpf, error)) + return FALSE; + + /* Resize tmpfile to content_size */ + if (ftruncate (tmpf.fd, state->content_size) < 0) + return glnx_throw_errno_prefix (error, "ftruncate"); + + /* Memory-map the tmpfile for bspatch to write to */ + void *buf = mmap (NULL, state->content_size, PROT_READ | PROT_WRITE, MAP_SHARED, tmpf.fd, 0); + if (buf == MAP_FAILED) + return glnx_throw_errno_prefix (error, "mmap"); struct bzpatch_opaque_s opaque; opaque.state = state; @@ -423,14 +441,45 @@ dispatch_bspatch (OstreeRepo *repo, StaticDeltaExecutionState *state, GCancellab struct bspatch_stream stream; stream.read = bspatch_read; stream.opaque = &opaque; - if (bspatch ((const guint8 *)g_mapped_file_get_contents (input_mfile), - g_mapped_file_get_length (input_mfile), buf, state->content_size, &stream) - < 0) + + int bspatch_result = bspatch ((const guint8 *)g_mapped_file_get_contents (input_mfile), + g_mapped_file_get_length (input_mfile), buf, + state->content_size, &stream); + + /* Unmap before checking result */ + if (munmap (buf, state->content_size) < 0) + return glnx_throw_errno_prefix (error, "munmap"); + + if (bspatch_result < 0) return glnx_throw (error, "bsdiff patch failed"); - if (!_ostree_repo_bare_content_write (repo, &state->content_out, buf, state->content_size, - cancellable, error)) - return FALSE; + /* Now read from tmpfile and write to repository */ + if (lseek (tmpf.fd, 0, SEEK_SET) < 0) + return glnx_throw_errno_prefix (error, "lseek"); + + /* Read and write in chunks to avoid loading entire file into memory */ + guint64 bytes_remaining = state->content_size; + while (bytes_remaining > 0) + { + guchar chunk_buf[8192]; + gsize chunk_size = MIN (sizeof (chunk_buf), bytes_remaining); + gssize bytes_read; + + do + bytes_read = read (tmpf.fd, chunk_buf, chunk_size); + while (G_UNLIKELY (bytes_read == -1 && errno == EINTR)); + + if (bytes_read < 0) + return glnx_throw_errno_prefix (error, "read"); + if (bytes_read == 0) + return glnx_throw (error, "Unexpected EOF reading from tmpfile"); + + if (!_ostree_repo_bare_content_write (repo, &state->content_out, chunk_buf, bytes_read, + cancellable, error)) + return FALSE; + + bytes_remaining -= bytes_read; + } } return TRUE; From 85f906b8b76ce238ad33655f68d4dfce092c6ad8 Mon Sep 17 00:00:00 2001 From: Razze Date: Sat, 13 Dec 2025 17:53:23 +0100 Subject: [PATCH 2/4] feat: Handle zero-sized content in static delta processing by conditionally mapping and writing. --- .../ostree-repo-static-delta-processing.c | 81 +++++++++++-------- 1 file changed, 49 insertions(+), 32 deletions(-) diff --git a/src/libostree/ostree-repo-static-delta-processing.c b/src/libostree/ostree-repo-static-delta-processing.c index 9ed78f829d..c6f83a54fe 100644 --- a/src/libostree/ostree-repo-static-delta-processing.c +++ b/src/libostree/ostree-repo-static-delta-processing.c @@ -419,20 +419,34 @@ dispatch_bspatch (OstreeRepo *repo, StaticDeltaExecutionState *state, GCancellab * systems with limited RAM when processing large files. * See: https://github.com/flatpak/flatpak/issues/6255 */ + void *buf; g_auto (GLnxTmpfile) tmpf = { 0, }; - if (!glnx_open_anonymous_tmpfile (O_RDWR | O_CLOEXEC, &tmpf, error)) - return FALSE; - /* Resize tmpfile to content_size */ - if (ftruncate (tmpf.fd, state->content_size) < 0) - return glnx_throw_errno_prefix (error, "ftruncate"); + if (state->content_size > 0) + { + if (!glnx_open_anonymous_tmpfile (O_RDWR | O_CLOEXEC, &tmpf, error)) + return FALSE; - /* Memory-map the tmpfile for bspatch to write to */ - void *buf = mmap (NULL, state->content_size, PROT_READ | PROT_WRITE, MAP_SHARED, tmpf.fd, 0); - if (buf == MAP_FAILED) - return glnx_throw_errno_prefix (error, "mmap"); + /* Resize tmpfile to content_size */ + if (ftruncate (tmpf.fd, state->content_size) < 0) + return glnx_throw_errno_prefix (error, "ftruncate"); + + /* Memory-map the tmpfile for bspatch to write to */ + buf = mmap (NULL, state->content_size, PROT_READ | PROT_WRITE, MAP_SHARED, tmpf.fd, 0); + if (buf == MAP_FAILED) + return glnx_throw_errno_prefix (error, "mmap"); + } + else + { + /* mmap() with a zero size fails with EINVAL. The old code with + * g_malloc0(0) worked by returning a valid pointer. bspatch might + * not like a NULL buffer, so provide a dummy buffer. + */ + static guchar dummy_buf; + buf = &dummy_buf; + } struct bzpatch_opaque_s opaque; opaque.state = state; @@ -446,39 +460,42 @@ dispatch_bspatch (OstreeRepo *repo, StaticDeltaExecutionState *state, GCancellab g_mapped_file_get_length (input_mfile), buf, state->content_size, &stream); - /* Unmap before checking result */ - if (munmap (buf, state->content_size) < 0) + /* Unmap before checking result - only if we actually created a mapping */ + if (state->content_size > 0 && munmap (buf, state->content_size) < 0) return glnx_throw_errno_prefix (error, "munmap"); if (bspatch_result < 0) return glnx_throw (error, "bsdiff patch failed"); - /* Now read from tmpfile and write to repository */ - if (lseek (tmpf.fd, 0, SEEK_SET) < 0) - return glnx_throw_errno_prefix (error, "lseek"); - - /* Read and write in chunks to avoid loading entire file into memory */ - guint64 bytes_remaining = state->content_size; - while (bytes_remaining > 0) + /* Now read from tmpfile and write to repository (if content_size > 0) */ + if (state->content_size > 0) { - guchar chunk_buf[8192]; - gsize chunk_size = MIN (sizeof (chunk_buf), bytes_remaining); - gssize bytes_read; + if (lseek (tmpf.fd, 0, SEEK_SET) < 0) + return glnx_throw_errno_prefix (error, "lseek"); + + /* Read and write in chunks to avoid loading entire file into memory */ + guint64 bytes_remaining = state->content_size; + while (bytes_remaining > 0) + { + guchar chunk_buf[8192]; + gsize chunk_size = MIN (sizeof (chunk_buf), bytes_remaining); + gssize bytes_read; - do - bytes_read = read (tmpf.fd, chunk_buf, chunk_size); - while (G_UNLIKELY (bytes_read == -1 && errno == EINTR)); + do + bytes_read = read (tmpf.fd, chunk_buf, chunk_size); + while (G_UNLIKELY (bytes_read == -1 && errno == EINTR)); - if (bytes_read < 0) - return glnx_throw_errno_prefix (error, "read"); - if (bytes_read == 0) - return glnx_throw (error, "Unexpected EOF reading from tmpfile"); + if (bytes_read < 0) + return glnx_throw_errno_prefix (error, "read"); + if (bytes_read == 0) + return glnx_throw (error, "Unexpected EOF reading from tmpfile"); - if (!_ostree_repo_bare_content_write (repo, &state->content_out, chunk_buf, bytes_read, - cancellable, error)) - return FALSE; + if (!_ostree_repo_bare_content_write (repo, &state->content_out, chunk_buf, + bytes_read, cancellable, error)) + return FALSE; - bytes_remaining -= bytes_read; + bytes_remaining -= bytes_read; + } } } From 55f95424e8a75252450b51928b60f122bc1a4492 Mon Sep 17 00:00:00 2001 From: Razze Date: Sat, 13 Dec 2025 18:00:40 +0100 Subject: [PATCH 3/4] build: Add `sys/mman.h` include. --- src/libostree/ostree-repo-static-delta-processing.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libostree/ostree-repo-static-delta-processing.c b/src/libostree/ostree-repo-static-delta-processing.c index c6f83a54fe..f209962a52 100644 --- a/src/libostree/ostree-repo-static-delta-processing.c +++ b/src/libostree/ostree-repo-static-delta-processing.c @@ -21,6 +21,7 @@ #include "config.h" #include +#include #include #include From 6d4824f4d4d4039a0032299ec2f0e50422b1f4d9 Mon Sep 17 00:00:00 2001 From: Razze Date: Sat, 13 Dec 2025 19:08:32 +0100 Subject: [PATCH 4/4] style: run clang format --- src/libostree/ostree-repo-static-delta-processing.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libostree/ostree-repo-static-delta-processing.c b/src/libostree/ostree-repo-static-delta-processing.c index f209962a52..1c7b4adec8 100644 --- a/src/libostree/ostree-repo-static-delta-processing.c +++ b/src/libostree/ostree-repo-static-delta-processing.c @@ -457,9 +457,9 @@ dispatch_bspatch (OstreeRepo *repo, StaticDeltaExecutionState *state, GCancellab stream.read = bspatch_read; stream.opaque = &opaque; - int bspatch_result = bspatch ((const guint8 *)g_mapped_file_get_contents (input_mfile), - g_mapped_file_get_length (input_mfile), buf, - state->content_size, &stream); + int bspatch_result + = bspatch ((const guint8 *)g_mapped_file_get_contents (input_mfile), + g_mapped_file_get_length (input_mfile), buf, state->content_size, &stream); /* Unmap before checking result - only if we actually created a mapping */ if (state->content_size > 0 && munmap (buf, state->content_size) < 0)