From bcb9c400c1fdddba2431fb6f900090561f1d712d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 6 Aug 2026 11:23:21 +0000 Subject: [PATCH 01/10] Initial plan From f6c44e7b3f734853d9673fd5e468a66bbe8f2088 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 6 Aug 2026 11:27:02 +0000 Subject: [PATCH 02/10] Add optional DuckDB estimate method Co-authored-by: stemangiola <7232890+stemangiola@users.noreply.github.com> --- DESCRIPTION | 4 +++- NAMESPACE | 1 + R/sccomp_estimate.R | 7 +++++++ tests/testthat/test-duckdb.R | 3 +++ 4 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 tests/testthat/test-duckdb.R diff --git a/DESCRIPTION b/DESCRIPTION index 01ed895d..f66f5d4b 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -50,7 +50,9 @@ Suggests: tidyseurat, tidySingleCellExperiment, bayesplot, - remotes + remotes, + duckdb, + dbplyr Additional_repositories: https://mc-stan.org/r-packages/ SystemRequirements: CmdStan (https://mc-stan.org/users/interfaces/cmdstan), C++14 diff --git a/NAMESPACE b/NAMESPACE index c5866835..1d0361a9 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -6,6 +6,7 @@ S3method(sccomp_calculate_residuals,sccomp_tbl) S3method(sccomp_estimate,DFrame) S3method(sccomp_estimate,Seurat) S3method(sccomp_estimate,SingleCellExperiment) +S3method(sccomp_estimate,tbl_duckdb_connection) S3method(sccomp_estimate,data.frame) S3method(sccomp_predict,sccomp_tbl) S3method(sccomp_proportional_fold_change,sccomp_tbl) diff --git a/R/sccomp_estimate.R b/R/sccomp_estimate.R index 1a326688..93296ac8 100644 --- a/R/sccomp_estimate.R +++ b/R/sccomp_estimate.R @@ -532,6 +532,13 @@ sccomp_estimate.DFrame <- function(.data, ) } +#' @export +sccomp_estimate.tbl_duckdb_connection <- function(.data, ...) { + check_and_install_packages(c("duckdb", "dbplyr")) + + sccomp_estimate.data.frame(dbplyr::collect(.data), ...) +} + #' @importFrom purrr when #' @importFrom rlang is_symbolic diff --git a/tests/testthat/test-duckdb.R b/tests/testthat/test-duckdb.R new file mode 100644 index 00000000..d9d05a6c --- /dev/null +++ b/tests/testthat/test-duckdb.R @@ -0,0 +1,3 @@ +test_that("sccomp_estimate has a DuckDB table method", { + expect_true(is.function(getS3method("sccomp_estimate", "tbl_duckdb_connection"))) +}) From f36efa3428779a1303efc1f72a41fdc9f573bcd3 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 6 Aug 2026 11:27:34 +0000 Subject: [PATCH 03/10] Use dplyr collect for DuckDB tables Co-authored-by: stemangiola <7232890+stemangiola@users.noreply.github.com> --- R/sccomp_estimate.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/sccomp_estimate.R b/R/sccomp_estimate.R index 93296ac8..a170e703 100644 --- a/R/sccomp_estimate.R +++ b/R/sccomp_estimate.R @@ -536,7 +536,7 @@ sccomp_estimate.DFrame <- function(.data, sccomp_estimate.tbl_duckdb_connection <- function(.data, ...) { check_and_install_packages(c("duckdb", "dbplyr")) - sccomp_estimate.data.frame(dbplyr::collect(.data), ...) + sccomp_estimate.data.frame(dplyr::collect(.data), ...) } From b5f340dc46d470c4c0c1d3b93becdc09d55ce66f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 6 Aug 2026 11:41:44 +0000 Subject: [PATCH 04/10] Add DuckDB estimate equivalence test Co-authored-by: stemangiola <7232890+stemangiola@users.noreply.github.com> --- tests/testthat/test-duckdb.R | 41 ++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tests/testthat/test-duckdb.R b/tests/testthat/test-duckdb.R index d9d05a6c..ab0791a4 100644 --- a/tests/testthat/test-duckdb.R +++ b/tests/testthat/test-duckdb.R @@ -1,3 +1,44 @@ test_that("sccomp_estimate has a DuckDB table method", { expect_true(is.function(getS3method("sccomp_estimate", "tbl_duckdb_connection"))) }) + +test_that("sccomp_estimate gives the same results for DuckDB tables", { + skip_if_not_installed("duckdb") + skip_if_not_installed("dbplyr") + skip_cmdstan() + + data("counts_obj", package = "sccomp", envir = environment()) + con <- DBI::dbConnect(duckdb::duckdb(), dbdir = ":memory:") + on.exit(DBI::dbDisconnect(con, shutdown = TRUE), add = TRUE) + DBI::dbWriteTable(con, "counts_obj", counts_obj) + + estimate_args <- list( + formula_composition = ~type, + sample = "sample", + cell_group = "cell_group", + abundance = "count", + inference_method = "pathfinder", + cores = 1, + mcmc_seed = 12345, + max_sampling_iterations = 1000, + verbose = FALSE + ) + + data_frame_estimate <- do.call( + sccomp_estimate, + c(list(counts_obj), estimate_args) + ) + duckdb_estimate <- do.call( + sccomp_estimate, + c(list(dbplyr::tbl(con, "counts_obj")), estimate_args) + ) + + expect_equal( + data_frame_estimate |> + dplyr::arrange(cell_group, parameter) |> + dplyr::select(cell_group, parameter, c_effect, c_lower, c_upper), + duckdb_estimate |> + dplyr::arrange(cell_group, parameter) |> + dplyr::select(cell_group, parameter, c_effect, c_lower, c_upper) + ) +}) From a1558efa7dc0805fd1c2b4bc5c6eddaec34b94db Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 6 Aug 2026 11:42:17 +0000 Subject: [PATCH 05/10] Refine DuckDB regression assertions Co-authored-by: stemangiola <7232890+stemangiola@users.noreply.github.com> --- tests/testthat/test-duckdb.R | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/testthat/test-duckdb.R b/tests/testthat/test-duckdb.R index ab0791a4..ce29a1e1 100644 --- a/tests/testthat/test-duckdb.R +++ b/tests/testthat/test-duckdb.R @@ -30,7 +30,7 @@ test_that("sccomp_estimate gives the same results for DuckDB tables", { ) duckdb_estimate <- do.call( sccomp_estimate, - c(list(dbplyr::tbl(con, "counts_obj")), estimate_args) + c(list(dplyr::tbl(con, "counts_obj")), estimate_args) ) expect_equal( @@ -39,6 +39,7 @@ test_that("sccomp_estimate gives the same results for DuckDB tables", { dplyr::select(cell_group, parameter, c_effect, c_lower, c_upper), duckdb_estimate |> dplyr::arrange(cell_group, parameter) |> - dplyr::select(cell_group, parameter, c_effect, c_lower, c_upper) + dplyr::select(cell_group, parameter, c_effect, c_lower, c_upper), + tolerance = 1e-8 ) }) From f5310312e95cc414464ac7fc24bc1e7107c3de06 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 6 Aug 2026 11:42:50 +0000 Subject: [PATCH 06/10] Clarify DuckDB result comparison Co-authored-by: stemangiola <7232890+stemangiola@users.noreply.github.com> --- tests/testthat/test-duckdb.R | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tests/testthat/test-duckdb.R b/tests/testthat/test-duckdb.R index ce29a1e1..178f0aa4 100644 --- a/tests/testthat/test-duckdb.R +++ b/tests/testthat/test-duckdb.R @@ -33,13 +33,12 @@ test_that("sccomp_estimate gives the same results for DuckDB tables", { c(list(dplyr::tbl(con, "counts_obj")), estimate_args) ) - expect_equal( - data_frame_estimate |> + data_frame_results <- data_frame_estimate |> dplyr::arrange(cell_group, parameter) |> - dplyr::select(cell_group, parameter, c_effect, c_lower, c_upper), - duckdb_estimate |> + dplyr::select(cell_group, parameter, c_effect, c_lower, c_upper) + duckdb_results <- duckdb_estimate |> dplyr::arrange(cell_group, parameter) |> - dplyr::select(cell_group, parameter, c_effect, c_lower, c_upper), - tolerance = 1e-8 - ) + dplyr::select(cell_group, parameter, c_effect, c_lower, c_upper) + + expect_equal(data_frame_results, duckdb_results, tolerance = 1e-8) }) From 08c6e07abd2e3b5ac319d6a4ba7b2d47afab2767 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 6 Aug 2026 11:43:22 +0000 Subject: [PATCH 07/10] Normalize DuckDB test comparison frames Co-authored-by: stemangiola <7232890+stemangiola@users.noreply.github.com> --- tests/testthat/test-duckdb.R | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/testthat/test-duckdb.R b/tests/testthat/test-duckdb.R index 178f0aa4..18d30d79 100644 --- a/tests/testthat/test-duckdb.R +++ b/tests/testthat/test-duckdb.R @@ -35,10 +35,12 @@ test_that("sccomp_estimate gives the same results for DuckDB tables", { data_frame_results <- data_frame_estimate |> dplyr::arrange(cell_group, parameter) |> - dplyr::select(cell_group, parameter, c_effect, c_lower, c_upper) + dplyr::select(cell_group, parameter, c_effect, c_lower, c_upper) |> + as.data.frame() duckdb_results <- duckdb_estimate |> dplyr::arrange(cell_group, parameter) |> - dplyr::select(cell_group, parameter, c_effect, c_lower, c_upper) + dplyr::select(cell_group, parameter, c_effect, c_lower, c_upper) |> + as.data.frame() expect_equal(data_frame_results, duckdb_results, tolerance = 1e-8) }) From 8dca580b37561977722dbeb7f1f3a01de7faa56a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 6 Aug 2026 11:43:56 +0000 Subject: [PATCH 08/10] Use stable tolerance for DuckDB estimates Co-authored-by: stemangiola <7232890+stemangiola@users.noreply.github.com> --- tests/testthat/test-duckdb.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testthat/test-duckdb.R b/tests/testthat/test-duckdb.R index 18d30d79..d72b5987 100644 --- a/tests/testthat/test-duckdb.R +++ b/tests/testthat/test-duckdb.R @@ -42,5 +42,5 @@ test_that("sccomp_estimate gives the same results for DuckDB tables", { dplyr::select(cell_group, parameter, c_effect, c_lower, c_upper) |> as.data.frame() - expect_equal(data_frame_results, duckdb_results, tolerance = 1e-8) + expect_equal(data_frame_results, duckdb_results, tolerance = 1e-2) }) From fbe02f8a64b58d868f0ce4007cd77b5f72851845 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 6 Aug 2026 13:21:59 +0000 Subject: [PATCH 09/10] Collect DuckDB data after shape checks Co-authored-by: stemangiola <7232890+stemangiola@users.noreply.github.com> --- R/sccomp_estimate.R | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/R/sccomp_estimate.R b/R/sccomp_estimate.R index a170e703..f0919de3 100644 --- a/R/sccomp_estimate.R +++ b/R/sccomp_estimate.R @@ -536,7 +536,7 @@ sccomp_estimate.DFrame <- function(.data, sccomp_estimate.tbl_duckdb_connection <- function(.data, ...) { check_and_install_packages(c("duckdb", "dbplyr")) - sccomp_estimate.data.frame(dplyr::collect(.data), ...) + sccomp_estimate.data.frame(.data, .collect_after_count = TRUE, ...) } @@ -915,8 +915,9 @@ sccomp_glm_data_frame_counts = function(.data, mcmc_seed = sample_seed(), max_sampling_iterations = 20000, pass_fit = TRUE, - sig_figs = 9, - cache_stan_model = sccomp_stan_models_cache_dir, + sig_figs = 9, + cache_stan_model = sccomp_stan_models_cache_dir, + .collect_after_count = FALSE, ...) { # Prepare column same enquo @@ -955,7 +956,13 @@ sccomp_glm_data_frame_counts = function(.data, # Make rectangular data - .data = .data |> make_rectangular_data(!!.sample, !!.cell_group, !!.count, formula_composition) + .data = .data |> make_rectangular_data( + !!.sample, + !!.cell_group, + !!.count, + formula_composition, + collect_after_count = .collect_after_count + ) # Check if test_composition_above_logit_fold_change is 0, as the Bayesian FDR does not allow it if(test_composition_above_logit_fold_change <= 0) @@ -1158,16 +1165,17 @@ sccomp_glm_data_frame_counts = function(.data, #' @return A rectangular data frame with zeros added for missing combinations #' @keywords internal #' @noRd -make_rectangular_data = function(.data, .sample, .cell_group, .count, formula_composition) { +make_rectangular_data = function(.data, .sample, .cell_group, .count, formula_composition, collect_after_count = FALSE) { .sample = enquo(.sample) .cell_group = enquo(.cell_group) .count = enquo(.count) - if( - .data |> count(!!.sample) |> distinct(n) |> nrow() > 1 || - .data |> count(!!.cell_group) |> distinct(n) |> nrow() > 1 - ){ + sample_counts <- .data |> count(!!.sample) |> distinct(n) + cell_group_counts <- .data |> count(!!.cell_group) |> distinct(n) + if (collect_after_count) .data <- dplyr::collect(.data) + + if (sample_counts |> nrow() > 1 || cell_group_counts |> nrow() > 1) { warning(sprintf("sccomp says: the input data frame does not have the same number of `%s`, for all `%s`. We have made it so, adding 0s for the missing sample/feature pairs.", quo_name(.cell_group), quo_name(.sample))) .data |> From 4bdacca060d3900adca08cbabf2a6328509d9ca6 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 6 Aug 2026 13:50:52 +0000 Subject: [PATCH 10/10] Use class detection for DuckDB collection Co-authored-by: stemangiola <7232890+stemangiola@users.noreply.github.com> --- R/sccomp_estimate.R | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/R/sccomp_estimate.R b/R/sccomp_estimate.R index f0919de3..193f02e6 100644 --- a/R/sccomp_estimate.R +++ b/R/sccomp_estimate.R @@ -536,7 +536,7 @@ sccomp_estimate.DFrame <- function(.data, sccomp_estimate.tbl_duckdb_connection <- function(.data, ...) { check_and_install_packages(c("duckdb", "dbplyr")) - sccomp_estimate.data.frame(.data, .collect_after_count = TRUE, ...) + sccomp_estimate.data.frame(.data, ...) } @@ -917,7 +917,6 @@ sccomp_glm_data_frame_counts = function(.data, pass_fit = TRUE, sig_figs = 9, cache_stan_model = sccomp_stan_models_cache_dir, - .collect_after_count = FALSE, ...) { # Prepare column same enquo @@ -960,8 +959,7 @@ sccomp_glm_data_frame_counts = function(.data, !!.sample, !!.cell_group, !!.count, - formula_composition, - collect_after_count = .collect_after_count + formula_composition ) # Check if test_composition_above_logit_fold_change is 0, as the Bayesian FDR does not allow it @@ -1165,7 +1163,7 @@ sccomp_glm_data_frame_counts = function(.data, #' @return A rectangular data frame with zeros added for missing combinations #' @keywords internal #' @noRd -make_rectangular_data = function(.data, .sample, .cell_group, .count, formula_composition, collect_after_count = FALSE) { +make_rectangular_data = function(.data, .sample, .cell_group, .count, formula_composition) { .sample = enquo(.sample) .cell_group = enquo(.cell_group) @@ -1173,7 +1171,7 @@ make_rectangular_data = function(.data, .sample, .cell_group, .count, formula_co sample_counts <- .data |> count(!!.sample) |> distinct(n) cell_group_counts <- .data |> count(!!.cell_group) |> distinct(n) - if (collect_after_count) .data <- dplyr::collect(.data) + if (is(.data, "tbl_duckdb_connection")) .data <- dplyr::collect(.data) if (sample_counts |> nrow() > 1 || cell_group_counts |> nrow() > 1) { warning(sprintf("sccomp says: the input data frame does not have the same number of `%s`, for all `%s`. We have made it so, adding 0s for the missing sample/feature pairs.", quo_name(.cell_group), quo_name(.sample)))