diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 6abfeaad8dc0..4d0173a864dc 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -305,11 +305,13 @@ jobs: packages: read pull-requests: read secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@codex/caller-cache-directories if: fromJSON(needs.changed-files.outputs.changed_file_groups).cpp_build_inputs || fromJSON(needs.changed-files.outputs.changed_file_groups).cpp_test_files || fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp with: build_type: pull-request script: ci/test_cpp.sh + cache-paths: .cache/libcudf + cache-key-prefix: libcudf-rtcx-v1 # https://github.com/NVIDIA/cudf/issues/23498 matrix_filter: map(select(.GPU != "gb300" and .GPU != "gh200")) conda-python-build: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 7c73aad711c1..7c755c6caf96 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -47,13 +47,15 @@ jobs: packages: read pull-requests: read secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@codex/caller-cache-directories with: build_type: ${{ inputs.build_type }} branch: ${{ inputs.branch }} date: ${{ inputs.date }} script: ci/test_cpp.sh sha: ${{ inputs.sha }} + cache-paths: .cache/libcudf + cache-key-prefix: libcudf-rtcx-v1 # https://github.com/NVIDIA/cudf/issues/23498 matrix_filter: map(select(.GPU != "gb300" and .GPU != "gh200")) conda-cpp-benchmark-tests: diff --git a/ci/test_cpp_common.sh b/ci/test_cpp_common.sh index a0607b278911..4b348eae64f6 100755 --- a/ci/test_cpp_common.sh +++ b/ci/test_cpp_common.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail @@ -33,6 +33,11 @@ RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"} RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${RESULTS_DIR}/test-results"}/ mkdir -p "${RAPIDS_TESTS_DIR}" +# Keep JIT-compiled kernels in the workspace so CI can persist them across jobs. +JIT_CACHE_ROOT=${GITHUB_WORKSPACE:-${PWD}} +export LIBCUDF_KERNEL_CACHE_PATH="${LIBCUDF_KERNEL_CACHE_PATH:-${JIT_CACHE_ROOT}/.cache/libcudf}" +mkdir -p "${LIBCUDF_KERNEL_CACHE_PATH}" + rapids-print-env rapids-logger "Check GPU usage" diff --git a/cpp/tests/ast/jit_expressions_tests.cpp b/cpp/tests/ast/jit_expressions_tests.cpp index 794be830e0e4..55665a5d268d 100644 --- a/cpp/tests/ast/jit_expressions_tests.cpp +++ b/cpp/tests/ast/jit_expressions_tests.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -22,6 +23,7 @@ #include +#include #include #include @@ -586,48 +588,87 @@ constexpr cudf::ast::jit::op get_cast_op() } } -template -void test_cast() -{ - auto a = column_wrapper{{0, 1, 2, 3, 4, 5}}; - auto expected = column_wrapper{{0, 1, 2, 3, 4, 5}}; - auto table = cudf::table_view{{a}}; - auto a_ref = cudf::ast::column_reference(0); - auto tree = cudf::ast::tree{}; - auto result = - cudf::compute_column_jit(table, cudf::ast::jit::operation(tree, get_cast_op(), {a_ref})); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), VERBOSITY); -} - -template -void test_from_decimal_cast() -{ - auto a = decimal_column_wrapper{{0, 1, 2, 3, 4, 5}, numeric::scale_type{0}}; - auto expected = column_wrapper{0, 1, 2, 3, 4, 5}; - auto table = cudf::table_view{{a}}; - auto a_ref = cudf::ast::column_reference(0); - auto tree = cudf::ast::tree{}; - auto result = - cudf::compute_column_jit(table, cudf::ast::jit::operation(tree, get_cast_op(), {a_ref})); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), VERBOSITY); -} - template void test_cast_to() { - test_cast(); - test_cast(); - test_cast(); - test_cast(); - test_cast(); - test_cast(); - test_cast(); - test_cast(); - test_cast(); - test_cast(); - test_from_decimal_cast(); - test_from_decimal_cast(); - test_from_decimal_cast(); + auto const values = std::array{0, 1, 2, 3, 4, 5}; + + auto u8 = column_wrapper(values.begin(), values.end()); + auto u16 = column_wrapper(values.begin(), values.end()); + auto u32 = column_wrapper(values.begin(), values.end()); + auto u64 = column_wrapper(values.begin(), values.end()); + auto i8 = column_wrapper(values.begin(), values.end()); + auto i16 = column_wrapper(values.begin(), values.end()); + auto i32 = column_wrapper(values.begin(), values.end()); + auto i64 = column_wrapper(values.begin(), values.end()); + auto f32 = column_wrapper(values.begin(), values.end()); + auto f64 = column_wrapper(values.begin(), values.end()); + auto d32 = decimal_column_wrapper( + values.begin(), values.end(), numeric::scale_type{0}); + auto d64 = decimal_column_wrapper( + values.begin(), values.end(), numeric::scale_type{0}); + auto d128 = decimal_column_wrapper( + values.begin(), values.end(), numeric::scale_type{0}); + auto table = cudf::table_view{{u8, u16, u32, u64, i8, i16, i32, i64, f32, f64, d32, d64, d128}}; + + auto tree = cudf::ast::tree{}; + auto const op = get_cast_op(); + auto refs = std::array{cudf::ast::column_reference(0), + cudf::ast::column_reference(1), + cudf::ast::column_reference(2), + cudf::ast::column_reference(3), + cudf::ast::column_reference(4), + cudf::ast::column_reference(5), + cudf::ast::column_reference(6), + cudf::ast::column_reference(7), + cudf::ast::column_reference(8), + cudf::ast::column_reference(9), + cudf::ast::column_reference(10), + cudf::ast::column_reference(11), + cudf::ast::column_reference(12)}; + auto& cast_u8 = cudf::ast::jit::operation(tree, op, {refs[0]}); + auto& cast_u16 = cudf::ast::jit::operation(tree, op, {refs[1]}); + auto& cast_u32 = cudf::ast::jit::operation(tree, op, {refs[2]}); + auto& cast_u64 = cudf::ast::jit::operation(tree, op, {refs[3]}); + auto& cast_i8 = cudf::ast::jit::operation(tree, op, {refs[4]}); + auto& cast_i16 = cudf::ast::jit::operation(tree, op, {refs[5]}); + auto& cast_i32 = cudf::ast::jit::operation(tree, op, {refs[6]}); + auto& cast_i64 = cudf::ast::jit::operation(tree, op, {refs[7]}); + auto& cast_f32 = cudf::ast::jit::operation(tree, op, {refs[8]}); + auto& cast_f64 = cudf::ast::jit::operation(tree, op, {refs[9]}); + auto& cast_d32 = cudf::ast::jit::operation(tree, op, {refs[10]}); + auto& cast_d64 = cudf::ast::jit::operation(tree, op, {refs[11]}); + auto& cast_d128 = cudf::ast::jit::operation(tree, op, {refs[12]}); + auto expressions = std::array, 13>{cast_u8, + cast_u16, + cast_u32, + cast_u64, + cast_i8, + cast_i16, + cast_i32, + cast_i64, + cast_f32, + cast_f64, + cast_d32, + cast_d64, + cast_d128}; + auto result = cudf::compute_table_jit(table, expressions); + + auto expected = column_wrapper(values.begin(), values.end()); + auto expected_table = cudf::table_view{{expected, + expected, + expected, + expected, + expected, + expected, + expected, + expected, + expected, + expected, + expected, + expected, + expected}}; + CUDF_TEST_EXPECT_TABLES_EQUAL(expected_table, result->view()); } TEST_F(JITExpressionTest, Cast) diff --git a/cpp/tests/io/comp/comp_test.cpp b/cpp/tests/io/comp/comp_test.cpp index d80ad305023f..882f0bf9fa9e 100644 --- a/cpp/tests/io/comp/comp_test.cpp +++ b/cpp/tests/io/comp/comp_test.cpp @@ -22,6 +22,8 @@ #include +#include +#include #include using cudf::device_span; @@ -420,16 +422,23 @@ void roundtrip_test(cudf::io::compression_type compression) // Keep adding to the test data expected.insert(expected.end(), num_string.begin(), num_string.end()); } - if (cudf::io::detail::compress_max_allowed_chunk_size(compression) - .value_or(std::numeric_limits::max()) < expected.size()) { - // Skip if the data is too large for the compressor - return; - } + } + + // Exercise representative small, medium, and large inputs. The largest input preserves the + // previous test's maximum coverage without repeating the same round trip at every size in + // between. + auto const test_sizes = std::array{size_t{1 << 10}, size_t{1 << 20}, expected.size()}; + auto const max_input_size = cudf::io::detail::compress_max_allowed_chunk_size(compression) + .value_or(std::numeric_limits::max()); + for (auto const test_size : test_sizes) { + if (test_size > max_input_size) { continue; } + + auto const test_input = cudf::host_span{expected.data(), test_size}; auto d_comp = rmm::device_uvector( - cudf::io::detail::max_compressed_size(compression, expected.size()), stream, mr); + cudf::io::detail::max_compressed_size(compression, test_input.size()), stream, mr); { - auto const d_orig = cudf::detail::make_device_uvector_async(expected, stream, mr); + auto const d_orig = cudf::detail::make_device_uvector_async(test_input, stream, mr); auto hd_srcs = cudf::detail::hostdevice_vector>(1, stream); hd_srcs[0] = d_orig; hd_srcs.host_to_device_async(stream); @@ -448,7 +457,7 @@ void roundtrip_test(cudf::io::compression_type compression) d_comp.resize(hd_stats[0].bytes_written, stream); } - auto d_got = rmm::device_uvector(expected.size(), stream); + auto d_got = rmm::device_uvector(test_input.size(), stream); { auto hd_srcs = cudf::detail::hostdevice_vector>(1, stream); hd_srcs[0] = d_comp; @@ -463,14 +472,14 @@ void roundtrip_test(cudf::io::compression_type compression) hd_stats.host_to_device_async(stream); cudf::io::detail::decompress( - compression, hd_srcs, hd_dsts, hd_stats, expected.size(), expected.size(), stream); + compression, hd_srcs, hd_dsts, hd_stats, test_input.size(), test_input.size(), stream); hd_stats.device_to_host(stream); ASSERT_EQ(hd_stats[0].status, codec_status::SUCCESS); } auto const got = cudf::detail::make_std_vector(d_got, stream); - EXPECT_EQ(expected, got); + EXPECT_TRUE(std::equal(test_input.begin(), test_input.end(), got.begin(), got.end())); } } diff --git a/cpp/tests/io/cudftable_test.cpp b/cpp/tests/io/cudftable_test.cpp index 18afff9d41a0..4eb87fd6e18f 100644 --- a/cpp/tests/io/cudftable_test.cpp +++ b/cpp/tests/io/cudftable_test.cpp @@ -554,7 +554,9 @@ TEST_F(CudftableTest, LongStringColumns) TEST_F(CudftableTest, ManyColumns) { - constexpr int num_cols = 12'345; + // Exercise large packed metadata without creating enough tiny device allocations to + // dominate the test when it runs concurrently with other C++ test binaries. + constexpr int num_cols = 1'234; std::vector columns; for (int i = 0; i < num_cols; ++i) { cudf::test::fixed_width_column_wrapper col({i % 10, (i + 1) % 10, (i + 2) % 10}); diff --git a/cpp/tests/io/orc_chunked_reader_test.cu b/cpp/tests/io/orc_chunked_reader_test.cu index ee8c6c634870..2bb79e2d7a1b 100644 --- a/cpp/tests/io/orc_chunked_reader_test.cu +++ b/cpp/tests/io/orc_chunked_reader_test.cu @@ -1128,7 +1128,8 @@ void input_limit_test_read(int test_location, cudf::table_view const& input, output_limit output_limit_bytes, input_limit input_limit_bytes, - int const* expected_chunk_counts) + int const* expected_chunk_counts, + bool require_multiple_chunks = false) { CUDF_EXPECTS(test_files.size() == input_limit_expected_file_count, "Unexpected count of test filenames."); @@ -1142,6 +1143,7 @@ void input_limit_test_read(int test_location, // EXPECT_EQ(expected_chunk_counts[idx], num_chunks); // TODO: equal CUDF_TEST_EXPECT_TABLES_EQUIVALENT(*result, input); + if (require_multiple_chunks) { EXPECT_GT(num_chunks, 1); } } } @@ -1242,7 +1244,7 @@ TEST_F(OrcChunkedReaderInputLimitTest, ListType) // this test runs over 3 hours when racecheck is used if (getenv("LIBCUDF_RACECHECK_ENABLED")) { GTEST_SKIP(); } - int constexpr num_rows = 50'000'000; + int constexpr num_rows = 25'000'000; int constexpr list_size = 4; auto const stream = cudf::get_default_stream(); @@ -1274,13 +1276,18 @@ TEST_F(OrcChunkedReaderInputLimitTest, ListType) // Although we set `stripe_size_rows` to be very large, the writer only write // 250k rows (top level) per stripe due to having nested type. - // Thus, we have 200 stripes in total. + // Thus, we have 100 stripes in total. input_limit_test_write(test_files, input, cudf::io::default_stripe_size_rows); { int constexpr expected[] = {3, 40, 3}; - input_limit_test_read( - __LINE__, test_files, input, output_limit{0UL}, input_limit{5 * 1024 * 1024UL}, expected); + input_limit_test_read(__LINE__, + test_files, + input, + output_limit{0UL}, + input_limit{5 * 1024 * 1024UL}, + expected, + true); } { @@ -1299,7 +1306,7 @@ TEST_F(OrcChunkedReaderInputLimitTest, MixedColumnsHavingList) // this test runs over 3 hours when racecheck is used if (getenv("LIBCUDF_RACECHECK_ENABLED")) { GTEST_SKIP(); } - int constexpr num_rows = 50'000'000; + int constexpr num_rows = 25'000'000; int constexpr list_size = 4; int constexpr str_size = 3; @@ -1360,13 +1367,18 @@ TEST_F(OrcChunkedReaderInputLimitTest, MixedColumnsHavingList) // Although we set `stripe_size_rows` to be very large, the writer only write // 250k rows (top level) per stripe due to having nested type. - // Thus, we have 200 stripes in total. + // Thus, we have 100 stripes in total. input_limit_test_write(test_files, input, cudf::io::default_stripe_size_rows); { int constexpr expected[] = {13, 8, 6}; - input_limit_test_read( - __LINE__, test_files, input, output_limit{0UL}, input_limit{128 * 1024 * 1024UL}, expected); + input_limit_test_read(__LINE__, + test_files, + input, + output_limit{0UL}, + input_limit{128 * 1024 * 1024UL}, + expected, + true); } { @@ -1461,7 +1473,7 @@ TEST_F(OrcChunkedReaderInputLimitTest, SizeTypeRowsOverflow) int64_t constexpr num_rows = 500'000'000l; int constexpr rows_per_stripe = 1'000'000; - int constexpr num_reps = 10; + int constexpr num_reps = 5; int64_t constexpr total_rows = num_rows * num_reps; static_assert(total_rows > std::numeric_limits::max()); diff --git a/cpp/tests/io/parquet_chunked_reader_test.cu b/cpp/tests/io/parquet_chunked_reader_test.cu index 7abb3cf527af..01853f66dcae 100644 --- a/cpp/tests/io/parquet_chunked_reader_test.cu +++ b/cpp/tests/io/parquet_chunked_reader_test.cu @@ -1270,20 +1270,21 @@ void input_limit_test_write(std::vector const& test_filenames, test_filenames[3], t, cudf::io::compression_type::SNAPPY, cudf::io::dictionary_policy::ALWAYS); } -void input_limit_test_read(std::vector const& test_filenames, - cudf::table_view const& t, - std::size_t output_limit, - std::size_t input_limit, - int const expected_chunk_counts[input_limit_expected_file_count]) +void input_limit_test_read( + std::vector const& test_filenames, + cudf::table_view const& t, + std::size_t output_limit, + std::size_t input_limit, + [[maybe_unused]] int const expected_chunk_counts[input_limit_expected_file_count], + bool require_multiple_chunks = false) { CUDF_EXPECTS(test_filenames.size() == input_limit_expected_file_count, "Unexpected count of test filenames"); for (std::size_t idx = 0; idx < test_filenames.size(); idx++) { auto result = chunked_read(test_filenames[idx], output_limit, input_limit); - // CUDF_EXPECTS(result.second == expected_chunk_counts[idx], - // "Unexpected number of chunks produced in chunk read"); CUDF_TEST_EXPECT_TABLES_EQUIVALENT(*result.first, t); + if (require_multiple_chunks) { EXPECT_GT(result.second, 1); } } } } // namespace @@ -1528,7 +1529,7 @@ TEST_F(ParquetChunkedReaderInputLimitTest, List) auto base_path = temp_env->get_temp_filepath("list"); auto test_filenames = input_limit_get_test_names(base_path); - constexpr int num_rows = 10'000'000; + constexpr int num_rows = 2'500'000; constexpr int list_size = 4; auto const stream = cudf::get_default_stream(); @@ -1575,16 +1576,13 @@ TEST_F(ParquetChunkedReaderInputLimitTest, List) // size of the decompressed data. so 2 GB is actually not enough to hold the whole thing at // once. // - // Note that in the dictionary cases, both of these revert down to 1 chunk because the - // dictionaries dramatically shrink the size of the uncompressed data. constexpr int expected_a[] = {3, 3, 1, 1}; - input_limit_test_read(test_filenames, tbl, 0, 256 * 1024 * 1024, expected_a); - // smaller limit + input_limit_test_read(test_filenames, tbl, 0, 64 * 1024 * 1024, expected_a); constexpr int expected_b[] = {5, 5, 2, 1}; - input_limit_test_read(test_filenames, tbl, 0, 128 * 1024 * 1024, expected_b); - // include output chunking as well + input_limit_test_read(test_filenames, tbl, 0, 32 * 1024 * 1024, expected_b); + // Include output chunking as well, and verify each input format is split. constexpr int expected_c[] = {10, 9, 8, 7}; - input_limit_test_read(test_filenames, tbl, 32 * 1024 * 1024, 64 * 1024 * 1024, expected_c); + input_limit_test_read(test_filenames, tbl, 8 * 1024 * 1024, 16 * 1024 * 1024, expected_c, true); } namespace { @@ -1678,7 +1676,7 @@ TEST_F(ParquetChunkedReaderInputLimitTest, Mixed) auto base_path = temp_env->get_temp_filepath("mixed_types"); auto test_filenames = input_limit_get_test_names(base_path); - constexpr int num_rows = 10'000'000; + constexpr int num_rows = 2'500'000; constexpr int list_size = 4; constexpr int str_size = 3; @@ -1757,16 +1755,13 @@ TEST_F(ParquetChunkedReaderInputLimitTest, Mixed) // size of the decompressed data. so 2 GB is actually not enough to hold the whole thing at // once. // - // Note that in the dictionary cases, both of these revert down to 1 chunk because the - // dictionaries dramatically shrink the size of the uncompressed data. constexpr int expected_a[] = {5, 5, 2, 1}; - input_limit_test_read(test_filenames, tbl, 0, 256 * 1024 * 1024, expected_a); - // smaller limit + input_limit_test_read(test_filenames, tbl, 0, 64 * 1024 * 1024, expected_a); constexpr int expected_b[] = {10, 9, 3, 1}; - input_limit_test_read(test_filenames, tbl, 0, 128 * 1024 * 1024, expected_b); - // include output chunking as well + input_limit_test_read(test_filenames, tbl, 0, 32 * 1024 * 1024, expected_b); + // Include output chunking as well, and verify each input format is split. constexpr int expected_c[] = {20, 18, 15, 12}; - input_limit_test_read(test_filenames, tbl, 32 * 1024 * 1024, 64 * 1024 * 1024, expected_c); + input_limit_test_read(test_filenames, tbl, 8 * 1024 * 1024, 16 * 1024 * 1024, expected_c, true); } TEST_F(ParquetChunkedReaderTest, TestChunkedReadOutOfBoundChunks) diff --git a/cpp/tests/io/parquet_writer_test.cpp b/cpp/tests/io/parquet_writer_test.cpp index cb83f6e5d226..426e5e09b04f 100644 --- a/cpp/tests/io/parquet_writer_test.cpp +++ b/cpp/tests/io/parquet_writer_test.cpp @@ -2816,136 +2816,69 @@ TYPED_TEST(ParquetWriterTimestampTypeTest, TimestampsByteStreamSplit) // Base test fixture for "stress" tests struct ParquetWriterStressTest : public cudf::test::BaseFixture {}; -TEST_F(ParquetWriterStressTest, LargeTableWeakCompression) +// Keep row groups aligned with the default 5,000-row page-fragment size. +constexpr cudf::size_type stress_rows_per_row_group = 65'000; +constexpr cudf::size_type stress_num_row_groups = 4; +constexpr cudf::size_type stress_num_rows = stress_rows_per_row_group * stress_num_row_groups; + +template +void write_stress_table(std::unique_ptr const& expected) { std::vector mm_buf; mm_buf.reserve(4 * 1024 * 1024 * 16); - custom_test_memmap_sink custom_sink(&mm_buf); - - // exercises multiple rowgroups - srand(31337); - auto expected = create_random_fixed_table(16, 4 * 1024 * 1024, false); + custom_test_memmap_sink custom_sink(&mm_buf); - // write out using the custom sink (which uses device writes) + // Exercise multiple row groups without depending on the default row-group size. cudf::io::parquet_writer_options args = - cudf::io::parquet_writer_options::builder(cudf::io::sink_info{&custom_sink}, *expected); + cudf::io::parquet_writer_options::builder(cudf::io::sink_info{&custom_sink}, *expected) + .row_group_size_rows(stress_rows_per_row_group); cudf::io::write_parquet(args); - cudf::io::parquet_reader_options custom_args = cudf::io::parquet_reader_options::builder( - cudf::io::source_info{cudf::host_span{ - reinterpret_cast(mm_buf.data()), mm_buf.size()}}); - auto custom_tbl = cudf::io::read_parquet(custom_args); + auto const source = cudf::io::source_info{cudf::host_span{ + reinterpret_cast(mm_buf.data()), mm_buf.size()}}; + EXPECT_EQ(cudf::io::read_parquet_metadata(source).num_rowgroups(), stress_num_row_groups); + + cudf::io::parquet_reader_options custom_args = cudf::io::parquet_reader_options::builder(source); + auto custom_tbl = cudf::io::read_parquet(custom_args); CUDF_TEST_EXPECT_TABLES_EQUAL(custom_tbl.tbl->view(), expected->view()); } -TEST_F(ParquetWriterStressTest, LargeTableGoodCompression) +TEST_F(ParquetWriterStressTest, LargeTableWeakCompression) { - std::vector mm_buf; - mm_buf.reserve(4 * 1024 * 1024 * 16); - custom_test_memmap_sink custom_sink(&mm_buf); - - // exercises multiple rowgroups srand(31337); - auto expected = create_compressible_fixed_table(16, 4 * 1024 * 1024, 128 * 1024, false); - - // write out using the custom sink (which uses device writes) - cudf::io::parquet_writer_options args = - cudf::io::parquet_writer_options::builder(cudf::io::sink_info{&custom_sink}, *expected); - cudf::io::write_parquet(args); + write_stress_table(create_random_fixed_table(16, stress_num_rows, false)); +} - cudf::io::parquet_reader_options custom_args = cudf::io::parquet_reader_options::builder( - cudf::io::source_info{cudf::host_span{ - reinterpret_cast(mm_buf.data()), mm_buf.size()}}); - auto custom_tbl = cudf::io::read_parquet(custom_args); - CUDF_TEST_EXPECT_TABLES_EQUAL(custom_tbl.tbl->view(), expected->view()); +TEST_F(ParquetWriterStressTest, LargeTableGoodCompression) +{ + srand(31337); + write_stress_table( + create_compressible_fixed_table(16, stress_num_rows, 128 * 1024, false)); } TEST_F(ParquetWriterStressTest, LargeTableWithValids) { - std::vector mm_buf; - mm_buf.reserve(4 * 1024 * 1024 * 16); - custom_test_memmap_sink custom_sink(&mm_buf); - - // exercises multiple rowgroups srand(31337); - auto expected = create_compressible_fixed_table(16, 4 * 1024 * 1024, 6, true); - - // write out using the custom sink (which uses device writes) - cudf::io::parquet_writer_options args = - cudf::io::parquet_writer_options::builder(cudf::io::sink_info{&custom_sink}, *expected); - cudf::io::write_parquet(args); - - cudf::io::parquet_reader_options custom_args = cudf::io::parquet_reader_options::builder( - cudf::io::source_info{cudf::host_span{ - reinterpret_cast(mm_buf.data()), mm_buf.size()}}); - auto custom_tbl = cudf::io::read_parquet(custom_args); - CUDF_TEST_EXPECT_TABLES_EQUAL(custom_tbl.tbl->view(), expected->view()); + write_stress_table(create_compressible_fixed_table(16, stress_num_rows, 6, true)); } TEST_F(ParquetWriterStressTest, DeviceWriteLargeTableWeakCompression) { - std::vector mm_buf; - mm_buf.reserve(4 * 1024 * 1024 * 16); - custom_test_memmap_sink custom_sink(&mm_buf); - - // exercises multiple rowgroups srand(31337); - auto expected = create_random_fixed_table(16, 4 * 1024 * 1024, false); - - // write out using the custom sink (which uses device writes) - cudf::io::parquet_writer_options args = - cudf::io::parquet_writer_options::builder(cudf::io::sink_info{&custom_sink}, *expected); - cudf::io::write_parquet(args); - - cudf::io::parquet_reader_options custom_args = cudf::io::parquet_reader_options::builder( - cudf::io::source_info{cudf::host_span{ - reinterpret_cast(mm_buf.data()), mm_buf.size()}}); - auto custom_tbl = cudf::io::read_parquet(custom_args); - CUDF_TEST_EXPECT_TABLES_EQUAL(custom_tbl.tbl->view(), expected->view()); + write_stress_table(create_random_fixed_table(16, stress_num_rows, false)); } TEST_F(ParquetWriterStressTest, DeviceWriteLargeTableGoodCompression) { - std::vector mm_buf; - mm_buf.reserve(4 * 1024 * 1024 * 16); - custom_test_memmap_sink custom_sink(&mm_buf); - - // exercises multiple rowgroups srand(31337); - auto expected = create_compressible_fixed_table(16, 4 * 1024 * 1024, 128 * 1024, false); - - // write out using the custom sink (which uses device writes) - cudf::io::parquet_writer_options args = - cudf::io::parquet_writer_options::builder(cudf::io::sink_info{&custom_sink}, *expected); - cudf::io::write_parquet(args); - - cudf::io::parquet_reader_options custom_args = cudf::io::parquet_reader_options::builder( - cudf::io::source_info{cudf::host_span{ - reinterpret_cast(mm_buf.data()), mm_buf.size()}}); - auto custom_tbl = cudf::io::read_parquet(custom_args); - CUDF_TEST_EXPECT_TABLES_EQUAL(custom_tbl.tbl->view(), expected->view()); + write_stress_table( + create_compressible_fixed_table(16, stress_num_rows, 128 * 1024, false)); } TEST_F(ParquetWriterStressTest, DeviceWriteLargeTableWithValids) { - std::vector mm_buf; - mm_buf.reserve(4 * 1024 * 1024 * 16); - custom_test_memmap_sink custom_sink(&mm_buf); - - // exercises multiple rowgroups srand(31337); - auto expected = create_compressible_fixed_table(16, 4 * 1024 * 1024, 6, true); - - // write out using the custom sink (which uses device writes) - cudf::io::parquet_writer_options args = - cudf::io::parquet_writer_options::builder(cudf::io::sink_info{&custom_sink}, *expected); - cudf::io::write_parquet(args); - - cudf::io::parquet_reader_options custom_args = cudf::io::parquet_reader_options::builder( - cudf::io::source_info{cudf::host_span{ - reinterpret_cast(mm_buf.data()), mm_buf.size()}}); - auto custom_tbl = cudf::io::read_parquet(custom_args); - CUDF_TEST_EXPECT_TABLES_EQUAL(custom_tbl.tbl->view(), expected->view()); + write_stress_table(create_compressible_fixed_table(16, stress_num_rows, 6, true)); } TEST_F(ParquetWriterTest, ReturnedFooterMetadata) diff --git a/cpp/tests/io/text/data_chunk_source_test.cpp b/cpp/tests/io/text/data_chunk_source_test.cpp index 561be9b4d1bf..ddc796563bae 100644 --- a/cpp/tests/io/text/data_chunk_source_test.cpp +++ b/cpp/tests/io/text/data_chunk_source_test.cpp @@ -157,6 +157,19 @@ enum class compression { ENABLED, DISABLED }; enum class eof { ADD_EOF_BLOCK, NO_EOF_BLOCK }; +// 40 MiB exercises two full 16 MiB BGZIP reader loads plus a partial third load. +constexpr int bgzip_input_doublings = 22; + +std::string make_bgzip_test_input() +{ + std::string input{"bananarama"}; + input.reserve(input.size() << bgzip_input_doublings); + for (int i = 0; i < bgzip_input_doublings; ++i) { + input += input; + } + return input; +} + uint64_t virtual_offset(std::size_t block_offset, std::size_t local_offset) { return (block_offset << 16) | local_offset; @@ -225,11 +238,7 @@ using DataChunkDecompressionTest = DecompressionTest; TEST_P(DataChunkDecompressionTest, BgzipSource) { auto const filename = temp_env->get_temp_filepath("bgzip_source"); - std::string input{"bananarama"}; - input.reserve(input.size() << 25); - for (int i = 0; i < 24; i++) { - input = input + input; - } + auto const input = make_bgzip_test_input(); { std::ofstream output_stream{filename}; std::default_random_engine rng{}; @@ -244,11 +253,7 @@ TEST_P(DataChunkDecompressionTest, BgzipSource) TEST_F(DataChunkSourceTest, BgzipSourceVirtualOffsets) { auto const filename = temp_env->get_temp_filepath("bgzip_source_offsets"); - std::string input{"bananarama"}; - input.reserve(input.size() << 25); - for (int i = 0; i < 24; i++) { - input = input + input; - } + auto input = make_bgzip_test_input(); std::string const padding_garbage(10000, 'g'); std::string const data_garbage{"GARBAGE"}; std::string const begininput{"begin of bananarama"}; @@ -332,11 +337,7 @@ TEST_F(DataChunkSourceTest, BgzipSourceVirtualOffsetsSingleChunk) TEST_F(DataChunkSourceTest, BgzipCompressedSourceVirtualOffsets) { auto const filename = temp_env->get_temp_filepath("bgzip_source_compressed_offsets"); - std::string input{"bananarama"}; - input.reserve(input.size() << 25); - for (int i = 0; i < 24; i++) { - input = input + input; - } + auto input = make_bgzip_test_input(); std::string const padding_garbage(10000, 'g'); std::string const data_garbage{"GARBAGE"}; std::string const begininput{"begin of bananarama"}; diff --git a/cpp/tests/large_strings/json_tests.cpp b/cpp/tests/large_strings/json_tests.cpp index d802958ff9a7..436299ea4242 100644 --- a/cpp/tests/large_strings/json_tests.cpp +++ b/cpp/tests/large_strings/json_tests.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -103,10 +103,8 @@ TEST_P(JsonLargeReaderTest, MultiBatch) auto datasources = cudf::io::datasource::create(json_lines_options.get_source().host_buffers()); auto cdatasources = cudf::io::datasource::create(cjson_lines_options.get_source().host_buffers()); - // Test for different chunk sizes + // Test small byte ranges that require many batches and large ranges that exceed a batch. std::vector chunk_sizes{batch_size_upper_bound / 4, - batch_size_upper_bound / 2, - batch_size_upper_bound, static_cast(batch_size_upper_bound * 2)}; for (auto chunk_size : chunk_sizes) {