Skip to content
Merged
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 20 additions & 8 deletions parquet/src/encodings/rle.rs
Original file line number Diff line number Diff line change
Expand Up @@ -514,16 +514,28 @@ impl RleDecoder {
break;
}
{
#[cold]
#[inline(never)]
fn oob(max_idx: u32, dict_len: usize) -> ! {
panic!(
"dictionary index out of bounds: the len is {dict_len} but the index is {max_idx}"
)
}
const CHUNK: usize = 16;
let out = &mut buffer[values_read..values_read + num_values];
let idx = &index_buf[..num_values];
let mut out_chunks = out.chunks_exact_mut(8);
let idx_chunks = idx.chunks_exact(8);
let dict_len = dict.len();
let mut out_chunks = out.chunks_exact_mut(CHUNK);
let idx_chunks = idx.chunks_exact(CHUNK);
for (out_chunk, idx_chunk) in out_chunks.by_ref().zip(idx_chunks) {
let dict_len = dict.len();
assert!(
idx_chunk.iter().all(|&i| (i as usize) < dict_len),
"dictionary index out of bounds"
);
// u32 max-reduction instead of `.all(|&i| ..)`: `.all`
Comment thread
Dandandan marked this conversation as resolved.
// short-circuits and blocks autovectorisation. Negative
// i32 cast to u32 becomes a large value so the bounds
// check still rejects it.
let max_idx = idx_chunk.iter().fold(0u32, |acc, &i| acc.max(i as u32));
if (max_idx as usize) >= dict_len {
oob(max_idx, dict_len);
Comment thread
Dandandan marked this conversation as resolved.
Outdated
}
for (b, i) in out_chunk.iter_mut().zip(idx_chunk.iter()) {
// SAFETY: all indices checked above to be in bounds
b.clone_from(unsafe { dict.get_unchecked(*i as usize) });
Expand All @@ -532,7 +544,7 @@ impl RleDecoder {
for (b, i) in out_chunks
.into_remainder()
.iter_mut()
.zip(idx.chunks_exact(8).remainder().iter())
.zip(idx.chunks_exact(CHUNK).remainder().iter())
{
b.clone_from(&dict[*i as usize]);
}
Expand Down
Loading