Skip to content

Commit a7404a9

Browse files
authored
match: bound hash_search collision work (#1132)
* Fix hash_search() duplicate-chain regression (#1115) and collision flood When a hash chain contains > MAX_CHAIN_LEN identical colliding blocks (such as 0x00 sparse regions colliding with 0xFF blocks at 64K/128K block sizes), walking every duplicate entry exhausts MAX_CHAIN_LEN before reaching valid matching blocks further down the chain (#1115). Track the strong checksum of the last failed chain entry (last_false_sum2) and unlink subsequent duplicate entries sharing the same strong checksum (unless protected by SUMFLG_SAME_OFFSET in --inplace mode) so duplicates are neither re-walked nor counted repeatedly against MAX_CHAIN_LEN. Additionally, if 4 consecutive 1-byte rolling offsets experience a weak-checksum collision where the entire chain fails to match, advance offset by a full block rather than computing a full strong checksum on every single byte. * Split hash_search() collision tests and add boundary/--inplace coverage Split testsuite/collision_test.py into two dedicated regression suites: 1. testsuite/hashsearch-dup-chain_test.py: - Verifies forward chains (> MAX_CHAIN_LEN duplicates), multi-group interleaved duplicate chains, and --inplace SUMFLG_SAME_OFFSET preservation when mutating the chain after a subsequent duplicate is overwritten on disk. 2. testsuite/hashsearch-collision-skip_test.py: - Asserts explicit O(N/B) hash_hits CPU bounds and exact literal/matched bandwidth bounds across skip boundaries (3, BLOCK, BLOCK+3, BLOCK+4, 2*BLOCK+3, 2*BLOCK+6, and 10*BLOCK+137) in both standard and --inplace modes, and documents the bunched-perturbation hazard case. * Unlink redundant SUMFLG_SAME_OFFSET duplicates in hash_search() In --inplace mode, when > MAX_CHAIN_LEN identical basis blocks match at their aligned offsets (such as 0x00 sparse runs), every matching entry acquires SUMFLG_SAME_OFFSET. Skipping deduplication for all SUMFLG_SAME_OFFSET entries left all pinned duplicates in the chain, exhausting MAX_CHAIN_LEN, inflating false_alarms, and hiding valid basis blocks behind them. Track the flags of the entry that set last_false_sum2 (last_false_flags). Once a SUMFLG_SAME_OFFSET entry has been retained as last_false_sum2, unlink subsequent duplicate entries carrying SUMFLG_SAME_OFFSET as well, while still preventing unpinned entries from unlinking a pinned entry. Add Case 4 to testsuite/hashsearch-dup-chain_test.py to verify that > MAX_CHAIN_LEN pinned duplicate blocks are unlinked and trailing valid blocks behind them are matched with 0 extra literal bytes.
1 parent 9dc0ff6 commit a7404a9

3 files changed

Lines changed: 504 additions & 9 deletions

File tree

‎match.c‎

Lines changed: 45 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -166,7 +166,9 @@ static void hash_search(int f,struct sum_struct *s,
166166
OFF_T offset, aligned_offset, end;
167167
int32 k, want_i, aligned_i, backup;
168168
char sum2[MAX_DIGEST_LEN];
169+
int32 consecutive_false_offsets = 0;
169170
uint32 s1, s2, sum;
171+
uchar last_false_flags = 0;
170172
int more;
171173
schar *map;
172174

@@ -203,6 +205,8 @@ static void hash_search(int f,struct sum_struct *s,
203205

204206
do {
205207
int done_csum2 = 0;
208+
int matched_block = 0;
209+
const char *last_false_sum2 = NULL;
206210
uint32 hash_entry;
207211
int32 i, *prev;
208212
int32 chain_len = 0;
@@ -214,14 +218,18 @@ static void hash_search(int f,struct sum_struct *s,
214218

215219
if (tablesize == TRADITIONAL_TABLESIZE) {
216220
hash_entry = SUM2HASH2(s1,s2);
217-
if ((i = hash_table[hash_entry]) < 0)
221+
if ((i = hash_table[hash_entry]) < 0) {
222+
consecutive_false_offsets = 0;
218223
goto null_hash;
224+
}
219225
sum = (s1 & 0xffff) | (s2 << 16);
220226
} else {
221227
sum = (s1 & 0xffff) | (s2 << 16);
222228
hash_entry = BIG_SUM2HASH(sum);
223-
if ((i = hash_table[hash_entry]) < 0)
229+
if ((i = hash_table[hash_entry]) < 0) {
230+
consecutive_false_offsets = 0;
224231
goto null_hash;
232+
}
225233
}
226234
prev = &hash_table[hash_entry];
227235

@@ -237,10 +245,20 @@ static void hash_search(int f,struct sum_struct *s,
237245
*prev = s->sums[i].chain;
238246
continue;
239247
}
240-
prev = &s->sums[i].chain;
248+
l = (int32)MIN((OFF_T)s->blength, len-offset);
249+
if (sum != s->sums[i].sum1 || l != s->sums[i].len) {
250+
prev = &s->sums[i].chain;
251+
continue;
252+
}
241253

242-
if (sum != s->sums[i].sum1)
254+
if (last_false_sum2
255+
&& (!(s->sums[i].flags & SUMFLG_SAME_OFFSET)
256+
|| (last_false_flags & SUMFLG_SAME_OFFSET))
257+
&& memcmp(sum2_at(s, i), last_false_sum2, s->s2length) == 0) {
258+
*prev = s->sums[i].chain;
243259
continue;
260+
}
261+
prev = &s->sums[i].chain;
244262

245263
/* Bound the work spent on a single pathological hash
246264
* bucket. If this weak checksum matches more than
@@ -250,11 +268,6 @@ static void hash_search(int f,struct sum_struct *s,
250268
if (++chain_len > MAX_CHAIN_LEN)
251269
break;
252270

253-
/* also make sure the two blocks are the same length */
254-
l = (int32)MIN((OFF_T)s->blength, len-offset);
255-
if (l != s->sums[i].len)
256-
continue;
257-
258271
if (DEBUG_GTE(DELTASUM, 3)) {
259272
rprintf(FINFO,
260273
"potential match at %s i=%ld sum=%08x\n",
@@ -269,6 +282,8 @@ static void hash_search(int f,struct sum_struct *s,
269282

270283
if (memcmp(sum2, sum2_at(s, i), s->s2length) != 0) {
271284
false_alarms++;
285+
last_false_sum2 = sum2_at(s, i);
286+
last_false_flags = s->sums[i].flags;
272287
continue;
273288
}
274289

@@ -341,8 +356,29 @@ static void hash_search(int f,struct sum_struct *s,
341356
s1 = sum & 0xFFFF;
342357
s2 = sum >> 16;
343358
matches++;
359+
matched_block = 1;
360+
consecutive_false_offsets = 0;
344361
break;
345362
} while ((i = s->sums[i].chain) >= 0);
363+
/* If we computed a strong checksum (weak collision) and the entire
364+
* chain failed to match, don't let repeated collisions force a
365+
* 1-byte walk with a full strong-checksum calculation on every byte. */
366+
if (!matched_block && (done_csum2 || chain_len > MAX_CHAIN_LEN)) {
367+
if (++consecutive_false_offsets >= 4 && k > 1) {
368+
OFF_T skip = MIN((OFF_T)(k - 1), end - 1 - offset);
369+
if (skip > 0) {
370+
offset += skip;
371+
k = (int32)MIN((OFF_T)s->blength, len-offset);
372+
map = (schar *)map_ptr(buf, offset, k);
373+
sum = get_checksum1((char *)map, k);
374+
s1 = sum & 0xFFFF;
375+
s2 = sum >> 16;
376+
}
377+
consecutive_false_offsets = 3;
378+
}
379+
}
380+
else if (!matched_block)
381+
consecutive_false_offsets = 0;
346382

347383
null_hash:
348384
backup = (int32)(offset - last_match);
Lines changed: 234 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,234 @@
1+
#!/usr/bin/env python3
2+
"""Regression test for hash_search() collision-flood bounds and skip boundaries.
3+
4+
Verifies that when rolling across unmatched constant-checksum collision runs:
5+
1. CPU work (hash_hits and false_alarms) is strictly bounded at O(N / blength)
6+
rather than computing a strong checksum on every byte O(N).
7+
2. Bandwidth (Literal data vs Matched data) is exact: valid basis blocks placed
8+
immediately after colliding runs at every critical skip boundary are 100%
9+
matched without skipping a single valid block:
10+
- Sub-threshold collision run (3 bytes < 4-offset threshold)
11+
- Exact 1-block boundary (BLOCK bytes)
12+
- Exact 1st-skip landing boundary (BLOCK + 3 bytes)
13+
- 1-byte past 1st-skip landing boundary (BLOCK + 4 bytes)
14+
- Exact 2nd-skip landing boundary (2 * BLOCK + 3 bytes)
15+
- Multi-skip + sub-threshold roll (2 * BLOCK + 6 bytes)
16+
- Long multi-block flood + arbitrary unaligned offset (10 * BLOCK + 137 bytes)
17+
3. `--inplace` mode preserves exact boundary matches across collision skips.
18+
4. Crafted weak-checksum collisions with bunched perturbations (e.g., (+1, -2, +1)
19+
at bytes 0..2) trigger premature boundary skips due to partial window overlap,
20+
resulting in *increased literal data* (safely falling back to literal transmission
21+
rather than corrupting the file or hanging the CPU).
22+
"""
23+
24+
import re
25+
26+
from rsyncfns import (
27+
FROMDIR, TODIR, assert_same, rmtree, run_rsync, test_fail,
28+
)
29+
30+
BLOCK = 704
31+
C_BYTE = 100
32+
33+
34+
def make_spread_colliding_block(delta: int, fill: int = C_BYTE) -> bytes:
35+
b = bytearray([fill]) * BLOCK
36+
b[0] = fill + delta
37+
b[351] = fill - delta
38+
b[352] = fill - delta
39+
b[703] = fill + delta
40+
return bytes(b)
41+
42+
43+
def make_block(perturb_offset: int = -1, fill: int = C_BYTE) -> bytes:
44+
"""Crafted weak-checksum collision block ((+1, -2, +1) perturbation at 0..2).
45+
Because perturbations are bunched at the start, partial boundary windows
46+
falsely collide, triggering premature skips and resulting in increased literal data.
47+
"""
48+
b = bytearray([fill]) * BLOCK
49+
if perturb_offset >= 0:
50+
b[perturb_offset] = fill + 1
51+
b[perturb_offset + 1] = fill - 2
52+
b[perturb_offset + 2] = fill + 1
53+
return bytes(b)
54+
55+
56+
def parse_stats(out: str):
57+
m_delta = re.search(r'hash_hits=(\d+)\s+false_alarms=(\d+)', out)
58+
m_lit = re.search(r'Literal data:\s*([\d,]+)\s*bytes', out)
59+
m_match = re.search(r'Matched data:\s*([\d,]+)\s*bytes', out)
60+
if not m_delta or not m_lit or not m_match:
61+
test_fail(f"could not find deltasum/stats in rsync output:\n{out}")
62+
return (
63+
int(m_delta.group(1)),
64+
int(m_delta.group(2)),
65+
int(m_lit.group(1).replace(',', '')),
66+
int(m_match.group(1).replace(',', '')),
67+
)
68+
69+
70+
block_a = make_spread_colliding_block(5)
71+
block_d = make_spread_colliding_block(9)
72+
73+
rmtree(FROMDIR)
74+
rmtree(TODIR)
75+
FROMDIR.mkdir(parents=True, exist_ok=True)
76+
TODIR.mkdir(parents=True, exist_ok=True)
77+
78+
# ---------------------------------------------------------------------------
79+
# Case 1: Skip-boundary match preservation + CPU/bandwidth bounds
80+
# ---------------------------------------------------------------------------
81+
src1 = FROMDIR / 'skip_boundaries.bin'
82+
dst1 = TODIR / 'skip_boundaries.bin'
83+
84+
dst1_payload = (block_a * 2) + (block_d * 2) + (block_a * 14) + (block_d * 14)
85+
dst1.write_bytes(dst1_payload)
86+
87+
boundary_runs = [
88+
bytes([C_BYTE]) * 3, # < 4 misses (no skip, rolls into match)
89+
bytes([C_BYTE]) * BLOCK, # Exact 1-block boundary
90+
bytes([C_BYTE]) * (BLOCK + 3), # Exact landing offset of 1st skip (3 + BLOCK)
91+
bytes([C_BYTE]) * (BLOCK + 4), # 1 byte after 1st skip landing
92+
bytes([C_BYTE]) * (2 * BLOCK + 3), # Exact landing offset of 2nd consecutive skip
93+
bytes([C_BYTE]) * (2 * BLOCK + 6), # 2 skips + 3-byte sub-threshold roll
94+
bytes([C_BYTE]) * (10 * BLOCK + 137), # Long flood + unaligned remainder
95+
]
96+
97+
src1_parts = [(block_a * 2) + (block_d * 2)]
98+
for idx, run in enumerate(boundary_runs):
99+
valid_group = (block_d * 4) if (idx % 2 == 0) else (block_a * 4)
100+
src1_parts.append(run)
101+
src1_parts.append(valid_group)
102+
103+
src1_payload = b''.join(src1_parts)
104+
src1.write_bytes(src1_payload)
105+
106+
proc1 = run_rsync('-a', '-I', '--no-whole-file', f'--block-size={BLOCK}',
107+
'--no-compress', '--stats', '--debug=deltasum1',
108+
f'{src1}', f'{dst1}', capture_output=True)
109+
110+
hash_hits1, false_alarms1, literal1, matched1 = parse_stats(
111+
proc1.stdout + proc1.stderr
112+
)
113+
114+
total_valid_blocks = 4 * (1 + len(boundary_runs)) # 32 blocks = 22,528 bytes
115+
expected_matched1 = total_valid_blocks * BLOCK
116+
expected_literal1 = sum(len(r) for r in boundary_runs)
117+
118+
if matched1 != expected_matched1 or literal1 != expected_literal1:
119+
test_fail(
120+
f"Case 1 skip boundary missed valid basis blocks: "
121+
f"matched={matched1} (expected {expected_matched1}), "
122+
f"literal={literal1} (expected {expected_literal1})"
123+
)
124+
125+
MAX_EXPECTED_HITS1 = 160
126+
if hash_hits1 > MAX_EXPECTED_HITS1:
127+
test_fail(
128+
f"Case 1 collision flood rolled 1 byte at a time: hash_hits={hash_hits1} "
129+
f"(expected <= {MAX_EXPECTED_HITS1})"
130+
)
131+
132+
assert_same(dst1, src1, label='Case 1: collision skip-boundary transfer')
133+
134+
# ---------------------------------------------------------------------------
135+
# Case 2: Pure unmatched collision flood CPU bound (O(N/B) vs O(N))
136+
# ---------------------------------------------------------------------------
137+
src2 = FROMDIR / 'pure_flood.bin'
138+
dst2 = TODIR / 'pure_flood.bin'
139+
140+
N_FLOOD = 12
141+
dst2.write_bytes(block_a * N_FLOOD)
142+
src2.write_bytes(bytes([C_BYTE]) * (N_FLOOD * BLOCK))
143+
144+
proc2 = run_rsync('-a', '-I', '--no-whole-file', f'--block-size={BLOCK}',
145+
'--no-compress', '--stats', '--debug=deltasum1',
146+
f'{src2}', f'{dst2}', capture_output=True)
147+
148+
hash_hits2, false_alarms2, literal2, matched2 = parse_stats(
149+
proc2.stdout + proc2.stderr
150+
)
151+
152+
MAX_EXPECTED_HITS2 = N_FLOOD * 3
153+
if hash_hits2 > MAX_EXPECTED_HITS2:
154+
test_fail(
155+
f"Case 2 pure flood rolled 1 byte at a time: hash_hits={hash_hits2} "
156+
f"(expected <= {MAX_EXPECTED_HITS2})"
157+
)
158+
159+
assert_same(dst2, src2, label='Case 2: pure collision flood transfer')
160+
161+
# ---------------------------------------------------------------------------
162+
# Case 3: Skip-boundary match preservation under --inplace
163+
# ---------------------------------------------------------------------------
164+
src3 = FROMDIR / 'inplace_skip.bin'
165+
dst3 = TODIR / 'inplace_skip.bin'
166+
167+
dst3.write_bytes(dst1_payload)
168+
src3.write_bytes(src1_payload)
169+
170+
proc3 = run_rsync('-a', '-I', '--inplace', '--no-whole-file',
171+
f'--block-size={BLOCK}', '--no-compress',
172+
'--stats', '--debug=deltasum1',
173+
f'{src3}', f'{dst3}', capture_output=True)
174+
175+
hash_hits3, _, literal3, matched3 = parse_stats(proc3.stdout + proc3.stderr)
176+
177+
if matched3 != expected_matched1 or literal3 != expected_literal1:
178+
test_fail(
179+
f"Case 3 (--inplace skip boundaries) missed valid matches: "
180+
f"matched={matched3} (expected {expected_matched1}), "
181+
f"literal={literal3} (expected {expected_literal1})"
182+
)
183+
184+
if hash_hits3 > MAX_EXPECTED_HITS1:
185+
test_fail(
186+
f"Case 3 (--inplace) rolled 1 byte at a time: hash_hits={hash_hits3} "
187+
f"(expected <= {MAX_EXPECTED_HITS1})"
188+
)
189+
190+
assert_same(dst3, src3, label='Case 3: --inplace collision skip-boundary transfer')
191+
192+
# ---------------------------------------------------------------------------
193+
# Case 4: Bunched-perturbation hazard demonstration (make_block vs spread)
194+
# ---------------------------------------------------------------------------
195+
# Documents that if a colliding block uses bunched perturbations (like (+1, -2, +1)
196+
# at bytes 0..2), partial overlapping boundary windows falsely trigger premature
197+
# skips, jumping over valid blocks and resulting in *increased literal data*
198+
# (safe fallback to network transfer) while preserving 100% file integrity.
199+
# Odd block sizes here should not result in more literal data.
200+
src4 = FROMDIR / 'bunched_hazard.bin'
201+
dst4 = TODIR / 'bunched_hazard.bin'
202+
203+
# Valid block built with bunched perturbation (make_block)
204+
bunched_block = make_block(0)
205+
206+
dst4.write_bytes(bunched_block * 16)
207+
208+
# Interleave pure C_BYTE gaps with bunched blocks
209+
src4_payload = (
210+
(bunched_block * 2)
211+
+ (bytes([C_BYTE]) * BLOCK) # 1-block boundary gap
212+
+ (bunched_block * 4)
213+
)
214+
src4.write_bytes(src4_payload)
215+
216+
proc4 = run_rsync('-a', '-I', '--no-whole-file', f'--block-size={BLOCK}',
217+
'--no-compress', '--stats', '--debug=deltasum1',
218+
f'{src4}', f'{dst4}', capture_output=True)
219+
220+
hash_hits4, _, literal4, matched4 = parse_stats(proc4.stdout + proc4.stderr)
221+
222+
# Because bunched perturbations trigger premature skips, literal data is
223+
# higher than the exact gap length (> 704 bytes), but file integrity holds.
224+
EXPECTED_BASE_LITERAL = BLOCK + 10
225+
if literal4 <= EXPECTED_BASE_LITERAL:
226+
test_fail(f"Case 4 expected bunched perturbation to cause literal data inflation, got {literal4} bytes")
227+
228+
assert_same(dst4, src4, label='Case 4: bunched-perturbation hazard transfer')
229+
230+
print(
231+
f"collision-skip test passed: boundary_hits={hash_hits1}, "
232+
f"flood_hits={hash_hits2}, inplace_hits={hash_hits3}, "
233+
f"bunched_literal={literal4} (exceeded {EXPECTED_BASE_LITERAL} due to premature skip)"
234+
)

0 commit comments

Comments
 (0)