From c59112a1107c4d028df75811a86875a0c13c157a Mon Sep 17 00:00:00 2001 From: Saurav Joshi Date: Thu, 12 Nov 2026 21:53:08 +0000 Subject: [PATCH 01/13] xdiff: add Myers regression and performance tests Cover unchanged-span boundaries, common records that also appear in the middle, and equal fingerprints with different line contents. Add a Myers shortstat performance case over 8,192 non-merge commits from the perf test repository. Signed-off-by: Saurav Joshi --- t/perf/p4000-diff-algorithms.sh | 10 +++++ t/t4061-diff-indent.sh | 74 +++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) diff --git a/t/perf/p4000-diff-algorithms.sh b/t/perf/p4000-diff-algorithms.sh index 7e00c9da47d7a3..a41963c5fb462a 100755 --- a/t/perf/p4000-diff-algorithms.sh +++ b/t/perf/p4000-diff-algorithms.sh @@ -18,6 +18,16 @@ test_perf 'log -p -3000 (Myers)' ' git log -p -3000 >/dev/null ' +test_expect_success 'prepare revisions for Myers shortstat' ' + git rev-list --no-merges --max-count=8192 HEAD >commits +' + +test_perf 'diff-tree --shortstat (Myers)' ' + git diff-tree --stdin -r --root --no-renames --no-ext-diff \ + --no-textconv --no-color --diff-algorithm=myers --shortstat \ + /dev/null +' + test_perf 'log -p -3000 --histogram' ' git log -p -3000 --histogram >/dev/null ' diff --git a/t/t4061-diff-indent.sh b/t/t4061-diff-indent.sh index 7750b87ca16ab3..524cd59e7e3333 100755 --- a/t/t4061-diff-indent.sh +++ b/t/t4061-diff-indent.sh @@ -368,4 +368,78 @@ test_expect_success 'diff-files: --no-indent-heuristic overrides config' ' git checkout -f main ' +for prefix in 1 7 8 9 15 16 17 +do + test_expect_success "unchanged spans preserve insertion and deletion positions at $prefix" ' + test_seq 1 "$prefix" >prefix && + test_seq 100 180 >suffix && + cat prefix suffix >before.txt && + { + cat prefix && + echo inserted && + cat suffix + } >after.txt && + printf "@@ -%s,0 +%s @@\n+inserted\n" \ + "$prefix" "$((prefix + 1))" >expect && + test_expect_code 1 git diff --no-index --diff-algorithm=myers \ + --unified=0 before.txt after.txt >output && + sed -n "/^@@/,\$p" output >actual && + test_cmp expect actual && + printf "@@ -%s +%s,0 @@\n-inserted\n" \ + "$((prefix + 1))" "$prefix" >expect && + test_expect_code 1 git diff --no-index --diff-algorithm=myers \ + --unified=0 after.txt before.txt >output && + sed -n "/^@@/,\$p" output >actual && + test_cmp expect actual + ' +done + +test_expect_success 'common records that also occur in the middle retain their classes' ' + cat >before.txt <<-\EOF && + anchor + shared + changed-before + shared + end + EOF + cat >after.txt <<-\EOF && + anchor + shared + shared + shared + end + EOF + cat >expect <<-\EOF && + @@ -3 +3 @@ shared + -changed-before + +shared + EOF + test_expect_code 1 git diff --no-index --diff-algorithm=myers \ + --no-indent-heuristic --unified=0 before.txt after.txt >output && + sed -n "/^@@/,\$p" output >actual && + test_cmp expect actual +' + +test_expect_success 'matching line fingerprints do not equate different middle bytes' ' + cat >before.txt <<-\EOF && + abcdefghAAAAAAAAAAAAAAAAtuvwxyz + abcdefghBBBBBBBBBBBBBBBBtuvwxyz + abcdefghAAAAAAAAAAAAAAAAtuvwxyz + EOF + cat >after.txt <<-\EOF && + abcdefghAAAAAAAAAAAAAAAAtuvwxyz + abcdefghCCCCCCCCCCCCCCCCtuvwxyz + abcdefghAAAAAAAAAAAAAAAAtuvwxyz + EOF + cat >expect <<-\EOF && + @@ -2 +2 @@ abcdefghAAAAAAAAAAAAAAAAtuvwxyz + -abcdefghBBBBBBBBBBBBBBBBtuvwxyz + +abcdefghCCCCCCCCCCCCCCCCtuvwxyz + EOF + test_expect_code 1 git diff --no-index --diff-algorithm=myers \ + --no-indent-heuristic --unified=0 before.txt after.txt >output && + sed -n "/^@@/,\$p" output >actual && + test_cmp expect actual +' + test_done From 257c10864aef2705026d2dd08061f64149de2ad0 Mon Sep 17 00:00:00 2001 From: Saurav Joshi Date: Thu, 12 Nov 2026 21:53:09 +0000 Subject: [PATCH 02/13] xdiff: batch diff output callbacks Emitting records individually adds callback and buffer-walking overhead. Buffer complete records and flush before hunk headers and large records. Coalesce small prefixes, record contents and optional missing-newline markers into a single callback buffer. Preserve callback failures and the existing large-record fallback. Signed-off-by: Saurav Joshi --- xdiff/xemit.c | 84 ++++++++++++++++++++++++++++++++++++++------------ xdiff/xutils.c | 23 ++++++++++++++ 2 files changed, 88 insertions(+), 19 deletions(-) diff --git a/xdiff/xemit.c b/xdiff/xemit.c index 7cd9cf0a44f4d6..b73828359279fb 100644 --- a/xdiff/xemit.c +++ b/xdiff/xemit.c @@ -23,13 +23,52 @@ #include "xinclude.h" -static int xdl_emit_record(xdfile_t *xdf, long ri, char const *pre, xdemitcb_t *ecb) -{ - xrecord_t *rec = &xdf->recs[ri]; +/* + * Batch complete diff lines without changing their order or bytes. + * Flush before headers, exceptional records and the final return. + */ +struct xdl_emit_buf { + size_t n; + char buf[16384]; +}; - if (xdl_emit_diffrec((char const *)rec->ptr, (long)rec->size, pre, strlen(pre), ecb) < 0) +static int xdl_emit_flush(struct xdl_emit_buf *eb, xdemitcb_t *ecb) +{ + mmfile_t mb; + + if (!eb->n) + return 0; + mb.ptr = eb->buf; + mb.size = eb->n; + eb->n = 0; + if (ecb->out_line(ecb->priv, &mb, 1) < 0) return -1; + return 0; +} + +static int xdl_emit_records(struct xdl_emit_buf *eb, xdfile_t *xdf, + long s, long e, char pre, xdemitcb_t *ecb) +{ + for (; s < e; s++) { + xrecord_t *rec = &xdf->recs[s]; + size_t size = rec->size; + + if (!size || rec->ptr[size - 1] != '\n' || + size >= sizeof(eb->buf)) { + if (xdl_emit_flush(eb, ecb) < 0 || + xdl_emit_diffrec((char const *)rec->ptr, (long)size, + &pre, 1, ecb) < 0) + return -1; + continue; + } + if (eb->n + size + 1 > sizeof(eb->buf) && + xdl_emit_flush(eb, ecb) < 0) + return -1; + eb->buf[eb->n++] = pre; + memcpy(eb->buf + eb->n, rec->ptr, size); + eb->n += size; + } return 0; } @@ -172,6 +211,9 @@ int xdl_emit_diff(xdfenv_t *xe, xdchange_t *xscr, xdemitcb_t *ecb, xdchange_t *xch, *xche; long funclineprev = -1; struct func_line func_line = { 0 }; + struct xdl_emit_buf eb; + + eb.n = 0; for (xch = xscr; xch; xch = xche->next) { xdchange_t *xchp = xch; @@ -280,6 +322,8 @@ int xdl_emit_diff(xdfenv_t *xe, xdchange_t *xscr, xdemitcb_t *ecb, s1 - 1, funclineprev); funclineprev = s1 - 1; } + if (xdl_emit_flush(&eb, ecb) < 0) + return -1; if (!(xecfg->flags & XDL_EMIT_NO_HUNK_HDR) && xdl_emit_hunk_hdr(s1 + 1, e1 - s1, s2 + 1, e2 - s2, func_line.buf, func_line.len, ecb) < 0) @@ -288,31 +332,33 @@ int xdl_emit_diff(xdfenv_t *xe, xdchange_t *xscr, xdemitcb_t *ecb, /* * Emit pre-context. */ - for (; s2 < xch->i2; s2++) - if (xdl_emit_record(&xe->xdf2, s2, " ", ecb) < 0) - return -1; + if (xdl_emit_records(&eb, &xe->xdf2, s2, xch->i2, ' ', ecb) < 0) + return -1; for (s1 = xch->i1, s2 = xch->i2;; xch = xch->next) { /* * Merge previous with current change atom. */ - for (; s1 < xch->i1 && s2 < xch->i2; s1++, s2++) - if (xdl_emit_record(&xe->xdf2, s2, " ", ecb) < 0) + if (s1 < xch->i1 && s2 < xch->i2) { + long k = XDL_MIN(xch->i1 - s1, xch->i2 - s2); + + if (xdl_emit_records(&eb, &xe->xdf2, s2, s2 + k, ' ', ecb) < 0) return -1; + } /* * Removes lines from the first file. */ - for (s1 = xch->i1; s1 < xch->i1 + xch->chg1; s1++) - if (xdl_emit_record(&xe->xdf1, s1, "-", ecb) < 0) - return -1; + if (xdl_emit_records(&eb, &xe->xdf1, xch->i1, + xch->i1 + xch->chg1, '-', ecb) < 0) + return -1; /* * Adds lines from the second file. */ - for (s2 = xch->i2; s2 < xch->i2 + xch->chg2; s2++) - if (xdl_emit_record(&xe->xdf2, s2, "+", ecb) < 0) - return -1; + if (xdl_emit_records(&eb, &xe->xdf2, xch->i2, + xch->i2 + xch->chg2, '+', ecb) < 0) + return -1; if (xch == xche) break; @@ -323,10 +369,10 @@ int xdl_emit_diff(xdfenv_t *xe, xdchange_t *xscr, xdemitcb_t *ecb, /* * Emit post-context. */ - for (s2 = xche->i2 + xche->chg2; s2 < e2; s2++) - if (xdl_emit_record(&xe->xdf2, s2, " ", ecb) < 0) - return -1; + if (xdl_emit_records(&eb, &xe->xdf2, xche->i2 + xche->chg2, + e2, ' ', ecb) < 0) + return -1; } - return 0; + return xdl_emit_flush(&eb, ecb); } diff --git a/xdiff/xutils.c b/xdiff/xutils.c index 4215f646c5f846..4ed45bf47d9d77 100644 --- a/xdiff/xutils.c +++ b/xdiff/xutils.c @@ -40,6 +40,29 @@ int xdl_emit_diffrec(char const *rec, long size, char const *pre, long psize, xdemitcb_t *ecb) { int i = 2; mmfile_t mb[3]; + char buf[512]; + static const char nonl[] = "\n\\ No newline at end of file\n"; + + /* + * Small records, including an optional no-newline marker, fit in + * one callback buffer. Bound each length before adding them. + */ + if (size > 0 && psize >= 0 && + (size_t)psize <= sizeof(buf) - sizeof(nonl) && + (size_t)size <= sizeof(buf) - sizeof(nonl) - (size_t)psize) { + long n = psize; + + memcpy(buf, pre, psize); + memcpy(buf + n, rec, size); + n += size; + if (rec[size - 1] != '\n') { + memcpy(buf + n, nonl, sizeof(nonl) - 1); + n += sizeof(nonl) - 1; + } + mb[0].ptr = buf; + mb[0].size = n; + return ecb->out_line(ecb->priv, mb, 1) < 0 ? -1 : 0; + } mb[0].ptr = (char *) pre; mb[0].size = psize; From 74bb10c44bffdf668e9d0639f46a0b9ae1389cc8 Mon Sep 17 00:00:00 2001 From: Saurav Joshi Date: Thu, 12 Nov 2026 15:16:47 -0800 Subject: [PATCH 03/13] xdiff: store classes in an open-addressed table Each new line class is allocated from a chastore and chained into a hash bucket, and every lookup walks that chain through pointers. Keep the classes in one contiguous array and index them from a compact open-addressed table of 32-bit slots that grows when half full. For verbatim comparisons, check the length before comparing bytes, and compare eight bytes at a time. The table uses the high bits of the line hash, so mix the existing hash before lookup. The mix is bijective and does not change which lines are considered equal. Signed-off-by: Saurav Joshi --- xdiff/xprepare.c | 186 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 129 insertions(+), 57 deletions(-) diff --git a/xdiff/xprepare.c b/xdiff/xprepare.c index 11bada2608a7a4..8091446f22324e 100644 --- a/xdiff/xprepare.c +++ b/xdiff/xprepare.c @@ -35,102 +35,170 @@ typedef struct s_xdlclass { uint64_t line_hash; - struct s_xdlclass *next; const uint8_t *ptr; size_t size; - long idx; long len1, len2; } xdlclass_t; +/* + * A slot holds a class index plus one; zero denotes an empty slot. + * Keep hashes in the contiguous class array rather than in each slot. + */ +typedef uint32_t xdlslot_t; + typedef struct s_xdlclassifier { - unsigned int hbits; - long hsize; - xdlclass_t **rchash; - chastore_t ncha; - xdlclass_t **rcrecs; + unsigned int tbits; + size_t tmask; + xdlslot_t *table; + xdlclass_t *rcrecs; long alloc; long count; long flags; } xdlclassifier_t; +#define XDL_CLASS_MAX ((long)0x7ffffffe) +/* + * SIZE estimates the record count; TSIZE estimates the number of classes + * actually inserted into the table. Both arrays grow when necessary. + */ +static int xdl_init_classifier_sized(xdlclassifier_t *cf, long size, long tsize, + long flags) { + unsigned int bits; -static int xdl_init_classifier(xdlclassifier_t *cf, long size, long flags) { cf->flags = flags; + bits = xdl_hashbits((unsigned int)tsize) + 1; + if (bits > 31) + bits = 31; + cf->tbits = bits; + cf->tmask = ((size_t)1 << bits) - 1; - cf->hbits = xdl_hashbits((unsigned int) size); - cf->hsize = 1 << cf->hbits; - - if (xdl_cha_init(&cf->ncha, sizeof(xdlclass_t), size / 4 + 1) < 0) { - - return -1; - } - if (!XDL_CALLOC_ARRAY(cf->rchash, cf->hsize)) { - - xdl_cha_free(&cf->ncha); + if (!XDL_CALLOC_ARRAY(cf->table, cf->tmask + 1)) return -1; - } cf->alloc = size; if (!XDL_ALLOC_ARRAY(cf->rcrecs, cf->alloc)) { - - xdl_free(cf->rchash); - xdl_cha_free(&cf->ncha); + xdl_free(cf->table); + cf->table = NULL; return -1; } cf->count = 0; - return 0; } -static void xdl_free_classifier(xdlclassifier_t *cf) { +static int xdl_init_classifier(xdlclassifier_t *cf, long size, long flags) { + return xdl_init_classifier_sized(cf, size, size, flags); +} + +static void xdl_free_classifier(xdlclassifier_t *cf) { xdl_free(cf->rcrecs); - xdl_free(cf->rchash); - xdl_cha_free(&cf->ncha); + xdl_free(cf->table); } -static int xdl_classify_record(unsigned int pass, xdlclassifier_t *cf, xrecord_t *rec, - uint64_t line_hash) { - size_t hi; - xdlclass_t *rcrec; +static inline size_t xdl_slot_index(xdlclassifier_t *cf, uint64_t line_hash) { + return (size_t)(line_hash >> (64 - cf->tbits)); +} - hi = XDL_HASHLONG(line_hash, cf->hbits); - for (rcrec = cf->rchash[hi]; rcrec; rcrec = rcrec->next) - if (rcrec->line_hash == line_hash && - xdl_recmatch((const char *)rcrec->ptr, (long)rcrec->size, - (const char *)rec->ptr, (long)rec->size, cf->flags)) - break; - if (!rcrec) { - if (!(rcrec = xdl_cha_alloc(&cf->ncha))) { +static int xdl_grow_table(xdlclassifier_t *cf) { + xdlslot_t *ntab; + unsigned int nbits = cf->tbits + 1; + size_t nmask; + long i; - return -1; - } - rcrec->idx = cf->count++; - if (XDL_ALLOC_GROW(cf->rcrecs, cf->count, cf->alloc)) - return -1; - cf->rcrecs[rcrec->idx] = rcrec; - rcrec->line_hash = line_hash; - rcrec->ptr = rec->ptr; - rcrec->size = rec->size; - rcrec->len1 = rcrec->len2 = 0; - rcrec->next = cf->rchash[hi]; - cf->rchash[hi] = rcrec; + if (nbits > 32 || nbits >= sizeof(size_t) * CHAR_BIT) + return -1; + nmask = ((size_t)1 << nbits) - 1; + if (!XDL_CALLOC_ARRAY(ntab, nmask + 1)) + return -1; + + cf->tbits = nbits; + cf->tmask = nmask; + for (i = 0; i < cf->count; i++) { + size_t hi = xdl_slot_index(cf, cf->rcrecs[i].line_hash); + + while (ntab[hi]) + hi = (hi + 1) & nmask; + ntab[hi] = (xdlslot_t)(i + 1); } + xdl_free(cf->table); + cf->table = ntab; + return 0; +} - (pass == 1) ? rcrec->len1++ : rcrec->len2++; - rec->minimal_perfect_hash = (size_t)rcrec->idx; +static inline int xdl_bytes_equal(uint8_t const *a, uint8_t const *b, size_t n) { + uint64_t x, y; + size_t i; + if (n < sizeof(uint64_t)) { + for (i = 0; i < n; i++) + if (a[i] != b[i]) + return 0; + return 1; + } + for (i = 0; i + sizeof(uint64_t) < n; i += sizeof(uint64_t)) { + memcpy(&x, a + i, sizeof(x)); + memcpy(&y, b + i, sizeof(y)); + if (x != y) + return 0; + } + memcpy(&x, a + n - sizeof(x), sizeof(x)); + memcpy(&y, b + n - sizeof(y), sizeof(y)); + return x == y; +} + + +static int xdl_classify_record(unsigned int pass, xdlclassifier_t *cf, + xrecord_t *rec, uint64_t line_hash) { + size_t hi, mask = cf->tmask; + xdlslot_t slot; + xdlslot_t *table = cf->table; + xdlclass_t *rcrec; + long idx; + + hi = xdl_slot_index(cf, line_hash); + while ((slot = table[hi])) { + rcrec = &cf->rcrecs[slot - 1]; + if (rcrec->line_hash == line_hash && + (cf->flags & XDF_WHITESPACE_FLAGS + ? xdl_recmatch((const char *)rcrec->ptr, (long)rcrec->size, + (const char *)rec->ptr, (long)rec->size, cf->flags) + : (rcrec->size == rec->size && + xdl_bytes_equal(rcrec->ptr, rec->ptr, rec->size)))) + goto found; + hi = (hi + 1) & mask; + } + + if (cf->count >= XDL_CLASS_MAX) + return -1; + idx = cf->count++; + if (XDL_ALLOC_GROW(cf->rcrecs, cf->count, cf->alloc)) + return -1; + rcrec = &cf->rcrecs[idx]; + rcrec->line_hash = line_hash; + rcrec->ptr = rec->ptr; + rcrec->size = rec->size; + rcrec->len1 = rcrec->len2 = 0; + table[hi] = (xdlslot_t)(idx + 1); + if ((size_t)cf->count * 2 > mask + 1 && xdl_grow_table(cf) < 0) + return -1; + +found: + (pass == 1) ? rcrec->len1++ : rcrec->len2++; + rec->minimal_perfect_hash = (size_t)(rcrec - cf->rcrecs); return 0; } +#define XDL_HMUL 0x9e3779b97f4a7c15ULL + + static void xdl_free_ctx(xdfile_t *xdf) { xdl_free(xdf->reference_index); @@ -158,6 +226,13 @@ static int xdl_prepare_ctx(unsigned int pass, mmfile_t *mf, long narec, xpparam_ for (top = blk + bsize; cur < top; ) { prev = cur; hav = xdl_hash_record(&cur, top, xpp->flags); + /* + * Slots use the high hash bits. This bijective mix + * preserves equality while distributing short lines. + */ + hav ^= hav >> 32; + hav *= XDL_HMUL; + hav ^= hav >> 29; if (XDL_ALLOC_GROW(xdf->recs, (long)xdf->nrec + 1, narec)) goto abort; crec = &xdf->recs[xdf->nrec++]; @@ -264,7 +339,6 @@ static bool xdl_clean_mmatch(uint8_t const *action, ptrdiff_t i, ptrdiff_t len) */ static int xdl_cleanup_records(xdlclassifier_t *cf, xdfile_t *xdf1, xdfile_t *xdf2) { ptrdiff_t i, nm, mlim1, mlim2; - xdlclass_t *rcrec; uint8_t *action1 = NULL, *action2 = NULL; bool need_min = !!(cf->flags & XDF_NEED_MINIMAL); int ret = 0; @@ -298,8 +372,7 @@ static int xdl_cleanup_records(xdlclassifier_t *cf, xdfile_t *xdf1, xdfile_t *xd } for (i = 0; i < len1; i++) { size_t mph1 = xdf1->recs[i + off].minimal_perfect_hash; - rcrec = cf->rcrecs[mph1]; - nm = rcrec ? rcrec->len2 : 0; + nm = cf->rcrecs[mph1].len2; if (nm == 0) action1[i] = DISCARD; else if (nm < mlim1) @@ -318,8 +391,7 @@ static int xdl_cleanup_records(xdlclassifier_t *cf, xdfile_t *xdf1, xdfile_t *xd } for (i = 0; i < len2; i++) { size_t mph2 = xdf2->recs[i + off].minimal_perfect_hash; - rcrec = cf->rcrecs[mph2]; - nm = rcrec ? rcrec->len1 : 0; + nm = cf->rcrecs[mph2].len1; if (nm == 0) action2[i] = DISCARD; else if (nm < mlim2) From 624c1a26af06178e09827b405ececdf62285812b Mon Sep 17 00:00:00 2001 From: Saurav Joshi Date: Thu, 12 Nov 2026 15:16:47 -0800 Subject: [PATCH 04/13] xdiff: hash verbatim lines eight bytes at a time Hashing a line one byte at a time is a large part of record preparation for long files. When no whitespace option is in effect, hash each line by mixing eight-byte words and find the newline with a word-at-a-time check on little-endian GCC-compatible compilers, with a byte loop elsewhere. Equal byte sequences always hash equally, and a matching hash is still followed by a full comparison. Signed-off-by: Saurav Joshi --- xdiff/xprepare.c | 82 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 74 insertions(+), 8 deletions(-) diff --git a/xdiff/xprepare.c b/xdiff/xprepare.c index 8091446f22324e..4ef0debfa3280c 100644 --- a/xdiff/xprepare.c +++ b/xdiff/xprepare.c @@ -198,6 +198,67 @@ static int xdl_classify_record(unsigned int pass, xdlclassifier_t *cf, #define XDL_HMUL 0x9e3779b97f4a7c15ULL +static inline uint64_t xdl_hmix(uint64_t h, uint64_t w) { + h = (h << 23) | (h >> 41); + return (h ^ w) * XDL_HMUL; +} + +/* + * Equal byte sequences hash equally regardless of their buffer alignment. + * A matching hash is always followed by a byte comparison. + */ +static inline uint64_t xdl_hash_line_fast(uint8_t const **data, + uint8_t const *top) { + uint8_t const *start = *data, *ptr = start; + uint64_t h = 0, w; + unsigned int s; + +#if defined(__GNUC__) && defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + while (top - ptr >= 8) { + uint64_t x, m; + + memcpy(&w, ptr, 8); + x = w ^ 0x0a0a0a0a0a0a0a0aULL; + m = (x - 0x0101010101010101ULL) & ~x & 0x8080808080808080ULL; + if (m) { + unsigned int n = (unsigned int)__builtin_ctzll(m) >> 3; + + if (n) + h = xdl_hmix(h, w & (~0ULL >> (64 - 8 * n))); + ptr += n; + *data = ptr + 1; + goto done; + } + h = xdl_hmix(h, w); + ptr += 8; + } +#endif + w = 0; + s = 0; + for (; ptr < top && *ptr != '\n'; ptr++) { + w |= (uint64_t)*ptr << s; + s += 8; + if (s == 64) { + h = xdl_hmix(h, w); + w = 0; + s = 0; + } + } + if (s) + h = xdl_hmix(h, w); + *data = ptr < top ? ptr + 1 : ptr; + +#if defined(__GNUC__) && defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +done: +#endif + h ^= (uint64_t)(ptr - start); + h *= XDL_HMUL; + h ^= h >> 29; + h *= 0xbf58476d1ce4e5b9ULL; + h ^= h >> 32; + return h; +} + static void xdl_free_ctx(xdfile_t *xdf) { @@ -225,14 +286,19 @@ static int xdl_prepare_ctx(unsigned int pass, mmfile_t *mf, long narec, xpparam_ if ((cur = blk = xdl_mmfile_first(mf, &bsize))) { for (top = blk + bsize; cur < top; ) { prev = cur; - hav = xdl_hash_record(&cur, top, xpp->flags); - /* - * Slots use the high hash bits. This bijective mix - * preserves equality while distributing short lines. - */ - hav ^= hav >> 32; - hav *= XDL_HMUL; - hav ^= hav >> 29; + if (xpp->flags & XDF_WHITESPACE_FLAGS) { + hav = xdl_hash_record_with_whitespace(&cur, top, xpp->flags); + /* + * The whitespace hash has weak high bits for + * short lines. This bijective mix preserves + * equality while distributing the table slots. + */ + hav ^= hav >> 32; + hav *= XDL_HMUL; + hav ^= hav >> 29; + } else { + hav = xdl_hash_line_fast(&cur, top); + } if (XDL_ALLOC_GROW(xdf->recs, (long)xdf->nrec + 1, narec)) goto abort; crec = &xdf->recs[xdf->nrec++]; From 245d81e7e3d016c271cd0902daf5486474441b53 Mon Sep 17 00:00:00 2001 From: Saurav Joshi Date: Thu, 12 Nov 2026 15:16:47 -0800 Subject: [PATCH 05/13] xdiff: simplify the cleanup_records() loops xdl_cleanup_records() allocates two zeroed action arrays and checks for an impossible state on every record. Allocate both arrays at once without clearing them, since every slot is written before it is read, and only run the "investigate" pass on a side that has such records. Signed-off-by: Saurav Joshi --- xdiff/xprepare.c | 122 ++++++++++++++++++++++++++--------------------- 1 file changed, 67 insertions(+), 55 deletions(-) diff --git a/xdiff/xprepare.c b/xdiff/xprepare.c index 4ef0debfa3280c..e622cc808c10e0 100644 --- a/xdiff/xprepare.c +++ b/xdiff/xprepare.c @@ -404,26 +404,29 @@ static bool xdl_clean_mmatch(uint8_t const *action, ptrdiff_t i, ptrdiff_t len) * might be potentially discarded if they appear in a run of discardable. */ static int xdl_cleanup_records(xdlclassifier_t *cf, xdfile_t *xdf1, xdfile_t *xdf2) { - ptrdiff_t i, nm, mlim1, mlim2; + ptrdiff_t i, mlim1, mlim2; uint8_t *action1 = NULL, *action2 = NULL; bool need_min = !!(cf->flags & XDF_NEED_MINIMAL); int ret = 0; ptrdiff_t off = xdf1->dstart; ptrdiff_t len1 = xdf1->dend - off + 1; ptrdiff_t len2 = xdf2->dend - off + 1; + bool investigate1 = false, investigate2 = false; /* * Create temporary arrays that will help us decide if - * changed[i] should remain false, or become true. + * changed[i] should remain false, or become true. Every slot is + * initialized below, so one allocation serves both sides. */ - if (!XDL_CALLOC_ARRAY(action1, len1) || - !XDL_CALLOC_ARRAY(action2, len2) || + if (len1 > PTRDIFF_MAX - len2 - 1 || + !XDL_ALLOC_ARRAY(action1, len1 + len2 + 1) || !XDL_ALLOC_ARRAY(xdf1->reference_index, len1) || !XDL_ALLOC_ARRAY(xdf2->reference_index, len2)) { ret = -1; goto cleanup; } + action2 = action1 + len1; /* * Initialize temporary arrays with DISCARD, KEEP, or INVESTIGATE. @@ -436,15 +439,23 @@ static int xdl_cleanup_records(xdlclassifier_t *cf, xdfile_t *xdf1, xdfile_t *xd if (mlim1 > XDL_MAX_EQLIMIT) mlim1 = XDL_MAX_EQLIMIT; } - for (i = 0; i < len1; i++) { - size_t mph1 = xdf1->recs[i + off].minimal_perfect_hash; - nm = cf->rcrecs[mph1].len2; - if (nm == 0) - action1[i] = DISCARD; - else if (nm < mlim1) - action1[i] = KEEP; - else /* nm >= mlim1 */ - action1[i] = INVESTIGATE; + { + xrecord_t const *recs = xdf1->recs + off; + + for (i = 0; i < len1; i++) { + ptrdiff_t nm = cf->rcrecs[recs[i].minimal_perfect_hash].len2; + uint8_t action; + + if (!nm) + action = DISCARD; + else if (nm < mlim1) + action = KEEP; + else { + action = INVESTIGATE; + investigate1 = true; + } + action1[i] = action; + } } if (need_min) { @@ -455,66 +466,67 @@ static int xdl_cleanup_records(xdlclassifier_t *cf, xdfile_t *xdf1, xdfile_t *xd if (mlim2 > XDL_MAX_EQLIMIT) mlim2 = XDL_MAX_EQLIMIT; } - for (i = 0; i < len2; i++) { - size_t mph2 = xdf2->recs[i + off].minimal_perfect_hash; - nm = cf->rcrecs[mph2].len1; - if (nm == 0) - action2[i] = DISCARD; - else if (nm < mlim2) - action2[i] = KEEP; - else /* nm >= mlim2 */ - action2[i] = INVESTIGATE; + { + xrecord_t const *recs = xdf2->recs + off; + + for (i = 0; i < len2; i++) { + ptrdiff_t nm = cf->rcrecs[recs[i].minimal_perfect_hash].len1; + uint8_t action; + + if (!nm) + action = DISCARD; + else if (nm < mlim2) + action = KEEP; + else { + action = INVESTIGATE; + investigate2 = true; + } + action2[i] = action; + } } /* * Use temporary arrays to decide if changed[i] should remain * false, or become true. */ - xdf1->nreff = 0; - for (i = 0; i < len1; i++) { - uint8_t action = action1[i]; + { + size_t nreff = 0; + size_t *ref = xdf1->reference_index; + bool *changed = xdf1->changed + off; - if (action == INVESTIGATE) { - if (!xdl_clean_mmatch(action1, i, len1)) - action = KEEP; - else - action = DISCARD; - } + for (i = 0; i < len1; i++) { + uint8_t action = action1[i]; - if (action == KEEP) { - xdf1->reference_index[xdf1->nreff++] = i + off; - /* changed[i] remains false */ - } else if (action == DISCARD) { - xdf1->changed[i + off] = true; - } else { - BUG("Illegal state for action"); + if (investigate1 && action == INVESTIGATE) + action = xdl_clean_mmatch(action1, i, len1) ? DISCARD : KEEP; + if (action == KEEP) + ref[nreff++] = i + off; + else + changed[i] = true; } + xdf1->nreff = nreff; } - xdf2->nreff = 0; - for (i = 0; i < len2; i++) { - uint8_t action = action2[i]; + { + size_t nreff = 0; + size_t *ref = xdf2->reference_index; + bool *changed = xdf2->changed + off; - if (action == INVESTIGATE) { - if (!xdl_clean_mmatch(action2, i, len2)) - action = KEEP; - else - action = DISCARD; - } + for (i = 0; i < len2; i++) { + uint8_t action = action2[i]; - if (action == KEEP) { - xdf2->reference_index[xdf2->nreff++] = i + off; - /* changed[i] remains false */ - } else if (action == DISCARD) { - xdf2->changed[i + off] = true; - } else { - BUG("Illegal state for action"); + if (investigate2 && action == INVESTIGATE) + action = xdl_clean_mmatch(action2, i, len2) ? DISCARD : KEEP; + if (action == KEEP) + ref[nreff++] = i + off; + else + changed[i] = true; } + xdf2->nreff = nreff; } cleanup: xdl_free(action1); - xdl_free(action2); return ret; } From 1af98d8473df9e9bddb0fc569864fa4cdb3600b6 Mon Sep 17 00:00:00 2001 From: Saurav Joshi Date: Thu, 12 Nov 2026 15:16:47 -0800 Subject: [PATCH 06/13] xdiff: improve record allocation estimates The second file's record count is guessed from a small sample, so large files often grow the record array several times. Estimate it from the first file's line density when the two sizes are similar, refine it once the first file has been split, and when the array must grow, extrapolate from the bytes consumed so far. Size the class table from the first file's records. Signed-off-by: Saurav Joshi --- xdiff/xprepare.c | 57 ++++++++++++++++++++++++++++++++++++------------ 1 file changed, 43 insertions(+), 14 deletions(-) diff --git a/xdiff/xprepare.c b/xdiff/xprepare.c index e622cc808c10e0..f3460dd7f5a797 100644 --- a/xdiff/xprepare.c +++ b/xdiff/xprepare.c @@ -89,11 +89,6 @@ static int xdl_init_classifier_sized(xdlclassifier_t *cf, long size, long tsize, } -static int xdl_init_classifier(xdlclassifier_t *cf, long size, long flags) { - return xdl_init_classifier_sized(cf, size, size, flags); -} - - static void xdl_free_classifier(xdlclassifier_t *cf) { xdl_free(cf->rcrecs); xdl_free(cf->table); @@ -299,8 +294,22 @@ static int xdl_prepare_ctx(unsigned int pass, mmfile_t *mf, long narec, xpparam_ } else { hav = xdl_hash_line_fast(&cur, top); } - if (XDL_ALLOC_GROW(xdf->recs, (long)xdf->nrec + 1, narec)) - goto abort; + if ((long)xdf->nrec + 1 > narec) { + long want = (long)xdf->nrec + 1; + long used = (long)(cur - blk); + + /* Extrapolate from bytes already consumed. */ + if (used > 0 && want <= LONG_MAX / bsize) { + long est = want * bsize / used; + + if (est <= LONG_MAX - est / 32 - 16) + est += est / 32 + 16; + if (est > want) + want = est; + } + if (XDL_ALLOC_GROW(xdf->recs, want, narec)) + goto abort; + } crec = &xdf->recs[xdf->nrec++]; crec->ptr = prev; crec->size = cur - prev; @@ -580,19 +589,26 @@ int xdl_prepare_env(mmfile_t *mf1, mmfile_t *mf2, xpparam_t const *xpp, memset(&cf, 0, sizeof(cf)); /* - * For histogram diff, we can afford a smaller sample size and - * thus a poorer estimate of the number of lines, as the hash - * table (rhash) won't be filled up/grown. The number of lines - * (nrecs) will be updated correctly anyway by - * xdl_prepare_ctx(). + * Estimates only control allocations, not record boundaries. + * Both the record array and classifier can grow when necessary. */ sample = (XDF_DIFF_ALG(xpp->flags) == XDF_HISTOGRAM_DIFF ? XDL_GUESS_NLINES2 : XDL_GUESS_NLINES1); enl1 = xdl_guess_lines(mf1, sample) + 1; - enl2 = xdl_guess_lines(mf2, sample) + 1; + { + long sz1 = xdl_mmfile_size(mf1), sz2 = xdl_mmfile_size(mf2); - if (xdl_init_classifier(&cf, enl1 + enl2 + 1, xpp->flags) < 0) + if (sz1 > 0 && sz2 > 0 && + sz1 <= LONG_MAX / 4 && sz2 <= LONG_MAX / 4 && + sz2 <= 4 * sz1 && sz1 <= 4 * sz2 && + sz2 <= LONG_MAX / enl1) + enl2 = enl1 * sz2 / sz1 + 1; + else + enl2 = xdl_guess_lines(mf2, sample) + 1; + } + + if (xdl_init_classifier_sized(&cf, enl1 + enl2 + 1, enl1, xpp->flags) < 0) return -1; if (xdl_prepare_ctx(1, mf1, enl1, xpp, &cf, &xe->xdf1) < 0) { @@ -600,6 +616,19 @@ int xdl_prepare_env(mmfile_t *mf1, mmfile_t *mf2, xpparam_t const *xpp, xdl_free_classifier(&cf); return -1; } + if (xe->xdf1.nrec && xdl_mmfile_size(mf1) > 0) { + long sz1 = xdl_mmfile_size(mf1), sz2 = xdl_mmfile_size(mf2); + long nr1 = (long)xe->xdf1.nrec; + + if (sz2 <= LONG_MAX / nr1) { + long est = nr1 * sz2 / sz1; + + if (est <= LONG_MAX - est / 32 - 16) + est += est / 32 + 16; + if (est > enl2) + enl2 = est; + } + } if (xdl_prepare_ctx(2, mf2, enl2, xpp, &cf, &xe->xdf2) < 0) { xdl_free_ctx(&xe->xdf1); From 7f32ed648f1300e5106acd48b49ef6c42b4f1914 Mon Sep 17 00:00:00 2001 From: Saurav Joshi Date: Thu, 12 Nov 2026 15:16:48 -0800 Subject: [PATCH 07/13] xdiff: skip classification when one side is empty If one input is empty, no line can match, yet both sides are still hashed and classified. Only split the non-empty side into records and mark every record as changed. Signed-off-by: Saurav Joshi --- xdiff/xprepare.c | 80 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 79 insertions(+), 1 deletion(-) diff --git a/xdiff/xprepare.c b/xdiff/xprepare.c index f3460dd7f5a797..ea619103df9d7e 100644 --- a/xdiff/xprepare.c +++ b/xdiff/xprepare.c @@ -258,7 +258,8 @@ static inline uint64_t xdl_hash_line_fast(uint8_t const **data, static void xdl_free_ctx(xdfile_t *xdf) { xdl_free(xdf->reference_index); - xdl_free(xdf->changed - 1); + if (xdf->changed) + xdl_free(xdf->changed - 1); xdl_free(xdf->recs); } @@ -334,6 +335,50 @@ static int xdl_prepare_ctx(unsigned int pass, mmfile_t *mf, long narec, xpparam_ } +/* + * With an empty peer, no equivalence class can be consulted. Only split + * the nonempty input into the same newline-delimited records as usual. + */ +static int xdl_prepare_ctx_unmatched(mmfile_t *mf, long narec, xdfile_t *xdf) { + long bsize; + uint8_t const *blk, *cur, *top, *eol; + xrecord_t *crec; + + xdf->reference_index = NULL; + xdf->changed = NULL; + xdf->recs = NULL; + + if (!XDL_ALLOC_ARRAY(xdf->recs, narec)) + goto abort; + + xdf->nrec = 0; + if ((cur = blk = xdl_mmfile_first(mf, &bsize))) { + for (top = blk + bsize; cur < top; ) { + eol = memchr(cur, '\n', top - cur); + if (XDL_ALLOC_GROW(xdf->recs, (long)xdf->nrec + 1, narec)) + goto abort; + crec = &xdf->recs[xdf->nrec++]; + crec->ptr = cur; + cur = eol ? eol + 1 : top; + crec->size = cur - crec->ptr; + crec->minimal_perfect_hash = 0; + } + } + + if (!XDL_CALLOC_ARRAY(xdf->changed, xdf->nrec + 2)) + goto abort; + xdf->changed += 1; + xdf->nreff = 0; + xdf->dstart = 0; + xdf->dend = xdf->nrec - 1; + return 0; + +abort: + xdl_free_ctx(xdf); + return -1; +} + + void xdl_free_env(xdfenv_t *xe) { xdl_free_ctx(&xe->xdf2); @@ -581,6 +626,36 @@ static int xdl_optimize_ctxs(xdlclassifier_t *cf, xdfile_t *xdf1, xdfile_t *xdf2 return 0; } + +static int xdl_prepare_env_one_empty(mmfile_t *mf1, mmfile_t *mf2, + long enl1, long enl2, + xpparam_t const *xpp, xdfenv_t *xe) { + long i; + + if (xdl_prepare_ctx_unmatched(mf1, enl1, &xe->xdf1) < 0) + return -1; + if (xdl_prepare_ctx_unmatched(mf2, enl2, &xe->xdf2) < 0) { + xdl_free_ctx(&xe->xdf1); + return -1; + } + + if (XDF_DIFF_ALG(xpp->flags) != XDF_PATIENCE_DIFF && + XDF_DIFF_ALG(xpp->flags) != XDF_HISTOGRAM_DIFF) { + if (!XDL_ALLOC_ARRAY(xe->xdf1.reference_index, xe->xdf1.nrec) || + !XDL_ALLOC_ARRAY(xe->xdf2.reference_index, xe->xdf2.nrec)) { + xdl_free_ctx(&xe->xdf2); + xdl_free_ctx(&xe->xdf1); + return -1; + } + for (i = 0; i < (long)xe->xdf1.nrec; i++) + xe->xdf1.changed[i] = true; + for (i = 0; i < (long)xe->xdf2.nrec; i++) + xe->xdf2.changed[i] = true; + } + return 0; +} + + int xdl_prepare_env(mmfile_t *mf1, mmfile_t *mf2, xpparam_t const *xpp, xdfenv_t *xe) { long enl1, enl2, sample; @@ -608,6 +683,9 @@ int xdl_prepare_env(mmfile_t *mf1, mmfile_t *mf2, xpparam_t const *xpp, enl2 = xdl_guess_lines(mf2, sample) + 1; } + if (xdl_mmfile_size(mf1) <= 0 || xdl_mmfile_size(mf2) <= 0) + return xdl_prepare_env_one_empty(mf1, mf2, enl1, enl2, xpp, xe); + if (xdl_init_classifier_sized(&cf, enl1 + enl2 + 1, enl1, xpp->flags) < 0) return -1; From a2b90f10ca616a4a46154ea458ad9c06c7c17bee Mon Sep 17 00:00:00 2001 From: Saurav Joshi Date: Thu, 12 Nov 2026 15:16:48 -0800 Subject: [PATCH 08/13] xdiff: reuse classes for common leading and trailing lines Records that are byte-identical at the start and end of both files are hashed and classified twice. When preparing the second file, find the common prefix and suffix with block comparisons and give those records the class already assigned for the first file, keeping the occurrence counts that cleanup relies on. Signed-off-by: Saurav Joshi --- xdiff/xprepare.c | 148 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 143 insertions(+), 5 deletions(-) diff --git a/xdiff/xprepare.c b/xdiff/xprepare.c index ea619103df9d7e..850a751d118b31 100644 --- a/xdiff/xprepare.c +++ b/xdiff/xprepare.c @@ -264,11 +264,78 @@ static void xdl_free_ctx(xdfile_t *xdf) } +#define XDL_CMP_BLOCK 256 + +static size_t xdl_common_prefix_bytes(uint8_t const *a, uint8_t const *b, + size_t n) { + size_t i = 0; + + while (i + XDL_CMP_BLOCK <= n && + !memcmp(a + i, b + i, XDL_CMP_BLOCK)) + i += XDL_CMP_BLOCK; + while (i + sizeof(uint64_t) <= n) { + uint64_t x, y; + + memcpy(&x, a + i, sizeof(x)); + memcpy(&y, b + i, sizeof(y)); + if (x != y) + break; + i += sizeof(uint64_t); + } + while (i < n && a[i] == b[i]) + i++; + return i; +} + + +static size_t xdl_common_suffix_bytes(uint8_t const *a, uint8_t const *b, + size_t n) { + size_t i = 0; + + while (i + XDL_CMP_BLOCK <= n && + !memcmp(a + n - i - XDL_CMP_BLOCK, + b + n - i - XDL_CMP_BLOCK, XDL_CMP_BLOCK)) + i += XDL_CMP_BLOCK; + while (i + sizeof(uint64_t) <= n) { + uint64_t x, y; + + memcpy(&x, a + n - i - sizeof(x), sizeof(x)); + memcpy(&y, b + n - i - sizeof(y), sizeof(y)); + if (x != y) + break; + i += sizeof(uint64_t); + } + while (i < n && a[n - i - 1] == b[n - i - 1]) + i++; + return i; +} + + +static int xdl_add_rec(xdfile_t *xdf, long *narec, uint8_t const *ptr, + size_t size, size_t mph) { + xrecord_t *crec; + + if (XDL_ALLOC_GROW(xdf->recs, (long)xdf->nrec + 1, *narec)) + return -1; + crec = &xdf->recs[xdf->nrec++]; + crec->ptr = ptr; + crec->size = size; + crec->minimal_perfect_hash = mph; + return 0; +} + + +/* + * Byte-identical common head/tail records on side two can reuse side + * one's classes. Keep their occurrence counts, including when a middle + * record shares the same class. + */ static int xdl_prepare_ctx(unsigned int pass, mmfile_t *mf, long narec, xpparam_t const *xpp, - xdlclassifier_t *cf, xdfile_t *xdf) { + xdlclassifier_t *cf, xdfile_t *xdf, + mmfile_t *mfref, xdfile_t const *xdfref) { long bsize; uint64_t hav; - uint8_t const *blk, *cur, *top, *prev; + uint8_t const *blk, *cur, *top, *prev, *mid_end; xrecord_t *crec; xdf->reference_index = NULL; @@ -280,7 +347,53 @@ static int xdl_prepare_ctx(unsigned int pass, mmfile_t *mf, long narec, xpparam_ xdf->nrec = 0; if ((cur = blk = xdl_mmfile_first(mf, &bsize))) { - for (top = blk + bsize; cur < top; ) { + top = blk + bsize; + mid_end = top; + if (xdfref && xdfref->nrec) { + long rsize; + uint8_t const *rblk = xdl_mmfile_first(mfref, &rsize); + size_t n = (size_t)XDL_MIN(rsize, bsize); + size_t d = xdl_common_prefix_bytes(rblk, blk, n); + size_t i, hend = 0, l, s1, s2; + + for (i = 0; i < xdfref->nrec; i++) { + xrecord_t const *r = &xdfref->recs[i]; + size_t e = (size_t)(r->ptr - rblk) + r->size; + + if (e > d) + break; + if (rblk[e - 1] != '\n' && (size_t)bsize != e) + break; + if (xdl_add_rec(xdf, &narec, blk + (r->ptr - rblk), + r->size, r->minimal_perfect_hash) < 0) + goto abort; + cf->rcrecs[r->minimal_perfect_hash].len2++; + hend = e; + } + cur = blk + hend; + + s1 = (size_t)rsize - hend; + s2 = (size_t)bsize - hend; + n = XDL_MIN(s1, s2); + l = xdl_common_suffix_bytes(rblk + rsize - n, + blk + bsize - n, n); + if (l) { + size_t t2 = (size_t)bsize - l; + size_t t1 = (size_t)rsize - l; + + /* Advance to a line start in both files. */ + while (t2 < (size_t)bsize && + !((t2 == hend || blk[t2 - 1] == '\n') && + (t1 == hend || rblk[t1 - 1] == '\n'))) { + t2++; + t1++; + } + if (t2 < (size_t)bsize) + mid_end = blk + t2; + } + } + + for (; cur < mid_end; ) { prev = cur; if (xpp->flags & XDF_WHITESPACE_FLAGS) { hav = xdl_hash_record_with_whitespace(&cur, top, xpp->flags); @@ -317,6 +430,31 @@ static int xdl_prepare_ctx(unsigned int pass, mmfile_t *mf, long narec, xpparam_ if (xdl_classify_record(pass, cf, crec, hav) < 0) goto abort; } + + if (cur < top) { + long rsize; + uint8_t const *rblk = xdl_mmfile_first(mfref, &rsize); + uint8_t const *p; + size_t ntail = 0, j; + + for (p = cur; p < top; ntail++) { + p = memchr(p, '\n', top - p); + p = p ? p + 1 : top; + } + j = xdfref->nrec - ntail; + if (ntail > xdfref->nrec || + xdfref->recs[j].ptr - rblk != rsize - (top - cur)) + BUG("xdiff: common tail records do not line up"); + for (; j < xdfref->nrec; j++) { + xrecord_t const *r = &xdfref->recs[j]; + + if (xdl_add_rec(xdf, &narec, cur, r->size, + r->minimal_perfect_hash) < 0) + goto abort; + cf->rcrecs[r->minimal_perfect_hash].len2++; + cur += r->size; + } + } } if (!XDL_CALLOC_ARRAY(xdf->changed, xdf->nrec + 2)) @@ -689,7 +827,7 @@ int xdl_prepare_env(mmfile_t *mf1, mmfile_t *mf2, xpparam_t const *xpp, if (xdl_init_classifier_sized(&cf, enl1 + enl2 + 1, enl1, xpp->flags) < 0) return -1; - if (xdl_prepare_ctx(1, mf1, enl1, xpp, &cf, &xe->xdf1) < 0) { + if (xdl_prepare_ctx(1, mf1, enl1, xpp, &cf, &xe->xdf1, NULL, NULL) < 0) { xdl_free_classifier(&cf); return -1; @@ -707,7 +845,7 @@ int xdl_prepare_env(mmfile_t *mf1, mmfile_t *mf2, xpparam_t const *xpp, enl2 = est; } } - if (xdl_prepare_ctx(2, mf2, enl2, xpp, &cf, &xe->xdf2) < 0) { + if (xdl_prepare_ctx(2, mf2, enl2, xpp, &cf, &xe->xdf2, mf1, &xe->xdf1) < 0) { xdl_free_ctx(&xe->xdf1); xdl_free_classifier(&cf); From 9173f51612b9519911f207fa6bc2658a4f4ab309 Mon Sep 17 00:00:00 2001 From: Saurav Joshi Date: Thu, 12 Nov 2026 15:16:48 -0800 Subject: [PATCH 09/13] xdiff: avoid classifying common head and tail lines for Myers For a verbatim Myers diff, common leading and trailing lines are trimmed before the diff and never compared, but they are still classified. Split the common head and tail from the middle first and classify only the middle. Use a fingerprint filter to find common lines that might also appear in the middle; only those are classified, the rest get private class numbers. Fingerprints are only a negative filter: a possible match still goes through the normal hash and full comparison. Signed-off-by: Saurav Joshi --- xdiff/xprepare.c | 262 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 262 insertions(+) diff --git a/xdiff/xprepare.c b/xdiff/xprepare.c index 850a751d118b31..15c062cfdd5fe0 100644 --- a/xdiff/xprepare.c +++ b/xdiff/xprepare.c @@ -89,6 +89,11 @@ static int xdl_init_classifier_sized(xdlclassifier_t *cf, long size, long tsize, } +static int xdl_init_classifier(xdlclassifier_t *cf, long size, long flags) { + return xdl_init_classifier_sized(cf, size, size, flags); +} + + static void xdl_free_classifier(xdlclassifier_t *cf) { xdl_free(cf->rcrecs); xdl_free(cf->table); @@ -794,6 +799,247 @@ static int xdl_prepare_env_one_empty(mmfile_t *mf1, mmfile_t *mf2, } +#define XDL_REC_PENDING SIZE_MAX + +static uint64_t xdl_line_fingerprint(uint8_t const *p, size_t n) { + uint64_t a = 0, b = 0; + + if (n >= 8) { + memcpy(&a, p, 8); + memcpy(&b, p + n - 8, 8); + } else { + size_t i; + + for (i = 0; i < n; i++) + a = (a << 8) | p[i]; + } + a = (a ^ (b * XDL_HMUL) ^ ((uint64_t)n << 56)) * 0xff51afd7ed558ccdULL; + return a ^ (a >> 29); +} + + +/* + * For verbatim Myers diffs, first separate byte-identical head/tail + * records from the middle. A fingerprint filter tells us which common + * records might also occur in the middle; only those need classification. + * + * The remaining common pairs receive private class numbers. This is safe + * because trimming removes those pairs before cleanup consults the class + * counters. Compaction compares a common record only with a changed + * record, which necessarily belongs to a middle class. + * + * Fingerprints are only a negative filter. False positives take the + * normal hashing and byte-comparison path; they cannot equate two lines. + */ +static int xdl_prepare_myers_records(mmfile_t *mf1, mmfile_t *mf2, + long enl1, long enl2, xpparam_t const *xpp, + xdlclassifier_t *cf, xdfenv_t *xe) { + xdfile_t *xdf1 = &xe->xdf1, *xdf2 = &xe->xdf2; + long s1, s2; + uint8_t const *b1 = xdl_mmfile_first(mf1, &s1); + uint8_t const *b2 = xdl_mmfile_first(mf2, &s2); + uint8_t const *top1, *top2, *p, *cur, *prev; + size_t d, hend, t1, t2, nhead, i, j, nmid1, nmid2, tail1, tail2; + uint64_t *bits = NULL; + unsigned int fpbits = 0; + size_t next; + + xdf1->reference_index = xdf2->reference_index = NULL; + xdf1->changed = xdf2->changed = NULL; + xdf1->recs = xdf2->recs = NULL; + xdf1->nrec = xdf2->nrec = 0; + + if (!XDL_ALLOC_ARRAY(xdf1->recs, enl1) || + !XDL_ALLOC_ARRAY(xdf2->recs, enl2)) + return -1; + top1 = b1 + s1; + top2 = b2 + s2; + + d = xdl_common_prefix_bytes(b1, b2, (size_t)XDL_MIN(s1, s2)); + hend = 0; + for (p = b1; p < top1; ) { + uint8_t const *e = memchr(p, '\n', top1 - p); + size_t end = e ? (size_t)(e - b1) + 1 : (size_t)s1; + + if (end > d || (!e && (size_t)s2 != end)) + break; + if (xdl_add_rec(xdf1, &enl1, p, b1 + end - p, XDL_REC_PENDING) < 0 || + xdl_add_rec(xdf2, &enl2, b2 + (p - b1), b1 + end - p, + XDL_REC_PENDING) < 0) + return -1; + hend = end; + p = b1 + end; + } + nhead = xdf1->nrec; + + t1 = (size_t)s1; + t2 = (size_t)s2; + { + size_t n = XDL_MIN((size_t)s1 - hend, (size_t)s2 - hend); + size_t l = xdl_common_suffix_bytes(b1 + s1 - n, b2 + s2 - n, n); + + if (l) { + size_t u1 = (size_t)s1 - l, u2 = (size_t)s2 - l; + + while (u2 < (size_t)s2 && + !((u2 == hend || b2[u2 - 1] == '\n') && + (u1 == hend || b1[u1 - 1] == '\n'))) { + u1++; + u2++; + } + if (u2 < (size_t)s2) { + t1 = u1; + t2 = u2; + } + } + } + + /* Size for middle records; possible common-class matches grow it. */ + { + long est = 1; + + est += (long)((double)enl1 * (t1 - hend) / s1); + est += (long)((double)enl2 * (t2 - hend) / s2); + if (est < 32) + est = 32; + if (xdl_init_classifier(cf, est, xpp->flags) < 0) + return -1; + } + + for (cur = b1 + hend; cur < b1 + t1; ) { + uint64_t hav; + + prev = cur; + hav = xdl_hash_line_fast(&cur, top1); + if (xdl_add_rec(xdf1, &enl1, prev, cur - prev, 0) < 0 || + xdl_classify_record(1, cf, &xdf1->recs[xdf1->nrec - 1], hav) < 0) + return -1; + } + nmid1 = xdf1->nrec - nhead; + for (cur = b2 + hend; cur < b2 + t2; ) { + uint64_t hav; + + prev = cur; + hav = xdl_hash_line_fast(&cur, top2); + if (xdl_add_rec(xdf2, &enl2, prev, cur - prev, 0) < 0 || + xdl_classify_record(2, cf, &xdf2->recs[xdf2->nrec - 1], hav) < 0) + return -1; + } + nmid2 = xdf2->nrec - nhead; + + tail1 = xdf1->nrec; + tail2 = xdf2->nrec; + for (p = b1 + t1; p < top1; ) { + uint8_t const *e = memchr(p, '\n', top1 - p); + uint8_t const *end = e ? e + 1 : top1; + + if (xdl_add_rec(xdf1, &enl1, p, end - p, XDL_REC_PENDING) < 0 || + xdl_add_rec(xdf2, &enl2, b2 + t2 + (p - (b1 + t1)), + end - p, XDL_REC_PENDING) < 0) + return -1; + p = end; + } + + if (nmid1 + nmid2) { + size_t nmid = nmid1 + nmid2; + size_t want = nmid < ((size_t)1 << 26) ? + nmid * 16 : (size_t)1 << 30; + + for (fpbits = 10; + fpbits < 30 && ((size_t)1 << fpbits) < want; + fpbits++) + ; /* Keep the filter bounded even for very large inputs. */ + if (!XDL_CALLOC_ARRAY(bits, ((size_t)1 << fpbits) / 64)) + return -1; + for (i = nhead; i < tail1; i++) { + uint64_t h = xdl_line_fingerprint(xdf1->recs[i].ptr, + xdf1->recs[i].size) >> (64 - fpbits); + + bits[h >> 6] |= (uint64_t)1 << (h & 63); + } + for (i = nhead; i < tail2; i++) { + uint64_t h = xdl_line_fingerprint(xdf2->recs[i].ptr, + xdf2->recs[i].size) >> (64 - fpbits); + + bits[h >> 6] |= (uint64_t)1 << (h & 63); + } + } + + /* Classify any common pair that might share a middle class. */ + for (i = 0, j = 0; i < xdf1->nrec; i++, j++) { + xrecord_t *r1, *r2; + + if (i == nhead) { + i = tail1; + j = tail2; + if (i >= xdf1->nrec) + break; + } + r1 = &xdf1->recs[i]; + r2 = &xdf2->recs[j]; + if (bits) { + uint64_t h = xdl_line_fingerprint(r1->ptr, r1->size) >> (64 - fpbits); + + if (bits[h >> 6] & ((uint64_t)1 << (h & 63))) { + uint8_t const *q = r1->ptr; + uint64_t hav = xdl_hash_line_fast(&q, top1); + + if (xdl_classify_record(1, cf, r1, hav) < 0) + goto abort; + cf->rcrecs[r1->minimal_perfect_hash].len2++; + r2->minimal_perfect_hash = r1->minimal_perfect_hash; + } + } + } + + next = (size_t)cf->count; + for (i = 0, j = 0; i < xdf1->nrec; i++, j++) { + if (i == nhead) { + i = tail1; + j = tail2; + if (i >= xdf1->nrec) + break; + } + if (xdf1->recs[i].minimal_perfect_hash == XDL_REC_PENDING) { + xdf1->recs[i].minimal_perfect_hash = next; + xdf2->recs[j].minimal_perfect_hash = next++; + } + } + xdl_free(bits); + bits = NULL; + + if (!XDL_CALLOC_ARRAY(xdf1->changed, xdf1->nrec + 2)) + return -1; + xdf1->changed += 1; + if (!XDL_CALLOC_ARRAY(xdf2->changed, xdf2->nrec + 2)) + return -1; + xdf2->changed += 1; + xdf1->nreff = xdf2->nreff = 0; + xdf1->dstart = xdf2->dstart = 0; + xdf1->dend = xdf1->nrec - 1; + xdf2->dend = xdf2->nrec - 1; + return 0; + +abort: + xdl_free(bits); + return -1; +} + + +static int xdl_prepare_myers_verbatim(mmfile_t *mf1, mmfile_t *mf2, + long enl1, long enl2, xpparam_t const *xpp, + xdlclassifier_t *cf, xdfenv_t *xe) { + if (xdl_prepare_myers_records(mf1, mf2, enl1, enl2, xpp, cf, xe) < 0) { + xdl_free_ctx(&xe->xdf1); + xdl_free_ctx(&xe->xdf2); + xe->xdf1.recs = xe->xdf2.recs = NULL; + xe->xdf1.changed = xe->xdf2.changed = NULL; + return -1; + } + return 0; +} + + int xdl_prepare_env(mmfile_t *mf1, mmfile_t *mf2, xpparam_t const *xpp, xdfenv_t *xe) { long enl1, enl2, sample; @@ -824,6 +1070,22 @@ int xdl_prepare_env(mmfile_t *mf1, mmfile_t *mf2, xpparam_t const *xpp, if (xdl_mmfile_size(mf1) <= 0 || xdl_mmfile_size(mf2) <= 0) return xdl_prepare_env_one_empty(mf1, mf2, enl1, enl2, xpp, xe); + if (!(XDF_DIFF_ALG(xpp->flags) & (XDF_PATIENCE_DIFF | XDF_HISTOGRAM_DIFF)) && + !(xpp->flags & XDF_WHITESPACE_FLAGS)) { + if (xdl_prepare_myers_verbatim(mf1, mf2, enl1, enl2, xpp, &cf, xe) < 0) { + xdl_free_classifier(&cf); + return -1; + } + if (xdl_optimize_ctxs(&cf, &xe->xdf1, &xe->xdf2) < 0) { + xdl_free_ctx(&xe->xdf2); + xdl_free_ctx(&xe->xdf1); + xdl_free_classifier(&cf); + return -1; + } + xdl_free_classifier(&cf); + return 0; + } + if (xdl_init_classifier_sized(&cf, enl1 + enl2 + 1, enl1, xpp->flags) < 0) return -1; From 2ddb82fa912f072a024bf701cda86a122182c626 Mon Sep 17 00:00:00 2001 From: Saurav Joshi Date: Thu, 12 Nov 2026 15:18:38 -0800 Subject: [PATCH 10/13] xdiff: compare contiguous class arrays in the Myers search Every comparison in xdl_split() and xdl_recs_cmp() looks up a line's class through reference_index and then the record array. Copy the class of each line taking part in the search into a flat array once, and compare those arrays instead. The search, its recursion and its linear memory use are unchanged; the copies add one array per side. Signed-off-by: Saurav Joshi --- xdiff/xdiffi.c | 81 +++++++++++++++++++++++++++++++++++++------------- 1 file changed, 61 insertions(+), 20 deletions(-) diff --git a/xdiff/xdiffi.c b/xdiff/xdiffi.c index c5a892f91e00c0..dd70a390f1f394 100644 --- a/xdiff/xdiffi.c +++ b/xdiff/xdiffi.c @@ -22,10 +22,13 @@ #include "xinclude.h" -static size_t get_hash(xdfile_t *xdf, long index) -{ - return xdf->recs[xdf->reference_index[index]].minimal_perfect_hash; -} +/* + * Contiguous copies of the class indexes participating in the search. + * Comparisons no longer need reference_index followed by a record lookup. + */ +typedef struct s_xdmyers { + size_t const *ha1, *ha2; +} xdmyers_t; #define XDL_MAX_COST_MIN 256 #define XDL_HEUR_MIN_COST 256 @@ -47,8 +50,8 @@ typedef struct s_xdpsplit { * using this algorithm, so a little bit of heuristic is needed to cut the * search and to return a suboptimal point. */ -static long xdl_split(xdfile_t *xdf1, long off1, long lim1, - xdfile_t *xdf2, long off2, long lim2, +static long xdl_split(xdmyers_t const *mc, long off1, long lim1, + long off2, long lim2, long *kvdf, long *kvdb, int need_min, xdpsplit_t *spl, xdalgoenv_t *xenv) { long dmin = off1 - lim2, dmax = lim1 - off2; @@ -92,7 +95,7 @@ static long xdl_split(xdfile_t *xdf1, long off1, long lim1, i1 = kvdf[d + 1]; prev1 = i1; i2 = i1 - d; - for (; i1 < lim1 && i2 < lim2 && get_hash(xdf1, i1) == get_hash(xdf2, i2); i1++, i2++); + for (; i1 < lim1 && i2 < lim2 && mc->ha1[i1] == mc->ha2[i2]; i1++, i2++); if (i1 - prev1 > xenv->snake_cnt) got_snake = 1; kvdf[d] = i1; @@ -129,7 +132,7 @@ static long xdl_split(xdfile_t *xdf1, long off1, long lim1, i1 = kvdb[d + 1] - 1; prev1 = i1; i2 = i1 - d; - for (; i1 > off1 && i2 > off2 && get_hash(xdf1, i1 - 1) == get_hash(xdf2, i2 - 1); i1--, i2--); + for (; i1 > off1 && i2 > off2 && mc->ha1[i1 - 1] == mc->ha2[i2 - 1]; i1--, i2--); if (prev1 - i1 > xenv->snake_cnt) got_snake = 1; kvdb[d] = i1; @@ -164,7 +167,7 @@ static long xdl_split(xdfile_t *xdf1, long off1, long lim1, if (v > XDL_K_HEUR * ec && v > best && off1 + xenv->snake_cnt <= i1 && i1 < lim1 && off2 + xenv->snake_cnt <= i2 && i2 < lim2) { - for (k = 1; get_hash(xdf1, i1 - k) == get_hash(xdf2, i2 - k); k++) + for (k = 1; mc->ha1[i1 - k] == mc->ha2[i2 - k]; k++) if (k == xenv->snake_cnt) { best = v; spl->i1 = i1; @@ -188,7 +191,7 @@ static long xdl_split(xdfile_t *xdf1, long off1, long lim1, if (v > XDL_K_HEUR * ec && v > best && off1 < i1 && i1 <= lim1 - xenv->snake_cnt && off2 < i2 && i2 <= lim2 - xenv->snake_cnt) { - for (k = 0; get_hash(xdf1, i1 + k) == get_hash(xdf2, i2 + k); k++) + for (k = 0; mc->ha1[i1 + k] == mc->ha2[i2 + k]; k++) if (k == xenv->snake_cnt - 1) { best = v; spl->i1 = i1; @@ -262,15 +265,17 @@ static long xdl_split(xdfile_t *xdf1, long off1, long lim1, * sub-boxes by calling the box splitting function. Note that the real job * (marking changed lines) is done in the two boundary reaching checks. */ -int xdl_recs_cmp(xdfile_t *xdf1, long off1, long lim1, - xdfile_t *xdf2, long off2, long lim2, - long *kvdf, long *kvdb, int need_min, xdalgoenv_t *xenv) { +static int xdl_recs_cmp_flat(xdmyers_t const *mc, + xdfile_t *xdf1, long off1, long lim1, + xdfile_t *xdf2, long off2, long lim2, + long *kvdf, long *kvdb, int need_min, + xdalgoenv_t *xenv) { /* * Shrink the box by walking through each diagonal snake (SW and NE). */ - for (; off1 < lim1 && off2 < lim2 && get_hash(xdf1, off1) == get_hash(xdf2, off2); off1++, off2++); - for (; off1 < lim1 && off2 < lim2 && get_hash(xdf1, lim1 - 1) == get_hash(xdf2, lim2 - 1); lim1--, lim2--); + for (; off1 < lim1 && off2 < lim2 && mc->ha1[off1] == mc->ha2[off2]; off1++, off2++); + for (; off1 < lim1 && off2 < lim2 && mc->ha1[lim1 - 1] == mc->ha2[lim2 - 1]; lim1--, lim2--); /* * If one dimension is empty, then all records on the other one must @@ -289,7 +294,7 @@ int xdl_recs_cmp(xdfile_t *xdf1, long off1, long lim1, /* * Divide ... */ - if (xdl_split(xdf1, off1, lim1, xdf2, off2, lim2, kvdf, kvdb, + if (xdl_split(mc, off1, lim1, off2, lim2, kvdf, kvdb, need_min, &spl, xenv) < 0) { return -1; @@ -298,10 +303,10 @@ int xdl_recs_cmp(xdfile_t *xdf1, long off1, long lim1, /* * ... et Impera. */ - if (xdl_recs_cmp(xdf1, off1, spl.i1, xdf2, off2, spl.i2, - kvdf, kvdb, spl.min_lo, xenv) < 0 || - xdl_recs_cmp(xdf1, spl.i1, lim1, xdf2, spl.i2, lim2, - kvdf, kvdb, spl.min_hi, xenv) < 0) { + if (xdl_recs_cmp_flat(mc, xdf1, off1, spl.i1, xdf2, off2, spl.i2, + kvdf, kvdb, spl.min_lo, xenv) < 0 || + xdl_recs_cmp_flat(mc, xdf1, spl.i1, lim1, xdf2, spl.i2, lim2, + kvdf, kvdb, spl.min_hi, xenv) < 0) { return -1; } @@ -311,6 +316,42 @@ int xdl_recs_cmp(xdfile_t *xdf1, long off1, long lim1, } +static size_t *xdl_gather_hashes(xdfile_t *xdf, long off, long lim) +{ + size_t *ha; + long i; + + if (!XDL_ALLOC_ARRAY(ha, lim > 0 ? lim : 1)) + return NULL; + for (i = off; i < lim; i++) + ha[i] = xdf->recs[xdf->reference_index[i]].minimal_perfect_hash; + return ha; +} + + +int xdl_recs_cmp(xdfile_t *xdf1, long off1, long lim1, + xdfile_t *xdf2, long off2, long lim2, + long *kvdf, long *kvdb, int need_min, xdalgoenv_t *xenv) { + xdmyers_t mc; + size_t *ha1, *ha2; + int res; + + if (!(ha1 = xdl_gather_hashes(xdf1, off1, lim1))) + return -1; + if (!(ha2 = xdl_gather_hashes(xdf2, off2, lim2))) { + xdl_free(ha1); + return -1; + } + mc.ha1 = ha1; + mc.ha2 = ha2; + res = xdl_recs_cmp_flat(&mc, xdf1, off1, lim1, xdf2, off2, lim2, + kvdf, kvdb, need_min, xenv); + xdl_free(ha2); + xdl_free(ha1); + return res; +} + + int xdl_do_diff(mmfile_t *mf1, mmfile_t *mf2, xpparam_t const *xpp, xdfenv_t *xe) { long ndiags; From 4c1212f5bb35dbf84bd0488ba415b0ef96f24332 Mon Sep 17 00:00:00 2001 From: Saurav Joshi Date: Thu, 12 Nov 2026 15:19:28 -0800 Subject: [PATCH 11/13] xdiff: loop over the second half in xdl_recs_cmp() After splitting a box, the second recursive call is the last thing xdl_recs_cmp() does. Handle the first half recursively as before, then continue with the second half in a loop instead of another recursive call. This is the same divide-and-conquer search in the same order; it only avoids one stack frame per split. Signed-off-by: Saurav Joshi --- xdiff/xdiffi.c | 61 +++++++++++++++++++++++--------------------------- 1 file changed, 28 insertions(+), 33 deletions(-) diff --git a/xdiff/xdiffi.c b/xdiff/xdiffi.c index dd70a390f1f394..a984f8b7f8cebf 100644 --- a/xdiff/xdiffi.c +++ b/xdiff/xdiffi.c @@ -270,49 +270,44 @@ static int xdl_recs_cmp_flat(xdmyers_t const *mc, xdfile_t *xdf2, long off2, long lim2, long *kvdf, long *kvdb, int need_min, xdalgoenv_t *xenv) { - - /* - * Shrink the box by walking through each diagonal snake (SW and NE). - */ - for (; off1 < lim1 && off2 < lim2 && mc->ha1[off1] == mc->ha2[off2]; off1++, off2++); - for (; off1 < lim1 && off2 < lim2 && mc->ha1[lim1 - 1] == mc->ha2[lim2 - 1]; lim1--, lim2--); - - /* - * If one dimension is empty, then all records on the other one must - * be obviously changed. - */ - if (off1 == lim1) { - for (; off2 < lim2; off2++) - xdf2->changed[xdf2->reference_index[off2]] = true; - } else if (off2 == lim2) { - for (; off1 < lim1; off1++) - xdf1->changed[xdf1->reference_index[off1]] = true; - } else { + for (;;) { xdpsplit_t spl; + + for (; off1 < lim1 && off2 < lim2 && + mc->ha1[off1] == mc->ha2[off2]; off1++, off2++) + ; + for (; off1 < lim1 && off2 < lim2 && + mc->ha1[lim1 - 1] == mc->ha2[lim2 - 1]; lim1--, lim2--) + ; + + if (off1 == lim1) { + for (; off2 < lim2; off2++) + xdf2->changed[xdf2->reference_index[off2]] = true; + return 0; + } + if (off2 == lim2) { + for (; off1 < lim1; off1++) + xdf1->changed[xdf1->reference_index[off1]] = true; + return 0; + } spl.i1 = spl.i2 = 0; - /* - * Divide ... - */ if (xdl_split(mc, off1, lim1, off2, lim2, kvdf, kvdb, - need_min, &spl, xenv) < 0) { - + need_min, &spl, xenv) < 0) return -1; - } /* - * ... et Impera. + * Preserve the recursive traversal order, but handle the + * second half by looping rather than adding a stack frame. */ - if (xdl_recs_cmp_flat(mc, xdf1, off1, spl.i1, xdf2, off2, spl.i2, - kvdf, kvdb, spl.min_lo, xenv) < 0 || - xdl_recs_cmp_flat(mc, xdf1, spl.i1, lim1, xdf2, spl.i2, lim2, - kvdf, kvdb, spl.min_hi, xenv) < 0) { - + if (xdl_recs_cmp_flat(mc, xdf1, off1, spl.i1, + xdf2, off2, spl.i2, + kvdf, kvdb, spl.min_lo, xenv) < 0) return -1; - } + off1 = spl.i1; + off2 = spl.i2; + need_min = spl.min_hi; } - - return 0; } From 10c62a5e8409a088e577c9106a36ed625da5596f Mon Sep 17 00:00:00 2001 From: Saurav Joshi Date: Thu, 12 Nov 2026 15:20:17 -0800 Subject: [PATCH 12/13] xdiff: skip unchanged lines faster when building the script xdl_build_script() walks every line from the end, even long runs that are unchanged on both sides. When bool is one byte, check eight entries of changed[] at a time and skip a word that is all zero on both sides. Signed-off-by: Saurav Joshi --- xdiff/xdiffi.c | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/xdiff/xdiffi.c b/xdiff/xdiffi.c index a984f8b7f8cebf..265f04c1c08653 100644 --- a/xdiff/xdiffi.c +++ b/xdiff/xdiffi.c @@ -821,6 +821,19 @@ static int group_slide_up(xdfile_t *xdf, struct xdlgroup *g) } } +/* + * changed[] holds bool values (0 or 1), so a word is zero iff all of its + * lines are unchanged, and the sum of its bytes counts changed lines. + * Callers use these word operations only when bool occupies one byte. + */ +static inline uint64_t load_changed8(const bool *p) +{ + uint64_t w; + + memcpy(&w, p, sizeof(w)); + return w; +} + /* * Move back and forward change groups for a consistent and pretty diff output. * This also helps in finding joinable change groups and reducing the diff @@ -1016,7 +1029,14 @@ int xdl_build_script(xdfenv_t *xe, xdchange_t **xscr) { /* * Trivial. Collects "groups" of changes and creates an edit script. */ - for (i1 = xe->xdf1.nrec, i2 = xe->xdf2.nrec; i1 >= 0 || i2 >= 0; i1--, i2--) + for (i1 = xe->xdf1.nrec, i2 = xe->xdf2.nrec; i1 >= 0 || i2 >= 0; i1--, i2--) { + /* Skip lines unchanged on both sides, eight at a time. */ + while (sizeof(bool) == 1 && i1 >= 8 && i2 >= 8 && + !load_changed8(changed1 + i1 - 8) && + !load_changed8(changed2 + i2 - 8)) { + i1 -= 8; + i2 -= 8; + } if (changed1[i1 - 1] || changed2[i2 - 1]) { for (l1 = i1; changed1[i1 - 1]; i1--); for (l2 = i2; changed2[i2 - 1]; i2--); @@ -1027,6 +1047,7 @@ int xdl_build_script(xdfenv_t *xe, xdchange_t **xscr) { } cscr = xch; } + } *xscr = cscr; From 4af06e0115427bad9b1f0164c4defd4dc2b18ccc Mon Sep 17 00:00:00 2001 From: Saurav Joshi Date: Thu, 12 Nov 2026 15:21:09 -0800 Subject: [PATCH 13/13] xdiff: skip empty groups quickly in xdl_change_compact() When a file has long unchanged stretches, xdl_change_compact() advances through empty groups one group_next() call at a time. Jump straight to the next changed line, and move the other file's group to the position the repeated group_next() calls would reach, counting unchanged lines eight at a time. Signed-off-by: Saurav Joshi --- xdiff/xdiffi.c | 77 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 76 insertions(+), 1 deletion(-) diff --git a/xdiff/xdiffi.c b/xdiff/xdiffi.c index 265f04c1c08653..b09bc9f502390f 100644 --- a/xdiff/xdiffi.c +++ b/xdiff/xdiffi.c @@ -834,6 +834,79 @@ static inline uint64_t load_changed8(const bool *p) return w; } +/* Return the first changed index in [from, lim), or lim. */ +static long next_changed(const bool *changed, long from, long lim) +{ + long i = from; + + while (sizeof(bool) == 1 && lim - i >= 8 && + !load_changed8(changed + i)) + i += 8; + while (i < lim && !changed[i]) + i++; + return i; +} + +/* + * Return the k-th (k >= 1) unchanged index after from. The false sentinel + * at changed[lim] bounds the search. + */ +static long kth_unchanged_after(const bool *changed, long from, long k, long lim) +{ + long i = from + 1; + + while (i <= lim) { + if (sizeof(bool) == 1 && lim - i >= 8) { + uint64_t w = load_changed8(changed + i); + long zeros = 8 - (long)((w * 0x0101010101010101ULL) >> 56); + + if (zeros < k) { + k -= zeros; + i += 8; + continue; + } + } + if (!changed[i] && !--k) + return i; + i++; + } + return lim; +} + +/* + * Advance an empty group and its counterpart as repeated group_next() + * calls would. Stop before the next non-empty group, or at end of file. + */ +static void skip_empty_groups(xdfile_t *xdf, struct xdlgroup *g, + xdfile_t *xdfo, struct xdlgroup *go) +{ + long nrec = (long)xdf->nrec, nreco = (long)xdfo->nrec; + long p = g->end, q, k, prev, e; + + if (p >= nrec) + return; + q = next_changed(xdf->changed, p + 1, nrec); + if (q >= nrec) + q = nrec + 1; /* changed[nrec] is a false sentinel */ + k = q - 1 - p; + if (k <= 0) + return; + + /* + * Each group_next() moves end to the next unchanged index, and + * start to the previous end plus one. + */ + if (k > nreco - go->end) + return; /* let group_next() report the broken sync */ + prev = k > 1 ? + kth_unchanged_after(xdfo->changed, go->end, k - 1, nreco) : + go->end; + e = kth_unchanged_after(xdfo->changed, prev, 1, nreco); + go->start = prev + 1; + go->end = e; + g->start = g->end = q - 1; +} + /* * Move back and forward change groups for a consistent and pretty diff output. * This also helps in finding joinable change groups and reducing the diff @@ -852,8 +925,10 @@ int xdl_change_compact(xdfile_t *xdf, xdfile_t *xdfo, long flags) { /* * If the group is empty in the to-be-compacted file, skip it: */ - if (g.end == g.start) + if (g.end == g.start) { + skip_empty_groups(xdf, &g, xdfo, &go); goto next; + } g_orig = g;