Skip to content

Commit 843d2ac

Browse files
gitstergitster
committed
Merge branch 'js/objects-larger-than-4gb-on-windows' into next
Update code paths that assumed "unsigned long" was long enough for "size_t". * js/objects-larger-than-4gb-on-windows: ci: run expensive tests on push builds to integration branches t5608: mark >4GB tests as EXPENSIVE test-tool synthesize: add precomputed SHA-256 pack for 4 GiB + 1 test-tool synthesize: precompute pack for 4 GiB + 1 test-tool synthesize: use the unsafe hash for speed t5608: add regression test for >4GB object clone test-tool: add a helper to synthesize large packfiles delta, packfile: use size_t for delta header sizes odb, packfile: use size_t for streaming object sizes git-zlib: handle data streams larger than 4GB index-pack, unpack-objects: use size_t for object size
2 parents 59709fa + 7a094d6 commit 843d2ac

22 files changed

Lines changed: 724 additions & 56 deletions

‎Makefile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -872,6 +872,7 @@ TEST_BUILTINS_OBJS += test-submodule-config.o
872872
TEST_BUILTINS_OBJS += test-submodule-nested-repo-config.o
873873
TEST_BUILTINS_OBJS += test-submodule.o
874874
TEST_BUILTINS_OBJS += test-subprocess.o
875+
TEST_BUILTINS_OBJS += test-synthesize.o
875876
TEST_BUILTINS_OBJS += test-trace2.o
876877
TEST_BUILTINS_OBJS += test-truncate.o
877878
TEST_BUILTINS_OBJS += test-userdiff.o

‎builtin/index-pack.c‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ static const char index_pack_usage[] =
3737

3838
struct object_entry {
3939
struct pack_idx_entry idx;
40-
unsigned long size;
40+
size_t size;
4141
unsigned char hdr_size;
4242
signed char type;
4343
signed char real_type;
@@ -469,7 +469,7 @@ static int is_delta_type(enum object_type type)
469469
return (type == OBJ_REF_DELTA || type == OBJ_OFS_DELTA);
470470
}
471471

472-
static void *unpack_entry_data(off_t offset, unsigned long size,
472+
static void *unpack_entry_data(off_t offset, size_t size,
473473
enum object_type type, struct object_id *oid)
474474
{
475475
static char fixed_buf[8192];
@@ -524,7 +524,7 @@ static void *unpack_raw_entry(struct object_entry *obj,
524524
struct object_id *oid)
525525
{
526526
unsigned char *p;
527-
unsigned long size, c;
527+
size_t size, c;
528528
off_t base_offset;
529529
unsigned shift;
530530
void *data;
@@ -539,6 +539,8 @@ static void *unpack_raw_entry(struct object_entry *obj,
539539
size = (c & 15);
540540
shift = 4;
541541
while (c & 0x80) {
542+
if ((bitsizeof(size_t) - 7) < shift)
543+
die(_("object size too large for this platform"));
542544
p = fill(1);
543545
c = *p;
544546
use(1);

‎builtin/pack-objects.c‎

Lines changed: 22 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -629,14 +629,21 @@ static off_t write_reuse_object(struct hashfile *f, struct object_entry *entry,
629629
struct packed_git *p = IN_PACK(entry);
630630
struct pack_window *w_curs = NULL;
631631
uint32_t pos;
632-
off_t offset;
632+
off_t offset, cur;
633633
enum object_type type = oe_type(entry);
634+
enum object_type in_pack_type;
634635
off_t datalen;
635636
unsigned char header[MAX_PACK_OBJECT_HEADER],
636637
dheader[MAX_PACK_OBJECT_HEADER];
637638
unsigned hdrlen;
638639
const unsigned hashsz = the_hash_algo->rawsz;
639-
unsigned long entry_size = SIZE(entry);
640+
size_t entry_size;
641+
642+
cur = entry->in_pack_offset;
643+
in_pack_type = unpack_object_header(p, &w_curs, &cur, &entry_size);
644+
if (in_pack_type < 0)
645+
die(_("write_reuse_object: unable to parse object header of %s"),
646+
oid_to_hex(&entry->idx.oid));
640647

641648
if (DELTA(entry))
642649
type = (allow_ofs_delta && DELTA(entry)->idx.offset) ?
@@ -664,7 +671,8 @@ static off_t write_reuse_object(struct hashfile *f, struct object_entry *entry,
664671
datalen -= entry->in_pack_header_size;
665672

666673
if (!pack_to_stdout && p->index_version == 1 &&
667-
check_pack_inflate(p, &w_curs, offset, datalen, entry_size)) {
674+
check_pack_inflate(p, &w_curs, offset, datalen,
675+
cast_size_t_to_ulong(entry_size))) {
668676
error(_("corrupt packed object for %s"),
669677
oid_to_hex(&entry->idx.oid));
670678
unuse_pack(&w_curs);
@@ -1087,7 +1095,7 @@ static void write_reused_pack_one(struct packed_git *reuse_packfile,
10871095
{
10881096
off_t offset, next, cur;
10891097
enum object_type type;
1090-
unsigned long size;
1098+
size_t size;
10911099

10921100
offset = pack_pos_to_offset(reuse_packfile, pos);
10931101
next = pack_pos_to_offset(reuse_packfile, pos + 1);
@@ -2243,7 +2251,7 @@ static void check_object(struct object_entry *entry, uint32_t object_index)
22432251
off_t ofs;
22442252
unsigned char *buf, c;
22452253
enum object_type type;
2246-
unsigned long in_pack_size;
2254+
size_t in_pack_size;
22472255

22482256
buf = use_pack(p, &w_curs, entry->in_pack_offset, &avail);
22492257

@@ -2270,7 +2278,7 @@ static void check_object(struct object_entry *entry, uint32_t object_index)
22702278
default:
22712279
/* Not a delta hence we've already got all we need. */
22722280
oe_set_type(entry, entry->in_pack_type);
2273-
SET_SIZE(entry, in_pack_size);
2281+
SET_SIZE(entry, cast_size_t_to_ulong(in_pack_size));
22742282
entry->in_pack_header_size = used;
22752283
if (oe_type(entry) < OBJ_COMMIT || oe_type(entry) > OBJ_BLOB)
22762284
goto give_up;
@@ -2324,8 +2332,8 @@ static void check_object(struct object_entry *entry, uint32_t object_index)
23242332
if (have_base &&
23252333
can_reuse_delta(&base_ref, entry, &base_entry)) {
23262334
oe_set_type(entry, entry->in_pack_type);
2327-
SET_SIZE(entry, in_pack_size); /* delta size */
2328-
SET_DELTA_SIZE(entry, in_pack_size);
2335+
SET_SIZE(entry, cast_size_t_to_ulong(in_pack_size)); /* delta size */
2336+
SET_DELTA_SIZE(entry, cast_size_t_to_ulong(in_pack_size));
23292337

23302338
if (base_entry) {
23312339
SET_DELTA(entry, base_entry);
@@ -2734,16 +2742,18 @@ unsigned long oe_get_size_slow(struct packing_data *pack,
27342742
struct pack_window *w_curs;
27352743
unsigned char *buf;
27362744
enum object_type type;
2737-
unsigned long used, avail, size;
2745+
unsigned long used, avail;
2746+
size_t size;
27382747

27392748
if (e->type_ != OBJ_OFS_DELTA && e->type_ != OBJ_REF_DELTA) {
2749+
unsigned long sz;
27402750
packing_data_lock(&to_pack);
27412751
if (odb_read_object_info(the_repository->objects,
2742-
&e->idx.oid, &size) < 0)
2752+
&e->idx.oid, &sz) < 0)
27432753
die(_("unable to get size of %s"),
27442754
oid_to_hex(&e->idx.oid));
27452755
packing_data_unlock(&to_pack);
2746-
return size;
2756+
return sz;
27472757
}
27482758

27492759
p = oe_in_pack(pack, e);
@@ -2760,7 +2770,7 @@ unsigned long oe_get_size_slow(struct packing_data *pack,
27602770

27612771
unuse_pack(&w_curs);
27622772
packing_data_unlock(&to_pack);
2763-
return size;
2773+
return cast_size_t_to_ulong(size);
27642774
}
27652775

27662776
static int try_delta(struct unpacked *trg, struct unpacked *src,

‎builtin/unpack-objects.c‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -533,7 +533,7 @@ static void unpack_one(unsigned nr)
533533
{
534534
unsigned shift;
535535
unsigned char *pack;
536-
unsigned long size, c;
536+
size_t size, c;
537537
enum object_type type;
538538

539539
obj_list[nr].offset = consumed_bytes;
@@ -545,6 +545,8 @@ static void unpack_one(unsigned nr)
545545
size = (c & 15);
546546
shift = 4;
547547
while (c & 0x80) {
548+
if ((bitsizeof(size_t) - 7) < shift)
549+
die(_("object size too large for this platform"));
548550
pack = fill(1);
549551
c = *pack;
550552
use(1);

‎ci/lib.sh‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -314,6 +314,15 @@ export DEFAULT_TEST_TARGET=prove
314314
export GIT_TEST_CLONE_2GB=true
315315
export SKIP_DASHED_BUILT_INS=YesPlease
316316

317+
# Enable expensive tests on push builds to integration branches, but
318+
# not on PR builds where the extra time is not justified for every
319+
# iteration.
320+
case "$GITHUB_EVENT_NAME,$CI_BRANCH" in
321+
push,*next*|push,*master*|push,*main*|push,*maint*)
322+
export GIT_TEST_LONG=YesPlease
323+
;;
324+
esac
325+
317326
case "$distro" in
318327
ubuntu-*)
319328
# Python 2 is end of life, and Ubuntu 23.04 and newer don't actually

‎compat/zlib-compat.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77
# define z_stream_s zng_stream_s
88
# define gz_header_s zng_gz_header_s
99

10+
# define adler32(adler, buf, len) zng_adler32(adler, buf, len)
11+
1012
# define crc32(crc, buf, len) zng_crc32(crc, buf, len)
1113

1214
# define inflate(strm, bits) zng_inflate(strm, bits)

‎delta.h‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -86,8 +86,11 @@ void *patch_delta(const void *src_buf, unsigned long src_size,
8686
* This must be called twice on the delta data buffer, first to get the
8787
* expected source buffer size, and again to get the target buffer size.
8888
*/
89-
static inline unsigned long get_delta_hdr_size(const unsigned char **datap,
90-
const unsigned char *top)
89+
/*
90+
* Size_t variant that doesn't truncate - use for >4GB objects on Windows.
91+
*/
92+
static inline size_t get_delta_hdr_size_sz(const unsigned char **datap,
93+
const unsigned char *top)
9194
{
9295
const unsigned char *data = *datap;
9396
size_t cmd, size = 0;
@@ -98,6 +101,13 @@ static inline unsigned long get_delta_hdr_size(const unsigned char **datap,
98101
i += 7;
99102
} while (cmd & 0x80 && data < top);
100103
*datap = data;
104+
return size;
105+
}
106+
107+
static inline unsigned long get_delta_hdr_size(const unsigned char **datap,
108+
const unsigned char *top)
109+
{
110+
size_t size = get_delta_hdr_size_sz(datap, top);
101111
return cast_size_t_to_ulong(size);
102112
}
103113

‎git-zlib.c‎

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,9 @@ static const char *zerr_to_string(int status)
3030
*/
3131
/* #define ZLIB_BUF_MAX ((uInt)-1) */
3232
#define ZLIB_BUF_MAX ((uInt) 1024 * 1024 * 1024) /* 1GB */
33+
34+
/* uLong is 32-bit on Windows, even on 64-bit systems */
35+
#define ULONG_MAX_VALUE maximum_unsigned_value_of_type(uLong)
3336
static inline uInt zlib_buf_cap(unsigned long len)
3437
{
3538
return (ZLIB_BUF_MAX < len) ? ZLIB_BUF_MAX : len;
@@ -39,31 +42,37 @@ static void zlib_pre_call(git_zstream *s)
3942
{
4043
s->z.next_in = s->next_in;
4144
s->z.next_out = s->next_out;
42-
s->z.total_in = s->total_in;
43-
s->z.total_out = s->total_out;
45+
s->z.total_in = (uLong)(s->total_in & ULONG_MAX_VALUE);
46+
s->z.total_out = (uLong)(s->total_out & ULONG_MAX_VALUE);
4447
s->z.avail_in = zlib_buf_cap(s->avail_in);
4548
s->z.avail_out = zlib_buf_cap(s->avail_out);
4649
}
4750

4851
static void zlib_post_call(git_zstream *s, int status)
4952
{
50-
unsigned long bytes_consumed;
51-
unsigned long bytes_produced;
53+
size_t bytes_consumed;
54+
size_t bytes_produced;
5255

5356
bytes_consumed = s->z.next_in - s->next_in;
5457
bytes_produced = s->z.next_out - s->next_out;
55-
if (s->z.total_out != s->total_out + bytes_produced)
58+
/*
59+
* zlib's total_out/total_in are uLong which may wrap for >4GB.
60+
* We track our own totals and verify only the low bits match.
61+
*/
62+
if ((s->z.total_out & ULONG_MAX_VALUE) !=
63+
((s->total_out + bytes_produced) & ULONG_MAX_VALUE))
5664
BUG("total_out mismatch");
5765
/*
5866
* zlib does not update total_in when it returns Z_NEED_DICT,
5967
* causing a mismatch here. Skip the sanity check in that case.
6068
*/
6169
if (status != Z_NEED_DICT &&
62-
s->z.total_in != s->total_in + bytes_consumed)
70+
(s->z.total_in & ULONG_MAX_VALUE) !=
71+
((s->total_in + bytes_consumed) & ULONG_MAX_VALUE))
6372
BUG("total_in mismatch");
6473

65-
s->total_out = s->z.total_out;
66-
s->total_in = s->z.total_in;
74+
s->total_out += bytes_produced;
75+
s->total_in += bytes_consumed;
6776
/* zlib-ng marks `next_in` as `const`, so we have to cast it away. */
6877
s->next_in = (unsigned char *) s->z.next_in;
6978
s->next_out = s->z.next_out;

‎git-zlib.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@ typedef struct git_zstream {
77
struct z_stream_s z;
88
unsigned long avail_in;
99
unsigned long avail_out;
10-
unsigned long total_in;
11-
unsigned long total_out;
10+
size_t total_in;
11+
size_t total_out;
1212
unsigned char *next_in;
1313
unsigned char *next_out;
1414
} git_zstream;

‎object-file.c‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1118,7 +1118,7 @@ int odb_source_loose_write_stream(struct odb_source *source,
11181118
} while (ret == Z_OK || ret == Z_BUF_ERROR);
11191119

11201120
if (stream.total_in != len + hdrlen)
1121-
die(_("write stream object %ld != %"PRIuMAX), stream.total_in,
1121+
die(_("write stream object %"PRIuMAX" != %"PRIuMAX), (uintmax_t)stream.total_in,
11221122
(uintmax_t)len + hdrlen);
11231123

11241124
/*
@@ -2326,6 +2326,7 @@ int odb_source_loose_read_object_stream(struct odb_read_stream **out,
23262326
struct object_info oi = OBJECT_INFO_INIT;
23272327
struct odb_loose_read_stream *st;
23282328
unsigned long mapsize;
2329+
unsigned long size_ul;
23292330
void *mapped;
23302331

23312332
mapped = odb_source_loose_map_object(source, oid, &mapsize);
@@ -2349,11 +2350,18 @@ int odb_source_loose_read_object_stream(struct odb_read_stream **out,
23492350
goto error;
23502351
}
23512352

2352-
oi.sizep = &st->base.size;
2353+
/*
2354+
* object_info.sizep is unsigned long* (32-bit on Windows), but
2355+
* st->base.size is size_t (64-bit). Use temporary variable.
2356+
* Note: loose objects >4GB would still truncate here, but such
2357+
* large loose objects are uncommon (they'd normally be packed).
2358+
*/
2359+
oi.sizep = &size_ul;
23532360
oi.typep = &st->base.type;
23542361

23552362
if (parse_loose_header(st->hdr, &oi) < 0 || st->base.type < 0)
23562363
goto error;
2364+
st->base.size = size_ul;
23572365

23582366
st->mapped = mapped;
23592367
st->mapsize = mapsize;

0 commit comments

Comments
 (0)