1
2
3
4
5
6
7
8
9
10
11
12
13
14 package org.eclipse.jgit.transport;
15
16 import static org.junit.Assert.assertEquals;
17 import static org.junit.Assert.assertTrue;
18 import static org.junit.Assert.fail;
19
20 import java.io.ByteArrayInputStream;
21 import java.io.File;
22 import java.io.FileInputStream;
23 import java.io.IOException;
24 import java.io.InputStream;
25 import java.security.MessageDigest;
26 import java.text.MessageFormat;
27 import java.util.HashMap;
28 import java.util.List;
29 import java.util.Map;
30 import java.util.zip.Deflater;
31
32 import org.eclipse.jgit.errors.TooLargeObjectInPackException;
33 import org.eclipse.jgit.internal.JGitText;
34 import org.eclipse.jgit.internal.storage.file.ObjectDirectoryPackParser;
35 import org.eclipse.jgit.internal.storage.file.Pack;
36 import org.eclipse.jgit.junit.JGitTestUtil;
37 import org.eclipse.jgit.junit.RepositoryTestCase;
38 import org.eclipse.jgit.junit.TestRepository;
39 import org.eclipse.jgit.lib.Constants;
40 import org.eclipse.jgit.lib.NullProgressMonitor;
41 import org.eclipse.jgit.lib.ObjectId;
42 import org.eclipse.jgit.lib.ObjectInserter;
43 import org.eclipse.jgit.lib.Repository;
44 import org.eclipse.jgit.revwalk.RevBlob;
45 import org.eclipse.jgit.util.NB;
46 import org.eclipse.jgit.util.TemporaryBuffer;
47 import org.eclipse.jgit.util.io.UnionInputStream;
48 import org.junit.After;
49 import org.junit.Test;
50
51
52
53
54
55
56
57 public class PackParserTest extends RepositoryTestCase {
58
59
60
61
62
63 @Test
64 public void test1() throws IOException {
65 File packFile = JGitTestUtil.getTestResourceFile("pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.pack");
66 try (InputStream is = new FileInputStream(packFile)) {
67 ObjectDirectoryPackParser p = (ObjectDirectoryPackParser) index(is);
68 p.parse(NullProgressMonitor.INSTANCE);
69 Pack pack = p.getPack();
70
71 assertTrue(pack.hasObject(ObjectId.fromString("4b825dc642cb6eb9a060e54bf8d69288fbee4904")));
72 assertTrue(pack.hasObject(ObjectId.fromString("540a36d136cf413e4b064c2b0e0a4db60f77feab")));
73 assertTrue(pack.hasObject(ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259")));
74 assertTrue(pack.hasObject(ObjectId.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3")));
75 assertTrue(pack.hasObject(ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7")));
76 assertTrue(pack.hasObject(ObjectId.fromString("902d5476fa249b7abc9d84c611577a81381f0327")));
77 assertTrue(pack.hasObject(ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035")));
78 assertTrue(pack.hasObject(ObjectId.fromString("c59759f143fb1fe21c197981df75a7ee00290799")));
79 }
80 }
81
82 @Test
83 public void testParsePack1ReadsObjectSizes() throws IOException {
84 File packFile = JGitTestUtil.getTestResourceFile(
85 "pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.pack");
86
87
88 Map<String, Long> expected = new HashMap<>();
89
90 expected.put("540a36d136cf413e4b064c2b0e0a4db60f77feab",
91 Long.valueOf(191));
92 expected.put("c59759f143fb1fe21c197981df75a7ee00290799",
93 Long.valueOf(240));
94 expected.put("82c6b885ff600be425b4ea96dee75dca255b69e7",
95 Long.valueOf(245));
96
97
98 expected.put("4b825dc642cb6eb9a060e54bf8d69288fbee4904",
99 Long.valueOf(0));
100 expected.put("902d5476fa249b7abc9d84c611577a81381f0327",
101 Long.valueOf(35));
102 expected.put("aabf2ffaec9b497f0950352b3e582d73035c2035",
103 Long.valueOf(35));
104
105
106 expected.put("6ff87c4664981e4397625791c8ea3bbb5f2279a3",
107 Long.valueOf(18787));
108
109
110 expected.put("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259",
111 Long.valueOf(18009));
112
113
114 try (InputStream is = new FileInputStream(packFile)) {
115 ObjectDirectoryPackParser p = (ObjectDirectoryPackParser) index(is);
116 p.parse(NullProgressMonitor.INSTANCE);
117 List<PackedObjectInfo> parsedObjects = p.getSortedObjectList(null);
118 for (PackedObjectInfo objInfo: parsedObjects) {
119 assertEquals(objInfo.getName(), objInfo.getFullSize(),
120 expected.get(objInfo.getName()).longValue());
121 }
122 }
123 }
124
125
126
127
128
129
130
131 @Test
132 public void test2() throws IOException {
133 File packFile = JGitTestUtil.getTestResourceFile("pack-df2982f284bbabb6bdb59ee3fcc6eb0983e20371.pack");
134 try (InputStream is = new FileInputStream(packFile)) {
135 ObjectDirectoryPackParser p = (ObjectDirectoryPackParser) index(is);
136 p.parse(NullProgressMonitor.INSTANCE);
137 Pack pack = p.getPack();
138
139 assertTrue(pack.hasObject(ObjectId.fromString("02ba32d3649e510002c21651936b7077aa75ffa9")));
140 assertTrue(pack.hasObject(ObjectId.fromString("0966a434eb1a025db6b71485ab63a3bfbea520b6")));
141 assertTrue(pack.hasObject(ObjectId.fromString("09efc7e59a839528ac7bda9fa020dc9101278680")));
142 assertTrue(pack.hasObject(ObjectId.fromString("0a3d7772488b6b106fb62813c4d6d627918d9181")));
143 assertTrue(pack.hasObject(ObjectId.fromString("1004d0d7ac26fbf63050a234c9b88a46075719d3")));
144 assertTrue(pack.hasObject(ObjectId.fromString("10da5895682013006950e7da534b705252b03be6")));
145 assertTrue(pack.hasObject(ObjectId.fromString("1203b03dc816ccbb67773f28b3c19318654b0bc8")));
146 assertTrue(pack.hasObject(ObjectId.fromString("15fae9e651043de0fd1deef588aa3fbf5a7a41c6")));
147 assertTrue(pack.hasObject(ObjectId.fromString("16f9ec009e5568c435f473ba3a1df732d49ce8c3")));
148 assertTrue(pack.hasObject(ObjectId.fromString("1fd7d579fb6ae3fe942dc09c2c783443d04cf21e")));
149 assertTrue(pack.hasObject(ObjectId.fromString("20a8ade77639491ea0bd667bf95de8abf3a434c8")));
150 assertTrue(pack.hasObject(ObjectId.fromString("2675188fd86978d5bc4d7211698b2118ae3bf658")));
151
152 }
153 }
154
155 @Test
156 public void testParsePack2ReadsObjectSizes() throws IOException {
157 File packFile = JGitTestUtil.getTestResourceFile(
158 "pack-df2982f284bbabb6bdb59ee3fcc6eb0983e20371.pack");
159 Map<String, Long> expected = new HashMap<>();
160
161 expected.put("d0114ab8ac326bab30e3a657a0397578c5a1af88",
162 Long.valueOf(222));
163
164 expected.put("f73b95671f326616d66b2afb3bdfcdbbce110b44",
165 Long.valueOf(221));
166
167 expected.put("be9b45333b66013bde1c7314efc50fabd9b39c6d",
168 Long.valueOf(94));
169
170 try (InputStream is = new FileInputStream(packFile)) {
171 ObjectDirectoryPackParser p = (ObjectDirectoryPackParser) index(is);
172 p.parse(NullProgressMonitor.INSTANCE);
173 List<PackedObjectInfo> parsedObjects = p.getSortedObjectList(null);
174
175 int assertedObjs = 0;
176 for (PackedObjectInfo objInfo : parsedObjects) {
177 if (!expected.containsKey(objInfo.getName())) {
178 continue;
179 }
180 assertEquals(objInfo.getName(), objInfo.getFullSize(),
181 expected.get(objInfo.getName()).longValue());
182 assertedObjs += 1;
183 }
184 assertEquals(assertedObjs, expected.size());
185 }
186 }
187
188 @Test
189 public void testTinyThinPack() throws Exception {
190 RevBlob a;
191 try (TestRepository d = new TestRepository<Repository>(db)) {
192 db.incrementOpen();
193 a = d.blob("a");
194 }
195
196 TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(1024);
197
198 packHeader(pack, 1);
199
200 pack.write((Constants.OBJ_REF_DELTA) << 4 | 4);
201 a.copyRawTo(pack);
202 deflate(pack, new byte[] { 0x1, 0x1, 0x1, 'b' });
203
204 digest(pack);
205
206 PackParser p = index(new ByteArrayInputStream(pack.toByteArray()));
207 p.setAllowThin(true);
208 p.parse(NullProgressMonitor.INSTANCE);
209 }
210
211 @Test
212 public void testPackWithDuplicateBlob() throws Exception {
213 final byte[] data = Constants.encode("0123456789abcdefg");
214 try (TestRepository<Repository> d = new TestRepository<>(db)) {
215 db.incrementOpen();
216 assertTrue(db.getObjectDatabase().has(d.blob(data)));
217 }
218
219 TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(1024);
220 packHeader(pack, 1);
221 pack.write((Constants.OBJ_BLOB) << 4 | 0x80 | 1);
222 pack.write(1);
223 deflate(pack, data);
224 digest(pack);
225
226 PackParser p = index(new ByteArrayInputStream(pack.toByteArray()));
227 p.setAllowThin(false);
228 p.parse(NullProgressMonitor.INSTANCE);
229 }
230
231 @Test
232 public void testParseOfsDeltaFullSize() throws Exception {
233 final byte[] data = Constants.encode("0123456789");
234 try (TestRepository<Repository> d = new TestRepository<>(db)) {
235 db.incrementOpen();
236 assertTrue(db.getObjectDatabase().has(d.blob(data)));
237 }
238
239 TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(1024);
240 packHeader(pack, 2);
241 pack.write((Constants.OBJ_BLOB) << 4 | 10);
242 deflate(pack, data);
243 pack.write((Constants.OBJ_OFS_DELTA) << 4 | 4);
244 pack.write(19);
245 deflate(pack, new byte[] { 0xA, 0xB, 0x1, 'b' });
246 digest(pack);
247
248 PackParser p = index(new ByteArrayInputStream(pack.toByteArray()));
249 p.parse(NullProgressMonitor.INSTANCE);
250
251 List<PackedObjectInfo> sortedObjectList = p.getSortedObjectList(null);
252 assertEquals(sortedObjectList.size(), 2);
253
254
255 PackedObjectInfo deltifiedObj = sortedObjectList.get(0);
256 assertEquals(deltifiedObj.getName(),
257 "16646543f87fb53e30b032eec7dfc88f2e717966");
258 assertEquals(deltifiedObj.getOffset(), 31);
259 assertEquals(deltifiedObj.getType(), Constants.OBJ_BLOB);
260 assertEquals(deltifiedObj.getFullSize(), 11);
261
262 PackedObjectInfo baseObj = sortedObjectList.get(1);
263 assertEquals(baseObj.getName(),
264 "ad471007bd7f5983d273b9584e5629230150fd54");
265 assertEquals(baseObj.getOffset(), 12);
266 assertEquals(baseObj.getType(), Constants.OBJ_BLOB);
267 assertEquals(baseObj.getFullSize(), 10);
268 }
269
270 @Test
271 public void testPackWithTrailingGarbage() throws Exception {
272 RevBlob a;
273 try (TestRepository d = new TestRepository<Repository>(db)) {
274 db.incrementOpen();
275 a = d.blob("a");
276 }
277
278 TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(1024);
279 packHeader(pack, 1);
280 pack.write((Constants.OBJ_REF_DELTA) << 4 | 4);
281 a.copyRawTo(pack);
282 deflate(pack, new byte[] { 0x1, 0x1, 0x1, 'b' });
283 digest(pack);
284
285 PackParser p = index(new UnionInputStream(
286 new ByteArrayInputStream(pack.toByteArray()),
287 new ByteArrayInputStream(new byte[] { 0x7e })));
288 p.setAllowThin(true);
289 p.setCheckEofAfterPackFooter(true);
290 try {
291 p.parse(NullProgressMonitor.INSTANCE);
292 fail("Pack with trailing garbage was accepted");
293 } catch (IOException err) {
294 assertEquals(
295 MessageFormat.format(JGitText.get().expectedEOFReceived, "\\x7e"),
296 err.getMessage());
297 }
298 }
299
300 @Test
301 public void testMaxObjectSizeFullBlob() throws Exception {
302 final byte[] data = Constants.encode("0123456789");
303 try (TestRepository d = new TestRepository<Repository>(db)) {
304 db.incrementOpen();
305 d.blob(data);
306 }
307
308 TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(1024);
309
310 packHeader(pack, 1);
311 pack.write((Constants.OBJ_BLOB) << 4 | 10);
312 deflate(pack, data);
313 digest(pack);
314
315 PackParser p = index(new ByteArrayInputStream(pack.toByteArray()));
316 p.setMaxObjectSizeLimit(11);
317 p.parse(NullProgressMonitor.INSTANCE);
318
319 p = index(new ByteArrayInputStream(pack.toByteArray()));
320 p.setMaxObjectSizeLimit(10);
321 p.parse(NullProgressMonitor.INSTANCE);
322
323 p = index(new ByteArrayInputStream(pack.toByteArray()));
324 p.setMaxObjectSizeLimit(9);
325 try {
326 p.parse(NullProgressMonitor.INSTANCE);
327 fail("PackParser should have failed");
328 } catch (TooLargeObjectInPackException e) {
329 assertTrue(e.getMessage().contains("10"));
330 assertTrue(e.getMessage().contains("9"));
331 }
332 }
333
334 @Test
335 public void testMaxObjectSizeDeltaBlock() throws Exception {
336 RevBlob a;
337 try (TestRepository d = new TestRepository<Repository>(db)) {
338 db.incrementOpen();
339 a = d.blob("a");
340 }
341
342 TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(1024);
343
344 packHeader(pack, 1);
345 pack.write((Constants.OBJ_REF_DELTA) << 4 | 14);
346 a.copyRawTo(pack);
347 deflate(pack, new byte[] { 1, 11, 11, 'a', '0', '1', '2', '3', '4',
348 '5', '6', '7', '8', '9' });
349 digest(pack);
350
351 PackParser p = index(new ByteArrayInputStream(pack.toByteArray()));
352 p.setAllowThin(true);
353 p.setMaxObjectSizeLimit(14);
354 p.parse(NullProgressMonitor.INSTANCE);
355
356 p = index(new ByteArrayInputStream(pack.toByteArray()));
357 p.setAllowThin(true);
358 p.setMaxObjectSizeLimit(13);
359 try {
360 p.parse(NullProgressMonitor.INSTANCE);
361 fail("PackParser should have failed");
362 } catch (TooLargeObjectInPackException e) {
363 assertTrue(e.getMessage().contains("13"));
364 assertTrue(e.getMessage().contains("14"));
365 }
366 }
367
368 @Test
369 public void testMaxObjectSizeDeltaResultSize() throws Exception {
370 RevBlob a;
371 try (TestRepository d = new TestRepository<Repository>(db)) {
372 db.incrementOpen();
373 a = d.blob("0123456789");
374 }
375
376 TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(1024);
377
378 packHeader(pack, 1);
379 pack.write((Constants.OBJ_REF_DELTA) << 4 | 4);
380 a.copyRawTo(pack);
381 deflate(pack, new byte[] { 10, 11, 1, 'a' });
382 digest(pack);
383
384 PackParser p = index(new ByteArrayInputStream(pack.toByteArray()));
385 p.setAllowThin(true);
386 p.setMaxObjectSizeLimit(11);
387 p.parse(NullProgressMonitor.INSTANCE);
388
389 p = index(new ByteArrayInputStream(pack.toByteArray()));
390 p.setAllowThin(true);
391 p.setMaxObjectSizeLimit(10);
392 try {
393 p.parse(NullProgressMonitor.INSTANCE);
394 fail("PackParser should have failed");
395 } catch (TooLargeObjectInPackException e) {
396 assertTrue(e.getMessage().contains("11"));
397 assertTrue(e.getMessage().contains("10"));
398 }
399 }
400
401 @Test
402 public void testNonMarkingInputStream() throws Exception {
403 RevBlob a;
404 try (TestRepository d = new TestRepository<Repository>(db)) {
405 db.incrementOpen();
406 a = d.blob("a");
407 }
408
409 TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(1024);
410 packHeader(pack, 1);
411 pack.write((Constants.OBJ_REF_DELTA) << 4 | 4);
412 a.copyRawTo(pack);
413 deflate(pack, new byte[] { 0x1, 0x1, 0x1, 'b' });
414 digest(pack);
415
416 InputStream in = new ByteArrayInputStream(pack.toByteArray()) {
417 @Override
418 public boolean markSupported() {
419 return false;
420 }
421
422 @Override
423 public void mark(int maxlength) {
424 fail("Mark should not be called");
425 }
426 };
427
428 PackParser p = index(in);
429 p.setAllowThin(true);
430 p.setCheckEofAfterPackFooter(false);
431 p.setExpectDataAfterPackFooter(true);
432
433 try {
434 p.parse(NullProgressMonitor.INSTANCE);
435 fail("PackParser should have failed");
436 } catch (IOException e) {
437 assertEquals(e.getMessage(),
438 JGitText.get().inputStreamMustSupportMark);
439 }
440 }
441
442 @Test
443 public void testDataAfterPackFooterSingleRead() throws Exception {
444 RevBlob a;
445 try (TestRepository d = new TestRepository<Repository>(db)) {
446 db.incrementOpen();
447 a = d.blob("a");
448 }
449
450 TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(32*1024);
451 packHeader(pack, 1);
452 pack.write((Constants.OBJ_REF_DELTA) << 4 | 4);
453 a.copyRawTo(pack);
454 deflate(pack, new byte[] { 0x1, 0x1, 0x1, 'b' });
455 digest(pack);
456
457 byte packData[] = pack.toByteArray();
458 byte streamData[] = new byte[packData.length + 1];
459 System.arraycopy(packData, 0, streamData, 0, packData.length);
460 streamData[packData.length] = 0x7e;
461
462 InputStream in = new ByteArrayInputStream(streamData);
463 PackParser p = index(in);
464 p.setAllowThin(true);
465 p.setCheckEofAfterPackFooter(false);
466 p.setExpectDataAfterPackFooter(true);
467
468 p.parse(NullProgressMonitor.INSTANCE);
469
470 assertEquals(0x7e, in.read());
471 }
472
473 @Test
474 public void testDataAfterPackFooterSplitObjectRead() throws Exception {
475 final byte[] data = Constants.encode("0123456789");
476
477
478 int objects = 900;
479 TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(32 * 1024);
480 packHeader(pack, objects);
481
482 for (int i = 0; i < objects; i++) {
483 pack.write((Constants.OBJ_BLOB) << 4 | 10);
484 deflate(pack, data);
485 }
486 digest(pack);
487
488 byte packData[] = pack.toByteArray();
489 byte streamData[] = new byte[packData.length + 1];
490 System.arraycopy(packData, 0, streamData, 0, packData.length);
491 streamData[packData.length] = 0x7e;
492 InputStream in = new ByteArrayInputStream(streamData);
493 PackParser p = index(in);
494 p.setAllowThin(true);
495 p.setCheckEofAfterPackFooter(false);
496 p.setExpectDataAfterPackFooter(true);
497
498 p.parse(NullProgressMonitor.INSTANCE);
499
500 assertEquals(0x7e, in.read());
501 }
502
503 @Test
504 public void testDataAfterPackFooterSplitHeaderRead() throws Exception {
505 final byte[] data = Constants.encode("a");
506 RevBlob b;
507 try (TestRepository d = new TestRepository<Repository>(db)) {
508 db.incrementOpen();
509 b = d.blob(data);
510 }
511
512 int objects = 248;
513 TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(32 * 1024);
514 packHeader(pack, objects + 1);
515 int offset = 13;
516 StringBuilder sb = new StringBuilder();
517 for (int i = 0; i < offset; i++)
518 sb.append(i);
519 offset = sb.toString().length();
520 int lenByte = (Constants.OBJ_BLOB) << 4 | (offset & 0x0F);
521 offset >>= 4;
522 if (offset > 0)
523 lenByte |= 1 << 7;
524 pack.write(lenByte);
525 while (offset > 0) {
526 lenByte = offset & 0x7F;
527 offset >>= 6;
528 if (offset > 0)
529 lenByte |= 1 << 7;
530 pack.write(lenByte);
531 }
532 deflate(pack, Constants.encode(sb.toString()));
533
534 for (int i = 0; i < objects; i++) {
535
536
537 pack.write((Constants.OBJ_REF_DELTA) << 4 | 4);
538 b.copyRawTo(pack);
539 deflate(pack, new byte[] { 0x1, 0x1, 0x1, 'b' });
540 }
541 digest(pack);
542
543 byte packData[] = pack.toByteArray();
544 byte streamData[] = new byte[packData.length + 1];
545 System.arraycopy(packData, 0, streamData, 0, packData.length);
546 streamData[packData.length] = 0x7e;
547 InputStream in = new ByteArrayInputStream(streamData);
548 PackParser p = index(in);
549 p.setAllowThin(true);
550 p.setCheckEofAfterPackFooter(false);
551 p.setExpectDataAfterPackFooter(true);
552
553 p.parse(NullProgressMonitor.INSTANCE);
554
555 assertEquals(0x7e, in.read());
556 }
557
558 private static void packHeader(TemporaryBuffer.Heap tinyPack, int cnt)
559 throws IOException {
560 final byte[] hdr = new byte[8];
561 NB.encodeInt32(hdr, 0, 2);
562 NB.encodeInt32(hdr, 4, cnt);
563
564 tinyPack.write(Constants.PACK_SIGNATURE);
565 tinyPack.write(hdr, 0, 8);
566 }
567
568 private static void deflate(TemporaryBuffer.Heap tinyPack,
569 final byte[] content)
570 throws IOException {
571 final Deflater deflater = new Deflater();
572 final byte[] buf = new byte[128];
573 deflater.setInput(content, 0, content.length);
574 deflater.finish();
575 do {
576 final int n = deflater.deflate(buf, 0, buf.length);
577 if (n > 0)
578 tinyPack.write(buf, 0, n);
579 } while (!deflater.finished());
580 }
581
582 private static void digest(TemporaryBuffer.Heap buf) throws IOException {
583 MessageDigest md = Constants.newMessageDigest();
584 md.update(buf.toByteArray());
585 buf.write(md.digest());
586 }
587
588 private ObjectInserter inserter;
589
590 @After
591 public void release() {
592 if (inserter != null) {
593 inserter.close();
594 }
595 }
596
597 private PackParser index(InputStream in) throws IOException {
598 if (inserter == null)
599 inserter = db.newObjectInserter();
600 return inserter.newPackParser(in);
601 }
602 }