Fix binary chunk decoding and inflating

Git uses a unique Base85 encoding with different characters than the
ascii85 encoding implemented by Go, so add a custom decoding function.
Once decoded, use zlib instead of the raw DEFLATE algorithm to
decompress the data.

These issues were caught by some basic parsing tests which are added
here as well.

Author: bluekeyes

gitdiff/base85.go

@@ -0,0 +1,56 @@
+package gitdiff
+
+import (
+	"fmt"
+)
+
+const (
+	base85Alphabet = "0123456789" +
+		"ABCDEFGHIJKLMNOPQRSTUVWXYZ" +
+		"abcdefghijklmnopqrstuvwxyz" +
+		"!#$%&()*+-;<=>?@^_`{|}~"
+)
+
+var (
+	de85 map[byte]byte
+)
+
+func init() {
+	de85 = make(map[byte]byte)
+	for i, c := range base85Alphabet {
+		de85[byte(c)] = byte(i)
+	}
+}
+
+// base85Decode decodes Base85-encoded data from src into dst. It uses the
+// alphabet defined by base85.c in the Git source tree, which appears to be
+// unique. src must contain at least len(dst) bytes of encoded data.
+func base85Decode(dst, src []byte) error {
+	var v uint32
+	var n, ndst int
+	for i, b := range src {
+		if b, ok := de85[b]; ok {
+			v = 85*v + uint32(b)
+			n++
+		} else {
+			return fmt.Errorf("invalid base85 byte at index %d: 0x%x", i, b)
+		}
+		if n == 5 {
+			rem := len(dst) - ndst
+			for j := 0; j < 4 && j < rem; j++ {
+				dst[ndst] = byte(v >> 24)
+				ndst++
+				v <<= 8
+			}
+			v = 0
+			n = 0
+		}
+	}
+	if n > 0 {
+		return fmt.Errorf("base85 data terminated by underpadded sequence")
+	}
+	if ndst < len(dst) {
+		return fmt.Errorf("base85 data is too short: %d < %d", ndst, len(dst))
+	}
+	return nil
+}

gitdiff/parser.go

@@ -3,8 +3,7 @@ package gitdiff
 import (
 	"bufio"
 	"bytes"
-	"compress/flate"
-	"encoding/ascii85"
+	"compress/zlib"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -423,35 +422,30 @@ func (p *parser) ParseBinaryChunk(frag *BinaryFragment) error {
 		if line == "\n" {
 			break
 		}
-
 		if len(line) < len(shortestValidLine) || (len(line)-2)%5 != 0 {
 			return p.Errorf(0, "binary patch: corrupt data line")
 		}
 
-		byteCount := int(line[0])
+		byteCount, seq := int(line[0]), line[1:len(line)-1]
 		switch {
 		case 'A' <= byteCount && byteCount <= 'Z':
 			byteCount = byteCount - 'A' + 1
 		case 'a' <= byteCount && byteCount <= 'z':
 			byteCount = byteCount - 'a' + 27
 		default:
-			return p.Errorf(0, "binary patch: invalid length byte: %q", line[0])
+			return p.Errorf(0, "binary patch: invalid length byte")
 		}
 
 		// base85 encodes every 4 bytes into 5 characters, with up to 3 bytes of end padding
-		maxByteCount := (len(line) - 2) / 5 * 4
-		if byteCount >= maxByteCount || byteCount < maxByteCount-3 {
-			return p.Errorf(0, "binary patch: incorrect byte count: %d", byteCount)
+		maxByteCount := len(seq) / 5 * 4
+		if byteCount > maxByteCount || byteCount < maxByteCount-3 {
+			return p.Errorf(0, "binary patch: incorrect byte count")
 		}
 
-		ndst, _, err := ascii85.Decode(buf, []byte(line[1:]), byteCount < maxBytesPerLine)
-		if err != nil {
+		if err := base85Decode(buf[:byteCount], []byte(seq)); err != nil {
 			return p.Errorf(0, "binary patch: %v", err)
 		}
-		if ndst != byteCount {
-			return p.Errorf(0, "binary patch: %d byte line decoded as %d", byteCount, ndst)
-		}
-		data.Write(buf[:ndst])
+		data.Write(buf[:byteCount])
 
 		if err := p.Next(); err != nil {
 			if err == io.EOF {
@@ -472,18 +466,20 @@ func (p *parser) ParseBinaryChunk(frag *BinaryFragment) error {
 	return nil
 }
 
-func inflateBinaryChunk(frag *BinaryFragment, r io.Reader) (err error) {
-	inflater := flate.NewReader(r)
-	defer func() {
-		if cerr := inflater.Close(); cerr != nil && err == nil {
-			err = cerr
-		}
-	}()
+func inflateBinaryChunk(frag *BinaryFragment, r io.Reader) error {
+	zr, err := zlib.NewReader(r)
+	if err != nil {
+		return err
+	}
 
-	data, err := ioutil.ReadAll(inflater)
+	data, err := ioutil.ReadAll(zr)
 	if err != nil {
 		return err
 	}
+	if err := zr.Close(); err != nil {
+		return err
+	}
+
 	if int64(len(data)) != frag.Size {
 		return fmt.Errorf("%d byte fragment inflated to %d", frag.Size, len(data))
 	}

gitdiff/parser_binary_test.go

@@ -1,6 +1,7 @@
 package gitdiff
 
 import (
+	"encoding/binary"
 	"io"
 	"reflect"
 	"testing"
@@ -108,3 +109,63 @@ func TestParseBinaryFragmentHeader(t *testing.T) {
 		})
 	}
 }
+
+func TestParseBinaryChunk(t *testing.T) {
+	tests := map[string]struct {
+		Input    string
+		Fragment BinaryFragment
+		Output   []byte
+		Err      bool
+	}{
+		"newFile": {
+			Input: "gcmZQzU|?i`U?w2V48*KJ%mKu_Kr9NxN<eH500b)lkN^Mx\n\n",
+			Fragment: BinaryFragment{
+				Size: 40,
+			},
+			Output: fib(10),
+		},
+		"newFileMultiline": {
+			Input: "zcmZQzU|?i`U?w2V48*KJ%mKu_Kr9NxN<eH5#F0Qe0f=7$l~*z_FeL$%-)3N7vt?l5\n" +
+				"zl3-vE2xVZ9%4J~CI>f->s?WfX|B-=Vs{#X~svra7Ekg#T|4s}nH;WnAZ)|1Y*`&cB\n" +
+				"s(sh?X(Uz6L^!Ou&aF*u`J!eibJifSrv0z>$Q%Hd(^HIJ<Y?5`S0gT5UE&u=k\n\n",
+			Fragment: BinaryFragment{
+				Size: 160,
+			},
+			Output: fib(40),
+		},
+	}
+
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			p := newTestParser(test.Input, true)
+
+			frag := test.Fragment
+			err := p.ParseBinaryChunk(&frag)
+			if test.Err {
+				if err == nil || err == io.EOF {
+					t.Fatalf("expected error parsing binary chunk, but got %v", err)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("unexpected error parsing binary chunk: %v", err)
+			}
+			if !reflect.DeepEqual(test.Output, frag.Data) {
+				t.Errorf("incorrect binary chunk\nexpected: %+v\n  actual: %+v", test.Output, frag.Data)
+			}
+		})
+	}
+}
+
+func fib(n int) []byte {
+	seq := []uint32{1, 1}
+	for i := 2; i < n; i++ {
+		seq = append(seq, seq[i-1]+seq[i-2])
+	}
+
+	buf := make([]byte, 4*n)
+	for i, v := range seq[:n] {
+		binary.BigEndian.PutUint32(buf[i*4:], v)
+	}
+	return buf
+}