Merge pull request #10 from ipfs/kevina/multibase4

Changes needed for `--cid-base` option in go-ipfs (simplified vesion)

Author: Stebalien

.travis.yml

@@ -0,0 +1,24 @@
+sudo: false
+
+
+language: go
+go:
+  - 'tip'
+  
+install:
+  - go get github.com/whyrusleeping/gx
+  - go get github.com/whyrusleeping/gx-go
+  - gx install --global
+script:
+  - gx test -v -race -coverprofile=coverage.txt -covermode=atomic .
+  
+after_success:
+  - bash <(curl -s https://codecov.io/bash)
+
+cache:
+  directories:
+    - $GOPATH/src/gx
+
+notifications:
+email: false
+  

ci/Jenkinsfile

@@ -0,0 +1 @@
+golang()

cid-fmt/main.go

@@ -1,7 +1,9 @@
 package main
 
 import (
+	"bufio"
 	"fmt"
+	"io"
 	"os"
 	"strings"
 
@@ -12,7 +14,9 @@ import (
 )
 
 func usage() {
-	fmt.Fprintf(os.Stderr, "usage: %s [-b multibase-code] [-v cid-version] <fmt-str> <cid> ...\n\n", os.Args[0])
+	fmt.Fprintf(os.Stderr, "usage: %s [-b multibase-code] [-v cid-version] [--filter] <fmt-str> <cid> ...\n", os.Args[0])
+	fmt.Fprintf(os.Stderr, "--filter will read from stdin and convert anything that looks like a <cid>\n")
+	fmt.Fprintf(os.Stderr, "         -- including any non-cids that are valid Multihashes).\n")
 	fmt.Fprintf(os.Stderr, "<fmt-str> is either 'prefix' or a printf style format string:\n%s", cidutil.FormatRef)
 	os.Exit(2)
 }
@@ -24,8 +28,9 @@ func main() {
 	newBase := mb.Encoding(-1)
 	var verConv func(cid c.Cid) (c.Cid, error)
 	args := os.Args[1:]
+	filter := false
 outer:
-	for {
+	for len(args) > 0 {
 		switch args[0] {
 		case "-b":
 			if len(args) < 2 {
@@ -52,11 +57,14 @@ outer:
 				os.Exit(2)
 			}
 			args = args[2:]
+		case "--filter":
+			filter = true
+			args = args[1:]
 		default:
 			break outer
 		}
 	}
-	if len(args) < 2 {
+	if len(args) < 1 {
 		usage()
 	}
 	fmtStr := args[0]
@@ -69,41 +77,73 @@ outer:
 			os.Exit(2)
 		}
 	}
-	for _, cidStr := range args[1:] {
-		cid, err := c.Decode(cidStr)
-		if err != nil {
-			fmt.Fprintf(os.Stdout, "!INVALID_CID!\n")
-			errorMsg("%s: %v", cidStr, err)
-			// Don't abort on a bad cid
-			continue
-		}
+	format := func(cid c.Cid, cidStr string) (string, error) {
 		base := newBase
-		if newBase == -1 {
+		if base == -1 {
 			base, _ = c.ExtractEncoding(cidStr)
 		}
+		var err error
 		if verConv != nil {
 			cid, err = verConv(cid)
 			if err != nil {
-				fmt.Fprintf(os.Stdout, "!ERROR!\n")
-				errorMsg("%s: %v", cidStr, err)
-				// Don't abort on a bad conversion
-				continue
+				return "", err
+			}
+		}
+		return cidutil.Format(fmtStr, base, cid)
+	}
+	if filter {
+		scanner := bufio.NewScanner(os.Stdin)
+		for scanner.Scan() {
+			buf := scanner.Bytes()
+			for {
+				i, j, cid, cidStr := cidutil.ScanForCid(buf)
+				os.Stdout.Write(buf[0:i])
+				if i == len(buf) {
+					os.Stdout.Write([]byte{'\n'})
+					break
+				}
+				str, err := format(cid, cidStr)
+				switch err.(type) {
+				case cidutil.FormatStringError:
+					fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+					os.Exit(2)
+				default:
+					// just use the orignal sting on non-fatal error
+					str = cidStr
+				case nil:
+				}
+				io.WriteString(os.Stdout, str)
+				buf = buf[j:]
 			}
 		}
-		str, err := cidutil.Format(fmtStr, base, cid)
-		switch err.(type) {
-		case cidutil.FormatStringError:
+		if err := scanner.Err(); err != nil {
 			fmt.Fprintf(os.Stderr, "Error: %v\n", err)
 			os.Exit(2)
-		default:
-			fmt.Fprintf(os.Stdout, "!ERROR!\n")
-			errorMsg("%s: %v", cidStr, err)
-			// Don't abort on cid specific errors
-			continue
-		case nil:
-			// no error
 		}
-		fmt.Fprintf(os.Stdout, "%s\n", str)
+	} else {
+		for _, cidStr := range args[1:] {
+			cid, err := c.Decode(cidStr)
+			if err != nil {
+				fmt.Fprintf(os.Stdout, "!INVALID_CID!\n")
+				errorMsg("%s: %v", cidStr, err)
+				// Don't abort on a bad cid
+				continue
+			}
+			str, err := format(cid, cidStr)
+			switch err.(type) {
+			case cidutil.FormatStringError:
+				fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+				os.Exit(2)
+			default:
+				fmt.Fprintf(os.Stdout, "!ERROR!\n")
+				errorMsg("%s: %v", cidStr, err)
+				// Don't abort on cid specific errors
+				continue
+			case nil:
+				// no error
+			}
+			fmt.Fprintf(os.Stdout, "%s\n", str)
+		}
 	}
 	os.Exit(exitCode)
 }

cidenc/encoder.go

@@ -0,0 +1,59 @@
+package cidenc
+
+import (
+	cid "github.com/ipfs/go-cid"
+	mbase "github.com/multiformats/go-multibase"
+)
+
+// Encoder is a basic Encoder that will encode CIDs using a specified
+// base and optionally upgrade a CIDv0 to CIDv1
+type Encoder struct {
+	Base    mbase.Encoder // The multibase to use
+	Upgrade bool          // If true upgrade CIDv0 to CIDv1 when encoding
+}
+
+// Default return a new default encoder
+func Default() Encoder {
+	return Encoder{Base: mbase.MustNewEncoder(mbase.Base58BTC)}
+}
+
+// Encode encodes the cid using the parameters of the Encoder
+func (enc Encoder) Encode(c cid.Cid) string {
+	if enc.Upgrade && c.Version() == 0 {
+		c = cid.NewCidV1(c.Type(), c.Hash())
+	}
+	return c.Encode(enc.Base)
+}
+
+// Recode reencodes the cid string to match the parameters of the
+// encoder
+func (enc Encoder) Recode(v string) (string, error) {
+	skip, err := enc.noopRecode(v)
+	if skip || err != nil {
+		return v, err
+	}
+
+	c, err := cid.Decode(v)
+	if err != nil {
+		return v, err
+	}
+
+	return enc.Encode(c), nil
+}
+
+func (enc Encoder) noopRecode(v string) (bool, error) {
+	if len(v) < 2 {
+		return false, cid.ErrCidTooShort
+	}
+	ver := cidVer(v)
+	skip := ver == 0 && !enc.Upgrade || ver == 1 && v[0] == byte(enc.Base.Encoding())
+	return skip, nil
+}
+
+func cidVer(v string) int {
+	if len(v) == 46 && v[:2] == "Qm" {
+		return 0
+	} else {
+		return 1
+	}
+}

cidenc/encoder_test.go

@@ -0,0 +1,62 @@
+package cidenc
+
+import (
+	"testing"
+
+	cid "github.com/ipfs/go-cid"
+	mbase "github.com/multiformats/go-multibase"
+)
+
+func TestCidEncoder(t *testing.T) {
+	cidv0str := "QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n"
+	cidv1str := "zdj7Wkkhxcu2rsiN6GUyHCLsSLL47kdUNfjbFqBUUhMFTZKBi"
+	cidb32str := "bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku"
+	cidv0, _ := cid.Decode(cidv0str)
+	cidv1, _ := cid.Decode(cidv1str)
+
+	testEncode := func(enc Encoder, cid cid.Cid, expect string) {
+		actual := enc.Encode(cid)
+		if actual != expect {
+			t.Errorf("%+v.Encode(%s): expected %s but got %s", enc, cid, expect, actual)
+		}
+	}
+
+	testRecode := func(enc Encoder, cid string, expect string) {
+		actual, err := enc.Recode(cid)
+		if err != nil {
+			t.Errorf("%+v.Recode(%s): %s", enc, cid, err)
+			return
+		}
+		if actual != expect {
+			t.Errorf("%+v.Recode(%s): expected %s but got %s", enc, cid, expect, actual)
+		}
+	}
+
+	enc := Encoder{Base: mbase.MustNewEncoder(mbase.Base58BTC), Upgrade: false}
+	testEncode(enc, cidv0, cidv0str)
+	testEncode(enc, cidv1, cidv1str)
+	testRecode(enc, cidv0str, cidv0str)
+	testRecode(enc, cidv1str, cidv1str)
+	testRecode(enc, cidb32str, cidv1str)
+
+	enc = Encoder{Base: mbase.MustNewEncoder(mbase.Base58BTC), Upgrade: true}
+	testEncode(enc, cidv0, cidv1str)
+	testEncode(enc, cidv1, cidv1str)
+	testRecode(enc, cidv0str, cidv1str)
+	testRecode(enc, cidv1str, cidv1str)
+	testRecode(enc, cidb32str, cidv1str)
+
+	enc = Encoder{Base: mbase.MustNewEncoder(mbase.Base32), Upgrade: false}
+	testEncode(enc, cidv0, cidv0str)
+	testEncode(enc, cidv1, cidb32str)
+	testRecode(enc, cidv0str, cidv0str)
+	testRecode(enc, cidv1str, cidb32str)
+	testRecode(enc, cidb32str, cidb32str)
+
+	enc = Encoder{Base: mbase.MustNewEncoder(mbase.Base32), Upgrade: true}
+	testEncode(enc, cidv0, cidb32str)
+	testEncode(enc, cidv1, cidb32str)
+	testRecode(enc, cidv0str, cidb32str)
+	testRecode(enc, cidv1str, cidb32str)
+	testRecode(enc, cidb32str, cidb32str)
+}

format.go

@@ -150,3 +150,48 @@ func encode(base mb.Encoder, data []byte, strip bool) string {
 	}
 	return str
 }
+
+// ScanForCid scans bytes for anything resembling a CID. If one is
+// found `i` will point to the begging of the cid and `j` to to the
+// end and the cid will be returned, otherwise `i` and `j` will point
+// the end of the buffer and the cid will be `Undef`.
+func ScanForCid(buf []byte) (i, j int, cid c.Cid, cidStr string) {
+	i = 0
+	for {
+		i = j
+		for i < len(buf) && !asciiIsAlpha(buf[i]) {
+			i++
+		}
+		j = i
+		if i == len(buf) {
+			return
+		}
+		for j < len(buf) && asciiIsAlpha(buf[j]) {
+			j++
+		}
+		if j-i <= 1 || j-i > 128 || !supported[buf[i]] {
+			continue
+		}
+		var err error
+		cidStr = string(buf[i:j])
+		cid, err = c.Decode(cidStr)
+		if err == nil {
+			return
+		}
+	}
+}
+
+var supported = make([]bool, 256)
+
+func init() {
+	// for now base64 encoding are not supported as they contain non
+	// alhphanumeric characters
+	supportedPrefixes := []byte("QfFbBcCvVtThzZ")
+	for _, b := range supportedPrefixes {
+		supported[b] = true
+	}
+}
+
+func asciiIsAlpha(b byte) bool {
+	return ('A' <= b && b <= 'Z') || ('a' <= b && b <= 'z') || ('0' <= b && b <= '9')
+}

format_test.go

@@ -72,3 +72,35 @@ func testFmt(t *testing.T, cidStr string, newBase mb.Encoding, fmtStr string, re
 		t.Error(fmt.Sprintf("expected: %s; but got: %s", result, str))
 	}
 }
+
+func TestScanForCid(t *testing.T) {
+	testStr := []byte(`
+/ipfs/QmYFbmndVP7QqAVWyKhpmMuQHMaD88pkK57RgYVimmoh5H 22 45
+/ipfs/zdj7WfLr9DhLrb1hsoSi4fSdjjxuZmeqgEtBPWxMLtPbDNbFD/foobar
+BAFYBEIETJGSRL3EQPQPCABV3G6IUBYTSIFVQ24XRRHD3JUETSKLTGQ7DJA
+skip me (too long): QmYFbmndVP7QqAVWyKhpmMuQHMaD88pkK57RgYVimmoh5H876
+skip me (too short): bafybeietjgsrl3eqpqpcabv3g6iubytsifvq
+bafybeietjgsrl3eqpqpcabv3g6iubytsifvq24xrrhd3juetskltgq7dja
+bafybeietjgsrl3eqpqpcabv3g6iubytsifvq24xrrhd3juetskltgq7dja.
+`)
+	cids := []string{
+		"QmYFbmndVP7QqAVWyKhpmMuQHMaD88pkK57RgYVimmoh5H",
+		"zdj7WfLr9DhLrb1hsoSi4fSdjjxuZmeqgEtBPWxMLtPbDNbFD",
+		"BAFYBEIETJGSRL3EQPQPCABV3G6IUBYTSIFVQ24XRRHD3JUETSKLTGQ7DJA",
+		"bafybeietjgsrl3eqpqpcabv3g6iubytsifvq24xrrhd3juetskltgq7dja",
+		"bafybeietjgsrl3eqpqpcabv3g6iubytsifvq24xrrhd3juetskltgq7dja",
+	}
+
+	buf := testStr
+	idx := 0
+	offset := 0
+	for len(buf) > 0 {
+		_, j, _, cidStr := ScanForCid(buf)
+		if cidStr != "" && cids[idx]  != cidStr {
+			t.Fatalf("Scan failed, expected %s, got %s (idx=%d offset=%d)", cids[idx], cidStr, idx, offset)
+		}
+		buf = buf[j:]
+		offset += j
+		idx++
+	}
+}