add reader.readByHeaderFooterEnvelope (#116)

Author: jf-tech

extensions/omniv21/fileformat/fixedlength/reader.go

@@ -6,15 +6,18 @@ import (
 	"io"
 
 	"github.com/antchfx/xpath"
+	"github.com/jf-tech/go-corelib/caches"
 	"github.com/jf-tech/go-corelib/ios"
 
 	"github.com/jf-tech/omniparser/idr"
 )
 
+// ErrInvalidEnvelope indicates a fixed-length input envelope is invalid. This is a fatal, non-continuable error.
 type ErrInvalidEnvelope string
 
 func (e ErrInvalidEnvelope) Error() string { return string(e) }
 
+// IsErrInvalidEnvelope checks if an error is of ErrInvalidEnvelope type.
 func IsErrInvalidEnvelope(err error) bool {
 	switch err.(type) {
 	case ErrInvalidEnvelope:
@@ -42,10 +45,7 @@ func (r *reader) readLine() ([]byte, error) {
 		switch err {
 		case nil:
 			r.line++
-		case io.EOF:
-			return nil, err
 		default:
-			r.line++
 			return nil, err
 		}
 		// skip only truly empty lines.
@@ -87,6 +87,54 @@ func (r *reader) readByRowsEnvelope() (*idr.Node, error) {
 	return node, nil
 }
 
+func (r *reader) readByHeaderFooterEnvelope() (*idr.Node, error) {
+	line, err := r.readLine()
+	if err != nil {
+		if err == io.EOF {
+			return nil, err
+		}
+		return nil, ErrInvalidEnvelope(r.fmtErrStr("incomplete envelope: %s", err.Error()))
+	}
+	for ; r.envelopeIndex < len(r.decl.Envelopes); r.envelopeIndex++ {
+		// regex's are already validated
+		headerRegex, _ := caches.GetRegex(r.decl.Envelopes[r.envelopeIndex].ByHeaderFooter.Header)
+		if headerRegex.Match(line) {
+			break
+		}
+	}
+	if r.envelopeIndex >= len(r.decl.Envelopes) {
+		return nil, io.EOF
+	}
+	envelopeDecl := r.decl.Envelopes[r.envelopeIndex]
+	footerRegex, _ := caches.GetRegex(envelopeDecl.ByHeaderFooter.Footer)
+	node := idr.CreateNode(idr.ElementNode, *envelopeDecl.Name)
+	columnsDone := make([]bool, len(envelopeDecl.Columns))
+	for {
+		for col := range envelopeDecl.Columns {
+			if columnsDone[col] {
+				continue
+			}
+			colDecl := envelopeDecl.Columns[col]
+			if !colDecl.lineMatch(line) {
+				continue
+			}
+			colNode := idr.CreateNode(idr.ElementNode, colDecl.Name)
+			idr.AddChild(node, colNode)
+			colVal := idr.CreateNode(idr.TextNode, colDecl.lineToColumnValue(line))
+			idr.AddChild(colNode, colVal)
+			columnsDone[col] = true
+		}
+		if footerRegex.Match(line) {
+			return node, nil
+		}
+		line, err = r.readLine()
+		// Since the envelope has started, any reading error, including EOF, indicates incomplete envelope error.
+		if err != nil {
+			return nil, ErrInvalidEnvelope(r.fmtErrStr("incomplete envelope: %s", err.Error()))
+		}
+	}
+}
+
 func (r *reader) fmtErrStr(format string, args ...interface{}) string {
 	return fmt.Sprintf("input '%s' line %d: %s", r.inputName, r.line, fmt.Sprintf(format, args...))
 }

extensions/omniv21/fileformat/fixedlength/reader_test.go

@@ -72,8 +72,7 @@ func TestReadLine(t *testing.T) {
 	assert.Error(t, err)
 	assert.Equal(t, "read error", err.Error())
 	assert.Nil(t, line)
-	// reading error (unless it's EOF) bumps current line number
-	assert.Equal(t, 2, r.line)
+	assert.Equal(t, 1, r.line)
 }
 
 func TestReadByRowsEnvelope_ByRowsDefault(t *testing.T) {
@@ -133,6 +132,7 @@ func TestReadByRowsEnvelope_ByRowsNonDefault(t *testing.T) {
 
 	n, err = r.readByRowsEnvelope()
 	assert.Error(t, err)
+	assert.True(t, IsErrInvalidEnvelope(err))
 	assert.Equal(t, "input 'test' line 6: incomplete envelope, missing 2 row(s)", err.Error())
 	assert.Nil(t, n)
 }
@@ -171,3 +171,156 @@ func BenchmarkReadByRowsEnvelope(b *testing.B) {
 		}
 	}
 }
+
+func TestReadByHeaderFooterEnvelope_EOFBeforeStart(t *testing.T) {
+	r := testReader(strings.NewReader(""), &fileDecl{Envelopes: []*envelopeDecl{{Name: strs.StrPtr("env1")}}})
+	n, err := r.readByHeaderFooterEnvelope()
+	assert.Equal(t, io.EOF, err)
+	assert.Nil(t, n)
+}
+
+func TestReadByHeaderFooterEnvelope_ReadErrorBeforeStart(t *testing.T) {
+	r := testReader(
+		testlib.NewMockReadCloser("read error", nil),
+		&fileDecl{Envelopes: []*envelopeDecl{{Name: strs.StrPtr("env1")}}})
+	n, err := r.readByHeaderFooterEnvelope()
+	assert.Error(t, err)
+	assert.True(t, IsErrInvalidEnvelope(err))
+	assert.Equal(t, `input 'test' line 1: incomplete envelope: read error`, err.Error())
+	assert.Nil(t, n)
+}
+
+func TestReadByHeaderFooterEnvelope_NoEnvelopeMatch(t *testing.T) {
+	r := testReader(
+		strings.NewReader("efg"),
+		&fileDecl{Envelopes: []*envelopeDecl{{
+			Name:           strs.StrPtr("env1"),
+			ByHeaderFooter: &byHeaderFooterDecl{Header: "abc", Footer: "abc"},
+		}}})
+	n, err := r.readByHeaderFooterEnvelope()
+	assert.Equal(t, io.EOF, err)
+	assert.Nil(t, n)
+}
+
+func TestReadByHeaderFooterEnvelope_IncompleteEnvelope(t *testing.T) {
+	r := testReader(
+		strings.NewReader("abc"),
+		&fileDecl{Envelopes: []*envelopeDecl{{
+			Name:           strs.StrPtr("env1"),
+			ByHeaderFooter: &byHeaderFooterDecl{Header: "abc", Footer: "efg"},
+		}}})
+	n, err := r.readByHeaderFooterEnvelope()
+	assert.Error(t, err)
+	assert.True(t, IsErrInvalidEnvelope(err))
+	assert.Equal(t, `input 'test' line 2: incomplete envelope: EOF`, err.Error())
+	assert.Nil(t, n)
+}
+
+func lf(s string) string { return s + "\n" }
+
+func TestReadByHeaderFooterEnvelope_Success1(t *testing.T) {
+	r := testReader(
+		strings.NewReader(
+			lf("begin")+
+				lf("header-01")+
+				lf("a001-abc")+
+				lf("a002-def")+
+				lf("a003-ghi")+
+				lf("footer")+
+				lf("header-02")+
+				lf("a001-012")+
+				lf("a002-345")+
+				lf("a003-678")+
+				lf("footer")+
+				lf("end")),
+		&fileDecl{Envelopes: []*envelopeDecl{
+			{
+				Name:           strs.StrPtr("begin"),
+				ByHeaderFooter: &byHeaderFooterDecl{Header: "^begin", Footer: "^begin"},
+			},
+			{
+				Name:           strs.StrPtr("data"),
+				ByHeaderFooter: &byHeaderFooterDecl{Header: "^header", Footer: "^footer"},
+				Columns: []*columnDecl{
+					{Name: "data_id", StartPos: 8, Length: 2, LinePattern: strs.StrPtr("^header")},
+					{Name: "a001_first2chars", StartPos: 6, Length: 2, LinePattern: strs.StrPtr("^a001")},
+					{Name: "a003_last2chars", StartPos: 7, Length: 2, LinePattern: strs.StrPtr("^a003")},
+					{Name: "a001_last1char", StartPos: 8, Length: 1, LinePattern: strs.StrPtr("^a001")},
+				},
+			},
+			{
+				Name:           strs.StrPtr("end"),
+				ByHeaderFooter: &byHeaderFooterDecl{Header: "^end", Footer: "^end"},
+			},
+		}})
+	n, err := r.readByHeaderFooterEnvelope()
+	assert.NoError(t, err)
+	assert.Equal(t, "begin", n.Data)
+
+	n, err = r.readByHeaderFooterEnvelope()
+	assert.NoError(t, err)
+	assert.Equal(t,
+		`{"a001_first2chars":"ab","a001_last1char":"c","a003_last2chars":"hi","data_id":"01"}`, idr.JSONify2(n))
+
+	n, err = r.readByHeaderFooterEnvelope()
+	assert.NoError(t, err)
+	assert.Equal(t,
+		`{"a001_first2chars":"01","a001_last1char":"2","a003_last2chars":"78","data_id":"02"}`, idr.JSONify2(n))
+
+	n, err = r.readByHeaderFooterEnvelope()
+	assert.NoError(t, err)
+	assert.Equal(t, "end", n.Data)
+
+	n, err = r.readByHeaderFooterEnvelope()
+	assert.Equal(t, io.EOF, err)
+	assert.Nil(t, n)
+}
+
+var (
+	benchReadByHeaderFooterEnvelopeInput = lf("begin") +
+		strings.Repeat(
+			lf("header")+
+				lf("a001-abcdefghijklmnopqrstuvwxyz0123456789")+
+				lf("a002-abcdefghijklmnopqrstuvwxyz0123456789")+
+				lf("a003-abcdefghijklmnopqrstuvwxyz0123456789")+
+				lf("footer"), 1000) +
+		lf("end")
+	benchReadByHeaderFooterEnvelopeDecl = &fileDecl{
+		Envelopes: []*envelopeDecl{
+			{
+				Name:           strs.StrPtr("begin"),
+				ByHeaderFooter: &byHeaderFooterDecl{Header: "^begin", Footer: "^begin"},
+			},
+			{
+				Name:           strs.StrPtr("data"),
+				ByHeaderFooter: &byHeaderFooterDecl{Header: "^header", Footer: "^footer"},
+				Columns: []*columnDecl{
+					{Name: "a001_1", StartPos: 6, Length: 12, LinePattern: strs.StrPtr("^a001")},
+					{Name: "a003_1", StartPos: 7, Length: 9, LinePattern: strs.StrPtr("^a003")},
+					{Name: "a001_2", StartPos: 30, Length: 20, LinePattern: strs.StrPtr("^a001")},
+				},
+			},
+			{
+				Name:           strs.StrPtr("end"),
+				ByHeaderFooter: &byHeaderFooterDecl{Header: "^end", Footer: "^end"},
+			},
+		},
+	}
+)
+
+// BenchmarkReadByHeaderFooterEnvelope-8   	     310	   3819649 ns/op	  213840 B/op	   14009 allocs/op
+func BenchmarkReadByHeaderFooterEnvelope(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		r := testReader(strings.NewReader(benchReadByHeaderFooterEnvelopeInput), benchReadByHeaderFooterEnvelopeDecl)
+		for {
+			n, err := r.readByHeaderFooterEnvelope()
+			if err != nil {
+				if err == io.EOF {
+					break
+				}
+				b.FailNow()
+			}
+			idr.RemoveAndReleaseTree(n)
+		}
+	}
+}