Adding new flatfile.csv decls (#173)

* Adding new flatfile.csv decls.
* Adding a few design/implementation notes; adding csv2 json schema validation

Author: jf-tech

extensions/omniv21/fileformat/flatfile/csv/decl.go

@@ -0,0 +1,139 @@
+package csv
+
+import (
+	"fmt"
+	"regexp"
+
+	"github.com/jf-tech/go-corelib/maths"
+
+	"github.com/jf-tech/omniparser/extensions/omniv21/fileformat/flatfile"
+)
+
+// ColumnDecl describes a column of an csv record column.
+type ColumnDecl struct {
+	Name string `json:"name,omitempty"`
+}
+
+// Design note: given currently ColumnDecl contains only Name field, we could've simply
+// change RecordDecl.Columns into a []string. But in the future, if we ever need to add
+// anything to a column decl, we'd have to introduce a breaking schema change.
+
+const (
+	typeRecord = "record"
+	typeGroup  = "record_group"
+)
+
+// RecordDecl describes an record of a csv/delimited input.
+// If Rows/Header/Footer are all nil, then it defaults to Rows = 1.
+// If Rows specified, then Header/Footer must be nil. (JSON schema validation will ensure this.)
+// If Header is specified, Rows must be nil. (JSON schema validation will ensure this.)
+// Footer is optional; If not specified, Header will be used for a single-line record matching.
+type RecordDecl struct {
+	Name     string        `json:"name,omitempty"`
+	Rows     *int          `json:"rows,omitempty"`
+	Header   *string       `json:"header,omitempty"`
+	Footer   *string       `json:"footer,omitempty"`
+	Type     *string       `json:"type,omitempty"`
+	IsTarget bool          `json:"is_target,omitempty"`
+	Min      *int          `json:"min,omitempty"`
+	Max      *int          `json:"max,omitempty"`
+	Columns  []*ColumnDecl `json:"columns,omitempty"`
+	Children []*RecordDecl `json:"child_records,omitempty"`
+
+	fqdn          string // fully hierarchical name to the record.
+	childRecDecls []flatfile.RecDecl
+	headerRegexp  *regexp.Regexp
+	footerRegexp  *regexp.Regexp
+}
+
+func (r *RecordDecl) DeclName() string {
+	return r.Name
+}
+
+func (r *RecordDecl) Target() bool {
+	return r.IsTarget
+}
+
+func (r *RecordDecl) Group() bool {
+	return r.Type != nil && *r.Type == typeGroup
+}
+
+// MinOccurs defaults to 0. CSV/delimited input most common scenario is min=0/max=unbounded.
+func (r *RecordDecl) MinOccurs() int {
+	switch r.Min {
+	case nil:
+		return 0
+	default:
+		return *r.Min
+	}
+}
+
+// MaxOccurs defaults to unbounded. CSV/delimited input most common scenario is min=0/max=unbounded.
+func (r *RecordDecl) MaxOccurs() int {
+	switch {
+	case r.Max == nil:
+		fallthrough
+	case *r.Max < 0:
+		return maths.MaxIntValue
+	default:
+		return *r.Max
+	}
+}
+
+func (r *RecordDecl) ChildDecls() []flatfile.RecDecl {
+	return r.childRecDecls
+}
+
+func (r *RecordDecl) rowsBased() bool {
+	if r.Group() {
+		panic("record_group is neither rows based nor header/footer based")
+	}
+	// for header/footer based record, header must be specified; otherwise, it's rows based.
+	return r.Header == nil
+}
+
+// rows() defaults to 1. csv/delimited most common scenario is rows-based single line record.
+func (r *RecordDecl) rows() int {
+	if !r.rowsBased() {
+		panic(fmt.Sprintf("record '%s' is not rows based", r.fqdn))
+	}
+	if r.Rows == nil {
+		return 1
+	}
+	return *r.Rows
+}
+
+func (r *RecordDecl) matchHeader(line []byte) bool {
+	if r.headerRegexp == nil {
+		panic(fmt.Sprintf("record '%s' is not header/footer based", r.fqdn))
+	}
+	return r.headerRegexp.Match(line)
+}
+
+// Footer is optional. If not specified, it always matches. Thus for a header/footer record,
+// if the footer isn't specified, it effectively becomes a single-row record matched by header,
+// given that after the header matches a line, matchFooter is called on the same line.
+func (r *RecordDecl) matchFooter(line []byte) bool {
+	if r.footerRegexp == nil {
+		return true
+	}
+	return r.footerRegexp.Match(line)
+}
+
+func toFlatFileRecDecls(rs []*RecordDecl) []flatfile.RecDecl {
+	if len(rs) == 0 {
+		return nil
+	}
+	ret := make([]flatfile.RecDecl, len(rs))
+	for i, r := range rs {
+		ret[i] = r
+	}
+	return ret
+}
+
+// FileDecl describes csv/delimited schema `file_declaration` setting.
+type FileDecl struct {
+	Delimiter           string        `json:"delimiter,omitempty"`
+	ReplaceDoubleQuotes bool          `json:"replace_double_quotes,omitempty"`
+	Records             []*RecordDecl `json:"records,omitempty"`
+}

extensions/omniv21/fileformat/flatfile/csv/decl_test.go

@@ -0,0 +1,89 @@
+package csv
+
+import (
+	"regexp"
+	"testing"
+
+	"github.com/jf-tech/go-corelib/maths"
+	"github.com/jf-tech/go-corelib/strs"
+	"github.com/jf-tech/go-corelib/testlib"
+	"github.com/jf-tech/omniparser/extensions/omniv21/fileformat/flatfile"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRecordDecl(t *testing.T) {
+	// DeclName()
+	r := &RecordDecl{Name: "r1"}
+	assert.Equal(t, "r1", r.DeclName())
+	r.fqdn = r.DeclName()
+
+	// Target()
+	assert.False(t, r.Target())
+	r.IsTarget = true
+	assert.True(t, r.Target())
+
+	// Group()
+	assert.False(t, r.Group())
+	r.Type = strs.StrPtr(typeRecord)
+	assert.False(t, r.Group())
+	r.Type = strs.StrPtr(typeGroup)
+	assert.True(t, r.Group())
+
+	// MinOccurs()
+	assert.Equal(t, 0, r.MinOccurs())
+	r.Min = testlib.IntPtr(42)
+	assert.Equal(t, 42, r.MinOccurs())
+
+	// MaxOccurs()
+	assert.Equal(t, maths.MaxIntValue, r.MaxOccurs())
+	r.Max = testlib.IntPtr(-1)
+	assert.Equal(t, maths.MaxIntValue, r.MaxOccurs())
+	r.Max = testlib.IntPtr(42)
+	assert.Equal(t, 42, r.MaxOccurs())
+
+	// ChildDecls()
+	assert.Nil(t, r.ChildDecls())
+	r.childRecDecls = []flatfile.RecDecl{}
+	assert.Equal(t, r.childRecDecls, r.ChildDecls())
+
+	// rowsBased()
+	assert.PanicsWithValue(t, "record_group is neither rows based nor header/footer based",
+		func() { r.rowsBased() })
+	r.Type = strs.StrPtr(typeRecord)
+	assert.True(t, r.rowsBased())
+	r.Header = strs.StrPtr("^ABC$")
+	assert.False(t, r.rowsBased())
+
+	// rows()
+	assert.PanicsWithValue(t, "record 'r1' is not rows based", func() { r.rows() })
+	r.Header = nil
+	assert.Equal(t, 1, r.rows())
+	r.Rows = testlib.IntPtr(42)
+	assert.Equal(t, 42, r.rows())
+
+	// matchHeader()
+	assert.PanicsWithValue(
+		t, "record 'r1' is not header/footer based", func() { r.matchHeader(nil) })
+	r.headerRegexp = regexp.MustCompile("^ABC$")
+	assert.False(t, r.matchHeader([]byte("ABCD")))
+	assert.True(t, r.matchHeader([]byte("ABC")))
+
+	// matchFooter()
+	assert.True(t, r.matchFooter([]byte("ABCD")))
+	r.footerRegexp = regexp.MustCompile("^ABC$")
+	assert.False(t, r.matchFooter([]byte("ABCD")))
+	assert.True(t, r.matchFooter([]byte("ABC")))
+}
+
+func TestToFlatFileRecDecls(t *testing.T) {
+	assert.Nil(t, toFlatFileRecDecls(nil))
+	assert.Nil(t, toFlatFileRecDecls([]*RecordDecl{}))
+	rs := []*RecordDecl{
+		{},
+		{},
+	}
+	ds := toFlatFileRecDecls(rs)
+	for i := range ds {
+		assert.Same(t, rs[i], ds[i].(*RecordDecl))
+	}
+}

extensions/omniv21/fileformat/flatfile/fixedlength/decl.go

@@ -76,7 +76,7 @@ type EnvelopeDecl struct {
 	Columns  []*ColumnDecl   `json:"columns,omitempty"`
 	Children []*EnvelopeDecl `json:"child_envelopes,omitempty"`
 
-	fqdn          string // fullly hierarchical name to the envelope.
+	fqdn          string // fully hierarchical name to the envelope.
 	childRecDecls []flatfile.RecDecl
 	headerRegexp  *regexp.Regexp
 	footerRegexp  *regexp.Regexp

extensions/omniv21/fileformat/flatfile/recdecl.go

@@ -12,6 +12,19 @@ type RecDecl interface {
 	ChildDecls() []RecDecl
 }
 
+// Design note: flatfile.fixedlength, flatfile.csv, etc all have similar structs that contain name,
+// is_target, type, min, max, etc and implementation of RecDecl interface all very similar. So why
+// not just change RecDecl into a struct and embed that struct into format specific decl structs?
+// We chose not to do that and rather take a small hit of code duplication in preference to
+// flexibility. Note RecDecl's ChildDecls need to return child decls and each format specific decl
+// is different, so that's the first incompatibility here that ChildDecls cannot be even be included
+// in the struct if we go down the RecDecl being struct route. Yes, generics can do that but we
+// don't want to move omniparser 1.14 dependency up all the way to 1.18 simply because of this.
+// Second, depending on formats, the default values for min/max are different: csv/fixed-length
+// min/max default to 0/-1, but for EDI min/max default to 1/1. Given these incompatibility and
+// loss of flexibility, we chose to stick with the RecDecl interface route and have each format
+// somewhat duplicate a small amount of trivial code.
+
 const (
 	rootName = "#root"
 )