Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
114 changes: 109 additions & 5 deletions pkg/sql/colexec/external/external.go
Original file line number Diff line number Diff line change
Expand Up @@ -891,9 +891,24 @@ func getNullFlag(nullMap map[string][]string, attr, field string) bool {
func shouldLoadEmptyNumericAsZero(param *ExternalParam, id types.T) bool {
if param == nil || param.Extern == nil ||
param.Extern.ExternType != int32(plan.ExternType_LOAD) ||
(!param.ParallelLoad && !param.LoadEmptyNumericAsZero) {
(!param.ParallelLoad && !param.LoadEmptyNumericAsZero && !shouldApplyLoadDataNonStrictAdjustments(param)) {
return false
}
if !param.ParallelLoad && !param.LoadEmptyNumericAsZero {
return isLoadNumericAdjustedValueType(id)
}
return isLoadNumericZeroFillType(id)
}

func shouldApplyLoadDataNonStrictAdjustments(param *ExternalParam) bool {
return param != nil && param.Extern != nil &&
param.Extern.ExternType == int32(plan.ExternType_LOAD) &&
param.Extern.Local &&
param.Extern.Format == tree.CSV &&
!param.StrictSqlMode
}

func isLoadNumericZeroFillType(id types.T) bool {
switch id {
case types.T_bool,
types.T_int8, types.T_int16, types.T_int32, types.T_int64,
Expand All @@ -906,6 +921,69 @@ func shouldLoadEmptyNumericAsZero(param *ExternalParam, id types.T) bool {
}
}

func isLoadNumericAdjustedValueType(id types.T) bool {
switch id {
case types.T_int8, types.T_int16, types.T_int32, types.T_int64,
types.T_uint8, types.T_uint16, types.T_uint32, types.T_uint64,
types.T_float32, types.T_float64,
types.T_decimal64, types.T_decimal128:
return true
default:
return false
}
}

func loadDataNonStrictNumericPrefix(val string) string {
val = strings.TrimSpace(val)
if val == "" {
return "0"
}

i := 0
if val[i] == '+' || val[i] == '-' {
i++
}
digitsStart := i
for i < len(val) && val[i] >= '0' && val[i] <= '9' {
i++
}
digits := i > digitsStart
if i < len(val) && val[i] == '.' {
i++
fracStart := i
for i < len(val) && val[i] >= '0' && val[i] <= '9' {
i++
}
digits = digits || i > fracStart
}
if !digits {
return "0"
}

end := i
if i < len(val) && (val[i] == 'e' || val[i] == 'E') {
exp := i + 1
if exp < len(val) && (val[exp] == '+' || val[exp] == '-') {
exp++
}
expDigits := exp
for exp < len(val) && val[exp] >= '0' && val[exp] <= '9' {
exp++
}
if exp > expDigits {
end = exp
}
}
return val[:end]
}

func truncateLoadDataStringValue(val string, width int32) string {
if width <= 0 || utf8.RuneCountInString(val) <= int(width) {
return val
}
return string([]rune(val)[:width])
}

func appendLoadEmptyNumericZero(vec *vector.Vector, id types.T, asBytes bool, mp *mpool.MPool) error {
if asBytes {
return vector.AppendBytes(vec, loadZeroBytes, false, mp)
Expand Down Expand Up @@ -995,6 +1073,7 @@ func getColData(bat *batch.Batch, line []csvparser.Field, rowIdx int, param *Ext

field := getFieldFromLine(line, colName, param, fieldIdx)
id := types.T(col.Typ.Id)
loadDataNonStrictAdjustments := shouldApplyLoadDataNonStrictAdjustments(param)
trimSpace := false
// T_bit carries raw bytes (parsed byte-by-byte, not as text), so whitespace
// bytes are data and must not be trimmed (a whitespace-only bit value would
Expand All @@ -1019,6 +1098,25 @@ func getColData(bat *batch.Batch, line []csvparser.Field, rowIdx int, param *Ext
return nil
}

zeroDateAdjusted := false
if loadDataNonStrictAdjustments {
switch {
case isLoadNumericAdjustedValueType(id):
if !field.HasStringQuote {
field.Val = loadDataNonStrictNumericPrefix(field.Val)
}
case id == types.T_date:
if _, err := types.ParseDateCast(field.Val); err != nil {
zeroDateAdjusted = true
if param.ParallelLoad {
field.Val = "0000-00-00"
}
}
case id == types.T_char || id == types.T_varchar:
field.Val = truncateLoadDataStringValue(field.Val, col.Typ.Width)
}
}

// In strict SQL mode (non-LOCAL load), an over-width CHAR/VARCHAR value is
// rejected instead of silently truncated, matching strict assignment casts
// (cast_strict) and MySQL's "Data too long" behavior. Uses rune count, like
Expand Down Expand Up @@ -1326,10 +1424,16 @@ func getColData(bat *batch.Batch, line []csvparser.Field, rowIdx int, param *Ext
return err
}
case types.T_date:
d, err := types.ParseDateCast(field.Val)
if err != nil {
logutil.Errorf("parse field[%v] err:%v", field.Val, err)
return moerr.NewInternalErrorf(param.Ctx, "the input value '%v' is not Date type for column %d", field.Val, colIdx)
var d types.Date
if zeroDateAdjusted {
d = types.Date(0)
} else {
var err error
d, err = types.ParseDateCast(field.Val)
if err != nil {
logutil.Errorf("parse field[%v] err:%v", field.Val, err)
return moerr.NewInternalErrorf(param.Ctx, "the input value '%v' is not Date type for column %d", field.Val, colIdx)
}
}

if err := vector.AppendFixed(vec, d, false, mp); err != nil {
Expand Down
Loading
Loading