diff --git a/spark-cobol/src/main/scala/za/co/absa/cobrix/spark/cobol/schema/CobolSchema.scala b/spark-cobol/src/main/scala/za/co/absa/cobrix/spark/cobol/schema/CobolSchema.scala index 3beb8d8f..ed1b6d60 100644 --- a/spark-cobol/src/main/scala/za/co/absa/cobrix/spark/cobol/schema/CobolSchema.scala +++ b/spark-cobol/src/main/scala/za/co/absa/cobrix/spark/cobol/schema/CobolSchema.scala @@ -135,7 +135,7 @@ class CobolSchema(copybook: Copybook, StructField(Constants.fieldNameColumn, StringType, nullable = false), StructField(Constants.rawValueColumn, BinaryType, nullable = false) ) - ), containsNull = false), nullable = false) + ), containsNull = false), nullable = true) } else { recordsWithRecordId } diff --git a/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/CobolSchemaSpec.scala b/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/CobolSchemaSpec.scala index 34d1244d..92a5744d 100644 --- a/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/CobolSchemaSpec.scala +++ b/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/CobolSchemaSpec.scala @@ -306,7 +306,7 @@ class CobolSchemaSpec extends AnyWordSpec with SimpleComparisonBase { | | |-- IntValue: integer (nullable = true) | |-- STRUCT2: struct (nullable = true) | | |-- STR_FLD: string (nullable = true) - | |-- _corrupt_fields: array (nullable = false) + | |-- _corrupt_fields: array (nullable = true) | | |-- element: struct (containsNull = false) | | | |-- field_name: string (nullable = false) | | | |-- raw_value: binary (nullable = false) diff --git a/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/source/integration/Test41CorruptFieldsSpec.scala b/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/source/integration/Test41CorruptFieldsSpec.scala index 77dabe73..a82476ea 100644 --- a/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/source/integration/Test41CorruptFieldsSpec.scala +++ b/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/source/integration/Test41CorruptFieldsSpec.scala @@ -51,7 +51,7 @@ class Test41CorruptFieldsSpec extends AnyWordSpec with SparkTestBase with Binary | |-- F3: integer (nullable = true) | |-- F4: array (nullable = true) | | |-- element: integer (containsNull = true) - | |-- _corrupt_fields: array (nullable = false) + | |-- _corrupt_fields: array (nullable = true) | | |-- element: struct (containsNull = false) | | | |-- field_name: string (nullable = false) | | | |-- raw_value: binary (nullable = false)