Change array data types to use cardinality property instead of appending ARRAY to typename

dbptk-model/src/main/java/com/databasepreservation/model/modules/DatatypeImporter.java

@@ -7,6 +7,7 @@
  */
 package com.databasepreservation.model.modules;
 
+import java.math.BigInteger;
 import java.sql.Connection;
 import java.sql.SQLException;
 
@@ -46,6 +47,13 @@ public void setOnceReporter(Reporter reporter) {
     this.reporter = reporter;
   }
 
+  public Type getCheckedType(DatabaseStructure database, SchemaStructure currentSchema, String tableName,
+    String columnName, int dataType, String typeName, int columnSize, int decimalDigits, int numPrecRadix) {
+
+    return getCheckedType(database, currentSchema, tableName, columnName, dataType, typeName, columnSize, decimalDigits,
+      numPrecRadix, null);
+  }
+
   /**
    * Map the original type to the normalized type model
    *
@@ -66,19 +74,22 @@ public void setOnceReporter(Reporter reporter) {
    * @param numPrecRadix
    *          Indicates the numeric radix of this data type, which is usually 2 or
    *          10
+   * @param cardinality
+   *          Cardinality of this type (max number of elements if it's an array)
    * @return the normalized type
    * @throws UnknownTypeException
    *           the original type is unknown and cannot be mapped
    * @throws ClassNotFoundException
    * @throws SQLException
    */
   public Type getCheckedType(DatabaseStructure database, SchemaStructure currentSchema, String tableName,
-    String columnName, int dataType, String typeName, int columnSize, int decimalDigits, int numPrecRadix) {
+    String columnName, int dataType, String typeName, int columnSize, int decimalDigits, int numPrecRadix,
+    BigInteger cardinality) {
 
     Type type = getFallbackType(typeName);
     try {
       type = getType(database, currentSchema, tableName, columnName, dataType, typeName, columnSize, decimalDigits,
-        numPrecRadix);
+        numPrecRadix, cardinality);
     } catch (UnknownTypeException e) {
       LOGGER.debug("Got an UnknownTypeException while getting the source database type", e);
     } catch (SQLException e) {
@@ -132,6 +143,10 @@ protected abstract Type getType(DatabaseStructure database, SchemaStructure curr
     String columnName, int dataType, String typeName, int columnSize, int decimalDigits, int numPrecRadix)
     throws UnknownTypeException, SQLException, ClassNotFoundException;
 
+  protected abstract Type getType(DatabaseStructure database, SchemaStructure currentSchema, String tableName,
+    String columnName, int dataType, String typeName, int columnSize, int decimalDigits, int numPrecRadix,
+    BigInteger cardinality) throws UnknownTypeException, SQLException, ClassNotFoundException;
+
   protected abstract Type getArray(String typeName, int columnSize, int decimalDigits, int numPrecRadix, int dataType,
     Type subType) throws UnknownTypeException;
 

dbptk-model/src/main/java/com/databasepreservation/model/structure/ColumnStructure.java

@@ -7,6 +7,7 @@
  */
 package com.databasepreservation.model.structure;
 
+import java.math.BigInteger;
 import java.util.Objects;
 
 import com.databasepreservation.model.structure.type.Type;
@@ -33,6 +34,8 @@ public class ColumnStructure {
 
   private Boolean isAutoIncrement;
 
+  private BigInteger cardinality;
+
   /**
    * ColumnStructure empty constructor
    */
@@ -180,11 +183,19 @@ public void setNillable(Boolean nillable) {
     this.nillable = nillable;
   }
 
+  public BigInteger getCardinality() {
+    return cardinality;
+  }
+
+  public void setCardinality(BigInteger cardinality) {
+    this.cardinality = cardinality;
+  }
+
   @Override
   public String toString() {
     return "ColumnStructure{" + "id='" + id + '\'' + ", name='" + name + '\'' + ", type=" + type + ", defaultValue='"
       + defaultValue + '\'' + ", nillable=" + nillable + ", description='" + description + '\'' + ", isAutoIncrement="
-      + isAutoIncrement + '}';
+      + isAutoIncrement + ", cardinality=" + cardinality + "}";
   }
 
   @Override
@@ -253,6 +264,9 @@ public boolean equals(Object obj) {
     } else if (!type.equals(other.type)) {
       return false;
     }
+    if (cardinality != other.cardinality) {
+      return false;
+    }
     return true;
   }
 }

dbptk-modules/dbptk-module-jdbc/src/main/java/com/databasepreservation/modules/jdbc/in/JDBCDatatypeImporter.java

@@ -7,6 +7,7 @@
  */
 package com.databasepreservation.modules.jdbc.in;
 
+import java.math.BigInteger;
 import java.sql.SQLException;
 import java.sql.Types;
 
@@ -39,6 +40,14 @@ public class JDBCDatatypeImporter extends DatatypeImporter {
   protected Type getType(DatabaseStructure database, SchemaStructure currentSchema, String tableName, String columnName,
     int dataType, String typeName, int columnSize, int decimalDigits, int numPrecRadix)
     throws UnknownTypeException, SQLException, ClassNotFoundException {
+    return getType(database, currentSchema, tableName, columnName, dataType, typeName, columnSize, decimalDigits,
+      numPrecRadix, null);
+  }
+
+  @Override
+  protected Type getType(DatabaseStructure database, SchemaStructure currentSchema, String tableName, String columnName,
+    int dataType, String typeName, int columnSize, int decimalDigits, int numPrecRadix, BigInteger cardinality)
+    throws UnknownTypeException, SQLException, ClassNotFoundException {
     Type type;
 
     boolean maxSize = (columnSize == Integer.MAX_VALUE);
@@ -159,8 +168,8 @@ protected Type getType(DatabaseStructure database, SchemaStructure currentSchema
   protected Type getArray(String typeName, int columnSize, int decimalDigits, int numPrecRadix, int dataType,
     Type subType) {
     ComposedTypeArray type = new ComposedTypeArray(subType);
-    type.setSql99TypeName(subType.getSql99TypeName() + " ARRAY");
-    type.setSql2008TypeName(subType.getSql2008TypeName() + " ARRAY");
+    type.setSql99TypeName(subType.getSql99TypeName());
+    type.setSql2008TypeName(subType.getSql2008TypeName());
     return type;
   }
 

dbptk-modules/dbptk-module-jdbc/src/main/java/com/databasepreservation/modules/jdbc/in/JDBCImportModule.java

@@ -13,6 +13,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.math.BigDecimal;
+import java.math.BigInteger;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.nio.file.Files;
@@ -1642,6 +1643,22 @@ protected String processTriggerName(String string) {
     return string;
   }
 
+  private void updateMaxCardinalities(Row row, HashMap<String, Integer> cardinalities) {
+    for (Cell cell : row.getCells()) {
+      if (cell instanceof ArrayCell arrayCell) {
+        String[] cellIdParts = cell.getId().split("\\.");
+        String cellColumnName = cellIdParts[cellIdParts.length - 2];
+        int cellCardinality = 0;
+        try {
+          cellCardinality = arrayCell.toArray((c -> c), Cell.class).length;
+        } catch (InvalidDataException e) {
+          LOGGER.warn("Couldn't get array cell data for cell {}", cell);
+        }
+        cardinalities.put(cellColumnName, Math.max(cardinalities.getOrDefault(cellColumnName, 0), cellCardinality));
+      }
+    }
+  }
+
   protected Row convertRawToRow(ResultSet rawData, TableStructure tableStructure)
     throws InvalidDataException, SQLException, ModuleException {
     Row row = null;
@@ -2088,6 +2105,7 @@ public DatabaseFilterModule migrateDatabaseTo(DatabaseFilterModule exportModule)
         for (TableStructure table : schema.getTables()) {
           exportModule.handleDataOpenTable(table.getId());
 
+          HashMap<String, Integer> tableColumnsCardinality = new HashMap<>();
           long nRows = 0;
           if (getModuleConfiguration().isFetchRows()) {
             if (table.isFromCustomView()) {
@@ -2096,7 +2114,9 @@ public DatabaseFilterModule migrateDatabaseTo(DatabaseFilterModule exportModule)
               if (customView != null) {
                 try (ResultSet tableRawData = getTableRawData(customView.getQuery(), table.getId())) {
                   while (resultSetNext(tableRawData)) {
-                    exportModule.handleDataRow(convertRawToRow(tableRawData, table));
+                    Row row = convertRawToRow(tableRawData, table);
+                    exportModule.handleDataRow(row);
+                    updateMaxCardinalities(row, tableColumnsCardinality);
                     nRows++;
                   }
                 } catch (SQLException | ModuleException e) {
@@ -2108,7 +2128,9 @@ public DatabaseFilterModule migrateDatabaseTo(DatabaseFilterModule exportModule)
             } else {
               try (ResultSet tableRawData = getTableRawData(table)) {
                 while (resultSetNext(tableRawData)) {
-                  exportModule.handleDataRow(convertRawToRow(tableRawData, table));
+                  Row row = convertRawToRow(tableRawData, table);
+                  exportModule.handleDataRow(row);
+                  updateMaxCardinalities(row, tableColumnsCardinality);
                   nRows++;
                 }
               } catch (SQLException e) {
@@ -2131,6 +2153,12 @@ public DatabaseFilterModule migrateDatabaseTo(DatabaseFilterModule exportModule)
           // problem.");
           // }
 
+          for (Map.Entry<String, Integer> arrayCellCardinalityPair : tableColumnsCardinality.entrySet()) {
+            ColumnStructure column = table.getColumnByName(arrayCellCardinalityPair.getKey());
+            if (column != null && column.getType() instanceof ComposedTypeArray) {
+              column.setCardinality(BigInteger.valueOf(arrayCellCardinalityPair.getValue()));
+            }
+          }
           getDatabaseStructure().getTableById(table.getId()).setRows(nRows);
 
           exportModule.handleDataCloseTable(table.getId());

dbptk-modules/dbptk-module-postgresql/src/main/java/com/databasepreservation/modules/postgresql/in/PostgreSQLJDBCDatatypeImporter.java

@@ -160,8 +160,10 @@ protected Type getNumericType(String typeName, int columnSize, int decimalDigits
     // declaration is 1000, so if we find more than that it means that this type was
     // declared without precision nor scale
 
-    //20240626 alindo: when upgrading the conector to version 42.7.3 it sets the data type
-    // scale and precision to 0 when they are not specified which is against SIARD specification
+    // 20240626 alindo: when upgrading the conector to version 42.7.3 it sets the
+    // data type
+    // scale and precision to 0 when they are not specified which is against SIARD
+    // specification
     // https://github.com/pgjdbc/pgjdbc/issues/2188
     if (columnSize == 0) {
       type.setSql99TypeName("NUMERIC", NUMERIC_MAX_PRECISION_NUMBER, NUMERIC_MAX_SCALE_NUMBER);

dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARD1MetadataImportStrategy.java

@@ -13,9 +13,6 @@
 import java.util.List;
 
 import javax.xml.XMLConstants;
-import jakarta.xml.bind.JAXBContext;
-import jakarta.xml.bind.JAXBException;
-import jakarta.xml.bind.Unmarshaller;
 import javax.xml.transform.stream.StreamSource;
 import javax.xml.validation.Schema;
 import javax.xml.validation.SchemaFactory;
@@ -24,9 +21,9 @@
 import org.slf4j.LoggerFactory;
 import org.xml.sax.SAXException;
 
-import com.databasepreservation.model.reporters.Reporter;
 import com.databasepreservation.model.exception.ModuleException;
 import com.databasepreservation.model.modules.configuration.ModuleConfiguration;
+import com.databasepreservation.model.reporters.Reporter;
 import com.databasepreservation.model.structure.CandidateKey;
 import com.databasepreservation.model.structure.CheckConstraint;
 import com.databasepreservation.model.structure.ColumnStructure;
@@ -81,6 +78,10 @@
 import com.databasepreservation.utils.JodaUtils;
 import com.databasepreservation.utils.XMLUtils;
 
+import jakarta.xml.bind.JAXBContext;
+import jakarta.xml.bind.JAXBException;
+import jakarta.xml.bind.Unmarshaller;
+
 /**
  * @author Bruno Ferreira <bferreira@keep.pt>
  */

dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARD21MetadataImportStrategy.java

@@ -399,7 +399,7 @@ private Parameter getParameter(ParameterType parameterType) throws ModuleExcepti
       result.setMode(parameterType.getMode());
       result.setType(sqlStandardDatatypeImporter.getCheckedType(metadataCurrentDatabaseName, metadataCurrentSchemaName,
         metadataCurrentTableName + " (routine)", parameterType.getName() + " (parameter)", parameterType.getType(),
-        parameterType.getTypeOriginal()));
+        parameterType.getTypeOriginal(), parameterType.getCardinality()));
       result.setDescription(XMLUtils.decode(parameterType.getDescription()));
 
       // todo: deal with these fields (related to complex types)
@@ -670,7 +670,7 @@ private ColumnStructure getColumnStructure(ColumnType column, String tableId) th
       contentPathStrategy.associateColumnWithFolder(result.getId(), column.getLobFolder());
 
       result.setType(sqlStandardDatatypeImporter.getCheckedType(metadataCurrentDatabaseName, metadataCurrentSchemaName,
-        metadataCurrentTableName, column.getName(), column.getType(), column.getTypeOriginal()));
+        metadataCurrentTableName, column.getName(), column.getType(), column.getTypeOriginal(), column.getCardinality()));
 
       result.setNillable(column.isNullable());
       result.setDefaultValue(column.getDefaultValue());

dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARD22MetadataImportStrategy.java

@@ -398,7 +398,7 @@ private Parameter getParameter(ParameterType parameterType) throws ModuleExcepti
       result.setMode(parameterType.getMode());
       result.setType(sqlStandardDatatypeImporter.getCheckedType(metadataCurrentDatabaseName, metadataCurrentSchemaName,
         metadataCurrentTableName + " (routine)", parameterType.getName() + " (parameter)", parameterType.getType(),
-        parameterType.getTypeOriginal()));
+        parameterType.getTypeOriginal(), parameterType.getCardinality()));
       result.setDescription(XMLUtils.decode(parameterType.getDescription()));
 
       // todo: deal with these fields (related to complex types)
@@ -664,12 +664,13 @@ private ColumnStructure getColumnStructure(ColumnType column, String tableId) th
       contentPathStrategy.associateColumnWithFolder(result.getId(), column.getLobFolder());
 
       result.setType(sqlStandardDatatypeImporter.getCheckedType(metadataCurrentDatabaseName, metadataCurrentSchemaName,
-        metadataCurrentTableName, column.getName(), column.getType(), column.getTypeOriginal()));
+        metadataCurrentTableName, column.getName(), column.getType(), column.getTypeOriginal(), column.getCardinality()));
 
       result.setNillable(column.isNullable());
       result.setDefaultValue(column.getDefaultValue());
       result.setDescription(XMLUtils.decode(column.getDescription()));
       result.setLobFolder(column.getLobFolder());
+      result.setCardinality(column.getCardinality());
 
       // todo: deal with these fields
       // column.getLobFolder();

dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK1007MetadataImportStrategy.java

@@ -536,7 +536,8 @@ protected List<ColumnStructure> getTblColumns(ColumnsType columnsXml, String tab
         String typeOriginal = StringUtils.isNotBlank(columnXml.getTypeOriginal()) ? columnXml.getTypeOriginal() : null;
         columnDptkl
           .setType(sqlStandardDatatypeImporter.getCheckedType("<information unavailable>", "<information unavailable>",
-            "<information unavailable>", "<information unavailable>", columnXml.getType(), typeOriginal));
+            "<information unavailable>", "<information unavailable>", columnXml.getType(), typeOriginal,
+            columnDptkl.getCardinality()));
         columnDptkl.setDescription(columnXml.getDescription());
         String defaultValue = StringUtils.isNotBlank(columnXml.getDefaultValue()) ? columnXml.getDefaultValue() : null;
         columnDptkl.setDefaultValue(defaultValue);

dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK128MetadataImportStrategy.java

@@ -534,9 +534,9 @@ protected List<ColumnStructure> getTblColumns(ColumnsType columnsXml, String tab
         columnDptkl.setName(columnXml.getName());
         columnDptkl.setId(String.format("%s.%s", tableId, columnDptkl.getName()));
         String typeOriginal = StringUtils.isNotBlank(columnXml.getTypeOriginal()) ? columnXml.getTypeOriginal() : null;
-        columnDptkl
-          .setType(sqlStandardDatatypeImporter.getCheckedType("<information unavailable>", "<information unavailable>",
-            "<information unavailable>", "<information unavailable>", columnXml.getType(), typeOriginal));
+        columnDptkl.setType(sqlStandardDatatypeImporter.getCheckedType("<information unavailable>",
+          "<information unavailable>", "<information unavailable>", "<information unavailable>", columnXml.getType(),
+          typeOriginal, columnDptkl.getCardinality()));
         columnDptkl.setDescription(columnXml.getDescription());
         String defaultValue = StringUtils.isNotBlank(columnXml.getDefaultValue()) ? columnXml.getDefaultValue() : null;
         columnDptkl.setDefaultValue(defaultValue);

dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/typeConverter/SQL2008StandardDatatypeImporter.java

@@ -7,6 +7,7 @@
  */
 package com.databasepreservation.modules.siard.in.metadata.typeConverter;
 
+import java.math.BigInteger;
 import java.sql.SQLException;
 
 import org.apache.commons.lang3.StringUtils;
@@ -27,7 +28,15 @@ public class SQL2008StandardDatatypeImporter extends SQLStandardDatatypeImporter
 
   @Override
   protected Type getType(DatabaseStructure database, SchemaStructure currentSchema, String tableName, String columnName,
-    int dataType, String sql2008TypeName, int columnSize, int decimalDigits, int numPrecRadix)
+    int dataType, String typeName, int columnSize, int decimalDigits, int numPrecRadix)
+    throws UnknownTypeException, SQLException, ClassNotFoundException {
+    return getType(database, currentSchema, tableName, columnName, dataType, typeName, columnSize, decimalDigits,
+      numPrecRadix, null);
+  }
+
+  @Override
+  protected Type getType(DatabaseStructure database, SchemaStructure currentSchema, String tableName, String columnName,
+    int dataType, String sql2008TypeName, int columnSize, int decimalDigits, int numPrecRadix, BigInteger cardinality)
     throws UnknownTypeException, SQLException, ClassNotFoundException {
 
     SqlStandardType standardType = new SqlStandardType(sql2008TypeName);
@@ -157,17 +166,16 @@ protected Type getType(DatabaseStructure database, SchemaStructure currentSchema
       type.setOriginalTypeName(standardType.original);
     }
 
-    if (standardType.isArray) {
+    if (cardinality != null) {
       Type subtype = type;
       type = new ComposedTypeArray(subtype);
       type.setOriginalTypeName(subtype.getOriginalTypeName());
       type.setDescription(subtype.getDescription());
       type.setSql99TypeName(subtype.getSql99TypeName());
       type.setSql2008TypeName(subtype.getSql2008TypeName());
 
-      String typeNameWithoutArrayPart = standardType.normalized.substring(0, standardType.normalized.indexOf(" ARRAY"));
-      subtype.setSql99TypeName(typeNameWithoutArrayPart);
-      subtype.setSql2008TypeName(typeNameWithoutArrayPart);
+      subtype.setSql99TypeName(standardType.normalized);
+      subtype.setSql2008TypeName(standardType.normalized);
     }
     return type;
   }