diff --git a/yaml/README_Iceberg_To_MySql_Yaml.md b/yaml/README_Iceberg_To_MySql_Yaml.md index a2d3b938ee..3db7cce6c6 100644 --- a/yaml/README_Iceberg_To_MySql_Yaml.md +++ b/yaml/README_Iceberg_To_MySql_Yaml.md @@ -31,7 +31,7 @@ on [Metadata Annotations](https://github.com/GoogleCloudPlatform/DataflowTemplat * **jdbcType**: Specifies the type of JDBC source. An appropriate default driver will be packaged. For example, `mysql`. * **writeStatement**: The SQL query for inserting records, with placeholders for values. For example, `INSERT INTO my_table (col1, col2) VALUES(?, ?)`. * **batchSize**: The number of records to group together for each write. For example, `1000`. Defaults to: 1000. -* **autosharding**: If true, a dynamic number of shards will be used for writing. For example, `False`. +* **autoSharding**: If true, a dynamic number of shards will be used for writing. For example, `False`. * **configProperties**: A map of properties to pass to the Hadoop Configuration. For example, `{"fs.gs.impl": "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem"}`. * **drop**: A list of field names to drop. Mutually exclusive with 'keep' and 'only'. For example, `["field_to_drop_1", "field_to_drop_2"]`. * **filter**: A filter expression to apply to records from the Iceberg table. For example, `age > 18`. @@ -144,7 +144,7 @@ export CONNECTION_INIT_SQL= export JDBC_TYPE=mysql export WRITE_STATEMENT= export BATCH_SIZE=1000 -export AUTOSHARDING= +export AUTO_SHARDING= export CONFIG_PROPERTIES= export DROP= export FILTER= @@ -165,7 +165,7 @@ gcloud dataflow flex-template run "iceberg-to-mysql-yaml-job" \ --parameters "location=$LOCATION" \ --parameters "writeStatement=$WRITE_STATEMENT" \ --parameters "batchSize=$BATCH_SIZE" \ - --parameters "autosharding=$AUTOSHARDING" \ + --parameters "autoSharding=$AUTO_SHARDING" \ --parameters "table=$TABLE" \ --parameters "catalogName=$CATALOG_NAME" \ --parameters "catalogProperties=$CATALOG_PROPERTIES" \ @@ -207,7 +207,7 @@ export CONNECTION_INIT_SQL= export JDBC_TYPE=mysql export WRITE_STATEMENT= export BATCH_SIZE=1000 -export AUTOSHARDING= +export AUTO_SHARDING= export CONFIG_PROPERTIES= export DROP= export FILTER= @@ -220,7 +220,7 @@ mvn clean package -PtemplatesRun \ -Dregion="$REGION" \ -DjobName="iceberg-to-mysql-yaml-job" \ -DtemplateName="Iceberg_To_MySql_Yaml" \ --Dparameters="jdbcUrl=$JDBC_URL,username=$USERNAME,password=$PASSWORD,driverClassName=$DRIVER_CLASS_NAME,driverJars=$DRIVER_JARS,connectionProperties=$CONNECTION_PROPERTIES,connectionInitSql=$CONNECTION_INIT_SQL,jdbcType=$JDBC_TYPE,location=$LOCATION,writeStatement=$WRITE_STATEMENT,batchSize=$BATCH_SIZE,autosharding=$AUTOSHARDING,table=$TABLE,catalogName=$CATALOG_NAME,catalogProperties=$CATALOG_PROPERTIES,configProperties=$CONFIG_PROPERTIES,drop=$DROP,filter=$FILTER,keep=$KEEP" \ +-Dparameters="jdbcUrl=$JDBC_URL,username=$USERNAME,password=$PASSWORD,driverClassName=$DRIVER_CLASS_NAME,driverJars=$DRIVER_JARS,connectionProperties=$CONNECTION_PROPERTIES,connectionInitSql=$CONNECTION_INIT_SQL,jdbcType=$JDBC_TYPE,location=$LOCATION,writeStatement=$WRITE_STATEMENT,batchSize=$BATCH_SIZE,autoSharding=$AUTO_SHARDING,table=$TABLE,catalogName=$CATALOG_NAME,catalogProperties=$CATALOG_PROPERTIES,configProperties=$CONFIG_PROPERTIES,drop=$DROP,filter=$FILTER,keep=$KEEP" \ -f yaml ``` @@ -279,7 +279,7 @@ resource "google_dataflow_flex_template_job" "iceberg_to_mysql_yaml" { # jdbcType = "mysql" # writeStatement = "" # batchSize = "1000" - # autosharding = "" + # autoSharding = "" # configProperties = "" # drop = "" # filter = "" diff --git a/yaml/README_Iceberg_To_PostgreSQL_Yaml.md b/yaml/README_Iceberg_To_PostgreSQL_Yaml.md index 41c4f96367..10caca3c7c 100644 --- a/yaml/README_Iceberg_To_PostgreSQL_Yaml.md +++ b/yaml/README_Iceberg_To_PostgreSQL_Yaml.md @@ -31,7 +31,7 @@ on [Metadata Annotations](https://github.com/GoogleCloudPlatform/DataflowTemplat * **postgresTable**: The name of the database table. For example, `public.my_table`. * **query**: The SQL query/statement to execute on the source/sink. For example, `SELECT * FROM my_table WHERE status = 'active'`. * **batchSize**: The number of records to group together for each write. For example, `1000`. Defaults to: 1000. -* **autosharding**: If true, a dynamic number of shards will be used for writing. For example, `False`. +* **autoSharding**: If true, a dynamic number of shards will be used for writing. For example, `False`. @@ -140,7 +140,7 @@ export CONNECTION_PROPERTIES= export POSTGRES_TABLE= export QUERY= export BATCH_SIZE=1000 -export AUTOSHARDING= +export AUTO_SHARDING= gcloud dataflow flex-template run "iceberg-to-postgresql-yaml-job" \ --project "$PROJECT" \ @@ -160,7 +160,7 @@ gcloud dataflow flex-template run "iceberg-to-postgresql-yaml-job" \ --parameters "postgresTable=$POSTGRES_TABLE" \ --parameters "query=$QUERY" \ --parameters "batchSize=$BATCH_SIZE" \ - --parameters "autosharding=$AUTOSHARDING" + --parameters "autoSharding=$AUTO_SHARDING" ``` For more information about the command, please check: @@ -195,7 +195,7 @@ export CONNECTION_PROPERTIES= export POSTGRES_TABLE= export QUERY= export BATCH_SIZE=1000 -export AUTOSHARDING= +export AUTO_SHARDING= mvn clean package -PtemplatesRun \ -DskipTests \ @@ -204,7 +204,7 @@ mvn clean package -PtemplatesRun \ -Dregion="$REGION" \ -DjobName="iceberg-to-postgresql-yaml-job" \ -DtemplateName="Iceberg_To_PostgreSQL_Yaml" \ --Dparameters="table=$TABLE,catalogName=$CATALOG_NAME,catalogProperties=$CATALOG_PROPERTIES,configProperties=$CONFIG_PROPERTIES,drop=$DROP,filter=$FILTER,keep=$KEEP,jdbcUrl=$JDBC_URL,username=$USERNAME,password=$PASSWORD,connectionProperties=$CONNECTION_PROPERTIES,postgresTable=$POSTGRES_TABLE,query=$QUERY,batchSize=$BATCH_SIZE,autosharding=$AUTOSHARDING" \ +-Dparameters="table=$TABLE,catalogName=$CATALOG_NAME,catalogProperties=$CATALOG_PROPERTIES,configProperties=$CONFIG_PROPERTIES,drop=$DROP,filter=$FILTER,keep=$KEEP,jdbcUrl=$JDBC_URL,username=$USERNAME,password=$PASSWORD,connectionProperties=$CONNECTION_PROPERTIES,postgresTable=$POSTGRES_TABLE,query=$QUERY,batchSize=$BATCH_SIZE,autoSharding=$AUTO_SHARDING" \ -f yaml ``` @@ -263,7 +263,7 @@ resource "google_dataflow_flex_template_job" "iceberg_to_postgresql_yaml" { # postgresTable = "" # query = "" # batchSize = "1000" - # autosharding = "" + # autoSharding = "" } } ``` diff --git a/yaml/README_Iceberg_To_SqlServer_Yaml.md b/yaml/README_Iceberg_To_SqlServer_Yaml.md index 9b78c03cc3..7d370255c6 100644 --- a/yaml/README_Iceberg_To_SqlServer_Yaml.md +++ b/yaml/README_Iceberg_To_SqlServer_Yaml.md @@ -35,7 +35,7 @@ on [Metadata Annotations](https://github.com/GoogleCloudPlatform/DataflowTemplat * **jdbcType**: Specifies the type of JDBC source. An appropriate default driver will be packaged. For example, `mssql`. * **query**: The SQL query for inserting records, with placeholders for values. For example, `INSERT INTO my_table (col1, col2) VALUES(?, ?)`. * **batchSize**: The number of records to group together for each write. For example, `1000`. Defaults to: 1000. -* **autosharding**: If true, a dynamic number of shards will be used for writing. For example, `False`. +* **autoSharding**: If true, a dynamic number of shards will be used for writing. For example, `False`. @@ -148,7 +148,7 @@ export CONNECTION_INIT_SQL= export JDBC_TYPE=mssql export QUERY= export BATCH_SIZE=1000 -export AUTOSHARDING= +export AUTO_SHARDING= gcloud dataflow flex-template run "iceberg-to-sqlserver-yaml-job" \ --project "$PROJECT" \ @@ -172,7 +172,7 @@ gcloud dataflow flex-template run "iceberg-to-sqlserver-yaml-job" \ --parameters "location=$LOCATION" \ --parameters "query=$QUERY" \ --parameters "batchSize=$BATCH_SIZE" \ - --parameters "autosharding=$AUTOSHARDING" + --parameters "autoSharding=$AUTO_SHARDING" ``` For more information about the command, please check: @@ -211,7 +211,7 @@ export CONNECTION_INIT_SQL= export JDBC_TYPE=mssql export QUERY= export BATCH_SIZE=1000 -export AUTOSHARDING= +export AUTO_SHARDING= mvn clean package -PtemplatesRun \ -DskipTests \ @@ -220,7 +220,7 @@ mvn clean package -PtemplatesRun \ -Dregion="$REGION" \ -DjobName="iceberg-to-sqlserver-yaml-job" \ -DtemplateName="Iceberg_To_SqlServer_Yaml" \ --Dparameters="table=$TABLE,catalogName=$CATALOG_NAME,catalogProperties=$CATALOG_PROPERTIES,configProperties=$CONFIG_PROPERTIES,drop=$DROP,filter=$FILTER,keep=$KEEP,jdbcUrl=$JDBC_URL,username=$USERNAME,password=$PASSWORD,driverClassName=$DRIVER_CLASS_NAME,driverJars=$DRIVER_JARS,connectionProperties=$CONNECTION_PROPERTIES,connectionInitSql=$CONNECTION_INIT_SQL,jdbcType=$JDBC_TYPE,location=$LOCATION,query=$QUERY,batchSize=$BATCH_SIZE,autosharding=$AUTOSHARDING" \ +-Dparameters="table=$TABLE,catalogName=$CATALOG_NAME,catalogProperties=$CATALOG_PROPERTIES,configProperties=$CONFIG_PROPERTIES,drop=$DROP,filter=$FILTER,keep=$KEEP,jdbcUrl=$JDBC_URL,username=$USERNAME,password=$PASSWORD,driverClassName=$DRIVER_CLASS_NAME,driverJars=$DRIVER_JARS,connectionProperties=$CONNECTION_PROPERTIES,connectionInitSql=$CONNECTION_INIT_SQL,jdbcType=$JDBC_TYPE,location=$LOCATION,query=$QUERY,batchSize=$BATCH_SIZE,autoSharding=$AUTO_SHARDING" \ -f yaml ``` @@ -283,7 +283,7 @@ resource "google_dataflow_flex_template_job" "iceberg_to_sqlserver_yaml" { # jdbcType = "mssql" # query = "" # batchSize = "1000" - # autosharding = "" + # autoSharding = "" } } ``` diff --git a/yaml/src/main/java/com/google/cloud/teleport/templates/yaml/IcebergToMySQLYaml.java b/yaml/src/main/java/com/google/cloud/teleport/templates/yaml/IcebergToMySQLYaml.java index 156592b394..bad2d20632 100644 --- a/yaml/src/main/java/com/google/cloud/teleport/templates/yaml/IcebergToMySQLYaml.java +++ b/yaml/src/main/java/com/google/cloud/teleport/templates/yaml/IcebergToMySQLYaml.java @@ -154,12 +154,12 @@ public interface IcebergToMySQLYaml { @TemplateParameter.Boolean( order = 12, - name = "autosharding", + name = "autoSharding", optional = true, description = "If true, enables using a dynamically determined number of shards to write.", helpText = "If true, a dynamic number of shards will be used for writing.", example = "False") - Boolean getAutosharding(); + Boolean getAutoSharding(); @TemplateParameter.Text( order = 13, diff --git a/yaml/src/main/java/com/google/cloud/teleport/templates/yaml/IcebergToPostgreSQLYaml.java b/yaml/src/main/java/com/google/cloud/teleport/templates/yaml/IcebergToPostgreSQLYaml.java index 908b6265d8..7a75df62c5 100644 --- a/yaml/src/main/java/com/google/cloud/teleport/templates/yaml/IcebergToPostgreSQLYaml.java +++ b/yaml/src/main/java/com/google/cloud/teleport/templates/yaml/IcebergToPostgreSQLYaml.java @@ -179,10 +179,10 @@ public interface IcebergToPostgreSQLYaml { @TemplateParameter.Boolean( order = 15, - name = "autosharding", + name = "autoSharding", optional = true, description = "If true, enables using a dynamically determined number of shards to write.", helpText = "If true, a dynamic number of shards will be used for writing.", example = "False") - Boolean getAutosharding(); + Boolean getAutoSharding(); } diff --git a/yaml/src/main/java/com/google/cloud/teleport/templates/yaml/IcebergToSQLServerYaml.java b/yaml/src/main/java/com/google/cloud/teleport/templates/yaml/IcebergToSQLServerYaml.java index 51b87d3aff..f2f3b46710 100644 --- a/yaml/src/main/java/com/google/cloud/teleport/templates/yaml/IcebergToSQLServerYaml.java +++ b/yaml/src/main/java/com/google/cloud/teleport/templates/yaml/IcebergToSQLServerYaml.java @@ -220,10 +220,10 @@ public interface IcebergToSQLServerYaml { @TemplateParameter.Boolean( order = 19, - name = "autosharding", + name = "autoSharding", optional = true, description = "If true, enables using a dynamically determined number of shards to write.", helpText = "If true, a dynamic number of shards will be used for writing.", example = "False") - Boolean getAutosharding(); + Boolean getAutoSharding(); } diff --git a/yaml/src/main/python/options/mysql_options.yaml b/yaml/src/main/python/options/mysql_options.yaml index 7acc11934d..c1082cc1be 100644 --- a/yaml/src/main/python/options/mysql_options.yaml +++ b/yaml/src/main/python/options/mysql_options.yaml @@ -137,7 +137,7 @@ options: type: integer default: 1000 - order: 4 - name: "autosharding" + name: "autoSharding" description: "If true, enables using a dynamically determined number of shards to write." help: "If true, a dynamic number of shards will be used for writing." example: false diff --git a/yaml/src/main/python/options/postgres_options.yaml b/yaml/src/main/python/options/postgres_options.yaml index 823bef1930..fe10327932 100644 --- a/yaml/src/main/python/options/postgres_options.yaml +++ b/yaml/src/main/python/options/postgres_options.yaml @@ -93,7 +93,7 @@ options: type: integer default: 1000 - order: 2 - name: "autosharding" + name: "autoSharding" description: "If true, enables using a dynamically determined number of shards to write." help: "If true, a dynamic number of shards will be used for writing." example: false diff --git a/yaml/src/main/python/options/sqlserver_options.yaml b/yaml/src/main/python/options/sqlserver_options.yaml index 96824c81d7..0488e85573 100644 --- a/yaml/src/main/python/options/sqlserver_options.yaml +++ b/yaml/src/main/python/options/sqlserver_options.yaml @@ -137,7 +137,7 @@ options: type: integer default: 1000 - order: 4 - name: "autosharding" + name: "autoSharding" description: "If true, enables using a dynamically determined number of shards to write." help: "If true, a dynamic number of shards will be used for writing." example: false diff --git a/yaml/src/main/yaml/IcebergToMySQL.yaml b/yaml/src/main/yaml/IcebergToMySQL.yaml index 59b90cf483..2099d497ca 100644 --- a/yaml/src/main/yaml/IcebergToMySQL.yaml +++ b/yaml/src/main/yaml/IcebergToMySQL.yaml @@ -48,7 +48,7 @@ pipeline: url: "{{ jdbcUrl }}" username: "{{ username }}" password: "{{ password }}" - auto_sharding: {{ autosharding }} + auto_sharding: {{ autoSharding }} connection_init_sql: {{ connectionInitSql }} connection_properties: {{ connectionProperties }} query: {{ insertQuery }} diff --git a/yaml/src/main/yaml/IcebergToPostgreSQL.yaml b/yaml/src/main/yaml/IcebergToPostgreSQL.yaml index 2d540cae6a..b74021acf2 100644 --- a/yaml/src/main/yaml/IcebergToPostgreSQL.yaml +++ b/yaml/src/main/yaml/IcebergToPostgreSQL.yaml @@ -53,7 +53,7 @@ pipeline: connection_properties: {{ connectionProperties }} table: {{ postgresTable }} batch_size: {{ batchSize }} - autosharding: {{ autosharding }} + auto_sharding: {{ autoSharding }} options: streaming: false diff --git a/yaml/src/main/yaml/IcebergToSQLServer.yaml b/yaml/src/main/yaml/IcebergToSQLServer.yaml index 4b0e10645c..acc49db048 100644 --- a/yaml/src/main/yaml/IcebergToSQLServer.yaml +++ b/yaml/src/main/yaml/IcebergToSQLServer.yaml @@ -53,7 +53,7 @@ pipeline: query: {{ query }} table: {{ location }} batch_size: {{ batchSize }} - autosharding: {{ autosharding }} + auto_sharding: {{ autoSharding }} options: streaming: false \ No newline at end of file