From c9724e0db8ff27e9a41077ef96f06eaa31edbb55 Mon Sep 17 00:00:00 2001 From: Roland Praml Date: Wed, 22 Nov 2023 14:58:10 +0100 Subject: [PATCH] Add control options to override and/or validate index file --- .../io/ebean/migration/MigrationConfig.java | 37 +++++++++++++- .../io/ebean/migration/MigrationVersion.java | 16 +++++++ .../runner/LocalDdlMigrationResource.java | 4 ++ .../runner/LocalMigrationResource.java | 5 ++ .../runner/LocalMigrationResources.java | 48 +++++++++++++++++++ .../migration/runner/MigrationTable.java | 4 +- .../MigrationRunner_FastCheckTest.java | 44 +++++++++++++++++ .../indexB_check_invalid/1.0__initial.sql | 6 +++ .../resources/indexB_check_invalid/1.1.sql | 1 + .../resources/indexB_check_invalid/1.2.sql | 1 + .../resources/indexB_check_invalid/1.3.sql | 1 + .../indexB_check_invalid/idx_h2.migrations | 5 ++ .../indexB_check_valid/1.0__initial.sql | 6 +++ .../test/resources/indexB_check_valid/1.1.sql | 1 + .../test/resources/indexB_check_valid/1.2.sql | 1 + .../test/resources/indexB_check_valid/1.3.sql | 1 + .../indexB_check_valid/idx_h2.migrations | 5 ++ 17 files changed, 183 insertions(+), 3 deletions(-) create mode 100644 ebean-migration/src/test/resources/indexB_check_invalid/1.0__initial.sql create mode 100644 ebean-migration/src/test/resources/indexB_check_invalid/1.1.sql create mode 100644 ebean-migration/src/test/resources/indexB_check_invalid/1.2.sql create mode 100644 ebean-migration/src/test/resources/indexB_check_invalid/1.3.sql create mode 100644 ebean-migration/src/test/resources/indexB_check_invalid/idx_h2.migrations create mode 100644 ebean-migration/src/test/resources/indexB_check_valid/1.0__initial.sql create mode 100644 ebean-migration/src/test/resources/indexB_check_valid/1.1.sql create mode 100644 ebean-migration/src/test/resources/indexB_check_valid/1.2.sql create mode 100644 ebean-migration/src/test/resources/indexB_check_valid/1.3.sql create mode 100644 ebean-migration/src/test/resources/indexB_check_valid/idx_h2.migrations diff --git a/ebean-migration/src/main/java/io/ebean/migration/MigrationConfig.java b/ebean-migration/src/main/java/io/ebean/migration/MigrationConfig.java index d33696f..4dad274 100644 --- a/ebean-migration/src/main/java/io/ebean/migration/MigrationConfig.java +++ b/ebean-migration/src/main/java/io/ebean/migration/MigrationConfig.java @@ -12,7 +12,6 @@ * Configuration used to run the migration. */ public class MigrationConfig { - private String migrationPath = "dbmigration"; private String migrationInitPath = "dbinit"; private String metaTable = "db_migration"; @@ -67,6 +66,8 @@ public class MigrationConfig { private Properties properties; private boolean earlyChecksumMode; private boolean fastMode; + private boolean forceLocal = false; + private boolean forceLocalCheck = false; /** * Return the name of the migration table. @@ -457,6 +458,38 @@ public void setMinVersionFailMessage(String minVersionFailMessage) { this.minVersionFailMessage = minVersionFailMessage; } + /** + * Determines, if the local files should be used, although if there is an index file present. + */ + public boolean isForceLocal() { + return forceLocal; + } + + /** + * If this option is set, migrations are read from local resources and validated against the idx file. + */ + public void setForceLocal(boolean forceLocal) { + this.forceLocal = forceLocal; + } + + /** + * Determines, if the local files should be checked against the index. + */ + public boolean isForceLocalCheck() { + return forceLocalCheck; + } + + /** + * Determines, if local files should be checked. This means, the checksums of local files are validated against + * the ones in the index file. If this option is set, a MigrationException is thrown, when the index file does not + * match to local resources. (Note: Setting this option implies forceLocal. If this option is set to false, + * and forceLocal = true, then index mismatches will be logged as warning only) + */ + public void setForceLocalCheck(boolean forceLocalCheck) { + this.forceLocalCheck = forceLocalCheck; + } + + /** * Load configuration from standard properties. */ @@ -483,6 +516,8 @@ public void load(Properties props) { runPlaceholders = property("placeholders", runPlaceholders); minVersion = property("minVersion", minVersion); minVersionFailMessage = property("minVersionFailMessage", minVersionFailMessage); + forceLocal = property("forceLocal", forceLocal); + forceLocalCheck = property("forceLocalCheck", forceLocalCheck); String patchInsertOn = property("patchInsertOn"); if (patchInsertOn != null) { diff --git a/ebean-migration/src/main/java/io/ebean/migration/MigrationVersion.java b/ebean-migration/src/main/java/io/ebean/migration/MigrationVersion.java index c04059c..ff59055 100644 --- a/ebean-migration/src/main/java/io/ebean/migration/MigrationVersion.java +++ b/ebean-migration/src/main/java/io/ebean/migration/MigrationVersion.java @@ -1,6 +1,7 @@ package io.ebean.migration; import java.util.Arrays; +import java.util.Objects; import static java.lang.System.Logger.Level.*; @@ -265,4 +266,19 @@ public String type() { public String getType() { return type(); } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + MigrationVersion that = (MigrationVersion) o; + return Arrays.equals(ordering, that.ordering) && Objects.equals(comment, that.comment); + } + + @Override + public int hashCode() { + int result = Objects.hash(comment); + result = 31 * result + Arrays.hashCode(ordering); + return result; + } } diff --git a/ebean-migration/src/main/java/io/ebean/migration/runner/LocalDdlMigrationResource.java b/ebean-migration/src/main/java/io/ebean/migration/runner/LocalDdlMigrationResource.java index 5109b35..8290b40 100644 --- a/ebean-migration/src/main/java/io/ebean/migration/runner/LocalDdlMigrationResource.java +++ b/ebean-migration/src/main/java/io/ebean/migration/runner/LocalDdlMigrationResource.java @@ -37,4 +37,8 @@ private String missingOpensMessage() { return "NPE reading DB migration content at [" + location + "] Probably missing an 'opens dbmigration;' in module-info.java"; } + @Override + int checksum() { + return Checksum.calculate(content()); + } } diff --git a/ebean-migration/src/main/java/io/ebean/migration/runner/LocalMigrationResource.java b/ebean-migration/src/main/java/io/ebean/migration/runner/LocalMigrationResource.java index b467924..b67c5bd 100644 --- a/ebean-migration/src/main/java/io/ebean/migration/runner/LocalMigrationResource.java +++ b/ebean-migration/src/main/java/io/ebean/migration/runner/LocalMigrationResource.java @@ -91,4 +91,9 @@ public String type() { void setInitType() { this.type = MigrationVersion.BOOTINIT_TYPE; } + + /** + * The checksum of this resource (without parameter replacement). + */ + abstract int checksum(); } diff --git a/ebean-migration/src/main/java/io/ebean/migration/runner/LocalMigrationResources.java b/ebean-migration/src/main/java/io/ebean/migration/runner/LocalMigrationResources.java index cd605bb..b13d0a6 100644 --- a/ebean-migration/src/main/java/io/ebean/migration/runner/LocalMigrationResources.java +++ b/ebean-migration/src/main/java/io/ebean/migration/runner/LocalMigrationResources.java @@ -4,16 +4,21 @@ import io.avaje.classpath.scanner.core.Scanner; import io.ebean.migration.JdbcMigration; import io.ebean.migration.MigrationConfig; +import io.ebean.migration.MigrationException; import io.ebean.migration.MigrationVersion; import java.io.*; import java.net.URL; import java.util.ArrayList; import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; +import java.util.StringJoiner; import java.util.function.Predicate; import static java.lang.System.Logger.Level.DEBUG; +import static java.lang.System.Logger.Level.WARNING; /** * Loads the DB migration resources and sorts them into execution order. @@ -47,14 +52,57 @@ boolean readInitResources() { * Read all the migration resources (SQL scripts) returning true if there are versions. */ boolean readResources() { + if (readFromIndex()) { // automatically enable earlyChecksumMode when using index file with pre-computed checksums migrationConfig.setEarlyChecksumMode(true); + if (migrationConfig.isForceLocal() || migrationConfig.isForceLocalCheck()) { + return useLocalAndCheck(migrationConfig.isForceLocalCheck()); + } return true; } return readResourcesForPath(migrationConfig.getMigrationPath()); } + private boolean useLocalAndCheck(boolean fail) { + // throw away, what we have read from index. + Map index = new LinkedHashMap<>(); + for (LocalMigrationResource version : versions) { + index.put(version.version(), version); + } + versions.clear(); + // and re-read the local resources + if (!readResourcesForPath(migrationConfig.getMigrationPath())) { + if (fail) { + throw new MigrationException("Could not read local resources"); + } else { + log.log(WARNING, "Could not read local resources"); + return false; + } + } + + StringJoiner errors = new StringJoiner("; "); + for (LocalMigrationResource version : versions) { + LocalMigrationResource refVersion = index.remove(version.version); + if (refVersion == null) { + errors.add("'" + version + "' not in index file"); + } else if (refVersion.checksum() != version.checksum()) { + errors.add("'" + version + "' checksum mismatch (index " + refVersion.checksum() + ", local " + version.checksum() + ")"); + } + } + for (LocalMigrationResource refVersion : index.values()) { + errors.add("'" + refVersion + "' not in local migrations"); + } + if (errors.length() != 0) { + if (fail) { + throw new MigrationException("Index validation failed: " + errors); + } else { + log.log(WARNING, "Index validation failed: {0}", errors); + } + } + return true; + } + private boolean readFromIndex() { final var base = "/" + migrationConfig.getMigrationPath() + "/"; final var basePlatform = migrationConfig.getBasePlatform(); diff --git a/ebean-migration/src/main/java/io/ebean/migration/runner/MigrationTable.java b/ebean-migration/src/main/java/io/ebean/migration/runner/MigrationTable.java index 93550fe..105cfd8 100644 --- a/ebean-migration/src/main/java/io/ebean/migration/runner/MigrationTable.java +++ b/ebean-migration/src/main/java/io/ebean/migration/runner/MigrationTable.java @@ -302,7 +302,7 @@ private boolean runMigration(LocalMigrationResource local, MigrationMetaRow exis int checksum; int checksum2 = 0; if (local instanceof LocalUriMigrationResource) { - checksum = ((LocalUriMigrationResource)local).checksum(); + checksum = local.checksum(); checksum2 = patchLegacyChecksums ? AUTO_PATCH_CHECKSUM : 0; script = convertScript(local.content()); } else if (local instanceof LocalDdlMigrationResource) { @@ -312,7 +312,7 @@ private boolean runMigration(LocalMigrationResource local, MigrationMetaRow exis checksum = Checksum.calculate(earlyChecksumMode ? content : script); checksum2 = patchLegacyChecksums ? Checksum.calculate(script) : 0; } else { - checksum = ((LocalJdbcMigrationResource) local).checksum(); + checksum = local.checksum(); } if (existing == null && patchInsertMigration(local, checksum)) { diff --git a/ebean-migration/src/test/java/io/ebean/migration/MigrationRunner_FastCheckTest.java b/ebean-migration/src/test/java/io/ebean/migration/MigrationRunner_FastCheckTest.java index 6c158c3..1234ed4 100644 --- a/ebean-migration/src/test/java/io/ebean/migration/MigrationRunner_FastCheckTest.java +++ b/ebean-migration/src/test/java/io/ebean/migration/MigrationRunner_FastCheckTest.java @@ -9,6 +9,7 @@ import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; public class MigrationRunner_FastCheckTest { @@ -97,4 +98,47 @@ public void autoEnableEarlyMode() { assertThat(config.isEarlyChecksumMode()).isTrue(); } + @Test + public void checkIndex_valid() { + + MigrationConfig config = createMigrationConfig(); + + config.setPlatform("h2"); + config.setMigrationPath("indexB_check_valid"); + config.setForceLocalCheck(true); + config.setDbUrl("jdbc:h2:mem:checkindex_valid"); + config.setRunPlaceholderMap(Map.of("my_table_name", "bar")); + + MigrationRunner runner = new MigrationRunner(config); + runner.run(); + + } + + @Test + public void checkIndex_invalid() { + + MigrationConfig config = createMigrationConfig(); + + config.setPlatform("h2"); + config.setMigrationPath("indexB_check_invalid"); + config.setForceLocalCheck(true); + config.setDbUrl("jdbc:h2:mem:checkindex_invalid"); + config.setRunPlaceholderMap(Map.of("my_table_name", "bar")); + + MigrationRunner runner = new MigrationRunner(config); + assertThatThrownBy(runner::checkState) + .isInstanceOf(MigrationException.class) + .hasMessageContaining("'1.2' checksum mismatch (index -123456, local -212580746)") + .hasMessageContaining("'1.3' not in index file"); + + // switch to forceLocal only. This means, we get a warning about index validations on the console, + // but the local files are used + config.setForceLocalCheck(false); + config.setForceLocal(true); + + List state = runner.checkState(); + assertThat(state).hasSize(4); + // 1.3 not mentioned in the idx. + assertThat(state.get(3).location()).isEqualTo("indexB_check_invalid/1.3.sql"); + } } diff --git a/ebean-migration/src/test/resources/indexB_check_invalid/1.0__initial.sql b/ebean-migration/src/test/resources/indexB_check_invalid/1.0__initial.sql new file mode 100644 index 0000000..fd5f737 --- /dev/null +++ b/ebean-migration/src/test/resources/indexB_check_invalid/1.0__initial.sql @@ -0,0 +1,6 @@ +-- apply changes +create table ${my_table_name} ( + id integer generated by default as identity not null, + name varchar(255), + constraint pk_${my_table_name} primary key (id) +); diff --git a/ebean-migration/src/test/resources/indexB_check_invalid/1.1.sql b/ebean-migration/src/test/resources/indexB_check_invalid/1.1.sql new file mode 100644 index 0000000..38e24be --- /dev/null +++ b/ebean-migration/src/test/resources/indexB_check_invalid/1.1.sql @@ -0,0 +1 @@ +insert into ${my_table_name} (name) values ('hi'); diff --git a/ebean-migration/src/test/resources/indexB_check_invalid/1.2.sql b/ebean-migration/src/test/resources/indexB_check_invalid/1.2.sql new file mode 100644 index 0000000..b3fa4db --- /dev/null +++ b/ebean-migration/src/test/resources/indexB_check_invalid/1.2.sql @@ -0,0 +1 @@ +create table foo (acol varchar(10)); diff --git a/ebean-migration/src/test/resources/indexB_check_invalid/1.3.sql b/ebean-migration/src/test/resources/indexB_check_invalid/1.3.sql new file mode 100644 index 0000000..1095ac9 --- /dev/null +++ b/ebean-migration/src/test/resources/indexB_check_invalid/1.3.sql @@ -0,0 +1 @@ +create table bazz (acol varchar(10)); diff --git a/ebean-migration/src/test/resources/indexB_check_invalid/idx_h2.migrations b/ebean-migration/src/test/resources/indexB_check_invalid/idx_h2.migrations new file mode 100644 index 0000000..af6d736 --- /dev/null +++ b/ebean-migration/src/test/resources/indexB_check_invalid/idx_h2.migrations @@ -0,0 +1,5 @@ +-354472720, 1.0__initial.sql +-443571620, 1.1.sql +-123456, 1.2.sql + + diff --git a/ebean-migration/src/test/resources/indexB_check_valid/1.0__initial.sql b/ebean-migration/src/test/resources/indexB_check_valid/1.0__initial.sql new file mode 100644 index 0000000..fd5f737 --- /dev/null +++ b/ebean-migration/src/test/resources/indexB_check_valid/1.0__initial.sql @@ -0,0 +1,6 @@ +-- apply changes +create table ${my_table_name} ( + id integer generated by default as identity not null, + name varchar(255), + constraint pk_${my_table_name} primary key (id) +); diff --git a/ebean-migration/src/test/resources/indexB_check_valid/1.1.sql b/ebean-migration/src/test/resources/indexB_check_valid/1.1.sql new file mode 100644 index 0000000..38e24be --- /dev/null +++ b/ebean-migration/src/test/resources/indexB_check_valid/1.1.sql @@ -0,0 +1 @@ +insert into ${my_table_name} (name) values ('hi'); diff --git a/ebean-migration/src/test/resources/indexB_check_valid/1.2.sql b/ebean-migration/src/test/resources/indexB_check_valid/1.2.sql new file mode 100644 index 0000000..b3fa4db --- /dev/null +++ b/ebean-migration/src/test/resources/indexB_check_valid/1.2.sql @@ -0,0 +1 @@ +create table foo (acol varchar(10)); diff --git a/ebean-migration/src/test/resources/indexB_check_valid/1.3.sql b/ebean-migration/src/test/resources/indexB_check_valid/1.3.sql new file mode 100644 index 0000000..1095ac9 --- /dev/null +++ b/ebean-migration/src/test/resources/indexB_check_valid/1.3.sql @@ -0,0 +1 @@ +create table bazz (acol varchar(10)); diff --git a/ebean-migration/src/test/resources/indexB_check_valid/idx_h2.migrations b/ebean-migration/src/test/resources/indexB_check_valid/idx_h2.migrations new file mode 100644 index 0000000..eb407fa --- /dev/null +++ b/ebean-migration/src/test/resources/indexB_check_valid/idx_h2.migrations @@ -0,0 +1,5 @@ +-354472720, 1.0__initial.sql +-443571620, 1.1.sql +-212580746, 1.2.sql +-1779180902, 1.3.sql +