|
15 | 15 | */ |
16 | 16 | package org.labkey.experiment; |
17 | 17 |
|
| 18 | +import org.apache.commons.collections4.MultiValuedMap; |
| 19 | +import org.apache.commons.collections4.multimap.ArrayListValuedHashMap; |
18 | 20 | import org.apache.commons.lang3.StringUtils; |
19 | 21 | import org.apache.logging.log4j.Logger; |
20 | 22 | import org.jetbrains.annotations.NotNull; |
|
25 | 27 | import org.labkey.api.audit.SampleTimelineAuditEvent; |
26 | 28 | import org.labkey.api.audit.TransactionAuditProvider; |
27 | 29 | import org.labkey.api.collections.CaseInsensitiveHashSet; |
| 30 | +import org.labkey.api.collections.CsvSet; |
| 31 | +import org.labkey.api.collections.LabKeyCollectors; |
28 | 32 | import org.labkey.api.data.ColumnInfo; |
| 33 | +import org.labkey.api.data.CompareType; |
29 | 34 | import org.labkey.api.data.Container; |
30 | 35 | import org.labkey.api.data.ContainerManager; |
31 | 36 | import org.labkey.api.data.DbSchema; |
|
39 | 44 | import org.labkey.api.data.SQLFragment; |
40 | 45 | import org.labkey.api.data.SchemaTableInfo; |
41 | 46 | import org.labkey.api.data.Selector; |
| 47 | +import org.labkey.api.data.SimpleFilter; |
42 | 48 | import org.labkey.api.data.SqlExecutor; |
43 | 49 | import org.labkey.api.data.SqlSelector; |
| 50 | +import org.labkey.api.data.Table; |
44 | 51 | import org.labkey.api.data.TableInfo; |
45 | 52 | import org.labkey.api.data.TableSelector; |
46 | 53 | import org.labkey.api.data.UpgradeCode; |
| 54 | +import org.labkey.api.data.dialect.BasePostgreSqlDialect; |
| 55 | +import org.labkey.api.data.dialect.PostgreSqlService; |
| 56 | +import org.labkey.api.exp.OntologyManager; |
47 | 57 | import org.labkey.api.exp.PropertyDescriptor; |
48 | 58 | import org.labkey.api.exp.api.ExpSampleType; |
49 | 59 | import org.labkey.api.exp.api.ExperimentService; |
|
64 | 74 | import org.labkey.api.security.User; |
65 | 75 | import org.labkey.api.security.roles.SiteAdminRole; |
66 | 76 | import org.labkey.api.settings.AppProps; |
| 77 | +import org.labkey.api.util.PageFlowUtil; |
| 78 | +import org.labkey.api.util.StringUtilsLabKey; |
67 | 79 | import org.labkey.api.util.logging.LogHelper; |
68 | 80 | import org.labkey.experiment.api.DataClass; |
69 | 81 | import org.labkey.experiment.api.DataClassDomainKind; |
|
73 | 85 | import org.labkey.experiment.api.MaterialSource; |
74 | 86 | import org.labkey.experiment.api.property.DomainImpl; |
75 | 87 | import org.labkey.experiment.api.property.DomainPropertyImpl; |
| 88 | +import org.labkey.experiment.api.property.StorageNameGenerator; |
76 | 89 | import org.labkey.experiment.api.property.StorageProvisionerImpl; |
77 | 90 |
|
| 91 | +import java.nio.charset.StandardCharsets; |
78 | 92 | import java.sql.Connection; |
79 | 93 | import java.sql.SQLException; |
80 | 94 | import java.util.ArrayList; |
| 95 | +import java.util.Collection; |
81 | 96 | import java.util.Collections; |
82 | 97 | import java.util.HashMap; |
83 | 98 | import java.util.HashSet; |
@@ -656,5 +671,147 @@ private static void fillRowId(ExpDataClassImpl dc, Domain domain, DbScope scope) |
656 | 671 | LOG.info("DataClass '{}' ({}) populated 'rowId' column, count={}", dc.getName(), dc.getRowId(), count); |
657 | 672 | } |
658 | 673 |
|
| 674 | + record DomainRecord(Container container, int domainId, String name, String storageSchemaName, String storageTableName) |
| 675 | + { |
| 676 | + String fullName() |
| 677 | + { |
| 678 | + return storageSchemaName + "." + storageTableName; |
| 679 | + } |
| 680 | + } |
| 681 | + |
| 682 | + record Property(int domainId, int propertyId, String domainName, String name, String storageSchemaName, String storageTableName, String storageColumnName) |
| 683 | + { |
| 684 | + String fullName() |
| 685 | + { |
| 686 | + // Have to bracket storage column name since it could have special characters |
| 687 | + return storageSchemaName + "." + storageTableName + "." + bracketIt(storageColumnName); |
| 688 | + } |
| 689 | + } |
| 690 | + |
| 691 | + /** |
| 692 | + * Called from exp-????.sql, SQL Server only |
| 693 | + * Query all table & column storage names and rename the ones that are too long for PostgreSQL. |
| 694 | + * TODO: When this upgrade code is removed, get rid of the StorageProvisionerImpl.makeTableName() method it uses. |
| 695 | + */ |
| 696 | + @SuppressWarnings("unused") |
| 697 | + @DeferredUpgrade |
| 698 | + public static void shortenAllStorageNames(ModuleContext context) |
| 699 | + { |
| 700 | + if (context.isNewInstall()) |
| 701 | + return; |
| 702 | + |
| 703 | + // The PostgreSQL dialect knows which names are too long |
| 704 | + BasePostgreSqlDialect dialect = PostgreSqlService.get().getDialect(); |
| 705 | + DbScope scope = DbScope.getLabKeyScope(); |
| 706 | + SqlExecutor executor = new SqlExecutor(scope); |
659 | 707 |
|
| 708 | + // Stream all the storage table names and rename the ones that are too long for PostgreSQL. The filtering must |
| 709 | + // be done in code by the dialect; SQL Server has BYTELENGTH(), but that function returns values that are not |
| 710 | + // consistent with our dialect check. Also, it looks like the function's behavior changed starting in SS 2019. |
| 711 | + TableInfo tinfoDomainDescriptor = OntologyManager.getTinfoDomainDescriptor(); |
| 712 | + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("StorageSchemaName"), null, CompareType.NONBLANK); |
| 713 | + filter.addCondition(FieldKey.fromString("StorageTableName"), null, CompareType.NONBLANK); |
| 714 | + try (DbScope.Transaction t = scope.beginTransaction()) |
| 715 | + { |
| 716 | + new TableSelector(tinfoDomainDescriptor, new CsvSet("Container, DomainId, Name, StorageSchemaName, StorageTableName"), filter, null) |
| 717 | + .setJdbcCaching(false) |
| 718 | + .stream(DomainRecord.class) |
| 719 | + .filter(domain -> dialect.isIdentifierTooLong(domain.storageTableName())) |
| 720 | + .forEach(domain -> { |
| 721 | + String oldName = domain.fullName(); |
| 722 | + String newName = StorageProvisionerImpl.get().makeTableName(dialect, domain.container(), domain.domainId(), domain.name()); |
| 723 | + |
| 724 | + executor.execute(new SQLFragment("EXEC sp_rename ?, ?").add(oldName).add(newName)); |
| 725 | + Table.update(null, tinfoDomainDescriptor, PageFlowUtil.map("StorageTableName", newName), domain.domainId()); |
| 726 | + |
| 727 | + LOG.info(" Table \"{}\" renamed to \"{}\" ({} bytes)", oldName, newName, newName.getBytes(StandardCharsets.UTF_8).length); |
| 728 | + }); |
| 729 | + |
| 730 | + List<String> badTableNames = new TableSelector(tinfoDomainDescriptor, new CsvSet("StorageTableName"), filter, null) |
| 731 | + .setJdbcCaching(false) |
| 732 | + .stream(String.class) |
| 733 | + .filter(dialect::isIdentifierTooLong) |
| 734 | + .toList(); |
| 735 | + |
| 736 | + if (!badTableNames.isEmpty()) |
| 737 | + LOG.error("Some storage table names are still too long!! {}", badTableNames); |
| 738 | + } |
| 739 | + |
| 740 | + TableInfo tinfoPropertyDomain = OntologyManager.getTinfoPropertyDomain(); |
| 741 | + TableInfo tinfoPropertyDescriptor = OntologyManager.getTinfoPropertyDescriptor(); |
| 742 | + SQLFragment sql = new SQLFragment("SELECT dd.DomainId, dd.Name AS DomainName, px.PropertyId, StorageSchemaName, StorageTableName, StorageColumnName, px.Name FROM ") |
| 743 | + .append(tinfoDomainDescriptor, "dd") |
| 744 | + .append(" INNER JOIN ") |
| 745 | + .append(tinfoPropertyDomain, "pd") |
| 746 | + .append(" ON dd.DomainId = pd.DomainId INNER JOIN ") |
| 747 | + .append(tinfoPropertyDescriptor, "px") |
| 748 | + .append(" ON pd.PropertyId = px.PropertyId ") |
| 749 | + .append("WHERE StorageSchemaName IS NOT NULL AND StorageTableName IS NOT NULL AND StorageColumnName IS NOT NULL"); |
| 750 | + |
| 751 | + filter = new SimpleFilter(FieldKey.fromString("StorageSchemaName"), null, CompareType.NONBLANK); |
| 752 | + filter.addCondition(FieldKey.fromString("StorageTableName"), null, CompareType.NONBLANK); |
| 753 | + filter.addCondition(FieldKey.fromString("StorageColumnName"), null, CompareType.NONBLANK); |
| 754 | + MultiValuedMap<DomainRecord, Property> badDomainMap = new SqlSelector(scope, sql) |
| 755 | + .setJdbcCaching(false) |
| 756 | + .stream(Property.class) |
| 757 | + .filter(property -> dialect.isIdentifierTooLong(property.storageColumnName())) |
| 758 | + .collect(LabKeyCollectors.toMultiValuedMap( |
| 759 | + property -> new DomainRecord(null, property.domainId(), property.domainName(), property.storageSchemaName(), property.storageTableName()), |
| 760 | + property -> property, |
| 761 | + ArrayListValuedHashMap::new) |
| 762 | + ); |
| 763 | + |
| 764 | + LOG.info(" Found {} with storage column names that are too long for PostgreSQL:", StringUtilsLabKey.pluralize(badDomainMap.keySet().size(), "domain")); |
| 765 | + |
| 766 | + try (DbScope.Transaction t = scope.beginTransaction()) |
| 767 | + { |
| 768 | + // Now enumerate the bad domains and rename their bad storage columns |
| 769 | + badDomainMap.keySet() |
| 770 | + .forEach(domain -> { |
| 771 | + Collection<Property> badColumns = badDomainMap.get(domain); |
| 772 | + List<String> badColumnNames = badColumns.stream().map(Property::storageColumnName).toList(); |
| 773 | + |
| 774 | + // First, populate a new StorageNameGenerator with all the "good" names in this domain so we don't |
| 775 | + // accidentally try to re-use one of them |
| 776 | + StorageNameGenerator nameGenerator = new StorageNameGenerator(dialect); |
| 777 | + SQLFragment domainSql = new SQLFragment("SELECT StorageColumnName FROM ") |
| 778 | + .append(tinfoPropertyDomain, "pd") |
| 779 | + .append(" INNER JOIN ") |
| 780 | + .append(tinfoPropertyDescriptor, "px") |
| 781 | + .append(" ON pd.PropertyId = px.PropertyId ") |
| 782 | + .append("WHERE DomainId = ? AND StorageColumnName NOT ") |
| 783 | + .add(domain.domainId()) |
| 784 | + .appendInClause(badColumnNames, scope.getSqlDialect()); |
| 785 | + new SqlSelector(scope, domainSql).forEach(String.class, nameGenerator::claimName); |
| 786 | + |
| 787 | + LOG.info(" Renaming {} in table \"{}\"", StringUtilsLabKey.pluralize(badColumns.size(), "column"), domain.fullName()); |
| 788 | + |
| 789 | + // Now use that StorageNameGenerator to create new names. Rename the column and update the PropertyDescriptor table. |
| 790 | + badColumns.forEach(property -> { |
| 791 | + String oldName = property.fullName(); |
| 792 | + String newName = bracketIt(nameGenerator.generateColumnName(property.name())); // Could have special characters, so bracket it |
| 793 | + |
| 794 | + executor.execute(new SQLFragment("EXEC sp_rename ?, ?, 'COLUMN'").add(oldName).add(newName)); |
| 795 | + Table.update(null, tinfoPropertyDescriptor, PageFlowUtil.map("StorageColumnName", newName), property.propertyId()); |
| 796 | + |
| 797 | + LOG.info(" Column \"{}\" renamed to \"{}\" ({} bytes)", oldName, newName, newName.getBytes(StandardCharsets.UTF_8).length); |
| 798 | + }); |
| 799 | + }); |
| 800 | + |
| 801 | + List<String> badColumnNames = new TableSelector(tinfoPropertyDescriptor, new CsvSet("StorageColumnName"), new SimpleFilter(FieldKey.fromString("StorageColumnName"), null, CompareType.NONBLANK), null) |
| 802 | + .setJdbcCaching(false) |
| 803 | + .stream(String.class) |
| 804 | + .filter(dialect::isIdentifierTooLong) |
| 805 | + .toList(); |
| 806 | + |
| 807 | + if (!badColumnNames.isEmpty()) |
| 808 | + LOG.error("Some storage column names are still too long!! {}", badColumnNames); |
| 809 | + } |
| 810 | + } |
| 811 | + |
| 812 | + // Bracket name and escape any internal ending brackets |
| 813 | + static String bracketIt(String name) |
| 814 | + { |
| 815 | + return "[" + name.replace("]", "]]") + "]"; |
| 816 | + } |
660 | 817 | } |
0 commit comments