From 9cb8909489c67b79cc5f56744fdb644a710f8535 Mon Sep 17 00:00:00 2001 From: Adam Rauch Date: Thu, 18 Dec 2025 15:25:54 -0800 Subject: [PATCH] Remove unescape experimental feature --- .../labkey/api/exp/api/ExperimentService.java | 1 - .../labkey/api/qc/TsvDataExchangeHandler.java | 18 ++-- api/src/org/labkey/api/reader/TabLoader.java | 85 +------------------ core/src/org/labkey/core/CoreModule.java | 5 -- 4 files changed, 9 insertions(+), 100 deletions(-) diff --git a/api/src/org/labkey/api/exp/api/ExperimentService.java b/api/src/org/labkey/api/exp/api/ExperimentService.java index c7321129920..2a3ecc53443 100644 --- a/api/src/org/labkey/api/exp/api/ExperimentService.java +++ b/api/src/org/labkey/api/exp/api/ExperimentService.java @@ -526,7 +526,6 @@ static boolean isAliquotedFromColumn(String columnName) try (TabLoader tabLoader = new TabLoader(valueStr)) { tabLoader.setDelimiterCharacter(','); - tabLoader.setUnescapeBackslashes(false); // Issue 50924: LKSM: Importing samples using naming expression referencing parent inputs with # result in error tabLoader.setIncludeComments(true); // Issue 51056 Samples with single double quotes in the name will not resolve if added as parent samples. diff --git a/api/src/org/labkey/api/qc/TsvDataExchangeHandler.java b/api/src/org/labkey/api/qc/TsvDataExchangeHandler.java index a4d1880d417..9fff1b0e1f0 100644 --- a/api/src/org/labkey/api/qc/TsvDataExchangeHandler.java +++ b/api/src/org/labkey/api/qc/TsvDataExchangeHandler.java @@ -513,12 +513,10 @@ private RunInfo processWarningsFile(FileLike runInfoFile) throws ValidationExcep { try (TabLoader loader = new TabLoader(runInfoFile.toNioPathForRead().toFile(), false)) { - // Don't unescape file path names on windows (C:\foo\bar.tsv) - loader.setUnescapeBackslashes(false); loader.setColumns(new ColumnDescriptor[]{ - new ColumnDescriptor("name", String.class), - new ColumnDescriptor("value", String.class), - new ColumnDescriptor("type", String.class) + new ColumnDescriptor("name", String.class), + new ColumnDescriptor("value", String.class), + new ColumnDescriptor("type", String.class) }); for (Map row : loader) @@ -790,13 +788,11 @@ private List> parseRunInfo(File runInfo) { try (TabLoader loader = new TabLoader(runInfo, false)) { - // Don't unescape file path names on windows (C:\foo\bar.tsv) - loader.setUnescapeBackslashes(false); loader.setColumns(new ColumnDescriptor[]{ - new ColumnDescriptor("name", String.class), - new ColumnDescriptor("value", String.class), - new ColumnDescriptor("type", String.class), - new ColumnDescriptor("transformedData", String.class) + new ColumnDescriptor("name", String.class), + new ColumnDescriptor("value", String.class), + new ColumnDescriptor("type", String.class), + new ColumnDescriptor("transformedData", String.class) }); return loader.load(); } diff --git a/api/src/org/labkey/api/reader/TabLoader.java b/api/src/org/labkey/api/reader/TabLoader.java index f827106f888..865abbf6091 100644 --- a/api/src/org/labkey/api/reader/TabLoader.java +++ b/api/src/org/labkey/api/reader/TabLoader.java @@ -20,7 +20,6 @@ import org.apache.commons.io.input.CharSequenceReader; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.text.StringEscapeUtils; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -30,7 +29,6 @@ import org.labkey.api.dataiterator.HashDataIterator; import org.labkey.api.iterator.BeanIterator; import org.labkey.api.iterator.CloseableIterator; -import org.labkey.api.settings.AppProps; import org.labkey.api.util.FileType; import org.labkey.api.util.FileUtil; import org.labkey.api.util.JunitUtil; @@ -68,8 +66,6 @@ */ public class TabLoader extends DataLoader { - public static final String FEATUREFLAG_UNESCAPE_BACKSLASH = "dataloader-unescape-backslashes"; - public static final FileType TSV_FILE_TYPE = new TabFileType(Arrays.asList(".tsv", ".txt"), ".tsv", "text/tab-separated-values"); public static final FileType CSV_FILE_TYPE = new TabFileType(Collections.singletonList(".csv"), ".csv", "text/comma-separated-values"); @@ -138,7 +134,6 @@ private TabLoader configParsing(TabLoader loader) { loader.setInferTypes(false); // Issue 43661 - Excessive logging when indexing a .log file containing backslash followed by "u" that confuses TabLoader - loader.setUnescapeBackslashes(false); return loader; } @@ -160,7 +155,6 @@ public static class MysqlFactory extends AbstractDataLoaderFactory public DataLoader createLoader(File file, boolean hasColumnHeaders, Container mvIndicatorContainer) { TabLoader loader = new TabLoader(file, hasColumnHeaders, mvIndicatorContainer); - loader.setUnescapeBackslashes(false); loader.setDelimiters(fieldTerminator, lineTerminator); loader.setParseQuotes(false); return loader; @@ -170,7 +164,6 @@ public DataLoader createLoader(File file, boolean hasColumnHeaders, Container mv public DataLoader createLoader(InputStream is, boolean hasColumnHeaders, Container mvIndicatorContainer) throws IOException { TabLoader loader = new TabLoader(Readers.getBOMDetectingReader(is), hasColumnHeaders, mvIndicatorContainer); - loader.setUnescapeBackslashes(false); loader.setDelimiters(fieldTerminator,lineTerminator); loader.setParseQuotes(false); return loader; @@ -203,7 +196,6 @@ public FileType getFileType() private boolean _parseQuotes = true; private boolean _parseEnclosedQuotes = false; // only treat quote as quote if it comes in pairs, otherwise treat it as a regular character - private boolean _unescapeBackslashes = AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_UNESCAPE_BACKSLASH); // Infer whether there are headers public TabLoader(File inputFile) @@ -338,26 +330,6 @@ protected String parseValue(String value) value = StringUtils.trimToEmpty(value); if ("\\N".equals(value)) return _preserveEmptyString ? null : ""; - if (_unescapeBackslashes && value.indexOf('\\') >= 0) // unescapeJava() is really slow - { - try - { - return StringEscapeUtils.unescapeJava(value); - } - catch (IllegalArgumentException e) - { - // Issue 16691: OctalUnescaper or UnicodeUnescaper translators will throw NumberFormatException for illegal sequences such as '\' followed by octal '9' or unicode 'zzzz'. - // StringEscapeUtils can also throw IllegalArgumentException - String msg = "Error reading data. Can't unescape value '" + value + "'. "; - if (e instanceof NumberFormatException) - msg += "Number format error "; - msg += e.getMessage(); - if (isThrowOnErrors()) - throw new IllegalArgumentException(msg, e); - else - _log.warn(msg); - } - } return value; } @@ -591,11 +563,6 @@ public void setParseEnclosedQuotes(boolean parseEnclosedQuotes) _parseEnclosedQuotes = parseEnclosedQuotes; } - public void setUnescapeBackslashes(boolean unescapeBackslashes) - { - _unescapeBackslashes = unescapeBackslashes; - } - @Override public void close() { @@ -1093,27 +1060,9 @@ public void testUnescape() \tthis\\nis\\tmulti-line\tb """; + // test no-unescaping try (TabLoader loader = new TabLoader(data, true)) { - loader.setUnescapeBackslashes(true); - List> rows = loader.load(); - assertEquals(2, rows.size()); - - Map row = rows.get(0); - assertEquals("a", row.get("A")); - assertEquals("this\nis\tmulti-line", row.get("Multi-Line")); - assertEquals("b", row.get("B")); - - row = rows.get(1); - assertNull(row.get("A")); - assertEquals("this\nis\tmulti-line", row.get("Multi-Line")); - assertEquals("b", row.get("B")); - } - - // now test no-unescaping - try (TabLoader loader = new TabLoader(data, true)) - { - loader.setUnescapeBackslashes(false); List> rows = loader.load(); assertEquals(2, rows.size()); @@ -1196,40 +1145,10 @@ public void testParseQuotes() Fred\t"quoted stuff" unquoted\t1"""; data = data + "\nAlice\t\"\"\"quoted stuff\"\" unquoted"; + // test no-unescaping try (TabLoader loader = new TabLoader(data, true)) { loader.setParseQuotes(true); - loader.setUnescapeBackslashes(true); - - List> rows = loader.load(); - assertEquals(6, rows.size()); - - Map row = rows.get(0); - assertEquals("Bob", row.get("Name")); - assertEquals("with\ttab\nwith\"quote", row.get("Multi-Line")); - assertEquals(10, row.get("Age")); - - row = rows.get(1); - assertEquals("Bob", row.get("Name")); - assertEquals("apple \norange\tgrape", row.get("Multi-Line")); - assertEquals(3, row.get("Age")); - - row = rows.get(2); - assertEquals("Bob", row.get("Name")); - assertEquals("one\n\"two\" \tthree", row.get("Multi-Line")); - assertNull(row.get("Age")); - - row = rows.get(3); - assertNull(row.get("Name")); - assertEquals("red\nblue\tgreen", row.get("Multi-Line")); - assertEquals(4, row.get("Age")); - } - - // now test no-unescaping - try (TabLoader loader = new TabLoader(data, true)) - { - loader.setParseQuotes(true); - loader.setUnescapeBackslashes(false); List> rows = loader.load(); assertEquals(6, rows.size()); diff --git a/core/src/org/labkey/core/CoreModule.java b/core/src/org/labkey/core/CoreModule.java index ba52b1e2989..ee5d38b3ba9 100644 --- a/core/src/org/labkey/core/CoreModule.java +++ b/core/src/org/labkey/core/CoreModule.java @@ -535,11 +535,6 @@ public QuerySchema createSchema(DefaultSchema schema, Module module) "Short-circuit robots", "Save resources by not rendering pages marked as 'noindex' for robots. This is experimental as not all robots are search engines.", false); - OptionalFeatureService.get().addFeatureFlag(new OptionalFeatureFlag(TabLoader.FEATUREFLAG_UNESCAPE_BACKSLASH, - "Unescape backslash character on import", - "Treat backslash '\\' character as an escape character when loading data from file. This option will be removed in LabKey Server v26.3.", - false, false, OptionalFeatureService.FeatureType.Deprecated - )); OptionalFeatureService.get().addFeatureFlag(new OptionalFeatureFlag(AppProps.GENERATE_CONTROLLER_FIRST_URLS, "Restore controller-first URLs", "Generate URLs in a legacy format that puts the controller name before the folder path. This option will be removed in LabKey Server 26.3.",