From d3bb905c29609b31f9464df5ced4cf9c64214348 Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 30 Jun 2025 12:33:00 -0700 Subject: [PATCH 01/29] Add developer link to ShowUtilityActions --- discvrcore/src/org/labkey/discvrcore/DiscvrCoreModule.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/discvrcore/src/org/labkey/discvrcore/DiscvrCoreModule.java b/discvrcore/src/org/labkey/discvrcore/DiscvrCoreModule.java index a99627487..152241120 100644 --- a/discvrcore/src/org/labkey/discvrcore/DiscvrCoreModule.java +++ b/discvrcore/src/org/labkey/discvrcore/DiscvrCoreModule.java @@ -24,6 +24,9 @@ import org.labkey.api.query.DetailsURL; import org.labkey.api.settings.AdminConsole; import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.view.DeveloperMenuNavTrees; +import org.labkey.api.view.NavTree; +import org.labkey.api.view.PopupDeveloperView; import org.labkey.api.view.WebPartFactory; import java.util.Collection; @@ -70,6 +73,9 @@ protected void init() public void doStartup(ModuleContext moduleContext) { AdminConsole.addLink(AdminConsole.SettingsLinkType.Management, "site utility actions", DetailsURL.fromString("discvrcore/showUtilityActions.view", ContainerManager.getRoot()).getActionURL()); + PopupDeveloperView.registerMenuProvider((c, user, trees) -> { + trees.add(DeveloperMenuNavTrees.Section.tools, new NavTree("Show Utility Actions", DetailsURL.fromString("discvrcore/showUtilityActions.view", c).getActionURL())); + }); } @Override From e39489aa8cb12378cfa2cdda88dce14e2b0520dc Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 1 Jul 2025 22:03:43 -0700 Subject: [PATCH 02/29] Use remote API to load JBrowse search results (#339) * Use remote API to load JBrowse search results --- jbrowse/package-lock.json | 255 ++++++++++---- .../external/labModules/JBrowseTest.java | 313 ++++-------------- 2 files changed, 253 insertions(+), 315 deletions(-) diff --git a/jbrowse/package-lock.json b/jbrowse/package-lock.json index a59c3c97c..b2b8218e0 100644 --- a/jbrowse/package-lock.json +++ b/jbrowse/package-lock.json @@ -3091,9 +3091,9 @@ } }, "node_modules/@labkey/api": { - "version": "1.41.2", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/api/-/@labkey/api-1.41.2.tgz", - "integrity": "sha512-ninfc/+Sj5+8Zla9bY2j/4fSy41OS27YAHKtDFPnu52QkC8WsOYh3JFI5PkU6Rn+xIp0In4P6d5Qn/yluJRC/w==" + "version": "1.42.0", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/api/-/@labkey/api-1.42.0.tgz", + "integrity": "sha512-fh65cogl+8HNe9+3OHzI6ygkaa+ARJbKyUopguy3NqtOWIAXAA21GNxayRoeVf9m1n2Kpubqk4QS2z/+WCzf8A==" }, "node_modules/@labkey/build": { "version": "8.5.0", @@ -3133,12 +3133,12 @@ } }, "node_modules/@labkey/components": { - "version": "6.50.1", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-6.50.1.tgz", - "integrity": "sha512-g6DDCg3rsoCzkJDTRAtUFdGskyufFipJB50mmgQhEgtyEEbaXB6rF/Ny8ytPCZOmpA5yL2+pEtqFoXlvyXBMDg==", + "version": "6.52.5", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-6.52.5.tgz", + "integrity": "sha512-hkesy0zyVEpADMfK4CQN9ZPWRjGFGfqS7h1QgRazp0QiHgXI3i6Va39lMSBdZb27bDeQd5V78Xze4tclHO/xIQ==", "dependencies": { "@hello-pangea/dnd": "18.0.1", - "@labkey/api": "1.41.2", + "@labkey/api": "1.42.0", "@testing-library/dom": "~10.4.0", "@testing-library/jest-dom": "~6.6.3", "@testing-library/react": "~16.3.0", @@ -5361,14 +5361,41 @@ } }, "node_modules/call-bind": { - "version": "1.0.7", - "license": "MIT", + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.8.tgz", + "integrity": "sha512-oKlSFMcMwpUg2ednkhQ454wfWiU/ul3CkJe/PEHcTKuiX6RpbehUiFMXu13HalGZxfUwCQzZG747YXBn1im9ww==", "dependencies": { + "call-bind-apply-helpers": "^1.0.0", "es-define-property": "^1.0.0", - "es-errors": "^1.3.0", - "function-bind": "^1.1.2", "get-intrinsic": "^1.2.4", - "set-function-length": "^1.2.1" + "set-function-length": "^1.2.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" }, "engines": { "node": ">= 0.4" @@ -6495,6 +6522,19 @@ "tslib": "^2.0.3" } }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/duplexer": { "version": "0.1.2", "dev": true, @@ -6619,11 +6659,9 @@ } }, "node_modules/es-define-property": { - "version": "1.0.0", - "license": "MIT", - "dependencies": { - "get-intrinsic": "^1.2.4" - }, + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", "engines": { "node": ">= 0.4" } @@ -6639,6 +6677,17 @@ "version": "1.5.4", "license": "MIT" }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/escalade": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", @@ -6949,10 +6998,17 @@ } }, "node_modules/for-each": { - "version": "0.3.3", - "license": "MIT", + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.5.tgz", + "integrity": "sha512-dKx12eRCVIzqCxFGplyFKJMPvLEWgmNtUrpTiJIR5u97zEhRG8ySrtboPHZXx7daLxQVrl643cTzbab2tkQjxg==", "dependencies": { - "is-callable": "^1.1.3" + "is-callable": "^1.2.7" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/foreground-child": { @@ -7252,14 +7308,20 @@ } }, "node_modules/get-intrinsic": { - "version": "1.2.4", - "license": "MIT", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", "function-bind": "^1.1.2", - "has-proto": "^1.0.1", - "has-symbols": "^1.0.3", - "hasown": "^2.0.0" + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" }, "engines": { "node": ">= 0.4" @@ -7268,6 +7330,18 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/get-value": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/get-value/-/get-value-3.0.1.tgz", @@ -7352,10 +7426,11 @@ } }, "node_modules/gopd": { - "version": "1.0.1", - "license": "MIT", - "dependencies": { - "get-intrinsic": "^1.1.3" + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "engines": { + "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -7403,19 +7478,10 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/has-proto": { - "version": "1.0.3", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/has-symbols": { - "version": "1.0.3", - "license": "MIT", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", "engines": { "node": ">= 0.4" }, @@ -7887,7 +7953,8 @@ }, "node_modules/is-callable": { "version": "1.2.7", - "license": "MIT", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", + "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==", "engines": { "node": ">= 0.4" }, @@ -8076,10 +8143,11 @@ } }, "node_modules/is-typed-array": { - "version": "1.1.13", - "license": "MIT", + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.15.tgz", + "integrity": "sha512-p3EcsicXjit7SaskXHs1hA91QxgTw46Fv6EFKKGS5DRFLD8yKnohjF3hxoju94b/OcMZoQukzpPpBE9uLVKzgQ==", "dependencies": { - "which-typed-array": "^1.1.14" + "which-typed-array": "^1.1.16" }, "engines": { "node": ">= 0.4" @@ -8411,6 +8479,14 @@ "react": "^16.8.0 || ^17.0.0 || ^18.0.0" } }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/md5": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz", @@ -9404,20 +9480,49 @@ } }, "node_modules/pbkdf2": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/pbkdf2/-/pbkdf2-3.1.2.tgz", - "integrity": "sha512-iuh7L6jA7JEGu2WxDwtQP1ddOpaJNC4KlDEFfdQajSGgGPNi4OyDc2R7QnbY2bR9QjBVGwgvTdNJZoE7RaxUMA==", + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/pbkdf2/-/pbkdf2-3.1.3.tgz", + "integrity": "sha512-wfRLBZ0feWRhCIkoMB6ete7czJcnNnqRpcoWQBLqatqXXmelSRqfdDK4F3u9T2s2cXas/hQJcryI/4lAL+XTlA==", "dependencies": { - "create-hash": "^1.1.2", - "create-hmac": "^1.1.4", - "ripemd160": "^2.0.1", - "safe-buffer": "^5.0.1", - "sha.js": "^2.4.8" + "create-hash": "~1.1.3", + "create-hmac": "^1.1.7", + "ripemd160": "=2.0.1", + "safe-buffer": "^5.2.1", + "sha.js": "^2.4.11", + "to-buffer": "^1.2.0" }, "engines": { "node": ">=0.12" } }, + "node_modules/pbkdf2/node_modules/create-hash": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/create-hash/-/create-hash-1.1.3.tgz", + "integrity": "sha512-snRpch/kwQhcdlnZKYanNF1m0RDlrCdSKQaH87w1FCFPVPNCQ/Il9QJKAX2jVBZddRdaHBMC+zXa9Gw9tmkNUA==", + "dependencies": { + "cipher-base": "^1.0.1", + "inherits": "^2.0.1", + "ripemd160": "^2.0.0", + "sha.js": "^2.4.0" + } + }, + "node_modules/pbkdf2/node_modules/hash-base": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hash-base/-/hash-base-2.0.2.tgz", + "integrity": "sha512-0TROgQ1/SxE6KmxWSvXHvRj90/Xo1JvZShofnYF+f6ZsGtR4eES7WfrQzPalmyagfKZCXpVnitiRebZulWsbiw==", + "dependencies": { + "inherits": "^2.0.1" + } + }, + "node_modules/pbkdf2/node_modules/ripemd160": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/ripemd160/-/ripemd160-2.0.1.tgz", + "integrity": "sha512-J7f4wutN8mdbV08MJnXibYpCOPHR+yzy+iQ/AsjMv2j8cLavQ8VGagDFUwwTAdF8FmRKVeNpbTTEwNHCW1g94w==", + "dependencies": { + "hash-base": "^2.0.0", + "inherits": "^2.0.1" + } + }, "node_modules/performance-now": { "version": "2.1.0", "license": "MIT", @@ -11555,6 +11660,24 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/to-buffer": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/to-buffer/-/to-buffer-1.2.1.tgz", + "integrity": "sha512-tB82LpAIWjhLYbqjx3X4zEeHN6M8CiuOEy2JY8SEQVdYRe3CCHOFaqrBW1doLDrfpWhplcW7BL+bO3/6S3pcDQ==", + "dependencies": { + "isarray": "^2.0.5", + "safe-buffer": "^5.2.1", + "typed-array-buffer": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/to-buffer/node_modules/isarray": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz", + "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==" + }, "node_modules/to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", @@ -11676,6 +11799,19 @@ "node": ">= 0.6" } }, + "node_modules/typed-array-buffer": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/typed-array-buffer/-/typed-array-buffer-1.0.3.tgz", + "integrity": "sha512-nAYYwfY3qnzX30IkA6AQZjVbtK6duGontcQm1WSG1MD94YLqK0515GNApXkoxKOWMusVssAHWLh9SeaoefYFGw==", + "dependencies": { + "call-bound": "^1.0.3", + "es-errors": "^1.3.0", + "is-typed-array": "^1.1.14" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/typescript": { "version": "5.8.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.3.tgz", @@ -12328,13 +12464,16 @@ } }, "node_modules/which-typed-array": { - "version": "1.1.15", - "license": "MIT", + "version": "1.1.19", + "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.19.tgz", + "integrity": "sha512-rEvr90Bck4WZt9HHFC4DJMsjvu7x+r6bImz0/BrbWb7A2djJ8hnZMrWnHo9F8ssv0OMErasDhftrfROTyqSDrw==", "dependencies": { "available-typed-arrays": "^1.0.7", - "call-bind": "^1.0.7", - "for-each": "^0.3.3", - "gopd": "^1.0.1", + "call-bind": "^1.0.8", + "call-bound": "^1.0.4", + "for-each": "^0.3.5", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", "has-tostringtag": "^1.0.2" }, "engines": { diff --git a/jbrowse/test/src/org/labkey/test/tests/external/labModules/JBrowseTest.java b/jbrowse/test/src/org/labkey/test/tests/external/labModules/JBrowseTest.java index 5f03b05a6..87dd973f6 100644 --- a/jbrowse/test/src/org/labkey/test/tests/external/labModules/JBrowseTest.java +++ b/jbrowse/test/src/org/labkey/test/tests/external/labModules/JBrowseTest.java @@ -17,6 +17,7 @@ import au.com.bytecode.opencsv.CSVReader; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.DurationFormatUtils; import org.apache.commons.lang3.tuple.Pair; import org.json.JSONArray; import org.json.JSONException; @@ -25,7 +26,9 @@ import org.junit.Test; import org.junit.experimental.categories.Category; import org.labkey.remoteapi.CommandException; +import org.labkey.remoteapi.CommandResponse; import org.labkey.remoteapi.Connection; +import org.labkey.remoteapi.SimplePostCommand; import org.labkey.remoteapi.query.Filter; import org.labkey.remoteapi.query.InsertRowsCommand; import org.labkey.remoteapi.query.SelectRowsCommand; @@ -53,6 +56,7 @@ import java.io.File; import java.io.FileReader; import java.io.IOException; +import java.util.Date; import java.util.List; import java.util.Map; @@ -542,6 +546,22 @@ private void createGenomeFeatures(int genomeId) throws IOException, CommandExcep ic.execute(cn, getProjectName()); } + private JSONArray getSearchResults(Map queryParams) throws CommandException, IOException + { + Connection connection = createDefaultConnection(); + + Date start = new Date(); + SimplePostCommand command = new SimplePostCommand("jbrowse", "luceneQuery"); + command.setParameters(queryParams); + command.setTimeout(WAIT_FOR_PAGE * 3); + CommandResponse response = command.execute(connection, getProjectName()); + log("JBrowse search time: " + DurationFormatUtils.formatDurationWords(new Date().getTime() - start.getTime(), true, true)); + log(response.getText()); + + // NOTE: the response is ndjson. This converts it into a more standard JSONArray form: + return new JSONArray("[" + StringUtils.join(response.getText().split("\n"), ",") + "]"); + } + @Override public List getAssociatedModules() { @@ -573,24 +593,12 @@ private void testFullTextSearch() throws Exception // all // this should return 143 results. We can't make any other assumptions about the content - String url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=all&pageSize=143"; - beginAt(url, WAIT_FOR_PAGE * 2); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - String jsonString = getText(Locator.tagWithClass("pre", "data")); - JSONObject mainJsonObject = new JSONObject(jsonString); - JSONArray jsonArray = mainJsonObject.getJSONArray("data"); + JSONArray jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "all", "pageSize", 143)); Assert.assertEquals(143, jsonArray.length()); // stringType: // ref equals A - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=ref%3AA"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "ref:A")); Assert.assertEquals(100, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -598,13 +606,7 @@ private void testFullTextSearch() throws Exception } // alt does not equal C - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=*%3A*%20-alt%3AC"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "*:* -alt:C")); Assert.assertEquals(100, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -612,13 +614,7 @@ private void testFullTextSearch() throws Exception } // ref contains A - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=ref%3A*A*"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "ref:*A*")); Assert.assertEquals(100, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -626,13 +622,7 @@ private void testFullTextSearch() throws Exception } // alt does not contain AA - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=*%3A*%20-alt%3A*AA*"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "*:* -alt:*AA*")); Assert.assertEquals(100, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -640,13 +630,7 @@ private void testFullTextSearch() throws Exception } // IMPACT starts with HI - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=IMPACT%3AHI*"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "IMPACT:HI*")); Assert.assertEquals(1, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -654,13 +638,7 @@ private void testFullTextSearch() throws Exception } // ref ends with TA - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=ref%3A*TA"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "ref:*TA")); Assert.assertEquals(5, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -668,13 +646,7 @@ private void testFullTextSearch() throws Exception } // IMPACT is empty - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=*%3A*%20-IMPACT%3A*"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "*:* -IMPACT:*")); Assert.assertEquals(100, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -682,13 +654,7 @@ private void testFullTextSearch() throws Exception } // IMPACT is not empty - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=IMPACT%3A*"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "IMPACT:*")); Assert.assertEquals(3, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -696,23 +662,11 @@ private void testFullTextSearch() throws Exception } // variableSamplesType in set TestGroup - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=variableSamples%3A~!TestGroup!~"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "variableSamples:~!TestGroup!~")); Assert.assertEquals(100, jsonArray.length()); // variable in m00004 - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=variableSamples%3Am00004"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "variableSamples:m00004")); Assert.assertEquals(79, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { @@ -731,13 +685,7 @@ private void testFullTextSearch() throws Exception } // not variable in m00004 - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=*%3A*%20-variableSamples%3Am00004"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "*:* -variableSamples:m00004")); Assert.assertEquals(100, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { @@ -767,13 +715,7 @@ private void testFullTextSearch() throws Exception } // variable in all of m00004, m00013, m00029 - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=%252BvariableSamples%3Am00004%20%252BvariableSamples%3Am00013%20%252BvariableSamples%3Am00029"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "%2BvariableSamples:m00004 %2BvariableSamples:m00013 %2BvariableSamples:m00029")); Assert.assertEquals(69, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { @@ -801,13 +743,7 @@ private void testFullTextSearch() throws Exception } // variable in any of m00004, m00013, m00029 - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=variableSamples%3Am00004%20OR%20variableSamples%3Am00013%20OR%20variableSamples%3Am00029"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "variableSamples:m00004 OR variableSamples:m00013 OR variableSamples:m00029")); Assert.assertEquals(100, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { @@ -833,13 +769,7 @@ private void testFullTextSearch() throws Exception } // not variable in any of m00004, m00013, m00029 - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=*%3A*%20-variableSamples%3Am00004%20AND%20*%3A*%20-variableSamples%3Am00013%20AND%20*%3A*%20-variableSamples%3Am00029"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "*:* -variableSamples:m00004 AND *:* -variableSamples:m00013 AND *:* -variableSamples:m00029")); Assert.assertEquals(100, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { @@ -885,13 +815,7 @@ private void testFullTextSearch() throws Exception } // not variable in one of m00004, m00013, m00029 - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=*%3A*%20-variableSamples%3Am00004%20OR%20*%3A*%20-variableSamples%3Am00013%20OR%20*%3A*%20-variableSamples%3Am00029"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "*:* -variableSamples:m00004 OR *:* -variableSamples:m00013 OR *:* -variableSamples:m00029")); Assert.assertEquals(100, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { @@ -935,13 +859,7 @@ private void testFullTextSearch() throws Exception } // is empty - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=*%3A*%20-variableSamples%3A*"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "*:* -variableSamples:*")); Assert.assertEquals(5, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -949,13 +867,7 @@ private void testFullTextSearch() throws Exception } // is not empty - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=variableSamples%3A*"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "variableSamples:*")); Assert.assertEquals(100, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -965,13 +877,7 @@ private void testFullTextSearch() throws Exception // numericType, int and float: // AC = 12 - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=AC%3A%5B12%20TO%2012%5D"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "AC:[12 TO 12]")); Assert.assertEquals(3, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -979,13 +885,7 @@ private void testFullTextSearch() throws Exception } // AC != 88 - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=AC%3A%5B*%20TO%2088%7D%20OR%20AC%3A%7B88%20TO%20*%5D"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "AC:[* TO 88} OR AC:{88 TO *]")); Assert.assertEquals(100, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -993,13 +893,7 @@ private void testFullTextSearch() throws Exception } // AC > 88 - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=AC%3A%7B88%20TO%20*%5D"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "AC:{88 TO *]")); Assert.assertEquals(100, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -1007,13 +901,7 @@ private void testFullTextSearch() throws Exception } // AC >= 88 - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=AC%3A%5B88%20TO%20*%5D"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "AC:[88 TO *]")); Assert.assertEquals(100, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -1021,13 +909,7 @@ private void testFullTextSearch() throws Exception } // start < 137 - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=start%3A%5B*%20TO%20137%7D"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "start:[* TO 137}")); Assert.assertEquals(2, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -1035,13 +917,7 @@ private void testFullTextSearch() throws Exception } // end <= 440 - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=end%3A%5B*%20TO%20440%5D"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "end:[* TO 440]")); Assert.assertEquals(7, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -1049,13 +925,7 @@ private void testFullTextSearch() throws Exception } // AF = 0.532 - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=AF%3A%5B0.531999%20TO%200.5320010000000001%5D"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "AF:[0.531999 TO 0.5320010000000001]")); Assert.assertEquals(1, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -1063,13 +933,7 @@ private void testFullTextSearch() throws Exception } // AF != 0.029 - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=AF%3A%5B*%20TO%200.028999%5D%20OR%20AF%3A%5B0.029001000000000002%20TO%20*%5D"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "AF:[* TO 0.028999] OR AF:[0.029001000000000002 TO *]")); Assert.assertEquals(100, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -1077,13 +941,7 @@ private void testFullTextSearch() throws Exception } // AF > 0.532 - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=AF%3A%5B0.5320010000000001%20TO%20*%5D"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "AF:[0.5320010000000001 TO *]")); Assert.assertEquals(18, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -1091,13 +949,7 @@ private void testFullTextSearch() throws Exception } // AF >= 0.029 - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=AF%3A%5B0.029%20TO%20*%5D"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "AF:[0.029 TO *]")); Assert.assertEquals(100, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -1105,13 +957,7 @@ private void testFullTextSearch() throws Exception } // AF < 0.029 - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=AF%3A%5B*%20TO%200.028999%5D"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "AF:[* TO 0.028999]")); Assert.assertEquals(100, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -1119,13 +965,7 @@ private void testFullTextSearch() throws Exception } // AF <= 0.029 - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=AF%3A%5B*%20TO%200.029%5D"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "AF:[* TO 0.029]")); Assert.assertEquals(100, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -1137,13 +977,7 @@ private void testFullTextSearch() throws Exception // contig := 1 // ref := A // should be 100 results and each should be ref = A - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=contig%3A%3D1%26ref%3A%3DA&pageSize=200"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "contig:=1&ref:=A", "pageSize", 200)); Assert.assertEquals(104, jsonArray.length()); for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -1152,14 +986,7 @@ private void testFullTextSearch() throws Exception } // Default genomic position sort (ascending) - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=all&pageSize=100"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); - + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "all", "pageSize", 100)); long previousGenomicPosition = Long.MIN_VALUE; for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -1169,14 +996,7 @@ private void testFullTextSearch() throws Exception } // Sort by alt, ascending - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=all&pageSize=100&sortField=alt"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); - + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "all", "pageSize", 100, "sortField", "alt")); String previousAlt = ""; for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -1186,14 +1006,7 @@ private void testFullTextSearch() throws Exception } // Sort by alt, descending - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=all&pageSize=100&sortField=alt&sortReverse=true"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); - + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "all", "pageSize", 100, "sortField", "alt", "sortReverse", "true")); previousAlt = "ZZZZ"; // Assuming 'Z' is higher than any character in your data for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -1203,14 +1016,7 @@ private void testFullTextSearch() throws Exception } // Sort by af, ascending - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=all&pageSize=100&sortField=AF"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); - + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "all", "pageSize", 100, "sortField", "AF")); double previousAf = -1.0; for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); @@ -1220,14 +1026,7 @@ private void testFullTextSearch() throws Exception } // Sort by af, descending - url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=all&pageSize=100&sortField=AF&sortReverse=true"; - beginAt(url); - waitForText("data"); - waitAndClick(Locator.tagWithId("a", "rawdata-tab")); - jsonString = getText(Locator.tagWithClass("pre", "data")); - mainJsonObject = new JSONObject(jsonString); - jsonArray = mainJsonObject.getJSONArray("data"); - + jsonArray = getSearchResults(Map.of("sessionId", sessionId, "trackId", trackId, "searchString", "all", "pageSize", 100, "sortField", "AF", "sortReverse", "true")); previousAf = 2.0; // Assuming 'af' is <= 1.0 for (int i = 0; i < jsonArray.length(); i++) { JSONObject jsonObject = jsonArray.getJSONObject(i); From 63a447438eab88682a9dd9be6a46badad838fdba Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 2 Jul 2025 15:53:54 -0700 Subject: [PATCH 03/29] Allow subset to use dplyr --- .../pipeline/singlecell/SubsetSeurat.java | 33 ++++++++++++------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/SubsetSeurat.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/SubsetSeurat.java index 13ae96797..9ec09710e 100644 --- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/SubsetSeurat.java +++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/SubsetSeurat.java @@ -9,7 +9,6 @@ import org.labkey.api.singlecell.pipeline.SingleCellStep; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.HashSet; import java.util.List; @@ -34,7 +33,8 @@ public Provider() put("height", 150); put("width", 600); put("delimiter", DELIM); - }}, null) + }}, null), + ToolParameterDescriptor.create("useDplyr", "Use dplyr", "If checked, the subset will be executed using dplyr::filter rather than Seurat::subset. This should allow more complex expressions to be used, including negations", "checkbox", null, false) ), List.of("/sequenceanalysis/field/TrimmingTextArea.js"), null); } @@ -71,6 +71,9 @@ protected List loadChunkFromFile() throws PipelineJobException final String[] values = val.split(DELIM); + ToolParameterDescriptor pd2 = getProvider().getParameterByName("useDplyr"); + final boolean useDplyr = pd2.extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Boolean.class, false); + List ret = new ArrayList<>(); for (String line : super.loadChunkFromFile()) { @@ -82,15 +85,23 @@ protected List loadChunkFromFile() throws PipelineJobException ret.add("\tif (!is.null(seuratObj)) {"); ret.add("\tprint(paste0('Subsetting dataset: ', datasetId, ' with the expression: " + subsetEscaped + "'))"); - ret.add("\t\tcells <- c()"); - ret.add("\t\ttryCatch({"); - ret.add("\t\t\tcells <- WhichCells(seuratObj, expression = " + subset + ")"); - ret.add("\t\t}, error = function(e){"); - ret.add("\t\t\tif (!is.null(e) && e$message == 'Cannot find cells provided') {"); - ret.add("\t\t\t\tprint(paste0('There were no cells remaining after the subset: ', '" + subsetEscaped + "'))"); - ret.add("\t\t\t}"); - ret.add("\t\t})"); - ret.add(""); + if (useDplyr) + { + ret.add("\t\t\tcells <- rownames(seuratObj@meta.data %>% dplyr::filter( " + subset + " ))"); + } + else + { + ret.add("\t\tcells <- c()"); + ret.add("\t\ttryCatch({"); + ret.add("\t\t\tcells <- WhichCells(seuratObj, expression = " + subset + ")"); + ret.add("\t\t}, error = function(e){"); + ret.add("\t\t\tif (!is.null(e) && e$message == 'Cannot find cells provided') {"); + ret.add("\t\t\t\tprint(paste0('There were no cells remaining after the subset: ', '" + subsetEscaped + "'))"); + ret.add("\t\t\t}"); + ret.add("\t\t})"); + ret.add(""); + } + ret.add("\t\tif (length(cells) == 0) {"); ret.add("\t\t\tprint(paste0('There were no cells after subsetting for dataset: ', datasetId, ', with subset: ', '" + subsetEscaped + "'))"); ret.add("\t\t\tseuratObj <- NULL"); From 5f7d8c32c1cfd665956eaea410466c84f99bf638 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 2 Jul 2025 16:26:53 -0700 Subject: [PATCH 04/29] Bugfix to vireo when using donor file --- .../pipeline/singlecell/VireoHandler.java | 70 ++++++++++++++----- 1 file changed, 54 insertions(+), 16 deletions(-) diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java index eb7ae055d..28da76fae 100644 --- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java +++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java @@ -12,6 +12,7 @@ import org.labkey.api.sequenceanalysis.SequenceAnalysisService; import org.labkey.api.sequenceanalysis.SequenceOutputFile; import org.labkey.api.sequenceanalysis.pipeline.AbstractParameterizedOutputHandler; +import org.labkey.api.sequenceanalysis.pipeline.BcftoolsRunner; import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenome; import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport; import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputHandler; @@ -256,6 +257,49 @@ public void processFilesRemote(List inputFiles, JobContext c } } + File cellSnpBaseVcf = new File(cellsnpDir, "cellSNP.base.vcf.gz"); + if (!cellSnpBaseVcf.exists()) + { + throw new PipelineJobException("Unable to find cellsnp base VCF"); + } + + + File cellSnpCellsVcf = new File(cellsnpDir, "cellSNP.cells.vcf.gz"); + if (!cellSnpCellsVcf.exists()) + { + throw new PipelineJobException("Unable to find cellsnp calls VCF"); + } + + sortAndFixVcf(cellSnpBaseVcf, genome, ctx.getLogger()); + sortAndFixVcf(cellSnpCellsVcf, genome, ctx.getLogger()); + + int vcfFile = ctx.getParams().optInt(REF_VCF, -1); + File refVcfSubset = null; + if (vcfFile > -1) + { + File vcf = ctx.getSequenceSupport().getCachedData(vcfFile); + if (vcf == null || !vcf.exists()) + { + throw new PipelineJobException("Unable to find file with ID: " + vcfFile); + } + + refVcfSubset = new File(ctx.getWorkingDirectory(), vcf.getName()); + BcftoolsRunner bcftoolsRunner = new BcftoolsRunner(ctx.getLogger()); + bcftoolsRunner.execute(Arrays.asList( + BcftoolsRunner.getBcfToolsPath().getAbsolutePath(), + "view", + vcf.getPath(), + "-R", + cellSnpCellsVcf.getPath(), + "-Oz", + "-o", + refVcfSubset.getPath() + )); + + ctx.getFileManager().addIntermediateFile(refVcfSubset); + ctx.getFileManager().addIntermediateFile(new File(refVcfSubset.getPath() + ".tbi")); + } + List vireo = new ArrayList<>(); vireo.add("vireo"); vireo.add("-c"); @@ -277,6 +321,12 @@ public void processFilesRemote(List inputFiles, JobContext c throw new PipelineJobException("Must provide nDonors"); } + if (refVcfSubset != null) + { + vireo.add("-d"); + vireo.add(refVcfSubset.getPath()); + } + vireo.add("-N"); vireo.add(String.valueOf(nDonors)); @@ -312,25 +362,13 @@ else if (outFiles.length > 1) so.setName(inputFiles.get(0).getName() + ": Vireo Demultiplexing"); } so.setCategory("Vireo Demultiplexing"); + if (vcfFile > -1) + { + so.setDescription("Reference VCF ID: " + vcfFile); + } ctx.addSequenceOutput(so); } - File cellSnpBaseVcf = new File(cellsnpDir, "cellSNP.base.vcf.gz"); - if (!cellSnpBaseVcf.exists()) - { - throw new PipelineJobException("Unable to find cellsnp base VCF"); - } - - - File cellSnpCellsVcf = new File(cellsnpDir, "cellSNP.cells.vcf.gz"); - if (!cellSnpCellsVcf.exists()) - { - throw new PipelineJobException("Unable to find cellsnp calls VCF"); - } - - sortAndFixVcf(cellSnpBaseVcf, genome, ctx.getLogger()); - sortAndFixVcf(cellSnpCellsVcf, genome, ctx.getLogger()); - if (storeCellSnpVcf) { SequenceOutputFile so = new SequenceOutputFile(); From c83a8e94e35cc2dc256fad0cf70dedab312c676b Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 2 Jul 2025 16:41:53 -0700 Subject: [PATCH 05/29] Avoid dplyr loading --- .../org/labkey/singlecell/pipeline/singlecell/SubsetSeurat.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/SubsetSeurat.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/SubsetSeurat.java index 9ec09710e..7a646eb38 100644 --- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/SubsetSeurat.java +++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/SubsetSeurat.java @@ -87,7 +87,7 @@ protected List loadChunkFromFile() throws PipelineJobException ret.add("\tprint(paste0('Subsetting dataset: ', datasetId, ' with the expression: " + subsetEscaped + "'))"); if (useDplyr) { - ret.add("\t\t\tcells <- rownames(seuratObj@meta.data %>% dplyr::filter( " + subset + " ))"); + ret.add("\t\t\tcells <- rownames(seuratObj@meta.data |> dplyr::filter( " + subset + " ))"); } else { From 16e9682b42f78ecb85b5cc5827da8dce2d2ddf23 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 3 Jul 2025 05:29:30 -0700 Subject: [PATCH 06/29] Handle missing values --- singlecell/resources/chunks/FindClustersAndDimRedux.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/singlecell/resources/chunks/FindClustersAndDimRedux.R b/singlecell/resources/chunks/FindClustersAndDimRedux.R index b7fdc7c0f..175472c6c 100644 --- a/singlecell/resources/chunks/FindClustersAndDimRedux.R +++ b/singlecell/resources/chunks/FindClustersAndDimRedux.R @@ -19,7 +19,7 @@ for (datasetId in names(seuratObjects)) { printName(datasetId) seuratObj <- readSeuratRDS(seuratObjects[[datasetId]]) - if (all(is.null(clusterResolutions))) { + if (all(is.null(clusterResolutions)) || clusterResolutions == '') { clusterResolutions <- c(0.2, 0.4, 0.6, 0.8, 1.2) } else if (is.character(clusterResolutions)) { clusterResolutionsOrig <- clusterResolutions From a45c0c0cf60d5de04b749d377a4fcbab3a5aad32 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 3 Jul 2025 06:03:24 -0700 Subject: [PATCH 07/29] Bugfix to FindClustersAndDimRedux --- .../chunks/FindClustersAndDimRedux.R | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/singlecell/resources/chunks/FindClustersAndDimRedux.R b/singlecell/resources/chunks/FindClustersAndDimRedux.R index 175472c6c..baba68718 100644 --- a/singlecell/resources/chunks/FindClustersAndDimRedux.R +++ b/singlecell/resources/chunks/FindClustersAndDimRedux.R @@ -15,24 +15,26 @@ if (!reticulate::py_module_available(module = 'leidenalg')) { } } +if (all(is.null(clusterResolutions)) || clusterResolutions == '') { + clusterResolutions <- c(0.2, 0.4, 0.6, 0.8, 1.2) +} else if (is.character(clusterResolutions)) { + clusterResolutionsOrig <- clusterResolutions + clusterResolutions <- gsub(clusterResolutions, pattern = ' ', replacement = '') + clusterResolutions <- unlist(strsplit(clusterResolutions, split = ',')) + clusterResolutions <- as.numeric(clusterResolutions) + if (any(is.na(clusterResolutions))) { + stop(paste0('Some values for clusterResolutions were not numeric: ', clusterResolutionsOrig)) + } +else if (is.numeric(clusterResolutions)) { + # No action needed +} else { + stop('Must provide a value for clusterResolutions') +} + for (datasetId in names(seuratObjects)) { printName(datasetId) seuratObj <- readSeuratRDS(seuratObjects[[datasetId]]) - if (all(is.null(clusterResolutions)) || clusterResolutions == '') { - clusterResolutions <- c(0.2, 0.4, 0.6, 0.8, 1.2) - } else if (is.character(clusterResolutions)) { - clusterResolutionsOrig <- clusterResolutions - clusterResolutions <- gsub(clusterResolutions, pattern = ' ', replacement = '') - clusterResolutions <- unlist(strsplit(clusterResolutions, split = ',')) - clusterResolutions <- as.numeric(clusterResolutions) - if (any(is.na(clusterResolutions))) { - stop(paste0('Some values for clusterResolutions were not numeric: ', clusterResolutionsOrig)) - } - } else { - stop('Must provide a value for clusterResolutions') - } - seuratObj <- CellMembrane::FindClustersAndDimRedux(seuratObj, minDimsToUse = minDimsToUse, useLeiden = useLeiden, clusterResolutions = clusterResolutions) saveData(seuratObj, datasetId) From 5a42760aa647544ab800494364c78856ae755604 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 3 Jul 2025 15:52:32 -0700 Subject: [PATCH 08/29] Bugfix to FindClustersAndDimRedux --- singlecell/resources/chunks/FindClustersAndDimRedux.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/singlecell/resources/chunks/FindClustersAndDimRedux.R b/singlecell/resources/chunks/FindClustersAndDimRedux.R index baba68718..47ecfa6c2 100644 --- a/singlecell/resources/chunks/FindClustersAndDimRedux.R +++ b/singlecell/resources/chunks/FindClustersAndDimRedux.R @@ -25,7 +25,7 @@ if (all(is.null(clusterResolutions)) || clusterResolutions == '') { if (any(is.na(clusterResolutions))) { stop(paste0('Some values for clusterResolutions were not numeric: ', clusterResolutionsOrig)) } -else if (is.numeric(clusterResolutions)) { +} else if (is.numeric(clusterResolutions)) { # No action needed } else { stop('Must provide a value for clusterResolutions') From 057d0276ffefe93103a4fcc64adb26b1df439fd9 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 8 Jul 2025 20:42:07 -0700 Subject: [PATCH 09/29] Update vireo args --- .../singlecell/pipeline/singlecell/VireoHandler.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java index 28da76fae..b37f73ac2 100644 --- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java +++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java @@ -202,12 +202,6 @@ public void processFilesRemote(List inputFiles, JobContext c cellsnp.add(maxThreads.toString()); } - cellsnp.add("--minMAF"); - cellsnp.add("0.1"); - - cellsnp.add("--minCOUNT"); - cellsnp.add("100"); - String maxDepth = StringUtils.trimToNull(ctx.getParams().optString("maxDepth")); if (maxDepth != null) { @@ -240,6 +234,12 @@ public void processFilesRemote(List inputFiles, JobContext c cellsnp.add("--chrom"); cellsnp.add(contigs); } + + cellsnp.add("--minMAF"); + cellsnp.add("0.1"); + + cellsnp.add("--minCOUNT"); + cellsnp.add("100"); } new SimpleScriptWrapper(ctx.getLogger()).execute(cellsnp); From 77203cec29edbc9812d4a1f6b25eb05d79f1307a Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 8 Jul 2025 21:44:12 -0700 Subject: [PATCH 10/29] Update vireo args --- .../pipeline/singlecell/VireoHandler.java | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java index b37f73ac2..341126b95 100644 --- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java +++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java @@ -41,8 +41,8 @@ public class VireoHandler extends AbstractParameterizedOutputHandler(PageFlowUtil.set("sequenceanalysis/field/SequenceOutputFileSelectorField.js")), Arrays.asList( - ToolParameterDescriptor.create("nDonors", "# Donors", "The number of donors to demultiplex", "ldk-integerfield", new JSONObject(){{ - put("allowBlank", false); + ToolParameterDescriptor.create("nDonors", "# Donors", "The number of donors to demultiplex. This can be blank only if a reference VCF is provided.", "ldk-integerfield", new JSONObject(){{ + }}, null), ToolParameterDescriptor.create("maxDepth", "Max Depth", "At a position, read maximally INT reads per input file, to avoid excessive memory usage", "ldk-integerfield", new JSONObject(){{ put("minValue", 0); @@ -112,6 +112,13 @@ public void init(JobContext ctx, List inputFiles, List inputFiles, JobContext c File barcodesGz = getBarcodesFile(inputFiles.get(0).getFile()); File bam = getBamFile(inputFiles.get(0).getFile()); - File barcodes = new File(ctx.getWorkingDirectory(), "barcodes.csv"); + File barcodes = new File(ctx.getWorkingDirectory(), "barcodes.tsv"); try (BufferedReader reader = IOUtil.openFileForBufferedUtf8Reading(barcodesGz); PrintWriter writer = PrintWriters.getPrintWriter(barcodes)) { String line; @@ -314,21 +321,20 @@ public void processFilesRemote(List inputFiles, JobContext c vireo.add("-o"); vireo.add(ctx.getWorkingDirectory().getPath()); - int nDonors = ctx.getParams().optInt("nDonors", 0); boolean storeCellSnpVcf = ctx.getParams().optBoolean("storeCellSnpVcf", false); - if (nDonors == 0) - { - throw new PipelineJobException("Must provide nDonors"); - } - if (refVcfSubset != null) { vireo.add("-d"); vireo.add(refVcfSubset.getPath()); } - vireo.add("-N"); - vireo.add(String.valueOf(nDonors)); + // Note: this value should be checked in init(). It is required only if refVCF is null + int nDonors = ctx.getParams().optInt("nDonors", -1); + if (nDonors > -1) + { + vireo.add("-N"); + vireo.add(String.valueOf(nDonors)); + } if (nDonors == 1) { From 392881bd7fcc764e654e564025f17a1e22a9d96b Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 9 Jul 2025 07:39:25 -0700 Subject: [PATCH 11/29] Reduce logging --- .../sequenceanalysis/run/AbstractCommandWrapper.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/AbstractCommandWrapper.java b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/AbstractCommandWrapper.java index e9b0df727..74e665c07 100644 --- a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/AbstractCommandWrapper.java +++ b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/AbstractCommandWrapper.java @@ -44,6 +44,7 @@ abstract public class AbstractCommandWrapper implements CommandWrapper private File _outputDir = null; private File _workingDir = null; private Logger _log = null; + private boolean _logPath = false; private Level _logLevel = Level.DEBUG; private boolean _warnNonZeroExits = true; private boolean _throwNonZeroExits = true; @@ -229,11 +230,19 @@ private void setPath(ProcessBuilder pb) path = fileExe.getParent() + File.pathSeparatorChar + path; } - getLogger().debug("using path: " + path); + if (_logPath) + { + getLogger().debug("using path: " + path); + } pb.environment().put("PATH", path); } } + public void setLogPath(boolean logPath) + { + _logPath = logPath; + } + public void setOutputDir(File outputDir) { _outputDir = outputDir; From c57d7e792afc530d6dcc0fd03e927465cc408f7f Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 9 Jul 2025 07:56:23 -0700 Subject: [PATCH 12/29] Defer sorting of cellsnp VCF --- .../pipeline/singlecell/VireoHandler.java | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java index 341126b95..42db6bc8a 100644 --- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java +++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java @@ -277,9 +277,6 @@ public void processFilesRemote(List inputFiles, JobContext c throw new PipelineJobException("Unable to find cellsnp calls VCF"); } - sortAndFixVcf(cellSnpBaseVcf, genome, ctx.getLogger()); - sortAndFixVcf(cellSnpCellsVcf, genome, ctx.getLogger()); - int vcfFile = ctx.getParams().optInt(REF_VCF, -1); File refVcfSubset = null; if (vcfFile > -1) @@ -377,33 +374,44 @@ else if (outFiles.length > 1) if (storeCellSnpVcf) { + File fixedVcf = sortAndFixVcf(cellSnpBaseVcf, genome, ctx.getLogger(), ctx.getWorkingDirectory()); + SequenceOutputFile so = new SequenceOutputFile(); so.setReadset(inputFiles.get(0).getReadset()); so.setLibrary_id(inputFiles.get(0).getLibrary_id()); - so.setFile(cellSnpCellsVcf); + so.setFile(fixedVcf); if (so.getReadset() != null) { so.setName(ctx.getSequenceSupport().getCachedReadset(so.getReadset()).getName() + ": Cellsnp-lite VCF"); } else { - so.setName(inputFiles.get(0).getName() + ": Cellsnp-lite VCF"); + so.setName(inputFiles.get(0).getName() + ": Cellsnp-lite Base VCF"); } so.setCategory("VCF File"); ctx.addSequenceOutput(so); } + else + { + ctx.getFileManager().addIntermediateFile(cellSnpBaseVcf.getParentFile()); + } } - private void sortAndFixVcf(File vcf, ReferenceGenome genome, Logger log) throws PipelineJobException + private File sortAndFixVcf(File vcf, ReferenceGenome genome, Logger log, File outDir) throws PipelineJobException { + File outVcf = new File(outDir, vcf.getName()); + // This original VCF is generally not properly sorted, and has an invalid index. This is redundant, the VCF is not that large: try { - SequencePipelineService.get().sortROD(vcf, log, 2); - SequenceAnalysisService.get().ensureVcfIndex(vcf, log, true); + FileUtils.copyFile(vcf, outVcf); + SequencePipelineService.get().sortROD(outVcf, log, 2); + SequenceAnalysisService.get().ensureVcfIndex(outVcf, log, true); + + new UpdateVCFSequenceDictionary(log).execute(outVcf, genome.getSequenceDictionary()); + SequenceAnalysisService.get().ensureVcfIndex(outVcf, log); - new UpdateVCFSequenceDictionary(log).execute(vcf, genome.getSequenceDictionary()); - SequenceAnalysisService.get().ensureVcfIndex(vcf, log); + return outVcf; } catch (IOException e) { From ac24a995012775a5bf5ea56c6f563a6ba0575a84 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 9 Jul 2025 10:31:40 -0700 Subject: [PATCH 13/29] More obvious link to ManageFileRootAction --- .../org/labkey/discvrcore/DiscvrCoreController.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/discvrcore/src/org/labkey/discvrcore/DiscvrCoreController.java b/discvrcore/src/org/labkey/discvrcore/DiscvrCoreController.java index ea2596b7f..3382d1a0d 100644 --- a/discvrcore/src/org/labkey/discvrcore/DiscvrCoreController.java +++ b/discvrcore/src/org/labkey/discvrcore/DiscvrCoreController.java @@ -291,4 +291,16 @@ public URLHelper getRedirectURL(Object o) throws Exception return DetailsURL.fromString("laboratory/setTableIncrementValue.view", getContainer()).getActionURL(); } } + + // This allows registration of this action without creating a dependency between laboratory and discvrcore + @UtilityAction(label = "Manage File Roots", description = "This standalone file root management action can be used on folder types that do not support the normal 'Manage Folder' UI.") + @RequiresPermission(AdminPermission.class) + public class ManageFileRootAction extends SimpleRedirectAction + { + @Override + public URLHelper getRedirectURL(Object o) throws Exception + { + return DetailsURL.fromString("admin/manageFileRoot.view", getContainer()).getActionURL(); + } + } } From fdfaab204781567692df01d8a1f084f5d32c04ae Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 9 Jul 2025 10:32:23 -0700 Subject: [PATCH 14/29] Report summary for vireo --- .../pipeline/singlecell/VireoHandler.java | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java index 42db6bc8a..75f26c2e6 100644 --- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java +++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java @@ -1,5 +1,6 @@ package org.labkey.singlecell.pipeline.singlecell; +import au.com.bytecode.opencsv.CSVReader; import htsjdk.samtools.util.IOUtil; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; @@ -9,6 +10,7 @@ import org.labkey.api.pipeline.PipelineJob; import org.labkey.api.pipeline.PipelineJobException; import org.labkey.api.pipeline.RecordedAction; +import org.labkey.api.reader.Readers; import org.labkey.api.sequenceanalysis.SequenceAnalysisService; import org.labkey.api.sequenceanalysis.SequenceOutputFile; import org.labkey.api.sequenceanalysis.pipeline.AbstractParameterizedOutputHandler; @@ -365,10 +367,38 @@ else if (outFiles.length > 1) so.setName(inputFiles.get(0).getName() + ": Vireo Demultiplexing"); } so.setCategory("Vireo Demultiplexing"); + StringBuilder description = new StringBuilder(); if (vcfFile > -1) { - so.setDescription("Reference VCF ID: " + vcfFile); + description.append("Reference VCF ID: \n").append(vcfFile); } + + File summary = new File(ctx.getOutputDir(), "summary.tsv"); + if (!summary.exists()) + { + throw new PipelineJobException("Missing file: " + summary.getPath()); + } + + description.append("Results:\n"); + try (CSVReader reader = new CSVReader(Readers.getReader(summary))) + { + String[] line; + while ((line = reader.readNext()) != null) + { + if ("Var1".equals(line[0])) + { + continue; + } + + description.append(line[0]).append(": ").append(line[1]).append("\n"); + } + } + catch (IOException e) + { + throw new PipelineJobException(e); + } + + so.setDescription(StringUtils.trimToEmpty(description.toString())); ctx.addSequenceOutput(so); } From 05be6caa1f0243b0af634c583bacc820eb9657d1 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 9 Jul 2025 11:13:12 -0700 Subject: [PATCH 15/29] Always re-cache lookups after clear --- .../src/org/labkey/studies/query/LookupSetsTable.java | 6 +++--- .../org/labkey/studies/query/StudiesUserSchema.java | 10 ++++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/Studies/src/org/labkey/studies/query/LookupSetsTable.java b/Studies/src/org/labkey/studies/query/LookupSetsTable.java index 4d6ff8230..83c8c78ef 100644 --- a/Studies/src/org/labkey/studies/query/LookupSetsTable.java +++ b/Studies/src/org/labkey/studies/query/LookupSetsTable.java @@ -38,14 +38,14 @@ public UpdateService(SimpleUserSchema.SimpleTable ti) @Override protected void afterInsertUpdate(int count, BatchValidationException errors) { - LookupSetsManager.get().getCache().clear(); + StudiesUserSchema.repopulateCaches(getUserSchema().getUser(), getUserSchema().getContainer()); } @Override protected Map deleteRow(User user, Container container, Map oldRowMap) throws QueryUpdateServiceException, SQLException, InvalidKeyException { Map row = super.deleteRow(user, container, oldRowMap); - LookupSetsManager.get().getCache().clear(); + StudiesUserSchema.repopulateCaches(getUserSchema().getUser(), getUserSchema().getContainer()); return row; } @@ -53,7 +53,7 @@ protected Map deleteRow(User user, Container container, Map> getPropertySetNames() { Map> nameMap = (Map>) LookupSetsManager.get().getCache().get(LookupSetTable.getCacheKey(getTargetContainer())); @@ -81,6 +90,7 @@ private Map> getPropertySetNames() return nameMap; } + _log.debug("Populating lookup tables in StudiesUserSchema.getPropertySetNames() for container: " + getTargetContainer().getName()); nameMap = new CaseInsensitiveHashMap<>(); TableSelector ts = new TableSelector(_dbSchema.getTable(TABLE_LOOKUP_SETS), new SimpleFilter(FieldKey.fromString("container"), getTargetContainer().getId()), null); From 7382511e4abe9ef5ab9534eb83812280b147f473 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 9 Jul 2025 16:38:34 -0700 Subject: [PATCH 16/29] TSVs need tab --- .../org/labkey/singlecell/pipeline/singlecell/VireoHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java index 75f26c2e6..d4ddc7326 100644 --- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java +++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java @@ -380,7 +380,7 @@ else if (outFiles.length > 1) } description.append("Results:\n"); - try (CSVReader reader = new CSVReader(Readers.getReader(summary))) + try (CSVReader reader = new CSVReader(Readers.getReader(summary), '\t')) { String[] line; while ((line = reader.readNext()) != null) From 7d11ce99a224c527e733f742bf70aa53cd81a572 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 10 Jul 2025 15:08:48 -0700 Subject: [PATCH 17/29] Support sawfish --- .../pipeline_code/extra_tools_install.sh | 14 ++ .../SequenceAnalysisModule.java | 4 + .../run/analysis/SawfishAnalysis.java | 90 +++++++++ .../analysis/SawfishJointCallingHandler.java | 183 ++++++++++++++++++ 4 files changed, 291 insertions(+) create mode 100644 SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishAnalysis.java create mode 100644 SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishJointCallingHandler.java diff --git a/SequenceAnalysis/pipeline_code/extra_tools_install.sh b/SequenceAnalysis/pipeline_code/extra_tools_install.sh index e3cce3c55..8ecd60f37 100755 --- a/SequenceAnalysis/pipeline_code/extra_tools_install.sh +++ b/SequenceAnalysis/pipeline_code/extra_tools_install.sh @@ -319,3 +319,17 @@ then else echo "Already installed" fi + +if [[ ! -e ${LKTOOLS_DIR}/sawfish || ! -z $FORCE_REINSTALL ]]; +then + echo "Cleaning up previous installs" + rm -Rf $LKTOOLS_DIR/sawfish* + + wget https://github.com/PacificBiosciences/sawfish/releases/download/v2.0.0/sawfish-v2.0.0-x86_64-unknown-linux-gnu.tar.gz + tar -xzf sawfish-v2.0.0-x86_64-unknown-linux-gnu.tar.gz + + mv sawfish-v2.0.0-x86_64-unknown-linux-gnu $LKTOOLS_DIR/ + ln -s $LKTOOLS_DIR/sawfish-v2.0.0/bin/sawfish $LKTOOLS_DIR/ +else + echo "Already installed" +fi diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisModule.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisModule.java index 7a10338fd..8dd0d99b8 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisModule.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisModule.java @@ -123,6 +123,8 @@ import org.labkey.sequenceanalysis.run.analysis.PbsvAnalysis; import org.labkey.sequenceanalysis.run.analysis.PbsvJointCallingHandler; import org.labkey.sequenceanalysis.run.analysis.PindelAnalysis; +import org.labkey.sequenceanalysis.run.analysis.SawfishAnalysis; +import org.labkey.sequenceanalysis.run.analysis.SawfishJointCallingHandler; import org.labkey.sequenceanalysis.run.analysis.SequenceBasedTypingAnalysis; import org.labkey.sequenceanalysis.run.analysis.SnpCountAnalysis; import org.labkey.sequenceanalysis.run.analysis.SubreadAnalysis; @@ -342,6 +344,7 @@ public static void registerPipelineSteps() SequencePipelineService.get().registerPipelineStep(new PindelAnalysis.Provider()); SequencePipelineService.get().registerPipelineStep(new PbsvAnalysis.Provider()); SequencePipelineService.get().registerPipelineStep(new GenrichStep.Provider()); + SequencePipelineService.get().registerPipelineStep(new SawfishAnalysis.Provider()); SequencePipelineService.get().registerPipelineStep(new PARalyzerAnalysis.Provider()); SequencePipelineService.get().registerPipelineStep(new RnaSeQCStep.Provider()); @@ -400,6 +403,7 @@ public static void registerPipelineSteps() SequenceAnalysisService.get().registerFileHandler(new NextCladeHandler()); SequenceAnalysisService.get().registerFileHandler(new ConvertToCramHandler()); SequenceAnalysisService.get().registerFileHandler(new PbsvJointCallingHandler()); + SequenceAnalysisService.get().registerFileHandler(new SawfishJointCallingHandler()); SequenceAnalysisService.get().registerFileHandler(new DeepVariantHandler()); SequenceAnalysisService.get().registerFileHandler(new GLNexusHandler()); SequenceAnalysisService.get().registerFileHandler(new ParagraphStep()); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishAnalysis.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishAnalysis.java new file mode 100644 index 000000000..04abb26ef --- /dev/null +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishAnalysis.java @@ -0,0 +1,90 @@ +package org.labkey.sequenceanalysis.run.analysis; + +import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.sequenceanalysis.model.AnalysisModel; +import org.labkey.api.sequenceanalysis.model.Readset; +import org.labkey.api.sequenceanalysis.pipeline.AbstractAnalysisStepProvider; +import org.labkey.api.sequenceanalysis.pipeline.AbstractPipelineStep; +import org.labkey.api.sequenceanalysis.pipeline.AnalysisOutputImpl; +import org.labkey.api.sequenceanalysis.pipeline.AnalysisStep; +import org.labkey.api.sequenceanalysis.pipeline.PipelineContext; +import org.labkey.api.sequenceanalysis.pipeline.PipelineStepProvider; +import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenome; +import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService; +import org.labkey.api.sequenceanalysis.run.SimpleScriptWrapper; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +public class SawfishAnalysis extends AbstractPipelineStep implements AnalysisStep +{ + public SawfishAnalysis(PipelineStepProvider provider, PipelineContext ctx) + { + super(provider, ctx); + } + + public static class Provider extends AbstractAnalysisStepProvider + { + public Provider() + { + super("sawfish", "Sawfish Analysis", null, "This will run sawfish SV dicvoery and calling on the selected BAMs", List.of(), null, null); + } + + + @Override + public SawfishAnalysis create(PipelineContext ctx) + { + return new SawfishAnalysis(this, ctx); + } + } + + @Override + public Output performAnalysisPerSampleRemote(Readset rs, File inputBam, ReferenceGenome referenceGenome, File outputDir) throws PipelineJobException + { + AnalysisOutputImpl output = new AnalysisOutputImpl(); + + List args = new ArrayList<>(); + args.add(getExe().getPath()); + args.add("discover"); + + args.add("--bam"); + args.add(inputBam.getPath()); + + args.add("--ref"); + args.add(referenceGenome.getWorkingFastaFile().getPath()); + + File svOutDir = new File(outputDir, "sawfish"); + args.add("--output-dir"); + args.add(svOutDir.getPath()); + + Integer maxThreads = SequencePipelineService.get().getMaxThreads(getPipelineCtx().getLogger()); + if (maxThreads != null) + { + args.add("-threads"); + args.add(String.valueOf(maxThreads)); + } + + new SimpleScriptWrapper(getPipelineCtx().getLogger()).execute(args); + + File vcf = new File(svOutDir, "genotyped.sv.vcf.gz"); + if (!vcf.exists()) + { + throw new PipelineJobException("Unable to find file: " + vcf.getPath()); + } + + output.addSequenceOutput(vcf, rs.getName() + ": sawfish", "Sawfish SV Discovery", rs.getReadsetId(), null, referenceGenome.getGenomeId(), null); + return output; + } + + @Override + public Output performAnalysisPerSampleLocal(AnalysisModel model, File inputBam, File referenceFasta, File outDir) throws PipelineJobException + { + return null; + } + + private File getExe() + { + return SequencePipelineService.get().getExeForPackage("SAWFISHPATH", "sawfish"); + } +} \ No newline at end of file diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishJointCallingHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishJointCallingHandler.java new file mode 100644 index 000000000..50957ddd3 --- /dev/null +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishJointCallingHandler.java @@ -0,0 +1,183 @@ +package org.labkey.sequenceanalysis.run.analysis; + +import org.apache.commons.io.FileUtils; +import org.json.JSONObject; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.pipeline.PipelineJob; +import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.pipeline.RecordedAction; +import org.labkey.api.sequenceanalysis.SequenceAnalysisService; +import org.labkey.api.sequenceanalysis.SequenceOutputFile; +import org.labkey.api.sequenceanalysis.pipeline.AbstractParameterizedOutputHandler; +import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenome; +import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport; +import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputHandler; +import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService; +import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor; +import org.labkey.api.sequenceanalysis.run.SimpleScriptWrapper; +import org.labkey.sequenceanalysis.SequenceAnalysisModule; +import org.labkey.sequenceanalysis.util.SequenceUtil; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.stream.Collectors; + +public class SawfishJointCallingHandler extends AbstractParameterizedOutputHandler +{ + private static final String OUTPUT_CATEGORY = "Sawfish VCF"; + + public SawfishJointCallingHandler() + { + super(ModuleLoader.getInstance().getModule(SequenceAnalysisModule.NAME), "Sawfish Joint-Call", "Runs sawfish joint-call, which jointly calls SVs from PacBio CCS data", new LinkedHashSet<>(List.of("sequenceanalysis/panel/VariantScatterGatherPanel.js")), Arrays.asList( + ToolParameterDescriptor.create("fileName", "VCF Filename", "The name of the resulting file.", "textfield", new JSONObject(){{ + put("allowBlank", false); + put("doNotIncludeInTemplates", true); + }}, null) + )); + } + + @Override + public boolean canProcess(SequenceOutputFile o) + { + return o.getFile() != null && SequenceUtil.FILETYPE.vcf.getFileType().isType(o.getFile()); + } + + @Override + public boolean doRunRemote() + { + return true; + } + + @Override + public boolean doRunLocal() + { + return false; + } + + @Override + public SequenceOutputProcessor getProcessor() + { + return new Processor(); + } + + public static class Processor implements SequenceOutputProcessor + { + @Override + public void processFilesOnWebserver(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException + { + + } + + @Override + public void processFilesRemote(List inputFiles, JobContext ctx) throws UnsupportedOperationException, PipelineJobException + { + List filesToProcess = inputFiles.stream().map(SequenceOutputFile::getFile).collect(Collectors.toList()); + + ReferenceGenome genome = ctx.getSequenceSupport().getCachedGenomes().iterator().next(); + String outputBaseName = ctx.getParams().getString("fileName"); + if (!outputBaseName.toLowerCase().endsWith(".gz")) + { + outputBaseName = outputBaseName.replaceAll(".gz$", ""); + } + + if (!outputBaseName.toLowerCase().endsWith(".vcf")) + { + outputBaseName = outputBaseName.replaceAll(".vcf$", ""); + } + + File expectedFinalOutput = new File(ctx.getOutputDir(), outputBaseName + ".vcf.gz"); + File expectedFinalOutputIdx = new File(expectedFinalOutput.getPath() + ".tbi"); + boolean jobCompleted = expectedFinalOutputIdx.exists(); // this would occur if the job died during the cleanup phase + + File ouputVcf = runSawfishCall(ctx, filesToProcess, genome, outputBaseName); + + SequenceOutputFile so = new SequenceOutputFile(); + so.setName("Sawfish call: " + outputBaseName); + so.setFile(ouputVcf); + so.setCategory(OUTPUT_CATEGORY); + so.setLibrary_id(genome.getGenomeId()); + + ctx.addSequenceOutput(so); + } + + private File runSawfishCall(JobContext ctx, List inputs, ReferenceGenome genome, String outputBaseName) throws PipelineJobException + { + if (inputs.isEmpty()) + { + throw new PipelineJobException("No inputs provided"); + } + + List args = new ArrayList<>(); + args.add(getExe().getPath()); + args.add("joint-call"); + + Integer maxThreads = SequencePipelineService.get().getMaxThreads(ctx.getLogger()); + if (maxThreads != null) + { + args.add("--threads"); + args.add(String.valueOf(maxThreads)); + } + + args.add("--ref"); + args.add(genome.getWorkingFastaFile().getPath()); + + for (File sample : inputs) + { + args.add("--sample"); + args.add(sample.getParentFile().getPath()); + } + + File outDir = new File(ctx.getOutputDir(), "sawfish"); + args.add("--output-dir"); + args.add(outDir.getPath()); + + new SimpleScriptWrapper(ctx.getLogger()).execute(args); + + File vcfOut = new File(outDir, "genotyped.sv.vcf.gz"); + if (!vcfOut.exists()) + { + throw new PipelineJobException("Unable to find file: " + vcfOut.getPath()); + } + + File vcfOutFinal = new File(ctx.getOutputDir(), outputBaseName + ".vcf.gz"); + + try + { + if (vcfOutFinal.exists()) + { + vcfOutFinal.delete(); + FileUtils.moveFile(vcfOut, vcfOutFinal); + + File targetIndex = new File(vcfOutFinal.getPath() + ".tbi"); + if (targetIndex.exists()) + { + targetIndex.delete(); + } + + File origIndex = new File(vcfOut.getPath() + ".tbi"); + if (origIndex.exists()) + { + FileUtils.moveFile(origIndex, targetIndex); + } + + SequenceAnalysisService.get().ensureVcfIndex(vcfOutFinal, ctx.getLogger(), true); + } + } + catch (IOException e) + { + throw new PipelineJobException(e); + } + + return vcfOutFinal; + } + + private File getExe() + { + return SequencePipelineService.get().getExeForPackage("PBSVPATH", "pbsv"); + } + } +} From 6b1801611e54d5a34530261498633bd8b30f7bde Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 10 Jul 2025 16:12:53 -0700 Subject: [PATCH 18/29] Correct sawfish argument --- .../labkey/sequenceanalysis/run/analysis/SawfishAnalysis.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishAnalysis.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishAnalysis.java index 04abb26ef..df2e534cb 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishAnalysis.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishAnalysis.java @@ -61,7 +61,7 @@ public Output performAnalysisPerSampleRemote(Readset rs, File inputBam, Referenc Integer maxThreads = SequencePipelineService.get().getMaxThreads(getPipelineCtx().getLogger()); if (maxThreads != null) { - args.add("-threads"); + args.add("--threads"); args.add(String.valueOf(maxThreads)); } From 23bf99992f3b28b854db0b619f91015868fa4577 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 10 Jul 2025 21:51:01 -0700 Subject: [PATCH 19/29] Provide sawfish with BAM input --- .../run/analysis/SawfishAnalysis.java | 55 ++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishAnalysis.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishAnalysis.java index df2e534cb..1e6c7ef4c 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishAnalysis.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishAnalysis.java @@ -1,5 +1,7 @@ package org.labkey.sequenceanalysis.run.analysis; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.Nullable; import org.labkey.api.pipeline.PipelineJobException; import org.labkey.api.sequenceanalysis.model.AnalysisModel; import org.labkey.api.sequenceanalysis.model.Readset; @@ -10,8 +12,10 @@ import org.labkey.api.sequenceanalysis.pipeline.PipelineContext; import org.labkey.api.sequenceanalysis.pipeline.PipelineStepProvider; import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenome; +import org.labkey.api.sequenceanalysis.pipeline.SamtoolsRunner; import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService; import org.labkey.api.sequenceanalysis.run.SimpleScriptWrapper; +import org.labkey.sequenceanalysis.util.SequenceUtil; import java.io.File; import java.util.ArrayList; @@ -44,12 +48,24 @@ public Output performAnalysisPerSampleRemote(Readset rs, File inputBam, Referenc { AnalysisOutputImpl output = new AnalysisOutputImpl(); + File inputFile = inputBam; + if (SequenceUtil.FILETYPE.cram.getFileType().isType(inputFile)) + { + CramToBam samtoolsRunner = new CramToBam(getPipelineCtx().getLogger()); + File bam = new File(getPipelineCtx().getWorkingDirectory(), inputFile.getName().replaceAll(".cram$", ".bam")); + samtoolsRunner.convert(inputFile, bam, referenceGenome.getWorkingFastaFile(), SequencePipelineService.get().getMaxThreads(getPipelineCtx().getLogger())); + inputFile = bam; + + output.addIntermediateFile(bam); + output.addIntermediateFile(new File(bam.getPath() + ".bai")); + } + List args = new ArrayList<>(); args.add(getExe().getPath()); args.add("discover"); args.add("--bam"); - args.add(inputBam.getPath()); + args.add(inputFile.getPath()); args.add("--ref"); args.add(referenceGenome.getWorkingFastaFile().getPath()); @@ -87,4 +103,41 @@ private File getExe() { return SequencePipelineService.get().getExeForPackage("SAWFISHPATH", "sawfish"); } + + private static class CramToBam extends SamtoolsRunner + { + public CramToBam(Logger log) + { + super(log); + } + + public void convert(File inputCram, File outputBam, File fasta, @Nullable Integer threads) throws PipelineJobException + { + getLogger().info("Converting CRAM to BAM"); + + execute(getParams(inputCram, outputBam, fasta, threads)); + } + + private List getParams(File inputCram, File outputBam, File fasta, @Nullable Integer threads) + { + List params = new ArrayList<>(); + params.add(getSamtoolsPath().getPath()); + params.add("view"); + params.add("-b"); + params.add("-T"); + params.add(fasta.getPath()); + params.add("-o"); + params.add(outputBam.getPath()); + + if (threads != null) + { + params.add("-@"); + params.add(String.valueOf(threads)); + } + + params.add(inputCram.getPath()); + + return params; + } + } } \ No newline at end of file From 311f538917afb8de6b70c288c6828afa8aebc990 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 10 Jul 2025 22:28:09 -0700 Subject: [PATCH 20/29] Reduce logging --- .../api/sequenceanalysis/run/AbstractCommandWrapper.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/AbstractCommandWrapper.java b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/AbstractCommandWrapper.java index 74e665c07..5b5a830ec 100644 --- a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/AbstractCommandWrapper.java +++ b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/AbstractCommandWrapper.java @@ -206,9 +206,11 @@ private void setPath(ProcessBuilder pb) { String path = System.getenv("PATH"); - getLogger().debug("Existing PATH: " + path); - getLogger().debug("toolDir: " + toolDir); - + if (_logPath) + { + getLogger().debug("Existing PATH: " + path); + getLogger().debug("toolDir: " + toolDir); + } if (path == null) { From dccf4c7248de90266f90aaacdcbc2a02a660f404 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 10 Jul 2025 22:34:44 -0700 Subject: [PATCH 21/29] Ensure BAM index exists --- .../run/analysis/SawfishAnalysis.java | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishAnalysis.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishAnalysis.java index 1e6c7ef4c..bc346470b 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishAnalysis.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishAnalysis.java @@ -12,6 +12,7 @@ import org.labkey.api.sequenceanalysis.pipeline.PipelineContext; import org.labkey.api.sequenceanalysis.pipeline.PipelineStepProvider; import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenome; +import org.labkey.api.sequenceanalysis.pipeline.SamtoolsIndexer; import org.labkey.api.sequenceanalysis.pipeline.SamtoolsRunner; import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService; import org.labkey.api.sequenceanalysis.run.SimpleScriptWrapper; @@ -53,11 +54,21 @@ public Output performAnalysisPerSampleRemote(Readset rs, File inputBam, Referenc { CramToBam samtoolsRunner = new CramToBam(getPipelineCtx().getLogger()); File bam = new File(getPipelineCtx().getWorkingDirectory(), inputFile.getName().replaceAll(".cram$", ".bam")); - samtoolsRunner.convert(inputFile, bam, referenceGenome.getWorkingFastaFile(), SequencePipelineService.get().getMaxThreads(getPipelineCtx().getLogger())); + File bamIdx = new File(bam.getPath() + ".bai"); + if (!bamIdx.exists()) + { + samtoolsRunner.convert(inputFile, bam, referenceGenome.getWorkingFastaFile(), SequencePipelineService.get().getMaxThreads(getPipelineCtx().getLogger())); + new SamtoolsIndexer(getPipelineCtx().getLogger()).execute(bam); + } + else + { + getPipelineCtx().getLogger().debug("BAM index exists, will not re-convert CRAM"); + } + inputFile = bam; output.addIntermediateFile(bam); - output.addIntermediateFile(new File(bam.getPath() + ".bai")); + output.addIntermediateFile(bamIdx); } List args = new ArrayList<>(); From 3c454e35be5b70edb7d6d152212970209550a436 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 11 Jul 2025 06:52:40 -0700 Subject: [PATCH 22/29] Bugfix to sawfish --- .../run/analysis/SawfishAnalysis.java | 19 ++++++++---- .../analysis/SawfishJointCallingHandler.java | 30 +++++++++---------- .../sequenceanalysis/util/SequenceUtil.java | 1 + 3 files changed, 30 insertions(+), 20 deletions(-) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishAnalysis.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishAnalysis.java index bc346470b..421f77ffb 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishAnalysis.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishAnalysis.java @@ -92,15 +92,24 @@ public Output performAnalysisPerSampleRemote(Readset rs, File inputBam, Referenc args.add(String.valueOf(maxThreads)); } - new SimpleScriptWrapper(getPipelineCtx().getLogger()).execute(args); + File bcf = new File(svOutDir, "candidate.sv.bcf"); + File bcfIdx = new File(bcf.getPath() + ".csi"); + if (bcfIdx.exists()) + { + getPipelineCtx().getLogger().debug("BCF index already exists, reusing output"); + } + else + { + new SimpleScriptWrapper(getPipelineCtx().getLogger()).execute(args); + } - File vcf = new File(svOutDir, "genotyped.sv.vcf.gz"); - if (!vcf.exists()) + if (!bcf.exists()) { - throw new PipelineJobException("Unable to find file: " + vcf.getPath()); + throw new PipelineJobException("Unable to find file: " + bcf.getPath()); } - output.addSequenceOutput(vcf, rs.getName() + ": sawfish", "Sawfish SV Discovery", rs.getReadsetId(), null, referenceGenome.getGenomeId(), null); + output.addSequenceOutput(bcf, rs.getName() + ": sawfish", "Sawfish SV Discovery", rs.getReadsetId(), null, referenceGenome.getGenomeId(), null); + return output; } diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishJointCallingHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishJointCallingHandler.java index 50957ddd3..ce3b066a9 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishJointCallingHandler.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishJointCallingHandler.java @@ -43,7 +43,7 @@ public SawfishJointCallingHandler() @Override public boolean canProcess(SequenceOutputFile o) { - return o.getFile() != null && SequenceUtil.FILETYPE.vcf.getFileType().isType(o.getFile()); + return o.getFile() != null && SequenceUtil.FILETYPE.bcf.getFileType().isType(o.getFile()); } @Override @@ -90,8 +90,6 @@ public void processFilesRemote(List inputFiles, JobContext c } File expectedFinalOutput = new File(ctx.getOutputDir(), outputBaseName + ".vcf.gz"); - File expectedFinalOutputIdx = new File(expectedFinalOutput.getPath() + ".tbi"); - boolean jobCompleted = expectedFinalOutputIdx.exists(); // this would occur if the job died during the cleanup phase File ouputVcf = runSawfishCall(ctx, filesToProcess, genome, outputBaseName); @@ -150,20 +148,22 @@ private File runSawfishCall(JobContext ctx, List inputs, ReferenceGenome g if (vcfOutFinal.exists()) { vcfOutFinal.delete(); - FileUtils.moveFile(vcfOut, vcfOutFinal); - - File targetIndex = new File(vcfOutFinal.getPath() + ".tbi"); - if (targetIndex.exists()) - { - targetIndex.delete(); - } + } + FileUtils.moveFile(vcfOut, vcfOutFinal); - File origIndex = new File(vcfOut.getPath() + ".tbi"); - if (origIndex.exists()) - { - FileUtils.moveFile(origIndex, targetIndex); - } + File targetIndex = new File(vcfOutFinal.getPath() + ".tbi"); + if (targetIndex.exists()) + { + targetIndex.delete(); + } + File origIndex = new File(vcfOut.getPath() + ".tbi"); + if (origIndex.exists()) + { + FileUtils.moveFile(origIndex, targetIndex); + } + else + { SequenceAnalysisService.get().ensureVcfIndex(vcfOutFinal, ctx.getLogger(), true); } } diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/util/SequenceUtil.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/util/SequenceUtil.java index 0a2ce0784..a5357d176 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/util/SequenceUtil.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/util/SequenceUtil.java @@ -96,6 +96,7 @@ public enum FILETYPE bed(Collections.singletonList(".bed"), true), bw(Collections.singletonList(".bw"), false), vcf(List.of(".vcf"), true), + bcf(List.of(".bcf"), true), gvcf(List.of(".g.vcf"), true); private final List _extensions; From a7abcb9b154ad7f62c3586ae6ca386802e9f4f89 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 11 Jul 2025 07:58:42 -0700 Subject: [PATCH 23/29] Bugfix to sawfish --- .../run/analysis/SawfishJointCallingHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishJointCallingHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishJointCallingHandler.java index ce3b066a9..337ee8113 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishJointCallingHandler.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishJointCallingHandler.java @@ -177,7 +177,7 @@ private File runSawfishCall(JobContext ctx, List inputs, ReferenceGenome g private File getExe() { - return SequencePipelineService.get().getExeForPackage("PBSVPATH", "pbsv"); + return SequencePipelineService.get().getExeForPackage("SAWFISHPATH", "sawfish"); } } } From 6c3015bcc5175a0e68215edd91762e60e73bab28 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 11 Jul 2025 09:06:51 -0700 Subject: [PATCH 24/29] Drop ref arg to sawfish --- .../run/analysis/SawfishJointCallingHandler.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishJointCallingHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishJointCallingHandler.java index 337ee8113..9beae27c6 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishJointCallingHandler.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishJointCallingHandler.java @@ -120,9 +120,6 @@ private File runSawfishCall(JobContext ctx, List inputs, ReferenceGenome g args.add(String.valueOf(maxThreads)); } - args.add("--ref"); - args.add(genome.getWorkingFastaFile().getPath()); - for (File sample : inputs) { args.add("--sample"); From 2914e9fc5a80b1658241a264b92e8dbf36234f95 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 11 Jul 2025 10:01:06 -0700 Subject: [PATCH 25/29] Allow sawfish to process CRAMs --- .../run/analysis/SawfishAnalysis.java | 64 +------------------ .../run/util/CramToBamWrapper.java | 47 ++++++++++++++ 2 files changed, 50 insertions(+), 61 deletions(-) create mode 100644 SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/CramToBamWrapper.java diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishAnalysis.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishAnalysis.java index 421f77ffb..8039ff338 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishAnalysis.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishAnalysis.java @@ -49,37 +49,16 @@ public Output performAnalysisPerSampleRemote(Readset rs, File inputBam, Referenc { AnalysisOutputImpl output = new AnalysisOutputImpl(); - File inputFile = inputBam; - if (SequenceUtil.FILETYPE.cram.getFileType().isType(inputFile)) - { - CramToBam samtoolsRunner = new CramToBam(getPipelineCtx().getLogger()); - File bam = new File(getPipelineCtx().getWorkingDirectory(), inputFile.getName().replaceAll(".cram$", ".bam")); - File bamIdx = new File(bam.getPath() + ".bai"); - if (!bamIdx.exists()) - { - samtoolsRunner.convert(inputFile, bam, referenceGenome.getWorkingFastaFile(), SequencePipelineService.get().getMaxThreads(getPipelineCtx().getLogger())); - new SamtoolsIndexer(getPipelineCtx().getLogger()).execute(bam); - } - else - { - getPipelineCtx().getLogger().debug("BAM index exists, will not re-convert CRAM"); - } - - inputFile = bam; - - output.addIntermediateFile(bam); - output.addIntermediateFile(bamIdx); - } - List args = new ArrayList<>(); args.add(getExe().getPath()); args.add("discover"); args.add("--bam"); - args.add(inputFile.getPath()); + args.add(inputBam.getPath()); + // NOTE: sawfish stores the absolute path of the FASTA in the output JSON, so dont rely on working copies: args.add("--ref"); - args.add(referenceGenome.getWorkingFastaFile().getPath()); + args.add(referenceGenome.getSourceFastaFile().getPath()); File svOutDir = new File(outputDir, "sawfish"); args.add("--output-dir"); @@ -123,41 +102,4 @@ private File getExe() { return SequencePipelineService.get().getExeForPackage("SAWFISHPATH", "sawfish"); } - - private static class CramToBam extends SamtoolsRunner - { - public CramToBam(Logger log) - { - super(log); - } - - public void convert(File inputCram, File outputBam, File fasta, @Nullable Integer threads) throws PipelineJobException - { - getLogger().info("Converting CRAM to BAM"); - - execute(getParams(inputCram, outputBam, fasta, threads)); - } - - private List getParams(File inputCram, File outputBam, File fasta, @Nullable Integer threads) - { - List params = new ArrayList<>(); - params.add(getSamtoolsPath().getPath()); - params.add("view"); - params.add("-b"); - params.add("-T"); - params.add(fasta.getPath()); - params.add("-o"); - params.add(outputBam.getPath()); - - if (threads != null) - { - params.add("-@"); - params.add(String.valueOf(threads)); - } - - params.add(inputCram.getPath()); - - return params; - } - } } \ No newline at end of file diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/CramToBamWrapper.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/CramToBamWrapper.java new file mode 100644 index 000000000..1e6cf749d --- /dev/null +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/CramToBamWrapper.java @@ -0,0 +1,47 @@ +package org.labkey.sequenceanalysis.run.util; + +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.sequenceanalysis.pipeline.SamtoolsRunner; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +public class CramToBamWrapper extends SamtoolsRunner +{ + public CramToBamWrapper(Logger log) + { + super(log); + } + + public void convert(File inputCram, File outputBam, File fasta, @Nullable Integer threads) throws PipelineJobException + { + getLogger().info("Converting CRAM to BAM"); + + execute(getParams(inputCram, outputBam, fasta, threads)); + } + + private List getParams(File inputCram, File outputBam, File fasta, @Nullable Integer threads) + { + List params = new ArrayList<>(); + params.add(getSamtoolsPath().getPath()); + params.add("view"); + params.add("-b"); + params.add("-T"); + params.add(fasta.getPath()); + params.add("-o"); + params.add(outputBam.getPath()); + + if (threads != null) + { + params.add("-@"); + params.add(String.valueOf(threads)); + } + + params.add(inputCram.getPath()); + + return params; + } +} From 3213c67173eb386781ca93bee1853b0ff90bf0ef Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 22 Jul 2025 14:05:15 -0700 Subject: [PATCH 26/29] Remove unused code --- jbrowse/resources/views/begin.html | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/jbrowse/resources/views/begin.html b/jbrowse/resources/views/begin.html index d42ddad7b..9032801a3 100644 --- a/jbrowse/resources/views/begin.html +++ b/jbrowse/resources/views/begin.html @@ -1,13 +1,3 @@ - - The JBrowse module is part of DISCVR-Seq. It provides a wrapper around the JBrowse Genome Browser, which lets users rapidly take data generated or uploaded into DISCRV-Seq and view it using JBrowse. The module ships with a version of JBrowse, meaning very little extra configuration is required in order to start using these tools. The primary reasons we created this wrapper around JBrowse are:

From 6a3389a0a075601457e6115ebb4bce2e14bf817e Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 22 Jul 2025 18:05:09 -0700 Subject: [PATCH 27/29] Initial commit for JSON-based study definition (#336) * Initial commit for JSON-based study definition * Clean up generics * Refinements to customizer code, and many new SIV-related ETLs * Implement upsert + test * Fix unicode hyphen --------- Co-authored-by: Sebastian Benjamin --- .../api/studies/study/StudyDefinition.java | 669 ++++++++++++++++++ Studies/resources/study/DemoStudy.json | 43 ++ .../org/labkey/studies/StudiesController.java | 51 +- .../org/labkey/studies/StudiesManager.java | 402 ++++++++++- .../src/org/labkey/studies/StudiesModule.java | 10 + .../studies/query/StudiesTableCustomizer.java | 2 + 6 files changed, 1151 insertions(+), 26 deletions(-) create mode 100644 Studies/api-src/org/labkey/api/studies/study/StudyDefinition.java create mode 100644 Studies/resources/study/DemoStudy.json diff --git a/Studies/api-src/org/labkey/api/studies/study/StudyDefinition.java b/Studies/api-src/org/labkey/api/studies/study/StudyDefinition.java new file mode 100644 index 000000000..cff4174ba --- /dev/null +++ b/Studies/api-src/org/labkey/api/studies/study/StudyDefinition.java @@ -0,0 +1,669 @@ +package org.labkey.api.studies.study; + +import com.fasterxml.jackson.annotation.JsonGetter; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSetter; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import org.json.JSONObject; + +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +public class StudyDefinition +{ + private Integer _rowId; + private String _studyName; + private String _label; + private String _category; + private String _description; + + private String _container; + private Integer _createdBy; + private Date _created; + private Integer _modifiedBy; + private Date _modified; + + private List _cohorts; + private List _anchorEvents; + private List _timepoints; + + public StudyDefinition() + { + + } + + public Integer getRowId() + { + return _rowId; + } + + public void setRowId(Integer rowId) + { + _rowId = rowId; + } + + public String getStudyName() + { + return _studyName; + } + + public void setStudyName(String studyName) + { + _studyName = studyName; + } + + public String getLabel() + { + return _label; + } + + public void setLabel(String label) + { + _label = label; + } + + public String getCategory() + { + return _category; + } + + public void setCategory(String category) + { + _category = category; + } + + public String getDescription() + { + return _description; + } + + public void setDescription(String description) + { + _description = description; + } + + public String getContainer() + { + return _container; + } + + public void setContainer(String container) + { + _container = container; + } + + public Integer getCreatedBy() + { + return _createdBy; + } + + public void setCreatedBy(Integer createdBy) + { + _createdBy = createdBy; + } + + public Date getCreated() + { + return _created; + } + + public void setCreated(Date created) + { + _created = created; + } + + public Integer getModifiedBy() + { + return _modifiedBy; + } + + public void setModifiedBy(Integer modifiedBy) + { + _modifiedBy = modifiedBy; + } + + public Date getModified() + { + return _modified; + } + + public void setModified(Date modified) + { + _modified = modified; + } + + public List getCohorts() + { + return _cohorts; + } + + public void setCohorts(List cohorts) + { + _cohorts = cohorts; + } + + public List getAnchorEvents() + { + return _anchorEvents; + } + + public void setAnchorEvents(List anchorEvents) + { + _anchorEvents = anchorEvents; + } + + public List getTimepoints() + { + return _timepoints; + } + + public void setTimepoints(List timepoints) + { + _timepoints = timepoints; + } + + public static class StudyCohort + { + private Integer _rowId; + private Integer _studyId; + + private String _cohortName; + private String _label; + private String _category; + private String _description; + private Boolean _isControlGroup = false; + private Integer _sortOrder; + + private String _container; + private Integer _createdBy; + private Date _created; + + private Integer _modifiedBy; + private Date _modified; + + public StudyCohort() + { + + } + + public Integer getRowId() + { + return _rowId; + } + + public void setRowId(Integer rowId) + { + _rowId = rowId; + } + + public Integer getStudyId() + { + return _studyId; + } + + public void setStudyId(Integer studyId) + { + _studyId = studyId; + } + + public String getCohortName() + { + return _cohortName; + } + + public void setCohortName(String cohortName) + { + _cohortName = cohortName; + } + + public String getLabel() + { + return _label; + } + + public void setLabel(String label) + { + _label = label; + } + + public String getCategory() + { + return _category; + } + + public void setCategory(String category) + { + _category = category; + } + + public String getDescription() + { + return _description; + } + + public void setDescription(String description) + { + _description = description; + } + + public Boolean getIsControlGroup() + { + return _isControlGroup; + } + + public void setIsControlGroup(Boolean controlGroup) + { + _isControlGroup = controlGroup; + } + + public Integer getSortOrder() + { + return _sortOrder; + } + + public void setSortOrder(Integer sortOrder) + { + _sortOrder = sortOrder; + } + + public String getContainer() + { + return _container; + } + + public void setContainer(String container) + { + _container = container; + } + + public Integer getCreatedBy() + { + return _createdBy; + } + + public void setCreatedBy(Integer createdBy) + { + _createdBy = createdBy; + } + + public Date getCreated() + { + return _created; + } + + public void setCreated(Date created) + { + _created = created; + } + + public Integer getModifiedBy() + { + return _modifiedBy; + } + + public void setModifiedBy(Integer modifiedBy) + { + _modifiedBy = modifiedBy; + } + + public Date getModified() + { + return _modified; + } + + public void setModified(Date modified) + { + _modified = modified; + } + } + + public static class AnchorEvent + { + private Integer _rowId; + private Integer _studyId; + + private String _label; + private String _description; + private String _eventProviderName; + + private String _container; + private Integer _createdBy; + private Date _created; + + private Integer _modifiedBy; + private Date _modified; + + public AnchorEvent() + { + + } + + public Integer getRowId() + { + return _rowId; + } + + public void setRowId(Integer rowId) + { + _rowId = rowId; + } + + public Integer getStudyId() + { + return _studyId; + } + + public void setStudyId(Integer studyId) + { + _studyId = studyId; + } + + public String getLabel() + { + return _label; + } + + public void setLabel(String label) + { + _label = label; + } + + public String getDescription() + { + return _description; + } + + public void setDescription(String description) + { + _description = description; + } + + public String getEventProviderName() + { + return _eventProviderName; + } + + public void setEventProviderName(String eventProviderName) + { + _eventProviderName = eventProviderName; + } + + public String getContainer() + { + return _container; + } + + public void setContainer(String container) + { + _container = container; + } + + public Integer getCreatedBy() + { + return _createdBy; + } + + public void setCreatedBy(Integer createdBy) + { + _createdBy = createdBy; + } + + public Date getCreated() + { + return _created; + } + + public void setCreated(Date created) + { + _created = created; + } + + public Integer getModifiedBy() + { + return _modifiedBy; + } + + public void setModifiedBy(Integer modifiedBy) + { + _modifiedBy = modifiedBy; + } + + public Date getModified() + { + return _modified; + } + + public void setModified(Date modified) + { + _modified = modified; + } + } + + public static class Timepoint + { + private Integer _rowId; + private Integer _studyId; + private Integer _cohortId; + private String _label; + private String _labelShort; + private String _description; + + @JsonIgnore + private Integer _anchorEvent; + + @JsonIgnore + private String _anchorEventLabel; + + private String _cohortName; + private Integer _rangeMin; + private Integer _rangeMax; + + private String _container; + private Integer _createdBy; + private Date _created; + + private Integer _modifiedBy; + private Date _modified; + + public Timepoint() + { + + } + + // When reading from a JSON object, store the anchorEvent label + @JsonSetter("anchorEvent") + void readAnchorEvent(String lbl) { _anchorEventLabel = lbl; } + + @JsonGetter("anchorEvent") + String writeAnchorEvent() { return _anchorEventLabel; } + + // Call this to translate from label to index in order to fit the DB schema + void resolveAnchorEvent(Map idxByLabel) + { + Integer idx = idxByLabel.get(_anchorEventLabel); + if (idx == null) + throw new IllegalArgumentException( + "Unknown anchorEvent label '" + _anchorEventLabel + "'"); + _anchorEvent = idx; + } + + public Integer getAnchorEvent() { return _anchorEvent; } + + public Integer getRowId() + { + return _rowId; + } + + public void setRowId(Integer rowId) + { + _rowId = rowId; + } + + public Integer getStudyId() + { + return _studyId; + } + + public void setStudyId(Integer studyId) + { + _studyId = studyId; + } + + public String getCohortName() + { + return _cohortName; + } + + public void setCohortName(String cohortName) + { + _cohortName = cohortName; + } + + public Integer getCohortId() + { + return _cohortId; + } + + public void setCohortId(Integer cohortId) + { + _cohortId = cohortId; + } + + public String getLabel() + { + return _label; + } + + public void setLabel(String label) + { + _label = label; + } + + public String getLabelShort() + { + return _labelShort; + } + + public void setLabelShort(String labelShort) + { + _labelShort = labelShort; + } + + public String getDescription() + { + return _description; + } + + public void setDescription(String description) + { + _description = description; + } + + public Integer getRangeMin() + { + return _rangeMin; + } + + public void setRangeMin(Integer rangeMin) + { + _rangeMin = rangeMin; + } + + public Integer getRangeMax() + { + return _rangeMax; + } + + public void setRangeMax(Integer rangeMax) + { + _rangeMax = rangeMax; + } + + public String getContainer() + { + return _container; + } + + public void setContainer(String container) + { + _container = container; + } + + public Integer getCreatedBy() + { + return _createdBy; + } + + public void setCreatedBy(Integer createdBy) + { + _createdBy = createdBy; + } + + public Date getCreated() + { + return _created; + } + + public void setCreated(Date created) + { + _created = created; + } + + public Integer getModifiedBy() + { + return _modifiedBy; + } + + public void setModifiedBy(Integer modifiedBy) + { + _modifiedBy = modifiedBy; + } + + public Date getModified() + { + return _modified; + } + + public void setModified(Date modified) + { + _modified = modified; + } + } + + public static StudyDefinition fromJson(JSONObject json) + { + ObjectMapper mapper = new ObjectMapper(); + StudyDefinition sd = mapper.convertValue(json.toMap(), StudyDefinition.class); + + // In our JSON, Timepoints store the anchorEvent label, not an ID. Since the DB schema requires an int, we need + // to do that translation manually. Here, we store the anchorEvent by its index in the anchorEvent list. + Map idxByLabel = IntStream.range(0, sd.getAnchorEvents().size()) + .boxed() + .collect(Collectors.toMap( + i -> sd.getAnchorEvents().get(i).getLabel(), + i -> i)); + + sd.getTimepoints().forEach(tp -> tp.resolveAnchorEvent(idxByLabel)); + + return sd; + } + + public String toJson() throws JsonProcessingException + { + ObjectWriter ow = new ObjectMapper().writer(); + return ow.writeValueAsString(this); + } + + public static StudyDefinition getForId(int studyId) + { + // TODO: implement this. This should query the DB and return a populated StudyDefinition + + return null; + } +} diff --git a/Studies/resources/study/DemoStudy.json b/Studies/resources/study/DemoStudy.json new file mode 100644 index 000000000..07f419b06 --- /dev/null +++ b/Studies/resources/study/DemoStudy.json @@ -0,0 +1,43 @@ +{ + "studyName": "DemoStudy", + "label": "Demo Study", + "description": "This is a demo study", + "cohorts": [{ + "cohortName": "Group1", + "label": "Group 1", + "description": "This is the first group", + "isControlGroup": false, + "sortOrder": 1 + },{ + "cohortName": "Control", + "label": "Control Group", + "description": "This is the control group", + "isControlGroup": true, + "sortOrder": 2 + }], + "anchorEvents": [{ + "label": "Study Enrollment", + "description": "The represents Day 0 of the study", + "eventProviderName": "EnrollmentStart" + }], + "timepoints": [{ + "cohortId": null, + "label": "Day 0", + "labelShort": "D0", + "anchorEvent": "Study Enrollment" + },{ + "cohortName": "Group1", + "label": "Vaccination", + "labelShort": "V", + "anchorEvent": "Study Enrollment", + "rangeMin": 7, + "rangeMax": 10 + },{ + "cohortName": "Control", + "label": "Mock-Vaccination", + "labelShort": "V", + "anchorEvent": "Study Enrollment", + "rangeMin": 7, + "rangeMax": 10 + }] +} \ No newline at end of file diff --git a/Studies/src/org/labkey/studies/StudiesController.java b/Studies/src/org/labkey/studies/StudiesController.java index 58d6b51dd..8de8e05be 100644 --- a/Studies/src/org/labkey/studies/StudiesController.java +++ b/Studies/src/org/labkey/studies/StudiesController.java @@ -1,37 +1,16 @@ package org.labkey.studies; import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.labkey.api.action.ConfirmAction; +import org.labkey.api.action.ApiSimpleResponse; +import org.labkey.api.action.MutatingApiAction; +import org.labkey.api.action.SimpleApiJsonForm; import org.labkey.api.action.SpringActionController; -import org.labkey.api.data.TableInfo; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.pipeline.PipelineUrls; -import org.labkey.api.query.BatchValidationException; -import org.labkey.api.query.DuplicateKeyException; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.QueryUpdateService; -import org.labkey.api.query.QueryUpdateServiceException; -import org.labkey.api.reader.DataLoader; -import org.labkey.api.reader.TabLoader; -import org.labkey.api.resource.Resource; import org.labkey.api.security.RequiresPermission; import org.labkey.api.security.permissions.AdminPermission; -import org.labkey.api.studies.StudiesService; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.HtmlString; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.Path; -import org.labkey.api.util.URLHelper; +import org.labkey.api.studies.study.StudyDefinition; import org.labkey.api.util.logging.LogHelper; -import org.labkey.api.view.HtmlView; import org.springframework.validation.BindException; -import org.springframework.validation.Errors; -import org.springframework.web.servlet.ModelAndView; -import java.io.IOException; -import java.sql.SQLException; -import java.util.List; import java.util.Map; public class StudiesController extends SpringActionController @@ -45,4 +24,26 @@ public StudiesController() { setActionResolver(_actionResolver); } + + + @RequiresPermission(AdminPermission.class) + public static class UpdateStudyDefinitionAction extends MutatingApiAction + { + @Override + public Object execute(SimpleApiJsonForm json, BindException errors) throws Exception + { + try + { + StudyDefinition sd = StudyDefinition.fromJson(json.getJsonObject()); + sd = StudiesManager.get().insertOrUpdateStudyDefinition(sd, getContainer(), getUser()); + + return new ApiSimpleResponse(Map.of("success", true, "studyDefinition", sd)); + } + catch (Exception e) + { + _log.error("Unable to import study definition", e); + return new ApiSimpleResponse("success", false); + } + } + } } diff --git a/Studies/src/org/labkey/studies/StudiesManager.java b/Studies/src/org/labkey/studies/StudiesManager.java index 469f52781..bf7926dcc 100644 --- a/Studies/src/org/labkey/studies/StudiesManager.java +++ b/Studies/src/org/labkey/studies/StudiesManager.java @@ -1,16 +1,416 @@ package org.labkey.studies; +import org.apache.commons.collections.map.CaseInsensitiveMap; +import org.apache.commons.io.IOUtils; +import org.json.JSONObject; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.runners.Enclosed; +import org.junit.runner.RunWith; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.discvrcore.test.AbstractIntegrationTest; +import org.labkey.api.module.Module; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.DuplicateKeyException; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.InvalidKeyException; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.QueryUpdateService; +import org.labkey.api.query.QueryUpdateServiceException; +import org.labkey.api.query.SimpleUserSchema; +import org.labkey.api.query.UserSchema; +import org.labkey.api.resource.FileResource; +import org.labkey.api.resource.Resource; +import org.labkey.api.security.User; +import org.labkey.api.studies.study.StudyDefinition; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.TestContext; +import org.labkey.studies.query.StudiesUserSchema; + + +import java.io.FileInputStream; +import java.io.InputStream; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static com.fasterxml.jackson.databind.type.LogicalType.Collection; + +@RunWith(Enclosed.class) public class StudiesManager { private static final StudiesManager _instance = new StudiesManager(); private StudiesManager() { - // prevent external construction with a private default constructor + } public static StudiesManager get() { return _instance; } + + public StudyDefinition insertOrUpdateStudyDefinition(StudyDefinition sd, Container c, User u) + { + StudiesSchema ss = StudiesSchema.getInstance(); + DbSchema schema = ss.getSchema(); + DbScope scope = schema.getScope(); + + UserSchema us = QueryService.get().getUserSchema(u, c, StudiesSchema.NAME); + + TableInfo tblStudies = us.getTable(StudiesSchema.TABLE_STUDIES); + TableInfo tblCohorts = us.getTable(StudiesSchema.TABLE_COHORTS); + TableInfo tblAnchorEvents = us.getTable(StudiesSchema.TABLE_ANCHOR_EVENTS); + TableInfo tblTimepoints = us.getTable(StudiesSchema.TABLE_EXPECTED_TIMEPOINTS); + + try (DbScope.Transaction tx = scope.ensureTransaction()) + { + sd.setContainer(c.getEntityId().toString()); + sd = upsertStudy(sd, tblStudies, c, u); + + upsertChildRecords( + sd.getRowId(), + sd.getCohorts(), + tblCohorts, + c, + u, + this::cohortToMap, + StudyDefinition.StudyCohort::getRowId, + StudyDefinition.StudyCohort::setRowId + ); + + upsertChildRecords( + sd.getRowId(), + sd.getAnchorEvents(), + tblAnchorEvents, + c, + u, + this::anchorToMap, + StudyDefinition.AnchorEvent::getRowId, + StudyDefinition.AnchorEvent::setRowId + ); + + upsertChildRecords( + sd.getRowId(), + sd.getTimepoints(), + tblTimepoints, + c, + u, + this::timepointToMap, + StudyDefinition.Timepoint::getRowId, + StudyDefinition.Timepoint::setRowId + ); + + tx.commit(); + } + catch (Exception x) + { + throw new RuntimeException("Failed to up‑sert StudyDefinition", x); + } + + return sd; + } + + private StudyDefinition upsertStudy(StudyDefinition sd, + TableInfo tbl, + Container c, + User u) throws Exception + { + Map map = studyToMap(sd); + BatchValidationException bve = new BatchValidationException(); + QueryUpdateService qus = tbl.getUpdateService(); + + List> rows = List.of(map); + List> ret; + + if (sd.getRowId() == null) + ret = qus.insertRows(u, c, rows, bve, null, null); + else + { + ret = qus.updateRows(u, c, rows, null, bve, null, null); + } + + if (bve.hasErrors()) + throw bve; + + sd.setRowId((Integer) ret.get(0).get("rowId")); + return sd; + } + + + private void upsertChildRecords(int studyRowId, + List incoming, + TableInfo tbl, + Container c, + User u, + Mapper mapper, + RowIdGetter getRowId, + RowIdSetter setRowId) throws Exception + { + QueryUpdateService qus = tbl.getUpdateService(); + + Set existing = new HashSet<>( + new TableSelector(tbl, PageFlowUtil.set("rowId"), + new SimpleFilter(FieldKey.fromString("studyId"), studyRowId), + null).getCollection(Integer.class) + ); + + List> inserts = new ArrayList<>(); + List insertBeans = new ArrayList<>(); + List> updates = new ArrayList<>(); + + for (T bean : incoming) + { + Map row = mapper.apply(bean); + row.put("studyId", studyRowId); + + Integer rk = getRowId.get(bean); + if (rk == null) + { + inserts.add(row); + insertBeans.add(bean); + } + else + { + updates.add(row); + existing.remove(rk); + } + } + + if (!existing.isEmpty()) + { + List> keys = existing.stream() + .map(rid -> Map.of("rowid", (Object) rid)) + .toList(); + + qus.deleteRows(u, c, keys, null, null); + } + + BatchValidationException bve = new BatchValidationException(); + + if (!inserts.isEmpty()) + { + List> ret = qus.insertRows(u, c, inserts, bve, null, null); + for (int i = 0; i < ret.size(); i++) + setRowId.set(insertBeans.get(i), (Integer) ret.get(i).get("rowId")); + } + + if (!updates.isEmpty()) + { + qus.updateRows(u, c, updates, null, bve, null, null); + } + + if (bve.hasErrors()) + throw bve; + } + + private Map studyToMap(StudyDefinition s) + { + Map m = new HashMap<>(); + if (s.getRowId() != null) + m.put("rowId", s.getRowId()); + m.put("name", s.getStudyName()); + m.put("label", s.getLabel()); + m.put("category", s.getCategory()); + m.put("description", s.getDescription()); + m.put("container", s.getContainer()); + return m; + } + + private Map cohortToMap(StudyDefinition.StudyCohort c) + { + Map m = new HashMap<>(); + if (c.getRowId() != null) + m.put("rowId", c.getRowId()); + m.put("cohortName", c.getCohortName()); + m.put("label", c.getLabel()); + m.put("category", c.getCategory()); + m.put("description", c.getDescription()); + m.put("isControlGroup",c.getIsControlGroup()); + m.put("sortOrder", c.getSortOrder()); + m.put("container", c.getContainer()); + return m; + } + + private Map anchorToMap(StudyDefinition.AnchorEvent a) + { + Map m = new HashMap<>(); + if (a.getRowId() != null) + m.put("rowId", a.getRowId()); + m.put("label", a.getLabel()); + m.put("description", a.getDescription()); + m.put("eventProviderName",a.getEventProviderName()); + m.put("container", a.getContainer()); + return m; + } + + private Map timepointToMap(StudyDefinition.Timepoint t) + { + Map m = new HashMap<>(); + if (t.getRowId() != null) + m.put("rowId", t.getRowId()); + m.put("cohortId", t.getCohortId()); + m.put("cohortName", t.getCohortName()); + m.put("label", t.getLabel()); + m.put("labelShort", t.getLabelShort()); + m.put("description", t.getDescription()); + m.put("anchorEvent", t.getAnchorEvent()); + m.put("rangeMin", t.getRangeMin()); + m.put("rangeMax", t.getRangeMax()); + m.put("container", t.getContainer()); + return m; + } + + @FunctionalInterface private interface Mapper { Map apply(T t); } + @FunctionalInterface private interface RowIdGetter { Integer get(T t); } + @FunctionalInterface private interface RowIdSetter { void set(T t, Integer id); } + + public static class TestCase extends AbstractIntegrationTest + { + public static final String PROJECT_NAME = "StudiesIntegrationTestFolder"; + + @BeforeClass + public static void setup() throws Exception + { + doInitialSetUp(PROJECT_NAME); + + Container project = ContainerManager.getForPath(PROJECT_NAME); + Set active = new HashSet<>(project.getActiveModules()); + active.add(ModuleLoader.getInstance().getModule(StudiesModule.NAME)); + project.setActiveModules(active); + } + + @AfterClass + public static void cleanup() + { + doCleanup(PROJECT_NAME); + } + + @Test + public void testStudyInsert() throws Exception + { + Container c = ContainerManager.getForPath(PROJECT_NAME); + Resource r = ModuleLoader.getInstance() + .getModule(StudiesModule.NAME) + .getModuleResource("study/DemoStudy.json"); + + if (!(r instanceof FileResource fr)) + throw new IllegalStateException("Expected a FileResource; got " + r); + + StudyDefinition sd; + try (InputStream is = new FileInputStream(fr.getFile())) + { + String jsonTxt = IOUtils.toString(is, StringUtilsLabKey.DEFAULT_CHARSET); + sd = StudyDefinition.fromJson(new JSONObject(jsonTxt)); + } + + sd = StudiesManager.get().insertOrUpdateStudyDefinition(sd, c, TestContext.get().getUser()); + + // 1. Verify insert + assertNotNull( "study rowId null after insert", sd.getRowId()); + sd.getCohorts().forEach(co -> assertNotNull("cohort rowId null", co.getRowId())); + sd.getAnchorEvents().forEach(ev -> assertNotNull("anchor rowId null", ev.getRowId())); + sd.getTimepoints().forEach(tp -> assertNotNull("timepoint rowId null", tp.getRowId())); + + StudiesSchema ss = StudiesSchema.getInstance(); + DbSchema schema = ss.getSchema(); + TableInfo tblStudies = schema.getTable(StudiesSchema.TABLE_STUDIES); + TableInfo tblCohorts = schema.getTable(StudiesSchema.TABLE_COHORTS); + TableInfo tblTP = schema.getTable(StudiesSchema.TABLE_EXPECTED_TIMEPOINTS); + + assertEquals(1, + new TableSelector(tblStudies, + PageFlowUtil.set("rowId"), + new SimpleFilter(FieldKey.fromString("rowId"), sd.getRowId()), + null).getRowCount()); + + int cohortCount = sd.getCohorts().size(); + int timepointCount = sd.getTimepoints().size(); + + assertEquals(cohortCount, + new TableSelector(tblCohorts, + PageFlowUtil.set("rowId"), + new SimpleFilter(FieldKey.fromString("studyId"), sd.getRowId()), + null).getRowCount()); + + assertEquals(timepointCount, + new TableSelector(tblTP, + PageFlowUtil.set("rowId"), + new SimpleFilter(FieldKey.fromString("studyId"), sd.getRowId()), + null).getRowCount()); + + // 2. Update some values, add a cohort, delete a timepoint + sd.setLabel(sd.getLabel() + " (updated)"); + + StudyDefinition.StudyCohort firstCohort = sd.getCohorts().get(0); + firstCohort.setLabel(firstCohort.getLabel() + "-updated"); + + StudyDefinition.StudyCohort newCohort = new StudyDefinition.StudyCohort(); + newCohort.setCohortName("NEW"); + newCohort.setLabel("Brand-new cohort"); + sd.getCohorts().add(newCohort); + + StudyDefinition.Timepoint removedTp = sd.getTimepoints().remove(0); + + sd = StudiesManager.get().insertOrUpdateStudyDefinition(sd, c, TestContext.get().getUser()); + + assertNotNull("new cohort did not receive rowId", newCohort.getRowId()); + + assertEquals(cohortCount + 1, + new TableSelector(tblCohorts, + PageFlowUtil.set("rowId"), + new SimpleFilter(FieldKey.fromString("studyId"), sd.getRowId()), + null).getRowCount()); + + assertEquals(timepointCount - 1, + new TableSelector(tblTP, + PageFlowUtil.set("rowId"), + new SimpleFilter(FieldKey.fromString("studyId"), sd.getRowId()), + null).getRowCount()); + + String dbLabel = new TableSelector(tblStudies, + PageFlowUtil.set("label"), + new SimpleFilter(FieldKey.fromString("rowId"), sd.getRowId()), + null).getObject(String.class); + assertEquals(sd.getLabel(), dbLabel); + + String dbCohortLabel = new TableSelector(tblCohorts, + PageFlowUtil.set("label"), + new SimpleFilter(FieldKey.fromString("rowId"), firstCohort.getRowId()), + null).getObject(String.class); + assertEquals(firstCohort.getLabel(), dbCohortLabel); + + // 3. Delete the new cohort + sd.getCohorts().remove(newCohort); + sd = StudiesManager.get().insertOrUpdateStudyDefinition(sd, c, TestContext.get().getUser()); + + assertEquals(cohortCount, + new TableSelector(tblCohorts, + PageFlowUtil.set("rowId"), + new SimpleFilter(FieldKey.fromString("studyId"), sd.getRowId()), + null).getRowCount()); + + // 4. Round-trip JSON export + JSONObject roundTrip = new JSONObject(sd.toJson()); + assertEquals(sd.getLabel(), roundTrip.getString("label")); + assertEquals(sd.getCohorts().size(), roundTrip.getJSONArray("cohorts").length()); + assertEquals(sd.getTimepoints().size(), roundTrip.getJSONArray("timepoints").length()); + } + } } \ No newline at end of file diff --git a/Studies/src/org/labkey/studies/StudiesModule.java b/Studies/src/org/labkey/studies/StudiesModule.java index f68d3746d..b4d204551 100644 --- a/Studies/src/org/labkey/studies/StudiesModule.java +++ b/Studies/src/org/labkey/studies/StudiesModule.java @@ -11,6 +11,8 @@ import org.labkey.api.query.QuerySchema; import org.labkey.api.security.roles.RoleManager; import org.labkey.api.studies.StudiesService; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.studies.query.StudiesUserSchema; import org.labkey.api.studies.security.StudiesDataAdminRole; import org.labkey.studies.query.StudiesUserSchema; import org.labkey.studies.study.StudiesFilterProvider; @@ -79,4 +81,12 @@ public QuerySchema createSchema(final DefaultSchema schema, Module module) } }); } + + @Override + public @NotNull Set getIntegrationTests() + { + return PageFlowUtil.set( + StudiesManager.TestCase.class + ); + } } \ No newline at end of file diff --git a/Studies/src/org/labkey/studies/query/StudiesTableCustomizer.java b/Studies/src/org/labkey/studies/query/StudiesTableCustomizer.java index 9b360fe1b..fdf8f8754 100644 --- a/Studies/src/org/labkey/studies/query/StudiesTableCustomizer.java +++ b/Studies/src/org/labkey/studies/query/StudiesTableCustomizer.java @@ -1,8 +1,10 @@ package org.labkey.studies.query; import org.apache.commons.collections4.MultiValuedMap; +import org.apache.commons.collections4.SetValuedMap; import org.apache.logging.log4j.Logger; import org.labkey.api.collections.CaseInsensitiveHashMap; + import org.labkey.api.collections.CaseInsensitiveKeyedHashSetValuedMap; import org.labkey.api.data.AbstractTableInfo; import org.labkey.api.data.TableCustomizer; From f522a7686de82607a98fe2b2058413f4543dc913 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 23 Jul 2025 12:36:23 -0700 Subject: [PATCH 28/29] Correct typo --- jbrowse/resources/views/begin.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jbrowse/resources/views/begin.html b/jbrowse/resources/views/begin.html index 9032801a3..50c216bb6 100644 --- a/jbrowse/resources/views/begin.html +++ b/jbrowse/resources/views/begin.html @@ -1,9 +1,9 @@ The JBrowse module is part of DISCVR-Seq. It provides a wrapper around the JBrowse Genome Browser, which lets users rapidly take data generated or uploaded into DISCRV-Seq and view it using JBrowse. -The module ships with a version of JBrowse, meaning very little extra configuration is required in order to start using these tools. The primary reasons we created this wrapper around JBrowse are: +The module ships with a version of JBrowse 2, meaning very little extra configuration is required in order to start using these tools. The primary reasons we created this wrapper around JBrowse are:

  • We selected JBrowse as the browser because it is a modern web-based genome viewer, with a large support base
  • This wrapper makes is very simple for non-technical users to take data uploaded into DISCVR-Seq and use the power of JBrowse to see your data. While we find JBrowse easy to use, preparing samples for the browser often requires command-line tools or was otherwise beyond the level of a typical scientist
  • For our usage, we find it important to be able to quickly view different combinations of files. For example, after running an alignment or calling SNPs on a set of samples, you might want to view these specific samples in conjunction with a specific set of genome annotations (coding regions, regulatory regions, site of known variation, etc).
    • -
  • The files createde by genome-scale projects can become large. If we allow users to create many different JBrowse databases, we need to be careful about the amount of files this creates. The JBrowse module does a fair amount of work behind the scenes to avoid duplicating files. For example, if the same genome is used as the base for 100s of JBrowse sessions, there will only be one copy of the processed sequences files saved. The same is true for all tracks that are loaded.
    • +
  • The files created by genome-scale projects can become large. If we allow users to create many different JBrowse databases, we need to be careful about the amount of files this creates. The JBrowse module does a fair amount of work behind the scenes to avoid duplicating files. For example, if the same genome is used as the base for 100s of JBrowse sessions, there will only be one copy of the processed sequences files saved. The same is true for all tracks that are loaded.
\ No newline at end of file From 6790e2d7646478da3884a93ffa4f3d748f6c389b Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 23 Jul 2025 12:37:30 -0700 Subject: [PATCH 29/29] Improve test logging --- jbrowse/src/org/labkey/jbrowse/JBrowseManager.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/jbrowse/src/org/labkey/jbrowse/JBrowseManager.java b/jbrowse/src/org/labkey/jbrowse/JBrowseManager.java index f3e56434b..d727be3a8 100644 --- a/jbrowse/src/org/labkey/jbrowse/JBrowseManager.java +++ b/jbrowse/src/org/labkey/jbrowse/JBrowseManager.java @@ -238,7 +238,19 @@ public static class TestCase extends Assert public void testJBrowseCli() throws Exception { File exe = JBrowseManager.get().getJbrowseCli(); - String output = new SimpleScriptWrapper(_log).executeWithOutput(Arrays.asList(exe.getPath(), "help")); + SimpleScriptWrapper wrapper = new SimpleScriptWrapper(_log); + wrapper.setThrowNonZeroExits(false); + + String output = wrapper.executeWithOutput(Arrays.asList(exe.getPath(), "help")); + if (wrapper.getLastReturnCode() != 0) + { + _log.error("Non-zero exit from testJBrowseCli: " + wrapper.getLastReturnCode()); + wrapper.getCommandsExecuted().forEach(_log::error); + _log.error("output: "); + _log.error(output); + + throw new RuntimeException("Non-zero exit running testJBrowseCli: " + wrapper.getLastReturnCode()); + } assertTrue("Malformed output", output.contains("Add an assembly to a JBrowse 2 configuration")); }